1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include "iavf.h" 5 #include "iavf_prototype.h" 6 #include "iavf_client.h" 7 /* All iavf tracepoints are defined by the include below, which must 8 * be included exactly once across the whole kernel with 9 * CREATE_TRACE_POINTS defined 10 */ 11 #define CREATE_TRACE_POINTS 12 #include "iavf_trace.h" 13 14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter); 15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter); 16 static int iavf_close(struct net_device *netdev); 17 static void iavf_init_get_resources(struct iavf_adapter *adapter); 18 static int iavf_check_reset_complete(struct iavf_hw *hw); 19 20 char iavf_driver_name[] = "iavf"; 21 static const char iavf_driver_string[] = 22 "Intel(R) Ethernet Adaptive Virtual Function Network Driver"; 23 24 static const char iavf_copyright[] = 25 "Copyright (c) 2013 - 2018 Intel Corporation."; 26 27 /* iavf_pci_tbl - PCI Device ID Table 28 * 29 * Wildcard entries (PCI_ANY_ID) should come last 30 * Last entry must be all 0s 31 * 32 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 33 * Class, Class Mask, private data (not used) } 34 */ 35 static const struct pci_device_id iavf_pci_tbl[] = { 36 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0}, 37 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0}, 38 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0}, 39 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0}, 40 /* required last entry */ 41 {0, } 42 }; 43 44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl); 45 46 MODULE_ALIAS("i40evf"); 47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver"); 49 MODULE_LICENSE("GPL v2"); 50 51 static const struct net_device_ops iavf_netdev_ops; 52 struct workqueue_struct *iavf_wq; 53 54 /** 55 * iavf_pdev_to_adapter - go from pci_dev to adapter 56 * @pdev: pci_dev pointer 57 */ 58 static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev) 59 { 60 return netdev_priv(pci_get_drvdata(pdev)); 61 } 62 63 /** 64 * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code 65 * @hw: pointer to the HW structure 66 * @mem: ptr to mem struct to fill out 67 * @size: size of memory requested 68 * @alignment: what to align the allocation to 69 **/ 70 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw, 71 struct iavf_dma_mem *mem, 72 u64 size, u32 alignment) 73 { 74 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 75 76 if (!mem) 77 return IAVF_ERR_PARAM; 78 79 mem->size = ALIGN(size, alignment); 80 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size, 81 (dma_addr_t *)&mem->pa, GFP_KERNEL); 82 if (mem->va) 83 return 0; 84 else 85 return IAVF_ERR_NO_MEMORY; 86 } 87 88 /** 89 * iavf_free_dma_mem_d - OS specific memory free for shared code 90 * @hw: pointer to the HW structure 91 * @mem: ptr to mem struct to free 92 **/ 93 enum iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw, 94 struct iavf_dma_mem *mem) 95 { 96 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 97 98 if (!mem || !mem->va) 99 return IAVF_ERR_PARAM; 100 dma_free_coherent(&adapter->pdev->dev, mem->size, 101 mem->va, (dma_addr_t)mem->pa); 102 return 0; 103 } 104 105 /** 106 * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code 107 * @hw: pointer to the HW structure 108 * @mem: ptr to mem struct to fill out 109 * @size: size of memory requested 110 **/ 111 enum iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw, 112 struct iavf_virt_mem *mem, u32 size) 113 { 114 if (!mem) 115 return IAVF_ERR_PARAM; 116 117 mem->size = size; 118 mem->va = kzalloc(size, GFP_KERNEL); 119 120 if (mem->va) 121 return 0; 122 else 123 return IAVF_ERR_NO_MEMORY; 124 } 125 126 /** 127 * iavf_free_virt_mem_d - OS specific memory free for shared code 128 * @hw: pointer to the HW structure 129 * @mem: ptr to mem struct to free 130 **/ 131 enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw, 132 struct iavf_virt_mem *mem) 133 { 134 if (!mem) 135 return IAVF_ERR_PARAM; 136 137 /* it's ok to kfree a NULL pointer */ 138 kfree(mem->va); 139 140 return 0; 141 } 142 143 /** 144 * iavf_lock_timeout - try to lock mutex but give up after timeout 145 * @lock: mutex that should be locked 146 * @msecs: timeout in msecs 147 * 148 * Returns 0 on success, negative on failure 149 **/ 150 int iavf_lock_timeout(struct mutex *lock, unsigned int msecs) 151 { 152 unsigned int wait, delay = 10; 153 154 for (wait = 0; wait < msecs; wait += delay) { 155 if (mutex_trylock(lock)) 156 return 0; 157 158 msleep(delay); 159 } 160 161 return -1; 162 } 163 164 /** 165 * iavf_schedule_reset - Set the flags and schedule a reset event 166 * @adapter: board private structure 167 **/ 168 void iavf_schedule_reset(struct iavf_adapter *adapter) 169 { 170 if (!(adapter->flags & 171 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) { 172 adapter->flags |= IAVF_FLAG_RESET_NEEDED; 173 queue_work(iavf_wq, &adapter->reset_task); 174 } 175 } 176 177 /** 178 * iavf_schedule_request_stats - Set the flags and schedule statistics request 179 * @adapter: board private structure 180 * 181 * Sets IAVF_FLAG_AQ_REQUEST_STATS flag so iavf_watchdog_task() will explicitly 182 * request and refresh ethtool stats 183 **/ 184 void iavf_schedule_request_stats(struct iavf_adapter *adapter) 185 { 186 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_STATS; 187 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 188 } 189 190 /** 191 * iavf_tx_timeout - Respond to a Tx Hang 192 * @netdev: network interface device structure 193 * @txqueue: queue number that is timing out 194 **/ 195 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue) 196 { 197 struct iavf_adapter *adapter = netdev_priv(netdev); 198 199 adapter->tx_timeout_count++; 200 iavf_schedule_reset(adapter); 201 } 202 203 /** 204 * iavf_misc_irq_disable - Mask off interrupt generation on the NIC 205 * @adapter: board private structure 206 **/ 207 static void iavf_misc_irq_disable(struct iavf_adapter *adapter) 208 { 209 struct iavf_hw *hw = &adapter->hw; 210 211 if (!adapter->msix_entries) 212 return; 213 214 wr32(hw, IAVF_VFINT_DYN_CTL01, 0); 215 216 iavf_flush(hw); 217 218 synchronize_irq(adapter->msix_entries[0].vector); 219 } 220 221 /** 222 * iavf_misc_irq_enable - Enable default interrupt generation settings 223 * @adapter: board private structure 224 **/ 225 static void iavf_misc_irq_enable(struct iavf_adapter *adapter) 226 { 227 struct iavf_hw *hw = &adapter->hw; 228 229 wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK | 230 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 231 wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 232 233 iavf_flush(hw); 234 } 235 236 /** 237 * iavf_irq_disable - Mask off interrupt generation on the NIC 238 * @adapter: board private structure 239 **/ 240 static void iavf_irq_disable(struct iavf_adapter *adapter) 241 { 242 int i; 243 struct iavf_hw *hw = &adapter->hw; 244 245 if (!adapter->msix_entries) 246 return; 247 248 for (i = 1; i < adapter->num_msix_vectors; i++) { 249 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0); 250 synchronize_irq(adapter->msix_entries[i].vector); 251 } 252 iavf_flush(hw); 253 } 254 255 /** 256 * iavf_irq_enable_queues - Enable interrupt for specified queues 257 * @adapter: board private structure 258 * @mask: bitmap of queues to enable 259 **/ 260 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask) 261 { 262 struct iavf_hw *hw = &adapter->hw; 263 int i; 264 265 for (i = 1; i < adapter->num_msix_vectors; i++) { 266 if (mask & BIT(i - 1)) { 267 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 268 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 269 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 270 } 271 } 272 } 273 274 /** 275 * iavf_irq_enable - Enable default interrupt generation settings 276 * @adapter: board private structure 277 * @flush: boolean value whether to run rd32() 278 **/ 279 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush) 280 { 281 struct iavf_hw *hw = &adapter->hw; 282 283 iavf_misc_irq_enable(adapter); 284 iavf_irq_enable_queues(adapter, ~0); 285 286 if (flush) 287 iavf_flush(hw); 288 } 289 290 /** 291 * iavf_msix_aq - Interrupt handler for vector 0 292 * @irq: interrupt number 293 * @data: pointer to netdev 294 **/ 295 static irqreturn_t iavf_msix_aq(int irq, void *data) 296 { 297 struct net_device *netdev = data; 298 struct iavf_adapter *adapter = netdev_priv(netdev); 299 struct iavf_hw *hw = &adapter->hw; 300 301 /* handle non-queue interrupts, these reads clear the registers */ 302 rd32(hw, IAVF_VFINT_ICR01); 303 rd32(hw, IAVF_VFINT_ICR0_ENA1); 304 305 /* schedule work on the private workqueue */ 306 queue_work(iavf_wq, &adapter->adminq_task); 307 308 return IRQ_HANDLED; 309 } 310 311 /** 312 * iavf_msix_clean_rings - MSIX mode Interrupt Handler 313 * @irq: interrupt number 314 * @data: pointer to a q_vector 315 **/ 316 static irqreturn_t iavf_msix_clean_rings(int irq, void *data) 317 { 318 struct iavf_q_vector *q_vector = data; 319 320 if (!q_vector->tx.ring && !q_vector->rx.ring) 321 return IRQ_HANDLED; 322 323 napi_schedule_irqoff(&q_vector->napi); 324 325 return IRQ_HANDLED; 326 } 327 328 /** 329 * iavf_map_vector_to_rxq - associate irqs with rx queues 330 * @adapter: board private structure 331 * @v_idx: interrupt number 332 * @r_idx: queue number 333 **/ 334 static void 335 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx) 336 { 337 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 338 struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx]; 339 struct iavf_hw *hw = &adapter->hw; 340 341 rx_ring->q_vector = q_vector; 342 rx_ring->next = q_vector->rx.ring; 343 rx_ring->vsi = &adapter->vsi; 344 q_vector->rx.ring = rx_ring; 345 q_vector->rx.count++; 346 q_vector->rx.next_update = jiffies + 1; 347 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting); 348 q_vector->ring_mask |= BIT(r_idx); 349 wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx), 350 q_vector->rx.current_itr >> 1); 351 q_vector->rx.current_itr = q_vector->rx.target_itr; 352 } 353 354 /** 355 * iavf_map_vector_to_txq - associate irqs with tx queues 356 * @adapter: board private structure 357 * @v_idx: interrupt number 358 * @t_idx: queue number 359 **/ 360 static void 361 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx) 362 { 363 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 364 struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx]; 365 struct iavf_hw *hw = &adapter->hw; 366 367 tx_ring->q_vector = q_vector; 368 tx_ring->next = q_vector->tx.ring; 369 tx_ring->vsi = &adapter->vsi; 370 q_vector->tx.ring = tx_ring; 371 q_vector->tx.count++; 372 q_vector->tx.next_update = jiffies + 1; 373 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting); 374 q_vector->num_ringpairs++; 375 wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx), 376 q_vector->tx.target_itr >> 1); 377 q_vector->tx.current_itr = q_vector->tx.target_itr; 378 } 379 380 /** 381 * iavf_map_rings_to_vectors - Maps descriptor rings to vectors 382 * @adapter: board private structure to initialize 383 * 384 * This function maps descriptor rings to the queue-specific vectors 385 * we were allotted through the MSI-X enabling code. Ideally, we'd have 386 * one vector per ring/queue, but on a constrained vector budget, we 387 * group the rings as "efficiently" as possible. You would add new 388 * mapping configurations in here. 389 **/ 390 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter) 391 { 392 int rings_remaining = adapter->num_active_queues; 393 int ridx = 0, vidx = 0; 394 int q_vectors; 395 396 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 397 398 for (; ridx < rings_remaining; ridx++) { 399 iavf_map_vector_to_rxq(adapter, vidx, ridx); 400 iavf_map_vector_to_txq(adapter, vidx, ridx); 401 402 /* In the case where we have more queues than vectors, continue 403 * round-robin on vectors until all queues are mapped. 404 */ 405 if (++vidx >= q_vectors) 406 vidx = 0; 407 } 408 409 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 410 } 411 412 /** 413 * iavf_irq_affinity_notify - Callback for affinity changes 414 * @notify: context as to what irq was changed 415 * @mask: the new affinity mask 416 * 417 * This is a callback function used by the irq_set_affinity_notifier function 418 * so that we may register to receive changes to the irq affinity masks. 419 **/ 420 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify, 421 const cpumask_t *mask) 422 { 423 struct iavf_q_vector *q_vector = 424 container_of(notify, struct iavf_q_vector, affinity_notify); 425 426 cpumask_copy(&q_vector->affinity_mask, mask); 427 } 428 429 /** 430 * iavf_irq_affinity_release - Callback for affinity notifier release 431 * @ref: internal core kernel usage 432 * 433 * This is a callback function used by the irq_set_affinity_notifier function 434 * to inform the current notification subscriber that they will no longer 435 * receive notifications. 436 **/ 437 static void iavf_irq_affinity_release(struct kref *ref) {} 438 439 /** 440 * iavf_request_traffic_irqs - Initialize MSI-X interrupts 441 * @adapter: board private structure 442 * @basename: device basename 443 * 444 * Allocates MSI-X vectors for tx and rx handling, and requests 445 * interrupts from the kernel. 446 **/ 447 static int 448 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename) 449 { 450 unsigned int vector, q_vectors; 451 unsigned int rx_int_idx = 0, tx_int_idx = 0; 452 int irq_num, err; 453 int cpu; 454 455 iavf_irq_disable(adapter); 456 /* Decrement for Other and TCP Timer vectors */ 457 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 458 459 for (vector = 0; vector < q_vectors; vector++) { 460 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector]; 461 462 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 463 464 if (q_vector->tx.ring && q_vector->rx.ring) { 465 snprintf(q_vector->name, sizeof(q_vector->name), 466 "iavf-%s-TxRx-%u", basename, rx_int_idx++); 467 tx_int_idx++; 468 } else if (q_vector->rx.ring) { 469 snprintf(q_vector->name, sizeof(q_vector->name), 470 "iavf-%s-rx-%u", basename, rx_int_idx++); 471 } else if (q_vector->tx.ring) { 472 snprintf(q_vector->name, sizeof(q_vector->name), 473 "iavf-%s-tx-%u", basename, tx_int_idx++); 474 } else { 475 /* skip this unused q_vector */ 476 continue; 477 } 478 err = request_irq(irq_num, 479 iavf_msix_clean_rings, 480 0, 481 q_vector->name, 482 q_vector); 483 if (err) { 484 dev_info(&adapter->pdev->dev, 485 "Request_irq failed, error: %d\n", err); 486 goto free_queue_irqs; 487 } 488 /* register for affinity change notifications */ 489 q_vector->affinity_notify.notify = iavf_irq_affinity_notify; 490 q_vector->affinity_notify.release = 491 iavf_irq_affinity_release; 492 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 493 /* Spread the IRQ affinity hints across online CPUs. Note that 494 * get_cpu_mask returns a mask with a permanent lifetime so 495 * it's safe to use as a hint for irq_update_affinity_hint. 496 */ 497 cpu = cpumask_local_spread(q_vector->v_idx, -1); 498 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu)); 499 } 500 501 return 0; 502 503 free_queue_irqs: 504 while (vector) { 505 vector--; 506 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 507 irq_set_affinity_notifier(irq_num, NULL); 508 irq_update_affinity_hint(irq_num, NULL); 509 free_irq(irq_num, &adapter->q_vectors[vector]); 510 } 511 return err; 512 } 513 514 /** 515 * iavf_request_misc_irq - Initialize MSI-X interrupts 516 * @adapter: board private structure 517 * 518 * Allocates MSI-X vector 0 and requests interrupts from the kernel. This 519 * vector is only for the admin queue, and stays active even when the netdev 520 * is closed. 521 **/ 522 static int iavf_request_misc_irq(struct iavf_adapter *adapter) 523 { 524 struct net_device *netdev = adapter->netdev; 525 int err; 526 527 snprintf(adapter->misc_vector_name, 528 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx", 529 dev_name(&adapter->pdev->dev)); 530 err = request_irq(adapter->msix_entries[0].vector, 531 &iavf_msix_aq, 0, 532 adapter->misc_vector_name, netdev); 533 if (err) { 534 dev_err(&adapter->pdev->dev, 535 "request_irq for %s failed: %d\n", 536 adapter->misc_vector_name, err); 537 free_irq(adapter->msix_entries[0].vector, netdev); 538 } 539 return err; 540 } 541 542 /** 543 * iavf_free_traffic_irqs - Free MSI-X interrupts 544 * @adapter: board private structure 545 * 546 * Frees all MSI-X vectors other than 0. 547 **/ 548 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter) 549 { 550 int vector, irq_num, q_vectors; 551 552 if (!adapter->msix_entries) 553 return; 554 555 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 556 557 for (vector = 0; vector < q_vectors; vector++) { 558 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 559 irq_set_affinity_notifier(irq_num, NULL); 560 irq_update_affinity_hint(irq_num, NULL); 561 free_irq(irq_num, &adapter->q_vectors[vector]); 562 } 563 } 564 565 /** 566 * iavf_free_misc_irq - Free MSI-X miscellaneous vector 567 * @adapter: board private structure 568 * 569 * Frees MSI-X vector 0. 570 **/ 571 static void iavf_free_misc_irq(struct iavf_adapter *adapter) 572 { 573 struct net_device *netdev = adapter->netdev; 574 575 if (!adapter->msix_entries) 576 return; 577 578 free_irq(adapter->msix_entries[0].vector, netdev); 579 } 580 581 /** 582 * iavf_configure_tx - Configure Transmit Unit after Reset 583 * @adapter: board private structure 584 * 585 * Configure the Tx unit of the MAC after a reset. 586 **/ 587 static void iavf_configure_tx(struct iavf_adapter *adapter) 588 { 589 struct iavf_hw *hw = &adapter->hw; 590 int i; 591 592 for (i = 0; i < adapter->num_active_queues; i++) 593 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i); 594 } 595 596 /** 597 * iavf_configure_rx - Configure Receive Unit after Reset 598 * @adapter: board private structure 599 * 600 * Configure the Rx unit of the MAC after a reset. 601 **/ 602 static void iavf_configure_rx(struct iavf_adapter *adapter) 603 { 604 unsigned int rx_buf_len = IAVF_RXBUFFER_2048; 605 struct iavf_hw *hw = &adapter->hw; 606 int i; 607 608 /* Legacy Rx will always default to a 2048 buffer size. */ 609 #if (PAGE_SIZE < 8192) 610 if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) { 611 struct net_device *netdev = adapter->netdev; 612 613 /* For jumbo frames on systems with 4K pages we have to use 614 * an order 1 page, so we might as well increase the size 615 * of our Rx buffer to make better use of the available space 616 */ 617 rx_buf_len = IAVF_RXBUFFER_3072; 618 619 /* We use a 1536 buffer size for configurations with 620 * standard Ethernet mtu. On x86 this gives us enough room 621 * for shared info and 192 bytes of padding. 622 */ 623 if (!IAVF_2K_TOO_SMALL_WITH_PADDING && 624 (netdev->mtu <= ETH_DATA_LEN)) 625 rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN; 626 } 627 #endif 628 629 for (i = 0; i < adapter->num_active_queues; i++) { 630 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i); 631 adapter->rx_rings[i].rx_buf_len = rx_buf_len; 632 633 if (adapter->flags & IAVF_FLAG_LEGACY_RX) 634 clear_ring_build_skb_enabled(&adapter->rx_rings[i]); 635 else 636 set_ring_build_skb_enabled(&adapter->rx_rings[i]); 637 } 638 } 639 640 /** 641 * iavf_find_vlan - Search filter list for specific vlan filter 642 * @adapter: board private structure 643 * @vlan: vlan tag 644 * 645 * Returns ptr to the filter object or NULL. Must be called while holding the 646 * mac_vlan_list_lock. 647 **/ 648 static struct 649 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, 650 struct iavf_vlan vlan) 651 { 652 struct iavf_vlan_filter *f; 653 654 list_for_each_entry(f, &adapter->vlan_filter_list, list) { 655 if (f->vlan.vid == vlan.vid && 656 f->vlan.tpid == vlan.tpid) 657 return f; 658 } 659 660 return NULL; 661 } 662 663 /** 664 * iavf_add_vlan - Add a vlan filter to the list 665 * @adapter: board private structure 666 * @vlan: VLAN tag 667 * 668 * Returns ptr to the filter object or NULL when no memory available. 669 **/ 670 static struct 671 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, 672 struct iavf_vlan vlan) 673 { 674 struct iavf_vlan_filter *f = NULL; 675 676 spin_lock_bh(&adapter->mac_vlan_list_lock); 677 678 f = iavf_find_vlan(adapter, vlan); 679 if (!f) { 680 f = kzalloc(sizeof(*f), GFP_ATOMIC); 681 if (!f) 682 goto clearout; 683 684 f->vlan = vlan; 685 686 list_add_tail(&f->list, &adapter->vlan_filter_list); 687 f->add = true; 688 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; 689 } 690 691 clearout: 692 spin_unlock_bh(&adapter->mac_vlan_list_lock); 693 return f; 694 } 695 696 /** 697 * iavf_del_vlan - Remove a vlan filter from the list 698 * @adapter: board private structure 699 * @vlan: VLAN tag 700 **/ 701 static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan) 702 { 703 struct iavf_vlan_filter *f; 704 705 spin_lock_bh(&adapter->mac_vlan_list_lock); 706 707 f = iavf_find_vlan(adapter, vlan); 708 if (f) { 709 f->remove = true; 710 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 711 } 712 713 spin_unlock_bh(&adapter->mac_vlan_list_lock); 714 } 715 716 /** 717 * iavf_restore_filters 718 * @adapter: board private structure 719 * 720 * Restore existing non MAC filters when VF netdev comes back up 721 **/ 722 static void iavf_restore_filters(struct iavf_adapter *adapter) 723 { 724 u16 vid; 725 726 /* re-add all VLAN filters */ 727 for_each_set_bit(vid, adapter->vsi.active_cvlans, VLAN_N_VID) 728 iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021Q)); 729 730 for_each_set_bit(vid, adapter->vsi.active_svlans, VLAN_N_VID) 731 iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021AD)); 732 } 733 734 /** 735 * iavf_get_num_vlans_added - get number of VLANs added 736 * @adapter: board private structure 737 */ 738 static u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter) 739 { 740 return bitmap_weight(adapter->vsi.active_cvlans, VLAN_N_VID) + 741 bitmap_weight(adapter->vsi.active_svlans, VLAN_N_VID); 742 } 743 744 /** 745 * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF 746 * @adapter: board private structure 747 * 748 * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN, 749 * do not impose a limit as that maintains current behavior and for 750 * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF. 751 **/ 752 static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter) 753 { 754 /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has 755 * never been a limit on the VF driver side 756 */ 757 if (VLAN_ALLOWED(adapter)) 758 return VLAN_N_VID; 759 else if (VLAN_V2_ALLOWED(adapter)) 760 return adapter->vlan_v2_caps.filtering.max_filters; 761 762 return 0; 763 } 764 765 /** 766 * iavf_max_vlans_added - check if maximum VLANs allowed already exist 767 * @adapter: board private structure 768 **/ 769 static bool iavf_max_vlans_added(struct iavf_adapter *adapter) 770 { 771 if (iavf_get_num_vlans_added(adapter) < 772 iavf_get_max_vlans_allowed(adapter)) 773 return false; 774 775 return true; 776 } 777 778 /** 779 * iavf_vlan_rx_add_vid - Add a VLAN filter to a device 780 * @netdev: network device struct 781 * @proto: unused protocol data 782 * @vid: VLAN tag 783 **/ 784 static int iavf_vlan_rx_add_vid(struct net_device *netdev, 785 __always_unused __be16 proto, u16 vid) 786 { 787 struct iavf_adapter *adapter = netdev_priv(netdev); 788 789 if (!VLAN_FILTERING_ALLOWED(adapter)) 790 return -EIO; 791 792 if (iavf_max_vlans_added(adapter)) { 793 netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n", 794 iavf_get_max_vlans_allowed(adapter)); 795 return -EIO; 796 } 797 798 if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)))) 799 return -ENOMEM; 800 801 if (proto == cpu_to_be16(ETH_P_8021Q)) 802 set_bit(vid, adapter->vsi.active_cvlans); 803 else 804 set_bit(vid, adapter->vsi.active_svlans); 805 806 return 0; 807 } 808 809 /** 810 * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device 811 * @netdev: network device struct 812 * @proto: unused protocol data 813 * @vid: VLAN tag 814 **/ 815 static int iavf_vlan_rx_kill_vid(struct net_device *netdev, 816 __always_unused __be16 proto, u16 vid) 817 { 818 struct iavf_adapter *adapter = netdev_priv(netdev); 819 820 iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))); 821 if (proto == cpu_to_be16(ETH_P_8021Q)) 822 clear_bit(vid, adapter->vsi.active_cvlans); 823 else 824 clear_bit(vid, adapter->vsi.active_svlans); 825 826 return 0; 827 } 828 829 /** 830 * iavf_find_filter - Search filter list for specific mac filter 831 * @adapter: board private structure 832 * @macaddr: the MAC address 833 * 834 * Returns ptr to the filter object or NULL. Must be called while holding the 835 * mac_vlan_list_lock. 836 **/ 837 static struct 838 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter, 839 const u8 *macaddr) 840 { 841 struct iavf_mac_filter *f; 842 843 if (!macaddr) 844 return NULL; 845 846 list_for_each_entry(f, &adapter->mac_filter_list, list) { 847 if (ether_addr_equal(macaddr, f->macaddr)) 848 return f; 849 } 850 return NULL; 851 } 852 853 /** 854 * iavf_add_filter - Add a mac filter to the filter list 855 * @adapter: board private structure 856 * @macaddr: the MAC address 857 * 858 * Returns ptr to the filter object or NULL when no memory available. 859 **/ 860 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, 861 const u8 *macaddr) 862 { 863 struct iavf_mac_filter *f; 864 865 if (!macaddr) 866 return NULL; 867 868 f = iavf_find_filter(adapter, macaddr); 869 if (!f) { 870 f = kzalloc(sizeof(*f), GFP_ATOMIC); 871 if (!f) 872 return f; 873 874 ether_addr_copy(f->macaddr, macaddr); 875 876 list_add_tail(&f->list, &adapter->mac_filter_list); 877 f->add = true; 878 f->is_new_mac = true; 879 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 880 } else { 881 f->remove = false; 882 } 883 884 return f; 885 } 886 887 /** 888 * iavf_set_mac - NDO callback to set port mac address 889 * @netdev: network interface device structure 890 * @p: pointer to an address structure 891 * 892 * Returns 0 on success, negative on failure 893 **/ 894 static int iavf_set_mac(struct net_device *netdev, void *p) 895 { 896 struct iavf_adapter *adapter = netdev_priv(netdev); 897 struct iavf_hw *hw = &adapter->hw; 898 struct iavf_mac_filter *f; 899 struct sockaddr *addr = p; 900 901 if (!is_valid_ether_addr(addr->sa_data)) 902 return -EADDRNOTAVAIL; 903 904 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) 905 return 0; 906 907 spin_lock_bh(&adapter->mac_vlan_list_lock); 908 909 f = iavf_find_filter(adapter, hw->mac.addr); 910 if (f) { 911 f->remove = true; 912 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 913 } 914 915 f = iavf_add_filter(adapter, addr->sa_data); 916 917 spin_unlock_bh(&adapter->mac_vlan_list_lock); 918 919 if (f) { 920 ether_addr_copy(hw->mac.addr, addr->sa_data); 921 } 922 923 return (f == NULL) ? -ENOMEM : 0; 924 } 925 926 /** 927 * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address 928 * @netdev: the netdevice 929 * @addr: address to add 930 * 931 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 932 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 933 */ 934 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr) 935 { 936 struct iavf_adapter *adapter = netdev_priv(netdev); 937 938 if (iavf_add_filter(adapter, addr)) 939 return 0; 940 else 941 return -ENOMEM; 942 } 943 944 /** 945 * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 946 * @netdev: the netdevice 947 * @addr: address to add 948 * 949 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 950 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 951 */ 952 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr) 953 { 954 struct iavf_adapter *adapter = netdev_priv(netdev); 955 struct iavf_mac_filter *f; 956 957 /* Under some circumstances, we might receive a request to delete 958 * our own device address from our uc list. Because we store the 959 * device address in the VSI's MAC/VLAN filter list, we need to ignore 960 * such requests and not delete our device address from this list. 961 */ 962 if (ether_addr_equal(addr, netdev->dev_addr)) 963 return 0; 964 965 f = iavf_find_filter(adapter, addr); 966 if (f) { 967 f->remove = true; 968 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 969 } 970 return 0; 971 } 972 973 /** 974 * iavf_set_rx_mode - NDO callback to set the netdev filters 975 * @netdev: network interface device structure 976 **/ 977 static void iavf_set_rx_mode(struct net_device *netdev) 978 { 979 struct iavf_adapter *adapter = netdev_priv(netdev); 980 981 spin_lock_bh(&adapter->mac_vlan_list_lock); 982 __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 983 __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 984 spin_unlock_bh(&adapter->mac_vlan_list_lock); 985 986 if (netdev->flags & IFF_PROMISC && 987 !(adapter->flags & IAVF_FLAG_PROMISC_ON)) 988 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC; 989 else if (!(netdev->flags & IFF_PROMISC) && 990 adapter->flags & IAVF_FLAG_PROMISC_ON) 991 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC; 992 993 if (netdev->flags & IFF_ALLMULTI && 994 !(adapter->flags & IAVF_FLAG_ALLMULTI_ON)) 995 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI; 996 else if (!(netdev->flags & IFF_ALLMULTI) && 997 adapter->flags & IAVF_FLAG_ALLMULTI_ON) 998 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI; 999 } 1000 1001 /** 1002 * iavf_napi_enable_all - enable NAPI on all queue vectors 1003 * @adapter: board private structure 1004 **/ 1005 static void iavf_napi_enable_all(struct iavf_adapter *adapter) 1006 { 1007 int q_idx; 1008 struct iavf_q_vector *q_vector; 1009 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1010 1011 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1012 struct napi_struct *napi; 1013 1014 q_vector = &adapter->q_vectors[q_idx]; 1015 napi = &q_vector->napi; 1016 napi_enable(napi); 1017 } 1018 } 1019 1020 /** 1021 * iavf_napi_disable_all - disable NAPI on all queue vectors 1022 * @adapter: board private structure 1023 **/ 1024 static void iavf_napi_disable_all(struct iavf_adapter *adapter) 1025 { 1026 int q_idx; 1027 struct iavf_q_vector *q_vector; 1028 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1029 1030 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1031 q_vector = &adapter->q_vectors[q_idx]; 1032 napi_disable(&q_vector->napi); 1033 } 1034 } 1035 1036 /** 1037 * iavf_configure - set up transmit and receive data structures 1038 * @adapter: board private structure 1039 **/ 1040 static void iavf_configure(struct iavf_adapter *adapter) 1041 { 1042 struct net_device *netdev = adapter->netdev; 1043 int i; 1044 1045 iavf_set_rx_mode(netdev); 1046 1047 iavf_configure_tx(adapter); 1048 iavf_configure_rx(adapter); 1049 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES; 1050 1051 for (i = 0; i < adapter->num_active_queues; i++) { 1052 struct iavf_ring *ring = &adapter->rx_rings[i]; 1053 1054 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring)); 1055 } 1056 } 1057 1058 /** 1059 * iavf_up_complete - Finish the last steps of bringing up a connection 1060 * @adapter: board private structure 1061 * 1062 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. 1063 **/ 1064 static void iavf_up_complete(struct iavf_adapter *adapter) 1065 { 1066 iavf_change_state(adapter, __IAVF_RUNNING); 1067 clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1068 1069 iavf_napi_enable_all(adapter); 1070 1071 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES; 1072 if (CLIENT_ENABLED(adapter)) 1073 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN; 1074 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 1075 } 1076 1077 /** 1078 * iavf_down - Shutdown the connection processing 1079 * @adapter: board private structure 1080 * 1081 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. 1082 **/ 1083 void iavf_down(struct iavf_adapter *adapter) 1084 { 1085 struct net_device *netdev = adapter->netdev; 1086 struct iavf_vlan_filter *vlf; 1087 struct iavf_cloud_filter *cf; 1088 struct iavf_fdir_fltr *fdir; 1089 struct iavf_mac_filter *f; 1090 struct iavf_adv_rss *rss; 1091 1092 if (adapter->state <= __IAVF_DOWN_PENDING) 1093 return; 1094 1095 netif_carrier_off(netdev); 1096 netif_tx_disable(netdev); 1097 adapter->link_up = false; 1098 iavf_napi_disable_all(adapter); 1099 iavf_irq_disable(adapter); 1100 1101 spin_lock_bh(&adapter->mac_vlan_list_lock); 1102 1103 /* clear the sync flag on all filters */ 1104 __dev_uc_unsync(adapter->netdev, NULL); 1105 __dev_mc_unsync(adapter->netdev, NULL); 1106 1107 /* remove all MAC filters */ 1108 list_for_each_entry(f, &adapter->mac_filter_list, list) { 1109 f->remove = true; 1110 } 1111 1112 /* remove all VLAN filters */ 1113 list_for_each_entry(vlf, &adapter->vlan_filter_list, list) { 1114 vlf->remove = true; 1115 } 1116 1117 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1118 1119 /* remove all cloud filters */ 1120 spin_lock_bh(&adapter->cloud_filter_list_lock); 1121 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 1122 cf->del = true; 1123 } 1124 spin_unlock_bh(&adapter->cloud_filter_list_lock); 1125 1126 /* remove all Flow Director filters */ 1127 spin_lock_bh(&adapter->fdir_fltr_lock); 1128 list_for_each_entry(fdir, &adapter->fdir_list_head, list) { 1129 fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST; 1130 } 1131 spin_unlock_bh(&adapter->fdir_fltr_lock); 1132 1133 /* remove all advance RSS configuration */ 1134 spin_lock_bh(&adapter->adv_rss_lock); 1135 list_for_each_entry(rss, &adapter->adv_rss_list_head, list) 1136 rss->state = IAVF_ADV_RSS_DEL_REQUEST; 1137 spin_unlock_bh(&adapter->adv_rss_lock); 1138 1139 if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) && 1140 adapter->state != __IAVF_RESETTING) { 1141 /* cancel any current operation */ 1142 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1143 /* Schedule operations to close down the HW. Don't wait 1144 * here for this to complete. The watchdog is still running 1145 * and it will take care of this. 1146 */ 1147 adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER; 1148 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 1149 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 1150 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER; 1151 adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG; 1152 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES; 1153 } 1154 1155 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 1156 } 1157 1158 /** 1159 * iavf_acquire_msix_vectors - Setup the MSIX capability 1160 * @adapter: board private structure 1161 * @vectors: number of vectors to request 1162 * 1163 * Work with the OS to set up the MSIX vectors needed. 1164 * 1165 * Returns 0 on success, negative on failure 1166 **/ 1167 static int 1168 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors) 1169 { 1170 int err, vector_threshold; 1171 1172 /* We'll want at least 3 (vector_threshold): 1173 * 0) Other (Admin Queue and link, mostly) 1174 * 1) TxQ[0] Cleanup 1175 * 2) RxQ[0] Cleanup 1176 */ 1177 vector_threshold = MIN_MSIX_COUNT; 1178 1179 /* The more we get, the more we will assign to Tx/Rx Cleanup 1180 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1181 * Right now, we simply care about how many we'll get; we'll 1182 * set them up later while requesting irq's. 1183 */ 1184 err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 1185 vector_threshold, vectors); 1186 if (err < 0) { 1187 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n"); 1188 kfree(adapter->msix_entries); 1189 adapter->msix_entries = NULL; 1190 return err; 1191 } 1192 1193 /* Adjust for only the vectors we'll use, which is minimum 1194 * of max_msix_q_vectors + NONQ_VECS, or the number of 1195 * vectors we were allocated. 1196 */ 1197 adapter->num_msix_vectors = err; 1198 return 0; 1199 } 1200 1201 /** 1202 * iavf_free_queues - Free memory for all rings 1203 * @adapter: board private structure to initialize 1204 * 1205 * Free all of the memory associated with queue pairs. 1206 **/ 1207 static void iavf_free_queues(struct iavf_adapter *adapter) 1208 { 1209 if (!adapter->vsi_res) 1210 return; 1211 adapter->num_active_queues = 0; 1212 kfree(adapter->tx_rings); 1213 adapter->tx_rings = NULL; 1214 kfree(adapter->rx_rings); 1215 adapter->rx_rings = NULL; 1216 } 1217 1218 /** 1219 * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload 1220 * @adapter: board private structure 1221 * 1222 * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or 1223 * stripped in certain descriptor fields. Instead of checking the offload 1224 * capability bits in the hot path, cache the location the ring specific 1225 * flags. 1226 */ 1227 void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter) 1228 { 1229 int i; 1230 1231 for (i = 0; i < adapter->num_active_queues; i++) { 1232 struct iavf_ring *tx_ring = &adapter->tx_rings[i]; 1233 struct iavf_ring *rx_ring = &adapter->rx_rings[i]; 1234 1235 /* prevent multiple L2TAG bits being set after VFR */ 1236 tx_ring->flags &= 1237 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 | 1238 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2); 1239 rx_ring->flags &= 1240 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 | 1241 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2); 1242 1243 if (VLAN_ALLOWED(adapter)) { 1244 tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1245 rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1246 } else if (VLAN_V2_ALLOWED(adapter)) { 1247 struct virtchnl_vlan_supported_caps *stripping_support; 1248 struct virtchnl_vlan_supported_caps *insertion_support; 1249 1250 stripping_support = 1251 &adapter->vlan_v2_caps.offloads.stripping_support; 1252 insertion_support = 1253 &adapter->vlan_v2_caps.offloads.insertion_support; 1254 1255 if (stripping_support->outer) { 1256 if (stripping_support->outer & 1257 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1258 rx_ring->flags |= 1259 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1260 else if (stripping_support->outer & 1261 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) 1262 rx_ring->flags |= 1263 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2; 1264 } else if (stripping_support->inner) { 1265 if (stripping_support->inner & 1266 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1267 rx_ring->flags |= 1268 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1269 else if (stripping_support->inner & 1270 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) 1271 rx_ring->flags |= 1272 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2; 1273 } 1274 1275 if (insertion_support->outer) { 1276 if (insertion_support->outer & 1277 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1278 tx_ring->flags |= 1279 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1280 else if (insertion_support->outer & 1281 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) 1282 tx_ring->flags |= 1283 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2; 1284 } else if (insertion_support->inner) { 1285 if (insertion_support->inner & 1286 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1287 tx_ring->flags |= 1288 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1289 else if (insertion_support->inner & 1290 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) 1291 tx_ring->flags |= 1292 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2; 1293 } 1294 } 1295 } 1296 } 1297 1298 /** 1299 * iavf_alloc_queues - Allocate memory for all rings 1300 * @adapter: board private structure to initialize 1301 * 1302 * We allocate one ring per queue at run-time since we don't know the 1303 * number of queues at compile-time. The polling_netdev array is 1304 * intended for Multiqueue, but should work fine with a single queue. 1305 **/ 1306 static int iavf_alloc_queues(struct iavf_adapter *adapter) 1307 { 1308 int i, num_active_queues; 1309 1310 /* If we're in reset reallocating queues we don't actually know yet for 1311 * certain the PF gave us the number of queues we asked for but we'll 1312 * assume it did. Once basic reset is finished we'll confirm once we 1313 * start negotiating config with PF. 1314 */ 1315 if (adapter->num_req_queues) 1316 num_active_queues = adapter->num_req_queues; 1317 else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1318 adapter->num_tc) 1319 num_active_queues = adapter->ch_config.total_qps; 1320 else 1321 num_active_queues = min_t(int, 1322 adapter->vsi_res->num_queue_pairs, 1323 (int)(num_online_cpus())); 1324 1325 1326 adapter->tx_rings = kcalloc(num_active_queues, 1327 sizeof(struct iavf_ring), GFP_KERNEL); 1328 if (!adapter->tx_rings) 1329 goto err_out; 1330 adapter->rx_rings = kcalloc(num_active_queues, 1331 sizeof(struct iavf_ring), GFP_KERNEL); 1332 if (!adapter->rx_rings) 1333 goto err_out; 1334 1335 for (i = 0; i < num_active_queues; i++) { 1336 struct iavf_ring *tx_ring; 1337 struct iavf_ring *rx_ring; 1338 1339 tx_ring = &adapter->tx_rings[i]; 1340 1341 tx_ring->queue_index = i; 1342 tx_ring->netdev = adapter->netdev; 1343 tx_ring->dev = &adapter->pdev->dev; 1344 tx_ring->count = adapter->tx_desc_count; 1345 tx_ring->itr_setting = IAVF_ITR_TX_DEF; 1346 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE) 1347 tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR; 1348 1349 rx_ring = &adapter->rx_rings[i]; 1350 rx_ring->queue_index = i; 1351 rx_ring->netdev = adapter->netdev; 1352 rx_ring->dev = &adapter->pdev->dev; 1353 rx_ring->count = adapter->rx_desc_count; 1354 rx_ring->itr_setting = IAVF_ITR_RX_DEF; 1355 } 1356 1357 adapter->num_active_queues = num_active_queues; 1358 1359 iavf_set_queue_vlan_tag_loc(adapter); 1360 1361 return 0; 1362 1363 err_out: 1364 iavf_free_queues(adapter); 1365 return -ENOMEM; 1366 } 1367 1368 /** 1369 * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported 1370 * @adapter: board private structure to initialize 1371 * 1372 * Attempt to configure the interrupts using the best available 1373 * capabilities of the hardware and the kernel. 1374 **/ 1375 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter) 1376 { 1377 int vector, v_budget; 1378 int pairs = 0; 1379 int err = 0; 1380 1381 if (!adapter->vsi_res) { 1382 err = -EIO; 1383 goto out; 1384 } 1385 pairs = adapter->num_active_queues; 1386 1387 /* It's easy to be greedy for MSI-X vectors, but it really doesn't do 1388 * us much good if we have more vectors than CPUs. However, we already 1389 * limit the total number of queues by the number of CPUs so we do not 1390 * need any further limiting here. 1391 */ 1392 v_budget = min_t(int, pairs + NONQ_VECS, 1393 (int)adapter->vf_res->max_vectors); 1394 1395 adapter->msix_entries = kcalloc(v_budget, 1396 sizeof(struct msix_entry), GFP_KERNEL); 1397 if (!adapter->msix_entries) { 1398 err = -ENOMEM; 1399 goto out; 1400 } 1401 1402 for (vector = 0; vector < v_budget; vector++) 1403 adapter->msix_entries[vector].entry = vector; 1404 1405 err = iavf_acquire_msix_vectors(adapter, v_budget); 1406 1407 out: 1408 netif_set_real_num_rx_queues(adapter->netdev, pairs); 1409 netif_set_real_num_tx_queues(adapter->netdev, pairs); 1410 return err; 1411 } 1412 1413 /** 1414 * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands 1415 * @adapter: board private structure 1416 * 1417 * Return 0 on success, negative on failure 1418 **/ 1419 static int iavf_config_rss_aq(struct iavf_adapter *adapter) 1420 { 1421 struct iavf_aqc_get_set_rss_key_data *rss_key = 1422 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key; 1423 struct iavf_hw *hw = &adapter->hw; 1424 int ret = 0; 1425 1426 if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { 1427 /* bail because we already have a command pending */ 1428 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n", 1429 adapter->current_op); 1430 return -EBUSY; 1431 } 1432 1433 ret = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key); 1434 if (ret) { 1435 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n", 1436 iavf_stat_str(hw, ret), 1437 iavf_aq_str(hw, hw->aq.asq_last_status)); 1438 return ret; 1439 1440 } 1441 1442 ret = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false, 1443 adapter->rss_lut, adapter->rss_lut_size); 1444 if (ret) { 1445 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n", 1446 iavf_stat_str(hw, ret), 1447 iavf_aq_str(hw, hw->aq.asq_last_status)); 1448 } 1449 1450 return ret; 1451 1452 } 1453 1454 /** 1455 * iavf_config_rss_reg - Configure RSS keys and lut by writing registers 1456 * @adapter: board private structure 1457 * 1458 * Returns 0 on success, negative on failure 1459 **/ 1460 static int iavf_config_rss_reg(struct iavf_adapter *adapter) 1461 { 1462 struct iavf_hw *hw = &adapter->hw; 1463 u32 *dw; 1464 u16 i; 1465 1466 dw = (u32 *)adapter->rss_key; 1467 for (i = 0; i <= adapter->rss_key_size / 4; i++) 1468 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]); 1469 1470 dw = (u32 *)adapter->rss_lut; 1471 for (i = 0; i <= adapter->rss_lut_size / 4; i++) 1472 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]); 1473 1474 iavf_flush(hw); 1475 1476 return 0; 1477 } 1478 1479 /** 1480 * iavf_config_rss - Configure RSS keys and lut 1481 * @adapter: board private structure 1482 * 1483 * Returns 0 on success, negative on failure 1484 **/ 1485 int iavf_config_rss(struct iavf_adapter *adapter) 1486 { 1487 1488 if (RSS_PF(adapter)) { 1489 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT | 1490 IAVF_FLAG_AQ_SET_RSS_KEY; 1491 return 0; 1492 } else if (RSS_AQ(adapter)) { 1493 return iavf_config_rss_aq(adapter); 1494 } else { 1495 return iavf_config_rss_reg(adapter); 1496 } 1497 } 1498 1499 /** 1500 * iavf_fill_rss_lut - Fill the lut with default values 1501 * @adapter: board private structure 1502 **/ 1503 static void iavf_fill_rss_lut(struct iavf_adapter *adapter) 1504 { 1505 u16 i; 1506 1507 for (i = 0; i < adapter->rss_lut_size; i++) 1508 adapter->rss_lut[i] = i % adapter->num_active_queues; 1509 } 1510 1511 /** 1512 * iavf_init_rss - Prepare for RSS 1513 * @adapter: board private structure 1514 * 1515 * Return 0 on success, negative on failure 1516 **/ 1517 static int iavf_init_rss(struct iavf_adapter *adapter) 1518 { 1519 struct iavf_hw *hw = &adapter->hw; 1520 int ret; 1521 1522 if (!RSS_PF(adapter)) { 1523 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */ 1524 if (adapter->vf_res->vf_cap_flags & 1525 VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2) 1526 adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED; 1527 else 1528 adapter->hena = IAVF_DEFAULT_RSS_HENA; 1529 1530 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena); 1531 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32)); 1532 } 1533 1534 iavf_fill_rss_lut(adapter); 1535 netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size); 1536 ret = iavf_config_rss(adapter); 1537 1538 return ret; 1539 } 1540 1541 /** 1542 * iavf_alloc_q_vectors - Allocate memory for interrupt vectors 1543 * @adapter: board private structure to initialize 1544 * 1545 * We allocate one q_vector per queue interrupt. If allocation fails we 1546 * return -ENOMEM. 1547 **/ 1548 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter) 1549 { 1550 int q_idx = 0, num_q_vectors; 1551 struct iavf_q_vector *q_vector; 1552 1553 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1554 adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector), 1555 GFP_KERNEL); 1556 if (!adapter->q_vectors) 1557 return -ENOMEM; 1558 1559 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1560 q_vector = &adapter->q_vectors[q_idx]; 1561 q_vector->adapter = adapter; 1562 q_vector->vsi = &adapter->vsi; 1563 q_vector->v_idx = q_idx; 1564 q_vector->reg_idx = q_idx; 1565 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 1566 netif_napi_add(adapter->netdev, &q_vector->napi, 1567 iavf_napi_poll, NAPI_POLL_WEIGHT); 1568 } 1569 1570 return 0; 1571 } 1572 1573 /** 1574 * iavf_free_q_vectors - Free memory allocated for interrupt vectors 1575 * @adapter: board private structure to initialize 1576 * 1577 * This function frees the memory allocated to the q_vectors. In addition if 1578 * NAPI is enabled it will delete any references to the NAPI struct prior 1579 * to freeing the q_vector. 1580 **/ 1581 static void iavf_free_q_vectors(struct iavf_adapter *adapter) 1582 { 1583 int q_idx, num_q_vectors; 1584 int napi_vectors; 1585 1586 if (!adapter->q_vectors) 1587 return; 1588 1589 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1590 napi_vectors = adapter->num_active_queues; 1591 1592 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1593 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx]; 1594 1595 if (q_idx < napi_vectors) 1596 netif_napi_del(&q_vector->napi); 1597 } 1598 kfree(adapter->q_vectors); 1599 adapter->q_vectors = NULL; 1600 } 1601 1602 /** 1603 * iavf_reset_interrupt_capability - Reset MSIX setup 1604 * @adapter: board private structure 1605 * 1606 **/ 1607 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter) 1608 { 1609 if (!adapter->msix_entries) 1610 return; 1611 1612 pci_disable_msix(adapter->pdev); 1613 kfree(adapter->msix_entries); 1614 adapter->msix_entries = NULL; 1615 } 1616 1617 /** 1618 * iavf_init_interrupt_scheme - Determine if MSIX is supported and init 1619 * @adapter: board private structure to initialize 1620 * 1621 **/ 1622 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter) 1623 { 1624 int err; 1625 1626 err = iavf_alloc_queues(adapter); 1627 if (err) { 1628 dev_err(&adapter->pdev->dev, 1629 "Unable to allocate memory for queues\n"); 1630 goto err_alloc_queues; 1631 } 1632 1633 rtnl_lock(); 1634 err = iavf_set_interrupt_capability(adapter); 1635 rtnl_unlock(); 1636 if (err) { 1637 dev_err(&adapter->pdev->dev, 1638 "Unable to setup interrupt capabilities\n"); 1639 goto err_set_interrupt; 1640 } 1641 1642 err = iavf_alloc_q_vectors(adapter); 1643 if (err) { 1644 dev_err(&adapter->pdev->dev, 1645 "Unable to allocate memory for queue vectors\n"); 1646 goto err_alloc_q_vectors; 1647 } 1648 1649 /* If we've made it so far while ADq flag being ON, then we haven't 1650 * bailed out anywhere in middle. And ADq isn't just enabled but actual 1651 * resources have been allocated in the reset path. 1652 * Now we can truly claim that ADq is enabled. 1653 */ 1654 if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1655 adapter->num_tc) 1656 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created", 1657 adapter->num_tc); 1658 1659 dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u", 1660 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled", 1661 adapter->num_active_queues); 1662 1663 return 0; 1664 err_alloc_q_vectors: 1665 iavf_reset_interrupt_capability(adapter); 1666 err_set_interrupt: 1667 iavf_free_queues(adapter); 1668 err_alloc_queues: 1669 return err; 1670 } 1671 1672 /** 1673 * iavf_free_rss - Free memory used by RSS structs 1674 * @adapter: board private structure 1675 **/ 1676 static void iavf_free_rss(struct iavf_adapter *adapter) 1677 { 1678 kfree(adapter->rss_key); 1679 adapter->rss_key = NULL; 1680 1681 kfree(adapter->rss_lut); 1682 adapter->rss_lut = NULL; 1683 } 1684 1685 /** 1686 * iavf_reinit_interrupt_scheme - Reallocate queues and vectors 1687 * @adapter: board private structure 1688 * 1689 * Returns 0 on success, negative on failure 1690 **/ 1691 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter) 1692 { 1693 struct net_device *netdev = adapter->netdev; 1694 int err; 1695 1696 if (netif_running(netdev)) 1697 iavf_free_traffic_irqs(adapter); 1698 iavf_free_misc_irq(adapter); 1699 iavf_reset_interrupt_capability(adapter); 1700 iavf_free_q_vectors(adapter); 1701 iavf_free_queues(adapter); 1702 1703 err = iavf_init_interrupt_scheme(adapter); 1704 if (err) 1705 goto err; 1706 1707 netif_tx_stop_all_queues(netdev); 1708 1709 err = iavf_request_misc_irq(adapter); 1710 if (err) 1711 goto err; 1712 1713 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1714 1715 iavf_map_rings_to_vectors(adapter); 1716 err: 1717 return err; 1718 } 1719 1720 /** 1721 * iavf_process_aq_command - process aq_required flags 1722 * and sends aq command 1723 * @adapter: pointer to iavf adapter structure 1724 * 1725 * Returns 0 on success 1726 * Returns error code if no command was sent 1727 * or error code if the command failed. 1728 **/ 1729 static int iavf_process_aq_command(struct iavf_adapter *adapter) 1730 { 1731 if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG) 1732 return iavf_send_vf_config_msg(adapter); 1733 if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS) 1734 return iavf_send_vf_offload_vlan_v2_msg(adapter); 1735 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) { 1736 iavf_disable_queues(adapter); 1737 return 0; 1738 } 1739 1740 if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) { 1741 iavf_map_queues(adapter); 1742 return 0; 1743 } 1744 1745 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) { 1746 iavf_add_ether_addrs(adapter); 1747 return 0; 1748 } 1749 1750 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) { 1751 iavf_add_vlans(adapter); 1752 return 0; 1753 } 1754 1755 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) { 1756 iavf_del_ether_addrs(adapter); 1757 return 0; 1758 } 1759 1760 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) { 1761 iavf_del_vlans(adapter); 1762 return 0; 1763 } 1764 1765 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) { 1766 iavf_enable_vlan_stripping(adapter); 1767 return 0; 1768 } 1769 1770 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) { 1771 iavf_disable_vlan_stripping(adapter); 1772 return 0; 1773 } 1774 1775 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) { 1776 iavf_configure_queues(adapter); 1777 return 0; 1778 } 1779 1780 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) { 1781 iavf_enable_queues(adapter); 1782 return 0; 1783 } 1784 1785 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) { 1786 /* This message goes straight to the firmware, not the 1787 * PF, so we don't have to set current_op as we will 1788 * not get a response through the ARQ. 1789 */ 1790 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS; 1791 return 0; 1792 } 1793 if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) { 1794 iavf_get_hena(adapter); 1795 return 0; 1796 } 1797 if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) { 1798 iavf_set_hena(adapter); 1799 return 0; 1800 } 1801 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) { 1802 iavf_set_rss_key(adapter); 1803 return 0; 1804 } 1805 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) { 1806 iavf_set_rss_lut(adapter); 1807 return 0; 1808 } 1809 1810 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) { 1811 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC | 1812 FLAG_VF_MULTICAST_PROMISC); 1813 return 0; 1814 } 1815 1816 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) { 1817 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC); 1818 return 0; 1819 } 1820 if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) || 1821 (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) { 1822 iavf_set_promiscuous(adapter, 0); 1823 return 0; 1824 } 1825 1826 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) { 1827 iavf_enable_channels(adapter); 1828 return 0; 1829 } 1830 1831 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) { 1832 iavf_disable_channels(adapter); 1833 return 0; 1834 } 1835 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) { 1836 iavf_add_cloud_filter(adapter); 1837 return 0; 1838 } 1839 1840 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) { 1841 iavf_del_cloud_filter(adapter); 1842 return 0; 1843 } 1844 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) { 1845 iavf_del_cloud_filter(adapter); 1846 return 0; 1847 } 1848 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) { 1849 iavf_add_cloud_filter(adapter); 1850 return 0; 1851 } 1852 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) { 1853 iavf_add_fdir_filter(adapter); 1854 return IAVF_SUCCESS; 1855 } 1856 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) { 1857 iavf_del_fdir_filter(adapter); 1858 return IAVF_SUCCESS; 1859 } 1860 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) { 1861 iavf_add_adv_rss_cfg(adapter); 1862 return 0; 1863 } 1864 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) { 1865 iavf_del_adv_rss_cfg(adapter); 1866 return 0; 1867 } 1868 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) { 1869 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q); 1870 return 0; 1871 } 1872 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) { 1873 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD); 1874 return 0; 1875 } 1876 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) { 1877 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q); 1878 return 0; 1879 } 1880 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) { 1881 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD); 1882 return 0; 1883 } 1884 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) { 1885 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q); 1886 return 0; 1887 } 1888 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) { 1889 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD); 1890 return 0; 1891 } 1892 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) { 1893 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q); 1894 return 0; 1895 } 1896 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) { 1897 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD); 1898 return 0; 1899 } 1900 1901 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) { 1902 iavf_request_stats(adapter); 1903 return 0; 1904 } 1905 1906 return -EAGAIN; 1907 } 1908 1909 /** 1910 * iavf_set_vlan_offload_features - set VLAN offload configuration 1911 * @adapter: board private structure 1912 * @prev_features: previous features used for comparison 1913 * @features: updated features used for configuration 1914 * 1915 * Set the aq_required bit(s) based on the requested features passed in to 1916 * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule 1917 * the watchdog if any changes are requested to expedite the request via 1918 * virtchnl. 1919 **/ 1920 void 1921 iavf_set_vlan_offload_features(struct iavf_adapter *adapter, 1922 netdev_features_t prev_features, 1923 netdev_features_t features) 1924 { 1925 bool enable_stripping = true, enable_insertion = true; 1926 u16 vlan_ethertype = 0; 1927 u64 aq_required = 0; 1928 1929 /* keep cases separate because one ethertype for offloads can be 1930 * disabled at the same time as another is disabled, so check for an 1931 * enabled ethertype first, then check for disabled. Default to 1932 * ETH_P_8021Q so an ethertype is specified if disabling insertion and 1933 * stripping. 1934 */ 1935 if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 1936 vlan_ethertype = ETH_P_8021AD; 1937 else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 1938 vlan_ethertype = ETH_P_8021Q; 1939 else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 1940 vlan_ethertype = ETH_P_8021AD; 1941 else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 1942 vlan_ethertype = ETH_P_8021Q; 1943 else 1944 vlan_ethertype = ETH_P_8021Q; 1945 1946 if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX))) 1947 enable_stripping = false; 1948 if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX))) 1949 enable_insertion = false; 1950 1951 if (VLAN_ALLOWED(adapter)) { 1952 /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN 1953 * stripping via virtchnl. VLAN insertion can be toggled on the 1954 * netdev, but it doesn't require a virtchnl message 1955 */ 1956 if (enable_stripping) 1957 aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING; 1958 else 1959 aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING; 1960 1961 } else if (VLAN_V2_ALLOWED(adapter)) { 1962 switch (vlan_ethertype) { 1963 case ETH_P_8021Q: 1964 if (enable_stripping) 1965 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING; 1966 else 1967 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING; 1968 1969 if (enable_insertion) 1970 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION; 1971 else 1972 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION; 1973 break; 1974 case ETH_P_8021AD: 1975 if (enable_stripping) 1976 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING; 1977 else 1978 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING; 1979 1980 if (enable_insertion) 1981 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION; 1982 else 1983 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION; 1984 break; 1985 } 1986 } 1987 1988 if (aq_required) { 1989 adapter->aq_required |= aq_required; 1990 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 1991 } 1992 } 1993 1994 /** 1995 * iavf_startup - first step of driver startup 1996 * @adapter: board private structure 1997 * 1998 * Function process __IAVF_STARTUP driver state. 1999 * When success the state is changed to __IAVF_INIT_VERSION_CHECK 2000 * when fails the state is changed to __IAVF_INIT_FAILED 2001 **/ 2002 static void iavf_startup(struct iavf_adapter *adapter) 2003 { 2004 struct pci_dev *pdev = adapter->pdev; 2005 struct iavf_hw *hw = &adapter->hw; 2006 int err; 2007 2008 WARN_ON(adapter->state != __IAVF_STARTUP); 2009 2010 /* driver loaded, probe complete */ 2011 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 2012 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2013 err = iavf_set_mac_type(hw); 2014 if (err) { 2015 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", err); 2016 goto err; 2017 } 2018 2019 err = iavf_check_reset_complete(hw); 2020 if (err) { 2021 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n", 2022 err); 2023 goto err; 2024 } 2025 hw->aq.num_arq_entries = IAVF_AQ_LEN; 2026 hw->aq.num_asq_entries = IAVF_AQ_LEN; 2027 hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 2028 hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 2029 2030 err = iavf_init_adminq(hw); 2031 if (err) { 2032 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n", err); 2033 goto err; 2034 } 2035 err = iavf_send_api_ver(adapter); 2036 if (err) { 2037 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", err); 2038 iavf_shutdown_adminq(hw); 2039 goto err; 2040 } 2041 iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK); 2042 return; 2043 err: 2044 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2045 } 2046 2047 /** 2048 * iavf_init_version_check - second step of driver startup 2049 * @adapter: board private structure 2050 * 2051 * Function process __IAVF_INIT_VERSION_CHECK driver state. 2052 * When success the state is changed to __IAVF_INIT_GET_RESOURCES 2053 * when fails the state is changed to __IAVF_INIT_FAILED 2054 **/ 2055 static void iavf_init_version_check(struct iavf_adapter *adapter) 2056 { 2057 struct pci_dev *pdev = adapter->pdev; 2058 struct iavf_hw *hw = &adapter->hw; 2059 int err = -EAGAIN; 2060 2061 WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK); 2062 2063 if (!iavf_asq_done(hw)) { 2064 dev_err(&pdev->dev, "Admin queue command never completed\n"); 2065 iavf_shutdown_adminq(hw); 2066 iavf_change_state(adapter, __IAVF_STARTUP); 2067 goto err; 2068 } 2069 2070 /* aq msg sent, awaiting reply */ 2071 err = iavf_verify_api_ver(adapter); 2072 if (err) { 2073 if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK) 2074 err = iavf_send_api_ver(adapter); 2075 else 2076 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n", 2077 adapter->pf_version.major, 2078 adapter->pf_version.minor, 2079 VIRTCHNL_VERSION_MAJOR, 2080 VIRTCHNL_VERSION_MINOR); 2081 goto err; 2082 } 2083 err = iavf_send_vf_config_msg(adapter); 2084 if (err) { 2085 dev_err(&pdev->dev, "Unable to send config request (%d)\n", 2086 err); 2087 goto err; 2088 } 2089 iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES); 2090 return; 2091 err: 2092 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2093 } 2094 2095 /** 2096 * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES 2097 * @adapter: board private structure 2098 */ 2099 int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter) 2100 { 2101 int i, num_req_queues = adapter->num_req_queues; 2102 struct iavf_vsi *vsi = &adapter->vsi; 2103 2104 for (i = 0; i < adapter->vf_res->num_vsis; i++) { 2105 if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 2106 adapter->vsi_res = &adapter->vf_res->vsi_res[i]; 2107 } 2108 if (!adapter->vsi_res) { 2109 dev_err(&adapter->pdev->dev, "No LAN VSI found\n"); 2110 return -ENODEV; 2111 } 2112 2113 if (num_req_queues && 2114 num_req_queues > adapter->vsi_res->num_queue_pairs) { 2115 /* Problem. The PF gave us fewer queues than what we had 2116 * negotiated in our request. Need a reset to see if we can't 2117 * get back to a working state. 2118 */ 2119 dev_err(&adapter->pdev->dev, 2120 "Requested %d queues, but PF only gave us %d.\n", 2121 num_req_queues, 2122 adapter->vsi_res->num_queue_pairs); 2123 adapter->flags |= IAVF_FLAG_REINIT_ITR_NEEDED; 2124 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs; 2125 iavf_schedule_reset(adapter); 2126 2127 return -EAGAIN; 2128 } 2129 adapter->num_req_queues = 0; 2130 adapter->vsi.id = adapter->vsi_res->vsi_id; 2131 2132 adapter->vsi.back = adapter; 2133 adapter->vsi.base_vector = 1; 2134 adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK; 2135 vsi->netdev = adapter->netdev; 2136 vsi->qs_handle = adapter->vsi_res->qset_handle; 2137 if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2138 adapter->rss_key_size = adapter->vf_res->rss_key_size; 2139 adapter->rss_lut_size = adapter->vf_res->rss_lut_size; 2140 } else { 2141 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE; 2142 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE; 2143 } 2144 2145 return 0; 2146 } 2147 2148 /** 2149 * iavf_init_get_resources - third step of driver startup 2150 * @adapter: board private structure 2151 * 2152 * Function process __IAVF_INIT_GET_RESOURCES driver state and 2153 * finishes driver initialization procedure. 2154 * When success the state is changed to __IAVF_DOWN 2155 * when fails the state is changed to __IAVF_INIT_FAILED 2156 **/ 2157 static void iavf_init_get_resources(struct iavf_adapter *adapter) 2158 { 2159 struct pci_dev *pdev = adapter->pdev; 2160 struct iavf_hw *hw = &adapter->hw; 2161 int err; 2162 2163 WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES); 2164 /* aq msg sent, awaiting reply */ 2165 if (!adapter->vf_res) { 2166 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE, 2167 GFP_KERNEL); 2168 if (!adapter->vf_res) { 2169 err = -ENOMEM; 2170 goto err; 2171 } 2172 } 2173 err = iavf_get_vf_config(adapter); 2174 if (err == IAVF_ERR_ADMIN_QUEUE_NO_WORK) { 2175 err = iavf_send_vf_config_msg(adapter); 2176 goto err_alloc; 2177 } else if (err == IAVF_ERR_PARAM) { 2178 /* We only get ERR_PARAM if the device is in a very bad 2179 * state or if we've been disabled for previous bad 2180 * behavior. Either way, we're done now. 2181 */ 2182 iavf_shutdown_adminq(hw); 2183 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n"); 2184 return; 2185 } 2186 if (err) { 2187 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err); 2188 goto err_alloc; 2189 } 2190 2191 err = iavf_parse_vf_resource_msg(adapter); 2192 if (err) 2193 goto err_alloc; 2194 2195 err = iavf_send_vf_offload_vlan_v2_msg(adapter); 2196 if (err == -EOPNOTSUPP) { 2197 /* underlying PF doesn't support VIRTCHNL_VF_OFFLOAD_VLAN_V2, so 2198 * go directly to finishing initialization 2199 */ 2200 iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER); 2201 return; 2202 } else if (err) { 2203 dev_err(&pdev->dev, "Unable to send offload vlan v2 request (%d)\n", 2204 err); 2205 goto err_alloc; 2206 } 2207 2208 /* underlying PF supports VIRTCHNL_VF_OFFLOAD_VLAN_V2, so update the 2209 * state accordingly 2210 */ 2211 iavf_change_state(adapter, __IAVF_INIT_GET_OFFLOAD_VLAN_V2_CAPS); 2212 return; 2213 2214 err_alloc: 2215 kfree(adapter->vf_res); 2216 adapter->vf_res = NULL; 2217 err: 2218 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2219 } 2220 2221 /** 2222 * iavf_init_get_offload_vlan_v2_caps - part of driver startup 2223 * @adapter: board private structure 2224 * 2225 * Function processes __IAVF_INIT_GET_OFFLOAD_VLAN_V2_CAPS driver state if the 2226 * VF negotiates VIRTCHNL_VF_OFFLOAD_VLAN_V2. If VIRTCHNL_VF_OFFLOAD_VLAN_V2 is 2227 * not negotiated, then this state will never be entered. 2228 **/ 2229 static void iavf_init_get_offload_vlan_v2_caps(struct iavf_adapter *adapter) 2230 { 2231 int ret; 2232 2233 WARN_ON(adapter->state != __IAVF_INIT_GET_OFFLOAD_VLAN_V2_CAPS); 2234 2235 memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps)); 2236 2237 ret = iavf_get_vf_vlan_v2_caps(adapter); 2238 if (ret) { 2239 if (ret == IAVF_ERR_ADMIN_QUEUE_NO_WORK) 2240 iavf_send_vf_offload_vlan_v2_msg(adapter); 2241 goto err; 2242 } 2243 2244 iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER); 2245 return; 2246 err: 2247 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2248 } 2249 2250 /** 2251 * iavf_init_config_adapter - last part of driver startup 2252 * @adapter: board private structure 2253 * 2254 * After all the supported capabilities are negotiated, then the 2255 * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization. 2256 */ 2257 static void iavf_init_config_adapter(struct iavf_adapter *adapter) 2258 { 2259 struct net_device *netdev = adapter->netdev; 2260 struct pci_dev *pdev = adapter->pdev; 2261 int err; 2262 2263 WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER); 2264 2265 if (iavf_process_config(adapter)) 2266 goto err; 2267 2268 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2269 2270 adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED; 2271 2272 netdev->netdev_ops = &iavf_netdev_ops; 2273 iavf_set_ethtool_ops(netdev); 2274 netdev->watchdog_timeo = 5 * HZ; 2275 2276 /* MTU range: 68 - 9710 */ 2277 netdev->min_mtu = ETH_MIN_MTU; 2278 netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD; 2279 2280 if (!is_valid_ether_addr(adapter->hw.mac.addr)) { 2281 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n", 2282 adapter->hw.mac.addr); 2283 eth_hw_addr_random(netdev); 2284 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); 2285 } else { 2286 eth_hw_addr_set(netdev, adapter->hw.mac.addr); 2287 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 2288 } 2289 2290 adapter->tx_desc_count = IAVF_DEFAULT_TXD; 2291 adapter->rx_desc_count = IAVF_DEFAULT_RXD; 2292 err = iavf_init_interrupt_scheme(adapter); 2293 if (err) 2294 goto err_sw_init; 2295 iavf_map_rings_to_vectors(adapter); 2296 if (adapter->vf_res->vf_cap_flags & 2297 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 2298 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE; 2299 2300 err = iavf_request_misc_irq(adapter); 2301 if (err) 2302 goto err_sw_init; 2303 2304 netif_carrier_off(netdev); 2305 adapter->link_up = false; 2306 2307 /* set the semaphore to prevent any callbacks after device registration 2308 * up to time when state of driver will be set to __IAVF_DOWN 2309 */ 2310 rtnl_lock(); 2311 if (!adapter->netdev_registered) { 2312 err = register_netdevice(netdev); 2313 if (err) { 2314 rtnl_unlock(); 2315 goto err_register; 2316 } 2317 } 2318 2319 adapter->netdev_registered = true; 2320 2321 netif_tx_stop_all_queues(netdev); 2322 if (CLIENT_ALLOWED(adapter)) { 2323 err = iavf_lan_add_device(adapter); 2324 if (err) 2325 dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n", 2326 err); 2327 } 2328 dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr); 2329 if (netdev->features & NETIF_F_GRO) 2330 dev_info(&pdev->dev, "GRO is enabled\n"); 2331 2332 iavf_change_state(adapter, __IAVF_DOWN); 2333 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 2334 rtnl_unlock(); 2335 2336 iavf_misc_irq_enable(adapter); 2337 wake_up(&adapter->down_waitqueue); 2338 2339 adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL); 2340 adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL); 2341 if (!adapter->rss_key || !adapter->rss_lut) { 2342 err = -ENOMEM; 2343 goto err_mem; 2344 } 2345 if (RSS_AQ(adapter)) 2346 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 2347 else 2348 iavf_init_rss(adapter); 2349 2350 if (VLAN_V2_ALLOWED(adapter)) 2351 /* request initial VLAN offload settings */ 2352 iavf_set_vlan_offload_features(adapter, 0, netdev->features); 2353 2354 return; 2355 err_mem: 2356 iavf_free_rss(adapter); 2357 err_register: 2358 iavf_free_misc_irq(adapter); 2359 err_sw_init: 2360 iavf_reset_interrupt_capability(adapter); 2361 err: 2362 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2363 } 2364 2365 /** 2366 * iavf_watchdog_task - Periodic call-back task 2367 * @work: pointer to work_struct 2368 **/ 2369 static void iavf_watchdog_task(struct work_struct *work) 2370 { 2371 struct iavf_adapter *adapter = container_of(work, 2372 struct iavf_adapter, 2373 watchdog_task.work); 2374 struct iavf_hw *hw = &adapter->hw; 2375 u32 reg_val; 2376 2377 if (!mutex_trylock(&adapter->crit_lock)) 2378 goto restart_watchdog; 2379 2380 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 2381 iavf_change_state(adapter, __IAVF_COMM_FAILED); 2382 2383 if (adapter->flags & IAVF_FLAG_RESET_NEEDED && 2384 adapter->state != __IAVF_RESETTING) { 2385 iavf_change_state(adapter, __IAVF_RESETTING); 2386 adapter->aq_required = 0; 2387 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2388 } 2389 2390 switch (adapter->state) { 2391 case __IAVF_STARTUP: 2392 iavf_startup(adapter); 2393 mutex_unlock(&adapter->crit_lock); 2394 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2395 msecs_to_jiffies(30)); 2396 return; 2397 case __IAVF_INIT_VERSION_CHECK: 2398 iavf_init_version_check(adapter); 2399 mutex_unlock(&adapter->crit_lock); 2400 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2401 msecs_to_jiffies(30)); 2402 return; 2403 case __IAVF_INIT_GET_RESOURCES: 2404 iavf_init_get_resources(adapter); 2405 mutex_unlock(&adapter->crit_lock); 2406 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2407 msecs_to_jiffies(1)); 2408 return; 2409 case __IAVF_INIT_GET_OFFLOAD_VLAN_V2_CAPS: 2410 iavf_init_get_offload_vlan_v2_caps(adapter); 2411 mutex_unlock(&adapter->crit_lock); 2412 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2413 msecs_to_jiffies(1)); 2414 return; 2415 case __IAVF_INIT_CONFIG_ADAPTER: 2416 iavf_init_config_adapter(adapter); 2417 mutex_unlock(&adapter->crit_lock); 2418 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2419 msecs_to_jiffies(1)); 2420 return; 2421 case __IAVF_INIT_FAILED: 2422 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) { 2423 dev_err(&adapter->pdev->dev, 2424 "Failed to communicate with PF; waiting before retry\n"); 2425 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 2426 iavf_shutdown_adminq(hw); 2427 mutex_unlock(&adapter->crit_lock); 2428 queue_delayed_work(iavf_wq, 2429 &adapter->watchdog_task, (5 * HZ)); 2430 return; 2431 } 2432 /* Try again from failed step*/ 2433 iavf_change_state(adapter, adapter->last_state); 2434 mutex_unlock(&adapter->crit_lock); 2435 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ); 2436 return; 2437 case __IAVF_COMM_FAILED: 2438 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 2439 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 2440 if (reg_val == VIRTCHNL_VFR_VFACTIVE || 2441 reg_val == VIRTCHNL_VFR_COMPLETED) { 2442 /* A chance for redemption! */ 2443 dev_err(&adapter->pdev->dev, 2444 "Hardware came out of reset. Attempting reinit.\n"); 2445 /* When init task contacts the PF and 2446 * gets everything set up again, it'll restart the 2447 * watchdog for us. Down, boy. Sit. Stay. Woof. 2448 */ 2449 iavf_change_state(adapter, __IAVF_STARTUP); 2450 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 2451 } 2452 adapter->aq_required = 0; 2453 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2454 mutex_unlock(&adapter->crit_lock); 2455 queue_delayed_work(iavf_wq, 2456 &adapter->watchdog_task, 2457 msecs_to_jiffies(10)); 2458 return; 2459 case __IAVF_RESETTING: 2460 mutex_unlock(&adapter->crit_lock); 2461 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2); 2462 return; 2463 case __IAVF_DOWN: 2464 case __IAVF_DOWN_PENDING: 2465 case __IAVF_TESTING: 2466 case __IAVF_RUNNING: 2467 if (adapter->current_op) { 2468 if (!iavf_asq_done(hw)) { 2469 dev_dbg(&adapter->pdev->dev, 2470 "Admin queue timeout\n"); 2471 iavf_send_api_ver(adapter); 2472 } 2473 } else { 2474 int ret = iavf_process_aq_command(adapter); 2475 2476 /* An error will be returned if no commands were 2477 * processed; use this opportunity to update stats 2478 * if the error isn't -ENOTSUPP 2479 */ 2480 if (ret && ret != -EOPNOTSUPP && 2481 adapter->state == __IAVF_RUNNING) 2482 iavf_request_stats(adapter); 2483 } 2484 if (adapter->state == __IAVF_RUNNING) 2485 iavf_detect_recover_hung(&adapter->vsi); 2486 break; 2487 case __IAVF_REMOVE: 2488 default: 2489 mutex_unlock(&adapter->crit_lock); 2490 return; 2491 } 2492 2493 /* check for hw reset */ 2494 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK; 2495 if (!reg_val) { 2496 adapter->flags |= IAVF_FLAG_RESET_PENDING; 2497 adapter->aq_required = 0; 2498 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2499 dev_err(&adapter->pdev->dev, "Hardware reset detected\n"); 2500 queue_work(iavf_wq, &adapter->reset_task); 2501 mutex_unlock(&adapter->crit_lock); 2502 queue_delayed_work(iavf_wq, 2503 &adapter->watchdog_task, HZ * 2); 2504 return; 2505 } 2506 2507 schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5)); 2508 mutex_unlock(&adapter->crit_lock); 2509 restart_watchdog: 2510 queue_work(iavf_wq, &adapter->adminq_task); 2511 if (adapter->aq_required) 2512 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2513 msecs_to_jiffies(20)); 2514 else 2515 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2); 2516 } 2517 2518 static void iavf_disable_vf(struct iavf_adapter *adapter) 2519 { 2520 struct iavf_mac_filter *f, *ftmp; 2521 struct iavf_vlan_filter *fv, *fvtmp; 2522 struct iavf_cloud_filter *cf, *cftmp; 2523 2524 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 2525 2526 /* We don't use netif_running() because it may be true prior to 2527 * ndo_open() returning, so we can't assume it means all our open 2528 * tasks have finished, since we're not holding the rtnl_lock here. 2529 */ 2530 if (adapter->state == __IAVF_RUNNING) { 2531 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 2532 netif_carrier_off(adapter->netdev); 2533 netif_tx_disable(adapter->netdev); 2534 adapter->link_up = false; 2535 iavf_napi_disable_all(adapter); 2536 iavf_irq_disable(adapter); 2537 iavf_free_traffic_irqs(adapter); 2538 iavf_free_all_tx_resources(adapter); 2539 iavf_free_all_rx_resources(adapter); 2540 } 2541 2542 spin_lock_bh(&adapter->mac_vlan_list_lock); 2543 2544 /* Delete all of the filters */ 2545 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 2546 list_del(&f->list); 2547 kfree(f); 2548 } 2549 2550 list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) { 2551 list_del(&fv->list); 2552 kfree(fv); 2553 } 2554 2555 spin_unlock_bh(&adapter->mac_vlan_list_lock); 2556 2557 spin_lock_bh(&adapter->cloud_filter_list_lock); 2558 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 2559 list_del(&cf->list); 2560 kfree(cf); 2561 adapter->num_cloud_filters--; 2562 } 2563 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2564 2565 iavf_free_misc_irq(adapter); 2566 iavf_reset_interrupt_capability(adapter); 2567 iavf_free_q_vectors(adapter); 2568 iavf_free_queues(adapter); 2569 memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE); 2570 iavf_shutdown_adminq(&adapter->hw); 2571 adapter->netdev->flags &= ~IFF_UP; 2572 mutex_unlock(&adapter->crit_lock); 2573 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2574 iavf_change_state(adapter, __IAVF_DOWN); 2575 wake_up(&adapter->down_waitqueue); 2576 dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n"); 2577 } 2578 2579 /** 2580 * iavf_reset_task - Call-back task to handle hardware reset 2581 * @work: pointer to work_struct 2582 * 2583 * During reset we need to shut down and reinitialize the admin queue 2584 * before we can use it to communicate with the PF again. We also clear 2585 * and reinit the rings because that context is lost as well. 2586 **/ 2587 static void iavf_reset_task(struct work_struct *work) 2588 { 2589 struct iavf_adapter *adapter = container_of(work, 2590 struct iavf_adapter, 2591 reset_task); 2592 struct virtchnl_vf_resource *vfres = adapter->vf_res; 2593 struct net_device *netdev = adapter->netdev; 2594 struct iavf_hw *hw = &adapter->hw; 2595 struct iavf_mac_filter *f, *ftmp; 2596 struct iavf_cloud_filter *cf; 2597 u32 reg_val; 2598 int i = 0, err; 2599 bool running; 2600 2601 /* When device is being removed it doesn't make sense to run the reset 2602 * task, just return in such a case. 2603 */ 2604 if (mutex_is_locked(&adapter->remove_lock)) 2605 return; 2606 2607 if (iavf_lock_timeout(&adapter->crit_lock, 200)) { 2608 schedule_work(&adapter->reset_task); 2609 return; 2610 } 2611 while (!mutex_trylock(&adapter->client_lock)) 2612 usleep_range(500, 1000); 2613 if (CLIENT_ENABLED(adapter)) { 2614 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN | 2615 IAVF_FLAG_CLIENT_NEEDS_CLOSE | 2616 IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS | 2617 IAVF_FLAG_SERVICE_CLIENT_REQUESTED); 2618 cancel_delayed_work_sync(&adapter->client_task); 2619 iavf_notify_client_close(&adapter->vsi, true); 2620 } 2621 iavf_misc_irq_disable(adapter); 2622 if (adapter->flags & IAVF_FLAG_RESET_NEEDED) { 2623 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED; 2624 /* Restart the AQ here. If we have been reset but didn't 2625 * detect it, or if the PF had to reinit, our AQ will be hosed. 2626 */ 2627 iavf_shutdown_adminq(hw); 2628 iavf_init_adminq(hw); 2629 iavf_request_reset(adapter); 2630 } 2631 adapter->flags |= IAVF_FLAG_RESET_PENDING; 2632 2633 /* poll until we see the reset actually happen */ 2634 for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) { 2635 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & 2636 IAVF_VF_ARQLEN1_ARQENABLE_MASK; 2637 if (!reg_val) 2638 break; 2639 usleep_range(5000, 10000); 2640 } 2641 if (i == IAVF_RESET_WAIT_DETECTED_COUNT) { 2642 dev_info(&adapter->pdev->dev, "Never saw reset\n"); 2643 goto continue_reset; /* act like the reset happened */ 2644 } 2645 2646 /* wait until the reset is complete and the PF is responding to us */ 2647 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { 2648 /* sleep first to make sure a minimum wait time is met */ 2649 msleep(IAVF_RESET_WAIT_MS); 2650 2651 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 2652 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 2653 if (reg_val == VIRTCHNL_VFR_VFACTIVE) 2654 break; 2655 } 2656 2657 pci_set_master(adapter->pdev); 2658 pci_restore_msi_state(adapter->pdev); 2659 2660 if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) { 2661 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n", 2662 reg_val); 2663 iavf_disable_vf(adapter); 2664 mutex_unlock(&adapter->client_lock); 2665 return; /* Do not attempt to reinit. It's dead, Jim. */ 2666 } 2667 2668 continue_reset: 2669 /* We don't use netif_running() because it may be true prior to 2670 * ndo_open() returning, so we can't assume it means all our open 2671 * tasks have finished, since we're not holding the rtnl_lock here. 2672 */ 2673 running = ((adapter->state == __IAVF_RUNNING) || 2674 (adapter->state == __IAVF_RESETTING)); 2675 2676 if (running) { 2677 netdev->flags &= ~IFF_UP; 2678 netif_carrier_off(netdev); 2679 netif_tx_stop_all_queues(netdev); 2680 adapter->link_up = false; 2681 iavf_napi_disable_all(adapter); 2682 } 2683 iavf_irq_disable(adapter); 2684 2685 iavf_change_state(adapter, __IAVF_RESETTING); 2686 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2687 2688 /* free the Tx/Rx rings and descriptors, might be better to just 2689 * re-use them sometime in the future 2690 */ 2691 iavf_free_all_rx_resources(adapter); 2692 iavf_free_all_tx_resources(adapter); 2693 2694 adapter->flags |= IAVF_FLAG_QUEUES_DISABLED; 2695 /* kill and reinit the admin queue */ 2696 iavf_shutdown_adminq(hw); 2697 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2698 err = iavf_init_adminq(hw); 2699 if (err) 2700 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n", 2701 err); 2702 adapter->aq_required = 0; 2703 2704 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) { 2705 err = iavf_reinit_interrupt_scheme(adapter); 2706 if (err) 2707 goto reset_err; 2708 } 2709 2710 if (RSS_AQ(adapter)) { 2711 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 2712 } else { 2713 err = iavf_init_rss(adapter); 2714 if (err) 2715 goto reset_err; 2716 } 2717 2718 adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG; 2719 /* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been 2720 * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here, 2721 * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until 2722 * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have 2723 * been successfully sent and negotiated 2724 */ 2725 adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS; 2726 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 2727 2728 spin_lock_bh(&adapter->mac_vlan_list_lock); 2729 2730 /* Delete filter for the current MAC address, it could have 2731 * been changed by the PF via administratively set MAC. 2732 * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES. 2733 */ 2734 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 2735 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) { 2736 list_del(&f->list); 2737 kfree(f); 2738 } 2739 } 2740 /* re-add all MAC filters */ 2741 list_for_each_entry(f, &adapter->mac_filter_list, list) { 2742 f->add = true; 2743 } 2744 spin_unlock_bh(&adapter->mac_vlan_list_lock); 2745 2746 /* check if TCs are running and re-add all cloud filters */ 2747 spin_lock_bh(&adapter->cloud_filter_list_lock); 2748 if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 2749 adapter->num_tc) { 2750 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 2751 cf->add = true; 2752 } 2753 } 2754 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2755 2756 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 2757 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 2758 iavf_misc_irq_enable(adapter); 2759 2760 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 2); 2761 2762 /* We were running when the reset started, so we need to restore some 2763 * state here. 2764 */ 2765 if (running) { 2766 /* allocate transmit descriptors */ 2767 err = iavf_setup_all_tx_resources(adapter); 2768 if (err) 2769 goto reset_err; 2770 2771 /* allocate receive descriptors */ 2772 err = iavf_setup_all_rx_resources(adapter); 2773 if (err) 2774 goto reset_err; 2775 2776 if (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED) { 2777 err = iavf_request_traffic_irqs(adapter, netdev->name); 2778 if (err) 2779 goto reset_err; 2780 2781 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 2782 } 2783 2784 iavf_configure(adapter); 2785 2786 /* iavf_up_complete() will switch device back 2787 * to __IAVF_RUNNING 2788 */ 2789 iavf_up_complete(adapter); 2790 netdev->flags |= IFF_UP; 2791 iavf_irq_enable(adapter, true); 2792 } else { 2793 iavf_change_state(adapter, __IAVF_DOWN); 2794 wake_up(&adapter->down_waitqueue); 2795 } 2796 mutex_unlock(&adapter->client_lock); 2797 mutex_unlock(&adapter->crit_lock); 2798 2799 return; 2800 reset_err: 2801 mutex_unlock(&adapter->client_lock); 2802 mutex_unlock(&adapter->crit_lock); 2803 if (running) { 2804 iavf_change_state(adapter, __IAVF_RUNNING); 2805 netdev->flags |= IFF_UP; 2806 } 2807 dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); 2808 iavf_close(netdev); 2809 } 2810 2811 /** 2812 * iavf_adminq_task - worker thread to clean the admin queue 2813 * @work: pointer to work_struct containing our data 2814 **/ 2815 static void iavf_adminq_task(struct work_struct *work) 2816 { 2817 struct iavf_adapter *adapter = 2818 container_of(work, struct iavf_adapter, adminq_task); 2819 struct iavf_hw *hw = &adapter->hw; 2820 struct iavf_arq_event_info event; 2821 enum virtchnl_ops v_op; 2822 enum iavf_status ret, v_ret; 2823 u32 val, oldval; 2824 u16 pending; 2825 2826 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 2827 goto out; 2828 2829 event.buf_len = IAVF_MAX_AQ_BUF_SIZE; 2830 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 2831 if (!event.msg_buf) 2832 goto out; 2833 2834 if (iavf_lock_timeout(&adapter->crit_lock, 200)) 2835 goto freedom; 2836 do { 2837 ret = iavf_clean_arq_element(hw, &event, &pending); 2838 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high); 2839 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low); 2840 2841 if (ret || !v_op) 2842 break; /* No event to process or error cleaning ARQ */ 2843 2844 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf, 2845 event.msg_len); 2846 if (pending != 0) 2847 memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE); 2848 } while (pending); 2849 mutex_unlock(&adapter->crit_lock); 2850 2851 if ((adapter->flags & 2852 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) || 2853 adapter->state == __IAVF_RESETTING) 2854 goto freedom; 2855 2856 /* check for error indications */ 2857 val = rd32(hw, hw->aq.arq.len); 2858 if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */ 2859 goto freedom; 2860 oldval = val; 2861 if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) { 2862 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n"); 2863 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK; 2864 } 2865 if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) { 2866 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n"); 2867 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK; 2868 } 2869 if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) { 2870 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n"); 2871 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK; 2872 } 2873 if (oldval != val) 2874 wr32(hw, hw->aq.arq.len, val); 2875 2876 val = rd32(hw, hw->aq.asq.len); 2877 oldval = val; 2878 if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) { 2879 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n"); 2880 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK; 2881 } 2882 if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) { 2883 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n"); 2884 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK; 2885 } 2886 if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) { 2887 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n"); 2888 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK; 2889 } 2890 if (oldval != val) 2891 wr32(hw, hw->aq.asq.len, val); 2892 2893 freedom: 2894 kfree(event.msg_buf); 2895 out: 2896 /* re-enable Admin queue interrupt cause */ 2897 iavf_misc_irq_enable(adapter); 2898 } 2899 2900 /** 2901 * iavf_client_task - worker thread to perform client work 2902 * @work: pointer to work_struct containing our data 2903 * 2904 * This task handles client interactions. Because client calls can be 2905 * reentrant, we can't handle them in the watchdog. 2906 **/ 2907 static void iavf_client_task(struct work_struct *work) 2908 { 2909 struct iavf_adapter *adapter = 2910 container_of(work, struct iavf_adapter, client_task.work); 2911 2912 /* If we can't get the client bit, just give up. We'll be rescheduled 2913 * later. 2914 */ 2915 2916 if (!mutex_trylock(&adapter->client_lock)) 2917 return; 2918 2919 if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) { 2920 iavf_client_subtask(adapter); 2921 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED; 2922 goto out; 2923 } 2924 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) { 2925 iavf_notify_client_l2_params(&adapter->vsi); 2926 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS; 2927 goto out; 2928 } 2929 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) { 2930 iavf_notify_client_close(&adapter->vsi, false); 2931 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE; 2932 goto out; 2933 } 2934 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) { 2935 iavf_notify_client_open(&adapter->vsi); 2936 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN; 2937 } 2938 out: 2939 mutex_unlock(&adapter->client_lock); 2940 } 2941 2942 /** 2943 * iavf_free_all_tx_resources - Free Tx Resources for All Queues 2944 * @adapter: board private structure 2945 * 2946 * Free all transmit software resources 2947 **/ 2948 void iavf_free_all_tx_resources(struct iavf_adapter *adapter) 2949 { 2950 int i; 2951 2952 if (!adapter->tx_rings) 2953 return; 2954 2955 for (i = 0; i < adapter->num_active_queues; i++) 2956 if (adapter->tx_rings[i].desc) 2957 iavf_free_tx_resources(&adapter->tx_rings[i]); 2958 } 2959 2960 /** 2961 * iavf_setup_all_tx_resources - allocate all queues Tx resources 2962 * @adapter: board private structure 2963 * 2964 * If this function returns with an error, then it's possible one or 2965 * more of the rings is populated (while the rest are not). It is the 2966 * callers duty to clean those orphaned rings. 2967 * 2968 * Return 0 on success, negative on failure 2969 **/ 2970 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter) 2971 { 2972 int i, err = 0; 2973 2974 for (i = 0; i < adapter->num_active_queues; i++) { 2975 adapter->tx_rings[i].count = adapter->tx_desc_count; 2976 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]); 2977 if (!err) 2978 continue; 2979 dev_err(&adapter->pdev->dev, 2980 "Allocation for Tx Queue %u failed\n", i); 2981 break; 2982 } 2983 2984 return err; 2985 } 2986 2987 /** 2988 * iavf_setup_all_rx_resources - allocate all queues Rx resources 2989 * @adapter: board private structure 2990 * 2991 * If this function returns with an error, then it's possible one or 2992 * more of the rings is populated (while the rest are not). It is the 2993 * callers duty to clean those orphaned rings. 2994 * 2995 * Return 0 on success, negative on failure 2996 **/ 2997 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter) 2998 { 2999 int i, err = 0; 3000 3001 for (i = 0; i < adapter->num_active_queues; i++) { 3002 adapter->rx_rings[i].count = adapter->rx_desc_count; 3003 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]); 3004 if (!err) 3005 continue; 3006 dev_err(&adapter->pdev->dev, 3007 "Allocation for Rx Queue %u failed\n", i); 3008 break; 3009 } 3010 return err; 3011 } 3012 3013 /** 3014 * iavf_free_all_rx_resources - Free Rx Resources for All Queues 3015 * @adapter: board private structure 3016 * 3017 * Free all receive software resources 3018 **/ 3019 void iavf_free_all_rx_resources(struct iavf_adapter *adapter) 3020 { 3021 int i; 3022 3023 if (!adapter->rx_rings) 3024 return; 3025 3026 for (i = 0; i < adapter->num_active_queues; i++) 3027 if (adapter->rx_rings[i].desc) 3028 iavf_free_rx_resources(&adapter->rx_rings[i]); 3029 } 3030 3031 /** 3032 * iavf_validate_tx_bandwidth - validate the max Tx bandwidth 3033 * @adapter: board private structure 3034 * @max_tx_rate: max Tx bw for a tc 3035 **/ 3036 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter, 3037 u64 max_tx_rate) 3038 { 3039 int speed = 0, ret = 0; 3040 3041 if (ADV_LINK_SUPPORT(adapter)) { 3042 if (adapter->link_speed_mbps < U32_MAX) { 3043 speed = adapter->link_speed_mbps; 3044 goto validate_bw; 3045 } else { 3046 dev_err(&adapter->pdev->dev, "Unknown link speed\n"); 3047 return -EINVAL; 3048 } 3049 } 3050 3051 switch (adapter->link_speed) { 3052 case VIRTCHNL_LINK_SPEED_40GB: 3053 speed = SPEED_40000; 3054 break; 3055 case VIRTCHNL_LINK_SPEED_25GB: 3056 speed = SPEED_25000; 3057 break; 3058 case VIRTCHNL_LINK_SPEED_20GB: 3059 speed = SPEED_20000; 3060 break; 3061 case VIRTCHNL_LINK_SPEED_10GB: 3062 speed = SPEED_10000; 3063 break; 3064 case VIRTCHNL_LINK_SPEED_5GB: 3065 speed = SPEED_5000; 3066 break; 3067 case VIRTCHNL_LINK_SPEED_2_5GB: 3068 speed = SPEED_2500; 3069 break; 3070 case VIRTCHNL_LINK_SPEED_1GB: 3071 speed = SPEED_1000; 3072 break; 3073 case VIRTCHNL_LINK_SPEED_100MB: 3074 speed = SPEED_100; 3075 break; 3076 default: 3077 break; 3078 } 3079 3080 validate_bw: 3081 if (max_tx_rate > speed) { 3082 dev_err(&adapter->pdev->dev, 3083 "Invalid tx rate specified\n"); 3084 ret = -EINVAL; 3085 } 3086 3087 return ret; 3088 } 3089 3090 /** 3091 * iavf_validate_ch_config - validate queue mapping info 3092 * @adapter: board private structure 3093 * @mqprio_qopt: queue parameters 3094 * 3095 * This function validates if the config provided by the user to 3096 * configure queue channels is valid or not. Returns 0 on a valid 3097 * config. 3098 **/ 3099 static int iavf_validate_ch_config(struct iavf_adapter *adapter, 3100 struct tc_mqprio_qopt_offload *mqprio_qopt) 3101 { 3102 u64 total_max_rate = 0; 3103 int i, num_qps = 0; 3104 u64 tx_rate = 0; 3105 int ret = 0; 3106 3107 if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS || 3108 mqprio_qopt->qopt.num_tc < 1) 3109 return -EINVAL; 3110 3111 for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) { 3112 if (!mqprio_qopt->qopt.count[i] || 3113 mqprio_qopt->qopt.offset[i] != num_qps) 3114 return -EINVAL; 3115 if (mqprio_qopt->min_rate[i]) { 3116 dev_err(&adapter->pdev->dev, 3117 "Invalid min tx rate (greater than 0) specified\n"); 3118 return -EINVAL; 3119 } 3120 /*convert to Mbps */ 3121 tx_rate = div_u64(mqprio_qopt->max_rate[i], 3122 IAVF_MBPS_DIVISOR); 3123 total_max_rate += tx_rate; 3124 num_qps += mqprio_qopt->qopt.count[i]; 3125 } 3126 if (num_qps > adapter->num_active_queues) { 3127 dev_err(&adapter->pdev->dev, 3128 "Cannot support requested number of queues\n"); 3129 return -EINVAL; 3130 } 3131 3132 ret = iavf_validate_tx_bandwidth(adapter, total_max_rate); 3133 return ret; 3134 } 3135 3136 /** 3137 * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes 3138 * @adapter: board private structure 3139 **/ 3140 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter) 3141 { 3142 struct iavf_cloud_filter *cf, *cftmp; 3143 3144 spin_lock_bh(&adapter->cloud_filter_list_lock); 3145 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, 3146 list) { 3147 list_del(&cf->list); 3148 kfree(cf); 3149 adapter->num_cloud_filters--; 3150 } 3151 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3152 } 3153 3154 /** 3155 * __iavf_setup_tc - configure multiple traffic classes 3156 * @netdev: network interface device structure 3157 * @type_data: tc offload data 3158 * 3159 * This function processes the config information provided by the 3160 * user to configure traffic classes/queue channels and packages the 3161 * information to request the PF to setup traffic classes. 3162 * 3163 * Returns 0 on success. 3164 **/ 3165 static int __iavf_setup_tc(struct net_device *netdev, void *type_data) 3166 { 3167 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 3168 struct iavf_adapter *adapter = netdev_priv(netdev); 3169 struct virtchnl_vf_resource *vfres = adapter->vf_res; 3170 u8 num_tc = 0, total_qps = 0; 3171 int ret = 0, netdev_tc = 0; 3172 u64 max_tx_rate; 3173 u16 mode; 3174 int i; 3175 3176 num_tc = mqprio_qopt->qopt.num_tc; 3177 mode = mqprio_qopt->mode; 3178 3179 /* delete queue_channel */ 3180 if (!mqprio_qopt->qopt.hw) { 3181 if (adapter->ch_config.state == __IAVF_TC_RUNNING) { 3182 /* reset the tc configuration */ 3183 netdev_reset_tc(netdev); 3184 adapter->num_tc = 0; 3185 netif_tx_stop_all_queues(netdev); 3186 netif_tx_disable(netdev); 3187 iavf_del_all_cloud_filters(adapter); 3188 adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS; 3189 goto exit; 3190 } else { 3191 return -EINVAL; 3192 } 3193 } 3194 3195 /* add queue channel */ 3196 if (mode == TC_MQPRIO_MODE_CHANNEL) { 3197 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) { 3198 dev_err(&adapter->pdev->dev, "ADq not supported\n"); 3199 return -EOPNOTSUPP; 3200 } 3201 if (adapter->ch_config.state != __IAVF_TC_INVALID) { 3202 dev_err(&adapter->pdev->dev, "TC configuration already exists\n"); 3203 return -EINVAL; 3204 } 3205 3206 ret = iavf_validate_ch_config(adapter, mqprio_qopt); 3207 if (ret) 3208 return ret; 3209 /* Return if same TC config is requested */ 3210 if (adapter->num_tc == num_tc) 3211 return 0; 3212 adapter->num_tc = num_tc; 3213 3214 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 3215 if (i < num_tc) { 3216 adapter->ch_config.ch_info[i].count = 3217 mqprio_qopt->qopt.count[i]; 3218 adapter->ch_config.ch_info[i].offset = 3219 mqprio_qopt->qopt.offset[i]; 3220 total_qps += mqprio_qopt->qopt.count[i]; 3221 max_tx_rate = mqprio_qopt->max_rate[i]; 3222 /* convert to Mbps */ 3223 max_tx_rate = div_u64(max_tx_rate, 3224 IAVF_MBPS_DIVISOR); 3225 adapter->ch_config.ch_info[i].max_tx_rate = 3226 max_tx_rate; 3227 } else { 3228 adapter->ch_config.ch_info[i].count = 1; 3229 adapter->ch_config.ch_info[i].offset = 0; 3230 } 3231 } 3232 adapter->ch_config.total_qps = total_qps; 3233 netif_tx_stop_all_queues(netdev); 3234 netif_tx_disable(netdev); 3235 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS; 3236 netdev_reset_tc(netdev); 3237 /* Report the tc mapping up the stack */ 3238 netdev_set_num_tc(adapter->netdev, num_tc); 3239 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 3240 u16 qcount = mqprio_qopt->qopt.count[i]; 3241 u16 qoffset = mqprio_qopt->qopt.offset[i]; 3242 3243 if (i < num_tc) 3244 netdev_set_tc_queue(netdev, netdev_tc++, qcount, 3245 qoffset); 3246 } 3247 } 3248 exit: 3249 return ret; 3250 } 3251 3252 /** 3253 * iavf_parse_cls_flower - Parse tc flower filters provided by kernel 3254 * @adapter: board private structure 3255 * @f: pointer to struct flow_cls_offload 3256 * @filter: pointer to cloud filter structure 3257 */ 3258 static int iavf_parse_cls_flower(struct iavf_adapter *adapter, 3259 struct flow_cls_offload *f, 3260 struct iavf_cloud_filter *filter) 3261 { 3262 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 3263 struct flow_dissector *dissector = rule->match.dissector; 3264 u16 n_proto_mask = 0; 3265 u16 n_proto_key = 0; 3266 u8 field_flags = 0; 3267 u16 addr_type = 0; 3268 u16 n_proto = 0; 3269 int i = 0; 3270 struct virtchnl_filter *vf = &filter->f; 3271 3272 if (dissector->used_keys & 3273 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 3274 BIT(FLOW_DISSECTOR_KEY_BASIC) | 3275 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 3276 BIT(FLOW_DISSECTOR_KEY_VLAN) | 3277 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 3278 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 3279 BIT(FLOW_DISSECTOR_KEY_PORTS) | 3280 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 3281 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n", 3282 dissector->used_keys); 3283 return -EOPNOTSUPP; 3284 } 3285 3286 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 3287 struct flow_match_enc_keyid match; 3288 3289 flow_rule_match_enc_keyid(rule, &match); 3290 if (match.mask->keyid != 0) 3291 field_flags |= IAVF_CLOUD_FIELD_TEN_ID; 3292 } 3293 3294 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 3295 struct flow_match_basic match; 3296 3297 flow_rule_match_basic(rule, &match); 3298 n_proto_key = ntohs(match.key->n_proto); 3299 n_proto_mask = ntohs(match.mask->n_proto); 3300 3301 if (n_proto_key == ETH_P_ALL) { 3302 n_proto_key = 0; 3303 n_proto_mask = 0; 3304 } 3305 n_proto = n_proto_key & n_proto_mask; 3306 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6) 3307 return -EINVAL; 3308 if (n_proto == ETH_P_IPV6) { 3309 /* specify flow type as TCP IPv6 */ 3310 vf->flow_type = VIRTCHNL_TCP_V6_FLOW; 3311 } 3312 3313 if (match.key->ip_proto != IPPROTO_TCP) { 3314 dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n"); 3315 return -EINVAL; 3316 } 3317 } 3318 3319 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 3320 struct flow_match_eth_addrs match; 3321 3322 flow_rule_match_eth_addrs(rule, &match); 3323 3324 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 3325 if (!is_zero_ether_addr(match.mask->dst)) { 3326 if (is_broadcast_ether_addr(match.mask->dst)) { 3327 field_flags |= IAVF_CLOUD_FIELD_OMAC; 3328 } else { 3329 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n", 3330 match.mask->dst); 3331 return -EINVAL; 3332 } 3333 } 3334 3335 if (!is_zero_ether_addr(match.mask->src)) { 3336 if (is_broadcast_ether_addr(match.mask->src)) { 3337 field_flags |= IAVF_CLOUD_FIELD_IMAC; 3338 } else { 3339 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n", 3340 match.mask->src); 3341 return -EINVAL; 3342 } 3343 } 3344 3345 if (!is_zero_ether_addr(match.key->dst)) 3346 if (is_valid_ether_addr(match.key->dst) || 3347 is_multicast_ether_addr(match.key->dst)) { 3348 /* set the mask if a valid dst_mac address */ 3349 for (i = 0; i < ETH_ALEN; i++) 3350 vf->mask.tcp_spec.dst_mac[i] |= 0xff; 3351 ether_addr_copy(vf->data.tcp_spec.dst_mac, 3352 match.key->dst); 3353 } 3354 3355 if (!is_zero_ether_addr(match.key->src)) 3356 if (is_valid_ether_addr(match.key->src) || 3357 is_multicast_ether_addr(match.key->src)) { 3358 /* set the mask if a valid dst_mac address */ 3359 for (i = 0; i < ETH_ALEN; i++) 3360 vf->mask.tcp_spec.src_mac[i] |= 0xff; 3361 ether_addr_copy(vf->data.tcp_spec.src_mac, 3362 match.key->src); 3363 } 3364 } 3365 3366 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 3367 struct flow_match_vlan match; 3368 3369 flow_rule_match_vlan(rule, &match); 3370 if (match.mask->vlan_id) { 3371 if (match.mask->vlan_id == VLAN_VID_MASK) { 3372 field_flags |= IAVF_CLOUD_FIELD_IVLAN; 3373 } else { 3374 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n", 3375 match.mask->vlan_id); 3376 return -EINVAL; 3377 } 3378 } 3379 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff); 3380 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id); 3381 } 3382 3383 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 3384 struct flow_match_control match; 3385 3386 flow_rule_match_control(rule, &match); 3387 addr_type = match.key->addr_type; 3388 } 3389 3390 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 3391 struct flow_match_ipv4_addrs match; 3392 3393 flow_rule_match_ipv4_addrs(rule, &match); 3394 if (match.mask->dst) { 3395 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 3396 field_flags |= IAVF_CLOUD_FIELD_IIP; 3397 } else { 3398 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n", 3399 be32_to_cpu(match.mask->dst)); 3400 return -EINVAL; 3401 } 3402 } 3403 3404 if (match.mask->src) { 3405 if (match.mask->src == cpu_to_be32(0xffffffff)) { 3406 field_flags |= IAVF_CLOUD_FIELD_IIP; 3407 } else { 3408 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n", 3409 be32_to_cpu(match.mask->dst)); 3410 return -EINVAL; 3411 } 3412 } 3413 3414 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) { 3415 dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n"); 3416 return -EINVAL; 3417 } 3418 if (match.key->dst) { 3419 vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff); 3420 vf->data.tcp_spec.dst_ip[0] = match.key->dst; 3421 } 3422 if (match.key->src) { 3423 vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff); 3424 vf->data.tcp_spec.src_ip[0] = match.key->src; 3425 } 3426 } 3427 3428 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 3429 struct flow_match_ipv6_addrs match; 3430 3431 flow_rule_match_ipv6_addrs(rule, &match); 3432 3433 /* validate mask, make sure it is not IPV6_ADDR_ANY */ 3434 if (ipv6_addr_any(&match.mask->dst)) { 3435 dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n", 3436 IPV6_ADDR_ANY); 3437 return -EINVAL; 3438 } 3439 3440 /* src and dest IPv6 address should not be LOOPBACK 3441 * (0:0:0:0:0:0:0:1) which can be represented as ::1 3442 */ 3443 if (ipv6_addr_loopback(&match.key->dst) || 3444 ipv6_addr_loopback(&match.key->src)) { 3445 dev_err(&adapter->pdev->dev, 3446 "ipv6 addr should not be loopback\n"); 3447 return -EINVAL; 3448 } 3449 if (!ipv6_addr_any(&match.mask->dst) || 3450 !ipv6_addr_any(&match.mask->src)) 3451 field_flags |= IAVF_CLOUD_FIELD_IIP; 3452 3453 for (i = 0; i < 4; i++) 3454 vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff); 3455 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32, 3456 sizeof(vf->data.tcp_spec.dst_ip)); 3457 for (i = 0; i < 4; i++) 3458 vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff); 3459 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32, 3460 sizeof(vf->data.tcp_spec.src_ip)); 3461 } 3462 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 3463 struct flow_match_ports match; 3464 3465 flow_rule_match_ports(rule, &match); 3466 if (match.mask->src) { 3467 if (match.mask->src == cpu_to_be16(0xffff)) { 3468 field_flags |= IAVF_CLOUD_FIELD_IIP; 3469 } else { 3470 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n", 3471 be16_to_cpu(match.mask->src)); 3472 return -EINVAL; 3473 } 3474 } 3475 3476 if (match.mask->dst) { 3477 if (match.mask->dst == cpu_to_be16(0xffff)) { 3478 field_flags |= IAVF_CLOUD_FIELD_IIP; 3479 } else { 3480 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n", 3481 be16_to_cpu(match.mask->dst)); 3482 return -EINVAL; 3483 } 3484 } 3485 if (match.key->dst) { 3486 vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff); 3487 vf->data.tcp_spec.dst_port = match.key->dst; 3488 } 3489 3490 if (match.key->src) { 3491 vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff); 3492 vf->data.tcp_spec.src_port = match.key->src; 3493 } 3494 } 3495 vf->field_flags = field_flags; 3496 3497 return 0; 3498 } 3499 3500 /** 3501 * iavf_handle_tclass - Forward to a traffic class on the device 3502 * @adapter: board private structure 3503 * @tc: traffic class index on the device 3504 * @filter: pointer to cloud filter structure 3505 */ 3506 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc, 3507 struct iavf_cloud_filter *filter) 3508 { 3509 if (tc == 0) 3510 return 0; 3511 if (tc < adapter->num_tc) { 3512 if (!filter->f.data.tcp_spec.dst_port) { 3513 dev_err(&adapter->pdev->dev, 3514 "Specify destination port to redirect to traffic class other than TC0\n"); 3515 return -EINVAL; 3516 } 3517 } 3518 /* redirect to a traffic class on the same device */ 3519 filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT; 3520 filter->f.action_meta = tc; 3521 return 0; 3522 } 3523 3524 /** 3525 * iavf_configure_clsflower - Add tc flower filters 3526 * @adapter: board private structure 3527 * @cls_flower: Pointer to struct flow_cls_offload 3528 */ 3529 static int iavf_configure_clsflower(struct iavf_adapter *adapter, 3530 struct flow_cls_offload *cls_flower) 3531 { 3532 int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid); 3533 struct iavf_cloud_filter *filter = NULL; 3534 int err = -EINVAL, count = 50; 3535 3536 if (tc < 0) { 3537 dev_err(&adapter->pdev->dev, "Invalid traffic class\n"); 3538 return -EINVAL; 3539 } 3540 3541 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 3542 if (!filter) 3543 return -ENOMEM; 3544 3545 while (!mutex_trylock(&adapter->crit_lock)) { 3546 if (--count == 0) { 3547 kfree(filter); 3548 return err; 3549 } 3550 udelay(1); 3551 } 3552 3553 filter->cookie = cls_flower->cookie; 3554 3555 /* set the mask to all zeroes to begin with */ 3556 memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec)); 3557 /* start out with flow type and eth type IPv4 to begin with */ 3558 filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW; 3559 err = iavf_parse_cls_flower(adapter, cls_flower, filter); 3560 if (err) 3561 goto err; 3562 3563 err = iavf_handle_tclass(adapter, tc, filter); 3564 if (err) 3565 goto err; 3566 3567 /* add filter to the list */ 3568 spin_lock_bh(&adapter->cloud_filter_list_lock); 3569 list_add_tail(&filter->list, &adapter->cloud_filter_list); 3570 adapter->num_cloud_filters++; 3571 filter->add = true; 3572 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 3573 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3574 err: 3575 if (err) 3576 kfree(filter); 3577 3578 mutex_unlock(&adapter->crit_lock); 3579 return err; 3580 } 3581 3582 /* iavf_find_cf - Find the cloud filter in the list 3583 * @adapter: Board private structure 3584 * @cookie: filter specific cookie 3585 * 3586 * Returns ptr to the filter object or NULL. Must be called while holding the 3587 * cloud_filter_list_lock. 3588 */ 3589 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter, 3590 unsigned long *cookie) 3591 { 3592 struct iavf_cloud_filter *filter = NULL; 3593 3594 if (!cookie) 3595 return NULL; 3596 3597 list_for_each_entry(filter, &adapter->cloud_filter_list, list) { 3598 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 3599 return filter; 3600 } 3601 return NULL; 3602 } 3603 3604 /** 3605 * iavf_delete_clsflower - Remove tc flower filters 3606 * @adapter: board private structure 3607 * @cls_flower: Pointer to struct flow_cls_offload 3608 */ 3609 static int iavf_delete_clsflower(struct iavf_adapter *adapter, 3610 struct flow_cls_offload *cls_flower) 3611 { 3612 struct iavf_cloud_filter *filter = NULL; 3613 int err = 0; 3614 3615 spin_lock_bh(&adapter->cloud_filter_list_lock); 3616 filter = iavf_find_cf(adapter, &cls_flower->cookie); 3617 if (filter) { 3618 filter->del = true; 3619 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 3620 } else { 3621 err = -EINVAL; 3622 } 3623 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3624 3625 return err; 3626 } 3627 3628 /** 3629 * iavf_setup_tc_cls_flower - flower classifier offloads 3630 * @adapter: board private structure 3631 * @cls_flower: pointer to flow_cls_offload struct with flow info 3632 */ 3633 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, 3634 struct flow_cls_offload *cls_flower) 3635 { 3636 switch (cls_flower->command) { 3637 case FLOW_CLS_REPLACE: 3638 return iavf_configure_clsflower(adapter, cls_flower); 3639 case FLOW_CLS_DESTROY: 3640 return iavf_delete_clsflower(adapter, cls_flower); 3641 case FLOW_CLS_STATS: 3642 return -EOPNOTSUPP; 3643 default: 3644 return -EOPNOTSUPP; 3645 } 3646 } 3647 3648 /** 3649 * iavf_setup_tc_block_cb - block callback for tc 3650 * @type: type of offload 3651 * @type_data: offload data 3652 * @cb_priv: 3653 * 3654 * This function is the block callback for traffic classes 3655 **/ 3656 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 3657 void *cb_priv) 3658 { 3659 struct iavf_adapter *adapter = cb_priv; 3660 3661 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data)) 3662 return -EOPNOTSUPP; 3663 3664 switch (type) { 3665 case TC_SETUP_CLSFLOWER: 3666 return iavf_setup_tc_cls_flower(cb_priv, type_data); 3667 default: 3668 return -EOPNOTSUPP; 3669 } 3670 } 3671 3672 static LIST_HEAD(iavf_block_cb_list); 3673 3674 /** 3675 * iavf_setup_tc - configure multiple traffic classes 3676 * @netdev: network interface device structure 3677 * @type: type of offload 3678 * @type_data: tc offload data 3679 * 3680 * This function is the callback to ndo_setup_tc in the 3681 * netdev_ops. 3682 * 3683 * Returns 0 on success 3684 **/ 3685 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type, 3686 void *type_data) 3687 { 3688 struct iavf_adapter *adapter = netdev_priv(netdev); 3689 3690 switch (type) { 3691 case TC_SETUP_QDISC_MQPRIO: 3692 return __iavf_setup_tc(netdev, type_data); 3693 case TC_SETUP_BLOCK: 3694 return flow_block_cb_setup_simple(type_data, 3695 &iavf_block_cb_list, 3696 iavf_setup_tc_block_cb, 3697 adapter, adapter, true); 3698 default: 3699 return -EOPNOTSUPP; 3700 } 3701 } 3702 3703 /** 3704 * iavf_open - Called when a network interface is made active 3705 * @netdev: network interface device structure 3706 * 3707 * Returns 0 on success, negative value on failure 3708 * 3709 * The open entry point is called when a network interface is made 3710 * active by the system (IFF_UP). At this point all resources needed 3711 * for transmit and receive operations are allocated, the interrupt 3712 * handler is registered with the OS, the watchdog is started, 3713 * and the stack is notified that the interface is ready. 3714 **/ 3715 static int iavf_open(struct net_device *netdev) 3716 { 3717 struct iavf_adapter *adapter = netdev_priv(netdev); 3718 int err; 3719 3720 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) { 3721 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n"); 3722 return -EIO; 3723 } 3724 3725 while (!mutex_trylock(&adapter->crit_lock)) 3726 usleep_range(500, 1000); 3727 3728 if (adapter->state != __IAVF_DOWN) { 3729 err = -EBUSY; 3730 goto err_unlock; 3731 } 3732 3733 if (adapter->state == __IAVF_RUNNING && 3734 !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) { 3735 dev_dbg(&adapter->pdev->dev, "VF is already open.\n"); 3736 err = 0; 3737 goto err_unlock; 3738 } 3739 3740 /* allocate transmit descriptors */ 3741 err = iavf_setup_all_tx_resources(adapter); 3742 if (err) 3743 goto err_setup_tx; 3744 3745 /* allocate receive descriptors */ 3746 err = iavf_setup_all_rx_resources(adapter); 3747 if (err) 3748 goto err_setup_rx; 3749 3750 /* clear any pending interrupts, may auto mask */ 3751 err = iavf_request_traffic_irqs(adapter, netdev->name); 3752 if (err) 3753 goto err_req_irq; 3754 3755 spin_lock_bh(&adapter->mac_vlan_list_lock); 3756 3757 iavf_add_filter(adapter, adapter->hw.mac.addr); 3758 3759 spin_unlock_bh(&adapter->mac_vlan_list_lock); 3760 3761 /* Restore VLAN filters that were removed with IFF_DOWN */ 3762 iavf_restore_filters(adapter); 3763 3764 iavf_configure(adapter); 3765 3766 iavf_up_complete(adapter); 3767 3768 iavf_irq_enable(adapter, true); 3769 3770 mutex_unlock(&adapter->crit_lock); 3771 3772 return 0; 3773 3774 err_req_irq: 3775 iavf_down(adapter); 3776 iavf_free_traffic_irqs(adapter); 3777 err_setup_rx: 3778 iavf_free_all_rx_resources(adapter); 3779 err_setup_tx: 3780 iavf_free_all_tx_resources(adapter); 3781 err_unlock: 3782 mutex_unlock(&adapter->crit_lock); 3783 3784 return err; 3785 } 3786 3787 /** 3788 * iavf_close - Disables a network interface 3789 * @netdev: network interface device structure 3790 * 3791 * Returns 0, this is not allowed to fail 3792 * 3793 * The close entry point is called when an interface is de-activated 3794 * by the OS. The hardware is still under the drivers control, but 3795 * needs to be disabled. All IRQs except vector 0 (reserved for admin queue) 3796 * are freed, along with all transmit and receive resources. 3797 **/ 3798 static int iavf_close(struct net_device *netdev) 3799 { 3800 struct iavf_adapter *adapter = netdev_priv(netdev); 3801 int status; 3802 3803 if (adapter->state <= __IAVF_DOWN_PENDING) 3804 return 0; 3805 3806 while (!mutex_trylock(&adapter->crit_lock)) 3807 usleep_range(500, 1000); 3808 3809 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 3810 if (CLIENT_ENABLED(adapter)) 3811 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE; 3812 3813 iavf_down(adapter); 3814 iavf_change_state(adapter, __IAVF_DOWN_PENDING); 3815 iavf_free_traffic_irqs(adapter); 3816 3817 mutex_unlock(&adapter->crit_lock); 3818 3819 /* We explicitly don't free resources here because the hardware is 3820 * still active and can DMA into memory. Resources are cleared in 3821 * iavf_virtchnl_completion() after we get confirmation from the PF 3822 * driver that the rings have been stopped. 3823 * 3824 * Also, we wait for state to transition to __IAVF_DOWN before 3825 * returning. State change occurs in iavf_virtchnl_completion() after 3826 * VF resources are released (which occurs after PF driver processes and 3827 * responds to admin queue commands). 3828 */ 3829 3830 status = wait_event_timeout(adapter->down_waitqueue, 3831 adapter->state == __IAVF_DOWN, 3832 msecs_to_jiffies(500)); 3833 if (!status) 3834 netdev_warn(netdev, "Device resources not yet released\n"); 3835 return 0; 3836 } 3837 3838 /** 3839 * iavf_change_mtu - Change the Maximum Transfer Unit 3840 * @netdev: network interface device structure 3841 * @new_mtu: new value for maximum frame size 3842 * 3843 * Returns 0 on success, negative on failure 3844 **/ 3845 static int iavf_change_mtu(struct net_device *netdev, int new_mtu) 3846 { 3847 struct iavf_adapter *adapter = netdev_priv(netdev); 3848 3849 netdev_dbg(netdev, "changing MTU from %d to %d\n", 3850 netdev->mtu, new_mtu); 3851 netdev->mtu = new_mtu; 3852 if (CLIENT_ENABLED(adapter)) { 3853 iavf_notify_client_l2_params(&adapter->vsi); 3854 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED; 3855 } 3856 adapter->flags |= IAVF_FLAG_RESET_NEEDED; 3857 queue_work(iavf_wq, &adapter->reset_task); 3858 3859 return 0; 3860 } 3861 3862 #define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 3863 NETIF_F_HW_VLAN_CTAG_TX | \ 3864 NETIF_F_HW_VLAN_STAG_RX | \ 3865 NETIF_F_HW_VLAN_STAG_TX) 3866 3867 /** 3868 * iavf_set_features - set the netdev feature flags 3869 * @netdev: ptr to the netdev being adjusted 3870 * @features: the feature set that the stack is suggesting 3871 * Note: expects to be called while under rtnl_lock() 3872 **/ 3873 static int iavf_set_features(struct net_device *netdev, 3874 netdev_features_t features) 3875 { 3876 struct iavf_adapter *adapter = netdev_priv(netdev); 3877 3878 /* trigger update on any VLAN feature change */ 3879 if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^ 3880 (features & NETIF_VLAN_OFFLOAD_FEATURES)) 3881 iavf_set_vlan_offload_features(adapter, netdev->features, 3882 features); 3883 3884 return 0; 3885 } 3886 3887 /** 3888 * iavf_features_check - Validate encapsulated packet conforms to limits 3889 * @skb: skb buff 3890 * @dev: This physical port's netdev 3891 * @features: Offload features that the stack believes apply 3892 **/ 3893 static netdev_features_t iavf_features_check(struct sk_buff *skb, 3894 struct net_device *dev, 3895 netdev_features_t features) 3896 { 3897 size_t len; 3898 3899 /* No point in doing any of this if neither checksum nor GSO are 3900 * being requested for this frame. We can rule out both by just 3901 * checking for CHECKSUM_PARTIAL 3902 */ 3903 if (skb->ip_summed != CHECKSUM_PARTIAL) 3904 return features; 3905 3906 /* We cannot support GSO if the MSS is going to be less than 3907 * 64 bytes. If it is then we need to drop support for GSO. 3908 */ 3909 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 3910 features &= ~NETIF_F_GSO_MASK; 3911 3912 /* MACLEN can support at most 63 words */ 3913 len = skb_network_header(skb) - skb->data; 3914 if (len & ~(63 * 2)) 3915 goto out_err; 3916 3917 /* IPLEN and EIPLEN can support at most 127 dwords */ 3918 len = skb_transport_header(skb) - skb_network_header(skb); 3919 if (len & ~(127 * 4)) 3920 goto out_err; 3921 3922 if (skb->encapsulation) { 3923 /* L4TUNLEN can support 127 words */ 3924 len = skb_inner_network_header(skb) - skb_transport_header(skb); 3925 if (len & ~(127 * 2)) 3926 goto out_err; 3927 3928 /* IPLEN can support at most 127 dwords */ 3929 len = skb_inner_transport_header(skb) - 3930 skb_inner_network_header(skb); 3931 if (len & ~(127 * 4)) 3932 goto out_err; 3933 } 3934 3935 /* No need to validate L4LEN as TCP is the only protocol with a 3936 * a flexible value and we support all possible values supported 3937 * by TCP, which is at most 15 dwords 3938 */ 3939 3940 return features; 3941 out_err: 3942 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3943 } 3944 3945 /** 3946 * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off 3947 * @adapter: board private structure 3948 * 3949 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2 3950 * were negotiated determine the VLAN features that can be toggled on and off. 3951 **/ 3952 static netdev_features_t 3953 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter) 3954 { 3955 netdev_features_t hw_features = 0; 3956 3957 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags) 3958 return hw_features; 3959 3960 /* Enable VLAN features if supported */ 3961 if (VLAN_ALLOWED(adapter)) { 3962 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX | 3963 NETIF_F_HW_VLAN_CTAG_RX); 3964 } else if (VLAN_V2_ALLOWED(adapter)) { 3965 struct virtchnl_vlan_caps *vlan_v2_caps = 3966 &adapter->vlan_v2_caps; 3967 struct virtchnl_vlan_supported_caps *stripping_support = 3968 &vlan_v2_caps->offloads.stripping_support; 3969 struct virtchnl_vlan_supported_caps *insertion_support = 3970 &vlan_v2_caps->offloads.insertion_support; 3971 3972 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED && 3973 stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) { 3974 if (stripping_support->outer & 3975 VIRTCHNL_VLAN_ETHERTYPE_8100) 3976 hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3977 if (stripping_support->outer & 3978 VIRTCHNL_VLAN_ETHERTYPE_88A8) 3979 hw_features |= NETIF_F_HW_VLAN_STAG_RX; 3980 } else if (stripping_support->inner != 3981 VIRTCHNL_VLAN_UNSUPPORTED && 3982 stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) { 3983 if (stripping_support->inner & 3984 VIRTCHNL_VLAN_ETHERTYPE_8100) 3985 hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 3986 } 3987 3988 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED && 3989 insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) { 3990 if (insertion_support->outer & 3991 VIRTCHNL_VLAN_ETHERTYPE_8100) 3992 hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 3993 if (insertion_support->outer & 3994 VIRTCHNL_VLAN_ETHERTYPE_88A8) 3995 hw_features |= NETIF_F_HW_VLAN_STAG_TX; 3996 } else if (insertion_support->inner && 3997 insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) { 3998 if (insertion_support->inner & 3999 VIRTCHNL_VLAN_ETHERTYPE_8100) 4000 hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 4001 } 4002 } 4003 4004 return hw_features; 4005 } 4006 4007 /** 4008 * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures 4009 * @adapter: board private structure 4010 * 4011 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2 4012 * were negotiated determine the VLAN features that are enabled by default. 4013 **/ 4014 static netdev_features_t 4015 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter) 4016 { 4017 netdev_features_t features = 0; 4018 4019 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags) 4020 return features; 4021 4022 if (VLAN_ALLOWED(adapter)) { 4023 features |= NETIF_F_HW_VLAN_CTAG_FILTER | 4024 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX; 4025 } else if (VLAN_V2_ALLOWED(adapter)) { 4026 struct virtchnl_vlan_caps *vlan_v2_caps = 4027 &adapter->vlan_v2_caps; 4028 struct virtchnl_vlan_supported_caps *filtering_support = 4029 &vlan_v2_caps->filtering.filtering_support; 4030 struct virtchnl_vlan_supported_caps *stripping_support = 4031 &vlan_v2_caps->offloads.stripping_support; 4032 struct virtchnl_vlan_supported_caps *insertion_support = 4033 &vlan_v2_caps->offloads.insertion_support; 4034 u32 ethertype_init; 4035 4036 /* give priority to outer stripping and don't support both outer 4037 * and inner stripping 4038 */ 4039 ethertype_init = vlan_v2_caps->offloads.ethertype_init; 4040 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4041 if (stripping_support->outer & 4042 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4043 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4044 features |= NETIF_F_HW_VLAN_CTAG_RX; 4045 else if (stripping_support->outer & 4046 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4047 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4048 features |= NETIF_F_HW_VLAN_STAG_RX; 4049 } else if (stripping_support->inner != 4050 VIRTCHNL_VLAN_UNSUPPORTED) { 4051 if (stripping_support->inner & 4052 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4053 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4054 features |= NETIF_F_HW_VLAN_CTAG_RX; 4055 } 4056 4057 /* give priority to outer insertion and don't support both outer 4058 * and inner insertion 4059 */ 4060 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4061 if (insertion_support->outer & 4062 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4063 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4064 features |= NETIF_F_HW_VLAN_CTAG_TX; 4065 else if (insertion_support->outer & 4066 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4067 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4068 features |= NETIF_F_HW_VLAN_STAG_TX; 4069 } else if (insertion_support->inner != 4070 VIRTCHNL_VLAN_UNSUPPORTED) { 4071 if (insertion_support->inner & 4072 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4073 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4074 features |= NETIF_F_HW_VLAN_CTAG_TX; 4075 } 4076 4077 /* give priority to outer filtering and don't bother if both 4078 * outer and inner filtering are enabled 4079 */ 4080 ethertype_init = vlan_v2_caps->filtering.ethertype_init; 4081 if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4082 if (filtering_support->outer & 4083 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4084 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4085 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4086 if (filtering_support->outer & 4087 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4088 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4089 features |= NETIF_F_HW_VLAN_STAG_FILTER; 4090 } else if (filtering_support->inner != 4091 VIRTCHNL_VLAN_UNSUPPORTED) { 4092 if (filtering_support->inner & 4093 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4094 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4095 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4096 if (filtering_support->inner & 4097 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4098 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4099 features |= NETIF_F_HW_VLAN_STAG_FILTER; 4100 } 4101 } 4102 4103 return features; 4104 } 4105 4106 #define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \ 4107 (!(((requested) & (feature_bit)) && \ 4108 !((allowed) & (feature_bit)))) 4109 4110 /** 4111 * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support 4112 * @adapter: board private structure 4113 * @requested_features: stack requested NETDEV features 4114 **/ 4115 static netdev_features_t 4116 iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter, 4117 netdev_features_t requested_features) 4118 { 4119 netdev_features_t allowed_features; 4120 4121 allowed_features = iavf_get_netdev_vlan_hw_features(adapter) | 4122 iavf_get_netdev_vlan_features(adapter); 4123 4124 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4125 allowed_features, 4126 NETIF_F_HW_VLAN_CTAG_TX)) 4127 requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX; 4128 4129 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4130 allowed_features, 4131 NETIF_F_HW_VLAN_CTAG_RX)) 4132 requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX; 4133 4134 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4135 allowed_features, 4136 NETIF_F_HW_VLAN_STAG_TX)) 4137 requested_features &= ~NETIF_F_HW_VLAN_STAG_TX; 4138 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4139 allowed_features, 4140 NETIF_F_HW_VLAN_STAG_RX)) 4141 requested_features &= ~NETIF_F_HW_VLAN_STAG_RX; 4142 4143 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4144 allowed_features, 4145 NETIF_F_HW_VLAN_CTAG_FILTER)) 4146 requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4147 4148 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4149 allowed_features, 4150 NETIF_F_HW_VLAN_STAG_FILTER)) 4151 requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER; 4152 4153 if ((requested_features & 4154 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) && 4155 (requested_features & 4156 (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) && 4157 adapter->vlan_v2_caps.offloads.ethertype_match == 4158 VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) { 4159 netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n"); 4160 requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX | 4161 NETIF_F_HW_VLAN_STAG_TX); 4162 } 4163 4164 return requested_features; 4165 } 4166 4167 /** 4168 * iavf_fix_features - fix up the netdev feature bits 4169 * @netdev: our net device 4170 * @features: desired feature bits 4171 * 4172 * Returns fixed-up features bits 4173 **/ 4174 static netdev_features_t iavf_fix_features(struct net_device *netdev, 4175 netdev_features_t features) 4176 { 4177 struct iavf_adapter *adapter = netdev_priv(netdev); 4178 4179 return iavf_fix_netdev_vlan_features(adapter, features); 4180 } 4181 4182 static const struct net_device_ops iavf_netdev_ops = { 4183 .ndo_open = iavf_open, 4184 .ndo_stop = iavf_close, 4185 .ndo_start_xmit = iavf_xmit_frame, 4186 .ndo_set_rx_mode = iavf_set_rx_mode, 4187 .ndo_validate_addr = eth_validate_addr, 4188 .ndo_set_mac_address = iavf_set_mac, 4189 .ndo_change_mtu = iavf_change_mtu, 4190 .ndo_tx_timeout = iavf_tx_timeout, 4191 .ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid, 4192 .ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid, 4193 .ndo_features_check = iavf_features_check, 4194 .ndo_fix_features = iavf_fix_features, 4195 .ndo_set_features = iavf_set_features, 4196 .ndo_setup_tc = iavf_setup_tc, 4197 }; 4198 4199 /** 4200 * iavf_check_reset_complete - check that VF reset is complete 4201 * @hw: pointer to hw struct 4202 * 4203 * Returns 0 if device is ready to use, or -EBUSY if it's in reset. 4204 **/ 4205 static int iavf_check_reset_complete(struct iavf_hw *hw) 4206 { 4207 u32 rstat; 4208 int i; 4209 4210 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { 4211 rstat = rd32(hw, IAVF_VFGEN_RSTAT) & 4212 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 4213 if ((rstat == VIRTCHNL_VFR_VFACTIVE) || 4214 (rstat == VIRTCHNL_VFR_COMPLETED)) 4215 return 0; 4216 usleep_range(10, 20); 4217 } 4218 return -EBUSY; 4219 } 4220 4221 /** 4222 * iavf_process_config - Process the config information we got from the PF 4223 * @adapter: board private structure 4224 * 4225 * Verify that we have a valid config struct, and set up our netdev features 4226 * and our VSI struct. 4227 **/ 4228 int iavf_process_config(struct iavf_adapter *adapter) 4229 { 4230 struct virtchnl_vf_resource *vfres = adapter->vf_res; 4231 netdev_features_t hw_vlan_features, vlan_features; 4232 struct net_device *netdev = adapter->netdev; 4233 netdev_features_t hw_enc_features; 4234 netdev_features_t hw_features; 4235 4236 hw_enc_features = NETIF_F_SG | 4237 NETIF_F_IP_CSUM | 4238 NETIF_F_IPV6_CSUM | 4239 NETIF_F_HIGHDMA | 4240 NETIF_F_SOFT_FEATURES | 4241 NETIF_F_TSO | 4242 NETIF_F_TSO_ECN | 4243 NETIF_F_TSO6 | 4244 NETIF_F_SCTP_CRC | 4245 NETIF_F_RXHASH | 4246 NETIF_F_RXCSUM | 4247 0; 4248 4249 /* advertise to stack only if offloads for encapsulated packets is 4250 * supported 4251 */ 4252 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) { 4253 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 4254 NETIF_F_GSO_GRE | 4255 NETIF_F_GSO_GRE_CSUM | 4256 NETIF_F_GSO_IPXIP4 | 4257 NETIF_F_GSO_IPXIP6 | 4258 NETIF_F_GSO_UDP_TUNNEL_CSUM | 4259 NETIF_F_GSO_PARTIAL | 4260 0; 4261 4262 if (!(vfres->vf_cap_flags & 4263 VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM)) 4264 netdev->gso_partial_features |= 4265 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4266 4267 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 4268 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 4269 netdev->hw_enc_features |= hw_enc_features; 4270 } 4271 /* record features VLANs can make use of */ 4272 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 4273 4274 /* Write features and hw_features separately to avoid polluting 4275 * with, or dropping, features that are set when we registered. 4276 */ 4277 hw_features = hw_enc_features; 4278 4279 /* get HW VLAN features that can be toggled */ 4280 hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter); 4281 4282 /* Enable cloud filter if ADQ is supported */ 4283 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) 4284 hw_features |= NETIF_F_HW_TC; 4285 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO) 4286 hw_features |= NETIF_F_GSO_UDP_L4; 4287 4288 netdev->hw_features |= hw_features | hw_vlan_features; 4289 vlan_features = iavf_get_netdev_vlan_features(adapter); 4290 4291 netdev->features |= hw_features | vlan_features; 4292 4293 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) 4294 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4295 4296 netdev->priv_flags |= IFF_UNICAST_FLT; 4297 4298 /* Do not turn on offloads when they are requested to be turned off. 4299 * TSO needs minimum 576 bytes to work correctly. 4300 */ 4301 if (netdev->wanted_features) { 4302 if (!(netdev->wanted_features & NETIF_F_TSO) || 4303 netdev->mtu < 576) 4304 netdev->features &= ~NETIF_F_TSO; 4305 if (!(netdev->wanted_features & NETIF_F_TSO6) || 4306 netdev->mtu < 576) 4307 netdev->features &= ~NETIF_F_TSO6; 4308 if (!(netdev->wanted_features & NETIF_F_TSO_ECN)) 4309 netdev->features &= ~NETIF_F_TSO_ECN; 4310 if (!(netdev->wanted_features & NETIF_F_GRO)) 4311 netdev->features &= ~NETIF_F_GRO; 4312 if (!(netdev->wanted_features & NETIF_F_GSO)) 4313 netdev->features &= ~NETIF_F_GSO; 4314 } 4315 4316 return 0; 4317 } 4318 4319 /** 4320 * iavf_shutdown - Shutdown the device in preparation for a reboot 4321 * @pdev: pci device structure 4322 **/ 4323 static void iavf_shutdown(struct pci_dev *pdev) 4324 { 4325 struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev); 4326 struct net_device *netdev = adapter->netdev; 4327 4328 netif_device_detach(netdev); 4329 4330 if (netif_running(netdev)) 4331 iavf_close(netdev); 4332 4333 if (iavf_lock_timeout(&adapter->crit_lock, 5000)) 4334 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__); 4335 /* Prevent the watchdog from running. */ 4336 iavf_change_state(adapter, __IAVF_REMOVE); 4337 adapter->aq_required = 0; 4338 mutex_unlock(&adapter->crit_lock); 4339 4340 #ifdef CONFIG_PM 4341 pci_save_state(pdev); 4342 4343 #endif 4344 pci_disable_device(pdev); 4345 } 4346 4347 /** 4348 * iavf_probe - Device Initialization Routine 4349 * @pdev: PCI device information struct 4350 * @ent: entry in iavf_pci_tbl 4351 * 4352 * Returns 0 on success, negative on failure 4353 * 4354 * iavf_probe initializes an adapter identified by a pci_dev structure. 4355 * The OS initialization, configuring of the adapter private structure, 4356 * and a hardware reset occur. 4357 **/ 4358 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 4359 { 4360 struct net_device *netdev; 4361 struct iavf_adapter *adapter = NULL; 4362 struct iavf_hw *hw = NULL; 4363 int err; 4364 4365 err = pci_enable_device(pdev); 4366 if (err) 4367 return err; 4368 4369 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4370 if (err) { 4371 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 4372 if (err) { 4373 dev_err(&pdev->dev, 4374 "DMA configuration failed: 0x%x\n", err); 4375 goto err_dma; 4376 } 4377 } 4378 4379 err = pci_request_regions(pdev, iavf_driver_name); 4380 if (err) { 4381 dev_err(&pdev->dev, 4382 "pci_request_regions failed 0x%x\n", err); 4383 goto err_pci_reg; 4384 } 4385 4386 pci_enable_pcie_error_reporting(pdev); 4387 4388 pci_set_master(pdev); 4389 4390 netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter), 4391 IAVF_MAX_REQ_QUEUES); 4392 if (!netdev) { 4393 err = -ENOMEM; 4394 goto err_alloc_etherdev; 4395 } 4396 4397 SET_NETDEV_DEV(netdev, &pdev->dev); 4398 4399 pci_set_drvdata(pdev, netdev); 4400 adapter = netdev_priv(netdev); 4401 4402 adapter->netdev = netdev; 4403 adapter->pdev = pdev; 4404 4405 hw = &adapter->hw; 4406 hw->back = adapter; 4407 4408 adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1; 4409 iavf_change_state(adapter, __IAVF_STARTUP); 4410 4411 /* Call save state here because it relies on the adapter struct. */ 4412 pci_save_state(pdev); 4413 4414 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 4415 pci_resource_len(pdev, 0)); 4416 if (!hw->hw_addr) { 4417 err = -EIO; 4418 goto err_ioremap; 4419 } 4420 hw->vendor_id = pdev->vendor; 4421 hw->device_id = pdev->device; 4422 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 4423 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4424 hw->subsystem_device_id = pdev->subsystem_device; 4425 hw->bus.device = PCI_SLOT(pdev->devfn); 4426 hw->bus.func = PCI_FUNC(pdev->devfn); 4427 hw->bus.bus_id = pdev->bus->number; 4428 4429 /* set up the locks for the AQ, do this only once in probe 4430 * and destroy them only once in remove 4431 */ 4432 mutex_init(&adapter->crit_lock); 4433 mutex_init(&adapter->client_lock); 4434 mutex_init(&adapter->remove_lock); 4435 mutex_init(&hw->aq.asq_mutex); 4436 mutex_init(&hw->aq.arq_mutex); 4437 4438 spin_lock_init(&adapter->mac_vlan_list_lock); 4439 spin_lock_init(&adapter->cloud_filter_list_lock); 4440 spin_lock_init(&adapter->fdir_fltr_lock); 4441 spin_lock_init(&adapter->adv_rss_lock); 4442 4443 INIT_LIST_HEAD(&adapter->mac_filter_list); 4444 INIT_LIST_HEAD(&adapter->vlan_filter_list); 4445 INIT_LIST_HEAD(&adapter->cloud_filter_list); 4446 INIT_LIST_HEAD(&adapter->fdir_list_head); 4447 INIT_LIST_HEAD(&adapter->adv_rss_list_head); 4448 4449 INIT_WORK(&adapter->reset_task, iavf_reset_task); 4450 INIT_WORK(&adapter->adminq_task, iavf_adminq_task); 4451 INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task); 4452 INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task); 4453 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 4454 msecs_to_jiffies(5 * (pdev->devfn & 0x07))); 4455 4456 /* Setup the wait queue for indicating transition to down status */ 4457 init_waitqueue_head(&adapter->down_waitqueue); 4458 4459 return 0; 4460 4461 err_ioremap: 4462 free_netdev(netdev); 4463 err_alloc_etherdev: 4464 pci_disable_pcie_error_reporting(pdev); 4465 pci_release_regions(pdev); 4466 err_pci_reg: 4467 err_dma: 4468 pci_disable_device(pdev); 4469 return err; 4470 } 4471 4472 /** 4473 * iavf_suspend - Power management suspend routine 4474 * @dev_d: device info pointer 4475 * 4476 * Called when the system (VM) is entering sleep/suspend. 4477 **/ 4478 static int __maybe_unused iavf_suspend(struct device *dev_d) 4479 { 4480 struct net_device *netdev = dev_get_drvdata(dev_d); 4481 struct iavf_adapter *adapter = netdev_priv(netdev); 4482 4483 netif_device_detach(netdev); 4484 4485 while (!mutex_trylock(&adapter->crit_lock)) 4486 usleep_range(500, 1000); 4487 4488 if (netif_running(netdev)) { 4489 rtnl_lock(); 4490 iavf_down(adapter); 4491 rtnl_unlock(); 4492 } 4493 iavf_free_misc_irq(adapter); 4494 iavf_reset_interrupt_capability(adapter); 4495 4496 mutex_unlock(&adapter->crit_lock); 4497 4498 return 0; 4499 } 4500 4501 /** 4502 * iavf_resume - Power management resume routine 4503 * @dev_d: device info pointer 4504 * 4505 * Called when the system (VM) is resumed from sleep/suspend. 4506 **/ 4507 static int __maybe_unused iavf_resume(struct device *dev_d) 4508 { 4509 struct pci_dev *pdev = to_pci_dev(dev_d); 4510 struct iavf_adapter *adapter; 4511 u32 err; 4512 4513 adapter = iavf_pdev_to_adapter(pdev); 4514 4515 pci_set_master(pdev); 4516 4517 rtnl_lock(); 4518 err = iavf_set_interrupt_capability(adapter); 4519 if (err) { 4520 rtnl_unlock(); 4521 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n"); 4522 return err; 4523 } 4524 err = iavf_request_misc_irq(adapter); 4525 rtnl_unlock(); 4526 if (err) { 4527 dev_err(&pdev->dev, "Cannot get interrupt vector.\n"); 4528 return err; 4529 } 4530 4531 queue_work(iavf_wq, &adapter->reset_task); 4532 4533 netif_device_attach(adapter->netdev); 4534 4535 return err; 4536 } 4537 4538 /** 4539 * iavf_remove - Device Removal Routine 4540 * @pdev: PCI device information struct 4541 * 4542 * iavf_remove is called by the PCI subsystem to alert the driver 4543 * that it should release a PCI device. The could be caused by a 4544 * Hot-Plug event, or because the driver is going to be removed from 4545 * memory. 4546 **/ 4547 static void iavf_remove(struct pci_dev *pdev) 4548 { 4549 struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev); 4550 enum iavf_state_t prev_state = adapter->last_state; 4551 struct net_device *netdev = adapter->netdev; 4552 struct iavf_fdir_fltr *fdir, *fdirtmp; 4553 struct iavf_vlan_filter *vlf, *vlftmp; 4554 struct iavf_adv_rss *rss, *rsstmp; 4555 struct iavf_mac_filter *f, *ftmp; 4556 struct iavf_cloud_filter *cf, *cftmp; 4557 struct iavf_hw *hw = &adapter->hw; 4558 int err; 4559 /* Indicate we are in remove and not to run reset_task */ 4560 mutex_lock(&adapter->remove_lock); 4561 cancel_work_sync(&adapter->reset_task); 4562 cancel_delayed_work_sync(&adapter->watchdog_task); 4563 cancel_delayed_work_sync(&adapter->client_task); 4564 if (adapter->netdev_registered) { 4565 unregister_netdev(netdev); 4566 adapter->netdev_registered = false; 4567 } 4568 if (CLIENT_ALLOWED(adapter)) { 4569 err = iavf_lan_del_device(adapter); 4570 if (err) 4571 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 4572 err); 4573 } 4574 4575 iavf_request_reset(adapter); 4576 msleep(50); 4577 /* If the FW isn't responding, kick it once, but only once. */ 4578 if (!iavf_asq_done(hw)) { 4579 iavf_request_reset(adapter); 4580 msleep(50); 4581 } 4582 if (iavf_lock_timeout(&adapter->crit_lock, 5000)) 4583 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__); 4584 4585 dev_info(&adapter->pdev->dev, "Removing device\n"); 4586 /* Shut down all the garbage mashers on the detention level */ 4587 iavf_change_state(adapter, __IAVF_REMOVE); 4588 adapter->aq_required = 0; 4589 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 4590 4591 iavf_free_all_tx_resources(adapter); 4592 iavf_free_all_rx_resources(adapter); 4593 iavf_misc_irq_disable(adapter); 4594 iavf_free_misc_irq(adapter); 4595 4596 /* In case we enter iavf_remove from erroneous state, free traffic irqs 4597 * here, so as to not cause a kernel crash, when calling 4598 * iavf_reset_interrupt_capability. 4599 */ 4600 if ((adapter->last_state == __IAVF_RESETTING && 4601 prev_state != __IAVF_DOWN) || 4602 (adapter->last_state == __IAVF_RUNNING && 4603 !(netdev->flags & IFF_UP))) 4604 iavf_free_traffic_irqs(adapter); 4605 4606 iavf_reset_interrupt_capability(adapter); 4607 iavf_free_q_vectors(adapter); 4608 4609 cancel_delayed_work_sync(&adapter->watchdog_task); 4610 4611 cancel_work_sync(&adapter->adminq_task); 4612 4613 iavf_free_rss(adapter); 4614 4615 if (hw->aq.asq.count) 4616 iavf_shutdown_adminq(hw); 4617 4618 /* destroy the locks only once, here */ 4619 mutex_destroy(&hw->aq.arq_mutex); 4620 mutex_destroy(&hw->aq.asq_mutex); 4621 mutex_destroy(&adapter->client_lock); 4622 mutex_unlock(&adapter->crit_lock); 4623 mutex_destroy(&adapter->crit_lock); 4624 mutex_unlock(&adapter->remove_lock); 4625 mutex_destroy(&adapter->remove_lock); 4626 4627 iounmap(hw->hw_addr); 4628 pci_release_regions(pdev); 4629 iavf_free_queues(adapter); 4630 kfree(adapter->vf_res); 4631 spin_lock_bh(&adapter->mac_vlan_list_lock); 4632 /* If we got removed before an up/down sequence, we've got a filter 4633 * hanging out there that we need to get rid of. 4634 */ 4635 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 4636 list_del(&f->list); 4637 kfree(f); 4638 } 4639 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list, 4640 list) { 4641 list_del(&vlf->list); 4642 kfree(vlf); 4643 } 4644 4645 spin_unlock_bh(&adapter->mac_vlan_list_lock); 4646 4647 spin_lock_bh(&adapter->cloud_filter_list_lock); 4648 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 4649 list_del(&cf->list); 4650 kfree(cf); 4651 } 4652 spin_unlock_bh(&adapter->cloud_filter_list_lock); 4653 4654 spin_lock_bh(&adapter->fdir_fltr_lock); 4655 list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) { 4656 list_del(&fdir->list); 4657 kfree(fdir); 4658 } 4659 spin_unlock_bh(&adapter->fdir_fltr_lock); 4660 4661 spin_lock_bh(&adapter->adv_rss_lock); 4662 list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head, 4663 list) { 4664 list_del(&rss->list); 4665 kfree(rss); 4666 } 4667 spin_unlock_bh(&adapter->adv_rss_lock); 4668 4669 free_netdev(netdev); 4670 4671 pci_disable_pcie_error_reporting(pdev); 4672 4673 pci_disable_device(pdev); 4674 } 4675 4676 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume); 4677 4678 static struct pci_driver iavf_driver = { 4679 .name = iavf_driver_name, 4680 .id_table = iavf_pci_tbl, 4681 .probe = iavf_probe, 4682 .remove = iavf_remove, 4683 .driver.pm = &iavf_pm_ops, 4684 .shutdown = iavf_shutdown, 4685 }; 4686 4687 /** 4688 * iavf_init_module - Driver Registration Routine 4689 * 4690 * iavf_init_module is the first routine called when the driver is 4691 * loaded. All it does is register with the PCI subsystem. 4692 **/ 4693 static int __init iavf_init_module(void) 4694 { 4695 int ret; 4696 4697 pr_info("iavf: %s\n", iavf_driver_string); 4698 4699 pr_info("%s\n", iavf_copyright); 4700 4701 iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1, 4702 iavf_driver_name); 4703 if (!iavf_wq) { 4704 pr_err("%s: Failed to create workqueue\n", iavf_driver_name); 4705 return -ENOMEM; 4706 } 4707 ret = pci_register_driver(&iavf_driver); 4708 return ret; 4709 } 4710 4711 module_init(iavf_init_module); 4712 4713 /** 4714 * iavf_exit_module - Driver Exit Cleanup Routine 4715 * 4716 * iavf_exit_module is called just before the driver is removed 4717 * from memory. 4718 **/ 4719 static void __exit iavf_exit_module(void) 4720 { 4721 pci_unregister_driver(&iavf_driver); 4722 destroy_workqueue(iavf_wq); 4723 } 4724 4725 module_exit(iavf_exit_module); 4726 4727 /* iavf_main.c */ 4728