1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include "iavf.h" 5 #include "iavf_prototype.h" 6 #include "iavf_client.h" 7 /* All iavf tracepoints are defined by the include below, which must 8 * be included exactly once across the whole kernel with 9 * CREATE_TRACE_POINTS defined 10 */ 11 #define CREATE_TRACE_POINTS 12 #include "iavf_trace.h" 13 14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter); 15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter); 16 static int iavf_close(struct net_device *netdev); 17 static void iavf_init_get_resources(struct iavf_adapter *adapter); 18 static int iavf_check_reset_complete(struct iavf_hw *hw); 19 20 char iavf_driver_name[] = "iavf"; 21 static const char iavf_driver_string[] = 22 "Intel(R) Ethernet Adaptive Virtual Function Network Driver"; 23 24 static const char iavf_copyright[] = 25 "Copyright (c) 2013 - 2018 Intel Corporation."; 26 27 /* iavf_pci_tbl - PCI Device ID Table 28 * 29 * Wildcard entries (PCI_ANY_ID) should come last 30 * Last entry must be all 0s 31 * 32 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 33 * Class, Class Mask, private data (not used) } 34 */ 35 static const struct pci_device_id iavf_pci_tbl[] = { 36 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0}, 37 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0}, 38 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0}, 39 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0}, 40 /* required last entry */ 41 {0, } 42 }; 43 44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl); 45 46 MODULE_ALIAS("i40evf"); 47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver"); 49 MODULE_LICENSE("GPL v2"); 50 51 static const struct net_device_ops iavf_netdev_ops; 52 struct workqueue_struct *iavf_wq; 53 54 int iavf_status_to_errno(enum iavf_status status) 55 { 56 switch (status) { 57 case IAVF_SUCCESS: 58 return 0; 59 case IAVF_ERR_PARAM: 60 case IAVF_ERR_MAC_TYPE: 61 case IAVF_ERR_INVALID_MAC_ADDR: 62 case IAVF_ERR_INVALID_LINK_SETTINGS: 63 case IAVF_ERR_INVALID_PD_ID: 64 case IAVF_ERR_INVALID_QP_ID: 65 case IAVF_ERR_INVALID_CQ_ID: 66 case IAVF_ERR_INVALID_CEQ_ID: 67 case IAVF_ERR_INVALID_AEQ_ID: 68 case IAVF_ERR_INVALID_SIZE: 69 case IAVF_ERR_INVALID_ARP_INDEX: 70 case IAVF_ERR_INVALID_FPM_FUNC_ID: 71 case IAVF_ERR_QP_INVALID_MSG_SIZE: 72 case IAVF_ERR_INVALID_FRAG_COUNT: 73 case IAVF_ERR_INVALID_ALIGNMENT: 74 case IAVF_ERR_INVALID_PUSH_PAGE_INDEX: 75 case IAVF_ERR_INVALID_IMM_DATA_SIZE: 76 case IAVF_ERR_INVALID_VF_ID: 77 case IAVF_ERR_INVALID_HMCFN_ID: 78 case IAVF_ERR_INVALID_PBLE_INDEX: 79 case IAVF_ERR_INVALID_SD_INDEX: 80 case IAVF_ERR_INVALID_PAGE_DESC_INDEX: 81 case IAVF_ERR_INVALID_SD_TYPE: 82 case IAVF_ERR_INVALID_HMC_OBJ_INDEX: 83 case IAVF_ERR_INVALID_HMC_OBJ_COUNT: 84 case IAVF_ERR_INVALID_SRQ_ARM_LIMIT: 85 return -EINVAL; 86 case IAVF_ERR_NVM: 87 case IAVF_ERR_NVM_CHECKSUM: 88 case IAVF_ERR_PHY: 89 case IAVF_ERR_CONFIG: 90 case IAVF_ERR_UNKNOWN_PHY: 91 case IAVF_ERR_LINK_SETUP: 92 case IAVF_ERR_ADAPTER_STOPPED: 93 case IAVF_ERR_PRIMARY_REQUESTS_PENDING: 94 case IAVF_ERR_AUTONEG_NOT_COMPLETE: 95 case IAVF_ERR_RESET_FAILED: 96 case IAVF_ERR_BAD_PTR: 97 case IAVF_ERR_SWFW_SYNC: 98 case IAVF_ERR_QP_TOOMANY_WRS_POSTED: 99 case IAVF_ERR_QUEUE_EMPTY: 100 case IAVF_ERR_FLUSHED_QUEUE: 101 case IAVF_ERR_OPCODE_MISMATCH: 102 case IAVF_ERR_CQP_COMPL_ERROR: 103 case IAVF_ERR_BACKING_PAGE_ERROR: 104 case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE: 105 case IAVF_ERR_MEMCPY_FAILED: 106 case IAVF_ERR_SRQ_ENABLED: 107 case IAVF_ERR_ADMIN_QUEUE_ERROR: 108 case IAVF_ERR_ADMIN_QUEUE_FULL: 109 case IAVF_ERR_BAD_IWARP_CQE: 110 case IAVF_ERR_NVM_BLANK_MODE: 111 case IAVF_ERR_PE_DOORBELL_NOT_ENABLED: 112 case IAVF_ERR_DIAG_TEST_FAILED: 113 case IAVF_ERR_FIRMWARE_API_VERSION: 114 case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR: 115 return -EIO; 116 case IAVF_ERR_DEVICE_NOT_SUPPORTED: 117 return -ENODEV; 118 case IAVF_ERR_NO_AVAILABLE_VSI: 119 case IAVF_ERR_RING_FULL: 120 return -ENOSPC; 121 case IAVF_ERR_NO_MEMORY: 122 return -ENOMEM; 123 case IAVF_ERR_TIMEOUT: 124 case IAVF_ERR_ADMIN_QUEUE_TIMEOUT: 125 return -ETIMEDOUT; 126 case IAVF_ERR_NOT_IMPLEMENTED: 127 case IAVF_NOT_SUPPORTED: 128 return -EOPNOTSUPP; 129 case IAVF_ERR_ADMIN_QUEUE_NO_WORK: 130 return -EALREADY; 131 case IAVF_ERR_NOT_READY: 132 return -EBUSY; 133 case IAVF_ERR_BUF_TOO_SHORT: 134 return -EMSGSIZE; 135 } 136 137 return -EIO; 138 } 139 140 int virtchnl_status_to_errno(enum virtchnl_status_code v_status) 141 { 142 switch (v_status) { 143 case VIRTCHNL_STATUS_SUCCESS: 144 return 0; 145 case VIRTCHNL_STATUS_ERR_PARAM: 146 case VIRTCHNL_STATUS_ERR_INVALID_VF_ID: 147 return -EINVAL; 148 case VIRTCHNL_STATUS_ERR_NO_MEMORY: 149 return -ENOMEM; 150 case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH: 151 case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR: 152 case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR: 153 return -EIO; 154 case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED: 155 return -EOPNOTSUPP; 156 } 157 158 return -EIO; 159 } 160 161 /** 162 * iavf_pdev_to_adapter - go from pci_dev to adapter 163 * @pdev: pci_dev pointer 164 */ 165 static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev) 166 { 167 return netdev_priv(pci_get_drvdata(pdev)); 168 } 169 170 /** 171 * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code 172 * @hw: pointer to the HW structure 173 * @mem: ptr to mem struct to fill out 174 * @size: size of memory requested 175 * @alignment: what to align the allocation to 176 **/ 177 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw, 178 struct iavf_dma_mem *mem, 179 u64 size, u32 alignment) 180 { 181 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 182 183 if (!mem) 184 return IAVF_ERR_PARAM; 185 186 mem->size = ALIGN(size, alignment); 187 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size, 188 (dma_addr_t *)&mem->pa, GFP_KERNEL); 189 if (mem->va) 190 return 0; 191 else 192 return IAVF_ERR_NO_MEMORY; 193 } 194 195 /** 196 * iavf_free_dma_mem_d - OS specific memory free for shared code 197 * @hw: pointer to the HW structure 198 * @mem: ptr to mem struct to free 199 **/ 200 enum iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw, 201 struct iavf_dma_mem *mem) 202 { 203 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back; 204 205 if (!mem || !mem->va) 206 return IAVF_ERR_PARAM; 207 dma_free_coherent(&adapter->pdev->dev, mem->size, 208 mem->va, (dma_addr_t)mem->pa); 209 return 0; 210 } 211 212 /** 213 * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code 214 * @hw: pointer to the HW structure 215 * @mem: ptr to mem struct to fill out 216 * @size: size of memory requested 217 **/ 218 enum iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw, 219 struct iavf_virt_mem *mem, u32 size) 220 { 221 if (!mem) 222 return IAVF_ERR_PARAM; 223 224 mem->size = size; 225 mem->va = kzalloc(size, GFP_KERNEL); 226 227 if (mem->va) 228 return 0; 229 else 230 return IAVF_ERR_NO_MEMORY; 231 } 232 233 /** 234 * iavf_free_virt_mem_d - OS specific memory free for shared code 235 * @hw: pointer to the HW structure 236 * @mem: ptr to mem struct to free 237 **/ 238 enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw, 239 struct iavf_virt_mem *mem) 240 { 241 if (!mem) 242 return IAVF_ERR_PARAM; 243 244 /* it's ok to kfree a NULL pointer */ 245 kfree(mem->va); 246 247 return 0; 248 } 249 250 /** 251 * iavf_lock_timeout - try to lock mutex but give up after timeout 252 * @lock: mutex that should be locked 253 * @msecs: timeout in msecs 254 * 255 * Returns 0 on success, negative on failure 256 **/ 257 int iavf_lock_timeout(struct mutex *lock, unsigned int msecs) 258 { 259 unsigned int wait, delay = 10; 260 261 for (wait = 0; wait < msecs; wait += delay) { 262 if (mutex_trylock(lock)) 263 return 0; 264 265 msleep(delay); 266 } 267 268 return -1; 269 } 270 271 /** 272 * iavf_schedule_reset - Set the flags and schedule a reset event 273 * @adapter: board private structure 274 **/ 275 void iavf_schedule_reset(struct iavf_adapter *adapter) 276 { 277 if (!(adapter->flags & 278 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) { 279 adapter->flags |= IAVF_FLAG_RESET_NEEDED; 280 queue_work(iavf_wq, &adapter->reset_task); 281 } 282 } 283 284 /** 285 * iavf_schedule_request_stats - Set the flags and schedule statistics request 286 * @adapter: board private structure 287 * 288 * Sets IAVF_FLAG_AQ_REQUEST_STATS flag so iavf_watchdog_task() will explicitly 289 * request and refresh ethtool stats 290 **/ 291 void iavf_schedule_request_stats(struct iavf_adapter *adapter) 292 { 293 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_STATS; 294 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 295 } 296 297 /** 298 * iavf_tx_timeout - Respond to a Tx Hang 299 * @netdev: network interface device structure 300 * @txqueue: queue number that is timing out 301 **/ 302 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue) 303 { 304 struct iavf_adapter *adapter = netdev_priv(netdev); 305 306 adapter->tx_timeout_count++; 307 iavf_schedule_reset(adapter); 308 } 309 310 /** 311 * iavf_misc_irq_disable - Mask off interrupt generation on the NIC 312 * @adapter: board private structure 313 **/ 314 static void iavf_misc_irq_disable(struct iavf_adapter *adapter) 315 { 316 struct iavf_hw *hw = &adapter->hw; 317 318 if (!adapter->msix_entries) 319 return; 320 321 wr32(hw, IAVF_VFINT_DYN_CTL01, 0); 322 323 iavf_flush(hw); 324 325 synchronize_irq(adapter->msix_entries[0].vector); 326 } 327 328 /** 329 * iavf_misc_irq_enable - Enable default interrupt generation settings 330 * @adapter: board private structure 331 **/ 332 static void iavf_misc_irq_enable(struct iavf_adapter *adapter) 333 { 334 struct iavf_hw *hw = &adapter->hw; 335 336 wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK | 337 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK); 338 wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK); 339 340 iavf_flush(hw); 341 } 342 343 /** 344 * iavf_irq_disable - Mask off interrupt generation on the NIC 345 * @adapter: board private structure 346 **/ 347 static void iavf_irq_disable(struct iavf_adapter *adapter) 348 { 349 int i; 350 struct iavf_hw *hw = &adapter->hw; 351 352 if (!adapter->msix_entries) 353 return; 354 355 for (i = 1; i < adapter->num_msix_vectors; i++) { 356 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0); 357 synchronize_irq(adapter->msix_entries[i].vector); 358 } 359 iavf_flush(hw); 360 } 361 362 /** 363 * iavf_irq_enable_queues - Enable interrupt for specified queues 364 * @adapter: board private structure 365 * @mask: bitmap of queues to enable 366 **/ 367 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask) 368 { 369 struct iavf_hw *hw = &adapter->hw; 370 int i; 371 372 for (i = 1; i < adapter->num_msix_vectors; i++) { 373 if (mask & BIT(i - 1)) { 374 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 375 IAVF_VFINT_DYN_CTLN1_INTENA_MASK | 376 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK); 377 } 378 } 379 } 380 381 /** 382 * iavf_irq_enable - Enable default interrupt generation settings 383 * @adapter: board private structure 384 * @flush: boolean value whether to run rd32() 385 **/ 386 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush) 387 { 388 struct iavf_hw *hw = &adapter->hw; 389 390 iavf_misc_irq_enable(adapter); 391 iavf_irq_enable_queues(adapter, ~0); 392 393 if (flush) 394 iavf_flush(hw); 395 } 396 397 /** 398 * iavf_msix_aq - Interrupt handler for vector 0 399 * @irq: interrupt number 400 * @data: pointer to netdev 401 **/ 402 static irqreturn_t iavf_msix_aq(int irq, void *data) 403 { 404 struct net_device *netdev = data; 405 struct iavf_adapter *adapter = netdev_priv(netdev); 406 struct iavf_hw *hw = &adapter->hw; 407 408 /* handle non-queue interrupts, these reads clear the registers */ 409 rd32(hw, IAVF_VFINT_ICR01); 410 rd32(hw, IAVF_VFINT_ICR0_ENA1); 411 412 if (adapter->state != __IAVF_REMOVE) 413 /* schedule work on the private workqueue */ 414 queue_work(iavf_wq, &adapter->adminq_task); 415 416 return IRQ_HANDLED; 417 } 418 419 /** 420 * iavf_msix_clean_rings - MSIX mode Interrupt Handler 421 * @irq: interrupt number 422 * @data: pointer to a q_vector 423 **/ 424 static irqreturn_t iavf_msix_clean_rings(int irq, void *data) 425 { 426 struct iavf_q_vector *q_vector = data; 427 428 if (!q_vector->tx.ring && !q_vector->rx.ring) 429 return IRQ_HANDLED; 430 431 napi_schedule_irqoff(&q_vector->napi); 432 433 return IRQ_HANDLED; 434 } 435 436 /** 437 * iavf_map_vector_to_rxq - associate irqs with rx queues 438 * @adapter: board private structure 439 * @v_idx: interrupt number 440 * @r_idx: queue number 441 **/ 442 static void 443 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx) 444 { 445 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 446 struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx]; 447 struct iavf_hw *hw = &adapter->hw; 448 449 rx_ring->q_vector = q_vector; 450 rx_ring->next = q_vector->rx.ring; 451 rx_ring->vsi = &adapter->vsi; 452 q_vector->rx.ring = rx_ring; 453 q_vector->rx.count++; 454 q_vector->rx.next_update = jiffies + 1; 455 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting); 456 q_vector->ring_mask |= BIT(r_idx); 457 wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx), 458 q_vector->rx.current_itr >> 1); 459 q_vector->rx.current_itr = q_vector->rx.target_itr; 460 } 461 462 /** 463 * iavf_map_vector_to_txq - associate irqs with tx queues 464 * @adapter: board private structure 465 * @v_idx: interrupt number 466 * @t_idx: queue number 467 **/ 468 static void 469 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx) 470 { 471 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx]; 472 struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx]; 473 struct iavf_hw *hw = &adapter->hw; 474 475 tx_ring->q_vector = q_vector; 476 tx_ring->next = q_vector->tx.ring; 477 tx_ring->vsi = &adapter->vsi; 478 q_vector->tx.ring = tx_ring; 479 q_vector->tx.count++; 480 q_vector->tx.next_update = jiffies + 1; 481 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting); 482 q_vector->num_ringpairs++; 483 wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx), 484 q_vector->tx.target_itr >> 1); 485 q_vector->tx.current_itr = q_vector->tx.target_itr; 486 } 487 488 /** 489 * iavf_map_rings_to_vectors - Maps descriptor rings to vectors 490 * @adapter: board private structure to initialize 491 * 492 * This function maps descriptor rings to the queue-specific vectors 493 * we were allotted through the MSI-X enabling code. Ideally, we'd have 494 * one vector per ring/queue, but on a constrained vector budget, we 495 * group the rings as "efficiently" as possible. You would add new 496 * mapping configurations in here. 497 **/ 498 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter) 499 { 500 int rings_remaining = adapter->num_active_queues; 501 int ridx = 0, vidx = 0; 502 int q_vectors; 503 504 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 505 506 for (; ridx < rings_remaining; ridx++) { 507 iavf_map_vector_to_rxq(adapter, vidx, ridx); 508 iavf_map_vector_to_txq(adapter, vidx, ridx); 509 510 /* In the case where we have more queues than vectors, continue 511 * round-robin on vectors until all queues are mapped. 512 */ 513 if (++vidx >= q_vectors) 514 vidx = 0; 515 } 516 517 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 518 } 519 520 /** 521 * iavf_irq_affinity_notify - Callback for affinity changes 522 * @notify: context as to what irq was changed 523 * @mask: the new affinity mask 524 * 525 * This is a callback function used by the irq_set_affinity_notifier function 526 * so that we may register to receive changes to the irq affinity masks. 527 **/ 528 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify, 529 const cpumask_t *mask) 530 { 531 struct iavf_q_vector *q_vector = 532 container_of(notify, struct iavf_q_vector, affinity_notify); 533 534 cpumask_copy(&q_vector->affinity_mask, mask); 535 } 536 537 /** 538 * iavf_irq_affinity_release - Callback for affinity notifier release 539 * @ref: internal core kernel usage 540 * 541 * This is a callback function used by the irq_set_affinity_notifier function 542 * to inform the current notification subscriber that they will no longer 543 * receive notifications. 544 **/ 545 static void iavf_irq_affinity_release(struct kref *ref) {} 546 547 /** 548 * iavf_request_traffic_irqs - Initialize MSI-X interrupts 549 * @adapter: board private structure 550 * @basename: device basename 551 * 552 * Allocates MSI-X vectors for tx and rx handling, and requests 553 * interrupts from the kernel. 554 **/ 555 static int 556 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename) 557 { 558 unsigned int vector, q_vectors; 559 unsigned int rx_int_idx = 0, tx_int_idx = 0; 560 int irq_num, err; 561 int cpu; 562 563 iavf_irq_disable(adapter); 564 /* Decrement for Other and TCP Timer vectors */ 565 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 566 567 for (vector = 0; vector < q_vectors; vector++) { 568 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector]; 569 570 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 571 572 if (q_vector->tx.ring && q_vector->rx.ring) { 573 snprintf(q_vector->name, sizeof(q_vector->name), 574 "iavf-%s-TxRx-%u", basename, rx_int_idx++); 575 tx_int_idx++; 576 } else if (q_vector->rx.ring) { 577 snprintf(q_vector->name, sizeof(q_vector->name), 578 "iavf-%s-rx-%u", basename, rx_int_idx++); 579 } else if (q_vector->tx.ring) { 580 snprintf(q_vector->name, sizeof(q_vector->name), 581 "iavf-%s-tx-%u", basename, tx_int_idx++); 582 } else { 583 /* skip this unused q_vector */ 584 continue; 585 } 586 err = request_irq(irq_num, 587 iavf_msix_clean_rings, 588 0, 589 q_vector->name, 590 q_vector); 591 if (err) { 592 dev_info(&adapter->pdev->dev, 593 "Request_irq failed, error: %d\n", err); 594 goto free_queue_irqs; 595 } 596 /* register for affinity change notifications */ 597 q_vector->affinity_notify.notify = iavf_irq_affinity_notify; 598 q_vector->affinity_notify.release = 599 iavf_irq_affinity_release; 600 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 601 /* Spread the IRQ affinity hints across online CPUs. Note that 602 * get_cpu_mask returns a mask with a permanent lifetime so 603 * it's safe to use as a hint for irq_update_affinity_hint. 604 */ 605 cpu = cpumask_local_spread(q_vector->v_idx, -1); 606 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu)); 607 } 608 609 return 0; 610 611 free_queue_irqs: 612 while (vector) { 613 vector--; 614 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 615 irq_set_affinity_notifier(irq_num, NULL); 616 irq_update_affinity_hint(irq_num, NULL); 617 free_irq(irq_num, &adapter->q_vectors[vector]); 618 } 619 return err; 620 } 621 622 /** 623 * iavf_request_misc_irq - Initialize MSI-X interrupts 624 * @adapter: board private structure 625 * 626 * Allocates MSI-X vector 0 and requests interrupts from the kernel. This 627 * vector is only for the admin queue, and stays active even when the netdev 628 * is closed. 629 **/ 630 static int iavf_request_misc_irq(struct iavf_adapter *adapter) 631 { 632 struct net_device *netdev = adapter->netdev; 633 int err; 634 635 snprintf(adapter->misc_vector_name, 636 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx", 637 dev_name(&adapter->pdev->dev)); 638 err = request_irq(adapter->msix_entries[0].vector, 639 &iavf_msix_aq, 0, 640 adapter->misc_vector_name, netdev); 641 if (err) { 642 dev_err(&adapter->pdev->dev, 643 "request_irq for %s failed: %d\n", 644 adapter->misc_vector_name, err); 645 free_irq(adapter->msix_entries[0].vector, netdev); 646 } 647 return err; 648 } 649 650 /** 651 * iavf_free_traffic_irqs - Free MSI-X interrupts 652 * @adapter: board private structure 653 * 654 * Frees all MSI-X vectors other than 0. 655 **/ 656 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter) 657 { 658 int vector, irq_num, q_vectors; 659 660 if (!adapter->msix_entries) 661 return; 662 663 q_vectors = adapter->num_msix_vectors - NONQ_VECS; 664 665 for (vector = 0; vector < q_vectors; vector++) { 666 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector; 667 irq_set_affinity_notifier(irq_num, NULL); 668 irq_update_affinity_hint(irq_num, NULL); 669 free_irq(irq_num, &adapter->q_vectors[vector]); 670 } 671 } 672 673 /** 674 * iavf_free_misc_irq - Free MSI-X miscellaneous vector 675 * @adapter: board private structure 676 * 677 * Frees MSI-X vector 0. 678 **/ 679 static void iavf_free_misc_irq(struct iavf_adapter *adapter) 680 { 681 struct net_device *netdev = adapter->netdev; 682 683 if (!adapter->msix_entries) 684 return; 685 686 free_irq(adapter->msix_entries[0].vector, netdev); 687 } 688 689 /** 690 * iavf_configure_tx - Configure Transmit Unit after Reset 691 * @adapter: board private structure 692 * 693 * Configure the Tx unit of the MAC after a reset. 694 **/ 695 static void iavf_configure_tx(struct iavf_adapter *adapter) 696 { 697 struct iavf_hw *hw = &adapter->hw; 698 int i; 699 700 for (i = 0; i < adapter->num_active_queues; i++) 701 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i); 702 } 703 704 /** 705 * iavf_configure_rx - Configure Receive Unit after Reset 706 * @adapter: board private structure 707 * 708 * Configure the Rx unit of the MAC after a reset. 709 **/ 710 static void iavf_configure_rx(struct iavf_adapter *adapter) 711 { 712 unsigned int rx_buf_len = IAVF_RXBUFFER_2048; 713 struct iavf_hw *hw = &adapter->hw; 714 int i; 715 716 /* Legacy Rx will always default to a 2048 buffer size. */ 717 #if (PAGE_SIZE < 8192) 718 if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) { 719 struct net_device *netdev = adapter->netdev; 720 721 /* For jumbo frames on systems with 4K pages we have to use 722 * an order 1 page, so we might as well increase the size 723 * of our Rx buffer to make better use of the available space 724 */ 725 rx_buf_len = IAVF_RXBUFFER_3072; 726 727 /* We use a 1536 buffer size for configurations with 728 * standard Ethernet mtu. On x86 this gives us enough room 729 * for shared info and 192 bytes of padding. 730 */ 731 if (!IAVF_2K_TOO_SMALL_WITH_PADDING && 732 (netdev->mtu <= ETH_DATA_LEN)) 733 rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN; 734 } 735 #endif 736 737 for (i = 0; i < adapter->num_active_queues; i++) { 738 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i); 739 adapter->rx_rings[i].rx_buf_len = rx_buf_len; 740 741 if (adapter->flags & IAVF_FLAG_LEGACY_RX) 742 clear_ring_build_skb_enabled(&adapter->rx_rings[i]); 743 else 744 set_ring_build_skb_enabled(&adapter->rx_rings[i]); 745 } 746 } 747 748 /** 749 * iavf_find_vlan - Search filter list for specific vlan filter 750 * @adapter: board private structure 751 * @vlan: vlan tag 752 * 753 * Returns ptr to the filter object or NULL. Must be called while holding the 754 * mac_vlan_list_lock. 755 **/ 756 static struct 757 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter, 758 struct iavf_vlan vlan) 759 { 760 struct iavf_vlan_filter *f; 761 762 list_for_each_entry(f, &adapter->vlan_filter_list, list) { 763 if (f->vlan.vid == vlan.vid && 764 f->vlan.tpid == vlan.tpid) 765 return f; 766 } 767 768 return NULL; 769 } 770 771 /** 772 * iavf_add_vlan - Add a vlan filter to the list 773 * @adapter: board private structure 774 * @vlan: VLAN tag 775 * 776 * Returns ptr to the filter object or NULL when no memory available. 777 **/ 778 static struct 779 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter, 780 struct iavf_vlan vlan) 781 { 782 struct iavf_vlan_filter *f = NULL; 783 784 spin_lock_bh(&adapter->mac_vlan_list_lock); 785 786 f = iavf_find_vlan(adapter, vlan); 787 if (!f) { 788 f = kzalloc(sizeof(*f), GFP_ATOMIC); 789 if (!f) 790 goto clearout; 791 792 f->vlan = vlan; 793 794 list_add_tail(&f->list, &adapter->vlan_filter_list); 795 f->add = true; 796 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER; 797 } 798 799 clearout: 800 spin_unlock_bh(&adapter->mac_vlan_list_lock); 801 return f; 802 } 803 804 /** 805 * iavf_del_vlan - Remove a vlan filter from the list 806 * @adapter: board private structure 807 * @vlan: VLAN tag 808 **/ 809 static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan) 810 { 811 struct iavf_vlan_filter *f; 812 813 spin_lock_bh(&adapter->mac_vlan_list_lock); 814 815 f = iavf_find_vlan(adapter, vlan); 816 if (f) { 817 f->remove = true; 818 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 819 } 820 821 spin_unlock_bh(&adapter->mac_vlan_list_lock); 822 } 823 824 /** 825 * iavf_restore_filters 826 * @adapter: board private structure 827 * 828 * Restore existing non MAC filters when VF netdev comes back up 829 **/ 830 static void iavf_restore_filters(struct iavf_adapter *adapter) 831 { 832 u16 vid; 833 834 /* re-add all VLAN filters */ 835 for_each_set_bit(vid, adapter->vsi.active_cvlans, VLAN_N_VID) 836 iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021Q)); 837 838 for_each_set_bit(vid, adapter->vsi.active_svlans, VLAN_N_VID) 839 iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021AD)); 840 } 841 842 /** 843 * iavf_get_num_vlans_added - get number of VLANs added 844 * @adapter: board private structure 845 */ 846 static u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter) 847 { 848 return bitmap_weight(adapter->vsi.active_cvlans, VLAN_N_VID) + 849 bitmap_weight(adapter->vsi.active_svlans, VLAN_N_VID); 850 } 851 852 /** 853 * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF 854 * @adapter: board private structure 855 * 856 * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN, 857 * do not impose a limit as that maintains current behavior and for 858 * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF. 859 **/ 860 static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter) 861 { 862 /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has 863 * never been a limit on the VF driver side 864 */ 865 if (VLAN_ALLOWED(adapter)) 866 return VLAN_N_VID; 867 else if (VLAN_V2_ALLOWED(adapter)) 868 return adapter->vlan_v2_caps.filtering.max_filters; 869 870 return 0; 871 } 872 873 /** 874 * iavf_max_vlans_added - check if maximum VLANs allowed already exist 875 * @adapter: board private structure 876 **/ 877 static bool iavf_max_vlans_added(struct iavf_adapter *adapter) 878 { 879 if (iavf_get_num_vlans_added(adapter) < 880 iavf_get_max_vlans_allowed(adapter)) 881 return false; 882 883 return true; 884 } 885 886 /** 887 * iavf_vlan_rx_add_vid - Add a VLAN filter to a device 888 * @netdev: network device struct 889 * @proto: unused protocol data 890 * @vid: VLAN tag 891 **/ 892 static int iavf_vlan_rx_add_vid(struct net_device *netdev, 893 __always_unused __be16 proto, u16 vid) 894 { 895 struct iavf_adapter *adapter = netdev_priv(netdev); 896 897 if (!VLAN_FILTERING_ALLOWED(adapter)) 898 return -EIO; 899 900 if (iavf_max_vlans_added(adapter)) { 901 netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n", 902 iavf_get_max_vlans_allowed(adapter)); 903 return -EIO; 904 } 905 906 if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)))) 907 return -ENOMEM; 908 909 if (proto == cpu_to_be16(ETH_P_8021Q)) 910 set_bit(vid, adapter->vsi.active_cvlans); 911 else 912 set_bit(vid, adapter->vsi.active_svlans); 913 914 return 0; 915 } 916 917 /** 918 * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device 919 * @netdev: network device struct 920 * @proto: unused protocol data 921 * @vid: VLAN tag 922 **/ 923 static int iavf_vlan_rx_kill_vid(struct net_device *netdev, 924 __always_unused __be16 proto, u16 vid) 925 { 926 struct iavf_adapter *adapter = netdev_priv(netdev); 927 928 iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))); 929 if (proto == cpu_to_be16(ETH_P_8021Q)) 930 clear_bit(vid, adapter->vsi.active_cvlans); 931 else 932 clear_bit(vid, adapter->vsi.active_svlans); 933 934 return 0; 935 } 936 937 /** 938 * iavf_find_filter - Search filter list for specific mac filter 939 * @adapter: board private structure 940 * @macaddr: the MAC address 941 * 942 * Returns ptr to the filter object or NULL. Must be called while holding the 943 * mac_vlan_list_lock. 944 **/ 945 static struct 946 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter, 947 const u8 *macaddr) 948 { 949 struct iavf_mac_filter *f; 950 951 if (!macaddr) 952 return NULL; 953 954 list_for_each_entry(f, &adapter->mac_filter_list, list) { 955 if (ether_addr_equal(macaddr, f->macaddr)) 956 return f; 957 } 958 return NULL; 959 } 960 961 /** 962 * iavf_add_filter - Add a mac filter to the filter list 963 * @adapter: board private structure 964 * @macaddr: the MAC address 965 * 966 * Returns ptr to the filter object or NULL when no memory available. 967 **/ 968 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter, 969 const u8 *macaddr) 970 { 971 struct iavf_mac_filter *f; 972 973 if (!macaddr) 974 return NULL; 975 976 f = iavf_find_filter(adapter, macaddr); 977 if (!f) { 978 f = kzalloc(sizeof(*f), GFP_ATOMIC); 979 if (!f) 980 return f; 981 982 ether_addr_copy(f->macaddr, macaddr); 983 984 list_add_tail(&f->list, &adapter->mac_filter_list); 985 f->add = true; 986 f->add_handled = false; 987 f->is_new_mac = true; 988 f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr); 989 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 990 } else { 991 f->remove = false; 992 } 993 994 return f; 995 } 996 997 /** 998 * iavf_replace_primary_mac - Replace current primary address 999 * @adapter: board private structure 1000 * @new_mac: new MAC address to be applied 1001 * 1002 * Replace current dev_addr and send request to PF for removal of previous 1003 * primary MAC address filter and addition of new primary MAC filter. 1004 * Return 0 for success, -ENOMEM for failure. 1005 * 1006 * Do not call this with mac_vlan_list_lock! 1007 **/ 1008 int iavf_replace_primary_mac(struct iavf_adapter *adapter, 1009 const u8 *new_mac) 1010 { 1011 struct iavf_hw *hw = &adapter->hw; 1012 struct iavf_mac_filter *f; 1013 1014 spin_lock_bh(&adapter->mac_vlan_list_lock); 1015 1016 list_for_each_entry(f, &adapter->mac_filter_list, list) { 1017 f->is_primary = false; 1018 } 1019 1020 f = iavf_find_filter(adapter, hw->mac.addr); 1021 if (f) { 1022 f->remove = true; 1023 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 1024 } 1025 1026 f = iavf_add_filter(adapter, new_mac); 1027 1028 if (f) { 1029 /* Always send the request to add if changing primary MAC 1030 * even if filter is already present on the list 1031 */ 1032 f->is_primary = true; 1033 f->add = true; 1034 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 1035 ether_addr_copy(hw->mac.addr, new_mac); 1036 } 1037 1038 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1039 1040 /* schedule the watchdog task to immediately process the request */ 1041 if (f) { 1042 queue_work(iavf_wq, &adapter->watchdog_task.work); 1043 return 0; 1044 } 1045 return -ENOMEM; 1046 } 1047 1048 /** 1049 * iavf_is_mac_set_handled - wait for a response to set MAC from PF 1050 * @netdev: network interface device structure 1051 * @macaddr: MAC address to set 1052 * 1053 * Returns true on success, false on failure 1054 */ 1055 static bool iavf_is_mac_set_handled(struct net_device *netdev, 1056 const u8 *macaddr) 1057 { 1058 struct iavf_adapter *adapter = netdev_priv(netdev); 1059 struct iavf_mac_filter *f; 1060 bool ret = false; 1061 1062 spin_lock_bh(&adapter->mac_vlan_list_lock); 1063 1064 f = iavf_find_filter(adapter, macaddr); 1065 1066 if (!f || (!f->add && f->add_handled)) 1067 ret = true; 1068 1069 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1070 1071 return ret; 1072 } 1073 1074 /** 1075 * iavf_set_mac - NDO callback to set port MAC address 1076 * @netdev: network interface device structure 1077 * @p: pointer to an address structure 1078 * 1079 * Returns 0 on success, negative on failure 1080 */ 1081 static int iavf_set_mac(struct net_device *netdev, void *p) 1082 { 1083 struct iavf_adapter *adapter = netdev_priv(netdev); 1084 struct sockaddr *addr = p; 1085 bool handle_mac = iavf_is_mac_set_handled(netdev, addr->sa_data); 1086 int ret; 1087 1088 if (!is_valid_ether_addr(addr->sa_data)) 1089 return -EADDRNOTAVAIL; 1090 1091 ret = iavf_replace_primary_mac(adapter, addr->sa_data); 1092 1093 if (ret) 1094 return ret; 1095 1096 /* If this is an initial set MAC during VF spawn do not wait */ 1097 if (adapter->flags & IAVF_FLAG_INITIAL_MAC_SET) { 1098 adapter->flags &= ~IAVF_FLAG_INITIAL_MAC_SET; 1099 return 0; 1100 } 1101 1102 if (handle_mac) 1103 goto done; 1104 1105 ret = wait_event_interruptible_timeout(adapter->vc_waitqueue, false, msecs_to_jiffies(2500)); 1106 1107 /* If ret < 0 then it means wait was interrupted. 1108 * If ret == 0 then it means we got a timeout. 1109 * else it means we got response for set MAC from PF, 1110 * check if netdev MAC was updated to requested MAC, 1111 * if yes then set MAC succeeded otherwise it failed return -EACCES 1112 */ 1113 if (ret < 0) 1114 return ret; 1115 1116 if (!ret) 1117 return -EAGAIN; 1118 1119 done: 1120 if (!ether_addr_equal(netdev->dev_addr, addr->sa_data)) 1121 return -EACCES; 1122 1123 return 0; 1124 } 1125 1126 /** 1127 * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address 1128 * @netdev: the netdevice 1129 * @addr: address to add 1130 * 1131 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1132 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1133 */ 1134 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr) 1135 { 1136 struct iavf_adapter *adapter = netdev_priv(netdev); 1137 1138 if (iavf_add_filter(adapter, addr)) 1139 return 0; 1140 else 1141 return -ENOMEM; 1142 } 1143 1144 /** 1145 * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1146 * @netdev: the netdevice 1147 * @addr: address to add 1148 * 1149 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1150 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1151 */ 1152 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr) 1153 { 1154 struct iavf_adapter *adapter = netdev_priv(netdev); 1155 struct iavf_mac_filter *f; 1156 1157 /* Under some circumstances, we might receive a request to delete 1158 * our own device address from our uc list. Because we store the 1159 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1160 * such requests and not delete our device address from this list. 1161 */ 1162 if (ether_addr_equal(addr, netdev->dev_addr)) 1163 return 0; 1164 1165 f = iavf_find_filter(adapter, addr); 1166 if (f) { 1167 f->remove = true; 1168 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER; 1169 } 1170 return 0; 1171 } 1172 1173 /** 1174 * iavf_set_rx_mode - NDO callback to set the netdev filters 1175 * @netdev: network interface device structure 1176 **/ 1177 static void iavf_set_rx_mode(struct net_device *netdev) 1178 { 1179 struct iavf_adapter *adapter = netdev_priv(netdev); 1180 1181 spin_lock_bh(&adapter->mac_vlan_list_lock); 1182 __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 1183 __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync); 1184 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1185 1186 if (netdev->flags & IFF_PROMISC && 1187 !(adapter->flags & IAVF_FLAG_PROMISC_ON)) 1188 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC; 1189 else if (!(netdev->flags & IFF_PROMISC) && 1190 adapter->flags & IAVF_FLAG_PROMISC_ON) 1191 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC; 1192 1193 if (netdev->flags & IFF_ALLMULTI && 1194 !(adapter->flags & IAVF_FLAG_ALLMULTI_ON)) 1195 adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI; 1196 else if (!(netdev->flags & IFF_ALLMULTI) && 1197 adapter->flags & IAVF_FLAG_ALLMULTI_ON) 1198 adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI; 1199 } 1200 1201 /** 1202 * iavf_napi_enable_all - enable NAPI on all queue vectors 1203 * @adapter: board private structure 1204 **/ 1205 static void iavf_napi_enable_all(struct iavf_adapter *adapter) 1206 { 1207 int q_idx; 1208 struct iavf_q_vector *q_vector; 1209 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1210 1211 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1212 struct napi_struct *napi; 1213 1214 q_vector = &adapter->q_vectors[q_idx]; 1215 napi = &q_vector->napi; 1216 napi_enable(napi); 1217 } 1218 } 1219 1220 /** 1221 * iavf_napi_disable_all - disable NAPI on all queue vectors 1222 * @adapter: board private structure 1223 **/ 1224 static void iavf_napi_disable_all(struct iavf_adapter *adapter) 1225 { 1226 int q_idx; 1227 struct iavf_q_vector *q_vector; 1228 int q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1229 1230 for (q_idx = 0; q_idx < q_vectors; q_idx++) { 1231 q_vector = &adapter->q_vectors[q_idx]; 1232 napi_disable(&q_vector->napi); 1233 } 1234 } 1235 1236 /** 1237 * iavf_configure - set up transmit and receive data structures 1238 * @adapter: board private structure 1239 **/ 1240 static void iavf_configure(struct iavf_adapter *adapter) 1241 { 1242 struct net_device *netdev = adapter->netdev; 1243 int i; 1244 1245 iavf_set_rx_mode(netdev); 1246 1247 iavf_configure_tx(adapter); 1248 iavf_configure_rx(adapter); 1249 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES; 1250 1251 for (i = 0; i < adapter->num_active_queues; i++) { 1252 struct iavf_ring *ring = &adapter->rx_rings[i]; 1253 1254 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring)); 1255 } 1256 } 1257 1258 /** 1259 * iavf_up_complete - Finish the last steps of bringing up a connection 1260 * @adapter: board private structure 1261 * 1262 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. 1263 **/ 1264 static void iavf_up_complete(struct iavf_adapter *adapter) 1265 { 1266 iavf_change_state(adapter, __IAVF_RUNNING); 1267 clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1268 1269 iavf_napi_enable_all(adapter); 1270 1271 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES; 1272 if (CLIENT_ENABLED(adapter)) 1273 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN; 1274 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 1275 } 1276 1277 /** 1278 * iavf_down - Shutdown the connection processing 1279 * @adapter: board private structure 1280 * 1281 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock. 1282 **/ 1283 void iavf_down(struct iavf_adapter *adapter) 1284 { 1285 struct net_device *netdev = adapter->netdev; 1286 struct iavf_vlan_filter *vlf; 1287 struct iavf_cloud_filter *cf; 1288 struct iavf_fdir_fltr *fdir; 1289 struct iavf_mac_filter *f; 1290 struct iavf_adv_rss *rss; 1291 1292 if (adapter->state <= __IAVF_DOWN_PENDING) 1293 return; 1294 1295 netif_carrier_off(netdev); 1296 netif_tx_disable(netdev); 1297 adapter->link_up = false; 1298 iavf_napi_disable_all(adapter); 1299 iavf_irq_disable(adapter); 1300 1301 spin_lock_bh(&adapter->mac_vlan_list_lock); 1302 1303 /* clear the sync flag on all filters */ 1304 __dev_uc_unsync(adapter->netdev, NULL); 1305 __dev_mc_unsync(adapter->netdev, NULL); 1306 1307 /* remove all MAC filters */ 1308 list_for_each_entry(f, &adapter->mac_filter_list, list) { 1309 f->remove = true; 1310 } 1311 1312 /* remove all VLAN filters */ 1313 list_for_each_entry(vlf, &adapter->vlan_filter_list, list) { 1314 vlf->remove = true; 1315 } 1316 1317 spin_unlock_bh(&adapter->mac_vlan_list_lock); 1318 1319 /* remove all cloud filters */ 1320 spin_lock_bh(&adapter->cloud_filter_list_lock); 1321 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 1322 cf->del = true; 1323 } 1324 spin_unlock_bh(&adapter->cloud_filter_list_lock); 1325 1326 /* remove all Flow Director filters */ 1327 spin_lock_bh(&adapter->fdir_fltr_lock); 1328 list_for_each_entry(fdir, &adapter->fdir_list_head, list) { 1329 fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST; 1330 } 1331 spin_unlock_bh(&adapter->fdir_fltr_lock); 1332 1333 /* remove all advance RSS configuration */ 1334 spin_lock_bh(&adapter->adv_rss_lock); 1335 list_for_each_entry(rss, &adapter->adv_rss_list_head, list) 1336 rss->state = IAVF_ADV_RSS_DEL_REQUEST; 1337 spin_unlock_bh(&adapter->adv_rss_lock); 1338 1339 if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)) { 1340 /* cancel any current operation */ 1341 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 1342 /* Schedule operations to close down the HW. Don't wait 1343 * here for this to complete. The watchdog is still running 1344 * and it will take care of this. 1345 */ 1346 adapter->aq_required = IAVF_FLAG_AQ_DEL_MAC_FILTER; 1347 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER; 1348 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 1349 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER; 1350 adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG; 1351 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES; 1352 } 1353 1354 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 1355 } 1356 1357 /** 1358 * iavf_acquire_msix_vectors - Setup the MSIX capability 1359 * @adapter: board private structure 1360 * @vectors: number of vectors to request 1361 * 1362 * Work with the OS to set up the MSIX vectors needed. 1363 * 1364 * Returns 0 on success, negative on failure 1365 **/ 1366 static int 1367 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors) 1368 { 1369 int err, vector_threshold; 1370 1371 /* We'll want at least 3 (vector_threshold): 1372 * 0) Other (Admin Queue and link, mostly) 1373 * 1) TxQ[0] Cleanup 1374 * 2) RxQ[0] Cleanup 1375 */ 1376 vector_threshold = MIN_MSIX_COUNT; 1377 1378 /* The more we get, the more we will assign to Tx/Rx Cleanup 1379 * for the separate queues...where Rx Cleanup >= Tx Cleanup. 1380 * Right now, we simply care about how many we'll get; we'll 1381 * set them up later while requesting irq's. 1382 */ 1383 err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 1384 vector_threshold, vectors); 1385 if (err < 0) { 1386 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n"); 1387 kfree(adapter->msix_entries); 1388 adapter->msix_entries = NULL; 1389 return err; 1390 } 1391 1392 /* Adjust for only the vectors we'll use, which is minimum 1393 * of max_msix_q_vectors + NONQ_VECS, or the number of 1394 * vectors we were allocated. 1395 */ 1396 adapter->num_msix_vectors = err; 1397 return 0; 1398 } 1399 1400 /** 1401 * iavf_free_queues - Free memory for all rings 1402 * @adapter: board private structure to initialize 1403 * 1404 * Free all of the memory associated with queue pairs. 1405 **/ 1406 static void iavf_free_queues(struct iavf_adapter *adapter) 1407 { 1408 if (!adapter->vsi_res) 1409 return; 1410 adapter->num_active_queues = 0; 1411 kfree(adapter->tx_rings); 1412 adapter->tx_rings = NULL; 1413 kfree(adapter->rx_rings); 1414 adapter->rx_rings = NULL; 1415 } 1416 1417 /** 1418 * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload 1419 * @adapter: board private structure 1420 * 1421 * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or 1422 * stripped in certain descriptor fields. Instead of checking the offload 1423 * capability bits in the hot path, cache the location the ring specific 1424 * flags. 1425 */ 1426 void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter) 1427 { 1428 int i; 1429 1430 for (i = 0; i < adapter->num_active_queues; i++) { 1431 struct iavf_ring *tx_ring = &adapter->tx_rings[i]; 1432 struct iavf_ring *rx_ring = &adapter->rx_rings[i]; 1433 1434 /* prevent multiple L2TAG bits being set after VFR */ 1435 tx_ring->flags &= 1436 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 | 1437 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2); 1438 rx_ring->flags &= 1439 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 | 1440 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2); 1441 1442 if (VLAN_ALLOWED(adapter)) { 1443 tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1444 rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1445 } else if (VLAN_V2_ALLOWED(adapter)) { 1446 struct virtchnl_vlan_supported_caps *stripping_support; 1447 struct virtchnl_vlan_supported_caps *insertion_support; 1448 1449 stripping_support = 1450 &adapter->vlan_v2_caps.offloads.stripping_support; 1451 insertion_support = 1452 &adapter->vlan_v2_caps.offloads.insertion_support; 1453 1454 if (stripping_support->outer) { 1455 if (stripping_support->outer & 1456 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1457 rx_ring->flags |= 1458 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1459 else if (stripping_support->outer & 1460 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) 1461 rx_ring->flags |= 1462 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2; 1463 } else if (stripping_support->inner) { 1464 if (stripping_support->inner & 1465 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1466 rx_ring->flags |= 1467 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1468 else if (stripping_support->inner & 1469 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2) 1470 rx_ring->flags |= 1471 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2; 1472 } 1473 1474 if (insertion_support->outer) { 1475 if (insertion_support->outer & 1476 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1477 tx_ring->flags |= 1478 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1479 else if (insertion_support->outer & 1480 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) 1481 tx_ring->flags |= 1482 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2; 1483 } else if (insertion_support->inner) { 1484 if (insertion_support->inner & 1485 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1) 1486 tx_ring->flags |= 1487 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1; 1488 else if (insertion_support->inner & 1489 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2) 1490 tx_ring->flags |= 1491 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2; 1492 } 1493 } 1494 } 1495 } 1496 1497 /** 1498 * iavf_alloc_queues - Allocate memory for all rings 1499 * @adapter: board private structure to initialize 1500 * 1501 * We allocate one ring per queue at run-time since we don't know the 1502 * number of queues at compile-time. The polling_netdev array is 1503 * intended for Multiqueue, but should work fine with a single queue. 1504 **/ 1505 static int iavf_alloc_queues(struct iavf_adapter *adapter) 1506 { 1507 int i, num_active_queues; 1508 1509 /* If we're in reset reallocating queues we don't actually know yet for 1510 * certain the PF gave us the number of queues we asked for but we'll 1511 * assume it did. Once basic reset is finished we'll confirm once we 1512 * start negotiating config with PF. 1513 */ 1514 if (adapter->num_req_queues) 1515 num_active_queues = adapter->num_req_queues; 1516 else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1517 adapter->num_tc) 1518 num_active_queues = adapter->ch_config.total_qps; 1519 else 1520 num_active_queues = min_t(int, 1521 adapter->vsi_res->num_queue_pairs, 1522 (int)(num_online_cpus())); 1523 1524 1525 adapter->tx_rings = kcalloc(num_active_queues, 1526 sizeof(struct iavf_ring), GFP_KERNEL); 1527 if (!adapter->tx_rings) 1528 goto err_out; 1529 adapter->rx_rings = kcalloc(num_active_queues, 1530 sizeof(struct iavf_ring), GFP_KERNEL); 1531 if (!adapter->rx_rings) 1532 goto err_out; 1533 1534 for (i = 0; i < num_active_queues; i++) { 1535 struct iavf_ring *tx_ring; 1536 struct iavf_ring *rx_ring; 1537 1538 tx_ring = &adapter->tx_rings[i]; 1539 1540 tx_ring->queue_index = i; 1541 tx_ring->netdev = adapter->netdev; 1542 tx_ring->dev = &adapter->pdev->dev; 1543 tx_ring->count = adapter->tx_desc_count; 1544 tx_ring->itr_setting = IAVF_ITR_TX_DEF; 1545 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE) 1546 tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR; 1547 1548 rx_ring = &adapter->rx_rings[i]; 1549 rx_ring->queue_index = i; 1550 rx_ring->netdev = adapter->netdev; 1551 rx_ring->dev = &adapter->pdev->dev; 1552 rx_ring->count = adapter->rx_desc_count; 1553 rx_ring->itr_setting = IAVF_ITR_RX_DEF; 1554 } 1555 1556 adapter->num_active_queues = num_active_queues; 1557 1558 iavf_set_queue_vlan_tag_loc(adapter); 1559 1560 return 0; 1561 1562 err_out: 1563 iavf_free_queues(adapter); 1564 return -ENOMEM; 1565 } 1566 1567 /** 1568 * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported 1569 * @adapter: board private structure to initialize 1570 * 1571 * Attempt to configure the interrupts using the best available 1572 * capabilities of the hardware and the kernel. 1573 **/ 1574 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter) 1575 { 1576 int vector, v_budget; 1577 int pairs = 0; 1578 int err = 0; 1579 1580 if (!adapter->vsi_res) { 1581 err = -EIO; 1582 goto out; 1583 } 1584 pairs = adapter->num_active_queues; 1585 1586 /* It's easy to be greedy for MSI-X vectors, but it really doesn't do 1587 * us much good if we have more vectors than CPUs. However, we already 1588 * limit the total number of queues by the number of CPUs so we do not 1589 * need any further limiting here. 1590 */ 1591 v_budget = min_t(int, pairs + NONQ_VECS, 1592 (int)adapter->vf_res->max_vectors); 1593 1594 adapter->msix_entries = kcalloc(v_budget, 1595 sizeof(struct msix_entry), GFP_KERNEL); 1596 if (!adapter->msix_entries) { 1597 err = -ENOMEM; 1598 goto out; 1599 } 1600 1601 for (vector = 0; vector < v_budget; vector++) 1602 adapter->msix_entries[vector].entry = vector; 1603 1604 err = iavf_acquire_msix_vectors(adapter, v_budget); 1605 1606 out: 1607 netif_set_real_num_rx_queues(adapter->netdev, pairs); 1608 netif_set_real_num_tx_queues(adapter->netdev, pairs); 1609 return err; 1610 } 1611 1612 /** 1613 * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands 1614 * @adapter: board private structure 1615 * 1616 * Return 0 on success, negative on failure 1617 **/ 1618 static int iavf_config_rss_aq(struct iavf_adapter *adapter) 1619 { 1620 struct iavf_aqc_get_set_rss_key_data *rss_key = 1621 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key; 1622 struct iavf_hw *hw = &adapter->hw; 1623 enum iavf_status status; 1624 1625 if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) { 1626 /* bail because we already have a command pending */ 1627 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n", 1628 adapter->current_op); 1629 return -EBUSY; 1630 } 1631 1632 status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key); 1633 if (status) { 1634 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n", 1635 iavf_stat_str(hw, status), 1636 iavf_aq_str(hw, hw->aq.asq_last_status)); 1637 return iavf_status_to_errno(status); 1638 1639 } 1640 1641 status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false, 1642 adapter->rss_lut, adapter->rss_lut_size); 1643 if (status) { 1644 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n", 1645 iavf_stat_str(hw, status), 1646 iavf_aq_str(hw, hw->aq.asq_last_status)); 1647 return iavf_status_to_errno(status); 1648 } 1649 1650 return 0; 1651 1652 } 1653 1654 /** 1655 * iavf_config_rss_reg - Configure RSS keys and lut by writing registers 1656 * @adapter: board private structure 1657 * 1658 * Returns 0 on success, negative on failure 1659 **/ 1660 static int iavf_config_rss_reg(struct iavf_adapter *adapter) 1661 { 1662 struct iavf_hw *hw = &adapter->hw; 1663 u32 *dw; 1664 u16 i; 1665 1666 dw = (u32 *)adapter->rss_key; 1667 for (i = 0; i <= adapter->rss_key_size / 4; i++) 1668 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]); 1669 1670 dw = (u32 *)adapter->rss_lut; 1671 for (i = 0; i <= adapter->rss_lut_size / 4; i++) 1672 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]); 1673 1674 iavf_flush(hw); 1675 1676 return 0; 1677 } 1678 1679 /** 1680 * iavf_config_rss - Configure RSS keys and lut 1681 * @adapter: board private structure 1682 * 1683 * Returns 0 on success, negative on failure 1684 **/ 1685 int iavf_config_rss(struct iavf_adapter *adapter) 1686 { 1687 1688 if (RSS_PF(adapter)) { 1689 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT | 1690 IAVF_FLAG_AQ_SET_RSS_KEY; 1691 return 0; 1692 } else if (RSS_AQ(adapter)) { 1693 return iavf_config_rss_aq(adapter); 1694 } else { 1695 return iavf_config_rss_reg(adapter); 1696 } 1697 } 1698 1699 /** 1700 * iavf_fill_rss_lut - Fill the lut with default values 1701 * @adapter: board private structure 1702 **/ 1703 static void iavf_fill_rss_lut(struct iavf_adapter *adapter) 1704 { 1705 u16 i; 1706 1707 for (i = 0; i < adapter->rss_lut_size; i++) 1708 adapter->rss_lut[i] = i % adapter->num_active_queues; 1709 } 1710 1711 /** 1712 * iavf_init_rss - Prepare for RSS 1713 * @adapter: board private structure 1714 * 1715 * Return 0 on success, negative on failure 1716 **/ 1717 static int iavf_init_rss(struct iavf_adapter *adapter) 1718 { 1719 struct iavf_hw *hw = &adapter->hw; 1720 1721 if (!RSS_PF(adapter)) { 1722 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */ 1723 if (adapter->vf_res->vf_cap_flags & 1724 VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2) 1725 adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED; 1726 else 1727 adapter->hena = IAVF_DEFAULT_RSS_HENA; 1728 1729 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena); 1730 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32)); 1731 } 1732 1733 iavf_fill_rss_lut(adapter); 1734 netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size); 1735 1736 return iavf_config_rss(adapter); 1737 } 1738 1739 /** 1740 * iavf_alloc_q_vectors - Allocate memory for interrupt vectors 1741 * @adapter: board private structure to initialize 1742 * 1743 * We allocate one q_vector per queue interrupt. If allocation fails we 1744 * return -ENOMEM. 1745 **/ 1746 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter) 1747 { 1748 int q_idx = 0, num_q_vectors; 1749 struct iavf_q_vector *q_vector; 1750 1751 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1752 adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector), 1753 GFP_KERNEL); 1754 if (!adapter->q_vectors) 1755 return -ENOMEM; 1756 1757 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1758 q_vector = &adapter->q_vectors[q_idx]; 1759 q_vector->adapter = adapter; 1760 q_vector->vsi = &adapter->vsi; 1761 q_vector->v_idx = q_idx; 1762 q_vector->reg_idx = q_idx; 1763 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 1764 netif_napi_add(adapter->netdev, &q_vector->napi, 1765 iavf_napi_poll, NAPI_POLL_WEIGHT); 1766 } 1767 1768 return 0; 1769 } 1770 1771 /** 1772 * iavf_free_q_vectors - Free memory allocated for interrupt vectors 1773 * @adapter: board private structure to initialize 1774 * 1775 * This function frees the memory allocated to the q_vectors. In addition if 1776 * NAPI is enabled it will delete any references to the NAPI struct prior 1777 * to freeing the q_vector. 1778 **/ 1779 static void iavf_free_q_vectors(struct iavf_adapter *adapter) 1780 { 1781 int q_idx, num_q_vectors; 1782 int napi_vectors; 1783 1784 if (!adapter->q_vectors) 1785 return; 1786 1787 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS; 1788 napi_vectors = adapter->num_active_queues; 1789 1790 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) { 1791 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx]; 1792 1793 if (q_idx < napi_vectors) 1794 netif_napi_del(&q_vector->napi); 1795 } 1796 kfree(adapter->q_vectors); 1797 adapter->q_vectors = NULL; 1798 } 1799 1800 /** 1801 * iavf_reset_interrupt_capability - Reset MSIX setup 1802 * @adapter: board private structure 1803 * 1804 **/ 1805 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter) 1806 { 1807 if (!adapter->msix_entries) 1808 return; 1809 1810 pci_disable_msix(adapter->pdev); 1811 kfree(adapter->msix_entries); 1812 adapter->msix_entries = NULL; 1813 } 1814 1815 /** 1816 * iavf_init_interrupt_scheme - Determine if MSIX is supported and init 1817 * @adapter: board private structure to initialize 1818 * 1819 **/ 1820 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter) 1821 { 1822 int err; 1823 1824 err = iavf_alloc_queues(adapter); 1825 if (err) { 1826 dev_err(&adapter->pdev->dev, 1827 "Unable to allocate memory for queues\n"); 1828 goto err_alloc_queues; 1829 } 1830 1831 rtnl_lock(); 1832 err = iavf_set_interrupt_capability(adapter); 1833 rtnl_unlock(); 1834 if (err) { 1835 dev_err(&adapter->pdev->dev, 1836 "Unable to setup interrupt capabilities\n"); 1837 goto err_set_interrupt; 1838 } 1839 1840 err = iavf_alloc_q_vectors(adapter); 1841 if (err) { 1842 dev_err(&adapter->pdev->dev, 1843 "Unable to allocate memory for queue vectors\n"); 1844 goto err_alloc_q_vectors; 1845 } 1846 1847 /* If we've made it so far while ADq flag being ON, then we haven't 1848 * bailed out anywhere in middle. And ADq isn't just enabled but actual 1849 * resources have been allocated in the reset path. 1850 * Now we can truly claim that ADq is enabled. 1851 */ 1852 if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 1853 adapter->num_tc) 1854 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created", 1855 adapter->num_tc); 1856 1857 dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u", 1858 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled", 1859 adapter->num_active_queues); 1860 1861 return 0; 1862 err_alloc_q_vectors: 1863 iavf_reset_interrupt_capability(adapter); 1864 err_set_interrupt: 1865 iavf_free_queues(adapter); 1866 err_alloc_queues: 1867 return err; 1868 } 1869 1870 /** 1871 * iavf_free_rss - Free memory used by RSS structs 1872 * @adapter: board private structure 1873 **/ 1874 static void iavf_free_rss(struct iavf_adapter *adapter) 1875 { 1876 kfree(adapter->rss_key); 1877 adapter->rss_key = NULL; 1878 1879 kfree(adapter->rss_lut); 1880 adapter->rss_lut = NULL; 1881 } 1882 1883 /** 1884 * iavf_reinit_interrupt_scheme - Reallocate queues and vectors 1885 * @adapter: board private structure 1886 * 1887 * Returns 0 on success, negative on failure 1888 **/ 1889 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter) 1890 { 1891 struct net_device *netdev = adapter->netdev; 1892 int err; 1893 1894 if (netif_running(netdev)) 1895 iavf_free_traffic_irqs(adapter); 1896 iavf_free_misc_irq(adapter); 1897 iavf_reset_interrupt_capability(adapter); 1898 iavf_free_q_vectors(adapter); 1899 iavf_free_queues(adapter); 1900 1901 err = iavf_init_interrupt_scheme(adapter); 1902 if (err) 1903 goto err; 1904 1905 netif_tx_stop_all_queues(netdev); 1906 1907 err = iavf_request_misc_irq(adapter); 1908 if (err) 1909 goto err; 1910 1911 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 1912 1913 iavf_map_rings_to_vectors(adapter); 1914 err: 1915 return err; 1916 } 1917 1918 /** 1919 * iavf_process_aq_command - process aq_required flags 1920 * and sends aq command 1921 * @adapter: pointer to iavf adapter structure 1922 * 1923 * Returns 0 on success 1924 * Returns error code if no command was sent 1925 * or error code if the command failed. 1926 **/ 1927 static int iavf_process_aq_command(struct iavf_adapter *adapter) 1928 { 1929 if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG) 1930 return iavf_send_vf_config_msg(adapter); 1931 if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS) 1932 return iavf_send_vf_offload_vlan_v2_msg(adapter); 1933 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) { 1934 iavf_disable_queues(adapter); 1935 return 0; 1936 } 1937 1938 if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) { 1939 iavf_map_queues(adapter); 1940 return 0; 1941 } 1942 1943 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) { 1944 iavf_add_ether_addrs(adapter); 1945 return 0; 1946 } 1947 1948 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) { 1949 iavf_add_vlans(adapter); 1950 return 0; 1951 } 1952 1953 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) { 1954 iavf_del_ether_addrs(adapter); 1955 return 0; 1956 } 1957 1958 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) { 1959 iavf_del_vlans(adapter); 1960 return 0; 1961 } 1962 1963 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) { 1964 iavf_enable_vlan_stripping(adapter); 1965 return 0; 1966 } 1967 1968 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) { 1969 iavf_disable_vlan_stripping(adapter); 1970 return 0; 1971 } 1972 1973 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) { 1974 iavf_configure_queues(adapter); 1975 return 0; 1976 } 1977 1978 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) { 1979 iavf_enable_queues(adapter); 1980 return 0; 1981 } 1982 1983 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) { 1984 /* This message goes straight to the firmware, not the 1985 * PF, so we don't have to set current_op as we will 1986 * not get a response through the ARQ. 1987 */ 1988 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS; 1989 return 0; 1990 } 1991 if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) { 1992 iavf_get_hena(adapter); 1993 return 0; 1994 } 1995 if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) { 1996 iavf_set_hena(adapter); 1997 return 0; 1998 } 1999 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) { 2000 iavf_set_rss_key(adapter); 2001 return 0; 2002 } 2003 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) { 2004 iavf_set_rss_lut(adapter); 2005 return 0; 2006 } 2007 2008 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) { 2009 iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC | 2010 FLAG_VF_MULTICAST_PROMISC); 2011 return 0; 2012 } 2013 2014 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) { 2015 iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC); 2016 return 0; 2017 } 2018 if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) || 2019 (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) { 2020 iavf_set_promiscuous(adapter, 0); 2021 return 0; 2022 } 2023 2024 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) { 2025 iavf_enable_channels(adapter); 2026 return 0; 2027 } 2028 2029 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) { 2030 iavf_disable_channels(adapter); 2031 return 0; 2032 } 2033 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) { 2034 iavf_add_cloud_filter(adapter); 2035 return 0; 2036 } 2037 2038 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) { 2039 iavf_del_cloud_filter(adapter); 2040 return 0; 2041 } 2042 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) { 2043 iavf_del_cloud_filter(adapter); 2044 return 0; 2045 } 2046 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) { 2047 iavf_add_cloud_filter(adapter); 2048 return 0; 2049 } 2050 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) { 2051 iavf_add_fdir_filter(adapter); 2052 return IAVF_SUCCESS; 2053 } 2054 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) { 2055 iavf_del_fdir_filter(adapter); 2056 return IAVF_SUCCESS; 2057 } 2058 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) { 2059 iavf_add_adv_rss_cfg(adapter); 2060 return 0; 2061 } 2062 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) { 2063 iavf_del_adv_rss_cfg(adapter); 2064 return 0; 2065 } 2066 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) { 2067 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q); 2068 return 0; 2069 } 2070 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) { 2071 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD); 2072 return 0; 2073 } 2074 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) { 2075 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q); 2076 return 0; 2077 } 2078 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) { 2079 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD); 2080 return 0; 2081 } 2082 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) { 2083 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q); 2084 return 0; 2085 } 2086 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) { 2087 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD); 2088 return 0; 2089 } 2090 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) { 2091 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q); 2092 return 0; 2093 } 2094 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) { 2095 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD); 2096 return 0; 2097 } 2098 2099 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) { 2100 iavf_request_stats(adapter); 2101 return 0; 2102 } 2103 2104 return -EAGAIN; 2105 } 2106 2107 /** 2108 * iavf_set_vlan_offload_features - set VLAN offload configuration 2109 * @adapter: board private structure 2110 * @prev_features: previous features used for comparison 2111 * @features: updated features used for configuration 2112 * 2113 * Set the aq_required bit(s) based on the requested features passed in to 2114 * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule 2115 * the watchdog if any changes are requested to expedite the request via 2116 * virtchnl. 2117 **/ 2118 void 2119 iavf_set_vlan_offload_features(struct iavf_adapter *adapter, 2120 netdev_features_t prev_features, 2121 netdev_features_t features) 2122 { 2123 bool enable_stripping = true, enable_insertion = true; 2124 u16 vlan_ethertype = 0; 2125 u64 aq_required = 0; 2126 2127 /* keep cases separate because one ethertype for offloads can be 2128 * disabled at the same time as another is disabled, so check for an 2129 * enabled ethertype first, then check for disabled. Default to 2130 * ETH_P_8021Q so an ethertype is specified if disabling insertion and 2131 * stripping. 2132 */ 2133 if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 2134 vlan_ethertype = ETH_P_8021AD; 2135 else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 2136 vlan_ethertype = ETH_P_8021Q; 2137 else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 2138 vlan_ethertype = ETH_P_8021AD; 2139 else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 2140 vlan_ethertype = ETH_P_8021Q; 2141 else 2142 vlan_ethertype = ETH_P_8021Q; 2143 2144 if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX))) 2145 enable_stripping = false; 2146 if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX))) 2147 enable_insertion = false; 2148 2149 if (VLAN_ALLOWED(adapter)) { 2150 /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN 2151 * stripping via virtchnl. VLAN insertion can be toggled on the 2152 * netdev, but it doesn't require a virtchnl message 2153 */ 2154 if (enable_stripping) 2155 aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING; 2156 else 2157 aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING; 2158 2159 } else if (VLAN_V2_ALLOWED(adapter)) { 2160 switch (vlan_ethertype) { 2161 case ETH_P_8021Q: 2162 if (enable_stripping) 2163 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING; 2164 else 2165 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING; 2166 2167 if (enable_insertion) 2168 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION; 2169 else 2170 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION; 2171 break; 2172 case ETH_P_8021AD: 2173 if (enable_stripping) 2174 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING; 2175 else 2176 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING; 2177 2178 if (enable_insertion) 2179 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION; 2180 else 2181 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION; 2182 break; 2183 } 2184 } 2185 2186 if (aq_required) { 2187 adapter->aq_required |= aq_required; 2188 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0); 2189 } 2190 } 2191 2192 /** 2193 * iavf_startup - first step of driver startup 2194 * @adapter: board private structure 2195 * 2196 * Function process __IAVF_STARTUP driver state. 2197 * When success the state is changed to __IAVF_INIT_VERSION_CHECK 2198 * when fails the state is changed to __IAVF_INIT_FAILED 2199 **/ 2200 static void iavf_startup(struct iavf_adapter *adapter) 2201 { 2202 struct pci_dev *pdev = adapter->pdev; 2203 struct iavf_hw *hw = &adapter->hw; 2204 enum iavf_status status; 2205 int ret; 2206 2207 WARN_ON(adapter->state != __IAVF_STARTUP); 2208 2209 /* driver loaded, probe complete */ 2210 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 2211 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2212 status = iavf_set_mac_type(hw); 2213 if (status) { 2214 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", status); 2215 goto err; 2216 } 2217 2218 ret = iavf_check_reset_complete(hw); 2219 if (ret) { 2220 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n", 2221 ret); 2222 goto err; 2223 } 2224 hw->aq.num_arq_entries = IAVF_AQ_LEN; 2225 hw->aq.num_asq_entries = IAVF_AQ_LEN; 2226 hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 2227 hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE; 2228 2229 status = iavf_init_adminq(hw); 2230 if (status) { 2231 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n", 2232 status); 2233 goto err; 2234 } 2235 ret = iavf_send_api_ver(adapter); 2236 if (ret) { 2237 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret); 2238 iavf_shutdown_adminq(hw); 2239 goto err; 2240 } 2241 iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK); 2242 return; 2243 err: 2244 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2245 } 2246 2247 /** 2248 * iavf_init_version_check - second step of driver startup 2249 * @adapter: board private structure 2250 * 2251 * Function process __IAVF_INIT_VERSION_CHECK driver state. 2252 * When success the state is changed to __IAVF_INIT_GET_RESOURCES 2253 * when fails the state is changed to __IAVF_INIT_FAILED 2254 **/ 2255 static void iavf_init_version_check(struct iavf_adapter *adapter) 2256 { 2257 struct pci_dev *pdev = adapter->pdev; 2258 struct iavf_hw *hw = &adapter->hw; 2259 int err = -EAGAIN; 2260 2261 WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK); 2262 2263 if (!iavf_asq_done(hw)) { 2264 dev_err(&pdev->dev, "Admin queue command never completed\n"); 2265 iavf_shutdown_adminq(hw); 2266 iavf_change_state(adapter, __IAVF_STARTUP); 2267 goto err; 2268 } 2269 2270 /* aq msg sent, awaiting reply */ 2271 err = iavf_verify_api_ver(adapter); 2272 if (err) { 2273 if (err == -EALREADY) 2274 err = iavf_send_api_ver(adapter); 2275 else 2276 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n", 2277 adapter->pf_version.major, 2278 adapter->pf_version.minor, 2279 VIRTCHNL_VERSION_MAJOR, 2280 VIRTCHNL_VERSION_MINOR); 2281 goto err; 2282 } 2283 err = iavf_send_vf_config_msg(adapter); 2284 if (err) { 2285 dev_err(&pdev->dev, "Unable to send config request (%d)\n", 2286 err); 2287 goto err; 2288 } 2289 iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES); 2290 return; 2291 err: 2292 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2293 } 2294 2295 /** 2296 * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES 2297 * @adapter: board private structure 2298 */ 2299 int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter) 2300 { 2301 int i, num_req_queues = adapter->num_req_queues; 2302 struct iavf_vsi *vsi = &adapter->vsi; 2303 2304 for (i = 0; i < adapter->vf_res->num_vsis; i++) { 2305 if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV) 2306 adapter->vsi_res = &adapter->vf_res->vsi_res[i]; 2307 } 2308 if (!adapter->vsi_res) { 2309 dev_err(&adapter->pdev->dev, "No LAN VSI found\n"); 2310 return -ENODEV; 2311 } 2312 2313 if (num_req_queues && 2314 num_req_queues > adapter->vsi_res->num_queue_pairs) { 2315 /* Problem. The PF gave us fewer queues than what we had 2316 * negotiated in our request. Need a reset to see if we can't 2317 * get back to a working state. 2318 */ 2319 dev_err(&adapter->pdev->dev, 2320 "Requested %d queues, but PF only gave us %d.\n", 2321 num_req_queues, 2322 adapter->vsi_res->num_queue_pairs); 2323 adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED; 2324 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs; 2325 iavf_schedule_reset(adapter); 2326 2327 return -EAGAIN; 2328 } 2329 adapter->num_req_queues = 0; 2330 adapter->vsi.id = adapter->vsi_res->vsi_id; 2331 2332 adapter->vsi.back = adapter; 2333 adapter->vsi.base_vector = 1; 2334 adapter->vsi.work_limit = IAVF_DEFAULT_IRQ_WORK; 2335 vsi->netdev = adapter->netdev; 2336 vsi->qs_handle = adapter->vsi_res->qset_handle; 2337 if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) { 2338 adapter->rss_key_size = adapter->vf_res->rss_key_size; 2339 adapter->rss_lut_size = adapter->vf_res->rss_lut_size; 2340 } else { 2341 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE; 2342 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE; 2343 } 2344 2345 return 0; 2346 } 2347 2348 /** 2349 * iavf_init_get_resources - third step of driver startup 2350 * @adapter: board private structure 2351 * 2352 * Function process __IAVF_INIT_GET_RESOURCES driver state and 2353 * finishes driver initialization procedure. 2354 * When success the state is changed to __IAVF_DOWN 2355 * when fails the state is changed to __IAVF_INIT_FAILED 2356 **/ 2357 static void iavf_init_get_resources(struct iavf_adapter *adapter) 2358 { 2359 struct pci_dev *pdev = adapter->pdev; 2360 struct iavf_hw *hw = &adapter->hw; 2361 int err; 2362 2363 WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES); 2364 /* aq msg sent, awaiting reply */ 2365 if (!adapter->vf_res) { 2366 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE, 2367 GFP_KERNEL); 2368 if (!adapter->vf_res) { 2369 err = -ENOMEM; 2370 goto err; 2371 } 2372 } 2373 err = iavf_get_vf_config(adapter); 2374 if (err == -EALREADY) { 2375 err = iavf_send_vf_config_msg(adapter); 2376 goto err_alloc; 2377 } else if (err == -EINVAL) { 2378 /* We only get -EINVAL if the device is in a very bad 2379 * state or if we've been disabled for previous bad 2380 * behavior. Either way, we're done now. 2381 */ 2382 iavf_shutdown_adminq(hw); 2383 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n"); 2384 return; 2385 } 2386 if (err) { 2387 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err); 2388 goto err_alloc; 2389 } 2390 2391 err = iavf_parse_vf_resource_msg(adapter); 2392 if (err) { 2393 dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n", 2394 err); 2395 goto err_alloc; 2396 } 2397 /* Some features require additional messages to negotiate extended 2398 * capabilities. These are processed in sequence by the 2399 * __IAVF_INIT_EXTENDED_CAPS driver state. 2400 */ 2401 adapter->extended_caps = IAVF_EXTENDED_CAPS; 2402 2403 iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS); 2404 return; 2405 2406 err_alloc: 2407 kfree(adapter->vf_res); 2408 adapter->vf_res = NULL; 2409 err: 2410 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2411 } 2412 2413 /** 2414 * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps 2415 * @adapter: board private structure 2416 * 2417 * Function processes send of the extended VLAN V2 capability message to the 2418 * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent, 2419 * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2. 2420 */ 2421 static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter) 2422 { 2423 int ret; 2424 2425 WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2)); 2426 2427 ret = iavf_send_vf_offload_vlan_v2_msg(adapter); 2428 if (ret && ret == -EOPNOTSUPP) { 2429 /* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case, 2430 * we did not send the capability exchange message and do not 2431 * expect a response. 2432 */ 2433 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2; 2434 } 2435 2436 /* We sent the message, so move on to the next step */ 2437 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2; 2438 } 2439 2440 /** 2441 * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps 2442 * @adapter: board private structure 2443 * 2444 * Function processes receipt of the extended VLAN V2 capability message from 2445 * the PF. 2446 **/ 2447 static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter) 2448 { 2449 int ret; 2450 2451 WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2)); 2452 2453 memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps)); 2454 2455 ret = iavf_get_vf_vlan_v2_caps(adapter); 2456 if (ret) 2457 goto err; 2458 2459 /* We've processed receipt of the VLAN V2 caps message */ 2460 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2; 2461 return; 2462 err: 2463 /* We didn't receive a reply. Make sure we try sending again when 2464 * __IAVF_INIT_FAILED attempts to recover. 2465 */ 2466 adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2; 2467 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2468 } 2469 2470 /** 2471 * iavf_init_process_extended_caps - Part of driver startup 2472 * @adapter: board private structure 2473 * 2474 * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state 2475 * handles negotiating capabilities for features which require an additional 2476 * message. 2477 * 2478 * Once all extended capabilities exchanges are finished, the driver will 2479 * transition into __IAVF_INIT_CONFIG_ADAPTER. 2480 */ 2481 static void iavf_init_process_extended_caps(struct iavf_adapter *adapter) 2482 { 2483 WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS); 2484 2485 /* Process capability exchange for VLAN V2 */ 2486 if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) { 2487 iavf_init_send_offload_vlan_v2_caps(adapter); 2488 return; 2489 } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) { 2490 iavf_init_recv_offload_vlan_v2_caps(adapter); 2491 return; 2492 } 2493 2494 /* When we reach here, no further extended capabilities exchanges are 2495 * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER 2496 */ 2497 iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER); 2498 } 2499 2500 /** 2501 * iavf_init_config_adapter - last part of driver startup 2502 * @adapter: board private structure 2503 * 2504 * After all the supported capabilities are negotiated, then the 2505 * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization. 2506 */ 2507 static void iavf_init_config_adapter(struct iavf_adapter *adapter) 2508 { 2509 struct net_device *netdev = adapter->netdev; 2510 struct pci_dev *pdev = adapter->pdev; 2511 int err; 2512 2513 WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER); 2514 2515 if (iavf_process_config(adapter)) 2516 goto err; 2517 2518 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2519 2520 adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED; 2521 2522 netdev->netdev_ops = &iavf_netdev_ops; 2523 iavf_set_ethtool_ops(netdev); 2524 netdev->watchdog_timeo = 5 * HZ; 2525 2526 /* MTU range: 68 - 9710 */ 2527 netdev->min_mtu = ETH_MIN_MTU; 2528 netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD; 2529 2530 if (!is_valid_ether_addr(adapter->hw.mac.addr)) { 2531 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n", 2532 adapter->hw.mac.addr); 2533 eth_hw_addr_random(netdev); 2534 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr); 2535 } else { 2536 eth_hw_addr_set(netdev, adapter->hw.mac.addr); 2537 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr); 2538 } 2539 2540 adapter->flags |= IAVF_FLAG_INITIAL_MAC_SET; 2541 2542 adapter->tx_desc_count = IAVF_DEFAULT_TXD; 2543 adapter->rx_desc_count = IAVF_DEFAULT_RXD; 2544 err = iavf_init_interrupt_scheme(adapter); 2545 if (err) 2546 goto err_sw_init; 2547 iavf_map_rings_to_vectors(adapter); 2548 if (adapter->vf_res->vf_cap_flags & 2549 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR) 2550 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE; 2551 2552 err = iavf_request_misc_irq(adapter); 2553 if (err) 2554 goto err_sw_init; 2555 2556 netif_carrier_off(netdev); 2557 adapter->link_up = false; 2558 2559 /* set the semaphore to prevent any callbacks after device registration 2560 * up to time when state of driver will be set to __IAVF_DOWN 2561 */ 2562 rtnl_lock(); 2563 if (!adapter->netdev_registered) { 2564 err = register_netdevice(netdev); 2565 if (err) { 2566 rtnl_unlock(); 2567 goto err_register; 2568 } 2569 } 2570 2571 adapter->netdev_registered = true; 2572 2573 netif_tx_stop_all_queues(netdev); 2574 if (CLIENT_ALLOWED(adapter)) { 2575 err = iavf_lan_add_device(adapter); 2576 if (err) 2577 dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n", 2578 err); 2579 } 2580 dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr); 2581 if (netdev->features & NETIF_F_GRO) 2582 dev_info(&pdev->dev, "GRO is enabled\n"); 2583 2584 iavf_change_state(adapter, __IAVF_DOWN); 2585 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 2586 rtnl_unlock(); 2587 2588 iavf_misc_irq_enable(adapter); 2589 wake_up(&adapter->down_waitqueue); 2590 2591 adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL); 2592 adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL); 2593 if (!adapter->rss_key || !adapter->rss_lut) { 2594 err = -ENOMEM; 2595 goto err_mem; 2596 } 2597 if (RSS_AQ(adapter)) 2598 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 2599 else 2600 iavf_init_rss(adapter); 2601 2602 if (VLAN_V2_ALLOWED(adapter)) 2603 /* request initial VLAN offload settings */ 2604 iavf_set_vlan_offload_features(adapter, 0, netdev->features); 2605 2606 return; 2607 err_mem: 2608 iavf_free_rss(adapter); 2609 err_register: 2610 iavf_free_misc_irq(adapter); 2611 err_sw_init: 2612 iavf_reset_interrupt_capability(adapter); 2613 err: 2614 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2615 } 2616 2617 /** 2618 * iavf_watchdog_task - Periodic call-back task 2619 * @work: pointer to work_struct 2620 **/ 2621 static void iavf_watchdog_task(struct work_struct *work) 2622 { 2623 struct iavf_adapter *adapter = container_of(work, 2624 struct iavf_adapter, 2625 watchdog_task.work); 2626 struct iavf_hw *hw = &adapter->hw; 2627 u32 reg_val; 2628 2629 if (!mutex_trylock(&adapter->crit_lock)) { 2630 if (adapter->state == __IAVF_REMOVE) 2631 return; 2632 2633 goto restart_watchdog; 2634 } 2635 2636 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 2637 iavf_change_state(adapter, __IAVF_COMM_FAILED); 2638 2639 if (adapter->flags & IAVF_FLAG_RESET_NEEDED) { 2640 adapter->aq_required = 0; 2641 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2642 mutex_unlock(&adapter->crit_lock); 2643 queue_work(iavf_wq, &adapter->reset_task); 2644 return; 2645 } 2646 2647 switch (adapter->state) { 2648 case __IAVF_STARTUP: 2649 iavf_startup(adapter); 2650 mutex_unlock(&adapter->crit_lock); 2651 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2652 msecs_to_jiffies(30)); 2653 return; 2654 case __IAVF_INIT_VERSION_CHECK: 2655 iavf_init_version_check(adapter); 2656 mutex_unlock(&adapter->crit_lock); 2657 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2658 msecs_to_jiffies(30)); 2659 return; 2660 case __IAVF_INIT_GET_RESOURCES: 2661 iavf_init_get_resources(adapter); 2662 mutex_unlock(&adapter->crit_lock); 2663 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2664 msecs_to_jiffies(1)); 2665 return; 2666 case __IAVF_INIT_EXTENDED_CAPS: 2667 iavf_init_process_extended_caps(adapter); 2668 mutex_unlock(&adapter->crit_lock); 2669 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2670 msecs_to_jiffies(1)); 2671 return; 2672 case __IAVF_INIT_CONFIG_ADAPTER: 2673 iavf_init_config_adapter(adapter); 2674 mutex_unlock(&adapter->crit_lock); 2675 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2676 msecs_to_jiffies(1)); 2677 return; 2678 case __IAVF_INIT_FAILED: 2679 if (test_bit(__IAVF_IN_REMOVE_TASK, 2680 &adapter->crit_section)) { 2681 /* Do not update the state and do not reschedule 2682 * watchdog task, iavf_remove should handle this state 2683 * as it can loop forever 2684 */ 2685 mutex_unlock(&adapter->crit_lock); 2686 return; 2687 } 2688 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) { 2689 dev_err(&adapter->pdev->dev, 2690 "Failed to communicate with PF; waiting before retry\n"); 2691 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 2692 iavf_shutdown_adminq(hw); 2693 mutex_unlock(&adapter->crit_lock); 2694 queue_delayed_work(iavf_wq, 2695 &adapter->watchdog_task, (5 * HZ)); 2696 return; 2697 } 2698 /* Try again from failed step*/ 2699 iavf_change_state(adapter, adapter->last_state); 2700 mutex_unlock(&adapter->crit_lock); 2701 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ); 2702 return; 2703 case __IAVF_COMM_FAILED: 2704 if (test_bit(__IAVF_IN_REMOVE_TASK, 2705 &adapter->crit_section)) { 2706 /* Set state to __IAVF_INIT_FAILED and perform remove 2707 * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task 2708 * doesn't bring the state back to __IAVF_COMM_FAILED. 2709 */ 2710 iavf_change_state(adapter, __IAVF_INIT_FAILED); 2711 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 2712 mutex_unlock(&adapter->crit_lock); 2713 return; 2714 } 2715 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 2716 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 2717 if (reg_val == VIRTCHNL_VFR_VFACTIVE || 2718 reg_val == VIRTCHNL_VFR_COMPLETED) { 2719 /* A chance for redemption! */ 2720 dev_err(&adapter->pdev->dev, 2721 "Hardware came out of reset. Attempting reinit.\n"); 2722 /* When init task contacts the PF and 2723 * gets everything set up again, it'll restart the 2724 * watchdog for us. Down, boy. Sit. Stay. Woof. 2725 */ 2726 iavf_change_state(adapter, __IAVF_STARTUP); 2727 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED; 2728 } 2729 adapter->aq_required = 0; 2730 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2731 mutex_unlock(&adapter->crit_lock); 2732 queue_delayed_work(iavf_wq, 2733 &adapter->watchdog_task, 2734 msecs_to_jiffies(10)); 2735 return; 2736 case __IAVF_RESETTING: 2737 mutex_unlock(&adapter->crit_lock); 2738 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2); 2739 return; 2740 case __IAVF_DOWN: 2741 case __IAVF_DOWN_PENDING: 2742 case __IAVF_TESTING: 2743 case __IAVF_RUNNING: 2744 if (adapter->current_op) { 2745 if (!iavf_asq_done(hw)) { 2746 dev_dbg(&adapter->pdev->dev, 2747 "Admin queue timeout\n"); 2748 iavf_send_api_ver(adapter); 2749 } 2750 } else { 2751 int ret = iavf_process_aq_command(adapter); 2752 2753 /* An error will be returned if no commands were 2754 * processed; use this opportunity to update stats 2755 * if the error isn't -ENOTSUPP 2756 */ 2757 if (ret && ret != -EOPNOTSUPP && 2758 adapter->state == __IAVF_RUNNING) 2759 iavf_request_stats(adapter); 2760 } 2761 if (adapter->state == __IAVF_RUNNING) 2762 iavf_detect_recover_hung(&adapter->vsi); 2763 break; 2764 case __IAVF_REMOVE: 2765 default: 2766 mutex_unlock(&adapter->crit_lock); 2767 return; 2768 } 2769 2770 /* check for hw reset */ 2771 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK; 2772 if (!reg_val) { 2773 adapter->flags |= IAVF_FLAG_RESET_PENDING; 2774 adapter->aq_required = 0; 2775 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2776 dev_err(&adapter->pdev->dev, "Hardware reset detected\n"); 2777 queue_work(iavf_wq, &adapter->reset_task); 2778 mutex_unlock(&adapter->crit_lock); 2779 queue_delayed_work(iavf_wq, 2780 &adapter->watchdog_task, HZ * 2); 2781 return; 2782 } 2783 2784 schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5)); 2785 mutex_unlock(&adapter->crit_lock); 2786 restart_watchdog: 2787 if (adapter->state >= __IAVF_DOWN) 2788 queue_work(iavf_wq, &adapter->adminq_task); 2789 if (adapter->aq_required) 2790 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 2791 msecs_to_jiffies(20)); 2792 else 2793 queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2); 2794 } 2795 2796 /** 2797 * iavf_disable_vf - disable VF 2798 * @adapter: board private structure 2799 * 2800 * Set communication failed flag and free all resources. 2801 * NOTE: This function is expected to be called with crit_lock being held. 2802 **/ 2803 static void iavf_disable_vf(struct iavf_adapter *adapter) 2804 { 2805 struct iavf_mac_filter *f, *ftmp; 2806 struct iavf_vlan_filter *fv, *fvtmp; 2807 struct iavf_cloud_filter *cf, *cftmp; 2808 2809 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED; 2810 2811 /* We don't use netif_running() because it may be true prior to 2812 * ndo_open() returning, so we can't assume it means all our open 2813 * tasks have finished, since we're not holding the rtnl_lock here. 2814 */ 2815 if (adapter->state == __IAVF_RUNNING) { 2816 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 2817 netif_carrier_off(adapter->netdev); 2818 netif_tx_disable(adapter->netdev); 2819 adapter->link_up = false; 2820 iavf_napi_disable_all(adapter); 2821 iavf_irq_disable(adapter); 2822 iavf_free_traffic_irqs(adapter); 2823 iavf_free_all_tx_resources(adapter); 2824 iavf_free_all_rx_resources(adapter); 2825 } 2826 2827 spin_lock_bh(&adapter->mac_vlan_list_lock); 2828 2829 /* Delete all of the filters */ 2830 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 2831 list_del(&f->list); 2832 kfree(f); 2833 } 2834 2835 list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) { 2836 list_del(&fv->list); 2837 kfree(fv); 2838 } 2839 2840 spin_unlock_bh(&adapter->mac_vlan_list_lock); 2841 2842 spin_lock_bh(&adapter->cloud_filter_list_lock); 2843 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 2844 list_del(&cf->list); 2845 kfree(cf); 2846 adapter->num_cloud_filters--; 2847 } 2848 spin_unlock_bh(&adapter->cloud_filter_list_lock); 2849 2850 iavf_free_misc_irq(adapter); 2851 iavf_reset_interrupt_capability(adapter); 2852 iavf_free_q_vectors(adapter); 2853 iavf_free_queues(adapter); 2854 memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE); 2855 iavf_shutdown_adminq(&adapter->hw); 2856 adapter->netdev->flags &= ~IFF_UP; 2857 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2858 iavf_change_state(adapter, __IAVF_DOWN); 2859 wake_up(&adapter->down_waitqueue); 2860 dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n"); 2861 } 2862 2863 /** 2864 * iavf_reset_task - Call-back task to handle hardware reset 2865 * @work: pointer to work_struct 2866 * 2867 * During reset we need to shut down and reinitialize the admin queue 2868 * before we can use it to communicate with the PF again. We also clear 2869 * and reinit the rings because that context is lost as well. 2870 **/ 2871 static void iavf_reset_task(struct work_struct *work) 2872 { 2873 struct iavf_adapter *adapter = container_of(work, 2874 struct iavf_adapter, 2875 reset_task); 2876 struct virtchnl_vf_resource *vfres = adapter->vf_res; 2877 struct net_device *netdev = adapter->netdev; 2878 struct iavf_hw *hw = &adapter->hw; 2879 struct iavf_mac_filter *f, *ftmp; 2880 struct iavf_cloud_filter *cf; 2881 enum iavf_status status; 2882 u32 reg_val; 2883 int i = 0, err; 2884 bool running; 2885 2886 /* When device is being removed it doesn't make sense to run the reset 2887 * task, just return in such a case. 2888 */ 2889 if (!mutex_trylock(&adapter->crit_lock)) { 2890 if (adapter->state != __IAVF_REMOVE) 2891 queue_work(iavf_wq, &adapter->reset_task); 2892 2893 return; 2894 } 2895 2896 while (!mutex_trylock(&adapter->client_lock)) 2897 usleep_range(500, 1000); 2898 if (CLIENT_ENABLED(adapter)) { 2899 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN | 2900 IAVF_FLAG_CLIENT_NEEDS_CLOSE | 2901 IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS | 2902 IAVF_FLAG_SERVICE_CLIENT_REQUESTED); 2903 cancel_delayed_work_sync(&adapter->client_task); 2904 iavf_notify_client_close(&adapter->vsi, true); 2905 } 2906 iavf_misc_irq_disable(adapter); 2907 if (adapter->flags & IAVF_FLAG_RESET_NEEDED) { 2908 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED; 2909 /* Restart the AQ here. If we have been reset but didn't 2910 * detect it, or if the PF had to reinit, our AQ will be hosed. 2911 */ 2912 iavf_shutdown_adminq(hw); 2913 iavf_init_adminq(hw); 2914 iavf_request_reset(adapter); 2915 } 2916 adapter->flags |= IAVF_FLAG_RESET_PENDING; 2917 2918 /* poll until we see the reset actually happen */ 2919 for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) { 2920 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & 2921 IAVF_VF_ARQLEN1_ARQENABLE_MASK; 2922 if (!reg_val) 2923 break; 2924 usleep_range(5000, 10000); 2925 } 2926 if (i == IAVF_RESET_WAIT_DETECTED_COUNT) { 2927 dev_info(&adapter->pdev->dev, "Never saw reset\n"); 2928 goto continue_reset; /* act like the reset happened */ 2929 } 2930 2931 /* wait until the reset is complete and the PF is responding to us */ 2932 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { 2933 /* sleep first to make sure a minimum wait time is met */ 2934 msleep(IAVF_RESET_WAIT_MS); 2935 2936 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) & 2937 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 2938 if (reg_val == VIRTCHNL_VFR_VFACTIVE) 2939 break; 2940 } 2941 2942 pci_set_master(adapter->pdev); 2943 pci_restore_msi_state(adapter->pdev); 2944 2945 if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) { 2946 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n", 2947 reg_val); 2948 iavf_disable_vf(adapter); 2949 mutex_unlock(&adapter->client_lock); 2950 mutex_unlock(&adapter->crit_lock); 2951 return; /* Do not attempt to reinit. It's dead, Jim. */ 2952 } 2953 2954 continue_reset: 2955 /* We don't use netif_running() because it may be true prior to 2956 * ndo_open() returning, so we can't assume it means all our open 2957 * tasks have finished, since we're not holding the rtnl_lock here. 2958 */ 2959 running = adapter->state == __IAVF_RUNNING; 2960 2961 if (running) { 2962 netif_carrier_off(netdev); 2963 netif_tx_stop_all_queues(netdev); 2964 adapter->link_up = false; 2965 iavf_napi_disable_all(adapter); 2966 } 2967 iavf_irq_disable(adapter); 2968 2969 iavf_change_state(adapter, __IAVF_RESETTING); 2970 adapter->flags &= ~IAVF_FLAG_RESET_PENDING; 2971 2972 /* free the Tx/Rx rings and descriptors, might be better to just 2973 * re-use them sometime in the future 2974 */ 2975 iavf_free_all_rx_resources(adapter); 2976 iavf_free_all_tx_resources(adapter); 2977 2978 adapter->flags |= IAVF_FLAG_QUEUES_DISABLED; 2979 /* kill and reinit the admin queue */ 2980 iavf_shutdown_adminq(hw); 2981 adapter->current_op = VIRTCHNL_OP_UNKNOWN; 2982 status = iavf_init_adminq(hw); 2983 if (status) { 2984 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n", 2985 status); 2986 goto reset_err; 2987 } 2988 adapter->aq_required = 0; 2989 2990 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) || 2991 (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) { 2992 err = iavf_reinit_interrupt_scheme(adapter); 2993 if (err) 2994 goto reset_err; 2995 } 2996 2997 if (RSS_AQ(adapter)) { 2998 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS; 2999 } else { 3000 err = iavf_init_rss(adapter); 3001 if (err) 3002 goto reset_err; 3003 } 3004 3005 adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG; 3006 /* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been 3007 * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here, 3008 * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until 3009 * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have 3010 * been successfully sent and negotiated 3011 */ 3012 adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS; 3013 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS; 3014 3015 spin_lock_bh(&adapter->mac_vlan_list_lock); 3016 3017 /* Delete filter for the current MAC address, it could have 3018 * been changed by the PF via administratively set MAC. 3019 * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES. 3020 */ 3021 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 3022 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) { 3023 list_del(&f->list); 3024 kfree(f); 3025 } 3026 } 3027 /* re-add all MAC filters */ 3028 list_for_each_entry(f, &adapter->mac_filter_list, list) { 3029 f->add = true; 3030 } 3031 spin_unlock_bh(&adapter->mac_vlan_list_lock); 3032 3033 /* check if TCs are running and re-add all cloud filters */ 3034 spin_lock_bh(&adapter->cloud_filter_list_lock); 3035 if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) && 3036 adapter->num_tc) { 3037 list_for_each_entry(cf, &adapter->cloud_filter_list, list) { 3038 cf->add = true; 3039 } 3040 } 3041 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3042 3043 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER; 3044 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 3045 iavf_misc_irq_enable(adapter); 3046 3047 mod_delayed_work(iavf_wq, &adapter->watchdog_task, 2); 3048 3049 /* We were running when the reset started, so we need to restore some 3050 * state here. 3051 */ 3052 if (running) { 3053 /* allocate transmit descriptors */ 3054 err = iavf_setup_all_tx_resources(adapter); 3055 if (err) 3056 goto reset_err; 3057 3058 /* allocate receive descriptors */ 3059 err = iavf_setup_all_rx_resources(adapter); 3060 if (err) 3061 goto reset_err; 3062 3063 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) || 3064 (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) { 3065 err = iavf_request_traffic_irqs(adapter, netdev->name); 3066 if (err) 3067 goto reset_err; 3068 3069 adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED; 3070 } 3071 3072 iavf_configure(adapter); 3073 3074 /* iavf_up_complete() will switch device back 3075 * to __IAVF_RUNNING 3076 */ 3077 iavf_up_complete(adapter); 3078 3079 iavf_irq_enable(adapter, true); 3080 } else { 3081 iavf_change_state(adapter, __IAVF_DOWN); 3082 wake_up(&adapter->down_waitqueue); 3083 } 3084 3085 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 3086 3087 mutex_unlock(&adapter->client_lock); 3088 mutex_unlock(&adapter->crit_lock); 3089 3090 return; 3091 reset_err: 3092 mutex_unlock(&adapter->client_lock); 3093 mutex_unlock(&adapter->crit_lock); 3094 if (running) 3095 iavf_change_state(adapter, __IAVF_RUNNING); 3096 dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n"); 3097 iavf_close(netdev); 3098 } 3099 3100 /** 3101 * iavf_adminq_task - worker thread to clean the admin queue 3102 * @work: pointer to work_struct containing our data 3103 **/ 3104 static void iavf_adminq_task(struct work_struct *work) 3105 { 3106 struct iavf_adapter *adapter = 3107 container_of(work, struct iavf_adapter, adminq_task); 3108 struct iavf_hw *hw = &adapter->hw; 3109 struct iavf_arq_event_info event; 3110 enum virtchnl_ops v_op; 3111 enum iavf_status ret, v_ret; 3112 u32 val, oldval; 3113 u16 pending; 3114 3115 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) 3116 goto out; 3117 3118 if (!mutex_trylock(&adapter->crit_lock)) { 3119 if (adapter->state == __IAVF_REMOVE) 3120 return; 3121 3122 queue_work(iavf_wq, &adapter->adminq_task); 3123 goto out; 3124 } 3125 3126 event.buf_len = IAVF_MAX_AQ_BUF_SIZE; 3127 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 3128 if (!event.msg_buf) 3129 goto out; 3130 3131 do { 3132 ret = iavf_clean_arq_element(hw, &event, &pending); 3133 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high); 3134 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low); 3135 3136 if (ret || !v_op) 3137 break; /* No event to process or error cleaning ARQ */ 3138 3139 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf, 3140 event.msg_len); 3141 if (pending != 0) 3142 memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE); 3143 } while (pending); 3144 mutex_unlock(&adapter->crit_lock); 3145 3146 if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES)) { 3147 if (adapter->netdev_registered || 3148 !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) { 3149 struct net_device *netdev = adapter->netdev; 3150 3151 rtnl_lock(); 3152 netdev_update_features(netdev); 3153 rtnl_unlock(); 3154 /* Request VLAN offload settings */ 3155 if (VLAN_V2_ALLOWED(adapter)) 3156 iavf_set_vlan_offload_features 3157 (adapter, 0, netdev->features); 3158 3159 iavf_set_queue_vlan_tag_loc(adapter); 3160 } 3161 3162 adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES; 3163 } 3164 if ((adapter->flags & 3165 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) || 3166 adapter->state == __IAVF_RESETTING) 3167 goto freedom; 3168 3169 /* check for error indications */ 3170 val = rd32(hw, hw->aq.arq.len); 3171 if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */ 3172 goto freedom; 3173 oldval = val; 3174 if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) { 3175 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n"); 3176 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK; 3177 } 3178 if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) { 3179 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n"); 3180 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK; 3181 } 3182 if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) { 3183 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n"); 3184 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK; 3185 } 3186 if (oldval != val) 3187 wr32(hw, hw->aq.arq.len, val); 3188 3189 val = rd32(hw, hw->aq.asq.len); 3190 oldval = val; 3191 if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) { 3192 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n"); 3193 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK; 3194 } 3195 if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) { 3196 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n"); 3197 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK; 3198 } 3199 if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) { 3200 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n"); 3201 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK; 3202 } 3203 if (oldval != val) 3204 wr32(hw, hw->aq.asq.len, val); 3205 3206 freedom: 3207 kfree(event.msg_buf); 3208 out: 3209 /* re-enable Admin queue interrupt cause */ 3210 iavf_misc_irq_enable(adapter); 3211 } 3212 3213 /** 3214 * iavf_client_task - worker thread to perform client work 3215 * @work: pointer to work_struct containing our data 3216 * 3217 * This task handles client interactions. Because client calls can be 3218 * reentrant, we can't handle them in the watchdog. 3219 **/ 3220 static void iavf_client_task(struct work_struct *work) 3221 { 3222 struct iavf_adapter *adapter = 3223 container_of(work, struct iavf_adapter, client_task.work); 3224 3225 /* If we can't get the client bit, just give up. We'll be rescheduled 3226 * later. 3227 */ 3228 3229 if (!mutex_trylock(&adapter->client_lock)) 3230 return; 3231 3232 if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) { 3233 iavf_client_subtask(adapter); 3234 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED; 3235 goto out; 3236 } 3237 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) { 3238 iavf_notify_client_l2_params(&adapter->vsi); 3239 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS; 3240 goto out; 3241 } 3242 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) { 3243 iavf_notify_client_close(&adapter->vsi, false); 3244 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE; 3245 goto out; 3246 } 3247 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) { 3248 iavf_notify_client_open(&adapter->vsi); 3249 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN; 3250 } 3251 out: 3252 mutex_unlock(&adapter->client_lock); 3253 } 3254 3255 /** 3256 * iavf_free_all_tx_resources - Free Tx Resources for All Queues 3257 * @adapter: board private structure 3258 * 3259 * Free all transmit software resources 3260 **/ 3261 void iavf_free_all_tx_resources(struct iavf_adapter *adapter) 3262 { 3263 int i; 3264 3265 if (!adapter->tx_rings) 3266 return; 3267 3268 for (i = 0; i < adapter->num_active_queues; i++) 3269 if (adapter->tx_rings[i].desc) 3270 iavf_free_tx_resources(&adapter->tx_rings[i]); 3271 } 3272 3273 /** 3274 * iavf_setup_all_tx_resources - allocate all queues Tx resources 3275 * @adapter: board private structure 3276 * 3277 * If this function returns with an error, then it's possible one or 3278 * more of the rings is populated (while the rest are not). It is the 3279 * callers duty to clean those orphaned rings. 3280 * 3281 * Return 0 on success, negative on failure 3282 **/ 3283 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter) 3284 { 3285 int i, err = 0; 3286 3287 for (i = 0; i < adapter->num_active_queues; i++) { 3288 adapter->tx_rings[i].count = adapter->tx_desc_count; 3289 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]); 3290 if (!err) 3291 continue; 3292 dev_err(&adapter->pdev->dev, 3293 "Allocation for Tx Queue %u failed\n", i); 3294 break; 3295 } 3296 3297 return err; 3298 } 3299 3300 /** 3301 * iavf_setup_all_rx_resources - allocate all queues Rx resources 3302 * @adapter: board private structure 3303 * 3304 * If this function returns with an error, then it's possible one or 3305 * more of the rings is populated (while the rest are not). It is the 3306 * callers duty to clean those orphaned rings. 3307 * 3308 * Return 0 on success, negative on failure 3309 **/ 3310 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter) 3311 { 3312 int i, err = 0; 3313 3314 for (i = 0; i < adapter->num_active_queues; i++) { 3315 adapter->rx_rings[i].count = adapter->rx_desc_count; 3316 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]); 3317 if (!err) 3318 continue; 3319 dev_err(&adapter->pdev->dev, 3320 "Allocation for Rx Queue %u failed\n", i); 3321 break; 3322 } 3323 return err; 3324 } 3325 3326 /** 3327 * iavf_free_all_rx_resources - Free Rx Resources for All Queues 3328 * @adapter: board private structure 3329 * 3330 * Free all receive software resources 3331 **/ 3332 void iavf_free_all_rx_resources(struct iavf_adapter *adapter) 3333 { 3334 int i; 3335 3336 if (!adapter->rx_rings) 3337 return; 3338 3339 for (i = 0; i < adapter->num_active_queues; i++) 3340 if (adapter->rx_rings[i].desc) 3341 iavf_free_rx_resources(&adapter->rx_rings[i]); 3342 } 3343 3344 /** 3345 * iavf_validate_tx_bandwidth - validate the max Tx bandwidth 3346 * @adapter: board private structure 3347 * @max_tx_rate: max Tx bw for a tc 3348 **/ 3349 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter, 3350 u64 max_tx_rate) 3351 { 3352 int speed = 0, ret = 0; 3353 3354 if (ADV_LINK_SUPPORT(adapter)) { 3355 if (adapter->link_speed_mbps < U32_MAX) { 3356 speed = adapter->link_speed_mbps; 3357 goto validate_bw; 3358 } else { 3359 dev_err(&adapter->pdev->dev, "Unknown link speed\n"); 3360 return -EINVAL; 3361 } 3362 } 3363 3364 switch (adapter->link_speed) { 3365 case VIRTCHNL_LINK_SPEED_40GB: 3366 speed = SPEED_40000; 3367 break; 3368 case VIRTCHNL_LINK_SPEED_25GB: 3369 speed = SPEED_25000; 3370 break; 3371 case VIRTCHNL_LINK_SPEED_20GB: 3372 speed = SPEED_20000; 3373 break; 3374 case VIRTCHNL_LINK_SPEED_10GB: 3375 speed = SPEED_10000; 3376 break; 3377 case VIRTCHNL_LINK_SPEED_5GB: 3378 speed = SPEED_5000; 3379 break; 3380 case VIRTCHNL_LINK_SPEED_2_5GB: 3381 speed = SPEED_2500; 3382 break; 3383 case VIRTCHNL_LINK_SPEED_1GB: 3384 speed = SPEED_1000; 3385 break; 3386 case VIRTCHNL_LINK_SPEED_100MB: 3387 speed = SPEED_100; 3388 break; 3389 default: 3390 break; 3391 } 3392 3393 validate_bw: 3394 if (max_tx_rate > speed) { 3395 dev_err(&adapter->pdev->dev, 3396 "Invalid tx rate specified\n"); 3397 ret = -EINVAL; 3398 } 3399 3400 return ret; 3401 } 3402 3403 /** 3404 * iavf_validate_ch_config - validate queue mapping info 3405 * @adapter: board private structure 3406 * @mqprio_qopt: queue parameters 3407 * 3408 * This function validates if the config provided by the user to 3409 * configure queue channels is valid or not. Returns 0 on a valid 3410 * config. 3411 **/ 3412 static int iavf_validate_ch_config(struct iavf_adapter *adapter, 3413 struct tc_mqprio_qopt_offload *mqprio_qopt) 3414 { 3415 u64 total_max_rate = 0; 3416 int i, num_qps = 0; 3417 u64 tx_rate = 0; 3418 int ret = 0; 3419 3420 if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS || 3421 mqprio_qopt->qopt.num_tc < 1) 3422 return -EINVAL; 3423 3424 for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) { 3425 if (!mqprio_qopt->qopt.count[i] || 3426 mqprio_qopt->qopt.offset[i] != num_qps) 3427 return -EINVAL; 3428 if (mqprio_qopt->min_rate[i]) { 3429 dev_err(&adapter->pdev->dev, 3430 "Invalid min tx rate (greater than 0) specified\n"); 3431 return -EINVAL; 3432 } 3433 /*convert to Mbps */ 3434 tx_rate = div_u64(mqprio_qopt->max_rate[i], 3435 IAVF_MBPS_DIVISOR); 3436 total_max_rate += tx_rate; 3437 num_qps += mqprio_qopt->qopt.count[i]; 3438 } 3439 if (num_qps > adapter->num_active_queues) { 3440 dev_err(&adapter->pdev->dev, 3441 "Cannot support requested number of queues\n"); 3442 return -EINVAL; 3443 } 3444 3445 ret = iavf_validate_tx_bandwidth(adapter, total_max_rate); 3446 return ret; 3447 } 3448 3449 /** 3450 * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes 3451 * @adapter: board private structure 3452 **/ 3453 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter) 3454 { 3455 struct iavf_cloud_filter *cf, *cftmp; 3456 3457 spin_lock_bh(&adapter->cloud_filter_list_lock); 3458 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, 3459 list) { 3460 list_del(&cf->list); 3461 kfree(cf); 3462 adapter->num_cloud_filters--; 3463 } 3464 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3465 } 3466 3467 /** 3468 * __iavf_setup_tc - configure multiple traffic classes 3469 * @netdev: network interface device structure 3470 * @type_data: tc offload data 3471 * 3472 * This function processes the config information provided by the 3473 * user to configure traffic classes/queue channels and packages the 3474 * information to request the PF to setup traffic classes. 3475 * 3476 * Returns 0 on success. 3477 **/ 3478 static int __iavf_setup_tc(struct net_device *netdev, void *type_data) 3479 { 3480 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 3481 struct iavf_adapter *adapter = netdev_priv(netdev); 3482 struct virtchnl_vf_resource *vfres = adapter->vf_res; 3483 u8 num_tc = 0, total_qps = 0; 3484 int ret = 0, netdev_tc = 0; 3485 u64 max_tx_rate; 3486 u16 mode; 3487 int i; 3488 3489 num_tc = mqprio_qopt->qopt.num_tc; 3490 mode = mqprio_qopt->mode; 3491 3492 /* delete queue_channel */ 3493 if (!mqprio_qopt->qopt.hw) { 3494 if (adapter->ch_config.state == __IAVF_TC_RUNNING) { 3495 /* reset the tc configuration */ 3496 netdev_reset_tc(netdev); 3497 adapter->num_tc = 0; 3498 netif_tx_stop_all_queues(netdev); 3499 netif_tx_disable(netdev); 3500 iavf_del_all_cloud_filters(adapter); 3501 adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS; 3502 goto exit; 3503 } else { 3504 return -EINVAL; 3505 } 3506 } 3507 3508 /* add queue channel */ 3509 if (mode == TC_MQPRIO_MODE_CHANNEL) { 3510 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) { 3511 dev_err(&adapter->pdev->dev, "ADq not supported\n"); 3512 return -EOPNOTSUPP; 3513 } 3514 if (adapter->ch_config.state != __IAVF_TC_INVALID) { 3515 dev_err(&adapter->pdev->dev, "TC configuration already exists\n"); 3516 return -EINVAL; 3517 } 3518 3519 ret = iavf_validate_ch_config(adapter, mqprio_qopt); 3520 if (ret) 3521 return ret; 3522 /* Return if same TC config is requested */ 3523 if (adapter->num_tc == num_tc) 3524 return 0; 3525 adapter->num_tc = num_tc; 3526 3527 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 3528 if (i < num_tc) { 3529 adapter->ch_config.ch_info[i].count = 3530 mqprio_qopt->qopt.count[i]; 3531 adapter->ch_config.ch_info[i].offset = 3532 mqprio_qopt->qopt.offset[i]; 3533 total_qps += mqprio_qopt->qopt.count[i]; 3534 max_tx_rate = mqprio_qopt->max_rate[i]; 3535 /* convert to Mbps */ 3536 max_tx_rate = div_u64(max_tx_rate, 3537 IAVF_MBPS_DIVISOR); 3538 adapter->ch_config.ch_info[i].max_tx_rate = 3539 max_tx_rate; 3540 } else { 3541 adapter->ch_config.ch_info[i].count = 1; 3542 adapter->ch_config.ch_info[i].offset = 0; 3543 } 3544 } 3545 adapter->ch_config.total_qps = total_qps; 3546 netif_tx_stop_all_queues(netdev); 3547 netif_tx_disable(netdev); 3548 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS; 3549 netdev_reset_tc(netdev); 3550 /* Report the tc mapping up the stack */ 3551 netdev_set_num_tc(adapter->netdev, num_tc); 3552 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) { 3553 u16 qcount = mqprio_qopt->qopt.count[i]; 3554 u16 qoffset = mqprio_qopt->qopt.offset[i]; 3555 3556 if (i < num_tc) 3557 netdev_set_tc_queue(netdev, netdev_tc++, qcount, 3558 qoffset); 3559 } 3560 } 3561 exit: 3562 return ret; 3563 } 3564 3565 /** 3566 * iavf_parse_cls_flower - Parse tc flower filters provided by kernel 3567 * @adapter: board private structure 3568 * @f: pointer to struct flow_cls_offload 3569 * @filter: pointer to cloud filter structure 3570 */ 3571 static int iavf_parse_cls_flower(struct iavf_adapter *adapter, 3572 struct flow_cls_offload *f, 3573 struct iavf_cloud_filter *filter) 3574 { 3575 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 3576 struct flow_dissector *dissector = rule->match.dissector; 3577 u16 n_proto_mask = 0; 3578 u16 n_proto_key = 0; 3579 u8 field_flags = 0; 3580 u16 addr_type = 0; 3581 u16 n_proto = 0; 3582 int i = 0; 3583 struct virtchnl_filter *vf = &filter->f; 3584 3585 if (dissector->used_keys & 3586 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 3587 BIT(FLOW_DISSECTOR_KEY_BASIC) | 3588 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 3589 BIT(FLOW_DISSECTOR_KEY_VLAN) | 3590 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 3591 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 3592 BIT(FLOW_DISSECTOR_KEY_PORTS) | 3593 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 3594 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n", 3595 dissector->used_keys); 3596 return -EOPNOTSUPP; 3597 } 3598 3599 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 3600 struct flow_match_enc_keyid match; 3601 3602 flow_rule_match_enc_keyid(rule, &match); 3603 if (match.mask->keyid != 0) 3604 field_flags |= IAVF_CLOUD_FIELD_TEN_ID; 3605 } 3606 3607 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 3608 struct flow_match_basic match; 3609 3610 flow_rule_match_basic(rule, &match); 3611 n_proto_key = ntohs(match.key->n_proto); 3612 n_proto_mask = ntohs(match.mask->n_proto); 3613 3614 if (n_proto_key == ETH_P_ALL) { 3615 n_proto_key = 0; 3616 n_proto_mask = 0; 3617 } 3618 n_proto = n_proto_key & n_proto_mask; 3619 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6) 3620 return -EINVAL; 3621 if (n_proto == ETH_P_IPV6) { 3622 /* specify flow type as TCP IPv6 */ 3623 vf->flow_type = VIRTCHNL_TCP_V6_FLOW; 3624 } 3625 3626 if (match.key->ip_proto != IPPROTO_TCP) { 3627 dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n"); 3628 return -EINVAL; 3629 } 3630 } 3631 3632 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 3633 struct flow_match_eth_addrs match; 3634 3635 flow_rule_match_eth_addrs(rule, &match); 3636 3637 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 3638 if (!is_zero_ether_addr(match.mask->dst)) { 3639 if (is_broadcast_ether_addr(match.mask->dst)) { 3640 field_flags |= IAVF_CLOUD_FIELD_OMAC; 3641 } else { 3642 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n", 3643 match.mask->dst); 3644 return -EINVAL; 3645 } 3646 } 3647 3648 if (!is_zero_ether_addr(match.mask->src)) { 3649 if (is_broadcast_ether_addr(match.mask->src)) { 3650 field_flags |= IAVF_CLOUD_FIELD_IMAC; 3651 } else { 3652 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n", 3653 match.mask->src); 3654 return -EINVAL; 3655 } 3656 } 3657 3658 if (!is_zero_ether_addr(match.key->dst)) 3659 if (is_valid_ether_addr(match.key->dst) || 3660 is_multicast_ether_addr(match.key->dst)) { 3661 /* set the mask if a valid dst_mac address */ 3662 for (i = 0; i < ETH_ALEN; i++) 3663 vf->mask.tcp_spec.dst_mac[i] |= 0xff; 3664 ether_addr_copy(vf->data.tcp_spec.dst_mac, 3665 match.key->dst); 3666 } 3667 3668 if (!is_zero_ether_addr(match.key->src)) 3669 if (is_valid_ether_addr(match.key->src) || 3670 is_multicast_ether_addr(match.key->src)) { 3671 /* set the mask if a valid dst_mac address */ 3672 for (i = 0; i < ETH_ALEN; i++) 3673 vf->mask.tcp_spec.src_mac[i] |= 0xff; 3674 ether_addr_copy(vf->data.tcp_spec.src_mac, 3675 match.key->src); 3676 } 3677 } 3678 3679 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 3680 struct flow_match_vlan match; 3681 3682 flow_rule_match_vlan(rule, &match); 3683 if (match.mask->vlan_id) { 3684 if (match.mask->vlan_id == VLAN_VID_MASK) { 3685 field_flags |= IAVF_CLOUD_FIELD_IVLAN; 3686 } else { 3687 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n", 3688 match.mask->vlan_id); 3689 return -EINVAL; 3690 } 3691 } 3692 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff); 3693 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id); 3694 } 3695 3696 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 3697 struct flow_match_control match; 3698 3699 flow_rule_match_control(rule, &match); 3700 addr_type = match.key->addr_type; 3701 } 3702 3703 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 3704 struct flow_match_ipv4_addrs match; 3705 3706 flow_rule_match_ipv4_addrs(rule, &match); 3707 if (match.mask->dst) { 3708 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 3709 field_flags |= IAVF_CLOUD_FIELD_IIP; 3710 } else { 3711 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n", 3712 be32_to_cpu(match.mask->dst)); 3713 return -EINVAL; 3714 } 3715 } 3716 3717 if (match.mask->src) { 3718 if (match.mask->src == cpu_to_be32(0xffffffff)) { 3719 field_flags |= IAVF_CLOUD_FIELD_IIP; 3720 } else { 3721 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n", 3722 be32_to_cpu(match.mask->dst)); 3723 return -EINVAL; 3724 } 3725 } 3726 3727 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) { 3728 dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n"); 3729 return -EINVAL; 3730 } 3731 if (match.key->dst) { 3732 vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff); 3733 vf->data.tcp_spec.dst_ip[0] = match.key->dst; 3734 } 3735 if (match.key->src) { 3736 vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff); 3737 vf->data.tcp_spec.src_ip[0] = match.key->src; 3738 } 3739 } 3740 3741 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 3742 struct flow_match_ipv6_addrs match; 3743 3744 flow_rule_match_ipv6_addrs(rule, &match); 3745 3746 /* validate mask, make sure it is not IPV6_ADDR_ANY */ 3747 if (ipv6_addr_any(&match.mask->dst)) { 3748 dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n", 3749 IPV6_ADDR_ANY); 3750 return -EINVAL; 3751 } 3752 3753 /* src and dest IPv6 address should not be LOOPBACK 3754 * (0:0:0:0:0:0:0:1) which can be represented as ::1 3755 */ 3756 if (ipv6_addr_loopback(&match.key->dst) || 3757 ipv6_addr_loopback(&match.key->src)) { 3758 dev_err(&adapter->pdev->dev, 3759 "ipv6 addr should not be loopback\n"); 3760 return -EINVAL; 3761 } 3762 if (!ipv6_addr_any(&match.mask->dst) || 3763 !ipv6_addr_any(&match.mask->src)) 3764 field_flags |= IAVF_CLOUD_FIELD_IIP; 3765 3766 for (i = 0; i < 4; i++) 3767 vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff); 3768 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32, 3769 sizeof(vf->data.tcp_spec.dst_ip)); 3770 for (i = 0; i < 4; i++) 3771 vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff); 3772 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32, 3773 sizeof(vf->data.tcp_spec.src_ip)); 3774 } 3775 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 3776 struct flow_match_ports match; 3777 3778 flow_rule_match_ports(rule, &match); 3779 if (match.mask->src) { 3780 if (match.mask->src == cpu_to_be16(0xffff)) { 3781 field_flags |= IAVF_CLOUD_FIELD_IIP; 3782 } else { 3783 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n", 3784 be16_to_cpu(match.mask->src)); 3785 return -EINVAL; 3786 } 3787 } 3788 3789 if (match.mask->dst) { 3790 if (match.mask->dst == cpu_to_be16(0xffff)) { 3791 field_flags |= IAVF_CLOUD_FIELD_IIP; 3792 } else { 3793 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n", 3794 be16_to_cpu(match.mask->dst)); 3795 return -EINVAL; 3796 } 3797 } 3798 if (match.key->dst) { 3799 vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff); 3800 vf->data.tcp_spec.dst_port = match.key->dst; 3801 } 3802 3803 if (match.key->src) { 3804 vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff); 3805 vf->data.tcp_spec.src_port = match.key->src; 3806 } 3807 } 3808 vf->field_flags = field_flags; 3809 3810 return 0; 3811 } 3812 3813 /** 3814 * iavf_handle_tclass - Forward to a traffic class on the device 3815 * @adapter: board private structure 3816 * @tc: traffic class index on the device 3817 * @filter: pointer to cloud filter structure 3818 */ 3819 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc, 3820 struct iavf_cloud_filter *filter) 3821 { 3822 if (tc == 0) 3823 return 0; 3824 if (tc < adapter->num_tc) { 3825 if (!filter->f.data.tcp_spec.dst_port) { 3826 dev_err(&adapter->pdev->dev, 3827 "Specify destination port to redirect to traffic class other than TC0\n"); 3828 return -EINVAL; 3829 } 3830 } 3831 /* redirect to a traffic class on the same device */ 3832 filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT; 3833 filter->f.action_meta = tc; 3834 return 0; 3835 } 3836 3837 /** 3838 * iavf_configure_clsflower - Add tc flower filters 3839 * @adapter: board private structure 3840 * @cls_flower: Pointer to struct flow_cls_offload 3841 */ 3842 static int iavf_configure_clsflower(struct iavf_adapter *adapter, 3843 struct flow_cls_offload *cls_flower) 3844 { 3845 int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid); 3846 struct iavf_cloud_filter *filter = NULL; 3847 int err = -EINVAL, count = 50; 3848 3849 if (tc < 0) { 3850 dev_err(&adapter->pdev->dev, "Invalid traffic class\n"); 3851 return -EINVAL; 3852 } 3853 3854 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 3855 if (!filter) 3856 return -ENOMEM; 3857 3858 while (!mutex_trylock(&adapter->crit_lock)) { 3859 if (--count == 0) { 3860 kfree(filter); 3861 return err; 3862 } 3863 udelay(1); 3864 } 3865 3866 filter->cookie = cls_flower->cookie; 3867 3868 /* set the mask to all zeroes to begin with */ 3869 memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec)); 3870 /* start out with flow type and eth type IPv4 to begin with */ 3871 filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW; 3872 err = iavf_parse_cls_flower(adapter, cls_flower, filter); 3873 if (err) 3874 goto err; 3875 3876 err = iavf_handle_tclass(adapter, tc, filter); 3877 if (err) 3878 goto err; 3879 3880 /* add filter to the list */ 3881 spin_lock_bh(&adapter->cloud_filter_list_lock); 3882 list_add_tail(&filter->list, &adapter->cloud_filter_list); 3883 adapter->num_cloud_filters++; 3884 filter->add = true; 3885 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER; 3886 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3887 err: 3888 if (err) 3889 kfree(filter); 3890 3891 mutex_unlock(&adapter->crit_lock); 3892 return err; 3893 } 3894 3895 /* iavf_find_cf - Find the cloud filter in the list 3896 * @adapter: Board private structure 3897 * @cookie: filter specific cookie 3898 * 3899 * Returns ptr to the filter object or NULL. Must be called while holding the 3900 * cloud_filter_list_lock. 3901 */ 3902 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter, 3903 unsigned long *cookie) 3904 { 3905 struct iavf_cloud_filter *filter = NULL; 3906 3907 if (!cookie) 3908 return NULL; 3909 3910 list_for_each_entry(filter, &adapter->cloud_filter_list, list) { 3911 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 3912 return filter; 3913 } 3914 return NULL; 3915 } 3916 3917 /** 3918 * iavf_delete_clsflower - Remove tc flower filters 3919 * @adapter: board private structure 3920 * @cls_flower: Pointer to struct flow_cls_offload 3921 */ 3922 static int iavf_delete_clsflower(struct iavf_adapter *adapter, 3923 struct flow_cls_offload *cls_flower) 3924 { 3925 struct iavf_cloud_filter *filter = NULL; 3926 int err = 0; 3927 3928 spin_lock_bh(&adapter->cloud_filter_list_lock); 3929 filter = iavf_find_cf(adapter, &cls_flower->cookie); 3930 if (filter) { 3931 filter->del = true; 3932 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER; 3933 } else { 3934 err = -EINVAL; 3935 } 3936 spin_unlock_bh(&adapter->cloud_filter_list_lock); 3937 3938 return err; 3939 } 3940 3941 /** 3942 * iavf_setup_tc_cls_flower - flower classifier offloads 3943 * @adapter: board private structure 3944 * @cls_flower: pointer to flow_cls_offload struct with flow info 3945 */ 3946 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, 3947 struct flow_cls_offload *cls_flower) 3948 { 3949 switch (cls_flower->command) { 3950 case FLOW_CLS_REPLACE: 3951 return iavf_configure_clsflower(adapter, cls_flower); 3952 case FLOW_CLS_DESTROY: 3953 return iavf_delete_clsflower(adapter, cls_flower); 3954 case FLOW_CLS_STATS: 3955 return -EOPNOTSUPP; 3956 default: 3957 return -EOPNOTSUPP; 3958 } 3959 } 3960 3961 /** 3962 * iavf_setup_tc_block_cb - block callback for tc 3963 * @type: type of offload 3964 * @type_data: offload data 3965 * @cb_priv: 3966 * 3967 * This function is the block callback for traffic classes 3968 **/ 3969 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 3970 void *cb_priv) 3971 { 3972 struct iavf_adapter *adapter = cb_priv; 3973 3974 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data)) 3975 return -EOPNOTSUPP; 3976 3977 switch (type) { 3978 case TC_SETUP_CLSFLOWER: 3979 return iavf_setup_tc_cls_flower(cb_priv, type_data); 3980 default: 3981 return -EOPNOTSUPP; 3982 } 3983 } 3984 3985 static LIST_HEAD(iavf_block_cb_list); 3986 3987 /** 3988 * iavf_setup_tc - configure multiple traffic classes 3989 * @netdev: network interface device structure 3990 * @type: type of offload 3991 * @type_data: tc offload data 3992 * 3993 * This function is the callback to ndo_setup_tc in the 3994 * netdev_ops. 3995 * 3996 * Returns 0 on success 3997 **/ 3998 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type, 3999 void *type_data) 4000 { 4001 struct iavf_adapter *adapter = netdev_priv(netdev); 4002 4003 switch (type) { 4004 case TC_SETUP_QDISC_MQPRIO: 4005 return __iavf_setup_tc(netdev, type_data); 4006 case TC_SETUP_BLOCK: 4007 return flow_block_cb_setup_simple(type_data, 4008 &iavf_block_cb_list, 4009 iavf_setup_tc_block_cb, 4010 adapter, adapter, true); 4011 default: 4012 return -EOPNOTSUPP; 4013 } 4014 } 4015 4016 /** 4017 * iavf_open - Called when a network interface is made active 4018 * @netdev: network interface device structure 4019 * 4020 * Returns 0 on success, negative value on failure 4021 * 4022 * The open entry point is called when a network interface is made 4023 * active by the system (IFF_UP). At this point all resources needed 4024 * for transmit and receive operations are allocated, the interrupt 4025 * handler is registered with the OS, the watchdog is started, 4026 * and the stack is notified that the interface is ready. 4027 **/ 4028 static int iavf_open(struct net_device *netdev) 4029 { 4030 struct iavf_adapter *adapter = netdev_priv(netdev); 4031 int err; 4032 4033 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) { 4034 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n"); 4035 return -EIO; 4036 } 4037 4038 while (!mutex_trylock(&adapter->crit_lock)) 4039 usleep_range(500, 1000); 4040 4041 if (adapter->state != __IAVF_DOWN) { 4042 err = -EBUSY; 4043 goto err_unlock; 4044 } 4045 4046 if (adapter->state == __IAVF_RUNNING && 4047 !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) { 4048 dev_dbg(&adapter->pdev->dev, "VF is already open.\n"); 4049 err = 0; 4050 goto err_unlock; 4051 } 4052 4053 /* allocate transmit descriptors */ 4054 err = iavf_setup_all_tx_resources(adapter); 4055 if (err) 4056 goto err_setup_tx; 4057 4058 /* allocate receive descriptors */ 4059 err = iavf_setup_all_rx_resources(adapter); 4060 if (err) 4061 goto err_setup_rx; 4062 4063 /* clear any pending interrupts, may auto mask */ 4064 err = iavf_request_traffic_irqs(adapter, netdev->name); 4065 if (err) 4066 goto err_req_irq; 4067 4068 spin_lock_bh(&adapter->mac_vlan_list_lock); 4069 4070 iavf_add_filter(adapter, adapter->hw.mac.addr); 4071 4072 spin_unlock_bh(&adapter->mac_vlan_list_lock); 4073 4074 /* Restore VLAN filters that were removed with IFF_DOWN */ 4075 iavf_restore_filters(adapter); 4076 4077 iavf_configure(adapter); 4078 4079 iavf_up_complete(adapter); 4080 4081 iavf_irq_enable(adapter, true); 4082 4083 mutex_unlock(&adapter->crit_lock); 4084 4085 return 0; 4086 4087 err_req_irq: 4088 iavf_down(adapter); 4089 iavf_free_traffic_irqs(adapter); 4090 err_setup_rx: 4091 iavf_free_all_rx_resources(adapter); 4092 err_setup_tx: 4093 iavf_free_all_tx_resources(adapter); 4094 err_unlock: 4095 mutex_unlock(&adapter->crit_lock); 4096 4097 return err; 4098 } 4099 4100 /** 4101 * iavf_close - Disables a network interface 4102 * @netdev: network interface device structure 4103 * 4104 * Returns 0, this is not allowed to fail 4105 * 4106 * The close entry point is called when an interface is de-activated 4107 * by the OS. The hardware is still under the drivers control, but 4108 * needs to be disabled. All IRQs except vector 0 (reserved for admin queue) 4109 * are freed, along with all transmit and receive resources. 4110 **/ 4111 static int iavf_close(struct net_device *netdev) 4112 { 4113 struct iavf_adapter *adapter = netdev_priv(netdev); 4114 int status; 4115 4116 mutex_lock(&adapter->crit_lock); 4117 4118 if (adapter->state <= __IAVF_DOWN_PENDING) { 4119 mutex_unlock(&adapter->crit_lock); 4120 return 0; 4121 } 4122 4123 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state); 4124 if (CLIENT_ENABLED(adapter)) 4125 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE; 4126 4127 iavf_down(adapter); 4128 iavf_change_state(adapter, __IAVF_DOWN_PENDING); 4129 iavf_free_traffic_irqs(adapter); 4130 4131 mutex_unlock(&adapter->crit_lock); 4132 4133 /* We explicitly don't free resources here because the hardware is 4134 * still active and can DMA into memory. Resources are cleared in 4135 * iavf_virtchnl_completion() after we get confirmation from the PF 4136 * driver that the rings have been stopped. 4137 * 4138 * Also, we wait for state to transition to __IAVF_DOWN before 4139 * returning. State change occurs in iavf_virtchnl_completion() after 4140 * VF resources are released (which occurs after PF driver processes and 4141 * responds to admin queue commands). 4142 */ 4143 4144 status = wait_event_timeout(adapter->down_waitqueue, 4145 adapter->state == __IAVF_DOWN, 4146 msecs_to_jiffies(500)); 4147 if (!status) 4148 netdev_warn(netdev, "Device resources not yet released\n"); 4149 return 0; 4150 } 4151 4152 /** 4153 * iavf_change_mtu - Change the Maximum Transfer Unit 4154 * @netdev: network interface device structure 4155 * @new_mtu: new value for maximum frame size 4156 * 4157 * Returns 0 on success, negative on failure 4158 **/ 4159 static int iavf_change_mtu(struct net_device *netdev, int new_mtu) 4160 { 4161 struct iavf_adapter *adapter = netdev_priv(netdev); 4162 4163 netdev_dbg(netdev, "changing MTU from %d to %d\n", 4164 netdev->mtu, new_mtu); 4165 netdev->mtu = new_mtu; 4166 if (CLIENT_ENABLED(adapter)) { 4167 iavf_notify_client_l2_params(&adapter->vsi); 4168 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED; 4169 } 4170 4171 if (netif_running(netdev)) { 4172 adapter->flags |= IAVF_FLAG_RESET_NEEDED; 4173 queue_work(iavf_wq, &adapter->reset_task); 4174 } 4175 4176 return 0; 4177 } 4178 4179 #define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 4180 NETIF_F_HW_VLAN_CTAG_TX | \ 4181 NETIF_F_HW_VLAN_STAG_RX | \ 4182 NETIF_F_HW_VLAN_STAG_TX) 4183 4184 /** 4185 * iavf_set_features - set the netdev feature flags 4186 * @netdev: ptr to the netdev being adjusted 4187 * @features: the feature set that the stack is suggesting 4188 * Note: expects to be called while under rtnl_lock() 4189 **/ 4190 static int iavf_set_features(struct net_device *netdev, 4191 netdev_features_t features) 4192 { 4193 struct iavf_adapter *adapter = netdev_priv(netdev); 4194 4195 /* trigger update on any VLAN feature change */ 4196 if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^ 4197 (features & NETIF_VLAN_OFFLOAD_FEATURES)) 4198 iavf_set_vlan_offload_features(adapter, netdev->features, 4199 features); 4200 4201 return 0; 4202 } 4203 4204 /** 4205 * iavf_features_check - Validate encapsulated packet conforms to limits 4206 * @skb: skb buff 4207 * @dev: This physical port's netdev 4208 * @features: Offload features that the stack believes apply 4209 **/ 4210 static netdev_features_t iavf_features_check(struct sk_buff *skb, 4211 struct net_device *dev, 4212 netdev_features_t features) 4213 { 4214 size_t len; 4215 4216 /* No point in doing any of this if neither checksum nor GSO are 4217 * being requested for this frame. We can rule out both by just 4218 * checking for CHECKSUM_PARTIAL 4219 */ 4220 if (skb->ip_summed != CHECKSUM_PARTIAL) 4221 return features; 4222 4223 /* We cannot support GSO if the MSS is going to be less than 4224 * 64 bytes. If it is then we need to drop support for GSO. 4225 */ 4226 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 4227 features &= ~NETIF_F_GSO_MASK; 4228 4229 /* MACLEN can support at most 63 words */ 4230 len = skb_network_header(skb) - skb->data; 4231 if (len & ~(63 * 2)) 4232 goto out_err; 4233 4234 /* IPLEN and EIPLEN can support at most 127 dwords */ 4235 len = skb_transport_header(skb) - skb_network_header(skb); 4236 if (len & ~(127 * 4)) 4237 goto out_err; 4238 4239 if (skb->encapsulation) { 4240 /* L4TUNLEN can support 127 words */ 4241 len = skb_inner_network_header(skb) - skb_transport_header(skb); 4242 if (len & ~(127 * 2)) 4243 goto out_err; 4244 4245 /* IPLEN can support at most 127 dwords */ 4246 len = skb_inner_transport_header(skb) - 4247 skb_inner_network_header(skb); 4248 if (len & ~(127 * 4)) 4249 goto out_err; 4250 } 4251 4252 /* No need to validate L4LEN as TCP is the only protocol with a 4253 * flexible value and we support all possible values supported 4254 * by TCP, which is at most 15 dwords 4255 */ 4256 4257 return features; 4258 out_err: 4259 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 4260 } 4261 4262 /** 4263 * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off 4264 * @adapter: board private structure 4265 * 4266 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2 4267 * were negotiated determine the VLAN features that can be toggled on and off. 4268 **/ 4269 static netdev_features_t 4270 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter) 4271 { 4272 netdev_features_t hw_features = 0; 4273 4274 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags) 4275 return hw_features; 4276 4277 /* Enable VLAN features if supported */ 4278 if (VLAN_ALLOWED(adapter)) { 4279 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX | 4280 NETIF_F_HW_VLAN_CTAG_RX); 4281 } else if (VLAN_V2_ALLOWED(adapter)) { 4282 struct virtchnl_vlan_caps *vlan_v2_caps = 4283 &adapter->vlan_v2_caps; 4284 struct virtchnl_vlan_supported_caps *stripping_support = 4285 &vlan_v2_caps->offloads.stripping_support; 4286 struct virtchnl_vlan_supported_caps *insertion_support = 4287 &vlan_v2_caps->offloads.insertion_support; 4288 4289 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED && 4290 stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) { 4291 if (stripping_support->outer & 4292 VIRTCHNL_VLAN_ETHERTYPE_8100) 4293 hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 4294 if (stripping_support->outer & 4295 VIRTCHNL_VLAN_ETHERTYPE_88A8) 4296 hw_features |= NETIF_F_HW_VLAN_STAG_RX; 4297 } else if (stripping_support->inner != 4298 VIRTCHNL_VLAN_UNSUPPORTED && 4299 stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) { 4300 if (stripping_support->inner & 4301 VIRTCHNL_VLAN_ETHERTYPE_8100) 4302 hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 4303 } 4304 4305 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED && 4306 insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) { 4307 if (insertion_support->outer & 4308 VIRTCHNL_VLAN_ETHERTYPE_8100) 4309 hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 4310 if (insertion_support->outer & 4311 VIRTCHNL_VLAN_ETHERTYPE_88A8) 4312 hw_features |= NETIF_F_HW_VLAN_STAG_TX; 4313 } else if (insertion_support->inner && 4314 insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) { 4315 if (insertion_support->inner & 4316 VIRTCHNL_VLAN_ETHERTYPE_8100) 4317 hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 4318 } 4319 } 4320 4321 return hw_features; 4322 } 4323 4324 /** 4325 * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures 4326 * @adapter: board private structure 4327 * 4328 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2 4329 * were negotiated determine the VLAN features that are enabled by default. 4330 **/ 4331 static netdev_features_t 4332 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter) 4333 { 4334 netdev_features_t features = 0; 4335 4336 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags) 4337 return features; 4338 4339 if (VLAN_ALLOWED(adapter)) { 4340 features |= NETIF_F_HW_VLAN_CTAG_FILTER | 4341 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX; 4342 } else if (VLAN_V2_ALLOWED(adapter)) { 4343 struct virtchnl_vlan_caps *vlan_v2_caps = 4344 &adapter->vlan_v2_caps; 4345 struct virtchnl_vlan_supported_caps *filtering_support = 4346 &vlan_v2_caps->filtering.filtering_support; 4347 struct virtchnl_vlan_supported_caps *stripping_support = 4348 &vlan_v2_caps->offloads.stripping_support; 4349 struct virtchnl_vlan_supported_caps *insertion_support = 4350 &vlan_v2_caps->offloads.insertion_support; 4351 u32 ethertype_init; 4352 4353 /* give priority to outer stripping and don't support both outer 4354 * and inner stripping 4355 */ 4356 ethertype_init = vlan_v2_caps->offloads.ethertype_init; 4357 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4358 if (stripping_support->outer & 4359 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4360 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4361 features |= NETIF_F_HW_VLAN_CTAG_RX; 4362 else if (stripping_support->outer & 4363 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4364 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4365 features |= NETIF_F_HW_VLAN_STAG_RX; 4366 } else if (stripping_support->inner != 4367 VIRTCHNL_VLAN_UNSUPPORTED) { 4368 if (stripping_support->inner & 4369 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4370 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4371 features |= NETIF_F_HW_VLAN_CTAG_RX; 4372 } 4373 4374 /* give priority to outer insertion and don't support both outer 4375 * and inner insertion 4376 */ 4377 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4378 if (insertion_support->outer & 4379 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4380 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4381 features |= NETIF_F_HW_VLAN_CTAG_TX; 4382 else if (insertion_support->outer & 4383 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4384 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4385 features |= NETIF_F_HW_VLAN_STAG_TX; 4386 } else if (insertion_support->inner != 4387 VIRTCHNL_VLAN_UNSUPPORTED) { 4388 if (insertion_support->inner & 4389 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4390 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4391 features |= NETIF_F_HW_VLAN_CTAG_TX; 4392 } 4393 4394 /* give priority to outer filtering and don't bother if both 4395 * outer and inner filtering are enabled 4396 */ 4397 ethertype_init = vlan_v2_caps->filtering.ethertype_init; 4398 if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) { 4399 if (filtering_support->outer & 4400 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4401 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4402 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4403 if (filtering_support->outer & 4404 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4405 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4406 features |= NETIF_F_HW_VLAN_STAG_FILTER; 4407 } else if (filtering_support->inner != 4408 VIRTCHNL_VLAN_UNSUPPORTED) { 4409 if (filtering_support->inner & 4410 VIRTCHNL_VLAN_ETHERTYPE_8100 && 4411 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100) 4412 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4413 if (filtering_support->inner & 4414 VIRTCHNL_VLAN_ETHERTYPE_88A8 && 4415 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8) 4416 features |= NETIF_F_HW_VLAN_STAG_FILTER; 4417 } 4418 } 4419 4420 return features; 4421 } 4422 4423 #define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \ 4424 (!(((requested) & (feature_bit)) && \ 4425 !((allowed) & (feature_bit)))) 4426 4427 /** 4428 * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support 4429 * @adapter: board private structure 4430 * @requested_features: stack requested NETDEV features 4431 **/ 4432 static netdev_features_t 4433 iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter, 4434 netdev_features_t requested_features) 4435 { 4436 netdev_features_t allowed_features; 4437 4438 allowed_features = iavf_get_netdev_vlan_hw_features(adapter) | 4439 iavf_get_netdev_vlan_features(adapter); 4440 4441 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4442 allowed_features, 4443 NETIF_F_HW_VLAN_CTAG_TX)) 4444 requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX; 4445 4446 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4447 allowed_features, 4448 NETIF_F_HW_VLAN_CTAG_RX)) 4449 requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX; 4450 4451 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4452 allowed_features, 4453 NETIF_F_HW_VLAN_STAG_TX)) 4454 requested_features &= ~NETIF_F_HW_VLAN_STAG_TX; 4455 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4456 allowed_features, 4457 NETIF_F_HW_VLAN_STAG_RX)) 4458 requested_features &= ~NETIF_F_HW_VLAN_STAG_RX; 4459 4460 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4461 allowed_features, 4462 NETIF_F_HW_VLAN_CTAG_FILTER)) 4463 requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER; 4464 4465 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features, 4466 allowed_features, 4467 NETIF_F_HW_VLAN_STAG_FILTER)) 4468 requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER; 4469 4470 if ((requested_features & 4471 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) && 4472 (requested_features & 4473 (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) && 4474 adapter->vlan_v2_caps.offloads.ethertype_match == 4475 VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) { 4476 netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n"); 4477 requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX | 4478 NETIF_F_HW_VLAN_STAG_TX); 4479 } 4480 4481 return requested_features; 4482 } 4483 4484 /** 4485 * iavf_fix_features - fix up the netdev feature bits 4486 * @netdev: our net device 4487 * @features: desired feature bits 4488 * 4489 * Returns fixed-up features bits 4490 **/ 4491 static netdev_features_t iavf_fix_features(struct net_device *netdev, 4492 netdev_features_t features) 4493 { 4494 struct iavf_adapter *adapter = netdev_priv(netdev); 4495 4496 return iavf_fix_netdev_vlan_features(adapter, features); 4497 } 4498 4499 static const struct net_device_ops iavf_netdev_ops = { 4500 .ndo_open = iavf_open, 4501 .ndo_stop = iavf_close, 4502 .ndo_start_xmit = iavf_xmit_frame, 4503 .ndo_set_rx_mode = iavf_set_rx_mode, 4504 .ndo_validate_addr = eth_validate_addr, 4505 .ndo_set_mac_address = iavf_set_mac, 4506 .ndo_change_mtu = iavf_change_mtu, 4507 .ndo_tx_timeout = iavf_tx_timeout, 4508 .ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid, 4509 .ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid, 4510 .ndo_features_check = iavf_features_check, 4511 .ndo_fix_features = iavf_fix_features, 4512 .ndo_set_features = iavf_set_features, 4513 .ndo_setup_tc = iavf_setup_tc, 4514 }; 4515 4516 /** 4517 * iavf_check_reset_complete - check that VF reset is complete 4518 * @hw: pointer to hw struct 4519 * 4520 * Returns 0 if device is ready to use, or -EBUSY if it's in reset. 4521 **/ 4522 static int iavf_check_reset_complete(struct iavf_hw *hw) 4523 { 4524 u32 rstat; 4525 int i; 4526 4527 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) { 4528 rstat = rd32(hw, IAVF_VFGEN_RSTAT) & 4529 IAVF_VFGEN_RSTAT_VFR_STATE_MASK; 4530 if ((rstat == VIRTCHNL_VFR_VFACTIVE) || 4531 (rstat == VIRTCHNL_VFR_COMPLETED)) 4532 return 0; 4533 usleep_range(10, 20); 4534 } 4535 return -EBUSY; 4536 } 4537 4538 /** 4539 * iavf_process_config - Process the config information we got from the PF 4540 * @adapter: board private structure 4541 * 4542 * Verify that we have a valid config struct, and set up our netdev features 4543 * and our VSI struct. 4544 **/ 4545 int iavf_process_config(struct iavf_adapter *adapter) 4546 { 4547 struct virtchnl_vf_resource *vfres = adapter->vf_res; 4548 netdev_features_t hw_vlan_features, vlan_features; 4549 struct net_device *netdev = adapter->netdev; 4550 netdev_features_t hw_enc_features; 4551 netdev_features_t hw_features; 4552 4553 hw_enc_features = NETIF_F_SG | 4554 NETIF_F_IP_CSUM | 4555 NETIF_F_IPV6_CSUM | 4556 NETIF_F_HIGHDMA | 4557 NETIF_F_SOFT_FEATURES | 4558 NETIF_F_TSO | 4559 NETIF_F_TSO_ECN | 4560 NETIF_F_TSO6 | 4561 NETIF_F_SCTP_CRC | 4562 NETIF_F_RXHASH | 4563 NETIF_F_RXCSUM | 4564 0; 4565 4566 /* advertise to stack only if offloads for encapsulated packets is 4567 * supported 4568 */ 4569 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) { 4570 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 4571 NETIF_F_GSO_GRE | 4572 NETIF_F_GSO_GRE_CSUM | 4573 NETIF_F_GSO_IPXIP4 | 4574 NETIF_F_GSO_IPXIP6 | 4575 NETIF_F_GSO_UDP_TUNNEL_CSUM | 4576 NETIF_F_GSO_PARTIAL | 4577 0; 4578 4579 if (!(vfres->vf_cap_flags & 4580 VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM)) 4581 netdev->gso_partial_features |= 4582 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4583 4584 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 4585 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 4586 netdev->hw_enc_features |= hw_enc_features; 4587 } 4588 /* record features VLANs can make use of */ 4589 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 4590 4591 /* Write features and hw_features separately to avoid polluting 4592 * with, or dropping, features that are set when we registered. 4593 */ 4594 hw_features = hw_enc_features; 4595 4596 /* get HW VLAN features that can be toggled */ 4597 hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter); 4598 4599 /* Enable cloud filter if ADQ is supported */ 4600 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) 4601 hw_features |= NETIF_F_HW_TC; 4602 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO) 4603 hw_features |= NETIF_F_GSO_UDP_L4; 4604 4605 netdev->hw_features |= hw_features | hw_vlan_features; 4606 vlan_features = iavf_get_netdev_vlan_features(adapter); 4607 4608 netdev->features |= hw_features | vlan_features; 4609 4610 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN) 4611 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4612 4613 netdev->priv_flags |= IFF_UNICAST_FLT; 4614 4615 /* Do not turn on offloads when they are requested to be turned off. 4616 * TSO needs minimum 576 bytes to work correctly. 4617 */ 4618 if (netdev->wanted_features) { 4619 if (!(netdev->wanted_features & NETIF_F_TSO) || 4620 netdev->mtu < 576) 4621 netdev->features &= ~NETIF_F_TSO; 4622 if (!(netdev->wanted_features & NETIF_F_TSO6) || 4623 netdev->mtu < 576) 4624 netdev->features &= ~NETIF_F_TSO6; 4625 if (!(netdev->wanted_features & NETIF_F_TSO_ECN)) 4626 netdev->features &= ~NETIF_F_TSO_ECN; 4627 if (!(netdev->wanted_features & NETIF_F_GRO)) 4628 netdev->features &= ~NETIF_F_GRO; 4629 if (!(netdev->wanted_features & NETIF_F_GSO)) 4630 netdev->features &= ~NETIF_F_GSO; 4631 } 4632 4633 return 0; 4634 } 4635 4636 /** 4637 * iavf_shutdown - Shutdown the device in preparation for a reboot 4638 * @pdev: pci device structure 4639 **/ 4640 static void iavf_shutdown(struct pci_dev *pdev) 4641 { 4642 struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev); 4643 struct net_device *netdev = adapter->netdev; 4644 4645 netif_device_detach(netdev); 4646 4647 if (netif_running(netdev)) 4648 iavf_close(netdev); 4649 4650 if (iavf_lock_timeout(&adapter->crit_lock, 5000)) 4651 dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__); 4652 /* Prevent the watchdog from running. */ 4653 iavf_change_state(adapter, __IAVF_REMOVE); 4654 adapter->aq_required = 0; 4655 mutex_unlock(&adapter->crit_lock); 4656 4657 #ifdef CONFIG_PM 4658 pci_save_state(pdev); 4659 4660 #endif 4661 pci_disable_device(pdev); 4662 } 4663 4664 /** 4665 * iavf_probe - Device Initialization Routine 4666 * @pdev: PCI device information struct 4667 * @ent: entry in iavf_pci_tbl 4668 * 4669 * Returns 0 on success, negative on failure 4670 * 4671 * iavf_probe initializes an adapter identified by a pci_dev structure. 4672 * The OS initialization, configuring of the adapter private structure, 4673 * and a hardware reset occur. 4674 **/ 4675 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 4676 { 4677 struct net_device *netdev; 4678 struct iavf_adapter *adapter = NULL; 4679 struct iavf_hw *hw = NULL; 4680 int err; 4681 4682 err = pci_enable_device(pdev); 4683 if (err) 4684 return err; 4685 4686 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 4687 if (err) { 4688 dev_err(&pdev->dev, 4689 "DMA configuration failed: 0x%x\n", err); 4690 goto err_dma; 4691 } 4692 4693 err = pci_request_regions(pdev, iavf_driver_name); 4694 if (err) { 4695 dev_err(&pdev->dev, 4696 "pci_request_regions failed 0x%x\n", err); 4697 goto err_pci_reg; 4698 } 4699 4700 pci_enable_pcie_error_reporting(pdev); 4701 4702 pci_set_master(pdev); 4703 4704 netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter), 4705 IAVF_MAX_REQ_QUEUES); 4706 if (!netdev) { 4707 err = -ENOMEM; 4708 goto err_alloc_etherdev; 4709 } 4710 4711 SET_NETDEV_DEV(netdev, &pdev->dev); 4712 4713 pci_set_drvdata(pdev, netdev); 4714 adapter = netdev_priv(netdev); 4715 4716 adapter->netdev = netdev; 4717 adapter->pdev = pdev; 4718 4719 hw = &adapter->hw; 4720 hw->back = adapter; 4721 4722 adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1; 4723 iavf_change_state(adapter, __IAVF_STARTUP); 4724 4725 /* Call save state here because it relies on the adapter struct. */ 4726 pci_save_state(pdev); 4727 4728 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 4729 pci_resource_len(pdev, 0)); 4730 if (!hw->hw_addr) { 4731 err = -EIO; 4732 goto err_ioremap; 4733 } 4734 hw->vendor_id = pdev->vendor; 4735 hw->device_id = pdev->device; 4736 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 4737 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4738 hw->subsystem_device_id = pdev->subsystem_device; 4739 hw->bus.device = PCI_SLOT(pdev->devfn); 4740 hw->bus.func = PCI_FUNC(pdev->devfn); 4741 hw->bus.bus_id = pdev->bus->number; 4742 4743 /* set up the locks for the AQ, do this only once in probe 4744 * and destroy them only once in remove 4745 */ 4746 mutex_init(&adapter->crit_lock); 4747 mutex_init(&adapter->client_lock); 4748 mutex_init(&hw->aq.asq_mutex); 4749 mutex_init(&hw->aq.arq_mutex); 4750 4751 spin_lock_init(&adapter->mac_vlan_list_lock); 4752 spin_lock_init(&adapter->cloud_filter_list_lock); 4753 spin_lock_init(&adapter->fdir_fltr_lock); 4754 spin_lock_init(&adapter->adv_rss_lock); 4755 4756 INIT_LIST_HEAD(&adapter->mac_filter_list); 4757 INIT_LIST_HEAD(&adapter->vlan_filter_list); 4758 INIT_LIST_HEAD(&adapter->cloud_filter_list); 4759 INIT_LIST_HEAD(&adapter->fdir_list_head); 4760 INIT_LIST_HEAD(&adapter->adv_rss_list_head); 4761 4762 INIT_WORK(&adapter->reset_task, iavf_reset_task); 4763 INIT_WORK(&adapter->adminq_task, iavf_adminq_task); 4764 INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task); 4765 INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task); 4766 queue_delayed_work(iavf_wq, &adapter->watchdog_task, 4767 msecs_to_jiffies(5 * (pdev->devfn & 0x07))); 4768 4769 /* Setup the wait queue for indicating transition to down status */ 4770 init_waitqueue_head(&adapter->down_waitqueue); 4771 4772 /* Setup the wait queue for indicating virtchannel events */ 4773 init_waitqueue_head(&adapter->vc_waitqueue); 4774 4775 return 0; 4776 4777 err_ioremap: 4778 free_netdev(netdev); 4779 err_alloc_etherdev: 4780 pci_disable_pcie_error_reporting(pdev); 4781 pci_release_regions(pdev); 4782 err_pci_reg: 4783 err_dma: 4784 pci_disable_device(pdev); 4785 return err; 4786 } 4787 4788 /** 4789 * iavf_suspend - Power management suspend routine 4790 * @dev_d: device info pointer 4791 * 4792 * Called when the system (VM) is entering sleep/suspend. 4793 **/ 4794 static int __maybe_unused iavf_suspend(struct device *dev_d) 4795 { 4796 struct net_device *netdev = dev_get_drvdata(dev_d); 4797 struct iavf_adapter *adapter = netdev_priv(netdev); 4798 4799 netif_device_detach(netdev); 4800 4801 while (!mutex_trylock(&adapter->crit_lock)) 4802 usleep_range(500, 1000); 4803 4804 if (netif_running(netdev)) { 4805 rtnl_lock(); 4806 iavf_down(adapter); 4807 rtnl_unlock(); 4808 } 4809 iavf_free_misc_irq(adapter); 4810 iavf_reset_interrupt_capability(adapter); 4811 4812 mutex_unlock(&adapter->crit_lock); 4813 4814 return 0; 4815 } 4816 4817 /** 4818 * iavf_resume - Power management resume routine 4819 * @dev_d: device info pointer 4820 * 4821 * Called when the system (VM) is resumed from sleep/suspend. 4822 **/ 4823 static int __maybe_unused iavf_resume(struct device *dev_d) 4824 { 4825 struct pci_dev *pdev = to_pci_dev(dev_d); 4826 struct iavf_adapter *adapter; 4827 u32 err; 4828 4829 adapter = iavf_pdev_to_adapter(pdev); 4830 4831 pci_set_master(pdev); 4832 4833 rtnl_lock(); 4834 err = iavf_set_interrupt_capability(adapter); 4835 if (err) { 4836 rtnl_unlock(); 4837 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n"); 4838 return err; 4839 } 4840 err = iavf_request_misc_irq(adapter); 4841 rtnl_unlock(); 4842 if (err) { 4843 dev_err(&pdev->dev, "Cannot get interrupt vector.\n"); 4844 return err; 4845 } 4846 4847 queue_work(iavf_wq, &adapter->reset_task); 4848 4849 netif_device_attach(adapter->netdev); 4850 4851 return err; 4852 } 4853 4854 /** 4855 * iavf_remove - Device Removal Routine 4856 * @pdev: PCI device information struct 4857 * 4858 * iavf_remove is called by the PCI subsystem to alert the driver 4859 * that it should release a PCI device. The could be caused by a 4860 * Hot-Plug event, or because the driver is going to be removed from 4861 * memory. 4862 **/ 4863 static void iavf_remove(struct pci_dev *pdev) 4864 { 4865 struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev); 4866 struct net_device *netdev = adapter->netdev; 4867 struct iavf_fdir_fltr *fdir, *fdirtmp; 4868 struct iavf_vlan_filter *vlf, *vlftmp; 4869 struct iavf_adv_rss *rss, *rsstmp; 4870 struct iavf_mac_filter *f, *ftmp; 4871 struct iavf_cloud_filter *cf, *cftmp; 4872 struct iavf_hw *hw = &adapter->hw; 4873 int err; 4874 4875 /* When reboot/shutdown is in progress no need to do anything 4876 * as the adapter is already REMOVE state that was set during 4877 * iavf_shutdown() callback. 4878 */ 4879 if (adapter->state == __IAVF_REMOVE) 4880 return; 4881 4882 set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section); 4883 /* Wait until port initialization is complete. 4884 * There are flows where register/unregister netdev may race. 4885 */ 4886 while (1) { 4887 mutex_lock(&adapter->crit_lock); 4888 if (adapter->state == __IAVF_RUNNING || 4889 adapter->state == __IAVF_DOWN || 4890 adapter->state == __IAVF_INIT_FAILED) { 4891 mutex_unlock(&adapter->crit_lock); 4892 break; 4893 } 4894 4895 mutex_unlock(&adapter->crit_lock); 4896 usleep_range(500, 1000); 4897 } 4898 cancel_delayed_work_sync(&adapter->watchdog_task); 4899 4900 if (adapter->netdev_registered) { 4901 rtnl_lock(); 4902 unregister_netdevice(netdev); 4903 adapter->netdev_registered = false; 4904 rtnl_unlock(); 4905 } 4906 if (CLIENT_ALLOWED(adapter)) { 4907 err = iavf_lan_del_device(adapter); 4908 if (err) 4909 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 4910 err); 4911 } 4912 4913 mutex_lock(&adapter->crit_lock); 4914 dev_info(&adapter->pdev->dev, "Remove device\n"); 4915 iavf_change_state(adapter, __IAVF_REMOVE); 4916 4917 iavf_request_reset(adapter); 4918 msleep(50); 4919 /* If the FW isn't responding, kick it once, but only once. */ 4920 if (!iavf_asq_done(hw)) { 4921 iavf_request_reset(adapter); 4922 msleep(50); 4923 } 4924 4925 iavf_misc_irq_disable(adapter); 4926 /* Shut down all the garbage mashers on the detention level */ 4927 cancel_work_sync(&adapter->reset_task); 4928 cancel_delayed_work_sync(&adapter->watchdog_task); 4929 cancel_work_sync(&adapter->adminq_task); 4930 cancel_delayed_work_sync(&adapter->client_task); 4931 4932 adapter->aq_required = 0; 4933 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED; 4934 4935 iavf_free_all_tx_resources(adapter); 4936 iavf_free_all_rx_resources(adapter); 4937 iavf_free_misc_irq(adapter); 4938 4939 iavf_reset_interrupt_capability(adapter); 4940 iavf_free_q_vectors(adapter); 4941 4942 iavf_free_rss(adapter); 4943 4944 if (hw->aq.asq.count) 4945 iavf_shutdown_adminq(hw); 4946 4947 /* destroy the locks only once, here */ 4948 mutex_destroy(&hw->aq.arq_mutex); 4949 mutex_destroy(&hw->aq.asq_mutex); 4950 mutex_destroy(&adapter->client_lock); 4951 mutex_unlock(&adapter->crit_lock); 4952 mutex_destroy(&adapter->crit_lock); 4953 4954 iounmap(hw->hw_addr); 4955 pci_release_regions(pdev); 4956 iavf_free_queues(adapter); 4957 kfree(adapter->vf_res); 4958 spin_lock_bh(&adapter->mac_vlan_list_lock); 4959 /* If we got removed before an up/down sequence, we've got a filter 4960 * hanging out there that we need to get rid of. 4961 */ 4962 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) { 4963 list_del(&f->list); 4964 kfree(f); 4965 } 4966 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list, 4967 list) { 4968 list_del(&vlf->list); 4969 kfree(vlf); 4970 } 4971 4972 spin_unlock_bh(&adapter->mac_vlan_list_lock); 4973 4974 spin_lock_bh(&adapter->cloud_filter_list_lock); 4975 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) { 4976 list_del(&cf->list); 4977 kfree(cf); 4978 } 4979 spin_unlock_bh(&adapter->cloud_filter_list_lock); 4980 4981 spin_lock_bh(&adapter->fdir_fltr_lock); 4982 list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) { 4983 list_del(&fdir->list); 4984 kfree(fdir); 4985 } 4986 spin_unlock_bh(&adapter->fdir_fltr_lock); 4987 4988 spin_lock_bh(&adapter->adv_rss_lock); 4989 list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head, 4990 list) { 4991 list_del(&rss->list); 4992 kfree(rss); 4993 } 4994 spin_unlock_bh(&adapter->adv_rss_lock); 4995 4996 free_netdev(netdev); 4997 4998 pci_disable_pcie_error_reporting(pdev); 4999 5000 pci_disable_device(pdev); 5001 } 5002 5003 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume); 5004 5005 static struct pci_driver iavf_driver = { 5006 .name = iavf_driver_name, 5007 .id_table = iavf_pci_tbl, 5008 .probe = iavf_probe, 5009 .remove = iavf_remove, 5010 .driver.pm = &iavf_pm_ops, 5011 .shutdown = iavf_shutdown, 5012 }; 5013 5014 /** 5015 * iavf_init_module - Driver Registration Routine 5016 * 5017 * iavf_init_module is the first routine called when the driver is 5018 * loaded. All it does is register with the PCI subsystem. 5019 **/ 5020 static int __init iavf_init_module(void) 5021 { 5022 pr_info("iavf: %s\n", iavf_driver_string); 5023 5024 pr_info("%s\n", iavf_copyright); 5025 5026 iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1, 5027 iavf_driver_name); 5028 if (!iavf_wq) { 5029 pr_err("%s: Failed to create workqueue\n", iavf_driver_name); 5030 return -ENOMEM; 5031 } 5032 return pci_register_driver(&iavf_driver); 5033 } 5034 5035 module_init(iavf_init_module); 5036 5037 /** 5038 * iavf_exit_module - Driver Exit Cleanup Routine 5039 * 5040 * iavf_exit_module is called just before the driver is removed 5041 * from memory. 5042 **/ 5043 static void __exit iavf_exit_module(void) 5044 { 5045 pci_unregister_driver(&iavf_driver); 5046 destroy_workqueue(iavf_wq); 5047 } 5048 5049 module_exit(iavf_exit_module); 5050 5051 /* iavf_main.c */ 5052