1 /* Intel(R) Ethernet Switch Host Interface Driver 2 * Copyright(c) 2013 - 2017 Intel Corporation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms and conditions of the GNU General Public License, 6 * version 2, as published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope it will be useful, but WITHOUT 9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 10 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 11 * more details. 12 * 13 * The full GNU General Public License is included in this distribution in 14 * the file called "COPYING". 15 * 16 * Contact Information: 17 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 18 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 19 */ 20 21 #include <linux/module.h> 22 #include <linux/interrupt.h> 23 #include <linux/aer.h> 24 25 #include "fm10k.h" 26 27 static const struct fm10k_info *fm10k_info_tbl[] = { 28 [fm10k_device_pf] = &fm10k_pf_info, 29 [fm10k_device_vf] = &fm10k_vf_info, 30 }; 31 32 /** 33 * fm10k_pci_tbl - PCI Device ID Table 34 * 35 * Wildcard entries (PCI_ANY_ID) should come last 36 * Last entry must be all 0s 37 * 38 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 39 * Class, Class Mask, private data (not used) } 40 */ 41 static const struct pci_device_id fm10k_pci_tbl[] = { 42 { PCI_VDEVICE(INTEL, FM10K_DEV_ID_PF), fm10k_device_pf }, 43 { PCI_VDEVICE(INTEL, FM10K_DEV_ID_VF), fm10k_device_vf }, 44 /* required last entry */ 45 { 0, } 46 }; 47 MODULE_DEVICE_TABLE(pci, fm10k_pci_tbl); 48 49 u16 fm10k_read_pci_cfg_word(struct fm10k_hw *hw, u32 reg) 50 { 51 struct fm10k_intfc *interface = hw->back; 52 u16 value = 0; 53 54 if (FM10K_REMOVED(hw->hw_addr)) 55 return ~value; 56 57 pci_read_config_word(interface->pdev, reg, &value); 58 if (value == 0xFFFF) 59 fm10k_write_flush(hw); 60 61 return value; 62 } 63 64 u32 fm10k_read_reg(struct fm10k_hw *hw, int reg) 65 { 66 u32 __iomem *hw_addr = READ_ONCE(hw->hw_addr); 67 u32 value = 0; 68 69 if (FM10K_REMOVED(hw_addr)) 70 return ~value; 71 72 value = readl(&hw_addr[reg]); 73 if (!(~value) && (!reg || !(~readl(hw_addr)))) { 74 struct fm10k_intfc *interface = hw->back; 75 struct net_device *netdev = interface->netdev; 76 77 hw->hw_addr = NULL; 78 netif_device_detach(netdev); 79 netdev_err(netdev, "PCIe link lost, device now detached\n"); 80 } 81 82 return value; 83 } 84 85 static int fm10k_hw_ready(struct fm10k_intfc *interface) 86 { 87 struct fm10k_hw *hw = &interface->hw; 88 89 fm10k_write_flush(hw); 90 91 return FM10K_REMOVED(hw->hw_addr) ? -ENODEV : 0; 92 } 93 94 void fm10k_service_event_schedule(struct fm10k_intfc *interface) 95 { 96 if (!test_bit(__FM10K_SERVICE_DISABLE, interface->state) && 97 !test_and_set_bit(__FM10K_SERVICE_SCHED, interface->state)) { 98 clear_bit(__FM10K_SERVICE_REQUEST, interface->state); 99 queue_work(fm10k_workqueue, &interface->service_task); 100 } else { 101 set_bit(__FM10K_SERVICE_REQUEST, interface->state); 102 } 103 } 104 105 static void fm10k_service_event_complete(struct fm10k_intfc *interface) 106 { 107 WARN_ON(!test_bit(__FM10K_SERVICE_SCHED, interface->state)); 108 109 /* flush memory to make sure state is correct before next watchog */ 110 smp_mb__before_atomic(); 111 clear_bit(__FM10K_SERVICE_SCHED, interface->state); 112 113 /* If a service event was requested since we started, immediately 114 * re-schedule now. This ensures we don't drop a request until the 115 * next timer event. 116 */ 117 if (test_bit(__FM10K_SERVICE_REQUEST, interface->state)) 118 fm10k_service_event_schedule(interface); 119 } 120 121 /** 122 * fm10k_service_timer - Timer Call-back 123 * @data: pointer to interface cast into an unsigned long 124 **/ 125 static void fm10k_service_timer(unsigned long data) 126 { 127 struct fm10k_intfc *interface = (struct fm10k_intfc *)data; 128 129 /* Reset the timer */ 130 mod_timer(&interface->service_timer, (HZ * 2) + jiffies); 131 132 fm10k_service_event_schedule(interface); 133 } 134 135 static void fm10k_detach_subtask(struct fm10k_intfc *interface) 136 { 137 struct net_device *netdev = interface->netdev; 138 u32 __iomem *hw_addr; 139 u32 value; 140 141 /* do nothing if device is still present or hw_addr is set */ 142 if (netif_device_present(netdev) || interface->hw.hw_addr) 143 return; 144 145 /* check the real address space to see if we've recovered */ 146 hw_addr = READ_ONCE(interface->uc_addr); 147 value = readl(hw_addr); 148 if (~value) { 149 interface->hw.hw_addr = interface->uc_addr; 150 netif_device_attach(netdev); 151 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 152 netdev_warn(netdev, "PCIe link restored, device now attached\n"); 153 return; 154 } 155 156 rtnl_lock(); 157 158 if (netif_running(netdev)) 159 dev_close(netdev); 160 161 rtnl_unlock(); 162 } 163 164 static void fm10k_prepare_for_reset(struct fm10k_intfc *interface) 165 { 166 struct net_device *netdev = interface->netdev; 167 168 WARN_ON(in_interrupt()); 169 170 /* put off any impending NetWatchDogTimeout */ 171 netif_trans_update(netdev); 172 173 while (test_and_set_bit(__FM10K_RESETTING, interface->state)) 174 usleep_range(1000, 2000); 175 176 rtnl_lock(); 177 178 fm10k_iov_suspend(interface->pdev); 179 180 if (netif_running(netdev)) 181 fm10k_close(netdev); 182 183 fm10k_mbx_free_irq(interface); 184 185 /* free interrupts */ 186 fm10k_clear_queueing_scheme(interface); 187 188 /* delay any future reset requests */ 189 interface->last_reset = jiffies + (10 * HZ); 190 191 rtnl_unlock(); 192 } 193 194 static int fm10k_handle_reset(struct fm10k_intfc *interface) 195 { 196 struct net_device *netdev = interface->netdev; 197 struct fm10k_hw *hw = &interface->hw; 198 int err; 199 200 rtnl_lock(); 201 202 pci_set_master(interface->pdev); 203 204 /* reset and initialize the hardware so it is in a known state */ 205 err = hw->mac.ops.reset_hw(hw); 206 if (err) { 207 dev_err(&interface->pdev->dev, "reset_hw failed: %d\n", err); 208 goto reinit_err; 209 } 210 211 err = hw->mac.ops.init_hw(hw); 212 if (err) { 213 dev_err(&interface->pdev->dev, "init_hw failed: %d\n", err); 214 goto reinit_err; 215 } 216 217 err = fm10k_init_queueing_scheme(interface); 218 if (err) { 219 dev_err(&interface->pdev->dev, 220 "init_queueing_scheme failed: %d\n", err); 221 goto reinit_err; 222 } 223 224 /* re-associate interrupts */ 225 err = fm10k_mbx_request_irq(interface); 226 if (err) 227 goto err_mbx_irq; 228 229 err = fm10k_hw_ready(interface); 230 if (err) 231 goto err_open; 232 233 /* update hardware address for VFs if perm_addr has changed */ 234 if (hw->mac.type == fm10k_mac_vf) { 235 if (is_valid_ether_addr(hw->mac.perm_addr)) { 236 ether_addr_copy(hw->mac.addr, hw->mac.perm_addr); 237 ether_addr_copy(netdev->perm_addr, hw->mac.perm_addr); 238 ether_addr_copy(netdev->dev_addr, hw->mac.perm_addr); 239 netdev->addr_assign_type &= ~NET_ADDR_RANDOM; 240 } 241 242 if (hw->mac.vlan_override) 243 netdev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 244 else 245 netdev->features |= NETIF_F_HW_VLAN_CTAG_RX; 246 } 247 248 err = netif_running(netdev) ? fm10k_open(netdev) : 0; 249 if (err) 250 goto err_open; 251 252 fm10k_iov_resume(interface->pdev); 253 254 rtnl_unlock(); 255 256 clear_bit(__FM10K_RESETTING, interface->state); 257 258 return err; 259 err_open: 260 fm10k_mbx_free_irq(interface); 261 err_mbx_irq: 262 fm10k_clear_queueing_scheme(interface); 263 reinit_err: 264 netif_device_detach(netdev); 265 266 rtnl_unlock(); 267 268 clear_bit(__FM10K_RESETTING, interface->state); 269 270 return err; 271 } 272 273 static void fm10k_reinit(struct fm10k_intfc *interface) 274 { 275 int err; 276 277 fm10k_prepare_for_reset(interface); 278 279 err = fm10k_handle_reset(interface); 280 if (err) 281 dev_err(&interface->pdev->dev, 282 "fm10k_handle_reset failed: %d\n", err); 283 } 284 285 static void fm10k_reset_subtask(struct fm10k_intfc *interface) 286 { 287 if (!test_and_clear_bit(FM10K_FLAG_RESET_REQUESTED, 288 interface->flags)) 289 return; 290 291 netdev_err(interface->netdev, "Reset interface\n"); 292 293 fm10k_reinit(interface); 294 } 295 296 /** 297 * fm10k_configure_swpri_map - Configure Receive SWPRI to PC mapping 298 * @interface: board private structure 299 * 300 * Configure the SWPRI to PC mapping for the port. 301 **/ 302 static void fm10k_configure_swpri_map(struct fm10k_intfc *interface) 303 { 304 struct net_device *netdev = interface->netdev; 305 struct fm10k_hw *hw = &interface->hw; 306 int i; 307 308 /* clear flag indicating update is needed */ 309 clear_bit(FM10K_FLAG_SWPRI_CONFIG, interface->flags); 310 311 /* these registers are only available on the PF */ 312 if (hw->mac.type != fm10k_mac_pf) 313 return; 314 315 /* configure SWPRI to PC map */ 316 for (i = 0; i < FM10K_SWPRI_MAX; i++) 317 fm10k_write_reg(hw, FM10K_SWPRI_MAP(i), 318 netdev_get_prio_tc_map(netdev, i)); 319 } 320 321 /** 322 * fm10k_watchdog_update_host_state - Update the link status based on host. 323 * @interface: board private structure 324 **/ 325 static void fm10k_watchdog_update_host_state(struct fm10k_intfc *interface) 326 { 327 struct fm10k_hw *hw = &interface->hw; 328 s32 err; 329 330 if (test_bit(__FM10K_LINK_DOWN, interface->state)) { 331 interface->host_ready = false; 332 if (time_is_after_jiffies(interface->link_down_event)) 333 return; 334 clear_bit(__FM10K_LINK_DOWN, interface->state); 335 } 336 337 if (test_bit(FM10K_FLAG_SWPRI_CONFIG, interface->flags)) { 338 if (rtnl_trylock()) { 339 fm10k_configure_swpri_map(interface); 340 rtnl_unlock(); 341 } 342 } 343 344 /* lock the mailbox for transmit and receive */ 345 fm10k_mbx_lock(interface); 346 347 err = hw->mac.ops.get_host_state(hw, &interface->host_ready); 348 if (err && time_is_before_jiffies(interface->last_reset)) 349 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 350 351 /* free the lock */ 352 fm10k_mbx_unlock(interface); 353 } 354 355 /** 356 * fm10k_mbx_subtask - Process upstream and downstream mailboxes 357 * @interface: board private structure 358 * 359 * This function will process both the upstream and downstream mailboxes. 360 **/ 361 static void fm10k_mbx_subtask(struct fm10k_intfc *interface) 362 { 363 /* process upstream mailbox and update device state */ 364 fm10k_watchdog_update_host_state(interface); 365 366 /* process downstream mailboxes */ 367 fm10k_iov_mbx(interface); 368 } 369 370 /** 371 * fm10k_watchdog_host_is_ready - Update netdev status based on host ready 372 * @interface: board private structure 373 **/ 374 static void fm10k_watchdog_host_is_ready(struct fm10k_intfc *interface) 375 { 376 struct net_device *netdev = interface->netdev; 377 378 /* only continue if link state is currently down */ 379 if (netif_carrier_ok(netdev)) 380 return; 381 382 netif_info(interface, drv, netdev, "NIC Link is up\n"); 383 384 netif_carrier_on(netdev); 385 netif_tx_wake_all_queues(netdev); 386 } 387 388 /** 389 * fm10k_watchdog_host_not_ready - Update netdev status based on host not ready 390 * @interface: board private structure 391 **/ 392 static void fm10k_watchdog_host_not_ready(struct fm10k_intfc *interface) 393 { 394 struct net_device *netdev = interface->netdev; 395 396 /* only continue if link state is currently up */ 397 if (!netif_carrier_ok(netdev)) 398 return; 399 400 netif_info(interface, drv, netdev, "NIC Link is down\n"); 401 402 netif_carrier_off(netdev); 403 netif_tx_stop_all_queues(netdev); 404 } 405 406 /** 407 * fm10k_update_stats - Update the board statistics counters. 408 * @interface: board private structure 409 **/ 410 void fm10k_update_stats(struct fm10k_intfc *interface) 411 { 412 struct net_device_stats *net_stats = &interface->netdev->stats; 413 struct fm10k_hw *hw = &interface->hw; 414 u64 hw_csum_tx_good = 0, hw_csum_rx_good = 0, rx_length_errors = 0; 415 u64 rx_switch_errors = 0, rx_drops = 0, rx_pp_errors = 0; 416 u64 rx_link_errors = 0; 417 u64 rx_errors = 0, rx_csum_errors = 0, tx_csum_errors = 0; 418 u64 restart_queue = 0, tx_busy = 0, alloc_failed = 0; 419 u64 rx_bytes_nic = 0, rx_pkts_nic = 0, rx_drops_nic = 0; 420 u64 tx_bytes_nic = 0, tx_pkts_nic = 0; 421 u64 bytes, pkts; 422 int i; 423 424 /* ensure only one thread updates stats at a time */ 425 if (test_and_set_bit(__FM10K_UPDATING_STATS, interface->state)) 426 return; 427 428 /* do not allow stats update via service task for next second */ 429 interface->next_stats_update = jiffies + HZ; 430 431 /* gather some stats to the interface struct that are per queue */ 432 for (bytes = 0, pkts = 0, i = 0; i < interface->num_tx_queues; i++) { 433 struct fm10k_ring *tx_ring = READ_ONCE(interface->tx_ring[i]); 434 435 if (!tx_ring) 436 continue; 437 438 restart_queue += tx_ring->tx_stats.restart_queue; 439 tx_busy += tx_ring->tx_stats.tx_busy; 440 tx_csum_errors += tx_ring->tx_stats.csum_err; 441 bytes += tx_ring->stats.bytes; 442 pkts += tx_ring->stats.packets; 443 hw_csum_tx_good += tx_ring->tx_stats.csum_good; 444 } 445 446 interface->restart_queue = restart_queue; 447 interface->tx_busy = tx_busy; 448 net_stats->tx_bytes = bytes; 449 net_stats->tx_packets = pkts; 450 interface->tx_csum_errors = tx_csum_errors; 451 interface->hw_csum_tx_good = hw_csum_tx_good; 452 453 /* gather some stats to the interface struct that are per queue */ 454 for (bytes = 0, pkts = 0, i = 0; i < interface->num_rx_queues; i++) { 455 struct fm10k_ring *rx_ring = READ_ONCE(interface->rx_ring[i]); 456 457 if (!rx_ring) 458 continue; 459 460 bytes += rx_ring->stats.bytes; 461 pkts += rx_ring->stats.packets; 462 alloc_failed += rx_ring->rx_stats.alloc_failed; 463 rx_csum_errors += rx_ring->rx_stats.csum_err; 464 rx_errors += rx_ring->rx_stats.errors; 465 hw_csum_rx_good += rx_ring->rx_stats.csum_good; 466 rx_switch_errors += rx_ring->rx_stats.switch_errors; 467 rx_drops += rx_ring->rx_stats.drops; 468 rx_pp_errors += rx_ring->rx_stats.pp_errors; 469 rx_link_errors += rx_ring->rx_stats.link_errors; 470 rx_length_errors += rx_ring->rx_stats.length_errors; 471 } 472 473 net_stats->rx_bytes = bytes; 474 net_stats->rx_packets = pkts; 475 interface->alloc_failed = alloc_failed; 476 interface->rx_csum_errors = rx_csum_errors; 477 interface->hw_csum_rx_good = hw_csum_rx_good; 478 interface->rx_switch_errors = rx_switch_errors; 479 interface->rx_drops = rx_drops; 480 interface->rx_pp_errors = rx_pp_errors; 481 interface->rx_link_errors = rx_link_errors; 482 interface->rx_length_errors = rx_length_errors; 483 484 hw->mac.ops.update_hw_stats(hw, &interface->stats); 485 486 for (i = 0; i < hw->mac.max_queues; i++) { 487 struct fm10k_hw_stats_q *q = &interface->stats.q[i]; 488 489 tx_bytes_nic += q->tx_bytes.count; 490 tx_pkts_nic += q->tx_packets.count; 491 rx_bytes_nic += q->rx_bytes.count; 492 rx_pkts_nic += q->rx_packets.count; 493 rx_drops_nic += q->rx_drops.count; 494 } 495 496 interface->tx_bytes_nic = tx_bytes_nic; 497 interface->tx_packets_nic = tx_pkts_nic; 498 interface->rx_bytes_nic = rx_bytes_nic; 499 interface->rx_packets_nic = rx_pkts_nic; 500 interface->rx_drops_nic = rx_drops_nic; 501 502 /* Fill out the OS statistics structure */ 503 net_stats->rx_errors = rx_errors; 504 net_stats->rx_dropped = interface->stats.nodesc_drop.count; 505 506 clear_bit(__FM10K_UPDATING_STATS, interface->state); 507 } 508 509 /** 510 * fm10k_watchdog_flush_tx - flush queues on host not ready 511 * @interface - pointer to the device interface structure 512 **/ 513 static void fm10k_watchdog_flush_tx(struct fm10k_intfc *interface) 514 { 515 int some_tx_pending = 0; 516 int i; 517 518 /* nothing to do if carrier is up */ 519 if (netif_carrier_ok(interface->netdev)) 520 return; 521 522 for (i = 0; i < interface->num_tx_queues; i++) { 523 struct fm10k_ring *tx_ring = interface->tx_ring[i]; 524 525 if (tx_ring->next_to_use != tx_ring->next_to_clean) { 526 some_tx_pending = 1; 527 break; 528 } 529 } 530 531 /* We've lost link, so the controller stops DMA, but we've got 532 * queued Tx work that's never going to get done, so reset 533 * controller to flush Tx. 534 */ 535 if (some_tx_pending) 536 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 537 } 538 539 /** 540 * fm10k_watchdog_subtask - check and bring link up 541 * @interface - pointer to the device interface structure 542 **/ 543 static void fm10k_watchdog_subtask(struct fm10k_intfc *interface) 544 { 545 /* if interface is down do nothing */ 546 if (test_bit(__FM10K_DOWN, interface->state) || 547 test_bit(__FM10K_RESETTING, interface->state)) 548 return; 549 550 if (interface->host_ready) 551 fm10k_watchdog_host_is_ready(interface); 552 else 553 fm10k_watchdog_host_not_ready(interface); 554 555 /* update stats only once every second */ 556 if (time_is_before_jiffies(interface->next_stats_update)) 557 fm10k_update_stats(interface); 558 559 /* flush any uncompleted work */ 560 fm10k_watchdog_flush_tx(interface); 561 } 562 563 /** 564 * fm10k_check_hang_subtask - check for hung queues and dropped interrupts 565 * @interface - pointer to the device interface structure 566 * 567 * This function serves two purposes. First it strobes the interrupt lines 568 * in order to make certain interrupts are occurring. Secondly it sets the 569 * bits needed to check for TX hangs. As a result we should immediately 570 * determine if a hang has occurred. 571 */ 572 static void fm10k_check_hang_subtask(struct fm10k_intfc *interface) 573 { 574 int i; 575 576 /* If we're down or resetting, just bail */ 577 if (test_bit(__FM10K_DOWN, interface->state) || 578 test_bit(__FM10K_RESETTING, interface->state)) 579 return; 580 581 /* rate limit tx hang checks to only once every 2 seconds */ 582 if (time_is_after_eq_jiffies(interface->next_tx_hang_check)) 583 return; 584 interface->next_tx_hang_check = jiffies + (2 * HZ); 585 586 if (netif_carrier_ok(interface->netdev)) { 587 /* Force detection of hung controller */ 588 for (i = 0; i < interface->num_tx_queues; i++) 589 set_check_for_tx_hang(interface->tx_ring[i]); 590 591 /* Rearm all in-use q_vectors for immediate firing */ 592 for (i = 0; i < interface->num_q_vectors; i++) { 593 struct fm10k_q_vector *qv = interface->q_vector[i]; 594 595 if (!qv->tx.count && !qv->rx.count) 596 continue; 597 writel(FM10K_ITR_ENABLE | FM10K_ITR_PENDING2, qv->itr); 598 } 599 } 600 } 601 602 /** 603 * fm10k_service_task - manages and runs subtasks 604 * @work: pointer to work_struct containing our data 605 **/ 606 static void fm10k_service_task(struct work_struct *work) 607 { 608 struct fm10k_intfc *interface; 609 610 interface = container_of(work, struct fm10k_intfc, service_task); 611 612 /* tasks run even when interface is down */ 613 fm10k_mbx_subtask(interface); 614 fm10k_detach_subtask(interface); 615 fm10k_reset_subtask(interface); 616 617 /* tasks only run when interface is up */ 618 fm10k_watchdog_subtask(interface); 619 fm10k_check_hang_subtask(interface); 620 621 /* release lock on service events to allow scheduling next event */ 622 fm10k_service_event_complete(interface); 623 } 624 625 /** 626 * fm10k_configure_tx_ring - Configure Tx ring after Reset 627 * @interface: board private structure 628 * @ring: structure containing ring specific data 629 * 630 * Configure the Tx descriptor ring after a reset. 631 **/ 632 static void fm10k_configure_tx_ring(struct fm10k_intfc *interface, 633 struct fm10k_ring *ring) 634 { 635 struct fm10k_hw *hw = &interface->hw; 636 u64 tdba = ring->dma; 637 u32 size = ring->count * sizeof(struct fm10k_tx_desc); 638 u32 txint = FM10K_INT_MAP_DISABLE; 639 u32 txdctl = BIT(FM10K_TXDCTL_MAX_TIME_SHIFT) | FM10K_TXDCTL_ENABLE; 640 u8 reg_idx = ring->reg_idx; 641 642 /* disable queue to avoid issues while updating state */ 643 fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), 0); 644 fm10k_write_flush(hw); 645 646 /* possible poll here to verify ring resources have been cleaned */ 647 648 /* set location and size for descriptor ring */ 649 fm10k_write_reg(hw, FM10K_TDBAL(reg_idx), tdba & DMA_BIT_MASK(32)); 650 fm10k_write_reg(hw, FM10K_TDBAH(reg_idx), tdba >> 32); 651 fm10k_write_reg(hw, FM10K_TDLEN(reg_idx), size); 652 653 /* reset head and tail pointers */ 654 fm10k_write_reg(hw, FM10K_TDH(reg_idx), 0); 655 fm10k_write_reg(hw, FM10K_TDT(reg_idx), 0); 656 657 /* store tail pointer */ 658 ring->tail = &interface->uc_addr[FM10K_TDT(reg_idx)]; 659 660 /* reset ntu and ntc to place SW in sync with hardware */ 661 ring->next_to_clean = 0; 662 ring->next_to_use = 0; 663 664 /* Map interrupt */ 665 if (ring->q_vector) { 666 txint = ring->q_vector->v_idx + NON_Q_VECTORS(hw); 667 txint |= FM10K_INT_MAP_TIMER0; 668 } 669 670 fm10k_write_reg(hw, FM10K_TXINT(reg_idx), txint); 671 672 /* enable use of FTAG bit in Tx descriptor, register is RO for VF */ 673 fm10k_write_reg(hw, FM10K_PFVTCTL(reg_idx), 674 FM10K_PFVTCTL_FTAG_DESC_ENABLE); 675 676 /* Initialize XPS */ 677 if (!test_and_set_bit(__FM10K_TX_XPS_INIT_DONE, ring->state) && 678 ring->q_vector) 679 netif_set_xps_queue(ring->netdev, 680 &ring->q_vector->affinity_mask, 681 ring->queue_index); 682 683 /* enable queue */ 684 fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), txdctl); 685 } 686 687 /** 688 * fm10k_enable_tx_ring - Verify Tx ring is enabled after configuration 689 * @interface: board private structure 690 * @ring: structure containing ring specific data 691 * 692 * Verify the Tx descriptor ring is ready for transmit. 693 **/ 694 static void fm10k_enable_tx_ring(struct fm10k_intfc *interface, 695 struct fm10k_ring *ring) 696 { 697 struct fm10k_hw *hw = &interface->hw; 698 int wait_loop = 10; 699 u32 txdctl; 700 u8 reg_idx = ring->reg_idx; 701 702 /* if we are already enabled just exit */ 703 if (fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)) & FM10K_TXDCTL_ENABLE) 704 return; 705 706 /* poll to verify queue is enabled */ 707 do { 708 usleep_range(1000, 2000); 709 txdctl = fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)); 710 } while (!(txdctl & FM10K_TXDCTL_ENABLE) && --wait_loop); 711 if (!wait_loop) 712 netif_err(interface, drv, interface->netdev, 713 "Could not enable Tx Queue %d\n", reg_idx); 714 } 715 716 /** 717 * fm10k_configure_tx - Configure Transmit Unit after Reset 718 * @interface: board private structure 719 * 720 * Configure the Tx unit of the MAC after a reset. 721 **/ 722 static void fm10k_configure_tx(struct fm10k_intfc *interface) 723 { 724 int i; 725 726 /* Setup the HW Tx Head and Tail descriptor pointers */ 727 for (i = 0; i < interface->num_tx_queues; i++) 728 fm10k_configure_tx_ring(interface, interface->tx_ring[i]); 729 730 /* poll here to verify that Tx rings are now enabled */ 731 for (i = 0; i < interface->num_tx_queues; i++) 732 fm10k_enable_tx_ring(interface, interface->tx_ring[i]); 733 } 734 735 /** 736 * fm10k_configure_rx_ring - Configure Rx ring after Reset 737 * @interface: board private structure 738 * @ring: structure containing ring specific data 739 * 740 * Configure the Rx descriptor ring after a reset. 741 **/ 742 static void fm10k_configure_rx_ring(struct fm10k_intfc *interface, 743 struct fm10k_ring *ring) 744 { 745 u64 rdba = ring->dma; 746 struct fm10k_hw *hw = &interface->hw; 747 u32 size = ring->count * sizeof(union fm10k_rx_desc); 748 u32 rxqctl, rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 749 u32 srrctl = FM10K_SRRCTL_BUFFER_CHAINING_EN; 750 u32 rxint = FM10K_INT_MAP_DISABLE; 751 u8 rx_pause = interface->rx_pause; 752 u8 reg_idx = ring->reg_idx; 753 754 /* disable queue to avoid issues while updating state */ 755 rxqctl = fm10k_read_reg(hw, FM10K_RXQCTL(reg_idx)); 756 rxqctl &= ~FM10K_RXQCTL_ENABLE; 757 fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl); 758 fm10k_write_flush(hw); 759 760 /* possible poll here to verify ring resources have been cleaned */ 761 762 /* set location and size for descriptor ring */ 763 fm10k_write_reg(hw, FM10K_RDBAL(reg_idx), rdba & DMA_BIT_MASK(32)); 764 fm10k_write_reg(hw, FM10K_RDBAH(reg_idx), rdba >> 32); 765 fm10k_write_reg(hw, FM10K_RDLEN(reg_idx), size); 766 767 /* reset head and tail pointers */ 768 fm10k_write_reg(hw, FM10K_RDH(reg_idx), 0); 769 fm10k_write_reg(hw, FM10K_RDT(reg_idx), 0); 770 771 /* store tail pointer */ 772 ring->tail = &interface->uc_addr[FM10K_RDT(reg_idx)]; 773 774 /* reset ntu and ntc to place SW in sync with hardware */ 775 ring->next_to_clean = 0; 776 ring->next_to_use = 0; 777 ring->next_to_alloc = 0; 778 779 /* Configure the Rx buffer size for one buff without split */ 780 srrctl |= FM10K_RX_BUFSZ >> FM10K_SRRCTL_BSIZEPKT_SHIFT; 781 782 /* Configure the Rx ring to suppress loopback packets */ 783 srrctl |= FM10K_SRRCTL_LOOPBACK_SUPPRESS; 784 fm10k_write_reg(hw, FM10K_SRRCTL(reg_idx), srrctl); 785 786 /* Enable drop on empty */ 787 #ifdef CONFIG_DCB 788 if (interface->pfc_en) 789 rx_pause = interface->pfc_en; 790 #endif 791 if (!(rx_pause & BIT(ring->qos_pc))) 792 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 793 794 fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl); 795 796 /* assign default VLAN to queue */ 797 ring->vid = hw->mac.default_vid; 798 799 /* if we have an active VLAN, disable default VLAN ID */ 800 if (test_bit(hw->mac.default_vid, interface->active_vlans)) 801 ring->vid |= FM10K_VLAN_CLEAR; 802 803 /* Map interrupt */ 804 if (ring->q_vector) { 805 rxint = ring->q_vector->v_idx + NON_Q_VECTORS(hw); 806 rxint |= FM10K_INT_MAP_TIMER1; 807 } 808 809 fm10k_write_reg(hw, FM10K_RXINT(reg_idx), rxint); 810 811 /* enable queue */ 812 rxqctl = fm10k_read_reg(hw, FM10K_RXQCTL(reg_idx)); 813 rxqctl |= FM10K_RXQCTL_ENABLE; 814 fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl); 815 816 /* place buffers on ring for receive data */ 817 fm10k_alloc_rx_buffers(ring, fm10k_desc_unused(ring)); 818 } 819 820 /** 821 * fm10k_update_rx_drop_en - Configures the drop enable bits for Rx rings 822 * @interface: board private structure 823 * 824 * Configure the drop enable bits for the Rx rings. 825 **/ 826 void fm10k_update_rx_drop_en(struct fm10k_intfc *interface) 827 { 828 struct fm10k_hw *hw = &interface->hw; 829 u8 rx_pause = interface->rx_pause; 830 int i; 831 832 #ifdef CONFIG_DCB 833 if (interface->pfc_en) 834 rx_pause = interface->pfc_en; 835 836 #endif 837 for (i = 0; i < interface->num_rx_queues; i++) { 838 struct fm10k_ring *ring = interface->rx_ring[i]; 839 u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 840 u8 reg_idx = ring->reg_idx; 841 842 if (!(rx_pause & BIT(ring->qos_pc))) 843 rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY; 844 845 fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl); 846 } 847 } 848 849 /** 850 * fm10k_configure_dglort - Configure Receive DGLORT after reset 851 * @interface: board private structure 852 * 853 * Configure the DGLORT description and RSS tables. 854 **/ 855 static void fm10k_configure_dglort(struct fm10k_intfc *interface) 856 { 857 struct fm10k_dglort_cfg dglort = { 0 }; 858 struct fm10k_hw *hw = &interface->hw; 859 int i; 860 u32 mrqc; 861 862 /* Fill out hash function seeds */ 863 for (i = 0; i < FM10K_RSSRK_SIZE; i++) 864 fm10k_write_reg(hw, FM10K_RSSRK(0, i), interface->rssrk[i]); 865 866 /* Write RETA table to hardware */ 867 for (i = 0; i < FM10K_RETA_SIZE; i++) 868 fm10k_write_reg(hw, FM10K_RETA(0, i), interface->reta[i]); 869 870 /* Generate RSS hash based on packet types, TCP/UDP 871 * port numbers and/or IPv4/v6 src and dst addresses 872 */ 873 mrqc = FM10K_MRQC_IPV4 | 874 FM10K_MRQC_TCP_IPV4 | 875 FM10K_MRQC_IPV6 | 876 FM10K_MRQC_TCP_IPV6; 877 878 if (test_bit(FM10K_FLAG_RSS_FIELD_IPV4_UDP, interface->flags)) 879 mrqc |= FM10K_MRQC_UDP_IPV4; 880 if (test_bit(FM10K_FLAG_RSS_FIELD_IPV6_UDP, interface->flags)) 881 mrqc |= FM10K_MRQC_UDP_IPV6; 882 883 fm10k_write_reg(hw, FM10K_MRQC(0), mrqc); 884 885 /* configure default DGLORT mapping for RSS/DCB */ 886 dglort.inner_rss = 1; 887 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask); 888 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask); 889 hw->mac.ops.configure_dglort_map(hw, &dglort); 890 891 /* assign GLORT per queue for queue mapped testing */ 892 if (interface->glort_count > 64) { 893 memset(&dglort, 0, sizeof(dglort)); 894 dglort.inner_rss = 1; 895 dglort.glort = interface->glort + 64; 896 dglort.idx = fm10k_dglort_pf_queue; 897 dglort.queue_l = fls(interface->num_rx_queues - 1); 898 hw->mac.ops.configure_dglort_map(hw, &dglort); 899 } 900 901 /* assign glort value for RSS/DCB specific to this interface */ 902 memset(&dglort, 0, sizeof(dglort)); 903 dglort.inner_rss = 1; 904 dglort.glort = interface->glort; 905 dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask); 906 dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask); 907 /* configure DGLORT mapping for RSS/DCB */ 908 dglort.idx = fm10k_dglort_pf_rss; 909 if (interface->l2_accel) 910 dglort.shared_l = fls(interface->l2_accel->size); 911 hw->mac.ops.configure_dglort_map(hw, &dglort); 912 } 913 914 /** 915 * fm10k_configure_rx - Configure Receive Unit after Reset 916 * @interface: board private structure 917 * 918 * Configure the Rx unit of the MAC after a reset. 919 **/ 920 static void fm10k_configure_rx(struct fm10k_intfc *interface) 921 { 922 int i; 923 924 /* Configure SWPRI to PC map */ 925 fm10k_configure_swpri_map(interface); 926 927 /* Configure RSS and DGLORT map */ 928 fm10k_configure_dglort(interface); 929 930 /* Setup the HW Rx Head and Tail descriptor pointers */ 931 for (i = 0; i < interface->num_rx_queues; i++) 932 fm10k_configure_rx_ring(interface, interface->rx_ring[i]); 933 934 /* possible poll here to verify that Rx rings are now enabled */ 935 } 936 937 static void fm10k_napi_enable_all(struct fm10k_intfc *interface) 938 { 939 struct fm10k_q_vector *q_vector; 940 int q_idx; 941 942 for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) { 943 q_vector = interface->q_vector[q_idx]; 944 napi_enable(&q_vector->napi); 945 } 946 } 947 948 static irqreturn_t fm10k_msix_clean_rings(int __always_unused irq, void *data) 949 { 950 struct fm10k_q_vector *q_vector = data; 951 952 if (q_vector->rx.count || q_vector->tx.count) 953 napi_schedule_irqoff(&q_vector->napi); 954 955 return IRQ_HANDLED; 956 } 957 958 static irqreturn_t fm10k_msix_mbx_vf(int __always_unused irq, void *data) 959 { 960 struct fm10k_intfc *interface = data; 961 struct fm10k_hw *hw = &interface->hw; 962 struct fm10k_mbx_info *mbx = &hw->mbx; 963 964 /* re-enable mailbox interrupt and indicate 20us delay */ 965 fm10k_write_reg(hw, FM10K_VFITR(FM10K_MBX_VECTOR), 966 (FM10K_MBX_INT_DELAY >> hw->mac.itr_scale) | 967 FM10K_ITR_ENABLE); 968 969 /* service upstream mailbox */ 970 if (fm10k_mbx_trylock(interface)) { 971 mbx->ops.process(hw, mbx); 972 fm10k_mbx_unlock(interface); 973 } 974 975 hw->mac.get_host_state = true; 976 fm10k_service_event_schedule(interface); 977 978 return IRQ_HANDLED; 979 } 980 981 #ifdef CONFIG_NET_POLL_CONTROLLER 982 /** 983 * fm10k_netpoll - A Polling 'interrupt' handler 984 * @netdev: network interface device structure 985 * 986 * This is used by netconsole to send skbs without having to re-enable 987 * interrupts. It's not called while the normal interrupt routine is executing. 988 **/ 989 void fm10k_netpoll(struct net_device *netdev) 990 { 991 struct fm10k_intfc *interface = netdev_priv(netdev); 992 int i; 993 994 /* if interface is down do nothing */ 995 if (test_bit(__FM10K_DOWN, interface->state)) 996 return; 997 998 for (i = 0; i < interface->num_q_vectors; i++) 999 fm10k_msix_clean_rings(0, interface->q_vector[i]); 1000 } 1001 1002 #endif 1003 #define FM10K_ERR_MSG(type) case (type): error = #type; break 1004 static void fm10k_handle_fault(struct fm10k_intfc *interface, int type, 1005 struct fm10k_fault *fault) 1006 { 1007 struct pci_dev *pdev = interface->pdev; 1008 struct fm10k_hw *hw = &interface->hw; 1009 struct fm10k_iov_data *iov_data = interface->iov_data; 1010 char *error; 1011 1012 switch (type) { 1013 case FM10K_PCA_FAULT: 1014 switch (fault->type) { 1015 default: 1016 error = "Unknown PCA error"; 1017 break; 1018 FM10K_ERR_MSG(PCA_NO_FAULT); 1019 FM10K_ERR_MSG(PCA_UNMAPPED_ADDR); 1020 FM10K_ERR_MSG(PCA_BAD_QACCESS_PF); 1021 FM10K_ERR_MSG(PCA_BAD_QACCESS_VF); 1022 FM10K_ERR_MSG(PCA_MALICIOUS_REQ); 1023 FM10K_ERR_MSG(PCA_POISONED_TLP); 1024 FM10K_ERR_MSG(PCA_TLP_ABORT); 1025 } 1026 break; 1027 case FM10K_THI_FAULT: 1028 switch (fault->type) { 1029 default: 1030 error = "Unknown THI error"; 1031 break; 1032 FM10K_ERR_MSG(THI_NO_FAULT); 1033 FM10K_ERR_MSG(THI_MAL_DIS_Q_FAULT); 1034 } 1035 break; 1036 case FM10K_FUM_FAULT: 1037 switch (fault->type) { 1038 default: 1039 error = "Unknown FUM error"; 1040 break; 1041 FM10K_ERR_MSG(FUM_NO_FAULT); 1042 FM10K_ERR_MSG(FUM_UNMAPPED_ADDR); 1043 FM10K_ERR_MSG(FUM_BAD_VF_QACCESS); 1044 FM10K_ERR_MSG(FUM_ADD_DECODE_ERR); 1045 FM10K_ERR_MSG(FUM_RO_ERROR); 1046 FM10K_ERR_MSG(FUM_QPRC_CRC_ERROR); 1047 FM10K_ERR_MSG(FUM_CSR_TIMEOUT); 1048 FM10K_ERR_MSG(FUM_INVALID_TYPE); 1049 FM10K_ERR_MSG(FUM_INVALID_LENGTH); 1050 FM10K_ERR_MSG(FUM_INVALID_BE); 1051 FM10K_ERR_MSG(FUM_INVALID_ALIGN); 1052 } 1053 break; 1054 default: 1055 error = "Undocumented fault"; 1056 break; 1057 } 1058 1059 dev_warn(&pdev->dev, 1060 "%s Address: 0x%llx SpecInfo: 0x%x Func: %02x.%0x\n", 1061 error, fault->address, fault->specinfo, 1062 PCI_SLOT(fault->func), PCI_FUNC(fault->func)); 1063 1064 /* For VF faults, clear out the respective LPORT, reset the queue 1065 * resources, and then reconnect to the mailbox. This allows the 1066 * VF in question to resume behavior. For transient faults that are 1067 * the result of non-malicious behavior this will log the fault and 1068 * allow the VF to resume functionality. Obviously for malicious VFs 1069 * they will be able to attempt malicious behavior again. In this 1070 * case, the system administrator will need to step in and manually 1071 * remove or disable the VF in question. 1072 */ 1073 if (fault->func && iov_data) { 1074 int vf = fault->func - 1; 1075 struct fm10k_vf_info *vf_info = &iov_data->vf_info[vf]; 1076 1077 hw->iov.ops.reset_lport(hw, vf_info); 1078 hw->iov.ops.reset_resources(hw, vf_info); 1079 1080 /* reset_lport disables the VF, so re-enable it */ 1081 hw->iov.ops.set_lport(hw, vf_info, vf, 1082 FM10K_VF_FLAG_MULTI_CAPABLE); 1083 1084 /* reset_resources will disconnect from the mbx */ 1085 vf_info->mbx.ops.connect(hw, &vf_info->mbx); 1086 } 1087 } 1088 1089 static void fm10k_report_fault(struct fm10k_intfc *interface, u32 eicr) 1090 { 1091 struct fm10k_hw *hw = &interface->hw; 1092 struct fm10k_fault fault = { 0 }; 1093 int type, err; 1094 1095 for (eicr &= FM10K_EICR_FAULT_MASK, type = FM10K_PCA_FAULT; 1096 eicr; 1097 eicr >>= 1, type += FM10K_FAULT_SIZE) { 1098 /* only check if there is an error reported */ 1099 if (!(eicr & 0x1)) 1100 continue; 1101 1102 /* retrieve fault info */ 1103 err = hw->mac.ops.get_fault(hw, type, &fault); 1104 if (err) { 1105 dev_err(&interface->pdev->dev, 1106 "error reading fault\n"); 1107 continue; 1108 } 1109 1110 fm10k_handle_fault(interface, type, &fault); 1111 } 1112 } 1113 1114 static void fm10k_reset_drop_on_empty(struct fm10k_intfc *interface, u32 eicr) 1115 { 1116 struct fm10k_hw *hw = &interface->hw; 1117 const u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY; 1118 u32 maxholdq; 1119 int q; 1120 1121 if (!(eicr & FM10K_EICR_MAXHOLDTIME)) 1122 return; 1123 1124 maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(7)); 1125 if (maxholdq) 1126 fm10k_write_reg(hw, FM10K_MAXHOLDQ(7), maxholdq); 1127 for (q = 255;;) { 1128 if (maxholdq & BIT(31)) { 1129 if (q < FM10K_MAX_QUEUES_PF) { 1130 interface->rx_overrun_pf++; 1131 fm10k_write_reg(hw, FM10K_RXDCTL(q), rxdctl); 1132 } else { 1133 interface->rx_overrun_vf++; 1134 } 1135 } 1136 1137 maxholdq *= 2; 1138 if (!maxholdq) 1139 q &= ~(32 - 1); 1140 1141 if (!q) 1142 break; 1143 1144 if (q-- % 32) 1145 continue; 1146 1147 maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(q / 32)); 1148 if (maxholdq) 1149 fm10k_write_reg(hw, FM10K_MAXHOLDQ(q / 32), maxholdq); 1150 } 1151 } 1152 1153 static irqreturn_t fm10k_msix_mbx_pf(int __always_unused irq, void *data) 1154 { 1155 struct fm10k_intfc *interface = data; 1156 struct fm10k_hw *hw = &interface->hw; 1157 struct fm10k_mbx_info *mbx = &hw->mbx; 1158 u32 eicr; 1159 s32 err = 0; 1160 1161 /* unmask any set bits related to this interrupt */ 1162 eicr = fm10k_read_reg(hw, FM10K_EICR); 1163 fm10k_write_reg(hw, FM10K_EICR, eicr & (FM10K_EICR_MAILBOX | 1164 FM10K_EICR_SWITCHREADY | 1165 FM10K_EICR_SWITCHNOTREADY)); 1166 1167 /* report any faults found to the message log */ 1168 fm10k_report_fault(interface, eicr); 1169 1170 /* reset any queues disabled due to receiver overrun */ 1171 fm10k_reset_drop_on_empty(interface, eicr); 1172 1173 /* service mailboxes */ 1174 if (fm10k_mbx_trylock(interface)) { 1175 err = mbx->ops.process(hw, mbx); 1176 /* handle VFLRE events */ 1177 fm10k_iov_event(interface); 1178 fm10k_mbx_unlock(interface); 1179 } 1180 1181 if (err == FM10K_ERR_RESET_REQUESTED) 1182 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 1183 1184 /* if switch toggled state we should reset GLORTs */ 1185 if (eicr & FM10K_EICR_SWITCHNOTREADY) { 1186 /* force link down for at least 4 seconds */ 1187 interface->link_down_event = jiffies + (4 * HZ); 1188 set_bit(__FM10K_LINK_DOWN, interface->state); 1189 1190 /* reset dglort_map back to no config */ 1191 hw->mac.dglort_map = FM10K_DGLORTMAP_NONE; 1192 } 1193 1194 /* we should validate host state after interrupt event */ 1195 hw->mac.get_host_state = true; 1196 1197 /* validate host state, and handle VF mailboxes in the service task */ 1198 fm10k_service_event_schedule(interface); 1199 1200 /* re-enable mailbox interrupt and indicate 20us delay */ 1201 fm10k_write_reg(hw, FM10K_ITR(FM10K_MBX_VECTOR), 1202 (FM10K_MBX_INT_DELAY >> hw->mac.itr_scale) | 1203 FM10K_ITR_ENABLE); 1204 1205 return IRQ_HANDLED; 1206 } 1207 1208 void fm10k_mbx_free_irq(struct fm10k_intfc *interface) 1209 { 1210 struct fm10k_hw *hw = &interface->hw; 1211 struct msix_entry *entry; 1212 int itr_reg; 1213 1214 /* no mailbox IRQ to free if MSI-X is not enabled */ 1215 if (!interface->msix_entries) 1216 return; 1217 1218 entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1219 1220 /* disconnect the mailbox */ 1221 hw->mbx.ops.disconnect(hw, &hw->mbx); 1222 1223 /* disable Mailbox cause */ 1224 if (hw->mac.type == fm10k_mac_pf) { 1225 fm10k_write_reg(hw, FM10K_EIMR, 1226 FM10K_EIMR_DISABLE(PCA_FAULT) | 1227 FM10K_EIMR_DISABLE(FUM_FAULT) | 1228 FM10K_EIMR_DISABLE(MAILBOX) | 1229 FM10K_EIMR_DISABLE(SWITCHREADY) | 1230 FM10K_EIMR_DISABLE(SWITCHNOTREADY) | 1231 FM10K_EIMR_DISABLE(SRAMERROR) | 1232 FM10K_EIMR_DISABLE(VFLR) | 1233 FM10K_EIMR_DISABLE(MAXHOLDTIME)); 1234 itr_reg = FM10K_ITR(FM10K_MBX_VECTOR); 1235 } else { 1236 itr_reg = FM10K_VFITR(FM10K_MBX_VECTOR); 1237 } 1238 1239 fm10k_write_reg(hw, itr_reg, FM10K_ITR_MASK_SET); 1240 1241 free_irq(entry->vector, interface); 1242 } 1243 1244 static s32 fm10k_mbx_mac_addr(struct fm10k_hw *hw, u32 **results, 1245 struct fm10k_mbx_info *mbx) 1246 { 1247 bool vlan_override = hw->mac.vlan_override; 1248 u16 default_vid = hw->mac.default_vid; 1249 struct fm10k_intfc *interface; 1250 s32 err; 1251 1252 err = fm10k_msg_mac_vlan_vf(hw, results, mbx); 1253 if (err) 1254 return err; 1255 1256 interface = container_of(hw, struct fm10k_intfc, hw); 1257 1258 /* MAC was changed so we need reset */ 1259 if (is_valid_ether_addr(hw->mac.perm_addr) && 1260 !ether_addr_equal(hw->mac.perm_addr, hw->mac.addr)) 1261 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 1262 1263 /* VLAN override was changed, or default VLAN changed */ 1264 if ((vlan_override != hw->mac.vlan_override) || 1265 (default_vid != hw->mac.default_vid)) 1266 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 1267 1268 return 0; 1269 } 1270 1271 /* generic error handler for mailbox issues */ 1272 static s32 fm10k_mbx_error(struct fm10k_hw *hw, u32 **results, 1273 struct fm10k_mbx_info __always_unused *mbx) 1274 { 1275 struct fm10k_intfc *interface; 1276 struct pci_dev *pdev; 1277 1278 interface = container_of(hw, struct fm10k_intfc, hw); 1279 pdev = interface->pdev; 1280 1281 dev_err(&pdev->dev, "Unknown message ID %u\n", 1282 **results & FM10K_TLV_ID_MASK); 1283 1284 return 0; 1285 } 1286 1287 static const struct fm10k_msg_data vf_mbx_data[] = { 1288 FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test), 1289 FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_mbx_mac_addr), 1290 FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf), 1291 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error), 1292 }; 1293 1294 static int fm10k_mbx_request_irq_vf(struct fm10k_intfc *interface) 1295 { 1296 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1297 struct net_device *dev = interface->netdev; 1298 struct fm10k_hw *hw = &interface->hw; 1299 int err; 1300 1301 /* Use timer0 for interrupt moderation on the mailbox */ 1302 u32 itr = entry->entry | FM10K_INT_MAP_TIMER0; 1303 1304 /* register mailbox handlers */ 1305 err = hw->mbx.ops.register_handlers(&hw->mbx, vf_mbx_data); 1306 if (err) 1307 return err; 1308 1309 /* request the IRQ */ 1310 err = request_irq(entry->vector, fm10k_msix_mbx_vf, 0, 1311 dev->name, interface); 1312 if (err) { 1313 netif_err(interface, probe, dev, 1314 "request_irq for msix_mbx failed: %d\n", err); 1315 return err; 1316 } 1317 1318 /* map all of the interrupt sources */ 1319 fm10k_write_reg(hw, FM10K_VFINT_MAP, itr); 1320 1321 /* enable interrupt */ 1322 fm10k_write_reg(hw, FM10K_VFITR(entry->entry), FM10K_ITR_ENABLE); 1323 1324 return 0; 1325 } 1326 1327 static s32 fm10k_lport_map(struct fm10k_hw *hw, u32 **results, 1328 struct fm10k_mbx_info *mbx) 1329 { 1330 struct fm10k_intfc *interface; 1331 u32 dglort_map = hw->mac.dglort_map; 1332 s32 err; 1333 1334 interface = container_of(hw, struct fm10k_intfc, hw); 1335 1336 err = fm10k_msg_err_pf(hw, results, mbx); 1337 if (!err && hw->swapi.status) { 1338 /* force link down for a reasonable delay */ 1339 interface->link_down_event = jiffies + (2 * HZ); 1340 set_bit(__FM10K_LINK_DOWN, interface->state); 1341 1342 /* reset dglort_map back to no config */ 1343 hw->mac.dglort_map = FM10K_DGLORTMAP_NONE; 1344 1345 fm10k_service_event_schedule(interface); 1346 1347 /* prevent overloading kernel message buffer */ 1348 if (interface->lport_map_failed) 1349 return 0; 1350 1351 interface->lport_map_failed = true; 1352 1353 if (hw->swapi.status == FM10K_MSG_ERR_PEP_NOT_SCHEDULED) 1354 dev_warn(&interface->pdev->dev, 1355 "cannot obtain link because the host interface is configured for a PCIe host interface bandwidth of zero\n"); 1356 dev_warn(&interface->pdev->dev, 1357 "request logical port map failed: %d\n", 1358 hw->swapi.status); 1359 1360 return 0; 1361 } 1362 1363 err = fm10k_msg_lport_map_pf(hw, results, mbx); 1364 if (err) 1365 return err; 1366 1367 interface->lport_map_failed = false; 1368 1369 /* we need to reset if port count was just updated */ 1370 if (dglort_map != hw->mac.dglort_map) 1371 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 1372 1373 return 0; 1374 } 1375 1376 static s32 fm10k_update_pvid(struct fm10k_hw *hw, u32 **results, 1377 struct fm10k_mbx_info __always_unused *mbx) 1378 { 1379 struct fm10k_intfc *interface; 1380 u16 glort, pvid; 1381 u32 pvid_update; 1382 s32 err; 1383 1384 err = fm10k_tlv_attr_get_u32(results[FM10K_PF_ATTR_ID_UPDATE_PVID], 1385 &pvid_update); 1386 if (err) 1387 return err; 1388 1389 /* extract values from the pvid update */ 1390 glort = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_GLORT); 1391 pvid = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_PVID); 1392 1393 /* if glort is not valid return error */ 1394 if (!fm10k_glort_valid_pf(hw, glort)) 1395 return FM10K_ERR_PARAM; 1396 1397 /* verify VLAN ID is valid */ 1398 if (pvid >= FM10K_VLAN_TABLE_VID_MAX) 1399 return FM10K_ERR_PARAM; 1400 1401 interface = container_of(hw, struct fm10k_intfc, hw); 1402 1403 /* check to see if this belongs to one of the VFs */ 1404 err = fm10k_iov_update_pvid(interface, glort, pvid); 1405 if (!err) 1406 return 0; 1407 1408 /* we need to reset if default VLAN was just updated */ 1409 if (pvid != hw->mac.default_vid) 1410 set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags); 1411 1412 hw->mac.default_vid = pvid; 1413 1414 return 0; 1415 } 1416 1417 static const struct fm10k_msg_data pf_mbx_data[] = { 1418 FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf), 1419 FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf), 1420 FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_lport_map), 1421 FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf), 1422 FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf), 1423 FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_update_pvid), 1424 FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error), 1425 }; 1426 1427 static int fm10k_mbx_request_irq_pf(struct fm10k_intfc *interface) 1428 { 1429 struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR]; 1430 struct net_device *dev = interface->netdev; 1431 struct fm10k_hw *hw = &interface->hw; 1432 int err; 1433 1434 /* Use timer0 for interrupt moderation on the mailbox */ 1435 u32 mbx_itr = entry->entry | FM10K_INT_MAP_TIMER0; 1436 u32 other_itr = entry->entry | FM10K_INT_MAP_IMMEDIATE; 1437 1438 /* register mailbox handlers */ 1439 err = hw->mbx.ops.register_handlers(&hw->mbx, pf_mbx_data); 1440 if (err) 1441 return err; 1442 1443 /* request the IRQ */ 1444 err = request_irq(entry->vector, fm10k_msix_mbx_pf, 0, 1445 dev->name, interface); 1446 if (err) { 1447 netif_err(interface, probe, dev, 1448 "request_irq for msix_mbx failed: %d\n", err); 1449 return err; 1450 } 1451 1452 /* Enable interrupts w/ no moderation for "other" interrupts */ 1453 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), other_itr); 1454 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), other_itr); 1455 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_sram), other_itr); 1456 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_max_hold_time), other_itr); 1457 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_vflr), other_itr); 1458 1459 /* Enable interrupts w/ moderation for mailbox */ 1460 fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_mailbox), mbx_itr); 1461 1462 /* Enable individual interrupt causes */ 1463 fm10k_write_reg(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) | 1464 FM10K_EIMR_ENABLE(FUM_FAULT) | 1465 FM10K_EIMR_ENABLE(MAILBOX) | 1466 FM10K_EIMR_ENABLE(SWITCHREADY) | 1467 FM10K_EIMR_ENABLE(SWITCHNOTREADY) | 1468 FM10K_EIMR_ENABLE(SRAMERROR) | 1469 FM10K_EIMR_ENABLE(VFLR) | 1470 FM10K_EIMR_ENABLE(MAXHOLDTIME)); 1471 1472 /* enable interrupt */ 1473 fm10k_write_reg(hw, FM10K_ITR(entry->entry), FM10K_ITR_ENABLE); 1474 1475 return 0; 1476 } 1477 1478 int fm10k_mbx_request_irq(struct fm10k_intfc *interface) 1479 { 1480 struct fm10k_hw *hw = &interface->hw; 1481 int err; 1482 1483 /* enable Mailbox cause */ 1484 if (hw->mac.type == fm10k_mac_pf) 1485 err = fm10k_mbx_request_irq_pf(interface); 1486 else 1487 err = fm10k_mbx_request_irq_vf(interface); 1488 if (err) 1489 return err; 1490 1491 /* connect mailbox */ 1492 err = hw->mbx.ops.connect(hw, &hw->mbx); 1493 1494 /* if the mailbox failed to connect, then free IRQ */ 1495 if (err) 1496 fm10k_mbx_free_irq(interface); 1497 1498 return err; 1499 } 1500 1501 /** 1502 * fm10k_qv_free_irq - release interrupts associated with queue vectors 1503 * @interface: board private structure 1504 * 1505 * Release all interrupts associated with this interface 1506 **/ 1507 void fm10k_qv_free_irq(struct fm10k_intfc *interface) 1508 { 1509 int vector = interface->num_q_vectors; 1510 struct fm10k_hw *hw = &interface->hw; 1511 struct msix_entry *entry; 1512 1513 entry = &interface->msix_entries[NON_Q_VECTORS(hw) + vector]; 1514 1515 while (vector) { 1516 struct fm10k_q_vector *q_vector; 1517 1518 vector--; 1519 entry--; 1520 q_vector = interface->q_vector[vector]; 1521 1522 if (!q_vector->tx.count && !q_vector->rx.count) 1523 continue; 1524 1525 /* clear the affinity_mask in the IRQ descriptor */ 1526 irq_set_affinity_hint(entry->vector, NULL); 1527 1528 /* disable interrupts */ 1529 writel(FM10K_ITR_MASK_SET, q_vector->itr); 1530 1531 free_irq(entry->vector, q_vector); 1532 } 1533 } 1534 1535 /** 1536 * fm10k_qv_request_irq - initialize interrupts for queue vectors 1537 * @interface: board private structure 1538 * 1539 * Attempts to configure interrupts using the best available 1540 * capabilities of the hardware and kernel. 1541 **/ 1542 int fm10k_qv_request_irq(struct fm10k_intfc *interface) 1543 { 1544 struct net_device *dev = interface->netdev; 1545 struct fm10k_hw *hw = &interface->hw; 1546 struct msix_entry *entry; 1547 int ri = 0, ti = 0; 1548 int vector, err; 1549 1550 entry = &interface->msix_entries[NON_Q_VECTORS(hw)]; 1551 1552 for (vector = 0; vector < interface->num_q_vectors; vector++) { 1553 struct fm10k_q_vector *q_vector = interface->q_vector[vector]; 1554 1555 /* name the vector */ 1556 if (q_vector->tx.count && q_vector->rx.count) { 1557 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1558 "%s-TxRx-%d", dev->name, ri++); 1559 ti++; 1560 } else if (q_vector->rx.count) { 1561 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1562 "%s-rx-%d", dev->name, ri++); 1563 } else if (q_vector->tx.count) { 1564 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1565 "%s-tx-%d", dev->name, ti++); 1566 } else { 1567 /* skip this unused q_vector */ 1568 continue; 1569 } 1570 1571 /* Assign ITR register to q_vector */ 1572 q_vector->itr = (hw->mac.type == fm10k_mac_pf) ? 1573 &interface->uc_addr[FM10K_ITR(entry->entry)] : 1574 &interface->uc_addr[FM10K_VFITR(entry->entry)]; 1575 1576 /* request the IRQ */ 1577 err = request_irq(entry->vector, &fm10k_msix_clean_rings, 0, 1578 q_vector->name, q_vector); 1579 if (err) { 1580 netif_err(interface, probe, dev, 1581 "request_irq failed for MSIX interrupt Error: %d\n", 1582 err); 1583 goto err_out; 1584 } 1585 1586 /* assign the mask for this irq */ 1587 irq_set_affinity_hint(entry->vector, &q_vector->affinity_mask); 1588 1589 /* Enable q_vector */ 1590 writel(FM10K_ITR_ENABLE, q_vector->itr); 1591 1592 entry++; 1593 } 1594 1595 return 0; 1596 1597 err_out: 1598 /* wind through the ring freeing all entries and vectors */ 1599 while (vector) { 1600 struct fm10k_q_vector *q_vector; 1601 1602 entry--; 1603 vector--; 1604 q_vector = interface->q_vector[vector]; 1605 1606 if (!q_vector->tx.count && !q_vector->rx.count) 1607 continue; 1608 1609 /* clear the affinity_mask in the IRQ descriptor */ 1610 irq_set_affinity_hint(entry->vector, NULL); 1611 1612 /* disable interrupts */ 1613 writel(FM10K_ITR_MASK_SET, q_vector->itr); 1614 1615 free_irq(entry->vector, q_vector); 1616 } 1617 1618 return err; 1619 } 1620 1621 void fm10k_up(struct fm10k_intfc *interface) 1622 { 1623 struct fm10k_hw *hw = &interface->hw; 1624 1625 /* Enable Tx/Rx DMA */ 1626 hw->mac.ops.start_hw(hw); 1627 1628 /* configure Tx descriptor rings */ 1629 fm10k_configure_tx(interface); 1630 1631 /* configure Rx descriptor rings */ 1632 fm10k_configure_rx(interface); 1633 1634 /* configure interrupts */ 1635 hw->mac.ops.update_int_moderator(hw); 1636 1637 /* enable statistics capture again */ 1638 clear_bit(__FM10K_UPDATING_STATS, interface->state); 1639 1640 /* clear down bit to indicate we are ready to go */ 1641 clear_bit(__FM10K_DOWN, interface->state); 1642 1643 /* enable polling cleanups */ 1644 fm10k_napi_enable_all(interface); 1645 1646 /* re-establish Rx filters */ 1647 fm10k_restore_rx_state(interface); 1648 1649 /* enable transmits */ 1650 netif_tx_start_all_queues(interface->netdev); 1651 1652 /* kick off the service timer now */ 1653 hw->mac.get_host_state = true; 1654 mod_timer(&interface->service_timer, jiffies); 1655 } 1656 1657 static void fm10k_napi_disable_all(struct fm10k_intfc *interface) 1658 { 1659 struct fm10k_q_vector *q_vector; 1660 int q_idx; 1661 1662 for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) { 1663 q_vector = interface->q_vector[q_idx]; 1664 napi_disable(&q_vector->napi); 1665 } 1666 } 1667 1668 void fm10k_down(struct fm10k_intfc *interface) 1669 { 1670 struct net_device *netdev = interface->netdev; 1671 struct fm10k_hw *hw = &interface->hw; 1672 int err, i = 0, count = 0; 1673 1674 /* signal that we are down to the interrupt handler and service task */ 1675 if (test_and_set_bit(__FM10K_DOWN, interface->state)) 1676 return; 1677 1678 /* call carrier off first to avoid false dev_watchdog timeouts */ 1679 netif_carrier_off(netdev); 1680 1681 /* disable transmits */ 1682 netif_tx_stop_all_queues(netdev); 1683 netif_tx_disable(netdev); 1684 1685 /* reset Rx filters */ 1686 fm10k_reset_rx_state(interface); 1687 1688 /* disable polling routines */ 1689 fm10k_napi_disable_all(interface); 1690 1691 /* capture stats one last time before stopping interface */ 1692 fm10k_update_stats(interface); 1693 1694 /* prevent updating statistics while we're down */ 1695 while (test_and_set_bit(__FM10K_UPDATING_STATS, interface->state)) 1696 usleep_range(1000, 2000); 1697 1698 /* skip waiting for TX DMA if we lost PCIe link */ 1699 if (FM10K_REMOVED(hw->hw_addr)) 1700 goto skip_tx_dma_drain; 1701 1702 /* In some rare circumstances it can take a while for Tx queues to 1703 * quiesce and be fully disabled. Attempt to .stop_hw() first, and 1704 * then if we get ERR_REQUESTS_PENDING, go ahead and wait in a loop 1705 * until the Tx queues have emptied, or until a number of retries. If 1706 * we fail to clear within the retry loop, we will issue a warning 1707 * indicating that Tx DMA is probably hung. Note this means we call 1708 * .stop_hw() twice but this shouldn't cause any problems. 1709 */ 1710 err = hw->mac.ops.stop_hw(hw); 1711 if (err != FM10K_ERR_REQUESTS_PENDING) 1712 goto skip_tx_dma_drain; 1713 1714 #define TX_DMA_DRAIN_RETRIES 25 1715 for (count = 0; count < TX_DMA_DRAIN_RETRIES; count++) { 1716 usleep_range(10000, 20000); 1717 1718 /* start checking at the last ring to have pending Tx */ 1719 for (; i < interface->num_tx_queues; i++) 1720 if (fm10k_get_tx_pending(interface->tx_ring[i], false)) 1721 break; 1722 1723 /* if all the queues are drained, we can break now */ 1724 if (i == interface->num_tx_queues) 1725 break; 1726 } 1727 1728 if (count >= TX_DMA_DRAIN_RETRIES) 1729 dev_err(&interface->pdev->dev, 1730 "Tx queues failed to drain after %d tries. Tx DMA is probably hung.\n", 1731 count); 1732 skip_tx_dma_drain: 1733 /* Disable DMA engine for Tx/Rx */ 1734 err = hw->mac.ops.stop_hw(hw); 1735 if (err == FM10K_ERR_REQUESTS_PENDING) 1736 dev_err(&interface->pdev->dev, 1737 "due to pending requests hw was not shut down gracefully\n"); 1738 else if (err) 1739 dev_err(&interface->pdev->dev, "stop_hw failed: %d\n", err); 1740 1741 /* free any buffers still on the rings */ 1742 fm10k_clean_all_tx_rings(interface); 1743 fm10k_clean_all_rx_rings(interface); 1744 } 1745 1746 /** 1747 * fm10k_sw_init - Initialize general software structures 1748 * @interface: host interface private structure to initialize 1749 * 1750 * fm10k_sw_init initializes the interface private data structure. 1751 * Fields are initialized based on PCI device information and 1752 * OS network device settings (MTU size). 1753 **/ 1754 static int fm10k_sw_init(struct fm10k_intfc *interface, 1755 const struct pci_device_id *ent) 1756 { 1757 const struct fm10k_info *fi = fm10k_info_tbl[ent->driver_data]; 1758 struct fm10k_hw *hw = &interface->hw; 1759 struct pci_dev *pdev = interface->pdev; 1760 struct net_device *netdev = interface->netdev; 1761 u32 rss_key[FM10K_RSSRK_SIZE]; 1762 unsigned int rss; 1763 int err; 1764 1765 /* initialize back pointer */ 1766 hw->back = interface; 1767 hw->hw_addr = interface->uc_addr; 1768 1769 /* PCI config space info */ 1770 hw->vendor_id = pdev->vendor; 1771 hw->device_id = pdev->device; 1772 hw->revision_id = pdev->revision; 1773 hw->subsystem_vendor_id = pdev->subsystem_vendor; 1774 hw->subsystem_device_id = pdev->subsystem_device; 1775 1776 /* Setup hw api */ 1777 memcpy(&hw->mac.ops, fi->mac_ops, sizeof(hw->mac.ops)); 1778 hw->mac.type = fi->mac; 1779 1780 /* Setup IOV handlers */ 1781 if (fi->iov_ops) 1782 memcpy(&hw->iov.ops, fi->iov_ops, sizeof(hw->iov.ops)); 1783 1784 /* Set common capability flags and settings */ 1785 rss = min_t(int, FM10K_MAX_RSS_INDICES, num_online_cpus()); 1786 interface->ring_feature[RING_F_RSS].limit = rss; 1787 fi->get_invariants(hw); 1788 1789 /* pick up the PCIe bus settings for reporting later */ 1790 if (hw->mac.ops.get_bus_info) 1791 hw->mac.ops.get_bus_info(hw); 1792 1793 /* limit the usable DMA range */ 1794 if (hw->mac.ops.set_dma_mask) 1795 hw->mac.ops.set_dma_mask(hw, dma_get_mask(&pdev->dev)); 1796 1797 /* update netdev with DMA restrictions */ 1798 if (dma_get_mask(&pdev->dev) > DMA_BIT_MASK(32)) { 1799 netdev->features |= NETIF_F_HIGHDMA; 1800 netdev->vlan_features |= NETIF_F_HIGHDMA; 1801 } 1802 1803 /* delay any future reset requests */ 1804 interface->last_reset = jiffies + (10 * HZ); 1805 1806 /* reset and initialize the hardware so it is in a known state */ 1807 err = hw->mac.ops.reset_hw(hw); 1808 if (err) { 1809 dev_err(&pdev->dev, "reset_hw failed: %d\n", err); 1810 return err; 1811 } 1812 1813 err = hw->mac.ops.init_hw(hw); 1814 if (err) { 1815 dev_err(&pdev->dev, "init_hw failed: %d\n", err); 1816 return err; 1817 } 1818 1819 /* initialize hardware statistics */ 1820 hw->mac.ops.update_hw_stats(hw, &interface->stats); 1821 1822 /* Set upper limit on IOV VFs that can be allocated */ 1823 pci_sriov_set_totalvfs(pdev, hw->iov.total_vfs); 1824 1825 /* Start with random Ethernet address */ 1826 eth_random_addr(hw->mac.addr); 1827 1828 /* Initialize MAC address from hardware */ 1829 err = hw->mac.ops.read_mac_addr(hw); 1830 if (err) { 1831 dev_warn(&pdev->dev, 1832 "Failed to obtain MAC address defaulting to random\n"); 1833 /* tag address assignment as random */ 1834 netdev->addr_assign_type |= NET_ADDR_RANDOM; 1835 } 1836 1837 ether_addr_copy(netdev->dev_addr, hw->mac.addr); 1838 ether_addr_copy(netdev->perm_addr, hw->mac.addr); 1839 1840 if (!is_valid_ether_addr(netdev->perm_addr)) { 1841 dev_err(&pdev->dev, "Invalid MAC Address\n"); 1842 return -EIO; 1843 } 1844 1845 /* initialize DCBNL interface */ 1846 fm10k_dcbnl_set_ops(netdev); 1847 1848 /* set default ring sizes */ 1849 interface->tx_ring_count = FM10K_DEFAULT_TXD; 1850 interface->rx_ring_count = FM10K_DEFAULT_RXD; 1851 1852 /* set default interrupt moderation */ 1853 interface->tx_itr = FM10K_TX_ITR_DEFAULT; 1854 interface->rx_itr = FM10K_ITR_ADAPTIVE | FM10K_RX_ITR_DEFAULT; 1855 1856 /* initialize udp port lists */ 1857 INIT_LIST_HEAD(&interface->vxlan_port); 1858 INIT_LIST_HEAD(&interface->geneve_port); 1859 1860 netdev_rss_key_fill(rss_key, sizeof(rss_key)); 1861 memcpy(interface->rssrk, rss_key, sizeof(rss_key)); 1862 1863 /* Start off interface as being down */ 1864 set_bit(__FM10K_DOWN, interface->state); 1865 set_bit(__FM10K_UPDATING_STATS, interface->state); 1866 1867 return 0; 1868 } 1869 1870 static void fm10k_slot_warn(struct fm10k_intfc *interface) 1871 { 1872 enum pcie_link_width width = PCIE_LNK_WIDTH_UNKNOWN; 1873 enum pci_bus_speed speed = PCI_SPEED_UNKNOWN; 1874 struct fm10k_hw *hw = &interface->hw; 1875 int max_gts = 0, expected_gts = 0; 1876 1877 if (pcie_get_minimum_link(interface->pdev, &speed, &width) || 1878 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { 1879 dev_warn(&interface->pdev->dev, 1880 "Unable to determine PCI Express bandwidth.\n"); 1881 return; 1882 } 1883 1884 switch (speed) { 1885 case PCIE_SPEED_2_5GT: 1886 /* 8b/10b encoding reduces max throughput by 20% */ 1887 max_gts = 2 * width; 1888 break; 1889 case PCIE_SPEED_5_0GT: 1890 /* 8b/10b encoding reduces max throughput by 20% */ 1891 max_gts = 4 * width; 1892 break; 1893 case PCIE_SPEED_8_0GT: 1894 /* 128b/130b encoding has less than 2% impact on throughput */ 1895 max_gts = 8 * width; 1896 break; 1897 default: 1898 dev_warn(&interface->pdev->dev, 1899 "Unable to determine PCI Express bandwidth.\n"); 1900 return; 1901 } 1902 1903 dev_info(&interface->pdev->dev, 1904 "PCI Express bandwidth of %dGT/s available\n", 1905 max_gts); 1906 dev_info(&interface->pdev->dev, 1907 "(Speed:%s, Width: x%d, Encoding Loss:%s, Payload:%s)\n", 1908 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : 1909 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : 1910 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : 1911 "Unknown"), 1912 hw->bus.width, 1913 (speed == PCIE_SPEED_2_5GT ? "20%" : 1914 speed == PCIE_SPEED_5_0GT ? "20%" : 1915 speed == PCIE_SPEED_8_0GT ? "<2%" : 1916 "Unknown"), 1917 (hw->bus.payload == fm10k_bus_payload_128 ? "128B" : 1918 hw->bus.payload == fm10k_bus_payload_256 ? "256B" : 1919 hw->bus.payload == fm10k_bus_payload_512 ? "512B" : 1920 "Unknown")); 1921 1922 switch (hw->bus_caps.speed) { 1923 case fm10k_bus_speed_2500: 1924 /* 8b/10b encoding reduces max throughput by 20% */ 1925 expected_gts = 2 * hw->bus_caps.width; 1926 break; 1927 case fm10k_bus_speed_5000: 1928 /* 8b/10b encoding reduces max throughput by 20% */ 1929 expected_gts = 4 * hw->bus_caps.width; 1930 break; 1931 case fm10k_bus_speed_8000: 1932 /* 128b/130b encoding has less than 2% impact on throughput */ 1933 expected_gts = 8 * hw->bus_caps.width; 1934 break; 1935 default: 1936 dev_warn(&interface->pdev->dev, 1937 "Unable to determine expected PCI Express bandwidth.\n"); 1938 return; 1939 } 1940 1941 if (max_gts >= expected_gts) 1942 return; 1943 1944 dev_warn(&interface->pdev->dev, 1945 "This device requires %dGT/s of bandwidth for optimal performance.\n", 1946 expected_gts); 1947 dev_warn(&interface->pdev->dev, 1948 "A %sslot with x%d lanes is suggested.\n", 1949 (hw->bus_caps.speed == fm10k_bus_speed_2500 ? "2.5GT/s " : 1950 hw->bus_caps.speed == fm10k_bus_speed_5000 ? "5.0GT/s " : 1951 hw->bus_caps.speed == fm10k_bus_speed_8000 ? "8.0GT/s " : ""), 1952 hw->bus_caps.width); 1953 } 1954 1955 /** 1956 * fm10k_probe - Device Initialization Routine 1957 * @pdev: PCI device information struct 1958 * @ent: entry in fm10k_pci_tbl 1959 * 1960 * Returns 0 on success, negative on failure 1961 * 1962 * fm10k_probe initializes an interface identified by a pci_dev structure. 1963 * The OS initialization, configuring of the interface private structure, 1964 * and a hardware reset occur. 1965 **/ 1966 static int fm10k_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 1967 { 1968 struct net_device *netdev; 1969 struct fm10k_intfc *interface; 1970 int err; 1971 1972 if (pdev->error_state != pci_channel_io_normal) { 1973 dev_err(&pdev->dev, 1974 "PCI device still in an error state. Unable to load...\n"); 1975 return -EIO; 1976 } 1977 1978 err = pci_enable_device_mem(pdev); 1979 if (err) { 1980 dev_err(&pdev->dev, 1981 "PCI enable device failed: %d\n", err); 1982 return err; 1983 } 1984 1985 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)); 1986 if (err) 1987 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 1988 if (err) { 1989 dev_err(&pdev->dev, 1990 "DMA configuration failed: %d\n", err); 1991 goto err_dma; 1992 } 1993 1994 err = pci_request_mem_regions(pdev, fm10k_driver_name); 1995 if (err) { 1996 dev_err(&pdev->dev, 1997 "pci_request_selected_regions failed: %d\n", err); 1998 goto err_pci_reg; 1999 } 2000 2001 pci_enable_pcie_error_reporting(pdev); 2002 2003 pci_set_master(pdev); 2004 pci_save_state(pdev); 2005 2006 netdev = fm10k_alloc_netdev(fm10k_info_tbl[ent->driver_data]); 2007 if (!netdev) { 2008 err = -ENOMEM; 2009 goto err_alloc_netdev; 2010 } 2011 2012 SET_NETDEV_DEV(netdev, &pdev->dev); 2013 2014 interface = netdev_priv(netdev); 2015 pci_set_drvdata(pdev, interface); 2016 2017 interface->netdev = netdev; 2018 interface->pdev = pdev; 2019 2020 interface->uc_addr = ioremap(pci_resource_start(pdev, 0), 2021 FM10K_UC_ADDR_SIZE); 2022 if (!interface->uc_addr) { 2023 err = -EIO; 2024 goto err_ioremap; 2025 } 2026 2027 err = fm10k_sw_init(interface, ent); 2028 if (err) 2029 goto err_sw_init; 2030 2031 /* enable debugfs support */ 2032 fm10k_dbg_intfc_init(interface); 2033 2034 err = fm10k_init_queueing_scheme(interface); 2035 if (err) 2036 goto err_sw_init; 2037 2038 /* the mbx interrupt might attempt to schedule the service task, so we 2039 * must ensure it is disabled since we haven't yet requested the timer 2040 * or work item. 2041 */ 2042 set_bit(__FM10K_SERVICE_DISABLE, interface->state); 2043 2044 err = fm10k_mbx_request_irq(interface); 2045 if (err) 2046 goto err_mbx_interrupt; 2047 2048 /* final check of hardware state before registering the interface */ 2049 err = fm10k_hw_ready(interface); 2050 if (err) 2051 goto err_register; 2052 2053 err = register_netdev(netdev); 2054 if (err) 2055 goto err_register; 2056 2057 /* carrier off reporting is important to ethtool even BEFORE open */ 2058 netif_carrier_off(netdev); 2059 2060 /* stop all the transmit queues from transmitting until link is up */ 2061 netif_tx_stop_all_queues(netdev); 2062 2063 /* Initialize service timer and service task late in order to avoid 2064 * cleanup issues. 2065 */ 2066 setup_timer(&interface->service_timer, &fm10k_service_timer, 2067 (unsigned long)interface); 2068 INIT_WORK(&interface->service_task, fm10k_service_task); 2069 2070 /* kick off service timer now, even when interface is down */ 2071 mod_timer(&interface->service_timer, (HZ * 2) + jiffies); 2072 2073 /* print warning for non-optimal configurations */ 2074 fm10k_slot_warn(interface); 2075 2076 /* report MAC address for logging */ 2077 dev_info(&pdev->dev, "%pM\n", netdev->dev_addr); 2078 2079 /* enable SR-IOV after registering netdev to enforce PF/VF ordering */ 2080 fm10k_iov_configure(pdev, 0); 2081 2082 /* clear the service task disable bit to allow service task to start */ 2083 clear_bit(__FM10K_SERVICE_DISABLE, interface->state); 2084 2085 return 0; 2086 2087 err_register: 2088 fm10k_mbx_free_irq(interface); 2089 err_mbx_interrupt: 2090 fm10k_clear_queueing_scheme(interface); 2091 err_sw_init: 2092 if (interface->sw_addr) 2093 iounmap(interface->sw_addr); 2094 iounmap(interface->uc_addr); 2095 err_ioremap: 2096 free_netdev(netdev); 2097 err_alloc_netdev: 2098 pci_release_mem_regions(pdev); 2099 err_pci_reg: 2100 err_dma: 2101 pci_disable_device(pdev); 2102 return err; 2103 } 2104 2105 /** 2106 * fm10k_remove - Device Removal Routine 2107 * @pdev: PCI device information struct 2108 * 2109 * fm10k_remove is called by the PCI subsystem to alert the driver 2110 * that it should release a PCI device. The could be caused by a 2111 * Hot-Plug event, or because the driver is going to be removed from 2112 * memory. 2113 **/ 2114 static void fm10k_remove(struct pci_dev *pdev) 2115 { 2116 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2117 struct net_device *netdev = interface->netdev; 2118 2119 del_timer_sync(&interface->service_timer); 2120 2121 set_bit(__FM10K_SERVICE_DISABLE, interface->state); 2122 cancel_work_sync(&interface->service_task); 2123 2124 /* free netdev, this may bounce the interrupts due to setup_tc */ 2125 if (netdev->reg_state == NETREG_REGISTERED) 2126 unregister_netdev(netdev); 2127 2128 /* release VFs */ 2129 fm10k_iov_disable(pdev); 2130 2131 /* disable mailbox interrupt */ 2132 fm10k_mbx_free_irq(interface); 2133 2134 /* free interrupts */ 2135 fm10k_clear_queueing_scheme(interface); 2136 2137 /* remove any debugfs interfaces */ 2138 fm10k_dbg_intfc_exit(interface); 2139 2140 if (interface->sw_addr) 2141 iounmap(interface->sw_addr); 2142 iounmap(interface->uc_addr); 2143 2144 free_netdev(netdev); 2145 2146 pci_release_mem_regions(pdev); 2147 2148 pci_disable_pcie_error_reporting(pdev); 2149 2150 pci_disable_device(pdev); 2151 } 2152 2153 static void fm10k_prepare_suspend(struct fm10k_intfc *interface) 2154 { 2155 /* the watchdog task reads from registers, which might appear like 2156 * a surprise remove if the PCIe device is disabled while we're 2157 * stopped. We stop the watchdog task until after we resume software 2158 * activity. 2159 */ 2160 set_bit(__FM10K_SERVICE_DISABLE, interface->state); 2161 cancel_work_sync(&interface->service_task); 2162 2163 fm10k_prepare_for_reset(interface); 2164 } 2165 2166 static int fm10k_handle_resume(struct fm10k_intfc *interface) 2167 { 2168 struct fm10k_hw *hw = &interface->hw; 2169 int err; 2170 2171 /* reset statistics starting values */ 2172 hw->mac.ops.rebind_hw_stats(hw, &interface->stats); 2173 2174 err = fm10k_handle_reset(interface); 2175 if (err) 2176 return err; 2177 2178 /* assume host is not ready, to prevent race with watchdog in case we 2179 * actually don't have connection to the switch 2180 */ 2181 interface->host_ready = false; 2182 fm10k_watchdog_host_not_ready(interface); 2183 2184 /* force link to stay down for a second to prevent link flutter */ 2185 interface->link_down_event = jiffies + (HZ); 2186 set_bit(__FM10K_LINK_DOWN, interface->state); 2187 2188 /* clear the service task disable bit to allow service task to start */ 2189 clear_bit(__FM10K_SERVICE_DISABLE, interface->state); 2190 fm10k_service_event_schedule(interface); 2191 2192 return err; 2193 } 2194 2195 #ifdef CONFIG_PM 2196 /** 2197 * fm10k_resume - Restore device to pre-sleep state 2198 * @pdev: PCI device information struct 2199 * 2200 * fm10k_resume is called after the system has powered back up from a sleep 2201 * state and is ready to resume operation. This function is meant to restore 2202 * the device back to its pre-sleep state. 2203 **/ 2204 static int fm10k_resume(struct pci_dev *pdev) 2205 { 2206 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2207 struct net_device *netdev = interface->netdev; 2208 struct fm10k_hw *hw = &interface->hw; 2209 u32 err; 2210 2211 pci_set_power_state(pdev, PCI_D0); 2212 pci_restore_state(pdev); 2213 2214 /* pci_restore_state clears dev->state_saved so call 2215 * pci_save_state to restore it. 2216 */ 2217 pci_save_state(pdev); 2218 2219 err = pci_enable_device_mem(pdev); 2220 if (err) { 2221 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 2222 return err; 2223 } 2224 pci_set_master(pdev); 2225 2226 pci_wake_from_d3(pdev, false); 2227 2228 /* refresh hw_addr in case it was dropped */ 2229 hw->hw_addr = interface->uc_addr; 2230 2231 err = fm10k_handle_resume(interface); 2232 if (err) 2233 return err; 2234 2235 netif_device_attach(netdev); 2236 2237 return 0; 2238 } 2239 2240 /** 2241 * fm10k_suspend - Prepare the device for a system sleep state 2242 * @pdev: PCI device information struct 2243 * 2244 * fm10k_suspend is meant to shutdown the device prior to the system entering 2245 * a sleep state. The fm10k hardware does not support wake on lan so the 2246 * driver simply needs to shut down the device so it is in a low power state. 2247 **/ 2248 static int fm10k_suspend(struct pci_dev *pdev, 2249 pm_message_t __always_unused state) 2250 { 2251 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2252 struct net_device *netdev = interface->netdev; 2253 int err = 0; 2254 2255 netif_device_detach(netdev); 2256 2257 fm10k_prepare_suspend(interface); 2258 2259 err = pci_save_state(pdev); 2260 if (err) 2261 return err; 2262 2263 pci_disable_device(pdev); 2264 pci_wake_from_d3(pdev, false); 2265 pci_set_power_state(pdev, PCI_D3hot); 2266 2267 return 0; 2268 } 2269 2270 #endif /* CONFIG_PM */ 2271 /** 2272 * fm10k_io_error_detected - called when PCI error is detected 2273 * @pdev: Pointer to PCI device 2274 * @state: The current pci connection state 2275 * 2276 * This function is called after a PCI bus error affecting 2277 * this device has been detected. 2278 */ 2279 static pci_ers_result_t fm10k_io_error_detected(struct pci_dev *pdev, 2280 pci_channel_state_t state) 2281 { 2282 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2283 struct net_device *netdev = interface->netdev; 2284 2285 netif_device_detach(netdev); 2286 2287 if (state == pci_channel_io_perm_failure) 2288 return PCI_ERS_RESULT_DISCONNECT; 2289 2290 fm10k_prepare_suspend(interface); 2291 2292 /* Request a slot reset. */ 2293 return PCI_ERS_RESULT_NEED_RESET; 2294 } 2295 2296 /** 2297 * fm10k_io_slot_reset - called after the pci bus has been reset. 2298 * @pdev: Pointer to PCI device 2299 * 2300 * Restart the card from scratch, as if from a cold-boot. 2301 */ 2302 static pci_ers_result_t fm10k_io_slot_reset(struct pci_dev *pdev) 2303 { 2304 pci_ers_result_t result; 2305 2306 if (pci_reenable_device(pdev)) { 2307 dev_err(&pdev->dev, 2308 "Cannot re-enable PCI device after reset.\n"); 2309 result = PCI_ERS_RESULT_DISCONNECT; 2310 } else { 2311 pci_set_master(pdev); 2312 pci_restore_state(pdev); 2313 2314 /* After second error pci->state_saved is false, this 2315 * resets it so EEH doesn't break. 2316 */ 2317 pci_save_state(pdev); 2318 2319 pci_wake_from_d3(pdev, false); 2320 2321 result = PCI_ERS_RESULT_RECOVERED; 2322 } 2323 2324 pci_cleanup_aer_uncorrect_error_status(pdev); 2325 2326 return result; 2327 } 2328 2329 /** 2330 * fm10k_io_resume - called when traffic can start flowing again. 2331 * @pdev: Pointer to PCI device 2332 * 2333 * This callback is called when the error recovery driver tells us that 2334 * its OK to resume normal operation. 2335 */ 2336 static void fm10k_io_resume(struct pci_dev *pdev) 2337 { 2338 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2339 struct net_device *netdev = interface->netdev; 2340 int err; 2341 2342 err = fm10k_handle_resume(interface); 2343 2344 if (err) 2345 dev_warn(&pdev->dev, 2346 "fm10k_io_resume failed: %d\n", err); 2347 else 2348 netif_device_attach(netdev); 2349 } 2350 2351 static void fm10k_io_reset_prepare(struct pci_dev *pdev) 2352 { 2353 /* warn incase we have any active VF devices */ 2354 if (pci_num_vf(pdev)) 2355 dev_warn(&pdev->dev, 2356 "PCIe FLR may cause issues for any active VF devices\n"); 2357 fm10k_prepare_suspend(pci_get_drvdata(pdev)); 2358 } 2359 2360 static void fm10k_io_reset_done(struct pci_dev *pdev) 2361 { 2362 struct fm10k_intfc *interface = pci_get_drvdata(pdev); 2363 int err = fm10k_handle_resume(interface); 2364 2365 if (err) { 2366 dev_warn(&pdev->dev, 2367 "fm10k_io_reset_notify failed: %d\n", err); 2368 netif_device_detach(interface->netdev); 2369 } 2370 } 2371 2372 static const struct pci_error_handlers fm10k_err_handler = { 2373 .error_detected = fm10k_io_error_detected, 2374 .slot_reset = fm10k_io_slot_reset, 2375 .resume = fm10k_io_resume, 2376 .reset_prepare = fm10k_io_reset_prepare, 2377 .reset_done = fm10k_io_reset_done, 2378 }; 2379 2380 static struct pci_driver fm10k_driver = { 2381 .name = fm10k_driver_name, 2382 .id_table = fm10k_pci_tbl, 2383 .probe = fm10k_probe, 2384 .remove = fm10k_remove, 2385 #ifdef CONFIG_PM 2386 .suspend = fm10k_suspend, 2387 .resume = fm10k_resume, 2388 #endif 2389 .sriov_configure = fm10k_iov_configure, 2390 .err_handler = &fm10k_err_handler 2391 }; 2392 2393 /** 2394 * fm10k_register_pci_driver - register driver interface 2395 * 2396 * This function is called on module load in order to register the driver. 2397 **/ 2398 int fm10k_register_pci_driver(void) 2399 { 2400 return pci_register_driver(&fm10k_driver); 2401 } 2402 2403 /** 2404 * fm10k_unregister_pci_driver - unregister driver interface 2405 * 2406 * This function is called on module unload in order to remove the driver. 2407 **/ 2408 void fm10k_unregister_pci_driver(void) 2409 { 2410 pci_unregister_driver(&fm10k_driver); 2411 } 2412