1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, Intel Corporation. */ 3 4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include "ice.h" 9 #include "ice_lib.h" 10 11 #define DRV_VERSION "0.7.2-k" 12 #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver" 13 const char ice_drv_ver[] = DRV_VERSION; 14 static const char ice_driver_string[] = DRV_SUMMARY; 15 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation."; 16 17 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 18 MODULE_DESCRIPTION(DRV_SUMMARY); 19 MODULE_LICENSE("GPL v2"); 20 MODULE_VERSION(DRV_VERSION); 21 22 static int debug = -1; 23 module_param(debug, int, 0644); 24 #ifndef CONFIG_DYNAMIC_DEBUG 25 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)"); 26 #else 27 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)"); 28 #endif /* !CONFIG_DYNAMIC_DEBUG */ 29 30 static struct workqueue_struct *ice_wq; 31 static const struct net_device_ops ice_netdev_ops; 32 33 static void ice_pf_dis_all_vsi(struct ice_pf *pf); 34 static void ice_rebuild(struct ice_pf *pf); 35 36 static void ice_vsi_release_all(struct ice_pf *pf); 37 static void ice_update_vsi_stats(struct ice_vsi *vsi); 38 static void ice_update_pf_stats(struct ice_pf *pf); 39 40 /** 41 * ice_get_tx_pending - returns number of Tx descriptors not processed 42 * @ring: the ring of descriptors 43 */ 44 static u32 ice_get_tx_pending(struct ice_ring *ring) 45 { 46 u32 head, tail; 47 48 head = ring->next_to_clean; 49 tail = readl(ring->tail); 50 51 if (head != tail) 52 return (head < tail) ? 53 tail - head : (tail + ring->count - head); 54 return 0; 55 } 56 57 /** 58 * ice_check_for_hang_subtask - check for and recover hung queues 59 * @pf: pointer to PF struct 60 */ 61 static void ice_check_for_hang_subtask(struct ice_pf *pf) 62 { 63 struct ice_vsi *vsi = NULL; 64 unsigned int i; 65 u32 v, v_idx; 66 int packets; 67 68 ice_for_each_vsi(pf, v) 69 if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) { 70 vsi = pf->vsi[v]; 71 break; 72 } 73 74 if (!vsi || test_bit(__ICE_DOWN, vsi->state)) 75 return; 76 77 if (!(vsi->netdev && netif_carrier_ok(vsi->netdev))) 78 return; 79 80 for (i = 0; i < vsi->num_txq; i++) { 81 struct ice_ring *tx_ring = vsi->tx_rings[i]; 82 83 if (tx_ring && tx_ring->desc) { 84 int itr = ICE_ITR_NONE; 85 86 /* If packet counter has not changed the queue is 87 * likely stalled, so force an interrupt for this 88 * queue. 89 * 90 * prev_pkt would be negative if there was no 91 * pending work. 92 */ 93 packets = tx_ring->stats.pkts & INT_MAX; 94 if (tx_ring->tx_stats.prev_pkt == packets) { 95 /* Trigger sw interrupt to revive the queue */ 96 v_idx = tx_ring->q_vector->v_idx; 97 wr32(&vsi->back->hw, 98 GLINT_DYN_CTL(vsi->hw_base_vector + v_idx), 99 (itr << GLINT_DYN_CTL_ITR_INDX_S) | 100 GLINT_DYN_CTL_SWINT_TRIG_M | 101 GLINT_DYN_CTL_INTENA_MSK_M); 102 continue; 103 } 104 105 /* Memory barrier between read of packet count and call 106 * to ice_get_tx_pending() 107 */ 108 smp_rmb(); 109 tx_ring->tx_stats.prev_pkt = 110 ice_get_tx_pending(tx_ring) ? packets : -1; 111 } 112 } 113 } 114 115 /** 116 * ice_add_mac_to_sync_list - creates list of mac addresses to be synced 117 * @netdev: the net device on which the sync is happening 118 * @addr: mac address to sync 119 * 120 * This is a callback function which is called by the in kernel device sync 121 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only 122 * populates the tmp_sync_list, which is later used by ice_add_mac to add the 123 * mac filters from the hardware. 124 */ 125 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr) 126 { 127 struct ice_netdev_priv *np = netdev_priv(netdev); 128 struct ice_vsi *vsi = np->vsi; 129 130 if (ice_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr)) 131 return -EINVAL; 132 133 return 0; 134 } 135 136 /** 137 * ice_add_mac_to_unsync_list - creates list of mac addresses to be unsynced 138 * @netdev: the net device on which the unsync is happening 139 * @addr: mac address to unsync 140 * 141 * This is a callback function which is called by the in kernel device unsync 142 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only 143 * populates the tmp_unsync_list, which is later used by ice_remove_mac to 144 * delete the mac filters from the hardware. 145 */ 146 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) 147 { 148 struct ice_netdev_priv *np = netdev_priv(netdev); 149 struct ice_vsi *vsi = np->vsi; 150 151 if (ice_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr)) 152 return -EINVAL; 153 154 return 0; 155 } 156 157 /** 158 * ice_vsi_fltr_changed - check if filter state changed 159 * @vsi: VSI to be checked 160 * 161 * returns true if filter state has changed, false otherwise. 162 */ 163 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi) 164 { 165 return test_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags) || 166 test_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags) || 167 test_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 168 } 169 170 /** 171 * ice_cfg_promisc - Enable or disable promiscuous mode for a given PF 172 * @vsi: the VSI being configured 173 * @promisc_m: mask of promiscuous config bits 174 * @set_promisc: enable or disable promisc flag request 175 * 176 */ 177 static int ice_cfg_promisc(struct ice_vsi *vsi, u8 promisc_m, bool set_promisc) 178 { 179 struct ice_hw *hw = &vsi->back->hw; 180 enum ice_status status = 0; 181 182 if (vsi->type != ICE_VSI_PF) 183 return 0; 184 185 if (vsi->vlan_ena) { 186 status = ice_set_vlan_vsi_promisc(hw, vsi->idx, promisc_m, 187 set_promisc); 188 } else { 189 if (set_promisc) 190 status = ice_set_vsi_promisc(hw, vsi->idx, promisc_m, 191 0); 192 else 193 status = ice_clear_vsi_promisc(hw, vsi->idx, promisc_m, 194 0); 195 } 196 197 if (status) 198 return -EIO; 199 200 return 0; 201 } 202 203 /** 204 * ice_vsi_sync_fltr - Update the VSI filter list to the HW 205 * @vsi: ptr to the VSI 206 * 207 * Push any outstanding VSI filter changes through the AdminQ. 208 */ 209 static int ice_vsi_sync_fltr(struct ice_vsi *vsi) 210 { 211 struct device *dev = &vsi->back->pdev->dev; 212 struct net_device *netdev = vsi->netdev; 213 bool promisc_forced_on = false; 214 struct ice_pf *pf = vsi->back; 215 struct ice_hw *hw = &pf->hw; 216 enum ice_status status = 0; 217 u32 changed_flags = 0; 218 u8 promisc_m; 219 int err = 0; 220 221 if (!vsi->netdev) 222 return -EINVAL; 223 224 while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state)) 225 usleep_range(1000, 2000); 226 227 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 228 vsi->current_netdev_flags = vsi->netdev->flags; 229 230 INIT_LIST_HEAD(&vsi->tmp_sync_list); 231 INIT_LIST_HEAD(&vsi->tmp_unsync_list); 232 233 if (ice_vsi_fltr_changed(vsi)) { 234 clear_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 235 clear_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 236 clear_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 237 238 /* grab the netdev's addr_list_lock */ 239 netif_addr_lock_bh(netdev); 240 __dev_uc_sync(netdev, ice_add_mac_to_sync_list, 241 ice_add_mac_to_unsync_list); 242 __dev_mc_sync(netdev, ice_add_mac_to_sync_list, 243 ice_add_mac_to_unsync_list); 244 /* our temp lists are populated. release lock */ 245 netif_addr_unlock_bh(netdev); 246 } 247 248 /* Remove mac addresses in the unsync list */ 249 status = ice_remove_mac(hw, &vsi->tmp_unsync_list); 250 ice_free_fltr_list(dev, &vsi->tmp_unsync_list); 251 if (status) { 252 netdev_err(netdev, "Failed to delete MAC filters\n"); 253 /* if we failed because of alloc failures, just bail */ 254 if (status == ICE_ERR_NO_MEMORY) { 255 err = -ENOMEM; 256 goto out; 257 } 258 } 259 260 /* Add mac addresses in the sync list */ 261 status = ice_add_mac(hw, &vsi->tmp_sync_list); 262 ice_free_fltr_list(dev, &vsi->tmp_sync_list); 263 /* If filter is added successfully or already exists, do not go into 264 * 'if' condition and report it as error. Instead continue processing 265 * rest of the function. 266 */ 267 if (status && status != ICE_ERR_ALREADY_EXISTS) { 268 netdev_err(netdev, "Failed to add MAC filters\n"); 269 /* If there is no more space for new umac filters, vsi 270 * should go into promiscuous mode. There should be some 271 * space reserved for promiscuous filters. 272 */ 273 if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC && 274 !test_and_set_bit(__ICE_FLTR_OVERFLOW_PROMISC, 275 vsi->state)) { 276 promisc_forced_on = true; 277 netdev_warn(netdev, 278 "Reached MAC filter limit, forcing promisc mode on VSI %d\n", 279 vsi->vsi_num); 280 } else { 281 err = -EIO; 282 goto out; 283 } 284 } 285 /* check for changes in promiscuous modes */ 286 if (changed_flags & IFF_ALLMULTI) { 287 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 288 if (vsi->vlan_ena) 289 promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; 290 else 291 promisc_m = ICE_MCAST_PROMISC_BITS; 292 293 err = ice_cfg_promisc(vsi, promisc_m, true); 294 if (err) { 295 netdev_err(netdev, "Error setting Multicast promiscuous mode on VSI %i\n", 296 vsi->vsi_num); 297 vsi->current_netdev_flags &= ~IFF_ALLMULTI; 298 goto out_promisc; 299 } 300 } else if (!(vsi->current_netdev_flags & IFF_ALLMULTI)) { 301 if (vsi->vlan_ena) 302 promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; 303 else 304 promisc_m = ICE_MCAST_PROMISC_BITS; 305 306 err = ice_cfg_promisc(vsi, promisc_m, false); 307 if (err) { 308 netdev_err(netdev, "Error clearing Multicast promiscuous mode on VSI %i\n", 309 vsi->vsi_num); 310 vsi->current_netdev_flags |= IFF_ALLMULTI; 311 goto out_promisc; 312 } 313 } 314 } 315 316 if (((changed_flags & IFF_PROMISC) || promisc_forced_on) || 317 test_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags)) { 318 clear_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags); 319 if (vsi->current_netdev_flags & IFF_PROMISC) { 320 /* Apply TX filter rule to get traffic from VMs */ 321 status = ice_cfg_dflt_vsi(hw, vsi->idx, true, 322 ICE_FLTR_TX); 323 if (status) { 324 netdev_err(netdev, "Error setting default VSI %i tx rule\n", 325 vsi->vsi_num); 326 vsi->current_netdev_flags &= ~IFF_PROMISC; 327 err = -EIO; 328 goto out_promisc; 329 } 330 /* Apply RX filter rule to get traffic from wire */ 331 status = ice_cfg_dflt_vsi(hw, vsi->idx, true, 332 ICE_FLTR_RX); 333 if (status) { 334 netdev_err(netdev, "Error setting default VSI %i rx rule\n", 335 vsi->vsi_num); 336 vsi->current_netdev_flags &= ~IFF_PROMISC; 337 err = -EIO; 338 goto out_promisc; 339 } 340 } else { 341 /* Clear TX filter rule to stop traffic from VMs */ 342 status = ice_cfg_dflt_vsi(hw, vsi->idx, false, 343 ICE_FLTR_TX); 344 if (status) { 345 netdev_err(netdev, "Error clearing default VSI %i tx rule\n", 346 vsi->vsi_num); 347 vsi->current_netdev_flags |= IFF_PROMISC; 348 err = -EIO; 349 goto out_promisc; 350 } 351 /* Clear RX filter to remove traffic from wire */ 352 status = ice_cfg_dflt_vsi(hw, vsi->idx, false, 353 ICE_FLTR_RX); 354 if (status) { 355 netdev_err(netdev, "Error clearing default VSI %i rx rule\n", 356 vsi->vsi_num); 357 vsi->current_netdev_flags |= IFF_PROMISC; 358 err = -EIO; 359 goto out_promisc; 360 } 361 } 362 } 363 goto exit; 364 365 out_promisc: 366 set_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags); 367 goto exit; 368 out: 369 /* if something went wrong then set the changed flag so we try again */ 370 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 371 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 372 exit: 373 clear_bit(__ICE_CFG_BUSY, vsi->state); 374 return err; 375 } 376 377 /** 378 * ice_sync_fltr_subtask - Sync the VSI filter list with HW 379 * @pf: board private structure 380 */ 381 static void ice_sync_fltr_subtask(struct ice_pf *pf) 382 { 383 int v; 384 385 if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags))) 386 return; 387 388 clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 389 390 ice_for_each_vsi(pf, v) 391 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) && 392 ice_vsi_sync_fltr(pf->vsi[v])) { 393 /* come back and try again later */ 394 set_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 395 break; 396 } 397 } 398 399 /** 400 * ice_prepare_for_reset - prep for the core to reset 401 * @pf: board private structure 402 * 403 * Inform or close all dependent features in prep for reset. 404 */ 405 static void 406 ice_prepare_for_reset(struct ice_pf *pf) 407 { 408 struct ice_hw *hw = &pf->hw; 409 410 /* already prepared for reset */ 411 if (test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) 412 return; 413 414 /* Notify VFs of impending reset */ 415 if (ice_check_sq_alive(hw, &hw->mailboxq)) 416 ice_vc_notify_reset(pf); 417 418 /* disable the VSIs and their queues that are not already DOWN */ 419 ice_pf_dis_all_vsi(pf); 420 421 if (hw->port_info) 422 ice_sched_clear_port(hw->port_info); 423 424 ice_shutdown_all_ctrlq(hw); 425 426 set_bit(__ICE_PREPARED_FOR_RESET, pf->state); 427 } 428 429 /** 430 * ice_do_reset - Initiate one of many types of resets 431 * @pf: board private structure 432 * @reset_type: reset type requested 433 * before this function was called. 434 */ 435 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 436 { 437 struct device *dev = &pf->pdev->dev; 438 struct ice_hw *hw = &pf->hw; 439 440 dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); 441 WARN_ON(in_interrupt()); 442 443 ice_prepare_for_reset(pf); 444 445 /* trigger the reset */ 446 if (ice_reset(hw, reset_type)) { 447 dev_err(dev, "reset %d failed\n", reset_type); 448 set_bit(__ICE_RESET_FAILED, pf->state); 449 clear_bit(__ICE_RESET_OICR_RECV, pf->state); 450 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 451 clear_bit(__ICE_PFR_REQ, pf->state); 452 clear_bit(__ICE_CORER_REQ, pf->state); 453 clear_bit(__ICE_GLOBR_REQ, pf->state); 454 return; 455 } 456 457 /* PFR is a bit of a special case because it doesn't result in an OICR 458 * interrupt. So for PFR, rebuild after the reset and clear the reset- 459 * associated state bits. 460 */ 461 if (reset_type == ICE_RESET_PFR) { 462 pf->pfr_count++; 463 ice_rebuild(pf); 464 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 465 clear_bit(__ICE_PFR_REQ, pf->state); 466 ice_reset_all_vfs(pf, true); 467 } 468 } 469 470 /** 471 * ice_reset_subtask - Set up for resetting the device and driver 472 * @pf: board private structure 473 */ 474 static void ice_reset_subtask(struct ice_pf *pf) 475 { 476 enum ice_reset_req reset_type = ICE_RESET_INVAL; 477 478 /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an 479 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type 480 * of reset is pending and sets bits in pf->state indicating the reset 481 * type and __ICE_RESET_OICR_RECV. So, if the latter bit is set 482 * prepare for pending reset if not already (for PF software-initiated 483 * global resets the software should already be prepared for it as 484 * indicated by __ICE_PREPARED_FOR_RESET; for global resets initiated 485 * by firmware or software on other PFs, that bit is not set so prepare 486 * for the reset now), poll for reset done, rebuild and return. 487 */ 488 if (test_bit(__ICE_RESET_OICR_RECV, pf->state)) { 489 /* Perform the largest reset requested */ 490 if (test_and_clear_bit(__ICE_CORER_RECV, pf->state)) 491 reset_type = ICE_RESET_CORER; 492 if (test_and_clear_bit(__ICE_GLOBR_RECV, pf->state)) 493 reset_type = ICE_RESET_GLOBR; 494 /* return if no valid reset type requested */ 495 if (reset_type == ICE_RESET_INVAL) 496 return; 497 ice_prepare_for_reset(pf); 498 499 /* make sure we are ready to rebuild */ 500 if (ice_check_reset(&pf->hw)) { 501 set_bit(__ICE_RESET_FAILED, pf->state); 502 } else { 503 /* done with reset. start rebuild */ 504 pf->hw.reset_ongoing = false; 505 ice_rebuild(pf); 506 /* clear bit to resume normal operations, but 507 * ICE_NEEDS_RESTART bit is set incase rebuild failed 508 */ 509 clear_bit(__ICE_RESET_OICR_RECV, pf->state); 510 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 511 clear_bit(__ICE_PFR_REQ, pf->state); 512 clear_bit(__ICE_CORER_REQ, pf->state); 513 clear_bit(__ICE_GLOBR_REQ, pf->state); 514 ice_reset_all_vfs(pf, true); 515 } 516 517 return; 518 } 519 520 /* No pending resets to finish processing. Check for new resets */ 521 if (test_bit(__ICE_PFR_REQ, pf->state)) 522 reset_type = ICE_RESET_PFR; 523 if (test_bit(__ICE_CORER_REQ, pf->state)) 524 reset_type = ICE_RESET_CORER; 525 if (test_bit(__ICE_GLOBR_REQ, pf->state)) 526 reset_type = ICE_RESET_GLOBR; 527 /* If no valid reset type requested just return */ 528 if (reset_type == ICE_RESET_INVAL) 529 return; 530 531 /* reset if not already down or busy */ 532 if (!test_bit(__ICE_DOWN, pf->state) && 533 !test_bit(__ICE_CFG_BUSY, pf->state)) { 534 ice_do_reset(pf, reset_type); 535 } 536 } 537 538 /** 539 * ice_print_link_msg - print link up or down message 540 * @vsi: the VSI whose link status is being queried 541 * @isup: boolean for if the link is now up or down 542 */ 543 void ice_print_link_msg(struct ice_vsi *vsi, bool isup) 544 { 545 const char *speed; 546 const char *fc; 547 548 if (vsi->current_isup == isup) 549 return; 550 551 vsi->current_isup = isup; 552 553 if (!isup) { 554 netdev_info(vsi->netdev, "NIC Link is Down\n"); 555 return; 556 } 557 558 switch (vsi->port_info->phy.link_info.link_speed) { 559 case ICE_AQ_LINK_SPEED_40GB: 560 speed = "40 G"; 561 break; 562 case ICE_AQ_LINK_SPEED_25GB: 563 speed = "25 G"; 564 break; 565 case ICE_AQ_LINK_SPEED_20GB: 566 speed = "20 G"; 567 break; 568 case ICE_AQ_LINK_SPEED_10GB: 569 speed = "10 G"; 570 break; 571 case ICE_AQ_LINK_SPEED_5GB: 572 speed = "5 G"; 573 break; 574 case ICE_AQ_LINK_SPEED_2500MB: 575 speed = "2.5 G"; 576 break; 577 case ICE_AQ_LINK_SPEED_1000MB: 578 speed = "1 G"; 579 break; 580 case ICE_AQ_LINK_SPEED_100MB: 581 speed = "100 M"; 582 break; 583 default: 584 speed = "Unknown"; 585 break; 586 } 587 588 switch (vsi->port_info->fc.current_mode) { 589 case ICE_FC_FULL: 590 fc = "RX/TX"; 591 break; 592 case ICE_FC_TX_PAUSE: 593 fc = "TX"; 594 break; 595 case ICE_FC_RX_PAUSE: 596 fc = "RX"; 597 break; 598 case ICE_FC_NONE: 599 fc = "None"; 600 break; 601 default: 602 fc = "Unknown"; 603 break; 604 } 605 606 netdev_info(vsi->netdev, "NIC Link is up %sbps, Flow Control: %s\n", 607 speed, fc); 608 } 609 610 /** 611 * ice_vsi_link_event - update the vsi's netdev 612 * @vsi: the vsi on which the link event occurred 613 * @link_up: whether or not the vsi needs to be set up or down 614 */ 615 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up) 616 { 617 if (!vsi || test_bit(__ICE_DOWN, vsi->state)) 618 return; 619 620 if (vsi->type == ICE_VSI_PF) { 621 if (!vsi->netdev) { 622 dev_dbg(&vsi->back->pdev->dev, 623 "vsi->netdev is not initialized!\n"); 624 return; 625 } 626 if (link_up) { 627 netif_carrier_on(vsi->netdev); 628 netif_tx_wake_all_queues(vsi->netdev); 629 } else { 630 netif_carrier_off(vsi->netdev); 631 netif_tx_stop_all_queues(vsi->netdev); 632 } 633 } 634 } 635 636 /** 637 * ice_link_event - process the link event 638 * @pf: pf that the link event is associated with 639 * @pi: port_info for the port that the link event is associated with 640 * 641 * Returns -EIO if ice_get_link_status() fails 642 * Returns 0 on success 643 */ 644 static int 645 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi) 646 { 647 u8 new_link_speed, old_link_speed; 648 struct ice_phy_info *phy_info; 649 bool new_link_same_as_old; 650 bool new_link, old_link; 651 u8 lport; 652 u16 v; 653 654 phy_info = &pi->phy; 655 phy_info->link_info_old = phy_info->link_info; 656 /* Force ice_get_link_status() to update link info */ 657 phy_info->get_link_info = true; 658 659 old_link = (phy_info->link_info_old.link_info & ICE_AQ_LINK_UP); 660 old_link_speed = phy_info->link_info_old.link_speed; 661 662 lport = pi->lport; 663 if (ice_get_link_status(pi, &new_link)) { 664 dev_dbg(&pf->pdev->dev, 665 "Could not get link status for port %d\n", lport); 666 return -EIO; 667 } 668 669 new_link_speed = phy_info->link_info.link_speed; 670 671 new_link_same_as_old = (new_link == old_link && 672 new_link_speed == old_link_speed); 673 674 ice_for_each_vsi(pf, v) { 675 struct ice_vsi *vsi = pf->vsi[v]; 676 677 if (!vsi || !vsi->port_info) 678 continue; 679 680 if (new_link_same_as_old && 681 (test_bit(__ICE_DOWN, vsi->state) || 682 new_link == netif_carrier_ok(vsi->netdev))) 683 continue; 684 685 if (vsi->port_info->lport == lport) { 686 ice_print_link_msg(vsi, new_link); 687 ice_vsi_link_event(vsi, new_link); 688 } 689 } 690 691 if (!new_link_same_as_old && pf->num_alloc_vfs) 692 ice_vc_notify_link_state(pf); 693 694 return 0; 695 } 696 697 /** 698 * ice_watchdog_subtask - periodic tasks not using event driven scheduling 699 * @pf: board private structure 700 */ 701 static void ice_watchdog_subtask(struct ice_pf *pf) 702 { 703 int i; 704 705 /* if interface is down do nothing */ 706 if (test_bit(__ICE_DOWN, pf->state) || 707 test_bit(__ICE_CFG_BUSY, pf->state)) 708 return; 709 710 /* make sure we don't do these things too often */ 711 if (time_before(jiffies, 712 pf->serv_tmr_prev + pf->serv_tmr_period)) 713 return; 714 715 pf->serv_tmr_prev = jiffies; 716 717 /* Update the stats for active netdevs so the network stack 718 * can look at updated numbers whenever it cares to 719 */ 720 ice_update_pf_stats(pf); 721 ice_for_each_vsi(pf, i) 722 if (pf->vsi[i] && pf->vsi[i]->netdev) 723 ice_update_vsi_stats(pf->vsi[i]); 724 } 725 726 /** 727 * ice_init_link_events - enable/initialize link events 728 * @pi: pointer to the port_info instance 729 * 730 * Returns -EIO on failure, 0 on success 731 */ 732 static int ice_init_link_events(struct ice_port_info *pi) 733 { 734 u16 mask; 735 736 mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA | 737 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL)); 738 739 if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) { 740 dev_dbg(ice_hw_to_dev(pi->hw), 741 "Failed to set link event mask for port %d\n", 742 pi->lport); 743 return -EIO; 744 } 745 746 if (ice_aq_get_link_info(pi, true, NULL, NULL)) { 747 dev_dbg(ice_hw_to_dev(pi->hw), 748 "Failed to enable link events for port %d\n", 749 pi->lport); 750 return -EIO; 751 } 752 753 return 0; 754 } 755 756 /** 757 * ice_handle_link_event - handle link event via ARQ 758 * @pf: pf that the link event is associated with 759 * 760 * Return -EINVAL if port_info is null 761 * Return status on success 762 */ 763 static int ice_handle_link_event(struct ice_pf *pf) 764 { 765 struct ice_port_info *port_info; 766 int status; 767 768 port_info = pf->hw.port_info; 769 if (!port_info) 770 return -EINVAL; 771 772 status = ice_link_event(pf, port_info); 773 if (status) 774 dev_dbg(&pf->pdev->dev, 775 "Could not process link event, error %d\n", status); 776 777 return status; 778 } 779 780 /** 781 * __ice_clean_ctrlq - helper function to clean controlq rings 782 * @pf: ptr to struct ice_pf 783 * @q_type: specific Control queue type 784 */ 785 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) 786 { 787 struct ice_rq_event_info event; 788 struct ice_hw *hw = &pf->hw; 789 struct ice_ctl_q_info *cq; 790 u16 pending, i = 0; 791 const char *qtype; 792 u32 oldval, val; 793 794 /* Do not clean control queue if/when PF reset fails */ 795 if (test_bit(__ICE_RESET_FAILED, pf->state)) 796 return 0; 797 798 switch (q_type) { 799 case ICE_CTL_Q_ADMIN: 800 cq = &hw->adminq; 801 qtype = "Admin"; 802 break; 803 case ICE_CTL_Q_MAILBOX: 804 cq = &hw->mailboxq; 805 qtype = "Mailbox"; 806 break; 807 default: 808 dev_warn(&pf->pdev->dev, "Unknown control queue type 0x%x\n", 809 q_type); 810 return 0; 811 } 812 813 /* check for error indications - PF_xx_AxQLEN register layout for 814 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN. 815 */ 816 val = rd32(hw, cq->rq.len); 817 if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 818 PF_FW_ARQLEN_ARQCRIT_M)) { 819 oldval = val; 820 if (val & PF_FW_ARQLEN_ARQVFE_M) 821 dev_dbg(&pf->pdev->dev, 822 "%s Receive Queue VF Error detected\n", qtype); 823 if (val & PF_FW_ARQLEN_ARQOVFL_M) { 824 dev_dbg(&pf->pdev->dev, 825 "%s Receive Queue Overflow Error detected\n", 826 qtype); 827 } 828 if (val & PF_FW_ARQLEN_ARQCRIT_M) 829 dev_dbg(&pf->pdev->dev, 830 "%s Receive Queue Critical Error detected\n", 831 qtype); 832 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 833 PF_FW_ARQLEN_ARQCRIT_M); 834 if (oldval != val) 835 wr32(hw, cq->rq.len, val); 836 } 837 838 val = rd32(hw, cq->sq.len); 839 if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 840 PF_FW_ATQLEN_ATQCRIT_M)) { 841 oldval = val; 842 if (val & PF_FW_ATQLEN_ATQVFE_M) 843 dev_dbg(&pf->pdev->dev, 844 "%s Send Queue VF Error detected\n", qtype); 845 if (val & PF_FW_ATQLEN_ATQOVFL_M) { 846 dev_dbg(&pf->pdev->dev, 847 "%s Send Queue Overflow Error detected\n", 848 qtype); 849 } 850 if (val & PF_FW_ATQLEN_ATQCRIT_M) 851 dev_dbg(&pf->pdev->dev, 852 "%s Send Queue Critical Error detected\n", 853 qtype); 854 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 855 PF_FW_ATQLEN_ATQCRIT_M); 856 if (oldval != val) 857 wr32(hw, cq->sq.len, val); 858 } 859 860 event.buf_len = cq->rq_buf_size; 861 event.msg_buf = devm_kzalloc(&pf->pdev->dev, event.buf_len, 862 GFP_KERNEL); 863 if (!event.msg_buf) 864 return 0; 865 866 do { 867 enum ice_status ret; 868 u16 opcode; 869 870 ret = ice_clean_rq_elem(hw, cq, &event, &pending); 871 if (ret == ICE_ERR_AQ_NO_WORK) 872 break; 873 if (ret) { 874 dev_err(&pf->pdev->dev, 875 "%s Receive Queue event error %d\n", qtype, 876 ret); 877 break; 878 } 879 880 opcode = le16_to_cpu(event.desc.opcode); 881 882 switch (opcode) { 883 case ice_aqc_opc_get_link_status: 884 if (ice_handle_link_event(pf)) 885 dev_err(&pf->pdev->dev, 886 "Could not handle link event\n"); 887 break; 888 case ice_mbx_opc_send_msg_to_pf: 889 ice_vc_process_vf_msg(pf, &event); 890 break; 891 case ice_aqc_opc_fw_logging: 892 ice_output_fw_log(hw, &event.desc, event.msg_buf); 893 break; 894 default: 895 dev_dbg(&pf->pdev->dev, 896 "%s Receive Queue unknown event 0x%04x ignored\n", 897 qtype, opcode); 898 break; 899 } 900 } while (pending && (i++ < ICE_DFLT_IRQ_WORK)); 901 902 devm_kfree(&pf->pdev->dev, event.msg_buf); 903 904 return pending && (i == ICE_DFLT_IRQ_WORK); 905 } 906 907 /** 908 * ice_ctrlq_pending - check if there is a difference between ntc and ntu 909 * @hw: pointer to hardware info 910 * @cq: control queue information 911 * 912 * returns true if there are pending messages in a queue, false if there aren't 913 */ 914 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq) 915 { 916 u16 ntu; 917 918 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 919 return cq->rq.next_to_clean != ntu; 920 } 921 922 /** 923 * ice_clean_adminq_subtask - clean the AdminQ rings 924 * @pf: board private structure 925 */ 926 static void ice_clean_adminq_subtask(struct ice_pf *pf) 927 { 928 struct ice_hw *hw = &pf->hw; 929 930 if (!test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state)) 931 return; 932 933 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN)) 934 return; 935 936 clear_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state); 937 938 /* There might be a situation where new messages arrive to a control 939 * queue between processing the last message and clearing the 940 * EVENT_PENDING bit. So before exiting, check queue head again (using 941 * ice_ctrlq_pending) and process new messages if any. 942 */ 943 if (ice_ctrlq_pending(hw, &hw->adminq)) 944 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN); 945 946 ice_flush(hw); 947 } 948 949 /** 950 * ice_clean_mailboxq_subtask - clean the MailboxQ rings 951 * @pf: board private structure 952 */ 953 static void ice_clean_mailboxq_subtask(struct ice_pf *pf) 954 { 955 struct ice_hw *hw = &pf->hw; 956 957 if (!test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state)) 958 return; 959 960 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX)) 961 return; 962 963 clear_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state); 964 965 if (ice_ctrlq_pending(hw, &hw->mailboxq)) 966 __ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX); 967 968 ice_flush(hw); 969 } 970 971 /** 972 * ice_service_task_schedule - schedule the service task to wake up 973 * @pf: board private structure 974 * 975 * If not already scheduled, this puts the task into the work queue. 976 */ 977 static void ice_service_task_schedule(struct ice_pf *pf) 978 { 979 if (!test_bit(__ICE_SERVICE_DIS, pf->state) && 980 !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state) && 981 !test_bit(__ICE_NEEDS_RESTART, pf->state)) 982 queue_work(ice_wq, &pf->serv_task); 983 } 984 985 /** 986 * ice_service_task_complete - finish up the service task 987 * @pf: board private structure 988 */ 989 static void ice_service_task_complete(struct ice_pf *pf) 990 { 991 WARN_ON(!test_bit(__ICE_SERVICE_SCHED, pf->state)); 992 993 /* force memory (pf->state) to sync before next service task */ 994 smp_mb__before_atomic(); 995 clear_bit(__ICE_SERVICE_SCHED, pf->state); 996 } 997 998 /** 999 * ice_service_task_stop - stop service task and cancel works 1000 * @pf: board private structure 1001 */ 1002 static void ice_service_task_stop(struct ice_pf *pf) 1003 { 1004 set_bit(__ICE_SERVICE_DIS, pf->state); 1005 1006 if (pf->serv_tmr.function) 1007 del_timer_sync(&pf->serv_tmr); 1008 if (pf->serv_task.func) 1009 cancel_work_sync(&pf->serv_task); 1010 1011 clear_bit(__ICE_SERVICE_SCHED, pf->state); 1012 } 1013 1014 /** 1015 * ice_service_task_restart - restart service task and schedule works 1016 * @pf: board private structure 1017 * 1018 * This function is needed for suspend and resume works (e.g WoL scenario) 1019 */ 1020 static void ice_service_task_restart(struct ice_pf *pf) 1021 { 1022 clear_bit(__ICE_SERVICE_DIS, pf->state); 1023 ice_service_task_schedule(pf); 1024 } 1025 1026 /** 1027 * ice_service_timer - timer callback to schedule service task 1028 * @t: pointer to timer_list 1029 */ 1030 static void ice_service_timer(struct timer_list *t) 1031 { 1032 struct ice_pf *pf = from_timer(pf, t, serv_tmr); 1033 1034 mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies)); 1035 ice_service_task_schedule(pf); 1036 } 1037 1038 /** 1039 * ice_handle_mdd_event - handle malicious driver detect event 1040 * @pf: pointer to the PF structure 1041 * 1042 * Called from service task. OICR interrupt handler indicates MDD event 1043 */ 1044 static void ice_handle_mdd_event(struct ice_pf *pf) 1045 { 1046 struct ice_hw *hw = &pf->hw; 1047 bool mdd_detected = false; 1048 u32 reg; 1049 int i; 1050 1051 if (!test_bit(__ICE_MDD_EVENT_PENDING, pf->state)) 1052 return; 1053 1054 /* find what triggered the MDD event */ 1055 reg = rd32(hw, GL_MDET_TX_PQM); 1056 if (reg & GL_MDET_TX_PQM_VALID_M) { 1057 u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >> 1058 GL_MDET_TX_PQM_PF_NUM_S; 1059 u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >> 1060 GL_MDET_TX_PQM_VF_NUM_S; 1061 u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >> 1062 GL_MDET_TX_PQM_MAL_TYPE_S; 1063 u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >> 1064 GL_MDET_TX_PQM_QNUM_S); 1065 1066 if (netif_msg_tx_err(pf)) 1067 dev_info(&pf->pdev->dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1068 event, queue, pf_num, vf_num); 1069 wr32(hw, GL_MDET_TX_PQM, 0xffffffff); 1070 mdd_detected = true; 1071 } 1072 1073 reg = rd32(hw, GL_MDET_TX_TCLAN); 1074 if (reg & GL_MDET_TX_TCLAN_VALID_M) { 1075 u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >> 1076 GL_MDET_TX_TCLAN_PF_NUM_S; 1077 u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >> 1078 GL_MDET_TX_TCLAN_VF_NUM_S; 1079 u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >> 1080 GL_MDET_TX_TCLAN_MAL_TYPE_S; 1081 u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >> 1082 GL_MDET_TX_TCLAN_QNUM_S); 1083 1084 if (netif_msg_rx_err(pf)) 1085 dev_info(&pf->pdev->dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1086 event, queue, pf_num, vf_num); 1087 wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff); 1088 mdd_detected = true; 1089 } 1090 1091 reg = rd32(hw, GL_MDET_RX); 1092 if (reg & GL_MDET_RX_VALID_M) { 1093 u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >> 1094 GL_MDET_RX_PF_NUM_S; 1095 u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >> 1096 GL_MDET_RX_VF_NUM_S; 1097 u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >> 1098 GL_MDET_RX_MAL_TYPE_S; 1099 u16 queue = ((reg & GL_MDET_RX_QNUM_M) >> 1100 GL_MDET_RX_QNUM_S); 1101 1102 if (netif_msg_rx_err(pf)) 1103 dev_info(&pf->pdev->dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n", 1104 event, queue, pf_num, vf_num); 1105 wr32(hw, GL_MDET_RX, 0xffffffff); 1106 mdd_detected = true; 1107 } 1108 1109 if (mdd_detected) { 1110 bool pf_mdd_detected = false; 1111 1112 reg = rd32(hw, PF_MDET_TX_PQM); 1113 if (reg & PF_MDET_TX_PQM_VALID_M) { 1114 wr32(hw, PF_MDET_TX_PQM, 0xFFFF); 1115 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 1116 pf_mdd_detected = true; 1117 } 1118 1119 reg = rd32(hw, PF_MDET_TX_TCLAN); 1120 if (reg & PF_MDET_TX_TCLAN_VALID_M) { 1121 wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF); 1122 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 1123 pf_mdd_detected = true; 1124 } 1125 1126 reg = rd32(hw, PF_MDET_RX); 1127 if (reg & PF_MDET_RX_VALID_M) { 1128 wr32(hw, PF_MDET_RX, 0xFFFF); 1129 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 1130 pf_mdd_detected = true; 1131 } 1132 /* Queue belongs to the PF initiate a reset */ 1133 if (pf_mdd_detected) { 1134 set_bit(__ICE_NEEDS_RESTART, pf->state); 1135 ice_service_task_schedule(pf); 1136 } 1137 } 1138 1139 /* see if one of the VFs needs to be reset */ 1140 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 1141 struct ice_vf *vf = &pf->vf[i]; 1142 1143 reg = rd32(hw, VP_MDET_TX_PQM(i)); 1144 if (reg & VP_MDET_TX_PQM_VALID_M) { 1145 wr32(hw, VP_MDET_TX_PQM(i), 0xFFFF); 1146 vf->num_mdd_events++; 1147 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 1148 i); 1149 } 1150 1151 reg = rd32(hw, VP_MDET_TX_TCLAN(i)); 1152 if (reg & VP_MDET_TX_TCLAN_VALID_M) { 1153 wr32(hw, VP_MDET_TX_TCLAN(i), 0xFFFF); 1154 vf->num_mdd_events++; 1155 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 1156 i); 1157 } 1158 1159 reg = rd32(hw, VP_MDET_TX_TDPU(i)); 1160 if (reg & VP_MDET_TX_TDPU_VALID_M) { 1161 wr32(hw, VP_MDET_TX_TDPU(i), 0xFFFF); 1162 vf->num_mdd_events++; 1163 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 1164 i); 1165 } 1166 1167 reg = rd32(hw, VP_MDET_RX(i)); 1168 if (reg & VP_MDET_RX_VALID_M) { 1169 wr32(hw, VP_MDET_RX(i), 0xFFFF); 1170 vf->num_mdd_events++; 1171 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 1172 i); 1173 } 1174 1175 if (vf->num_mdd_events > ICE_DFLT_NUM_MDD_EVENTS_ALLOWED) { 1176 dev_info(&pf->pdev->dev, 1177 "Too many MDD events on VF %d, disabled\n", i); 1178 dev_info(&pf->pdev->dev, 1179 "Use PF Control I/F to re-enable the VF\n"); 1180 set_bit(ICE_VF_STATE_DIS, vf->vf_states); 1181 } 1182 } 1183 1184 /* re-enable MDD interrupt cause */ 1185 clear_bit(__ICE_MDD_EVENT_PENDING, pf->state); 1186 reg = rd32(hw, PFINT_OICR_ENA); 1187 reg |= PFINT_OICR_MAL_DETECT_M; 1188 wr32(hw, PFINT_OICR_ENA, reg); 1189 ice_flush(hw); 1190 } 1191 1192 /** 1193 * ice_service_task - manage and run subtasks 1194 * @work: pointer to work_struct contained by the PF struct 1195 */ 1196 static void ice_service_task(struct work_struct *work) 1197 { 1198 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task); 1199 unsigned long start_time = jiffies; 1200 1201 /* subtasks */ 1202 1203 /* process reset requests first */ 1204 ice_reset_subtask(pf); 1205 1206 /* bail if a reset/recovery cycle is pending or rebuild failed */ 1207 if (ice_is_reset_in_progress(pf->state) || 1208 test_bit(__ICE_SUSPENDED, pf->state) || 1209 test_bit(__ICE_NEEDS_RESTART, pf->state)) { 1210 ice_service_task_complete(pf); 1211 return; 1212 } 1213 1214 ice_check_for_hang_subtask(pf); 1215 ice_sync_fltr_subtask(pf); 1216 ice_handle_mdd_event(pf); 1217 ice_process_vflr_event(pf); 1218 ice_watchdog_subtask(pf); 1219 ice_clean_adminq_subtask(pf); 1220 ice_clean_mailboxq_subtask(pf); 1221 1222 /* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */ 1223 ice_service_task_complete(pf); 1224 1225 /* If the tasks have taken longer than one service timer period 1226 * or there is more work to be done, reset the service timer to 1227 * schedule the service task now. 1228 */ 1229 if (time_after(jiffies, (start_time + pf->serv_tmr_period)) || 1230 test_bit(__ICE_MDD_EVENT_PENDING, pf->state) || 1231 test_bit(__ICE_VFLR_EVENT_PENDING, pf->state) || 1232 test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state) || 1233 test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state)) 1234 mod_timer(&pf->serv_tmr, jiffies); 1235 } 1236 1237 /** 1238 * ice_set_ctrlq_len - helper function to set controlq length 1239 * @hw: pointer to the hw instance 1240 */ 1241 static void ice_set_ctrlq_len(struct ice_hw *hw) 1242 { 1243 hw->adminq.num_rq_entries = ICE_AQ_LEN; 1244 hw->adminq.num_sq_entries = ICE_AQ_LEN; 1245 hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN; 1246 hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN; 1247 hw->mailboxq.num_rq_entries = ICE_MBXQ_LEN; 1248 hw->mailboxq.num_sq_entries = ICE_MBXQ_LEN; 1249 hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 1250 hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 1251 } 1252 1253 /** 1254 * ice_irq_affinity_notify - Callback for affinity changes 1255 * @notify: context as to what irq was changed 1256 * @mask: the new affinity mask 1257 * 1258 * This is a callback function used by the irq_set_affinity_notifier function 1259 * so that we may register to receive changes to the irq affinity masks. 1260 */ 1261 static void 1262 ice_irq_affinity_notify(struct irq_affinity_notify *notify, 1263 const cpumask_t *mask) 1264 { 1265 struct ice_q_vector *q_vector = 1266 container_of(notify, struct ice_q_vector, affinity_notify); 1267 1268 cpumask_copy(&q_vector->affinity_mask, mask); 1269 } 1270 1271 /** 1272 * ice_irq_affinity_release - Callback for affinity notifier release 1273 * @ref: internal core kernel usage 1274 * 1275 * This is a callback function used by the irq_set_affinity_notifier function 1276 * to inform the current notification subscriber that they will no longer 1277 * receive notifications. 1278 */ 1279 static void ice_irq_affinity_release(struct kref __always_unused *ref) {} 1280 1281 /** 1282 * ice_vsi_ena_irq - Enable IRQ for the given VSI 1283 * @vsi: the VSI being configured 1284 */ 1285 static int ice_vsi_ena_irq(struct ice_vsi *vsi) 1286 { 1287 struct ice_pf *pf = vsi->back; 1288 struct ice_hw *hw = &pf->hw; 1289 1290 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) { 1291 int i; 1292 1293 for (i = 0; i < vsi->num_q_vectors; i++) 1294 ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]); 1295 } 1296 1297 ice_flush(hw); 1298 return 0; 1299 } 1300 1301 /** 1302 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI 1303 * @vsi: the VSI being configured 1304 * @basename: name for the vector 1305 */ 1306 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) 1307 { 1308 int q_vectors = vsi->num_q_vectors; 1309 struct ice_pf *pf = vsi->back; 1310 int base = vsi->sw_base_vector; 1311 int rx_int_idx = 0; 1312 int tx_int_idx = 0; 1313 int vector, err; 1314 int irq_num; 1315 1316 for (vector = 0; vector < q_vectors; vector++) { 1317 struct ice_q_vector *q_vector = vsi->q_vectors[vector]; 1318 1319 irq_num = pf->msix_entries[base + vector].vector; 1320 1321 if (q_vector->tx.ring && q_vector->rx.ring) { 1322 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1323 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 1324 tx_int_idx++; 1325 } else if (q_vector->rx.ring) { 1326 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1327 "%s-%s-%d", basename, "rx", rx_int_idx++); 1328 } else if (q_vector->tx.ring) { 1329 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 1330 "%s-%s-%d", basename, "tx", tx_int_idx++); 1331 } else { 1332 /* skip this unused q_vector */ 1333 continue; 1334 } 1335 err = devm_request_irq(&pf->pdev->dev, irq_num, 1336 vsi->irq_handler, 0, 1337 q_vector->name, q_vector); 1338 if (err) { 1339 netdev_err(vsi->netdev, 1340 "MSIX request_irq failed, error: %d\n", err); 1341 goto free_q_irqs; 1342 } 1343 1344 /* register for affinity change notifications */ 1345 q_vector->affinity_notify.notify = ice_irq_affinity_notify; 1346 q_vector->affinity_notify.release = ice_irq_affinity_release; 1347 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 1348 1349 /* assign the mask for this irq */ 1350 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 1351 } 1352 1353 vsi->irqs_ready = true; 1354 return 0; 1355 1356 free_q_irqs: 1357 while (vector) { 1358 vector--; 1359 irq_num = pf->msix_entries[base + vector].vector, 1360 irq_set_affinity_notifier(irq_num, NULL); 1361 irq_set_affinity_hint(irq_num, NULL); 1362 devm_free_irq(&pf->pdev->dev, irq_num, &vsi->q_vectors[vector]); 1363 } 1364 return err; 1365 } 1366 1367 /** 1368 * ice_ena_misc_vector - enable the non-queue interrupts 1369 * @pf: board private structure 1370 */ 1371 static void ice_ena_misc_vector(struct ice_pf *pf) 1372 { 1373 struct ice_hw *hw = &pf->hw; 1374 u32 val; 1375 1376 /* clear things first */ 1377 wr32(hw, PFINT_OICR_ENA, 0); /* disable all */ 1378 rd32(hw, PFINT_OICR); /* read to clear */ 1379 1380 val = (PFINT_OICR_ECC_ERR_M | 1381 PFINT_OICR_MAL_DETECT_M | 1382 PFINT_OICR_GRST_M | 1383 PFINT_OICR_PCI_EXCEPTION_M | 1384 PFINT_OICR_VFLR_M | 1385 PFINT_OICR_HMC_ERR_M | 1386 PFINT_OICR_PE_CRITERR_M); 1387 1388 wr32(hw, PFINT_OICR_ENA, val); 1389 1390 /* SW_ITR_IDX = 0, but don't change INTENA */ 1391 wr32(hw, GLINT_DYN_CTL(pf->hw_oicr_idx), 1392 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M); 1393 } 1394 1395 /** 1396 * ice_misc_intr - misc interrupt handler 1397 * @irq: interrupt number 1398 * @data: pointer to a q_vector 1399 */ 1400 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) 1401 { 1402 struct ice_pf *pf = (struct ice_pf *)data; 1403 struct ice_hw *hw = &pf->hw; 1404 irqreturn_t ret = IRQ_NONE; 1405 u32 oicr, ena_mask; 1406 1407 set_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state); 1408 set_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state); 1409 1410 oicr = rd32(hw, PFINT_OICR); 1411 ena_mask = rd32(hw, PFINT_OICR_ENA); 1412 1413 if (oicr & PFINT_OICR_MAL_DETECT_M) { 1414 ena_mask &= ~PFINT_OICR_MAL_DETECT_M; 1415 set_bit(__ICE_MDD_EVENT_PENDING, pf->state); 1416 } 1417 if (oicr & PFINT_OICR_VFLR_M) { 1418 ena_mask &= ~PFINT_OICR_VFLR_M; 1419 set_bit(__ICE_VFLR_EVENT_PENDING, pf->state); 1420 } 1421 1422 if (oicr & PFINT_OICR_GRST_M) { 1423 u32 reset; 1424 1425 /* we have a reset warning */ 1426 ena_mask &= ~PFINT_OICR_GRST_M; 1427 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 1428 GLGEN_RSTAT_RESET_TYPE_S; 1429 1430 if (reset == ICE_RESET_CORER) 1431 pf->corer_count++; 1432 else if (reset == ICE_RESET_GLOBR) 1433 pf->globr_count++; 1434 else if (reset == ICE_RESET_EMPR) 1435 pf->empr_count++; 1436 else 1437 dev_dbg(&pf->pdev->dev, "Invalid reset type %d\n", 1438 reset); 1439 1440 /* If a reset cycle isn't already in progress, we set a bit in 1441 * pf->state so that the service task can start a reset/rebuild. 1442 * We also make note of which reset happened so that peer 1443 * devices/drivers can be informed. 1444 */ 1445 if (!test_and_set_bit(__ICE_RESET_OICR_RECV, pf->state)) { 1446 if (reset == ICE_RESET_CORER) 1447 set_bit(__ICE_CORER_RECV, pf->state); 1448 else if (reset == ICE_RESET_GLOBR) 1449 set_bit(__ICE_GLOBR_RECV, pf->state); 1450 else 1451 set_bit(__ICE_EMPR_RECV, pf->state); 1452 1453 /* There are couple of different bits at play here. 1454 * hw->reset_ongoing indicates whether the hardware is 1455 * in reset. This is set to true when a reset interrupt 1456 * is received and set back to false after the driver 1457 * has determined that the hardware is out of reset. 1458 * 1459 * __ICE_RESET_OICR_RECV in pf->state indicates 1460 * that a post reset rebuild is required before the 1461 * driver is operational again. This is set above. 1462 * 1463 * As this is the start of the reset/rebuild cycle, set 1464 * both to indicate that. 1465 */ 1466 hw->reset_ongoing = true; 1467 } 1468 } 1469 1470 if (oicr & PFINT_OICR_HMC_ERR_M) { 1471 ena_mask &= ~PFINT_OICR_HMC_ERR_M; 1472 dev_dbg(&pf->pdev->dev, 1473 "HMC Error interrupt - info 0x%x, data 0x%x\n", 1474 rd32(hw, PFHMC_ERRORINFO), 1475 rd32(hw, PFHMC_ERRORDATA)); 1476 } 1477 1478 /* Report and mask off any remaining unexpected interrupts */ 1479 oicr &= ena_mask; 1480 if (oicr) { 1481 dev_dbg(&pf->pdev->dev, "unhandled interrupt oicr=0x%08x\n", 1482 oicr); 1483 /* If a critical error is pending there is no choice but to 1484 * reset the device. 1485 */ 1486 if (oicr & (PFINT_OICR_PE_CRITERR_M | 1487 PFINT_OICR_PCI_EXCEPTION_M | 1488 PFINT_OICR_ECC_ERR_M)) { 1489 set_bit(__ICE_PFR_REQ, pf->state); 1490 ice_service_task_schedule(pf); 1491 } 1492 ena_mask &= ~oicr; 1493 } 1494 ret = IRQ_HANDLED; 1495 1496 /* re-enable interrupt causes that are not handled during this pass */ 1497 wr32(hw, PFINT_OICR_ENA, ena_mask); 1498 if (!test_bit(__ICE_DOWN, pf->state)) { 1499 ice_service_task_schedule(pf); 1500 ice_irq_dynamic_ena(hw, NULL, NULL); 1501 } 1502 1503 return ret; 1504 } 1505 1506 /** 1507 * ice_dis_ctrlq_interrupts - disable control queue interrupts 1508 * @hw: pointer to HW structure 1509 */ 1510 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw) 1511 { 1512 /* disable Admin queue Interrupt causes */ 1513 wr32(hw, PFINT_FW_CTL, 1514 rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M); 1515 1516 /* disable Mailbox queue Interrupt causes */ 1517 wr32(hw, PFINT_MBX_CTL, 1518 rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M); 1519 1520 /* disable Control queue Interrupt causes */ 1521 wr32(hw, PFINT_OICR_CTL, 1522 rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M); 1523 1524 ice_flush(hw); 1525 } 1526 1527 /** 1528 * ice_free_irq_msix_misc - Unroll misc vector setup 1529 * @pf: board private structure 1530 */ 1531 static void ice_free_irq_msix_misc(struct ice_pf *pf) 1532 { 1533 struct ice_hw *hw = &pf->hw; 1534 1535 ice_dis_ctrlq_interrupts(hw); 1536 1537 /* disable OICR interrupt */ 1538 wr32(hw, PFINT_OICR_ENA, 0); 1539 ice_flush(hw); 1540 1541 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) { 1542 synchronize_irq(pf->msix_entries[pf->sw_oicr_idx].vector); 1543 devm_free_irq(&pf->pdev->dev, 1544 pf->msix_entries[pf->sw_oicr_idx].vector, pf); 1545 } 1546 1547 pf->num_avail_sw_msix += 1; 1548 ice_free_res(pf->sw_irq_tracker, pf->sw_oicr_idx, ICE_RES_MISC_VEC_ID); 1549 pf->num_avail_hw_msix += 1; 1550 ice_free_res(pf->hw_irq_tracker, pf->hw_oicr_idx, ICE_RES_MISC_VEC_ID); 1551 } 1552 1553 /** 1554 * ice_ena_ctrlq_interrupts - enable control queue interrupts 1555 * @hw: pointer to HW structure 1556 * @v_idx: HW vector index to associate the control queue interrupts with 1557 */ 1558 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 v_idx) 1559 { 1560 u32 val; 1561 1562 val = ((v_idx & PFINT_OICR_CTL_MSIX_INDX_M) | 1563 PFINT_OICR_CTL_CAUSE_ENA_M); 1564 wr32(hw, PFINT_OICR_CTL, val); 1565 1566 /* enable Admin queue Interrupt causes */ 1567 val = ((v_idx & PFINT_FW_CTL_MSIX_INDX_M) | 1568 PFINT_FW_CTL_CAUSE_ENA_M); 1569 wr32(hw, PFINT_FW_CTL, val); 1570 1571 /* enable Mailbox queue Interrupt causes */ 1572 val = ((v_idx & PFINT_MBX_CTL_MSIX_INDX_M) | 1573 PFINT_MBX_CTL_CAUSE_ENA_M); 1574 wr32(hw, PFINT_MBX_CTL, val); 1575 1576 ice_flush(hw); 1577 } 1578 1579 /** 1580 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events 1581 * @pf: board private structure 1582 * 1583 * This sets up the handler for MSIX 0, which is used to manage the 1584 * non-queue interrupts, e.g. AdminQ and errors. This is not used 1585 * when in MSI or Legacy interrupt mode. 1586 */ 1587 static int ice_req_irq_msix_misc(struct ice_pf *pf) 1588 { 1589 struct ice_hw *hw = &pf->hw; 1590 int oicr_idx, err = 0; 1591 1592 if (!pf->int_name[0]) 1593 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc", 1594 dev_driver_string(&pf->pdev->dev), 1595 dev_name(&pf->pdev->dev)); 1596 1597 /* Do not request IRQ but do enable OICR interrupt since settings are 1598 * lost during reset. Note that this function is called only during 1599 * rebuild path and not while reset is in progress. 1600 */ 1601 if (ice_is_reset_in_progress(pf->state)) 1602 goto skip_req_irq; 1603 1604 /* reserve one vector in sw_irq_tracker for misc interrupts */ 1605 oicr_idx = ice_get_res(pf, pf->sw_irq_tracker, 1, ICE_RES_MISC_VEC_ID); 1606 if (oicr_idx < 0) 1607 return oicr_idx; 1608 1609 pf->num_avail_sw_msix -= 1; 1610 pf->sw_oicr_idx = oicr_idx; 1611 1612 /* reserve one vector in hw_irq_tracker for misc interrupts */ 1613 oicr_idx = ice_get_res(pf, pf->hw_irq_tracker, 1, ICE_RES_MISC_VEC_ID); 1614 if (oicr_idx < 0) { 1615 ice_free_res(pf->sw_irq_tracker, 1, ICE_RES_MISC_VEC_ID); 1616 pf->num_avail_sw_msix += 1; 1617 return oicr_idx; 1618 } 1619 pf->num_avail_hw_msix -= 1; 1620 pf->hw_oicr_idx = oicr_idx; 1621 1622 err = devm_request_irq(&pf->pdev->dev, 1623 pf->msix_entries[pf->sw_oicr_idx].vector, 1624 ice_misc_intr, 0, pf->int_name, pf); 1625 if (err) { 1626 dev_err(&pf->pdev->dev, 1627 "devm_request_irq for %s failed: %d\n", 1628 pf->int_name, err); 1629 ice_free_res(pf->sw_irq_tracker, 1, ICE_RES_MISC_VEC_ID); 1630 pf->num_avail_sw_msix += 1; 1631 ice_free_res(pf->hw_irq_tracker, 1, ICE_RES_MISC_VEC_ID); 1632 pf->num_avail_hw_msix += 1; 1633 return err; 1634 } 1635 1636 skip_req_irq: 1637 ice_ena_misc_vector(pf); 1638 1639 ice_ena_ctrlq_interrupts(hw, pf->hw_oicr_idx); 1640 wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->hw_oicr_idx), 1641 ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S); 1642 1643 ice_flush(hw); 1644 ice_irq_dynamic_ena(hw, NULL, NULL); 1645 1646 return 0; 1647 } 1648 1649 /** 1650 * ice_napi_del - Remove NAPI handler for the VSI 1651 * @vsi: VSI for which NAPI handler is to be removed 1652 */ 1653 void ice_napi_del(struct ice_vsi *vsi) 1654 { 1655 int v_idx; 1656 1657 if (!vsi->netdev) 1658 return; 1659 1660 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 1661 netif_napi_del(&vsi->q_vectors[v_idx]->napi); 1662 } 1663 1664 /** 1665 * ice_napi_add - register NAPI handler for the VSI 1666 * @vsi: VSI for which NAPI handler is to be registered 1667 * 1668 * This function is only called in the driver's load path. Registering the NAPI 1669 * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, 1670 * reset/rebuild, etc.) 1671 */ 1672 static void ice_napi_add(struct ice_vsi *vsi) 1673 { 1674 int v_idx; 1675 1676 if (!vsi->netdev) 1677 return; 1678 1679 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 1680 netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, 1681 ice_napi_poll, NAPI_POLL_WEIGHT); 1682 } 1683 1684 /** 1685 * ice_cfg_netdev - Allocate, configure and register a netdev 1686 * @vsi: the VSI associated with the new netdev 1687 * 1688 * Returns 0 on success, negative value on failure 1689 */ 1690 static int ice_cfg_netdev(struct ice_vsi *vsi) 1691 { 1692 netdev_features_t csumo_features; 1693 netdev_features_t vlano_features; 1694 netdev_features_t dflt_features; 1695 netdev_features_t tso_features; 1696 struct ice_netdev_priv *np; 1697 struct net_device *netdev; 1698 u8 mac_addr[ETH_ALEN]; 1699 int err; 1700 1701 netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, 1702 vsi->alloc_rxq); 1703 if (!netdev) 1704 return -ENOMEM; 1705 1706 vsi->netdev = netdev; 1707 np = netdev_priv(netdev); 1708 np->vsi = vsi; 1709 1710 dflt_features = NETIF_F_SG | 1711 NETIF_F_HIGHDMA | 1712 NETIF_F_RXHASH; 1713 1714 csumo_features = NETIF_F_RXCSUM | 1715 NETIF_F_IP_CSUM | 1716 NETIF_F_SCTP_CRC | 1717 NETIF_F_IPV6_CSUM; 1718 1719 vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER | 1720 NETIF_F_HW_VLAN_CTAG_TX | 1721 NETIF_F_HW_VLAN_CTAG_RX; 1722 1723 tso_features = NETIF_F_TSO; 1724 1725 /* set features that user can change */ 1726 netdev->hw_features = dflt_features | csumo_features | 1727 vlano_features | tso_features; 1728 1729 /* enable features */ 1730 netdev->features |= netdev->hw_features; 1731 /* encap and VLAN devices inherit default, csumo and tso features */ 1732 netdev->hw_enc_features |= dflt_features | csumo_features | 1733 tso_features; 1734 netdev->vlan_features |= dflt_features | csumo_features | 1735 tso_features; 1736 1737 if (vsi->type == ICE_VSI_PF) { 1738 SET_NETDEV_DEV(netdev, &vsi->back->pdev->dev); 1739 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 1740 1741 ether_addr_copy(netdev->dev_addr, mac_addr); 1742 ether_addr_copy(netdev->perm_addr, mac_addr); 1743 } 1744 1745 netdev->priv_flags |= IFF_UNICAST_FLT; 1746 1747 /* assign netdev_ops */ 1748 netdev->netdev_ops = &ice_netdev_ops; 1749 1750 /* setup watchdog timeout value to be 5 second */ 1751 netdev->watchdog_timeo = 5 * HZ; 1752 1753 ice_set_ethtool_ops(netdev); 1754 1755 netdev->min_mtu = ETH_MIN_MTU; 1756 netdev->max_mtu = ICE_MAX_MTU; 1757 1758 err = register_netdev(vsi->netdev); 1759 if (err) 1760 return err; 1761 1762 netif_carrier_off(vsi->netdev); 1763 1764 /* make sure transmit queues start off as stopped */ 1765 netif_tx_stop_all_queues(vsi->netdev); 1766 1767 return 0; 1768 } 1769 1770 /** 1771 * ice_fill_rss_lut - Fill the RSS lookup table with default values 1772 * @lut: Lookup table 1773 * @rss_table_size: Lookup table size 1774 * @rss_size: Range of queue number for hashing 1775 */ 1776 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size) 1777 { 1778 u16 i; 1779 1780 for (i = 0; i < rss_table_size; i++) 1781 lut[i] = i % rss_size; 1782 } 1783 1784 /** 1785 * ice_pf_vsi_setup - Set up a PF VSI 1786 * @pf: board private structure 1787 * @pi: pointer to the port_info instance 1788 * 1789 * Returns pointer to the successfully allocated VSI sw struct on success, 1790 * otherwise returns NULL on failure. 1791 */ 1792 static struct ice_vsi * 1793 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 1794 { 1795 return ice_vsi_setup(pf, pi, ICE_VSI_PF, ICE_INVAL_VFID); 1796 } 1797 1798 /** 1799 * ice_vlan_rx_add_vid - Add a vlan id filter to HW offload 1800 * @netdev: network interface to be adjusted 1801 * @proto: unused protocol 1802 * @vid: vlan id to be added 1803 * 1804 * net_device_ops implementation for adding vlan ids 1805 */ 1806 static int 1807 ice_vlan_rx_add_vid(struct net_device *netdev, __always_unused __be16 proto, 1808 u16 vid) 1809 { 1810 struct ice_netdev_priv *np = netdev_priv(netdev); 1811 struct ice_vsi *vsi = np->vsi; 1812 int ret; 1813 1814 if (vid >= VLAN_N_VID) { 1815 netdev_err(netdev, "VLAN id requested %d is out of range %d\n", 1816 vid, VLAN_N_VID); 1817 return -EINVAL; 1818 } 1819 1820 if (vsi->info.pvid) 1821 return -EINVAL; 1822 1823 /* Enable VLAN pruning when VLAN 0 is added */ 1824 if (unlikely(!vid)) { 1825 ret = ice_cfg_vlan_pruning(vsi, true, false); 1826 if (ret) 1827 return ret; 1828 } 1829 1830 /* Add all VLAN ids including 0 to the switch filter. VLAN id 0 is 1831 * needed to continue allowing all untagged packets since VLAN prune 1832 * list is applied to all packets by the switch 1833 */ 1834 ret = ice_vsi_add_vlan(vsi, vid); 1835 if (!ret) { 1836 vsi->vlan_ena = true; 1837 set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 1838 } 1839 1840 return ret; 1841 } 1842 1843 /** 1844 * ice_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 1845 * @netdev: network interface to be adjusted 1846 * @proto: unused protocol 1847 * @vid: vlan id to be removed 1848 * 1849 * net_device_ops implementation for removing vlan ids 1850 */ 1851 static int 1852 ice_vlan_rx_kill_vid(struct net_device *netdev, __always_unused __be16 proto, 1853 u16 vid) 1854 { 1855 struct ice_netdev_priv *np = netdev_priv(netdev); 1856 struct ice_vsi *vsi = np->vsi; 1857 int ret; 1858 1859 if (vsi->info.pvid) 1860 return -EINVAL; 1861 1862 /* Make sure ice_vsi_kill_vlan is successful before updating VLAN 1863 * information 1864 */ 1865 ret = ice_vsi_kill_vlan(vsi, vid); 1866 if (ret) 1867 return ret; 1868 1869 /* Disable VLAN pruning when VLAN 0 is removed */ 1870 if (unlikely(!vid)) 1871 ret = ice_cfg_vlan_pruning(vsi, false, false); 1872 1873 vsi->vlan_ena = false; 1874 set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 1875 return ret; 1876 } 1877 1878 /** 1879 * ice_setup_pf_sw - Setup the HW switch on startup or after reset 1880 * @pf: board private structure 1881 * 1882 * Returns 0 on success, negative value on failure 1883 */ 1884 static int ice_setup_pf_sw(struct ice_pf *pf) 1885 { 1886 LIST_HEAD(tmp_add_list); 1887 u8 broadcast[ETH_ALEN]; 1888 struct ice_vsi *vsi; 1889 int status = 0; 1890 1891 if (ice_is_reset_in_progress(pf->state)) 1892 return -EBUSY; 1893 1894 vsi = ice_pf_vsi_setup(pf, pf->hw.port_info); 1895 if (!vsi) { 1896 status = -ENOMEM; 1897 goto unroll_vsi_setup; 1898 } 1899 1900 status = ice_cfg_netdev(vsi); 1901 if (status) { 1902 status = -ENODEV; 1903 goto unroll_vsi_setup; 1904 } 1905 1906 /* registering the NAPI handler requires both the queues and 1907 * netdev to be created, which are done in ice_pf_vsi_setup() 1908 * and ice_cfg_netdev() respectively 1909 */ 1910 ice_napi_add(vsi); 1911 1912 /* To add a MAC filter, first add the MAC to a list and then 1913 * pass the list to ice_add_mac. 1914 */ 1915 1916 /* Add a unicast MAC filter so the VSI can get its packets */ 1917 status = ice_add_mac_to_list(vsi, &tmp_add_list, 1918 vsi->port_info->mac.perm_addr); 1919 if (status) 1920 goto unroll_napi_add; 1921 1922 /* VSI needs to receive broadcast traffic, so add the broadcast 1923 * MAC address to the list as well. 1924 */ 1925 eth_broadcast_addr(broadcast); 1926 status = ice_add_mac_to_list(vsi, &tmp_add_list, broadcast); 1927 if (status) 1928 goto free_mac_list; 1929 1930 /* program MAC filters for entries in tmp_add_list */ 1931 status = ice_add_mac(&pf->hw, &tmp_add_list); 1932 if (status) { 1933 dev_err(&pf->pdev->dev, "Could not add MAC filters\n"); 1934 status = -ENOMEM; 1935 goto free_mac_list; 1936 } 1937 1938 ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list); 1939 return status; 1940 1941 free_mac_list: 1942 ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list); 1943 1944 unroll_napi_add: 1945 if (vsi) { 1946 ice_napi_del(vsi); 1947 if (vsi->netdev) { 1948 if (vsi->netdev->reg_state == NETREG_REGISTERED) 1949 unregister_netdev(vsi->netdev); 1950 free_netdev(vsi->netdev); 1951 vsi->netdev = NULL; 1952 } 1953 } 1954 1955 unroll_vsi_setup: 1956 if (vsi) { 1957 ice_vsi_free_q_vectors(vsi); 1958 ice_vsi_delete(vsi); 1959 ice_vsi_put_qs(vsi); 1960 pf->q_left_tx += vsi->alloc_txq; 1961 pf->q_left_rx += vsi->alloc_rxq; 1962 ice_vsi_clear(vsi); 1963 } 1964 return status; 1965 } 1966 1967 /** 1968 * ice_determine_q_usage - Calculate queue distribution 1969 * @pf: board private structure 1970 * 1971 * Return -ENOMEM if we don't get enough queues for all ports 1972 */ 1973 static void ice_determine_q_usage(struct ice_pf *pf) 1974 { 1975 u16 q_left_tx, q_left_rx; 1976 1977 q_left_tx = pf->hw.func_caps.common_cap.num_txq; 1978 q_left_rx = pf->hw.func_caps.common_cap.num_rxq; 1979 1980 pf->num_lan_tx = min_t(int, q_left_tx, num_online_cpus()); 1981 1982 /* only 1 Rx queue unless RSS is enabled */ 1983 if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) 1984 pf->num_lan_rx = 1; 1985 else 1986 pf->num_lan_rx = min_t(int, q_left_rx, num_online_cpus()); 1987 1988 pf->q_left_tx = q_left_tx - pf->num_lan_tx; 1989 pf->q_left_rx = q_left_rx - pf->num_lan_rx; 1990 } 1991 1992 /** 1993 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf 1994 * @pf: board private structure to initialize 1995 */ 1996 static void ice_deinit_pf(struct ice_pf *pf) 1997 { 1998 ice_service_task_stop(pf); 1999 mutex_destroy(&pf->sw_mutex); 2000 mutex_destroy(&pf->avail_q_mutex); 2001 } 2002 2003 /** 2004 * ice_init_pf - Initialize general software structures (struct ice_pf) 2005 * @pf: board private structure to initialize 2006 */ 2007 static void ice_init_pf(struct ice_pf *pf) 2008 { 2009 bitmap_zero(pf->flags, ICE_PF_FLAGS_NBITS); 2010 set_bit(ICE_FLAG_MSIX_ENA, pf->flags); 2011 #ifdef CONFIG_PCI_IOV 2012 if (pf->hw.func_caps.common_cap.sr_iov_1_1) { 2013 struct ice_hw *hw = &pf->hw; 2014 2015 set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 2016 pf->num_vfs_supported = min_t(int, hw->func_caps.num_allocd_vfs, 2017 ICE_MAX_VF_COUNT); 2018 } 2019 #endif /* CONFIG_PCI_IOV */ 2020 2021 mutex_init(&pf->sw_mutex); 2022 mutex_init(&pf->avail_q_mutex); 2023 2024 /* Clear avail_[t|r]x_qs bitmaps (set all to avail) */ 2025 mutex_lock(&pf->avail_q_mutex); 2026 bitmap_zero(pf->avail_txqs, ICE_MAX_TXQS); 2027 bitmap_zero(pf->avail_rxqs, ICE_MAX_RXQS); 2028 mutex_unlock(&pf->avail_q_mutex); 2029 2030 if (pf->hw.func_caps.common_cap.rss_table_size) 2031 set_bit(ICE_FLAG_RSS_ENA, pf->flags); 2032 2033 /* setup service timer and periodic service task */ 2034 timer_setup(&pf->serv_tmr, ice_service_timer, 0); 2035 pf->serv_tmr_period = HZ; 2036 INIT_WORK(&pf->serv_task, ice_service_task); 2037 clear_bit(__ICE_SERVICE_SCHED, pf->state); 2038 } 2039 2040 /** 2041 * ice_ena_msix_range - Request a range of MSIX vectors from the OS 2042 * @pf: board private structure 2043 * 2044 * compute the number of MSIX vectors required (v_budget) and request from 2045 * the OS. Return the number of vectors reserved or negative on failure 2046 */ 2047 static int ice_ena_msix_range(struct ice_pf *pf) 2048 { 2049 int v_left, v_actual, v_budget = 0; 2050 int needed, err, i; 2051 2052 v_left = pf->hw.func_caps.common_cap.num_msix_vectors; 2053 2054 /* reserve one vector for miscellaneous handler */ 2055 needed = 1; 2056 v_budget += needed; 2057 v_left -= needed; 2058 2059 /* reserve vectors for LAN traffic */ 2060 pf->num_lan_msix = min_t(int, num_online_cpus(), v_left); 2061 v_budget += pf->num_lan_msix; 2062 v_left -= pf->num_lan_msix; 2063 2064 pf->msix_entries = devm_kcalloc(&pf->pdev->dev, v_budget, 2065 sizeof(*pf->msix_entries), GFP_KERNEL); 2066 2067 if (!pf->msix_entries) { 2068 err = -ENOMEM; 2069 goto exit_err; 2070 } 2071 2072 for (i = 0; i < v_budget; i++) 2073 pf->msix_entries[i].entry = i; 2074 2075 /* actually reserve the vectors */ 2076 v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries, 2077 ICE_MIN_MSIX, v_budget); 2078 2079 if (v_actual < 0) { 2080 dev_err(&pf->pdev->dev, "unable to reserve MSI-X vectors\n"); 2081 err = v_actual; 2082 goto msix_err; 2083 } 2084 2085 if (v_actual < v_budget) { 2086 dev_warn(&pf->pdev->dev, 2087 "not enough vectors. requested = %d, obtained = %d\n", 2088 v_budget, v_actual); 2089 if (v_actual >= (pf->num_lan_msix + 1)) { 2090 pf->num_avail_sw_msix = v_actual - 2091 (pf->num_lan_msix + 1); 2092 } else if (v_actual >= 2) { 2093 pf->num_lan_msix = 1; 2094 pf->num_avail_sw_msix = v_actual - 2; 2095 } else { 2096 pci_disable_msix(pf->pdev); 2097 err = -ERANGE; 2098 goto msix_err; 2099 } 2100 } 2101 2102 return v_actual; 2103 2104 msix_err: 2105 devm_kfree(&pf->pdev->dev, pf->msix_entries); 2106 goto exit_err; 2107 2108 exit_err: 2109 pf->num_lan_msix = 0; 2110 clear_bit(ICE_FLAG_MSIX_ENA, pf->flags); 2111 return err; 2112 } 2113 2114 /** 2115 * ice_dis_msix - Disable MSI-X interrupt setup in OS 2116 * @pf: board private structure 2117 */ 2118 static void ice_dis_msix(struct ice_pf *pf) 2119 { 2120 pci_disable_msix(pf->pdev); 2121 devm_kfree(&pf->pdev->dev, pf->msix_entries); 2122 pf->msix_entries = NULL; 2123 clear_bit(ICE_FLAG_MSIX_ENA, pf->flags); 2124 } 2125 2126 /** 2127 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme 2128 * @pf: board private structure 2129 */ 2130 static void ice_clear_interrupt_scheme(struct ice_pf *pf) 2131 { 2132 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) 2133 ice_dis_msix(pf); 2134 2135 if (pf->sw_irq_tracker) { 2136 devm_kfree(&pf->pdev->dev, pf->sw_irq_tracker); 2137 pf->sw_irq_tracker = NULL; 2138 } 2139 2140 if (pf->hw_irq_tracker) { 2141 devm_kfree(&pf->pdev->dev, pf->hw_irq_tracker); 2142 pf->hw_irq_tracker = NULL; 2143 } 2144 } 2145 2146 /** 2147 * ice_init_interrupt_scheme - Determine proper interrupt scheme 2148 * @pf: board private structure to initialize 2149 */ 2150 static int ice_init_interrupt_scheme(struct ice_pf *pf) 2151 { 2152 int vectors = 0, hw_vectors = 0; 2153 2154 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) 2155 vectors = ice_ena_msix_range(pf); 2156 else 2157 return -ENODEV; 2158 2159 if (vectors < 0) 2160 return vectors; 2161 2162 /* set up vector assignment tracking */ 2163 pf->sw_irq_tracker = 2164 devm_kzalloc(&pf->pdev->dev, sizeof(*pf->sw_irq_tracker) + 2165 (sizeof(u16) * vectors), GFP_KERNEL); 2166 if (!pf->sw_irq_tracker) { 2167 ice_dis_msix(pf); 2168 return -ENOMEM; 2169 } 2170 2171 /* populate SW interrupts pool with number of OS granted IRQs. */ 2172 pf->num_avail_sw_msix = vectors; 2173 pf->sw_irq_tracker->num_entries = vectors; 2174 2175 /* set up HW vector assignment tracking */ 2176 hw_vectors = pf->hw.func_caps.common_cap.num_msix_vectors; 2177 pf->hw_irq_tracker = 2178 devm_kzalloc(&pf->pdev->dev, sizeof(*pf->hw_irq_tracker) + 2179 (sizeof(u16) * hw_vectors), GFP_KERNEL); 2180 if (!pf->hw_irq_tracker) { 2181 ice_clear_interrupt_scheme(pf); 2182 return -ENOMEM; 2183 } 2184 2185 /* populate HW interrupts pool with number of HW supported irqs. */ 2186 pf->num_avail_hw_msix = hw_vectors; 2187 pf->hw_irq_tracker->num_entries = hw_vectors; 2188 2189 return 0; 2190 } 2191 2192 /** 2193 * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines 2194 * @pf: pointer to the PF structure 2195 * 2196 * There is no error returned here because the driver should be able to handle 2197 * 128 Byte cache lines, so we only print a warning in case issues are seen, 2198 * specifically with Tx. 2199 */ 2200 static void ice_verify_cacheline_size(struct ice_pf *pf) 2201 { 2202 if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M) 2203 dev_warn(&pf->pdev->dev, 2204 "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n", 2205 ICE_CACHE_LINE_BYTES); 2206 } 2207 2208 /** 2209 * ice_probe - Device initialization routine 2210 * @pdev: PCI device information struct 2211 * @ent: entry in ice_pci_tbl 2212 * 2213 * Returns 0 on success, negative on failure 2214 */ 2215 static int 2216 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) 2217 { 2218 struct device *dev = &pdev->dev; 2219 struct ice_pf *pf; 2220 struct ice_hw *hw; 2221 int err; 2222 2223 /* this driver uses devres, see Documentation/driver-model/devres.txt */ 2224 err = pcim_enable_device(pdev); 2225 if (err) 2226 return err; 2227 2228 err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), pci_name(pdev)); 2229 if (err) { 2230 dev_err(dev, "BAR0 I/O map error %d\n", err); 2231 return err; 2232 } 2233 2234 pf = devm_kzalloc(dev, sizeof(*pf), GFP_KERNEL); 2235 if (!pf) 2236 return -ENOMEM; 2237 2238 /* set up for high or low dma */ 2239 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 2240 if (err) 2241 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 2242 if (err) { 2243 dev_err(dev, "DMA configuration failed: 0x%x\n", err); 2244 return err; 2245 } 2246 2247 pci_enable_pcie_error_reporting(pdev); 2248 pci_set_master(pdev); 2249 2250 pf->pdev = pdev; 2251 pci_set_drvdata(pdev, pf); 2252 set_bit(__ICE_DOWN, pf->state); 2253 /* Disable service task until DOWN bit is cleared */ 2254 set_bit(__ICE_SERVICE_DIS, pf->state); 2255 2256 hw = &pf->hw; 2257 hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0]; 2258 hw->back = pf; 2259 hw->vendor_id = pdev->vendor; 2260 hw->device_id = pdev->device; 2261 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 2262 hw->subsystem_vendor_id = pdev->subsystem_vendor; 2263 hw->subsystem_device_id = pdev->subsystem_device; 2264 hw->bus.device = PCI_SLOT(pdev->devfn); 2265 hw->bus.func = PCI_FUNC(pdev->devfn); 2266 ice_set_ctrlq_len(hw); 2267 2268 pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M); 2269 2270 #ifndef CONFIG_DYNAMIC_DEBUG 2271 if (debug < -1) 2272 hw->debug_mask = debug; 2273 #endif 2274 2275 err = ice_init_hw(hw); 2276 if (err) { 2277 dev_err(dev, "ice_init_hw failed: %d\n", err); 2278 err = -EIO; 2279 goto err_exit_unroll; 2280 } 2281 2282 dev_info(dev, "firmware %d.%d.%05d api %d.%d\n", 2283 hw->fw_maj_ver, hw->fw_min_ver, hw->fw_build, 2284 hw->api_maj_ver, hw->api_min_ver); 2285 2286 ice_init_pf(pf); 2287 2288 ice_determine_q_usage(pf); 2289 2290 pf->num_alloc_vsi = hw->func_caps.guar_num_vsi; 2291 if (!pf->num_alloc_vsi) { 2292 err = -EIO; 2293 goto err_init_pf_unroll; 2294 } 2295 2296 pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi), 2297 GFP_KERNEL); 2298 if (!pf->vsi) { 2299 err = -ENOMEM; 2300 goto err_init_pf_unroll; 2301 } 2302 2303 err = ice_init_interrupt_scheme(pf); 2304 if (err) { 2305 dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); 2306 err = -EIO; 2307 goto err_init_interrupt_unroll; 2308 } 2309 2310 /* Driver is mostly up */ 2311 clear_bit(__ICE_DOWN, pf->state); 2312 2313 /* In case of MSIX we are going to setup the misc vector right here 2314 * to handle admin queue events etc. In case of legacy and MSI 2315 * the misc functionality and queue processing is combined in 2316 * the same vector and that gets setup at open. 2317 */ 2318 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) { 2319 err = ice_req_irq_msix_misc(pf); 2320 if (err) { 2321 dev_err(dev, "setup of misc vector failed: %d\n", err); 2322 goto err_init_interrupt_unroll; 2323 } 2324 } 2325 2326 /* create switch struct for the switch element created by FW on boot */ 2327 pf->first_sw = devm_kzalloc(dev, sizeof(*pf->first_sw), GFP_KERNEL); 2328 if (!pf->first_sw) { 2329 err = -ENOMEM; 2330 goto err_msix_misc_unroll; 2331 } 2332 2333 if (hw->evb_veb) 2334 pf->first_sw->bridge_mode = BRIDGE_MODE_VEB; 2335 else 2336 pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA; 2337 2338 pf->first_sw->pf = pf; 2339 2340 /* record the sw_id available for later use */ 2341 pf->first_sw->sw_id = hw->port_info->sw_id; 2342 2343 err = ice_setup_pf_sw(pf); 2344 if (err) { 2345 dev_err(dev, "probe failed due to setup pf switch:%d\n", err); 2346 goto err_alloc_sw_unroll; 2347 } 2348 2349 clear_bit(__ICE_SERVICE_DIS, pf->state); 2350 2351 /* since everything is good, start the service timer */ 2352 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 2353 2354 err = ice_init_link_events(pf->hw.port_info); 2355 if (err) { 2356 dev_err(dev, "ice_init_link_events failed: %d\n", err); 2357 goto err_alloc_sw_unroll; 2358 } 2359 2360 ice_verify_cacheline_size(pf); 2361 2362 return 0; 2363 2364 err_alloc_sw_unroll: 2365 set_bit(__ICE_SERVICE_DIS, pf->state); 2366 set_bit(__ICE_DOWN, pf->state); 2367 devm_kfree(&pf->pdev->dev, pf->first_sw); 2368 err_msix_misc_unroll: 2369 ice_free_irq_msix_misc(pf); 2370 err_init_interrupt_unroll: 2371 ice_clear_interrupt_scheme(pf); 2372 devm_kfree(dev, pf->vsi); 2373 err_init_pf_unroll: 2374 ice_deinit_pf(pf); 2375 ice_deinit_hw(hw); 2376 err_exit_unroll: 2377 pci_disable_pcie_error_reporting(pdev); 2378 return err; 2379 } 2380 2381 /** 2382 * ice_remove - Device removal routine 2383 * @pdev: PCI device information struct 2384 */ 2385 static void ice_remove(struct pci_dev *pdev) 2386 { 2387 struct ice_pf *pf = pci_get_drvdata(pdev); 2388 int i; 2389 2390 if (!pf) 2391 return; 2392 2393 for (i = 0; i < ICE_MAX_RESET_WAIT; i++) { 2394 if (!ice_is_reset_in_progress(pf->state)) 2395 break; 2396 msleep(100); 2397 } 2398 2399 set_bit(__ICE_DOWN, pf->state); 2400 ice_service_task_stop(pf); 2401 2402 if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) 2403 ice_free_vfs(pf); 2404 ice_vsi_release_all(pf); 2405 ice_free_irq_msix_misc(pf); 2406 ice_for_each_vsi(pf, i) { 2407 if (!pf->vsi[i]) 2408 continue; 2409 ice_vsi_free_q_vectors(pf->vsi[i]); 2410 } 2411 ice_clear_interrupt_scheme(pf); 2412 ice_deinit_pf(pf); 2413 ice_deinit_hw(&pf->hw); 2414 pci_disable_pcie_error_reporting(pdev); 2415 } 2416 2417 /** 2418 * ice_pci_err_detected - warning that PCI error has been detected 2419 * @pdev: PCI device information struct 2420 * @err: the type of PCI error 2421 * 2422 * Called to warn that something happened on the PCI bus and the error handling 2423 * is in progress. Allows the driver to gracefully prepare/handle PCI errors. 2424 */ 2425 static pci_ers_result_t 2426 ice_pci_err_detected(struct pci_dev *pdev, enum pci_channel_state err) 2427 { 2428 struct ice_pf *pf = pci_get_drvdata(pdev); 2429 2430 if (!pf) { 2431 dev_err(&pdev->dev, "%s: unrecoverable device error %d\n", 2432 __func__, err); 2433 return PCI_ERS_RESULT_DISCONNECT; 2434 } 2435 2436 if (!test_bit(__ICE_SUSPENDED, pf->state)) { 2437 ice_service_task_stop(pf); 2438 2439 if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) { 2440 set_bit(__ICE_PFR_REQ, pf->state); 2441 ice_prepare_for_reset(pf); 2442 } 2443 } 2444 2445 return PCI_ERS_RESULT_NEED_RESET; 2446 } 2447 2448 /** 2449 * ice_pci_err_slot_reset - a PCI slot reset has just happened 2450 * @pdev: PCI device information struct 2451 * 2452 * Called to determine if the driver can recover from the PCI slot reset by 2453 * using a register read to determine if the device is recoverable. 2454 */ 2455 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) 2456 { 2457 struct ice_pf *pf = pci_get_drvdata(pdev); 2458 pci_ers_result_t result; 2459 int err; 2460 u32 reg; 2461 2462 err = pci_enable_device_mem(pdev); 2463 if (err) { 2464 dev_err(&pdev->dev, 2465 "Cannot re-enable PCI device after reset, error %d\n", 2466 err); 2467 result = PCI_ERS_RESULT_DISCONNECT; 2468 } else { 2469 pci_set_master(pdev); 2470 pci_restore_state(pdev); 2471 pci_save_state(pdev); 2472 pci_wake_from_d3(pdev, false); 2473 2474 /* Check for life */ 2475 reg = rd32(&pf->hw, GLGEN_RTRIG); 2476 if (!reg) 2477 result = PCI_ERS_RESULT_RECOVERED; 2478 else 2479 result = PCI_ERS_RESULT_DISCONNECT; 2480 } 2481 2482 err = pci_cleanup_aer_uncorrect_error_status(pdev); 2483 if (err) 2484 dev_dbg(&pdev->dev, 2485 "pci_cleanup_aer_uncorrect_error_status failed, error %d\n", 2486 err); 2487 /* non-fatal, continue */ 2488 2489 return result; 2490 } 2491 2492 /** 2493 * ice_pci_err_resume - restart operations after PCI error recovery 2494 * @pdev: PCI device information struct 2495 * 2496 * Called to allow the driver to bring things back up after PCI error and/or 2497 * reset recovery have finished 2498 */ 2499 static void ice_pci_err_resume(struct pci_dev *pdev) 2500 { 2501 struct ice_pf *pf = pci_get_drvdata(pdev); 2502 2503 if (!pf) { 2504 dev_err(&pdev->dev, 2505 "%s failed, device is unrecoverable\n", __func__); 2506 return; 2507 } 2508 2509 if (test_bit(__ICE_SUSPENDED, pf->state)) { 2510 dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n", 2511 __func__); 2512 return; 2513 } 2514 2515 ice_do_reset(pf, ICE_RESET_PFR); 2516 ice_service_task_restart(pf); 2517 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 2518 } 2519 2520 /** 2521 * ice_pci_err_reset_prepare - prepare device driver for PCI reset 2522 * @pdev: PCI device information struct 2523 */ 2524 static void ice_pci_err_reset_prepare(struct pci_dev *pdev) 2525 { 2526 struct ice_pf *pf = pci_get_drvdata(pdev); 2527 2528 if (!test_bit(__ICE_SUSPENDED, pf->state)) { 2529 ice_service_task_stop(pf); 2530 2531 if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) { 2532 set_bit(__ICE_PFR_REQ, pf->state); 2533 ice_prepare_for_reset(pf); 2534 } 2535 } 2536 } 2537 2538 /** 2539 * ice_pci_err_reset_done - PCI reset done, device driver reset can begin 2540 * @pdev: PCI device information struct 2541 */ 2542 static void ice_pci_err_reset_done(struct pci_dev *pdev) 2543 { 2544 ice_pci_err_resume(pdev); 2545 } 2546 2547 /* ice_pci_tbl - PCI Device ID Table 2548 * 2549 * Wildcard entries (PCI_ANY_ID) should come last 2550 * Last entry must be all 0s 2551 * 2552 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 2553 * Class, Class Mask, private data (not used) } 2554 */ 2555 static const struct pci_device_id ice_pci_tbl[] = { 2556 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE), 0 }, 2557 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP), 0 }, 2558 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP), 0 }, 2559 /* required last entry */ 2560 { 0, } 2561 }; 2562 MODULE_DEVICE_TABLE(pci, ice_pci_tbl); 2563 2564 static const struct pci_error_handlers ice_pci_err_handler = { 2565 .error_detected = ice_pci_err_detected, 2566 .slot_reset = ice_pci_err_slot_reset, 2567 .reset_prepare = ice_pci_err_reset_prepare, 2568 .reset_done = ice_pci_err_reset_done, 2569 .resume = ice_pci_err_resume 2570 }; 2571 2572 static struct pci_driver ice_driver = { 2573 .name = KBUILD_MODNAME, 2574 .id_table = ice_pci_tbl, 2575 .probe = ice_probe, 2576 .remove = ice_remove, 2577 .sriov_configure = ice_sriov_configure, 2578 .err_handler = &ice_pci_err_handler 2579 }; 2580 2581 /** 2582 * ice_module_init - Driver registration routine 2583 * 2584 * ice_module_init is the first routine called when the driver is 2585 * loaded. All it does is register with the PCI subsystem. 2586 */ 2587 static int __init ice_module_init(void) 2588 { 2589 int status; 2590 2591 pr_info("%s - version %s\n", ice_driver_string, ice_drv_ver); 2592 pr_info("%s\n", ice_copyright); 2593 2594 ice_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, KBUILD_MODNAME); 2595 if (!ice_wq) { 2596 pr_err("Failed to create workqueue\n"); 2597 return -ENOMEM; 2598 } 2599 2600 status = pci_register_driver(&ice_driver); 2601 if (status) { 2602 pr_err("failed to register pci driver, err %d\n", status); 2603 destroy_workqueue(ice_wq); 2604 } 2605 2606 return status; 2607 } 2608 module_init(ice_module_init); 2609 2610 /** 2611 * ice_module_exit - Driver exit cleanup routine 2612 * 2613 * ice_module_exit is called just before the driver is removed 2614 * from memory. 2615 */ 2616 static void __exit ice_module_exit(void) 2617 { 2618 pci_unregister_driver(&ice_driver); 2619 destroy_workqueue(ice_wq); 2620 pr_info("module unloaded\n"); 2621 } 2622 module_exit(ice_module_exit); 2623 2624 /** 2625 * ice_set_mac_address - NDO callback to set mac address 2626 * @netdev: network interface device structure 2627 * @pi: pointer to an address structure 2628 * 2629 * Returns 0 on success, negative on failure 2630 */ 2631 static int ice_set_mac_address(struct net_device *netdev, void *pi) 2632 { 2633 struct ice_netdev_priv *np = netdev_priv(netdev); 2634 struct ice_vsi *vsi = np->vsi; 2635 struct ice_pf *pf = vsi->back; 2636 struct ice_hw *hw = &pf->hw; 2637 struct sockaddr *addr = pi; 2638 enum ice_status status; 2639 LIST_HEAD(a_mac_list); 2640 LIST_HEAD(r_mac_list); 2641 u8 flags = 0; 2642 int err; 2643 u8 *mac; 2644 2645 mac = (u8 *)addr->sa_data; 2646 2647 if (!is_valid_ether_addr(mac)) 2648 return -EADDRNOTAVAIL; 2649 2650 if (ether_addr_equal(netdev->dev_addr, mac)) { 2651 netdev_warn(netdev, "already using mac %pM\n", mac); 2652 return 0; 2653 } 2654 2655 if (test_bit(__ICE_DOWN, pf->state) || 2656 ice_is_reset_in_progress(pf->state)) { 2657 netdev_err(netdev, "can't set mac %pM. device not ready\n", 2658 mac); 2659 return -EBUSY; 2660 } 2661 2662 /* When we change the mac address we also have to change the mac address 2663 * based filter rules that were created previously for the old mac 2664 * address. So first, we remove the old filter rule using ice_remove_mac 2665 * and then create a new filter rule using ice_add_mac. Note that for 2666 * both these operations, we first need to form a "list" of mac 2667 * addresses (even though in this case, we have only 1 mac address to be 2668 * added/removed) and this done using ice_add_mac_to_list. Depending on 2669 * the ensuing operation this "list" of mac addresses is either to be 2670 * added or removed from the filter. 2671 */ 2672 err = ice_add_mac_to_list(vsi, &r_mac_list, netdev->dev_addr); 2673 if (err) { 2674 err = -EADDRNOTAVAIL; 2675 goto free_lists; 2676 } 2677 2678 status = ice_remove_mac(hw, &r_mac_list); 2679 if (status) { 2680 err = -EADDRNOTAVAIL; 2681 goto free_lists; 2682 } 2683 2684 err = ice_add_mac_to_list(vsi, &a_mac_list, mac); 2685 if (err) { 2686 err = -EADDRNOTAVAIL; 2687 goto free_lists; 2688 } 2689 2690 status = ice_add_mac(hw, &a_mac_list); 2691 if (status) { 2692 err = -EADDRNOTAVAIL; 2693 goto free_lists; 2694 } 2695 2696 free_lists: 2697 /* free list entries */ 2698 ice_free_fltr_list(&pf->pdev->dev, &r_mac_list); 2699 ice_free_fltr_list(&pf->pdev->dev, &a_mac_list); 2700 2701 if (err) { 2702 netdev_err(netdev, "can't set mac %pM. filter update failed\n", 2703 mac); 2704 return err; 2705 } 2706 2707 /* change the netdev's mac address */ 2708 memcpy(netdev->dev_addr, mac, netdev->addr_len); 2709 netdev_dbg(vsi->netdev, "updated mac address to %pM\n", 2710 netdev->dev_addr); 2711 2712 /* write new mac address to the firmware */ 2713 flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL; 2714 status = ice_aq_manage_mac_write(hw, mac, flags, NULL); 2715 if (status) { 2716 netdev_err(netdev, "can't set mac %pM. write to firmware failed.\n", 2717 mac); 2718 } 2719 return 0; 2720 } 2721 2722 /** 2723 * ice_set_rx_mode - NDO callback to set the netdev filters 2724 * @netdev: network interface device structure 2725 */ 2726 static void ice_set_rx_mode(struct net_device *netdev) 2727 { 2728 struct ice_netdev_priv *np = netdev_priv(netdev); 2729 struct ice_vsi *vsi = np->vsi; 2730 2731 if (!vsi) 2732 return; 2733 2734 /* Set the flags to synchronize filters 2735 * ndo_set_rx_mode may be triggered even without a change in netdev 2736 * flags 2737 */ 2738 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 2739 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 2740 set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags); 2741 2742 /* schedule our worker thread which will take care of 2743 * applying the new filter changes 2744 */ 2745 ice_service_task_schedule(vsi->back); 2746 } 2747 2748 /** 2749 * ice_fdb_add - add an entry to the hardware database 2750 * @ndm: the input from the stack 2751 * @tb: pointer to array of nladdr (unused) 2752 * @dev: the net device pointer 2753 * @addr: the MAC address entry being added 2754 * @vid: VLAN id 2755 * @flags: instructions from stack about fdb operation 2756 * @extack: netlink extended ack 2757 */ 2758 static int 2759 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[], 2760 struct net_device *dev, const unsigned char *addr, u16 vid, 2761 u16 flags, struct netlink_ext_ack __always_unused *extack) 2762 { 2763 int err; 2764 2765 if (vid) { 2766 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n"); 2767 return -EINVAL; 2768 } 2769 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 2770 netdev_err(dev, "FDB only supports static addresses\n"); 2771 return -EINVAL; 2772 } 2773 2774 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 2775 err = dev_uc_add_excl(dev, addr); 2776 else if (is_multicast_ether_addr(addr)) 2777 err = dev_mc_add_excl(dev, addr); 2778 else 2779 err = -EINVAL; 2780 2781 /* Only return duplicate errors if NLM_F_EXCL is set */ 2782 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 2783 err = 0; 2784 2785 return err; 2786 } 2787 2788 /** 2789 * ice_fdb_del - delete an entry from the hardware database 2790 * @ndm: the input from the stack 2791 * @tb: pointer to array of nladdr (unused) 2792 * @dev: the net device pointer 2793 * @addr: the MAC address entry being added 2794 * @vid: VLAN id 2795 */ 2796 static int 2797 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[], 2798 struct net_device *dev, const unsigned char *addr, 2799 __always_unused u16 vid) 2800 { 2801 int err; 2802 2803 if (ndm->ndm_state & NUD_PERMANENT) { 2804 netdev_err(dev, "FDB only supports static addresses\n"); 2805 return -EINVAL; 2806 } 2807 2808 if (is_unicast_ether_addr(addr)) 2809 err = dev_uc_del(dev, addr); 2810 else if (is_multicast_ether_addr(addr)) 2811 err = dev_mc_del(dev, addr); 2812 else 2813 err = -EINVAL; 2814 2815 return err; 2816 } 2817 2818 /** 2819 * ice_set_features - set the netdev feature flags 2820 * @netdev: ptr to the netdev being adjusted 2821 * @features: the feature set that the stack is suggesting 2822 */ 2823 static int 2824 ice_set_features(struct net_device *netdev, netdev_features_t features) 2825 { 2826 struct ice_netdev_priv *np = netdev_priv(netdev); 2827 struct ice_vsi *vsi = np->vsi; 2828 int ret = 0; 2829 2830 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 2831 ret = ice_vsi_manage_rss_lut(vsi, true); 2832 else if (!(features & NETIF_F_RXHASH) && 2833 netdev->features & NETIF_F_RXHASH) 2834 ret = ice_vsi_manage_rss_lut(vsi, false); 2835 2836 if ((features & NETIF_F_HW_VLAN_CTAG_RX) && 2837 !(netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 2838 ret = ice_vsi_manage_vlan_stripping(vsi, true); 2839 else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && 2840 (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 2841 ret = ice_vsi_manage_vlan_stripping(vsi, false); 2842 else if ((features & NETIF_F_HW_VLAN_CTAG_TX) && 2843 !(netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) 2844 ret = ice_vsi_manage_vlan_insertion(vsi); 2845 else if (!(features & NETIF_F_HW_VLAN_CTAG_TX) && 2846 (netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) 2847 ret = ice_vsi_manage_vlan_insertion(vsi); 2848 2849 return ret; 2850 } 2851 2852 /** 2853 * ice_vsi_vlan_setup - Setup vlan offload properties on a VSI 2854 * @vsi: VSI to setup vlan properties for 2855 */ 2856 static int ice_vsi_vlan_setup(struct ice_vsi *vsi) 2857 { 2858 int ret = 0; 2859 2860 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2861 ret = ice_vsi_manage_vlan_stripping(vsi, true); 2862 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_TX) 2863 ret = ice_vsi_manage_vlan_insertion(vsi); 2864 2865 return ret; 2866 } 2867 2868 /** 2869 * ice_vsi_cfg - Setup the VSI 2870 * @vsi: the VSI being configured 2871 * 2872 * Return 0 on success and negative value on error 2873 */ 2874 static int ice_vsi_cfg(struct ice_vsi *vsi) 2875 { 2876 int err; 2877 2878 if (vsi->netdev) { 2879 ice_set_rx_mode(vsi->netdev); 2880 2881 err = ice_vsi_vlan_setup(vsi); 2882 2883 if (err) 2884 return err; 2885 } 2886 2887 err = ice_vsi_cfg_lan_txqs(vsi); 2888 if (!err) 2889 err = ice_vsi_cfg_rxqs(vsi); 2890 2891 return err; 2892 } 2893 2894 /** 2895 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI 2896 * @vsi: the VSI being configured 2897 */ 2898 static void ice_napi_enable_all(struct ice_vsi *vsi) 2899 { 2900 int q_idx; 2901 2902 if (!vsi->netdev) 2903 return; 2904 2905 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 2906 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 2907 2908 if (q_vector->rx.ring || q_vector->tx.ring) 2909 napi_enable(&q_vector->napi); 2910 } 2911 } 2912 2913 /** 2914 * ice_up_complete - Finish the last steps of bringing up a connection 2915 * @vsi: The VSI being configured 2916 * 2917 * Return 0 on success and negative value on error 2918 */ 2919 static int ice_up_complete(struct ice_vsi *vsi) 2920 { 2921 struct ice_pf *pf = vsi->back; 2922 int err; 2923 2924 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) 2925 ice_vsi_cfg_msix(vsi); 2926 else 2927 return -ENOTSUPP; 2928 2929 /* Enable only Rx rings, Tx rings were enabled by the FW when the 2930 * Tx queue group list was configured and the context bits were 2931 * programmed using ice_vsi_cfg_txqs 2932 */ 2933 err = ice_vsi_start_rx_rings(vsi); 2934 if (err) 2935 return err; 2936 2937 clear_bit(__ICE_DOWN, vsi->state); 2938 ice_napi_enable_all(vsi); 2939 ice_vsi_ena_irq(vsi); 2940 2941 if (vsi->port_info && 2942 (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) && 2943 vsi->netdev) { 2944 ice_print_link_msg(vsi, true); 2945 netif_tx_start_all_queues(vsi->netdev); 2946 netif_carrier_on(vsi->netdev); 2947 } 2948 2949 ice_service_task_schedule(pf); 2950 2951 return 0; 2952 } 2953 2954 /** 2955 * ice_up - Bring the connection back up after being down 2956 * @vsi: VSI being configured 2957 */ 2958 int ice_up(struct ice_vsi *vsi) 2959 { 2960 int err; 2961 2962 err = ice_vsi_cfg(vsi); 2963 if (!err) 2964 err = ice_up_complete(vsi); 2965 2966 return err; 2967 } 2968 2969 /** 2970 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring 2971 * @ring: Tx or Rx ring to read stats from 2972 * @pkts: packets stats counter 2973 * @bytes: bytes stats counter 2974 * 2975 * This function fetches stats from the ring considering the atomic operations 2976 * that needs to be performed to read u64 values in 32 bit machine. 2977 */ 2978 static void 2979 ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts, u64 *bytes) 2980 { 2981 unsigned int start; 2982 *pkts = 0; 2983 *bytes = 0; 2984 2985 if (!ring) 2986 return; 2987 do { 2988 start = u64_stats_fetch_begin_irq(&ring->syncp); 2989 *pkts = ring->stats.pkts; 2990 *bytes = ring->stats.bytes; 2991 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 2992 } 2993 2994 /** 2995 * ice_update_vsi_ring_stats - Update VSI stats counters 2996 * @vsi: the VSI to be updated 2997 */ 2998 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) 2999 { 3000 struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; 3001 struct ice_ring *ring; 3002 u64 pkts, bytes; 3003 int i; 3004 3005 /* reset netdev stats */ 3006 vsi_stats->tx_packets = 0; 3007 vsi_stats->tx_bytes = 0; 3008 vsi_stats->rx_packets = 0; 3009 vsi_stats->rx_bytes = 0; 3010 3011 /* reset non-netdev (extended) stats */ 3012 vsi->tx_restart = 0; 3013 vsi->tx_busy = 0; 3014 vsi->tx_linearize = 0; 3015 vsi->rx_buf_failed = 0; 3016 vsi->rx_page_failed = 0; 3017 3018 rcu_read_lock(); 3019 3020 /* update Tx rings counters */ 3021 ice_for_each_txq(vsi, i) { 3022 ring = READ_ONCE(vsi->tx_rings[i]); 3023 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); 3024 vsi_stats->tx_packets += pkts; 3025 vsi_stats->tx_bytes += bytes; 3026 vsi->tx_restart += ring->tx_stats.restart_q; 3027 vsi->tx_busy += ring->tx_stats.tx_busy; 3028 vsi->tx_linearize += ring->tx_stats.tx_linearize; 3029 } 3030 3031 /* update Rx rings counters */ 3032 ice_for_each_rxq(vsi, i) { 3033 ring = READ_ONCE(vsi->rx_rings[i]); 3034 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); 3035 vsi_stats->rx_packets += pkts; 3036 vsi_stats->rx_bytes += bytes; 3037 vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed; 3038 vsi->rx_page_failed += ring->rx_stats.alloc_page_failed; 3039 } 3040 3041 rcu_read_unlock(); 3042 } 3043 3044 /** 3045 * ice_update_vsi_stats - Update VSI stats counters 3046 * @vsi: the VSI to be updated 3047 */ 3048 static void ice_update_vsi_stats(struct ice_vsi *vsi) 3049 { 3050 struct rtnl_link_stats64 *cur_ns = &vsi->net_stats; 3051 struct ice_eth_stats *cur_es = &vsi->eth_stats; 3052 struct ice_pf *pf = vsi->back; 3053 3054 if (test_bit(__ICE_DOWN, vsi->state) || 3055 test_bit(__ICE_CFG_BUSY, pf->state)) 3056 return; 3057 3058 /* get stats as recorded by Tx/Rx rings */ 3059 ice_update_vsi_ring_stats(vsi); 3060 3061 /* get VSI stats as recorded by the hardware */ 3062 ice_update_eth_stats(vsi); 3063 3064 cur_ns->tx_errors = cur_es->tx_errors; 3065 cur_ns->rx_dropped = cur_es->rx_discards; 3066 cur_ns->tx_dropped = cur_es->tx_discards; 3067 cur_ns->multicast = cur_es->rx_multicast; 3068 3069 /* update some more netdev stats if this is main VSI */ 3070 if (vsi->type == ICE_VSI_PF) { 3071 cur_ns->rx_crc_errors = pf->stats.crc_errors; 3072 cur_ns->rx_errors = pf->stats.crc_errors + 3073 pf->stats.illegal_bytes; 3074 cur_ns->rx_length_errors = pf->stats.rx_len_errors; 3075 } 3076 } 3077 3078 /** 3079 * ice_update_pf_stats - Update PF port stats counters 3080 * @pf: PF whose stats needs to be updated 3081 */ 3082 static void ice_update_pf_stats(struct ice_pf *pf) 3083 { 3084 struct ice_hw_port_stats *prev_ps, *cur_ps; 3085 struct ice_hw *hw = &pf->hw; 3086 u8 pf_id; 3087 3088 prev_ps = &pf->stats_prev; 3089 cur_ps = &pf->stats; 3090 pf_id = hw->pf_id; 3091 3092 ice_stat_update40(hw, GLPRT_GORCH(pf_id), GLPRT_GORCL(pf_id), 3093 pf->stat_prev_loaded, &prev_ps->eth.rx_bytes, 3094 &cur_ps->eth.rx_bytes); 3095 3096 ice_stat_update40(hw, GLPRT_UPRCH(pf_id), GLPRT_UPRCL(pf_id), 3097 pf->stat_prev_loaded, &prev_ps->eth.rx_unicast, 3098 &cur_ps->eth.rx_unicast); 3099 3100 ice_stat_update40(hw, GLPRT_MPRCH(pf_id), GLPRT_MPRCL(pf_id), 3101 pf->stat_prev_loaded, &prev_ps->eth.rx_multicast, 3102 &cur_ps->eth.rx_multicast); 3103 3104 ice_stat_update40(hw, GLPRT_BPRCH(pf_id), GLPRT_BPRCL(pf_id), 3105 pf->stat_prev_loaded, &prev_ps->eth.rx_broadcast, 3106 &cur_ps->eth.rx_broadcast); 3107 3108 ice_stat_update40(hw, GLPRT_GOTCH(pf_id), GLPRT_GOTCL(pf_id), 3109 pf->stat_prev_loaded, &prev_ps->eth.tx_bytes, 3110 &cur_ps->eth.tx_bytes); 3111 3112 ice_stat_update40(hw, GLPRT_UPTCH(pf_id), GLPRT_UPTCL(pf_id), 3113 pf->stat_prev_loaded, &prev_ps->eth.tx_unicast, 3114 &cur_ps->eth.tx_unicast); 3115 3116 ice_stat_update40(hw, GLPRT_MPTCH(pf_id), GLPRT_MPTCL(pf_id), 3117 pf->stat_prev_loaded, &prev_ps->eth.tx_multicast, 3118 &cur_ps->eth.tx_multicast); 3119 3120 ice_stat_update40(hw, GLPRT_BPTCH(pf_id), GLPRT_BPTCL(pf_id), 3121 pf->stat_prev_loaded, &prev_ps->eth.tx_broadcast, 3122 &cur_ps->eth.tx_broadcast); 3123 3124 ice_stat_update32(hw, GLPRT_TDOLD(pf_id), pf->stat_prev_loaded, 3125 &prev_ps->tx_dropped_link_down, 3126 &cur_ps->tx_dropped_link_down); 3127 3128 ice_stat_update40(hw, GLPRT_PRC64H(pf_id), GLPRT_PRC64L(pf_id), 3129 pf->stat_prev_loaded, &prev_ps->rx_size_64, 3130 &cur_ps->rx_size_64); 3131 3132 ice_stat_update40(hw, GLPRT_PRC127H(pf_id), GLPRT_PRC127L(pf_id), 3133 pf->stat_prev_loaded, &prev_ps->rx_size_127, 3134 &cur_ps->rx_size_127); 3135 3136 ice_stat_update40(hw, GLPRT_PRC255H(pf_id), GLPRT_PRC255L(pf_id), 3137 pf->stat_prev_loaded, &prev_ps->rx_size_255, 3138 &cur_ps->rx_size_255); 3139 3140 ice_stat_update40(hw, GLPRT_PRC511H(pf_id), GLPRT_PRC511L(pf_id), 3141 pf->stat_prev_loaded, &prev_ps->rx_size_511, 3142 &cur_ps->rx_size_511); 3143 3144 ice_stat_update40(hw, GLPRT_PRC1023H(pf_id), 3145 GLPRT_PRC1023L(pf_id), pf->stat_prev_loaded, 3146 &prev_ps->rx_size_1023, &cur_ps->rx_size_1023); 3147 3148 ice_stat_update40(hw, GLPRT_PRC1522H(pf_id), 3149 GLPRT_PRC1522L(pf_id), pf->stat_prev_loaded, 3150 &prev_ps->rx_size_1522, &cur_ps->rx_size_1522); 3151 3152 ice_stat_update40(hw, GLPRT_PRC9522H(pf_id), 3153 GLPRT_PRC9522L(pf_id), pf->stat_prev_loaded, 3154 &prev_ps->rx_size_big, &cur_ps->rx_size_big); 3155 3156 ice_stat_update40(hw, GLPRT_PTC64H(pf_id), GLPRT_PTC64L(pf_id), 3157 pf->stat_prev_loaded, &prev_ps->tx_size_64, 3158 &cur_ps->tx_size_64); 3159 3160 ice_stat_update40(hw, GLPRT_PTC127H(pf_id), GLPRT_PTC127L(pf_id), 3161 pf->stat_prev_loaded, &prev_ps->tx_size_127, 3162 &cur_ps->tx_size_127); 3163 3164 ice_stat_update40(hw, GLPRT_PTC255H(pf_id), GLPRT_PTC255L(pf_id), 3165 pf->stat_prev_loaded, &prev_ps->tx_size_255, 3166 &cur_ps->tx_size_255); 3167 3168 ice_stat_update40(hw, GLPRT_PTC511H(pf_id), GLPRT_PTC511L(pf_id), 3169 pf->stat_prev_loaded, &prev_ps->tx_size_511, 3170 &cur_ps->tx_size_511); 3171 3172 ice_stat_update40(hw, GLPRT_PTC1023H(pf_id), 3173 GLPRT_PTC1023L(pf_id), pf->stat_prev_loaded, 3174 &prev_ps->tx_size_1023, &cur_ps->tx_size_1023); 3175 3176 ice_stat_update40(hw, GLPRT_PTC1522H(pf_id), 3177 GLPRT_PTC1522L(pf_id), pf->stat_prev_loaded, 3178 &prev_ps->tx_size_1522, &cur_ps->tx_size_1522); 3179 3180 ice_stat_update40(hw, GLPRT_PTC9522H(pf_id), 3181 GLPRT_PTC9522L(pf_id), pf->stat_prev_loaded, 3182 &prev_ps->tx_size_big, &cur_ps->tx_size_big); 3183 3184 ice_stat_update32(hw, GLPRT_LXONRXC(pf_id), pf->stat_prev_loaded, 3185 &prev_ps->link_xon_rx, &cur_ps->link_xon_rx); 3186 3187 ice_stat_update32(hw, GLPRT_LXOFFRXC(pf_id), pf->stat_prev_loaded, 3188 &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx); 3189 3190 ice_stat_update32(hw, GLPRT_LXONTXC(pf_id), pf->stat_prev_loaded, 3191 &prev_ps->link_xon_tx, &cur_ps->link_xon_tx); 3192 3193 ice_stat_update32(hw, GLPRT_LXOFFTXC(pf_id), pf->stat_prev_loaded, 3194 &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx); 3195 3196 ice_stat_update32(hw, GLPRT_CRCERRS(pf_id), pf->stat_prev_loaded, 3197 &prev_ps->crc_errors, &cur_ps->crc_errors); 3198 3199 ice_stat_update32(hw, GLPRT_ILLERRC(pf_id), pf->stat_prev_loaded, 3200 &prev_ps->illegal_bytes, &cur_ps->illegal_bytes); 3201 3202 ice_stat_update32(hw, GLPRT_MLFC(pf_id), pf->stat_prev_loaded, 3203 &prev_ps->mac_local_faults, 3204 &cur_ps->mac_local_faults); 3205 3206 ice_stat_update32(hw, GLPRT_MRFC(pf_id), pf->stat_prev_loaded, 3207 &prev_ps->mac_remote_faults, 3208 &cur_ps->mac_remote_faults); 3209 3210 ice_stat_update32(hw, GLPRT_RLEC(pf_id), pf->stat_prev_loaded, 3211 &prev_ps->rx_len_errors, &cur_ps->rx_len_errors); 3212 3213 ice_stat_update32(hw, GLPRT_RUC(pf_id), pf->stat_prev_loaded, 3214 &prev_ps->rx_undersize, &cur_ps->rx_undersize); 3215 3216 ice_stat_update32(hw, GLPRT_RFC(pf_id), pf->stat_prev_loaded, 3217 &prev_ps->rx_fragments, &cur_ps->rx_fragments); 3218 3219 ice_stat_update32(hw, GLPRT_ROC(pf_id), pf->stat_prev_loaded, 3220 &prev_ps->rx_oversize, &cur_ps->rx_oversize); 3221 3222 ice_stat_update32(hw, GLPRT_RJC(pf_id), pf->stat_prev_loaded, 3223 &prev_ps->rx_jabber, &cur_ps->rx_jabber); 3224 3225 pf->stat_prev_loaded = true; 3226 } 3227 3228 /** 3229 * ice_get_stats64 - get statistics for network device structure 3230 * @netdev: network interface device structure 3231 * @stats: main device statistics structure 3232 */ 3233 static 3234 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 3235 { 3236 struct ice_netdev_priv *np = netdev_priv(netdev); 3237 struct rtnl_link_stats64 *vsi_stats; 3238 struct ice_vsi *vsi = np->vsi; 3239 3240 vsi_stats = &vsi->net_stats; 3241 3242 if (test_bit(__ICE_DOWN, vsi->state) || !vsi->num_txq || !vsi->num_rxq) 3243 return; 3244 /* netdev packet/byte stats come from ring counter. These are obtained 3245 * by summing up ring counters (done by ice_update_vsi_ring_stats). 3246 */ 3247 ice_update_vsi_ring_stats(vsi); 3248 stats->tx_packets = vsi_stats->tx_packets; 3249 stats->tx_bytes = vsi_stats->tx_bytes; 3250 stats->rx_packets = vsi_stats->rx_packets; 3251 stats->rx_bytes = vsi_stats->rx_bytes; 3252 3253 /* The rest of the stats can be read from the hardware but instead we 3254 * just return values that the watchdog task has already obtained from 3255 * the hardware. 3256 */ 3257 stats->multicast = vsi_stats->multicast; 3258 stats->tx_errors = vsi_stats->tx_errors; 3259 stats->tx_dropped = vsi_stats->tx_dropped; 3260 stats->rx_errors = vsi_stats->rx_errors; 3261 stats->rx_dropped = vsi_stats->rx_dropped; 3262 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 3263 stats->rx_length_errors = vsi_stats->rx_length_errors; 3264 } 3265 3266 /** 3267 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI 3268 * @vsi: VSI having NAPI disabled 3269 */ 3270 static void ice_napi_disable_all(struct ice_vsi *vsi) 3271 { 3272 int q_idx; 3273 3274 if (!vsi->netdev) 3275 return; 3276 3277 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 3278 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 3279 3280 if (q_vector->rx.ring || q_vector->tx.ring) 3281 napi_disable(&q_vector->napi); 3282 } 3283 } 3284 3285 /** 3286 * ice_force_phys_link_state - Force the physical link state 3287 * @vsi: VSI to force the physical link state to up/down 3288 * @link_up: true/false indicates to set the physical link to up/down 3289 * 3290 * Force the physical link state by getting the current PHY capabilities from 3291 * hardware and setting the PHY config based on the determined capabilities. If 3292 * link changes a link event will be triggered because both the Enable Automatic 3293 * Link Update and LESM Enable bits are set when setting the PHY capabilities. 3294 * 3295 * Returns 0 on success, negative on failure 3296 */ 3297 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up) 3298 { 3299 struct ice_aqc_get_phy_caps_data *pcaps; 3300 struct ice_aqc_set_phy_cfg_data *cfg; 3301 struct ice_port_info *pi; 3302 struct device *dev; 3303 int retcode; 3304 3305 if (!vsi || !vsi->port_info || !vsi->back) 3306 return -EINVAL; 3307 if (vsi->type != ICE_VSI_PF) 3308 return 0; 3309 3310 dev = &vsi->back->pdev->dev; 3311 3312 pi = vsi->port_info; 3313 3314 pcaps = devm_kzalloc(dev, sizeof(*pcaps), GFP_KERNEL); 3315 if (!pcaps) 3316 return -ENOMEM; 3317 3318 retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_SW_CFG, pcaps, 3319 NULL); 3320 if (retcode) { 3321 dev_err(dev, 3322 "Failed to get phy capabilities, VSI %d error %d\n", 3323 vsi->vsi_num, retcode); 3324 retcode = -EIO; 3325 goto out; 3326 } 3327 3328 /* No change in link */ 3329 if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && 3330 link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) 3331 goto out; 3332 3333 cfg = devm_kzalloc(dev, sizeof(*cfg), GFP_KERNEL); 3334 if (!cfg) { 3335 retcode = -ENOMEM; 3336 goto out; 3337 } 3338 3339 cfg->phy_type_low = pcaps->phy_type_low; 3340 cfg->phy_type_high = pcaps->phy_type_high; 3341 cfg->caps = pcaps->caps | ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; 3342 cfg->low_power_ctrl = pcaps->low_power_ctrl; 3343 cfg->eee_cap = pcaps->eee_cap; 3344 cfg->eeer_value = pcaps->eeer_value; 3345 cfg->link_fec_opt = pcaps->link_fec_options; 3346 if (link_up) 3347 cfg->caps |= ICE_AQ_PHY_ENA_LINK; 3348 else 3349 cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; 3350 3351 retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi->lport, cfg, NULL); 3352 if (retcode) { 3353 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 3354 vsi->vsi_num, retcode); 3355 retcode = -EIO; 3356 } 3357 3358 devm_kfree(dev, cfg); 3359 out: 3360 devm_kfree(dev, pcaps); 3361 return retcode; 3362 } 3363 3364 /** 3365 * ice_down - Shutdown the connection 3366 * @vsi: The VSI being stopped 3367 */ 3368 int ice_down(struct ice_vsi *vsi) 3369 { 3370 int i, tx_err, rx_err, link_err = 0; 3371 3372 /* Caller of this function is expected to set the 3373 * vsi->state __ICE_DOWN bit 3374 */ 3375 if (vsi->netdev) { 3376 netif_carrier_off(vsi->netdev); 3377 netif_tx_disable(vsi->netdev); 3378 } 3379 3380 ice_vsi_dis_irq(vsi); 3381 3382 tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0); 3383 if (tx_err) 3384 netdev_err(vsi->netdev, 3385 "Failed stop Tx rings, VSI %d error %d\n", 3386 vsi->vsi_num, tx_err); 3387 3388 rx_err = ice_vsi_stop_rx_rings(vsi); 3389 if (rx_err) 3390 netdev_err(vsi->netdev, 3391 "Failed stop Rx rings, VSI %d error %d\n", 3392 vsi->vsi_num, rx_err); 3393 3394 ice_napi_disable_all(vsi); 3395 3396 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { 3397 link_err = ice_force_phys_link_state(vsi, false); 3398 if (link_err) 3399 netdev_err(vsi->netdev, 3400 "Failed to set physical link down, VSI %d error %d\n", 3401 vsi->vsi_num, link_err); 3402 } 3403 3404 ice_for_each_txq(vsi, i) 3405 ice_clean_tx_ring(vsi->tx_rings[i]); 3406 3407 ice_for_each_rxq(vsi, i) 3408 ice_clean_rx_ring(vsi->rx_rings[i]); 3409 3410 if (tx_err || rx_err || link_err) { 3411 netdev_err(vsi->netdev, 3412 "Failed to close VSI 0x%04X on switch 0x%04X\n", 3413 vsi->vsi_num, vsi->vsw->sw_id); 3414 return -EIO; 3415 } 3416 3417 return 0; 3418 } 3419 3420 /** 3421 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources 3422 * @vsi: VSI having resources allocated 3423 * 3424 * Return 0 on success, negative on failure 3425 */ 3426 static int ice_vsi_setup_tx_rings(struct ice_vsi *vsi) 3427 { 3428 int i, err = 0; 3429 3430 if (!vsi->num_txq) { 3431 dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Tx queues\n", 3432 vsi->vsi_num); 3433 return -EINVAL; 3434 } 3435 3436 ice_for_each_txq(vsi, i) { 3437 vsi->tx_rings[i]->netdev = vsi->netdev; 3438 err = ice_setup_tx_ring(vsi->tx_rings[i]); 3439 if (err) 3440 break; 3441 } 3442 3443 return err; 3444 } 3445 3446 /** 3447 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources 3448 * @vsi: VSI having resources allocated 3449 * 3450 * Return 0 on success, negative on failure 3451 */ 3452 static int ice_vsi_setup_rx_rings(struct ice_vsi *vsi) 3453 { 3454 int i, err = 0; 3455 3456 if (!vsi->num_rxq) { 3457 dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Rx queues\n", 3458 vsi->vsi_num); 3459 return -EINVAL; 3460 } 3461 3462 ice_for_each_rxq(vsi, i) { 3463 vsi->rx_rings[i]->netdev = vsi->netdev; 3464 err = ice_setup_rx_ring(vsi->rx_rings[i]); 3465 if (err) 3466 break; 3467 } 3468 3469 return err; 3470 } 3471 3472 /** 3473 * ice_vsi_req_irq - Request IRQ from the OS 3474 * @vsi: The VSI IRQ is being requested for 3475 * @basename: name for the vector 3476 * 3477 * Return 0 on success and a negative value on error 3478 */ 3479 static int ice_vsi_req_irq(struct ice_vsi *vsi, char *basename) 3480 { 3481 struct ice_pf *pf = vsi->back; 3482 int err = -EINVAL; 3483 3484 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) 3485 err = ice_vsi_req_irq_msix(vsi, basename); 3486 3487 return err; 3488 } 3489 3490 /** 3491 * ice_vsi_open - Called when a network interface is made active 3492 * @vsi: the VSI to open 3493 * 3494 * Initialization of the VSI 3495 * 3496 * Returns 0 on success, negative value on error 3497 */ 3498 static int ice_vsi_open(struct ice_vsi *vsi) 3499 { 3500 char int_name[ICE_INT_NAME_STR_LEN]; 3501 struct ice_pf *pf = vsi->back; 3502 int err; 3503 3504 /* allocate descriptors */ 3505 err = ice_vsi_setup_tx_rings(vsi); 3506 if (err) 3507 goto err_setup_tx; 3508 3509 err = ice_vsi_setup_rx_rings(vsi); 3510 if (err) 3511 goto err_setup_rx; 3512 3513 err = ice_vsi_cfg(vsi); 3514 if (err) 3515 goto err_setup_rx; 3516 3517 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 3518 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 3519 err = ice_vsi_req_irq(vsi, int_name); 3520 if (err) 3521 goto err_setup_rx; 3522 3523 /* Notify the stack of the actual queue counts. */ 3524 err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); 3525 if (err) 3526 goto err_set_qs; 3527 3528 err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq); 3529 if (err) 3530 goto err_set_qs; 3531 3532 err = ice_up_complete(vsi); 3533 if (err) 3534 goto err_up_complete; 3535 3536 return 0; 3537 3538 err_up_complete: 3539 ice_down(vsi); 3540 err_set_qs: 3541 ice_vsi_free_irq(vsi); 3542 err_setup_rx: 3543 ice_vsi_free_rx_rings(vsi); 3544 err_setup_tx: 3545 ice_vsi_free_tx_rings(vsi); 3546 3547 return err; 3548 } 3549 3550 /** 3551 * ice_vsi_release_all - Delete all VSIs 3552 * @pf: PF from which all VSIs are being removed 3553 */ 3554 static void ice_vsi_release_all(struct ice_pf *pf) 3555 { 3556 int err, i; 3557 3558 if (!pf->vsi) 3559 return; 3560 3561 ice_for_each_vsi(pf, i) { 3562 if (!pf->vsi[i]) 3563 continue; 3564 3565 err = ice_vsi_release(pf->vsi[i]); 3566 if (err) 3567 dev_dbg(&pf->pdev->dev, 3568 "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n", 3569 i, err, pf->vsi[i]->vsi_num); 3570 } 3571 } 3572 3573 /** 3574 * ice_dis_vsi - pause a VSI 3575 * @vsi: the VSI being paused 3576 * @locked: is the rtnl_lock already held 3577 */ 3578 static void ice_dis_vsi(struct ice_vsi *vsi, bool locked) 3579 { 3580 if (test_bit(__ICE_DOWN, vsi->state)) 3581 return; 3582 3583 set_bit(__ICE_NEEDS_RESTART, vsi->state); 3584 3585 if (vsi->type == ICE_VSI_PF && vsi->netdev) { 3586 if (netif_running(vsi->netdev)) { 3587 if (!locked) { 3588 rtnl_lock(); 3589 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 3590 rtnl_unlock(); 3591 } else { 3592 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 3593 } 3594 } else { 3595 ice_vsi_close(vsi); 3596 } 3597 } 3598 } 3599 3600 /** 3601 * ice_ena_vsi - resume a VSI 3602 * @vsi: the VSI being resume 3603 */ 3604 static int ice_ena_vsi(struct ice_vsi *vsi) 3605 { 3606 int err = 0; 3607 3608 if (test_and_clear_bit(__ICE_NEEDS_RESTART, vsi->state) && 3609 vsi->netdev) { 3610 if (netif_running(vsi->netdev)) { 3611 rtnl_lock(); 3612 err = vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 3613 rtnl_unlock(); 3614 } else { 3615 err = ice_vsi_open(vsi); 3616 } 3617 } 3618 3619 return err; 3620 } 3621 3622 /** 3623 * ice_pf_dis_all_vsi - Pause all VSIs on a PF 3624 * @pf: the PF 3625 */ 3626 static void ice_pf_dis_all_vsi(struct ice_pf *pf) 3627 { 3628 int v; 3629 3630 ice_for_each_vsi(pf, v) 3631 if (pf->vsi[v]) 3632 ice_dis_vsi(pf->vsi[v], false); 3633 } 3634 3635 /** 3636 * ice_pf_ena_all_vsi - Resume all VSIs on a PF 3637 * @pf: the PF 3638 */ 3639 static int ice_pf_ena_all_vsi(struct ice_pf *pf) 3640 { 3641 int v; 3642 3643 ice_for_each_vsi(pf, v) 3644 if (pf->vsi[v]) 3645 if (ice_ena_vsi(pf->vsi[v])) 3646 return -EIO; 3647 3648 return 0; 3649 } 3650 3651 /** 3652 * ice_vsi_rebuild_all - rebuild all VSIs in pf 3653 * @pf: the PF 3654 */ 3655 static int ice_vsi_rebuild_all(struct ice_pf *pf) 3656 { 3657 int i; 3658 3659 /* loop through pf->vsi array and reinit the VSI if found */ 3660 ice_for_each_vsi(pf, i) { 3661 int err; 3662 3663 if (!pf->vsi[i]) 3664 continue; 3665 3666 err = ice_vsi_rebuild(pf->vsi[i]); 3667 if (err) { 3668 dev_err(&pf->pdev->dev, 3669 "VSI at index %d rebuild failed\n", 3670 pf->vsi[i]->idx); 3671 return err; 3672 } 3673 3674 dev_info(&pf->pdev->dev, 3675 "VSI at index %d rebuilt. vsi_num = 0x%x\n", 3676 pf->vsi[i]->idx, pf->vsi[i]->vsi_num); 3677 } 3678 3679 return 0; 3680 } 3681 3682 /** 3683 * ice_vsi_replay_all - replay all VSIs configuration in the PF 3684 * @pf: the PF 3685 */ 3686 static int ice_vsi_replay_all(struct ice_pf *pf) 3687 { 3688 struct ice_hw *hw = &pf->hw; 3689 enum ice_status ret; 3690 int i; 3691 3692 /* loop through pf->vsi array and replay the VSI if found */ 3693 ice_for_each_vsi(pf, i) { 3694 if (!pf->vsi[i]) 3695 continue; 3696 3697 ret = ice_replay_vsi(hw, pf->vsi[i]->idx); 3698 if (ret) { 3699 dev_err(&pf->pdev->dev, 3700 "VSI at index %d replay failed %d\n", 3701 pf->vsi[i]->idx, ret); 3702 return -EIO; 3703 } 3704 3705 /* Re-map HW VSI number, using VSI handle that has been 3706 * previously validated in ice_replay_vsi() call above 3707 */ 3708 pf->vsi[i]->vsi_num = ice_get_hw_vsi_num(hw, pf->vsi[i]->idx); 3709 3710 dev_info(&pf->pdev->dev, 3711 "VSI at index %d filter replayed successfully - vsi_num %i\n", 3712 pf->vsi[i]->idx, pf->vsi[i]->vsi_num); 3713 } 3714 3715 /* Clean up replay filter after successful re-configuration */ 3716 ice_replay_post(hw); 3717 return 0; 3718 } 3719 3720 /** 3721 * ice_rebuild - rebuild after reset 3722 * @pf: pf to rebuild 3723 */ 3724 static void ice_rebuild(struct ice_pf *pf) 3725 { 3726 struct device *dev = &pf->pdev->dev; 3727 struct ice_hw *hw = &pf->hw; 3728 enum ice_status ret; 3729 int err, i; 3730 3731 if (test_bit(__ICE_DOWN, pf->state)) 3732 goto clear_recovery; 3733 3734 dev_dbg(dev, "rebuilding pf\n"); 3735 3736 ret = ice_init_all_ctrlq(hw); 3737 if (ret) { 3738 dev_err(dev, "control queues init failed %d\n", ret); 3739 goto err_init_ctrlq; 3740 } 3741 3742 ret = ice_clear_pf_cfg(hw); 3743 if (ret) { 3744 dev_err(dev, "clear PF configuration failed %d\n", ret); 3745 goto err_init_ctrlq; 3746 } 3747 3748 ice_clear_pxe_mode(hw); 3749 3750 ret = ice_get_caps(hw); 3751 if (ret) { 3752 dev_err(dev, "ice_get_caps failed %d\n", ret); 3753 goto err_init_ctrlq; 3754 } 3755 3756 err = ice_sched_init_port(hw->port_info); 3757 if (err) 3758 goto err_sched_init_port; 3759 3760 /* reset search_hint of irq_trackers to 0 since interrupts are 3761 * reclaimed and could be allocated from beginning during VSI rebuild 3762 */ 3763 pf->sw_irq_tracker->search_hint = 0; 3764 pf->hw_irq_tracker->search_hint = 0; 3765 3766 err = ice_vsi_rebuild_all(pf); 3767 if (err) { 3768 dev_err(dev, "ice_vsi_rebuild_all failed\n"); 3769 goto err_vsi_rebuild; 3770 } 3771 3772 err = ice_update_link_info(hw->port_info); 3773 if (err) 3774 dev_err(&pf->pdev->dev, "Get link status error %d\n", err); 3775 3776 /* Replay all VSIs Configuration, including filters after reset */ 3777 if (ice_vsi_replay_all(pf)) { 3778 dev_err(&pf->pdev->dev, 3779 "error replaying VSI configurations with switch filter rules\n"); 3780 goto err_vsi_rebuild; 3781 } 3782 3783 /* start misc vector */ 3784 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) { 3785 err = ice_req_irq_msix_misc(pf); 3786 if (err) { 3787 dev_err(dev, "misc vector setup failed: %d\n", err); 3788 goto err_vsi_rebuild; 3789 } 3790 } 3791 3792 /* restart the VSIs that were rebuilt and running before the reset */ 3793 err = ice_pf_ena_all_vsi(pf); 3794 if (err) { 3795 dev_err(&pf->pdev->dev, "error enabling VSIs\n"); 3796 /* no need to disable VSIs in tear down path in ice_rebuild() 3797 * since its already taken care in ice_vsi_open() 3798 */ 3799 goto err_vsi_rebuild; 3800 } 3801 3802 ice_for_each_vsi(pf, i) { 3803 bool link_up; 3804 3805 if (!pf->vsi[i] || pf->vsi[i]->type != ICE_VSI_PF) 3806 continue; 3807 ice_get_link_status(pf->vsi[i]->port_info, &link_up); 3808 if (link_up) { 3809 netif_carrier_on(pf->vsi[i]->netdev); 3810 netif_tx_wake_all_queues(pf->vsi[i]->netdev); 3811 } else { 3812 netif_carrier_off(pf->vsi[i]->netdev); 3813 netif_tx_stop_all_queues(pf->vsi[i]->netdev); 3814 } 3815 } 3816 3817 /* if we get here, reset flow is successful */ 3818 clear_bit(__ICE_RESET_FAILED, pf->state); 3819 return; 3820 3821 err_vsi_rebuild: 3822 ice_vsi_release_all(pf); 3823 err_sched_init_port: 3824 ice_sched_cleanup_all(hw); 3825 err_init_ctrlq: 3826 ice_shutdown_all_ctrlq(hw); 3827 set_bit(__ICE_RESET_FAILED, pf->state); 3828 clear_recovery: 3829 /* set this bit in PF state to control service task scheduling */ 3830 set_bit(__ICE_NEEDS_RESTART, pf->state); 3831 dev_err(dev, "Rebuild failed, unload and reload driver\n"); 3832 } 3833 3834 /** 3835 * ice_change_mtu - NDO callback to change the MTU 3836 * @netdev: network interface device structure 3837 * @new_mtu: new value for maximum frame size 3838 * 3839 * Returns 0 on success, negative on failure 3840 */ 3841 static int ice_change_mtu(struct net_device *netdev, int new_mtu) 3842 { 3843 struct ice_netdev_priv *np = netdev_priv(netdev); 3844 struct ice_vsi *vsi = np->vsi; 3845 struct ice_pf *pf = vsi->back; 3846 u8 count = 0; 3847 3848 if (new_mtu == netdev->mtu) { 3849 netdev_warn(netdev, "mtu is already %u\n", netdev->mtu); 3850 return 0; 3851 } 3852 3853 if (new_mtu < netdev->min_mtu) { 3854 netdev_err(netdev, "new mtu invalid. min_mtu is %d\n", 3855 netdev->min_mtu); 3856 return -EINVAL; 3857 } else if (new_mtu > netdev->max_mtu) { 3858 netdev_err(netdev, "new mtu invalid. max_mtu is %d\n", 3859 netdev->min_mtu); 3860 return -EINVAL; 3861 } 3862 /* if a reset is in progress, wait for some time for it to complete */ 3863 do { 3864 if (ice_is_reset_in_progress(pf->state)) { 3865 count++; 3866 usleep_range(1000, 2000); 3867 } else { 3868 break; 3869 } 3870 3871 } while (count < 100); 3872 3873 if (count == 100) { 3874 netdev_err(netdev, "can't change mtu. Device is busy\n"); 3875 return -EBUSY; 3876 } 3877 3878 netdev->mtu = new_mtu; 3879 3880 /* if VSI is up, bring it down and then back up */ 3881 if (!test_and_set_bit(__ICE_DOWN, vsi->state)) { 3882 int err; 3883 3884 err = ice_down(vsi); 3885 if (err) { 3886 netdev_err(netdev, "change mtu if_up err %d\n", err); 3887 return err; 3888 } 3889 3890 err = ice_up(vsi); 3891 if (err) { 3892 netdev_err(netdev, "change mtu if_up err %d\n", err); 3893 return err; 3894 } 3895 } 3896 3897 netdev_dbg(netdev, "changed mtu to %d\n", new_mtu); 3898 return 0; 3899 } 3900 3901 /** 3902 * ice_set_rss - Set RSS keys and lut 3903 * @vsi: Pointer to VSI structure 3904 * @seed: RSS hash seed 3905 * @lut: Lookup table 3906 * @lut_size: Lookup table size 3907 * 3908 * Returns 0 on success, negative on failure 3909 */ 3910 int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 3911 { 3912 struct ice_pf *pf = vsi->back; 3913 struct ice_hw *hw = &pf->hw; 3914 enum ice_status status; 3915 3916 if (seed) { 3917 struct ice_aqc_get_set_rss_keys *buf = 3918 (struct ice_aqc_get_set_rss_keys *)seed; 3919 3920 status = ice_aq_set_rss_key(hw, vsi->idx, buf); 3921 3922 if (status) { 3923 dev_err(&pf->pdev->dev, 3924 "Cannot set RSS key, err %d aq_err %d\n", 3925 status, hw->adminq.rq_last_status); 3926 return -EIO; 3927 } 3928 } 3929 3930 if (lut) { 3931 status = ice_aq_set_rss_lut(hw, vsi->idx, vsi->rss_lut_type, 3932 lut, lut_size); 3933 if (status) { 3934 dev_err(&pf->pdev->dev, 3935 "Cannot set RSS lut, err %d aq_err %d\n", 3936 status, hw->adminq.rq_last_status); 3937 return -EIO; 3938 } 3939 } 3940 3941 return 0; 3942 } 3943 3944 /** 3945 * ice_get_rss - Get RSS keys and lut 3946 * @vsi: Pointer to VSI structure 3947 * @seed: Buffer to store the keys 3948 * @lut: Buffer to store the lookup table entries 3949 * @lut_size: Size of buffer to store the lookup table entries 3950 * 3951 * Returns 0 on success, negative on failure 3952 */ 3953 int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 3954 { 3955 struct ice_pf *pf = vsi->back; 3956 struct ice_hw *hw = &pf->hw; 3957 enum ice_status status; 3958 3959 if (seed) { 3960 struct ice_aqc_get_set_rss_keys *buf = 3961 (struct ice_aqc_get_set_rss_keys *)seed; 3962 3963 status = ice_aq_get_rss_key(hw, vsi->idx, buf); 3964 if (status) { 3965 dev_err(&pf->pdev->dev, 3966 "Cannot get RSS key, err %d aq_err %d\n", 3967 status, hw->adminq.rq_last_status); 3968 return -EIO; 3969 } 3970 } 3971 3972 if (lut) { 3973 status = ice_aq_get_rss_lut(hw, vsi->idx, vsi->rss_lut_type, 3974 lut, lut_size); 3975 if (status) { 3976 dev_err(&pf->pdev->dev, 3977 "Cannot get RSS lut, err %d aq_err %d\n", 3978 status, hw->adminq.rq_last_status); 3979 return -EIO; 3980 } 3981 } 3982 3983 return 0; 3984 } 3985 3986 /** 3987 * ice_bridge_getlink - Get the hardware bridge mode 3988 * @skb: skb buff 3989 * @pid: process id 3990 * @seq: RTNL message seq 3991 * @dev: the netdev being configured 3992 * @filter_mask: filter mask passed in 3993 * @nlflags: netlink flags passed in 3994 * 3995 * Return the bridge mode (VEB/VEPA) 3996 */ 3997 static int 3998 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 3999 struct net_device *dev, u32 filter_mask, int nlflags) 4000 { 4001 struct ice_netdev_priv *np = netdev_priv(dev); 4002 struct ice_vsi *vsi = np->vsi; 4003 struct ice_pf *pf = vsi->back; 4004 u16 bmode; 4005 4006 bmode = pf->first_sw->bridge_mode; 4007 4008 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags, 4009 filter_mask, NULL); 4010 } 4011 4012 /** 4013 * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA) 4014 * @vsi: Pointer to VSI structure 4015 * @bmode: Hardware bridge mode (VEB/VEPA) 4016 * 4017 * Returns 0 on success, negative on failure 4018 */ 4019 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode) 4020 { 4021 struct device *dev = &vsi->back->pdev->dev; 4022 struct ice_aqc_vsi_props *vsi_props; 4023 struct ice_hw *hw = &vsi->back->hw; 4024 struct ice_vsi_ctx *ctxt; 4025 enum ice_status status; 4026 int ret = 0; 4027 4028 vsi_props = &vsi->info; 4029 4030 ctxt = devm_kzalloc(dev, sizeof(*ctxt), GFP_KERNEL); 4031 if (!ctxt) 4032 return -ENOMEM; 4033 4034 ctxt->info = vsi->info; 4035 4036 if (bmode == BRIDGE_MODE_VEB) 4037 /* change from VEPA to VEB mode */ 4038 ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 4039 else 4040 /* change from VEB to VEPA mode */ 4041 ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 4042 ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); 4043 4044 status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 4045 if (status) { 4046 dev_err(dev, "update VSI for bridge mode failed, bmode = %d err %d aq_err %d\n", 4047 bmode, status, hw->adminq.sq_last_status); 4048 ret = -EIO; 4049 goto out; 4050 } 4051 /* Update sw flags for book keeping */ 4052 vsi_props->sw_flags = ctxt->info.sw_flags; 4053 4054 out: 4055 devm_kfree(dev, ctxt); 4056 return ret; 4057 } 4058 4059 /** 4060 * ice_bridge_setlink - Set the hardware bridge mode 4061 * @dev: the netdev being configured 4062 * @nlh: RTNL message 4063 * @flags: bridge setlink flags 4064 * @extack: netlink extended ack 4065 * 4066 * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is 4067 * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if 4068 * not already set for all VSIs connected to this switch. And also update the 4069 * unicast switch filter rules for the corresponding switch of the netdev. 4070 */ 4071 static int 4072 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4073 u16 __always_unused flags, 4074 struct netlink_ext_ack __always_unused *extack) 4075 { 4076 struct ice_netdev_priv *np = netdev_priv(dev); 4077 struct ice_pf *pf = np->vsi->back; 4078 struct nlattr *attr, *br_spec; 4079 struct ice_hw *hw = &pf->hw; 4080 enum ice_status status; 4081 struct ice_sw *pf_sw; 4082 int rem, v, err = 0; 4083 4084 pf_sw = pf->first_sw; 4085 /* find the attribute in the netlink message */ 4086 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4087 4088 nla_for_each_nested(attr, br_spec, rem) { 4089 __u16 mode; 4090 4091 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4092 continue; 4093 mode = nla_get_u16(attr); 4094 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 4095 return -EINVAL; 4096 /* Continue if bridge mode is not being flipped */ 4097 if (mode == pf_sw->bridge_mode) 4098 continue; 4099 /* Iterates through the PF VSI list and update the loopback 4100 * mode of the VSI 4101 */ 4102 ice_for_each_vsi(pf, v) { 4103 if (!pf->vsi[v]) 4104 continue; 4105 err = ice_vsi_update_bridge_mode(pf->vsi[v], mode); 4106 if (err) 4107 return err; 4108 } 4109 4110 hw->evb_veb = (mode == BRIDGE_MODE_VEB); 4111 /* Update the unicast switch filter rules for the corresponding 4112 * switch of the netdev 4113 */ 4114 status = ice_update_sw_rule_bridge_mode(hw); 4115 if (status) { 4116 netdev_err(dev, "switch rule update failed, mode = %d err %d aq_err %d\n", 4117 mode, status, hw->adminq.sq_last_status); 4118 /* revert hw->evb_veb */ 4119 hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB); 4120 return -EIO; 4121 } 4122 4123 pf_sw->bridge_mode = mode; 4124 } 4125 4126 return 0; 4127 } 4128 4129 /** 4130 * ice_tx_timeout - Respond to a Tx Hang 4131 * @netdev: network interface device structure 4132 */ 4133 static void ice_tx_timeout(struct net_device *netdev) 4134 { 4135 struct ice_netdev_priv *np = netdev_priv(netdev); 4136 struct ice_ring *tx_ring = NULL; 4137 struct ice_vsi *vsi = np->vsi; 4138 struct ice_pf *pf = vsi->back; 4139 int hung_queue = -1; 4140 u32 i; 4141 4142 pf->tx_timeout_count++; 4143 4144 /* find the stopped queue the same way dev_watchdog() does */ 4145 for (i = 0; i < netdev->num_tx_queues; i++) { 4146 unsigned long trans_start; 4147 struct netdev_queue *q; 4148 4149 q = netdev_get_tx_queue(netdev, i); 4150 trans_start = q->trans_start; 4151 if (netif_xmit_stopped(q) && 4152 time_after(jiffies, 4153 trans_start + netdev->watchdog_timeo)) { 4154 hung_queue = i; 4155 break; 4156 } 4157 } 4158 4159 if (i == netdev->num_tx_queues) 4160 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 4161 else 4162 /* now that we have an index, find the tx_ring struct */ 4163 for (i = 0; i < vsi->num_txq; i++) 4164 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 4165 if (hung_queue == vsi->tx_rings[i]->q_index) { 4166 tx_ring = vsi->tx_rings[i]; 4167 break; 4168 } 4169 4170 /* Reset recovery level if enough time has elapsed after last timeout. 4171 * Also ensure no new reset action happens before next timeout period. 4172 */ 4173 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20))) 4174 pf->tx_timeout_recovery_level = 1; 4175 else if (time_before(jiffies, (pf->tx_timeout_last_recovery + 4176 netdev->watchdog_timeo))) 4177 return; 4178 4179 if (tx_ring) { 4180 struct ice_hw *hw = &pf->hw; 4181 u32 head, val = 0; 4182 4183 head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[hung_queue])) & 4184 QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S; 4185 /* Read interrupt register */ 4186 if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) 4187 val = rd32(hw, 4188 GLINT_DYN_CTL(tx_ring->q_vector->v_idx + 4189 tx_ring->vsi->hw_base_vector)); 4190 4191 netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %d, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n", 4192 vsi->vsi_num, hung_queue, tx_ring->next_to_clean, 4193 head, tx_ring->next_to_use, val); 4194 } 4195 4196 pf->tx_timeout_last_recovery = jiffies; 4197 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 4198 pf->tx_timeout_recovery_level, hung_queue); 4199 4200 switch (pf->tx_timeout_recovery_level) { 4201 case 1: 4202 set_bit(__ICE_PFR_REQ, pf->state); 4203 break; 4204 case 2: 4205 set_bit(__ICE_CORER_REQ, pf->state); 4206 break; 4207 case 3: 4208 set_bit(__ICE_GLOBR_REQ, pf->state); 4209 break; 4210 default: 4211 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n"); 4212 set_bit(__ICE_DOWN, pf->state); 4213 set_bit(__ICE_NEEDS_RESTART, vsi->state); 4214 set_bit(__ICE_SERVICE_DIS, pf->state); 4215 break; 4216 } 4217 4218 ice_service_task_schedule(pf); 4219 pf->tx_timeout_recovery_level++; 4220 } 4221 4222 /** 4223 * ice_open - Called when a network interface becomes active 4224 * @netdev: network interface device structure 4225 * 4226 * The open entry point is called when a network interface is made 4227 * active by the system (IFF_UP). At this point all resources needed 4228 * for transmit and receive operations are allocated, the interrupt 4229 * handler is registered with the OS, the netdev watchdog is enabled, 4230 * and the stack is notified that the interface is ready. 4231 * 4232 * Returns 0 on success, negative value on failure 4233 */ 4234 static int ice_open(struct net_device *netdev) 4235 { 4236 struct ice_netdev_priv *np = netdev_priv(netdev); 4237 struct ice_vsi *vsi = np->vsi; 4238 int err; 4239 4240 if (test_bit(__ICE_NEEDS_RESTART, vsi->back->state)) { 4241 netdev_err(netdev, "driver needs to be unloaded and reloaded\n"); 4242 return -EIO; 4243 } 4244 4245 netif_carrier_off(netdev); 4246 4247 err = ice_force_phys_link_state(vsi, true); 4248 if (err) { 4249 netdev_err(netdev, 4250 "Failed to set physical link up, error %d\n", err); 4251 return err; 4252 } 4253 4254 err = ice_vsi_open(vsi); 4255 if (err) 4256 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", 4257 vsi->vsi_num, vsi->vsw->sw_id); 4258 return err; 4259 } 4260 4261 /** 4262 * ice_stop - Disables a network interface 4263 * @netdev: network interface device structure 4264 * 4265 * The stop entry point is called when an interface is de-activated by the OS, 4266 * and the netdevice enters the DOWN state. The hardware is still under the 4267 * driver's control, but the netdev interface is disabled. 4268 * 4269 * Returns success only - not allowed to fail 4270 */ 4271 static int ice_stop(struct net_device *netdev) 4272 { 4273 struct ice_netdev_priv *np = netdev_priv(netdev); 4274 struct ice_vsi *vsi = np->vsi; 4275 4276 ice_vsi_close(vsi); 4277 4278 return 0; 4279 } 4280 4281 /** 4282 * ice_features_check - Validate encapsulated packet conforms to limits 4283 * @skb: skb buffer 4284 * @netdev: This port's netdev 4285 * @features: Offload features that the stack believes apply 4286 */ 4287 static netdev_features_t 4288 ice_features_check(struct sk_buff *skb, 4289 struct net_device __always_unused *netdev, 4290 netdev_features_t features) 4291 { 4292 size_t len; 4293 4294 /* No point in doing any of this if neither checksum nor GSO are 4295 * being requested for this frame. We can rule out both by just 4296 * checking for CHECKSUM_PARTIAL 4297 */ 4298 if (skb->ip_summed != CHECKSUM_PARTIAL) 4299 return features; 4300 4301 /* We cannot support GSO if the MSS is going to be less than 4302 * 64 bytes. If it is then we need to drop support for GSO. 4303 */ 4304 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 4305 features &= ~NETIF_F_GSO_MASK; 4306 4307 len = skb_network_header(skb) - skb->data; 4308 if (len & ~(ICE_TXD_MACLEN_MAX)) 4309 goto out_rm_features; 4310 4311 len = skb_transport_header(skb) - skb_network_header(skb); 4312 if (len & ~(ICE_TXD_IPLEN_MAX)) 4313 goto out_rm_features; 4314 4315 if (skb->encapsulation) { 4316 len = skb_inner_network_header(skb) - skb_transport_header(skb); 4317 if (len & ~(ICE_TXD_L4LEN_MAX)) 4318 goto out_rm_features; 4319 4320 len = skb_inner_transport_header(skb) - 4321 skb_inner_network_header(skb); 4322 if (len & ~(ICE_TXD_IPLEN_MAX)) 4323 goto out_rm_features; 4324 } 4325 4326 return features; 4327 out_rm_features: 4328 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 4329 } 4330 4331 static const struct net_device_ops ice_netdev_ops = { 4332 .ndo_open = ice_open, 4333 .ndo_stop = ice_stop, 4334 .ndo_start_xmit = ice_start_xmit, 4335 .ndo_features_check = ice_features_check, 4336 .ndo_set_rx_mode = ice_set_rx_mode, 4337 .ndo_set_mac_address = ice_set_mac_address, 4338 .ndo_validate_addr = eth_validate_addr, 4339 .ndo_change_mtu = ice_change_mtu, 4340 .ndo_get_stats64 = ice_get_stats64, 4341 .ndo_set_vf_spoofchk = ice_set_vf_spoofchk, 4342 .ndo_set_vf_mac = ice_set_vf_mac, 4343 .ndo_get_vf_config = ice_get_vf_cfg, 4344 .ndo_set_vf_trust = ice_set_vf_trust, 4345 .ndo_set_vf_vlan = ice_set_vf_port_vlan, 4346 .ndo_set_vf_link_state = ice_set_vf_link_state, 4347 .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid, 4348 .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid, 4349 .ndo_set_features = ice_set_features, 4350 .ndo_bridge_getlink = ice_bridge_getlink, 4351 .ndo_bridge_setlink = ice_bridge_setlink, 4352 .ndo_fdb_add = ice_fdb_add, 4353 .ndo_fdb_del = ice_fdb_del, 4354 .ndo_tx_timeout = ice_tx_timeout, 4355 }; 4356