1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, Intel Corporation. */ 3 4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <generated/utsrelease.h> 9 #include "ice.h" 10 #include "ice_base.h" 11 #include "ice_lib.h" 12 #include "ice_fltr.h" 13 #include "ice_dcb_lib.h" 14 #include "ice_dcb_nl.h" 15 #include "ice_devlink.h" 16 17 #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver" 18 static const char ice_driver_string[] = DRV_SUMMARY; 19 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation."; 20 21 /* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */ 22 #define ICE_DDP_PKG_PATH "intel/ice/ddp/" 23 #define ICE_DDP_PKG_FILE ICE_DDP_PKG_PATH "ice.pkg" 24 25 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 26 MODULE_DESCRIPTION(DRV_SUMMARY); 27 MODULE_LICENSE("GPL v2"); 28 MODULE_FIRMWARE(ICE_DDP_PKG_FILE); 29 30 static int debug = -1; 31 module_param(debug, int, 0644); 32 #ifndef CONFIG_DYNAMIC_DEBUG 33 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)"); 34 #else 35 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)"); 36 #endif /* !CONFIG_DYNAMIC_DEBUG */ 37 38 static struct workqueue_struct *ice_wq; 39 static const struct net_device_ops ice_netdev_safe_mode_ops; 40 static const struct net_device_ops ice_netdev_ops; 41 static int ice_vsi_open(struct ice_vsi *vsi); 42 43 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type); 44 45 static void ice_vsi_release_all(struct ice_pf *pf); 46 47 bool netif_is_ice(struct net_device *dev) 48 { 49 return dev && (dev->netdev_ops == &ice_netdev_ops); 50 } 51 52 /** 53 * ice_get_tx_pending - returns number of Tx descriptors not processed 54 * @ring: the ring of descriptors 55 */ 56 static u16 ice_get_tx_pending(struct ice_ring *ring) 57 { 58 u16 head, tail; 59 60 head = ring->next_to_clean; 61 tail = ring->next_to_use; 62 63 if (head != tail) 64 return (head < tail) ? 65 tail - head : (tail + ring->count - head); 66 return 0; 67 } 68 69 /** 70 * ice_check_for_hang_subtask - check for and recover hung queues 71 * @pf: pointer to PF struct 72 */ 73 static void ice_check_for_hang_subtask(struct ice_pf *pf) 74 { 75 struct ice_vsi *vsi = NULL; 76 struct ice_hw *hw; 77 unsigned int i; 78 int packets; 79 u32 v; 80 81 ice_for_each_vsi(pf, v) 82 if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) { 83 vsi = pf->vsi[v]; 84 break; 85 } 86 87 if (!vsi || test_bit(__ICE_DOWN, vsi->state)) 88 return; 89 90 if (!(vsi->netdev && netif_carrier_ok(vsi->netdev))) 91 return; 92 93 hw = &vsi->back->hw; 94 95 for (i = 0; i < vsi->num_txq; i++) { 96 struct ice_ring *tx_ring = vsi->tx_rings[i]; 97 98 if (tx_ring && tx_ring->desc) { 99 /* If packet counter has not changed the queue is 100 * likely stalled, so force an interrupt for this 101 * queue. 102 * 103 * prev_pkt would be negative if there was no 104 * pending work. 105 */ 106 packets = tx_ring->stats.pkts & INT_MAX; 107 if (tx_ring->tx_stats.prev_pkt == packets) { 108 /* Trigger sw interrupt to revive the queue */ 109 ice_trigger_sw_intr(hw, tx_ring->q_vector); 110 continue; 111 } 112 113 /* Memory barrier between read of packet count and call 114 * to ice_get_tx_pending() 115 */ 116 smp_rmb(); 117 tx_ring->tx_stats.prev_pkt = 118 ice_get_tx_pending(tx_ring) ? packets : -1; 119 } 120 } 121 } 122 123 /** 124 * ice_init_mac_fltr - Set initial MAC filters 125 * @pf: board private structure 126 * 127 * Set initial set of MAC filters for PF VSI; configure filters for permanent 128 * address and broadcast address. If an error is encountered, netdevice will be 129 * unregistered. 130 */ 131 static int ice_init_mac_fltr(struct ice_pf *pf) 132 { 133 enum ice_status status; 134 struct ice_vsi *vsi; 135 u8 *perm_addr; 136 137 vsi = ice_get_main_vsi(pf); 138 if (!vsi) 139 return -EINVAL; 140 141 perm_addr = vsi->port_info->mac.perm_addr; 142 status = ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI); 143 if (!status) 144 return 0; 145 146 /* We aren't useful with no MAC filters, so unregister if we 147 * had an error 148 */ 149 if (vsi->netdev->reg_state == NETREG_REGISTERED) { 150 dev_err(ice_pf_to_dev(pf), "Could not add MAC filters error %s. Unregistering device\n", 151 ice_stat_str(status)); 152 unregister_netdev(vsi->netdev); 153 free_netdev(vsi->netdev); 154 vsi->netdev = NULL; 155 } 156 157 return -EIO; 158 } 159 160 /** 161 * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced 162 * @netdev: the net device on which the sync is happening 163 * @addr: MAC address to sync 164 * 165 * This is a callback function which is called by the in kernel device sync 166 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only 167 * populates the tmp_sync_list, which is later used by ice_add_mac to add the 168 * MAC filters from the hardware. 169 */ 170 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr) 171 { 172 struct ice_netdev_priv *np = netdev_priv(netdev); 173 struct ice_vsi *vsi = np->vsi; 174 175 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr, 176 ICE_FWD_TO_VSI)) 177 return -EINVAL; 178 179 return 0; 180 } 181 182 /** 183 * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced 184 * @netdev: the net device on which the unsync is happening 185 * @addr: MAC address to unsync 186 * 187 * This is a callback function which is called by the in kernel device unsync 188 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only 189 * populates the tmp_unsync_list, which is later used by ice_remove_mac to 190 * delete the MAC filters from the hardware. 191 */ 192 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) 193 { 194 struct ice_netdev_priv *np = netdev_priv(netdev); 195 struct ice_vsi *vsi = np->vsi; 196 197 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr, 198 ICE_FWD_TO_VSI)) 199 return -EINVAL; 200 201 return 0; 202 } 203 204 /** 205 * ice_vsi_fltr_changed - check if filter state changed 206 * @vsi: VSI to be checked 207 * 208 * returns true if filter state has changed, false otherwise. 209 */ 210 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi) 211 { 212 return test_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags) || 213 test_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags) || 214 test_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 215 } 216 217 /** 218 * ice_cfg_promisc - Enable or disable promiscuous mode for a given PF 219 * @vsi: the VSI being configured 220 * @promisc_m: mask of promiscuous config bits 221 * @set_promisc: enable or disable promisc flag request 222 * 223 */ 224 static int ice_cfg_promisc(struct ice_vsi *vsi, u8 promisc_m, bool set_promisc) 225 { 226 struct ice_hw *hw = &vsi->back->hw; 227 enum ice_status status = 0; 228 229 if (vsi->type != ICE_VSI_PF) 230 return 0; 231 232 if (vsi->num_vlan > 1) { 233 status = ice_set_vlan_vsi_promisc(hw, vsi->idx, promisc_m, 234 set_promisc); 235 } else { 236 if (set_promisc) 237 status = ice_set_vsi_promisc(hw, vsi->idx, promisc_m, 238 0); 239 else 240 status = ice_clear_vsi_promisc(hw, vsi->idx, promisc_m, 241 0); 242 } 243 244 if (status) 245 return -EIO; 246 247 return 0; 248 } 249 250 /** 251 * ice_vsi_sync_fltr - Update the VSI filter list to the HW 252 * @vsi: ptr to the VSI 253 * 254 * Push any outstanding VSI filter changes through the AdminQ. 255 */ 256 static int ice_vsi_sync_fltr(struct ice_vsi *vsi) 257 { 258 struct device *dev = ice_pf_to_dev(vsi->back); 259 struct net_device *netdev = vsi->netdev; 260 bool promisc_forced_on = false; 261 struct ice_pf *pf = vsi->back; 262 struct ice_hw *hw = &pf->hw; 263 enum ice_status status = 0; 264 u32 changed_flags = 0; 265 u8 promisc_m; 266 int err = 0; 267 268 if (!vsi->netdev) 269 return -EINVAL; 270 271 while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state)) 272 usleep_range(1000, 2000); 273 274 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 275 vsi->current_netdev_flags = vsi->netdev->flags; 276 277 INIT_LIST_HEAD(&vsi->tmp_sync_list); 278 INIT_LIST_HEAD(&vsi->tmp_unsync_list); 279 280 if (ice_vsi_fltr_changed(vsi)) { 281 clear_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 282 clear_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 283 clear_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 284 285 /* grab the netdev's addr_list_lock */ 286 netif_addr_lock_bh(netdev); 287 __dev_uc_sync(netdev, ice_add_mac_to_sync_list, 288 ice_add_mac_to_unsync_list); 289 __dev_mc_sync(netdev, ice_add_mac_to_sync_list, 290 ice_add_mac_to_unsync_list); 291 /* our temp lists are populated. release lock */ 292 netif_addr_unlock_bh(netdev); 293 } 294 295 /* Remove MAC addresses in the unsync list */ 296 status = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list); 297 ice_fltr_free_list(dev, &vsi->tmp_unsync_list); 298 if (status) { 299 netdev_err(netdev, "Failed to delete MAC filters\n"); 300 /* if we failed because of alloc failures, just bail */ 301 if (status == ICE_ERR_NO_MEMORY) { 302 err = -ENOMEM; 303 goto out; 304 } 305 } 306 307 /* Add MAC addresses in the sync list */ 308 status = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list); 309 ice_fltr_free_list(dev, &vsi->tmp_sync_list); 310 /* If filter is added successfully or already exists, do not go into 311 * 'if' condition and report it as error. Instead continue processing 312 * rest of the function. 313 */ 314 if (status && status != ICE_ERR_ALREADY_EXISTS) { 315 netdev_err(netdev, "Failed to add MAC filters\n"); 316 /* If there is no more space for new umac filters, VSI 317 * should go into promiscuous mode. There should be some 318 * space reserved for promiscuous filters. 319 */ 320 if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC && 321 !test_and_set_bit(__ICE_FLTR_OVERFLOW_PROMISC, 322 vsi->state)) { 323 promisc_forced_on = true; 324 netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n", 325 vsi->vsi_num); 326 } else { 327 err = -EIO; 328 goto out; 329 } 330 } 331 /* check for changes in promiscuous modes */ 332 if (changed_flags & IFF_ALLMULTI) { 333 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 334 if (vsi->num_vlan > 1) 335 promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; 336 else 337 promisc_m = ICE_MCAST_PROMISC_BITS; 338 339 err = ice_cfg_promisc(vsi, promisc_m, true); 340 if (err) { 341 netdev_err(netdev, "Error setting Multicast promiscuous mode on VSI %i\n", 342 vsi->vsi_num); 343 vsi->current_netdev_flags &= ~IFF_ALLMULTI; 344 goto out_promisc; 345 } 346 } else { 347 /* !(vsi->current_netdev_flags & IFF_ALLMULTI) */ 348 if (vsi->num_vlan > 1) 349 promisc_m = ICE_MCAST_VLAN_PROMISC_BITS; 350 else 351 promisc_m = ICE_MCAST_PROMISC_BITS; 352 353 err = ice_cfg_promisc(vsi, promisc_m, false); 354 if (err) { 355 netdev_err(netdev, "Error clearing Multicast promiscuous mode on VSI %i\n", 356 vsi->vsi_num); 357 vsi->current_netdev_flags |= IFF_ALLMULTI; 358 goto out_promisc; 359 } 360 } 361 } 362 363 if (((changed_flags & IFF_PROMISC) || promisc_forced_on) || 364 test_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags)) { 365 clear_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags); 366 if (vsi->current_netdev_flags & IFF_PROMISC) { 367 /* Apply Rx filter rule to get traffic from wire */ 368 if (!ice_is_dflt_vsi_in_use(pf->first_sw)) { 369 err = ice_set_dflt_vsi(pf->first_sw, vsi); 370 if (err && err != -EEXIST) { 371 netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n", 372 err, vsi->vsi_num); 373 vsi->current_netdev_flags &= 374 ~IFF_PROMISC; 375 goto out_promisc; 376 } 377 ice_cfg_vlan_pruning(vsi, false, false); 378 } 379 } else { 380 /* Clear Rx filter to remove traffic from wire */ 381 if (ice_is_vsi_dflt_vsi(pf->first_sw, vsi)) { 382 err = ice_clear_dflt_vsi(pf->first_sw); 383 if (err) { 384 netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n", 385 err, vsi->vsi_num); 386 vsi->current_netdev_flags |= 387 IFF_PROMISC; 388 goto out_promisc; 389 } 390 if (vsi->num_vlan > 1) 391 ice_cfg_vlan_pruning(vsi, true, false); 392 } 393 } 394 } 395 goto exit; 396 397 out_promisc: 398 set_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags); 399 goto exit; 400 out: 401 /* if something went wrong then set the changed flag so we try again */ 402 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 403 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 404 exit: 405 clear_bit(__ICE_CFG_BUSY, vsi->state); 406 return err; 407 } 408 409 /** 410 * ice_sync_fltr_subtask - Sync the VSI filter list with HW 411 * @pf: board private structure 412 */ 413 static void ice_sync_fltr_subtask(struct ice_pf *pf) 414 { 415 int v; 416 417 if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags))) 418 return; 419 420 clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 421 422 ice_for_each_vsi(pf, v) 423 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) && 424 ice_vsi_sync_fltr(pf->vsi[v])) { 425 /* come back and try again later */ 426 set_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 427 break; 428 } 429 } 430 431 /** 432 * ice_pf_dis_all_vsi - Pause all VSIs on a PF 433 * @pf: the PF 434 * @locked: is the rtnl_lock already held 435 */ 436 static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked) 437 { 438 int node; 439 int v; 440 441 ice_for_each_vsi(pf, v) 442 if (pf->vsi[v]) 443 ice_dis_vsi(pf->vsi[v], locked); 444 445 for (node = 0; node < ICE_MAX_PF_AGG_NODES; node++) 446 pf->pf_agg_node[node].num_vsis = 0; 447 448 for (node = 0; node < ICE_MAX_VF_AGG_NODES; node++) 449 pf->vf_agg_node[node].num_vsis = 0; 450 451 } 452 453 /** 454 * ice_prepare_for_reset - prep for the core to reset 455 * @pf: board private structure 456 * 457 * Inform or close all dependent features in prep for reset. 458 */ 459 static void 460 ice_prepare_for_reset(struct ice_pf *pf) 461 { 462 struct ice_hw *hw = &pf->hw; 463 unsigned int i; 464 465 /* already prepared for reset */ 466 if (test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) 467 return; 468 469 /* Notify VFs of impending reset */ 470 if (ice_check_sq_alive(hw, &hw->mailboxq)) 471 ice_vc_notify_reset(pf); 472 473 /* Disable VFs until reset is completed */ 474 ice_for_each_vf(pf, i) 475 ice_set_vf_state_qs_dis(&pf->vf[i]); 476 477 /* clear SW filtering DB */ 478 ice_clear_hw_tbls(hw); 479 /* disable the VSIs and their queues that are not already DOWN */ 480 ice_pf_dis_all_vsi(pf, false); 481 482 if (hw->port_info) 483 ice_sched_clear_port(hw->port_info); 484 485 ice_shutdown_all_ctrlq(hw); 486 487 set_bit(__ICE_PREPARED_FOR_RESET, pf->state); 488 } 489 490 /** 491 * ice_do_reset - Initiate one of many types of resets 492 * @pf: board private structure 493 * @reset_type: reset type requested 494 * before this function was called. 495 */ 496 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 497 { 498 struct device *dev = ice_pf_to_dev(pf); 499 struct ice_hw *hw = &pf->hw; 500 501 dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); 502 503 ice_prepare_for_reset(pf); 504 505 /* trigger the reset */ 506 if (ice_reset(hw, reset_type)) { 507 dev_err(dev, "reset %d failed\n", reset_type); 508 set_bit(__ICE_RESET_FAILED, pf->state); 509 clear_bit(__ICE_RESET_OICR_RECV, pf->state); 510 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 511 clear_bit(__ICE_PFR_REQ, pf->state); 512 clear_bit(__ICE_CORER_REQ, pf->state); 513 clear_bit(__ICE_GLOBR_REQ, pf->state); 514 return; 515 } 516 517 /* PFR is a bit of a special case because it doesn't result in an OICR 518 * interrupt. So for PFR, rebuild after the reset and clear the reset- 519 * associated state bits. 520 */ 521 if (reset_type == ICE_RESET_PFR) { 522 pf->pfr_count++; 523 ice_rebuild(pf, reset_type); 524 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 525 clear_bit(__ICE_PFR_REQ, pf->state); 526 ice_reset_all_vfs(pf, true); 527 } 528 } 529 530 /** 531 * ice_reset_subtask - Set up for resetting the device and driver 532 * @pf: board private structure 533 */ 534 static void ice_reset_subtask(struct ice_pf *pf) 535 { 536 enum ice_reset_req reset_type = ICE_RESET_INVAL; 537 538 /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an 539 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type 540 * of reset is pending and sets bits in pf->state indicating the reset 541 * type and __ICE_RESET_OICR_RECV. So, if the latter bit is set 542 * prepare for pending reset if not already (for PF software-initiated 543 * global resets the software should already be prepared for it as 544 * indicated by __ICE_PREPARED_FOR_RESET; for global resets initiated 545 * by firmware or software on other PFs, that bit is not set so prepare 546 * for the reset now), poll for reset done, rebuild and return. 547 */ 548 if (test_bit(__ICE_RESET_OICR_RECV, pf->state)) { 549 /* Perform the largest reset requested */ 550 if (test_and_clear_bit(__ICE_CORER_RECV, pf->state)) 551 reset_type = ICE_RESET_CORER; 552 if (test_and_clear_bit(__ICE_GLOBR_RECV, pf->state)) 553 reset_type = ICE_RESET_GLOBR; 554 if (test_and_clear_bit(__ICE_EMPR_RECV, pf->state)) 555 reset_type = ICE_RESET_EMPR; 556 /* return if no valid reset type requested */ 557 if (reset_type == ICE_RESET_INVAL) 558 return; 559 ice_prepare_for_reset(pf); 560 561 /* make sure we are ready to rebuild */ 562 if (ice_check_reset(&pf->hw)) { 563 set_bit(__ICE_RESET_FAILED, pf->state); 564 } else { 565 /* done with reset. start rebuild */ 566 pf->hw.reset_ongoing = false; 567 ice_rebuild(pf, reset_type); 568 /* clear bit to resume normal operations, but 569 * ICE_NEEDS_RESTART bit is set in case rebuild failed 570 */ 571 clear_bit(__ICE_RESET_OICR_RECV, pf->state); 572 clear_bit(__ICE_PREPARED_FOR_RESET, pf->state); 573 clear_bit(__ICE_PFR_REQ, pf->state); 574 clear_bit(__ICE_CORER_REQ, pf->state); 575 clear_bit(__ICE_GLOBR_REQ, pf->state); 576 ice_reset_all_vfs(pf, true); 577 } 578 579 return; 580 } 581 582 /* No pending resets to finish processing. Check for new resets */ 583 if (test_bit(__ICE_PFR_REQ, pf->state)) 584 reset_type = ICE_RESET_PFR; 585 if (test_bit(__ICE_CORER_REQ, pf->state)) 586 reset_type = ICE_RESET_CORER; 587 if (test_bit(__ICE_GLOBR_REQ, pf->state)) 588 reset_type = ICE_RESET_GLOBR; 589 /* If no valid reset type requested just return */ 590 if (reset_type == ICE_RESET_INVAL) 591 return; 592 593 /* reset if not already down or busy */ 594 if (!test_bit(__ICE_DOWN, pf->state) && 595 !test_bit(__ICE_CFG_BUSY, pf->state)) { 596 ice_do_reset(pf, reset_type); 597 } 598 } 599 600 /** 601 * ice_print_topo_conflict - print topology conflict message 602 * @vsi: the VSI whose topology status is being checked 603 */ 604 static void ice_print_topo_conflict(struct ice_vsi *vsi) 605 { 606 switch (vsi->port_info->phy.link_info.topo_media_conflict) { 607 case ICE_AQ_LINK_TOPO_CONFLICT: 608 case ICE_AQ_LINK_MEDIA_CONFLICT: 609 case ICE_AQ_LINK_TOPO_UNREACH_PRT: 610 case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT: 611 case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA: 612 netdev_info(vsi->netdev, "Possible mis-configuration of the Ethernet port detected, please use the Intel(R) Ethernet Port Configuration Tool application to address the issue.\n"); 613 break; 614 case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA: 615 netdev_info(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 616 break; 617 default: 618 break; 619 } 620 } 621 622 /** 623 * ice_print_link_msg - print link up or down message 624 * @vsi: the VSI whose link status is being queried 625 * @isup: boolean for if the link is now up or down 626 */ 627 void ice_print_link_msg(struct ice_vsi *vsi, bool isup) 628 { 629 struct ice_aqc_get_phy_caps_data *caps; 630 const char *an_advertised; 631 enum ice_status status; 632 const char *fec_req; 633 const char *speed; 634 const char *fec; 635 const char *fc; 636 const char *an; 637 638 if (!vsi) 639 return; 640 641 if (vsi->current_isup == isup) 642 return; 643 644 vsi->current_isup = isup; 645 646 if (!isup) { 647 netdev_info(vsi->netdev, "NIC Link is Down\n"); 648 return; 649 } 650 651 switch (vsi->port_info->phy.link_info.link_speed) { 652 case ICE_AQ_LINK_SPEED_100GB: 653 speed = "100 G"; 654 break; 655 case ICE_AQ_LINK_SPEED_50GB: 656 speed = "50 G"; 657 break; 658 case ICE_AQ_LINK_SPEED_40GB: 659 speed = "40 G"; 660 break; 661 case ICE_AQ_LINK_SPEED_25GB: 662 speed = "25 G"; 663 break; 664 case ICE_AQ_LINK_SPEED_20GB: 665 speed = "20 G"; 666 break; 667 case ICE_AQ_LINK_SPEED_10GB: 668 speed = "10 G"; 669 break; 670 case ICE_AQ_LINK_SPEED_5GB: 671 speed = "5 G"; 672 break; 673 case ICE_AQ_LINK_SPEED_2500MB: 674 speed = "2.5 G"; 675 break; 676 case ICE_AQ_LINK_SPEED_1000MB: 677 speed = "1 G"; 678 break; 679 case ICE_AQ_LINK_SPEED_100MB: 680 speed = "100 M"; 681 break; 682 default: 683 speed = "Unknown "; 684 break; 685 } 686 687 switch (vsi->port_info->fc.current_mode) { 688 case ICE_FC_FULL: 689 fc = "Rx/Tx"; 690 break; 691 case ICE_FC_TX_PAUSE: 692 fc = "Tx"; 693 break; 694 case ICE_FC_RX_PAUSE: 695 fc = "Rx"; 696 break; 697 case ICE_FC_NONE: 698 fc = "None"; 699 break; 700 default: 701 fc = "Unknown"; 702 break; 703 } 704 705 /* Get FEC mode based on negotiated link info */ 706 switch (vsi->port_info->phy.link_info.fec_info) { 707 case ICE_AQ_LINK_25G_RS_528_FEC_EN: 708 case ICE_AQ_LINK_25G_RS_544_FEC_EN: 709 fec = "RS-FEC"; 710 break; 711 case ICE_AQ_LINK_25G_KR_FEC_EN: 712 fec = "FC-FEC/BASE-R"; 713 break; 714 default: 715 fec = "NONE"; 716 break; 717 } 718 719 /* check if autoneg completed, might be false due to not supported */ 720 if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED) 721 an = "True"; 722 else 723 an = "False"; 724 725 /* Get FEC mode requested based on PHY caps last SW configuration */ 726 caps = kzalloc(sizeof(*caps), GFP_KERNEL); 727 if (!caps) { 728 fec_req = "Unknown"; 729 an_advertised = "Unknown"; 730 goto done; 731 } 732 733 status = ice_aq_get_phy_caps(vsi->port_info, false, 734 ICE_AQC_REPORT_SW_CFG, caps, NULL); 735 if (status) 736 netdev_info(vsi->netdev, "Get phy capability failed.\n"); 737 738 an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off"; 739 740 if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ || 741 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ) 742 fec_req = "RS-FEC"; 743 else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ || 744 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ) 745 fec_req = "FC-FEC/BASE-R"; 746 else 747 fec_req = "NONE"; 748 749 kfree(caps); 750 751 done: 752 netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n", 753 speed, fec_req, fec, an_advertised, an, fc); 754 ice_print_topo_conflict(vsi); 755 } 756 757 /** 758 * ice_vsi_link_event - update the VSI's netdev 759 * @vsi: the VSI on which the link event occurred 760 * @link_up: whether or not the VSI needs to be set up or down 761 */ 762 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up) 763 { 764 if (!vsi) 765 return; 766 767 if (test_bit(__ICE_DOWN, vsi->state) || !vsi->netdev) 768 return; 769 770 if (vsi->type == ICE_VSI_PF) { 771 if (link_up == netif_carrier_ok(vsi->netdev)) 772 return; 773 774 if (link_up) { 775 netif_carrier_on(vsi->netdev); 776 netif_tx_wake_all_queues(vsi->netdev); 777 } else { 778 netif_carrier_off(vsi->netdev); 779 netif_tx_stop_all_queues(vsi->netdev); 780 } 781 } 782 } 783 784 /** 785 * ice_set_dflt_mib - send a default config MIB to the FW 786 * @pf: private PF struct 787 * 788 * This function sends a default configuration MIB to the FW. 789 * 790 * If this function errors out at any point, the driver is still able to 791 * function. The main impact is that LFC may not operate as expected. 792 * Therefore an error state in this function should be treated with a DBG 793 * message and continue on with driver rebuild/reenable. 794 */ 795 static void ice_set_dflt_mib(struct ice_pf *pf) 796 { 797 struct device *dev = ice_pf_to_dev(pf); 798 u8 mib_type, *buf, *lldpmib = NULL; 799 u16 len, typelen, offset = 0; 800 struct ice_lldp_org_tlv *tlv; 801 struct ice_hw *hw = &pf->hw; 802 u32 ouisubtype; 803 804 mib_type = SET_LOCAL_MIB_TYPE_LOCAL_MIB; 805 lldpmib = kzalloc(ICE_LLDPDU_SIZE, GFP_KERNEL); 806 if (!lldpmib) { 807 dev_dbg(dev, "%s Failed to allocate MIB memory\n", 808 __func__); 809 return; 810 } 811 812 /* Add ETS CFG TLV */ 813 tlv = (struct ice_lldp_org_tlv *)lldpmib; 814 typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | 815 ICE_IEEE_ETS_TLV_LEN); 816 tlv->typelen = htons(typelen); 817 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 818 ICE_IEEE_SUBTYPE_ETS_CFG); 819 tlv->ouisubtype = htonl(ouisubtype); 820 821 buf = tlv->tlvinfo; 822 buf[0] = 0; 823 824 /* ETS CFG all UPs map to TC 0. Next 4 (1 - 4) Octets = 0. 825 * Octets 5 - 12 are BW values, set octet 5 to 100% BW. 826 * Octets 13 - 20 are TSA values - leave as zeros 827 */ 828 buf[5] = 0x64; 829 len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S; 830 offset += len + 2; 831 tlv = (struct ice_lldp_org_tlv *) 832 ((char *)tlv + sizeof(tlv->typelen) + len); 833 834 /* Add ETS REC TLV */ 835 buf = tlv->tlvinfo; 836 tlv->typelen = htons(typelen); 837 838 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 839 ICE_IEEE_SUBTYPE_ETS_REC); 840 tlv->ouisubtype = htonl(ouisubtype); 841 842 /* First octet of buf is reserved 843 * Octets 1 - 4 map UP to TC - all UPs map to zero 844 * Octets 5 - 12 are BW values - set TC 0 to 100%. 845 * Octets 13 - 20 are TSA value - leave as zeros 846 */ 847 buf[5] = 0x64; 848 offset += len + 2; 849 tlv = (struct ice_lldp_org_tlv *) 850 ((char *)tlv + sizeof(tlv->typelen) + len); 851 852 /* Add PFC CFG TLV */ 853 typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | 854 ICE_IEEE_PFC_TLV_LEN); 855 tlv->typelen = htons(typelen); 856 857 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 858 ICE_IEEE_SUBTYPE_PFC_CFG); 859 tlv->ouisubtype = htonl(ouisubtype); 860 861 /* Octet 1 left as all zeros - PFC disabled */ 862 buf[0] = 0x08; 863 len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S; 864 offset += len + 2; 865 866 if (ice_aq_set_lldp_mib(hw, mib_type, (void *)lldpmib, offset, NULL)) 867 dev_dbg(dev, "%s Failed to set default LLDP MIB\n", __func__); 868 869 kfree(lldpmib); 870 } 871 872 /** 873 * ice_link_event - process the link event 874 * @pf: PF that the link event is associated with 875 * @pi: port_info for the port that the link event is associated with 876 * @link_up: true if the physical link is up and false if it is down 877 * @link_speed: current link speed received from the link event 878 * 879 * Returns 0 on success and negative on failure 880 */ 881 static int 882 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up, 883 u16 link_speed) 884 { 885 struct device *dev = ice_pf_to_dev(pf); 886 struct ice_phy_info *phy_info; 887 struct ice_vsi *vsi; 888 u16 old_link_speed; 889 bool old_link; 890 int result; 891 892 phy_info = &pi->phy; 893 phy_info->link_info_old = phy_info->link_info; 894 895 old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP); 896 old_link_speed = phy_info->link_info_old.link_speed; 897 898 /* update the link info structures and re-enable link events, 899 * don't bail on failure due to other book keeping needed 900 */ 901 result = ice_update_link_info(pi); 902 if (result) 903 dev_dbg(dev, "Failed to update link status and re-enable link events for port %d\n", 904 pi->lport); 905 906 /* Check if the link state is up after updating link info, and treat 907 * this event as an UP event since the link is actually UP now. 908 */ 909 if (phy_info->link_info.link_info & ICE_AQ_LINK_UP) 910 link_up = true; 911 912 vsi = ice_get_main_vsi(pf); 913 if (!vsi || !vsi->port_info) 914 return -EINVAL; 915 916 /* turn off PHY if media was removed */ 917 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) && 918 !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) { 919 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 920 921 result = ice_aq_set_link_restart_an(pi, false, NULL); 922 if (result) { 923 dev_dbg(dev, "Failed to set link down, VSI %d error %d\n", 924 vsi->vsi_num, result); 925 return result; 926 } 927 } 928 929 /* if the old link up/down and speed is the same as the new */ 930 if (link_up == old_link && link_speed == old_link_speed) 931 return result; 932 933 if (ice_is_dcb_active(pf)) { 934 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 935 ice_dcb_rebuild(pf); 936 } else { 937 if (link_up) 938 ice_set_dflt_mib(pf); 939 } 940 ice_vsi_link_event(vsi, link_up); 941 ice_print_link_msg(vsi, link_up); 942 943 ice_vc_notify_link_state(pf); 944 945 return result; 946 } 947 948 /** 949 * ice_watchdog_subtask - periodic tasks not using event driven scheduling 950 * @pf: board private structure 951 */ 952 static void ice_watchdog_subtask(struct ice_pf *pf) 953 { 954 int i; 955 956 /* if interface is down do nothing */ 957 if (test_bit(__ICE_DOWN, pf->state) || 958 test_bit(__ICE_CFG_BUSY, pf->state)) 959 return; 960 961 /* make sure we don't do these things too often */ 962 if (time_before(jiffies, 963 pf->serv_tmr_prev + pf->serv_tmr_period)) 964 return; 965 966 pf->serv_tmr_prev = jiffies; 967 968 /* Update the stats for active netdevs so the network stack 969 * can look at updated numbers whenever it cares to 970 */ 971 ice_update_pf_stats(pf); 972 ice_for_each_vsi(pf, i) 973 if (pf->vsi[i] && pf->vsi[i]->netdev) 974 ice_update_vsi_stats(pf->vsi[i]); 975 } 976 977 /** 978 * ice_init_link_events - enable/initialize link events 979 * @pi: pointer to the port_info instance 980 * 981 * Returns -EIO on failure, 0 on success 982 */ 983 static int ice_init_link_events(struct ice_port_info *pi) 984 { 985 u16 mask; 986 987 mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA | 988 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL)); 989 990 if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) { 991 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n", 992 pi->lport); 993 return -EIO; 994 } 995 996 if (ice_aq_get_link_info(pi, true, NULL, NULL)) { 997 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n", 998 pi->lport); 999 return -EIO; 1000 } 1001 1002 return 0; 1003 } 1004 1005 /** 1006 * ice_handle_link_event - handle link event via ARQ 1007 * @pf: PF that the link event is associated with 1008 * @event: event structure containing link status info 1009 */ 1010 static int 1011 ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event) 1012 { 1013 struct ice_aqc_get_link_status_data *link_data; 1014 struct ice_port_info *port_info; 1015 int status; 1016 1017 link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf; 1018 port_info = pf->hw.port_info; 1019 if (!port_info) 1020 return -EINVAL; 1021 1022 status = ice_link_event(pf, port_info, 1023 !!(link_data->link_info & ICE_AQ_LINK_UP), 1024 le16_to_cpu(link_data->link_speed)); 1025 if (status) 1026 dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n", 1027 status); 1028 1029 return status; 1030 } 1031 1032 enum ice_aq_task_state { 1033 ICE_AQ_TASK_WAITING = 0, 1034 ICE_AQ_TASK_COMPLETE, 1035 ICE_AQ_TASK_CANCELED, 1036 }; 1037 1038 struct ice_aq_task { 1039 struct hlist_node entry; 1040 1041 u16 opcode; 1042 struct ice_rq_event_info *event; 1043 enum ice_aq_task_state state; 1044 }; 1045 1046 /** 1047 * ice_wait_for_aq_event - Wait for an AdminQ event from firmware 1048 * @pf: pointer to the PF private structure 1049 * @opcode: the opcode to wait for 1050 * @timeout: how long to wait, in jiffies 1051 * @event: storage for the event info 1052 * 1053 * Waits for a specific AdminQ completion event on the ARQ for a given PF. The 1054 * current thread will be put to sleep until the specified event occurs or 1055 * until the given timeout is reached. 1056 * 1057 * To obtain only the descriptor contents, pass an event without an allocated 1058 * msg_buf. If the complete data buffer is desired, allocate the 1059 * event->msg_buf with enough space ahead of time. 1060 * 1061 * Returns: zero on success, or a negative error code on failure. 1062 */ 1063 int ice_aq_wait_for_event(struct ice_pf *pf, u16 opcode, unsigned long timeout, 1064 struct ice_rq_event_info *event) 1065 { 1066 struct device *dev = ice_pf_to_dev(pf); 1067 struct ice_aq_task *task; 1068 unsigned long start; 1069 long ret; 1070 int err; 1071 1072 task = kzalloc(sizeof(*task), GFP_KERNEL); 1073 if (!task) 1074 return -ENOMEM; 1075 1076 INIT_HLIST_NODE(&task->entry); 1077 task->opcode = opcode; 1078 task->event = event; 1079 task->state = ICE_AQ_TASK_WAITING; 1080 1081 spin_lock_bh(&pf->aq_wait_lock); 1082 hlist_add_head(&task->entry, &pf->aq_wait_list); 1083 spin_unlock_bh(&pf->aq_wait_lock); 1084 1085 start = jiffies; 1086 1087 ret = wait_event_interruptible_timeout(pf->aq_wait_queue, task->state, 1088 timeout); 1089 switch (task->state) { 1090 case ICE_AQ_TASK_WAITING: 1091 err = ret < 0 ? ret : -ETIMEDOUT; 1092 break; 1093 case ICE_AQ_TASK_CANCELED: 1094 err = ret < 0 ? ret : -ECANCELED; 1095 break; 1096 case ICE_AQ_TASK_COMPLETE: 1097 err = ret < 0 ? ret : 0; 1098 break; 1099 default: 1100 WARN(1, "Unexpected AdminQ wait task state %u", task->state); 1101 err = -EINVAL; 1102 break; 1103 } 1104 1105 dev_dbg(dev, "Waited %u msecs (max %u msecs) for firmware response to op 0x%04x\n", 1106 jiffies_to_msecs(jiffies - start), 1107 jiffies_to_msecs(timeout), 1108 opcode); 1109 1110 spin_lock_bh(&pf->aq_wait_lock); 1111 hlist_del(&task->entry); 1112 spin_unlock_bh(&pf->aq_wait_lock); 1113 kfree(task); 1114 1115 return err; 1116 } 1117 1118 /** 1119 * ice_aq_check_events - Check if any thread is waiting for an AdminQ event 1120 * @pf: pointer to the PF private structure 1121 * @opcode: the opcode of the event 1122 * @event: the event to check 1123 * 1124 * Loops over the current list of pending threads waiting for an AdminQ event. 1125 * For each matching task, copy the contents of the event into the task 1126 * structure and wake up the thread. 1127 * 1128 * If multiple threads wait for the same opcode, they will all be woken up. 1129 * 1130 * Note that event->msg_buf will only be duplicated if the event has a buffer 1131 * with enough space already allocated. Otherwise, only the descriptor and 1132 * message length will be copied. 1133 * 1134 * Returns: true if an event was found, false otherwise 1135 */ 1136 static void ice_aq_check_events(struct ice_pf *pf, u16 opcode, 1137 struct ice_rq_event_info *event) 1138 { 1139 struct ice_aq_task *task; 1140 bool found = false; 1141 1142 spin_lock_bh(&pf->aq_wait_lock); 1143 hlist_for_each_entry(task, &pf->aq_wait_list, entry) { 1144 if (task->state || task->opcode != opcode) 1145 continue; 1146 1147 memcpy(&task->event->desc, &event->desc, sizeof(event->desc)); 1148 task->event->msg_len = event->msg_len; 1149 1150 /* Only copy the data buffer if a destination was set */ 1151 if (task->event->msg_buf && 1152 task->event->buf_len > event->buf_len) { 1153 memcpy(task->event->msg_buf, event->msg_buf, 1154 event->buf_len); 1155 task->event->buf_len = event->buf_len; 1156 } 1157 1158 task->state = ICE_AQ_TASK_COMPLETE; 1159 found = true; 1160 } 1161 spin_unlock_bh(&pf->aq_wait_lock); 1162 1163 if (found) 1164 wake_up(&pf->aq_wait_queue); 1165 } 1166 1167 /** 1168 * ice_aq_cancel_waiting_tasks - Immediately cancel all waiting tasks 1169 * @pf: the PF private structure 1170 * 1171 * Set all waiting tasks to ICE_AQ_TASK_CANCELED, and wake up their threads. 1172 * This will then cause ice_aq_wait_for_event to exit with -ECANCELED. 1173 */ 1174 static void ice_aq_cancel_waiting_tasks(struct ice_pf *pf) 1175 { 1176 struct ice_aq_task *task; 1177 1178 spin_lock_bh(&pf->aq_wait_lock); 1179 hlist_for_each_entry(task, &pf->aq_wait_list, entry) 1180 task->state = ICE_AQ_TASK_CANCELED; 1181 spin_unlock_bh(&pf->aq_wait_lock); 1182 1183 wake_up(&pf->aq_wait_queue); 1184 } 1185 1186 /** 1187 * __ice_clean_ctrlq - helper function to clean controlq rings 1188 * @pf: ptr to struct ice_pf 1189 * @q_type: specific Control queue type 1190 */ 1191 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) 1192 { 1193 struct device *dev = ice_pf_to_dev(pf); 1194 struct ice_rq_event_info event; 1195 struct ice_hw *hw = &pf->hw; 1196 struct ice_ctl_q_info *cq; 1197 u16 pending, i = 0; 1198 const char *qtype; 1199 u32 oldval, val; 1200 1201 /* Do not clean control queue if/when PF reset fails */ 1202 if (test_bit(__ICE_RESET_FAILED, pf->state)) 1203 return 0; 1204 1205 switch (q_type) { 1206 case ICE_CTL_Q_ADMIN: 1207 cq = &hw->adminq; 1208 qtype = "Admin"; 1209 break; 1210 case ICE_CTL_Q_MAILBOX: 1211 cq = &hw->mailboxq; 1212 qtype = "Mailbox"; 1213 break; 1214 default: 1215 dev_warn(dev, "Unknown control queue type 0x%x\n", q_type); 1216 return 0; 1217 } 1218 1219 /* check for error indications - PF_xx_AxQLEN register layout for 1220 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN. 1221 */ 1222 val = rd32(hw, cq->rq.len); 1223 if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 1224 PF_FW_ARQLEN_ARQCRIT_M)) { 1225 oldval = val; 1226 if (val & PF_FW_ARQLEN_ARQVFE_M) 1227 dev_dbg(dev, "%s Receive Queue VF Error detected\n", 1228 qtype); 1229 if (val & PF_FW_ARQLEN_ARQOVFL_M) { 1230 dev_dbg(dev, "%s Receive Queue Overflow Error detected\n", 1231 qtype); 1232 } 1233 if (val & PF_FW_ARQLEN_ARQCRIT_M) 1234 dev_dbg(dev, "%s Receive Queue Critical Error detected\n", 1235 qtype); 1236 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 1237 PF_FW_ARQLEN_ARQCRIT_M); 1238 if (oldval != val) 1239 wr32(hw, cq->rq.len, val); 1240 } 1241 1242 val = rd32(hw, cq->sq.len); 1243 if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 1244 PF_FW_ATQLEN_ATQCRIT_M)) { 1245 oldval = val; 1246 if (val & PF_FW_ATQLEN_ATQVFE_M) 1247 dev_dbg(dev, "%s Send Queue VF Error detected\n", 1248 qtype); 1249 if (val & PF_FW_ATQLEN_ATQOVFL_M) { 1250 dev_dbg(dev, "%s Send Queue Overflow Error detected\n", 1251 qtype); 1252 } 1253 if (val & PF_FW_ATQLEN_ATQCRIT_M) 1254 dev_dbg(dev, "%s Send Queue Critical Error detected\n", 1255 qtype); 1256 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 1257 PF_FW_ATQLEN_ATQCRIT_M); 1258 if (oldval != val) 1259 wr32(hw, cq->sq.len, val); 1260 } 1261 1262 event.buf_len = cq->rq_buf_size; 1263 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 1264 if (!event.msg_buf) 1265 return 0; 1266 1267 do { 1268 enum ice_status ret; 1269 u16 opcode; 1270 1271 ret = ice_clean_rq_elem(hw, cq, &event, &pending); 1272 if (ret == ICE_ERR_AQ_NO_WORK) 1273 break; 1274 if (ret) { 1275 dev_err(dev, "%s Receive Queue event error %s\n", qtype, 1276 ice_stat_str(ret)); 1277 break; 1278 } 1279 1280 opcode = le16_to_cpu(event.desc.opcode); 1281 1282 /* Notify any thread that might be waiting for this event */ 1283 ice_aq_check_events(pf, opcode, &event); 1284 1285 switch (opcode) { 1286 case ice_aqc_opc_get_link_status: 1287 if (ice_handle_link_event(pf, &event)) 1288 dev_err(dev, "Could not handle link event\n"); 1289 break; 1290 case ice_aqc_opc_event_lan_overflow: 1291 ice_vf_lan_overflow_event(pf, &event); 1292 break; 1293 case ice_mbx_opc_send_msg_to_pf: 1294 ice_vc_process_vf_msg(pf, &event); 1295 break; 1296 case ice_aqc_opc_fw_logging: 1297 ice_output_fw_log(hw, &event.desc, event.msg_buf); 1298 break; 1299 case ice_aqc_opc_lldp_set_mib_change: 1300 ice_dcb_process_lldp_set_mib_change(pf, &event); 1301 break; 1302 default: 1303 dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n", 1304 qtype, opcode); 1305 break; 1306 } 1307 } while (pending && (i++ < ICE_DFLT_IRQ_WORK)); 1308 1309 kfree(event.msg_buf); 1310 1311 return pending && (i == ICE_DFLT_IRQ_WORK); 1312 } 1313 1314 /** 1315 * ice_ctrlq_pending - check if there is a difference between ntc and ntu 1316 * @hw: pointer to hardware info 1317 * @cq: control queue information 1318 * 1319 * returns true if there are pending messages in a queue, false if there aren't 1320 */ 1321 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq) 1322 { 1323 u16 ntu; 1324 1325 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 1326 return cq->rq.next_to_clean != ntu; 1327 } 1328 1329 /** 1330 * ice_clean_adminq_subtask - clean the AdminQ rings 1331 * @pf: board private structure 1332 */ 1333 static void ice_clean_adminq_subtask(struct ice_pf *pf) 1334 { 1335 struct ice_hw *hw = &pf->hw; 1336 1337 if (!test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state)) 1338 return; 1339 1340 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN)) 1341 return; 1342 1343 clear_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state); 1344 1345 /* There might be a situation where new messages arrive to a control 1346 * queue between processing the last message and clearing the 1347 * EVENT_PENDING bit. So before exiting, check queue head again (using 1348 * ice_ctrlq_pending) and process new messages if any. 1349 */ 1350 if (ice_ctrlq_pending(hw, &hw->adminq)) 1351 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN); 1352 1353 ice_flush(hw); 1354 } 1355 1356 /** 1357 * ice_clean_mailboxq_subtask - clean the MailboxQ rings 1358 * @pf: board private structure 1359 */ 1360 static void ice_clean_mailboxq_subtask(struct ice_pf *pf) 1361 { 1362 struct ice_hw *hw = &pf->hw; 1363 1364 if (!test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state)) 1365 return; 1366 1367 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX)) 1368 return; 1369 1370 clear_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state); 1371 1372 if (ice_ctrlq_pending(hw, &hw->mailboxq)) 1373 __ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX); 1374 1375 ice_flush(hw); 1376 } 1377 1378 /** 1379 * ice_service_task_schedule - schedule the service task to wake up 1380 * @pf: board private structure 1381 * 1382 * If not already scheduled, this puts the task into the work queue. 1383 */ 1384 void ice_service_task_schedule(struct ice_pf *pf) 1385 { 1386 if (!test_bit(__ICE_SERVICE_DIS, pf->state) && 1387 !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state) && 1388 !test_bit(__ICE_NEEDS_RESTART, pf->state)) 1389 queue_work(ice_wq, &pf->serv_task); 1390 } 1391 1392 /** 1393 * ice_service_task_complete - finish up the service task 1394 * @pf: board private structure 1395 */ 1396 static void ice_service_task_complete(struct ice_pf *pf) 1397 { 1398 WARN_ON(!test_bit(__ICE_SERVICE_SCHED, pf->state)); 1399 1400 /* force memory (pf->state) to sync before next service task */ 1401 smp_mb__before_atomic(); 1402 clear_bit(__ICE_SERVICE_SCHED, pf->state); 1403 } 1404 1405 /** 1406 * ice_service_task_stop - stop service task and cancel works 1407 * @pf: board private structure 1408 * 1409 * Return 0 if the __ICE_SERVICE_DIS bit was not already set, 1410 * 1 otherwise. 1411 */ 1412 static int ice_service_task_stop(struct ice_pf *pf) 1413 { 1414 int ret; 1415 1416 ret = test_and_set_bit(__ICE_SERVICE_DIS, pf->state); 1417 1418 if (pf->serv_tmr.function) 1419 del_timer_sync(&pf->serv_tmr); 1420 if (pf->serv_task.func) 1421 cancel_work_sync(&pf->serv_task); 1422 1423 clear_bit(__ICE_SERVICE_SCHED, pf->state); 1424 return ret; 1425 } 1426 1427 /** 1428 * ice_service_task_restart - restart service task and schedule works 1429 * @pf: board private structure 1430 * 1431 * This function is needed for suspend and resume works (e.g WoL scenario) 1432 */ 1433 static void ice_service_task_restart(struct ice_pf *pf) 1434 { 1435 clear_bit(__ICE_SERVICE_DIS, pf->state); 1436 ice_service_task_schedule(pf); 1437 } 1438 1439 /** 1440 * ice_service_timer - timer callback to schedule service task 1441 * @t: pointer to timer_list 1442 */ 1443 static void ice_service_timer(struct timer_list *t) 1444 { 1445 struct ice_pf *pf = from_timer(pf, t, serv_tmr); 1446 1447 mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies)); 1448 ice_service_task_schedule(pf); 1449 } 1450 1451 /** 1452 * ice_handle_mdd_event - handle malicious driver detect event 1453 * @pf: pointer to the PF structure 1454 * 1455 * Called from service task. OICR interrupt handler indicates MDD event. 1456 * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log 1457 * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events 1458 * disable the queue, the PF can be configured to reset the VF using ethtool 1459 * private flag mdd-auto-reset-vf. 1460 */ 1461 static void ice_handle_mdd_event(struct ice_pf *pf) 1462 { 1463 struct device *dev = ice_pf_to_dev(pf); 1464 struct ice_hw *hw = &pf->hw; 1465 unsigned int i; 1466 u32 reg; 1467 1468 if (!test_and_clear_bit(__ICE_MDD_EVENT_PENDING, pf->state)) { 1469 /* Since the VF MDD event logging is rate limited, check if 1470 * there are pending MDD events. 1471 */ 1472 ice_print_vfs_mdd_events(pf); 1473 return; 1474 } 1475 1476 /* find what triggered an MDD event */ 1477 reg = rd32(hw, GL_MDET_TX_PQM); 1478 if (reg & GL_MDET_TX_PQM_VALID_M) { 1479 u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >> 1480 GL_MDET_TX_PQM_PF_NUM_S; 1481 u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >> 1482 GL_MDET_TX_PQM_VF_NUM_S; 1483 u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >> 1484 GL_MDET_TX_PQM_MAL_TYPE_S; 1485 u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >> 1486 GL_MDET_TX_PQM_QNUM_S); 1487 1488 if (netif_msg_tx_err(pf)) 1489 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1490 event, queue, pf_num, vf_num); 1491 wr32(hw, GL_MDET_TX_PQM, 0xffffffff); 1492 } 1493 1494 reg = rd32(hw, GL_MDET_TX_TCLAN); 1495 if (reg & GL_MDET_TX_TCLAN_VALID_M) { 1496 u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >> 1497 GL_MDET_TX_TCLAN_PF_NUM_S; 1498 u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >> 1499 GL_MDET_TX_TCLAN_VF_NUM_S; 1500 u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >> 1501 GL_MDET_TX_TCLAN_MAL_TYPE_S; 1502 u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >> 1503 GL_MDET_TX_TCLAN_QNUM_S); 1504 1505 if (netif_msg_tx_err(pf)) 1506 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1507 event, queue, pf_num, vf_num); 1508 wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff); 1509 } 1510 1511 reg = rd32(hw, GL_MDET_RX); 1512 if (reg & GL_MDET_RX_VALID_M) { 1513 u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >> 1514 GL_MDET_RX_PF_NUM_S; 1515 u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >> 1516 GL_MDET_RX_VF_NUM_S; 1517 u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >> 1518 GL_MDET_RX_MAL_TYPE_S; 1519 u16 queue = ((reg & GL_MDET_RX_QNUM_M) >> 1520 GL_MDET_RX_QNUM_S); 1521 1522 if (netif_msg_rx_err(pf)) 1523 dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n", 1524 event, queue, pf_num, vf_num); 1525 wr32(hw, GL_MDET_RX, 0xffffffff); 1526 } 1527 1528 /* check to see if this PF caused an MDD event */ 1529 reg = rd32(hw, PF_MDET_TX_PQM); 1530 if (reg & PF_MDET_TX_PQM_VALID_M) { 1531 wr32(hw, PF_MDET_TX_PQM, 0xFFFF); 1532 if (netif_msg_tx_err(pf)) 1533 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n"); 1534 } 1535 1536 reg = rd32(hw, PF_MDET_TX_TCLAN); 1537 if (reg & PF_MDET_TX_TCLAN_VALID_M) { 1538 wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF); 1539 if (netif_msg_tx_err(pf)) 1540 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n"); 1541 } 1542 1543 reg = rd32(hw, PF_MDET_RX); 1544 if (reg & PF_MDET_RX_VALID_M) { 1545 wr32(hw, PF_MDET_RX, 0xFFFF); 1546 if (netif_msg_rx_err(pf)) 1547 dev_info(dev, "Malicious Driver Detection event RX detected on PF\n"); 1548 } 1549 1550 /* Check to see if one of the VFs caused an MDD event, and then 1551 * increment counters and set print pending 1552 */ 1553 ice_for_each_vf(pf, i) { 1554 struct ice_vf *vf = &pf->vf[i]; 1555 1556 reg = rd32(hw, VP_MDET_TX_PQM(i)); 1557 if (reg & VP_MDET_TX_PQM_VALID_M) { 1558 wr32(hw, VP_MDET_TX_PQM(i), 0xFFFF); 1559 vf->mdd_tx_events.count++; 1560 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1561 if (netif_msg_tx_err(pf)) 1562 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n", 1563 i); 1564 } 1565 1566 reg = rd32(hw, VP_MDET_TX_TCLAN(i)); 1567 if (reg & VP_MDET_TX_TCLAN_VALID_M) { 1568 wr32(hw, VP_MDET_TX_TCLAN(i), 0xFFFF); 1569 vf->mdd_tx_events.count++; 1570 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1571 if (netif_msg_tx_err(pf)) 1572 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n", 1573 i); 1574 } 1575 1576 reg = rd32(hw, VP_MDET_TX_TDPU(i)); 1577 if (reg & VP_MDET_TX_TDPU_VALID_M) { 1578 wr32(hw, VP_MDET_TX_TDPU(i), 0xFFFF); 1579 vf->mdd_tx_events.count++; 1580 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1581 if (netif_msg_tx_err(pf)) 1582 dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n", 1583 i); 1584 } 1585 1586 reg = rd32(hw, VP_MDET_RX(i)); 1587 if (reg & VP_MDET_RX_VALID_M) { 1588 wr32(hw, VP_MDET_RX(i), 0xFFFF); 1589 vf->mdd_rx_events.count++; 1590 set_bit(__ICE_MDD_VF_PRINT_PENDING, pf->state); 1591 if (netif_msg_rx_err(pf)) 1592 dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n", 1593 i); 1594 1595 /* Since the queue is disabled on VF Rx MDD events, the 1596 * PF can be configured to reset the VF through ethtool 1597 * private flag mdd-auto-reset-vf. 1598 */ 1599 if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) { 1600 /* VF MDD event counters will be cleared by 1601 * reset, so print the event prior to reset. 1602 */ 1603 ice_print_vf_rx_mdd_event(vf); 1604 ice_reset_vf(&pf->vf[i], false); 1605 } 1606 } 1607 } 1608 1609 ice_print_vfs_mdd_events(pf); 1610 } 1611 1612 /** 1613 * ice_force_phys_link_state - Force the physical link state 1614 * @vsi: VSI to force the physical link state to up/down 1615 * @link_up: true/false indicates to set the physical link to up/down 1616 * 1617 * Force the physical link state by getting the current PHY capabilities from 1618 * hardware and setting the PHY config based on the determined capabilities. If 1619 * link changes a link event will be triggered because both the Enable Automatic 1620 * Link Update and LESM Enable bits are set when setting the PHY capabilities. 1621 * 1622 * Returns 0 on success, negative on failure 1623 */ 1624 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up) 1625 { 1626 struct ice_aqc_get_phy_caps_data *pcaps; 1627 struct ice_aqc_set_phy_cfg_data *cfg; 1628 struct ice_port_info *pi; 1629 struct device *dev; 1630 int retcode; 1631 1632 if (!vsi || !vsi->port_info || !vsi->back) 1633 return -EINVAL; 1634 if (vsi->type != ICE_VSI_PF) 1635 return 0; 1636 1637 dev = ice_pf_to_dev(vsi->back); 1638 1639 pi = vsi->port_info; 1640 1641 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1642 if (!pcaps) 1643 return -ENOMEM; 1644 1645 retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_SW_CFG, pcaps, 1646 NULL); 1647 if (retcode) { 1648 dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n", 1649 vsi->vsi_num, retcode); 1650 retcode = -EIO; 1651 goto out; 1652 } 1653 1654 /* No change in link */ 1655 if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && 1656 link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) 1657 goto out; 1658 1659 /* Use the current user PHY configuration. The current user PHY 1660 * configuration is initialized during probe from PHY capabilities 1661 * software mode, and updated on set PHY configuration. 1662 */ 1663 cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL); 1664 if (!cfg) { 1665 retcode = -ENOMEM; 1666 goto out; 1667 } 1668 1669 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; 1670 if (link_up) 1671 cfg->caps |= ICE_AQ_PHY_ENA_LINK; 1672 else 1673 cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; 1674 1675 retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1676 if (retcode) { 1677 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 1678 vsi->vsi_num, retcode); 1679 retcode = -EIO; 1680 } 1681 1682 kfree(cfg); 1683 out: 1684 kfree(pcaps); 1685 return retcode; 1686 } 1687 1688 /** 1689 * ice_init_nvm_phy_type - Initialize the NVM PHY type 1690 * @pi: port info structure 1691 * 1692 * Initialize nvm_phy_type_[low|high] for link lenient mode support 1693 */ 1694 static int ice_init_nvm_phy_type(struct ice_port_info *pi) 1695 { 1696 struct ice_aqc_get_phy_caps_data *pcaps; 1697 struct ice_pf *pf = pi->hw->back; 1698 enum ice_status status; 1699 int err = 0; 1700 1701 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1702 if (!pcaps) 1703 return -ENOMEM; 1704 1705 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_NVM_CAP, pcaps, 1706 NULL); 1707 1708 if (status) { 1709 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 1710 err = -EIO; 1711 goto out; 1712 } 1713 1714 pf->nvm_phy_type_hi = pcaps->phy_type_high; 1715 pf->nvm_phy_type_lo = pcaps->phy_type_low; 1716 1717 out: 1718 kfree(pcaps); 1719 return err; 1720 } 1721 1722 /** 1723 * ice_init_link_dflt_override - Initialize link default override 1724 * @pi: port info structure 1725 * 1726 * Initialize link default override and PHY total port shutdown during probe 1727 */ 1728 static void ice_init_link_dflt_override(struct ice_port_info *pi) 1729 { 1730 struct ice_link_default_override_tlv *ldo; 1731 struct ice_pf *pf = pi->hw->back; 1732 1733 ldo = &pf->link_dflt_override; 1734 if (ice_get_link_default_override(ldo, pi)) 1735 return; 1736 1737 if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS)) 1738 return; 1739 1740 /* Enable Total Port Shutdown (override/replace link-down-on-close 1741 * ethtool private flag) for ports with Port Disable bit set. 1742 */ 1743 set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 1744 set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 1745 } 1746 1747 /** 1748 * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings 1749 * @pi: port info structure 1750 * 1751 * If default override is enabled, initialized the user PHY cfg speed and FEC 1752 * settings using the default override mask from the NVM. 1753 * 1754 * The PHY should only be configured with the default override settings the 1755 * first time media is available. The __ICE_LINK_DEFAULT_OVERRIDE_PENDING state 1756 * is used to indicate that the user PHY cfg default override is initialized 1757 * and the PHY has not been configured with the default override settings. The 1758 * state is set here, and cleared in ice_configure_phy the first time the PHY is 1759 * configured. 1760 */ 1761 static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi) 1762 { 1763 struct ice_link_default_override_tlv *ldo; 1764 struct ice_aqc_set_phy_cfg_data *cfg; 1765 struct ice_phy_info *phy = &pi->phy; 1766 struct ice_pf *pf = pi->hw->back; 1767 1768 ldo = &pf->link_dflt_override; 1769 1770 /* If link default override is enabled, use to mask NVM PHY capabilities 1771 * for speed and FEC default configuration. 1772 */ 1773 cfg = &phy->curr_user_phy_cfg; 1774 1775 if (ldo->phy_type_low || ldo->phy_type_high) { 1776 cfg->phy_type_low = pf->nvm_phy_type_lo & 1777 cpu_to_le64(ldo->phy_type_low); 1778 cfg->phy_type_high = pf->nvm_phy_type_hi & 1779 cpu_to_le64(ldo->phy_type_high); 1780 } 1781 cfg->link_fec_opt = ldo->fec_options; 1782 phy->curr_user_fec_req = ICE_FEC_AUTO; 1783 1784 set_bit(__ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state); 1785 } 1786 1787 /** 1788 * ice_init_phy_user_cfg - Initialize the PHY user configuration 1789 * @pi: port info structure 1790 * 1791 * Initialize the current user PHY configuration, speed, FEC, and FC requested 1792 * mode to default. The PHY defaults are from get PHY capabilities topology 1793 * with media so call when media is first available. An error is returned if 1794 * called when media is not available. The PHY initialization completed state is 1795 * set here. 1796 * 1797 * These configurations are used when setting PHY 1798 * configuration. The user PHY configuration is updated on set PHY 1799 * configuration. Returns 0 on success, negative on failure 1800 */ 1801 static int ice_init_phy_user_cfg(struct ice_port_info *pi) 1802 { 1803 struct ice_aqc_get_phy_caps_data *pcaps; 1804 struct ice_phy_info *phy = &pi->phy; 1805 struct ice_pf *pf = pi->hw->back; 1806 enum ice_status status; 1807 struct ice_vsi *vsi; 1808 int err = 0; 1809 1810 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 1811 return -EIO; 1812 1813 vsi = ice_get_main_vsi(pf); 1814 if (!vsi) 1815 return -EINVAL; 1816 1817 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1818 if (!pcaps) 1819 return -ENOMEM; 1820 1821 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP, pcaps, 1822 NULL); 1823 if (status) { 1824 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 1825 err = -EIO; 1826 goto err_out; 1827 } 1828 1829 ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg); 1830 1831 /* check if lenient mode is supported and enabled */ 1832 if (ice_fw_supports_link_override(&vsi->back->hw) && 1833 !(pcaps->module_compliance_enforcement & 1834 ICE_AQC_MOD_ENFORCE_STRICT_MODE)) { 1835 set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags); 1836 1837 /* if link default override is enabled, initialize user PHY 1838 * configuration with link default override values 1839 */ 1840 if (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN) { 1841 ice_init_phy_cfg_dflt_override(pi); 1842 goto out; 1843 } 1844 } 1845 1846 /* if link default override is not enabled, initialize PHY using 1847 * topology with media 1848 */ 1849 phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps, 1850 pcaps->link_fec_options); 1851 phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps); 1852 1853 out: 1854 phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M; 1855 set_bit(__ICE_PHY_INIT_COMPLETE, pf->state); 1856 err_out: 1857 kfree(pcaps); 1858 return err; 1859 } 1860 1861 /** 1862 * ice_configure_phy - configure PHY 1863 * @vsi: VSI of PHY 1864 * 1865 * Set the PHY configuration. If the current PHY configuration is the same as 1866 * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise 1867 * configure the based get PHY capabilities for topology with media. 1868 */ 1869 static int ice_configure_phy(struct ice_vsi *vsi) 1870 { 1871 struct device *dev = ice_pf_to_dev(vsi->back); 1872 struct ice_aqc_get_phy_caps_data *pcaps; 1873 struct ice_aqc_set_phy_cfg_data *cfg; 1874 struct ice_port_info *pi; 1875 enum ice_status status; 1876 int err = 0; 1877 1878 pi = vsi->port_info; 1879 if (!pi) 1880 return -EINVAL; 1881 1882 /* Ensure we have media as we cannot configure a medialess port */ 1883 if (!(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 1884 return -EPERM; 1885 1886 ice_print_topo_conflict(vsi); 1887 1888 if (vsi->port_info->phy.link_info.topo_media_conflict == 1889 ICE_AQ_LINK_TOPO_UNSUPP_MEDIA) 1890 return -EPERM; 1891 1892 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 1893 return ice_force_phys_link_state(vsi, true); 1894 1895 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1896 if (!pcaps) 1897 return -ENOMEM; 1898 1899 /* Get current PHY config */ 1900 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_SW_CFG, pcaps, 1901 NULL); 1902 if (status) { 1903 dev_err(dev, "Failed to get PHY configuration, VSI %d error %s\n", 1904 vsi->vsi_num, ice_stat_str(status)); 1905 err = -EIO; 1906 goto done; 1907 } 1908 1909 /* If PHY enable link is configured and configuration has not changed, 1910 * there's nothing to do 1911 */ 1912 if (pcaps->caps & ICE_AQC_PHY_EN_LINK && 1913 ice_phy_caps_equals_cfg(pcaps, &pi->phy.curr_user_phy_cfg)) 1914 goto done; 1915 1916 /* Use PHY topology as baseline for configuration */ 1917 memset(pcaps, 0, sizeof(*pcaps)); 1918 status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP, pcaps, 1919 NULL); 1920 if (status) { 1921 dev_err(dev, "Failed to get PHY topology, VSI %d error %s\n", 1922 vsi->vsi_num, ice_stat_str(status)); 1923 err = -EIO; 1924 goto done; 1925 } 1926 1927 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 1928 if (!cfg) { 1929 err = -ENOMEM; 1930 goto done; 1931 } 1932 1933 ice_copy_phy_caps_to_cfg(pi, pcaps, cfg); 1934 1935 /* Speed - If default override pending, use curr_user_phy_cfg set in 1936 * ice_init_phy_user_cfg_ldo. 1937 */ 1938 if (test_and_clear_bit(__ICE_LINK_DEFAULT_OVERRIDE_PENDING, 1939 vsi->back->state)) { 1940 cfg->phy_type_low = pi->phy.curr_user_phy_cfg.phy_type_low; 1941 cfg->phy_type_high = pi->phy.curr_user_phy_cfg.phy_type_high; 1942 } else { 1943 u64 phy_low = 0, phy_high = 0; 1944 1945 ice_update_phy_type(&phy_low, &phy_high, 1946 pi->phy.curr_user_speed_req); 1947 cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low); 1948 cfg->phy_type_high = pcaps->phy_type_high & 1949 cpu_to_le64(phy_high); 1950 } 1951 1952 /* Can't provide what was requested; use PHY capabilities */ 1953 if (!cfg->phy_type_low && !cfg->phy_type_high) { 1954 cfg->phy_type_low = pcaps->phy_type_low; 1955 cfg->phy_type_high = pcaps->phy_type_high; 1956 } 1957 1958 /* FEC */ 1959 ice_cfg_phy_fec(pi, cfg, pi->phy.curr_user_fec_req); 1960 1961 /* Can't provide what was requested; use PHY capabilities */ 1962 if (cfg->link_fec_opt != 1963 (cfg->link_fec_opt & pcaps->link_fec_options)) { 1964 cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC; 1965 cfg->link_fec_opt = pcaps->link_fec_options; 1966 } 1967 1968 /* Flow Control - always supported; no need to check against 1969 * capabilities 1970 */ 1971 ice_cfg_phy_fc(pi, cfg, pi->phy.curr_user_fc_req); 1972 1973 /* Enable link and link update */ 1974 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK; 1975 1976 status = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1977 if (status) { 1978 dev_err(dev, "Failed to set phy config, VSI %d error %s\n", 1979 vsi->vsi_num, ice_stat_str(status)); 1980 err = -EIO; 1981 } 1982 1983 kfree(cfg); 1984 done: 1985 kfree(pcaps); 1986 return err; 1987 } 1988 1989 /** 1990 * ice_check_media_subtask - Check for media 1991 * @pf: pointer to PF struct 1992 * 1993 * If media is available, then initialize PHY user configuration if it is not 1994 * been, and configure the PHY if the interface is up. 1995 */ 1996 static void ice_check_media_subtask(struct ice_pf *pf) 1997 { 1998 struct ice_port_info *pi; 1999 struct ice_vsi *vsi; 2000 int err; 2001 2002 /* No need to check for media if it's already present */ 2003 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags)) 2004 return; 2005 2006 vsi = ice_get_main_vsi(pf); 2007 if (!vsi) 2008 return; 2009 2010 /* Refresh link info and check if media is present */ 2011 pi = vsi->port_info; 2012 err = ice_update_link_info(pi); 2013 if (err) 2014 return; 2015 2016 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 2017 if (!test_bit(__ICE_PHY_INIT_COMPLETE, pf->state)) 2018 ice_init_phy_user_cfg(pi); 2019 2020 /* PHY settings are reset on media insertion, reconfigure 2021 * PHY to preserve settings. 2022 */ 2023 if (test_bit(__ICE_DOWN, vsi->state) && 2024 test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 2025 return; 2026 2027 err = ice_configure_phy(vsi); 2028 if (!err) 2029 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 2030 2031 /* A Link Status Event will be generated; the event handler 2032 * will complete bringing the interface up 2033 */ 2034 } 2035 } 2036 2037 /** 2038 * ice_service_task - manage and run subtasks 2039 * @work: pointer to work_struct contained by the PF struct 2040 */ 2041 static void ice_service_task(struct work_struct *work) 2042 { 2043 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task); 2044 unsigned long start_time = jiffies; 2045 2046 /* subtasks */ 2047 2048 /* process reset requests first */ 2049 ice_reset_subtask(pf); 2050 2051 /* bail if a reset/recovery cycle is pending or rebuild failed */ 2052 if (ice_is_reset_in_progress(pf->state) || 2053 test_bit(__ICE_SUSPENDED, pf->state) || 2054 test_bit(__ICE_NEEDS_RESTART, pf->state)) { 2055 ice_service_task_complete(pf); 2056 return; 2057 } 2058 2059 ice_clean_adminq_subtask(pf); 2060 ice_check_media_subtask(pf); 2061 ice_check_for_hang_subtask(pf); 2062 ice_sync_fltr_subtask(pf); 2063 ice_handle_mdd_event(pf); 2064 ice_watchdog_subtask(pf); 2065 2066 if (ice_is_safe_mode(pf)) { 2067 ice_service_task_complete(pf); 2068 return; 2069 } 2070 2071 ice_process_vflr_event(pf); 2072 ice_clean_mailboxq_subtask(pf); 2073 ice_sync_arfs_fltrs(pf); 2074 /* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */ 2075 ice_service_task_complete(pf); 2076 2077 /* If the tasks have taken longer than one service timer period 2078 * or there is more work to be done, reset the service timer to 2079 * schedule the service task now. 2080 */ 2081 if (time_after(jiffies, (start_time + pf->serv_tmr_period)) || 2082 test_bit(__ICE_MDD_EVENT_PENDING, pf->state) || 2083 test_bit(__ICE_VFLR_EVENT_PENDING, pf->state) || 2084 test_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state) || 2085 test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state)) 2086 mod_timer(&pf->serv_tmr, jiffies); 2087 } 2088 2089 /** 2090 * ice_set_ctrlq_len - helper function to set controlq length 2091 * @hw: pointer to the HW instance 2092 */ 2093 static void ice_set_ctrlq_len(struct ice_hw *hw) 2094 { 2095 hw->adminq.num_rq_entries = ICE_AQ_LEN; 2096 hw->adminq.num_sq_entries = ICE_AQ_LEN; 2097 hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN; 2098 hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN; 2099 hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M; 2100 hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN; 2101 hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 2102 hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 2103 } 2104 2105 /** 2106 * ice_schedule_reset - schedule a reset 2107 * @pf: board private structure 2108 * @reset: reset being requested 2109 */ 2110 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset) 2111 { 2112 struct device *dev = ice_pf_to_dev(pf); 2113 2114 /* bail out if earlier reset has failed */ 2115 if (test_bit(__ICE_RESET_FAILED, pf->state)) { 2116 dev_dbg(dev, "earlier reset has failed\n"); 2117 return -EIO; 2118 } 2119 /* bail if reset/recovery already in progress */ 2120 if (ice_is_reset_in_progress(pf->state)) { 2121 dev_dbg(dev, "Reset already in progress\n"); 2122 return -EBUSY; 2123 } 2124 2125 switch (reset) { 2126 case ICE_RESET_PFR: 2127 set_bit(__ICE_PFR_REQ, pf->state); 2128 break; 2129 case ICE_RESET_CORER: 2130 set_bit(__ICE_CORER_REQ, pf->state); 2131 break; 2132 case ICE_RESET_GLOBR: 2133 set_bit(__ICE_GLOBR_REQ, pf->state); 2134 break; 2135 default: 2136 return -EINVAL; 2137 } 2138 2139 ice_service_task_schedule(pf); 2140 return 0; 2141 } 2142 2143 /** 2144 * ice_irq_affinity_notify - Callback for affinity changes 2145 * @notify: context as to what irq was changed 2146 * @mask: the new affinity mask 2147 * 2148 * This is a callback function used by the irq_set_affinity_notifier function 2149 * so that we may register to receive changes to the irq affinity masks. 2150 */ 2151 static void 2152 ice_irq_affinity_notify(struct irq_affinity_notify *notify, 2153 const cpumask_t *mask) 2154 { 2155 struct ice_q_vector *q_vector = 2156 container_of(notify, struct ice_q_vector, affinity_notify); 2157 2158 cpumask_copy(&q_vector->affinity_mask, mask); 2159 } 2160 2161 /** 2162 * ice_irq_affinity_release - Callback for affinity notifier release 2163 * @ref: internal core kernel usage 2164 * 2165 * This is a callback function used by the irq_set_affinity_notifier function 2166 * to inform the current notification subscriber that they will no longer 2167 * receive notifications. 2168 */ 2169 static void ice_irq_affinity_release(struct kref __always_unused *ref) {} 2170 2171 /** 2172 * ice_vsi_ena_irq - Enable IRQ for the given VSI 2173 * @vsi: the VSI being configured 2174 */ 2175 static int ice_vsi_ena_irq(struct ice_vsi *vsi) 2176 { 2177 struct ice_hw *hw = &vsi->back->hw; 2178 int i; 2179 2180 ice_for_each_q_vector(vsi, i) 2181 ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]); 2182 2183 ice_flush(hw); 2184 return 0; 2185 } 2186 2187 /** 2188 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI 2189 * @vsi: the VSI being configured 2190 * @basename: name for the vector 2191 */ 2192 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) 2193 { 2194 int q_vectors = vsi->num_q_vectors; 2195 struct ice_pf *pf = vsi->back; 2196 int base = vsi->base_vector; 2197 struct device *dev; 2198 int rx_int_idx = 0; 2199 int tx_int_idx = 0; 2200 int vector, err; 2201 int irq_num; 2202 2203 dev = ice_pf_to_dev(pf); 2204 for (vector = 0; vector < q_vectors; vector++) { 2205 struct ice_q_vector *q_vector = vsi->q_vectors[vector]; 2206 2207 irq_num = pf->msix_entries[base + vector].vector; 2208 2209 if (q_vector->tx.ring && q_vector->rx.ring) { 2210 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2211 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 2212 tx_int_idx++; 2213 } else if (q_vector->rx.ring) { 2214 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2215 "%s-%s-%d", basename, "rx", rx_int_idx++); 2216 } else if (q_vector->tx.ring) { 2217 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2218 "%s-%s-%d", basename, "tx", tx_int_idx++); 2219 } else { 2220 /* skip this unused q_vector */ 2221 continue; 2222 } 2223 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 0, 2224 q_vector->name, q_vector); 2225 if (err) { 2226 netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n", 2227 err); 2228 goto free_q_irqs; 2229 } 2230 2231 /* register for affinity change notifications */ 2232 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) { 2233 struct irq_affinity_notify *affinity_notify; 2234 2235 affinity_notify = &q_vector->affinity_notify; 2236 affinity_notify->notify = ice_irq_affinity_notify; 2237 affinity_notify->release = ice_irq_affinity_release; 2238 irq_set_affinity_notifier(irq_num, affinity_notify); 2239 } 2240 2241 /* assign the mask for this irq */ 2242 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 2243 } 2244 2245 vsi->irqs_ready = true; 2246 return 0; 2247 2248 free_q_irqs: 2249 while (vector) { 2250 vector--; 2251 irq_num = pf->msix_entries[base + vector].vector; 2252 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) 2253 irq_set_affinity_notifier(irq_num, NULL); 2254 irq_set_affinity_hint(irq_num, NULL); 2255 devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]); 2256 } 2257 return err; 2258 } 2259 2260 /** 2261 * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP 2262 * @vsi: VSI to setup Tx rings used by XDP 2263 * 2264 * Return 0 on success and negative value on error 2265 */ 2266 static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) 2267 { 2268 struct device *dev = ice_pf_to_dev(vsi->back); 2269 int i; 2270 2271 for (i = 0; i < vsi->num_xdp_txq; i++) { 2272 u16 xdp_q_idx = vsi->alloc_txq + i; 2273 struct ice_ring *xdp_ring; 2274 2275 xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL); 2276 2277 if (!xdp_ring) 2278 goto free_xdp_rings; 2279 2280 xdp_ring->q_index = xdp_q_idx; 2281 xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx]; 2282 xdp_ring->ring_active = false; 2283 xdp_ring->vsi = vsi; 2284 xdp_ring->netdev = NULL; 2285 xdp_ring->dev = dev; 2286 xdp_ring->count = vsi->num_tx_desc; 2287 WRITE_ONCE(vsi->xdp_rings[i], xdp_ring); 2288 if (ice_setup_tx_ring(xdp_ring)) 2289 goto free_xdp_rings; 2290 ice_set_ring_xdp(xdp_ring); 2291 xdp_ring->xsk_pool = ice_xsk_pool(xdp_ring); 2292 } 2293 2294 return 0; 2295 2296 free_xdp_rings: 2297 for (; i >= 0; i--) 2298 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 2299 ice_free_tx_ring(vsi->xdp_rings[i]); 2300 return -ENOMEM; 2301 } 2302 2303 /** 2304 * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI 2305 * @vsi: VSI to set the bpf prog on 2306 * @prog: the bpf prog pointer 2307 */ 2308 static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) 2309 { 2310 struct bpf_prog *old_prog; 2311 int i; 2312 2313 old_prog = xchg(&vsi->xdp_prog, prog); 2314 if (old_prog) 2315 bpf_prog_put(old_prog); 2316 2317 ice_for_each_rxq(vsi, i) 2318 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 2319 } 2320 2321 /** 2322 * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP 2323 * @vsi: VSI to bring up Tx rings used by XDP 2324 * @prog: bpf program that will be assigned to VSI 2325 * 2326 * Return 0 on success and negative value on error 2327 */ 2328 int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) 2329 { 2330 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2331 int xdp_rings_rem = vsi->num_xdp_txq; 2332 struct ice_pf *pf = vsi->back; 2333 struct ice_qs_cfg xdp_qs_cfg = { 2334 .qs_mutex = &pf->avail_q_mutex, 2335 .pf_map = pf->avail_txqs, 2336 .pf_map_size = pf->max_pf_txqs, 2337 .q_count = vsi->num_xdp_txq, 2338 .scatter_count = ICE_MAX_SCATTER_TXQS, 2339 .vsi_map = vsi->txq_map, 2340 .vsi_map_offset = vsi->alloc_txq, 2341 .mapping_mode = ICE_VSI_MAP_CONTIG 2342 }; 2343 enum ice_status status; 2344 struct device *dev; 2345 int i, v_idx; 2346 2347 dev = ice_pf_to_dev(pf); 2348 vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq, 2349 sizeof(*vsi->xdp_rings), GFP_KERNEL); 2350 if (!vsi->xdp_rings) 2351 return -ENOMEM; 2352 2353 vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode; 2354 if (__ice_vsi_get_qs(&xdp_qs_cfg)) 2355 goto err_map_xdp; 2356 2357 if (ice_xdp_alloc_setup_rings(vsi)) 2358 goto clear_xdp_rings; 2359 2360 /* follow the logic from ice_vsi_map_rings_to_vectors */ 2361 ice_for_each_q_vector(vsi, v_idx) { 2362 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2363 int xdp_rings_per_v, q_id, q_base; 2364 2365 xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem, 2366 vsi->num_q_vectors - v_idx); 2367 q_base = vsi->num_xdp_txq - xdp_rings_rem; 2368 2369 for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) { 2370 struct ice_ring *xdp_ring = vsi->xdp_rings[q_id]; 2371 2372 xdp_ring->q_vector = q_vector; 2373 xdp_ring->next = q_vector->tx.ring; 2374 q_vector->tx.ring = xdp_ring; 2375 } 2376 xdp_rings_rem -= xdp_rings_per_v; 2377 } 2378 2379 /* omit the scheduler update if in reset path; XDP queues will be 2380 * taken into account at the end of ice_vsi_rebuild, where 2381 * ice_cfg_vsi_lan is being called 2382 */ 2383 if (ice_is_reset_in_progress(pf->state)) 2384 return 0; 2385 2386 /* tell the Tx scheduler that right now we have 2387 * additional queues 2388 */ 2389 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2390 max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq; 2391 2392 status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2393 max_txqs); 2394 if (status) { 2395 dev_err(dev, "Failed VSI LAN queue config for XDP, error: %s\n", 2396 ice_stat_str(status)); 2397 goto clear_xdp_rings; 2398 } 2399 ice_vsi_assign_bpf_prog(vsi, prog); 2400 2401 return 0; 2402 clear_xdp_rings: 2403 for (i = 0; i < vsi->num_xdp_txq; i++) 2404 if (vsi->xdp_rings[i]) { 2405 kfree_rcu(vsi->xdp_rings[i], rcu); 2406 vsi->xdp_rings[i] = NULL; 2407 } 2408 2409 err_map_xdp: 2410 mutex_lock(&pf->avail_q_mutex); 2411 for (i = 0; i < vsi->num_xdp_txq; i++) { 2412 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2413 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2414 } 2415 mutex_unlock(&pf->avail_q_mutex); 2416 2417 devm_kfree(dev, vsi->xdp_rings); 2418 return -ENOMEM; 2419 } 2420 2421 /** 2422 * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings 2423 * @vsi: VSI to remove XDP rings 2424 * 2425 * Detach XDP rings from irq vectors, clean up the PF bitmap and free 2426 * resources 2427 */ 2428 int ice_destroy_xdp_rings(struct ice_vsi *vsi) 2429 { 2430 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2431 struct ice_pf *pf = vsi->back; 2432 int i, v_idx; 2433 2434 /* q_vectors are freed in reset path so there's no point in detaching 2435 * rings; in case of rebuild being triggered not from reset bits 2436 * in pf->state won't be set, so additionally check first q_vector 2437 * against NULL 2438 */ 2439 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2440 goto free_qmap; 2441 2442 ice_for_each_q_vector(vsi, v_idx) { 2443 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2444 struct ice_ring *ring; 2445 2446 ice_for_each_ring(ring, q_vector->tx) 2447 if (!ring->tx_buf || !ice_ring_is_xdp(ring)) 2448 break; 2449 2450 /* restore the value of last node prior to XDP setup */ 2451 q_vector->tx.ring = ring; 2452 } 2453 2454 free_qmap: 2455 mutex_lock(&pf->avail_q_mutex); 2456 for (i = 0; i < vsi->num_xdp_txq; i++) { 2457 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2458 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2459 } 2460 mutex_unlock(&pf->avail_q_mutex); 2461 2462 for (i = 0; i < vsi->num_xdp_txq; i++) 2463 if (vsi->xdp_rings[i]) { 2464 if (vsi->xdp_rings[i]->desc) 2465 ice_free_tx_ring(vsi->xdp_rings[i]); 2466 kfree_rcu(vsi->xdp_rings[i], rcu); 2467 vsi->xdp_rings[i] = NULL; 2468 } 2469 2470 devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings); 2471 vsi->xdp_rings = NULL; 2472 2473 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2474 return 0; 2475 2476 ice_vsi_assign_bpf_prog(vsi, NULL); 2477 2478 /* notify Tx scheduler that we destroyed XDP queues and bring 2479 * back the old number of child nodes 2480 */ 2481 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2482 max_txqs[i] = vsi->num_txq; 2483 2484 /* change number of XDP Tx queues to 0 */ 2485 vsi->num_xdp_txq = 0; 2486 2487 return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2488 max_txqs); 2489 } 2490 2491 /** 2492 * ice_vsi_rx_napi_schedule - Schedule napi on RX queues from VSI 2493 * @vsi: VSI to schedule napi on 2494 */ 2495 static void ice_vsi_rx_napi_schedule(struct ice_vsi *vsi) 2496 { 2497 int i; 2498 2499 ice_for_each_rxq(vsi, i) { 2500 struct ice_ring *rx_ring = vsi->rx_rings[i]; 2501 2502 if (rx_ring->xsk_pool) 2503 napi_schedule(&rx_ring->q_vector->napi); 2504 } 2505 } 2506 2507 /** 2508 * ice_xdp_setup_prog - Add or remove XDP eBPF program 2509 * @vsi: VSI to setup XDP for 2510 * @prog: XDP program 2511 * @extack: netlink extended ack 2512 */ 2513 static int 2514 ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, 2515 struct netlink_ext_ack *extack) 2516 { 2517 int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; 2518 bool if_running = netif_running(vsi->netdev); 2519 int ret = 0, xdp_ring_err = 0; 2520 2521 if (frame_size > vsi->rx_buf_len) { 2522 NL_SET_ERR_MSG_MOD(extack, "MTU too large for loading XDP"); 2523 return -EOPNOTSUPP; 2524 } 2525 2526 /* need to stop netdev while setting up the program for Rx rings */ 2527 if (if_running && !test_and_set_bit(__ICE_DOWN, vsi->state)) { 2528 ret = ice_down(vsi); 2529 if (ret) { 2530 NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed"); 2531 return ret; 2532 } 2533 } 2534 2535 if (!ice_is_xdp_ena_vsi(vsi) && prog) { 2536 vsi->num_xdp_txq = vsi->alloc_rxq; 2537 xdp_ring_err = ice_prepare_xdp_rings(vsi, prog); 2538 if (xdp_ring_err) 2539 NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); 2540 } else if (ice_is_xdp_ena_vsi(vsi) && !prog) { 2541 xdp_ring_err = ice_destroy_xdp_rings(vsi); 2542 if (xdp_ring_err) 2543 NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed"); 2544 } else { 2545 ice_vsi_assign_bpf_prog(vsi, prog); 2546 } 2547 2548 if (if_running) 2549 ret = ice_up(vsi); 2550 2551 if (!ret && prog) 2552 ice_vsi_rx_napi_schedule(vsi); 2553 2554 return (ret || xdp_ring_err) ? -ENOMEM : 0; 2555 } 2556 2557 /** 2558 * ice_xdp - implements XDP handler 2559 * @dev: netdevice 2560 * @xdp: XDP command 2561 */ 2562 static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2563 { 2564 struct ice_netdev_priv *np = netdev_priv(dev); 2565 struct ice_vsi *vsi = np->vsi; 2566 2567 if (vsi->type != ICE_VSI_PF) { 2568 NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI"); 2569 return -EINVAL; 2570 } 2571 2572 switch (xdp->command) { 2573 case XDP_SETUP_PROG: 2574 return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); 2575 case XDP_SETUP_XSK_POOL: 2576 return ice_xsk_pool_setup(vsi, xdp->xsk.pool, 2577 xdp->xsk.queue_id); 2578 default: 2579 return -EINVAL; 2580 } 2581 } 2582 2583 /** 2584 * ice_ena_misc_vector - enable the non-queue interrupts 2585 * @pf: board private structure 2586 */ 2587 static void ice_ena_misc_vector(struct ice_pf *pf) 2588 { 2589 struct ice_hw *hw = &pf->hw; 2590 u32 val; 2591 2592 /* Disable anti-spoof detection interrupt to prevent spurious event 2593 * interrupts during a function reset. Anti-spoof functionally is 2594 * still supported. 2595 */ 2596 val = rd32(hw, GL_MDCK_TX_TDPU); 2597 val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M; 2598 wr32(hw, GL_MDCK_TX_TDPU, val); 2599 2600 /* clear things first */ 2601 wr32(hw, PFINT_OICR_ENA, 0); /* disable all */ 2602 rd32(hw, PFINT_OICR); /* read to clear */ 2603 2604 val = (PFINT_OICR_ECC_ERR_M | 2605 PFINT_OICR_MAL_DETECT_M | 2606 PFINT_OICR_GRST_M | 2607 PFINT_OICR_PCI_EXCEPTION_M | 2608 PFINT_OICR_VFLR_M | 2609 PFINT_OICR_HMC_ERR_M | 2610 PFINT_OICR_PE_CRITERR_M); 2611 2612 wr32(hw, PFINT_OICR_ENA, val); 2613 2614 /* SW_ITR_IDX = 0, but don't change INTENA */ 2615 wr32(hw, GLINT_DYN_CTL(pf->oicr_idx), 2616 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M); 2617 } 2618 2619 /** 2620 * ice_misc_intr - misc interrupt handler 2621 * @irq: interrupt number 2622 * @data: pointer to a q_vector 2623 */ 2624 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) 2625 { 2626 struct ice_pf *pf = (struct ice_pf *)data; 2627 struct ice_hw *hw = &pf->hw; 2628 irqreturn_t ret = IRQ_NONE; 2629 struct device *dev; 2630 u32 oicr, ena_mask; 2631 2632 dev = ice_pf_to_dev(pf); 2633 set_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state); 2634 set_bit(__ICE_MAILBOXQ_EVENT_PENDING, pf->state); 2635 2636 oicr = rd32(hw, PFINT_OICR); 2637 ena_mask = rd32(hw, PFINT_OICR_ENA); 2638 2639 if (oicr & PFINT_OICR_SWINT_M) { 2640 ena_mask &= ~PFINT_OICR_SWINT_M; 2641 pf->sw_int_count++; 2642 } 2643 2644 if (oicr & PFINT_OICR_MAL_DETECT_M) { 2645 ena_mask &= ~PFINT_OICR_MAL_DETECT_M; 2646 set_bit(__ICE_MDD_EVENT_PENDING, pf->state); 2647 } 2648 if (oicr & PFINT_OICR_VFLR_M) { 2649 /* disable any further VFLR event notifications */ 2650 if (test_bit(__ICE_VF_RESETS_DISABLED, pf->state)) { 2651 u32 reg = rd32(hw, PFINT_OICR_ENA); 2652 2653 reg &= ~PFINT_OICR_VFLR_M; 2654 wr32(hw, PFINT_OICR_ENA, reg); 2655 } else { 2656 ena_mask &= ~PFINT_OICR_VFLR_M; 2657 set_bit(__ICE_VFLR_EVENT_PENDING, pf->state); 2658 } 2659 } 2660 2661 if (oicr & PFINT_OICR_GRST_M) { 2662 u32 reset; 2663 2664 /* we have a reset warning */ 2665 ena_mask &= ~PFINT_OICR_GRST_M; 2666 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 2667 GLGEN_RSTAT_RESET_TYPE_S; 2668 2669 if (reset == ICE_RESET_CORER) 2670 pf->corer_count++; 2671 else if (reset == ICE_RESET_GLOBR) 2672 pf->globr_count++; 2673 else if (reset == ICE_RESET_EMPR) 2674 pf->empr_count++; 2675 else 2676 dev_dbg(dev, "Invalid reset type %d\n", reset); 2677 2678 /* If a reset cycle isn't already in progress, we set a bit in 2679 * pf->state so that the service task can start a reset/rebuild. 2680 * We also make note of which reset happened so that peer 2681 * devices/drivers can be informed. 2682 */ 2683 if (!test_and_set_bit(__ICE_RESET_OICR_RECV, pf->state)) { 2684 if (reset == ICE_RESET_CORER) 2685 set_bit(__ICE_CORER_RECV, pf->state); 2686 else if (reset == ICE_RESET_GLOBR) 2687 set_bit(__ICE_GLOBR_RECV, pf->state); 2688 else 2689 set_bit(__ICE_EMPR_RECV, pf->state); 2690 2691 /* There are couple of different bits at play here. 2692 * hw->reset_ongoing indicates whether the hardware is 2693 * in reset. This is set to true when a reset interrupt 2694 * is received and set back to false after the driver 2695 * has determined that the hardware is out of reset. 2696 * 2697 * __ICE_RESET_OICR_RECV in pf->state indicates 2698 * that a post reset rebuild is required before the 2699 * driver is operational again. This is set above. 2700 * 2701 * As this is the start of the reset/rebuild cycle, set 2702 * both to indicate that. 2703 */ 2704 hw->reset_ongoing = true; 2705 } 2706 } 2707 2708 if (oicr & PFINT_OICR_HMC_ERR_M) { 2709 ena_mask &= ~PFINT_OICR_HMC_ERR_M; 2710 dev_dbg(dev, "HMC Error interrupt - info 0x%x, data 0x%x\n", 2711 rd32(hw, PFHMC_ERRORINFO), 2712 rd32(hw, PFHMC_ERRORDATA)); 2713 } 2714 2715 /* Report any remaining unexpected interrupts */ 2716 oicr &= ena_mask; 2717 if (oicr) { 2718 dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr); 2719 /* If a critical error is pending there is no choice but to 2720 * reset the device. 2721 */ 2722 if (oicr & (PFINT_OICR_PE_CRITERR_M | 2723 PFINT_OICR_PCI_EXCEPTION_M | 2724 PFINT_OICR_ECC_ERR_M)) { 2725 set_bit(__ICE_PFR_REQ, pf->state); 2726 ice_service_task_schedule(pf); 2727 } 2728 } 2729 ret = IRQ_HANDLED; 2730 2731 ice_service_task_schedule(pf); 2732 ice_irq_dynamic_ena(hw, NULL, NULL); 2733 2734 return ret; 2735 } 2736 2737 /** 2738 * ice_dis_ctrlq_interrupts - disable control queue interrupts 2739 * @hw: pointer to HW structure 2740 */ 2741 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw) 2742 { 2743 /* disable Admin queue Interrupt causes */ 2744 wr32(hw, PFINT_FW_CTL, 2745 rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M); 2746 2747 /* disable Mailbox queue Interrupt causes */ 2748 wr32(hw, PFINT_MBX_CTL, 2749 rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M); 2750 2751 /* disable Control queue Interrupt causes */ 2752 wr32(hw, PFINT_OICR_CTL, 2753 rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M); 2754 2755 ice_flush(hw); 2756 } 2757 2758 /** 2759 * ice_free_irq_msix_misc - Unroll misc vector setup 2760 * @pf: board private structure 2761 */ 2762 static void ice_free_irq_msix_misc(struct ice_pf *pf) 2763 { 2764 struct ice_hw *hw = &pf->hw; 2765 2766 ice_dis_ctrlq_interrupts(hw); 2767 2768 /* disable OICR interrupt */ 2769 wr32(hw, PFINT_OICR_ENA, 0); 2770 ice_flush(hw); 2771 2772 if (pf->msix_entries) { 2773 synchronize_irq(pf->msix_entries[pf->oicr_idx].vector); 2774 devm_free_irq(ice_pf_to_dev(pf), 2775 pf->msix_entries[pf->oicr_idx].vector, pf); 2776 } 2777 2778 pf->num_avail_sw_msix += 1; 2779 ice_free_res(pf->irq_tracker, pf->oicr_idx, ICE_RES_MISC_VEC_ID); 2780 } 2781 2782 /** 2783 * ice_ena_ctrlq_interrupts - enable control queue interrupts 2784 * @hw: pointer to HW structure 2785 * @reg_idx: HW vector index to associate the control queue interrupts with 2786 */ 2787 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx) 2788 { 2789 u32 val; 2790 2791 val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) | 2792 PFINT_OICR_CTL_CAUSE_ENA_M); 2793 wr32(hw, PFINT_OICR_CTL, val); 2794 2795 /* enable Admin queue Interrupt causes */ 2796 val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) | 2797 PFINT_FW_CTL_CAUSE_ENA_M); 2798 wr32(hw, PFINT_FW_CTL, val); 2799 2800 /* enable Mailbox queue Interrupt causes */ 2801 val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) | 2802 PFINT_MBX_CTL_CAUSE_ENA_M); 2803 wr32(hw, PFINT_MBX_CTL, val); 2804 2805 ice_flush(hw); 2806 } 2807 2808 /** 2809 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events 2810 * @pf: board private structure 2811 * 2812 * This sets up the handler for MSIX 0, which is used to manage the 2813 * non-queue interrupts, e.g. AdminQ and errors. This is not used 2814 * when in MSI or Legacy interrupt mode. 2815 */ 2816 static int ice_req_irq_msix_misc(struct ice_pf *pf) 2817 { 2818 struct device *dev = ice_pf_to_dev(pf); 2819 struct ice_hw *hw = &pf->hw; 2820 int oicr_idx, err = 0; 2821 2822 if (!pf->int_name[0]) 2823 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc", 2824 dev_driver_string(dev), dev_name(dev)); 2825 2826 /* Do not request IRQ but do enable OICR interrupt since settings are 2827 * lost during reset. Note that this function is called only during 2828 * rebuild path and not while reset is in progress. 2829 */ 2830 if (ice_is_reset_in_progress(pf->state)) 2831 goto skip_req_irq; 2832 2833 /* reserve one vector in irq_tracker for misc interrupts */ 2834 oicr_idx = ice_get_res(pf, pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); 2835 if (oicr_idx < 0) 2836 return oicr_idx; 2837 2838 pf->num_avail_sw_msix -= 1; 2839 pf->oicr_idx = (u16)oicr_idx; 2840 2841 err = devm_request_irq(dev, pf->msix_entries[pf->oicr_idx].vector, 2842 ice_misc_intr, 0, pf->int_name, pf); 2843 if (err) { 2844 dev_err(dev, "devm_request_irq for %s failed: %d\n", 2845 pf->int_name, err); 2846 ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); 2847 pf->num_avail_sw_msix += 1; 2848 return err; 2849 } 2850 2851 skip_req_irq: 2852 ice_ena_misc_vector(pf); 2853 2854 ice_ena_ctrlq_interrupts(hw, pf->oicr_idx); 2855 wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_idx), 2856 ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S); 2857 2858 ice_flush(hw); 2859 ice_irq_dynamic_ena(hw, NULL, NULL); 2860 2861 return 0; 2862 } 2863 2864 /** 2865 * ice_napi_add - register NAPI handler for the VSI 2866 * @vsi: VSI for which NAPI handler is to be registered 2867 * 2868 * This function is only called in the driver's load path. Registering the NAPI 2869 * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, 2870 * reset/rebuild, etc.) 2871 */ 2872 static void ice_napi_add(struct ice_vsi *vsi) 2873 { 2874 int v_idx; 2875 2876 if (!vsi->netdev) 2877 return; 2878 2879 ice_for_each_q_vector(vsi, v_idx) 2880 netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, 2881 ice_napi_poll, NAPI_POLL_WEIGHT); 2882 } 2883 2884 /** 2885 * ice_set_ops - set netdev and ethtools ops for the given netdev 2886 * @netdev: netdev instance 2887 */ 2888 static void ice_set_ops(struct net_device *netdev) 2889 { 2890 struct ice_pf *pf = ice_netdev_to_pf(netdev); 2891 2892 if (ice_is_safe_mode(pf)) { 2893 netdev->netdev_ops = &ice_netdev_safe_mode_ops; 2894 ice_set_ethtool_safe_mode_ops(netdev); 2895 return; 2896 } 2897 2898 netdev->netdev_ops = &ice_netdev_ops; 2899 netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic; 2900 ice_set_ethtool_ops(netdev); 2901 } 2902 2903 /** 2904 * ice_set_netdev_features - set features for the given netdev 2905 * @netdev: netdev instance 2906 */ 2907 static void ice_set_netdev_features(struct net_device *netdev) 2908 { 2909 struct ice_pf *pf = ice_netdev_to_pf(netdev); 2910 netdev_features_t csumo_features; 2911 netdev_features_t vlano_features; 2912 netdev_features_t dflt_features; 2913 netdev_features_t tso_features; 2914 2915 if (ice_is_safe_mode(pf)) { 2916 /* safe mode */ 2917 netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA; 2918 netdev->hw_features = netdev->features; 2919 return; 2920 } 2921 2922 dflt_features = NETIF_F_SG | 2923 NETIF_F_HIGHDMA | 2924 NETIF_F_NTUPLE | 2925 NETIF_F_RXHASH; 2926 2927 csumo_features = NETIF_F_RXCSUM | 2928 NETIF_F_IP_CSUM | 2929 NETIF_F_SCTP_CRC | 2930 NETIF_F_IPV6_CSUM; 2931 2932 vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER | 2933 NETIF_F_HW_VLAN_CTAG_TX | 2934 NETIF_F_HW_VLAN_CTAG_RX; 2935 2936 tso_features = NETIF_F_TSO | 2937 NETIF_F_TSO_ECN | 2938 NETIF_F_TSO6 | 2939 NETIF_F_GSO_GRE | 2940 NETIF_F_GSO_UDP_TUNNEL | 2941 NETIF_F_GSO_GRE_CSUM | 2942 NETIF_F_GSO_UDP_TUNNEL_CSUM | 2943 NETIF_F_GSO_PARTIAL | 2944 NETIF_F_GSO_IPXIP4 | 2945 NETIF_F_GSO_IPXIP6 | 2946 NETIF_F_GSO_UDP_L4; 2947 2948 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM | 2949 NETIF_F_GSO_GRE_CSUM; 2950 /* set features that user can change */ 2951 netdev->hw_features = dflt_features | csumo_features | 2952 vlano_features | tso_features; 2953 2954 /* add support for HW_CSUM on packets with MPLS header */ 2955 netdev->mpls_features = NETIF_F_HW_CSUM; 2956 2957 /* enable features */ 2958 netdev->features |= netdev->hw_features; 2959 /* encap and VLAN devices inherit default, csumo and tso features */ 2960 netdev->hw_enc_features |= dflt_features | csumo_features | 2961 tso_features; 2962 netdev->vlan_features |= dflt_features | csumo_features | 2963 tso_features; 2964 } 2965 2966 /** 2967 * ice_cfg_netdev - Allocate, configure and register a netdev 2968 * @vsi: the VSI associated with the new netdev 2969 * 2970 * Returns 0 on success, negative value on failure 2971 */ 2972 static int ice_cfg_netdev(struct ice_vsi *vsi) 2973 { 2974 struct ice_pf *pf = vsi->back; 2975 struct ice_netdev_priv *np; 2976 struct net_device *netdev; 2977 u8 mac_addr[ETH_ALEN]; 2978 int err; 2979 2980 err = ice_devlink_create_port(vsi); 2981 if (err) 2982 return err; 2983 2984 netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, 2985 vsi->alloc_rxq); 2986 if (!netdev) { 2987 err = -ENOMEM; 2988 goto err_destroy_devlink_port; 2989 } 2990 2991 vsi->netdev = netdev; 2992 np = netdev_priv(netdev); 2993 np->vsi = vsi; 2994 2995 ice_set_netdev_features(netdev); 2996 2997 ice_set_ops(netdev); 2998 2999 if (vsi->type == ICE_VSI_PF) { 3000 SET_NETDEV_DEV(netdev, ice_pf_to_dev(pf)); 3001 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 3002 ether_addr_copy(netdev->dev_addr, mac_addr); 3003 ether_addr_copy(netdev->perm_addr, mac_addr); 3004 } 3005 3006 netdev->priv_flags |= IFF_UNICAST_FLT; 3007 3008 /* Setup netdev TC information */ 3009 ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); 3010 3011 /* setup watchdog timeout value to be 5 second */ 3012 netdev->watchdog_timeo = 5 * HZ; 3013 3014 netdev->min_mtu = ETH_MIN_MTU; 3015 netdev->max_mtu = ICE_MAX_MTU; 3016 3017 err = register_netdev(vsi->netdev); 3018 if (err) 3019 goto err_free_netdev; 3020 3021 devlink_port_type_eth_set(&vsi->devlink_port, vsi->netdev); 3022 3023 netif_carrier_off(vsi->netdev); 3024 3025 /* make sure transmit queues start off as stopped */ 3026 netif_tx_stop_all_queues(vsi->netdev); 3027 3028 return 0; 3029 3030 err_free_netdev: 3031 free_netdev(vsi->netdev); 3032 vsi->netdev = NULL; 3033 err_destroy_devlink_port: 3034 ice_devlink_destroy_port(vsi); 3035 return err; 3036 } 3037 3038 /** 3039 * ice_fill_rss_lut - Fill the RSS lookup table with default values 3040 * @lut: Lookup table 3041 * @rss_table_size: Lookup table size 3042 * @rss_size: Range of queue number for hashing 3043 */ 3044 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size) 3045 { 3046 u16 i; 3047 3048 for (i = 0; i < rss_table_size; i++) 3049 lut[i] = i % rss_size; 3050 } 3051 3052 /** 3053 * ice_pf_vsi_setup - Set up a PF VSI 3054 * @pf: board private structure 3055 * @pi: pointer to the port_info instance 3056 * 3057 * Returns pointer to the successfully allocated VSI software struct 3058 * on success, otherwise returns NULL on failure. 3059 */ 3060 static struct ice_vsi * 3061 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3062 { 3063 return ice_vsi_setup(pf, pi, ICE_VSI_PF, ICE_INVAL_VFID); 3064 } 3065 3066 /** 3067 * ice_ctrl_vsi_setup - Set up a control VSI 3068 * @pf: board private structure 3069 * @pi: pointer to the port_info instance 3070 * 3071 * Returns pointer to the successfully allocated VSI software struct 3072 * on success, otherwise returns NULL on failure. 3073 */ 3074 static struct ice_vsi * 3075 ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3076 { 3077 return ice_vsi_setup(pf, pi, ICE_VSI_CTRL, ICE_INVAL_VFID); 3078 } 3079 3080 /** 3081 * ice_lb_vsi_setup - Set up a loopback VSI 3082 * @pf: board private structure 3083 * @pi: pointer to the port_info instance 3084 * 3085 * Returns pointer to the successfully allocated VSI software struct 3086 * on success, otherwise returns NULL on failure. 3087 */ 3088 struct ice_vsi * 3089 ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3090 { 3091 return ice_vsi_setup(pf, pi, ICE_VSI_LB, ICE_INVAL_VFID); 3092 } 3093 3094 /** 3095 * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload 3096 * @netdev: network interface to be adjusted 3097 * @proto: unused protocol 3098 * @vid: VLAN ID to be added 3099 * 3100 * net_device_ops implementation for adding VLAN IDs 3101 */ 3102 static int 3103 ice_vlan_rx_add_vid(struct net_device *netdev, __always_unused __be16 proto, 3104 u16 vid) 3105 { 3106 struct ice_netdev_priv *np = netdev_priv(netdev); 3107 struct ice_vsi *vsi = np->vsi; 3108 int ret; 3109 3110 if (vid >= VLAN_N_VID) { 3111 netdev_err(netdev, "VLAN id requested %d is out of range %d\n", 3112 vid, VLAN_N_VID); 3113 return -EINVAL; 3114 } 3115 3116 if (vsi->info.pvid) 3117 return -EINVAL; 3118 3119 /* VLAN 0 is added by default during load/reset */ 3120 if (!vid) 3121 return 0; 3122 3123 /* Enable VLAN pruning when a VLAN other than 0 is added */ 3124 if (!ice_vsi_is_vlan_pruning_ena(vsi)) { 3125 ret = ice_cfg_vlan_pruning(vsi, true, false); 3126 if (ret) 3127 return ret; 3128 } 3129 3130 /* Add a switch rule for this VLAN ID so its corresponding VLAN tagged 3131 * packets aren't pruned by the device's internal switch on Rx 3132 */ 3133 ret = ice_vsi_add_vlan(vsi, vid, ICE_FWD_TO_VSI); 3134 if (!ret) 3135 set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 3136 3137 return ret; 3138 } 3139 3140 /** 3141 * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload 3142 * @netdev: network interface to be adjusted 3143 * @proto: unused protocol 3144 * @vid: VLAN ID to be removed 3145 * 3146 * net_device_ops implementation for removing VLAN IDs 3147 */ 3148 static int 3149 ice_vlan_rx_kill_vid(struct net_device *netdev, __always_unused __be16 proto, 3150 u16 vid) 3151 { 3152 struct ice_netdev_priv *np = netdev_priv(netdev); 3153 struct ice_vsi *vsi = np->vsi; 3154 int ret; 3155 3156 if (vsi->info.pvid) 3157 return -EINVAL; 3158 3159 /* don't allow removal of VLAN 0 */ 3160 if (!vid) 3161 return 0; 3162 3163 /* Make sure ice_vsi_kill_vlan is successful before updating VLAN 3164 * information 3165 */ 3166 ret = ice_vsi_kill_vlan(vsi, vid); 3167 if (ret) 3168 return ret; 3169 3170 /* Disable pruning when VLAN 0 is the only VLAN rule */ 3171 if (vsi->num_vlan == 1 && ice_vsi_is_vlan_pruning_ena(vsi)) 3172 ret = ice_cfg_vlan_pruning(vsi, false, false); 3173 3174 set_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags); 3175 return ret; 3176 } 3177 3178 /** 3179 * ice_setup_pf_sw - Setup the HW switch on startup or after reset 3180 * @pf: board private structure 3181 * 3182 * Returns 0 on success, negative value on failure 3183 */ 3184 static int ice_setup_pf_sw(struct ice_pf *pf) 3185 { 3186 struct ice_vsi *vsi; 3187 int status = 0; 3188 3189 if (ice_is_reset_in_progress(pf->state)) 3190 return -EBUSY; 3191 3192 vsi = ice_pf_vsi_setup(pf, pf->hw.port_info); 3193 if (!vsi) 3194 return -ENOMEM; 3195 3196 status = ice_cfg_netdev(vsi); 3197 if (status) { 3198 status = -ENODEV; 3199 goto unroll_vsi_setup; 3200 } 3201 /* netdev has to be configured before setting frame size */ 3202 ice_vsi_cfg_frame_size(vsi); 3203 3204 /* Setup DCB netlink interface */ 3205 ice_dcbnl_setup(vsi); 3206 3207 /* registering the NAPI handler requires both the queues and 3208 * netdev to be created, which are done in ice_pf_vsi_setup() 3209 * and ice_cfg_netdev() respectively 3210 */ 3211 ice_napi_add(vsi); 3212 3213 status = ice_set_cpu_rx_rmap(vsi); 3214 if (status) { 3215 dev_err(ice_pf_to_dev(pf), "Failed to set CPU Rx map VSI %d error %d\n", 3216 vsi->vsi_num, status); 3217 status = -EINVAL; 3218 goto unroll_napi_add; 3219 } 3220 status = ice_init_mac_fltr(pf); 3221 if (status) 3222 goto free_cpu_rx_map; 3223 3224 return status; 3225 3226 free_cpu_rx_map: 3227 ice_free_cpu_rx_rmap(vsi); 3228 3229 unroll_napi_add: 3230 if (vsi) { 3231 ice_napi_del(vsi); 3232 if (vsi->netdev) { 3233 if (vsi->netdev->reg_state == NETREG_REGISTERED) 3234 unregister_netdev(vsi->netdev); 3235 free_netdev(vsi->netdev); 3236 vsi->netdev = NULL; 3237 } 3238 } 3239 3240 unroll_vsi_setup: 3241 ice_vsi_release(vsi); 3242 return status; 3243 } 3244 3245 /** 3246 * ice_get_avail_q_count - Get count of queues in use 3247 * @pf_qmap: bitmap to get queue use count from 3248 * @lock: pointer to a mutex that protects access to pf_qmap 3249 * @size: size of the bitmap 3250 */ 3251 static u16 3252 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size) 3253 { 3254 unsigned long bit; 3255 u16 count = 0; 3256 3257 mutex_lock(lock); 3258 for_each_clear_bit(bit, pf_qmap, size) 3259 count++; 3260 mutex_unlock(lock); 3261 3262 return count; 3263 } 3264 3265 /** 3266 * ice_get_avail_txq_count - Get count of Tx queues in use 3267 * @pf: pointer to an ice_pf instance 3268 */ 3269 u16 ice_get_avail_txq_count(struct ice_pf *pf) 3270 { 3271 return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex, 3272 pf->max_pf_txqs); 3273 } 3274 3275 /** 3276 * ice_get_avail_rxq_count - Get count of Rx queues in use 3277 * @pf: pointer to an ice_pf instance 3278 */ 3279 u16 ice_get_avail_rxq_count(struct ice_pf *pf) 3280 { 3281 return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex, 3282 pf->max_pf_rxqs); 3283 } 3284 3285 /** 3286 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf 3287 * @pf: board private structure to initialize 3288 */ 3289 static void ice_deinit_pf(struct ice_pf *pf) 3290 { 3291 ice_service_task_stop(pf); 3292 mutex_destroy(&pf->sw_mutex); 3293 mutex_destroy(&pf->tc_mutex); 3294 mutex_destroy(&pf->avail_q_mutex); 3295 3296 if (pf->avail_txqs) { 3297 bitmap_free(pf->avail_txqs); 3298 pf->avail_txqs = NULL; 3299 } 3300 3301 if (pf->avail_rxqs) { 3302 bitmap_free(pf->avail_rxqs); 3303 pf->avail_rxqs = NULL; 3304 } 3305 } 3306 3307 /** 3308 * ice_set_pf_caps - set PFs capability flags 3309 * @pf: pointer to the PF instance 3310 */ 3311 static void ice_set_pf_caps(struct ice_pf *pf) 3312 { 3313 struct ice_hw_func_caps *func_caps = &pf->hw.func_caps; 3314 3315 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3316 if (func_caps->common_cap.dcb) 3317 set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3318 clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3319 if (func_caps->common_cap.sr_iov_1_1) { 3320 set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3321 pf->num_vfs_supported = min_t(int, func_caps->num_allocd_vfs, 3322 ICE_MAX_VF_COUNT); 3323 } 3324 clear_bit(ICE_FLAG_RSS_ENA, pf->flags); 3325 if (func_caps->common_cap.rss_table_size) 3326 set_bit(ICE_FLAG_RSS_ENA, pf->flags); 3327 3328 clear_bit(ICE_FLAG_FD_ENA, pf->flags); 3329 if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) { 3330 u16 unused; 3331 3332 /* ctrl_vsi_idx will be set to a valid value when flow director 3333 * is setup by ice_init_fdir 3334 */ 3335 pf->ctrl_vsi_idx = ICE_NO_VSI; 3336 set_bit(ICE_FLAG_FD_ENA, pf->flags); 3337 /* force guaranteed filter pool for PF */ 3338 ice_alloc_fd_guar_item(&pf->hw, &unused, 3339 func_caps->fd_fltr_guar); 3340 /* force shared filter pool for PF */ 3341 ice_alloc_fd_shrd_item(&pf->hw, &unused, 3342 func_caps->fd_fltr_best_effort); 3343 } 3344 3345 pf->max_pf_txqs = func_caps->common_cap.num_txq; 3346 pf->max_pf_rxqs = func_caps->common_cap.num_rxq; 3347 } 3348 3349 /** 3350 * ice_init_pf - Initialize general software structures (struct ice_pf) 3351 * @pf: board private structure to initialize 3352 */ 3353 static int ice_init_pf(struct ice_pf *pf) 3354 { 3355 ice_set_pf_caps(pf); 3356 3357 mutex_init(&pf->sw_mutex); 3358 mutex_init(&pf->tc_mutex); 3359 3360 INIT_HLIST_HEAD(&pf->aq_wait_list); 3361 spin_lock_init(&pf->aq_wait_lock); 3362 init_waitqueue_head(&pf->aq_wait_queue); 3363 3364 /* setup service timer and periodic service task */ 3365 timer_setup(&pf->serv_tmr, ice_service_timer, 0); 3366 pf->serv_tmr_period = HZ; 3367 INIT_WORK(&pf->serv_task, ice_service_task); 3368 clear_bit(__ICE_SERVICE_SCHED, pf->state); 3369 3370 mutex_init(&pf->avail_q_mutex); 3371 pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); 3372 if (!pf->avail_txqs) 3373 return -ENOMEM; 3374 3375 pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); 3376 if (!pf->avail_rxqs) { 3377 devm_kfree(ice_pf_to_dev(pf), pf->avail_txqs); 3378 pf->avail_txqs = NULL; 3379 return -ENOMEM; 3380 } 3381 3382 return 0; 3383 } 3384 3385 /** 3386 * ice_ena_msix_range - Request a range of MSIX vectors from the OS 3387 * @pf: board private structure 3388 * 3389 * compute the number of MSIX vectors required (v_budget) and request from 3390 * the OS. Return the number of vectors reserved or negative on failure 3391 */ 3392 static int ice_ena_msix_range(struct ice_pf *pf) 3393 { 3394 int v_left, v_actual, v_other, v_budget = 0; 3395 struct device *dev = ice_pf_to_dev(pf); 3396 int needed, err, i; 3397 3398 v_left = pf->hw.func_caps.common_cap.num_msix_vectors; 3399 3400 /* reserve for LAN miscellaneous handler */ 3401 needed = ICE_MIN_LAN_OICR_MSIX; 3402 if (v_left < needed) 3403 goto no_hw_vecs_left_err; 3404 v_budget += needed; 3405 v_left -= needed; 3406 3407 /* reserve for flow director */ 3408 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 3409 needed = ICE_FDIR_MSIX; 3410 if (v_left < needed) 3411 goto no_hw_vecs_left_err; 3412 v_budget += needed; 3413 v_left -= needed; 3414 } 3415 3416 /* total used for non-traffic vectors */ 3417 v_other = v_budget; 3418 3419 /* reserve vectors for LAN traffic */ 3420 needed = min_t(int, num_online_cpus(), v_left); 3421 if (v_left < needed) 3422 goto no_hw_vecs_left_err; 3423 pf->num_lan_msix = needed; 3424 v_budget += needed; 3425 v_left -= needed; 3426 3427 pf->msix_entries = devm_kcalloc(dev, v_budget, 3428 sizeof(*pf->msix_entries), GFP_KERNEL); 3429 if (!pf->msix_entries) { 3430 err = -ENOMEM; 3431 goto exit_err; 3432 } 3433 3434 for (i = 0; i < v_budget; i++) 3435 pf->msix_entries[i].entry = i; 3436 3437 /* actually reserve the vectors */ 3438 v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries, 3439 ICE_MIN_MSIX, v_budget); 3440 if (v_actual < 0) { 3441 dev_err(dev, "unable to reserve MSI-X vectors\n"); 3442 err = v_actual; 3443 goto msix_err; 3444 } 3445 3446 if (v_actual < v_budget) { 3447 dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n", 3448 v_budget, v_actual); 3449 3450 if (v_actual < ICE_MIN_MSIX) { 3451 /* error if we can't get minimum vectors */ 3452 pci_disable_msix(pf->pdev); 3453 err = -ERANGE; 3454 goto msix_err; 3455 } else { 3456 int v_traffic = v_actual - v_other; 3457 3458 if (v_actual == ICE_MIN_MSIX || 3459 v_traffic < ICE_MIN_LAN_TXRX_MSIX) 3460 pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX; 3461 else 3462 pf->num_lan_msix = v_traffic; 3463 3464 dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n", 3465 pf->num_lan_msix); 3466 } 3467 } 3468 3469 return v_actual; 3470 3471 msix_err: 3472 devm_kfree(dev, pf->msix_entries); 3473 goto exit_err; 3474 3475 no_hw_vecs_left_err: 3476 dev_err(dev, "not enough device MSI-X vectors. requested = %d, available = %d\n", 3477 needed, v_left); 3478 err = -ERANGE; 3479 exit_err: 3480 pf->num_lan_msix = 0; 3481 return err; 3482 } 3483 3484 /** 3485 * ice_dis_msix - Disable MSI-X interrupt setup in OS 3486 * @pf: board private structure 3487 */ 3488 static void ice_dis_msix(struct ice_pf *pf) 3489 { 3490 pci_disable_msix(pf->pdev); 3491 devm_kfree(ice_pf_to_dev(pf), pf->msix_entries); 3492 pf->msix_entries = NULL; 3493 } 3494 3495 /** 3496 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme 3497 * @pf: board private structure 3498 */ 3499 static void ice_clear_interrupt_scheme(struct ice_pf *pf) 3500 { 3501 ice_dis_msix(pf); 3502 3503 if (pf->irq_tracker) { 3504 devm_kfree(ice_pf_to_dev(pf), pf->irq_tracker); 3505 pf->irq_tracker = NULL; 3506 } 3507 } 3508 3509 /** 3510 * ice_init_interrupt_scheme - Determine proper interrupt scheme 3511 * @pf: board private structure to initialize 3512 */ 3513 static int ice_init_interrupt_scheme(struct ice_pf *pf) 3514 { 3515 int vectors; 3516 3517 vectors = ice_ena_msix_range(pf); 3518 3519 if (vectors < 0) 3520 return vectors; 3521 3522 /* set up vector assignment tracking */ 3523 pf->irq_tracker = devm_kzalloc(ice_pf_to_dev(pf), 3524 struct_size(pf->irq_tracker, list, vectors), 3525 GFP_KERNEL); 3526 if (!pf->irq_tracker) { 3527 ice_dis_msix(pf); 3528 return -ENOMEM; 3529 } 3530 3531 /* populate SW interrupts pool with number of OS granted IRQs. */ 3532 pf->num_avail_sw_msix = (u16)vectors; 3533 pf->irq_tracker->num_entries = (u16)vectors; 3534 pf->irq_tracker->end = pf->irq_tracker->num_entries; 3535 3536 return 0; 3537 } 3538 3539 /** 3540 * ice_is_wol_supported - get NVM state of WoL 3541 * @pf: board private structure 3542 * 3543 * Check if WoL is supported based on the HW configuration. 3544 * Returns true if NVM supports and enables WoL for this port, false otherwise 3545 */ 3546 bool ice_is_wol_supported(struct ice_pf *pf) 3547 { 3548 struct ice_hw *hw = &pf->hw; 3549 u16 wol_ctrl; 3550 3551 /* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control 3552 * word) indicates WoL is not supported on the corresponding PF ID. 3553 */ 3554 if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl)) 3555 return false; 3556 3557 return !(BIT(hw->pf_id) & wol_ctrl); 3558 } 3559 3560 /** 3561 * ice_vsi_recfg_qs - Change the number of queues on a VSI 3562 * @vsi: VSI being changed 3563 * @new_rx: new number of Rx queues 3564 * @new_tx: new number of Tx queues 3565 * 3566 * Only change the number of queues if new_tx, or new_rx is non-0. 3567 * 3568 * Returns 0 on success. 3569 */ 3570 int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx) 3571 { 3572 struct ice_pf *pf = vsi->back; 3573 int err = 0, timeout = 50; 3574 3575 if (!new_rx && !new_tx) 3576 return -EINVAL; 3577 3578 while (test_and_set_bit(__ICE_CFG_BUSY, pf->state)) { 3579 timeout--; 3580 if (!timeout) 3581 return -EBUSY; 3582 usleep_range(1000, 2000); 3583 } 3584 3585 if (new_tx) 3586 vsi->req_txq = (u16)new_tx; 3587 if (new_rx) 3588 vsi->req_rxq = (u16)new_rx; 3589 3590 /* set for the next time the netdev is started */ 3591 if (!netif_running(vsi->netdev)) { 3592 ice_vsi_rebuild(vsi, false); 3593 dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n"); 3594 goto done; 3595 } 3596 3597 ice_vsi_close(vsi); 3598 ice_vsi_rebuild(vsi, false); 3599 ice_pf_dcb_recfg(pf); 3600 ice_vsi_open(vsi); 3601 done: 3602 clear_bit(__ICE_CFG_BUSY, pf->state); 3603 return err; 3604 } 3605 3606 /** 3607 * ice_set_safe_mode_vlan_cfg - configure PF VSI to allow all VLANs in safe mode 3608 * @pf: PF to configure 3609 * 3610 * No VLAN offloads/filtering are advertised in safe mode so make sure the PF 3611 * VSI can still Tx/Rx VLAN tagged packets. 3612 */ 3613 static void ice_set_safe_mode_vlan_cfg(struct ice_pf *pf) 3614 { 3615 struct ice_vsi *vsi = ice_get_main_vsi(pf); 3616 struct ice_vsi_ctx *ctxt; 3617 enum ice_status status; 3618 struct ice_hw *hw; 3619 3620 if (!vsi) 3621 return; 3622 3623 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 3624 if (!ctxt) 3625 return; 3626 3627 hw = &pf->hw; 3628 ctxt->info = vsi->info; 3629 3630 ctxt->info.valid_sections = 3631 cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID | 3632 ICE_AQ_VSI_PROP_SECURITY_VALID | 3633 ICE_AQ_VSI_PROP_SW_VALID); 3634 3635 /* disable VLAN anti-spoof */ 3636 ctxt->info.sec_flags &= ~(ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA << 3637 ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S); 3638 3639 /* disable VLAN pruning and keep all other settings */ 3640 ctxt->info.sw_flags2 &= ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; 3641 3642 /* allow all VLANs on Tx and don't strip on Rx */ 3643 ctxt->info.vlan_flags = ICE_AQ_VSI_VLAN_MODE_ALL | 3644 ICE_AQ_VSI_VLAN_EMOD_NOTHING; 3645 3646 status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 3647 if (status) { 3648 dev_err(ice_pf_to_dev(vsi->back), "Failed to update VSI for safe mode VLANs, err %s aq_err %s\n", 3649 ice_stat_str(status), 3650 ice_aq_str(hw->adminq.sq_last_status)); 3651 } else { 3652 vsi->info.sec_flags = ctxt->info.sec_flags; 3653 vsi->info.sw_flags2 = ctxt->info.sw_flags2; 3654 vsi->info.vlan_flags = ctxt->info.vlan_flags; 3655 } 3656 3657 kfree(ctxt); 3658 } 3659 3660 /** 3661 * ice_log_pkg_init - log result of DDP package load 3662 * @hw: pointer to hardware info 3663 * @status: status of package load 3664 */ 3665 static void 3666 ice_log_pkg_init(struct ice_hw *hw, enum ice_status *status) 3667 { 3668 struct ice_pf *pf = (struct ice_pf *)hw->back; 3669 struct device *dev = ice_pf_to_dev(pf); 3670 3671 switch (*status) { 3672 case ICE_SUCCESS: 3673 /* The package download AdminQ command returned success because 3674 * this download succeeded or ICE_ERR_AQ_NO_WORK since there is 3675 * already a package loaded on the device. 3676 */ 3677 if (hw->pkg_ver.major == hw->active_pkg_ver.major && 3678 hw->pkg_ver.minor == hw->active_pkg_ver.minor && 3679 hw->pkg_ver.update == hw->active_pkg_ver.update && 3680 hw->pkg_ver.draft == hw->active_pkg_ver.draft && 3681 !memcmp(hw->pkg_name, hw->active_pkg_name, 3682 sizeof(hw->pkg_name))) { 3683 if (hw->pkg_dwnld_status == ICE_AQ_RC_EEXIST) 3684 dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n", 3685 hw->active_pkg_name, 3686 hw->active_pkg_ver.major, 3687 hw->active_pkg_ver.minor, 3688 hw->active_pkg_ver.update, 3689 hw->active_pkg_ver.draft); 3690 else 3691 dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n", 3692 hw->active_pkg_name, 3693 hw->active_pkg_ver.major, 3694 hw->active_pkg_ver.minor, 3695 hw->active_pkg_ver.update, 3696 hw->active_pkg_ver.draft); 3697 } else if (hw->active_pkg_ver.major != ICE_PKG_SUPP_VER_MAJ || 3698 hw->active_pkg_ver.minor != ICE_PKG_SUPP_VER_MNR) { 3699 dev_err(dev, "The device has a DDP package that is not supported by the driver. The device has package '%s' version %d.%d.x.x. The driver requires version %d.%d.x.x. Entering Safe Mode.\n", 3700 hw->active_pkg_name, 3701 hw->active_pkg_ver.major, 3702 hw->active_pkg_ver.minor, 3703 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 3704 *status = ICE_ERR_NOT_SUPPORTED; 3705 } else if (hw->active_pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3706 hw->active_pkg_ver.minor == ICE_PKG_SUPP_VER_MNR) { 3707 dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device. The device has package '%s' version %d.%d.%d.%d. The package file found by the driver: '%s' version %d.%d.%d.%d.\n", 3708 hw->active_pkg_name, 3709 hw->active_pkg_ver.major, 3710 hw->active_pkg_ver.minor, 3711 hw->active_pkg_ver.update, 3712 hw->active_pkg_ver.draft, 3713 hw->pkg_name, 3714 hw->pkg_ver.major, 3715 hw->pkg_ver.minor, 3716 hw->pkg_ver.update, 3717 hw->pkg_ver.draft); 3718 } else { 3719 dev_err(dev, "An unknown error occurred when loading the DDP package, please reboot the system. If the problem persists, update the NVM. Entering Safe Mode.\n"); 3720 *status = ICE_ERR_NOT_SUPPORTED; 3721 } 3722 break; 3723 case ICE_ERR_FW_DDP_MISMATCH: 3724 dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package. Please update the device's NVM. Entering safe mode.\n"); 3725 break; 3726 case ICE_ERR_BUF_TOO_SHORT: 3727 case ICE_ERR_CFG: 3728 dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n"); 3729 break; 3730 case ICE_ERR_NOT_SUPPORTED: 3731 /* Package File version not supported */ 3732 if (hw->pkg_ver.major > ICE_PKG_SUPP_VER_MAJ || 3733 (hw->pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3734 hw->pkg_ver.minor > ICE_PKG_SUPP_VER_MNR)) 3735 dev_err(dev, "The DDP package file version is higher than the driver supports. Please use an updated driver. Entering Safe Mode.\n"); 3736 else if (hw->pkg_ver.major < ICE_PKG_SUPP_VER_MAJ || 3737 (hw->pkg_ver.major == ICE_PKG_SUPP_VER_MAJ && 3738 hw->pkg_ver.minor < ICE_PKG_SUPP_VER_MNR)) 3739 dev_err(dev, "The DDP package file version is lower than the driver supports. The driver requires version %d.%d.x.x. Please use an updated DDP Package file. Entering Safe Mode.\n", 3740 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 3741 break; 3742 case ICE_ERR_AQ_ERROR: 3743 switch (hw->pkg_dwnld_status) { 3744 case ICE_AQ_RC_ENOSEC: 3745 case ICE_AQ_RC_EBADSIG: 3746 dev_err(dev, "The DDP package could not be loaded because its signature is not valid. Please use a valid DDP Package. Entering Safe Mode.\n"); 3747 return; 3748 case ICE_AQ_RC_ESVN: 3749 dev_err(dev, "The DDP Package could not be loaded because its security revision is too low. Please use an updated DDP Package. Entering Safe Mode.\n"); 3750 return; 3751 case ICE_AQ_RC_EBADMAN: 3752 case ICE_AQ_RC_EBADBUF: 3753 dev_err(dev, "An error occurred on the device while loading the DDP package. The device will be reset.\n"); 3754 /* poll for reset to complete */ 3755 if (ice_check_reset(hw)) 3756 dev_err(dev, "Error resetting device. Please reload the driver\n"); 3757 return; 3758 default: 3759 break; 3760 } 3761 fallthrough; 3762 default: 3763 dev_err(dev, "An unknown error (%d) occurred when loading the DDP package. Entering Safe Mode.\n", 3764 *status); 3765 break; 3766 } 3767 } 3768 3769 /** 3770 * ice_load_pkg - load/reload the DDP Package file 3771 * @firmware: firmware structure when firmware requested or NULL for reload 3772 * @pf: pointer to the PF instance 3773 * 3774 * Called on probe and post CORER/GLOBR rebuild to load DDP Package and 3775 * initialize HW tables. 3776 */ 3777 static void 3778 ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf) 3779 { 3780 enum ice_status status = ICE_ERR_PARAM; 3781 struct device *dev = ice_pf_to_dev(pf); 3782 struct ice_hw *hw = &pf->hw; 3783 3784 /* Load DDP Package */ 3785 if (firmware && !hw->pkg_copy) { 3786 status = ice_copy_and_init_pkg(hw, firmware->data, 3787 firmware->size); 3788 ice_log_pkg_init(hw, &status); 3789 } else if (!firmware && hw->pkg_copy) { 3790 /* Reload package during rebuild after CORER/GLOBR reset */ 3791 status = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 3792 ice_log_pkg_init(hw, &status); 3793 } else { 3794 dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n"); 3795 } 3796 3797 if (status) { 3798 /* Safe Mode */ 3799 clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 3800 return; 3801 } 3802 3803 /* Successful download package is the precondition for advanced 3804 * features, hence setting the ICE_FLAG_ADV_FEATURES flag 3805 */ 3806 set_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 3807 } 3808 3809 /** 3810 * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines 3811 * @pf: pointer to the PF structure 3812 * 3813 * There is no error returned here because the driver should be able to handle 3814 * 128 Byte cache lines, so we only print a warning in case issues are seen, 3815 * specifically with Tx. 3816 */ 3817 static void ice_verify_cacheline_size(struct ice_pf *pf) 3818 { 3819 if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M) 3820 dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n", 3821 ICE_CACHE_LINE_BYTES); 3822 } 3823 3824 /** 3825 * ice_send_version - update firmware with driver version 3826 * @pf: PF struct 3827 * 3828 * Returns ICE_SUCCESS on success, else error code 3829 */ 3830 static enum ice_status ice_send_version(struct ice_pf *pf) 3831 { 3832 struct ice_driver_ver dv; 3833 3834 dv.major_ver = 0xff; 3835 dv.minor_ver = 0xff; 3836 dv.build_ver = 0xff; 3837 dv.subbuild_ver = 0; 3838 strscpy((char *)dv.driver_string, UTS_RELEASE, 3839 sizeof(dv.driver_string)); 3840 return ice_aq_send_driver_ver(&pf->hw, &dv, NULL); 3841 } 3842 3843 /** 3844 * ice_init_fdir - Initialize flow director VSI and configuration 3845 * @pf: pointer to the PF instance 3846 * 3847 * returns 0 on success, negative on error 3848 */ 3849 static int ice_init_fdir(struct ice_pf *pf) 3850 { 3851 struct device *dev = ice_pf_to_dev(pf); 3852 struct ice_vsi *ctrl_vsi; 3853 int err; 3854 3855 /* Side Band Flow Director needs to have a control VSI. 3856 * Allocate it and store it in the PF. 3857 */ 3858 ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info); 3859 if (!ctrl_vsi) { 3860 dev_dbg(dev, "could not create control VSI\n"); 3861 return -ENOMEM; 3862 } 3863 3864 err = ice_vsi_open_ctrl(ctrl_vsi); 3865 if (err) { 3866 dev_dbg(dev, "could not open control VSI\n"); 3867 goto err_vsi_open; 3868 } 3869 3870 mutex_init(&pf->hw.fdir_fltr_lock); 3871 3872 err = ice_fdir_create_dflt_rules(pf); 3873 if (err) 3874 goto err_fdir_rule; 3875 3876 return 0; 3877 3878 err_fdir_rule: 3879 ice_fdir_release_flows(&pf->hw); 3880 ice_vsi_close(ctrl_vsi); 3881 err_vsi_open: 3882 ice_vsi_release(ctrl_vsi); 3883 if (pf->ctrl_vsi_idx != ICE_NO_VSI) { 3884 pf->vsi[pf->ctrl_vsi_idx] = NULL; 3885 pf->ctrl_vsi_idx = ICE_NO_VSI; 3886 } 3887 return err; 3888 } 3889 3890 /** 3891 * ice_get_opt_fw_name - return optional firmware file name or NULL 3892 * @pf: pointer to the PF instance 3893 */ 3894 static char *ice_get_opt_fw_name(struct ice_pf *pf) 3895 { 3896 /* Optional firmware name same as default with additional dash 3897 * followed by a EUI-64 identifier (PCIe Device Serial Number) 3898 */ 3899 struct pci_dev *pdev = pf->pdev; 3900 char *opt_fw_filename; 3901 u64 dsn; 3902 3903 /* Determine the name of the optional file using the DSN (two 3904 * dwords following the start of the DSN Capability). 3905 */ 3906 dsn = pci_get_dsn(pdev); 3907 if (!dsn) 3908 return NULL; 3909 3910 opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL); 3911 if (!opt_fw_filename) 3912 return NULL; 3913 3914 snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg", 3915 ICE_DDP_PKG_PATH, dsn); 3916 3917 return opt_fw_filename; 3918 } 3919 3920 /** 3921 * ice_request_fw - Device initialization routine 3922 * @pf: pointer to the PF instance 3923 */ 3924 static void ice_request_fw(struct ice_pf *pf) 3925 { 3926 char *opt_fw_filename = ice_get_opt_fw_name(pf); 3927 const struct firmware *firmware = NULL; 3928 struct device *dev = ice_pf_to_dev(pf); 3929 int err = 0; 3930 3931 /* optional device-specific DDP (if present) overrides the default DDP 3932 * package file. kernel logs a debug message if the file doesn't exist, 3933 * and warning messages for other errors. 3934 */ 3935 if (opt_fw_filename) { 3936 err = firmware_request_nowarn(&firmware, opt_fw_filename, dev); 3937 if (err) { 3938 kfree(opt_fw_filename); 3939 goto dflt_pkg_load; 3940 } 3941 3942 /* request for firmware was successful. Download to device */ 3943 ice_load_pkg(firmware, pf); 3944 kfree(opt_fw_filename); 3945 release_firmware(firmware); 3946 return; 3947 } 3948 3949 dflt_pkg_load: 3950 err = request_firmware(&firmware, ICE_DDP_PKG_FILE, dev); 3951 if (err) { 3952 dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n"); 3953 return; 3954 } 3955 3956 /* request for firmware was successful. Download to device */ 3957 ice_load_pkg(firmware, pf); 3958 release_firmware(firmware); 3959 } 3960 3961 /** 3962 * ice_print_wake_reason - show the wake up cause in the log 3963 * @pf: pointer to the PF struct 3964 */ 3965 static void ice_print_wake_reason(struct ice_pf *pf) 3966 { 3967 u32 wus = pf->wakeup_reason; 3968 const char *wake_str; 3969 3970 /* if no wake event, nothing to print */ 3971 if (!wus) 3972 return; 3973 3974 if (wus & PFPM_WUS_LNKC_M) 3975 wake_str = "Link\n"; 3976 else if (wus & PFPM_WUS_MAG_M) 3977 wake_str = "Magic Packet\n"; 3978 else if (wus & PFPM_WUS_MNG_M) 3979 wake_str = "Management\n"; 3980 else if (wus & PFPM_WUS_FW_RST_WK_M) 3981 wake_str = "Firmware Reset\n"; 3982 else 3983 wake_str = "Unknown\n"; 3984 3985 dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str); 3986 } 3987 3988 /** 3989 * ice_probe - Device initialization routine 3990 * @pdev: PCI device information struct 3991 * @ent: entry in ice_pci_tbl 3992 * 3993 * Returns 0 on success, negative on failure 3994 */ 3995 static int 3996 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) 3997 { 3998 struct device *dev = &pdev->dev; 3999 struct ice_pf *pf; 4000 struct ice_hw *hw; 4001 int i, err; 4002 4003 /* this driver uses devres, see 4004 * Documentation/driver-api/driver-model/devres.rst 4005 */ 4006 err = pcim_enable_device(pdev); 4007 if (err) 4008 return err; 4009 4010 err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), pci_name(pdev)); 4011 if (err) { 4012 dev_err(dev, "BAR0 I/O map error %d\n", err); 4013 return err; 4014 } 4015 4016 pf = ice_allocate_pf(dev); 4017 if (!pf) 4018 return -ENOMEM; 4019 4020 /* set up for high or low DMA */ 4021 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 4022 if (err) 4023 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 4024 if (err) { 4025 dev_err(dev, "DMA configuration failed: 0x%x\n", err); 4026 return err; 4027 } 4028 4029 pci_enable_pcie_error_reporting(pdev); 4030 pci_set_master(pdev); 4031 4032 pf->pdev = pdev; 4033 pci_set_drvdata(pdev, pf); 4034 set_bit(__ICE_DOWN, pf->state); 4035 /* Disable service task until DOWN bit is cleared */ 4036 set_bit(__ICE_SERVICE_DIS, pf->state); 4037 4038 hw = &pf->hw; 4039 hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0]; 4040 pci_save_state(pdev); 4041 4042 hw->back = pf; 4043 hw->vendor_id = pdev->vendor; 4044 hw->device_id = pdev->device; 4045 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 4046 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4047 hw->subsystem_device_id = pdev->subsystem_device; 4048 hw->bus.device = PCI_SLOT(pdev->devfn); 4049 hw->bus.func = PCI_FUNC(pdev->devfn); 4050 ice_set_ctrlq_len(hw); 4051 4052 pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M); 4053 4054 err = ice_devlink_register(pf); 4055 if (err) { 4056 dev_err(dev, "ice_devlink_register failed: %d\n", err); 4057 goto err_exit_unroll; 4058 } 4059 4060 #ifndef CONFIG_DYNAMIC_DEBUG 4061 if (debug < -1) 4062 hw->debug_mask = debug; 4063 #endif 4064 4065 err = ice_init_hw(hw); 4066 if (err) { 4067 dev_err(dev, "ice_init_hw failed: %d\n", err); 4068 err = -EIO; 4069 goto err_exit_unroll; 4070 } 4071 4072 ice_request_fw(pf); 4073 4074 /* if ice_request_fw fails, ICE_FLAG_ADV_FEATURES bit won't be 4075 * set in pf->state, which will cause ice_is_safe_mode to return 4076 * true 4077 */ 4078 if (ice_is_safe_mode(pf)) { 4079 dev_err(dev, "Package download failed. Advanced features disabled - Device now in Safe Mode\n"); 4080 /* we already got function/device capabilities but these don't 4081 * reflect what the driver needs to do in safe mode. Instead of 4082 * adding conditional logic everywhere to ignore these 4083 * device/function capabilities, override them. 4084 */ 4085 ice_set_safe_mode_caps(hw); 4086 } 4087 4088 err = ice_init_pf(pf); 4089 if (err) { 4090 dev_err(dev, "ice_init_pf failed: %d\n", err); 4091 goto err_init_pf_unroll; 4092 } 4093 4094 ice_devlink_init_regions(pf); 4095 4096 pf->hw.udp_tunnel_nic.set_port = ice_udp_tunnel_set_port; 4097 pf->hw.udp_tunnel_nic.unset_port = ice_udp_tunnel_unset_port; 4098 pf->hw.udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP; 4099 pf->hw.udp_tunnel_nic.shared = &pf->hw.udp_tunnel_shared; 4100 i = 0; 4101 if (pf->hw.tnl.valid_count[TNL_VXLAN]) { 4102 pf->hw.udp_tunnel_nic.tables[i].n_entries = 4103 pf->hw.tnl.valid_count[TNL_VXLAN]; 4104 pf->hw.udp_tunnel_nic.tables[i].tunnel_types = 4105 UDP_TUNNEL_TYPE_VXLAN; 4106 i++; 4107 } 4108 if (pf->hw.tnl.valid_count[TNL_GENEVE]) { 4109 pf->hw.udp_tunnel_nic.tables[i].n_entries = 4110 pf->hw.tnl.valid_count[TNL_GENEVE]; 4111 pf->hw.udp_tunnel_nic.tables[i].tunnel_types = 4112 UDP_TUNNEL_TYPE_GENEVE; 4113 i++; 4114 } 4115 4116 pf->num_alloc_vsi = hw->func_caps.guar_num_vsi; 4117 if (!pf->num_alloc_vsi) { 4118 err = -EIO; 4119 goto err_init_pf_unroll; 4120 } 4121 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 4122 dev_warn(&pf->pdev->dev, 4123 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 4124 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 4125 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 4126 } 4127 4128 pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi), 4129 GFP_KERNEL); 4130 if (!pf->vsi) { 4131 err = -ENOMEM; 4132 goto err_init_pf_unroll; 4133 } 4134 4135 err = ice_init_interrupt_scheme(pf); 4136 if (err) { 4137 dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); 4138 err = -EIO; 4139 goto err_init_vsi_unroll; 4140 } 4141 4142 /* In case of MSIX we are going to setup the misc vector right here 4143 * to handle admin queue events etc. In case of legacy and MSI 4144 * the misc functionality and queue processing is combined in 4145 * the same vector and that gets setup at open. 4146 */ 4147 err = ice_req_irq_msix_misc(pf); 4148 if (err) { 4149 dev_err(dev, "setup of misc vector failed: %d\n", err); 4150 goto err_init_interrupt_unroll; 4151 } 4152 4153 /* create switch struct for the switch element created by FW on boot */ 4154 pf->first_sw = devm_kzalloc(dev, sizeof(*pf->first_sw), GFP_KERNEL); 4155 if (!pf->first_sw) { 4156 err = -ENOMEM; 4157 goto err_msix_misc_unroll; 4158 } 4159 4160 if (hw->evb_veb) 4161 pf->first_sw->bridge_mode = BRIDGE_MODE_VEB; 4162 else 4163 pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA; 4164 4165 pf->first_sw->pf = pf; 4166 4167 /* record the sw_id available for later use */ 4168 pf->first_sw->sw_id = hw->port_info->sw_id; 4169 4170 err = ice_setup_pf_sw(pf); 4171 if (err) { 4172 dev_err(dev, "probe failed due to setup PF switch: %d\n", err); 4173 goto err_alloc_sw_unroll; 4174 } 4175 4176 clear_bit(__ICE_SERVICE_DIS, pf->state); 4177 4178 /* tell the firmware we are up */ 4179 err = ice_send_version(pf); 4180 if (err) { 4181 dev_err(dev, "probe failed sending driver version %s. error: %d\n", 4182 UTS_RELEASE, err); 4183 goto err_send_version_unroll; 4184 } 4185 4186 /* since everything is good, start the service timer */ 4187 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4188 4189 err = ice_init_link_events(pf->hw.port_info); 4190 if (err) { 4191 dev_err(dev, "ice_init_link_events failed: %d\n", err); 4192 goto err_send_version_unroll; 4193 } 4194 4195 err = ice_init_nvm_phy_type(pf->hw.port_info); 4196 if (err) { 4197 dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err); 4198 goto err_send_version_unroll; 4199 } 4200 4201 err = ice_update_link_info(pf->hw.port_info); 4202 if (err) { 4203 dev_err(dev, "ice_update_link_info failed: %d\n", err); 4204 goto err_send_version_unroll; 4205 } 4206 4207 ice_init_link_dflt_override(pf->hw.port_info); 4208 4209 /* if media available, initialize PHY settings */ 4210 if (pf->hw.port_info->phy.link_info.link_info & 4211 ICE_AQ_MEDIA_AVAILABLE) { 4212 err = ice_init_phy_user_cfg(pf->hw.port_info); 4213 if (err) { 4214 dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err); 4215 goto err_send_version_unroll; 4216 } 4217 4218 if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) { 4219 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4220 4221 if (vsi) 4222 ice_configure_phy(vsi); 4223 } 4224 } else { 4225 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 4226 } 4227 4228 ice_verify_cacheline_size(pf); 4229 4230 /* Save wakeup reason register for later use */ 4231 pf->wakeup_reason = rd32(hw, PFPM_WUS); 4232 4233 /* check for a power management event */ 4234 ice_print_wake_reason(pf); 4235 4236 /* clear wake status, all bits */ 4237 wr32(hw, PFPM_WUS, U32_MAX); 4238 4239 /* Disable WoL at init, wait for user to enable */ 4240 device_set_wakeup_enable(dev, false); 4241 4242 if (ice_is_safe_mode(pf)) { 4243 ice_set_safe_mode_vlan_cfg(pf); 4244 goto probe_done; 4245 } 4246 4247 /* initialize DDP driven features */ 4248 4249 /* Note: Flow director init failure is non-fatal to load */ 4250 if (ice_init_fdir(pf)) 4251 dev_err(dev, "could not initialize flow director\n"); 4252 4253 /* Note: DCB init failure is non-fatal to load */ 4254 if (ice_init_pf_dcb(pf, false)) { 4255 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 4256 clear_bit(ICE_FLAG_DCB_ENA, pf->flags); 4257 } else { 4258 ice_cfg_lldp_mib_change(&pf->hw, true); 4259 } 4260 4261 if (ice_init_lag(pf)) 4262 dev_warn(dev, "Failed to init link aggregation support\n"); 4263 4264 /* print PCI link speed and width */ 4265 pcie_print_link_status(pf->pdev); 4266 4267 probe_done: 4268 /* ready to go, so clear down state bit */ 4269 clear_bit(__ICE_DOWN, pf->state); 4270 return 0; 4271 4272 err_send_version_unroll: 4273 ice_vsi_release_all(pf); 4274 err_alloc_sw_unroll: 4275 set_bit(__ICE_SERVICE_DIS, pf->state); 4276 set_bit(__ICE_DOWN, pf->state); 4277 devm_kfree(dev, pf->first_sw); 4278 err_msix_misc_unroll: 4279 ice_free_irq_msix_misc(pf); 4280 err_init_interrupt_unroll: 4281 ice_clear_interrupt_scheme(pf); 4282 err_init_vsi_unroll: 4283 devm_kfree(dev, pf->vsi); 4284 err_init_pf_unroll: 4285 ice_deinit_pf(pf); 4286 ice_devlink_destroy_regions(pf); 4287 ice_deinit_hw(hw); 4288 err_exit_unroll: 4289 ice_devlink_unregister(pf); 4290 pci_disable_pcie_error_reporting(pdev); 4291 pci_disable_device(pdev); 4292 return err; 4293 } 4294 4295 /** 4296 * ice_set_wake - enable or disable Wake on LAN 4297 * @pf: pointer to the PF struct 4298 * 4299 * Simple helper for WoL control 4300 */ 4301 static void ice_set_wake(struct ice_pf *pf) 4302 { 4303 struct ice_hw *hw = &pf->hw; 4304 bool wol = pf->wol_ena; 4305 4306 /* clear wake state, otherwise new wake events won't fire */ 4307 wr32(hw, PFPM_WUS, U32_MAX); 4308 4309 /* enable / disable APM wake up, no RMW needed */ 4310 wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0); 4311 4312 /* set magic packet filter enabled */ 4313 wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0); 4314 } 4315 4316 /** 4317 * ice_setup_magic_mc_wake - setup device to wake on multicast magic packet 4318 * @pf: pointer to the PF struct 4319 * 4320 * Issue firmware command to enable multicast magic wake, making 4321 * sure that any locally administered address (LAA) is used for 4322 * wake, and that PF reset doesn't undo the LAA. 4323 */ 4324 static void ice_setup_mc_magic_wake(struct ice_pf *pf) 4325 { 4326 struct device *dev = ice_pf_to_dev(pf); 4327 struct ice_hw *hw = &pf->hw; 4328 enum ice_status status; 4329 u8 mac_addr[ETH_ALEN]; 4330 struct ice_vsi *vsi; 4331 u8 flags; 4332 4333 if (!pf->wol_ena) 4334 return; 4335 4336 vsi = ice_get_main_vsi(pf); 4337 if (!vsi) 4338 return; 4339 4340 /* Get current MAC address in case it's an LAA */ 4341 if (vsi->netdev) 4342 ether_addr_copy(mac_addr, vsi->netdev->dev_addr); 4343 else 4344 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 4345 4346 flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN | 4347 ICE_AQC_MAN_MAC_UPDATE_LAA_WOL | 4348 ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP; 4349 4350 status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL); 4351 if (status) 4352 dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %s aq_err %s\n", 4353 ice_stat_str(status), 4354 ice_aq_str(hw->adminq.sq_last_status)); 4355 } 4356 4357 /** 4358 * ice_remove - Device removal routine 4359 * @pdev: PCI device information struct 4360 */ 4361 static void ice_remove(struct pci_dev *pdev) 4362 { 4363 struct ice_pf *pf = pci_get_drvdata(pdev); 4364 int i; 4365 4366 if (!pf) 4367 return; 4368 4369 for (i = 0; i < ICE_MAX_RESET_WAIT; i++) { 4370 if (!ice_is_reset_in_progress(pf->state)) 4371 break; 4372 msleep(100); 4373 } 4374 4375 if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) { 4376 set_bit(__ICE_VF_RESETS_DISABLED, pf->state); 4377 ice_free_vfs(pf); 4378 } 4379 4380 set_bit(__ICE_DOWN, pf->state); 4381 ice_service_task_stop(pf); 4382 4383 ice_aq_cancel_waiting_tasks(pf); 4384 4385 mutex_destroy(&(&pf->hw)->fdir_fltr_lock); 4386 ice_deinit_lag(pf); 4387 if (!ice_is_safe_mode(pf)) 4388 ice_remove_arfs(pf); 4389 ice_setup_mc_magic_wake(pf); 4390 ice_vsi_release_all(pf); 4391 ice_set_wake(pf); 4392 ice_free_irq_msix_misc(pf); 4393 ice_for_each_vsi(pf, i) { 4394 if (!pf->vsi[i]) 4395 continue; 4396 ice_vsi_free_q_vectors(pf->vsi[i]); 4397 } 4398 ice_deinit_pf(pf); 4399 ice_devlink_destroy_regions(pf); 4400 ice_deinit_hw(&pf->hw); 4401 ice_devlink_unregister(pf); 4402 4403 /* Issue a PFR as part of the prescribed driver unload flow. Do not 4404 * do it via ice_schedule_reset() since there is no need to rebuild 4405 * and the service task is already stopped. 4406 */ 4407 ice_reset(&pf->hw, ICE_RESET_PFR); 4408 pci_wait_for_pending_transaction(pdev); 4409 ice_clear_interrupt_scheme(pf); 4410 pci_disable_pcie_error_reporting(pdev); 4411 pci_disable_device(pdev); 4412 } 4413 4414 /** 4415 * ice_shutdown - PCI callback for shutting down device 4416 * @pdev: PCI device information struct 4417 */ 4418 static void ice_shutdown(struct pci_dev *pdev) 4419 { 4420 struct ice_pf *pf = pci_get_drvdata(pdev); 4421 4422 ice_remove(pdev); 4423 4424 if (system_state == SYSTEM_POWER_OFF) { 4425 pci_wake_from_d3(pdev, pf->wol_ena); 4426 pci_set_power_state(pdev, PCI_D3hot); 4427 } 4428 } 4429 4430 #ifdef CONFIG_PM 4431 /** 4432 * ice_prepare_for_shutdown - prep for PCI shutdown 4433 * @pf: board private structure 4434 * 4435 * Inform or close all dependent features in prep for PCI device shutdown 4436 */ 4437 static void ice_prepare_for_shutdown(struct ice_pf *pf) 4438 { 4439 struct ice_hw *hw = &pf->hw; 4440 u32 v; 4441 4442 /* Notify VFs of impending reset */ 4443 if (ice_check_sq_alive(hw, &hw->mailboxq)) 4444 ice_vc_notify_reset(pf); 4445 4446 dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n"); 4447 4448 /* disable the VSIs and their queues that are not already DOWN */ 4449 ice_pf_dis_all_vsi(pf, false); 4450 4451 ice_for_each_vsi(pf, v) 4452 if (pf->vsi[v]) 4453 pf->vsi[v]->vsi_num = 0; 4454 4455 ice_shutdown_all_ctrlq(hw); 4456 } 4457 4458 /** 4459 * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme 4460 * @pf: board private structure to reinitialize 4461 * 4462 * This routine reinitialize interrupt scheme that was cleared during 4463 * power management suspend callback. 4464 * 4465 * This should be called during resume routine to re-allocate the q_vectors 4466 * and reacquire interrupts. 4467 */ 4468 static int ice_reinit_interrupt_scheme(struct ice_pf *pf) 4469 { 4470 struct device *dev = ice_pf_to_dev(pf); 4471 int ret, v; 4472 4473 /* Since we clear MSIX flag during suspend, we need to 4474 * set it back during resume... 4475 */ 4476 4477 ret = ice_init_interrupt_scheme(pf); 4478 if (ret) { 4479 dev_err(dev, "Failed to re-initialize interrupt %d\n", ret); 4480 return ret; 4481 } 4482 4483 /* Remap vectors and rings, after successful re-init interrupts */ 4484 ice_for_each_vsi(pf, v) { 4485 if (!pf->vsi[v]) 4486 continue; 4487 4488 ret = ice_vsi_alloc_q_vectors(pf->vsi[v]); 4489 if (ret) 4490 goto err_reinit; 4491 ice_vsi_map_rings_to_vectors(pf->vsi[v]); 4492 } 4493 4494 ret = ice_req_irq_msix_misc(pf); 4495 if (ret) { 4496 dev_err(dev, "Setting up misc vector failed after device suspend %d\n", 4497 ret); 4498 goto err_reinit; 4499 } 4500 4501 return 0; 4502 4503 err_reinit: 4504 while (v--) 4505 if (pf->vsi[v]) 4506 ice_vsi_free_q_vectors(pf->vsi[v]); 4507 4508 return ret; 4509 } 4510 4511 /** 4512 * ice_suspend 4513 * @dev: generic device information structure 4514 * 4515 * Power Management callback to quiesce the device and prepare 4516 * for D3 transition. 4517 */ 4518 static int __maybe_unused ice_suspend(struct device *dev) 4519 { 4520 struct pci_dev *pdev = to_pci_dev(dev); 4521 struct ice_pf *pf; 4522 int disabled, v; 4523 4524 pf = pci_get_drvdata(pdev); 4525 4526 if (!ice_pf_state_is_nominal(pf)) { 4527 dev_err(dev, "Device is not ready, no need to suspend it\n"); 4528 return -EBUSY; 4529 } 4530 4531 /* Stop watchdog tasks until resume completion. 4532 * Even though it is most likely that the service task is 4533 * disabled if the device is suspended or down, the service task's 4534 * state is controlled by a different state bit, and we should 4535 * store and honor whatever state that bit is in at this point. 4536 */ 4537 disabled = ice_service_task_stop(pf); 4538 4539 /* Already suspended?, then there is nothing to do */ 4540 if (test_and_set_bit(__ICE_SUSPENDED, pf->state)) { 4541 if (!disabled) 4542 ice_service_task_restart(pf); 4543 return 0; 4544 } 4545 4546 if (test_bit(__ICE_DOWN, pf->state) || 4547 ice_is_reset_in_progress(pf->state)) { 4548 dev_err(dev, "can't suspend device in reset or already down\n"); 4549 if (!disabled) 4550 ice_service_task_restart(pf); 4551 return 0; 4552 } 4553 4554 ice_setup_mc_magic_wake(pf); 4555 4556 ice_prepare_for_shutdown(pf); 4557 4558 ice_set_wake(pf); 4559 4560 /* Free vectors, clear the interrupt scheme and release IRQs 4561 * for proper hibernation, especially with large number of CPUs. 4562 * Otherwise hibernation might fail when mapping all the vectors back 4563 * to CPU0. 4564 */ 4565 ice_free_irq_msix_misc(pf); 4566 ice_for_each_vsi(pf, v) { 4567 if (!pf->vsi[v]) 4568 continue; 4569 ice_vsi_free_q_vectors(pf->vsi[v]); 4570 } 4571 ice_clear_interrupt_scheme(pf); 4572 4573 pci_save_state(pdev); 4574 pci_wake_from_d3(pdev, pf->wol_ena); 4575 pci_set_power_state(pdev, PCI_D3hot); 4576 return 0; 4577 } 4578 4579 /** 4580 * ice_resume - PM callback for waking up from D3 4581 * @dev: generic device information structure 4582 */ 4583 static int __maybe_unused ice_resume(struct device *dev) 4584 { 4585 struct pci_dev *pdev = to_pci_dev(dev); 4586 enum ice_reset_req reset_type; 4587 struct ice_pf *pf; 4588 struct ice_hw *hw; 4589 int ret; 4590 4591 pci_set_power_state(pdev, PCI_D0); 4592 pci_restore_state(pdev); 4593 pci_save_state(pdev); 4594 4595 if (!pci_device_is_present(pdev)) 4596 return -ENODEV; 4597 4598 ret = pci_enable_device_mem(pdev); 4599 if (ret) { 4600 dev_err(dev, "Cannot enable device after suspend\n"); 4601 return ret; 4602 } 4603 4604 pf = pci_get_drvdata(pdev); 4605 hw = &pf->hw; 4606 4607 pf->wakeup_reason = rd32(hw, PFPM_WUS); 4608 ice_print_wake_reason(pf); 4609 4610 /* We cleared the interrupt scheme when we suspended, so we need to 4611 * restore it now to resume device functionality. 4612 */ 4613 ret = ice_reinit_interrupt_scheme(pf); 4614 if (ret) 4615 dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret); 4616 4617 clear_bit(__ICE_DOWN, pf->state); 4618 /* Now perform PF reset and rebuild */ 4619 reset_type = ICE_RESET_PFR; 4620 /* re-enable service task for reset, but allow reset to schedule it */ 4621 clear_bit(__ICE_SERVICE_DIS, pf->state); 4622 4623 if (ice_schedule_reset(pf, reset_type)) 4624 dev_err(dev, "Reset during resume failed.\n"); 4625 4626 clear_bit(__ICE_SUSPENDED, pf->state); 4627 ice_service_task_restart(pf); 4628 4629 /* Restart the service task */ 4630 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4631 4632 return 0; 4633 } 4634 #endif /* CONFIG_PM */ 4635 4636 /** 4637 * ice_pci_err_detected - warning that PCI error has been detected 4638 * @pdev: PCI device information struct 4639 * @err: the type of PCI error 4640 * 4641 * Called to warn that something happened on the PCI bus and the error handling 4642 * is in progress. Allows the driver to gracefully prepare/handle PCI errors. 4643 */ 4644 static pci_ers_result_t 4645 ice_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t err) 4646 { 4647 struct ice_pf *pf = pci_get_drvdata(pdev); 4648 4649 if (!pf) { 4650 dev_err(&pdev->dev, "%s: unrecoverable device error %d\n", 4651 __func__, err); 4652 return PCI_ERS_RESULT_DISCONNECT; 4653 } 4654 4655 if (!test_bit(__ICE_SUSPENDED, pf->state)) { 4656 ice_service_task_stop(pf); 4657 4658 if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) { 4659 set_bit(__ICE_PFR_REQ, pf->state); 4660 ice_prepare_for_reset(pf); 4661 } 4662 } 4663 4664 return PCI_ERS_RESULT_NEED_RESET; 4665 } 4666 4667 /** 4668 * ice_pci_err_slot_reset - a PCI slot reset has just happened 4669 * @pdev: PCI device information struct 4670 * 4671 * Called to determine if the driver can recover from the PCI slot reset by 4672 * using a register read to determine if the device is recoverable. 4673 */ 4674 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) 4675 { 4676 struct ice_pf *pf = pci_get_drvdata(pdev); 4677 pci_ers_result_t result; 4678 int err; 4679 u32 reg; 4680 4681 err = pci_enable_device_mem(pdev); 4682 if (err) { 4683 dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n", 4684 err); 4685 result = PCI_ERS_RESULT_DISCONNECT; 4686 } else { 4687 pci_set_master(pdev); 4688 pci_restore_state(pdev); 4689 pci_save_state(pdev); 4690 pci_wake_from_d3(pdev, false); 4691 4692 /* Check for life */ 4693 reg = rd32(&pf->hw, GLGEN_RTRIG); 4694 if (!reg) 4695 result = PCI_ERS_RESULT_RECOVERED; 4696 else 4697 result = PCI_ERS_RESULT_DISCONNECT; 4698 } 4699 4700 err = pci_aer_clear_nonfatal_status(pdev); 4701 if (err) 4702 dev_dbg(&pdev->dev, "pci_aer_clear_nonfatal_status() failed, error %d\n", 4703 err); 4704 /* non-fatal, continue */ 4705 4706 return result; 4707 } 4708 4709 /** 4710 * ice_pci_err_resume - restart operations after PCI error recovery 4711 * @pdev: PCI device information struct 4712 * 4713 * Called to allow the driver to bring things back up after PCI error and/or 4714 * reset recovery have finished 4715 */ 4716 static void ice_pci_err_resume(struct pci_dev *pdev) 4717 { 4718 struct ice_pf *pf = pci_get_drvdata(pdev); 4719 4720 if (!pf) { 4721 dev_err(&pdev->dev, "%s failed, device is unrecoverable\n", 4722 __func__); 4723 return; 4724 } 4725 4726 if (test_bit(__ICE_SUSPENDED, pf->state)) { 4727 dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n", 4728 __func__); 4729 return; 4730 } 4731 4732 ice_restore_all_vfs_msi_state(pdev); 4733 4734 ice_do_reset(pf, ICE_RESET_PFR); 4735 ice_service_task_restart(pf); 4736 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4737 } 4738 4739 /** 4740 * ice_pci_err_reset_prepare - prepare device driver for PCI reset 4741 * @pdev: PCI device information struct 4742 */ 4743 static void ice_pci_err_reset_prepare(struct pci_dev *pdev) 4744 { 4745 struct ice_pf *pf = pci_get_drvdata(pdev); 4746 4747 if (!test_bit(__ICE_SUSPENDED, pf->state)) { 4748 ice_service_task_stop(pf); 4749 4750 if (!test_bit(__ICE_PREPARED_FOR_RESET, pf->state)) { 4751 set_bit(__ICE_PFR_REQ, pf->state); 4752 ice_prepare_for_reset(pf); 4753 } 4754 } 4755 } 4756 4757 /** 4758 * ice_pci_err_reset_done - PCI reset done, device driver reset can begin 4759 * @pdev: PCI device information struct 4760 */ 4761 static void ice_pci_err_reset_done(struct pci_dev *pdev) 4762 { 4763 ice_pci_err_resume(pdev); 4764 } 4765 4766 /* ice_pci_tbl - PCI Device ID Table 4767 * 4768 * Wildcard entries (PCI_ANY_ID) should come last 4769 * Last entry must be all 0s 4770 * 4771 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 4772 * Class, Class Mask, private data (not used) } 4773 */ 4774 static const struct pci_device_id ice_pci_tbl[] = { 4775 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE), 0 }, 4776 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP), 0 }, 4777 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP), 0 }, 4778 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP), 0 }, 4779 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE), 0 }, 4780 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP), 0 }, 4781 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP), 0 }, 4782 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T), 0 }, 4783 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII), 0 }, 4784 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE), 0 }, 4785 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP), 0 }, 4786 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP), 0 }, 4787 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T), 0 }, 4788 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII), 0 }, 4789 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE), 0 }, 4790 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP), 0 }, 4791 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T), 0 }, 4792 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII), 0 }, 4793 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE), 0 }, 4794 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP), 0 }, 4795 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T), 0 }, 4796 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE), 0 }, 4797 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP), 0 }, 4798 /* required last entry */ 4799 { 0, } 4800 }; 4801 MODULE_DEVICE_TABLE(pci, ice_pci_tbl); 4802 4803 static __maybe_unused SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume); 4804 4805 static const struct pci_error_handlers ice_pci_err_handler = { 4806 .error_detected = ice_pci_err_detected, 4807 .slot_reset = ice_pci_err_slot_reset, 4808 .reset_prepare = ice_pci_err_reset_prepare, 4809 .reset_done = ice_pci_err_reset_done, 4810 .resume = ice_pci_err_resume 4811 }; 4812 4813 static struct pci_driver ice_driver = { 4814 .name = KBUILD_MODNAME, 4815 .id_table = ice_pci_tbl, 4816 .probe = ice_probe, 4817 .remove = ice_remove, 4818 #ifdef CONFIG_PM 4819 .driver.pm = &ice_pm_ops, 4820 #endif /* CONFIG_PM */ 4821 .shutdown = ice_shutdown, 4822 .sriov_configure = ice_sriov_configure, 4823 .err_handler = &ice_pci_err_handler 4824 }; 4825 4826 /** 4827 * ice_module_init - Driver registration routine 4828 * 4829 * ice_module_init is the first routine called when the driver is 4830 * loaded. All it does is register with the PCI subsystem. 4831 */ 4832 static int __init ice_module_init(void) 4833 { 4834 int status; 4835 4836 pr_info("%s\n", ice_driver_string); 4837 pr_info("%s\n", ice_copyright); 4838 4839 ice_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, KBUILD_MODNAME); 4840 if (!ice_wq) { 4841 pr_err("Failed to create workqueue\n"); 4842 return -ENOMEM; 4843 } 4844 4845 status = pci_register_driver(&ice_driver); 4846 if (status) { 4847 pr_err("failed to register PCI driver, err %d\n", status); 4848 destroy_workqueue(ice_wq); 4849 } 4850 4851 return status; 4852 } 4853 module_init(ice_module_init); 4854 4855 /** 4856 * ice_module_exit - Driver exit cleanup routine 4857 * 4858 * ice_module_exit is called just before the driver is removed 4859 * from memory. 4860 */ 4861 static void __exit ice_module_exit(void) 4862 { 4863 pci_unregister_driver(&ice_driver); 4864 destroy_workqueue(ice_wq); 4865 pr_info("module unloaded\n"); 4866 } 4867 module_exit(ice_module_exit); 4868 4869 /** 4870 * ice_set_mac_address - NDO callback to set MAC address 4871 * @netdev: network interface device structure 4872 * @pi: pointer to an address structure 4873 * 4874 * Returns 0 on success, negative on failure 4875 */ 4876 static int ice_set_mac_address(struct net_device *netdev, void *pi) 4877 { 4878 struct ice_netdev_priv *np = netdev_priv(netdev); 4879 struct ice_vsi *vsi = np->vsi; 4880 struct ice_pf *pf = vsi->back; 4881 struct ice_hw *hw = &pf->hw; 4882 struct sockaddr *addr = pi; 4883 enum ice_status status; 4884 u8 flags = 0; 4885 int err = 0; 4886 u8 *mac; 4887 4888 mac = (u8 *)addr->sa_data; 4889 4890 if (!is_valid_ether_addr(mac)) 4891 return -EADDRNOTAVAIL; 4892 4893 if (ether_addr_equal(netdev->dev_addr, mac)) { 4894 netdev_warn(netdev, "already using mac %pM\n", mac); 4895 return 0; 4896 } 4897 4898 if (test_bit(__ICE_DOWN, pf->state) || 4899 ice_is_reset_in_progress(pf->state)) { 4900 netdev_err(netdev, "can't set mac %pM. device not ready\n", 4901 mac); 4902 return -EBUSY; 4903 } 4904 4905 /* Clean up old MAC filter. Not an error if old filter doesn't exist */ 4906 status = ice_fltr_remove_mac(vsi, netdev->dev_addr, ICE_FWD_TO_VSI); 4907 if (status && status != ICE_ERR_DOES_NOT_EXIST) { 4908 err = -EADDRNOTAVAIL; 4909 goto err_update_filters; 4910 } 4911 4912 /* Add filter for new MAC. If filter exists, return success */ 4913 status = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI); 4914 if (status == ICE_ERR_ALREADY_EXISTS) { 4915 /* Although this MAC filter is already present in hardware it's 4916 * possible in some cases (e.g. bonding) that dev_addr was 4917 * modified outside of the driver and needs to be restored back 4918 * to this value. 4919 */ 4920 memcpy(netdev->dev_addr, mac, netdev->addr_len); 4921 netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac); 4922 return 0; 4923 } 4924 4925 /* error if the new filter addition failed */ 4926 if (status) 4927 err = -EADDRNOTAVAIL; 4928 4929 err_update_filters: 4930 if (err) { 4931 netdev_err(netdev, "can't set MAC %pM. filter update failed\n", 4932 mac); 4933 return err; 4934 } 4935 4936 /* change the netdev's MAC address */ 4937 memcpy(netdev->dev_addr, mac, netdev->addr_len); 4938 netdev_dbg(vsi->netdev, "updated MAC address to %pM\n", 4939 netdev->dev_addr); 4940 4941 /* write new MAC address to the firmware */ 4942 flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL; 4943 status = ice_aq_manage_mac_write(hw, mac, flags, NULL); 4944 if (status) { 4945 netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %s\n", 4946 mac, ice_stat_str(status)); 4947 } 4948 return 0; 4949 } 4950 4951 /** 4952 * ice_set_rx_mode - NDO callback to set the netdev filters 4953 * @netdev: network interface device structure 4954 */ 4955 static void ice_set_rx_mode(struct net_device *netdev) 4956 { 4957 struct ice_netdev_priv *np = netdev_priv(netdev); 4958 struct ice_vsi *vsi = np->vsi; 4959 4960 if (!vsi) 4961 return; 4962 4963 /* Set the flags to synchronize filters 4964 * ndo_set_rx_mode may be triggered even without a change in netdev 4965 * flags 4966 */ 4967 set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags); 4968 set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags); 4969 set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags); 4970 4971 /* schedule our worker thread which will take care of 4972 * applying the new filter changes 4973 */ 4974 ice_service_task_schedule(vsi->back); 4975 } 4976 4977 /** 4978 * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate 4979 * @netdev: network interface device structure 4980 * @queue_index: Queue ID 4981 * @maxrate: maximum bandwidth in Mbps 4982 */ 4983 static int 4984 ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate) 4985 { 4986 struct ice_netdev_priv *np = netdev_priv(netdev); 4987 struct ice_vsi *vsi = np->vsi; 4988 enum ice_status status; 4989 u16 q_handle; 4990 u8 tc; 4991 4992 /* Validate maxrate requested is within permitted range */ 4993 if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) { 4994 netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n", 4995 maxrate, queue_index); 4996 return -EINVAL; 4997 } 4998 4999 q_handle = vsi->tx_rings[queue_index]->q_handle; 5000 tc = ice_dcb_get_tc(vsi, queue_index); 5001 5002 /* Set BW back to default, when user set maxrate to 0 */ 5003 if (!maxrate) 5004 status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc, 5005 q_handle, ICE_MAX_BW); 5006 else 5007 status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc, 5008 q_handle, ICE_MAX_BW, maxrate * 1000); 5009 if (status) { 5010 netdev_err(netdev, "Unable to set Tx max rate, error %s\n", 5011 ice_stat_str(status)); 5012 return -EIO; 5013 } 5014 5015 return 0; 5016 } 5017 5018 /** 5019 * ice_fdb_add - add an entry to the hardware database 5020 * @ndm: the input from the stack 5021 * @tb: pointer to array of nladdr (unused) 5022 * @dev: the net device pointer 5023 * @addr: the MAC address entry being added 5024 * @vid: VLAN ID 5025 * @flags: instructions from stack about fdb operation 5026 * @extack: netlink extended ack 5027 */ 5028 static int 5029 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[], 5030 struct net_device *dev, const unsigned char *addr, u16 vid, 5031 u16 flags, struct netlink_ext_ack __always_unused *extack) 5032 { 5033 int err; 5034 5035 if (vid) { 5036 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n"); 5037 return -EINVAL; 5038 } 5039 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 5040 netdev_err(dev, "FDB only supports static addresses\n"); 5041 return -EINVAL; 5042 } 5043 5044 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 5045 err = dev_uc_add_excl(dev, addr); 5046 else if (is_multicast_ether_addr(addr)) 5047 err = dev_mc_add_excl(dev, addr); 5048 else 5049 err = -EINVAL; 5050 5051 /* Only return duplicate errors if NLM_F_EXCL is set */ 5052 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 5053 err = 0; 5054 5055 return err; 5056 } 5057 5058 /** 5059 * ice_fdb_del - delete an entry from the hardware database 5060 * @ndm: the input from the stack 5061 * @tb: pointer to array of nladdr (unused) 5062 * @dev: the net device pointer 5063 * @addr: the MAC address entry being added 5064 * @vid: VLAN ID 5065 */ 5066 static int 5067 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[], 5068 struct net_device *dev, const unsigned char *addr, 5069 __always_unused u16 vid) 5070 { 5071 int err; 5072 5073 if (ndm->ndm_state & NUD_PERMANENT) { 5074 netdev_err(dev, "FDB only supports static addresses\n"); 5075 return -EINVAL; 5076 } 5077 5078 if (is_unicast_ether_addr(addr)) 5079 err = dev_uc_del(dev, addr); 5080 else if (is_multicast_ether_addr(addr)) 5081 err = dev_mc_del(dev, addr); 5082 else 5083 err = -EINVAL; 5084 5085 return err; 5086 } 5087 5088 /** 5089 * ice_set_features - set the netdev feature flags 5090 * @netdev: ptr to the netdev being adjusted 5091 * @features: the feature set that the stack is suggesting 5092 */ 5093 static int 5094 ice_set_features(struct net_device *netdev, netdev_features_t features) 5095 { 5096 struct ice_netdev_priv *np = netdev_priv(netdev); 5097 struct ice_vsi *vsi = np->vsi; 5098 struct ice_pf *pf = vsi->back; 5099 int ret = 0; 5100 5101 /* Don't set any netdev advanced features with device in Safe Mode */ 5102 if (ice_is_safe_mode(vsi->back)) { 5103 dev_err(ice_pf_to_dev(vsi->back), "Device is in Safe Mode - not enabling advanced netdev features\n"); 5104 return ret; 5105 } 5106 5107 /* Do not change setting during reset */ 5108 if (ice_is_reset_in_progress(pf->state)) { 5109 dev_err(ice_pf_to_dev(vsi->back), "Device is resetting, changing advanced netdev features temporarily unavailable.\n"); 5110 return -EBUSY; 5111 } 5112 5113 /* Multiple features can be changed in one call so keep features in 5114 * separate if/else statements to guarantee each feature is checked 5115 */ 5116 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 5117 ret = ice_vsi_manage_rss_lut(vsi, true); 5118 else if (!(features & NETIF_F_RXHASH) && 5119 netdev->features & NETIF_F_RXHASH) 5120 ret = ice_vsi_manage_rss_lut(vsi, false); 5121 5122 if ((features & NETIF_F_HW_VLAN_CTAG_RX) && 5123 !(netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 5124 ret = ice_vsi_manage_vlan_stripping(vsi, true); 5125 else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) && 5126 (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) 5127 ret = ice_vsi_manage_vlan_stripping(vsi, false); 5128 5129 if ((features & NETIF_F_HW_VLAN_CTAG_TX) && 5130 !(netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) 5131 ret = ice_vsi_manage_vlan_insertion(vsi); 5132 else if (!(features & NETIF_F_HW_VLAN_CTAG_TX) && 5133 (netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) 5134 ret = ice_vsi_manage_vlan_insertion(vsi); 5135 5136 if ((features & NETIF_F_HW_VLAN_CTAG_FILTER) && 5137 !(netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) 5138 ret = ice_cfg_vlan_pruning(vsi, true, false); 5139 else if (!(features & NETIF_F_HW_VLAN_CTAG_FILTER) && 5140 (netdev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) 5141 ret = ice_cfg_vlan_pruning(vsi, false, false); 5142 5143 if ((features & NETIF_F_NTUPLE) && 5144 !(netdev->features & NETIF_F_NTUPLE)) { 5145 ice_vsi_manage_fdir(vsi, true); 5146 ice_init_arfs(vsi); 5147 } else if (!(features & NETIF_F_NTUPLE) && 5148 (netdev->features & NETIF_F_NTUPLE)) { 5149 ice_vsi_manage_fdir(vsi, false); 5150 ice_clear_arfs(vsi); 5151 } 5152 5153 return ret; 5154 } 5155 5156 /** 5157 * ice_vsi_vlan_setup - Setup VLAN offload properties on a VSI 5158 * @vsi: VSI to setup VLAN properties for 5159 */ 5160 static int ice_vsi_vlan_setup(struct ice_vsi *vsi) 5161 { 5162 int ret = 0; 5163 5164 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 5165 ret = ice_vsi_manage_vlan_stripping(vsi, true); 5166 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_TX) 5167 ret = ice_vsi_manage_vlan_insertion(vsi); 5168 5169 return ret; 5170 } 5171 5172 /** 5173 * ice_vsi_cfg - Setup the VSI 5174 * @vsi: the VSI being configured 5175 * 5176 * Return 0 on success and negative value on error 5177 */ 5178 int ice_vsi_cfg(struct ice_vsi *vsi) 5179 { 5180 int err; 5181 5182 if (vsi->netdev) { 5183 ice_set_rx_mode(vsi->netdev); 5184 5185 err = ice_vsi_vlan_setup(vsi); 5186 5187 if (err) 5188 return err; 5189 } 5190 ice_vsi_cfg_dcb_rings(vsi); 5191 5192 err = ice_vsi_cfg_lan_txqs(vsi); 5193 if (!err && ice_is_xdp_ena_vsi(vsi)) 5194 err = ice_vsi_cfg_xdp_txqs(vsi); 5195 if (!err) 5196 err = ice_vsi_cfg_rxqs(vsi); 5197 5198 return err; 5199 } 5200 5201 /** 5202 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI 5203 * @vsi: the VSI being configured 5204 */ 5205 static void ice_napi_enable_all(struct ice_vsi *vsi) 5206 { 5207 int q_idx; 5208 5209 if (!vsi->netdev) 5210 return; 5211 5212 ice_for_each_q_vector(vsi, q_idx) { 5213 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 5214 5215 if (q_vector->rx.ring || q_vector->tx.ring) 5216 napi_enable(&q_vector->napi); 5217 } 5218 } 5219 5220 /** 5221 * ice_up_complete - Finish the last steps of bringing up a connection 5222 * @vsi: The VSI being configured 5223 * 5224 * Return 0 on success and negative value on error 5225 */ 5226 static int ice_up_complete(struct ice_vsi *vsi) 5227 { 5228 struct ice_pf *pf = vsi->back; 5229 int err; 5230 5231 ice_vsi_cfg_msix(vsi); 5232 5233 /* Enable only Rx rings, Tx rings were enabled by the FW when the 5234 * Tx queue group list was configured and the context bits were 5235 * programmed using ice_vsi_cfg_txqs 5236 */ 5237 err = ice_vsi_start_all_rx_rings(vsi); 5238 if (err) 5239 return err; 5240 5241 clear_bit(__ICE_DOWN, vsi->state); 5242 ice_napi_enable_all(vsi); 5243 ice_vsi_ena_irq(vsi); 5244 5245 if (vsi->port_info && 5246 (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) && 5247 vsi->netdev) { 5248 ice_print_link_msg(vsi, true); 5249 netif_tx_start_all_queues(vsi->netdev); 5250 netif_carrier_on(vsi->netdev); 5251 } 5252 5253 ice_service_task_schedule(pf); 5254 5255 return 0; 5256 } 5257 5258 /** 5259 * ice_up - Bring the connection back up after being down 5260 * @vsi: VSI being configured 5261 */ 5262 int ice_up(struct ice_vsi *vsi) 5263 { 5264 int err; 5265 5266 err = ice_vsi_cfg(vsi); 5267 if (!err) 5268 err = ice_up_complete(vsi); 5269 5270 return err; 5271 } 5272 5273 /** 5274 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring 5275 * @ring: Tx or Rx ring to read stats from 5276 * @pkts: packets stats counter 5277 * @bytes: bytes stats counter 5278 * 5279 * This function fetches stats from the ring considering the atomic operations 5280 * that needs to be performed to read u64 values in 32 bit machine. 5281 */ 5282 static void 5283 ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts, u64 *bytes) 5284 { 5285 unsigned int start; 5286 *pkts = 0; 5287 *bytes = 0; 5288 5289 if (!ring) 5290 return; 5291 do { 5292 start = u64_stats_fetch_begin_irq(&ring->syncp); 5293 *pkts = ring->stats.pkts; 5294 *bytes = ring->stats.bytes; 5295 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 5296 } 5297 5298 /** 5299 * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters 5300 * @vsi: the VSI to be updated 5301 * @rings: rings to work on 5302 * @count: number of rings 5303 */ 5304 static void 5305 ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi, struct ice_ring **rings, 5306 u16 count) 5307 { 5308 struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; 5309 u16 i; 5310 5311 for (i = 0; i < count; i++) { 5312 struct ice_ring *ring; 5313 u64 pkts, bytes; 5314 5315 ring = READ_ONCE(rings[i]); 5316 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); 5317 vsi_stats->tx_packets += pkts; 5318 vsi_stats->tx_bytes += bytes; 5319 vsi->tx_restart += ring->tx_stats.restart_q; 5320 vsi->tx_busy += ring->tx_stats.tx_busy; 5321 vsi->tx_linearize += ring->tx_stats.tx_linearize; 5322 } 5323 } 5324 5325 /** 5326 * ice_update_vsi_ring_stats - Update VSI stats counters 5327 * @vsi: the VSI to be updated 5328 */ 5329 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) 5330 { 5331 struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats; 5332 struct ice_ring *ring; 5333 u64 pkts, bytes; 5334 int i; 5335 5336 /* reset netdev stats */ 5337 vsi_stats->tx_packets = 0; 5338 vsi_stats->tx_bytes = 0; 5339 vsi_stats->rx_packets = 0; 5340 vsi_stats->rx_bytes = 0; 5341 5342 /* reset non-netdev (extended) stats */ 5343 vsi->tx_restart = 0; 5344 vsi->tx_busy = 0; 5345 vsi->tx_linearize = 0; 5346 vsi->rx_buf_failed = 0; 5347 vsi->rx_page_failed = 0; 5348 vsi->rx_gro_dropped = 0; 5349 5350 rcu_read_lock(); 5351 5352 /* update Tx rings counters */ 5353 ice_update_vsi_tx_ring_stats(vsi, vsi->tx_rings, vsi->num_txq); 5354 5355 /* update Rx rings counters */ 5356 ice_for_each_rxq(vsi, i) { 5357 ring = READ_ONCE(vsi->rx_rings[i]); 5358 ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes); 5359 vsi_stats->rx_packets += pkts; 5360 vsi_stats->rx_bytes += bytes; 5361 vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed; 5362 vsi->rx_page_failed += ring->rx_stats.alloc_page_failed; 5363 vsi->rx_gro_dropped += ring->rx_stats.gro_dropped; 5364 } 5365 5366 /* update XDP Tx rings counters */ 5367 if (ice_is_xdp_ena_vsi(vsi)) 5368 ice_update_vsi_tx_ring_stats(vsi, vsi->xdp_rings, 5369 vsi->num_xdp_txq); 5370 5371 rcu_read_unlock(); 5372 } 5373 5374 /** 5375 * ice_update_vsi_stats - Update VSI stats counters 5376 * @vsi: the VSI to be updated 5377 */ 5378 void ice_update_vsi_stats(struct ice_vsi *vsi) 5379 { 5380 struct rtnl_link_stats64 *cur_ns = &vsi->net_stats; 5381 struct ice_eth_stats *cur_es = &vsi->eth_stats; 5382 struct ice_pf *pf = vsi->back; 5383 5384 if (test_bit(__ICE_DOWN, vsi->state) || 5385 test_bit(__ICE_CFG_BUSY, pf->state)) 5386 return; 5387 5388 /* get stats as recorded by Tx/Rx rings */ 5389 ice_update_vsi_ring_stats(vsi); 5390 5391 /* get VSI stats as recorded by the hardware */ 5392 ice_update_eth_stats(vsi); 5393 5394 cur_ns->tx_errors = cur_es->tx_errors; 5395 cur_ns->rx_dropped = cur_es->rx_discards + vsi->rx_gro_dropped; 5396 cur_ns->tx_dropped = cur_es->tx_discards; 5397 cur_ns->multicast = cur_es->rx_multicast; 5398 5399 /* update some more netdev stats if this is main VSI */ 5400 if (vsi->type == ICE_VSI_PF) { 5401 cur_ns->rx_crc_errors = pf->stats.crc_errors; 5402 cur_ns->rx_errors = pf->stats.crc_errors + 5403 pf->stats.illegal_bytes + 5404 pf->stats.rx_len_errors + 5405 pf->stats.rx_undersize + 5406 pf->hw_csum_rx_error + 5407 pf->stats.rx_jabber + 5408 pf->stats.rx_fragments + 5409 pf->stats.rx_oversize; 5410 cur_ns->rx_length_errors = pf->stats.rx_len_errors; 5411 /* record drops from the port level */ 5412 cur_ns->rx_missed_errors = pf->stats.eth.rx_discards; 5413 } 5414 } 5415 5416 /** 5417 * ice_update_pf_stats - Update PF port stats counters 5418 * @pf: PF whose stats needs to be updated 5419 */ 5420 void ice_update_pf_stats(struct ice_pf *pf) 5421 { 5422 struct ice_hw_port_stats *prev_ps, *cur_ps; 5423 struct ice_hw *hw = &pf->hw; 5424 u16 fd_ctr_base; 5425 u8 port; 5426 5427 port = hw->port_info->lport; 5428 prev_ps = &pf->stats_prev; 5429 cur_ps = &pf->stats; 5430 5431 ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded, 5432 &prev_ps->eth.rx_bytes, 5433 &cur_ps->eth.rx_bytes); 5434 5435 ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded, 5436 &prev_ps->eth.rx_unicast, 5437 &cur_ps->eth.rx_unicast); 5438 5439 ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded, 5440 &prev_ps->eth.rx_multicast, 5441 &cur_ps->eth.rx_multicast); 5442 5443 ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded, 5444 &prev_ps->eth.rx_broadcast, 5445 &cur_ps->eth.rx_broadcast); 5446 5447 ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded, 5448 &prev_ps->eth.rx_discards, 5449 &cur_ps->eth.rx_discards); 5450 5451 ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded, 5452 &prev_ps->eth.tx_bytes, 5453 &cur_ps->eth.tx_bytes); 5454 5455 ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded, 5456 &prev_ps->eth.tx_unicast, 5457 &cur_ps->eth.tx_unicast); 5458 5459 ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded, 5460 &prev_ps->eth.tx_multicast, 5461 &cur_ps->eth.tx_multicast); 5462 5463 ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded, 5464 &prev_ps->eth.tx_broadcast, 5465 &cur_ps->eth.tx_broadcast); 5466 5467 ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded, 5468 &prev_ps->tx_dropped_link_down, 5469 &cur_ps->tx_dropped_link_down); 5470 5471 ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded, 5472 &prev_ps->rx_size_64, &cur_ps->rx_size_64); 5473 5474 ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded, 5475 &prev_ps->rx_size_127, &cur_ps->rx_size_127); 5476 5477 ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded, 5478 &prev_ps->rx_size_255, &cur_ps->rx_size_255); 5479 5480 ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded, 5481 &prev_ps->rx_size_511, &cur_ps->rx_size_511); 5482 5483 ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded, 5484 &prev_ps->rx_size_1023, &cur_ps->rx_size_1023); 5485 5486 ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded, 5487 &prev_ps->rx_size_1522, &cur_ps->rx_size_1522); 5488 5489 ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded, 5490 &prev_ps->rx_size_big, &cur_ps->rx_size_big); 5491 5492 ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded, 5493 &prev_ps->tx_size_64, &cur_ps->tx_size_64); 5494 5495 ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded, 5496 &prev_ps->tx_size_127, &cur_ps->tx_size_127); 5497 5498 ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded, 5499 &prev_ps->tx_size_255, &cur_ps->tx_size_255); 5500 5501 ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded, 5502 &prev_ps->tx_size_511, &cur_ps->tx_size_511); 5503 5504 ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded, 5505 &prev_ps->tx_size_1023, &cur_ps->tx_size_1023); 5506 5507 ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded, 5508 &prev_ps->tx_size_1522, &cur_ps->tx_size_1522); 5509 5510 ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded, 5511 &prev_ps->tx_size_big, &cur_ps->tx_size_big); 5512 5513 fd_ctr_base = hw->fd_ctr_base; 5514 5515 ice_stat_update40(hw, 5516 GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)), 5517 pf->stat_prev_loaded, &prev_ps->fd_sb_match, 5518 &cur_ps->fd_sb_match); 5519 ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded, 5520 &prev_ps->link_xon_rx, &cur_ps->link_xon_rx); 5521 5522 ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded, 5523 &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx); 5524 5525 ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded, 5526 &prev_ps->link_xon_tx, &cur_ps->link_xon_tx); 5527 5528 ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded, 5529 &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx); 5530 5531 ice_update_dcb_stats(pf); 5532 5533 ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded, 5534 &prev_ps->crc_errors, &cur_ps->crc_errors); 5535 5536 ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded, 5537 &prev_ps->illegal_bytes, &cur_ps->illegal_bytes); 5538 5539 ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded, 5540 &prev_ps->mac_local_faults, 5541 &cur_ps->mac_local_faults); 5542 5543 ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded, 5544 &prev_ps->mac_remote_faults, 5545 &cur_ps->mac_remote_faults); 5546 5547 ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded, 5548 &prev_ps->rx_len_errors, &cur_ps->rx_len_errors); 5549 5550 ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded, 5551 &prev_ps->rx_undersize, &cur_ps->rx_undersize); 5552 5553 ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded, 5554 &prev_ps->rx_fragments, &cur_ps->rx_fragments); 5555 5556 ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded, 5557 &prev_ps->rx_oversize, &cur_ps->rx_oversize); 5558 5559 ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded, 5560 &prev_ps->rx_jabber, &cur_ps->rx_jabber); 5561 5562 cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0; 5563 5564 pf->stat_prev_loaded = true; 5565 } 5566 5567 /** 5568 * ice_get_stats64 - get statistics for network device structure 5569 * @netdev: network interface device structure 5570 * @stats: main device statistics structure 5571 */ 5572 static 5573 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 5574 { 5575 struct ice_netdev_priv *np = netdev_priv(netdev); 5576 struct rtnl_link_stats64 *vsi_stats; 5577 struct ice_vsi *vsi = np->vsi; 5578 5579 vsi_stats = &vsi->net_stats; 5580 5581 if (!vsi->num_txq || !vsi->num_rxq) 5582 return; 5583 5584 /* netdev packet/byte stats come from ring counter. These are obtained 5585 * by summing up ring counters (done by ice_update_vsi_ring_stats). 5586 * But, only call the update routine and read the registers if VSI is 5587 * not down. 5588 */ 5589 if (!test_bit(__ICE_DOWN, vsi->state)) 5590 ice_update_vsi_ring_stats(vsi); 5591 stats->tx_packets = vsi_stats->tx_packets; 5592 stats->tx_bytes = vsi_stats->tx_bytes; 5593 stats->rx_packets = vsi_stats->rx_packets; 5594 stats->rx_bytes = vsi_stats->rx_bytes; 5595 5596 /* The rest of the stats can be read from the hardware but instead we 5597 * just return values that the watchdog task has already obtained from 5598 * the hardware. 5599 */ 5600 stats->multicast = vsi_stats->multicast; 5601 stats->tx_errors = vsi_stats->tx_errors; 5602 stats->tx_dropped = vsi_stats->tx_dropped; 5603 stats->rx_errors = vsi_stats->rx_errors; 5604 stats->rx_dropped = vsi_stats->rx_dropped; 5605 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 5606 stats->rx_length_errors = vsi_stats->rx_length_errors; 5607 } 5608 5609 /** 5610 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI 5611 * @vsi: VSI having NAPI disabled 5612 */ 5613 static void ice_napi_disable_all(struct ice_vsi *vsi) 5614 { 5615 int q_idx; 5616 5617 if (!vsi->netdev) 5618 return; 5619 5620 ice_for_each_q_vector(vsi, q_idx) { 5621 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 5622 5623 if (q_vector->rx.ring || q_vector->tx.ring) 5624 napi_disable(&q_vector->napi); 5625 } 5626 } 5627 5628 /** 5629 * ice_down - Shutdown the connection 5630 * @vsi: The VSI being stopped 5631 */ 5632 int ice_down(struct ice_vsi *vsi) 5633 { 5634 int i, tx_err, rx_err, link_err = 0; 5635 5636 /* Caller of this function is expected to set the 5637 * vsi->state __ICE_DOWN bit 5638 */ 5639 if (vsi->netdev) { 5640 netif_carrier_off(vsi->netdev); 5641 netif_tx_disable(vsi->netdev); 5642 } 5643 5644 ice_vsi_dis_irq(vsi); 5645 5646 tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0); 5647 if (tx_err) 5648 netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n", 5649 vsi->vsi_num, tx_err); 5650 if (!tx_err && ice_is_xdp_ena_vsi(vsi)) { 5651 tx_err = ice_vsi_stop_xdp_tx_rings(vsi); 5652 if (tx_err) 5653 netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n", 5654 vsi->vsi_num, tx_err); 5655 } 5656 5657 rx_err = ice_vsi_stop_all_rx_rings(vsi); 5658 if (rx_err) 5659 netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n", 5660 vsi->vsi_num, rx_err); 5661 5662 ice_napi_disable_all(vsi); 5663 5664 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { 5665 link_err = ice_force_phys_link_state(vsi, false); 5666 if (link_err) 5667 netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n", 5668 vsi->vsi_num, link_err); 5669 } 5670 5671 ice_for_each_txq(vsi, i) 5672 ice_clean_tx_ring(vsi->tx_rings[i]); 5673 5674 ice_for_each_rxq(vsi, i) 5675 ice_clean_rx_ring(vsi->rx_rings[i]); 5676 5677 if (tx_err || rx_err || link_err) { 5678 netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n", 5679 vsi->vsi_num, vsi->vsw->sw_id); 5680 return -EIO; 5681 } 5682 5683 return 0; 5684 } 5685 5686 /** 5687 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources 5688 * @vsi: VSI having resources allocated 5689 * 5690 * Return 0 on success, negative on failure 5691 */ 5692 int ice_vsi_setup_tx_rings(struct ice_vsi *vsi) 5693 { 5694 int i, err = 0; 5695 5696 if (!vsi->num_txq) { 5697 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n", 5698 vsi->vsi_num); 5699 return -EINVAL; 5700 } 5701 5702 ice_for_each_txq(vsi, i) { 5703 struct ice_ring *ring = vsi->tx_rings[i]; 5704 5705 if (!ring) 5706 return -EINVAL; 5707 5708 ring->netdev = vsi->netdev; 5709 err = ice_setup_tx_ring(ring); 5710 if (err) 5711 break; 5712 } 5713 5714 return err; 5715 } 5716 5717 /** 5718 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources 5719 * @vsi: VSI having resources allocated 5720 * 5721 * Return 0 on success, negative on failure 5722 */ 5723 int ice_vsi_setup_rx_rings(struct ice_vsi *vsi) 5724 { 5725 int i, err = 0; 5726 5727 if (!vsi->num_rxq) { 5728 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n", 5729 vsi->vsi_num); 5730 return -EINVAL; 5731 } 5732 5733 ice_for_each_rxq(vsi, i) { 5734 struct ice_ring *ring = vsi->rx_rings[i]; 5735 5736 if (!ring) 5737 return -EINVAL; 5738 5739 ring->netdev = vsi->netdev; 5740 err = ice_setup_rx_ring(ring); 5741 if (err) 5742 break; 5743 } 5744 5745 return err; 5746 } 5747 5748 /** 5749 * ice_vsi_open_ctrl - open control VSI for use 5750 * @vsi: the VSI to open 5751 * 5752 * Initialization of the Control VSI 5753 * 5754 * Returns 0 on success, negative value on error 5755 */ 5756 int ice_vsi_open_ctrl(struct ice_vsi *vsi) 5757 { 5758 char int_name[ICE_INT_NAME_STR_LEN]; 5759 struct ice_pf *pf = vsi->back; 5760 struct device *dev; 5761 int err; 5762 5763 dev = ice_pf_to_dev(pf); 5764 /* allocate descriptors */ 5765 err = ice_vsi_setup_tx_rings(vsi); 5766 if (err) 5767 goto err_setup_tx; 5768 5769 err = ice_vsi_setup_rx_rings(vsi); 5770 if (err) 5771 goto err_setup_rx; 5772 5773 err = ice_vsi_cfg(vsi); 5774 if (err) 5775 goto err_setup_rx; 5776 5777 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl", 5778 dev_driver_string(dev), dev_name(dev)); 5779 err = ice_vsi_req_irq_msix(vsi, int_name); 5780 if (err) 5781 goto err_setup_rx; 5782 5783 ice_vsi_cfg_msix(vsi); 5784 5785 err = ice_vsi_start_all_rx_rings(vsi); 5786 if (err) 5787 goto err_up_complete; 5788 5789 clear_bit(__ICE_DOWN, vsi->state); 5790 ice_vsi_ena_irq(vsi); 5791 5792 return 0; 5793 5794 err_up_complete: 5795 ice_down(vsi); 5796 err_setup_rx: 5797 ice_vsi_free_rx_rings(vsi); 5798 err_setup_tx: 5799 ice_vsi_free_tx_rings(vsi); 5800 5801 return err; 5802 } 5803 5804 /** 5805 * ice_vsi_open - Called when a network interface is made active 5806 * @vsi: the VSI to open 5807 * 5808 * Initialization of the VSI 5809 * 5810 * Returns 0 on success, negative value on error 5811 */ 5812 static int ice_vsi_open(struct ice_vsi *vsi) 5813 { 5814 char int_name[ICE_INT_NAME_STR_LEN]; 5815 struct ice_pf *pf = vsi->back; 5816 int err; 5817 5818 /* allocate descriptors */ 5819 err = ice_vsi_setup_tx_rings(vsi); 5820 if (err) 5821 goto err_setup_tx; 5822 5823 err = ice_vsi_setup_rx_rings(vsi); 5824 if (err) 5825 goto err_setup_rx; 5826 5827 err = ice_vsi_cfg(vsi); 5828 if (err) 5829 goto err_setup_rx; 5830 5831 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 5832 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name); 5833 err = ice_vsi_req_irq_msix(vsi, int_name); 5834 if (err) 5835 goto err_setup_rx; 5836 5837 /* Notify the stack of the actual queue counts. */ 5838 err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); 5839 if (err) 5840 goto err_set_qs; 5841 5842 err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq); 5843 if (err) 5844 goto err_set_qs; 5845 5846 err = ice_up_complete(vsi); 5847 if (err) 5848 goto err_up_complete; 5849 5850 return 0; 5851 5852 err_up_complete: 5853 ice_down(vsi); 5854 err_set_qs: 5855 ice_vsi_free_irq(vsi); 5856 err_setup_rx: 5857 ice_vsi_free_rx_rings(vsi); 5858 err_setup_tx: 5859 ice_vsi_free_tx_rings(vsi); 5860 5861 return err; 5862 } 5863 5864 /** 5865 * ice_vsi_release_all - Delete all VSIs 5866 * @pf: PF from which all VSIs are being removed 5867 */ 5868 static void ice_vsi_release_all(struct ice_pf *pf) 5869 { 5870 int err, i; 5871 5872 if (!pf->vsi) 5873 return; 5874 5875 ice_for_each_vsi(pf, i) { 5876 if (!pf->vsi[i]) 5877 continue; 5878 5879 err = ice_vsi_release(pf->vsi[i]); 5880 if (err) 5881 dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n", 5882 i, err, pf->vsi[i]->vsi_num); 5883 } 5884 } 5885 5886 /** 5887 * ice_vsi_rebuild_by_type - Rebuild VSI of a given type 5888 * @pf: pointer to the PF instance 5889 * @type: VSI type to rebuild 5890 * 5891 * Iterates through the pf->vsi array and rebuilds VSIs of the requested type 5892 */ 5893 static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type) 5894 { 5895 struct device *dev = ice_pf_to_dev(pf); 5896 enum ice_status status; 5897 int i, err; 5898 5899 ice_for_each_vsi(pf, i) { 5900 struct ice_vsi *vsi = pf->vsi[i]; 5901 5902 if (!vsi || vsi->type != type) 5903 continue; 5904 5905 /* rebuild the VSI */ 5906 err = ice_vsi_rebuild(vsi, true); 5907 if (err) { 5908 dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n", 5909 err, vsi->idx, ice_vsi_type_str(type)); 5910 return err; 5911 } 5912 5913 /* replay filters for the VSI */ 5914 status = ice_replay_vsi(&pf->hw, vsi->idx); 5915 if (status) { 5916 dev_err(dev, "replay VSI failed, status %s, VSI index %d, type %s\n", 5917 ice_stat_str(status), vsi->idx, 5918 ice_vsi_type_str(type)); 5919 return -EIO; 5920 } 5921 5922 /* Re-map HW VSI number, using VSI handle that has been 5923 * previously validated in ice_replay_vsi() call above 5924 */ 5925 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 5926 5927 /* enable the VSI */ 5928 err = ice_ena_vsi(vsi, false); 5929 if (err) { 5930 dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n", 5931 err, vsi->idx, ice_vsi_type_str(type)); 5932 return err; 5933 } 5934 5935 dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx, 5936 ice_vsi_type_str(type)); 5937 } 5938 5939 return 0; 5940 } 5941 5942 /** 5943 * ice_update_pf_netdev_link - Update PF netdev link status 5944 * @pf: pointer to the PF instance 5945 */ 5946 static void ice_update_pf_netdev_link(struct ice_pf *pf) 5947 { 5948 bool link_up; 5949 int i; 5950 5951 ice_for_each_vsi(pf, i) { 5952 struct ice_vsi *vsi = pf->vsi[i]; 5953 5954 if (!vsi || vsi->type != ICE_VSI_PF) 5955 return; 5956 5957 ice_get_link_status(pf->vsi[i]->port_info, &link_up); 5958 if (link_up) { 5959 netif_carrier_on(pf->vsi[i]->netdev); 5960 netif_tx_wake_all_queues(pf->vsi[i]->netdev); 5961 } else { 5962 netif_carrier_off(pf->vsi[i]->netdev); 5963 netif_tx_stop_all_queues(pf->vsi[i]->netdev); 5964 } 5965 } 5966 } 5967 5968 /** 5969 * ice_rebuild - rebuild after reset 5970 * @pf: PF to rebuild 5971 * @reset_type: type of reset 5972 * 5973 * Do not rebuild VF VSI in this flow because that is already handled via 5974 * ice_reset_all_vfs(). This is because requirements for resetting a VF after a 5975 * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want 5976 * to reset/rebuild all the VF VSI twice. 5977 */ 5978 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) 5979 { 5980 struct device *dev = ice_pf_to_dev(pf); 5981 struct ice_hw *hw = &pf->hw; 5982 enum ice_status ret; 5983 int err; 5984 5985 if (test_bit(__ICE_DOWN, pf->state)) 5986 goto clear_recovery; 5987 5988 dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type); 5989 5990 ret = ice_init_all_ctrlq(hw); 5991 if (ret) { 5992 dev_err(dev, "control queues init failed %s\n", 5993 ice_stat_str(ret)); 5994 goto err_init_ctrlq; 5995 } 5996 5997 /* if DDP was previously loaded successfully */ 5998 if (!ice_is_safe_mode(pf)) { 5999 /* reload the SW DB of filter tables */ 6000 if (reset_type == ICE_RESET_PFR) 6001 ice_fill_blk_tbls(hw); 6002 else 6003 /* Reload DDP Package after CORER/GLOBR reset */ 6004 ice_load_pkg(NULL, pf); 6005 } 6006 6007 ret = ice_clear_pf_cfg(hw); 6008 if (ret) { 6009 dev_err(dev, "clear PF configuration failed %s\n", 6010 ice_stat_str(ret)); 6011 goto err_init_ctrlq; 6012 } 6013 6014 if (pf->first_sw->dflt_vsi_ena) 6015 dev_info(dev, "Clearing default VSI, re-enable after reset completes\n"); 6016 /* clear the default VSI configuration if it exists */ 6017 pf->first_sw->dflt_vsi = NULL; 6018 pf->first_sw->dflt_vsi_ena = false; 6019 6020 ice_clear_pxe_mode(hw); 6021 6022 ret = ice_get_caps(hw); 6023 if (ret) { 6024 dev_err(dev, "ice_get_caps failed %s\n", ice_stat_str(ret)); 6025 goto err_init_ctrlq; 6026 } 6027 6028 ret = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL); 6029 if (ret) { 6030 dev_err(dev, "set_mac_cfg failed %s\n", ice_stat_str(ret)); 6031 goto err_init_ctrlq; 6032 } 6033 6034 err = ice_sched_init_port(hw->port_info); 6035 if (err) 6036 goto err_sched_init_port; 6037 6038 /* start misc vector */ 6039 err = ice_req_irq_msix_misc(pf); 6040 if (err) { 6041 dev_err(dev, "misc vector setup failed: %d\n", err); 6042 goto err_sched_init_port; 6043 } 6044 6045 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 6046 wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M); 6047 if (!rd32(hw, PFQF_FD_SIZE)) { 6048 u16 unused, guar, b_effort; 6049 6050 guar = hw->func_caps.fd_fltr_guar; 6051 b_effort = hw->func_caps.fd_fltr_best_effort; 6052 6053 /* force guaranteed filter pool for PF */ 6054 ice_alloc_fd_guar_item(hw, &unused, guar); 6055 /* force shared filter pool for PF */ 6056 ice_alloc_fd_shrd_item(hw, &unused, b_effort); 6057 } 6058 } 6059 6060 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 6061 ice_dcb_rebuild(pf); 6062 6063 /* rebuild PF VSI */ 6064 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF); 6065 if (err) { 6066 dev_err(dev, "PF VSI rebuild failed: %d\n", err); 6067 goto err_vsi_rebuild; 6068 } 6069 6070 /* If Flow Director is active */ 6071 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 6072 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL); 6073 if (err) { 6074 dev_err(dev, "control VSI rebuild failed: %d\n", err); 6075 goto err_vsi_rebuild; 6076 } 6077 6078 /* replay HW Flow Director recipes */ 6079 if (hw->fdir_prof) 6080 ice_fdir_replay_flows(hw); 6081 6082 /* replay Flow Director filters */ 6083 ice_fdir_replay_fltrs(pf); 6084 6085 ice_rebuild_arfs(pf); 6086 } 6087 6088 ice_update_pf_netdev_link(pf); 6089 6090 /* tell the firmware we are up */ 6091 ret = ice_send_version(pf); 6092 if (ret) { 6093 dev_err(dev, "Rebuild failed due to error sending driver version: %s\n", 6094 ice_stat_str(ret)); 6095 goto err_vsi_rebuild; 6096 } 6097 6098 ice_replay_post(hw); 6099 6100 /* if we get here, reset flow is successful */ 6101 clear_bit(__ICE_RESET_FAILED, pf->state); 6102 return; 6103 6104 err_vsi_rebuild: 6105 err_sched_init_port: 6106 ice_sched_cleanup_all(hw); 6107 err_init_ctrlq: 6108 ice_shutdown_all_ctrlq(hw); 6109 set_bit(__ICE_RESET_FAILED, pf->state); 6110 clear_recovery: 6111 /* set this bit in PF state to control service task scheduling */ 6112 set_bit(__ICE_NEEDS_RESTART, pf->state); 6113 dev_err(dev, "Rebuild failed, unload and reload driver\n"); 6114 } 6115 6116 /** 6117 * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP 6118 * @vsi: Pointer to VSI structure 6119 */ 6120 static int ice_max_xdp_frame_size(struct ice_vsi *vsi) 6121 { 6122 if (PAGE_SIZE >= 8192 || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) 6123 return ICE_RXBUF_2048 - XDP_PACKET_HEADROOM; 6124 else 6125 return ICE_RXBUF_3072; 6126 } 6127 6128 /** 6129 * ice_change_mtu - NDO callback to change the MTU 6130 * @netdev: network interface device structure 6131 * @new_mtu: new value for maximum frame size 6132 * 6133 * Returns 0 on success, negative on failure 6134 */ 6135 static int ice_change_mtu(struct net_device *netdev, int new_mtu) 6136 { 6137 struct ice_netdev_priv *np = netdev_priv(netdev); 6138 struct ice_vsi *vsi = np->vsi; 6139 struct ice_pf *pf = vsi->back; 6140 u8 count = 0; 6141 6142 if (new_mtu == (int)netdev->mtu) { 6143 netdev_warn(netdev, "MTU is already %u\n", netdev->mtu); 6144 return 0; 6145 } 6146 6147 if (ice_is_xdp_ena_vsi(vsi)) { 6148 int frame_size = ice_max_xdp_frame_size(vsi); 6149 6150 if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) { 6151 netdev_err(netdev, "max MTU for XDP usage is %d\n", 6152 frame_size - ICE_ETH_PKT_HDR_PAD); 6153 return -EINVAL; 6154 } 6155 } 6156 6157 /* if a reset is in progress, wait for some time for it to complete */ 6158 do { 6159 if (ice_is_reset_in_progress(pf->state)) { 6160 count++; 6161 usleep_range(1000, 2000); 6162 } else { 6163 break; 6164 } 6165 6166 } while (count < 100); 6167 6168 if (count == 100) { 6169 netdev_err(netdev, "can't change MTU. Device is busy\n"); 6170 return -EBUSY; 6171 } 6172 6173 netdev->mtu = (unsigned int)new_mtu; 6174 6175 /* if VSI is up, bring it down and then back up */ 6176 if (!test_and_set_bit(__ICE_DOWN, vsi->state)) { 6177 int err; 6178 6179 err = ice_down(vsi); 6180 if (err) { 6181 netdev_err(netdev, "change MTU if_down err %d\n", err); 6182 return err; 6183 } 6184 6185 err = ice_up(vsi); 6186 if (err) { 6187 netdev_err(netdev, "change MTU if_up err %d\n", err); 6188 return err; 6189 } 6190 } 6191 6192 netdev_dbg(netdev, "changed MTU to %d\n", new_mtu); 6193 return 0; 6194 } 6195 6196 /** 6197 * ice_aq_str - convert AQ err code to a string 6198 * @aq_err: the AQ error code to convert 6199 */ 6200 const char *ice_aq_str(enum ice_aq_err aq_err) 6201 { 6202 switch (aq_err) { 6203 case ICE_AQ_RC_OK: 6204 return "OK"; 6205 case ICE_AQ_RC_EPERM: 6206 return "ICE_AQ_RC_EPERM"; 6207 case ICE_AQ_RC_ENOENT: 6208 return "ICE_AQ_RC_ENOENT"; 6209 case ICE_AQ_RC_ENOMEM: 6210 return "ICE_AQ_RC_ENOMEM"; 6211 case ICE_AQ_RC_EBUSY: 6212 return "ICE_AQ_RC_EBUSY"; 6213 case ICE_AQ_RC_EEXIST: 6214 return "ICE_AQ_RC_EEXIST"; 6215 case ICE_AQ_RC_EINVAL: 6216 return "ICE_AQ_RC_EINVAL"; 6217 case ICE_AQ_RC_ENOSPC: 6218 return "ICE_AQ_RC_ENOSPC"; 6219 case ICE_AQ_RC_ENOSYS: 6220 return "ICE_AQ_RC_ENOSYS"; 6221 case ICE_AQ_RC_EMODE: 6222 return "ICE_AQ_RC_EMODE"; 6223 case ICE_AQ_RC_ENOSEC: 6224 return "ICE_AQ_RC_ENOSEC"; 6225 case ICE_AQ_RC_EBADSIG: 6226 return "ICE_AQ_RC_EBADSIG"; 6227 case ICE_AQ_RC_ESVN: 6228 return "ICE_AQ_RC_ESVN"; 6229 case ICE_AQ_RC_EBADMAN: 6230 return "ICE_AQ_RC_EBADMAN"; 6231 case ICE_AQ_RC_EBADBUF: 6232 return "ICE_AQ_RC_EBADBUF"; 6233 } 6234 6235 return "ICE_AQ_RC_UNKNOWN"; 6236 } 6237 6238 /** 6239 * ice_stat_str - convert status err code to a string 6240 * @stat_err: the status error code to convert 6241 */ 6242 const char *ice_stat_str(enum ice_status stat_err) 6243 { 6244 switch (stat_err) { 6245 case ICE_SUCCESS: 6246 return "OK"; 6247 case ICE_ERR_PARAM: 6248 return "ICE_ERR_PARAM"; 6249 case ICE_ERR_NOT_IMPL: 6250 return "ICE_ERR_NOT_IMPL"; 6251 case ICE_ERR_NOT_READY: 6252 return "ICE_ERR_NOT_READY"; 6253 case ICE_ERR_NOT_SUPPORTED: 6254 return "ICE_ERR_NOT_SUPPORTED"; 6255 case ICE_ERR_BAD_PTR: 6256 return "ICE_ERR_BAD_PTR"; 6257 case ICE_ERR_INVAL_SIZE: 6258 return "ICE_ERR_INVAL_SIZE"; 6259 case ICE_ERR_DEVICE_NOT_SUPPORTED: 6260 return "ICE_ERR_DEVICE_NOT_SUPPORTED"; 6261 case ICE_ERR_RESET_FAILED: 6262 return "ICE_ERR_RESET_FAILED"; 6263 case ICE_ERR_FW_API_VER: 6264 return "ICE_ERR_FW_API_VER"; 6265 case ICE_ERR_NO_MEMORY: 6266 return "ICE_ERR_NO_MEMORY"; 6267 case ICE_ERR_CFG: 6268 return "ICE_ERR_CFG"; 6269 case ICE_ERR_OUT_OF_RANGE: 6270 return "ICE_ERR_OUT_OF_RANGE"; 6271 case ICE_ERR_ALREADY_EXISTS: 6272 return "ICE_ERR_ALREADY_EXISTS"; 6273 case ICE_ERR_NVM: 6274 return "ICE_ERR_NVM"; 6275 case ICE_ERR_NVM_CHECKSUM: 6276 return "ICE_ERR_NVM_CHECKSUM"; 6277 case ICE_ERR_BUF_TOO_SHORT: 6278 return "ICE_ERR_BUF_TOO_SHORT"; 6279 case ICE_ERR_NVM_BLANK_MODE: 6280 return "ICE_ERR_NVM_BLANK_MODE"; 6281 case ICE_ERR_IN_USE: 6282 return "ICE_ERR_IN_USE"; 6283 case ICE_ERR_MAX_LIMIT: 6284 return "ICE_ERR_MAX_LIMIT"; 6285 case ICE_ERR_RESET_ONGOING: 6286 return "ICE_ERR_RESET_ONGOING"; 6287 case ICE_ERR_HW_TABLE: 6288 return "ICE_ERR_HW_TABLE"; 6289 case ICE_ERR_DOES_NOT_EXIST: 6290 return "ICE_ERR_DOES_NOT_EXIST"; 6291 case ICE_ERR_FW_DDP_MISMATCH: 6292 return "ICE_ERR_FW_DDP_MISMATCH"; 6293 case ICE_ERR_AQ_ERROR: 6294 return "ICE_ERR_AQ_ERROR"; 6295 case ICE_ERR_AQ_TIMEOUT: 6296 return "ICE_ERR_AQ_TIMEOUT"; 6297 case ICE_ERR_AQ_FULL: 6298 return "ICE_ERR_AQ_FULL"; 6299 case ICE_ERR_AQ_NO_WORK: 6300 return "ICE_ERR_AQ_NO_WORK"; 6301 case ICE_ERR_AQ_EMPTY: 6302 return "ICE_ERR_AQ_EMPTY"; 6303 case ICE_ERR_AQ_FW_CRITICAL: 6304 return "ICE_ERR_AQ_FW_CRITICAL"; 6305 } 6306 6307 return "ICE_ERR_UNKNOWN"; 6308 } 6309 6310 /** 6311 * ice_set_rss - Set RSS keys and lut 6312 * @vsi: Pointer to VSI structure 6313 * @seed: RSS hash seed 6314 * @lut: Lookup table 6315 * @lut_size: Lookup table size 6316 * 6317 * Returns 0 on success, negative on failure 6318 */ 6319 int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 6320 { 6321 struct ice_pf *pf = vsi->back; 6322 struct ice_hw *hw = &pf->hw; 6323 enum ice_status status; 6324 struct device *dev; 6325 6326 dev = ice_pf_to_dev(pf); 6327 if (seed) { 6328 struct ice_aqc_get_set_rss_keys *buf = 6329 (struct ice_aqc_get_set_rss_keys *)seed; 6330 6331 status = ice_aq_set_rss_key(hw, vsi->idx, buf); 6332 6333 if (status) { 6334 dev_err(dev, "Cannot set RSS key, err %s aq_err %s\n", 6335 ice_stat_str(status), 6336 ice_aq_str(hw->adminq.sq_last_status)); 6337 return -EIO; 6338 } 6339 } 6340 6341 if (lut) { 6342 status = ice_aq_set_rss_lut(hw, vsi->idx, vsi->rss_lut_type, 6343 lut, lut_size); 6344 if (status) { 6345 dev_err(dev, "Cannot set RSS lut, err %s aq_err %s\n", 6346 ice_stat_str(status), 6347 ice_aq_str(hw->adminq.sq_last_status)); 6348 return -EIO; 6349 } 6350 } 6351 6352 return 0; 6353 } 6354 6355 /** 6356 * ice_get_rss - Get RSS keys and lut 6357 * @vsi: Pointer to VSI structure 6358 * @seed: Buffer to store the keys 6359 * @lut: Buffer to store the lookup table entries 6360 * @lut_size: Size of buffer to store the lookup table entries 6361 * 6362 * Returns 0 on success, negative on failure 6363 */ 6364 int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 6365 { 6366 struct ice_pf *pf = vsi->back; 6367 struct ice_hw *hw = &pf->hw; 6368 enum ice_status status; 6369 struct device *dev; 6370 6371 dev = ice_pf_to_dev(pf); 6372 if (seed) { 6373 struct ice_aqc_get_set_rss_keys *buf = 6374 (struct ice_aqc_get_set_rss_keys *)seed; 6375 6376 status = ice_aq_get_rss_key(hw, vsi->idx, buf); 6377 if (status) { 6378 dev_err(dev, "Cannot get RSS key, err %s aq_err %s\n", 6379 ice_stat_str(status), 6380 ice_aq_str(hw->adminq.sq_last_status)); 6381 return -EIO; 6382 } 6383 } 6384 6385 if (lut) { 6386 status = ice_aq_get_rss_lut(hw, vsi->idx, vsi->rss_lut_type, 6387 lut, lut_size); 6388 if (status) { 6389 dev_err(dev, "Cannot get RSS lut, err %s aq_err %s\n", 6390 ice_stat_str(status), 6391 ice_aq_str(hw->adminq.sq_last_status)); 6392 return -EIO; 6393 } 6394 } 6395 6396 return 0; 6397 } 6398 6399 /** 6400 * ice_bridge_getlink - Get the hardware bridge mode 6401 * @skb: skb buff 6402 * @pid: process ID 6403 * @seq: RTNL message seq 6404 * @dev: the netdev being configured 6405 * @filter_mask: filter mask passed in 6406 * @nlflags: netlink flags passed in 6407 * 6408 * Return the bridge mode (VEB/VEPA) 6409 */ 6410 static int 6411 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 6412 struct net_device *dev, u32 filter_mask, int nlflags) 6413 { 6414 struct ice_netdev_priv *np = netdev_priv(dev); 6415 struct ice_vsi *vsi = np->vsi; 6416 struct ice_pf *pf = vsi->back; 6417 u16 bmode; 6418 6419 bmode = pf->first_sw->bridge_mode; 6420 6421 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags, 6422 filter_mask, NULL); 6423 } 6424 6425 /** 6426 * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA) 6427 * @vsi: Pointer to VSI structure 6428 * @bmode: Hardware bridge mode (VEB/VEPA) 6429 * 6430 * Returns 0 on success, negative on failure 6431 */ 6432 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode) 6433 { 6434 struct ice_aqc_vsi_props *vsi_props; 6435 struct ice_hw *hw = &vsi->back->hw; 6436 struct ice_vsi_ctx *ctxt; 6437 enum ice_status status; 6438 int ret = 0; 6439 6440 vsi_props = &vsi->info; 6441 6442 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 6443 if (!ctxt) 6444 return -ENOMEM; 6445 6446 ctxt->info = vsi->info; 6447 6448 if (bmode == BRIDGE_MODE_VEB) 6449 /* change from VEPA to VEB mode */ 6450 ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 6451 else 6452 /* change from VEB to VEPA mode */ 6453 ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 6454 ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); 6455 6456 status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 6457 if (status) { 6458 dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %s aq_err %s\n", 6459 bmode, ice_stat_str(status), 6460 ice_aq_str(hw->adminq.sq_last_status)); 6461 ret = -EIO; 6462 goto out; 6463 } 6464 /* Update sw flags for book keeping */ 6465 vsi_props->sw_flags = ctxt->info.sw_flags; 6466 6467 out: 6468 kfree(ctxt); 6469 return ret; 6470 } 6471 6472 /** 6473 * ice_bridge_setlink - Set the hardware bridge mode 6474 * @dev: the netdev being configured 6475 * @nlh: RTNL message 6476 * @flags: bridge setlink flags 6477 * @extack: netlink extended ack 6478 * 6479 * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is 6480 * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if 6481 * not already set for all VSIs connected to this switch. And also update the 6482 * unicast switch filter rules for the corresponding switch of the netdev. 6483 */ 6484 static int 6485 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 6486 u16 __always_unused flags, 6487 struct netlink_ext_ack __always_unused *extack) 6488 { 6489 struct ice_netdev_priv *np = netdev_priv(dev); 6490 struct ice_pf *pf = np->vsi->back; 6491 struct nlattr *attr, *br_spec; 6492 struct ice_hw *hw = &pf->hw; 6493 enum ice_status status; 6494 struct ice_sw *pf_sw; 6495 int rem, v, err = 0; 6496 6497 pf_sw = pf->first_sw; 6498 /* find the attribute in the netlink message */ 6499 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 6500 6501 nla_for_each_nested(attr, br_spec, rem) { 6502 __u16 mode; 6503 6504 if (nla_type(attr) != IFLA_BRIDGE_MODE) 6505 continue; 6506 mode = nla_get_u16(attr); 6507 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 6508 return -EINVAL; 6509 /* Continue if bridge mode is not being flipped */ 6510 if (mode == pf_sw->bridge_mode) 6511 continue; 6512 /* Iterates through the PF VSI list and update the loopback 6513 * mode of the VSI 6514 */ 6515 ice_for_each_vsi(pf, v) { 6516 if (!pf->vsi[v]) 6517 continue; 6518 err = ice_vsi_update_bridge_mode(pf->vsi[v], mode); 6519 if (err) 6520 return err; 6521 } 6522 6523 hw->evb_veb = (mode == BRIDGE_MODE_VEB); 6524 /* Update the unicast switch filter rules for the corresponding 6525 * switch of the netdev 6526 */ 6527 status = ice_update_sw_rule_bridge_mode(hw); 6528 if (status) { 6529 netdev_err(dev, "switch rule update failed, mode = %d err %s aq_err %s\n", 6530 mode, ice_stat_str(status), 6531 ice_aq_str(hw->adminq.sq_last_status)); 6532 /* revert hw->evb_veb */ 6533 hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB); 6534 return -EIO; 6535 } 6536 6537 pf_sw->bridge_mode = mode; 6538 } 6539 6540 return 0; 6541 } 6542 6543 /** 6544 * ice_tx_timeout - Respond to a Tx Hang 6545 * @netdev: network interface device structure 6546 * @txqueue: Tx queue 6547 */ 6548 static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) 6549 { 6550 struct ice_netdev_priv *np = netdev_priv(netdev); 6551 struct ice_ring *tx_ring = NULL; 6552 struct ice_vsi *vsi = np->vsi; 6553 struct ice_pf *pf = vsi->back; 6554 u32 i; 6555 6556 pf->tx_timeout_count++; 6557 6558 /* Check if PFC is enabled for the TC to which the queue belongs 6559 * to. If yes then Tx timeout is not caused by a hung queue, no 6560 * need to reset and rebuild 6561 */ 6562 if (ice_is_pfc_causing_hung_q(pf, txqueue)) { 6563 dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n", 6564 txqueue); 6565 return; 6566 } 6567 6568 /* now that we have an index, find the tx_ring struct */ 6569 for (i = 0; i < vsi->num_txq; i++) 6570 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 6571 if (txqueue == vsi->tx_rings[i]->q_index) { 6572 tx_ring = vsi->tx_rings[i]; 6573 break; 6574 } 6575 6576 /* Reset recovery level if enough time has elapsed after last timeout. 6577 * Also ensure no new reset action happens before next timeout period. 6578 */ 6579 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20))) 6580 pf->tx_timeout_recovery_level = 1; 6581 else if (time_before(jiffies, (pf->tx_timeout_last_recovery + 6582 netdev->watchdog_timeo))) 6583 return; 6584 6585 if (tx_ring) { 6586 struct ice_hw *hw = &pf->hw; 6587 u32 head, val = 0; 6588 6589 head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])) & 6590 QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S; 6591 /* Read interrupt register */ 6592 val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx)); 6593 6594 netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n", 6595 vsi->vsi_num, txqueue, tx_ring->next_to_clean, 6596 head, tx_ring->next_to_use, val); 6597 } 6598 6599 pf->tx_timeout_last_recovery = jiffies; 6600 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n", 6601 pf->tx_timeout_recovery_level, txqueue); 6602 6603 switch (pf->tx_timeout_recovery_level) { 6604 case 1: 6605 set_bit(__ICE_PFR_REQ, pf->state); 6606 break; 6607 case 2: 6608 set_bit(__ICE_CORER_REQ, pf->state); 6609 break; 6610 case 3: 6611 set_bit(__ICE_GLOBR_REQ, pf->state); 6612 break; 6613 default: 6614 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n"); 6615 set_bit(__ICE_DOWN, pf->state); 6616 set_bit(__ICE_NEEDS_RESTART, vsi->state); 6617 set_bit(__ICE_SERVICE_DIS, pf->state); 6618 break; 6619 } 6620 6621 ice_service_task_schedule(pf); 6622 pf->tx_timeout_recovery_level++; 6623 } 6624 6625 /** 6626 * ice_open - Called when a network interface becomes active 6627 * @netdev: network interface device structure 6628 * 6629 * The open entry point is called when a network interface is made 6630 * active by the system (IFF_UP). At this point all resources needed 6631 * for transmit and receive operations are allocated, the interrupt 6632 * handler is registered with the OS, the netdev watchdog is enabled, 6633 * and the stack is notified that the interface is ready. 6634 * 6635 * Returns 0 on success, negative value on failure 6636 */ 6637 int ice_open(struct net_device *netdev) 6638 { 6639 struct ice_netdev_priv *np = netdev_priv(netdev); 6640 struct ice_vsi *vsi = np->vsi; 6641 struct ice_pf *pf = vsi->back; 6642 struct ice_port_info *pi; 6643 int err; 6644 6645 if (test_bit(__ICE_NEEDS_RESTART, pf->state)) { 6646 netdev_err(netdev, "driver needs to be unloaded and reloaded\n"); 6647 return -EIO; 6648 } 6649 6650 if (test_bit(__ICE_DOWN, pf->state)) { 6651 netdev_err(netdev, "device is not ready yet\n"); 6652 return -EBUSY; 6653 } 6654 6655 netif_carrier_off(netdev); 6656 6657 pi = vsi->port_info; 6658 err = ice_update_link_info(pi); 6659 if (err) { 6660 netdev_err(netdev, "Failed to get link info, error %d\n", 6661 err); 6662 return err; 6663 } 6664 6665 /* Set PHY if there is media, otherwise, turn off PHY */ 6666 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 6667 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 6668 if (!test_bit(__ICE_PHY_INIT_COMPLETE, pf->state)) { 6669 err = ice_init_phy_user_cfg(pi); 6670 if (err) { 6671 netdev_err(netdev, "Failed to initialize PHY settings, error %d\n", 6672 err); 6673 return err; 6674 } 6675 } 6676 6677 err = ice_configure_phy(vsi); 6678 if (err) { 6679 netdev_err(netdev, "Failed to set physical link up, error %d\n", 6680 err); 6681 return err; 6682 } 6683 } else { 6684 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 6685 err = ice_aq_set_link_restart_an(pi, false, NULL); 6686 if (err) { 6687 netdev_err(netdev, "Failed to set PHY state, VSI %d error %d\n", 6688 vsi->vsi_num, err); 6689 return err; 6690 } 6691 } 6692 6693 err = ice_vsi_open(vsi); 6694 if (err) 6695 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", 6696 vsi->vsi_num, vsi->vsw->sw_id); 6697 6698 /* Update existing tunnels information */ 6699 udp_tunnel_get_rx_info(netdev); 6700 6701 return err; 6702 } 6703 6704 /** 6705 * ice_stop - Disables a network interface 6706 * @netdev: network interface device structure 6707 * 6708 * The stop entry point is called when an interface is de-activated by the OS, 6709 * and the netdevice enters the DOWN state. The hardware is still under the 6710 * driver's control, but the netdev interface is disabled. 6711 * 6712 * Returns success only - not allowed to fail 6713 */ 6714 int ice_stop(struct net_device *netdev) 6715 { 6716 struct ice_netdev_priv *np = netdev_priv(netdev); 6717 struct ice_vsi *vsi = np->vsi; 6718 6719 ice_vsi_close(vsi); 6720 6721 return 0; 6722 } 6723 6724 /** 6725 * ice_features_check - Validate encapsulated packet conforms to limits 6726 * @skb: skb buffer 6727 * @netdev: This port's netdev 6728 * @features: Offload features that the stack believes apply 6729 */ 6730 static netdev_features_t 6731 ice_features_check(struct sk_buff *skb, 6732 struct net_device __always_unused *netdev, 6733 netdev_features_t features) 6734 { 6735 size_t len; 6736 6737 /* No point in doing any of this if neither checksum nor GSO are 6738 * being requested for this frame. We can rule out both by just 6739 * checking for CHECKSUM_PARTIAL 6740 */ 6741 if (skb->ip_summed != CHECKSUM_PARTIAL) 6742 return features; 6743 6744 /* We cannot support GSO if the MSS is going to be less than 6745 * 64 bytes. If it is then we need to drop support for GSO. 6746 */ 6747 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 6748 features &= ~NETIF_F_GSO_MASK; 6749 6750 len = skb_network_header(skb) - skb->data; 6751 if (len > ICE_TXD_MACLEN_MAX || len & 0x1) 6752 goto out_rm_features; 6753 6754 len = skb_transport_header(skb) - skb_network_header(skb); 6755 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 6756 goto out_rm_features; 6757 6758 if (skb->encapsulation) { 6759 len = skb_inner_network_header(skb) - skb_transport_header(skb); 6760 if (len > ICE_TXD_L4LEN_MAX || len & 0x1) 6761 goto out_rm_features; 6762 6763 len = skb_inner_transport_header(skb) - 6764 skb_inner_network_header(skb); 6765 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 6766 goto out_rm_features; 6767 } 6768 6769 return features; 6770 out_rm_features: 6771 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 6772 } 6773 6774 static const struct net_device_ops ice_netdev_safe_mode_ops = { 6775 .ndo_open = ice_open, 6776 .ndo_stop = ice_stop, 6777 .ndo_start_xmit = ice_start_xmit, 6778 .ndo_set_mac_address = ice_set_mac_address, 6779 .ndo_validate_addr = eth_validate_addr, 6780 .ndo_change_mtu = ice_change_mtu, 6781 .ndo_get_stats64 = ice_get_stats64, 6782 .ndo_tx_timeout = ice_tx_timeout, 6783 }; 6784 6785 static const struct net_device_ops ice_netdev_ops = { 6786 .ndo_open = ice_open, 6787 .ndo_stop = ice_stop, 6788 .ndo_start_xmit = ice_start_xmit, 6789 .ndo_features_check = ice_features_check, 6790 .ndo_set_rx_mode = ice_set_rx_mode, 6791 .ndo_set_mac_address = ice_set_mac_address, 6792 .ndo_validate_addr = eth_validate_addr, 6793 .ndo_change_mtu = ice_change_mtu, 6794 .ndo_get_stats64 = ice_get_stats64, 6795 .ndo_set_tx_maxrate = ice_set_tx_maxrate, 6796 .ndo_set_vf_spoofchk = ice_set_vf_spoofchk, 6797 .ndo_set_vf_mac = ice_set_vf_mac, 6798 .ndo_get_vf_config = ice_get_vf_cfg, 6799 .ndo_set_vf_trust = ice_set_vf_trust, 6800 .ndo_set_vf_vlan = ice_set_vf_port_vlan, 6801 .ndo_set_vf_link_state = ice_set_vf_link_state, 6802 .ndo_get_vf_stats = ice_get_vf_stats, 6803 .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid, 6804 .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid, 6805 .ndo_set_features = ice_set_features, 6806 .ndo_bridge_getlink = ice_bridge_getlink, 6807 .ndo_bridge_setlink = ice_bridge_setlink, 6808 .ndo_fdb_add = ice_fdb_add, 6809 .ndo_fdb_del = ice_fdb_del, 6810 #ifdef CONFIG_RFS_ACCEL 6811 .ndo_rx_flow_steer = ice_rx_flow_steer, 6812 #endif 6813 .ndo_tx_timeout = ice_tx_timeout, 6814 .ndo_bpf = ice_xdp, 6815 .ndo_xdp_xmit = ice_xdp_xmit, 6816 .ndo_xsk_wakeup = ice_xsk_wakeup, 6817 }; 6818