1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2018, Intel Corporation. */ 3 4 /* Intel(R) Ethernet Connection E800 Series Linux Driver */ 5 6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 7 8 #include <generated/utsrelease.h> 9 #include "ice.h" 10 #include "ice_base.h" 11 #include "ice_lib.h" 12 #include "ice_fltr.h" 13 #include "ice_dcb_lib.h" 14 #include "ice_dcb_nl.h" 15 #include "ice_devlink.h" 16 /* Including ice_trace.h with CREATE_TRACE_POINTS defined will generate the 17 * ice tracepoint functions. This must be done exactly once across the 18 * ice driver. 19 */ 20 #define CREATE_TRACE_POINTS 21 #include "ice_trace.h" 22 #include "ice_eswitch.h" 23 #include "ice_tc_lib.h" 24 #include "ice_vsi_vlan_ops.h" 25 26 #define DRV_SUMMARY "Intel(R) Ethernet Connection E800 Series Linux Driver" 27 static const char ice_driver_string[] = DRV_SUMMARY; 28 static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation."; 29 30 /* DDP Package file located in firmware search paths (e.g. /lib/firmware/) */ 31 #define ICE_DDP_PKG_PATH "intel/ice/ddp/" 32 #define ICE_DDP_PKG_FILE ICE_DDP_PKG_PATH "ice.pkg" 33 34 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>"); 35 MODULE_DESCRIPTION(DRV_SUMMARY); 36 MODULE_LICENSE("GPL v2"); 37 MODULE_FIRMWARE(ICE_DDP_PKG_FILE); 38 39 static int debug = -1; 40 module_param(debug, int, 0644); 41 #ifndef CONFIG_DYNAMIC_DEBUG 42 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)"); 43 #else 44 MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)"); 45 #endif /* !CONFIG_DYNAMIC_DEBUG */ 46 47 static DEFINE_IDA(ice_aux_ida); 48 DEFINE_STATIC_KEY_FALSE(ice_xdp_locking_key); 49 EXPORT_SYMBOL(ice_xdp_locking_key); 50 51 /** 52 * ice_hw_to_dev - Get device pointer from the hardware structure 53 * @hw: pointer to the device HW structure 54 * 55 * Used to access the device pointer from compilation units which can't easily 56 * include the definition of struct ice_pf without leading to circular header 57 * dependencies. 58 */ 59 struct device *ice_hw_to_dev(struct ice_hw *hw) 60 { 61 struct ice_pf *pf = container_of(hw, struct ice_pf, hw); 62 63 return &pf->pdev->dev; 64 } 65 66 static struct workqueue_struct *ice_wq; 67 static const struct net_device_ops ice_netdev_safe_mode_ops; 68 static const struct net_device_ops ice_netdev_ops; 69 70 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type); 71 72 static void ice_vsi_release_all(struct ice_pf *pf); 73 74 static int ice_rebuild_channels(struct ice_pf *pf); 75 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_adv_fltr); 76 77 static int 78 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, 79 void *cb_priv, enum tc_setup_type type, void *type_data, 80 void *data, 81 void (*cleanup)(struct flow_block_cb *block_cb)); 82 83 bool netif_is_ice(struct net_device *dev) 84 { 85 return dev && (dev->netdev_ops == &ice_netdev_ops); 86 } 87 88 /** 89 * ice_get_tx_pending - returns number of Tx descriptors not processed 90 * @ring: the ring of descriptors 91 */ 92 static u16 ice_get_tx_pending(struct ice_tx_ring *ring) 93 { 94 u16 head, tail; 95 96 head = ring->next_to_clean; 97 tail = ring->next_to_use; 98 99 if (head != tail) 100 return (head < tail) ? 101 tail - head : (tail + ring->count - head); 102 return 0; 103 } 104 105 /** 106 * ice_check_for_hang_subtask - check for and recover hung queues 107 * @pf: pointer to PF struct 108 */ 109 static void ice_check_for_hang_subtask(struct ice_pf *pf) 110 { 111 struct ice_vsi *vsi = NULL; 112 struct ice_hw *hw; 113 unsigned int i; 114 int packets; 115 u32 v; 116 117 ice_for_each_vsi(pf, v) 118 if (pf->vsi[v] && pf->vsi[v]->type == ICE_VSI_PF) { 119 vsi = pf->vsi[v]; 120 break; 121 } 122 123 if (!vsi || test_bit(ICE_VSI_DOWN, vsi->state)) 124 return; 125 126 if (!(vsi->netdev && netif_carrier_ok(vsi->netdev))) 127 return; 128 129 hw = &vsi->back->hw; 130 131 ice_for_each_txq(vsi, i) { 132 struct ice_tx_ring *tx_ring = vsi->tx_rings[i]; 133 134 if (!tx_ring) 135 continue; 136 if (ice_ring_ch_enabled(tx_ring)) 137 continue; 138 139 if (tx_ring->desc) { 140 /* If packet counter has not changed the queue is 141 * likely stalled, so force an interrupt for this 142 * queue. 143 * 144 * prev_pkt would be negative if there was no 145 * pending work. 146 */ 147 packets = tx_ring->stats.pkts & INT_MAX; 148 if (tx_ring->tx_stats.prev_pkt == packets) { 149 /* Trigger sw interrupt to revive the queue */ 150 ice_trigger_sw_intr(hw, tx_ring->q_vector); 151 continue; 152 } 153 154 /* Memory barrier between read of packet count and call 155 * to ice_get_tx_pending() 156 */ 157 smp_rmb(); 158 tx_ring->tx_stats.prev_pkt = 159 ice_get_tx_pending(tx_ring) ? packets : -1; 160 } 161 } 162 } 163 164 /** 165 * ice_init_mac_fltr - Set initial MAC filters 166 * @pf: board private structure 167 * 168 * Set initial set of MAC filters for PF VSI; configure filters for permanent 169 * address and broadcast address. If an error is encountered, netdevice will be 170 * unregistered. 171 */ 172 static int ice_init_mac_fltr(struct ice_pf *pf) 173 { 174 struct ice_vsi *vsi; 175 u8 *perm_addr; 176 177 vsi = ice_get_main_vsi(pf); 178 if (!vsi) 179 return -EINVAL; 180 181 perm_addr = vsi->port_info->mac.perm_addr; 182 return ice_fltr_add_mac_and_broadcast(vsi, perm_addr, ICE_FWD_TO_VSI); 183 } 184 185 /** 186 * ice_add_mac_to_sync_list - creates list of MAC addresses to be synced 187 * @netdev: the net device on which the sync is happening 188 * @addr: MAC address to sync 189 * 190 * This is a callback function which is called by the in kernel device sync 191 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only 192 * populates the tmp_sync_list, which is later used by ice_add_mac to add the 193 * MAC filters from the hardware. 194 */ 195 static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr) 196 { 197 struct ice_netdev_priv *np = netdev_priv(netdev); 198 struct ice_vsi *vsi = np->vsi; 199 200 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr, 201 ICE_FWD_TO_VSI)) 202 return -EINVAL; 203 204 return 0; 205 } 206 207 /** 208 * ice_add_mac_to_unsync_list - creates list of MAC addresses to be unsynced 209 * @netdev: the net device on which the unsync is happening 210 * @addr: MAC address to unsync 211 * 212 * This is a callback function which is called by the in kernel device unsync 213 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only 214 * populates the tmp_unsync_list, which is later used by ice_remove_mac to 215 * delete the MAC filters from the hardware. 216 */ 217 static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr) 218 { 219 struct ice_netdev_priv *np = netdev_priv(netdev); 220 struct ice_vsi *vsi = np->vsi; 221 222 /* Under some circumstances, we might receive a request to delete our 223 * own device address from our uc list. Because we store the device 224 * address in the VSI's MAC filter list, we need to ignore such 225 * requests and not delete our device address from this list. 226 */ 227 if (ether_addr_equal(addr, netdev->dev_addr)) 228 return 0; 229 230 if (ice_fltr_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr, 231 ICE_FWD_TO_VSI)) 232 return -EINVAL; 233 234 return 0; 235 } 236 237 /** 238 * ice_vsi_fltr_changed - check if filter state changed 239 * @vsi: VSI to be checked 240 * 241 * returns true if filter state has changed, false otherwise. 242 */ 243 static bool ice_vsi_fltr_changed(struct ice_vsi *vsi) 244 { 245 return test_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state) || 246 test_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 247 } 248 249 /** 250 * ice_set_promisc - Enable promiscuous mode for a given PF 251 * @vsi: the VSI being configured 252 * @promisc_m: mask of promiscuous config bits 253 * 254 */ 255 static int ice_set_promisc(struct ice_vsi *vsi, u8 promisc_m) 256 { 257 int status; 258 259 if (vsi->type != ICE_VSI_PF) 260 return 0; 261 262 if (ice_vsi_has_non_zero_vlans(vsi)) { 263 promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX); 264 status = ice_fltr_set_vlan_vsi_promisc(&vsi->back->hw, vsi, 265 promisc_m); 266 } else { 267 status = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 268 promisc_m, 0); 269 } 270 if (status && status != -EEXIST) 271 return status; 272 273 return 0; 274 } 275 276 /** 277 * ice_clear_promisc - Disable promiscuous mode for a given PF 278 * @vsi: the VSI being configured 279 * @promisc_m: mask of promiscuous config bits 280 * 281 */ 282 static int ice_clear_promisc(struct ice_vsi *vsi, u8 promisc_m) 283 { 284 int status; 285 286 if (vsi->type != ICE_VSI_PF) 287 return 0; 288 289 if (ice_vsi_has_non_zero_vlans(vsi)) { 290 promisc_m |= (ICE_PROMISC_VLAN_RX | ICE_PROMISC_VLAN_TX); 291 status = ice_fltr_clear_vlan_vsi_promisc(&vsi->back->hw, vsi, 292 promisc_m); 293 } else { 294 status = ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 295 promisc_m, 0); 296 } 297 298 return status; 299 } 300 301 /** 302 * ice_get_devlink_port - Get devlink port from netdev 303 * @netdev: the netdevice structure 304 */ 305 static struct devlink_port *ice_get_devlink_port(struct net_device *netdev) 306 { 307 struct ice_pf *pf = ice_netdev_to_pf(netdev); 308 309 if (!ice_is_switchdev_running(pf)) 310 return NULL; 311 312 return &pf->devlink_port; 313 } 314 315 /** 316 * ice_vsi_sync_fltr - Update the VSI filter list to the HW 317 * @vsi: ptr to the VSI 318 * 319 * Push any outstanding VSI filter changes through the AdminQ. 320 */ 321 static int ice_vsi_sync_fltr(struct ice_vsi *vsi) 322 { 323 struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 324 struct device *dev = ice_pf_to_dev(vsi->back); 325 struct net_device *netdev = vsi->netdev; 326 bool promisc_forced_on = false; 327 struct ice_pf *pf = vsi->back; 328 struct ice_hw *hw = &pf->hw; 329 u32 changed_flags = 0; 330 int err; 331 332 if (!vsi->netdev) 333 return -EINVAL; 334 335 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 336 usleep_range(1000, 2000); 337 338 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 339 vsi->current_netdev_flags = vsi->netdev->flags; 340 341 INIT_LIST_HEAD(&vsi->tmp_sync_list); 342 INIT_LIST_HEAD(&vsi->tmp_unsync_list); 343 344 if (ice_vsi_fltr_changed(vsi)) { 345 clear_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 346 clear_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 347 348 /* grab the netdev's addr_list_lock */ 349 netif_addr_lock_bh(netdev); 350 __dev_uc_sync(netdev, ice_add_mac_to_sync_list, 351 ice_add_mac_to_unsync_list); 352 __dev_mc_sync(netdev, ice_add_mac_to_sync_list, 353 ice_add_mac_to_unsync_list); 354 /* our temp lists are populated. release lock */ 355 netif_addr_unlock_bh(netdev); 356 } 357 358 /* Remove MAC addresses in the unsync list */ 359 err = ice_fltr_remove_mac_list(vsi, &vsi->tmp_unsync_list); 360 ice_fltr_free_list(dev, &vsi->tmp_unsync_list); 361 if (err) { 362 netdev_err(netdev, "Failed to delete MAC filters\n"); 363 /* if we failed because of alloc failures, just bail */ 364 if (err == -ENOMEM) 365 goto out; 366 } 367 368 /* Add MAC addresses in the sync list */ 369 err = ice_fltr_add_mac_list(vsi, &vsi->tmp_sync_list); 370 ice_fltr_free_list(dev, &vsi->tmp_sync_list); 371 /* If filter is added successfully or already exists, do not go into 372 * 'if' condition and report it as error. Instead continue processing 373 * rest of the function. 374 */ 375 if (err && err != -EEXIST) { 376 netdev_err(netdev, "Failed to add MAC filters\n"); 377 /* If there is no more space for new umac filters, VSI 378 * should go into promiscuous mode. There should be some 379 * space reserved for promiscuous filters. 380 */ 381 if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC && 382 !test_and_set_bit(ICE_FLTR_OVERFLOW_PROMISC, 383 vsi->state)) { 384 promisc_forced_on = true; 385 netdev_warn(netdev, "Reached MAC filter limit, forcing promisc mode on VSI %d\n", 386 vsi->vsi_num); 387 } else { 388 goto out; 389 } 390 } 391 err = 0; 392 /* check for changes in promiscuous modes */ 393 if (changed_flags & IFF_ALLMULTI) { 394 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 395 err = ice_set_promisc(vsi, ICE_MCAST_PROMISC_BITS); 396 if (err) { 397 vsi->current_netdev_flags &= ~IFF_ALLMULTI; 398 goto out_promisc; 399 } 400 } else { 401 /* !(vsi->current_netdev_flags & IFF_ALLMULTI) */ 402 err = ice_clear_promisc(vsi, ICE_MCAST_PROMISC_BITS); 403 if (err) { 404 vsi->current_netdev_flags |= IFF_ALLMULTI; 405 goto out_promisc; 406 } 407 } 408 } 409 410 if (((changed_flags & IFF_PROMISC) || promisc_forced_on) || 411 test_bit(ICE_VSI_PROMISC_CHANGED, vsi->state)) { 412 clear_bit(ICE_VSI_PROMISC_CHANGED, vsi->state); 413 if (vsi->current_netdev_flags & IFF_PROMISC) { 414 /* Apply Rx filter rule to get traffic from wire */ 415 if (!ice_is_dflt_vsi_in_use(vsi->port_info)) { 416 err = ice_set_dflt_vsi(vsi); 417 if (err && err != -EEXIST) { 418 netdev_err(netdev, "Error %d setting default VSI %i Rx rule\n", 419 err, vsi->vsi_num); 420 vsi->current_netdev_flags &= 421 ~IFF_PROMISC; 422 goto out_promisc; 423 } 424 err = 0; 425 vlan_ops->dis_rx_filtering(vsi); 426 } 427 } else { 428 /* Clear Rx filter to remove traffic from wire */ 429 if (ice_is_vsi_dflt_vsi(vsi)) { 430 err = ice_clear_dflt_vsi(vsi); 431 if (err) { 432 netdev_err(netdev, "Error %d clearing default VSI %i Rx rule\n", 433 err, vsi->vsi_num); 434 vsi->current_netdev_flags |= 435 IFF_PROMISC; 436 goto out_promisc; 437 } 438 if (vsi->netdev->features & 439 NETIF_F_HW_VLAN_CTAG_FILTER) 440 vlan_ops->ena_rx_filtering(vsi); 441 } 442 } 443 } 444 goto exit; 445 446 out_promisc: 447 set_bit(ICE_VSI_PROMISC_CHANGED, vsi->state); 448 goto exit; 449 out: 450 /* if something went wrong then set the changed flag so we try again */ 451 set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 452 set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 453 exit: 454 clear_bit(ICE_CFG_BUSY, vsi->state); 455 return err; 456 } 457 458 /** 459 * ice_sync_fltr_subtask - Sync the VSI filter list with HW 460 * @pf: board private structure 461 */ 462 static void ice_sync_fltr_subtask(struct ice_pf *pf) 463 { 464 int v; 465 466 if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags))) 467 return; 468 469 clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 470 471 ice_for_each_vsi(pf, v) 472 if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) && 473 ice_vsi_sync_fltr(pf->vsi[v])) { 474 /* come back and try again later */ 475 set_bit(ICE_FLAG_FLTR_SYNC, pf->flags); 476 break; 477 } 478 } 479 480 /** 481 * ice_pf_dis_all_vsi - Pause all VSIs on a PF 482 * @pf: the PF 483 * @locked: is the rtnl_lock already held 484 */ 485 static void ice_pf_dis_all_vsi(struct ice_pf *pf, bool locked) 486 { 487 int node; 488 int v; 489 490 ice_for_each_vsi(pf, v) 491 if (pf->vsi[v]) 492 ice_dis_vsi(pf->vsi[v], locked); 493 494 for (node = 0; node < ICE_MAX_PF_AGG_NODES; node++) 495 pf->pf_agg_node[node].num_vsis = 0; 496 497 for (node = 0; node < ICE_MAX_VF_AGG_NODES; node++) 498 pf->vf_agg_node[node].num_vsis = 0; 499 } 500 501 /** 502 * ice_clear_sw_switch_recipes - clear switch recipes 503 * @pf: board private structure 504 * 505 * Mark switch recipes as not created in sw structures. There are cases where 506 * rules (especially advanced rules) need to be restored, either re-read from 507 * hardware or added again. For example after the reset. 'recp_created' flag 508 * prevents from doing that and need to be cleared upfront. 509 */ 510 static void ice_clear_sw_switch_recipes(struct ice_pf *pf) 511 { 512 struct ice_sw_recipe *recp; 513 u8 i; 514 515 recp = pf->hw.switch_info->recp_list; 516 for (i = 0; i < ICE_MAX_NUM_RECIPES; i++) 517 recp[i].recp_created = false; 518 } 519 520 /** 521 * ice_prepare_for_reset - prep for reset 522 * @pf: board private structure 523 * @reset_type: reset type requested 524 * 525 * Inform or close all dependent features in prep for reset. 526 */ 527 static void 528 ice_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 529 { 530 struct ice_hw *hw = &pf->hw; 531 struct ice_vsi *vsi; 532 struct ice_vf *vf; 533 unsigned int bkt; 534 535 dev_dbg(ice_pf_to_dev(pf), "reset_type=%d\n", reset_type); 536 537 /* already prepared for reset */ 538 if (test_bit(ICE_PREPARED_FOR_RESET, pf->state)) 539 return; 540 541 ice_unplug_aux_dev(pf); 542 543 /* Notify VFs of impending reset */ 544 if (ice_check_sq_alive(hw, &hw->mailboxq)) 545 ice_vc_notify_reset(pf); 546 547 /* Disable VFs until reset is completed */ 548 mutex_lock(&pf->vfs.table_lock); 549 ice_for_each_vf(pf, bkt, vf) 550 ice_set_vf_state_qs_dis(vf); 551 mutex_unlock(&pf->vfs.table_lock); 552 553 if (ice_is_eswitch_mode_switchdev(pf)) { 554 if (reset_type != ICE_RESET_PFR) 555 ice_clear_sw_switch_recipes(pf); 556 } 557 558 /* release ADQ specific HW and SW resources */ 559 vsi = ice_get_main_vsi(pf); 560 if (!vsi) 561 goto skip; 562 563 /* to be on safe side, reset orig_rss_size so that normal flow 564 * of deciding rss_size can take precedence 565 */ 566 vsi->orig_rss_size = 0; 567 568 if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 569 if (reset_type == ICE_RESET_PFR) { 570 vsi->old_ena_tc = vsi->all_enatc; 571 vsi->old_numtc = vsi->all_numtc; 572 } else { 573 ice_remove_q_channels(vsi, true); 574 575 /* for other reset type, do not support channel rebuild 576 * hence reset needed info 577 */ 578 vsi->old_ena_tc = 0; 579 vsi->all_enatc = 0; 580 vsi->old_numtc = 0; 581 vsi->all_numtc = 0; 582 vsi->req_txq = 0; 583 vsi->req_rxq = 0; 584 clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 585 memset(&vsi->mqprio_qopt, 0, sizeof(vsi->mqprio_qopt)); 586 } 587 } 588 skip: 589 590 /* clear SW filtering DB */ 591 ice_clear_hw_tbls(hw); 592 /* disable the VSIs and their queues that are not already DOWN */ 593 ice_pf_dis_all_vsi(pf, false); 594 595 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 596 ice_ptp_prepare_for_reset(pf); 597 598 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 599 ice_gnss_exit(pf); 600 601 if (hw->port_info) 602 ice_sched_clear_port(hw->port_info); 603 604 ice_shutdown_all_ctrlq(hw); 605 606 set_bit(ICE_PREPARED_FOR_RESET, pf->state); 607 } 608 609 /** 610 * ice_do_reset - Initiate one of many types of resets 611 * @pf: board private structure 612 * @reset_type: reset type requested before this function was called. 613 */ 614 static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type) 615 { 616 struct device *dev = ice_pf_to_dev(pf); 617 struct ice_hw *hw = &pf->hw; 618 619 dev_dbg(dev, "reset_type 0x%x requested\n", reset_type); 620 621 ice_prepare_for_reset(pf, reset_type); 622 623 /* trigger the reset */ 624 if (ice_reset(hw, reset_type)) { 625 dev_err(dev, "reset %d failed\n", reset_type); 626 set_bit(ICE_RESET_FAILED, pf->state); 627 clear_bit(ICE_RESET_OICR_RECV, pf->state); 628 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 629 clear_bit(ICE_PFR_REQ, pf->state); 630 clear_bit(ICE_CORER_REQ, pf->state); 631 clear_bit(ICE_GLOBR_REQ, pf->state); 632 wake_up(&pf->reset_wait_queue); 633 return; 634 } 635 636 /* PFR is a bit of a special case because it doesn't result in an OICR 637 * interrupt. So for PFR, rebuild after the reset and clear the reset- 638 * associated state bits. 639 */ 640 if (reset_type == ICE_RESET_PFR) { 641 pf->pfr_count++; 642 ice_rebuild(pf, reset_type); 643 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 644 clear_bit(ICE_PFR_REQ, pf->state); 645 wake_up(&pf->reset_wait_queue); 646 ice_reset_all_vfs(pf); 647 } 648 } 649 650 /** 651 * ice_reset_subtask - Set up for resetting the device and driver 652 * @pf: board private structure 653 */ 654 static void ice_reset_subtask(struct ice_pf *pf) 655 { 656 enum ice_reset_req reset_type = ICE_RESET_INVAL; 657 658 /* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an 659 * OICR interrupt. The OICR handler (ice_misc_intr) determines what type 660 * of reset is pending and sets bits in pf->state indicating the reset 661 * type and ICE_RESET_OICR_RECV. So, if the latter bit is set 662 * prepare for pending reset if not already (for PF software-initiated 663 * global resets the software should already be prepared for it as 664 * indicated by ICE_PREPARED_FOR_RESET; for global resets initiated 665 * by firmware or software on other PFs, that bit is not set so prepare 666 * for the reset now), poll for reset done, rebuild and return. 667 */ 668 if (test_bit(ICE_RESET_OICR_RECV, pf->state)) { 669 /* Perform the largest reset requested */ 670 if (test_and_clear_bit(ICE_CORER_RECV, pf->state)) 671 reset_type = ICE_RESET_CORER; 672 if (test_and_clear_bit(ICE_GLOBR_RECV, pf->state)) 673 reset_type = ICE_RESET_GLOBR; 674 if (test_and_clear_bit(ICE_EMPR_RECV, pf->state)) 675 reset_type = ICE_RESET_EMPR; 676 /* return if no valid reset type requested */ 677 if (reset_type == ICE_RESET_INVAL) 678 return; 679 ice_prepare_for_reset(pf, reset_type); 680 681 /* make sure we are ready to rebuild */ 682 if (ice_check_reset(&pf->hw)) { 683 set_bit(ICE_RESET_FAILED, pf->state); 684 } else { 685 /* done with reset. start rebuild */ 686 pf->hw.reset_ongoing = false; 687 ice_rebuild(pf, reset_type); 688 /* clear bit to resume normal operations, but 689 * ICE_NEEDS_RESTART bit is set in case rebuild failed 690 */ 691 clear_bit(ICE_RESET_OICR_RECV, pf->state); 692 clear_bit(ICE_PREPARED_FOR_RESET, pf->state); 693 clear_bit(ICE_PFR_REQ, pf->state); 694 clear_bit(ICE_CORER_REQ, pf->state); 695 clear_bit(ICE_GLOBR_REQ, pf->state); 696 wake_up(&pf->reset_wait_queue); 697 ice_reset_all_vfs(pf); 698 } 699 700 return; 701 } 702 703 /* No pending resets to finish processing. Check for new resets */ 704 if (test_bit(ICE_PFR_REQ, pf->state)) 705 reset_type = ICE_RESET_PFR; 706 if (test_bit(ICE_CORER_REQ, pf->state)) 707 reset_type = ICE_RESET_CORER; 708 if (test_bit(ICE_GLOBR_REQ, pf->state)) 709 reset_type = ICE_RESET_GLOBR; 710 /* If no valid reset type requested just return */ 711 if (reset_type == ICE_RESET_INVAL) 712 return; 713 714 /* reset if not already down or busy */ 715 if (!test_bit(ICE_DOWN, pf->state) && 716 !test_bit(ICE_CFG_BUSY, pf->state)) { 717 ice_do_reset(pf, reset_type); 718 } 719 } 720 721 /** 722 * ice_print_topo_conflict - print topology conflict message 723 * @vsi: the VSI whose topology status is being checked 724 */ 725 static void ice_print_topo_conflict(struct ice_vsi *vsi) 726 { 727 switch (vsi->port_info->phy.link_info.topo_media_conflict) { 728 case ICE_AQ_LINK_TOPO_CONFLICT: 729 case ICE_AQ_LINK_MEDIA_CONFLICT: 730 case ICE_AQ_LINK_TOPO_UNREACH_PRT: 731 case ICE_AQ_LINK_TOPO_UNDRUTIL_PRT: 732 case ICE_AQ_LINK_TOPO_UNDRUTIL_MEDIA: 733 netdev_info(vsi->netdev, "Potential misconfiguration of the Ethernet port detected. If it was not intended, please use the Intel (R) Ethernet Port Configuration Tool to address the issue.\n"); 734 break; 735 case ICE_AQ_LINK_TOPO_UNSUPP_MEDIA: 736 if (test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, vsi->back->flags)) 737 netdev_warn(vsi->netdev, "An unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules\n"); 738 else 739 netdev_err(vsi->netdev, "Rx/Tx is disabled on this device because an unsupported module type was detected. Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 740 break; 741 default: 742 break; 743 } 744 } 745 746 /** 747 * ice_print_link_msg - print link up or down message 748 * @vsi: the VSI whose link status is being queried 749 * @isup: boolean for if the link is now up or down 750 */ 751 void ice_print_link_msg(struct ice_vsi *vsi, bool isup) 752 { 753 struct ice_aqc_get_phy_caps_data *caps; 754 const char *an_advertised; 755 const char *fec_req; 756 const char *speed; 757 const char *fec; 758 const char *fc; 759 const char *an; 760 int status; 761 762 if (!vsi) 763 return; 764 765 if (vsi->current_isup == isup) 766 return; 767 768 vsi->current_isup = isup; 769 770 if (!isup) { 771 netdev_info(vsi->netdev, "NIC Link is Down\n"); 772 return; 773 } 774 775 switch (vsi->port_info->phy.link_info.link_speed) { 776 case ICE_AQ_LINK_SPEED_100GB: 777 speed = "100 G"; 778 break; 779 case ICE_AQ_LINK_SPEED_50GB: 780 speed = "50 G"; 781 break; 782 case ICE_AQ_LINK_SPEED_40GB: 783 speed = "40 G"; 784 break; 785 case ICE_AQ_LINK_SPEED_25GB: 786 speed = "25 G"; 787 break; 788 case ICE_AQ_LINK_SPEED_20GB: 789 speed = "20 G"; 790 break; 791 case ICE_AQ_LINK_SPEED_10GB: 792 speed = "10 G"; 793 break; 794 case ICE_AQ_LINK_SPEED_5GB: 795 speed = "5 G"; 796 break; 797 case ICE_AQ_LINK_SPEED_2500MB: 798 speed = "2.5 G"; 799 break; 800 case ICE_AQ_LINK_SPEED_1000MB: 801 speed = "1 G"; 802 break; 803 case ICE_AQ_LINK_SPEED_100MB: 804 speed = "100 M"; 805 break; 806 default: 807 speed = "Unknown "; 808 break; 809 } 810 811 switch (vsi->port_info->fc.current_mode) { 812 case ICE_FC_FULL: 813 fc = "Rx/Tx"; 814 break; 815 case ICE_FC_TX_PAUSE: 816 fc = "Tx"; 817 break; 818 case ICE_FC_RX_PAUSE: 819 fc = "Rx"; 820 break; 821 case ICE_FC_NONE: 822 fc = "None"; 823 break; 824 default: 825 fc = "Unknown"; 826 break; 827 } 828 829 /* Get FEC mode based on negotiated link info */ 830 switch (vsi->port_info->phy.link_info.fec_info) { 831 case ICE_AQ_LINK_25G_RS_528_FEC_EN: 832 case ICE_AQ_LINK_25G_RS_544_FEC_EN: 833 fec = "RS-FEC"; 834 break; 835 case ICE_AQ_LINK_25G_KR_FEC_EN: 836 fec = "FC-FEC/BASE-R"; 837 break; 838 default: 839 fec = "NONE"; 840 break; 841 } 842 843 /* check if autoneg completed, might be false due to not supported */ 844 if (vsi->port_info->phy.link_info.an_info & ICE_AQ_AN_COMPLETED) 845 an = "True"; 846 else 847 an = "False"; 848 849 /* Get FEC mode requested based on PHY caps last SW configuration */ 850 caps = kzalloc(sizeof(*caps), GFP_KERNEL); 851 if (!caps) { 852 fec_req = "Unknown"; 853 an_advertised = "Unknown"; 854 goto done; 855 } 856 857 status = ice_aq_get_phy_caps(vsi->port_info, false, 858 ICE_AQC_REPORT_ACTIVE_CFG, caps, NULL); 859 if (status) 860 netdev_info(vsi->netdev, "Get phy capability failed.\n"); 861 862 an_advertised = ice_is_phy_caps_an_enabled(caps) ? "On" : "Off"; 863 864 if (caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_528_REQ || 865 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_RS_544_REQ) 866 fec_req = "RS-FEC"; 867 else if (caps->link_fec_options & ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ || 868 caps->link_fec_options & ICE_AQC_PHY_FEC_25G_KR_REQ) 869 fec_req = "FC-FEC/BASE-R"; 870 else 871 fec_req = "NONE"; 872 873 kfree(caps); 874 875 done: 876 netdev_info(vsi->netdev, "NIC Link is up %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg Advertised: %s, Autoneg Negotiated: %s, Flow Control: %s\n", 877 speed, fec_req, fec, an_advertised, an, fc); 878 ice_print_topo_conflict(vsi); 879 } 880 881 /** 882 * ice_vsi_link_event - update the VSI's netdev 883 * @vsi: the VSI on which the link event occurred 884 * @link_up: whether or not the VSI needs to be set up or down 885 */ 886 static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up) 887 { 888 if (!vsi) 889 return; 890 891 if (test_bit(ICE_VSI_DOWN, vsi->state) || !vsi->netdev) 892 return; 893 894 if (vsi->type == ICE_VSI_PF) { 895 if (link_up == netif_carrier_ok(vsi->netdev)) 896 return; 897 898 if (link_up) { 899 netif_carrier_on(vsi->netdev); 900 netif_tx_wake_all_queues(vsi->netdev); 901 } else { 902 netif_carrier_off(vsi->netdev); 903 netif_tx_stop_all_queues(vsi->netdev); 904 } 905 } 906 } 907 908 /** 909 * ice_set_dflt_mib - send a default config MIB to the FW 910 * @pf: private PF struct 911 * 912 * This function sends a default configuration MIB to the FW. 913 * 914 * If this function errors out at any point, the driver is still able to 915 * function. The main impact is that LFC may not operate as expected. 916 * Therefore an error state in this function should be treated with a DBG 917 * message and continue on with driver rebuild/reenable. 918 */ 919 static void ice_set_dflt_mib(struct ice_pf *pf) 920 { 921 struct device *dev = ice_pf_to_dev(pf); 922 u8 mib_type, *buf, *lldpmib = NULL; 923 u16 len, typelen, offset = 0; 924 struct ice_lldp_org_tlv *tlv; 925 struct ice_hw *hw = &pf->hw; 926 u32 ouisubtype; 927 928 mib_type = SET_LOCAL_MIB_TYPE_LOCAL_MIB; 929 lldpmib = kzalloc(ICE_LLDPDU_SIZE, GFP_KERNEL); 930 if (!lldpmib) { 931 dev_dbg(dev, "%s Failed to allocate MIB memory\n", 932 __func__); 933 return; 934 } 935 936 /* Add ETS CFG TLV */ 937 tlv = (struct ice_lldp_org_tlv *)lldpmib; 938 typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | 939 ICE_IEEE_ETS_TLV_LEN); 940 tlv->typelen = htons(typelen); 941 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 942 ICE_IEEE_SUBTYPE_ETS_CFG); 943 tlv->ouisubtype = htonl(ouisubtype); 944 945 buf = tlv->tlvinfo; 946 buf[0] = 0; 947 948 /* ETS CFG all UPs map to TC 0. Next 4 (1 - 4) Octets = 0. 949 * Octets 5 - 12 are BW values, set octet 5 to 100% BW. 950 * Octets 13 - 20 are TSA values - leave as zeros 951 */ 952 buf[5] = 0x64; 953 len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S; 954 offset += len + 2; 955 tlv = (struct ice_lldp_org_tlv *) 956 ((char *)tlv + sizeof(tlv->typelen) + len); 957 958 /* Add ETS REC TLV */ 959 buf = tlv->tlvinfo; 960 tlv->typelen = htons(typelen); 961 962 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 963 ICE_IEEE_SUBTYPE_ETS_REC); 964 tlv->ouisubtype = htonl(ouisubtype); 965 966 /* First octet of buf is reserved 967 * Octets 1 - 4 map UP to TC - all UPs map to zero 968 * Octets 5 - 12 are BW values - set TC 0 to 100%. 969 * Octets 13 - 20 are TSA value - leave as zeros 970 */ 971 buf[5] = 0x64; 972 offset += len + 2; 973 tlv = (struct ice_lldp_org_tlv *) 974 ((char *)tlv + sizeof(tlv->typelen) + len); 975 976 /* Add PFC CFG TLV */ 977 typelen = ((ICE_TLV_TYPE_ORG << ICE_LLDP_TLV_TYPE_S) | 978 ICE_IEEE_PFC_TLV_LEN); 979 tlv->typelen = htons(typelen); 980 981 ouisubtype = ((ICE_IEEE_8021QAZ_OUI << ICE_LLDP_TLV_OUI_S) | 982 ICE_IEEE_SUBTYPE_PFC_CFG); 983 tlv->ouisubtype = htonl(ouisubtype); 984 985 /* Octet 1 left as all zeros - PFC disabled */ 986 buf[0] = 0x08; 987 len = (typelen & ICE_LLDP_TLV_LEN_M) >> ICE_LLDP_TLV_LEN_S; 988 offset += len + 2; 989 990 if (ice_aq_set_lldp_mib(hw, mib_type, (void *)lldpmib, offset, NULL)) 991 dev_dbg(dev, "%s Failed to set default LLDP MIB\n", __func__); 992 993 kfree(lldpmib); 994 } 995 996 /** 997 * ice_check_phy_fw_load - check if PHY FW load failed 998 * @pf: pointer to PF struct 999 * @link_cfg_err: bitmap from the link info structure 1000 * 1001 * check if external PHY FW load failed and print an error message if it did 1002 */ 1003 static void ice_check_phy_fw_load(struct ice_pf *pf, u8 link_cfg_err) 1004 { 1005 if (!(link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE)) { 1006 clear_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags); 1007 return; 1008 } 1009 1010 if (test_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags)) 1011 return; 1012 1013 if (link_cfg_err & ICE_AQ_LINK_EXTERNAL_PHY_LOAD_FAILURE) { 1014 dev_err(ice_pf_to_dev(pf), "Device failed to load the FW for the external PHY. Please download and install the latest NVM for your device and try again\n"); 1015 set_bit(ICE_FLAG_PHY_FW_LOAD_FAILED, pf->flags); 1016 } 1017 } 1018 1019 /** 1020 * ice_check_module_power 1021 * @pf: pointer to PF struct 1022 * @link_cfg_err: bitmap from the link info structure 1023 * 1024 * check module power level returned by a previous call to aq_get_link_info 1025 * and print error messages if module power level is not supported 1026 */ 1027 static void ice_check_module_power(struct ice_pf *pf, u8 link_cfg_err) 1028 { 1029 /* if module power level is supported, clear the flag */ 1030 if (!(link_cfg_err & (ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT | 1031 ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED))) { 1032 clear_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1033 return; 1034 } 1035 1036 /* if ICE_FLAG_MOD_POWER_UNSUPPORTED was previously set and the 1037 * above block didn't clear this bit, there's nothing to do 1038 */ 1039 if (test_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags)) 1040 return; 1041 1042 if (link_cfg_err & ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT) { 1043 dev_err(ice_pf_to_dev(pf), "The installed module is incompatible with the device's NVM image. Cannot start link\n"); 1044 set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1045 } else if (link_cfg_err & ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED) { 1046 dev_err(ice_pf_to_dev(pf), "The module's power requirements exceed the device's power supply. Cannot start link\n"); 1047 set_bit(ICE_FLAG_MOD_POWER_UNSUPPORTED, pf->flags); 1048 } 1049 } 1050 1051 /** 1052 * ice_check_link_cfg_err - check if link configuration failed 1053 * @pf: pointer to the PF struct 1054 * @link_cfg_err: bitmap from the link info structure 1055 * 1056 * print if any link configuration failure happens due to the value in the 1057 * link_cfg_err parameter in the link info structure 1058 */ 1059 static void ice_check_link_cfg_err(struct ice_pf *pf, u8 link_cfg_err) 1060 { 1061 ice_check_module_power(pf, link_cfg_err); 1062 ice_check_phy_fw_load(pf, link_cfg_err); 1063 } 1064 1065 /** 1066 * ice_link_event - process the link event 1067 * @pf: PF that the link event is associated with 1068 * @pi: port_info for the port that the link event is associated with 1069 * @link_up: true if the physical link is up and false if it is down 1070 * @link_speed: current link speed received from the link event 1071 * 1072 * Returns 0 on success and negative on failure 1073 */ 1074 static int 1075 ice_link_event(struct ice_pf *pf, struct ice_port_info *pi, bool link_up, 1076 u16 link_speed) 1077 { 1078 struct device *dev = ice_pf_to_dev(pf); 1079 struct ice_phy_info *phy_info; 1080 struct ice_vsi *vsi; 1081 u16 old_link_speed; 1082 bool old_link; 1083 int status; 1084 1085 phy_info = &pi->phy; 1086 phy_info->link_info_old = phy_info->link_info; 1087 1088 old_link = !!(phy_info->link_info_old.link_info & ICE_AQ_LINK_UP); 1089 old_link_speed = phy_info->link_info_old.link_speed; 1090 1091 /* update the link info structures and re-enable link events, 1092 * don't bail on failure due to other book keeping needed 1093 */ 1094 status = ice_update_link_info(pi); 1095 if (status) 1096 dev_dbg(dev, "Failed to update link status on port %d, err %d aq_err %s\n", 1097 pi->lport, status, 1098 ice_aq_str(pi->hw->adminq.sq_last_status)); 1099 1100 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 1101 1102 /* Check if the link state is up after updating link info, and treat 1103 * this event as an UP event since the link is actually UP now. 1104 */ 1105 if (phy_info->link_info.link_info & ICE_AQ_LINK_UP) 1106 link_up = true; 1107 1108 vsi = ice_get_main_vsi(pf); 1109 if (!vsi || !vsi->port_info) 1110 return -EINVAL; 1111 1112 /* turn off PHY if media was removed */ 1113 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags) && 1114 !(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) { 1115 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 1116 ice_set_link(vsi, false); 1117 } 1118 1119 /* if the old link up/down and speed is the same as the new */ 1120 if (link_up == old_link && link_speed == old_link_speed) 1121 return 0; 1122 1123 if (!ice_is_e810(&pf->hw)) 1124 ice_ptp_link_change(pf, pf->hw.pf_id, link_up); 1125 1126 if (ice_is_dcb_active(pf)) { 1127 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 1128 ice_dcb_rebuild(pf); 1129 } else { 1130 if (link_up) 1131 ice_set_dflt_mib(pf); 1132 } 1133 ice_vsi_link_event(vsi, link_up); 1134 ice_print_link_msg(vsi, link_up); 1135 1136 ice_vc_notify_link_state(pf); 1137 1138 return 0; 1139 } 1140 1141 /** 1142 * ice_watchdog_subtask - periodic tasks not using event driven scheduling 1143 * @pf: board private structure 1144 */ 1145 static void ice_watchdog_subtask(struct ice_pf *pf) 1146 { 1147 int i; 1148 1149 /* if interface is down do nothing */ 1150 if (test_bit(ICE_DOWN, pf->state) || 1151 test_bit(ICE_CFG_BUSY, pf->state)) 1152 return; 1153 1154 /* make sure we don't do these things too often */ 1155 if (time_before(jiffies, 1156 pf->serv_tmr_prev + pf->serv_tmr_period)) 1157 return; 1158 1159 pf->serv_tmr_prev = jiffies; 1160 1161 /* Update the stats for active netdevs so the network stack 1162 * can look at updated numbers whenever it cares to 1163 */ 1164 ice_update_pf_stats(pf); 1165 ice_for_each_vsi(pf, i) 1166 if (pf->vsi[i] && pf->vsi[i]->netdev) 1167 ice_update_vsi_stats(pf->vsi[i]); 1168 } 1169 1170 /** 1171 * ice_init_link_events - enable/initialize link events 1172 * @pi: pointer to the port_info instance 1173 * 1174 * Returns -EIO on failure, 0 on success 1175 */ 1176 static int ice_init_link_events(struct ice_port_info *pi) 1177 { 1178 u16 mask; 1179 1180 mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA | 1181 ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL | 1182 ICE_AQ_LINK_EVENT_PHY_FW_LOAD_FAIL)); 1183 1184 if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) { 1185 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to set link event mask for port %d\n", 1186 pi->lport); 1187 return -EIO; 1188 } 1189 1190 if (ice_aq_get_link_info(pi, true, NULL, NULL)) { 1191 dev_dbg(ice_hw_to_dev(pi->hw), "Failed to enable link events for port %d\n", 1192 pi->lport); 1193 return -EIO; 1194 } 1195 1196 return 0; 1197 } 1198 1199 /** 1200 * ice_handle_link_event - handle link event via ARQ 1201 * @pf: PF that the link event is associated with 1202 * @event: event structure containing link status info 1203 */ 1204 static int 1205 ice_handle_link_event(struct ice_pf *pf, struct ice_rq_event_info *event) 1206 { 1207 struct ice_aqc_get_link_status_data *link_data; 1208 struct ice_port_info *port_info; 1209 int status; 1210 1211 link_data = (struct ice_aqc_get_link_status_data *)event->msg_buf; 1212 port_info = pf->hw.port_info; 1213 if (!port_info) 1214 return -EINVAL; 1215 1216 status = ice_link_event(pf, port_info, 1217 !!(link_data->link_info & ICE_AQ_LINK_UP), 1218 le16_to_cpu(link_data->link_speed)); 1219 if (status) 1220 dev_dbg(ice_pf_to_dev(pf), "Could not process link event, error %d\n", 1221 status); 1222 1223 return status; 1224 } 1225 1226 enum ice_aq_task_state { 1227 ICE_AQ_TASK_WAITING = 0, 1228 ICE_AQ_TASK_COMPLETE, 1229 ICE_AQ_TASK_CANCELED, 1230 }; 1231 1232 struct ice_aq_task { 1233 struct hlist_node entry; 1234 1235 u16 opcode; 1236 struct ice_rq_event_info *event; 1237 enum ice_aq_task_state state; 1238 }; 1239 1240 /** 1241 * ice_aq_wait_for_event - Wait for an AdminQ event from firmware 1242 * @pf: pointer to the PF private structure 1243 * @opcode: the opcode to wait for 1244 * @timeout: how long to wait, in jiffies 1245 * @event: storage for the event info 1246 * 1247 * Waits for a specific AdminQ completion event on the ARQ for a given PF. The 1248 * current thread will be put to sleep until the specified event occurs or 1249 * until the given timeout is reached. 1250 * 1251 * To obtain only the descriptor contents, pass an event without an allocated 1252 * msg_buf. If the complete data buffer is desired, allocate the 1253 * event->msg_buf with enough space ahead of time. 1254 * 1255 * Returns: zero on success, or a negative error code on failure. 1256 */ 1257 int ice_aq_wait_for_event(struct ice_pf *pf, u16 opcode, unsigned long timeout, 1258 struct ice_rq_event_info *event) 1259 { 1260 struct device *dev = ice_pf_to_dev(pf); 1261 struct ice_aq_task *task; 1262 unsigned long start; 1263 long ret; 1264 int err; 1265 1266 task = kzalloc(sizeof(*task), GFP_KERNEL); 1267 if (!task) 1268 return -ENOMEM; 1269 1270 INIT_HLIST_NODE(&task->entry); 1271 task->opcode = opcode; 1272 task->event = event; 1273 task->state = ICE_AQ_TASK_WAITING; 1274 1275 spin_lock_bh(&pf->aq_wait_lock); 1276 hlist_add_head(&task->entry, &pf->aq_wait_list); 1277 spin_unlock_bh(&pf->aq_wait_lock); 1278 1279 start = jiffies; 1280 1281 ret = wait_event_interruptible_timeout(pf->aq_wait_queue, task->state, 1282 timeout); 1283 switch (task->state) { 1284 case ICE_AQ_TASK_WAITING: 1285 err = ret < 0 ? ret : -ETIMEDOUT; 1286 break; 1287 case ICE_AQ_TASK_CANCELED: 1288 err = ret < 0 ? ret : -ECANCELED; 1289 break; 1290 case ICE_AQ_TASK_COMPLETE: 1291 err = ret < 0 ? ret : 0; 1292 break; 1293 default: 1294 WARN(1, "Unexpected AdminQ wait task state %u", task->state); 1295 err = -EINVAL; 1296 break; 1297 } 1298 1299 dev_dbg(dev, "Waited %u msecs (max %u msecs) for firmware response to op 0x%04x\n", 1300 jiffies_to_msecs(jiffies - start), 1301 jiffies_to_msecs(timeout), 1302 opcode); 1303 1304 spin_lock_bh(&pf->aq_wait_lock); 1305 hlist_del(&task->entry); 1306 spin_unlock_bh(&pf->aq_wait_lock); 1307 kfree(task); 1308 1309 return err; 1310 } 1311 1312 /** 1313 * ice_aq_check_events - Check if any thread is waiting for an AdminQ event 1314 * @pf: pointer to the PF private structure 1315 * @opcode: the opcode of the event 1316 * @event: the event to check 1317 * 1318 * Loops over the current list of pending threads waiting for an AdminQ event. 1319 * For each matching task, copy the contents of the event into the task 1320 * structure and wake up the thread. 1321 * 1322 * If multiple threads wait for the same opcode, they will all be woken up. 1323 * 1324 * Note that event->msg_buf will only be duplicated if the event has a buffer 1325 * with enough space already allocated. Otherwise, only the descriptor and 1326 * message length will be copied. 1327 * 1328 * Returns: true if an event was found, false otherwise 1329 */ 1330 static void ice_aq_check_events(struct ice_pf *pf, u16 opcode, 1331 struct ice_rq_event_info *event) 1332 { 1333 struct ice_aq_task *task; 1334 bool found = false; 1335 1336 spin_lock_bh(&pf->aq_wait_lock); 1337 hlist_for_each_entry(task, &pf->aq_wait_list, entry) { 1338 if (task->state || task->opcode != opcode) 1339 continue; 1340 1341 memcpy(&task->event->desc, &event->desc, sizeof(event->desc)); 1342 task->event->msg_len = event->msg_len; 1343 1344 /* Only copy the data buffer if a destination was set */ 1345 if (task->event->msg_buf && 1346 task->event->buf_len > event->buf_len) { 1347 memcpy(task->event->msg_buf, event->msg_buf, 1348 event->buf_len); 1349 task->event->buf_len = event->buf_len; 1350 } 1351 1352 task->state = ICE_AQ_TASK_COMPLETE; 1353 found = true; 1354 } 1355 spin_unlock_bh(&pf->aq_wait_lock); 1356 1357 if (found) 1358 wake_up(&pf->aq_wait_queue); 1359 } 1360 1361 /** 1362 * ice_aq_cancel_waiting_tasks - Immediately cancel all waiting tasks 1363 * @pf: the PF private structure 1364 * 1365 * Set all waiting tasks to ICE_AQ_TASK_CANCELED, and wake up their threads. 1366 * This will then cause ice_aq_wait_for_event to exit with -ECANCELED. 1367 */ 1368 static void ice_aq_cancel_waiting_tasks(struct ice_pf *pf) 1369 { 1370 struct ice_aq_task *task; 1371 1372 spin_lock_bh(&pf->aq_wait_lock); 1373 hlist_for_each_entry(task, &pf->aq_wait_list, entry) 1374 task->state = ICE_AQ_TASK_CANCELED; 1375 spin_unlock_bh(&pf->aq_wait_lock); 1376 1377 wake_up(&pf->aq_wait_queue); 1378 } 1379 1380 /** 1381 * __ice_clean_ctrlq - helper function to clean controlq rings 1382 * @pf: ptr to struct ice_pf 1383 * @q_type: specific Control queue type 1384 */ 1385 static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type) 1386 { 1387 struct device *dev = ice_pf_to_dev(pf); 1388 struct ice_rq_event_info event; 1389 struct ice_hw *hw = &pf->hw; 1390 struct ice_ctl_q_info *cq; 1391 u16 pending, i = 0; 1392 const char *qtype; 1393 u32 oldval, val; 1394 1395 /* Do not clean control queue if/when PF reset fails */ 1396 if (test_bit(ICE_RESET_FAILED, pf->state)) 1397 return 0; 1398 1399 switch (q_type) { 1400 case ICE_CTL_Q_ADMIN: 1401 cq = &hw->adminq; 1402 qtype = "Admin"; 1403 break; 1404 case ICE_CTL_Q_SB: 1405 cq = &hw->sbq; 1406 qtype = "Sideband"; 1407 break; 1408 case ICE_CTL_Q_MAILBOX: 1409 cq = &hw->mailboxq; 1410 qtype = "Mailbox"; 1411 /* we are going to try to detect a malicious VF, so set the 1412 * state to begin detection 1413 */ 1414 hw->mbx_snapshot.mbx_buf.state = ICE_MAL_VF_DETECT_STATE_NEW_SNAPSHOT; 1415 break; 1416 default: 1417 dev_warn(dev, "Unknown control queue type 0x%x\n", q_type); 1418 return 0; 1419 } 1420 1421 /* check for error indications - PF_xx_AxQLEN register layout for 1422 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN. 1423 */ 1424 val = rd32(hw, cq->rq.len); 1425 if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 1426 PF_FW_ARQLEN_ARQCRIT_M)) { 1427 oldval = val; 1428 if (val & PF_FW_ARQLEN_ARQVFE_M) 1429 dev_dbg(dev, "%s Receive Queue VF Error detected\n", 1430 qtype); 1431 if (val & PF_FW_ARQLEN_ARQOVFL_M) { 1432 dev_dbg(dev, "%s Receive Queue Overflow Error detected\n", 1433 qtype); 1434 } 1435 if (val & PF_FW_ARQLEN_ARQCRIT_M) 1436 dev_dbg(dev, "%s Receive Queue Critical Error detected\n", 1437 qtype); 1438 val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M | 1439 PF_FW_ARQLEN_ARQCRIT_M); 1440 if (oldval != val) 1441 wr32(hw, cq->rq.len, val); 1442 } 1443 1444 val = rd32(hw, cq->sq.len); 1445 if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 1446 PF_FW_ATQLEN_ATQCRIT_M)) { 1447 oldval = val; 1448 if (val & PF_FW_ATQLEN_ATQVFE_M) 1449 dev_dbg(dev, "%s Send Queue VF Error detected\n", 1450 qtype); 1451 if (val & PF_FW_ATQLEN_ATQOVFL_M) { 1452 dev_dbg(dev, "%s Send Queue Overflow Error detected\n", 1453 qtype); 1454 } 1455 if (val & PF_FW_ATQLEN_ATQCRIT_M) 1456 dev_dbg(dev, "%s Send Queue Critical Error detected\n", 1457 qtype); 1458 val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M | 1459 PF_FW_ATQLEN_ATQCRIT_M); 1460 if (oldval != val) 1461 wr32(hw, cq->sq.len, val); 1462 } 1463 1464 event.buf_len = cq->rq_buf_size; 1465 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 1466 if (!event.msg_buf) 1467 return 0; 1468 1469 do { 1470 u16 opcode; 1471 int ret; 1472 1473 ret = ice_clean_rq_elem(hw, cq, &event, &pending); 1474 if (ret == -EALREADY) 1475 break; 1476 if (ret) { 1477 dev_err(dev, "%s Receive Queue event error %d\n", qtype, 1478 ret); 1479 break; 1480 } 1481 1482 opcode = le16_to_cpu(event.desc.opcode); 1483 1484 /* Notify any thread that might be waiting for this event */ 1485 ice_aq_check_events(pf, opcode, &event); 1486 1487 switch (opcode) { 1488 case ice_aqc_opc_get_link_status: 1489 if (ice_handle_link_event(pf, &event)) 1490 dev_err(dev, "Could not handle link event\n"); 1491 break; 1492 case ice_aqc_opc_event_lan_overflow: 1493 ice_vf_lan_overflow_event(pf, &event); 1494 break; 1495 case ice_mbx_opc_send_msg_to_pf: 1496 if (!ice_is_malicious_vf(pf, &event, i, pending)) 1497 ice_vc_process_vf_msg(pf, &event); 1498 break; 1499 case ice_aqc_opc_fw_logging: 1500 ice_output_fw_log(hw, &event.desc, event.msg_buf); 1501 break; 1502 case ice_aqc_opc_lldp_set_mib_change: 1503 ice_dcb_process_lldp_set_mib_change(pf, &event); 1504 break; 1505 default: 1506 dev_dbg(dev, "%s Receive Queue unknown event 0x%04x ignored\n", 1507 qtype, opcode); 1508 break; 1509 } 1510 } while (pending && (i++ < ICE_DFLT_IRQ_WORK)); 1511 1512 kfree(event.msg_buf); 1513 1514 return pending && (i == ICE_DFLT_IRQ_WORK); 1515 } 1516 1517 /** 1518 * ice_ctrlq_pending - check if there is a difference between ntc and ntu 1519 * @hw: pointer to hardware info 1520 * @cq: control queue information 1521 * 1522 * returns true if there are pending messages in a queue, false if there aren't 1523 */ 1524 static bool ice_ctrlq_pending(struct ice_hw *hw, struct ice_ctl_q_info *cq) 1525 { 1526 u16 ntu; 1527 1528 ntu = (u16)(rd32(hw, cq->rq.head) & cq->rq.head_mask); 1529 return cq->rq.next_to_clean != ntu; 1530 } 1531 1532 /** 1533 * ice_clean_adminq_subtask - clean the AdminQ rings 1534 * @pf: board private structure 1535 */ 1536 static void ice_clean_adminq_subtask(struct ice_pf *pf) 1537 { 1538 struct ice_hw *hw = &pf->hw; 1539 1540 if (!test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state)) 1541 return; 1542 1543 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN)) 1544 return; 1545 1546 clear_bit(ICE_ADMINQ_EVENT_PENDING, pf->state); 1547 1548 /* There might be a situation where new messages arrive to a control 1549 * queue between processing the last message and clearing the 1550 * EVENT_PENDING bit. So before exiting, check queue head again (using 1551 * ice_ctrlq_pending) and process new messages if any. 1552 */ 1553 if (ice_ctrlq_pending(hw, &hw->adminq)) 1554 __ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN); 1555 1556 ice_flush(hw); 1557 } 1558 1559 /** 1560 * ice_clean_mailboxq_subtask - clean the MailboxQ rings 1561 * @pf: board private structure 1562 */ 1563 static void ice_clean_mailboxq_subtask(struct ice_pf *pf) 1564 { 1565 struct ice_hw *hw = &pf->hw; 1566 1567 if (!test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state)) 1568 return; 1569 1570 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX)) 1571 return; 1572 1573 clear_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state); 1574 1575 if (ice_ctrlq_pending(hw, &hw->mailboxq)) 1576 __ice_clean_ctrlq(pf, ICE_CTL_Q_MAILBOX); 1577 1578 ice_flush(hw); 1579 } 1580 1581 /** 1582 * ice_clean_sbq_subtask - clean the Sideband Queue rings 1583 * @pf: board private structure 1584 */ 1585 static void ice_clean_sbq_subtask(struct ice_pf *pf) 1586 { 1587 struct ice_hw *hw = &pf->hw; 1588 1589 /* Nothing to do here if sideband queue is not supported */ 1590 if (!ice_is_sbq_supported(hw)) { 1591 clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 1592 return; 1593 } 1594 1595 if (!test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state)) 1596 return; 1597 1598 if (__ice_clean_ctrlq(pf, ICE_CTL_Q_SB)) 1599 return; 1600 1601 clear_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 1602 1603 if (ice_ctrlq_pending(hw, &hw->sbq)) 1604 __ice_clean_ctrlq(pf, ICE_CTL_Q_SB); 1605 1606 ice_flush(hw); 1607 } 1608 1609 /** 1610 * ice_service_task_schedule - schedule the service task to wake up 1611 * @pf: board private structure 1612 * 1613 * If not already scheduled, this puts the task into the work queue. 1614 */ 1615 void ice_service_task_schedule(struct ice_pf *pf) 1616 { 1617 if (!test_bit(ICE_SERVICE_DIS, pf->state) && 1618 !test_and_set_bit(ICE_SERVICE_SCHED, pf->state) && 1619 !test_bit(ICE_NEEDS_RESTART, pf->state)) 1620 queue_work(ice_wq, &pf->serv_task); 1621 } 1622 1623 /** 1624 * ice_service_task_complete - finish up the service task 1625 * @pf: board private structure 1626 */ 1627 static void ice_service_task_complete(struct ice_pf *pf) 1628 { 1629 WARN_ON(!test_bit(ICE_SERVICE_SCHED, pf->state)); 1630 1631 /* force memory (pf->state) to sync before next service task */ 1632 smp_mb__before_atomic(); 1633 clear_bit(ICE_SERVICE_SCHED, pf->state); 1634 } 1635 1636 /** 1637 * ice_service_task_stop - stop service task and cancel works 1638 * @pf: board private structure 1639 * 1640 * Return 0 if the ICE_SERVICE_DIS bit was not already set, 1641 * 1 otherwise. 1642 */ 1643 static int ice_service_task_stop(struct ice_pf *pf) 1644 { 1645 int ret; 1646 1647 ret = test_and_set_bit(ICE_SERVICE_DIS, pf->state); 1648 1649 if (pf->serv_tmr.function) 1650 del_timer_sync(&pf->serv_tmr); 1651 if (pf->serv_task.func) 1652 cancel_work_sync(&pf->serv_task); 1653 1654 clear_bit(ICE_SERVICE_SCHED, pf->state); 1655 return ret; 1656 } 1657 1658 /** 1659 * ice_service_task_restart - restart service task and schedule works 1660 * @pf: board private structure 1661 * 1662 * This function is needed for suspend and resume works (e.g WoL scenario) 1663 */ 1664 static void ice_service_task_restart(struct ice_pf *pf) 1665 { 1666 clear_bit(ICE_SERVICE_DIS, pf->state); 1667 ice_service_task_schedule(pf); 1668 } 1669 1670 /** 1671 * ice_service_timer - timer callback to schedule service task 1672 * @t: pointer to timer_list 1673 */ 1674 static void ice_service_timer(struct timer_list *t) 1675 { 1676 struct ice_pf *pf = from_timer(pf, t, serv_tmr); 1677 1678 mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies)); 1679 ice_service_task_schedule(pf); 1680 } 1681 1682 /** 1683 * ice_handle_mdd_event - handle malicious driver detect event 1684 * @pf: pointer to the PF structure 1685 * 1686 * Called from service task. OICR interrupt handler indicates MDD event. 1687 * VF MDD logging is guarded by net_ratelimit. Additional PF and VF log 1688 * messages are wrapped by netif_msg_[rx|tx]_err. Since VF Rx MDD events 1689 * disable the queue, the PF can be configured to reset the VF using ethtool 1690 * private flag mdd-auto-reset-vf. 1691 */ 1692 static void ice_handle_mdd_event(struct ice_pf *pf) 1693 { 1694 struct device *dev = ice_pf_to_dev(pf); 1695 struct ice_hw *hw = &pf->hw; 1696 struct ice_vf *vf; 1697 unsigned int bkt; 1698 u32 reg; 1699 1700 if (!test_and_clear_bit(ICE_MDD_EVENT_PENDING, pf->state)) { 1701 /* Since the VF MDD event logging is rate limited, check if 1702 * there are pending MDD events. 1703 */ 1704 ice_print_vfs_mdd_events(pf); 1705 return; 1706 } 1707 1708 /* find what triggered an MDD event */ 1709 reg = rd32(hw, GL_MDET_TX_PQM); 1710 if (reg & GL_MDET_TX_PQM_VALID_M) { 1711 u8 pf_num = (reg & GL_MDET_TX_PQM_PF_NUM_M) >> 1712 GL_MDET_TX_PQM_PF_NUM_S; 1713 u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >> 1714 GL_MDET_TX_PQM_VF_NUM_S; 1715 u8 event = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >> 1716 GL_MDET_TX_PQM_MAL_TYPE_S; 1717 u16 queue = ((reg & GL_MDET_TX_PQM_QNUM_M) >> 1718 GL_MDET_TX_PQM_QNUM_S); 1719 1720 if (netif_msg_tx_err(pf)) 1721 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1722 event, queue, pf_num, vf_num); 1723 wr32(hw, GL_MDET_TX_PQM, 0xffffffff); 1724 } 1725 1726 reg = rd32(hw, GL_MDET_TX_TCLAN); 1727 if (reg & GL_MDET_TX_TCLAN_VALID_M) { 1728 u8 pf_num = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >> 1729 GL_MDET_TX_TCLAN_PF_NUM_S; 1730 u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >> 1731 GL_MDET_TX_TCLAN_VF_NUM_S; 1732 u8 event = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >> 1733 GL_MDET_TX_TCLAN_MAL_TYPE_S; 1734 u16 queue = ((reg & GL_MDET_TX_TCLAN_QNUM_M) >> 1735 GL_MDET_TX_TCLAN_QNUM_S); 1736 1737 if (netif_msg_tx_err(pf)) 1738 dev_info(dev, "Malicious Driver Detection event %d on TX queue %d PF# %d VF# %d\n", 1739 event, queue, pf_num, vf_num); 1740 wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff); 1741 } 1742 1743 reg = rd32(hw, GL_MDET_RX); 1744 if (reg & GL_MDET_RX_VALID_M) { 1745 u8 pf_num = (reg & GL_MDET_RX_PF_NUM_M) >> 1746 GL_MDET_RX_PF_NUM_S; 1747 u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >> 1748 GL_MDET_RX_VF_NUM_S; 1749 u8 event = (reg & GL_MDET_RX_MAL_TYPE_M) >> 1750 GL_MDET_RX_MAL_TYPE_S; 1751 u16 queue = ((reg & GL_MDET_RX_QNUM_M) >> 1752 GL_MDET_RX_QNUM_S); 1753 1754 if (netif_msg_rx_err(pf)) 1755 dev_info(dev, "Malicious Driver Detection event %d on RX queue %d PF# %d VF# %d\n", 1756 event, queue, pf_num, vf_num); 1757 wr32(hw, GL_MDET_RX, 0xffffffff); 1758 } 1759 1760 /* check to see if this PF caused an MDD event */ 1761 reg = rd32(hw, PF_MDET_TX_PQM); 1762 if (reg & PF_MDET_TX_PQM_VALID_M) { 1763 wr32(hw, PF_MDET_TX_PQM, 0xFFFF); 1764 if (netif_msg_tx_err(pf)) 1765 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on PF\n"); 1766 } 1767 1768 reg = rd32(hw, PF_MDET_TX_TCLAN); 1769 if (reg & PF_MDET_TX_TCLAN_VALID_M) { 1770 wr32(hw, PF_MDET_TX_TCLAN, 0xFFFF); 1771 if (netif_msg_tx_err(pf)) 1772 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on PF\n"); 1773 } 1774 1775 reg = rd32(hw, PF_MDET_RX); 1776 if (reg & PF_MDET_RX_VALID_M) { 1777 wr32(hw, PF_MDET_RX, 0xFFFF); 1778 if (netif_msg_rx_err(pf)) 1779 dev_info(dev, "Malicious Driver Detection event RX detected on PF\n"); 1780 } 1781 1782 /* Check to see if one of the VFs caused an MDD event, and then 1783 * increment counters and set print pending 1784 */ 1785 mutex_lock(&pf->vfs.table_lock); 1786 ice_for_each_vf(pf, bkt, vf) { 1787 reg = rd32(hw, VP_MDET_TX_PQM(vf->vf_id)); 1788 if (reg & VP_MDET_TX_PQM_VALID_M) { 1789 wr32(hw, VP_MDET_TX_PQM(vf->vf_id), 0xFFFF); 1790 vf->mdd_tx_events.count++; 1791 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1792 if (netif_msg_tx_err(pf)) 1793 dev_info(dev, "Malicious Driver Detection event TX_PQM detected on VF %d\n", 1794 vf->vf_id); 1795 } 1796 1797 reg = rd32(hw, VP_MDET_TX_TCLAN(vf->vf_id)); 1798 if (reg & VP_MDET_TX_TCLAN_VALID_M) { 1799 wr32(hw, VP_MDET_TX_TCLAN(vf->vf_id), 0xFFFF); 1800 vf->mdd_tx_events.count++; 1801 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1802 if (netif_msg_tx_err(pf)) 1803 dev_info(dev, "Malicious Driver Detection event TX_TCLAN detected on VF %d\n", 1804 vf->vf_id); 1805 } 1806 1807 reg = rd32(hw, VP_MDET_TX_TDPU(vf->vf_id)); 1808 if (reg & VP_MDET_TX_TDPU_VALID_M) { 1809 wr32(hw, VP_MDET_TX_TDPU(vf->vf_id), 0xFFFF); 1810 vf->mdd_tx_events.count++; 1811 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1812 if (netif_msg_tx_err(pf)) 1813 dev_info(dev, "Malicious Driver Detection event TX_TDPU detected on VF %d\n", 1814 vf->vf_id); 1815 } 1816 1817 reg = rd32(hw, VP_MDET_RX(vf->vf_id)); 1818 if (reg & VP_MDET_RX_VALID_M) { 1819 wr32(hw, VP_MDET_RX(vf->vf_id), 0xFFFF); 1820 vf->mdd_rx_events.count++; 1821 set_bit(ICE_MDD_VF_PRINT_PENDING, pf->state); 1822 if (netif_msg_rx_err(pf)) 1823 dev_info(dev, "Malicious Driver Detection event RX detected on VF %d\n", 1824 vf->vf_id); 1825 1826 /* Since the queue is disabled on VF Rx MDD events, the 1827 * PF can be configured to reset the VF through ethtool 1828 * private flag mdd-auto-reset-vf. 1829 */ 1830 if (test_bit(ICE_FLAG_MDD_AUTO_RESET_VF, pf->flags)) { 1831 /* VF MDD event counters will be cleared by 1832 * reset, so print the event prior to reset. 1833 */ 1834 ice_print_vf_rx_mdd_event(vf); 1835 ice_reset_vf(vf, ICE_VF_RESET_LOCK); 1836 } 1837 } 1838 } 1839 mutex_unlock(&pf->vfs.table_lock); 1840 1841 ice_print_vfs_mdd_events(pf); 1842 } 1843 1844 /** 1845 * ice_force_phys_link_state - Force the physical link state 1846 * @vsi: VSI to force the physical link state to up/down 1847 * @link_up: true/false indicates to set the physical link to up/down 1848 * 1849 * Force the physical link state by getting the current PHY capabilities from 1850 * hardware and setting the PHY config based on the determined capabilities. If 1851 * link changes a link event will be triggered because both the Enable Automatic 1852 * Link Update and LESM Enable bits are set when setting the PHY capabilities. 1853 * 1854 * Returns 0 on success, negative on failure 1855 */ 1856 static int ice_force_phys_link_state(struct ice_vsi *vsi, bool link_up) 1857 { 1858 struct ice_aqc_get_phy_caps_data *pcaps; 1859 struct ice_aqc_set_phy_cfg_data *cfg; 1860 struct ice_port_info *pi; 1861 struct device *dev; 1862 int retcode; 1863 1864 if (!vsi || !vsi->port_info || !vsi->back) 1865 return -EINVAL; 1866 if (vsi->type != ICE_VSI_PF) 1867 return 0; 1868 1869 dev = ice_pf_to_dev(vsi->back); 1870 1871 pi = vsi->port_info; 1872 1873 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1874 if (!pcaps) 1875 return -ENOMEM; 1876 1877 retcode = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps, 1878 NULL); 1879 if (retcode) { 1880 dev_err(dev, "Failed to get phy capabilities, VSI %d error %d\n", 1881 vsi->vsi_num, retcode); 1882 retcode = -EIO; 1883 goto out; 1884 } 1885 1886 /* No change in link */ 1887 if (link_up == !!(pcaps->caps & ICE_AQC_PHY_EN_LINK) && 1888 link_up == !!(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) 1889 goto out; 1890 1891 /* Use the current user PHY configuration. The current user PHY 1892 * configuration is initialized during probe from PHY capabilities 1893 * software mode, and updated on set PHY configuration. 1894 */ 1895 cfg = kmemdup(&pi->phy.curr_user_phy_cfg, sizeof(*cfg), GFP_KERNEL); 1896 if (!cfg) { 1897 retcode = -ENOMEM; 1898 goto out; 1899 } 1900 1901 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT; 1902 if (link_up) 1903 cfg->caps |= ICE_AQ_PHY_ENA_LINK; 1904 else 1905 cfg->caps &= ~ICE_AQ_PHY_ENA_LINK; 1906 1907 retcode = ice_aq_set_phy_cfg(&vsi->back->hw, pi, cfg, NULL); 1908 if (retcode) { 1909 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 1910 vsi->vsi_num, retcode); 1911 retcode = -EIO; 1912 } 1913 1914 kfree(cfg); 1915 out: 1916 kfree(pcaps); 1917 return retcode; 1918 } 1919 1920 /** 1921 * ice_init_nvm_phy_type - Initialize the NVM PHY type 1922 * @pi: port info structure 1923 * 1924 * Initialize nvm_phy_type_[low|high] for link lenient mode support 1925 */ 1926 static int ice_init_nvm_phy_type(struct ice_port_info *pi) 1927 { 1928 struct ice_aqc_get_phy_caps_data *pcaps; 1929 struct ice_pf *pf = pi->hw->back; 1930 int err; 1931 1932 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 1933 if (!pcaps) 1934 return -ENOMEM; 1935 1936 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA, 1937 pcaps, NULL); 1938 1939 if (err) { 1940 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 1941 goto out; 1942 } 1943 1944 pf->nvm_phy_type_hi = pcaps->phy_type_high; 1945 pf->nvm_phy_type_lo = pcaps->phy_type_low; 1946 1947 out: 1948 kfree(pcaps); 1949 return err; 1950 } 1951 1952 /** 1953 * ice_init_link_dflt_override - Initialize link default override 1954 * @pi: port info structure 1955 * 1956 * Initialize link default override and PHY total port shutdown during probe 1957 */ 1958 static void ice_init_link_dflt_override(struct ice_port_info *pi) 1959 { 1960 struct ice_link_default_override_tlv *ldo; 1961 struct ice_pf *pf = pi->hw->back; 1962 1963 ldo = &pf->link_dflt_override; 1964 if (ice_get_link_default_override(ldo, pi)) 1965 return; 1966 1967 if (!(ldo->options & ICE_LINK_OVERRIDE_PORT_DIS)) 1968 return; 1969 1970 /* Enable Total Port Shutdown (override/replace link-down-on-close 1971 * ethtool private flag) for ports with Port Disable bit set. 1972 */ 1973 set_bit(ICE_FLAG_TOTAL_PORT_SHUTDOWN_ENA, pf->flags); 1974 set_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags); 1975 } 1976 1977 /** 1978 * ice_init_phy_cfg_dflt_override - Initialize PHY cfg default override settings 1979 * @pi: port info structure 1980 * 1981 * If default override is enabled, initialize the user PHY cfg speed and FEC 1982 * settings using the default override mask from the NVM. 1983 * 1984 * The PHY should only be configured with the default override settings the 1985 * first time media is available. The ICE_LINK_DEFAULT_OVERRIDE_PENDING state 1986 * is used to indicate that the user PHY cfg default override is initialized 1987 * and the PHY has not been configured with the default override settings. The 1988 * state is set here, and cleared in ice_configure_phy the first time the PHY is 1989 * configured. 1990 * 1991 * This function should be called only if the FW doesn't support default 1992 * configuration mode, as reported by ice_fw_supports_report_dflt_cfg. 1993 */ 1994 static void ice_init_phy_cfg_dflt_override(struct ice_port_info *pi) 1995 { 1996 struct ice_link_default_override_tlv *ldo; 1997 struct ice_aqc_set_phy_cfg_data *cfg; 1998 struct ice_phy_info *phy = &pi->phy; 1999 struct ice_pf *pf = pi->hw->back; 2000 2001 ldo = &pf->link_dflt_override; 2002 2003 /* If link default override is enabled, use to mask NVM PHY capabilities 2004 * for speed and FEC default configuration. 2005 */ 2006 cfg = &phy->curr_user_phy_cfg; 2007 2008 if (ldo->phy_type_low || ldo->phy_type_high) { 2009 cfg->phy_type_low = pf->nvm_phy_type_lo & 2010 cpu_to_le64(ldo->phy_type_low); 2011 cfg->phy_type_high = pf->nvm_phy_type_hi & 2012 cpu_to_le64(ldo->phy_type_high); 2013 } 2014 cfg->link_fec_opt = ldo->fec_options; 2015 phy->curr_user_fec_req = ICE_FEC_AUTO; 2016 2017 set_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, pf->state); 2018 } 2019 2020 /** 2021 * ice_init_phy_user_cfg - Initialize the PHY user configuration 2022 * @pi: port info structure 2023 * 2024 * Initialize the current user PHY configuration, speed, FEC, and FC requested 2025 * mode to default. The PHY defaults are from get PHY capabilities topology 2026 * with media so call when media is first available. An error is returned if 2027 * called when media is not available. The PHY initialization completed state is 2028 * set here. 2029 * 2030 * These configurations are used when setting PHY 2031 * configuration. The user PHY configuration is updated on set PHY 2032 * configuration. Returns 0 on success, negative on failure 2033 */ 2034 static int ice_init_phy_user_cfg(struct ice_port_info *pi) 2035 { 2036 struct ice_aqc_get_phy_caps_data *pcaps; 2037 struct ice_phy_info *phy = &pi->phy; 2038 struct ice_pf *pf = pi->hw->back; 2039 int err; 2040 2041 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 2042 return -EIO; 2043 2044 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 2045 if (!pcaps) 2046 return -ENOMEM; 2047 2048 if (ice_fw_supports_report_dflt_cfg(pi->hw)) 2049 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG, 2050 pcaps, NULL); 2051 else 2052 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA, 2053 pcaps, NULL); 2054 if (err) { 2055 dev_err(ice_pf_to_dev(pf), "Get PHY capability failed.\n"); 2056 goto err_out; 2057 } 2058 2059 ice_copy_phy_caps_to_cfg(pi, pcaps, &pi->phy.curr_user_phy_cfg); 2060 2061 /* check if lenient mode is supported and enabled */ 2062 if (ice_fw_supports_link_override(pi->hw) && 2063 !(pcaps->module_compliance_enforcement & 2064 ICE_AQC_MOD_ENFORCE_STRICT_MODE)) { 2065 set_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags); 2066 2067 /* if the FW supports default PHY configuration mode, then the driver 2068 * does not have to apply link override settings. If not, 2069 * initialize user PHY configuration with link override values 2070 */ 2071 if (!ice_fw_supports_report_dflt_cfg(pi->hw) && 2072 (pf->link_dflt_override.options & ICE_LINK_OVERRIDE_EN)) { 2073 ice_init_phy_cfg_dflt_override(pi); 2074 goto out; 2075 } 2076 } 2077 2078 /* if link default override is not enabled, set user flow control and 2079 * FEC settings based on what get_phy_caps returned 2080 */ 2081 phy->curr_user_fec_req = ice_caps_to_fec_mode(pcaps->caps, 2082 pcaps->link_fec_options); 2083 phy->curr_user_fc_req = ice_caps_to_fc_mode(pcaps->caps); 2084 2085 out: 2086 phy->curr_user_speed_req = ICE_AQ_LINK_SPEED_M; 2087 set_bit(ICE_PHY_INIT_COMPLETE, pf->state); 2088 err_out: 2089 kfree(pcaps); 2090 return err; 2091 } 2092 2093 /** 2094 * ice_configure_phy - configure PHY 2095 * @vsi: VSI of PHY 2096 * 2097 * Set the PHY configuration. If the current PHY configuration is the same as 2098 * the curr_user_phy_cfg, then do nothing to avoid link flap. Otherwise 2099 * configure the based get PHY capabilities for topology with media. 2100 */ 2101 static int ice_configure_phy(struct ice_vsi *vsi) 2102 { 2103 struct device *dev = ice_pf_to_dev(vsi->back); 2104 struct ice_port_info *pi = vsi->port_info; 2105 struct ice_aqc_get_phy_caps_data *pcaps; 2106 struct ice_aqc_set_phy_cfg_data *cfg; 2107 struct ice_phy_info *phy = &pi->phy; 2108 struct ice_pf *pf = vsi->back; 2109 int err; 2110 2111 /* Ensure we have media as we cannot configure a medialess port */ 2112 if (!(phy->link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) 2113 return -EPERM; 2114 2115 ice_print_topo_conflict(vsi); 2116 2117 if (!test_bit(ICE_FLAG_LINK_LENIENT_MODE_ENA, pf->flags) && 2118 phy->link_info.topo_media_conflict == ICE_AQ_LINK_TOPO_UNSUPP_MEDIA) 2119 return -EPERM; 2120 2121 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) 2122 return ice_force_phys_link_state(vsi, true); 2123 2124 pcaps = kzalloc(sizeof(*pcaps), GFP_KERNEL); 2125 if (!pcaps) 2126 return -ENOMEM; 2127 2128 /* Get current PHY config */ 2129 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG, pcaps, 2130 NULL); 2131 if (err) { 2132 dev_err(dev, "Failed to get PHY configuration, VSI %d error %d\n", 2133 vsi->vsi_num, err); 2134 goto done; 2135 } 2136 2137 /* If PHY enable link is configured and configuration has not changed, 2138 * there's nothing to do 2139 */ 2140 if (pcaps->caps & ICE_AQC_PHY_EN_LINK && 2141 ice_phy_caps_equals_cfg(pcaps, &phy->curr_user_phy_cfg)) 2142 goto done; 2143 2144 /* Use PHY topology as baseline for configuration */ 2145 memset(pcaps, 0, sizeof(*pcaps)); 2146 if (ice_fw_supports_report_dflt_cfg(pi->hw)) 2147 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_DFLT_CFG, 2148 pcaps, NULL); 2149 else 2150 err = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_TOPO_CAP_MEDIA, 2151 pcaps, NULL); 2152 if (err) { 2153 dev_err(dev, "Failed to get PHY caps, VSI %d error %d\n", 2154 vsi->vsi_num, err); 2155 goto done; 2156 } 2157 2158 cfg = kzalloc(sizeof(*cfg), GFP_KERNEL); 2159 if (!cfg) { 2160 err = -ENOMEM; 2161 goto done; 2162 } 2163 2164 ice_copy_phy_caps_to_cfg(pi, pcaps, cfg); 2165 2166 /* Speed - If default override pending, use curr_user_phy_cfg set in 2167 * ice_init_phy_user_cfg_ldo. 2168 */ 2169 if (test_and_clear_bit(ICE_LINK_DEFAULT_OVERRIDE_PENDING, 2170 vsi->back->state)) { 2171 cfg->phy_type_low = phy->curr_user_phy_cfg.phy_type_low; 2172 cfg->phy_type_high = phy->curr_user_phy_cfg.phy_type_high; 2173 } else { 2174 u64 phy_low = 0, phy_high = 0; 2175 2176 ice_update_phy_type(&phy_low, &phy_high, 2177 pi->phy.curr_user_speed_req); 2178 cfg->phy_type_low = pcaps->phy_type_low & cpu_to_le64(phy_low); 2179 cfg->phy_type_high = pcaps->phy_type_high & 2180 cpu_to_le64(phy_high); 2181 } 2182 2183 /* Can't provide what was requested; use PHY capabilities */ 2184 if (!cfg->phy_type_low && !cfg->phy_type_high) { 2185 cfg->phy_type_low = pcaps->phy_type_low; 2186 cfg->phy_type_high = pcaps->phy_type_high; 2187 } 2188 2189 /* FEC */ 2190 ice_cfg_phy_fec(pi, cfg, phy->curr_user_fec_req); 2191 2192 /* Can't provide what was requested; use PHY capabilities */ 2193 if (cfg->link_fec_opt != 2194 (cfg->link_fec_opt & pcaps->link_fec_options)) { 2195 cfg->caps |= pcaps->caps & ICE_AQC_PHY_EN_AUTO_FEC; 2196 cfg->link_fec_opt = pcaps->link_fec_options; 2197 } 2198 2199 /* Flow Control - always supported; no need to check against 2200 * capabilities 2201 */ 2202 ice_cfg_phy_fc(pi, cfg, phy->curr_user_fc_req); 2203 2204 /* Enable link and link update */ 2205 cfg->caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK; 2206 2207 err = ice_aq_set_phy_cfg(&pf->hw, pi, cfg, NULL); 2208 if (err) 2209 dev_err(dev, "Failed to set phy config, VSI %d error %d\n", 2210 vsi->vsi_num, err); 2211 2212 kfree(cfg); 2213 done: 2214 kfree(pcaps); 2215 return err; 2216 } 2217 2218 /** 2219 * ice_check_media_subtask - Check for media 2220 * @pf: pointer to PF struct 2221 * 2222 * If media is available, then initialize PHY user configuration if it is not 2223 * been, and configure the PHY if the interface is up. 2224 */ 2225 static void ice_check_media_subtask(struct ice_pf *pf) 2226 { 2227 struct ice_port_info *pi; 2228 struct ice_vsi *vsi; 2229 int err; 2230 2231 /* No need to check for media if it's already present */ 2232 if (!test_bit(ICE_FLAG_NO_MEDIA, pf->flags)) 2233 return; 2234 2235 vsi = ice_get_main_vsi(pf); 2236 if (!vsi) 2237 return; 2238 2239 /* Refresh link info and check if media is present */ 2240 pi = vsi->port_info; 2241 err = ice_update_link_info(pi); 2242 if (err) 2243 return; 2244 2245 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 2246 2247 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 2248 if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) 2249 ice_init_phy_user_cfg(pi); 2250 2251 /* PHY settings are reset on media insertion, reconfigure 2252 * PHY to preserve settings. 2253 */ 2254 if (test_bit(ICE_VSI_DOWN, vsi->state) && 2255 test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) 2256 return; 2257 2258 err = ice_configure_phy(vsi); 2259 if (!err) 2260 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 2261 2262 /* A Link Status Event will be generated; the event handler 2263 * will complete bringing the interface up 2264 */ 2265 } 2266 } 2267 2268 /** 2269 * ice_service_task - manage and run subtasks 2270 * @work: pointer to work_struct contained by the PF struct 2271 */ 2272 static void ice_service_task(struct work_struct *work) 2273 { 2274 struct ice_pf *pf = container_of(work, struct ice_pf, serv_task); 2275 unsigned long start_time = jiffies; 2276 2277 /* subtasks */ 2278 2279 /* process reset requests first */ 2280 ice_reset_subtask(pf); 2281 2282 /* bail if a reset/recovery cycle is pending or rebuild failed */ 2283 if (ice_is_reset_in_progress(pf->state) || 2284 test_bit(ICE_SUSPENDED, pf->state) || 2285 test_bit(ICE_NEEDS_RESTART, pf->state)) { 2286 ice_service_task_complete(pf); 2287 return; 2288 } 2289 2290 if (test_and_clear_bit(ICE_AUX_ERR_PENDING, pf->state)) { 2291 struct iidc_event *event; 2292 2293 event = kzalloc(sizeof(*event), GFP_KERNEL); 2294 if (event) { 2295 set_bit(IIDC_EVENT_CRIT_ERR, event->type); 2296 /* report the entire OICR value to AUX driver */ 2297 swap(event->reg, pf->oicr_err_reg); 2298 ice_send_event_to_aux(pf, event); 2299 kfree(event); 2300 } 2301 } 2302 2303 if (test_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags)) { 2304 /* Plug aux device per request */ 2305 ice_plug_aux_dev(pf); 2306 2307 /* Mark plugging as done but check whether unplug was 2308 * requested during ice_plug_aux_dev() call 2309 * (e.g. from ice_clear_rdma_cap()) and if so then 2310 * plug aux device. 2311 */ 2312 if (!test_and_clear_bit(ICE_FLAG_PLUG_AUX_DEV, pf->flags)) 2313 ice_unplug_aux_dev(pf); 2314 } 2315 2316 if (test_and_clear_bit(ICE_FLAG_MTU_CHANGED, pf->flags)) { 2317 struct iidc_event *event; 2318 2319 event = kzalloc(sizeof(*event), GFP_KERNEL); 2320 if (event) { 2321 set_bit(IIDC_EVENT_AFTER_MTU_CHANGE, event->type); 2322 ice_send_event_to_aux(pf, event); 2323 kfree(event); 2324 } 2325 } 2326 2327 ice_clean_adminq_subtask(pf); 2328 ice_check_media_subtask(pf); 2329 ice_check_for_hang_subtask(pf); 2330 ice_sync_fltr_subtask(pf); 2331 ice_handle_mdd_event(pf); 2332 ice_watchdog_subtask(pf); 2333 2334 if (ice_is_safe_mode(pf)) { 2335 ice_service_task_complete(pf); 2336 return; 2337 } 2338 2339 ice_process_vflr_event(pf); 2340 ice_clean_mailboxq_subtask(pf); 2341 ice_clean_sbq_subtask(pf); 2342 ice_sync_arfs_fltrs(pf); 2343 ice_flush_fdir_ctx(pf); 2344 2345 /* Clear ICE_SERVICE_SCHED flag to allow scheduling next event */ 2346 ice_service_task_complete(pf); 2347 2348 /* If the tasks have taken longer than one service timer period 2349 * or there is more work to be done, reset the service timer to 2350 * schedule the service task now. 2351 */ 2352 if (time_after(jiffies, (start_time + pf->serv_tmr_period)) || 2353 test_bit(ICE_MDD_EVENT_PENDING, pf->state) || 2354 test_bit(ICE_VFLR_EVENT_PENDING, pf->state) || 2355 test_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state) || 2356 test_bit(ICE_FD_VF_FLUSH_CTX, pf->state) || 2357 test_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state) || 2358 test_bit(ICE_ADMINQ_EVENT_PENDING, pf->state)) 2359 mod_timer(&pf->serv_tmr, jiffies); 2360 } 2361 2362 /** 2363 * ice_set_ctrlq_len - helper function to set controlq length 2364 * @hw: pointer to the HW instance 2365 */ 2366 static void ice_set_ctrlq_len(struct ice_hw *hw) 2367 { 2368 hw->adminq.num_rq_entries = ICE_AQ_LEN; 2369 hw->adminq.num_sq_entries = ICE_AQ_LEN; 2370 hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN; 2371 hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN; 2372 hw->mailboxq.num_rq_entries = PF_MBX_ARQLEN_ARQLEN_M; 2373 hw->mailboxq.num_sq_entries = ICE_MBXSQ_LEN; 2374 hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 2375 hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN; 2376 hw->sbq.num_rq_entries = ICE_SBQ_LEN; 2377 hw->sbq.num_sq_entries = ICE_SBQ_LEN; 2378 hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN; 2379 hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN; 2380 } 2381 2382 /** 2383 * ice_schedule_reset - schedule a reset 2384 * @pf: board private structure 2385 * @reset: reset being requested 2386 */ 2387 int ice_schedule_reset(struct ice_pf *pf, enum ice_reset_req reset) 2388 { 2389 struct device *dev = ice_pf_to_dev(pf); 2390 2391 /* bail out if earlier reset has failed */ 2392 if (test_bit(ICE_RESET_FAILED, pf->state)) { 2393 dev_dbg(dev, "earlier reset has failed\n"); 2394 return -EIO; 2395 } 2396 /* bail if reset/recovery already in progress */ 2397 if (ice_is_reset_in_progress(pf->state)) { 2398 dev_dbg(dev, "Reset already in progress\n"); 2399 return -EBUSY; 2400 } 2401 2402 ice_unplug_aux_dev(pf); 2403 2404 switch (reset) { 2405 case ICE_RESET_PFR: 2406 set_bit(ICE_PFR_REQ, pf->state); 2407 break; 2408 case ICE_RESET_CORER: 2409 set_bit(ICE_CORER_REQ, pf->state); 2410 break; 2411 case ICE_RESET_GLOBR: 2412 set_bit(ICE_GLOBR_REQ, pf->state); 2413 break; 2414 default: 2415 return -EINVAL; 2416 } 2417 2418 ice_service_task_schedule(pf); 2419 return 0; 2420 } 2421 2422 /** 2423 * ice_irq_affinity_notify - Callback for affinity changes 2424 * @notify: context as to what irq was changed 2425 * @mask: the new affinity mask 2426 * 2427 * This is a callback function used by the irq_set_affinity_notifier function 2428 * so that we may register to receive changes to the irq affinity masks. 2429 */ 2430 static void 2431 ice_irq_affinity_notify(struct irq_affinity_notify *notify, 2432 const cpumask_t *mask) 2433 { 2434 struct ice_q_vector *q_vector = 2435 container_of(notify, struct ice_q_vector, affinity_notify); 2436 2437 cpumask_copy(&q_vector->affinity_mask, mask); 2438 } 2439 2440 /** 2441 * ice_irq_affinity_release - Callback for affinity notifier release 2442 * @ref: internal core kernel usage 2443 * 2444 * This is a callback function used by the irq_set_affinity_notifier function 2445 * to inform the current notification subscriber that they will no longer 2446 * receive notifications. 2447 */ 2448 static void ice_irq_affinity_release(struct kref __always_unused *ref) {} 2449 2450 /** 2451 * ice_vsi_ena_irq - Enable IRQ for the given VSI 2452 * @vsi: the VSI being configured 2453 */ 2454 static int ice_vsi_ena_irq(struct ice_vsi *vsi) 2455 { 2456 struct ice_hw *hw = &vsi->back->hw; 2457 int i; 2458 2459 ice_for_each_q_vector(vsi, i) 2460 ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]); 2461 2462 ice_flush(hw); 2463 return 0; 2464 } 2465 2466 /** 2467 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI 2468 * @vsi: the VSI being configured 2469 * @basename: name for the vector 2470 */ 2471 static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename) 2472 { 2473 int q_vectors = vsi->num_q_vectors; 2474 struct ice_pf *pf = vsi->back; 2475 int base = vsi->base_vector; 2476 struct device *dev; 2477 int rx_int_idx = 0; 2478 int tx_int_idx = 0; 2479 int vector, err; 2480 int irq_num; 2481 2482 dev = ice_pf_to_dev(pf); 2483 for (vector = 0; vector < q_vectors; vector++) { 2484 struct ice_q_vector *q_vector = vsi->q_vectors[vector]; 2485 2486 irq_num = pf->msix_entries[base + vector].vector; 2487 2488 if (q_vector->tx.tx_ring && q_vector->rx.rx_ring) { 2489 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2490 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 2491 tx_int_idx++; 2492 } else if (q_vector->rx.rx_ring) { 2493 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2494 "%s-%s-%d", basename, "rx", rx_int_idx++); 2495 } else if (q_vector->tx.tx_ring) { 2496 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2497 "%s-%s-%d", basename, "tx", tx_int_idx++); 2498 } else { 2499 /* skip this unused q_vector */ 2500 continue; 2501 } 2502 if (vsi->type == ICE_VSI_CTRL && vsi->vf) 2503 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 2504 IRQF_SHARED, q_vector->name, 2505 q_vector); 2506 else 2507 err = devm_request_irq(dev, irq_num, vsi->irq_handler, 2508 0, q_vector->name, q_vector); 2509 if (err) { 2510 netdev_err(vsi->netdev, "MSIX request_irq failed, error: %d\n", 2511 err); 2512 goto free_q_irqs; 2513 } 2514 2515 /* register for affinity change notifications */ 2516 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) { 2517 struct irq_affinity_notify *affinity_notify; 2518 2519 affinity_notify = &q_vector->affinity_notify; 2520 affinity_notify->notify = ice_irq_affinity_notify; 2521 affinity_notify->release = ice_irq_affinity_release; 2522 irq_set_affinity_notifier(irq_num, affinity_notify); 2523 } 2524 2525 /* assign the mask for this irq */ 2526 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 2527 } 2528 2529 err = ice_set_cpu_rx_rmap(vsi); 2530 if (err) { 2531 netdev_err(vsi->netdev, "Failed to setup CPU RMAP on VSI %u: %pe\n", 2532 vsi->vsi_num, ERR_PTR(err)); 2533 goto free_q_irqs; 2534 } 2535 2536 vsi->irqs_ready = true; 2537 return 0; 2538 2539 free_q_irqs: 2540 while (vector) { 2541 vector--; 2542 irq_num = pf->msix_entries[base + vector].vector; 2543 if (!IS_ENABLED(CONFIG_RFS_ACCEL)) 2544 irq_set_affinity_notifier(irq_num, NULL); 2545 irq_set_affinity_hint(irq_num, NULL); 2546 devm_free_irq(dev, irq_num, &vsi->q_vectors[vector]); 2547 } 2548 return err; 2549 } 2550 2551 /** 2552 * ice_xdp_alloc_setup_rings - Allocate and setup Tx rings for XDP 2553 * @vsi: VSI to setup Tx rings used by XDP 2554 * 2555 * Return 0 on success and negative value on error 2556 */ 2557 static int ice_xdp_alloc_setup_rings(struct ice_vsi *vsi) 2558 { 2559 struct device *dev = ice_pf_to_dev(vsi->back); 2560 struct ice_tx_desc *tx_desc; 2561 int i, j; 2562 2563 ice_for_each_xdp_txq(vsi, i) { 2564 u16 xdp_q_idx = vsi->alloc_txq + i; 2565 struct ice_tx_ring *xdp_ring; 2566 2567 xdp_ring = kzalloc(sizeof(*xdp_ring), GFP_KERNEL); 2568 2569 if (!xdp_ring) 2570 goto free_xdp_rings; 2571 2572 xdp_ring->q_index = xdp_q_idx; 2573 xdp_ring->reg_idx = vsi->txq_map[xdp_q_idx]; 2574 xdp_ring->vsi = vsi; 2575 xdp_ring->netdev = NULL; 2576 xdp_ring->dev = dev; 2577 xdp_ring->count = vsi->num_tx_desc; 2578 xdp_ring->next_dd = ICE_RING_QUARTER(xdp_ring) - 1; 2579 xdp_ring->next_rs = ICE_RING_QUARTER(xdp_ring) - 1; 2580 WRITE_ONCE(vsi->xdp_rings[i], xdp_ring); 2581 if (ice_setup_tx_ring(xdp_ring)) 2582 goto free_xdp_rings; 2583 ice_set_ring_xdp(xdp_ring); 2584 xdp_ring->xsk_pool = ice_tx_xsk_pool(xdp_ring); 2585 spin_lock_init(&xdp_ring->tx_lock); 2586 for (j = 0; j < xdp_ring->count; j++) { 2587 tx_desc = ICE_TX_DESC(xdp_ring, j); 2588 tx_desc->cmd_type_offset_bsz = 0; 2589 } 2590 } 2591 2592 ice_for_each_rxq(vsi, i) { 2593 if (static_key_enabled(&ice_xdp_locking_key)) 2594 vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i % vsi->num_xdp_txq]; 2595 else 2596 vsi->rx_rings[i]->xdp_ring = vsi->xdp_rings[i]; 2597 } 2598 2599 return 0; 2600 2601 free_xdp_rings: 2602 for (; i >= 0; i--) 2603 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 2604 ice_free_tx_ring(vsi->xdp_rings[i]); 2605 return -ENOMEM; 2606 } 2607 2608 /** 2609 * ice_vsi_assign_bpf_prog - set or clear bpf prog pointer on VSI 2610 * @vsi: VSI to set the bpf prog on 2611 * @prog: the bpf prog pointer 2612 */ 2613 static void ice_vsi_assign_bpf_prog(struct ice_vsi *vsi, struct bpf_prog *prog) 2614 { 2615 struct bpf_prog *old_prog; 2616 int i; 2617 2618 old_prog = xchg(&vsi->xdp_prog, prog); 2619 if (old_prog) 2620 bpf_prog_put(old_prog); 2621 2622 ice_for_each_rxq(vsi, i) 2623 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 2624 } 2625 2626 /** 2627 * ice_prepare_xdp_rings - Allocate, configure and setup Tx rings for XDP 2628 * @vsi: VSI to bring up Tx rings used by XDP 2629 * @prog: bpf program that will be assigned to VSI 2630 * 2631 * Return 0 on success and negative value on error 2632 */ 2633 int ice_prepare_xdp_rings(struct ice_vsi *vsi, struct bpf_prog *prog) 2634 { 2635 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2636 int xdp_rings_rem = vsi->num_xdp_txq; 2637 struct ice_pf *pf = vsi->back; 2638 struct ice_qs_cfg xdp_qs_cfg = { 2639 .qs_mutex = &pf->avail_q_mutex, 2640 .pf_map = pf->avail_txqs, 2641 .pf_map_size = pf->max_pf_txqs, 2642 .q_count = vsi->num_xdp_txq, 2643 .scatter_count = ICE_MAX_SCATTER_TXQS, 2644 .vsi_map = vsi->txq_map, 2645 .vsi_map_offset = vsi->alloc_txq, 2646 .mapping_mode = ICE_VSI_MAP_CONTIG 2647 }; 2648 struct device *dev; 2649 int i, v_idx; 2650 int status; 2651 2652 dev = ice_pf_to_dev(pf); 2653 vsi->xdp_rings = devm_kcalloc(dev, vsi->num_xdp_txq, 2654 sizeof(*vsi->xdp_rings), GFP_KERNEL); 2655 if (!vsi->xdp_rings) 2656 return -ENOMEM; 2657 2658 vsi->xdp_mapping_mode = xdp_qs_cfg.mapping_mode; 2659 if (__ice_vsi_get_qs(&xdp_qs_cfg)) 2660 goto err_map_xdp; 2661 2662 if (static_key_enabled(&ice_xdp_locking_key)) 2663 netdev_warn(vsi->netdev, 2664 "Could not allocate one XDP Tx ring per CPU, XDP_TX/XDP_REDIRECT actions will be slower\n"); 2665 2666 if (ice_xdp_alloc_setup_rings(vsi)) 2667 goto clear_xdp_rings; 2668 2669 /* follow the logic from ice_vsi_map_rings_to_vectors */ 2670 ice_for_each_q_vector(vsi, v_idx) { 2671 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2672 int xdp_rings_per_v, q_id, q_base; 2673 2674 xdp_rings_per_v = DIV_ROUND_UP(xdp_rings_rem, 2675 vsi->num_q_vectors - v_idx); 2676 q_base = vsi->num_xdp_txq - xdp_rings_rem; 2677 2678 for (q_id = q_base; q_id < (q_base + xdp_rings_per_v); q_id++) { 2679 struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_id]; 2680 2681 xdp_ring->q_vector = q_vector; 2682 xdp_ring->next = q_vector->tx.tx_ring; 2683 q_vector->tx.tx_ring = xdp_ring; 2684 } 2685 xdp_rings_rem -= xdp_rings_per_v; 2686 } 2687 2688 /* omit the scheduler update if in reset path; XDP queues will be 2689 * taken into account at the end of ice_vsi_rebuild, where 2690 * ice_cfg_vsi_lan is being called 2691 */ 2692 if (ice_is_reset_in_progress(pf->state)) 2693 return 0; 2694 2695 /* tell the Tx scheduler that right now we have 2696 * additional queues 2697 */ 2698 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2699 max_txqs[i] = vsi->num_txq + vsi->num_xdp_txq; 2700 2701 status = ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2702 max_txqs); 2703 if (status) { 2704 dev_err(dev, "Failed VSI LAN queue config for XDP, error: %d\n", 2705 status); 2706 goto clear_xdp_rings; 2707 } 2708 2709 /* assign the prog only when it's not already present on VSI; 2710 * this flow is a subject of both ethtool -L and ndo_bpf flows; 2711 * VSI rebuild that happens under ethtool -L can expose us to 2712 * the bpf_prog refcount issues as we would be swapping same 2713 * bpf_prog pointers from vsi->xdp_prog and calling bpf_prog_put 2714 * on it as it would be treated as an 'old_prog'; for ndo_bpf 2715 * this is not harmful as dev_xdp_install bumps the refcount 2716 * before calling the op exposed by the driver; 2717 */ 2718 if (!ice_is_xdp_ena_vsi(vsi)) 2719 ice_vsi_assign_bpf_prog(vsi, prog); 2720 2721 return 0; 2722 clear_xdp_rings: 2723 ice_for_each_xdp_txq(vsi, i) 2724 if (vsi->xdp_rings[i]) { 2725 kfree_rcu(vsi->xdp_rings[i], rcu); 2726 vsi->xdp_rings[i] = NULL; 2727 } 2728 2729 err_map_xdp: 2730 mutex_lock(&pf->avail_q_mutex); 2731 ice_for_each_xdp_txq(vsi, i) { 2732 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2733 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2734 } 2735 mutex_unlock(&pf->avail_q_mutex); 2736 2737 devm_kfree(dev, vsi->xdp_rings); 2738 return -ENOMEM; 2739 } 2740 2741 /** 2742 * ice_destroy_xdp_rings - undo the configuration made by ice_prepare_xdp_rings 2743 * @vsi: VSI to remove XDP rings 2744 * 2745 * Detach XDP rings from irq vectors, clean up the PF bitmap and free 2746 * resources 2747 */ 2748 int ice_destroy_xdp_rings(struct ice_vsi *vsi) 2749 { 2750 u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 }; 2751 struct ice_pf *pf = vsi->back; 2752 int i, v_idx; 2753 2754 /* q_vectors are freed in reset path so there's no point in detaching 2755 * rings; in case of rebuild being triggered not from reset bits 2756 * in pf->state won't be set, so additionally check first q_vector 2757 * against NULL 2758 */ 2759 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2760 goto free_qmap; 2761 2762 ice_for_each_q_vector(vsi, v_idx) { 2763 struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; 2764 struct ice_tx_ring *ring; 2765 2766 ice_for_each_tx_ring(ring, q_vector->tx) 2767 if (!ring->tx_buf || !ice_ring_is_xdp(ring)) 2768 break; 2769 2770 /* restore the value of last node prior to XDP setup */ 2771 q_vector->tx.tx_ring = ring; 2772 } 2773 2774 free_qmap: 2775 mutex_lock(&pf->avail_q_mutex); 2776 ice_for_each_xdp_txq(vsi, i) { 2777 clear_bit(vsi->txq_map[i + vsi->alloc_txq], pf->avail_txqs); 2778 vsi->txq_map[i + vsi->alloc_txq] = ICE_INVAL_Q_INDEX; 2779 } 2780 mutex_unlock(&pf->avail_q_mutex); 2781 2782 ice_for_each_xdp_txq(vsi, i) 2783 if (vsi->xdp_rings[i]) { 2784 if (vsi->xdp_rings[i]->desc) { 2785 synchronize_rcu(); 2786 ice_free_tx_ring(vsi->xdp_rings[i]); 2787 } 2788 kfree_rcu(vsi->xdp_rings[i], rcu); 2789 vsi->xdp_rings[i] = NULL; 2790 } 2791 2792 devm_kfree(ice_pf_to_dev(pf), vsi->xdp_rings); 2793 vsi->xdp_rings = NULL; 2794 2795 if (static_key_enabled(&ice_xdp_locking_key)) 2796 static_branch_dec(&ice_xdp_locking_key); 2797 2798 if (ice_is_reset_in_progress(pf->state) || !vsi->q_vectors[0]) 2799 return 0; 2800 2801 ice_vsi_assign_bpf_prog(vsi, NULL); 2802 2803 /* notify Tx scheduler that we destroyed XDP queues and bring 2804 * back the old number of child nodes 2805 */ 2806 for (i = 0; i < vsi->tc_cfg.numtc; i++) 2807 max_txqs[i] = vsi->num_txq; 2808 2809 /* change number of XDP Tx queues to 0 */ 2810 vsi->num_xdp_txq = 0; 2811 2812 return ice_cfg_vsi_lan(vsi->port_info, vsi->idx, vsi->tc_cfg.ena_tc, 2813 max_txqs); 2814 } 2815 2816 /** 2817 * ice_vsi_rx_napi_schedule - Schedule napi on RX queues from VSI 2818 * @vsi: VSI to schedule napi on 2819 */ 2820 static void ice_vsi_rx_napi_schedule(struct ice_vsi *vsi) 2821 { 2822 int i; 2823 2824 ice_for_each_rxq(vsi, i) { 2825 struct ice_rx_ring *rx_ring = vsi->rx_rings[i]; 2826 2827 if (rx_ring->xsk_pool) 2828 napi_schedule(&rx_ring->q_vector->napi); 2829 } 2830 } 2831 2832 /** 2833 * ice_vsi_determine_xdp_res - figure out how many Tx qs can XDP have 2834 * @vsi: VSI to determine the count of XDP Tx qs 2835 * 2836 * returns 0 if Tx qs count is higher than at least half of CPU count, 2837 * -ENOMEM otherwise 2838 */ 2839 int ice_vsi_determine_xdp_res(struct ice_vsi *vsi) 2840 { 2841 u16 avail = ice_get_avail_txq_count(vsi->back); 2842 u16 cpus = num_possible_cpus(); 2843 2844 if (avail < cpus / 2) 2845 return -ENOMEM; 2846 2847 vsi->num_xdp_txq = min_t(u16, avail, cpus); 2848 2849 if (vsi->num_xdp_txq < cpus) 2850 static_branch_inc(&ice_xdp_locking_key); 2851 2852 return 0; 2853 } 2854 2855 /** 2856 * ice_xdp_setup_prog - Add or remove XDP eBPF program 2857 * @vsi: VSI to setup XDP for 2858 * @prog: XDP program 2859 * @extack: netlink extended ack 2860 */ 2861 static int 2862 ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, 2863 struct netlink_ext_ack *extack) 2864 { 2865 int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; 2866 bool if_running = netif_running(vsi->netdev); 2867 int ret = 0, xdp_ring_err = 0; 2868 2869 if (frame_size > vsi->rx_buf_len) { 2870 NL_SET_ERR_MSG_MOD(extack, "MTU too large for loading XDP"); 2871 return -EOPNOTSUPP; 2872 } 2873 2874 /* need to stop netdev while setting up the program for Rx rings */ 2875 if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { 2876 ret = ice_down(vsi); 2877 if (ret) { 2878 NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed"); 2879 return ret; 2880 } 2881 } 2882 2883 if (!ice_is_xdp_ena_vsi(vsi) && prog) { 2884 xdp_ring_err = ice_vsi_determine_xdp_res(vsi); 2885 if (xdp_ring_err) { 2886 NL_SET_ERR_MSG_MOD(extack, "Not enough Tx resources for XDP"); 2887 } else { 2888 xdp_ring_err = ice_prepare_xdp_rings(vsi, prog); 2889 if (xdp_ring_err) 2890 NL_SET_ERR_MSG_MOD(extack, "Setting up XDP Tx resources failed"); 2891 } 2892 } else if (ice_is_xdp_ena_vsi(vsi) && !prog) { 2893 xdp_ring_err = ice_destroy_xdp_rings(vsi); 2894 if (xdp_ring_err) 2895 NL_SET_ERR_MSG_MOD(extack, "Freeing XDP Tx resources failed"); 2896 } else { 2897 /* safe to call even when prog == vsi->xdp_prog as 2898 * dev_xdp_install in net/core/dev.c incremented prog's 2899 * refcount so corresponding bpf_prog_put won't cause 2900 * underflow 2901 */ 2902 ice_vsi_assign_bpf_prog(vsi, prog); 2903 } 2904 2905 if (if_running) 2906 ret = ice_up(vsi); 2907 2908 if (!ret && prog) 2909 ice_vsi_rx_napi_schedule(vsi); 2910 2911 return (ret || xdp_ring_err) ? -ENOMEM : 0; 2912 } 2913 2914 /** 2915 * ice_xdp_safe_mode - XDP handler for safe mode 2916 * @dev: netdevice 2917 * @xdp: XDP command 2918 */ 2919 static int ice_xdp_safe_mode(struct net_device __always_unused *dev, 2920 struct netdev_bpf *xdp) 2921 { 2922 NL_SET_ERR_MSG_MOD(xdp->extack, 2923 "Please provide working DDP firmware package in order to use XDP\n" 2924 "Refer to Documentation/networking/device_drivers/ethernet/intel/ice.rst"); 2925 return -EOPNOTSUPP; 2926 } 2927 2928 /** 2929 * ice_xdp - implements XDP handler 2930 * @dev: netdevice 2931 * @xdp: XDP command 2932 */ 2933 static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2934 { 2935 struct ice_netdev_priv *np = netdev_priv(dev); 2936 struct ice_vsi *vsi = np->vsi; 2937 2938 if (vsi->type != ICE_VSI_PF) { 2939 NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI"); 2940 return -EINVAL; 2941 } 2942 2943 switch (xdp->command) { 2944 case XDP_SETUP_PROG: 2945 return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); 2946 case XDP_SETUP_XSK_POOL: 2947 return ice_xsk_pool_setup(vsi, xdp->xsk.pool, 2948 xdp->xsk.queue_id); 2949 default: 2950 return -EINVAL; 2951 } 2952 } 2953 2954 /** 2955 * ice_ena_misc_vector - enable the non-queue interrupts 2956 * @pf: board private structure 2957 */ 2958 static void ice_ena_misc_vector(struct ice_pf *pf) 2959 { 2960 struct ice_hw *hw = &pf->hw; 2961 u32 val; 2962 2963 /* Disable anti-spoof detection interrupt to prevent spurious event 2964 * interrupts during a function reset. Anti-spoof functionally is 2965 * still supported. 2966 */ 2967 val = rd32(hw, GL_MDCK_TX_TDPU); 2968 val |= GL_MDCK_TX_TDPU_RCU_ANTISPOOF_ITR_DIS_M; 2969 wr32(hw, GL_MDCK_TX_TDPU, val); 2970 2971 /* clear things first */ 2972 wr32(hw, PFINT_OICR_ENA, 0); /* disable all */ 2973 rd32(hw, PFINT_OICR); /* read to clear */ 2974 2975 val = (PFINT_OICR_ECC_ERR_M | 2976 PFINT_OICR_MAL_DETECT_M | 2977 PFINT_OICR_GRST_M | 2978 PFINT_OICR_PCI_EXCEPTION_M | 2979 PFINT_OICR_VFLR_M | 2980 PFINT_OICR_HMC_ERR_M | 2981 PFINT_OICR_PE_PUSH_M | 2982 PFINT_OICR_PE_CRITERR_M); 2983 2984 wr32(hw, PFINT_OICR_ENA, val); 2985 2986 /* SW_ITR_IDX = 0, but don't change INTENA */ 2987 wr32(hw, GLINT_DYN_CTL(pf->oicr_idx), 2988 GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M); 2989 } 2990 2991 /** 2992 * ice_misc_intr - misc interrupt handler 2993 * @irq: interrupt number 2994 * @data: pointer to a q_vector 2995 */ 2996 static irqreturn_t ice_misc_intr(int __always_unused irq, void *data) 2997 { 2998 struct ice_pf *pf = (struct ice_pf *)data; 2999 struct ice_hw *hw = &pf->hw; 3000 irqreturn_t ret = IRQ_NONE; 3001 struct device *dev; 3002 u32 oicr, ena_mask; 3003 3004 dev = ice_pf_to_dev(pf); 3005 set_bit(ICE_ADMINQ_EVENT_PENDING, pf->state); 3006 set_bit(ICE_MAILBOXQ_EVENT_PENDING, pf->state); 3007 set_bit(ICE_SIDEBANDQ_EVENT_PENDING, pf->state); 3008 3009 oicr = rd32(hw, PFINT_OICR); 3010 ena_mask = rd32(hw, PFINT_OICR_ENA); 3011 3012 if (oicr & PFINT_OICR_SWINT_M) { 3013 ena_mask &= ~PFINT_OICR_SWINT_M; 3014 pf->sw_int_count++; 3015 } 3016 3017 if (oicr & PFINT_OICR_MAL_DETECT_M) { 3018 ena_mask &= ~PFINT_OICR_MAL_DETECT_M; 3019 set_bit(ICE_MDD_EVENT_PENDING, pf->state); 3020 } 3021 if (oicr & PFINT_OICR_VFLR_M) { 3022 /* disable any further VFLR event notifications */ 3023 if (test_bit(ICE_VF_RESETS_DISABLED, pf->state)) { 3024 u32 reg = rd32(hw, PFINT_OICR_ENA); 3025 3026 reg &= ~PFINT_OICR_VFLR_M; 3027 wr32(hw, PFINT_OICR_ENA, reg); 3028 } else { 3029 ena_mask &= ~PFINT_OICR_VFLR_M; 3030 set_bit(ICE_VFLR_EVENT_PENDING, pf->state); 3031 } 3032 } 3033 3034 if (oicr & PFINT_OICR_GRST_M) { 3035 u32 reset; 3036 3037 /* we have a reset warning */ 3038 ena_mask &= ~PFINT_OICR_GRST_M; 3039 reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >> 3040 GLGEN_RSTAT_RESET_TYPE_S; 3041 3042 if (reset == ICE_RESET_CORER) 3043 pf->corer_count++; 3044 else if (reset == ICE_RESET_GLOBR) 3045 pf->globr_count++; 3046 else if (reset == ICE_RESET_EMPR) 3047 pf->empr_count++; 3048 else 3049 dev_dbg(dev, "Invalid reset type %d\n", reset); 3050 3051 /* If a reset cycle isn't already in progress, we set a bit in 3052 * pf->state so that the service task can start a reset/rebuild. 3053 */ 3054 if (!test_and_set_bit(ICE_RESET_OICR_RECV, pf->state)) { 3055 if (reset == ICE_RESET_CORER) 3056 set_bit(ICE_CORER_RECV, pf->state); 3057 else if (reset == ICE_RESET_GLOBR) 3058 set_bit(ICE_GLOBR_RECV, pf->state); 3059 else 3060 set_bit(ICE_EMPR_RECV, pf->state); 3061 3062 /* There are couple of different bits at play here. 3063 * hw->reset_ongoing indicates whether the hardware is 3064 * in reset. This is set to true when a reset interrupt 3065 * is received and set back to false after the driver 3066 * has determined that the hardware is out of reset. 3067 * 3068 * ICE_RESET_OICR_RECV in pf->state indicates 3069 * that a post reset rebuild is required before the 3070 * driver is operational again. This is set above. 3071 * 3072 * As this is the start of the reset/rebuild cycle, set 3073 * both to indicate that. 3074 */ 3075 hw->reset_ongoing = true; 3076 } 3077 } 3078 3079 if (oicr & PFINT_OICR_TSYN_TX_M) { 3080 ena_mask &= ~PFINT_OICR_TSYN_TX_M; 3081 ice_ptp_process_ts(pf); 3082 } 3083 3084 if (oicr & PFINT_OICR_TSYN_EVNT_M) { 3085 u8 tmr_idx = hw->func_caps.ts_func_info.tmr_index_owned; 3086 u32 gltsyn_stat = rd32(hw, GLTSYN_STAT(tmr_idx)); 3087 3088 /* Save EVENTs from GTSYN register */ 3089 pf->ptp.ext_ts_irq |= gltsyn_stat & (GLTSYN_STAT_EVENT0_M | 3090 GLTSYN_STAT_EVENT1_M | 3091 GLTSYN_STAT_EVENT2_M); 3092 ena_mask &= ~PFINT_OICR_TSYN_EVNT_M; 3093 kthread_queue_work(pf->ptp.kworker, &pf->ptp.extts_work); 3094 } 3095 3096 #define ICE_AUX_CRIT_ERR (PFINT_OICR_PE_CRITERR_M | PFINT_OICR_HMC_ERR_M | PFINT_OICR_PE_PUSH_M) 3097 if (oicr & ICE_AUX_CRIT_ERR) { 3098 pf->oicr_err_reg |= oicr; 3099 set_bit(ICE_AUX_ERR_PENDING, pf->state); 3100 ena_mask &= ~ICE_AUX_CRIT_ERR; 3101 } 3102 3103 /* Report any remaining unexpected interrupts */ 3104 oicr &= ena_mask; 3105 if (oicr) { 3106 dev_dbg(dev, "unhandled interrupt oicr=0x%08x\n", oicr); 3107 /* If a critical error is pending there is no choice but to 3108 * reset the device. 3109 */ 3110 if (oicr & (PFINT_OICR_PCI_EXCEPTION_M | 3111 PFINT_OICR_ECC_ERR_M)) { 3112 set_bit(ICE_PFR_REQ, pf->state); 3113 ice_service_task_schedule(pf); 3114 } 3115 } 3116 ret = IRQ_HANDLED; 3117 3118 ice_service_task_schedule(pf); 3119 ice_irq_dynamic_ena(hw, NULL, NULL); 3120 3121 return ret; 3122 } 3123 3124 /** 3125 * ice_dis_ctrlq_interrupts - disable control queue interrupts 3126 * @hw: pointer to HW structure 3127 */ 3128 static void ice_dis_ctrlq_interrupts(struct ice_hw *hw) 3129 { 3130 /* disable Admin queue Interrupt causes */ 3131 wr32(hw, PFINT_FW_CTL, 3132 rd32(hw, PFINT_FW_CTL) & ~PFINT_FW_CTL_CAUSE_ENA_M); 3133 3134 /* disable Mailbox queue Interrupt causes */ 3135 wr32(hw, PFINT_MBX_CTL, 3136 rd32(hw, PFINT_MBX_CTL) & ~PFINT_MBX_CTL_CAUSE_ENA_M); 3137 3138 wr32(hw, PFINT_SB_CTL, 3139 rd32(hw, PFINT_SB_CTL) & ~PFINT_SB_CTL_CAUSE_ENA_M); 3140 3141 /* disable Control queue Interrupt causes */ 3142 wr32(hw, PFINT_OICR_CTL, 3143 rd32(hw, PFINT_OICR_CTL) & ~PFINT_OICR_CTL_CAUSE_ENA_M); 3144 3145 ice_flush(hw); 3146 } 3147 3148 /** 3149 * ice_free_irq_msix_misc - Unroll misc vector setup 3150 * @pf: board private structure 3151 */ 3152 static void ice_free_irq_msix_misc(struct ice_pf *pf) 3153 { 3154 struct ice_hw *hw = &pf->hw; 3155 3156 ice_dis_ctrlq_interrupts(hw); 3157 3158 /* disable OICR interrupt */ 3159 wr32(hw, PFINT_OICR_ENA, 0); 3160 ice_flush(hw); 3161 3162 if (pf->msix_entries) { 3163 synchronize_irq(pf->msix_entries[pf->oicr_idx].vector); 3164 devm_free_irq(ice_pf_to_dev(pf), 3165 pf->msix_entries[pf->oicr_idx].vector, pf); 3166 } 3167 3168 pf->num_avail_sw_msix += 1; 3169 ice_free_res(pf->irq_tracker, pf->oicr_idx, ICE_RES_MISC_VEC_ID); 3170 } 3171 3172 /** 3173 * ice_ena_ctrlq_interrupts - enable control queue interrupts 3174 * @hw: pointer to HW structure 3175 * @reg_idx: HW vector index to associate the control queue interrupts with 3176 */ 3177 static void ice_ena_ctrlq_interrupts(struct ice_hw *hw, u16 reg_idx) 3178 { 3179 u32 val; 3180 3181 val = ((reg_idx & PFINT_OICR_CTL_MSIX_INDX_M) | 3182 PFINT_OICR_CTL_CAUSE_ENA_M); 3183 wr32(hw, PFINT_OICR_CTL, val); 3184 3185 /* enable Admin queue Interrupt causes */ 3186 val = ((reg_idx & PFINT_FW_CTL_MSIX_INDX_M) | 3187 PFINT_FW_CTL_CAUSE_ENA_M); 3188 wr32(hw, PFINT_FW_CTL, val); 3189 3190 /* enable Mailbox queue Interrupt causes */ 3191 val = ((reg_idx & PFINT_MBX_CTL_MSIX_INDX_M) | 3192 PFINT_MBX_CTL_CAUSE_ENA_M); 3193 wr32(hw, PFINT_MBX_CTL, val); 3194 3195 /* This enables Sideband queue Interrupt causes */ 3196 val = ((reg_idx & PFINT_SB_CTL_MSIX_INDX_M) | 3197 PFINT_SB_CTL_CAUSE_ENA_M); 3198 wr32(hw, PFINT_SB_CTL, val); 3199 3200 ice_flush(hw); 3201 } 3202 3203 /** 3204 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events 3205 * @pf: board private structure 3206 * 3207 * This sets up the handler for MSIX 0, which is used to manage the 3208 * non-queue interrupts, e.g. AdminQ and errors. This is not used 3209 * when in MSI or Legacy interrupt mode. 3210 */ 3211 static int ice_req_irq_msix_misc(struct ice_pf *pf) 3212 { 3213 struct device *dev = ice_pf_to_dev(pf); 3214 struct ice_hw *hw = &pf->hw; 3215 int oicr_idx, err = 0; 3216 3217 if (!pf->int_name[0]) 3218 snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc", 3219 dev_driver_string(dev), dev_name(dev)); 3220 3221 /* Do not request IRQ but do enable OICR interrupt since settings are 3222 * lost during reset. Note that this function is called only during 3223 * rebuild path and not while reset is in progress. 3224 */ 3225 if (ice_is_reset_in_progress(pf->state)) 3226 goto skip_req_irq; 3227 3228 /* reserve one vector in irq_tracker for misc interrupts */ 3229 oicr_idx = ice_get_res(pf, pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); 3230 if (oicr_idx < 0) 3231 return oicr_idx; 3232 3233 pf->num_avail_sw_msix -= 1; 3234 pf->oicr_idx = (u16)oicr_idx; 3235 3236 err = devm_request_irq(dev, pf->msix_entries[pf->oicr_idx].vector, 3237 ice_misc_intr, 0, pf->int_name, pf); 3238 if (err) { 3239 dev_err(dev, "devm_request_irq for %s failed: %d\n", 3240 pf->int_name, err); 3241 ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID); 3242 pf->num_avail_sw_msix += 1; 3243 return err; 3244 } 3245 3246 skip_req_irq: 3247 ice_ena_misc_vector(pf); 3248 3249 ice_ena_ctrlq_interrupts(hw, pf->oicr_idx); 3250 wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_idx), 3251 ITR_REG_ALIGN(ICE_ITR_8K) >> ICE_ITR_GRAN_S); 3252 3253 ice_flush(hw); 3254 ice_irq_dynamic_ena(hw, NULL, NULL); 3255 3256 return 0; 3257 } 3258 3259 /** 3260 * ice_napi_add - register NAPI handler for the VSI 3261 * @vsi: VSI for which NAPI handler is to be registered 3262 * 3263 * This function is only called in the driver's load path. Registering the NAPI 3264 * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, 3265 * reset/rebuild, etc.) 3266 */ 3267 static void ice_napi_add(struct ice_vsi *vsi) 3268 { 3269 int v_idx; 3270 3271 if (!vsi->netdev) 3272 return; 3273 3274 ice_for_each_q_vector(vsi, v_idx) 3275 netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, 3276 ice_napi_poll, NAPI_POLL_WEIGHT); 3277 } 3278 3279 /** 3280 * ice_set_ops - set netdev and ethtools ops for the given netdev 3281 * @netdev: netdev instance 3282 */ 3283 static void ice_set_ops(struct net_device *netdev) 3284 { 3285 struct ice_pf *pf = ice_netdev_to_pf(netdev); 3286 3287 if (ice_is_safe_mode(pf)) { 3288 netdev->netdev_ops = &ice_netdev_safe_mode_ops; 3289 ice_set_ethtool_safe_mode_ops(netdev); 3290 return; 3291 } 3292 3293 netdev->netdev_ops = &ice_netdev_ops; 3294 netdev->udp_tunnel_nic_info = &pf->hw.udp_tunnel_nic; 3295 ice_set_ethtool_ops(netdev); 3296 } 3297 3298 /** 3299 * ice_set_netdev_features - set features for the given netdev 3300 * @netdev: netdev instance 3301 */ 3302 static void ice_set_netdev_features(struct net_device *netdev) 3303 { 3304 struct ice_pf *pf = ice_netdev_to_pf(netdev); 3305 bool is_dvm_ena = ice_is_dvm_ena(&pf->hw); 3306 netdev_features_t csumo_features; 3307 netdev_features_t vlano_features; 3308 netdev_features_t dflt_features; 3309 netdev_features_t tso_features; 3310 3311 if (ice_is_safe_mode(pf)) { 3312 /* safe mode */ 3313 netdev->features = NETIF_F_SG | NETIF_F_HIGHDMA; 3314 netdev->hw_features = netdev->features; 3315 return; 3316 } 3317 3318 dflt_features = NETIF_F_SG | 3319 NETIF_F_HIGHDMA | 3320 NETIF_F_NTUPLE | 3321 NETIF_F_RXHASH; 3322 3323 csumo_features = NETIF_F_RXCSUM | 3324 NETIF_F_IP_CSUM | 3325 NETIF_F_SCTP_CRC | 3326 NETIF_F_IPV6_CSUM; 3327 3328 vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER | 3329 NETIF_F_HW_VLAN_CTAG_TX | 3330 NETIF_F_HW_VLAN_CTAG_RX; 3331 3332 /* Enable CTAG/STAG filtering by default in Double VLAN Mode (DVM) */ 3333 if (is_dvm_ena) 3334 vlano_features |= NETIF_F_HW_VLAN_STAG_FILTER; 3335 3336 tso_features = NETIF_F_TSO | 3337 NETIF_F_TSO_ECN | 3338 NETIF_F_TSO6 | 3339 NETIF_F_GSO_GRE | 3340 NETIF_F_GSO_UDP_TUNNEL | 3341 NETIF_F_GSO_GRE_CSUM | 3342 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3343 NETIF_F_GSO_PARTIAL | 3344 NETIF_F_GSO_IPXIP4 | 3345 NETIF_F_GSO_IPXIP6 | 3346 NETIF_F_GSO_UDP_L4; 3347 3348 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM | 3349 NETIF_F_GSO_GRE_CSUM; 3350 /* set features that user can change */ 3351 netdev->hw_features = dflt_features | csumo_features | 3352 vlano_features | tso_features; 3353 3354 /* add support for HW_CSUM on packets with MPLS header */ 3355 netdev->mpls_features = NETIF_F_HW_CSUM | 3356 NETIF_F_TSO | 3357 NETIF_F_TSO6; 3358 3359 /* enable features */ 3360 netdev->features |= netdev->hw_features; 3361 3362 netdev->hw_features |= NETIF_F_HW_TC; 3363 netdev->hw_features |= NETIF_F_LOOPBACK; 3364 3365 /* encap and VLAN devices inherit default, csumo and tso features */ 3366 netdev->hw_enc_features |= dflt_features | csumo_features | 3367 tso_features; 3368 netdev->vlan_features |= dflt_features | csumo_features | 3369 tso_features; 3370 3371 /* advertise support but don't enable by default since only one type of 3372 * VLAN offload can be enabled at a time (i.e. CTAG or STAG). When one 3373 * type turns on the other has to be turned off. This is enforced by the 3374 * ice_fix_features() ndo callback. 3375 */ 3376 if (is_dvm_ena) 3377 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_RX | 3378 NETIF_F_HW_VLAN_STAG_TX; 3379 3380 /* Leave CRC / FCS stripping enabled by default, but allow the value to 3381 * be changed at runtime 3382 */ 3383 netdev->hw_features |= NETIF_F_RXFCS; 3384 } 3385 3386 /** 3387 * ice_cfg_netdev - Allocate, configure and register a netdev 3388 * @vsi: the VSI associated with the new netdev 3389 * 3390 * Returns 0 on success, negative value on failure 3391 */ 3392 static int ice_cfg_netdev(struct ice_vsi *vsi) 3393 { 3394 struct ice_netdev_priv *np; 3395 struct net_device *netdev; 3396 u8 mac_addr[ETH_ALEN]; 3397 3398 netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, 3399 vsi->alloc_rxq); 3400 if (!netdev) 3401 return -ENOMEM; 3402 3403 set_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); 3404 vsi->netdev = netdev; 3405 np = netdev_priv(netdev); 3406 np->vsi = vsi; 3407 3408 ice_set_netdev_features(netdev); 3409 3410 ice_set_ops(netdev); 3411 3412 if (vsi->type == ICE_VSI_PF) { 3413 SET_NETDEV_DEV(netdev, ice_pf_to_dev(vsi->back)); 3414 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 3415 eth_hw_addr_set(netdev, mac_addr); 3416 ether_addr_copy(netdev->perm_addr, mac_addr); 3417 } 3418 3419 netdev->priv_flags |= IFF_UNICAST_FLT; 3420 3421 /* Setup netdev TC information */ 3422 ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); 3423 3424 /* setup watchdog timeout value to be 5 second */ 3425 netdev->watchdog_timeo = 5 * HZ; 3426 3427 netdev->min_mtu = ETH_MIN_MTU; 3428 netdev->max_mtu = ICE_MAX_MTU; 3429 3430 return 0; 3431 } 3432 3433 /** 3434 * ice_fill_rss_lut - Fill the RSS lookup table with default values 3435 * @lut: Lookup table 3436 * @rss_table_size: Lookup table size 3437 * @rss_size: Range of queue number for hashing 3438 */ 3439 void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size) 3440 { 3441 u16 i; 3442 3443 for (i = 0; i < rss_table_size; i++) 3444 lut[i] = i % rss_size; 3445 } 3446 3447 /** 3448 * ice_pf_vsi_setup - Set up a PF VSI 3449 * @pf: board private structure 3450 * @pi: pointer to the port_info instance 3451 * 3452 * Returns pointer to the successfully allocated VSI software struct 3453 * on success, otherwise returns NULL on failure. 3454 */ 3455 static struct ice_vsi * 3456 ice_pf_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3457 { 3458 return ice_vsi_setup(pf, pi, ICE_VSI_PF, NULL, NULL); 3459 } 3460 3461 static struct ice_vsi * 3462 ice_chnl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi, 3463 struct ice_channel *ch) 3464 { 3465 return ice_vsi_setup(pf, pi, ICE_VSI_CHNL, NULL, ch); 3466 } 3467 3468 /** 3469 * ice_ctrl_vsi_setup - Set up a control VSI 3470 * @pf: board private structure 3471 * @pi: pointer to the port_info instance 3472 * 3473 * Returns pointer to the successfully allocated VSI software struct 3474 * on success, otherwise returns NULL on failure. 3475 */ 3476 static struct ice_vsi * 3477 ice_ctrl_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3478 { 3479 return ice_vsi_setup(pf, pi, ICE_VSI_CTRL, NULL, NULL); 3480 } 3481 3482 /** 3483 * ice_lb_vsi_setup - Set up a loopback VSI 3484 * @pf: board private structure 3485 * @pi: pointer to the port_info instance 3486 * 3487 * Returns pointer to the successfully allocated VSI software struct 3488 * on success, otherwise returns NULL on failure. 3489 */ 3490 struct ice_vsi * 3491 ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) 3492 { 3493 return ice_vsi_setup(pf, pi, ICE_VSI_LB, NULL, NULL); 3494 } 3495 3496 /** 3497 * ice_vlan_rx_add_vid - Add a VLAN ID filter to HW offload 3498 * @netdev: network interface to be adjusted 3499 * @proto: VLAN TPID 3500 * @vid: VLAN ID to be added 3501 * 3502 * net_device_ops implementation for adding VLAN IDs 3503 */ 3504 static int 3505 ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 3506 { 3507 struct ice_netdev_priv *np = netdev_priv(netdev); 3508 struct ice_vsi_vlan_ops *vlan_ops; 3509 struct ice_vsi *vsi = np->vsi; 3510 struct ice_vlan vlan; 3511 int ret; 3512 3513 /* VLAN 0 is added by default during load/reset */ 3514 if (!vid) 3515 return 0; 3516 3517 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 3518 usleep_range(1000, 2000); 3519 3520 /* Add multicast promisc rule for the VLAN ID to be added if 3521 * all-multicast is currently enabled. 3522 */ 3523 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 3524 ret = ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3525 ICE_MCAST_VLAN_PROMISC_BITS, 3526 vid); 3527 if (ret) 3528 goto finish; 3529 } 3530 3531 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 3532 3533 /* Add a switch rule for this VLAN ID so its corresponding VLAN tagged 3534 * packets aren't pruned by the device's internal switch on Rx 3535 */ 3536 vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0); 3537 ret = vlan_ops->add_vlan(vsi, &vlan); 3538 if (ret) 3539 goto finish; 3540 3541 /* If all-multicast is currently enabled and this VLAN ID is only one 3542 * besides VLAN-0 we have to update look-up type of multicast promisc 3543 * rule for VLAN-0 from ICE_SW_LKUP_PROMISC to ICE_SW_LKUP_PROMISC_VLAN. 3544 */ 3545 if ((vsi->current_netdev_flags & IFF_ALLMULTI) && 3546 ice_vsi_num_non_zero_vlans(vsi) == 1) { 3547 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3548 ICE_MCAST_PROMISC_BITS, 0); 3549 ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3550 ICE_MCAST_VLAN_PROMISC_BITS, 0); 3551 } 3552 3553 finish: 3554 clear_bit(ICE_CFG_BUSY, vsi->state); 3555 3556 return ret; 3557 } 3558 3559 /** 3560 * ice_vlan_rx_kill_vid - Remove a VLAN ID filter from HW offload 3561 * @netdev: network interface to be adjusted 3562 * @proto: VLAN TPID 3563 * @vid: VLAN ID to be removed 3564 * 3565 * net_device_ops implementation for removing VLAN IDs 3566 */ 3567 static int 3568 ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) 3569 { 3570 struct ice_netdev_priv *np = netdev_priv(netdev); 3571 struct ice_vsi_vlan_ops *vlan_ops; 3572 struct ice_vsi *vsi = np->vsi; 3573 struct ice_vlan vlan; 3574 int ret; 3575 3576 /* don't allow removal of VLAN 0 */ 3577 if (!vid) 3578 return 0; 3579 3580 while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) 3581 usleep_range(1000, 2000); 3582 3583 ret = ice_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3584 ICE_MCAST_VLAN_PROMISC_BITS, vid); 3585 if (ret) { 3586 netdev_err(netdev, "Error clearing multicast promiscuous mode on VSI %i\n", 3587 vsi->vsi_num); 3588 vsi->current_netdev_flags |= IFF_ALLMULTI; 3589 } 3590 3591 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 3592 3593 /* Make sure VLAN delete is successful before updating VLAN 3594 * information 3595 */ 3596 vlan = ICE_VLAN(be16_to_cpu(proto), vid, 0); 3597 ret = vlan_ops->del_vlan(vsi, &vlan); 3598 if (ret) 3599 goto finish; 3600 3601 /* Remove multicast promisc rule for the removed VLAN ID if 3602 * all-multicast is enabled. 3603 */ 3604 if (vsi->current_netdev_flags & IFF_ALLMULTI) 3605 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3606 ICE_MCAST_VLAN_PROMISC_BITS, vid); 3607 3608 if (!ice_vsi_has_non_zero_vlans(vsi)) { 3609 /* Update look-up type of multicast promisc rule for VLAN 0 3610 * from ICE_SW_LKUP_PROMISC_VLAN to ICE_SW_LKUP_PROMISC when 3611 * all-multicast is enabled and VLAN 0 is the only VLAN rule. 3612 */ 3613 if (vsi->current_netdev_flags & IFF_ALLMULTI) { 3614 ice_fltr_clear_vsi_promisc(&vsi->back->hw, vsi->idx, 3615 ICE_MCAST_VLAN_PROMISC_BITS, 3616 0); 3617 ice_fltr_set_vsi_promisc(&vsi->back->hw, vsi->idx, 3618 ICE_MCAST_PROMISC_BITS, 0); 3619 } 3620 } 3621 3622 finish: 3623 clear_bit(ICE_CFG_BUSY, vsi->state); 3624 3625 return ret; 3626 } 3627 3628 /** 3629 * ice_rep_indr_tc_block_unbind 3630 * @cb_priv: indirection block private data 3631 */ 3632 static void ice_rep_indr_tc_block_unbind(void *cb_priv) 3633 { 3634 struct ice_indr_block_priv *indr_priv = cb_priv; 3635 3636 list_del(&indr_priv->list); 3637 kfree(indr_priv); 3638 } 3639 3640 /** 3641 * ice_tc_indir_block_unregister - Unregister TC indirect block notifications 3642 * @vsi: VSI struct which has the netdev 3643 */ 3644 static void ice_tc_indir_block_unregister(struct ice_vsi *vsi) 3645 { 3646 struct ice_netdev_priv *np = netdev_priv(vsi->netdev); 3647 3648 flow_indr_dev_unregister(ice_indr_setup_tc_cb, np, 3649 ice_rep_indr_tc_block_unbind); 3650 } 3651 3652 /** 3653 * ice_tc_indir_block_remove - clean indirect TC block notifications 3654 * @pf: PF structure 3655 */ 3656 static void ice_tc_indir_block_remove(struct ice_pf *pf) 3657 { 3658 struct ice_vsi *pf_vsi = ice_get_main_vsi(pf); 3659 3660 if (!pf_vsi) 3661 return; 3662 3663 ice_tc_indir_block_unregister(pf_vsi); 3664 } 3665 3666 /** 3667 * ice_tc_indir_block_register - Register TC indirect block notifications 3668 * @vsi: VSI struct which has the netdev 3669 * 3670 * Returns 0 on success, negative value on failure 3671 */ 3672 static int ice_tc_indir_block_register(struct ice_vsi *vsi) 3673 { 3674 struct ice_netdev_priv *np; 3675 3676 if (!vsi || !vsi->netdev) 3677 return -EINVAL; 3678 3679 np = netdev_priv(vsi->netdev); 3680 3681 INIT_LIST_HEAD(&np->tc_indr_block_priv_list); 3682 return flow_indr_dev_register(ice_indr_setup_tc_cb, np); 3683 } 3684 3685 /** 3686 * ice_setup_pf_sw - Setup the HW switch on startup or after reset 3687 * @pf: board private structure 3688 * 3689 * Returns 0 on success, negative value on failure 3690 */ 3691 static int ice_setup_pf_sw(struct ice_pf *pf) 3692 { 3693 struct device *dev = ice_pf_to_dev(pf); 3694 bool dvm = ice_is_dvm_ena(&pf->hw); 3695 struct ice_vsi *vsi; 3696 int status; 3697 3698 if (ice_is_reset_in_progress(pf->state)) 3699 return -EBUSY; 3700 3701 status = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL); 3702 if (status) 3703 return -EIO; 3704 3705 vsi = ice_pf_vsi_setup(pf, pf->hw.port_info); 3706 if (!vsi) 3707 return -ENOMEM; 3708 3709 /* init channel list */ 3710 INIT_LIST_HEAD(&vsi->ch_list); 3711 3712 status = ice_cfg_netdev(vsi); 3713 if (status) 3714 goto unroll_vsi_setup; 3715 /* netdev has to be configured before setting frame size */ 3716 ice_vsi_cfg_frame_size(vsi); 3717 3718 /* init indirect block notifications */ 3719 status = ice_tc_indir_block_register(vsi); 3720 if (status) { 3721 dev_err(dev, "Failed to register netdev notifier\n"); 3722 goto unroll_cfg_netdev; 3723 } 3724 3725 /* Setup DCB netlink interface */ 3726 ice_dcbnl_setup(vsi); 3727 3728 /* registering the NAPI handler requires both the queues and 3729 * netdev to be created, which are done in ice_pf_vsi_setup() 3730 * and ice_cfg_netdev() respectively 3731 */ 3732 ice_napi_add(vsi); 3733 3734 status = ice_init_mac_fltr(pf); 3735 if (status) 3736 goto unroll_napi_add; 3737 3738 return 0; 3739 3740 unroll_napi_add: 3741 ice_tc_indir_block_unregister(vsi); 3742 unroll_cfg_netdev: 3743 if (vsi) { 3744 ice_napi_del(vsi); 3745 if (vsi->netdev) { 3746 clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); 3747 free_netdev(vsi->netdev); 3748 vsi->netdev = NULL; 3749 } 3750 } 3751 3752 unroll_vsi_setup: 3753 ice_vsi_release(vsi); 3754 return status; 3755 } 3756 3757 /** 3758 * ice_get_avail_q_count - Get count of queues in use 3759 * @pf_qmap: bitmap to get queue use count from 3760 * @lock: pointer to a mutex that protects access to pf_qmap 3761 * @size: size of the bitmap 3762 */ 3763 static u16 3764 ice_get_avail_q_count(unsigned long *pf_qmap, struct mutex *lock, u16 size) 3765 { 3766 unsigned long bit; 3767 u16 count = 0; 3768 3769 mutex_lock(lock); 3770 for_each_clear_bit(bit, pf_qmap, size) 3771 count++; 3772 mutex_unlock(lock); 3773 3774 return count; 3775 } 3776 3777 /** 3778 * ice_get_avail_txq_count - Get count of Tx queues in use 3779 * @pf: pointer to an ice_pf instance 3780 */ 3781 u16 ice_get_avail_txq_count(struct ice_pf *pf) 3782 { 3783 return ice_get_avail_q_count(pf->avail_txqs, &pf->avail_q_mutex, 3784 pf->max_pf_txqs); 3785 } 3786 3787 /** 3788 * ice_get_avail_rxq_count - Get count of Rx queues in use 3789 * @pf: pointer to an ice_pf instance 3790 */ 3791 u16 ice_get_avail_rxq_count(struct ice_pf *pf) 3792 { 3793 return ice_get_avail_q_count(pf->avail_rxqs, &pf->avail_q_mutex, 3794 pf->max_pf_rxqs); 3795 } 3796 3797 /** 3798 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf 3799 * @pf: board private structure to initialize 3800 */ 3801 static void ice_deinit_pf(struct ice_pf *pf) 3802 { 3803 ice_service_task_stop(pf); 3804 mutex_destroy(&pf->adev_mutex); 3805 mutex_destroy(&pf->sw_mutex); 3806 mutex_destroy(&pf->tc_mutex); 3807 mutex_destroy(&pf->avail_q_mutex); 3808 mutex_destroy(&pf->vfs.table_lock); 3809 3810 if (pf->avail_txqs) { 3811 bitmap_free(pf->avail_txqs); 3812 pf->avail_txqs = NULL; 3813 } 3814 3815 if (pf->avail_rxqs) { 3816 bitmap_free(pf->avail_rxqs); 3817 pf->avail_rxqs = NULL; 3818 } 3819 3820 if (pf->ptp.clock) 3821 ptp_clock_unregister(pf->ptp.clock); 3822 } 3823 3824 /** 3825 * ice_set_pf_caps - set PFs capability flags 3826 * @pf: pointer to the PF instance 3827 */ 3828 static void ice_set_pf_caps(struct ice_pf *pf) 3829 { 3830 struct ice_hw_func_caps *func_caps = &pf->hw.func_caps; 3831 3832 clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); 3833 if (func_caps->common_cap.rdma) 3834 set_bit(ICE_FLAG_RDMA_ENA, pf->flags); 3835 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3836 if (func_caps->common_cap.dcb) 3837 set_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 3838 clear_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3839 if (func_caps->common_cap.sr_iov_1_1) { 3840 set_bit(ICE_FLAG_SRIOV_CAPABLE, pf->flags); 3841 pf->vfs.num_supported = min_t(int, func_caps->num_allocd_vfs, 3842 ICE_MAX_SRIOV_VFS); 3843 } 3844 clear_bit(ICE_FLAG_RSS_ENA, pf->flags); 3845 if (func_caps->common_cap.rss_table_size) 3846 set_bit(ICE_FLAG_RSS_ENA, pf->flags); 3847 3848 clear_bit(ICE_FLAG_FD_ENA, pf->flags); 3849 if (func_caps->fd_fltr_guar > 0 || func_caps->fd_fltr_best_effort > 0) { 3850 u16 unused; 3851 3852 /* ctrl_vsi_idx will be set to a valid value when flow director 3853 * is setup by ice_init_fdir 3854 */ 3855 pf->ctrl_vsi_idx = ICE_NO_VSI; 3856 set_bit(ICE_FLAG_FD_ENA, pf->flags); 3857 /* force guaranteed filter pool for PF */ 3858 ice_alloc_fd_guar_item(&pf->hw, &unused, 3859 func_caps->fd_fltr_guar); 3860 /* force shared filter pool for PF */ 3861 ice_alloc_fd_shrd_item(&pf->hw, &unused, 3862 func_caps->fd_fltr_best_effort); 3863 } 3864 3865 clear_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags); 3866 if (func_caps->common_cap.ieee_1588) 3867 set_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags); 3868 3869 pf->max_pf_txqs = func_caps->common_cap.num_txq; 3870 pf->max_pf_rxqs = func_caps->common_cap.num_rxq; 3871 } 3872 3873 /** 3874 * ice_init_pf - Initialize general software structures (struct ice_pf) 3875 * @pf: board private structure to initialize 3876 */ 3877 static int ice_init_pf(struct ice_pf *pf) 3878 { 3879 ice_set_pf_caps(pf); 3880 3881 mutex_init(&pf->sw_mutex); 3882 mutex_init(&pf->tc_mutex); 3883 mutex_init(&pf->adev_mutex); 3884 3885 INIT_HLIST_HEAD(&pf->aq_wait_list); 3886 spin_lock_init(&pf->aq_wait_lock); 3887 init_waitqueue_head(&pf->aq_wait_queue); 3888 3889 init_waitqueue_head(&pf->reset_wait_queue); 3890 3891 /* setup service timer and periodic service task */ 3892 timer_setup(&pf->serv_tmr, ice_service_timer, 0); 3893 pf->serv_tmr_period = HZ; 3894 INIT_WORK(&pf->serv_task, ice_service_task); 3895 clear_bit(ICE_SERVICE_SCHED, pf->state); 3896 3897 mutex_init(&pf->avail_q_mutex); 3898 pf->avail_txqs = bitmap_zalloc(pf->max_pf_txqs, GFP_KERNEL); 3899 if (!pf->avail_txqs) 3900 return -ENOMEM; 3901 3902 pf->avail_rxqs = bitmap_zalloc(pf->max_pf_rxqs, GFP_KERNEL); 3903 if (!pf->avail_rxqs) { 3904 devm_kfree(ice_pf_to_dev(pf), pf->avail_txqs); 3905 pf->avail_txqs = NULL; 3906 return -ENOMEM; 3907 } 3908 3909 mutex_init(&pf->vfs.table_lock); 3910 hash_init(pf->vfs.table); 3911 3912 return 0; 3913 } 3914 3915 /** 3916 * ice_ena_msix_range - Request a range of MSIX vectors from the OS 3917 * @pf: board private structure 3918 * 3919 * compute the number of MSIX vectors required (v_budget) and request from 3920 * the OS. Return the number of vectors reserved or negative on failure 3921 */ 3922 static int ice_ena_msix_range(struct ice_pf *pf) 3923 { 3924 int num_cpus, v_left, v_actual, v_other, v_budget = 0; 3925 struct device *dev = ice_pf_to_dev(pf); 3926 int needed, err, i; 3927 3928 v_left = pf->hw.func_caps.common_cap.num_msix_vectors; 3929 num_cpus = num_online_cpus(); 3930 3931 /* reserve for LAN miscellaneous handler */ 3932 needed = ICE_MIN_LAN_OICR_MSIX; 3933 if (v_left < needed) 3934 goto no_hw_vecs_left_err; 3935 v_budget += needed; 3936 v_left -= needed; 3937 3938 /* reserve for flow director */ 3939 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 3940 needed = ICE_FDIR_MSIX; 3941 if (v_left < needed) 3942 goto no_hw_vecs_left_err; 3943 v_budget += needed; 3944 v_left -= needed; 3945 } 3946 3947 /* reserve for switchdev */ 3948 needed = ICE_ESWITCH_MSIX; 3949 if (v_left < needed) 3950 goto no_hw_vecs_left_err; 3951 v_budget += needed; 3952 v_left -= needed; 3953 3954 /* total used for non-traffic vectors */ 3955 v_other = v_budget; 3956 3957 /* reserve vectors for LAN traffic */ 3958 needed = num_cpus; 3959 if (v_left < needed) 3960 goto no_hw_vecs_left_err; 3961 pf->num_lan_msix = needed; 3962 v_budget += needed; 3963 v_left -= needed; 3964 3965 /* reserve vectors for RDMA auxiliary driver */ 3966 if (ice_is_rdma_ena(pf)) { 3967 needed = num_cpus + ICE_RDMA_NUM_AEQ_MSIX; 3968 if (v_left < needed) 3969 goto no_hw_vecs_left_err; 3970 pf->num_rdma_msix = needed; 3971 v_budget += needed; 3972 v_left -= needed; 3973 } 3974 3975 pf->msix_entries = devm_kcalloc(dev, v_budget, 3976 sizeof(*pf->msix_entries), GFP_KERNEL); 3977 if (!pf->msix_entries) { 3978 err = -ENOMEM; 3979 goto exit_err; 3980 } 3981 3982 for (i = 0; i < v_budget; i++) 3983 pf->msix_entries[i].entry = i; 3984 3985 /* actually reserve the vectors */ 3986 v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries, 3987 ICE_MIN_MSIX, v_budget); 3988 if (v_actual < 0) { 3989 dev_err(dev, "unable to reserve MSI-X vectors\n"); 3990 err = v_actual; 3991 goto msix_err; 3992 } 3993 3994 if (v_actual < v_budget) { 3995 dev_warn(dev, "not enough OS MSI-X vectors. requested = %d, obtained = %d\n", 3996 v_budget, v_actual); 3997 3998 if (v_actual < ICE_MIN_MSIX) { 3999 /* error if we can't get minimum vectors */ 4000 pci_disable_msix(pf->pdev); 4001 err = -ERANGE; 4002 goto msix_err; 4003 } else { 4004 int v_remain = v_actual - v_other; 4005 int v_rdma = 0, v_min_rdma = 0; 4006 4007 if (ice_is_rdma_ena(pf)) { 4008 /* Need at least 1 interrupt in addition to 4009 * AEQ MSIX 4010 */ 4011 v_rdma = ICE_RDMA_NUM_AEQ_MSIX + 1; 4012 v_min_rdma = ICE_MIN_RDMA_MSIX; 4013 } 4014 4015 if (v_actual == ICE_MIN_MSIX || 4016 v_remain < ICE_MIN_LAN_TXRX_MSIX + v_min_rdma) { 4017 dev_warn(dev, "Not enough MSI-X vectors to support RDMA.\n"); 4018 clear_bit(ICE_FLAG_RDMA_ENA, pf->flags); 4019 4020 pf->num_rdma_msix = 0; 4021 pf->num_lan_msix = ICE_MIN_LAN_TXRX_MSIX; 4022 } else if ((v_remain < ICE_MIN_LAN_TXRX_MSIX + v_rdma) || 4023 (v_remain - v_rdma < v_rdma)) { 4024 /* Support minimum RDMA and give remaining 4025 * vectors to LAN MSIX 4026 */ 4027 pf->num_rdma_msix = v_min_rdma; 4028 pf->num_lan_msix = v_remain - v_min_rdma; 4029 } else { 4030 /* Split remaining MSIX with RDMA after 4031 * accounting for AEQ MSIX 4032 */ 4033 pf->num_rdma_msix = (v_remain - ICE_RDMA_NUM_AEQ_MSIX) / 2 + 4034 ICE_RDMA_NUM_AEQ_MSIX; 4035 pf->num_lan_msix = v_remain - pf->num_rdma_msix; 4036 } 4037 4038 dev_notice(dev, "Enabled %d MSI-X vectors for LAN traffic.\n", 4039 pf->num_lan_msix); 4040 4041 if (ice_is_rdma_ena(pf)) 4042 dev_notice(dev, "Enabled %d MSI-X vectors for RDMA.\n", 4043 pf->num_rdma_msix); 4044 } 4045 } 4046 4047 return v_actual; 4048 4049 msix_err: 4050 devm_kfree(dev, pf->msix_entries); 4051 goto exit_err; 4052 4053 no_hw_vecs_left_err: 4054 dev_err(dev, "not enough device MSI-X vectors. requested = %d, available = %d\n", 4055 needed, v_left); 4056 err = -ERANGE; 4057 exit_err: 4058 pf->num_rdma_msix = 0; 4059 pf->num_lan_msix = 0; 4060 return err; 4061 } 4062 4063 /** 4064 * ice_dis_msix - Disable MSI-X interrupt setup in OS 4065 * @pf: board private structure 4066 */ 4067 static void ice_dis_msix(struct ice_pf *pf) 4068 { 4069 pci_disable_msix(pf->pdev); 4070 devm_kfree(ice_pf_to_dev(pf), pf->msix_entries); 4071 pf->msix_entries = NULL; 4072 } 4073 4074 /** 4075 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme 4076 * @pf: board private structure 4077 */ 4078 static void ice_clear_interrupt_scheme(struct ice_pf *pf) 4079 { 4080 ice_dis_msix(pf); 4081 4082 if (pf->irq_tracker) { 4083 devm_kfree(ice_pf_to_dev(pf), pf->irq_tracker); 4084 pf->irq_tracker = NULL; 4085 } 4086 } 4087 4088 /** 4089 * ice_init_interrupt_scheme - Determine proper interrupt scheme 4090 * @pf: board private structure to initialize 4091 */ 4092 static int ice_init_interrupt_scheme(struct ice_pf *pf) 4093 { 4094 int vectors; 4095 4096 vectors = ice_ena_msix_range(pf); 4097 4098 if (vectors < 0) 4099 return vectors; 4100 4101 /* set up vector assignment tracking */ 4102 pf->irq_tracker = devm_kzalloc(ice_pf_to_dev(pf), 4103 struct_size(pf->irq_tracker, list, vectors), 4104 GFP_KERNEL); 4105 if (!pf->irq_tracker) { 4106 ice_dis_msix(pf); 4107 return -ENOMEM; 4108 } 4109 4110 /* populate SW interrupts pool with number of OS granted IRQs. */ 4111 pf->num_avail_sw_msix = (u16)vectors; 4112 pf->irq_tracker->num_entries = (u16)vectors; 4113 pf->irq_tracker->end = pf->irq_tracker->num_entries; 4114 4115 return 0; 4116 } 4117 4118 /** 4119 * ice_is_wol_supported - check if WoL is supported 4120 * @hw: pointer to hardware info 4121 * 4122 * Check if WoL is supported based on the HW configuration. 4123 * Returns true if NVM supports and enables WoL for this port, false otherwise 4124 */ 4125 bool ice_is_wol_supported(struct ice_hw *hw) 4126 { 4127 u16 wol_ctrl; 4128 4129 /* A bit set to 1 in the NVM Software Reserved Word 2 (WoL control 4130 * word) indicates WoL is not supported on the corresponding PF ID. 4131 */ 4132 if (ice_read_sr_word(hw, ICE_SR_NVM_WOL_CFG, &wol_ctrl)) 4133 return false; 4134 4135 return !(BIT(hw->port_info->lport) & wol_ctrl); 4136 } 4137 4138 /** 4139 * ice_vsi_recfg_qs - Change the number of queues on a VSI 4140 * @vsi: VSI being changed 4141 * @new_rx: new number of Rx queues 4142 * @new_tx: new number of Tx queues 4143 * 4144 * Only change the number of queues if new_tx, or new_rx is non-0. 4145 * 4146 * Returns 0 on success. 4147 */ 4148 int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx) 4149 { 4150 struct ice_pf *pf = vsi->back; 4151 int err = 0, timeout = 50; 4152 4153 if (!new_rx && !new_tx) 4154 return -EINVAL; 4155 4156 while (test_and_set_bit(ICE_CFG_BUSY, pf->state)) { 4157 timeout--; 4158 if (!timeout) 4159 return -EBUSY; 4160 usleep_range(1000, 2000); 4161 } 4162 4163 if (new_tx) 4164 vsi->req_txq = (u16)new_tx; 4165 if (new_rx) 4166 vsi->req_rxq = (u16)new_rx; 4167 4168 /* set for the next time the netdev is started */ 4169 if (!netif_running(vsi->netdev)) { 4170 ice_vsi_rebuild(vsi, false); 4171 dev_dbg(ice_pf_to_dev(pf), "Link is down, queue count change happens when link is brought up\n"); 4172 goto done; 4173 } 4174 4175 ice_vsi_close(vsi); 4176 ice_vsi_rebuild(vsi, false); 4177 ice_pf_dcb_recfg(pf); 4178 ice_vsi_open(vsi); 4179 done: 4180 clear_bit(ICE_CFG_BUSY, pf->state); 4181 return err; 4182 } 4183 4184 /** 4185 * ice_set_safe_mode_vlan_cfg - configure PF VSI to allow all VLANs in safe mode 4186 * @pf: PF to configure 4187 * 4188 * No VLAN offloads/filtering are advertised in safe mode so make sure the PF 4189 * VSI can still Tx/Rx VLAN tagged packets. 4190 */ 4191 static void ice_set_safe_mode_vlan_cfg(struct ice_pf *pf) 4192 { 4193 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4194 struct ice_vsi_ctx *ctxt; 4195 struct ice_hw *hw; 4196 int status; 4197 4198 if (!vsi) 4199 return; 4200 4201 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 4202 if (!ctxt) 4203 return; 4204 4205 hw = &pf->hw; 4206 ctxt->info = vsi->info; 4207 4208 ctxt->info.valid_sections = 4209 cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID | 4210 ICE_AQ_VSI_PROP_SECURITY_VALID | 4211 ICE_AQ_VSI_PROP_SW_VALID); 4212 4213 /* disable VLAN anti-spoof */ 4214 ctxt->info.sec_flags &= ~(ICE_AQ_VSI_SEC_TX_VLAN_PRUNE_ENA << 4215 ICE_AQ_VSI_SEC_TX_PRUNE_ENA_S); 4216 4217 /* disable VLAN pruning and keep all other settings */ 4218 ctxt->info.sw_flags2 &= ~ICE_AQ_VSI_SW_FLAG_RX_VLAN_PRUNE_ENA; 4219 4220 /* allow all VLANs on Tx and don't strip on Rx */ 4221 ctxt->info.inner_vlan_flags = ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL | 4222 ICE_AQ_VSI_INNER_VLAN_EMODE_NOTHING; 4223 4224 status = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 4225 if (status) { 4226 dev_err(ice_pf_to_dev(vsi->back), "Failed to update VSI for safe mode VLANs, err %d aq_err %s\n", 4227 status, ice_aq_str(hw->adminq.sq_last_status)); 4228 } else { 4229 vsi->info.sec_flags = ctxt->info.sec_flags; 4230 vsi->info.sw_flags2 = ctxt->info.sw_flags2; 4231 vsi->info.inner_vlan_flags = ctxt->info.inner_vlan_flags; 4232 } 4233 4234 kfree(ctxt); 4235 } 4236 4237 /** 4238 * ice_log_pkg_init - log result of DDP package load 4239 * @hw: pointer to hardware info 4240 * @state: state of package load 4241 */ 4242 static void ice_log_pkg_init(struct ice_hw *hw, enum ice_ddp_state state) 4243 { 4244 struct ice_pf *pf = hw->back; 4245 struct device *dev; 4246 4247 dev = ice_pf_to_dev(pf); 4248 4249 switch (state) { 4250 case ICE_DDP_PKG_SUCCESS: 4251 dev_info(dev, "The DDP package was successfully loaded: %s version %d.%d.%d.%d\n", 4252 hw->active_pkg_name, 4253 hw->active_pkg_ver.major, 4254 hw->active_pkg_ver.minor, 4255 hw->active_pkg_ver.update, 4256 hw->active_pkg_ver.draft); 4257 break; 4258 case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED: 4259 dev_info(dev, "DDP package already present on device: %s version %d.%d.%d.%d\n", 4260 hw->active_pkg_name, 4261 hw->active_pkg_ver.major, 4262 hw->active_pkg_ver.minor, 4263 hw->active_pkg_ver.update, 4264 hw->active_pkg_ver.draft); 4265 break; 4266 case ICE_DDP_PKG_ALREADY_LOADED_NOT_SUPPORTED: 4267 dev_err(dev, "The device has a DDP package that is not supported by the driver. The device has package '%s' version %d.%d.x.x. The driver requires version %d.%d.x.x. Entering Safe Mode.\n", 4268 hw->active_pkg_name, 4269 hw->active_pkg_ver.major, 4270 hw->active_pkg_ver.minor, 4271 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 4272 break; 4273 case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED: 4274 dev_info(dev, "The driver could not load the DDP package file because a compatible DDP package is already present on the device. The device has package '%s' version %d.%d.%d.%d. The package file found by the driver: '%s' version %d.%d.%d.%d.\n", 4275 hw->active_pkg_name, 4276 hw->active_pkg_ver.major, 4277 hw->active_pkg_ver.minor, 4278 hw->active_pkg_ver.update, 4279 hw->active_pkg_ver.draft, 4280 hw->pkg_name, 4281 hw->pkg_ver.major, 4282 hw->pkg_ver.minor, 4283 hw->pkg_ver.update, 4284 hw->pkg_ver.draft); 4285 break; 4286 case ICE_DDP_PKG_FW_MISMATCH: 4287 dev_err(dev, "The firmware loaded on the device is not compatible with the DDP package. Please update the device's NVM. Entering safe mode.\n"); 4288 break; 4289 case ICE_DDP_PKG_INVALID_FILE: 4290 dev_err(dev, "The DDP package file is invalid. Entering Safe Mode.\n"); 4291 break; 4292 case ICE_DDP_PKG_FILE_VERSION_TOO_HIGH: 4293 dev_err(dev, "The DDP package file version is higher than the driver supports. Please use an updated driver. Entering Safe Mode.\n"); 4294 break; 4295 case ICE_DDP_PKG_FILE_VERSION_TOO_LOW: 4296 dev_err(dev, "The DDP package file version is lower than the driver supports. The driver requires version %d.%d.x.x. Please use an updated DDP Package file. Entering Safe Mode.\n", 4297 ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR); 4298 break; 4299 case ICE_DDP_PKG_FILE_SIGNATURE_INVALID: 4300 dev_err(dev, "The DDP package could not be loaded because its signature is not valid. Please use a valid DDP Package. Entering Safe Mode.\n"); 4301 break; 4302 case ICE_DDP_PKG_FILE_REVISION_TOO_LOW: 4303 dev_err(dev, "The DDP Package could not be loaded because its security revision is too low. Please use an updated DDP Package. Entering Safe Mode.\n"); 4304 break; 4305 case ICE_DDP_PKG_LOAD_ERROR: 4306 dev_err(dev, "An error occurred on the device while loading the DDP package. The device will be reset.\n"); 4307 /* poll for reset to complete */ 4308 if (ice_check_reset(hw)) 4309 dev_err(dev, "Error resetting device. Please reload the driver\n"); 4310 break; 4311 case ICE_DDP_PKG_ERR: 4312 default: 4313 dev_err(dev, "An unknown error occurred when loading the DDP package. Entering Safe Mode.\n"); 4314 break; 4315 } 4316 } 4317 4318 /** 4319 * ice_load_pkg - load/reload the DDP Package file 4320 * @firmware: firmware structure when firmware requested or NULL for reload 4321 * @pf: pointer to the PF instance 4322 * 4323 * Called on probe and post CORER/GLOBR rebuild to load DDP Package and 4324 * initialize HW tables. 4325 */ 4326 static void 4327 ice_load_pkg(const struct firmware *firmware, struct ice_pf *pf) 4328 { 4329 enum ice_ddp_state state = ICE_DDP_PKG_ERR; 4330 struct device *dev = ice_pf_to_dev(pf); 4331 struct ice_hw *hw = &pf->hw; 4332 4333 /* Load DDP Package */ 4334 if (firmware && !hw->pkg_copy) { 4335 state = ice_copy_and_init_pkg(hw, firmware->data, 4336 firmware->size); 4337 ice_log_pkg_init(hw, state); 4338 } else if (!firmware && hw->pkg_copy) { 4339 /* Reload package during rebuild after CORER/GLOBR reset */ 4340 state = ice_init_pkg(hw, hw->pkg_copy, hw->pkg_size); 4341 ice_log_pkg_init(hw, state); 4342 } else { 4343 dev_err(dev, "The DDP package file failed to load. Entering Safe Mode.\n"); 4344 } 4345 4346 if (!ice_is_init_pkg_successful(state)) { 4347 /* Safe Mode */ 4348 clear_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 4349 return; 4350 } 4351 4352 /* Successful download package is the precondition for advanced 4353 * features, hence setting the ICE_FLAG_ADV_FEATURES flag 4354 */ 4355 set_bit(ICE_FLAG_ADV_FEATURES, pf->flags); 4356 } 4357 4358 /** 4359 * ice_verify_cacheline_size - verify driver's assumption of 64 Byte cache lines 4360 * @pf: pointer to the PF structure 4361 * 4362 * There is no error returned here because the driver should be able to handle 4363 * 128 Byte cache lines, so we only print a warning in case issues are seen, 4364 * specifically with Tx. 4365 */ 4366 static void ice_verify_cacheline_size(struct ice_pf *pf) 4367 { 4368 if (rd32(&pf->hw, GLPCI_CNF2) & GLPCI_CNF2_CACHELINE_SIZE_M) 4369 dev_warn(ice_pf_to_dev(pf), "%d Byte cache line assumption is invalid, driver may have Tx timeouts!\n", 4370 ICE_CACHE_LINE_BYTES); 4371 } 4372 4373 /** 4374 * ice_send_version - update firmware with driver version 4375 * @pf: PF struct 4376 * 4377 * Returns 0 on success, else error code 4378 */ 4379 static int ice_send_version(struct ice_pf *pf) 4380 { 4381 struct ice_driver_ver dv; 4382 4383 dv.major_ver = 0xff; 4384 dv.minor_ver = 0xff; 4385 dv.build_ver = 0xff; 4386 dv.subbuild_ver = 0; 4387 strscpy((char *)dv.driver_string, UTS_RELEASE, 4388 sizeof(dv.driver_string)); 4389 return ice_aq_send_driver_ver(&pf->hw, &dv, NULL); 4390 } 4391 4392 /** 4393 * ice_init_fdir - Initialize flow director VSI and configuration 4394 * @pf: pointer to the PF instance 4395 * 4396 * returns 0 on success, negative on error 4397 */ 4398 static int ice_init_fdir(struct ice_pf *pf) 4399 { 4400 struct device *dev = ice_pf_to_dev(pf); 4401 struct ice_vsi *ctrl_vsi; 4402 int err; 4403 4404 /* Side Band Flow Director needs to have a control VSI. 4405 * Allocate it and store it in the PF. 4406 */ 4407 ctrl_vsi = ice_ctrl_vsi_setup(pf, pf->hw.port_info); 4408 if (!ctrl_vsi) { 4409 dev_dbg(dev, "could not create control VSI\n"); 4410 return -ENOMEM; 4411 } 4412 4413 err = ice_vsi_open_ctrl(ctrl_vsi); 4414 if (err) { 4415 dev_dbg(dev, "could not open control VSI\n"); 4416 goto err_vsi_open; 4417 } 4418 4419 mutex_init(&pf->hw.fdir_fltr_lock); 4420 4421 err = ice_fdir_create_dflt_rules(pf); 4422 if (err) 4423 goto err_fdir_rule; 4424 4425 return 0; 4426 4427 err_fdir_rule: 4428 ice_fdir_release_flows(&pf->hw); 4429 ice_vsi_close(ctrl_vsi); 4430 err_vsi_open: 4431 ice_vsi_release(ctrl_vsi); 4432 if (pf->ctrl_vsi_idx != ICE_NO_VSI) { 4433 pf->vsi[pf->ctrl_vsi_idx] = NULL; 4434 pf->ctrl_vsi_idx = ICE_NO_VSI; 4435 } 4436 return err; 4437 } 4438 4439 /** 4440 * ice_get_opt_fw_name - return optional firmware file name or NULL 4441 * @pf: pointer to the PF instance 4442 */ 4443 static char *ice_get_opt_fw_name(struct ice_pf *pf) 4444 { 4445 /* Optional firmware name same as default with additional dash 4446 * followed by a EUI-64 identifier (PCIe Device Serial Number) 4447 */ 4448 struct pci_dev *pdev = pf->pdev; 4449 char *opt_fw_filename; 4450 u64 dsn; 4451 4452 /* Determine the name of the optional file using the DSN (two 4453 * dwords following the start of the DSN Capability). 4454 */ 4455 dsn = pci_get_dsn(pdev); 4456 if (!dsn) 4457 return NULL; 4458 4459 opt_fw_filename = kzalloc(NAME_MAX, GFP_KERNEL); 4460 if (!opt_fw_filename) 4461 return NULL; 4462 4463 snprintf(opt_fw_filename, NAME_MAX, "%sice-%016llx.pkg", 4464 ICE_DDP_PKG_PATH, dsn); 4465 4466 return opt_fw_filename; 4467 } 4468 4469 /** 4470 * ice_request_fw - Device initialization routine 4471 * @pf: pointer to the PF instance 4472 */ 4473 static void ice_request_fw(struct ice_pf *pf) 4474 { 4475 char *opt_fw_filename = ice_get_opt_fw_name(pf); 4476 const struct firmware *firmware = NULL; 4477 struct device *dev = ice_pf_to_dev(pf); 4478 int err = 0; 4479 4480 /* optional device-specific DDP (if present) overrides the default DDP 4481 * package file. kernel logs a debug message if the file doesn't exist, 4482 * and warning messages for other errors. 4483 */ 4484 if (opt_fw_filename) { 4485 err = firmware_request_nowarn(&firmware, opt_fw_filename, dev); 4486 if (err) { 4487 kfree(opt_fw_filename); 4488 goto dflt_pkg_load; 4489 } 4490 4491 /* request for firmware was successful. Download to device */ 4492 ice_load_pkg(firmware, pf); 4493 kfree(opt_fw_filename); 4494 release_firmware(firmware); 4495 return; 4496 } 4497 4498 dflt_pkg_load: 4499 err = request_firmware(&firmware, ICE_DDP_PKG_FILE, dev); 4500 if (err) { 4501 dev_err(dev, "The DDP package file was not found or could not be read. Entering Safe Mode\n"); 4502 return; 4503 } 4504 4505 /* request for firmware was successful. Download to device */ 4506 ice_load_pkg(firmware, pf); 4507 release_firmware(firmware); 4508 } 4509 4510 /** 4511 * ice_print_wake_reason - show the wake up cause in the log 4512 * @pf: pointer to the PF struct 4513 */ 4514 static void ice_print_wake_reason(struct ice_pf *pf) 4515 { 4516 u32 wus = pf->wakeup_reason; 4517 const char *wake_str; 4518 4519 /* if no wake event, nothing to print */ 4520 if (!wus) 4521 return; 4522 4523 if (wus & PFPM_WUS_LNKC_M) 4524 wake_str = "Link\n"; 4525 else if (wus & PFPM_WUS_MAG_M) 4526 wake_str = "Magic Packet\n"; 4527 else if (wus & PFPM_WUS_MNG_M) 4528 wake_str = "Management\n"; 4529 else if (wus & PFPM_WUS_FW_RST_WK_M) 4530 wake_str = "Firmware Reset\n"; 4531 else 4532 wake_str = "Unknown\n"; 4533 4534 dev_info(ice_pf_to_dev(pf), "Wake reason: %s", wake_str); 4535 } 4536 4537 /** 4538 * ice_register_netdev - register netdev and devlink port 4539 * @pf: pointer to the PF struct 4540 */ 4541 static int ice_register_netdev(struct ice_pf *pf) 4542 { 4543 struct ice_vsi *vsi; 4544 int err = 0; 4545 4546 vsi = ice_get_main_vsi(pf); 4547 if (!vsi || !vsi->netdev) 4548 return -EIO; 4549 4550 err = register_netdev(vsi->netdev); 4551 if (err) 4552 goto err_register_netdev; 4553 4554 set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); 4555 netif_carrier_off(vsi->netdev); 4556 netif_tx_stop_all_queues(vsi->netdev); 4557 err = ice_devlink_create_pf_port(pf); 4558 if (err) 4559 goto err_devlink_create; 4560 4561 devlink_port_type_eth_set(&pf->devlink_port, vsi->netdev); 4562 4563 return 0; 4564 err_devlink_create: 4565 unregister_netdev(vsi->netdev); 4566 clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); 4567 err_register_netdev: 4568 free_netdev(vsi->netdev); 4569 vsi->netdev = NULL; 4570 clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); 4571 return err; 4572 } 4573 4574 /** 4575 * ice_probe - Device initialization routine 4576 * @pdev: PCI device information struct 4577 * @ent: entry in ice_pci_tbl 4578 * 4579 * Returns 0 on success, negative on failure 4580 */ 4581 static int 4582 ice_probe(struct pci_dev *pdev, const struct pci_device_id __always_unused *ent) 4583 { 4584 struct device *dev = &pdev->dev; 4585 struct ice_pf *pf; 4586 struct ice_hw *hw; 4587 int i, err; 4588 4589 if (pdev->is_virtfn) { 4590 dev_err(dev, "can't probe a virtual function\n"); 4591 return -EINVAL; 4592 } 4593 4594 /* this driver uses devres, see 4595 * Documentation/driver-api/driver-model/devres.rst 4596 */ 4597 err = pcim_enable_device(pdev); 4598 if (err) 4599 return err; 4600 4601 err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), dev_driver_string(dev)); 4602 if (err) { 4603 dev_err(dev, "BAR0 I/O map error %d\n", err); 4604 return err; 4605 } 4606 4607 pf = ice_allocate_pf(dev); 4608 if (!pf) 4609 return -ENOMEM; 4610 4611 /* initialize Auxiliary index to invalid value */ 4612 pf->aux_idx = -1; 4613 4614 /* set up for high or low DMA */ 4615 err = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64)); 4616 if (err) { 4617 dev_err(dev, "DMA configuration failed: 0x%x\n", err); 4618 return err; 4619 } 4620 4621 pci_enable_pcie_error_reporting(pdev); 4622 pci_set_master(pdev); 4623 4624 pf->pdev = pdev; 4625 pci_set_drvdata(pdev, pf); 4626 set_bit(ICE_DOWN, pf->state); 4627 /* Disable service task until DOWN bit is cleared */ 4628 set_bit(ICE_SERVICE_DIS, pf->state); 4629 4630 hw = &pf->hw; 4631 hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0]; 4632 pci_save_state(pdev); 4633 4634 hw->back = pf; 4635 hw->vendor_id = pdev->vendor; 4636 hw->device_id = pdev->device; 4637 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 4638 hw->subsystem_vendor_id = pdev->subsystem_vendor; 4639 hw->subsystem_device_id = pdev->subsystem_device; 4640 hw->bus.device = PCI_SLOT(pdev->devfn); 4641 hw->bus.func = PCI_FUNC(pdev->devfn); 4642 ice_set_ctrlq_len(hw); 4643 4644 pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M); 4645 4646 #ifndef CONFIG_DYNAMIC_DEBUG 4647 if (debug < -1) 4648 hw->debug_mask = debug; 4649 #endif 4650 4651 err = ice_init_hw(hw); 4652 if (err) { 4653 dev_err(dev, "ice_init_hw failed: %d\n", err); 4654 err = -EIO; 4655 goto err_exit_unroll; 4656 } 4657 4658 ice_init_feature_support(pf); 4659 4660 ice_request_fw(pf); 4661 4662 /* if ice_request_fw fails, ICE_FLAG_ADV_FEATURES bit won't be 4663 * set in pf->state, which will cause ice_is_safe_mode to return 4664 * true 4665 */ 4666 if (ice_is_safe_mode(pf)) { 4667 /* we already got function/device capabilities but these don't 4668 * reflect what the driver needs to do in safe mode. Instead of 4669 * adding conditional logic everywhere to ignore these 4670 * device/function capabilities, override them. 4671 */ 4672 ice_set_safe_mode_caps(hw); 4673 } 4674 4675 err = ice_init_pf(pf); 4676 if (err) { 4677 dev_err(dev, "ice_init_pf failed: %d\n", err); 4678 goto err_init_pf_unroll; 4679 } 4680 4681 ice_devlink_init_regions(pf); 4682 4683 pf->hw.udp_tunnel_nic.set_port = ice_udp_tunnel_set_port; 4684 pf->hw.udp_tunnel_nic.unset_port = ice_udp_tunnel_unset_port; 4685 pf->hw.udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP; 4686 pf->hw.udp_tunnel_nic.shared = &pf->hw.udp_tunnel_shared; 4687 i = 0; 4688 if (pf->hw.tnl.valid_count[TNL_VXLAN]) { 4689 pf->hw.udp_tunnel_nic.tables[i].n_entries = 4690 pf->hw.tnl.valid_count[TNL_VXLAN]; 4691 pf->hw.udp_tunnel_nic.tables[i].tunnel_types = 4692 UDP_TUNNEL_TYPE_VXLAN; 4693 i++; 4694 } 4695 if (pf->hw.tnl.valid_count[TNL_GENEVE]) { 4696 pf->hw.udp_tunnel_nic.tables[i].n_entries = 4697 pf->hw.tnl.valid_count[TNL_GENEVE]; 4698 pf->hw.udp_tunnel_nic.tables[i].tunnel_types = 4699 UDP_TUNNEL_TYPE_GENEVE; 4700 i++; 4701 } 4702 4703 pf->num_alloc_vsi = hw->func_caps.guar_num_vsi; 4704 if (!pf->num_alloc_vsi) { 4705 err = -EIO; 4706 goto err_init_pf_unroll; 4707 } 4708 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 4709 dev_warn(&pf->pdev->dev, 4710 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 4711 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 4712 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 4713 } 4714 4715 pf->vsi = devm_kcalloc(dev, pf->num_alloc_vsi, sizeof(*pf->vsi), 4716 GFP_KERNEL); 4717 if (!pf->vsi) { 4718 err = -ENOMEM; 4719 goto err_init_pf_unroll; 4720 } 4721 4722 err = ice_init_interrupt_scheme(pf); 4723 if (err) { 4724 dev_err(dev, "ice_init_interrupt_scheme failed: %d\n", err); 4725 err = -EIO; 4726 goto err_init_vsi_unroll; 4727 } 4728 4729 /* In case of MSIX we are going to setup the misc vector right here 4730 * to handle admin queue events etc. In case of legacy and MSI 4731 * the misc functionality and queue processing is combined in 4732 * the same vector and that gets setup at open. 4733 */ 4734 err = ice_req_irq_msix_misc(pf); 4735 if (err) { 4736 dev_err(dev, "setup of misc vector failed: %d\n", err); 4737 goto err_init_interrupt_unroll; 4738 } 4739 4740 /* create switch struct for the switch element created by FW on boot */ 4741 pf->first_sw = devm_kzalloc(dev, sizeof(*pf->first_sw), GFP_KERNEL); 4742 if (!pf->first_sw) { 4743 err = -ENOMEM; 4744 goto err_msix_misc_unroll; 4745 } 4746 4747 if (hw->evb_veb) 4748 pf->first_sw->bridge_mode = BRIDGE_MODE_VEB; 4749 else 4750 pf->first_sw->bridge_mode = BRIDGE_MODE_VEPA; 4751 4752 pf->first_sw->pf = pf; 4753 4754 /* record the sw_id available for later use */ 4755 pf->first_sw->sw_id = hw->port_info->sw_id; 4756 4757 err = ice_setup_pf_sw(pf); 4758 if (err) { 4759 dev_err(dev, "probe failed due to setup PF switch: %d\n", err); 4760 goto err_alloc_sw_unroll; 4761 } 4762 4763 clear_bit(ICE_SERVICE_DIS, pf->state); 4764 4765 /* tell the firmware we are up */ 4766 err = ice_send_version(pf); 4767 if (err) { 4768 dev_err(dev, "probe failed sending driver version %s. error: %d\n", 4769 UTS_RELEASE, err); 4770 goto err_send_version_unroll; 4771 } 4772 4773 /* since everything is good, start the service timer */ 4774 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 4775 4776 err = ice_init_link_events(pf->hw.port_info); 4777 if (err) { 4778 dev_err(dev, "ice_init_link_events failed: %d\n", err); 4779 goto err_send_version_unroll; 4780 } 4781 4782 /* not a fatal error if this fails */ 4783 err = ice_init_nvm_phy_type(pf->hw.port_info); 4784 if (err) 4785 dev_err(dev, "ice_init_nvm_phy_type failed: %d\n", err); 4786 4787 /* not a fatal error if this fails */ 4788 err = ice_update_link_info(pf->hw.port_info); 4789 if (err) 4790 dev_err(dev, "ice_update_link_info failed: %d\n", err); 4791 4792 ice_init_link_dflt_override(pf->hw.port_info); 4793 4794 ice_check_link_cfg_err(pf, 4795 pf->hw.port_info->phy.link_info.link_cfg_err); 4796 4797 /* if media available, initialize PHY settings */ 4798 if (pf->hw.port_info->phy.link_info.link_info & 4799 ICE_AQ_MEDIA_AVAILABLE) { 4800 /* not a fatal error if this fails */ 4801 err = ice_init_phy_user_cfg(pf->hw.port_info); 4802 if (err) 4803 dev_err(dev, "ice_init_phy_user_cfg failed: %d\n", err); 4804 4805 if (!test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, pf->flags)) { 4806 struct ice_vsi *vsi = ice_get_main_vsi(pf); 4807 4808 if (vsi) 4809 ice_configure_phy(vsi); 4810 } 4811 } else { 4812 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 4813 } 4814 4815 ice_verify_cacheline_size(pf); 4816 4817 /* Save wakeup reason register for later use */ 4818 pf->wakeup_reason = rd32(hw, PFPM_WUS); 4819 4820 /* check for a power management event */ 4821 ice_print_wake_reason(pf); 4822 4823 /* clear wake status, all bits */ 4824 wr32(hw, PFPM_WUS, U32_MAX); 4825 4826 /* Disable WoL at init, wait for user to enable */ 4827 device_set_wakeup_enable(dev, false); 4828 4829 if (ice_is_safe_mode(pf)) { 4830 ice_set_safe_mode_vlan_cfg(pf); 4831 goto probe_done; 4832 } 4833 4834 /* initialize DDP driven features */ 4835 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 4836 ice_ptp_init(pf); 4837 4838 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 4839 ice_gnss_init(pf); 4840 4841 /* Note: Flow director init failure is non-fatal to load */ 4842 if (ice_init_fdir(pf)) 4843 dev_err(dev, "could not initialize flow director\n"); 4844 4845 /* Note: DCB init failure is non-fatal to load */ 4846 if (ice_init_pf_dcb(pf, false)) { 4847 clear_bit(ICE_FLAG_DCB_CAPABLE, pf->flags); 4848 clear_bit(ICE_FLAG_DCB_ENA, pf->flags); 4849 } else { 4850 ice_cfg_lldp_mib_change(&pf->hw, true); 4851 } 4852 4853 if (ice_init_lag(pf)) 4854 dev_warn(dev, "Failed to init link aggregation support\n"); 4855 4856 /* print PCI link speed and width */ 4857 pcie_print_link_status(pf->pdev); 4858 4859 probe_done: 4860 err = ice_register_netdev(pf); 4861 if (err) 4862 goto err_netdev_reg; 4863 4864 err = ice_devlink_register_params(pf); 4865 if (err) 4866 goto err_netdev_reg; 4867 4868 /* ready to go, so clear down state bit */ 4869 clear_bit(ICE_DOWN, pf->state); 4870 if (ice_is_rdma_ena(pf)) { 4871 pf->aux_idx = ida_alloc(&ice_aux_ida, GFP_KERNEL); 4872 if (pf->aux_idx < 0) { 4873 dev_err(dev, "Failed to allocate device ID for AUX driver\n"); 4874 err = -ENOMEM; 4875 goto err_devlink_reg_param; 4876 } 4877 4878 err = ice_init_rdma(pf); 4879 if (err) { 4880 dev_err(dev, "Failed to initialize RDMA: %d\n", err); 4881 err = -EIO; 4882 goto err_init_aux_unroll; 4883 } 4884 } else { 4885 dev_warn(dev, "RDMA is not supported on this device\n"); 4886 } 4887 4888 ice_devlink_register(pf); 4889 return 0; 4890 4891 err_init_aux_unroll: 4892 pf->adev = NULL; 4893 ida_free(&ice_aux_ida, pf->aux_idx); 4894 err_devlink_reg_param: 4895 ice_devlink_unregister_params(pf); 4896 err_netdev_reg: 4897 err_send_version_unroll: 4898 ice_vsi_release_all(pf); 4899 err_alloc_sw_unroll: 4900 set_bit(ICE_SERVICE_DIS, pf->state); 4901 set_bit(ICE_DOWN, pf->state); 4902 devm_kfree(dev, pf->first_sw); 4903 err_msix_misc_unroll: 4904 ice_free_irq_msix_misc(pf); 4905 err_init_interrupt_unroll: 4906 ice_clear_interrupt_scheme(pf); 4907 err_init_vsi_unroll: 4908 devm_kfree(dev, pf->vsi); 4909 err_init_pf_unroll: 4910 ice_deinit_pf(pf); 4911 ice_devlink_destroy_regions(pf); 4912 ice_deinit_hw(hw); 4913 err_exit_unroll: 4914 pci_disable_pcie_error_reporting(pdev); 4915 pci_disable_device(pdev); 4916 return err; 4917 } 4918 4919 /** 4920 * ice_set_wake - enable or disable Wake on LAN 4921 * @pf: pointer to the PF struct 4922 * 4923 * Simple helper for WoL control 4924 */ 4925 static void ice_set_wake(struct ice_pf *pf) 4926 { 4927 struct ice_hw *hw = &pf->hw; 4928 bool wol = pf->wol_ena; 4929 4930 /* clear wake state, otherwise new wake events won't fire */ 4931 wr32(hw, PFPM_WUS, U32_MAX); 4932 4933 /* enable / disable APM wake up, no RMW needed */ 4934 wr32(hw, PFPM_APM, wol ? PFPM_APM_APME_M : 0); 4935 4936 /* set magic packet filter enabled */ 4937 wr32(hw, PFPM_WUFC, wol ? PFPM_WUFC_MAG_M : 0); 4938 } 4939 4940 /** 4941 * ice_setup_mc_magic_wake - setup device to wake on multicast magic packet 4942 * @pf: pointer to the PF struct 4943 * 4944 * Issue firmware command to enable multicast magic wake, making 4945 * sure that any locally administered address (LAA) is used for 4946 * wake, and that PF reset doesn't undo the LAA. 4947 */ 4948 static void ice_setup_mc_magic_wake(struct ice_pf *pf) 4949 { 4950 struct device *dev = ice_pf_to_dev(pf); 4951 struct ice_hw *hw = &pf->hw; 4952 u8 mac_addr[ETH_ALEN]; 4953 struct ice_vsi *vsi; 4954 int status; 4955 u8 flags; 4956 4957 if (!pf->wol_ena) 4958 return; 4959 4960 vsi = ice_get_main_vsi(pf); 4961 if (!vsi) 4962 return; 4963 4964 /* Get current MAC address in case it's an LAA */ 4965 if (vsi->netdev) 4966 ether_addr_copy(mac_addr, vsi->netdev->dev_addr); 4967 else 4968 ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr); 4969 4970 flags = ICE_AQC_MAN_MAC_WR_MC_MAG_EN | 4971 ICE_AQC_MAN_MAC_UPDATE_LAA_WOL | 4972 ICE_AQC_MAN_MAC_WR_WOL_LAA_PFR_KEEP; 4973 4974 status = ice_aq_manage_mac_write(hw, mac_addr, flags, NULL); 4975 if (status) 4976 dev_err(dev, "Failed to enable Multicast Magic Packet wake, err %d aq_err %s\n", 4977 status, ice_aq_str(hw->adminq.sq_last_status)); 4978 } 4979 4980 /** 4981 * ice_remove - Device removal routine 4982 * @pdev: PCI device information struct 4983 */ 4984 static void ice_remove(struct pci_dev *pdev) 4985 { 4986 struct ice_pf *pf = pci_get_drvdata(pdev); 4987 int i; 4988 4989 ice_devlink_unregister(pf); 4990 for (i = 0; i < ICE_MAX_RESET_WAIT; i++) { 4991 if (!ice_is_reset_in_progress(pf->state)) 4992 break; 4993 msleep(100); 4994 } 4995 4996 ice_tc_indir_block_remove(pf); 4997 4998 if (test_bit(ICE_FLAG_SRIOV_ENA, pf->flags)) { 4999 set_bit(ICE_VF_RESETS_DISABLED, pf->state); 5000 ice_free_vfs(pf); 5001 } 5002 5003 ice_service_task_stop(pf); 5004 5005 ice_aq_cancel_waiting_tasks(pf); 5006 ice_unplug_aux_dev(pf); 5007 if (pf->aux_idx >= 0) 5008 ida_free(&ice_aux_ida, pf->aux_idx); 5009 ice_devlink_unregister_params(pf); 5010 set_bit(ICE_DOWN, pf->state); 5011 5012 ice_deinit_lag(pf); 5013 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 5014 ice_ptp_release(pf); 5015 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 5016 ice_gnss_exit(pf); 5017 if (!ice_is_safe_mode(pf)) 5018 ice_remove_arfs(pf); 5019 ice_setup_mc_magic_wake(pf); 5020 ice_vsi_release_all(pf); 5021 mutex_destroy(&(&pf->hw)->fdir_fltr_lock); 5022 ice_set_wake(pf); 5023 ice_free_irq_msix_misc(pf); 5024 ice_for_each_vsi(pf, i) { 5025 if (!pf->vsi[i]) 5026 continue; 5027 ice_vsi_free_q_vectors(pf->vsi[i]); 5028 } 5029 ice_deinit_pf(pf); 5030 ice_devlink_destroy_regions(pf); 5031 ice_deinit_hw(&pf->hw); 5032 5033 /* Issue a PFR as part of the prescribed driver unload flow. Do not 5034 * do it via ice_schedule_reset() since there is no need to rebuild 5035 * and the service task is already stopped. 5036 */ 5037 ice_reset(&pf->hw, ICE_RESET_PFR); 5038 pci_wait_for_pending_transaction(pdev); 5039 ice_clear_interrupt_scheme(pf); 5040 pci_disable_pcie_error_reporting(pdev); 5041 pci_disable_device(pdev); 5042 } 5043 5044 /** 5045 * ice_shutdown - PCI callback for shutting down device 5046 * @pdev: PCI device information struct 5047 */ 5048 static void ice_shutdown(struct pci_dev *pdev) 5049 { 5050 struct ice_pf *pf = pci_get_drvdata(pdev); 5051 5052 ice_remove(pdev); 5053 5054 if (system_state == SYSTEM_POWER_OFF) { 5055 pci_wake_from_d3(pdev, pf->wol_ena); 5056 pci_set_power_state(pdev, PCI_D3hot); 5057 } 5058 } 5059 5060 #ifdef CONFIG_PM 5061 /** 5062 * ice_prepare_for_shutdown - prep for PCI shutdown 5063 * @pf: board private structure 5064 * 5065 * Inform or close all dependent features in prep for PCI device shutdown 5066 */ 5067 static void ice_prepare_for_shutdown(struct ice_pf *pf) 5068 { 5069 struct ice_hw *hw = &pf->hw; 5070 u32 v; 5071 5072 /* Notify VFs of impending reset */ 5073 if (ice_check_sq_alive(hw, &hw->mailboxq)) 5074 ice_vc_notify_reset(pf); 5075 5076 dev_dbg(ice_pf_to_dev(pf), "Tearing down internal switch for shutdown\n"); 5077 5078 /* disable the VSIs and their queues that are not already DOWN */ 5079 ice_pf_dis_all_vsi(pf, false); 5080 5081 ice_for_each_vsi(pf, v) 5082 if (pf->vsi[v]) 5083 pf->vsi[v]->vsi_num = 0; 5084 5085 ice_shutdown_all_ctrlq(hw); 5086 } 5087 5088 /** 5089 * ice_reinit_interrupt_scheme - Reinitialize interrupt scheme 5090 * @pf: board private structure to reinitialize 5091 * 5092 * This routine reinitialize interrupt scheme that was cleared during 5093 * power management suspend callback. 5094 * 5095 * This should be called during resume routine to re-allocate the q_vectors 5096 * and reacquire interrupts. 5097 */ 5098 static int ice_reinit_interrupt_scheme(struct ice_pf *pf) 5099 { 5100 struct device *dev = ice_pf_to_dev(pf); 5101 int ret, v; 5102 5103 /* Since we clear MSIX flag during suspend, we need to 5104 * set it back during resume... 5105 */ 5106 5107 ret = ice_init_interrupt_scheme(pf); 5108 if (ret) { 5109 dev_err(dev, "Failed to re-initialize interrupt %d\n", ret); 5110 return ret; 5111 } 5112 5113 /* Remap vectors and rings, after successful re-init interrupts */ 5114 ice_for_each_vsi(pf, v) { 5115 if (!pf->vsi[v]) 5116 continue; 5117 5118 ret = ice_vsi_alloc_q_vectors(pf->vsi[v]); 5119 if (ret) 5120 goto err_reinit; 5121 ice_vsi_map_rings_to_vectors(pf->vsi[v]); 5122 } 5123 5124 ret = ice_req_irq_msix_misc(pf); 5125 if (ret) { 5126 dev_err(dev, "Setting up misc vector failed after device suspend %d\n", 5127 ret); 5128 goto err_reinit; 5129 } 5130 5131 return 0; 5132 5133 err_reinit: 5134 while (v--) 5135 if (pf->vsi[v]) 5136 ice_vsi_free_q_vectors(pf->vsi[v]); 5137 5138 return ret; 5139 } 5140 5141 /** 5142 * ice_suspend 5143 * @dev: generic device information structure 5144 * 5145 * Power Management callback to quiesce the device and prepare 5146 * for D3 transition. 5147 */ 5148 static int __maybe_unused ice_suspend(struct device *dev) 5149 { 5150 struct pci_dev *pdev = to_pci_dev(dev); 5151 struct ice_pf *pf; 5152 int disabled, v; 5153 5154 pf = pci_get_drvdata(pdev); 5155 5156 if (!ice_pf_state_is_nominal(pf)) { 5157 dev_err(dev, "Device is not ready, no need to suspend it\n"); 5158 return -EBUSY; 5159 } 5160 5161 /* Stop watchdog tasks until resume completion. 5162 * Even though it is most likely that the service task is 5163 * disabled if the device is suspended or down, the service task's 5164 * state is controlled by a different state bit, and we should 5165 * store and honor whatever state that bit is in at this point. 5166 */ 5167 disabled = ice_service_task_stop(pf); 5168 5169 ice_unplug_aux_dev(pf); 5170 5171 /* Already suspended?, then there is nothing to do */ 5172 if (test_and_set_bit(ICE_SUSPENDED, pf->state)) { 5173 if (!disabled) 5174 ice_service_task_restart(pf); 5175 return 0; 5176 } 5177 5178 if (test_bit(ICE_DOWN, pf->state) || 5179 ice_is_reset_in_progress(pf->state)) { 5180 dev_err(dev, "can't suspend device in reset or already down\n"); 5181 if (!disabled) 5182 ice_service_task_restart(pf); 5183 return 0; 5184 } 5185 5186 ice_setup_mc_magic_wake(pf); 5187 5188 ice_prepare_for_shutdown(pf); 5189 5190 ice_set_wake(pf); 5191 5192 /* Free vectors, clear the interrupt scheme and release IRQs 5193 * for proper hibernation, especially with large number of CPUs. 5194 * Otherwise hibernation might fail when mapping all the vectors back 5195 * to CPU0. 5196 */ 5197 ice_free_irq_msix_misc(pf); 5198 ice_for_each_vsi(pf, v) { 5199 if (!pf->vsi[v]) 5200 continue; 5201 ice_vsi_free_q_vectors(pf->vsi[v]); 5202 } 5203 ice_clear_interrupt_scheme(pf); 5204 5205 pci_save_state(pdev); 5206 pci_wake_from_d3(pdev, pf->wol_ena); 5207 pci_set_power_state(pdev, PCI_D3hot); 5208 return 0; 5209 } 5210 5211 /** 5212 * ice_resume - PM callback for waking up from D3 5213 * @dev: generic device information structure 5214 */ 5215 static int __maybe_unused ice_resume(struct device *dev) 5216 { 5217 struct pci_dev *pdev = to_pci_dev(dev); 5218 enum ice_reset_req reset_type; 5219 struct ice_pf *pf; 5220 struct ice_hw *hw; 5221 int ret; 5222 5223 pci_set_power_state(pdev, PCI_D0); 5224 pci_restore_state(pdev); 5225 pci_save_state(pdev); 5226 5227 if (!pci_device_is_present(pdev)) 5228 return -ENODEV; 5229 5230 ret = pci_enable_device_mem(pdev); 5231 if (ret) { 5232 dev_err(dev, "Cannot enable device after suspend\n"); 5233 return ret; 5234 } 5235 5236 pf = pci_get_drvdata(pdev); 5237 hw = &pf->hw; 5238 5239 pf->wakeup_reason = rd32(hw, PFPM_WUS); 5240 ice_print_wake_reason(pf); 5241 5242 /* We cleared the interrupt scheme when we suspended, so we need to 5243 * restore it now to resume device functionality. 5244 */ 5245 ret = ice_reinit_interrupt_scheme(pf); 5246 if (ret) 5247 dev_err(dev, "Cannot restore interrupt scheme: %d\n", ret); 5248 5249 clear_bit(ICE_DOWN, pf->state); 5250 /* Now perform PF reset and rebuild */ 5251 reset_type = ICE_RESET_PFR; 5252 /* re-enable service task for reset, but allow reset to schedule it */ 5253 clear_bit(ICE_SERVICE_DIS, pf->state); 5254 5255 if (ice_schedule_reset(pf, reset_type)) 5256 dev_err(dev, "Reset during resume failed.\n"); 5257 5258 clear_bit(ICE_SUSPENDED, pf->state); 5259 ice_service_task_restart(pf); 5260 5261 /* Restart the service task */ 5262 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 5263 5264 return 0; 5265 } 5266 #endif /* CONFIG_PM */ 5267 5268 /** 5269 * ice_pci_err_detected - warning that PCI error has been detected 5270 * @pdev: PCI device information struct 5271 * @err: the type of PCI error 5272 * 5273 * Called to warn that something happened on the PCI bus and the error handling 5274 * is in progress. Allows the driver to gracefully prepare/handle PCI errors. 5275 */ 5276 static pci_ers_result_t 5277 ice_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t err) 5278 { 5279 struct ice_pf *pf = pci_get_drvdata(pdev); 5280 5281 if (!pf) { 5282 dev_err(&pdev->dev, "%s: unrecoverable device error %d\n", 5283 __func__, err); 5284 return PCI_ERS_RESULT_DISCONNECT; 5285 } 5286 5287 if (!test_bit(ICE_SUSPENDED, pf->state)) { 5288 ice_service_task_stop(pf); 5289 5290 if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) { 5291 set_bit(ICE_PFR_REQ, pf->state); 5292 ice_prepare_for_reset(pf, ICE_RESET_PFR); 5293 } 5294 } 5295 5296 return PCI_ERS_RESULT_NEED_RESET; 5297 } 5298 5299 /** 5300 * ice_pci_err_slot_reset - a PCI slot reset has just happened 5301 * @pdev: PCI device information struct 5302 * 5303 * Called to determine if the driver can recover from the PCI slot reset by 5304 * using a register read to determine if the device is recoverable. 5305 */ 5306 static pci_ers_result_t ice_pci_err_slot_reset(struct pci_dev *pdev) 5307 { 5308 struct ice_pf *pf = pci_get_drvdata(pdev); 5309 pci_ers_result_t result; 5310 int err; 5311 u32 reg; 5312 5313 err = pci_enable_device_mem(pdev); 5314 if (err) { 5315 dev_err(&pdev->dev, "Cannot re-enable PCI device after reset, error %d\n", 5316 err); 5317 result = PCI_ERS_RESULT_DISCONNECT; 5318 } else { 5319 pci_set_master(pdev); 5320 pci_restore_state(pdev); 5321 pci_save_state(pdev); 5322 pci_wake_from_d3(pdev, false); 5323 5324 /* Check for life */ 5325 reg = rd32(&pf->hw, GLGEN_RTRIG); 5326 if (!reg) 5327 result = PCI_ERS_RESULT_RECOVERED; 5328 else 5329 result = PCI_ERS_RESULT_DISCONNECT; 5330 } 5331 5332 return result; 5333 } 5334 5335 /** 5336 * ice_pci_err_resume - restart operations after PCI error recovery 5337 * @pdev: PCI device information struct 5338 * 5339 * Called to allow the driver to bring things back up after PCI error and/or 5340 * reset recovery have finished 5341 */ 5342 static void ice_pci_err_resume(struct pci_dev *pdev) 5343 { 5344 struct ice_pf *pf = pci_get_drvdata(pdev); 5345 5346 if (!pf) { 5347 dev_err(&pdev->dev, "%s failed, device is unrecoverable\n", 5348 __func__); 5349 return; 5350 } 5351 5352 if (test_bit(ICE_SUSPENDED, pf->state)) { 5353 dev_dbg(&pdev->dev, "%s failed to resume normal operations!\n", 5354 __func__); 5355 return; 5356 } 5357 5358 ice_restore_all_vfs_msi_state(pdev); 5359 5360 ice_do_reset(pf, ICE_RESET_PFR); 5361 ice_service_task_restart(pf); 5362 mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period)); 5363 } 5364 5365 /** 5366 * ice_pci_err_reset_prepare - prepare device driver for PCI reset 5367 * @pdev: PCI device information struct 5368 */ 5369 static void ice_pci_err_reset_prepare(struct pci_dev *pdev) 5370 { 5371 struct ice_pf *pf = pci_get_drvdata(pdev); 5372 5373 if (!test_bit(ICE_SUSPENDED, pf->state)) { 5374 ice_service_task_stop(pf); 5375 5376 if (!test_bit(ICE_PREPARED_FOR_RESET, pf->state)) { 5377 set_bit(ICE_PFR_REQ, pf->state); 5378 ice_prepare_for_reset(pf, ICE_RESET_PFR); 5379 } 5380 } 5381 } 5382 5383 /** 5384 * ice_pci_err_reset_done - PCI reset done, device driver reset can begin 5385 * @pdev: PCI device information struct 5386 */ 5387 static void ice_pci_err_reset_done(struct pci_dev *pdev) 5388 { 5389 ice_pci_err_resume(pdev); 5390 } 5391 5392 /* ice_pci_tbl - PCI Device ID Table 5393 * 5394 * Wildcard entries (PCI_ANY_ID) should come last 5395 * Last entry must be all 0s 5396 * 5397 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 5398 * Class, Class Mask, private data (not used) } 5399 */ 5400 static const struct pci_device_id ice_pci_tbl[] = { 5401 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_BACKPLANE), 0 }, 5402 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_QSFP), 0 }, 5403 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810C_SFP), 0 }, 5404 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_BACKPLANE), 0 }, 5405 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_QSFP), 0 }, 5406 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E810_XXV_SFP), 0 }, 5407 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_BACKPLANE), 0 }, 5408 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_QSFP), 0 }, 5409 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SFP), 0 }, 5410 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_10G_BASE_T), 0 }, 5411 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823C_SGMII), 0 }, 5412 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_BACKPLANE), 0 }, 5413 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_QSFP), 0 }, 5414 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SFP), 0 }, 5415 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_10G_BASE_T), 0 }, 5416 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822C_SGMII), 0 }, 5417 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_BACKPLANE), 0 }, 5418 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SFP), 0 }, 5419 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_10G_BASE_T), 0 }, 5420 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822L_SGMII), 0 }, 5421 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_BACKPLANE), 0 }, 5422 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_SFP), 0 }, 5423 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_10G_BASE_T), 0 }, 5424 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_1GBE), 0 }, 5425 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E823L_QSFP), 0 }, 5426 { PCI_VDEVICE(INTEL, ICE_DEV_ID_E822_SI_DFLT), 0 }, 5427 /* required last entry */ 5428 { 0, } 5429 }; 5430 MODULE_DEVICE_TABLE(pci, ice_pci_tbl); 5431 5432 static __maybe_unused SIMPLE_DEV_PM_OPS(ice_pm_ops, ice_suspend, ice_resume); 5433 5434 static const struct pci_error_handlers ice_pci_err_handler = { 5435 .error_detected = ice_pci_err_detected, 5436 .slot_reset = ice_pci_err_slot_reset, 5437 .reset_prepare = ice_pci_err_reset_prepare, 5438 .reset_done = ice_pci_err_reset_done, 5439 .resume = ice_pci_err_resume 5440 }; 5441 5442 static struct pci_driver ice_driver = { 5443 .name = KBUILD_MODNAME, 5444 .id_table = ice_pci_tbl, 5445 .probe = ice_probe, 5446 .remove = ice_remove, 5447 #ifdef CONFIG_PM 5448 .driver.pm = &ice_pm_ops, 5449 #endif /* CONFIG_PM */ 5450 .shutdown = ice_shutdown, 5451 .sriov_configure = ice_sriov_configure, 5452 .err_handler = &ice_pci_err_handler 5453 }; 5454 5455 /** 5456 * ice_module_init - Driver registration routine 5457 * 5458 * ice_module_init is the first routine called when the driver is 5459 * loaded. All it does is register with the PCI subsystem. 5460 */ 5461 static int __init ice_module_init(void) 5462 { 5463 int status; 5464 5465 pr_info("%s\n", ice_driver_string); 5466 pr_info("%s\n", ice_copyright); 5467 5468 ice_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, KBUILD_MODNAME); 5469 if (!ice_wq) { 5470 pr_err("Failed to create workqueue\n"); 5471 return -ENOMEM; 5472 } 5473 5474 status = pci_register_driver(&ice_driver); 5475 if (status) { 5476 pr_err("failed to register PCI driver, err %d\n", status); 5477 destroy_workqueue(ice_wq); 5478 } 5479 5480 return status; 5481 } 5482 module_init(ice_module_init); 5483 5484 /** 5485 * ice_module_exit - Driver exit cleanup routine 5486 * 5487 * ice_module_exit is called just before the driver is removed 5488 * from memory. 5489 */ 5490 static void __exit ice_module_exit(void) 5491 { 5492 pci_unregister_driver(&ice_driver); 5493 destroy_workqueue(ice_wq); 5494 pr_info("module unloaded\n"); 5495 } 5496 module_exit(ice_module_exit); 5497 5498 /** 5499 * ice_set_mac_address - NDO callback to set MAC address 5500 * @netdev: network interface device structure 5501 * @pi: pointer to an address structure 5502 * 5503 * Returns 0 on success, negative on failure 5504 */ 5505 static int ice_set_mac_address(struct net_device *netdev, void *pi) 5506 { 5507 struct ice_netdev_priv *np = netdev_priv(netdev); 5508 struct ice_vsi *vsi = np->vsi; 5509 struct ice_pf *pf = vsi->back; 5510 struct ice_hw *hw = &pf->hw; 5511 struct sockaddr *addr = pi; 5512 u8 old_mac[ETH_ALEN]; 5513 u8 flags = 0; 5514 u8 *mac; 5515 int err; 5516 5517 mac = (u8 *)addr->sa_data; 5518 5519 if (!is_valid_ether_addr(mac)) 5520 return -EADDRNOTAVAIL; 5521 5522 if (ether_addr_equal(netdev->dev_addr, mac)) { 5523 netdev_dbg(netdev, "already using mac %pM\n", mac); 5524 return 0; 5525 } 5526 5527 if (test_bit(ICE_DOWN, pf->state) || 5528 ice_is_reset_in_progress(pf->state)) { 5529 netdev_err(netdev, "can't set mac %pM. device not ready\n", 5530 mac); 5531 return -EBUSY; 5532 } 5533 5534 if (ice_chnl_dmac_fltr_cnt(pf)) { 5535 netdev_err(netdev, "can't set mac %pM. Device has tc-flower filters, delete all of them and try again\n", 5536 mac); 5537 return -EAGAIN; 5538 } 5539 5540 netif_addr_lock_bh(netdev); 5541 ether_addr_copy(old_mac, netdev->dev_addr); 5542 /* change the netdev's MAC address */ 5543 eth_hw_addr_set(netdev, mac); 5544 netif_addr_unlock_bh(netdev); 5545 5546 /* Clean up old MAC filter. Not an error if old filter doesn't exist */ 5547 err = ice_fltr_remove_mac(vsi, old_mac, ICE_FWD_TO_VSI); 5548 if (err && err != -ENOENT) { 5549 err = -EADDRNOTAVAIL; 5550 goto err_update_filters; 5551 } 5552 5553 /* Add filter for new MAC. If filter exists, return success */ 5554 err = ice_fltr_add_mac(vsi, mac, ICE_FWD_TO_VSI); 5555 if (err == -EEXIST) { 5556 /* Although this MAC filter is already present in hardware it's 5557 * possible in some cases (e.g. bonding) that dev_addr was 5558 * modified outside of the driver and needs to be restored back 5559 * to this value. 5560 */ 5561 netdev_dbg(netdev, "filter for MAC %pM already exists\n", mac); 5562 5563 return 0; 5564 } else if (err) { 5565 /* error if the new filter addition failed */ 5566 err = -EADDRNOTAVAIL; 5567 } 5568 5569 err_update_filters: 5570 if (err) { 5571 netdev_err(netdev, "can't set MAC %pM. filter update failed\n", 5572 mac); 5573 netif_addr_lock_bh(netdev); 5574 eth_hw_addr_set(netdev, old_mac); 5575 netif_addr_unlock_bh(netdev); 5576 return err; 5577 } 5578 5579 netdev_dbg(vsi->netdev, "updated MAC address to %pM\n", 5580 netdev->dev_addr); 5581 5582 /* write new MAC address to the firmware */ 5583 flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL; 5584 err = ice_aq_manage_mac_write(hw, mac, flags, NULL); 5585 if (err) { 5586 netdev_err(netdev, "can't set MAC %pM. write to firmware failed error %d\n", 5587 mac, err); 5588 } 5589 return 0; 5590 } 5591 5592 /** 5593 * ice_set_rx_mode - NDO callback to set the netdev filters 5594 * @netdev: network interface device structure 5595 */ 5596 static void ice_set_rx_mode(struct net_device *netdev) 5597 { 5598 struct ice_netdev_priv *np = netdev_priv(netdev); 5599 struct ice_vsi *vsi = np->vsi; 5600 5601 if (!vsi) 5602 return; 5603 5604 /* Set the flags to synchronize filters 5605 * ndo_set_rx_mode may be triggered even without a change in netdev 5606 * flags 5607 */ 5608 set_bit(ICE_VSI_UMAC_FLTR_CHANGED, vsi->state); 5609 set_bit(ICE_VSI_MMAC_FLTR_CHANGED, vsi->state); 5610 set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags); 5611 5612 /* schedule our worker thread which will take care of 5613 * applying the new filter changes 5614 */ 5615 ice_service_task_schedule(vsi->back); 5616 } 5617 5618 /** 5619 * ice_set_tx_maxrate - NDO callback to set the maximum per-queue bitrate 5620 * @netdev: network interface device structure 5621 * @queue_index: Queue ID 5622 * @maxrate: maximum bandwidth in Mbps 5623 */ 5624 static int 5625 ice_set_tx_maxrate(struct net_device *netdev, int queue_index, u32 maxrate) 5626 { 5627 struct ice_netdev_priv *np = netdev_priv(netdev); 5628 struct ice_vsi *vsi = np->vsi; 5629 u16 q_handle; 5630 int status; 5631 u8 tc; 5632 5633 /* Validate maxrate requested is within permitted range */ 5634 if (maxrate && (maxrate > (ICE_SCHED_MAX_BW / 1000))) { 5635 netdev_err(netdev, "Invalid max rate %d specified for the queue %d\n", 5636 maxrate, queue_index); 5637 return -EINVAL; 5638 } 5639 5640 q_handle = vsi->tx_rings[queue_index]->q_handle; 5641 tc = ice_dcb_get_tc(vsi, queue_index); 5642 5643 /* Set BW back to default, when user set maxrate to 0 */ 5644 if (!maxrate) 5645 status = ice_cfg_q_bw_dflt_lmt(vsi->port_info, vsi->idx, tc, 5646 q_handle, ICE_MAX_BW); 5647 else 5648 status = ice_cfg_q_bw_lmt(vsi->port_info, vsi->idx, tc, 5649 q_handle, ICE_MAX_BW, maxrate * 1000); 5650 if (status) 5651 netdev_err(netdev, "Unable to set Tx max rate, error %d\n", 5652 status); 5653 5654 return status; 5655 } 5656 5657 /** 5658 * ice_fdb_add - add an entry to the hardware database 5659 * @ndm: the input from the stack 5660 * @tb: pointer to array of nladdr (unused) 5661 * @dev: the net device pointer 5662 * @addr: the MAC address entry being added 5663 * @vid: VLAN ID 5664 * @flags: instructions from stack about fdb operation 5665 * @extack: netlink extended ack 5666 */ 5667 static int 5668 ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[], 5669 struct net_device *dev, const unsigned char *addr, u16 vid, 5670 u16 flags, struct netlink_ext_ack __always_unused *extack) 5671 { 5672 int err; 5673 5674 if (vid) { 5675 netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n"); 5676 return -EINVAL; 5677 } 5678 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 5679 netdev_err(dev, "FDB only supports static addresses\n"); 5680 return -EINVAL; 5681 } 5682 5683 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 5684 err = dev_uc_add_excl(dev, addr); 5685 else if (is_multicast_ether_addr(addr)) 5686 err = dev_mc_add_excl(dev, addr); 5687 else 5688 err = -EINVAL; 5689 5690 /* Only return duplicate errors if NLM_F_EXCL is set */ 5691 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 5692 err = 0; 5693 5694 return err; 5695 } 5696 5697 /** 5698 * ice_fdb_del - delete an entry from the hardware database 5699 * @ndm: the input from the stack 5700 * @tb: pointer to array of nladdr (unused) 5701 * @dev: the net device pointer 5702 * @addr: the MAC address entry being added 5703 * @vid: VLAN ID 5704 * @extack: netlink extended ack 5705 */ 5706 static int 5707 ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[], 5708 struct net_device *dev, const unsigned char *addr, 5709 __always_unused u16 vid, struct netlink_ext_ack *extack) 5710 { 5711 int err; 5712 5713 if (ndm->ndm_state & NUD_PERMANENT) { 5714 netdev_err(dev, "FDB only supports static addresses\n"); 5715 return -EINVAL; 5716 } 5717 5718 if (is_unicast_ether_addr(addr)) 5719 err = dev_uc_del(dev, addr); 5720 else if (is_multicast_ether_addr(addr)) 5721 err = dev_mc_del(dev, addr); 5722 else 5723 err = -EINVAL; 5724 5725 return err; 5726 } 5727 5728 #define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 5729 NETIF_F_HW_VLAN_CTAG_TX | \ 5730 NETIF_F_HW_VLAN_STAG_RX | \ 5731 NETIF_F_HW_VLAN_STAG_TX) 5732 5733 #define NETIF_VLAN_STRIPPING_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \ 5734 NETIF_F_HW_VLAN_STAG_RX) 5735 5736 #define NETIF_VLAN_FILTERING_FEATURES (NETIF_F_HW_VLAN_CTAG_FILTER | \ 5737 NETIF_F_HW_VLAN_STAG_FILTER) 5738 5739 /** 5740 * ice_fix_features - fix the netdev features flags based on device limitations 5741 * @netdev: ptr to the netdev that flags are being fixed on 5742 * @features: features that need to be checked and possibly fixed 5743 * 5744 * Make sure any fixups are made to features in this callback. This enables the 5745 * driver to not have to check unsupported configurations throughout the driver 5746 * because that's the responsiblity of this callback. 5747 * 5748 * Single VLAN Mode (SVM) Supported Features: 5749 * NETIF_F_HW_VLAN_CTAG_FILTER 5750 * NETIF_F_HW_VLAN_CTAG_RX 5751 * NETIF_F_HW_VLAN_CTAG_TX 5752 * 5753 * Double VLAN Mode (DVM) Supported Features: 5754 * NETIF_F_HW_VLAN_CTAG_FILTER 5755 * NETIF_F_HW_VLAN_CTAG_RX 5756 * NETIF_F_HW_VLAN_CTAG_TX 5757 * 5758 * NETIF_F_HW_VLAN_STAG_FILTER 5759 * NETIF_HW_VLAN_STAG_RX 5760 * NETIF_HW_VLAN_STAG_TX 5761 * 5762 * Features that need fixing: 5763 * Cannot simultaneously enable CTAG and STAG stripping and/or insertion. 5764 * These are mutually exlusive as the VSI context cannot support multiple 5765 * VLAN ethertypes simultaneously for stripping and/or insertion. If this 5766 * is not done, then default to clearing the requested STAG offload 5767 * settings. 5768 * 5769 * All supported filtering has to be enabled or disabled together. For 5770 * example, in DVM, CTAG and STAG filtering have to be enabled and disabled 5771 * together. If this is not done, then default to VLAN filtering disabled. 5772 * These are mutually exclusive as there is currently no way to 5773 * enable/disable VLAN filtering based on VLAN ethertype when using VLAN 5774 * prune rules. 5775 */ 5776 static netdev_features_t 5777 ice_fix_features(struct net_device *netdev, netdev_features_t features) 5778 { 5779 struct ice_netdev_priv *np = netdev_priv(netdev); 5780 netdev_features_t req_vlan_fltr, cur_vlan_fltr; 5781 bool cur_ctag, cur_stag, req_ctag, req_stag; 5782 5783 cur_vlan_fltr = netdev->features & NETIF_VLAN_FILTERING_FEATURES; 5784 cur_ctag = cur_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER; 5785 cur_stag = cur_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER; 5786 5787 req_vlan_fltr = features & NETIF_VLAN_FILTERING_FEATURES; 5788 req_ctag = req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER; 5789 req_stag = req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER; 5790 5791 if (req_vlan_fltr != cur_vlan_fltr) { 5792 if (ice_is_dvm_ena(&np->vsi->back->hw)) { 5793 if (req_ctag && req_stag) { 5794 features |= NETIF_VLAN_FILTERING_FEATURES; 5795 } else if (!req_ctag && !req_stag) { 5796 features &= ~NETIF_VLAN_FILTERING_FEATURES; 5797 } else if ((!cur_ctag && req_ctag && !cur_stag) || 5798 (!cur_stag && req_stag && !cur_ctag)) { 5799 features |= NETIF_VLAN_FILTERING_FEATURES; 5800 netdev_warn(netdev, "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been enabled for both types.\n"); 5801 } else if ((cur_ctag && !req_ctag && cur_stag) || 5802 (cur_stag && !req_stag && cur_ctag)) { 5803 features &= ~NETIF_VLAN_FILTERING_FEATURES; 5804 netdev_warn(netdev, "802.1Q and 802.1ad VLAN filtering must be either both on or both off. VLAN filtering has been disabled for both types.\n"); 5805 } 5806 } else { 5807 if (req_vlan_fltr & NETIF_F_HW_VLAN_STAG_FILTER) 5808 netdev_warn(netdev, "cannot support requested 802.1ad filtering setting in SVM mode\n"); 5809 5810 if (req_vlan_fltr & NETIF_F_HW_VLAN_CTAG_FILTER) 5811 features |= NETIF_F_HW_VLAN_CTAG_FILTER; 5812 } 5813 } 5814 5815 if ((features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) && 5816 (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))) { 5817 netdev_warn(netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n"); 5818 features &= ~(NETIF_F_HW_VLAN_STAG_RX | 5819 NETIF_F_HW_VLAN_STAG_TX); 5820 } 5821 5822 if (!(netdev->features & NETIF_F_RXFCS) && 5823 (features & NETIF_F_RXFCS) && 5824 (features & NETIF_VLAN_STRIPPING_FEATURES) && 5825 !ice_vsi_has_non_zero_vlans(np->vsi)) { 5826 netdev_warn(netdev, "Disabling VLAN stripping as FCS/CRC stripping is also disabled and there is no VLAN configured\n"); 5827 features &= ~NETIF_VLAN_STRIPPING_FEATURES; 5828 } 5829 5830 return features; 5831 } 5832 5833 /** 5834 * ice_set_vlan_offload_features - set VLAN offload features for the PF VSI 5835 * @vsi: PF's VSI 5836 * @features: features used to determine VLAN offload settings 5837 * 5838 * First, determine the vlan_ethertype based on the VLAN offload bits in 5839 * features. Then determine if stripping and insertion should be enabled or 5840 * disabled. Finally enable or disable VLAN stripping and insertion. 5841 */ 5842 static int 5843 ice_set_vlan_offload_features(struct ice_vsi *vsi, netdev_features_t features) 5844 { 5845 bool enable_stripping = true, enable_insertion = true; 5846 struct ice_vsi_vlan_ops *vlan_ops; 5847 int strip_err = 0, insert_err = 0; 5848 u16 vlan_ethertype = 0; 5849 5850 vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 5851 5852 if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) 5853 vlan_ethertype = ETH_P_8021AD; 5854 else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) 5855 vlan_ethertype = ETH_P_8021Q; 5856 5857 if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX))) 5858 enable_stripping = false; 5859 if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX))) 5860 enable_insertion = false; 5861 5862 if (enable_stripping) 5863 strip_err = vlan_ops->ena_stripping(vsi, vlan_ethertype); 5864 else 5865 strip_err = vlan_ops->dis_stripping(vsi); 5866 5867 if (enable_insertion) 5868 insert_err = vlan_ops->ena_insertion(vsi, vlan_ethertype); 5869 else 5870 insert_err = vlan_ops->dis_insertion(vsi); 5871 5872 if (strip_err || insert_err) 5873 return -EIO; 5874 5875 return 0; 5876 } 5877 5878 /** 5879 * ice_set_vlan_filtering_features - set VLAN filtering features for the PF VSI 5880 * @vsi: PF's VSI 5881 * @features: features used to determine VLAN filtering settings 5882 * 5883 * Enable or disable Rx VLAN filtering based on the VLAN filtering bits in the 5884 * features. 5885 */ 5886 static int 5887 ice_set_vlan_filtering_features(struct ice_vsi *vsi, netdev_features_t features) 5888 { 5889 struct ice_vsi_vlan_ops *vlan_ops = ice_get_compat_vsi_vlan_ops(vsi); 5890 int err = 0; 5891 5892 /* support Single VLAN Mode (SVM) and Double VLAN Mode (DVM) by checking 5893 * if either bit is set 5894 */ 5895 if (features & 5896 (NETIF_F_HW_VLAN_CTAG_FILTER | NETIF_F_HW_VLAN_STAG_FILTER)) 5897 err = vlan_ops->ena_rx_filtering(vsi); 5898 else 5899 err = vlan_ops->dis_rx_filtering(vsi); 5900 5901 return err; 5902 } 5903 5904 /** 5905 * ice_set_vlan_features - set VLAN settings based on suggested feature set 5906 * @netdev: ptr to the netdev being adjusted 5907 * @features: the feature set that the stack is suggesting 5908 * 5909 * Only update VLAN settings if the requested_vlan_features are different than 5910 * the current_vlan_features. 5911 */ 5912 static int 5913 ice_set_vlan_features(struct net_device *netdev, netdev_features_t features) 5914 { 5915 netdev_features_t current_vlan_features, requested_vlan_features; 5916 struct ice_netdev_priv *np = netdev_priv(netdev); 5917 struct ice_vsi *vsi = np->vsi; 5918 int err; 5919 5920 current_vlan_features = netdev->features & NETIF_VLAN_OFFLOAD_FEATURES; 5921 requested_vlan_features = features & NETIF_VLAN_OFFLOAD_FEATURES; 5922 if (current_vlan_features ^ requested_vlan_features) { 5923 if ((features & NETIF_F_RXFCS) && 5924 (features & NETIF_VLAN_STRIPPING_FEATURES)) { 5925 dev_err(ice_pf_to_dev(vsi->back), 5926 "To enable VLAN stripping, you must first enable FCS/CRC stripping\n"); 5927 return -EIO; 5928 } 5929 5930 err = ice_set_vlan_offload_features(vsi, features); 5931 if (err) 5932 return err; 5933 } 5934 5935 current_vlan_features = netdev->features & 5936 NETIF_VLAN_FILTERING_FEATURES; 5937 requested_vlan_features = features & NETIF_VLAN_FILTERING_FEATURES; 5938 if (current_vlan_features ^ requested_vlan_features) { 5939 err = ice_set_vlan_filtering_features(vsi, features); 5940 if (err) 5941 return err; 5942 } 5943 5944 return 0; 5945 } 5946 5947 /** 5948 * ice_set_loopback - turn on/off loopback mode on underlying PF 5949 * @vsi: ptr to VSI 5950 * @ena: flag to indicate the on/off setting 5951 */ 5952 static int ice_set_loopback(struct ice_vsi *vsi, bool ena) 5953 { 5954 bool if_running = netif_running(vsi->netdev); 5955 int ret; 5956 5957 if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { 5958 ret = ice_down(vsi); 5959 if (ret) { 5960 netdev_err(vsi->netdev, "Preparing device to toggle loopback failed\n"); 5961 return ret; 5962 } 5963 } 5964 ret = ice_aq_set_mac_loopback(&vsi->back->hw, ena, NULL); 5965 if (ret) 5966 netdev_err(vsi->netdev, "Failed to toggle loopback state\n"); 5967 if (if_running) 5968 ret = ice_up(vsi); 5969 5970 return ret; 5971 } 5972 5973 /** 5974 * ice_set_features - set the netdev feature flags 5975 * @netdev: ptr to the netdev being adjusted 5976 * @features: the feature set that the stack is suggesting 5977 */ 5978 static int 5979 ice_set_features(struct net_device *netdev, netdev_features_t features) 5980 { 5981 netdev_features_t changed = netdev->features ^ features; 5982 struct ice_netdev_priv *np = netdev_priv(netdev); 5983 struct ice_vsi *vsi = np->vsi; 5984 struct ice_pf *pf = vsi->back; 5985 int ret = 0; 5986 5987 /* Don't set any netdev advanced features with device in Safe Mode */ 5988 if (ice_is_safe_mode(pf)) { 5989 dev_err(ice_pf_to_dev(pf), 5990 "Device is in Safe Mode - not enabling advanced netdev features\n"); 5991 return ret; 5992 } 5993 5994 /* Do not change setting during reset */ 5995 if (ice_is_reset_in_progress(pf->state)) { 5996 dev_err(ice_pf_to_dev(pf), 5997 "Device is resetting, changing advanced netdev features temporarily unavailable.\n"); 5998 return -EBUSY; 5999 } 6000 6001 /* Multiple features can be changed in one call so keep features in 6002 * separate if/else statements to guarantee each feature is checked 6003 */ 6004 if (changed & NETIF_F_RXHASH) 6005 ice_vsi_manage_rss_lut(vsi, !!(features & NETIF_F_RXHASH)); 6006 6007 ret = ice_set_vlan_features(netdev, features); 6008 if (ret) 6009 return ret; 6010 6011 /* Turn on receive of FCS aka CRC, and after setting this 6012 * flag the packet data will have the 4 byte CRC appended 6013 */ 6014 if (changed & NETIF_F_RXFCS) { 6015 if ((features & NETIF_F_RXFCS) && 6016 (features & NETIF_VLAN_STRIPPING_FEATURES)) { 6017 dev_err(ice_pf_to_dev(vsi->back), 6018 "To disable FCS/CRC stripping, you must first disable VLAN stripping\n"); 6019 return -EIO; 6020 } 6021 6022 ice_vsi_cfg_crc_strip(vsi, !!(features & NETIF_F_RXFCS)); 6023 ret = ice_down_up(vsi); 6024 if (ret) 6025 return ret; 6026 } 6027 6028 if (changed & NETIF_F_NTUPLE) { 6029 bool ena = !!(features & NETIF_F_NTUPLE); 6030 6031 ice_vsi_manage_fdir(vsi, ena); 6032 ena ? ice_init_arfs(vsi) : ice_clear_arfs(vsi); 6033 } 6034 6035 /* don't turn off hw_tc_offload when ADQ is already enabled */ 6036 if (!(features & NETIF_F_HW_TC) && ice_is_adq_active(pf)) { 6037 dev_err(ice_pf_to_dev(pf), "ADQ is active, can't turn hw_tc_offload off\n"); 6038 return -EACCES; 6039 } 6040 6041 if (changed & NETIF_F_HW_TC) { 6042 bool ena = !!(features & NETIF_F_HW_TC); 6043 6044 ena ? set_bit(ICE_FLAG_CLS_FLOWER, pf->flags) : 6045 clear_bit(ICE_FLAG_CLS_FLOWER, pf->flags); 6046 } 6047 6048 if (changed & NETIF_F_LOOPBACK) 6049 ret = ice_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK)); 6050 6051 return ret; 6052 } 6053 6054 /** 6055 * ice_vsi_vlan_setup - Setup VLAN offload properties on a PF VSI 6056 * @vsi: VSI to setup VLAN properties for 6057 */ 6058 static int ice_vsi_vlan_setup(struct ice_vsi *vsi) 6059 { 6060 int err; 6061 6062 err = ice_set_vlan_offload_features(vsi, vsi->netdev->features); 6063 if (err) 6064 return err; 6065 6066 err = ice_set_vlan_filtering_features(vsi, vsi->netdev->features); 6067 if (err) 6068 return err; 6069 6070 return ice_vsi_add_vlan_zero(vsi); 6071 } 6072 6073 /** 6074 * ice_vsi_cfg - Setup the VSI 6075 * @vsi: the VSI being configured 6076 * 6077 * Return 0 on success and negative value on error 6078 */ 6079 int ice_vsi_cfg(struct ice_vsi *vsi) 6080 { 6081 int err; 6082 6083 if (vsi->netdev) { 6084 ice_set_rx_mode(vsi->netdev); 6085 6086 if (vsi->type != ICE_VSI_LB) { 6087 err = ice_vsi_vlan_setup(vsi); 6088 6089 if (err) 6090 return err; 6091 } 6092 } 6093 ice_vsi_cfg_dcb_rings(vsi); 6094 6095 err = ice_vsi_cfg_lan_txqs(vsi); 6096 if (!err && ice_is_xdp_ena_vsi(vsi)) 6097 err = ice_vsi_cfg_xdp_txqs(vsi); 6098 if (!err) 6099 err = ice_vsi_cfg_rxqs(vsi); 6100 6101 return err; 6102 } 6103 6104 /* THEORY OF MODERATION: 6105 * The ice driver hardware works differently than the hardware that DIMLIB was 6106 * originally made for. ice hardware doesn't have packet count limits that 6107 * can trigger an interrupt, but it *does* have interrupt rate limit support, 6108 * which is hard-coded to a limit of 250,000 ints/second. 6109 * If not using dynamic moderation, the INTRL value can be modified 6110 * by ethtool rx-usecs-high. 6111 */ 6112 struct ice_dim { 6113 /* the throttle rate for interrupts, basically worst case delay before 6114 * an initial interrupt fires, value is stored in microseconds. 6115 */ 6116 u16 itr; 6117 }; 6118 6119 /* Make a different profile for Rx that doesn't allow quite so aggressive 6120 * moderation at the high end (it maxes out at 126us or about 8k interrupts a 6121 * second. 6122 */ 6123 static const struct ice_dim rx_profile[] = { 6124 {2}, /* 500,000 ints/s, capped at 250K by INTRL */ 6125 {8}, /* 125,000 ints/s */ 6126 {16}, /* 62,500 ints/s */ 6127 {62}, /* 16,129 ints/s */ 6128 {126} /* 7,936 ints/s */ 6129 }; 6130 6131 /* The transmit profile, which has the same sorts of values 6132 * as the previous struct 6133 */ 6134 static const struct ice_dim tx_profile[] = { 6135 {2}, /* 500,000 ints/s, capped at 250K by INTRL */ 6136 {8}, /* 125,000 ints/s */ 6137 {40}, /* 16,125 ints/s */ 6138 {128}, /* 7,812 ints/s */ 6139 {256} /* 3,906 ints/s */ 6140 }; 6141 6142 static void ice_tx_dim_work(struct work_struct *work) 6143 { 6144 struct ice_ring_container *rc; 6145 struct dim *dim; 6146 u16 itr; 6147 6148 dim = container_of(work, struct dim, work); 6149 rc = (struct ice_ring_container *)dim->priv; 6150 6151 WARN_ON(dim->profile_ix >= ARRAY_SIZE(tx_profile)); 6152 6153 /* look up the values in our local table */ 6154 itr = tx_profile[dim->profile_ix].itr; 6155 6156 ice_trace(tx_dim_work, container_of(rc, struct ice_q_vector, tx), dim); 6157 ice_write_itr(rc, itr); 6158 6159 dim->state = DIM_START_MEASURE; 6160 } 6161 6162 static void ice_rx_dim_work(struct work_struct *work) 6163 { 6164 struct ice_ring_container *rc; 6165 struct dim *dim; 6166 u16 itr; 6167 6168 dim = container_of(work, struct dim, work); 6169 rc = (struct ice_ring_container *)dim->priv; 6170 6171 WARN_ON(dim->profile_ix >= ARRAY_SIZE(rx_profile)); 6172 6173 /* look up the values in our local table */ 6174 itr = rx_profile[dim->profile_ix].itr; 6175 6176 ice_trace(rx_dim_work, container_of(rc, struct ice_q_vector, rx), dim); 6177 ice_write_itr(rc, itr); 6178 6179 dim->state = DIM_START_MEASURE; 6180 } 6181 6182 #define ICE_DIM_DEFAULT_PROFILE_IX 1 6183 6184 /** 6185 * ice_init_moderation - set up interrupt moderation 6186 * @q_vector: the vector containing rings to be configured 6187 * 6188 * Set up interrupt moderation registers, with the intent to do the right thing 6189 * when called from reset or from probe, and whether or not dynamic moderation 6190 * is enabled or not. Take special care to write all the registers in both 6191 * dynamic moderation mode or not in order to make sure hardware is in a known 6192 * state. 6193 */ 6194 static void ice_init_moderation(struct ice_q_vector *q_vector) 6195 { 6196 struct ice_ring_container *rc; 6197 bool tx_dynamic, rx_dynamic; 6198 6199 rc = &q_vector->tx; 6200 INIT_WORK(&rc->dim.work, ice_tx_dim_work); 6201 rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6202 rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX; 6203 rc->dim.priv = rc; 6204 tx_dynamic = ITR_IS_DYNAMIC(rc); 6205 6206 /* set the initial TX ITR to match the above */ 6207 ice_write_itr(rc, tx_dynamic ? 6208 tx_profile[rc->dim.profile_ix].itr : rc->itr_setting); 6209 6210 rc = &q_vector->rx; 6211 INIT_WORK(&rc->dim.work, ice_rx_dim_work); 6212 rc->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 6213 rc->dim.profile_ix = ICE_DIM_DEFAULT_PROFILE_IX; 6214 rc->dim.priv = rc; 6215 rx_dynamic = ITR_IS_DYNAMIC(rc); 6216 6217 /* set the initial RX ITR to match the above */ 6218 ice_write_itr(rc, rx_dynamic ? rx_profile[rc->dim.profile_ix].itr : 6219 rc->itr_setting); 6220 6221 ice_set_q_vector_intrl(q_vector); 6222 } 6223 6224 /** 6225 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI 6226 * @vsi: the VSI being configured 6227 */ 6228 static void ice_napi_enable_all(struct ice_vsi *vsi) 6229 { 6230 int q_idx; 6231 6232 if (!vsi->netdev) 6233 return; 6234 6235 ice_for_each_q_vector(vsi, q_idx) { 6236 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 6237 6238 ice_init_moderation(q_vector); 6239 6240 if (q_vector->rx.rx_ring || q_vector->tx.tx_ring) 6241 napi_enable(&q_vector->napi); 6242 } 6243 } 6244 6245 /** 6246 * ice_up_complete - Finish the last steps of bringing up a connection 6247 * @vsi: The VSI being configured 6248 * 6249 * Return 0 on success and negative value on error 6250 */ 6251 static int ice_up_complete(struct ice_vsi *vsi) 6252 { 6253 struct ice_pf *pf = vsi->back; 6254 int err; 6255 6256 ice_vsi_cfg_msix(vsi); 6257 6258 /* Enable only Rx rings, Tx rings were enabled by the FW when the 6259 * Tx queue group list was configured and the context bits were 6260 * programmed using ice_vsi_cfg_txqs 6261 */ 6262 err = ice_vsi_start_all_rx_rings(vsi); 6263 if (err) 6264 return err; 6265 6266 clear_bit(ICE_VSI_DOWN, vsi->state); 6267 ice_napi_enable_all(vsi); 6268 ice_vsi_ena_irq(vsi); 6269 6270 if (vsi->port_info && 6271 (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) && 6272 vsi->netdev) { 6273 ice_print_link_msg(vsi, true); 6274 netif_tx_start_all_queues(vsi->netdev); 6275 netif_carrier_on(vsi->netdev); 6276 if (!ice_is_e810(&pf->hw)) 6277 ice_ptp_link_change(pf, pf->hw.pf_id, true); 6278 } 6279 6280 /* Perform an initial read of the statistics registers now to 6281 * set the baseline so counters are ready when interface is up 6282 */ 6283 ice_update_eth_stats(vsi); 6284 ice_service_task_schedule(pf); 6285 6286 return 0; 6287 } 6288 6289 /** 6290 * ice_up - Bring the connection back up after being down 6291 * @vsi: VSI being configured 6292 */ 6293 int ice_up(struct ice_vsi *vsi) 6294 { 6295 int err; 6296 6297 err = ice_vsi_cfg(vsi); 6298 if (!err) 6299 err = ice_up_complete(vsi); 6300 6301 return err; 6302 } 6303 6304 /** 6305 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring 6306 * @syncp: pointer to u64_stats_sync 6307 * @stats: stats that pkts and bytes count will be taken from 6308 * @pkts: packets stats counter 6309 * @bytes: bytes stats counter 6310 * 6311 * This function fetches stats from the ring considering the atomic operations 6312 * that needs to be performed to read u64 values in 32 bit machine. 6313 */ 6314 void 6315 ice_fetch_u64_stats_per_ring(struct u64_stats_sync *syncp, 6316 struct ice_q_stats stats, u64 *pkts, u64 *bytes) 6317 { 6318 unsigned int start; 6319 6320 do { 6321 start = u64_stats_fetch_begin_irq(syncp); 6322 *pkts = stats.pkts; 6323 *bytes = stats.bytes; 6324 } while (u64_stats_fetch_retry_irq(syncp, start)); 6325 } 6326 6327 /** 6328 * ice_update_vsi_tx_ring_stats - Update VSI Tx ring stats counters 6329 * @vsi: the VSI to be updated 6330 * @vsi_stats: the stats struct to be updated 6331 * @rings: rings to work on 6332 * @count: number of rings 6333 */ 6334 static void 6335 ice_update_vsi_tx_ring_stats(struct ice_vsi *vsi, 6336 struct rtnl_link_stats64 *vsi_stats, 6337 struct ice_tx_ring **rings, u16 count) 6338 { 6339 u16 i; 6340 6341 for (i = 0; i < count; i++) { 6342 struct ice_tx_ring *ring; 6343 u64 pkts = 0, bytes = 0; 6344 6345 ring = READ_ONCE(rings[i]); 6346 if (!ring) 6347 continue; 6348 ice_fetch_u64_stats_per_ring(&ring->syncp, ring->stats, &pkts, &bytes); 6349 vsi_stats->tx_packets += pkts; 6350 vsi_stats->tx_bytes += bytes; 6351 vsi->tx_restart += ring->tx_stats.restart_q; 6352 vsi->tx_busy += ring->tx_stats.tx_busy; 6353 vsi->tx_linearize += ring->tx_stats.tx_linearize; 6354 } 6355 } 6356 6357 /** 6358 * ice_update_vsi_ring_stats - Update VSI stats counters 6359 * @vsi: the VSI to be updated 6360 */ 6361 static void ice_update_vsi_ring_stats(struct ice_vsi *vsi) 6362 { 6363 struct rtnl_link_stats64 *vsi_stats; 6364 u64 pkts, bytes; 6365 int i; 6366 6367 vsi_stats = kzalloc(sizeof(*vsi_stats), GFP_ATOMIC); 6368 if (!vsi_stats) 6369 return; 6370 6371 /* reset non-netdev (extended) stats */ 6372 vsi->tx_restart = 0; 6373 vsi->tx_busy = 0; 6374 vsi->tx_linearize = 0; 6375 vsi->rx_buf_failed = 0; 6376 vsi->rx_page_failed = 0; 6377 6378 rcu_read_lock(); 6379 6380 /* update Tx rings counters */ 6381 ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->tx_rings, 6382 vsi->num_txq); 6383 6384 /* update Rx rings counters */ 6385 ice_for_each_rxq(vsi, i) { 6386 struct ice_rx_ring *ring = READ_ONCE(vsi->rx_rings[i]); 6387 6388 ice_fetch_u64_stats_per_ring(&ring->syncp, ring->stats, &pkts, &bytes); 6389 vsi_stats->rx_packets += pkts; 6390 vsi_stats->rx_bytes += bytes; 6391 vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed; 6392 vsi->rx_page_failed += ring->rx_stats.alloc_page_failed; 6393 } 6394 6395 /* update XDP Tx rings counters */ 6396 if (ice_is_xdp_ena_vsi(vsi)) 6397 ice_update_vsi_tx_ring_stats(vsi, vsi_stats, vsi->xdp_rings, 6398 vsi->num_xdp_txq); 6399 6400 rcu_read_unlock(); 6401 6402 vsi->net_stats.tx_packets = vsi_stats->tx_packets; 6403 vsi->net_stats.tx_bytes = vsi_stats->tx_bytes; 6404 vsi->net_stats.rx_packets = vsi_stats->rx_packets; 6405 vsi->net_stats.rx_bytes = vsi_stats->rx_bytes; 6406 6407 kfree(vsi_stats); 6408 } 6409 6410 /** 6411 * ice_update_vsi_stats - Update VSI stats counters 6412 * @vsi: the VSI to be updated 6413 */ 6414 void ice_update_vsi_stats(struct ice_vsi *vsi) 6415 { 6416 struct rtnl_link_stats64 *cur_ns = &vsi->net_stats; 6417 struct ice_eth_stats *cur_es = &vsi->eth_stats; 6418 struct ice_pf *pf = vsi->back; 6419 6420 if (test_bit(ICE_VSI_DOWN, vsi->state) || 6421 test_bit(ICE_CFG_BUSY, pf->state)) 6422 return; 6423 6424 /* get stats as recorded by Tx/Rx rings */ 6425 ice_update_vsi_ring_stats(vsi); 6426 6427 /* get VSI stats as recorded by the hardware */ 6428 ice_update_eth_stats(vsi); 6429 6430 cur_ns->tx_errors = cur_es->tx_errors; 6431 cur_ns->rx_dropped = cur_es->rx_discards; 6432 cur_ns->tx_dropped = cur_es->tx_discards; 6433 cur_ns->multicast = cur_es->rx_multicast; 6434 6435 /* update some more netdev stats if this is main VSI */ 6436 if (vsi->type == ICE_VSI_PF) { 6437 cur_ns->rx_crc_errors = pf->stats.crc_errors; 6438 cur_ns->rx_errors = pf->stats.crc_errors + 6439 pf->stats.illegal_bytes + 6440 pf->stats.rx_len_errors + 6441 pf->stats.rx_undersize + 6442 pf->hw_csum_rx_error + 6443 pf->stats.rx_jabber + 6444 pf->stats.rx_fragments + 6445 pf->stats.rx_oversize; 6446 cur_ns->rx_length_errors = pf->stats.rx_len_errors; 6447 /* record drops from the port level */ 6448 cur_ns->rx_missed_errors = pf->stats.eth.rx_discards; 6449 } 6450 } 6451 6452 /** 6453 * ice_update_pf_stats - Update PF port stats counters 6454 * @pf: PF whose stats needs to be updated 6455 */ 6456 void ice_update_pf_stats(struct ice_pf *pf) 6457 { 6458 struct ice_hw_port_stats *prev_ps, *cur_ps; 6459 struct ice_hw *hw = &pf->hw; 6460 u16 fd_ctr_base; 6461 u8 port; 6462 6463 port = hw->port_info->lport; 6464 prev_ps = &pf->stats_prev; 6465 cur_ps = &pf->stats; 6466 6467 ice_stat_update40(hw, GLPRT_GORCL(port), pf->stat_prev_loaded, 6468 &prev_ps->eth.rx_bytes, 6469 &cur_ps->eth.rx_bytes); 6470 6471 ice_stat_update40(hw, GLPRT_UPRCL(port), pf->stat_prev_loaded, 6472 &prev_ps->eth.rx_unicast, 6473 &cur_ps->eth.rx_unicast); 6474 6475 ice_stat_update40(hw, GLPRT_MPRCL(port), pf->stat_prev_loaded, 6476 &prev_ps->eth.rx_multicast, 6477 &cur_ps->eth.rx_multicast); 6478 6479 ice_stat_update40(hw, GLPRT_BPRCL(port), pf->stat_prev_loaded, 6480 &prev_ps->eth.rx_broadcast, 6481 &cur_ps->eth.rx_broadcast); 6482 6483 ice_stat_update32(hw, PRTRPB_RDPC, pf->stat_prev_loaded, 6484 &prev_ps->eth.rx_discards, 6485 &cur_ps->eth.rx_discards); 6486 6487 ice_stat_update40(hw, GLPRT_GOTCL(port), pf->stat_prev_loaded, 6488 &prev_ps->eth.tx_bytes, 6489 &cur_ps->eth.tx_bytes); 6490 6491 ice_stat_update40(hw, GLPRT_UPTCL(port), pf->stat_prev_loaded, 6492 &prev_ps->eth.tx_unicast, 6493 &cur_ps->eth.tx_unicast); 6494 6495 ice_stat_update40(hw, GLPRT_MPTCL(port), pf->stat_prev_loaded, 6496 &prev_ps->eth.tx_multicast, 6497 &cur_ps->eth.tx_multicast); 6498 6499 ice_stat_update40(hw, GLPRT_BPTCL(port), pf->stat_prev_loaded, 6500 &prev_ps->eth.tx_broadcast, 6501 &cur_ps->eth.tx_broadcast); 6502 6503 ice_stat_update32(hw, GLPRT_TDOLD(port), pf->stat_prev_loaded, 6504 &prev_ps->tx_dropped_link_down, 6505 &cur_ps->tx_dropped_link_down); 6506 6507 ice_stat_update40(hw, GLPRT_PRC64L(port), pf->stat_prev_loaded, 6508 &prev_ps->rx_size_64, &cur_ps->rx_size_64); 6509 6510 ice_stat_update40(hw, GLPRT_PRC127L(port), pf->stat_prev_loaded, 6511 &prev_ps->rx_size_127, &cur_ps->rx_size_127); 6512 6513 ice_stat_update40(hw, GLPRT_PRC255L(port), pf->stat_prev_loaded, 6514 &prev_ps->rx_size_255, &cur_ps->rx_size_255); 6515 6516 ice_stat_update40(hw, GLPRT_PRC511L(port), pf->stat_prev_loaded, 6517 &prev_ps->rx_size_511, &cur_ps->rx_size_511); 6518 6519 ice_stat_update40(hw, GLPRT_PRC1023L(port), pf->stat_prev_loaded, 6520 &prev_ps->rx_size_1023, &cur_ps->rx_size_1023); 6521 6522 ice_stat_update40(hw, GLPRT_PRC1522L(port), pf->stat_prev_loaded, 6523 &prev_ps->rx_size_1522, &cur_ps->rx_size_1522); 6524 6525 ice_stat_update40(hw, GLPRT_PRC9522L(port), pf->stat_prev_loaded, 6526 &prev_ps->rx_size_big, &cur_ps->rx_size_big); 6527 6528 ice_stat_update40(hw, GLPRT_PTC64L(port), pf->stat_prev_loaded, 6529 &prev_ps->tx_size_64, &cur_ps->tx_size_64); 6530 6531 ice_stat_update40(hw, GLPRT_PTC127L(port), pf->stat_prev_loaded, 6532 &prev_ps->tx_size_127, &cur_ps->tx_size_127); 6533 6534 ice_stat_update40(hw, GLPRT_PTC255L(port), pf->stat_prev_loaded, 6535 &prev_ps->tx_size_255, &cur_ps->tx_size_255); 6536 6537 ice_stat_update40(hw, GLPRT_PTC511L(port), pf->stat_prev_loaded, 6538 &prev_ps->tx_size_511, &cur_ps->tx_size_511); 6539 6540 ice_stat_update40(hw, GLPRT_PTC1023L(port), pf->stat_prev_loaded, 6541 &prev_ps->tx_size_1023, &cur_ps->tx_size_1023); 6542 6543 ice_stat_update40(hw, GLPRT_PTC1522L(port), pf->stat_prev_loaded, 6544 &prev_ps->tx_size_1522, &cur_ps->tx_size_1522); 6545 6546 ice_stat_update40(hw, GLPRT_PTC9522L(port), pf->stat_prev_loaded, 6547 &prev_ps->tx_size_big, &cur_ps->tx_size_big); 6548 6549 fd_ctr_base = hw->fd_ctr_base; 6550 6551 ice_stat_update40(hw, 6552 GLSTAT_FD_CNT0L(ICE_FD_SB_STAT_IDX(fd_ctr_base)), 6553 pf->stat_prev_loaded, &prev_ps->fd_sb_match, 6554 &cur_ps->fd_sb_match); 6555 ice_stat_update32(hw, GLPRT_LXONRXC(port), pf->stat_prev_loaded, 6556 &prev_ps->link_xon_rx, &cur_ps->link_xon_rx); 6557 6558 ice_stat_update32(hw, GLPRT_LXOFFRXC(port), pf->stat_prev_loaded, 6559 &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx); 6560 6561 ice_stat_update32(hw, GLPRT_LXONTXC(port), pf->stat_prev_loaded, 6562 &prev_ps->link_xon_tx, &cur_ps->link_xon_tx); 6563 6564 ice_stat_update32(hw, GLPRT_LXOFFTXC(port), pf->stat_prev_loaded, 6565 &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx); 6566 6567 ice_update_dcb_stats(pf); 6568 6569 ice_stat_update32(hw, GLPRT_CRCERRS(port), pf->stat_prev_loaded, 6570 &prev_ps->crc_errors, &cur_ps->crc_errors); 6571 6572 ice_stat_update32(hw, GLPRT_ILLERRC(port), pf->stat_prev_loaded, 6573 &prev_ps->illegal_bytes, &cur_ps->illegal_bytes); 6574 6575 ice_stat_update32(hw, GLPRT_MLFC(port), pf->stat_prev_loaded, 6576 &prev_ps->mac_local_faults, 6577 &cur_ps->mac_local_faults); 6578 6579 ice_stat_update32(hw, GLPRT_MRFC(port), pf->stat_prev_loaded, 6580 &prev_ps->mac_remote_faults, 6581 &cur_ps->mac_remote_faults); 6582 6583 ice_stat_update32(hw, GLPRT_RLEC(port), pf->stat_prev_loaded, 6584 &prev_ps->rx_len_errors, &cur_ps->rx_len_errors); 6585 6586 ice_stat_update32(hw, GLPRT_RUC(port), pf->stat_prev_loaded, 6587 &prev_ps->rx_undersize, &cur_ps->rx_undersize); 6588 6589 ice_stat_update32(hw, GLPRT_RFC(port), pf->stat_prev_loaded, 6590 &prev_ps->rx_fragments, &cur_ps->rx_fragments); 6591 6592 ice_stat_update32(hw, GLPRT_ROC(port), pf->stat_prev_loaded, 6593 &prev_ps->rx_oversize, &cur_ps->rx_oversize); 6594 6595 ice_stat_update32(hw, GLPRT_RJC(port), pf->stat_prev_loaded, 6596 &prev_ps->rx_jabber, &cur_ps->rx_jabber); 6597 6598 cur_ps->fd_sb_status = test_bit(ICE_FLAG_FD_ENA, pf->flags) ? 1 : 0; 6599 6600 pf->stat_prev_loaded = true; 6601 } 6602 6603 /** 6604 * ice_get_stats64 - get statistics for network device structure 6605 * @netdev: network interface device structure 6606 * @stats: main device statistics structure 6607 */ 6608 static 6609 void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) 6610 { 6611 struct ice_netdev_priv *np = netdev_priv(netdev); 6612 struct rtnl_link_stats64 *vsi_stats; 6613 struct ice_vsi *vsi = np->vsi; 6614 6615 vsi_stats = &vsi->net_stats; 6616 6617 if (!vsi->num_txq || !vsi->num_rxq) 6618 return; 6619 6620 /* netdev packet/byte stats come from ring counter. These are obtained 6621 * by summing up ring counters (done by ice_update_vsi_ring_stats). 6622 * But, only call the update routine and read the registers if VSI is 6623 * not down. 6624 */ 6625 if (!test_bit(ICE_VSI_DOWN, vsi->state)) 6626 ice_update_vsi_ring_stats(vsi); 6627 stats->tx_packets = vsi_stats->tx_packets; 6628 stats->tx_bytes = vsi_stats->tx_bytes; 6629 stats->rx_packets = vsi_stats->rx_packets; 6630 stats->rx_bytes = vsi_stats->rx_bytes; 6631 6632 /* The rest of the stats can be read from the hardware but instead we 6633 * just return values that the watchdog task has already obtained from 6634 * the hardware. 6635 */ 6636 stats->multicast = vsi_stats->multicast; 6637 stats->tx_errors = vsi_stats->tx_errors; 6638 stats->tx_dropped = vsi_stats->tx_dropped; 6639 stats->rx_errors = vsi_stats->rx_errors; 6640 stats->rx_dropped = vsi_stats->rx_dropped; 6641 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 6642 stats->rx_length_errors = vsi_stats->rx_length_errors; 6643 } 6644 6645 /** 6646 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI 6647 * @vsi: VSI having NAPI disabled 6648 */ 6649 static void ice_napi_disable_all(struct ice_vsi *vsi) 6650 { 6651 int q_idx; 6652 6653 if (!vsi->netdev) 6654 return; 6655 6656 ice_for_each_q_vector(vsi, q_idx) { 6657 struct ice_q_vector *q_vector = vsi->q_vectors[q_idx]; 6658 6659 if (q_vector->rx.rx_ring || q_vector->tx.tx_ring) 6660 napi_disable(&q_vector->napi); 6661 6662 cancel_work_sync(&q_vector->tx.dim.work); 6663 cancel_work_sync(&q_vector->rx.dim.work); 6664 } 6665 } 6666 6667 /** 6668 * ice_down - Shutdown the connection 6669 * @vsi: The VSI being stopped 6670 * 6671 * Caller of this function is expected to set the vsi->state ICE_DOWN bit 6672 */ 6673 int ice_down(struct ice_vsi *vsi) 6674 { 6675 int i, tx_err, rx_err, link_err = 0, vlan_err = 0; 6676 6677 WARN_ON(!test_bit(ICE_VSI_DOWN, vsi->state)); 6678 6679 if (vsi->netdev && vsi->type == ICE_VSI_PF) { 6680 vlan_err = ice_vsi_del_vlan_zero(vsi); 6681 if (!ice_is_e810(&vsi->back->hw)) 6682 ice_ptp_link_change(vsi->back, vsi->back->hw.pf_id, false); 6683 netif_carrier_off(vsi->netdev); 6684 netif_tx_disable(vsi->netdev); 6685 } else if (vsi->type == ICE_VSI_SWITCHDEV_CTRL) { 6686 ice_eswitch_stop_all_tx_queues(vsi->back); 6687 } 6688 6689 ice_vsi_dis_irq(vsi); 6690 6691 tx_err = ice_vsi_stop_lan_tx_rings(vsi, ICE_NO_RESET, 0); 6692 if (tx_err) 6693 netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n", 6694 vsi->vsi_num, tx_err); 6695 if (!tx_err && ice_is_xdp_ena_vsi(vsi)) { 6696 tx_err = ice_vsi_stop_xdp_tx_rings(vsi); 6697 if (tx_err) 6698 netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n", 6699 vsi->vsi_num, tx_err); 6700 } 6701 6702 rx_err = ice_vsi_stop_all_rx_rings(vsi); 6703 if (rx_err) 6704 netdev_err(vsi->netdev, "Failed stop Rx rings, VSI %d error %d\n", 6705 vsi->vsi_num, rx_err); 6706 6707 ice_napi_disable_all(vsi); 6708 6709 if (test_bit(ICE_FLAG_LINK_DOWN_ON_CLOSE_ENA, vsi->back->flags)) { 6710 link_err = ice_force_phys_link_state(vsi, false); 6711 if (link_err) 6712 netdev_err(vsi->netdev, "Failed to set physical link down, VSI %d error %d\n", 6713 vsi->vsi_num, link_err); 6714 } 6715 6716 ice_for_each_txq(vsi, i) 6717 ice_clean_tx_ring(vsi->tx_rings[i]); 6718 6719 ice_for_each_rxq(vsi, i) 6720 ice_clean_rx_ring(vsi->rx_rings[i]); 6721 6722 if (tx_err || rx_err || link_err || vlan_err) { 6723 netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n", 6724 vsi->vsi_num, vsi->vsw->sw_id); 6725 return -EIO; 6726 } 6727 6728 return 0; 6729 } 6730 6731 /** 6732 * ice_down_up - shutdown the VSI connection and bring it up 6733 * @vsi: the VSI to be reconnected 6734 */ 6735 int ice_down_up(struct ice_vsi *vsi) 6736 { 6737 int ret; 6738 6739 /* if DOWN already set, nothing to do */ 6740 if (test_and_set_bit(ICE_VSI_DOWN, vsi->state)) 6741 return 0; 6742 6743 ret = ice_down(vsi); 6744 if (ret) 6745 return ret; 6746 6747 ret = ice_up(vsi); 6748 if (ret) { 6749 netdev_err(vsi->netdev, "reallocating resources failed during netdev features change, may need to reload driver\n"); 6750 return ret; 6751 } 6752 6753 return 0; 6754 } 6755 6756 /** 6757 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources 6758 * @vsi: VSI having resources allocated 6759 * 6760 * Return 0 on success, negative on failure 6761 */ 6762 int ice_vsi_setup_tx_rings(struct ice_vsi *vsi) 6763 { 6764 int i, err = 0; 6765 6766 if (!vsi->num_txq) { 6767 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Tx queues\n", 6768 vsi->vsi_num); 6769 return -EINVAL; 6770 } 6771 6772 ice_for_each_txq(vsi, i) { 6773 struct ice_tx_ring *ring = vsi->tx_rings[i]; 6774 6775 if (!ring) 6776 return -EINVAL; 6777 6778 if (vsi->netdev) 6779 ring->netdev = vsi->netdev; 6780 err = ice_setup_tx_ring(ring); 6781 if (err) 6782 break; 6783 } 6784 6785 return err; 6786 } 6787 6788 /** 6789 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources 6790 * @vsi: VSI having resources allocated 6791 * 6792 * Return 0 on success, negative on failure 6793 */ 6794 int ice_vsi_setup_rx_rings(struct ice_vsi *vsi) 6795 { 6796 int i, err = 0; 6797 6798 if (!vsi->num_rxq) { 6799 dev_err(ice_pf_to_dev(vsi->back), "VSI %d has 0 Rx queues\n", 6800 vsi->vsi_num); 6801 return -EINVAL; 6802 } 6803 6804 ice_for_each_rxq(vsi, i) { 6805 struct ice_rx_ring *ring = vsi->rx_rings[i]; 6806 6807 if (!ring) 6808 return -EINVAL; 6809 6810 if (vsi->netdev) 6811 ring->netdev = vsi->netdev; 6812 err = ice_setup_rx_ring(ring); 6813 if (err) 6814 break; 6815 } 6816 6817 return err; 6818 } 6819 6820 /** 6821 * ice_vsi_open_ctrl - open control VSI for use 6822 * @vsi: the VSI to open 6823 * 6824 * Initialization of the Control VSI 6825 * 6826 * Returns 0 on success, negative value on error 6827 */ 6828 int ice_vsi_open_ctrl(struct ice_vsi *vsi) 6829 { 6830 char int_name[ICE_INT_NAME_STR_LEN]; 6831 struct ice_pf *pf = vsi->back; 6832 struct device *dev; 6833 int err; 6834 6835 dev = ice_pf_to_dev(pf); 6836 /* allocate descriptors */ 6837 err = ice_vsi_setup_tx_rings(vsi); 6838 if (err) 6839 goto err_setup_tx; 6840 6841 err = ice_vsi_setup_rx_rings(vsi); 6842 if (err) 6843 goto err_setup_rx; 6844 6845 err = ice_vsi_cfg(vsi); 6846 if (err) 6847 goto err_setup_rx; 6848 6849 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:ctrl", 6850 dev_driver_string(dev), dev_name(dev)); 6851 err = ice_vsi_req_irq_msix(vsi, int_name); 6852 if (err) 6853 goto err_setup_rx; 6854 6855 ice_vsi_cfg_msix(vsi); 6856 6857 err = ice_vsi_start_all_rx_rings(vsi); 6858 if (err) 6859 goto err_up_complete; 6860 6861 clear_bit(ICE_VSI_DOWN, vsi->state); 6862 ice_vsi_ena_irq(vsi); 6863 6864 return 0; 6865 6866 err_up_complete: 6867 ice_down(vsi); 6868 err_setup_rx: 6869 ice_vsi_free_rx_rings(vsi); 6870 err_setup_tx: 6871 ice_vsi_free_tx_rings(vsi); 6872 6873 return err; 6874 } 6875 6876 /** 6877 * ice_vsi_open - Called when a network interface is made active 6878 * @vsi: the VSI to open 6879 * 6880 * Initialization of the VSI 6881 * 6882 * Returns 0 on success, negative value on error 6883 */ 6884 int ice_vsi_open(struct ice_vsi *vsi) 6885 { 6886 char int_name[ICE_INT_NAME_STR_LEN]; 6887 struct ice_pf *pf = vsi->back; 6888 int err; 6889 6890 /* allocate descriptors */ 6891 err = ice_vsi_setup_tx_rings(vsi); 6892 if (err) 6893 goto err_setup_tx; 6894 6895 err = ice_vsi_setup_rx_rings(vsi); 6896 if (err) 6897 goto err_setup_rx; 6898 6899 err = ice_vsi_cfg(vsi); 6900 if (err) 6901 goto err_setup_rx; 6902 6903 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 6904 dev_driver_string(ice_pf_to_dev(pf)), vsi->netdev->name); 6905 err = ice_vsi_req_irq_msix(vsi, int_name); 6906 if (err) 6907 goto err_setup_rx; 6908 6909 if (vsi->type == ICE_VSI_PF) { 6910 /* Notify the stack of the actual queue counts. */ 6911 err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); 6912 if (err) 6913 goto err_set_qs; 6914 6915 err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq); 6916 if (err) 6917 goto err_set_qs; 6918 } 6919 6920 err = ice_up_complete(vsi); 6921 if (err) 6922 goto err_up_complete; 6923 6924 return 0; 6925 6926 err_up_complete: 6927 ice_down(vsi); 6928 err_set_qs: 6929 ice_vsi_free_irq(vsi); 6930 err_setup_rx: 6931 ice_vsi_free_rx_rings(vsi); 6932 err_setup_tx: 6933 ice_vsi_free_tx_rings(vsi); 6934 6935 return err; 6936 } 6937 6938 /** 6939 * ice_vsi_release_all - Delete all VSIs 6940 * @pf: PF from which all VSIs are being removed 6941 */ 6942 static void ice_vsi_release_all(struct ice_pf *pf) 6943 { 6944 int err, i; 6945 6946 if (!pf->vsi) 6947 return; 6948 6949 ice_for_each_vsi(pf, i) { 6950 if (!pf->vsi[i]) 6951 continue; 6952 6953 if (pf->vsi[i]->type == ICE_VSI_CHNL) 6954 continue; 6955 6956 err = ice_vsi_release(pf->vsi[i]); 6957 if (err) 6958 dev_dbg(ice_pf_to_dev(pf), "Failed to release pf->vsi[%d], err %d, vsi_num = %d\n", 6959 i, err, pf->vsi[i]->vsi_num); 6960 } 6961 } 6962 6963 /** 6964 * ice_vsi_rebuild_by_type - Rebuild VSI of a given type 6965 * @pf: pointer to the PF instance 6966 * @type: VSI type to rebuild 6967 * 6968 * Iterates through the pf->vsi array and rebuilds VSIs of the requested type 6969 */ 6970 static int ice_vsi_rebuild_by_type(struct ice_pf *pf, enum ice_vsi_type type) 6971 { 6972 struct device *dev = ice_pf_to_dev(pf); 6973 int i, err; 6974 6975 ice_for_each_vsi(pf, i) { 6976 struct ice_vsi *vsi = pf->vsi[i]; 6977 6978 if (!vsi || vsi->type != type) 6979 continue; 6980 6981 /* rebuild the VSI */ 6982 err = ice_vsi_rebuild(vsi, true); 6983 if (err) { 6984 dev_err(dev, "rebuild VSI failed, err %d, VSI index %d, type %s\n", 6985 err, vsi->idx, ice_vsi_type_str(type)); 6986 return err; 6987 } 6988 6989 /* replay filters for the VSI */ 6990 err = ice_replay_vsi(&pf->hw, vsi->idx); 6991 if (err) { 6992 dev_err(dev, "replay VSI failed, error %d, VSI index %d, type %s\n", 6993 err, vsi->idx, ice_vsi_type_str(type)); 6994 return err; 6995 } 6996 6997 /* Re-map HW VSI number, using VSI handle that has been 6998 * previously validated in ice_replay_vsi() call above 6999 */ 7000 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 7001 7002 /* enable the VSI */ 7003 err = ice_ena_vsi(vsi, false); 7004 if (err) { 7005 dev_err(dev, "enable VSI failed, err %d, VSI index %d, type %s\n", 7006 err, vsi->idx, ice_vsi_type_str(type)); 7007 return err; 7008 } 7009 7010 dev_info(dev, "VSI rebuilt. VSI index %d, type %s\n", vsi->idx, 7011 ice_vsi_type_str(type)); 7012 } 7013 7014 return 0; 7015 } 7016 7017 /** 7018 * ice_update_pf_netdev_link - Update PF netdev link status 7019 * @pf: pointer to the PF instance 7020 */ 7021 static void ice_update_pf_netdev_link(struct ice_pf *pf) 7022 { 7023 bool link_up; 7024 int i; 7025 7026 ice_for_each_vsi(pf, i) { 7027 struct ice_vsi *vsi = pf->vsi[i]; 7028 7029 if (!vsi || vsi->type != ICE_VSI_PF) 7030 return; 7031 7032 ice_get_link_status(pf->vsi[i]->port_info, &link_up); 7033 if (link_up) { 7034 netif_carrier_on(pf->vsi[i]->netdev); 7035 netif_tx_wake_all_queues(pf->vsi[i]->netdev); 7036 } else { 7037 netif_carrier_off(pf->vsi[i]->netdev); 7038 netif_tx_stop_all_queues(pf->vsi[i]->netdev); 7039 } 7040 } 7041 } 7042 7043 /** 7044 * ice_rebuild - rebuild after reset 7045 * @pf: PF to rebuild 7046 * @reset_type: type of reset 7047 * 7048 * Do not rebuild VF VSI in this flow because that is already handled via 7049 * ice_reset_all_vfs(). This is because requirements for resetting a VF after a 7050 * PFR/CORER/GLOBER/etc. are different than the normal flow. Also, we don't want 7051 * to reset/rebuild all the VF VSI twice. 7052 */ 7053 static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) 7054 { 7055 struct device *dev = ice_pf_to_dev(pf); 7056 struct ice_hw *hw = &pf->hw; 7057 bool dvm; 7058 int err; 7059 7060 if (test_bit(ICE_DOWN, pf->state)) 7061 goto clear_recovery; 7062 7063 dev_dbg(dev, "rebuilding PF after reset_type=%d\n", reset_type); 7064 7065 #define ICE_EMP_RESET_SLEEP_MS 5000 7066 if (reset_type == ICE_RESET_EMPR) { 7067 /* If an EMP reset has occurred, any previously pending flash 7068 * update will have completed. We no longer know whether or 7069 * not the NVM update EMP reset is restricted. 7070 */ 7071 pf->fw_emp_reset_disabled = false; 7072 7073 msleep(ICE_EMP_RESET_SLEEP_MS); 7074 } 7075 7076 err = ice_init_all_ctrlq(hw); 7077 if (err) { 7078 dev_err(dev, "control queues init failed %d\n", err); 7079 goto err_init_ctrlq; 7080 } 7081 7082 /* if DDP was previously loaded successfully */ 7083 if (!ice_is_safe_mode(pf)) { 7084 /* reload the SW DB of filter tables */ 7085 if (reset_type == ICE_RESET_PFR) 7086 ice_fill_blk_tbls(hw); 7087 else 7088 /* Reload DDP Package after CORER/GLOBR reset */ 7089 ice_load_pkg(NULL, pf); 7090 } 7091 7092 err = ice_clear_pf_cfg(hw); 7093 if (err) { 7094 dev_err(dev, "clear PF configuration failed %d\n", err); 7095 goto err_init_ctrlq; 7096 } 7097 7098 ice_clear_pxe_mode(hw); 7099 7100 err = ice_init_nvm(hw); 7101 if (err) { 7102 dev_err(dev, "ice_init_nvm failed %d\n", err); 7103 goto err_init_ctrlq; 7104 } 7105 7106 err = ice_get_caps(hw); 7107 if (err) { 7108 dev_err(dev, "ice_get_caps failed %d\n", err); 7109 goto err_init_ctrlq; 7110 } 7111 7112 err = ice_aq_set_mac_cfg(hw, ICE_AQ_SET_MAC_FRAME_SIZE_MAX, NULL); 7113 if (err) { 7114 dev_err(dev, "set_mac_cfg failed %d\n", err); 7115 goto err_init_ctrlq; 7116 } 7117 7118 dvm = ice_is_dvm_ena(hw); 7119 7120 err = ice_aq_set_port_params(pf->hw.port_info, dvm, NULL); 7121 if (err) 7122 goto err_init_ctrlq; 7123 7124 err = ice_sched_init_port(hw->port_info); 7125 if (err) 7126 goto err_sched_init_port; 7127 7128 /* start misc vector */ 7129 err = ice_req_irq_msix_misc(pf); 7130 if (err) { 7131 dev_err(dev, "misc vector setup failed: %d\n", err); 7132 goto err_sched_init_port; 7133 } 7134 7135 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 7136 wr32(hw, PFQF_FD_ENA, PFQF_FD_ENA_FD_ENA_M); 7137 if (!rd32(hw, PFQF_FD_SIZE)) { 7138 u16 unused, guar, b_effort; 7139 7140 guar = hw->func_caps.fd_fltr_guar; 7141 b_effort = hw->func_caps.fd_fltr_best_effort; 7142 7143 /* force guaranteed filter pool for PF */ 7144 ice_alloc_fd_guar_item(hw, &unused, guar); 7145 /* force shared filter pool for PF */ 7146 ice_alloc_fd_shrd_item(hw, &unused, b_effort); 7147 } 7148 } 7149 7150 if (test_bit(ICE_FLAG_DCB_ENA, pf->flags)) 7151 ice_dcb_rebuild(pf); 7152 7153 /* If the PF previously had enabled PTP, PTP init needs to happen before 7154 * the VSI rebuild. If not, this causes the PTP link status events to 7155 * fail. 7156 */ 7157 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 7158 ice_ptp_reset(pf); 7159 7160 if (ice_is_feature_supported(pf, ICE_F_GNSS)) 7161 ice_gnss_init(pf); 7162 7163 /* rebuild PF VSI */ 7164 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_PF); 7165 if (err) { 7166 dev_err(dev, "PF VSI rebuild failed: %d\n", err); 7167 goto err_vsi_rebuild; 7168 } 7169 7170 /* configure PTP timestamping after VSI rebuild */ 7171 if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) 7172 ice_ptp_cfg_timestamp(pf, false); 7173 7174 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_SWITCHDEV_CTRL); 7175 if (err) { 7176 dev_err(dev, "Switchdev CTRL VSI rebuild failed: %d\n", err); 7177 goto err_vsi_rebuild; 7178 } 7179 7180 if (reset_type == ICE_RESET_PFR) { 7181 err = ice_rebuild_channels(pf); 7182 if (err) { 7183 dev_err(dev, "failed to rebuild and replay ADQ VSIs, err %d\n", 7184 err); 7185 goto err_vsi_rebuild; 7186 } 7187 } 7188 7189 /* If Flow Director is active */ 7190 if (test_bit(ICE_FLAG_FD_ENA, pf->flags)) { 7191 err = ice_vsi_rebuild_by_type(pf, ICE_VSI_CTRL); 7192 if (err) { 7193 dev_err(dev, "control VSI rebuild failed: %d\n", err); 7194 goto err_vsi_rebuild; 7195 } 7196 7197 /* replay HW Flow Director recipes */ 7198 if (hw->fdir_prof) 7199 ice_fdir_replay_flows(hw); 7200 7201 /* replay Flow Director filters */ 7202 ice_fdir_replay_fltrs(pf); 7203 7204 ice_rebuild_arfs(pf); 7205 } 7206 7207 ice_update_pf_netdev_link(pf); 7208 7209 /* tell the firmware we are up */ 7210 err = ice_send_version(pf); 7211 if (err) { 7212 dev_err(dev, "Rebuild failed due to error sending driver version: %d\n", 7213 err); 7214 goto err_vsi_rebuild; 7215 } 7216 7217 ice_replay_post(hw); 7218 7219 /* if we get here, reset flow is successful */ 7220 clear_bit(ICE_RESET_FAILED, pf->state); 7221 7222 ice_plug_aux_dev(pf); 7223 return; 7224 7225 err_vsi_rebuild: 7226 err_sched_init_port: 7227 ice_sched_cleanup_all(hw); 7228 err_init_ctrlq: 7229 ice_shutdown_all_ctrlq(hw); 7230 set_bit(ICE_RESET_FAILED, pf->state); 7231 clear_recovery: 7232 /* set this bit in PF state to control service task scheduling */ 7233 set_bit(ICE_NEEDS_RESTART, pf->state); 7234 dev_err(dev, "Rebuild failed, unload and reload driver\n"); 7235 } 7236 7237 /** 7238 * ice_max_xdp_frame_size - returns the maximum allowed frame size for XDP 7239 * @vsi: Pointer to VSI structure 7240 */ 7241 static int ice_max_xdp_frame_size(struct ice_vsi *vsi) 7242 { 7243 if (PAGE_SIZE >= 8192 || test_bit(ICE_FLAG_LEGACY_RX, vsi->back->flags)) 7244 return ICE_RXBUF_2048 - XDP_PACKET_HEADROOM; 7245 else 7246 return ICE_RXBUF_3072; 7247 } 7248 7249 /** 7250 * ice_change_mtu - NDO callback to change the MTU 7251 * @netdev: network interface device structure 7252 * @new_mtu: new value for maximum frame size 7253 * 7254 * Returns 0 on success, negative on failure 7255 */ 7256 static int ice_change_mtu(struct net_device *netdev, int new_mtu) 7257 { 7258 struct ice_netdev_priv *np = netdev_priv(netdev); 7259 struct ice_vsi *vsi = np->vsi; 7260 struct ice_pf *pf = vsi->back; 7261 u8 count = 0; 7262 int err = 0; 7263 7264 if (new_mtu == (int)netdev->mtu) { 7265 netdev_warn(netdev, "MTU is already %u\n", netdev->mtu); 7266 return 0; 7267 } 7268 7269 if (ice_is_xdp_ena_vsi(vsi)) { 7270 int frame_size = ice_max_xdp_frame_size(vsi); 7271 7272 if (new_mtu + ICE_ETH_PKT_HDR_PAD > frame_size) { 7273 netdev_err(netdev, "max MTU for XDP usage is %d\n", 7274 frame_size - ICE_ETH_PKT_HDR_PAD); 7275 return -EINVAL; 7276 } 7277 } 7278 7279 /* if a reset is in progress, wait for some time for it to complete */ 7280 do { 7281 if (ice_is_reset_in_progress(pf->state)) { 7282 count++; 7283 usleep_range(1000, 2000); 7284 } else { 7285 break; 7286 } 7287 7288 } while (count < 100); 7289 7290 if (count == 100) { 7291 netdev_err(netdev, "can't change MTU. Device is busy\n"); 7292 return -EBUSY; 7293 } 7294 7295 netdev->mtu = (unsigned int)new_mtu; 7296 7297 /* if VSI is up, bring it down and then back up */ 7298 if (!test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { 7299 err = ice_down(vsi); 7300 if (err) { 7301 netdev_err(netdev, "change MTU if_down err %d\n", err); 7302 return err; 7303 } 7304 7305 err = ice_up(vsi); 7306 if (err) { 7307 netdev_err(netdev, "change MTU if_up err %d\n", err); 7308 return err; 7309 } 7310 } 7311 7312 netdev_dbg(netdev, "changed MTU to %d\n", new_mtu); 7313 set_bit(ICE_FLAG_MTU_CHANGED, pf->flags); 7314 7315 return err; 7316 } 7317 7318 /** 7319 * ice_eth_ioctl - Access the hwtstamp interface 7320 * @netdev: network interface device structure 7321 * @ifr: interface request data 7322 * @cmd: ioctl command 7323 */ 7324 static int ice_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 7325 { 7326 struct ice_netdev_priv *np = netdev_priv(netdev); 7327 struct ice_pf *pf = np->vsi->back; 7328 7329 switch (cmd) { 7330 case SIOCGHWTSTAMP: 7331 return ice_ptp_get_ts_config(pf, ifr); 7332 case SIOCSHWTSTAMP: 7333 return ice_ptp_set_ts_config(pf, ifr); 7334 default: 7335 return -EOPNOTSUPP; 7336 } 7337 } 7338 7339 /** 7340 * ice_aq_str - convert AQ err code to a string 7341 * @aq_err: the AQ error code to convert 7342 */ 7343 const char *ice_aq_str(enum ice_aq_err aq_err) 7344 { 7345 switch (aq_err) { 7346 case ICE_AQ_RC_OK: 7347 return "OK"; 7348 case ICE_AQ_RC_EPERM: 7349 return "ICE_AQ_RC_EPERM"; 7350 case ICE_AQ_RC_ENOENT: 7351 return "ICE_AQ_RC_ENOENT"; 7352 case ICE_AQ_RC_ENOMEM: 7353 return "ICE_AQ_RC_ENOMEM"; 7354 case ICE_AQ_RC_EBUSY: 7355 return "ICE_AQ_RC_EBUSY"; 7356 case ICE_AQ_RC_EEXIST: 7357 return "ICE_AQ_RC_EEXIST"; 7358 case ICE_AQ_RC_EINVAL: 7359 return "ICE_AQ_RC_EINVAL"; 7360 case ICE_AQ_RC_ENOSPC: 7361 return "ICE_AQ_RC_ENOSPC"; 7362 case ICE_AQ_RC_ENOSYS: 7363 return "ICE_AQ_RC_ENOSYS"; 7364 case ICE_AQ_RC_EMODE: 7365 return "ICE_AQ_RC_EMODE"; 7366 case ICE_AQ_RC_ENOSEC: 7367 return "ICE_AQ_RC_ENOSEC"; 7368 case ICE_AQ_RC_EBADSIG: 7369 return "ICE_AQ_RC_EBADSIG"; 7370 case ICE_AQ_RC_ESVN: 7371 return "ICE_AQ_RC_ESVN"; 7372 case ICE_AQ_RC_EBADMAN: 7373 return "ICE_AQ_RC_EBADMAN"; 7374 case ICE_AQ_RC_EBADBUF: 7375 return "ICE_AQ_RC_EBADBUF"; 7376 } 7377 7378 return "ICE_AQ_RC_UNKNOWN"; 7379 } 7380 7381 /** 7382 * ice_set_rss_lut - Set RSS LUT 7383 * @vsi: Pointer to VSI structure 7384 * @lut: Lookup table 7385 * @lut_size: Lookup table size 7386 * 7387 * Returns 0 on success, negative on failure 7388 */ 7389 int ice_set_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size) 7390 { 7391 struct ice_aq_get_set_rss_lut_params params = {}; 7392 struct ice_hw *hw = &vsi->back->hw; 7393 int status; 7394 7395 if (!lut) 7396 return -EINVAL; 7397 7398 params.vsi_handle = vsi->idx; 7399 params.lut_size = lut_size; 7400 params.lut_type = vsi->rss_lut_type; 7401 params.lut = lut; 7402 7403 status = ice_aq_set_rss_lut(hw, ¶ms); 7404 if (status) 7405 dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS lut, err %d aq_err %s\n", 7406 status, ice_aq_str(hw->adminq.sq_last_status)); 7407 7408 return status; 7409 } 7410 7411 /** 7412 * ice_set_rss_key - Set RSS key 7413 * @vsi: Pointer to the VSI structure 7414 * @seed: RSS hash seed 7415 * 7416 * Returns 0 on success, negative on failure 7417 */ 7418 int ice_set_rss_key(struct ice_vsi *vsi, u8 *seed) 7419 { 7420 struct ice_hw *hw = &vsi->back->hw; 7421 int status; 7422 7423 if (!seed) 7424 return -EINVAL; 7425 7426 status = ice_aq_set_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed); 7427 if (status) 7428 dev_err(ice_pf_to_dev(vsi->back), "Cannot set RSS key, err %d aq_err %s\n", 7429 status, ice_aq_str(hw->adminq.sq_last_status)); 7430 7431 return status; 7432 } 7433 7434 /** 7435 * ice_get_rss_lut - Get RSS LUT 7436 * @vsi: Pointer to VSI structure 7437 * @lut: Buffer to store the lookup table entries 7438 * @lut_size: Size of buffer to store the lookup table entries 7439 * 7440 * Returns 0 on success, negative on failure 7441 */ 7442 int ice_get_rss_lut(struct ice_vsi *vsi, u8 *lut, u16 lut_size) 7443 { 7444 struct ice_aq_get_set_rss_lut_params params = {}; 7445 struct ice_hw *hw = &vsi->back->hw; 7446 int status; 7447 7448 if (!lut) 7449 return -EINVAL; 7450 7451 params.vsi_handle = vsi->idx; 7452 params.lut_size = lut_size; 7453 params.lut_type = vsi->rss_lut_type; 7454 params.lut = lut; 7455 7456 status = ice_aq_get_rss_lut(hw, ¶ms); 7457 if (status) 7458 dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS lut, err %d aq_err %s\n", 7459 status, ice_aq_str(hw->adminq.sq_last_status)); 7460 7461 return status; 7462 } 7463 7464 /** 7465 * ice_get_rss_key - Get RSS key 7466 * @vsi: Pointer to VSI structure 7467 * @seed: Buffer to store the key in 7468 * 7469 * Returns 0 on success, negative on failure 7470 */ 7471 int ice_get_rss_key(struct ice_vsi *vsi, u8 *seed) 7472 { 7473 struct ice_hw *hw = &vsi->back->hw; 7474 int status; 7475 7476 if (!seed) 7477 return -EINVAL; 7478 7479 status = ice_aq_get_rss_key(hw, vsi->idx, (struct ice_aqc_get_set_rss_keys *)seed); 7480 if (status) 7481 dev_err(ice_pf_to_dev(vsi->back), "Cannot get RSS key, err %d aq_err %s\n", 7482 status, ice_aq_str(hw->adminq.sq_last_status)); 7483 7484 return status; 7485 } 7486 7487 /** 7488 * ice_bridge_getlink - Get the hardware bridge mode 7489 * @skb: skb buff 7490 * @pid: process ID 7491 * @seq: RTNL message seq 7492 * @dev: the netdev being configured 7493 * @filter_mask: filter mask passed in 7494 * @nlflags: netlink flags passed in 7495 * 7496 * Return the bridge mode (VEB/VEPA) 7497 */ 7498 static int 7499 ice_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 7500 struct net_device *dev, u32 filter_mask, int nlflags) 7501 { 7502 struct ice_netdev_priv *np = netdev_priv(dev); 7503 struct ice_vsi *vsi = np->vsi; 7504 struct ice_pf *pf = vsi->back; 7505 u16 bmode; 7506 7507 bmode = pf->first_sw->bridge_mode; 7508 7509 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, bmode, 0, 0, nlflags, 7510 filter_mask, NULL); 7511 } 7512 7513 /** 7514 * ice_vsi_update_bridge_mode - Update VSI for switching bridge mode (VEB/VEPA) 7515 * @vsi: Pointer to VSI structure 7516 * @bmode: Hardware bridge mode (VEB/VEPA) 7517 * 7518 * Returns 0 on success, negative on failure 7519 */ 7520 static int ice_vsi_update_bridge_mode(struct ice_vsi *vsi, u16 bmode) 7521 { 7522 struct ice_aqc_vsi_props *vsi_props; 7523 struct ice_hw *hw = &vsi->back->hw; 7524 struct ice_vsi_ctx *ctxt; 7525 int ret; 7526 7527 vsi_props = &vsi->info; 7528 7529 ctxt = kzalloc(sizeof(*ctxt), GFP_KERNEL); 7530 if (!ctxt) 7531 return -ENOMEM; 7532 7533 ctxt->info = vsi->info; 7534 7535 if (bmode == BRIDGE_MODE_VEB) 7536 /* change from VEPA to VEB mode */ 7537 ctxt->info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 7538 else 7539 /* change from VEB to VEPA mode */ 7540 ctxt->info.sw_flags &= ~ICE_AQ_VSI_SW_FLAG_ALLOW_LB; 7541 ctxt->info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_SW_VALID); 7542 7543 ret = ice_update_vsi(hw, vsi->idx, ctxt, NULL); 7544 if (ret) { 7545 dev_err(ice_pf_to_dev(vsi->back), "update VSI for bridge mode failed, bmode = %d err %d aq_err %s\n", 7546 bmode, ret, ice_aq_str(hw->adminq.sq_last_status)); 7547 goto out; 7548 } 7549 /* Update sw flags for book keeping */ 7550 vsi_props->sw_flags = ctxt->info.sw_flags; 7551 7552 out: 7553 kfree(ctxt); 7554 return ret; 7555 } 7556 7557 /** 7558 * ice_bridge_setlink - Set the hardware bridge mode 7559 * @dev: the netdev being configured 7560 * @nlh: RTNL message 7561 * @flags: bridge setlink flags 7562 * @extack: netlink extended ack 7563 * 7564 * Sets the bridge mode (VEB/VEPA) of the switch to which the netdev (VSI) is 7565 * hooked up to. Iterates through the PF VSI list and sets the loopback mode (if 7566 * not already set for all VSIs connected to this switch. And also update the 7567 * unicast switch filter rules for the corresponding switch of the netdev. 7568 */ 7569 static int 7570 ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 7571 u16 __always_unused flags, 7572 struct netlink_ext_ack __always_unused *extack) 7573 { 7574 struct ice_netdev_priv *np = netdev_priv(dev); 7575 struct ice_pf *pf = np->vsi->back; 7576 struct nlattr *attr, *br_spec; 7577 struct ice_hw *hw = &pf->hw; 7578 struct ice_sw *pf_sw; 7579 int rem, v, err = 0; 7580 7581 pf_sw = pf->first_sw; 7582 /* find the attribute in the netlink message */ 7583 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 7584 7585 nla_for_each_nested(attr, br_spec, rem) { 7586 __u16 mode; 7587 7588 if (nla_type(attr) != IFLA_BRIDGE_MODE) 7589 continue; 7590 mode = nla_get_u16(attr); 7591 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 7592 return -EINVAL; 7593 /* Continue if bridge mode is not being flipped */ 7594 if (mode == pf_sw->bridge_mode) 7595 continue; 7596 /* Iterates through the PF VSI list and update the loopback 7597 * mode of the VSI 7598 */ 7599 ice_for_each_vsi(pf, v) { 7600 if (!pf->vsi[v]) 7601 continue; 7602 err = ice_vsi_update_bridge_mode(pf->vsi[v], mode); 7603 if (err) 7604 return err; 7605 } 7606 7607 hw->evb_veb = (mode == BRIDGE_MODE_VEB); 7608 /* Update the unicast switch filter rules for the corresponding 7609 * switch of the netdev 7610 */ 7611 err = ice_update_sw_rule_bridge_mode(hw); 7612 if (err) { 7613 netdev_err(dev, "switch rule update failed, mode = %d err %d aq_err %s\n", 7614 mode, err, 7615 ice_aq_str(hw->adminq.sq_last_status)); 7616 /* revert hw->evb_veb */ 7617 hw->evb_veb = (pf_sw->bridge_mode == BRIDGE_MODE_VEB); 7618 return err; 7619 } 7620 7621 pf_sw->bridge_mode = mode; 7622 } 7623 7624 return 0; 7625 } 7626 7627 /** 7628 * ice_tx_timeout - Respond to a Tx Hang 7629 * @netdev: network interface device structure 7630 * @txqueue: Tx queue 7631 */ 7632 static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) 7633 { 7634 struct ice_netdev_priv *np = netdev_priv(netdev); 7635 struct ice_tx_ring *tx_ring = NULL; 7636 struct ice_vsi *vsi = np->vsi; 7637 struct ice_pf *pf = vsi->back; 7638 u32 i; 7639 7640 pf->tx_timeout_count++; 7641 7642 /* Check if PFC is enabled for the TC to which the queue belongs 7643 * to. If yes then Tx timeout is not caused by a hung queue, no 7644 * need to reset and rebuild 7645 */ 7646 if (ice_is_pfc_causing_hung_q(pf, txqueue)) { 7647 dev_info(ice_pf_to_dev(pf), "Fake Tx hang detected on queue %u, timeout caused by PFC storm\n", 7648 txqueue); 7649 return; 7650 } 7651 7652 /* now that we have an index, find the tx_ring struct */ 7653 ice_for_each_txq(vsi, i) 7654 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 7655 if (txqueue == vsi->tx_rings[i]->q_index) { 7656 tx_ring = vsi->tx_rings[i]; 7657 break; 7658 } 7659 7660 /* Reset recovery level if enough time has elapsed after last timeout. 7661 * Also ensure no new reset action happens before next timeout period. 7662 */ 7663 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ * 20))) 7664 pf->tx_timeout_recovery_level = 1; 7665 else if (time_before(jiffies, (pf->tx_timeout_last_recovery + 7666 netdev->watchdog_timeo))) 7667 return; 7668 7669 if (tx_ring) { 7670 struct ice_hw *hw = &pf->hw; 7671 u32 head, val = 0; 7672 7673 head = (rd32(hw, QTX_COMM_HEAD(vsi->txq_map[txqueue])) & 7674 QTX_COMM_HEAD_HEAD_M) >> QTX_COMM_HEAD_HEAD_S; 7675 /* Read interrupt register */ 7676 val = rd32(hw, GLINT_DYN_CTL(tx_ring->q_vector->reg_idx)); 7677 7678 netdev_info(netdev, "tx_timeout: VSI_num: %d, Q %u, NTC: 0x%x, HW_HEAD: 0x%x, NTU: 0x%x, INT: 0x%x\n", 7679 vsi->vsi_num, txqueue, tx_ring->next_to_clean, 7680 head, tx_ring->next_to_use, val); 7681 } 7682 7683 pf->tx_timeout_last_recovery = jiffies; 7684 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %u\n", 7685 pf->tx_timeout_recovery_level, txqueue); 7686 7687 switch (pf->tx_timeout_recovery_level) { 7688 case 1: 7689 set_bit(ICE_PFR_REQ, pf->state); 7690 break; 7691 case 2: 7692 set_bit(ICE_CORER_REQ, pf->state); 7693 break; 7694 case 3: 7695 set_bit(ICE_GLOBR_REQ, pf->state); 7696 break; 7697 default: 7698 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in unrecoverable state.\n"); 7699 set_bit(ICE_DOWN, pf->state); 7700 set_bit(ICE_VSI_NEEDS_RESTART, vsi->state); 7701 set_bit(ICE_SERVICE_DIS, pf->state); 7702 break; 7703 } 7704 7705 ice_service_task_schedule(pf); 7706 pf->tx_timeout_recovery_level++; 7707 } 7708 7709 /** 7710 * ice_setup_tc_cls_flower - flower classifier offloads 7711 * @np: net device to configure 7712 * @filter_dev: device on which filter is added 7713 * @cls_flower: offload data 7714 */ 7715 static int 7716 ice_setup_tc_cls_flower(struct ice_netdev_priv *np, 7717 struct net_device *filter_dev, 7718 struct flow_cls_offload *cls_flower) 7719 { 7720 struct ice_vsi *vsi = np->vsi; 7721 7722 if (cls_flower->common.chain_index) 7723 return -EOPNOTSUPP; 7724 7725 switch (cls_flower->command) { 7726 case FLOW_CLS_REPLACE: 7727 return ice_add_cls_flower(filter_dev, vsi, cls_flower); 7728 case FLOW_CLS_DESTROY: 7729 return ice_del_cls_flower(vsi, cls_flower); 7730 default: 7731 return -EINVAL; 7732 } 7733 } 7734 7735 /** 7736 * ice_setup_tc_block_cb - callback handler registered for TC block 7737 * @type: TC SETUP type 7738 * @type_data: TC flower offload data that contains user input 7739 * @cb_priv: netdev private data 7740 */ 7741 static int 7742 ice_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) 7743 { 7744 struct ice_netdev_priv *np = cb_priv; 7745 7746 switch (type) { 7747 case TC_SETUP_CLSFLOWER: 7748 return ice_setup_tc_cls_flower(np, np->vsi->netdev, 7749 type_data); 7750 default: 7751 return -EOPNOTSUPP; 7752 } 7753 } 7754 7755 /** 7756 * ice_validate_mqprio_qopt - Validate TCF input parameters 7757 * @vsi: Pointer to VSI 7758 * @mqprio_qopt: input parameters for mqprio queue configuration 7759 * 7760 * This function validates MQPRIO params, such as qcount (power of 2 wherever 7761 * needed), and make sure user doesn't specify qcount and BW rate limit 7762 * for TCs, which are more than "num_tc" 7763 */ 7764 static int 7765 ice_validate_mqprio_qopt(struct ice_vsi *vsi, 7766 struct tc_mqprio_qopt_offload *mqprio_qopt) 7767 { 7768 u64 sum_max_rate = 0, sum_min_rate = 0; 7769 int non_power_of_2_qcount = 0; 7770 struct ice_pf *pf = vsi->back; 7771 int max_rss_q_cnt = 0; 7772 struct device *dev; 7773 int i, speed; 7774 u8 num_tc; 7775 7776 if (vsi->type != ICE_VSI_PF) 7777 return -EINVAL; 7778 7779 if (mqprio_qopt->qopt.offset[0] != 0 || 7780 mqprio_qopt->qopt.num_tc < 1 || 7781 mqprio_qopt->qopt.num_tc > ICE_CHNL_MAX_TC) 7782 return -EINVAL; 7783 7784 dev = ice_pf_to_dev(pf); 7785 vsi->ch_rss_size = 0; 7786 num_tc = mqprio_qopt->qopt.num_tc; 7787 7788 for (i = 0; num_tc; i++) { 7789 int qcount = mqprio_qopt->qopt.count[i]; 7790 u64 max_rate, min_rate, rem; 7791 7792 if (!qcount) 7793 return -EINVAL; 7794 7795 if (is_power_of_2(qcount)) { 7796 if (non_power_of_2_qcount && 7797 qcount > non_power_of_2_qcount) { 7798 dev_err(dev, "qcount[%d] cannot be greater than non power of 2 qcount[%d]\n", 7799 qcount, non_power_of_2_qcount); 7800 return -EINVAL; 7801 } 7802 if (qcount > max_rss_q_cnt) 7803 max_rss_q_cnt = qcount; 7804 } else { 7805 if (non_power_of_2_qcount && 7806 qcount != non_power_of_2_qcount) { 7807 dev_err(dev, "Only one non power of 2 qcount allowed[%d,%d]\n", 7808 qcount, non_power_of_2_qcount); 7809 return -EINVAL; 7810 } 7811 if (qcount < max_rss_q_cnt) { 7812 dev_err(dev, "non power of 2 qcount[%d] cannot be less than other qcount[%d]\n", 7813 qcount, max_rss_q_cnt); 7814 return -EINVAL; 7815 } 7816 max_rss_q_cnt = qcount; 7817 non_power_of_2_qcount = qcount; 7818 } 7819 7820 /* TC command takes input in K/N/Gbps or K/M/Gbit etc but 7821 * converts the bandwidth rate limit into Bytes/s when 7822 * passing it down to the driver. So convert input bandwidth 7823 * from Bytes/s to Kbps 7824 */ 7825 max_rate = mqprio_qopt->max_rate[i]; 7826 max_rate = div_u64(max_rate, ICE_BW_KBPS_DIVISOR); 7827 sum_max_rate += max_rate; 7828 7829 /* min_rate is minimum guaranteed rate and it can't be zero */ 7830 min_rate = mqprio_qopt->min_rate[i]; 7831 min_rate = div_u64(min_rate, ICE_BW_KBPS_DIVISOR); 7832 sum_min_rate += min_rate; 7833 7834 if (min_rate && min_rate < ICE_MIN_BW_LIMIT) { 7835 dev_err(dev, "TC%d: min_rate(%llu Kbps) < %u Kbps\n", i, 7836 min_rate, ICE_MIN_BW_LIMIT); 7837 return -EINVAL; 7838 } 7839 7840 iter_div_u64_rem(min_rate, ICE_MIN_BW_LIMIT, &rem); 7841 if (rem) { 7842 dev_err(dev, "TC%d: Min Rate not multiple of %u Kbps", 7843 i, ICE_MIN_BW_LIMIT); 7844 return -EINVAL; 7845 } 7846 7847 iter_div_u64_rem(max_rate, ICE_MIN_BW_LIMIT, &rem); 7848 if (rem) { 7849 dev_err(dev, "TC%d: Max Rate not multiple of %u Kbps", 7850 i, ICE_MIN_BW_LIMIT); 7851 return -EINVAL; 7852 } 7853 7854 /* min_rate can't be more than max_rate, except when max_rate 7855 * is zero (implies max_rate sought is max line rate). In such 7856 * a case min_rate can be more than max. 7857 */ 7858 if (max_rate && min_rate > max_rate) { 7859 dev_err(dev, "min_rate %llu Kbps can't be more than max_rate %llu Kbps\n", 7860 min_rate, max_rate); 7861 return -EINVAL; 7862 } 7863 7864 if (i >= mqprio_qopt->qopt.num_tc - 1) 7865 break; 7866 if (mqprio_qopt->qopt.offset[i + 1] != 7867 (mqprio_qopt->qopt.offset[i] + qcount)) 7868 return -EINVAL; 7869 } 7870 if (vsi->num_rxq < 7871 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 7872 return -EINVAL; 7873 if (vsi->num_txq < 7874 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 7875 return -EINVAL; 7876 7877 speed = ice_get_link_speed_kbps(vsi); 7878 if (sum_max_rate && sum_max_rate > (u64)speed) { 7879 dev_err(dev, "Invalid max Tx rate(%llu) Kbps > speed(%u) Kbps specified\n", 7880 sum_max_rate, speed); 7881 return -EINVAL; 7882 } 7883 if (sum_min_rate && sum_min_rate > (u64)speed) { 7884 dev_err(dev, "Invalid min Tx rate(%llu) Kbps > speed (%u) Kbps specified\n", 7885 sum_min_rate, speed); 7886 return -EINVAL; 7887 } 7888 7889 /* make sure vsi->ch_rss_size is set correctly based on TC's qcount */ 7890 vsi->ch_rss_size = max_rss_q_cnt; 7891 7892 return 0; 7893 } 7894 7895 /** 7896 * ice_add_vsi_to_fdir - add a VSI to the flow director group for PF 7897 * @pf: ptr to PF device 7898 * @vsi: ptr to VSI 7899 */ 7900 static int ice_add_vsi_to_fdir(struct ice_pf *pf, struct ice_vsi *vsi) 7901 { 7902 struct device *dev = ice_pf_to_dev(pf); 7903 bool added = false; 7904 struct ice_hw *hw; 7905 int flow; 7906 7907 if (!(vsi->num_gfltr || vsi->num_bfltr)) 7908 return -EINVAL; 7909 7910 hw = &pf->hw; 7911 for (flow = 0; flow < ICE_FLTR_PTYPE_MAX; flow++) { 7912 struct ice_fd_hw_prof *prof; 7913 int tun, status; 7914 u64 entry_h; 7915 7916 if (!(hw->fdir_prof && hw->fdir_prof[flow] && 7917 hw->fdir_prof[flow]->cnt)) 7918 continue; 7919 7920 for (tun = 0; tun < ICE_FD_HW_SEG_MAX; tun++) { 7921 enum ice_flow_priority prio; 7922 u64 prof_id; 7923 7924 /* add this VSI to FDir profile for this flow */ 7925 prio = ICE_FLOW_PRIO_NORMAL; 7926 prof = hw->fdir_prof[flow]; 7927 prof_id = flow + tun * ICE_FLTR_PTYPE_MAX; 7928 status = ice_flow_add_entry(hw, ICE_BLK_FD, prof_id, 7929 prof->vsi_h[0], vsi->idx, 7930 prio, prof->fdir_seg[tun], 7931 &entry_h); 7932 if (status) { 7933 dev_err(dev, "channel VSI idx %d, not able to add to group %d\n", 7934 vsi->idx, flow); 7935 continue; 7936 } 7937 7938 prof->entry_h[prof->cnt][tun] = entry_h; 7939 } 7940 7941 /* store VSI for filter replay and delete */ 7942 prof->vsi_h[prof->cnt] = vsi->idx; 7943 prof->cnt++; 7944 7945 added = true; 7946 dev_dbg(dev, "VSI idx %d added to fdir group %d\n", vsi->idx, 7947 flow); 7948 } 7949 7950 if (!added) 7951 dev_dbg(dev, "VSI idx %d not added to fdir groups\n", vsi->idx); 7952 7953 return 0; 7954 } 7955 7956 /** 7957 * ice_add_channel - add a channel by adding VSI 7958 * @pf: ptr to PF device 7959 * @sw_id: underlying HW switching element ID 7960 * @ch: ptr to channel structure 7961 * 7962 * Add a channel (VSI) using add_vsi and queue_map 7963 */ 7964 static int ice_add_channel(struct ice_pf *pf, u16 sw_id, struct ice_channel *ch) 7965 { 7966 struct device *dev = ice_pf_to_dev(pf); 7967 struct ice_vsi *vsi; 7968 7969 if (ch->type != ICE_VSI_CHNL) { 7970 dev_err(dev, "add new VSI failed, ch->type %d\n", ch->type); 7971 return -EINVAL; 7972 } 7973 7974 vsi = ice_chnl_vsi_setup(pf, pf->hw.port_info, ch); 7975 if (!vsi || vsi->type != ICE_VSI_CHNL) { 7976 dev_err(dev, "create chnl VSI failure\n"); 7977 return -EINVAL; 7978 } 7979 7980 ice_add_vsi_to_fdir(pf, vsi); 7981 7982 ch->sw_id = sw_id; 7983 ch->vsi_num = vsi->vsi_num; 7984 ch->info.mapping_flags = vsi->info.mapping_flags; 7985 ch->ch_vsi = vsi; 7986 /* set the back pointer of channel for newly created VSI */ 7987 vsi->ch = ch; 7988 7989 memcpy(&ch->info.q_mapping, &vsi->info.q_mapping, 7990 sizeof(vsi->info.q_mapping)); 7991 memcpy(&ch->info.tc_mapping, vsi->info.tc_mapping, 7992 sizeof(vsi->info.tc_mapping)); 7993 7994 return 0; 7995 } 7996 7997 /** 7998 * ice_chnl_cfg_res 7999 * @vsi: the VSI being setup 8000 * @ch: ptr to channel structure 8001 * 8002 * Configure channel specific resources such as rings, vector. 8003 */ 8004 static void ice_chnl_cfg_res(struct ice_vsi *vsi, struct ice_channel *ch) 8005 { 8006 int i; 8007 8008 for (i = 0; i < ch->num_txq; i++) { 8009 struct ice_q_vector *tx_q_vector, *rx_q_vector; 8010 struct ice_ring_container *rc; 8011 struct ice_tx_ring *tx_ring; 8012 struct ice_rx_ring *rx_ring; 8013 8014 tx_ring = vsi->tx_rings[ch->base_q + i]; 8015 rx_ring = vsi->rx_rings[ch->base_q + i]; 8016 if (!tx_ring || !rx_ring) 8017 continue; 8018 8019 /* setup ring being channel enabled */ 8020 tx_ring->ch = ch; 8021 rx_ring->ch = ch; 8022 8023 /* following code block sets up vector specific attributes */ 8024 tx_q_vector = tx_ring->q_vector; 8025 rx_q_vector = rx_ring->q_vector; 8026 if (!tx_q_vector && !rx_q_vector) 8027 continue; 8028 8029 if (tx_q_vector) { 8030 tx_q_vector->ch = ch; 8031 /* setup Tx and Rx ITR setting if DIM is off */ 8032 rc = &tx_q_vector->tx; 8033 if (!ITR_IS_DYNAMIC(rc)) 8034 ice_write_itr(rc, rc->itr_setting); 8035 } 8036 if (rx_q_vector) { 8037 rx_q_vector->ch = ch; 8038 /* setup Tx and Rx ITR setting if DIM is off */ 8039 rc = &rx_q_vector->rx; 8040 if (!ITR_IS_DYNAMIC(rc)) 8041 ice_write_itr(rc, rc->itr_setting); 8042 } 8043 } 8044 8045 /* it is safe to assume that, if channel has non-zero num_t[r]xq, then 8046 * GLINT_ITR register would have written to perform in-context 8047 * update, hence perform flush 8048 */ 8049 if (ch->num_txq || ch->num_rxq) 8050 ice_flush(&vsi->back->hw); 8051 } 8052 8053 /** 8054 * ice_cfg_chnl_all_res - configure channel resources 8055 * @vsi: pte to main_vsi 8056 * @ch: ptr to channel structure 8057 * 8058 * This function configures channel specific resources such as flow-director 8059 * counter index, and other resources such as queues, vectors, ITR settings 8060 */ 8061 static void 8062 ice_cfg_chnl_all_res(struct ice_vsi *vsi, struct ice_channel *ch) 8063 { 8064 /* configure channel (aka ADQ) resources such as queues, vectors, 8065 * ITR settings for channel specific vectors and anything else 8066 */ 8067 ice_chnl_cfg_res(vsi, ch); 8068 } 8069 8070 /** 8071 * ice_setup_hw_channel - setup new channel 8072 * @pf: ptr to PF device 8073 * @vsi: the VSI being setup 8074 * @ch: ptr to channel structure 8075 * @sw_id: underlying HW switching element ID 8076 * @type: type of channel to be created (VMDq2/VF) 8077 * 8078 * Setup new channel (VSI) based on specified type (VMDq2/VF) 8079 * and configures Tx rings accordingly 8080 */ 8081 static int 8082 ice_setup_hw_channel(struct ice_pf *pf, struct ice_vsi *vsi, 8083 struct ice_channel *ch, u16 sw_id, u8 type) 8084 { 8085 struct device *dev = ice_pf_to_dev(pf); 8086 int ret; 8087 8088 ch->base_q = vsi->next_base_q; 8089 ch->type = type; 8090 8091 ret = ice_add_channel(pf, sw_id, ch); 8092 if (ret) { 8093 dev_err(dev, "failed to add_channel using sw_id %u\n", sw_id); 8094 return ret; 8095 } 8096 8097 /* configure/setup ADQ specific resources */ 8098 ice_cfg_chnl_all_res(vsi, ch); 8099 8100 /* make sure to update the next_base_q so that subsequent channel's 8101 * (aka ADQ) VSI queue map is correct 8102 */ 8103 vsi->next_base_q = vsi->next_base_q + ch->num_rxq; 8104 dev_dbg(dev, "added channel: vsi_num %u, num_rxq %u\n", ch->vsi_num, 8105 ch->num_rxq); 8106 8107 return 0; 8108 } 8109 8110 /** 8111 * ice_setup_channel - setup new channel using uplink element 8112 * @pf: ptr to PF device 8113 * @vsi: the VSI being setup 8114 * @ch: ptr to channel structure 8115 * 8116 * Setup new channel (VSI) based on specified type (VMDq2/VF) 8117 * and uplink switching element 8118 */ 8119 static bool 8120 ice_setup_channel(struct ice_pf *pf, struct ice_vsi *vsi, 8121 struct ice_channel *ch) 8122 { 8123 struct device *dev = ice_pf_to_dev(pf); 8124 u16 sw_id; 8125 int ret; 8126 8127 if (vsi->type != ICE_VSI_PF) { 8128 dev_err(dev, "unsupported parent VSI type(%d)\n", vsi->type); 8129 return false; 8130 } 8131 8132 sw_id = pf->first_sw->sw_id; 8133 8134 /* create channel (VSI) */ 8135 ret = ice_setup_hw_channel(pf, vsi, ch, sw_id, ICE_VSI_CHNL); 8136 if (ret) { 8137 dev_err(dev, "failed to setup hw_channel\n"); 8138 return false; 8139 } 8140 dev_dbg(dev, "successfully created channel()\n"); 8141 8142 return ch->ch_vsi ? true : false; 8143 } 8144 8145 /** 8146 * ice_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 8147 * @vsi: VSI to be configured 8148 * @max_tx_rate: max Tx rate in Kbps to be configured as maximum BW limit 8149 * @min_tx_rate: min Tx rate in Kbps to be configured as minimum BW limit 8150 */ 8151 static int 8152 ice_set_bw_limit(struct ice_vsi *vsi, u64 max_tx_rate, u64 min_tx_rate) 8153 { 8154 int err; 8155 8156 err = ice_set_min_bw_limit(vsi, min_tx_rate); 8157 if (err) 8158 return err; 8159 8160 return ice_set_max_bw_limit(vsi, max_tx_rate); 8161 } 8162 8163 /** 8164 * ice_create_q_channel - function to create channel 8165 * @vsi: VSI to be configured 8166 * @ch: ptr to channel (it contains channel specific params) 8167 * 8168 * This function creates channel (VSI) using num_queues specified by user, 8169 * reconfigs RSS if needed. 8170 */ 8171 static int ice_create_q_channel(struct ice_vsi *vsi, struct ice_channel *ch) 8172 { 8173 struct ice_pf *pf = vsi->back; 8174 struct device *dev; 8175 8176 if (!ch) 8177 return -EINVAL; 8178 8179 dev = ice_pf_to_dev(pf); 8180 if (!ch->num_txq || !ch->num_rxq) { 8181 dev_err(dev, "Invalid num_queues requested: %d\n", ch->num_rxq); 8182 return -EINVAL; 8183 } 8184 8185 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_txq) { 8186 dev_err(dev, "cnt_q_avail (%u) less than num_queues %d\n", 8187 vsi->cnt_q_avail, ch->num_txq); 8188 return -EINVAL; 8189 } 8190 8191 if (!ice_setup_channel(pf, vsi, ch)) { 8192 dev_info(dev, "Failed to setup channel\n"); 8193 return -EINVAL; 8194 } 8195 /* configure BW rate limit */ 8196 if (ch->ch_vsi && (ch->max_tx_rate || ch->min_tx_rate)) { 8197 int ret; 8198 8199 ret = ice_set_bw_limit(ch->ch_vsi, ch->max_tx_rate, 8200 ch->min_tx_rate); 8201 if (ret) 8202 dev_err(dev, "failed to set Tx rate of %llu Kbps for VSI(%u)\n", 8203 ch->max_tx_rate, ch->ch_vsi->vsi_num); 8204 else 8205 dev_dbg(dev, "set Tx rate of %llu Kbps for VSI(%u)\n", 8206 ch->max_tx_rate, ch->ch_vsi->vsi_num); 8207 } 8208 8209 vsi->cnt_q_avail -= ch->num_txq; 8210 8211 return 0; 8212 } 8213 8214 /** 8215 * ice_rem_all_chnl_fltrs - removes all channel filters 8216 * @pf: ptr to PF, TC-flower based filter are tracked at PF level 8217 * 8218 * Remove all advanced switch filters only if they are channel specific 8219 * tc-flower based filter 8220 */ 8221 static void ice_rem_all_chnl_fltrs(struct ice_pf *pf) 8222 { 8223 struct ice_tc_flower_fltr *fltr; 8224 struct hlist_node *node; 8225 8226 /* to remove all channel filters, iterate an ordered list of filters */ 8227 hlist_for_each_entry_safe(fltr, node, 8228 &pf->tc_flower_fltr_list, 8229 tc_flower_node) { 8230 struct ice_rule_query_data rule; 8231 int status; 8232 8233 /* for now process only channel specific filters */ 8234 if (!ice_is_chnl_fltr(fltr)) 8235 continue; 8236 8237 rule.rid = fltr->rid; 8238 rule.rule_id = fltr->rule_id; 8239 rule.vsi_handle = fltr->dest_id; 8240 status = ice_rem_adv_rule_by_id(&pf->hw, &rule); 8241 if (status) { 8242 if (status == -ENOENT) 8243 dev_dbg(ice_pf_to_dev(pf), "TC flower filter (rule_id %u) does not exist\n", 8244 rule.rule_id); 8245 else 8246 dev_err(ice_pf_to_dev(pf), "failed to delete TC flower filter, status %d\n", 8247 status); 8248 } else if (fltr->dest_vsi) { 8249 /* update advanced switch filter count */ 8250 if (fltr->dest_vsi->type == ICE_VSI_CHNL) { 8251 u32 flags = fltr->flags; 8252 8253 fltr->dest_vsi->num_chnl_fltr--; 8254 if (flags & (ICE_TC_FLWR_FIELD_DST_MAC | 8255 ICE_TC_FLWR_FIELD_ENC_DST_MAC)) 8256 pf->num_dmac_chnl_fltrs--; 8257 } 8258 } 8259 8260 hlist_del(&fltr->tc_flower_node); 8261 kfree(fltr); 8262 } 8263 } 8264 8265 /** 8266 * ice_remove_q_channels - Remove queue channels for the TCs 8267 * @vsi: VSI to be configured 8268 * @rem_fltr: delete advanced switch filter or not 8269 * 8270 * Remove queue channels for the TCs 8271 */ 8272 static void ice_remove_q_channels(struct ice_vsi *vsi, bool rem_fltr) 8273 { 8274 struct ice_channel *ch, *ch_tmp; 8275 struct ice_pf *pf = vsi->back; 8276 int i; 8277 8278 /* remove all tc-flower based filter if they are channel filters only */ 8279 if (rem_fltr) 8280 ice_rem_all_chnl_fltrs(pf); 8281 8282 /* remove ntuple filters since queue configuration is being changed */ 8283 if (vsi->netdev->features & NETIF_F_NTUPLE) { 8284 struct ice_hw *hw = &pf->hw; 8285 8286 mutex_lock(&hw->fdir_fltr_lock); 8287 ice_fdir_del_all_fltrs(vsi); 8288 mutex_unlock(&hw->fdir_fltr_lock); 8289 } 8290 8291 /* perform cleanup for channels if they exist */ 8292 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 8293 struct ice_vsi *ch_vsi; 8294 8295 list_del(&ch->list); 8296 ch_vsi = ch->ch_vsi; 8297 if (!ch_vsi) { 8298 kfree(ch); 8299 continue; 8300 } 8301 8302 /* Reset queue contexts */ 8303 for (i = 0; i < ch->num_rxq; i++) { 8304 struct ice_tx_ring *tx_ring; 8305 struct ice_rx_ring *rx_ring; 8306 8307 tx_ring = vsi->tx_rings[ch->base_q + i]; 8308 rx_ring = vsi->rx_rings[ch->base_q + i]; 8309 if (tx_ring) { 8310 tx_ring->ch = NULL; 8311 if (tx_ring->q_vector) 8312 tx_ring->q_vector->ch = NULL; 8313 } 8314 if (rx_ring) { 8315 rx_ring->ch = NULL; 8316 if (rx_ring->q_vector) 8317 rx_ring->q_vector->ch = NULL; 8318 } 8319 } 8320 8321 /* Release FD resources for the channel VSI */ 8322 ice_fdir_rem_adq_chnl(&pf->hw, ch->ch_vsi->idx); 8323 8324 /* clear the VSI from scheduler tree */ 8325 ice_rm_vsi_lan_cfg(ch->ch_vsi->port_info, ch->ch_vsi->idx); 8326 8327 /* Delete VSI from FW */ 8328 ice_vsi_delete(ch->ch_vsi); 8329 8330 /* Delete VSI from PF and HW VSI arrays */ 8331 ice_vsi_clear(ch->ch_vsi); 8332 8333 /* free the channel */ 8334 kfree(ch); 8335 } 8336 8337 /* clear the channel VSI map which is stored in main VSI */ 8338 ice_for_each_chnl_tc(i) 8339 vsi->tc_map_vsi[i] = NULL; 8340 8341 /* reset main VSI's all TC information */ 8342 vsi->all_enatc = 0; 8343 vsi->all_numtc = 0; 8344 } 8345 8346 /** 8347 * ice_rebuild_channels - rebuild channel 8348 * @pf: ptr to PF 8349 * 8350 * Recreate channel VSIs and replay filters 8351 */ 8352 static int ice_rebuild_channels(struct ice_pf *pf) 8353 { 8354 struct device *dev = ice_pf_to_dev(pf); 8355 struct ice_vsi *main_vsi; 8356 bool rem_adv_fltr = true; 8357 struct ice_channel *ch; 8358 struct ice_vsi *vsi; 8359 int tc_idx = 1; 8360 int i, err; 8361 8362 main_vsi = ice_get_main_vsi(pf); 8363 if (!main_vsi) 8364 return 0; 8365 8366 if (!test_bit(ICE_FLAG_TC_MQPRIO, pf->flags) || 8367 main_vsi->old_numtc == 1) 8368 return 0; /* nothing to be done */ 8369 8370 /* reconfigure main VSI based on old value of TC and cached values 8371 * for MQPRIO opts 8372 */ 8373 err = ice_vsi_cfg_tc(main_vsi, main_vsi->old_ena_tc); 8374 if (err) { 8375 dev_err(dev, "failed configuring TC(ena_tc:0x%02x) for HW VSI=%u\n", 8376 main_vsi->old_ena_tc, main_vsi->vsi_num); 8377 return err; 8378 } 8379 8380 /* rebuild ADQ VSIs */ 8381 ice_for_each_vsi(pf, i) { 8382 enum ice_vsi_type type; 8383 8384 vsi = pf->vsi[i]; 8385 if (!vsi || vsi->type != ICE_VSI_CHNL) 8386 continue; 8387 8388 type = vsi->type; 8389 8390 /* rebuild ADQ VSI */ 8391 err = ice_vsi_rebuild(vsi, true); 8392 if (err) { 8393 dev_err(dev, "VSI (type:%s) at index %d rebuild failed, err %d\n", 8394 ice_vsi_type_str(type), vsi->idx, err); 8395 goto cleanup; 8396 } 8397 8398 /* Re-map HW VSI number, using VSI handle that has been 8399 * previously validated in ice_replay_vsi() call above 8400 */ 8401 vsi->vsi_num = ice_get_hw_vsi_num(&pf->hw, vsi->idx); 8402 8403 /* replay filters for the VSI */ 8404 err = ice_replay_vsi(&pf->hw, vsi->idx); 8405 if (err) { 8406 dev_err(dev, "VSI (type:%s) replay failed, err %d, VSI index %d\n", 8407 ice_vsi_type_str(type), err, vsi->idx); 8408 rem_adv_fltr = false; 8409 goto cleanup; 8410 } 8411 dev_info(dev, "VSI (type:%s) at index %d rebuilt successfully\n", 8412 ice_vsi_type_str(type), vsi->idx); 8413 8414 /* store ADQ VSI at correct TC index in main VSI's 8415 * map of TC to VSI 8416 */ 8417 main_vsi->tc_map_vsi[tc_idx++] = vsi; 8418 } 8419 8420 /* ADQ VSI(s) has been rebuilt successfully, so setup 8421 * channel for main VSI's Tx and Rx rings 8422 */ 8423 list_for_each_entry(ch, &main_vsi->ch_list, list) { 8424 struct ice_vsi *ch_vsi; 8425 8426 ch_vsi = ch->ch_vsi; 8427 if (!ch_vsi) 8428 continue; 8429 8430 /* reconfig channel resources */ 8431 ice_cfg_chnl_all_res(main_vsi, ch); 8432 8433 /* replay BW rate limit if it is non-zero */ 8434 if (!ch->max_tx_rate && !ch->min_tx_rate) 8435 continue; 8436 8437 err = ice_set_bw_limit(ch_vsi, ch->max_tx_rate, 8438 ch->min_tx_rate); 8439 if (err) 8440 dev_err(dev, "failed (err:%d) to rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n", 8441 err, ch->max_tx_rate, ch->min_tx_rate, 8442 ch_vsi->vsi_num); 8443 else 8444 dev_dbg(dev, "successfully rebuild BW rate limit, max_tx_rate: %llu Kbps, min_tx_rate: %llu Kbps for VSI(%u)\n", 8445 ch->max_tx_rate, ch->min_tx_rate, 8446 ch_vsi->vsi_num); 8447 } 8448 8449 /* reconfig RSS for main VSI */ 8450 if (main_vsi->ch_rss_size) 8451 ice_vsi_cfg_rss_lut_key(main_vsi); 8452 8453 return 0; 8454 8455 cleanup: 8456 ice_remove_q_channels(main_vsi, rem_adv_fltr); 8457 return err; 8458 } 8459 8460 /** 8461 * ice_create_q_channels - Add queue channel for the given TCs 8462 * @vsi: VSI to be configured 8463 * 8464 * Configures queue channel mapping to the given TCs 8465 */ 8466 static int ice_create_q_channels(struct ice_vsi *vsi) 8467 { 8468 struct ice_pf *pf = vsi->back; 8469 struct ice_channel *ch; 8470 int ret = 0, i; 8471 8472 ice_for_each_chnl_tc(i) { 8473 if (!(vsi->all_enatc & BIT(i))) 8474 continue; 8475 8476 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 8477 if (!ch) { 8478 ret = -ENOMEM; 8479 goto err_free; 8480 } 8481 INIT_LIST_HEAD(&ch->list); 8482 ch->num_rxq = vsi->mqprio_qopt.qopt.count[i]; 8483 ch->num_txq = vsi->mqprio_qopt.qopt.count[i]; 8484 ch->base_q = vsi->mqprio_qopt.qopt.offset[i]; 8485 ch->max_tx_rate = vsi->mqprio_qopt.max_rate[i]; 8486 ch->min_tx_rate = vsi->mqprio_qopt.min_rate[i]; 8487 8488 /* convert to Kbits/s */ 8489 if (ch->max_tx_rate) 8490 ch->max_tx_rate = div_u64(ch->max_tx_rate, 8491 ICE_BW_KBPS_DIVISOR); 8492 if (ch->min_tx_rate) 8493 ch->min_tx_rate = div_u64(ch->min_tx_rate, 8494 ICE_BW_KBPS_DIVISOR); 8495 8496 ret = ice_create_q_channel(vsi, ch); 8497 if (ret) { 8498 dev_err(ice_pf_to_dev(pf), 8499 "failed creating channel TC:%d\n", i); 8500 kfree(ch); 8501 goto err_free; 8502 } 8503 list_add_tail(&ch->list, &vsi->ch_list); 8504 vsi->tc_map_vsi[i] = ch->ch_vsi; 8505 dev_dbg(ice_pf_to_dev(pf), 8506 "successfully created channel: VSI %pK\n", ch->ch_vsi); 8507 } 8508 return 0; 8509 8510 err_free: 8511 ice_remove_q_channels(vsi, false); 8512 8513 return ret; 8514 } 8515 8516 /** 8517 * ice_setup_tc_mqprio_qdisc - configure multiple traffic classes 8518 * @netdev: net device to configure 8519 * @type_data: TC offload data 8520 */ 8521 static int ice_setup_tc_mqprio_qdisc(struct net_device *netdev, void *type_data) 8522 { 8523 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 8524 struct ice_netdev_priv *np = netdev_priv(netdev); 8525 struct ice_vsi *vsi = np->vsi; 8526 struct ice_pf *pf = vsi->back; 8527 u16 mode, ena_tc_qdisc = 0; 8528 int cur_txq, cur_rxq; 8529 u8 hw = 0, num_tcf; 8530 struct device *dev; 8531 int ret, i; 8532 8533 dev = ice_pf_to_dev(pf); 8534 num_tcf = mqprio_qopt->qopt.num_tc; 8535 hw = mqprio_qopt->qopt.hw; 8536 mode = mqprio_qopt->mode; 8537 if (!hw) { 8538 clear_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 8539 vsi->ch_rss_size = 0; 8540 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 8541 goto config_tcf; 8542 } 8543 8544 /* Generate queue region map for number of TCF requested */ 8545 for (i = 0; i < num_tcf; i++) 8546 ena_tc_qdisc |= BIT(i); 8547 8548 switch (mode) { 8549 case TC_MQPRIO_MODE_CHANNEL: 8550 8551 ret = ice_validate_mqprio_qopt(vsi, mqprio_qopt); 8552 if (ret) { 8553 netdev_err(netdev, "failed to validate_mqprio_qopt(), ret %d\n", 8554 ret); 8555 return ret; 8556 } 8557 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 8558 set_bit(ICE_FLAG_TC_MQPRIO, pf->flags); 8559 /* don't assume state of hw_tc_offload during driver load 8560 * and set the flag for TC flower filter if hw_tc_offload 8561 * already ON 8562 */ 8563 if (vsi->netdev->features & NETIF_F_HW_TC) 8564 set_bit(ICE_FLAG_CLS_FLOWER, pf->flags); 8565 break; 8566 default: 8567 return -EINVAL; 8568 } 8569 8570 config_tcf: 8571 8572 /* Requesting same TCF configuration as already enabled */ 8573 if (ena_tc_qdisc == vsi->tc_cfg.ena_tc && 8574 mode != TC_MQPRIO_MODE_CHANNEL) 8575 return 0; 8576 8577 /* Pause VSI queues */ 8578 ice_dis_vsi(vsi, true); 8579 8580 if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) 8581 ice_remove_q_channels(vsi, true); 8582 8583 if (!hw && !test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 8584 vsi->req_txq = min_t(int, ice_get_avail_txq_count(pf), 8585 num_online_cpus()); 8586 vsi->req_rxq = min_t(int, ice_get_avail_rxq_count(pf), 8587 num_online_cpus()); 8588 } else { 8589 /* logic to rebuild VSI, same like ethtool -L */ 8590 u16 offset = 0, qcount_tx = 0, qcount_rx = 0; 8591 8592 for (i = 0; i < num_tcf; i++) { 8593 if (!(ena_tc_qdisc & BIT(i))) 8594 continue; 8595 8596 offset = vsi->mqprio_qopt.qopt.offset[i]; 8597 qcount_rx = vsi->mqprio_qopt.qopt.count[i]; 8598 qcount_tx = vsi->mqprio_qopt.qopt.count[i]; 8599 } 8600 vsi->req_txq = offset + qcount_tx; 8601 vsi->req_rxq = offset + qcount_rx; 8602 8603 /* store away original rss_size info, so that it gets reused 8604 * form ice_vsi_rebuild during tc-qdisc delete stage - to 8605 * determine, what should be the rss_sizefor main VSI 8606 */ 8607 vsi->orig_rss_size = vsi->rss_size; 8608 } 8609 8610 /* save current values of Tx and Rx queues before calling VSI rebuild 8611 * for fallback option 8612 */ 8613 cur_txq = vsi->num_txq; 8614 cur_rxq = vsi->num_rxq; 8615 8616 /* proceed with rebuild main VSI using correct number of queues */ 8617 ret = ice_vsi_rebuild(vsi, false); 8618 if (ret) { 8619 /* fallback to current number of queues */ 8620 dev_info(dev, "Rebuild failed with new queues, try with current number of queues\n"); 8621 vsi->req_txq = cur_txq; 8622 vsi->req_rxq = cur_rxq; 8623 clear_bit(ICE_RESET_FAILED, pf->state); 8624 if (ice_vsi_rebuild(vsi, false)) { 8625 dev_err(dev, "Rebuild of main VSI failed again\n"); 8626 return ret; 8627 } 8628 } 8629 8630 vsi->all_numtc = num_tcf; 8631 vsi->all_enatc = ena_tc_qdisc; 8632 ret = ice_vsi_cfg_tc(vsi, ena_tc_qdisc); 8633 if (ret) { 8634 netdev_err(netdev, "failed configuring TC for VSI id=%d\n", 8635 vsi->vsi_num); 8636 goto exit; 8637 } 8638 8639 if (test_bit(ICE_FLAG_TC_MQPRIO, pf->flags)) { 8640 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 8641 u64 min_tx_rate = vsi->mqprio_qopt.min_rate[0]; 8642 8643 /* set TC0 rate limit if specified */ 8644 if (max_tx_rate || min_tx_rate) { 8645 /* convert to Kbits/s */ 8646 if (max_tx_rate) 8647 max_tx_rate = div_u64(max_tx_rate, ICE_BW_KBPS_DIVISOR); 8648 if (min_tx_rate) 8649 min_tx_rate = div_u64(min_tx_rate, ICE_BW_KBPS_DIVISOR); 8650 8651 ret = ice_set_bw_limit(vsi, max_tx_rate, min_tx_rate); 8652 if (!ret) { 8653 dev_dbg(dev, "set Tx rate max %llu min %llu for VSI(%u)\n", 8654 max_tx_rate, min_tx_rate, vsi->vsi_num); 8655 } else { 8656 dev_err(dev, "failed to set Tx rate max %llu min %llu for VSI(%u)\n", 8657 max_tx_rate, min_tx_rate, vsi->vsi_num); 8658 goto exit; 8659 } 8660 } 8661 ret = ice_create_q_channels(vsi); 8662 if (ret) { 8663 netdev_err(netdev, "failed configuring queue channels\n"); 8664 goto exit; 8665 } else { 8666 netdev_dbg(netdev, "successfully configured channels\n"); 8667 } 8668 } 8669 8670 if (vsi->ch_rss_size) 8671 ice_vsi_cfg_rss_lut_key(vsi); 8672 8673 exit: 8674 /* if error, reset the all_numtc and all_enatc */ 8675 if (ret) { 8676 vsi->all_numtc = 0; 8677 vsi->all_enatc = 0; 8678 } 8679 /* resume VSI */ 8680 ice_ena_vsi(vsi, true); 8681 8682 return ret; 8683 } 8684 8685 static LIST_HEAD(ice_block_cb_list); 8686 8687 static int 8688 ice_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8689 void *type_data) 8690 { 8691 struct ice_netdev_priv *np = netdev_priv(netdev); 8692 struct ice_pf *pf = np->vsi->back; 8693 int err; 8694 8695 switch (type) { 8696 case TC_SETUP_BLOCK: 8697 return flow_block_cb_setup_simple(type_data, 8698 &ice_block_cb_list, 8699 ice_setup_tc_block_cb, 8700 np, np, true); 8701 case TC_SETUP_QDISC_MQPRIO: 8702 /* setup traffic classifier for receive side */ 8703 mutex_lock(&pf->tc_mutex); 8704 err = ice_setup_tc_mqprio_qdisc(netdev, type_data); 8705 mutex_unlock(&pf->tc_mutex); 8706 return err; 8707 default: 8708 return -EOPNOTSUPP; 8709 } 8710 return -EOPNOTSUPP; 8711 } 8712 8713 static struct ice_indr_block_priv * 8714 ice_indr_block_priv_lookup(struct ice_netdev_priv *np, 8715 struct net_device *netdev) 8716 { 8717 struct ice_indr_block_priv *cb_priv; 8718 8719 list_for_each_entry(cb_priv, &np->tc_indr_block_priv_list, list) { 8720 if (!cb_priv->netdev) 8721 return NULL; 8722 if (cb_priv->netdev == netdev) 8723 return cb_priv; 8724 } 8725 return NULL; 8726 } 8727 8728 static int 8729 ice_indr_setup_block_cb(enum tc_setup_type type, void *type_data, 8730 void *indr_priv) 8731 { 8732 struct ice_indr_block_priv *priv = indr_priv; 8733 struct ice_netdev_priv *np = priv->np; 8734 8735 switch (type) { 8736 case TC_SETUP_CLSFLOWER: 8737 return ice_setup_tc_cls_flower(np, priv->netdev, 8738 (struct flow_cls_offload *) 8739 type_data); 8740 default: 8741 return -EOPNOTSUPP; 8742 } 8743 } 8744 8745 static int 8746 ice_indr_setup_tc_block(struct net_device *netdev, struct Qdisc *sch, 8747 struct ice_netdev_priv *np, 8748 struct flow_block_offload *f, void *data, 8749 void (*cleanup)(struct flow_block_cb *block_cb)) 8750 { 8751 struct ice_indr_block_priv *indr_priv; 8752 struct flow_block_cb *block_cb; 8753 8754 if (!ice_is_tunnel_supported(netdev) && 8755 !(is_vlan_dev(netdev) && 8756 vlan_dev_real_dev(netdev) == np->vsi->netdev)) 8757 return -EOPNOTSUPP; 8758 8759 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 8760 return -EOPNOTSUPP; 8761 8762 switch (f->command) { 8763 case FLOW_BLOCK_BIND: 8764 indr_priv = ice_indr_block_priv_lookup(np, netdev); 8765 if (indr_priv) 8766 return -EEXIST; 8767 8768 indr_priv = kzalloc(sizeof(*indr_priv), GFP_KERNEL); 8769 if (!indr_priv) 8770 return -ENOMEM; 8771 8772 indr_priv->netdev = netdev; 8773 indr_priv->np = np; 8774 list_add(&indr_priv->list, &np->tc_indr_block_priv_list); 8775 8776 block_cb = 8777 flow_indr_block_cb_alloc(ice_indr_setup_block_cb, 8778 indr_priv, indr_priv, 8779 ice_rep_indr_tc_block_unbind, 8780 f, netdev, sch, data, np, 8781 cleanup); 8782 8783 if (IS_ERR(block_cb)) { 8784 list_del(&indr_priv->list); 8785 kfree(indr_priv); 8786 return PTR_ERR(block_cb); 8787 } 8788 flow_block_cb_add(block_cb, f); 8789 list_add_tail(&block_cb->driver_list, &ice_block_cb_list); 8790 break; 8791 case FLOW_BLOCK_UNBIND: 8792 indr_priv = ice_indr_block_priv_lookup(np, netdev); 8793 if (!indr_priv) 8794 return -ENOENT; 8795 8796 block_cb = flow_block_cb_lookup(f->block, 8797 ice_indr_setup_block_cb, 8798 indr_priv); 8799 if (!block_cb) 8800 return -ENOENT; 8801 8802 flow_indr_block_cb_remove(block_cb, f); 8803 8804 list_del(&block_cb->driver_list); 8805 break; 8806 default: 8807 return -EOPNOTSUPP; 8808 } 8809 return 0; 8810 } 8811 8812 static int 8813 ice_indr_setup_tc_cb(struct net_device *netdev, struct Qdisc *sch, 8814 void *cb_priv, enum tc_setup_type type, void *type_data, 8815 void *data, 8816 void (*cleanup)(struct flow_block_cb *block_cb)) 8817 { 8818 switch (type) { 8819 case TC_SETUP_BLOCK: 8820 return ice_indr_setup_tc_block(netdev, sch, cb_priv, type_data, 8821 data, cleanup); 8822 8823 default: 8824 return -EOPNOTSUPP; 8825 } 8826 } 8827 8828 /** 8829 * ice_open - Called when a network interface becomes active 8830 * @netdev: network interface device structure 8831 * 8832 * The open entry point is called when a network interface is made 8833 * active by the system (IFF_UP). At this point all resources needed 8834 * for transmit and receive operations are allocated, the interrupt 8835 * handler is registered with the OS, the netdev watchdog is enabled, 8836 * and the stack is notified that the interface is ready. 8837 * 8838 * Returns 0 on success, negative value on failure 8839 */ 8840 int ice_open(struct net_device *netdev) 8841 { 8842 struct ice_netdev_priv *np = netdev_priv(netdev); 8843 struct ice_pf *pf = np->vsi->back; 8844 8845 if (ice_is_reset_in_progress(pf->state)) { 8846 netdev_err(netdev, "can't open net device while reset is in progress"); 8847 return -EBUSY; 8848 } 8849 8850 return ice_open_internal(netdev); 8851 } 8852 8853 /** 8854 * ice_open_internal - Called when a network interface becomes active 8855 * @netdev: network interface device structure 8856 * 8857 * Internal ice_open implementation. Should not be used directly except for ice_open and reset 8858 * handling routine 8859 * 8860 * Returns 0 on success, negative value on failure 8861 */ 8862 int ice_open_internal(struct net_device *netdev) 8863 { 8864 struct ice_netdev_priv *np = netdev_priv(netdev); 8865 struct ice_vsi *vsi = np->vsi; 8866 struct ice_pf *pf = vsi->back; 8867 struct ice_port_info *pi; 8868 int err; 8869 8870 if (test_bit(ICE_NEEDS_RESTART, pf->state)) { 8871 netdev_err(netdev, "driver needs to be unloaded and reloaded\n"); 8872 return -EIO; 8873 } 8874 8875 netif_carrier_off(netdev); 8876 8877 pi = vsi->port_info; 8878 err = ice_update_link_info(pi); 8879 if (err) { 8880 netdev_err(netdev, "Failed to get link info, error %d\n", err); 8881 return err; 8882 } 8883 8884 ice_check_link_cfg_err(pf, pi->phy.link_info.link_cfg_err); 8885 8886 /* Set PHY if there is media, otherwise, turn off PHY */ 8887 if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) { 8888 clear_bit(ICE_FLAG_NO_MEDIA, pf->flags); 8889 if (!test_bit(ICE_PHY_INIT_COMPLETE, pf->state)) { 8890 err = ice_init_phy_user_cfg(pi); 8891 if (err) { 8892 netdev_err(netdev, "Failed to initialize PHY settings, error %d\n", 8893 err); 8894 return err; 8895 } 8896 } 8897 8898 err = ice_configure_phy(vsi); 8899 if (err) { 8900 netdev_err(netdev, "Failed to set physical link up, error %d\n", 8901 err); 8902 return err; 8903 } 8904 } else { 8905 set_bit(ICE_FLAG_NO_MEDIA, pf->flags); 8906 ice_set_link(vsi, false); 8907 } 8908 8909 err = ice_vsi_open(vsi); 8910 if (err) 8911 netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n", 8912 vsi->vsi_num, vsi->vsw->sw_id); 8913 8914 /* Update existing tunnels information */ 8915 udp_tunnel_get_rx_info(netdev); 8916 8917 return err; 8918 } 8919 8920 /** 8921 * ice_stop - Disables a network interface 8922 * @netdev: network interface device structure 8923 * 8924 * The stop entry point is called when an interface is de-activated by the OS, 8925 * and the netdevice enters the DOWN state. The hardware is still under the 8926 * driver's control, but the netdev interface is disabled. 8927 * 8928 * Returns success only - not allowed to fail 8929 */ 8930 int ice_stop(struct net_device *netdev) 8931 { 8932 struct ice_netdev_priv *np = netdev_priv(netdev); 8933 struct ice_vsi *vsi = np->vsi; 8934 struct ice_pf *pf = vsi->back; 8935 8936 if (ice_is_reset_in_progress(pf->state)) { 8937 netdev_err(netdev, "can't stop net device while reset is in progress"); 8938 return -EBUSY; 8939 } 8940 8941 ice_vsi_close(vsi); 8942 8943 return 0; 8944 } 8945 8946 /** 8947 * ice_features_check - Validate encapsulated packet conforms to limits 8948 * @skb: skb buffer 8949 * @netdev: This port's netdev 8950 * @features: Offload features that the stack believes apply 8951 */ 8952 static netdev_features_t 8953 ice_features_check(struct sk_buff *skb, 8954 struct net_device __always_unused *netdev, 8955 netdev_features_t features) 8956 { 8957 bool gso = skb_is_gso(skb); 8958 size_t len; 8959 8960 /* No point in doing any of this if neither checksum nor GSO are 8961 * being requested for this frame. We can rule out both by just 8962 * checking for CHECKSUM_PARTIAL 8963 */ 8964 if (skb->ip_summed != CHECKSUM_PARTIAL) 8965 return features; 8966 8967 /* We cannot support GSO if the MSS is going to be less than 8968 * 64 bytes. If it is then we need to drop support for GSO. 8969 */ 8970 if (gso && (skb_shinfo(skb)->gso_size < ICE_TXD_CTX_MIN_MSS)) 8971 features &= ~NETIF_F_GSO_MASK; 8972 8973 len = skb_network_offset(skb); 8974 if (len > ICE_TXD_MACLEN_MAX || len & 0x1) 8975 goto out_rm_features; 8976 8977 len = skb_network_header_len(skb); 8978 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 8979 goto out_rm_features; 8980 8981 if (skb->encapsulation) { 8982 /* this must work for VXLAN frames AND IPIP/SIT frames, and in 8983 * the case of IPIP frames, the transport header pointer is 8984 * after the inner header! So check to make sure that this 8985 * is a GRE or UDP_TUNNEL frame before doing that math. 8986 */ 8987 if (gso && (skb_shinfo(skb)->gso_type & 8988 (SKB_GSO_GRE | SKB_GSO_UDP_TUNNEL))) { 8989 len = skb_inner_network_header(skb) - 8990 skb_transport_header(skb); 8991 if (len > ICE_TXD_L4LEN_MAX || len & 0x1) 8992 goto out_rm_features; 8993 } 8994 8995 len = skb_inner_network_header_len(skb); 8996 if (len > ICE_TXD_IPLEN_MAX || len & 0x1) 8997 goto out_rm_features; 8998 } 8999 9000 return features; 9001 out_rm_features: 9002 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 9003 } 9004 9005 static const struct net_device_ops ice_netdev_safe_mode_ops = { 9006 .ndo_open = ice_open, 9007 .ndo_stop = ice_stop, 9008 .ndo_start_xmit = ice_start_xmit, 9009 .ndo_set_mac_address = ice_set_mac_address, 9010 .ndo_validate_addr = eth_validate_addr, 9011 .ndo_change_mtu = ice_change_mtu, 9012 .ndo_get_stats64 = ice_get_stats64, 9013 .ndo_tx_timeout = ice_tx_timeout, 9014 .ndo_bpf = ice_xdp_safe_mode, 9015 }; 9016 9017 static const struct net_device_ops ice_netdev_ops = { 9018 .ndo_open = ice_open, 9019 .ndo_stop = ice_stop, 9020 .ndo_start_xmit = ice_start_xmit, 9021 .ndo_select_queue = ice_select_queue, 9022 .ndo_features_check = ice_features_check, 9023 .ndo_fix_features = ice_fix_features, 9024 .ndo_set_rx_mode = ice_set_rx_mode, 9025 .ndo_set_mac_address = ice_set_mac_address, 9026 .ndo_validate_addr = eth_validate_addr, 9027 .ndo_change_mtu = ice_change_mtu, 9028 .ndo_get_stats64 = ice_get_stats64, 9029 .ndo_set_tx_maxrate = ice_set_tx_maxrate, 9030 .ndo_eth_ioctl = ice_eth_ioctl, 9031 .ndo_set_vf_spoofchk = ice_set_vf_spoofchk, 9032 .ndo_set_vf_mac = ice_set_vf_mac, 9033 .ndo_get_vf_config = ice_get_vf_cfg, 9034 .ndo_set_vf_trust = ice_set_vf_trust, 9035 .ndo_set_vf_vlan = ice_set_vf_port_vlan, 9036 .ndo_set_vf_link_state = ice_set_vf_link_state, 9037 .ndo_get_vf_stats = ice_get_vf_stats, 9038 .ndo_set_vf_rate = ice_set_vf_bw, 9039 .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid, 9040 .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid, 9041 .ndo_setup_tc = ice_setup_tc, 9042 .ndo_set_features = ice_set_features, 9043 .ndo_bridge_getlink = ice_bridge_getlink, 9044 .ndo_bridge_setlink = ice_bridge_setlink, 9045 .ndo_fdb_add = ice_fdb_add, 9046 .ndo_fdb_del = ice_fdb_del, 9047 #ifdef CONFIG_RFS_ACCEL 9048 .ndo_rx_flow_steer = ice_rx_flow_steer, 9049 #endif 9050 .ndo_tx_timeout = ice_tx_timeout, 9051 .ndo_bpf = ice_xdp, 9052 .ndo_xdp_xmit = ice_xdp_xmit, 9053 .ndo_xsk_wakeup = ice_xsk_wakeup, 9054 .ndo_get_devlink_port = ice_get_devlink_port, 9055 }; 9056