1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2021 Intel Corporation. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/of_net.h> 6 #include <linux/pci.h> 7 #include <linux/bpf.h> 8 #include <generated/utsrelease.h> 9 #include <linux/crash_dump.h> 10 11 /* Local includes */ 12 #include "i40e.h" 13 #include "i40e_diag.h" 14 #include "i40e_xsk.h" 15 #include <net/udp_tunnel.h> 16 #include <net/xdp_sock_drv.h> 17 /* All i40e tracepoints are defined by the include below, which 18 * must be included exactly once across the whole kernel with 19 * CREATE_TRACE_POINTS defined 20 */ 21 #define CREATE_TRACE_POINTS 22 #include "i40e_trace.h" 23 24 const char i40e_driver_name[] = "i40e"; 25 static const char i40e_driver_string[] = 26 "Intel(R) Ethernet Connection XL710 Network Driver"; 27 28 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation."; 29 30 /* a bit of forward declarations */ 31 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 32 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 33 static int i40e_add_vsi(struct i40e_vsi *vsi); 34 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 35 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired); 36 static int i40e_setup_misc_vector(struct i40e_pf *pf); 37 static void i40e_determine_queue_usage(struct i40e_pf *pf); 38 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 39 static void i40e_prep_for_reset(struct i40e_pf *pf); 40 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 41 bool lock_acquired); 42 static int i40e_reset(struct i40e_pf *pf); 43 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 44 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf); 45 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf); 46 static bool i40e_check_recovery_mode(struct i40e_pf *pf); 47 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw); 48 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 49 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 50 static int i40e_get_capabilities(struct i40e_pf *pf, 51 enum i40e_admin_queue_opc list_type); 52 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf); 53 54 /* i40e_pci_tbl - PCI Device ID Table 55 * 56 * Last entry must be all 0s 57 * 58 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 59 * Class, Class Mask, private data (not used) } 60 */ 61 static const struct pci_device_id i40e_pci_tbl[] = { 62 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 63 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 64 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_BC), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722_A), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 88 /* required last entry */ 89 {0, } 90 }; 91 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 92 93 #define I40E_MAX_VF_COUNT 128 94 static int debug = -1; 95 module_param(debug, uint, 0); 96 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 97 98 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 99 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 100 MODULE_LICENSE("GPL v2"); 101 102 static struct workqueue_struct *i40e_wq; 103 104 static void netdev_hw_addr_refcnt(struct i40e_mac_filter *f, 105 struct net_device *netdev, int delta) 106 { 107 struct netdev_hw_addr *ha; 108 109 if (!f || !netdev) 110 return; 111 112 netdev_for_each_mc_addr(ha, netdev) { 113 if (ether_addr_equal(ha->addr, f->macaddr)) { 114 ha->refcount += delta; 115 if (ha->refcount <= 0) 116 ha->refcount = 1; 117 break; 118 } 119 } 120 } 121 122 /** 123 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 124 * @hw: pointer to the HW structure 125 * @mem: ptr to mem struct to fill out 126 * @size: size of memory requested 127 * @alignment: what to align the allocation to 128 **/ 129 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 130 u64 size, u32 alignment) 131 { 132 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 133 134 mem->size = ALIGN(size, alignment); 135 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa, 136 GFP_KERNEL); 137 if (!mem->va) 138 return -ENOMEM; 139 140 return 0; 141 } 142 143 /** 144 * i40e_free_dma_mem_d - OS specific memory free for shared code 145 * @hw: pointer to the HW structure 146 * @mem: ptr to mem struct to free 147 **/ 148 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 149 { 150 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 151 152 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 153 mem->va = NULL; 154 mem->pa = 0; 155 mem->size = 0; 156 157 return 0; 158 } 159 160 /** 161 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 162 * @hw: pointer to the HW structure 163 * @mem: ptr to mem struct to fill out 164 * @size: size of memory requested 165 **/ 166 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 167 u32 size) 168 { 169 mem->size = size; 170 mem->va = kzalloc(size, GFP_KERNEL); 171 172 if (!mem->va) 173 return -ENOMEM; 174 175 return 0; 176 } 177 178 /** 179 * i40e_free_virt_mem_d - OS specific memory free for shared code 180 * @hw: pointer to the HW structure 181 * @mem: ptr to mem struct to free 182 **/ 183 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 184 { 185 /* it's ok to kfree a NULL pointer */ 186 kfree(mem->va); 187 mem->va = NULL; 188 mem->size = 0; 189 190 return 0; 191 } 192 193 /** 194 * i40e_get_lump - find a lump of free generic resource 195 * @pf: board private structure 196 * @pile: the pile of resource to search 197 * @needed: the number of items needed 198 * @id: an owner id to stick on the items assigned 199 * 200 * Returns the base item index of the lump, or negative for error 201 **/ 202 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 203 u16 needed, u16 id) 204 { 205 int ret = -ENOMEM; 206 int i, j; 207 208 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 209 dev_info(&pf->pdev->dev, 210 "param err: pile=%s needed=%d id=0x%04x\n", 211 pile ? "<valid>" : "<null>", needed, id); 212 return -EINVAL; 213 } 214 215 /* Allocate last queue in the pile for FDIR VSI queue 216 * so it doesn't fragment the qp_pile 217 */ 218 if (pile == pf->qp_pile && pf->vsi[id]->type == I40E_VSI_FDIR) { 219 if (pile->list[pile->num_entries - 1] & I40E_PILE_VALID_BIT) { 220 dev_err(&pf->pdev->dev, 221 "Cannot allocate queue %d for I40E_VSI_FDIR\n", 222 pile->num_entries - 1); 223 return -ENOMEM; 224 } 225 pile->list[pile->num_entries - 1] = id | I40E_PILE_VALID_BIT; 226 return pile->num_entries - 1; 227 } 228 229 i = 0; 230 while (i < pile->num_entries) { 231 /* skip already allocated entries */ 232 if (pile->list[i] & I40E_PILE_VALID_BIT) { 233 i++; 234 continue; 235 } 236 237 /* do we have enough in this lump? */ 238 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 239 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 240 break; 241 } 242 243 if (j == needed) { 244 /* there was enough, so assign it to the requestor */ 245 for (j = 0; j < needed; j++) 246 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 247 ret = i; 248 break; 249 } 250 251 /* not enough, so skip over it and continue looking */ 252 i += j; 253 } 254 255 return ret; 256 } 257 258 /** 259 * i40e_put_lump - return a lump of generic resource 260 * @pile: the pile of resource to search 261 * @index: the base item index 262 * @id: the owner id of the items assigned 263 * 264 * Returns the count of items in the lump 265 **/ 266 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 267 { 268 int valid_id = (id | I40E_PILE_VALID_BIT); 269 int count = 0; 270 u16 i; 271 272 if (!pile || index >= pile->num_entries) 273 return -EINVAL; 274 275 for (i = index; 276 i < pile->num_entries && pile->list[i] == valid_id; 277 i++) { 278 pile->list[i] = 0; 279 count++; 280 } 281 282 283 return count; 284 } 285 286 /** 287 * i40e_find_vsi_from_id - searches for the vsi with the given id 288 * @pf: the pf structure to search for the vsi 289 * @id: id of the vsi it is searching for 290 **/ 291 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 292 { 293 int i; 294 295 for (i = 0; i < pf->num_alloc_vsi; i++) 296 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 297 return pf->vsi[i]; 298 299 return NULL; 300 } 301 302 /** 303 * i40e_service_event_schedule - Schedule the service task to wake up 304 * @pf: board private structure 305 * 306 * If not already scheduled, this puts the task into the work queue 307 **/ 308 void i40e_service_event_schedule(struct i40e_pf *pf) 309 { 310 if ((!test_bit(__I40E_DOWN, pf->state) && 311 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) || 312 test_bit(__I40E_RECOVERY_MODE, pf->state)) 313 queue_work(i40e_wq, &pf->service_task); 314 } 315 316 /** 317 * i40e_tx_timeout - Respond to a Tx Hang 318 * @netdev: network interface device structure 319 * @txqueue: queue number timing out 320 * 321 * If any port has noticed a Tx timeout, it is likely that the whole 322 * device is munged, not just the one netdev port, so go for the full 323 * reset. 324 **/ 325 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue) 326 { 327 struct i40e_netdev_priv *np = netdev_priv(netdev); 328 struct i40e_vsi *vsi = np->vsi; 329 struct i40e_pf *pf = vsi->back; 330 struct i40e_ring *tx_ring = NULL; 331 unsigned int i; 332 u32 head, val; 333 334 pf->tx_timeout_count++; 335 336 /* with txqueue index, find the tx_ring struct */ 337 for (i = 0; i < vsi->num_queue_pairs; i++) { 338 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 339 if (txqueue == 340 vsi->tx_rings[i]->queue_index) { 341 tx_ring = vsi->tx_rings[i]; 342 break; 343 } 344 } 345 } 346 347 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 348 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 349 else if (time_before(jiffies, 350 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 351 return; /* don't do any new action before the next timeout */ 352 353 /* don't kick off another recovery if one is already pending */ 354 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state)) 355 return; 356 357 if (tx_ring) { 358 head = i40e_get_head(tx_ring); 359 /* Read interrupt register */ 360 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 361 val = rd32(&pf->hw, 362 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 363 tx_ring->vsi->base_vector - 1)); 364 else 365 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 366 367 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 368 vsi->seid, txqueue, tx_ring->next_to_clean, 369 head, tx_ring->next_to_use, 370 readl(tx_ring->tail), val); 371 } 372 373 pf->tx_timeout_last_recovery = jiffies; 374 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n", 375 pf->tx_timeout_recovery_level, txqueue); 376 377 switch (pf->tx_timeout_recovery_level) { 378 case 1: 379 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 380 break; 381 case 2: 382 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 383 break; 384 case 3: 385 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 386 break; 387 default: 388 netdev_err(netdev, "tx_timeout recovery unsuccessful, device is in non-recoverable state.\n"); 389 set_bit(__I40E_DOWN_REQUESTED, pf->state); 390 set_bit(__I40E_VSI_DOWN_REQUESTED, vsi->state); 391 break; 392 } 393 394 i40e_service_event_schedule(pf); 395 pf->tx_timeout_recovery_level++; 396 } 397 398 /** 399 * i40e_get_vsi_stats_struct - Get System Network Statistics 400 * @vsi: the VSI we care about 401 * 402 * Returns the address of the device statistics structure. 403 * The statistics are actually updated from the service task. 404 **/ 405 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 406 { 407 return &vsi->net_stats; 408 } 409 410 /** 411 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 412 * @ring: Tx ring to get statistics from 413 * @stats: statistics entry to be updated 414 **/ 415 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 416 struct rtnl_link_stats64 *stats) 417 { 418 u64 bytes, packets; 419 unsigned int start; 420 421 do { 422 start = u64_stats_fetch_begin(&ring->syncp); 423 packets = ring->stats.packets; 424 bytes = ring->stats.bytes; 425 } while (u64_stats_fetch_retry(&ring->syncp, start)); 426 427 stats->tx_packets += packets; 428 stats->tx_bytes += bytes; 429 } 430 431 /** 432 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 433 * @netdev: network interface device structure 434 * @stats: data structure to store statistics 435 * 436 * Returns the address of the device statistics structure. 437 * The statistics are actually updated from the service task. 438 **/ 439 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 440 struct rtnl_link_stats64 *stats) 441 { 442 struct i40e_netdev_priv *np = netdev_priv(netdev); 443 struct i40e_vsi *vsi = np->vsi; 444 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 445 struct i40e_ring *ring; 446 int i; 447 448 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 449 return; 450 451 if (!vsi->tx_rings) 452 return; 453 454 rcu_read_lock(); 455 for (i = 0; i < vsi->num_queue_pairs; i++) { 456 u64 bytes, packets; 457 unsigned int start; 458 459 ring = READ_ONCE(vsi->tx_rings[i]); 460 if (!ring) 461 continue; 462 i40e_get_netdev_stats_struct_tx(ring, stats); 463 464 if (i40e_enabled_xdp_vsi(vsi)) { 465 ring = READ_ONCE(vsi->xdp_rings[i]); 466 if (!ring) 467 continue; 468 i40e_get_netdev_stats_struct_tx(ring, stats); 469 } 470 471 ring = READ_ONCE(vsi->rx_rings[i]); 472 if (!ring) 473 continue; 474 do { 475 start = u64_stats_fetch_begin(&ring->syncp); 476 packets = ring->stats.packets; 477 bytes = ring->stats.bytes; 478 } while (u64_stats_fetch_retry(&ring->syncp, start)); 479 480 stats->rx_packets += packets; 481 stats->rx_bytes += bytes; 482 483 } 484 rcu_read_unlock(); 485 486 /* following stats updated by i40e_watchdog_subtask() */ 487 stats->multicast = vsi_stats->multicast; 488 stats->tx_errors = vsi_stats->tx_errors; 489 stats->tx_dropped = vsi_stats->tx_dropped; 490 stats->rx_errors = vsi_stats->rx_errors; 491 stats->rx_dropped = vsi_stats->rx_dropped; 492 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 493 stats->rx_length_errors = vsi_stats->rx_length_errors; 494 } 495 496 /** 497 * i40e_vsi_reset_stats - Resets all stats of the given vsi 498 * @vsi: the VSI to have its stats reset 499 **/ 500 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 501 { 502 struct rtnl_link_stats64 *ns; 503 int i; 504 505 if (!vsi) 506 return; 507 508 ns = i40e_get_vsi_stats_struct(vsi); 509 memset(ns, 0, sizeof(*ns)); 510 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 511 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 512 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 513 if (vsi->rx_rings && vsi->rx_rings[0]) { 514 for (i = 0; i < vsi->num_queue_pairs; i++) { 515 memset(&vsi->rx_rings[i]->stats, 0, 516 sizeof(vsi->rx_rings[i]->stats)); 517 memset(&vsi->rx_rings[i]->rx_stats, 0, 518 sizeof(vsi->rx_rings[i]->rx_stats)); 519 memset(&vsi->tx_rings[i]->stats, 0, 520 sizeof(vsi->tx_rings[i]->stats)); 521 memset(&vsi->tx_rings[i]->tx_stats, 0, 522 sizeof(vsi->tx_rings[i]->tx_stats)); 523 } 524 } 525 vsi->stat_offsets_loaded = false; 526 } 527 528 /** 529 * i40e_pf_reset_stats - Reset all of the stats for the given PF 530 * @pf: the PF to be reset 531 **/ 532 void i40e_pf_reset_stats(struct i40e_pf *pf) 533 { 534 int i; 535 536 memset(&pf->stats, 0, sizeof(pf->stats)); 537 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 538 pf->stat_offsets_loaded = false; 539 540 for (i = 0; i < I40E_MAX_VEB; i++) { 541 if (pf->veb[i]) { 542 memset(&pf->veb[i]->stats, 0, 543 sizeof(pf->veb[i]->stats)); 544 memset(&pf->veb[i]->stats_offsets, 0, 545 sizeof(pf->veb[i]->stats_offsets)); 546 memset(&pf->veb[i]->tc_stats, 0, 547 sizeof(pf->veb[i]->tc_stats)); 548 memset(&pf->veb[i]->tc_stats_offsets, 0, 549 sizeof(pf->veb[i]->tc_stats_offsets)); 550 pf->veb[i]->stat_offsets_loaded = false; 551 } 552 } 553 pf->hw_csum_rx_error = 0; 554 } 555 556 /** 557 * i40e_compute_pci_to_hw_id - compute index form PCI function. 558 * @vsi: ptr to the VSI to read from. 559 * @hw: ptr to the hardware info. 560 **/ 561 static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw) 562 { 563 int pf_count = i40e_get_pf_count(hw); 564 565 if (vsi->type == I40E_VSI_SRIOV) 566 return (hw->port * BIT(7)) / pf_count + vsi->vf_id; 567 568 return hw->port + BIT(7); 569 } 570 571 /** 572 * i40e_stat_update64 - read and update a 64 bit stat from the chip. 573 * @hw: ptr to the hardware info. 574 * @hireg: the high 32 bit reg to read. 575 * @loreg: the low 32 bit reg to read. 576 * @offset_loaded: has the initial offset been loaded yet. 577 * @offset: ptr to current offset value. 578 * @stat: ptr to the stat. 579 * 580 * Since the device stats are not reset at PFReset, they will not 581 * be zeroed when the driver starts. We'll save the first values read 582 * and use them as offsets to be subtracted from the raw values in order 583 * to report stats that count from zero. 584 **/ 585 static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg, 586 bool offset_loaded, u64 *offset, u64 *stat) 587 { 588 u64 new_data; 589 590 new_data = rd64(hw, loreg); 591 592 if (!offset_loaded || new_data < *offset) 593 *offset = new_data; 594 *stat = new_data - *offset; 595 } 596 597 /** 598 * i40e_stat_update48 - read and update a 48 bit stat from the chip 599 * @hw: ptr to the hardware info 600 * @hireg: the high 32 bit reg to read 601 * @loreg: the low 32 bit reg to read 602 * @offset_loaded: has the initial offset been loaded yet 603 * @offset: ptr to current offset value 604 * @stat: ptr to the stat 605 * 606 * Since the device stats are not reset at PFReset, they likely will not 607 * be zeroed when the driver starts. We'll save the first values read 608 * and use them as offsets to be subtracted from the raw values in order 609 * to report stats that count from zero. In the process, we also manage 610 * the potential roll-over. 611 **/ 612 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 613 bool offset_loaded, u64 *offset, u64 *stat) 614 { 615 u64 new_data; 616 617 if (hw->device_id == I40E_DEV_ID_QEMU) { 618 new_data = rd32(hw, loreg); 619 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 620 } else { 621 new_data = rd64(hw, loreg); 622 } 623 if (!offset_loaded) 624 *offset = new_data; 625 if (likely(new_data >= *offset)) 626 *stat = new_data - *offset; 627 else 628 *stat = (new_data + BIT_ULL(48)) - *offset; 629 *stat &= 0xFFFFFFFFFFFFULL; 630 } 631 632 /** 633 * i40e_stat_update32 - read and update a 32 bit stat from the chip 634 * @hw: ptr to the hardware info 635 * @reg: the hw reg to read 636 * @offset_loaded: has the initial offset been loaded yet 637 * @offset: ptr to current offset value 638 * @stat: ptr to the stat 639 **/ 640 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 641 bool offset_loaded, u64 *offset, u64 *stat) 642 { 643 u32 new_data; 644 645 new_data = rd32(hw, reg); 646 if (!offset_loaded) 647 *offset = new_data; 648 if (likely(new_data >= *offset)) 649 *stat = (u32)(new_data - *offset); 650 else 651 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 652 } 653 654 /** 655 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 656 * @hw: ptr to the hardware info 657 * @reg: the hw reg to read and clear 658 * @stat: ptr to the stat 659 **/ 660 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 661 { 662 u32 new_data = rd32(hw, reg); 663 664 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 665 *stat += new_data; 666 } 667 668 /** 669 * i40e_stats_update_rx_discards - update rx_discards. 670 * @vsi: ptr to the VSI to be updated. 671 * @hw: ptr to the hardware info. 672 * @stat_idx: VSI's stat_counter_idx. 673 * @offset_loaded: ptr to the VSI's stat_offsets_loaded. 674 * @stat_offset: ptr to stat_offset to store first read of specific register. 675 * @stat: ptr to VSI's stat to be updated. 676 **/ 677 static void 678 i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw, 679 int stat_idx, bool offset_loaded, 680 struct i40e_eth_stats *stat_offset, 681 struct i40e_eth_stats *stat) 682 { 683 u64 rx_rdpc, rx_rxerr; 684 685 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded, 686 &stat_offset->rx_discards, &rx_rdpc); 687 i40e_stat_update64(hw, 688 I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)), 689 I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)), 690 offset_loaded, &stat_offset->rx_discards_other, 691 &rx_rxerr); 692 693 stat->rx_discards = rx_rdpc + rx_rxerr; 694 } 695 696 /** 697 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 698 * @vsi: the VSI to be updated 699 **/ 700 void i40e_update_eth_stats(struct i40e_vsi *vsi) 701 { 702 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 703 struct i40e_pf *pf = vsi->back; 704 struct i40e_hw *hw = &pf->hw; 705 struct i40e_eth_stats *oes; 706 struct i40e_eth_stats *es; /* device's eth stats */ 707 708 es = &vsi->eth_stats; 709 oes = &vsi->eth_stats_offsets; 710 711 /* Gather up the stats that the hw collects */ 712 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 713 vsi->stat_offsets_loaded, 714 &oes->tx_errors, &es->tx_errors); 715 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 716 vsi->stat_offsets_loaded, 717 &oes->rx_discards, &es->rx_discards); 718 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 719 vsi->stat_offsets_loaded, 720 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 721 722 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 723 I40E_GLV_GORCL(stat_idx), 724 vsi->stat_offsets_loaded, 725 &oes->rx_bytes, &es->rx_bytes); 726 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 727 I40E_GLV_UPRCL(stat_idx), 728 vsi->stat_offsets_loaded, 729 &oes->rx_unicast, &es->rx_unicast); 730 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 731 I40E_GLV_MPRCL(stat_idx), 732 vsi->stat_offsets_loaded, 733 &oes->rx_multicast, &es->rx_multicast); 734 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 735 I40E_GLV_BPRCL(stat_idx), 736 vsi->stat_offsets_loaded, 737 &oes->rx_broadcast, &es->rx_broadcast); 738 739 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 740 I40E_GLV_GOTCL(stat_idx), 741 vsi->stat_offsets_loaded, 742 &oes->tx_bytes, &es->tx_bytes); 743 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 744 I40E_GLV_UPTCL(stat_idx), 745 vsi->stat_offsets_loaded, 746 &oes->tx_unicast, &es->tx_unicast); 747 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 748 I40E_GLV_MPTCL(stat_idx), 749 vsi->stat_offsets_loaded, 750 &oes->tx_multicast, &es->tx_multicast); 751 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 752 I40E_GLV_BPTCL(stat_idx), 753 vsi->stat_offsets_loaded, 754 &oes->tx_broadcast, &es->tx_broadcast); 755 756 i40e_stats_update_rx_discards(vsi, hw, stat_idx, 757 vsi->stat_offsets_loaded, oes, es); 758 759 vsi->stat_offsets_loaded = true; 760 } 761 762 /** 763 * i40e_update_veb_stats - Update Switch component statistics 764 * @veb: the VEB being updated 765 **/ 766 void i40e_update_veb_stats(struct i40e_veb *veb) 767 { 768 struct i40e_pf *pf = veb->pf; 769 struct i40e_hw *hw = &pf->hw; 770 struct i40e_eth_stats *oes; 771 struct i40e_eth_stats *es; /* device's eth stats */ 772 struct i40e_veb_tc_stats *veb_oes; 773 struct i40e_veb_tc_stats *veb_es; 774 int i, idx = 0; 775 776 idx = veb->stats_idx; 777 es = &veb->stats; 778 oes = &veb->stats_offsets; 779 veb_es = &veb->tc_stats; 780 veb_oes = &veb->tc_stats_offsets; 781 782 /* Gather up the stats that the hw collects */ 783 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 784 veb->stat_offsets_loaded, 785 &oes->tx_discards, &es->tx_discards); 786 if (hw->revision_id > 0) 787 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 788 veb->stat_offsets_loaded, 789 &oes->rx_unknown_protocol, 790 &es->rx_unknown_protocol); 791 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 792 veb->stat_offsets_loaded, 793 &oes->rx_bytes, &es->rx_bytes); 794 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 795 veb->stat_offsets_loaded, 796 &oes->rx_unicast, &es->rx_unicast); 797 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 798 veb->stat_offsets_loaded, 799 &oes->rx_multicast, &es->rx_multicast); 800 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 801 veb->stat_offsets_loaded, 802 &oes->rx_broadcast, &es->rx_broadcast); 803 804 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 805 veb->stat_offsets_loaded, 806 &oes->tx_bytes, &es->tx_bytes); 807 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 808 veb->stat_offsets_loaded, 809 &oes->tx_unicast, &es->tx_unicast); 810 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 811 veb->stat_offsets_loaded, 812 &oes->tx_multicast, &es->tx_multicast); 813 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 814 veb->stat_offsets_loaded, 815 &oes->tx_broadcast, &es->tx_broadcast); 816 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 817 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 818 I40E_GLVEBTC_RPCL(i, idx), 819 veb->stat_offsets_loaded, 820 &veb_oes->tc_rx_packets[i], 821 &veb_es->tc_rx_packets[i]); 822 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 823 I40E_GLVEBTC_RBCL(i, idx), 824 veb->stat_offsets_loaded, 825 &veb_oes->tc_rx_bytes[i], 826 &veb_es->tc_rx_bytes[i]); 827 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 828 I40E_GLVEBTC_TPCL(i, idx), 829 veb->stat_offsets_loaded, 830 &veb_oes->tc_tx_packets[i], 831 &veb_es->tc_tx_packets[i]); 832 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 833 I40E_GLVEBTC_TBCL(i, idx), 834 veb->stat_offsets_loaded, 835 &veb_oes->tc_tx_bytes[i], 836 &veb_es->tc_tx_bytes[i]); 837 } 838 veb->stat_offsets_loaded = true; 839 } 840 841 /** 842 * i40e_update_vsi_stats - Update the vsi statistics counters. 843 * @vsi: the VSI to be updated 844 * 845 * There are a few instances where we store the same stat in a 846 * couple of different structs. This is partly because we have 847 * the netdev stats that need to be filled out, which is slightly 848 * different from the "eth_stats" defined by the chip and used in 849 * VF communications. We sort it out here. 850 **/ 851 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 852 { 853 u64 rx_page, rx_buf, rx_reuse, rx_alloc, rx_waive, rx_busy; 854 struct i40e_pf *pf = vsi->back; 855 struct rtnl_link_stats64 *ons; 856 struct rtnl_link_stats64 *ns; /* netdev stats */ 857 struct i40e_eth_stats *oes; 858 struct i40e_eth_stats *es; /* device's eth stats */ 859 u64 tx_restart, tx_busy; 860 struct i40e_ring *p; 861 u64 bytes, packets; 862 unsigned int start; 863 u64 tx_linearize; 864 u64 tx_force_wb; 865 u64 tx_stopped; 866 u64 rx_p, rx_b; 867 u64 tx_p, tx_b; 868 u16 q; 869 870 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 871 test_bit(__I40E_CONFIG_BUSY, pf->state)) 872 return; 873 874 ns = i40e_get_vsi_stats_struct(vsi); 875 ons = &vsi->net_stats_offsets; 876 es = &vsi->eth_stats; 877 oes = &vsi->eth_stats_offsets; 878 879 /* Gather up the netdev and vsi stats that the driver collects 880 * on the fly during packet processing 881 */ 882 rx_b = rx_p = 0; 883 tx_b = tx_p = 0; 884 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 885 tx_stopped = 0; 886 rx_page = 0; 887 rx_buf = 0; 888 rx_reuse = 0; 889 rx_alloc = 0; 890 rx_waive = 0; 891 rx_busy = 0; 892 rcu_read_lock(); 893 for (q = 0; q < vsi->num_queue_pairs; q++) { 894 /* locate Tx ring */ 895 p = READ_ONCE(vsi->tx_rings[q]); 896 if (!p) 897 continue; 898 899 do { 900 start = u64_stats_fetch_begin(&p->syncp); 901 packets = p->stats.packets; 902 bytes = p->stats.bytes; 903 } while (u64_stats_fetch_retry(&p->syncp, start)); 904 tx_b += bytes; 905 tx_p += packets; 906 tx_restart += p->tx_stats.restart_queue; 907 tx_busy += p->tx_stats.tx_busy; 908 tx_linearize += p->tx_stats.tx_linearize; 909 tx_force_wb += p->tx_stats.tx_force_wb; 910 tx_stopped += p->tx_stats.tx_stopped; 911 912 /* locate Rx ring */ 913 p = READ_ONCE(vsi->rx_rings[q]); 914 if (!p) 915 continue; 916 917 do { 918 start = u64_stats_fetch_begin(&p->syncp); 919 packets = p->stats.packets; 920 bytes = p->stats.bytes; 921 } while (u64_stats_fetch_retry(&p->syncp, start)); 922 rx_b += bytes; 923 rx_p += packets; 924 rx_buf += p->rx_stats.alloc_buff_failed; 925 rx_page += p->rx_stats.alloc_page_failed; 926 rx_reuse += p->rx_stats.page_reuse_count; 927 rx_alloc += p->rx_stats.page_alloc_count; 928 rx_waive += p->rx_stats.page_waive_count; 929 rx_busy += p->rx_stats.page_busy_count; 930 931 if (i40e_enabled_xdp_vsi(vsi)) { 932 /* locate XDP ring */ 933 p = READ_ONCE(vsi->xdp_rings[q]); 934 if (!p) 935 continue; 936 937 do { 938 start = u64_stats_fetch_begin(&p->syncp); 939 packets = p->stats.packets; 940 bytes = p->stats.bytes; 941 } while (u64_stats_fetch_retry(&p->syncp, start)); 942 tx_b += bytes; 943 tx_p += packets; 944 tx_restart += p->tx_stats.restart_queue; 945 tx_busy += p->tx_stats.tx_busy; 946 tx_linearize += p->tx_stats.tx_linearize; 947 tx_force_wb += p->tx_stats.tx_force_wb; 948 } 949 } 950 rcu_read_unlock(); 951 vsi->tx_restart = tx_restart; 952 vsi->tx_busy = tx_busy; 953 vsi->tx_linearize = tx_linearize; 954 vsi->tx_force_wb = tx_force_wb; 955 vsi->tx_stopped = tx_stopped; 956 vsi->rx_page_failed = rx_page; 957 vsi->rx_buf_failed = rx_buf; 958 vsi->rx_page_reuse = rx_reuse; 959 vsi->rx_page_alloc = rx_alloc; 960 vsi->rx_page_waive = rx_waive; 961 vsi->rx_page_busy = rx_busy; 962 963 ns->rx_packets = rx_p; 964 ns->rx_bytes = rx_b; 965 ns->tx_packets = tx_p; 966 ns->tx_bytes = tx_b; 967 968 /* update netdev stats from eth stats */ 969 i40e_update_eth_stats(vsi); 970 ons->tx_errors = oes->tx_errors; 971 ns->tx_errors = es->tx_errors; 972 ons->multicast = oes->rx_multicast; 973 ns->multicast = es->rx_multicast; 974 ons->rx_dropped = oes->rx_discards; 975 ns->rx_dropped = es->rx_discards; 976 ons->tx_dropped = oes->tx_discards; 977 ns->tx_dropped = es->tx_discards; 978 979 /* pull in a couple PF stats if this is the main vsi */ 980 if (vsi == pf->vsi[pf->lan_vsi]) { 981 ns->rx_crc_errors = pf->stats.crc_errors; 982 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 983 ns->rx_length_errors = pf->stats.rx_length_errors; 984 } 985 } 986 987 /** 988 * i40e_update_pf_stats - Update the PF statistics counters. 989 * @pf: the PF to be updated 990 **/ 991 static void i40e_update_pf_stats(struct i40e_pf *pf) 992 { 993 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 994 struct i40e_hw_port_stats *nsd = &pf->stats; 995 struct i40e_hw *hw = &pf->hw; 996 u32 val; 997 int i; 998 999 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 1000 I40E_GLPRT_GORCL(hw->port), 1001 pf->stat_offsets_loaded, 1002 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 1003 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 1004 I40E_GLPRT_GOTCL(hw->port), 1005 pf->stat_offsets_loaded, 1006 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 1007 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 1008 pf->stat_offsets_loaded, 1009 &osd->eth.rx_discards, 1010 &nsd->eth.rx_discards); 1011 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 1012 I40E_GLPRT_UPRCL(hw->port), 1013 pf->stat_offsets_loaded, 1014 &osd->eth.rx_unicast, 1015 &nsd->eth.rx_unicast); 1016 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 1017 I40E_GLPRT_MPRCL(hw->port), 1018 pf->stat_offsets_loaded, 1019 &osd->eth.rx_multicast, 1020 &nsd->eth.rx_multicast); 1021 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 1022 I40E_GLPRT_BPRCL(hw->port), 1023 pf->stat_offsets_loaded, 1024 &osd->eth.rx_broadcast, 1025 &nsd->eth.rx_broadcast); 1026 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 1027 I40E_GLPRT_UPTCL(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->eth.tx_unicast, 1030 &nsd->eth.tx_unicast); 1031 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 1032 I40E_GLPRT_MPTCL(hw->port), 1033 pf->stat_offsets_loaded, 1034 &osd->eth.tx_multicast, 1035 &nsd->eth.tx_multicast); 1036 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 1037 I40E_GLPRT_BPTCL(hw->port), 1038 pf->stat_offsets_loaded, 1039 &osd->eth.tx_broadcast, 1040 &nsd->eth.tx_broadcast); 1041 1042 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 1043 pf->stat_offsets_loaded, 1044 &osd->tx_dropped_link_down, 1045 &nsd->tx_dropped_link_down); 1046 1047 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 1048 pf->stat_offsets_loaded, 1049 &osd->crc_errors, &nsd->crc_errors); 1050 1051 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 1052 pf->stat_offsets_loaded, 1053 &osd->illegal_bytes, &nsd->illegal_bytes); 1054 1055 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 1056 pf->stat_offsets_loaded, 1057 &osd->mac_local_faults, 1058 &nsd->mac_local_faults); 1059 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 1060 pf->stat_offsets_loaded, 1061 &osd->mac_remote_faults, 1062 &nsd->mac_remote_faults); 1063 1064 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 1065 pf->stat_offsets_loaded, 1066 &osd->rx_length_errors, 1067 &nsd->rx_length_errors); 1068 1069 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 1070 pf->stat_offsets_loaded, 1071 &osd->link_xon_rx, &nsd->link_xon_rx); 1072 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 1073 pf->stat_offsets_loaded, 1074 &osd->link_xon_tx, &nsd->link_xon_tx); 1075 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 1076 pf->stat_offsets_loaded, 1077 &osd->link_xoff_rx, &nsd->link_xoff_rx); 1078 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 1079 pf->stat_offsets_loaded, 1080 &osd->link_xoff_tx, &nsd->link_xoff_tx); 1081 1082 for (i = 0; i < 8; i++) { 1083 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 1084 pf->stat_offsets_loaded, 1085 &osd->priority_xoff_rx[i], 1086 &nsd->priority_xoff_rx[i]); 1087 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 1088 pf->stat_offsets_loaded, 1089 &osd->priority_xon_rx[i], 1090 &nsd->priority_xon_rx[i]); 1091 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 1092 pf->stat_offsets_loaded, 1093 &osd->priority_xon_tx[i], 1094 &nsd->priority_xon_tx[i]); 1095 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 1096 pf->stat_offsets_loaded, 1097 &osd->priority_xoff_tx[i], 1098 &nsd->priority_xoff_tx[i]); 1099 i40e_stat_update32(hw, 1100 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 1101 pf->stat_offsets_loaded, 1102 &osd->priority_xon_2_xoff[i], 1103 &nsd->priority_xon_2_xoff[i]); 1104 } 1105 1106 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 1107 I40E_GLPRT_PRC64L(hw->port), 1108 pf->stat_offsets_loaded, 1109 &osd->rx_size_64, &nsd->rx_size_64); 1110 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 1111 I40E_GLPRT_PRC127L(hw->port), 1112 pf->stat_offsets_loaded, 1113 &osd->rx_size_127, &nsd->rx_size_127); 1114 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 1115 I40E_GLPRT_PRC255L(hw->port), 1116 pf->stat_offsets_loaded, 1117 &osd->rx_size_255, &nsd->rx_size_255); 1118 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 1119 I40E_GLPRT_PRC511L(hw->port), 1120 pf->stat_offsets_loaded, 1121 &osd->rx_size_511, &nsd->rx_size_511); 1122 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1123 I40E_GLPRT_PRC1023L(hw->port), 1124 pf->stat_offsets_loaded, 1125 &osd->rx_size_1023, &nsd->rx_size_1023); 1126 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1127 I40E_GLPRT_PRC1522L(hw->port), 1128 pf->stat_offsets_loaded, 1129 &osd->rx_size_1522, &nsd->rx_size_1522); 1130 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1131 I40E_GLPRT_PRC9522L(hw->port), 1132 pf->stat_offsets_loaded, 1133 &osd->rx_size_big, &nsd->rx_size_big); 1134 1135 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1136 I40E_GLPRT_PTC64L(hw->port), 1137 pf->stat_offsets_loaded, 1138 &osd->tx_size_64, &nsd->tx_size_64); 1139 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1140 I40E_GLPRT_PTC127L(hw->port), 1141 pf->stat_offsets_loaded, 1142 &osd->tx_size_127, &nsd->tx_size_127); 1143 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1144 I40E_GLPRT_PTC255L(hw->port), 1145 pf->stat_offsets_loaded, 1146 &osd->tx_size_255, &nsd->tx_size_255); 1147 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1148 I40E_GLPRT_PTC511L(hw->port), 1149 pf->stat_offsets_loaded, 1150 &osd->tx_size_511, &nsd->tx_size_511); 1151 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1152 I40E_GLPRT_PTC1023L(hw->port), 1153 pf->stat_offsets_loaded, 1154 &osd->tx_size_1023, &nsd->tx_size_1023); 1155 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1156 I40E_GLPRT_PTC1522L(hw->port), 1157 pf->stat_offsets_loaded, 1158 &osd->tx_size_1522, &nsd->tx_size_1522); 1159 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1160 I40E_GLPRT_PTC9522L(hw->port), 1161 pf->stat_offsets_loaded, 1162 &osd->tx_size_big, &nsd->tx_size_big); 1163 1164 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1165 pf->stat_offsets_loaded, 1166 &osd->rx_undersize, &nsd->rx_undersize); 1167 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1168 pf->stat_offsets_loaded, 1169 &osd->rx_fragments, &nsd->rx_fragments); 1170 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1171 pf->stat_offsets_loaded, 1172 &osd->rx_oversize, &nsd->rx_oversize); 1173 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1174 pf->stat_offsets_loaded, 1175 &osd->rx_jabber, &nsd->rx_jabber); 1176 1177 /* FDIR stats */ 1178 i40e_stat_update_and_clear32(hw, 1179 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1180 &nsd->fd_atr_match); 1181 i40e_stat_update_and_clear32(hw, 1182 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1183 &nsd->fd_sb_match); 1184 i40e_stat_update_and_clear32(hw, 1185 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1186 &nsd->fd_atr_tunnel_match); 1187 1188 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1189 nsd->tx_lpi_status = 1190 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1191 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1192 nsd->rx_lpi_status = 1193 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1194 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1195 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1196 pf->stat_offsets_loaded, 1197 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1198 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1199 pf->stat_offsets_loaded, 1200 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1201 1202 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1203 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1204 nsd->fd_sb_status = true; 1205 else 1206 nsd->fd_sb_status = false; 1207 1208 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1209 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1210 nsd->fd_atr_status = true; 1211 else 1212 nsd->fd_atr_status = false; 1213 1214 pf->stat_offsets_loaded = true; 1215 } 1216 1217 /** 1218 * i40e_update_stats - Update the various statistics counters. 1219 * @vsi: the VSI to be updated 1220 * 1221 * Update the various stats for this VSI and its related entities. 1222 **/ 1223 void i40e_update_stats(struct i40e_vsi *vsi) 1224 { 1225 struct i40e_pf *pf = vsi->back; 1226 1227 if (vsi == pf->vsi[pf->lan_vsi]) 1228 i40e_update_pf_stats(pf); 1229 1230 i40e_update_vsi_stats(vsi); 1231 } 1232 1233 /** 1234 * i40e_count_filters - counts VSI mac filters 1235 * @vsi: the VSI to be searched 1236 * 1237 * Returns count of mac filters 1238 **/ 1239 int i40e_count_filters(struct i40e_vsi *vsi) 1240 { 1241 struct i40e_mac_filter *f; 1242 struct hlist_node *h; 1243 int bkt; 1244 int cnt = 0; 1245 1246 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 1247 ++cnt; 1248 1249 return cnt; 1250 } 1251 1252 /** 1253 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1254 * @vsi: the VSI to be searched 1255 * @macaddr: the MAC address 1256 * @vlan: the vlan 1257 * 1258 * Returns ptr to the filter object or NULL 1259 **/ 1260 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1261 const u8 *macaddr, s16 vlan) 1262 { 1263 struct i40e_mac_filter *f; 1264 u64 key; 1265 1266 if (!vsi || !macaddr) 1267 return NULL; 1268 1269 key = i40e_addr_to_hkey(macaddr); 1270 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1271 if ((ether_addr_equal(macaddr, f->macaddr)) && 1272 (vlan == f->vlan)) 1273 return f; 1274 } 1275 return NULL; 1276 } 1277 1278 /** 1279 * i40e_find_mac - Find a mac addr in the macvlan filters list 1280 * @vsi: the VSI to be searched 1281 * @macaddr: the MAC address we are searching for 1282 * 1283 * Returns the first filter with the provided MAC address or NULL if 1284 * MAC address was not found 1285 **/ 1286 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1287 { 1288 struct i40e_mac_filter *f; 1289 u64 key; 1290 1291 if (!vsi || !macaddr) 1292 return NULL; 1293 1294 key = i40e_addr_to_hkey(macaddr); 1295 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1296 if ((ether_addr_equal(macaddr, f->macaddr))) 1297 return f; 1298 } 1299 return NULL; 1300 } 1301 1302 /** 1303 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1304 * @vsi: the VSI to be searched 1305 * 1306 * Returns true if VSI is in vlan mode or false otherwise 1307 **/ 1308 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1309 { 1310 /* If we have a PVID, always operate in VLAN mode */ 1311 if (vsi->info.pvid) 1312 return true; 1313 1314 /* We need to operate in VLAN mode whenever we have any filters with 1315 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1316 * time, incurring search cost repeatedly. However, we can notice two 1317 * things: 1318 * 1319 * 1) the only place where we can gain a VLAN filter is in 1320 * i40e_add_filter. 1321 * 1322 * 2) the only place where filters are actually removed is in 1323 * i40e_sync_filters_subtask. 1324 * 1325 * Thus, we can simply use a boolean value, has_vlan_filters which we 1326 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1327 * we have to perform the full search after deleting filters in 1328 * i40e_sync_filters_subtask, but we already have to search 1329 * filters here and can perform the check at the same time. This 1330 * results in avoiding embedding a loop for VLAN mode inside another 1331 * loop over all the filters, and should maintain correctness as noted 1332 * above. 1333 */ 1334 return vsi->has_vlan_filter; 1335 } 1336 1337 /** 1338 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1339 * @vsi: the VSI to configure 1340 * @tmp_add_list: list of filters ready to be added 1341 * @tmp_del_list: list of filters ready to be deleted 1342 * @vlan_filters: the number of active VLAN filters 1343 * 1344 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1345 * behave as expected. If we have any active VLAN filters remaining or about 1346 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1347 * so that they only match against untagged traffic. If we no longer have any 1348 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1349 * so that they match against both tagged and untagged traffic. In this way, 1350 * we ensure that we correctly receive the desired traffic. This ensures that 1351 * when we have an active VLAN we will receive only untagged traffic and 1352 * traffic matching active VLANs. If we have no active VLANs then we will 1353 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1354 * 1355 * Finally, in a similar fashion, this function also corrects filters when 1356 * there is an active PVID assigned to this VSI. 1357 * 1358 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1359 * 1360 * This function is only expected to be called from within 1361 * i40e_sync_vsi_filters. 1362 * 1363 * NOTE: This function expects to be called while under the 1364 * mac_filter_hash_lock 1365 */ 1366 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1367 struct hlist_head *tmp_add_list, 1368 struct hlist_head *tmp_del_list, 1369 int vlan_filters) 1370 { 1371 s16 pvid = le16_to_cpu(vsi->info.pvid); 1372 struct i40e_mac_filter *f, *add_head; 1373 struct i40e_new_mac_filter *new; 1374 struct hlist_node *h; 1375 int bkt, new_vlan; 1376 1377 /* To determine if a particular filter needs to be replaced we 1378 * have the three following conditions: 1379 * 1380 * a) if we have a PVID assigned, then all filters which are 1381 * not marked as VLAN=PVID must be replaced with filters that 1382 * are. 1383 * b) otherwise, if we have any active VLANS, all filters 1384 * which are marked as VLAN=-1 must be replaced with 1385 * filters marked as VLAN=0 1386 * c) finally, if we do not have any active VLANS, all filters 1387 * which are marked as VLAN=0 must be replaced with filters 1388 * marked as VLAN=-1 1389 */ 1390 1391 /* Update the filters about to be added in place */ 1392 hlist_for_each_entry(new, tmp_add_list, hlist) { 1393 if (pvid && new->f->vlan != pvid) 1394 new->f->vlan = pvid; 1395 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1396 new->f->vlan = 0; 1397 else if (!vlan_filters && new->f->vlan == 0) 1398 new->f->vlan = I40E_VLAN_ANY; 1399 } 1400 1401 /* Update the remaining active filters */ 1402 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1403 /* Combine the checks for whether a filter needs to be changed 1404 * and then determine the new VLAN inside the if block, in 1405 * order to avoid duplicating code for adding the new filter 1406 * then deleting the old filter. 1407 */ 1408 if ((pvid && f->vlan != pvid) || 1409 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1410 (!vlan_filters && f->vlan == 0)) { 1411 /* Determine the new vlan we will be adding */ 1412 if (pvid) 1413 new_vlan = pvid; 1414 else if (vlan_filters) 1415 new_vlan = 0; 1416 else 1417 new_vlan = I40E_VLAN_ANY; 1418 1419 /* Create the new filter */ 1420 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1421 if (!add_head) 1422 return -ENOMEM; 1423 1424 /* Create a temporary i40e_new_mac_filter */ 1425 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1426 if (!new) 1427 return -ENOMEM; 1428 1429 new->f = add_head; 1430 new->state = add_head->state; 1431 1432 /* Add the new filter to the tmp list */ 1433 hlist_add_head(&new->hlist, tmp_add_list); 1434 1435 /* Put the original filter into the delete list */ 1436 f->state = I40E_FILTER_REMOVE; 1437 hash_del(&f->hlist); 1438 hlist_add_head(&f->hlist, tmp_del_list); 1439 } 1440 } 1441 1442 vsi->has_vlan_filter = !!vlan_filters; 1443 1444 return 0; 1445 } 1446 1447 /** 1448 * i40e_get_vf_new_vlan - Get new vlan id on a vf 1449 * @vsi: the vsi to configure 1450 * @new_mac: new mac filter to be added 1451 * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL 1452 * @vlan_filters: the number of active VLAN filters 1453 * @trusted: flag if the VF is trusted 1454 * 1455 * Get new VLAN id based on current VLAN filters, trust, PVID 1456 * and vf-vlan-prune-disable flag. 1457 * 1458 * Returns the value of the new vlan filter or 1459 * the old value if no new filter is needed. 1460 */ 1461 static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi, 1462 struct i40e_new_mac_filter *new_mac, 1463 struct i40e_mac_filter *f, 1464 int vlan_filters, 1465 bool trusted) 1466 { 1467 s16 pvid = le16_to_cpu(vsi->info.pvid); 1468 struct i40e_pf *pf = vsi->back; 1469 bool is_any; 1470 1471 if (new_mac) 1472 f = new_mac->f; 1473 1474 if (pvid && f->vlan != pvid) 1475 return pvid; 1476 1477 is_any = (trusted || 1478 !(pf->flags & I40E_FLAG_VF_VLAN_PRUNING)); 1479 1480 if ((vlan_filters && f->vlan == I40E_VLAN_ANY) || 1481 (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) || 1482 (is_any && !vlan_filters && f->vlan == 0)) { 1483 if (is_any) 1484 return I40E_VLAN_ANY; 1485 else 1486 return 0; 1487 } 1488 1489 return f->vlan; 1490 } 1491 1492 /** 1493 * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary 1494 * @vsi: the vsi to configure 1495 * @tmp_add_list: list of filters ready to be added 1496 * @tmp_del_list: list of filters ready to be deleted 1497 * @vlan_filters: the number of active VLAN filters 1498 * @trusted: flag if the VF is trusted 1499 * 1500 * Correct VF VLAN filters based on current VLAN filters, trust, PVID 1501 * and vf-vlan-prune-disable flag. 1502 * 1503 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1504 * 1505 * This function is only expected to be called from within 1506 * i40e_sync_vsi_filters. 1507 * 1508 * NOTE: This function expects to be called while under the 1509 * mac_filter_hash_lock 1510 */ 1511 static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi, 1512 struct hlist_head *tmp_add_list, 1513 struct hlist_head *tmp_del_list, 1514 int vlan_filters, 1515 bool trusted) 1516 { 1517 struct i40e_mac_filter *f, *add_head; 1518 struct i40e_new_mac_filter *new_mac; 1519 struct hlist_node *h; 1520 int bkt, new_vlan; 1521 1522 hlist_for_each_entry(new_mac, tmp_add_list, hlist) { 1523 new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL, 1524 vlan_filters, trusted); 1525 } 1526 1527 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1528 new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters, 1529 trusted); 1530 if (new_vlan != f->vlan) { 1531 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1532 if (!add_head) 1533 return -ENOMEM; 1534 /* Create a temporary i40e_new_mac_filter */ 1535 new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC); 1536 if (!new_mac) 1537 return -ENOMEM; 1538 new_mac->f = add_head; 1539 new_mac->state = add_head->state; 1540 1541 /* Add the new filter to the tmp list */ 1542 hlist_add_head(&new_mac->hlist, tmp_add_list); 1543 1544 /* Put the original filter into the delete list */ 1545 f->state = I40E_FILTER_REMOVE; 1546 hash_del(&f->hlist); 1547 hlist_add_head(&f->hlist, tmp_del_list); 1548 } 1549 } 1550 1551 vsi->has_vlan_filter = !!vlan_filters; 1552 return 0; 1553 } 1554 1555 /** 1556 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1557 * @vsi: the PF Main VSI - inappropriate for any other VSI 1558 * @macaddr: the MAC address 1559 * 1560 * Remove whatever filter the firmware set up so the driver can manage 1561 * its own filtering intelligently. 1562 **/ 1563 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1564 { 1565 struct i40e_aqc_remove_macvlan_element_data element; 1566 struct i40e_pf *pf = vsi->back; 1567 1568 /* Only appropriate for the PF main VSI */ 1569 if (vsi->type != I40E_VSI_MAIN) 1570 return; 1571 1572 memset(&element, 0, sizeof(element)); 1573 ether_addr_copy(element.mac_addr, macaddr); 1574 element.vlan_tag = 0; 1575 /* Ignore error returns, some firmware does it this way... */ 1576 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1577 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1578 1579 memset(&element, 0, sizeof(element)); 1580 ether_addr_copy(element.mac_addr, macaddr); 1581 element.vlan_tag = 0; 1582 /* ...and some firmware does it this way. */ 1583 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1584 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1585 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1586 } 1587 1588 /** 1589 * i40e_add_filter - Add a mac/vlan filter to the VSI 1590 * @vsi: the VSI to be searched 1591 * @macaddr: the MAC address 1592 * @vlan: the vlan 1593 * 1594 * Returns ptr to the filter object or NULL when no memory available. 1595 * 1596 * NOTE: This function is expected to be called with mac_filter_hash_lock 1597 * being held. 1598 **/ 1599 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1600 const u8 *macaddr, s16 vlan) 1601 { 1602 struct i40e_mac_filter *f; 1603 u64 key; 1604 1605 if (!vsi || !macaddr) 1606 return NULL; 1607 1608 f = i40e_find_filter(vsi, macaddr, vlan); 1609 if (!f) { 1610 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1611 if (!f) 1612 return NULL; 1613 1614 /* Update the boolean indicating if we need to function in 1615 * VLAN mode. 1616 */ 1617 if (vlan >= 0) 1618 vsi->has_vlan_filter = true; 1619 1620 ether_addr_copy(f->macaddr, macaddr); 1621 f->vlan = vlan; 1622 f->state = I40E_FILTER_NEW; 1623 INIT_HLIST_NODE(&f->hlist); 1624 1625 key = i40e_addr_to_hkey(macaddr); 1626 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1627 1628 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1629 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1630 } 1631 1632 /* If we're asked to add a filter that has been marked for removal, it 1633 * is safe to simply restore it to active state. __i40e_del_filter 1634 * will have simply deleted any filters which were previously marked 1635 * NEW or FAILED, so if it is currently marked REMOVE it must have 1636 * previously been ACTIVE. Since we haven't yet run the sync filters 1637 * task, just restore this filter to the ACTIVE state so that the 1638 * sync task leaves it in place 1639 */ 1640 if (f->state == I40E_FILTER_REMOVE) 1641 f->state = I40E_FILTER_ACTIVE; 1642 1643 return f; 1644 } 1645 1646 /** 1647 * __i40e_del_filter - Remove a specific filter from the VSI 1648 * @vsi: VSI to remove from 1649 * @f: the filter to remove from the list 1650 * 1651 * This function should be called instead of i40e_del_filter only if you know 1652 * the exact filter you will remove already, such as via i40e_find_filter or 1653 * i40e_find_mac. 1654 * 1655 * NOTE: This function is expected to be called with mac_filter_hash_lock 1656 * being held. 1657 * ANOTHER NOTE: This function MUST be called from within the context of 1658 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1659 * instead of list_for_each_entry(). 1660 **/ 1661 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1662 { 1663 if (!f) 1664 return; 1665 1666 /* If the filter was never added to firmware then we can just delete it 1667 * directly and we don't want to set the status to remove or else an 1668 * admin queue command will unnecessarily fire. 1669 */ 1670 if ((f->state == I40E_FILTER_FAILED) || 1671 (f->state == I40E_FILTER_NEW)) { 1672 hash_del(&f->hlist); 1673 kfree(f); 1674 } else { 1675 f->state = I40E_FILTER_REMOVE; 1676 } 1677 1678 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1679 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1680 } 1681 1682 /** 1683 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1684 * @vsi: the VSI to be searched 1685 * @macaddr: the MAC address 1686 * @vlan: the VLAN 1687 * 1688 * NOTE: This function is expected to be called with mac_filter_hash_lock 1689 * being held. 1690 * ANOTHER NOTE: This function MUST be called from within the context of 1691 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1692 * instead of list_for_each_entry(). 1693 **/ 1694 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1695 { 1696 struct i40e_mac_filter *f; 1697 1698 if (!vsi || !macaddr) 1699 return; 1700 1701 f = i40e_find_filter(vsi, macaddr, vlan); 1702 __i40e_del_filter(vsi, f); 1703 } 1704 1705 /** 1706 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1707 * @vsi: the VSI to be searched 1708 * @macaddr: the mac address to be filtered 1709 * 1710 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1711 * go through all the macvlan filters and add a macvlan filter for each 1712 * unique vlan that already exists. If a PVID has been assigned, instead only 1713 * add the macaddr to that VLAN. 1714 * 1715 * Returns last filter added on success, else NULL 1716 **/ 1717 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1718 const u8 *macaddr) 1719 { 1720 struct i40e_mac_filter *f, *add = NULL; 1721 struct hlist_node *h; 1722 int bkt; 1723 1724 if (vsi->info.pvid) 1725 return i40e_add_filter(vsi, macaddr, 1726 le16_to_cpu(vsi->info.pvid)); 1727 1728 if (!i40e_is_vsi_in_vlan(vsi)) 1729 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1730 1731 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1732 if (f->state == I40E_FILTER_REMOVE) 1733 continue; 1734 add = i40e_add_filter(vsi, macaddr, f->vlan); 1735 if (!add) 1736 return NULL; 1737 } 1738 1739 return add; 1740 } 1741 1742 /** 1743 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1744 * @vsi: the VSI to be searched 1745 * @macaddr: the mac address to be removed 1746 * 1747 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1748 * associated with. 1749 * 1750 * Returns 0 for success, or error 1751 **/ 1752 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1753 { 1754 struct i40e_mac_filter *f; 1755 struct hlist_node *h; 1756 bool found = false; 1757 int bkt; 1758 1759 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1760 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1761 if (ether_addr_equal(macaddr, f->macaddr)) { 1762 __i40e_del_filter(vsi, f); 1763 found = true; 1764 } 1765 } 1766 1767 if (found) 1768 return 0; 1769 else 1770 return -ENOENT; 1771 } 1772 1773 /** 1774 * i40e_set_mac - NDO callback to set mac address 1775 * @netdev: network interface device structure 1776 * @p: pointer to an address structure 1777 * 1778 * Returns 0 on success, negative on failure 1779 **/ 1780 static int i40e_set_mac(struct net_device *netdev, void *p) 1781 { 1782 struct i40e_netdev_priv *np = netdev_priv(netdev); 1783 struct i40e_vsi *vsi = np->vsi; 1784 struct i40e_pf *pf = vsi->back; 1785 struct i40e_hw *hw = &pf->hw; 1786 struct sockaddr *addr = p; 1787 1788 if (!is_valid_ether_addr(addr->sa_data)) 1789 return -EADDRNOTAVAIL; 1790 1791 if (test_bit(__I40E_DOWN, pf->state) || 1792 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 1793 return -EADDRNOTAVAIL; 1794 1795 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1796 netdev_info(netdev, "returning to hw mac address %pM\n", 1797 hw->mac.addr); 1798 else 1799 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1800 1801 /* Copy the address first, so that we avoid a possible race with 1802 * .set_rx_mode(). 1803 * - Remove old address from MAC filter 1804 * - Copy new address 1805 * - Add new address to MAC filter 1806 */ 1807 spin_lock_bh(&vsi->mac_filter_hash_lock); 1808 i40e_del_mac_filter(vsi, netdev->dev_addr); 1809 eth_hw_addr_set(netdev, addr->sa_data); 1810 i40e_add_mac_filter(vsi, netdev->dev_addr); 1811 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1812 1813 if (vsi->type == I40E_VSI_MAIN) { 1814 int ret; 1815 1816 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, 1817 addr->sa_data, NULL); 1818 if (ret) 1819 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %pe, AQ ret %s\n", 1820 ERR_PTR(ret), 1821 i40e_aq_str(hw, hw->aq.asq_last_status)); 1822 } 1823 1824 /* schedule our worker thread which will take care of 1825 * applying the new filter changes 1826 */ 1827 i40e_service_event_schedule(pf); 1828 return 0; 1829 } 1830 1831 /** 1832 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1833 * @vsi: vsi structure 1834 * @seed: RSS hash seed 1835 * @lut: pointer to lookup table of lut_size 1836 * @lut_size: size of the lookup table 1837 **/ 1838 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1839 u8 *lut, u16 lut_size) 1840 { 1841 struct i40e_pf *pf = vsi->back; 1842 struct i40e_hw *hw = &pf->hw; 1843 int ret = 0; 1844 1845 if (seed) { 1846 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1847 (struct i40e_aqc_get_set_rss_key_data *)seed; 1848 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1849 if (ret) { 1850 dev_info(&pf->pdev->dev, 1851 "Cannot set RSS key, err %pe aq_err %s\n", 1852 ERR_PTR(ret), 1853 i40e_aq_str(hw, hw->aq.asq_last_status)); 1854 return ret; 1855 } 1856 } 1857 if (lut) { 1858 bool pf_lut = vsi->type == I40E_VSI_MAIN; 1859 1860 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1861 if (ret) { 1862 dev_info(&pf->pdev->dev, 1863 "Cannot set RSS lut, err %pe aq_err %s\n", 1864 ERR_PTR(ret), 1865 i40e_aq_str(hw, hw->aq.asq_last_status)); 1866 return ret; 1867 } 1868 } 1869 return ret; 1870 } 1871 1872 /** 1873 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1874 * @vsi: VSI structure 1875 **/ 1876 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1877 { 1878 struct i40e_pf *pf = vsi->back; 1879 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1880 u8 *lut; 1881 int ret; 1882 1883 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1884 return 0; 1885 if (!vsi->rss_size) 1886 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1887 vsi->num_queue_pairs); 1888 if (!vsi->rss_size) 1889 return -EINVAL; 1890 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1891 if (!lut) 1892 return -ENOMEM; 1893 1894 /* Use the user configured hash keys and lookup table if there is one, 1895 * otherwise use default 1896 */ 1897 if (vsi->rss_lut_user) 1898 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1899 else 1900 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1901 if (vsi->rss_hkey_user) 1902 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1903 else 1904 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1905 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1906 kfree(lut); 1907 return ret; 1908 } 1909 1910 /** 1911 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1912 * @vsi: the VSI being configured, 1913 * @ctxt: VSI context structure 1914 * @enabled_tc: number of traffic classes to enable 1915 * 1916 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1917 **/ 1918 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1919 struct i40e_vsi_context *ctxt, 1920 u8 enabled_tc) 1921 { 1922 u16 qcount = 0, max_qcount, qmap, sections = 0; 1923 int i, override_q, pow, num_qps, ret; 1924 u8 netdev_tc = 0, offset = 0; 1925 1926 if (vsi->type != I40E_VSI_MAIN) 1927 return -EINVAL; 1928 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1929 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1930 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1931 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1932 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1933 1934 /* find the next higher power-of-2 of num queue pairs */ 1935 pow = ilog2(num_qps); 1936 if (!is_power_of_2(num_qps)) 1937 pow++; 1938 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1939 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1940 1941 /* Setup queue offset/count for all TCs for given VSI */ 1942 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1943 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1944 /* See if the given TC is enabled for the given VSI */ 1945 if (vsi->tc_config.enabled_tc & BIT(i)) { 1946 offset = vsi->mqprio_qopt.qopt.offset[i]; 1947 qcount = vsi->mqprio_qopt.qopt.count[i]; 1948 if (qcount > max_qcount) 1949 max_qcount = qcount; 1950 vsi->tc_config.tc_info[i].qoffset = offset; 1951 vsi->tc_config.tc_info[i].qcount = qcount; 1952 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1953 } else { 1954 /* TC is not enabled so set the offset to 1955 * default queue and allocate one queue 1956 * for the given TC. 1957 */ 1958 vsi->tc_config.tc_info[i].qoffset = 0; 1959 vsi->tc_config.tc_info[i].qcount = 1; 1960 vsi->tc_config.tc_info[i].netdev_tc = 0; 1961 } 1962 } 1963 1964 /* Set actual Tx/Rx queue pairs */ 1965 vsi->num_queue_pairs = offset + qcount; 1966 1967 /* Setup queue TC[0].qmap for given VSI context */ 1968 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1969 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1970 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1971 ctxt->info.valid_sections |= cpu_to_le16(sections); 1972 1973 /* Reconfigure RSS for main VSI with max queue count */ 1974 vsi->rss_size = max_qcount; 1975 ret = i40e_vsi_config_rss(vsi); 1976 if (ret) { 1977 dev_info(&vsi->back->pdev->dev, 1978 "Failed to reconfig rss for num_queues (%u)\n", 1979 max_qcount); 1980 return ret; 1981 } 1982 vsi->reconfig_rss = true; 1983 dev_dbg(&vsi->back->pdev->dev, 1984 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1985 1986 /* Find queue count available for channel VSIs and starting offset 1987 * for channel VSIs 1988 */ 1989 override_q = vsi->mqprio_qopt.qopt.count[0]; 1990 if (override_q && override_q < vsi->num_queue_pairs) { 1991 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1992 vsi->next_base_queue = override_q; 1993 } 1994 return 0; 1995 } 1996 1997 /** 1998 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1999 * @vsi: the VSI being setup 2000 * @ctxt: VSI context structure 2001 * @enabled_tc: Enabled TCs bitmap 2002 * @is_add: True if called before Add VSI 2003 * 2004 * Setup VSI queue mapping for enabled traffic classes. 2005 **/ 2006 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 2007 struct i40e_vsi_context *ctxt, 2008 u8 enabled_tc, 2009 bool is_add) 2010 { 2011 struct i40e_pf *pf = vsi->back; 2012 u16 num_tc_qps = 0; 2013 u16 sections = 0; 2014 u8 netdev_tc = 0; 2015 u16 numtc = 1; 2016 u16 qcount; 2017 u8 offset; 2018 u16 qmap; 2019 int i; 2020 2021 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 2022 offset = 0; 2023 /* zero out queue mapping, it will get updated on the end of the function */ 2024 memset(ctxt->info.queue_mapping, 0, sizeof(ctxt->info.queue_mapping)); 2025 2026 if (vsi->type == I40E_VSI_MAIN) { 2027 /* This code helps add more queue to the VSI if we have 2028 * more cores than RSS can support, the higher cores will 2029 * be served by ATR or other filters. Furthermore, the 2030 * non-zero req_queue_pairs says that user requested a new 2031 * queue count via ethtool's set_channels, so use this 2032 * value for queues distribution across traffic classes 2033 * We need at least one queue pair for the interface 2034 * to be usable as we see in else statement. 2035 */ 2036 if (vsi->req_queue_pairs > 0) 2037 vsi->num_queue_pairs = vsi->req_queue_pairs; 2038 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 2039 vsi->num_queue_pairs = pf->num_lan_msix; 2040 else 2041 vsi->num_queue_pairs = 1; 2042 } 2043 2044 /* Number of queues per enabled TC */ 2045 if (vsi->type == I40E_VSI_MAIN || 2046 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs != 0)) 2047 num_tc_qps = vsi->num_queue_pairs; 2048 else 2049 num_tc_qps = vsi->alloc_queue_pairs; 2050 2051 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 2052 /* Find numtc from enabled TC bitmap */ 2053 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2054 if (enabled_tc & BIT(i)) /* TC is enabled */ 2055 numtc++; 2056 } 2057 if (!numtc) { 2058 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 2059 numtc = 1; 2060 } 2061 num_tc_qps = num_tc_qps / numtc; 2062 num_tc_qps = min_t(int, num_tc_qps, 2063 i40e_pf_get_max_q_per_tc(pf)); 2064 } 2065 2066 vsi->tc_config.numtc = numtc; 2067 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 2068 2069 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 2070 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 2071 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 2072 2073 /* Setup queue offset/count for all TCs for given VSI */ 2074 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 2075 /* See if the given TC is enabled for the given VSI */ 2076 if (vsi->tc_config.enabled_tc & BIT(i)) { 2077 /* TC is enabled */ 2078 int pow, num_qps; 2079 2080 switch (vsi->type) { 2081 case I40E_VSI_MAIN: 2082 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | 2083 I40E_FLAG_FD_ATR_ENABLED)) || 2084 vsi->tc_config.enabled_tc != 1) { 2085 qcount = min_t(int, pf->alloc_rss_size, 2086 num_tc_qps); 2087 break; 2088 } 2089 fallthrough; 2090 case I40E_VSI_FDIR: 2091 case I40E_VSI_SRIOV: 2092 case I40E_VSI_VMDQ2: 2093 default: 2094 qcount = num_tc_qps; 2095 WARN_ON(i != 0); 2096 break; 2097 } 2098 vsi->tc_config.tc_info[i].qoffset = offset; 2099 vsi->tc_config.tc_info[i].qcount = qcount; 2100 2101 /* find the next higher power-of-2 of num queue pairs */ 2102 num_qps = qcount; 2103 pow = 0; 2104 while (num_qps && (BIT_ULL(pow) < qcount)) { 2105 pow++; 2106 num_qps >>= 1; 2107 } 2108 2109 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 2110 qmap = 2111 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 2112 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 2113 2114 offset += qcount; 2115 } else { 2116 /* TC is not enabled so set the offset to 2117 * default queue and allocate one queue 2118 * for the given TC. 2119 */ 2120 vsi->tc_config.tc_info[i].qoffset = 0; 2121 vsi->tc_config.tc_info[i].qcount = 1; 2122 vsi->tc_config.tc_info[i].netdev_tc = 0; 2123 2124 qmap = 0; 2125 } 2126 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 2127 } 2128 /* Do not change previously set num_queue_pairs for PFs and VFs*/ 2129 if ((vsi->type == I40E_VSI_MAIN && numtc != 1) || 2130 (vsi->type == I40E_VSI_SRIOV && vsi->num_queue_pairs == 0) || 2131 (vsi->type != I40E_VSI_MAIN && vsi->type != I40E_VSI_SRIOV)) 2132 vsi->num_queue_pairs = offset; 2133 2134 /* Scheduler section valid can only be set for ADD VSI */ 2135 if (is_add) { 2136 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 2137 2138 ctxt->info.up_enable_bits = enabled_tc; 2139 } 2140 if (vsi->type == I40E_VSI_SRIOV) { 2141 ctxt->info.mapping_flags |= 2142 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 2143 for (i = 0; i < vsi->num_queue_pairs; i++) 2144 ctxt->info.queue_mapping[i] = 2145 cpu_to_le16(vsi->base_queue + i); 2146 } else { 2147 ctxt->info.mapping_flags |= 2148 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 2149 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 2150 } 2151 ctxt->info.valid_sections |= cpu_to_le16(sections); 2152 } 2153 2154 /** 2155 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 2156 * @netdev: the netdevice 2157 * @addr: address to add 2158 * 2159 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 2160 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 2161 */ 2162 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 2163 { 2164 struct i40e_netdev_priv *np = netdev_priv(netdev); 2165 struct i40e_vsi *vsi = np->vsi; 2166 2167 if (i40e_add_mac_filter(vsi, addr)) 2168 return 0; 2169 else 2170 return -ENOMEM; 2171 } 2172 2173 /** 2174 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 2175 * @netdev: the netdevice 2176 * @addr: address to add 2177 * 2178 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 2179 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 2180 */ 2181 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 2182 { 2183 struct i40e_netdev_priv *np = netdev_priv(netdev); 2184 struct i40e_vsi *vsi = np->vsi; 2185 2186 /* Under some circumstances, we might receive a request to delete 2187 * our own device address from our uc list. Because we store the 2188 * device address in the VSI's MAC/VLAN filter list, we need to ignore 2189 * such requests and not delete our device address from this list. 2190 */ 2191 if (ether_addr_equal(addr, netdev->dev_addr)) 2192 return 0; 2193 2194 i40e_del_mac_filter(vsi, addr); 2195 2196 return 0; 2197 } 2198 2199 /** 2200 * i40e_set_rx_mode - NDO callback to set the netdev filters 2201 * @netdev: network interface device structure 2202 **/ 2203 static void i40e_set_rx_mode(struct net_device *netdev) 2204 { 2205 struct i40e_netdev_priv *np = netdev_priv(netdev); 2206 struct i40e_vsi *vsi = np->vsi; 2207 2208 spin_lock_bh(&vsi->mac_filter_hash_lock); 2209 2210 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 2211 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 2212 2213 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2214 2215 /* check for other flag changes */ 2216 if (vsi->current_netdev_flags != vsi->netdev->flags) { 2217 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2218 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 2219 } 2220 } 2221 2222 /** 2223 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 2224 * @vsi: Pointer to VSI struct 2225 * @from: Pointer to list which contains MAC filter entries - changes to 2226 * those entries needs to be undone. 2227 * 2228 * MAC filter entries from this list were slated for deletion. 2229 **/ 2230 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 2231 struct hlist_head *from) 2232 { 2233 struct i40e_mac_filter *f; 2234 struct hlist_node *h; 2235 2236 hlist_for_each_entry_safe(f, h, from, hlist) { 2237 u64 key = i40e_addr_to_hkey(f->macaddr); 2238 2239 /* Move the element back into MAC filter list*/ 2240 hlist_del(&f->hlist); 2241 hash_add(vsi->mac_filter_hash, &f->hlist, key); 2242 } 2243 } 2244 2245 /** 2246 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 2247 * @vsi: Pointer to vsi struct 2248 * @from: Pointer to list which contains MAC filter entries - changes to 2249 * those entries needs to be undone. 2250 * 2251 * MAC filter entries from this list were slated for addition. 2252 **/ 2253 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 2254 struct hlist_head *from) 2255 { 2256 struct i40e_new_mac_filter *new; 2257 struct hlist_node *h; 2258 2259 hlist_for_each_entry_safe(new, h, from, hlist) { 2260 /* We can simply free the wrapper structure */ 2261 hlist_del(&new->hlist); 2262 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1); 2263 kfree(new); 2264 } 2265 } 2266 2267 /** 2268 * i40e_next_filter - Get the next non-broadcast filter from a list 2269 * @next: pointer to filter in list 2270 * 2271 * Returns the next non-broadcast filter in the list. Required so that we 2272 * ignore broadcast filters within the list, since these are not handled via 2273 * the normal firmware update path. 2274 */ 2275 static 2276 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2277 { 2278 hlist_for_each_entry_continue(next, hlist) { 2279 if (!is_broadcast_ether_addr(next->f->macaddr)) 2280 return next; 2281 } 2282 2283 return NULL; 2284 } 2285 2286 /** 2287 * i40e_update_filter_state - Update filter state based on return data 2288 * from firmware 2289 * @count: Number of filters added 2290 * @add_list: return data from fw 2291 * @add_head: pointer to first filter in current batch 2292 * 2293 * MAC filter entries from list were slated to be added to device. Returns 2294 * number of successful filters. Note that 0 does NOT mean success! 2295 **/ 2296 static int 2297 i40e_update_filter_state(int count, 2298 struct i40e_aqc_add_macvlan_element_data *add_list, 2299 struct i40e_new_mac_filter *add_head) 2300 { 2301 int retval = 0; 2302 int i; 2303 2304 for (i = 0; i < count; i++) { 2305 /* Always check status of each filter. We don't need to check 2306 * the firmware return status because we pre-set the filter 2307 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2308 * request to the adminq. Thus, if it no longer matches then 2309 * we know the filter is active. 2310 */ 2311 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2312 add_head->state = I40E_FILTER_FAILED; 2313 } else { 2314 add_head->state = I40E_FILTER_ACTIVE; 2315 retval++; 2316 } 2317 2318 add_head = i40e_next_filter(add_head); 2319 if (!add_head) 2320 break; 2321 } 2322 2323 return retval; 2324 } 2325 2326 /** 2327 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2328 * @vsi: ptr to the VSI 2329 * @vsi_name: name to display in messages 2330 * @list: the list of filters to send to firmware 2331 * @num_del: the number of filters to delete 2332 * @retval: Set to -EIO on failure to delete 2333 * 2334 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2335 * *retval instead of a return value so that success does not force ret_val to 2336 * be set to 0. This ensures that a sequence of calls to this function 2337 * preserve the previous value of *retval on successful delete. 2338 */ 2339 static 2340 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2341 struct i40e_aqc_remove_macvlan_element_data *list, 2342 int num_del, int *retval) 2343 { 2344 struct i40e_hw *hw = &vsi->back->hw; 2345 enum i40e_admin_queue_err aq_status; 2346 int aq_ret; 2347 2348 aq_ret = i40e_aq_remove_macvlan_v2(hw, vsi->seid, list, num_del, NULL, 2349 &aq_status); 2350 2351 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2352 if (aq_ret && !(aq_status == I40E_AQ_RC_ENOENT)) { 2353 *retval = -EIO; 2354 dev_info(&vsi->back->pdev->dev, 2355 "ignoring delete macvlan error on %s, err %pe, aq_err %s\n", 2356 vsi_name, ERR_PTR(aq_ret), 2357 i40e_aq_str(hw, aq_status)); 2358 } 2359 } 2360 2361 /** 2362 * i40e_aqc_add_filters - Request firmware to add a set of filters 2363 * @vsi: ptr to the VSI 2364 * @vsi_name: name to display in messages 2365 * @list: the list of filters to send to firmware 2366 * @add_head: Position in the add hlist 2367 * @num_add: the number of filters to add 2368 * 2369 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2370 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2371 * space for more filters. 2372 */ 2373 static 2374 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2375 struct i40e_aqc_add_macvlan_element_data *list, 2376 struct i40e_new_mac_filter *add_head, 2377 int num_add) 2378 { 2379 struct i40e_hw *hw = &vsi->back->hw; 2380 enum i40e_admin_queue_err aq_status; 2381 int fcnt; 2382 2383 i40e_aq_add_macvlan_v2(hw, vsi->seid, list, num_add, NULL, &aq_status); 2384 fcnt = i40e_update_filter_state(num_add, list, add_head); 2385 2386 if (fcnt != num_add) { 2387 if (vsi->type == I40E_VSI_MAIN) { 2388 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2389 dev_warn(&vsi->back->pdev->dev, 2390 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2391 i40e_aq_str(hw, aq_status), vsi_name); 2392 } else if (vsi->type == I40E_VSI_SRIOV || 2393 vsi->type == I40E_VSI_VMDQ1 || 2394 vsi->type == I40E_VSI_VMDQ2) { 2395 dev_warn(&vsi->back->pdev->dev, 2396 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n", 2397 i40e_aq_str(hw, aq_status), vsi_name, 2398 vsi_name); 2399 } else { 2400 dev_warn(&vsi->back->pdev->dev, 2401 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n", 2402 i40e_aq_str(hw, aq_status), vsi_name, 2403 vsi->type); 2404 } 2405 } 2406 } 2407 2408 /** 2409 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2410 * @vsi: pointer to the VSI 2411 * @vsi_name: the VSI name 2412 * @f: filter data 2413 * 2414 * This function sets or clears the promiscuous broadcast flags for VLAN 2415 * filters in order to properly receive broadcast frames. Assumes that only 2416 * broadcast filters are passed. 2417 * 2418 * Returns status indicating success or failure; 2419 **/ 2420 static int 2421 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2422 struct i40e_mac_filter *f) 2423 { 2424 bool enable = f->state == I40E_FILTER_NEW; 2425 struct i40e_hw *hw = &vsi->back->hw; 2426 int aq_ret; 2427 2428 if (f->vlan == I40E_VLAN_ANY) { 2429 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2430 vsi->seid, 2431 enable, 2432 NULL); 2433 } else { 2434 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2435 vsi->seid, 2436 enable, 2437 f->vlan, 2438 NULL); 2439 } 2440 2441 if (aq_ret) { 2442 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2443 dev_warn(&vsi->back->pdev->dev, 2444 "Error %s, forcing overflow promiscuous on %s\n", 2445 i40e_aq_str(hw, hw->aq.asq_last_status), 2446 vsi_name); 2447 } 2448 2449 return aq_ret; 2450 } 2451 2452 /** 2453 * i40e_set_promiscuous - set promiscuous mode 2454 * @pf: board private structure 2455 * @promisc: promisc on or off 2456 * 2457 * There are different ways of setting promiscuous mode on a PF depending on 2458 * what state/environment we're in. This identifies and sets it appropriately. 2459 * Returns 0 on success. 2460 **/ 2461 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2462 { 2463 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2464 struct i40e_hw *hw = &pf->hw; 2465 int aq_ret; 2466 2467 if (vsi->type == I40E_VSI_MAIN && 2468 pf->lan_veb != I40E_NO_VEB && 2469 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2470 /* set defport ON for Main VSI instead of true promisc 2471 * this way we will get all unicast/multicast and VLAN 2472 * promisc behavior but will not get VF or VMDq traffic 2473 * replicated on the Main VSI. 2474 */ 2475 if (promisc) 2476 aq_ret = i40e_aq_set_default_vsi(hw, 2477 vsi->seid, 2478 NULL); 2479 else 2480 aq_ret = i40e_aq_clear_default_vsi(hw, 2481 vsi->seid, 2482 NULL); 2483 if (aq_ret) { 2484 dev_info(&pf->pdev->dev, 2485 "Set default VSI failed, err %pe, aq_err %s\n", 2486 ERR_PTR(aq_ret), 2487 i40e_aq_str(hw, hw->aq.asq_last_status)); 2488 } 2489 } else { 2490 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2491 hw, 2492 vsi->seid, 2493 promisc, NULL, 2494 true); 2495 if (aq_ret) { 2496 dev_info(&pf->pdev->dev, 2497 "set unicast promisc failed, err %pe, aq_err %s\n", 2498 ERR_PTR(aq_ret), 2499 i40e_aq_str(hw, hw->aq.asq_last_status)); 2500 } 2501 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2502 hw, 2503 vsi->seid, 2504 promisc, NULL); 2505 if (aq_ret) { 2506 dev_info(&pf->pdev->dev, 2507 "set multicast promisc failed, err %pe, aq_err %s\n", 2508 ERR_PTR(aq_ret), 2509 i40e_aq_str(hw, hw->aq.asq_last_status)); 2510 } 2511 } 2512 2513 if (!aq_ret) 2514 pf->cur_promisc = promisc; 2515 2516 return aq_ret; 2517 } 2518 2519 /** 2520 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2521 * @vsi: ptr to the VSI 2522 * 2523 * Push any outstanding VSI filter changes through the AdminQ. 2524 * 2525 * Returns 0 or error value 2526 **/ 2527 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2528 { 2529 struct hlist_head tmp_add_list, tmp_del_list; 2530 struct i40e_mac_filter *f; 2531 struct i40e_new_mac_filter *new, *add_head = NULL; 2532 struct i40e_hw *hw = &vsi->back->hw; 2533 bool old_overflow, new_overflow; 2534 unsigned int failed_filters = 0; 2535 unsigned int vlan_filters = 0; 2536 char vsi_name[16] = "PF"; 2537 int filter_list_len = 0; 2538 u32 changed_flags = 0; 2539 struct hlist_node *h; 2540 struct i40e_pf *pf; 2541 int num_add = 0; 2542 int num_del = 0; 2543 int aq_ret = 0; 2544 int retval = 0; 2545 u16 cmd_flags; 2546 int list_size; 2547 int bkt; 2548 2549 /* empty array typed pointers, kcalloc later */ 2550 struct i40e_aqc_add_macvlan_element_data *add_list; 2551 struct i40e_aqc_remove_macvlan_element_data *del_list; 2552 2553 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2554 usleep_range(1000, 2000); 2555 pf = vsi->back; 2556 2557 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2558 2559 if (vsi->netdev) { 2560 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2561 vsi->current_netdev_flags = vsi->netdev->flags; 2562 } 2563 2564 INIT_HLIST_HEAD(&tmp_add_list); 2565 INIT_HLIST_HEAD(&tmp_del_list); 2566 2567 if (vsi->type == I40E_VSI_SRIOV) 2568 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2569 else if (vsi->type != I40E_VSI_MAIN) 2570 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2571 2572 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2573 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2574 2575 spin_lock_bh(&vsi->mac_filter_hash_lock); 2576 /* Create a list of filters to delete. */ 2577 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2578 if (f->state == I40E_FILTER_REMOVE) { 2579 /* Move the element into temporary del_list */ 2580 hash_del(&f->hlist); 2581 hlist_add_head(&f->hlist, &tmp_del_list); 2582 2583 /* Avoid counting removed filters */ 2584 continue; 2585 } 2586 if (f->state == I40E_FILTER_NEW) { 2587 /* Create a temporary i40e_new_mac_filter */ 2588 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2589 if (!new) 2590 goto err_no_memory_locked; 2591 2592 /* Store pointer to the real filter */ 2593 new->f = f; 2594 new->state = f->state; 2595 2596 /* Add it to the hash list */ 2597 hlist_add_head(&new->hlist, &tmp_add_list); 2598 } 2599 2600 /* Count the number of active (current and new) VLAN 2601 * filters we have now. Does not count filters which 2602 * are marked for deletion. 2603 */ 2604 if (f->vlan > 0) 2605 vlan_filters++; 2606 } 2607 2608 if (vsi->type != I40E_VSI_SRIOV) 2609 retval = i40e_correct_mac_vlan_filters 2610 (vsi, &tmp_add_list, &tmp_del_list, 2611 vlan_filters); 2612 else 2613 retval = i40e_correct_vf_mac_vlan_filters 2614 (vsi, &tmp_add_list, &tmp_del_list, 2615 vlan_filters, pf->vf[vsi->vf_id].trusted); 2616 2617 hlist_for_each_entry(new, &tmp_add_list, hlist) 2618 netdev_hw_addr_refcnt(new->f, vsi->netdev, 1); 2619 2620 if (retval) 2621 goto err_no_memory_locked; 2622 2623 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2624 } 2625 2626 /* Now process 'del_list' outside the lock */ 2627 if (!hlist_empty(&tmp_del_list)) { 2628 filter_list_len = hw->aq.asq_buf_size / 2629 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2630 list_size = filter_list_len * 2631 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2632 del_list = kzalloc(list_size, GFP_ATOMIC); 2633 if (!del_list) 2634 goto err_no_memory; 2635 2636 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2637 cmd_flags = 0; 2638 2639 /* handle broadcast filters by updating the broadcast 2640 * promiscuous flag and release filter list. 2641 */ 2642 if (is_broadcast_ether_addr(f->macaddr)) { 2643 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2644 2645 hlist_del(&f->hlist); 2646 kfree(f); 2647 continue; 2648 } 2649 2650 /* add to delete list */ 2651 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2652 if (f->vlan == I40E_VLAN_ANY) { 2653 del_list[num_del].vlan_tag = 0; 2654 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2655 } else { 2656 del_list[num_del].vlan_tag = 2657 cpu_to_le16((u16)(f->vlan)); 2658 } 2659 2660 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2661 del_list[num_del].flags = cmd_flags; 2662 num_del++; 2663 2664 /* flush a full buffer */ 2665 if (num_del == filter_list_len) { 2666 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2667 num_del, &retval); 2668 memset(del_list, 0, list_size); 2669 num_del = 0; 2670 } 2671 /* Release memory for MAC filter entries which were 2672 * synced up with HW. 2673 */ 2674 hlist_del(&f->hlist); 2675 kfree(f); 2676 } 2677 2678 if (num_del) { 2679 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2680 num_del, &retval); 2681 } 2682 2683 kfree(del_list); 2684 del_list = NULL; 2685 } 2686 2687 if (!hlist_empty(&tmp_add_list)) { 2688 /* Do all the adds now. */ 2689 filter_list_len = hw->aq.asq_buf_size / 2690 sizeof(struct i40e_aqc_add_macvlan_element_data); 2691 list_size = filter_list_len * 2692 sizeof(struct i40e_aqc_add_macvlan_element_data); 2693 add_list = kzalloc(list_size, GFP_ATOMIC); 2694 if (!add_list) 2695 goto err_no_memory; 2696 2697 num_add = 0; 2698 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2699 /* handle broadcast filters by updating the broadcast 2700 * promiscuous flag instead of adding a MAC filter. 2701 */ 2702 if (is_broadcast_ether_addr(new->f->macaddr)) { 2703 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2704 new->f)) 2705 new->state = I40E_FILTER_FAILED; 2706 else 2707 new->state = I40E_FILTER_ACTIVE; 2708 continue; 2709 } 2710 2711 /* add to add array */ 2712 if (num_add == 0) 2713 add_head = new; 2714 cmd_flags = 0; 2715 ether_addr_copy(add_list[num_add].mac_addr, 2716 new->f->macaddr); 2717 if (new->f->vlan == I40E_VLAN_ANY) { 2718 add_list[num_add].vlan_tag = 0; 2719 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2720 } else { 2721 add_list[num_add].vlan_tag = 2722 cpu_to_le16((u16)(new->f->vlan)); 2723 } 2724 add_list[num_add].queue_number = 0; 2725 /* set invalid match method for later detection */ 2726 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2727 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2728 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2729 num_add++; 2730 2731 /* flush a full buffer */ 2732 if (num_add == filter_list_len) { 2733 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2734 add_head, num_add); 2735 memset(add_list, 0, list_size); 2736 num_add = 0; 2737 } 2738 } 2739 if (num_add) { 2740 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2741 num_add); 2742 } 2743 /* Now move all of the filters from the temp add list back to 2744 * the VSI's list. 2745 */ 2746 spin_lock_bh(&vsi->mac_filter_hash_lock); 2747 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2748 /* Only update the state if we're still NEW */ 2749 if (new->f->state == I40E_FILTER_NEW) 2750 new->f->state = new->state; 2751 hlist_del(&new->hlist); 2752 netdev_hw_addr_refcnt(new->f, vsi->netdev, -1); 2753 kfree(new); 2754 } 2755 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2756 kfree(add_list); 2757 add_list = NULL; 2758 } 2759 2760 /* Determine the number of active and failed filters. */ 2761 spin_lock_bh(&vsi->mac_filter_hash_lock); 2762 vsi->active_filters = 0; 2763 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2764 if (f->state == I40E_FILTER_ACTIVE) 2765 vsi->active_filters++; 2766 else if (f->state == I40E_FILTER_FAILED) 2767 failed_filters++; 2768 } 2769 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2770 2771 /* Check if we are able to exit overflow promiscuous mode. We can 2772 * safely exit if we didn't just enter, we no longer have any failed 2773 * filters, and we have reduced filters below the threshold value. 2774 */ 2775 if (old_overflow && !failed_filters && 2776 vsi->active_filters < vsi->promisc_threshold) { 2777 dev_info(&pf->pdev->dev, 2778 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2779 vsi_name); 2780 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2781 vsi->promisc_threshold = 0; 2782 } 2783 2784 /* if the VF is not trusted do not do promisc */ 2785 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2786 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2787 goto out; 2788 } 2789 2790 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2791 2792 /* If we are entering overflow promiscuous, we need to calculate a new 2793 * threshold for when we are safe to exit 2794 */ 2795 if (!old_overflow && new_overflow) 2796 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2797 2798 /* check for changes in promiscuous modes */ 2799 if (changed_flags & IFF_ALLMULTI) { 2800 bool cur_multipromisc; 2801 2802 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2803 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2804 vsi->seid, 2805 cur_multipromisc, 2806 NULL); 2807 if (aq_ret) { 2808 retval = i40e_aq_rc_to_posix(aq_ret, 2809 hw->aq.asq_last_status); 2810 dev_info(&pf->pdev->dev, 2811 "set multi promisc failed on %s, err %pe aq_err %s\n", 2812 vsi_name, 2813 ERR_PTR(aq_ret), 2814 i40e_aq_str(hw, hw->aq.asq_last_status)); 2815 } else { 2816 dev_info(&pf->pdev->dev, "%s allmulti mode.\n", 2817 cur_multipromisc ? "entering" : "leaving"); 2818 } 2819 } 2820 2821 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2822 bool cur_promisc; 2823 2824 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2825 new_overflow); 2826 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2827 if (aq_ret) { 2828 retval = i40e_aq_rc_to_posix(aq_ret, 2829 hw->aq.asq_last_status); 2830 dev_info(&pf->pdev->dev, 2831 "Setting promiscuous %s failed on %s, err %pe aq_err %s\n", 2832 cur_promisc ? "on" : "off", 2833 vsi_name, 2834 ERR_PTR(aq_ret), 2835 i40e_aq_str(hw, hw->aq.asq_last_status)); 2836 } 2837 } 2838 out: 2839 /* if something went wrong then set the changed flag so we try again */ 2840 if (retval) 2841 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2842 2843 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2844 return retval; 2845 2846 err_no_memory: 2847 /* Restore elements on the temporary add and delete lists */ 2848 spin_lock_bh(&vsi->mac_filter_hash_lock); 2849 err_no_memory_locked: 2850 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2851 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2852 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2853 2854 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2855 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2856 return -ENOMEM; 2857 } 2858 2859 /** 2860 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2861 * @pf: board private structure 2862 **/ 2863 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2864 { 2865 int v; 2866 2867 if (!pf) 2868 return; 2869 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2870 return; 2871 if (test_bit(__I40E_VF_DISABLE, pf->state)) { 2872 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 2873 return; 2874 } 2875 2876 for (v = 0; v < pf->num_alloc_vsi; v++) { 2877 if (pf->vsi[v] && 2878 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED) && 2879 !test_bit(__I40E_VSI_RELEASING, pf->vsi[v]->state)) { 2880 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2881 2882 if (ret) { 2883 /* come back and try again later */ 2884 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2885 pf->state); 2886 break; 2887 } 2888 } 2889 } 2890 } 2891 2892 /** 2893 * i40e_calculate_vsi_rx_buf_len - Calculates buffer length 2894 * 2895 * @vsi: VSI to calculate rx_buf_len from 2896 */ 2897 static u16 i40e_calculate_vsi_rx_buf_len(struct i40e_vsi *vsi) 2898 { 2899 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2900 return SKB_WITH_OVERHEAD(I40E_RXBUFFER_2048); 2901 2902 return PAGE_SIZE < 8192 ? I40E_RXBUFFER_3072 : I40E_RXBUFFER_2048; 2903 } 2904 2905 /** 2906 * i40e_max_vsi_frame_size - returns the maximum allowed frame size for VSI 2907 * @vsi: the vsi 2908 * @xdp_prog: XDP program 2909 **/ 2910 static int i40e_max_vsi_frame_size(struct i40e_vsi *vsi, 2911 struct bpf_prog *xdp_prog) 2912 { 2913 u16 rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi); 2914 u16 chain_len; 2915 2916 if (xdp_prog && !xdp_prog->aux->xdp_has_frags) 2917 chain_len = 1; 2918 else 2919 chain_len = I40E_MAX_CHAINED_RX_BUFFERS; 2920 2921 return min_t(u16, rx_buf_len * chain_len, I40E_MAX_RXBUFFER); 2922 } 2923 2924 /** 2925 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2926 * @netdev: network interface device structure 2927 * @new_mtu: new value for maximum frame size 2928 * 2929 * Returns 0 on success, negative on failure 2930 **/ 2931 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2932 { 2933 struct i40e_netdev_priv *np = netdev_priv(netdev); 2934 struct i40e_vsi *vsi = np->vsi; 2935 struct i40e_pf *pf = vsi->back; 2936 int frame_size; 2937 2938 frame_size = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog); 2939 if (new_mtu > frame_size - I40E_PACKET_HDR_PAD) { 2940 netdev_err(netdev, "Error changing mtu to %d, Max is %d\n", 2941 new_mtu, frame_size - I40E_PACKET_HDR_PAD); 2942 return -EINVAL; 2943 } 2944 2945 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2946 netdev->mtu, new_mtu); 2947 netdev->mtu = new_mtu; 2948 if (netif_running(netdev)) 2949 i40e_vsi_reinit_locked(vsi); 2950 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2951 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2952 return 0; 2953 } 2954 2955 /** 2956 * i40e_ioctl - Access the hwtstamp interface 2957 * @netdev: network interface device structure 2958 * @ifr: interface request data 2959 * @cmd: ioctl command 2960 **/ 2961 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2962 { 2963 struct i40e_netdev_priv *np = netdev_priv(netdev); 2964 struct i40e_pf *pf = np->vsi->back; 2965 2966 switch (cmd) { 2967 case SIOCGHWTSTAMP: 2968 return i40e_ptp_get_ts_config(pf, ifr); 2969 case SIOCSHWTSTAMP: 2970 return i40e_ptp_set_ts_config(pf, ifr); 2971 default: 2972 return -EOPNOTSUPP; 2973 } 2974 } 2975 2976 /** 2977 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2978 * @vsi: the vsi being adjusted 2979 **/ 2980 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2981 { 2982 struct i40e_vsi_context ctxt; 2983 int ret; 2984 2985 /* Don't modify stripping options if a port VLAN is active */ 2986 if (vsi->info.pvid) 2987 return; 2988 2989 if ((vsi->info.valid_sections & 2990 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2991 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2992 return; /* already enabled */ 2993 2994 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2995 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2996 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2997 2998 ctxt.seid = vsi->seid; 2999 ctxt.info = vsi->info; 3000 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3001 if (ret) { 3002 dev_info(&vsi->back->pdev->dev, 3003 "update vlan stripping failed, err %pe aq_err %s\n", 3004 ERR_PTR(ret), 3005 i40e_aq_str(&vsi->back->hw, 3006 vsi->back->hw.aq.asq_last_status)); 3007 } 3008 } 3009 3010 /** 3011 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 3012 * @vsi: the vsi being adjusted 3013 **/ 3014 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 3015 { 3016 struct i40e_vsi_context ctxt; 3017 int ret; 3018 3019 /* Don't modify stripping options if a port VLAN is active */ 3020 if (vsi->info.pvid) 3021 return; 3022 3023 if ((vsi->info.valid_sections & 3024 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 3025 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 3026 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 3027 return; /* already disabled */ 3028 3029 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3030 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 3031 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 3032 3033 ctxt.seid = vsi->seid; 3034 ctxt.info = vsi->info; 3035 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3036 if (ret) { 3037 dev_info(&vsi->back->pdev->dev, 3038 "update vlan stripping failed, err %pe aq_err %s\n", 3039 ERR_PTR(ret), 3040 i40e_aq_str(&vsi->back->hw, 3041 vsi->back->hw.aq.asq_last_status)); 3042 } 3043 } 3044 3045 /** 3046 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 3047 * @vsi: the vsi being configured 3048 * @vid: vlan id to be added (0 = untagged only , -1 = any) 3049 * 3050 * This is a helper function for adding a new MAC/VLAN filter with the 3051 * specified VLAN for each existing MAC address already in the hash table. 3052 * This function does *not* perform any accounting to update filters based on 3053 * VLAN mode. 3054 * 3055 * NOTE: this function expects to be called while under the 3056 * mac_filter_hash_lock 3057 **/ 3058 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 3059 { 3060 struct i40e_mac_filter *f, *add_f; 3061 struct hlist_node *h; 3062 int bkt; 3063 3064 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 3065 /* If we're asked to add a filter that has been marked for 3066 * removal, it is safe to simply restore it to active state. 3067 * __i40e_del_filter will have simply deleted any filters which 3068 * were previously marked NEW or FAILED, so if it is currently 3069 * marked REMOVE it must have previously been ACTIVE. Since we 3070 * haven't yet run the sync filters task, just restore this 3071 * filter to the ACTIVE state so that the sync task leaves it 3072 * in place. 3073 */ 3074 if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) { 3075 f->state = I40E_FILTER_ACTIVE; 3076 continue; 3077 } else if (f->state == I40E_FILTER_REMOVE) { 3078 continue; 3079 } 3080 add_f = i40e_add_filter(vsi, f->macaddr, vid); 3081 if (!add_f) { 3082 dev_info(&vsi->back->pdev->dev, 3083 "Could not add vlan filter %d for %pM\n", 3084 vid, f->macaddr); 3085 return -ENOMEM; 3086 } 3087 } 3088 3089 return 0; 3090 } 3091 3092 /** 3093 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 3094 * @vsi: the VSI being configured 3095 * @vid: VLAN id to be added 3096 **/ 3097 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 3098 { 3099 int err; 3100 3101 if (vsi->info.pvid) 3102 return -EINVAL; 3103 3104 /* The network stack will attempt to add VID=0, with the intention to 3105 * receive priority tagged packets with a VLAN of 0. Our HW receives 3106 * these packets by default when configured to receive untagged 3107 * packets, so we don't need to add a filter for this case. 3108 * Additionally, HW interprets adding a VID=0 filter as meaning to 3109 * receive *only* tagged traffic and stops receiving untagged traffic. 3110 * Thus, we do not want to actually add a filter for VID=0 3111 */ 3112 if (!vid) 3113 return 0; 3114 3115 /* Locked once because all functions invoked below iterates list*/ 3116 spin_lock_bh(&vsi->mac_filter_hash_lock); 3117 err = i40e_add_vlan_all_mac(vsi, vid); 3118 spin_unlock_bh(&vsi->mac_filter_hash_lock); 3119 if (err) 3120 return err; 3121 3122 /* schedule our worker thread which will take care of 3123 * applying the new filter changes 3124 */ 3125 i40e_service_event_schedule(vsi->back); 3126 return 0; 3127 } 3128 3129 /** 3130 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 3131 * @vsi: the vsi being configured 3132 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 3133 * 3134 * This function should be used to remove all VLAN filters which match the 3135 * given VID. It does not schedule the service event and does not take the 3136 * mac_filter_hash_lock so it may be combined with other operations under 3137 * a single invocation of the mac_filter_hash_lock. 3138 * 3139 * NOTE: this function expects to be called while under the 3140 * mac_filter_hash_lock 3141 */ 3142 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 3143 { 3144 struct i40e_mac_filter *f; 3145 struct hlist_node *h; 3146 int bkt; 3147 3148 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 3149 if (f->vlan == vid) 3150 __i40e_del_filter(vsi, f); 3151 } 3152 } 3153 3154 /** 3155 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 3156 * @vsi: the VSI being configured 3157 * @vid: VLAN id to be removed 3158 **/ 3159 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 3160 { 3161 if (!vid || vsi->info.pvid) 3162 return; 3163 3164 spin_lock_bh(&vsi->mac_filter_hash_lock); 3165 i40e_rm_vlan_all_mac(vsi, vid); 3166 spin_unlock_bh(&vsi->mac_filter_hash_lock); 3167 3168 /* schedule our worker thread which will take care of 3169 * applying the new filter changes 3170 */ 3171 i40e_service_event_schedule(vsi->back); 3172 } 3173 3174 /** 3175 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 3176 * @netdev: network interface to be adjusted 3177 * @proto: unused protocol value 3178 * @vid: vlan id to be added 3179 * 3180 * net_device_ops implementation for adding vlan ids 3181 **/ 3182 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 3183 __always_unused __be16 proto, u16 vid) 3184 { 3185 struct i40e_netdev_priv *np = netdev_priv(netdev); 3186 struct i40e_vsi *vsi = np->vsi; 3187 int ret = 0; 3188 3189 if (vid >= VLAN_N_VID) 3190 return -EINVAL; 3191 3192 ret = i40e_vsi_add_vlan(vsi, vid); 3193 if (!ret) 3194 set_bit(vid, vsi->active_vlans); 3195 3196 return ret; 3197 } 3198 3199 /** 3200 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 3201 * @netdev: network interface to be adjusted 3202 * @proto: unused protocol value 3203 * @vid: vlan id to be added 3204 **/ 3205 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 3206 __always_unused __be16 proto, u16 vid) 3207 { 3208 struct i40e_netdev_priv *np = netdev_priv(netdev); 3209 struct i40e_vsi *vsi = np->vsi; 3210 3211 if (vid >= VLAN_N_VID) 3212 return; 3213 set_bit(vid, vsi->active_vlans); 3214 } 3215 3216 /** 3217 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 3218 * @netdev: network interface to be adjusted 3219 * @proto: unused protocol value 3220 * @vid: vlan id to be removed 3221 * 3222 * net_device_ops implementation for removing vlan ids 3223 **/ 3224 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 3225 __always_unused __be16 proto, u16 vid) 3226 { 3227 struct i40e_netdev_priv *np = netdev_priv(netdev); 3228 struct i40e_vsi *vsi = np->vsi; 3229 3230 /* return code is ignored as there is nothing a user 3231 * can do about failure to remove and a log message was 3232 * already printed from the other function 3233 */ 3234 i40e_vsi_kill_vlan(vsi, vid); 3235 3236 clear_bit(vid, vsi->active_vlans); 3237 3238 return 0; 3239 } 3240 3241 /** 3242 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 3243 * @vsi: the vsi being brought back up 3244 **/ 3245 static void i40e_restore_vlan(struct i40e_vsi *vsi) 3246 { 3247 u16 vid; 3248 3249 if (!vsi->netdev) 3250 return; 3251 3252 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 3253 i40e_vlan_stripping_enable(vsi); 3254 else 3255 i40e_vlan_stripping_disable(vsi); 3256 3257 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 3258 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 3259 vid); 3260 } 3261 3262 /** 3263 * i40e_vsi_add_pvid - Add pvid for the VSI 3264 * @vsi: the vsi being adjusted 3265 * @vid: the vlan id to set as a PVID 3266 **/ 3267 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 3268 { 3269 struct i40e_vsi_context ctxt; 3270 int ret; 3271 3272 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 3273 vsi->info.pvid = cpu_to_le16(vid); 3274 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 3275 I40E_AQ_VSI_PVLAN_INSERT_PVID | 3276 I40E_AQ_VSI_PVLAN_EMOD_STR; 3277 3278 ctxt.seid = vsi->seid; 3279 ctxt.info = vsi->info; 3280 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 3281 if (ret) { 3282 dev_info(&vsi->back->pdev->dev, 3283 "add pvid failed, err %pe aq_err %s\n", 3284 ERR_PTR(ret), 3285 i40e_aq_str(&vsi->back->hw, 3286 vsi->back->hw.aq.asq_last_status)); 3287 return -ENOENT; 3288 } 3289 3290 return 0; 3291 } 3292 3293 /** 3294 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 3295 * @vsi: the vsi being adjusted 3296 * 3297 * Just use the vlan_rx_register() service to put it back to normal 3298 **/ 3299 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 3300 { 3301 vsi->info.pvid = 0; 3302 3303 i40e_vlan_stripping_disable(vsi); 3304 } 3305 3306 /** 3307 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 3308 * @vsi: ptr to the VSI 3309 * 3310 * If this function returns with an error, then it's possible one or 3311 * more of the rings is populated (while the rest are not). It is the 3312 * callers duty to clean those orphaned rings. 3313 * 3314 * Return 0 on success, negative on failure 3315 **/ 3316 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 3317 { 3318 int i, err = 0; 3319 3320 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3321 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 3322 3323 if (!i40e_enabled_xdp_vsi(vsi)) 3324 return err; 3325 3326 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3327 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3328 3329 return err; 3330 } 3331 3332 /** 3333 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3334 * @vsi: ptr to the VSI 3335 * 3336 * Free VSI's transmit software resources 3337 **/ 3338 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3339 { 3340 int i; 3341 3342 if (vsi->tx_rings) { 3343 for (i = 0; i < vsi->num_queue_pairs; i++) 3344 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3345 i40e_free_tx_resources(vsi->tx_rings[i]); 3346 } 3347 3348 if (vsi->xdp_rings) { 3349 for (i = 0; i < vsi->num_queue_pairs; i++) 3350 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3351 i40e_free_tx_resources(vsi->xdp_rings[i]); 3352 } 3353 } 3354 3355 /** 3356 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3357 * @vsi: ptr to the VSI 3358 * 3359 * If this function returns with an error, then it's possible one or 3360 * more of the rings is populated (while the rest are not). It is the 3361 * callers duty to clean those orphaned rings. 3362 * 3363 * Return 0 on success, negative on failure 3364 **/ 3365 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3366 { 3367 int i, err = 0; 3368 3369 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3370 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3371 return err; 3372 } 3373 3374 /** 3375 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3376 * @vsi: ptr to the VSI 3377 * 3378 * Free all receive software resources 3379 **/ 3380 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3381 { 3382 int i; 3383 3384 if (!vsi->rx_rings) 3385 return; 3386 3387 for (i = 0; i < vsi->num_queue_pairs; i++) 3388 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3389 i40e_free_rx_resources(vsi->rx_rings[i]); 3390 } 3391 3392 /** 3393 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3394 * @ring: The Tx ring to configure 3395 * 3396 * This enables/disables XPS for a given Tx descriptor ring 3397 * based on the TCs enabled for the VSI that ring belongs to. 3398 **/ 3399 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3400 { 3401 int cpu; 3402 3403 if (!ring->q_vector || !ring->netdev || ring->ch) 3404 return; 3405 3406 /* We only initialize XPS once, so as not to overwrite user settings */ 3407 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3408 return; 3409 3410 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3411 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3412 ring->queue_index); 3413 } 3414 3415 /** 3416 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled 3417 * @ring: The Tx or Rx ring 3418 * 3419 * Returns the AF_XDP buffer pool or NULL. 3420 **/ 3421 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring) 3422 { 3423 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi); 3424 int qid = ring->queue_index; 3425 3426 if (ring_is_xdp(ring)) 3427 qid -= ring->vsi->alloc_queue_pairs; 3428 3429 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps)) 3430 return NULL; 3431 3432 return xsk_get_pool_from_qid(ring->vsi->netdev, qid); 3433 } 3434 3435 /** 3436 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3437 * @ring: The Tx ring to configure 3438 * 3439 * Configure the Tx descriptor ring in the HMC context. 3440 **/ 3441 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3442 { 3443 struct i40e_vsi *vsi = ring->vsi; 3444 u16 pf_q = vsi->base_queue + ring->queue_index; 3445 struct i40e_hw *hw = &vsi->back->hw; 3446 struct i40e_hmc_obj_txq tx_ctx; 3447 u32 qtx_ctl = 0; 3448 int err = 0; 3449 3450 if (ring_is_xdp(ring)) 3451 ring->xsk_pool = i40e_xsk_pool(ring); 3452 3453 /* some ATR related tx ring init */ 3454 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3455 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3456 ring->atr_count = 0; 3457 } else { 3458 ring->atr_sample_rate = 0; 3459 } 3460 3461 /* configure XPS */ 3462 i40e_config_xps_tx_ring(ring); 3463 3464 /* clear the context structure first */ 3465 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3466 3467 tx_ctx.new_context = 1; 3468 tx_ctx.base = (ring->dma / 128); 3469 tx_ctx.qlen = ring->count; 3470 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3471 I40E_FLAG_FD_ATR_ENABLED)); 3472 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3473 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3474 if (vsi->type != I40E_VSI_FDIR) 3475 tx_ctx.head_wb_ena = 1; 3476 tx_ctx.head_wb_addr = ring->dma + 3477 (ring->count * sizeof(struct i40e_tx_desc)); 3478 3479 /* As part of VSI creation/update, FW allocates certain 3480 * Tx arbitration queue sets for each TC enabled for 3481 * the VSI. The FW returns the handles to these queue 3482 * sets as part of the response buffer to Add VSI, 3483 * Update VSI, etc. AQ commands. It is expected that 3484 * these queue set handles be associated with the Tx 3485 * queues by the driver as part of the TX queue context 3486 * initialization. This has to be done regardless of 3487 * DCB as by default everything is mapped to TC0. 3488 */ 3489 3490 if (ring->ch) 3491 tx_ctx.rdylist = 3492 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3493 3494 else 3495 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3496 3497 tx_ctx.rdylist_act = 0; 3498 3499 /* clear the context in the HMC */ 3500 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3501 if (err) { 3502 dev_info(&vsi->back->pdev->dev, 3503 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3504 ring->queue_index, pf_q, err); 3505 return -ENOMEM; 3506 } 3507 3508 /* set the context in the HMC */ 3509 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3510 if (err) { 3511 dev_info(&vsi->back->pdev->dev, 3512 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3513 ring->queue_index, pf_q, err); 3514 return -ENOMEM; 3515 } 3516 3517 /* Now associate this queue with this PCI function */ 3518 if (ring->ch) { 3519 if (ring->ch->type == I40E_VSI_VMDQ2) 3520 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3521 else 3522 return -EINVAL; 3523 3524 qtx_ctl |= (ring->ch->vsi_number << 3525 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3526 I40E_QTX_CTL_VFVM_INDX_MASK; 3527 } else { 3528 if (vsi->type == I40E_VSI_VMDQ2) { 3529 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3530 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3531 I40E_QTX_CTL_VFVM_INDX_MASK; 3532 } else { 3533 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3534 } 3535 } 3536 3537 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3538 I40E_QTX_CTL_PF_INDX_MASK); 3539 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3540 i40e_flush(hw); 3541 3542 /* cache tail off for easier writes later */ 3543 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3544 3545 return 0; 3546 } 3547 3548 /** 3549 * i40e_rx_offset - Return expected offset into page to access data 3550 * @rx_ring: Ring we are requesting offset of 3551 * 3552 * Returns the offset value for ring into the data buffer. 3553 */ 3554 static unsigned int i40e_rx_offset(struct i40e_ring *rx_ring) 3555 { 3556 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0; 3557 } 3558 3559 /** 3560 * i40e_configure_rx_ring - Configure a receive ring context 3561 * @ring: The Rx ring to configure 3562 * 3563 * Configure the Rx descriptor ring in the HMC context. 3564 **/ 3565 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3566 { 3567 struct i40e_vsi *vsi = ring->vsi; 3568 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3569 u16 pf_q = vsi->base_queue + ring->queue_index; 3570 struct i40e_hw *hw = &vsi->back->hw; 3571 struct i40e_hmc_obj_rxq rx_ctx; 3572 int err = 0; 3573 bool ok; 3574 int ret; 3575 3576 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3577 3578 /* clear the context structure first */ 3579 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3580 3581 if (ring->vsi->type == I40E_VSI_MAIN) 3582 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 3583 3584 ring->xsk_pool = i40e_xsk_pool(ring); 3585 if (ring->xsk_pool) { 3586 ring->rx_buf_len = 3587 xsk_pool_get_rx_frame_size(ring->xsk_pool); 3588 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3589 MEM_TYPE_XSK_BUFF_POOL, 3590 NULL); 3591 if (ret) 3592 return ret; 3593 dev_info(&vsi->back->pdev->dev, 3594 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 3595 ring->queue_index); 3596 3597 } else { 3598 ring->rx_buf_len = vsi->rx_buf_len; 3599 if (ring->vsi->type == I40E_VSI_MAIN) { 3600 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3601 MEM_TYPE_PAGE_SHARED, 3602 NULL); 3603 if (ret) 3604 return ret; 3605 } 3606 } 3607 3608 xdp_init_buff(&ring->xdp, i40e_rx_pg_size(ring) / 2, &ring->xdp_rxq); 3609 3610 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3611 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3612 3613 rx_ctx.base = (ring->dma / 128); 3614 rx_ctx.qlen = ring->count; 3615 3616 /* use 16 byte descriptors */ 3617 rx_ctx.dsize = 0; 3618 3619 /* descriptor type is always zero 3620 * rx_ctx.dtype = 0; 3621 */ 3622 rx_ctx.hsplit_0 = 0; 3623 3624 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3625 if (hw->revision_id == 0) 3626 rx_ctx.lrxqthresh = 0; 3627 else 3628 rx_ctx.lrxqthresh = 1; 3629 rx_ctx.crcstrip = 1; 3630 rx_ctx.l2tsel = 1; 3631 /* this controls whether VLAN is stripped from inner headers */ 3632 rx_ctx.showiv = 0; 3633 /* set the prefena field to 1 because the manual says to */ 3634 rx_ctx.prefena = 1; 3635 3636 /* clear the context in the HMC */ 3637 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3638 if (err) { 3639 dev_info(&vsi->back->pdev->dev, 3640 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3641 ring->queue_index, pf_q, err); 3642 return -ENOMEM; 3643 } 3644 3645 /* set the context in the HMC */ 3646 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3647 if (err) { 3648 dev_info(&vsi->back->pdev->dev, 3649 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3650 ring->queue_index, pf_q, err); 3651 return -ENOMEM; 3652 } 3653 3654 /* configure Rx buffer alignment */ 3655 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3656 if (I40E_2K_TOO_SMALL_WITH_PADDING) { 3657 dev_info(&vsi->back->pdev->dev, 3658 "2k Rx buffer is too small to fit standard MTU and skb_shared_info\n"); 3659 return -EOPNOTSUPP; 3660 } 3661 clear_ring_build_skb_enabled(ring); 3662 } else { 3663 set_ring_build_skb_enabled(ring); 3664 } 3665 3666 ring->rx_offset = i40e_rx_offset(ring); 3667 3668 /* cache tail for quicker writes, and clear the reg before use */ 3669 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3670 writel(0, ring->tail); 3671 3672 if (ring->xsk_pool) { 3673 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 3674 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); 3675 } else { 3676 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3677 } 3678 if (!ok) { 3679 /* Log this in case the user has forgotten to give the kernel 3680 * any buffers, even later in the application. 3681 */ 3682 dev_info(&vsi->back->pdev->dev, 3683 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n", 3684 ring->xsk_pool ? "AF_XDP ZC enabled " : "", 3685 ring->queue_index, pf_q); 3686 } 3687 3688 return 0; 3689 } 3690 3691 /** 3692 * i40e_vsi_configure_tx - Configure the VSI for Tx 3693 * @vsi: VSI structure describing this set of rings and resources 3694 * 3695 * Configure the Tx VSI for operation. 3696 **/ 3697 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3698 { 3699 int err = 0; 3700 u16 i; 3701 3702 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3703 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3704 3705 if (err || !i40e_enabled_xdp_vsi(vsi)) 3706 return err; 3707 3708 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3709 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3710 3711 return err; 3712 } 3713 3714 /** 3715 * i40e_vsi_configure_rx - Configure the VSI for Rx 3716 * @vsi: the VSI being configured 3717 * 3718 * Configure the Rx VSI for operation. 3719 **/ 3720 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3721 { 3722 int err = 0; 3723 u16 i; 3724 3725 vsi->max_frame = i40e_max_vsi_frame_size(vsi, vsi->xdp_prog); 3726 vsi->rx_buf_len = i40e_calculate_vsi_rx_buf_len(vsi); 3727 3728 #if (PAGE_SIZE < 8192) 3729 if (vsi->netdev && !I40E_2K_TOO_SMALL_WITH_PADDING && 3730 vsi->netdev->mtu <= ETH_DATA_LEN) { 3731 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3732 vsi->max_frame = vsi->rx_buf_len; 3733 } 3734 #endif 3735 3736 /* set up individual rings */ 3737 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3738 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3739 3740 return err; 3741 } 3742 3743 /** 3744 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3745 * @vsi: ptr to the VSI 3746 **/ 3747 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3748 { 3749 struct i40e_ring *tx_ring, *rx_ring; 3750 u16 qoffset, qcount; 3751 int i, n; 3752 3753 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3754 /* Reset the TC information */ 3755 for (i = 0; i < vsi->num_queue_pairs; i++) { 3756 rx_ring = vsi->rx_rings[i]; 3757 tx_ring = vsi->tx_rings[i]; 3758 rx_ring->dcb_tc = 0; 3759 tx_ring->dcb_tc = 0; 3760 } 3761 return; 3762 } 3763 3764 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3765 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3766 continue; 3767 3768 qoffset = vsi->tc_config.tc_info[n].qoffset; 3769 qcount = vsi->tc_config.tc_info[n].qcount; 3770 for (i = qoffset; i < (qoffset + qcount); i++) { 3771 rx_ring = vsi->rx_rings[i]; 3772 tx_ring = vsi->tx_rings[i]; 3773 rx_ring->dcb_tc = n; 3774 tx_ring->dcb_tc = n; 3775 } 3776 } 3777 } 3778 3779 /** 3780 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3781 * @vsi: ptr to the VSI 3782 **/ 3783 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3784 { 3785 if (vsi->netdev) 3786 i40e_set_rx_mode(vsi->netdev); 3787 } 3788 3789 /** 3790 * i40e_reset_fdir_filter_cnt - Reset flow director filter counters 3791 * @pf: Pointer to the targeted PF 3792 * 3793 * Set all flow director counters to 0. 3794 */ 3795 static void i40e_reset_fdir_filter_cnt(struct i40e_pf *pf) 3796 { 3797 pf->fd_tcp4_filter_cnt = 0; 3798 pf->fd_udp4_filter_cnt = 0; 3799 pf->fd_sctp4_filter_cnt = 0; 3800 pf->fd_ip4_filter_cnt = 0; 3801 pf->fd_tcp6_filter_cnt = 0; 3802 pf->fd_udp6_filter_cnt = 0; 3803 pf->fd_sctp6_filter_cnt = 0; 3804 pf->fd_ip6_filter_cnt = 0; 3805 } 3806 3807 /** 3808 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3809 * @vsi: Pointer to the targeted VSI 3810 * 3811 * This function replays the hlist on the hw where all the SB Flow Director 3812 * filters were saved. 3813 **/ 3814 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3815 { 3816 struct i40e_fdir_filter *filter; 3817 struct i40e_pf *pf = vsi->back; 3818 struct hlist_node *node; 3819 3820 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3821 return; 3822 3823 /* Reset FDir counters as we're replaying all existing filters */ 3824 i40e_reset_fdir_filter_cnt(pf); 3825 3826 hlist_for_each_entry_safe(filter, node, 3827 &pf->fdir_filter_list, fdir_node) { 3828 i40e_add_del_fdir(vsi, filter, true); 3829 } 3830 } 3831 3832 /** 3833 * i40e_vsi_configure - Set up the VSI for action 3834 * @vsi: the VSI being configured 3835 **/ 3836 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3837 { 3838 int err; 3839 3840 i40e_set_vsi_rx_mode(vsi); 3841 i40e_restore_vlan(vsi); 3842 i40e_vsi_config_dcb_rings(vsi); 3843 err = i40e_vsi_configure_tx(vsi); 3844 if (!err) 3845 err = i40e_vsi_configure_rx(vsi); 3846 3847 return err; 3848 } 3849 3850 /** 3851 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3852 * @vsi: the VSI being configured 3853 **/ 3854 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3855 { 3856 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3857 struct i40e_pf *pf = vsi->back; 3858 struct i40e_hw *hw = &pf->hw; 3859 u16 vector; 3860 int i, q; 3861 u32 qp; 3862 3863 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3864 * and PFINT_LNKLSTn registers, e.g.: 3865 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3866 */ 3867 qp = vsi->base_queue; 3868 vector = vsi->base_vector; 3869 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3870 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3871 3872 q_vector->rx.next_update = jiffies + 1; 3873 q_vector->rx.target_itr = 3874 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3875 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3876 q_vector->rx.target_itr >> 1); 3877 q_vector->rx.current_itr = q_vector->rx.target_itr; 3878 3879 q_vector->tx.next_update = jiffies + 1; 3880 q_vector->tx.target_itr = 3881 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3882 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3883 q_vector->tx.target_itr >> 1); 3884 q_vector->tx.current_itr = q_vector->tx.target_itr; 3885 3886 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3887 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3888 3889 /* begin of linked list for RX queue assigned to this vector */ 3890 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3891 for (q = 0; q < q_vector->num_ringpairs; q++) { 3892 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3893 u32 val; 3894 3895 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3896 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3897 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3898 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3899 (I40E_QUEUE_TYPE_TX << 3900 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3901 3902 wr32(hw, I40E_QINT_RQCTL(qp), val); 3903 3904 if (has_xdp) { 3905 /* TX queue with next queue set to TX */ 3906 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3907 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3908 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3909 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3910 (I40E_QUEUE_TYPE_TX << 3911 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3912 3913 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3914 } 3915 /* TX queue with next RX or end of linked list */ 3916 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3917 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3918 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3919 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3920 (I40E_QUEUE_TYPE_RX << 3921 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3922 3923 /* Terminate the linked list */ 3924 if (q == (q_vector->num_ringpairs - 1)) 3925 val |= (I40E_QUEUE_END_OF_LIST << 3926 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3927 3928 wr32(hw, I40E_QINT_TQCTL(qp), val); 3929 qp++; 3930 } 3931 } 3932 3933 i40e_flush(hw); 3934 } 3935 3936 /** 3937 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3938 * @pf: pointer to private device data structure 3939 **/ 3940 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3941 { 3942 struct i40e_hw *hw = &pf->hw; 3943 u32 val; 3944 3945 /* clear things first */ 3946 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3947 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3948 3949 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3950 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3951 I40E_PFINT_ICR0_ENA_GRST_MASK | 3952 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3953 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3954 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3955 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3956 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3957 3958 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3959 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3960 3961 if (pf->flags & I40E_FLAG_PTP) 3962 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3963 3964 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3965 3966 /* SW_ITR_IDX = 0, but don't change INTENA */ 3967 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3968 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3969 3970 /* OTHER_ITR_IDX = 0 */ 3971 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3972 } 3973 3974 /** 3975 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3976 * @vsi: the VSI being configured 3977 **/ 3978 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3979 { 3980 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3981 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3982 struct i40e_pf *pf = vsi->back; 3983 struct i40e_hw *hw = &pf->hw; 3984 3985 /* set the ITR configuration */ 3986 q_vector->rx.next_update = jiffies + 1; 3987 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 3988 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1); 3989 q_vector->rx.current_itr = q_vector->rx.target_itr; 3990 q_vector->tx.next_update = jiffies + 1; 3991 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 3992 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1); 3993 q_vector->tx.current_itr = q_vector->tx.target_itr; 3994 3995 i40e_enable_misc_int_causes(pf); 3996 3997 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3998 wr32(hw, I40E_PFINT_LNKLST0, 0); 3999 4000 /* Associate the queue pair to the vector and enable the queue 4001 * interrupt RX queue in linked list with next queue set to TX 4002 */ 4003 wr32(hw, I40E_QINT_RQCTL(0), I40E_QINT_RQCTL_VAL(nextqp, 0, TX)); 4004 4005 if (i40e_enabled_xdp_vsi(vsi)) { 4006 /* TX queue in linked list with next queue set to TX */ 4007 wr32(hw, I40E_QINT_TQCTL(nextqp), 4008 I40E_QINT_TQCTL_VAL(nextqp, 0, TX)); 4009 } 4010 4011 /* last TX queue so the next RX queue doesn't matter */ 4012 wr32(hw, I40E_QINT_TQCTL(0), 4013 I40E_QINT_TQCTL_VAL(I40E_QUEUE_END_OF_LIST, 0, RX)); 4014 i40e_flush(hw); 4015 } 4016 4017 /** 4018 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 4019 * @pf: board private structure 4020 **/ 4021 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 4022 { 4023 struct i40e_hw *hw = &pf->hw; 4024 4025 wr32(hw, I40E_PFINT_DYN_CTL0, 4026 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 4027 i40e_flush(hw); 4028 } 4029 4030 /** 4031 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 4032 * @pf: board private structure 4033 **/ 4034 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 4035 { 4036 struct i40e_hw *hw = &pf->hw; 4037 u32 val; 4038 4039 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 4040 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 4041 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 4042 4043 wr32(hw, I40E_PFINT_DYN_CTL0, val); 4044 i40e_flush(hw); 4045 } 4046 4047 /** 4048 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 4049 * @irq: interrupt number 4050 * @data: pointer to a q_vector 4051 **/ 4052 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 4053 { 4054 struct i40e_q_vector *q_vector = data; 4055 4056 if (!q_vector->tx.ring && !q_vector->rx.ring) 4057 return IRQ_HANDLED; 4058 4059 napi_schedule_irqoff(&q_vector->napi); 4060 4061 return IRQ_HANDLED; 4062 } 4063 4064 /** 4065 * i40e_irq_affinity_notify - Callback for affinity changes 4066 * @notify: context as to what irq was changed 4067 * @mask: the new affinity mask 4068 * 4069 * This is a callback function used by the irq_set_affinity_notifier function 4070 * so that we may register to receive changes to the irq affinity masks. 4071 **/ 4072 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 4073 const cpumask_t *mask) 4074 { 4075 struct i40e_q_vector *q_vector = 4076 container_of(notify, struct i40e_q_vector, affinity_notify); 4077 4078 cpumask_copy(&q_vector->affinity_mask, mask); 4079 } 4080 4081 /** 4082 * i40e_irq_affinity_release - Callback for affinity notifier release 4083 * @ref: internal core kernel usage 4084 * 4085 * This is a callback function used by the irq_set_affinity_notifier function 4086 * to inform the current notification subscriber that they will no longer 4087 * receive notifications. 4088 **/ 4089 static void i40e_irq_affinity_release(struct kref *ref) {} 4090 4091 /** 4092 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 4093 * @vsi: the VSI being configured 4094 * @basename: name for the vector 4095 * 4096 * Allocates MSI-X vectors and requests interrupts from the kernel. 4097 **/ 4098 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 4099 { 4100 int q_vectors = vsi->num_q_vectors; 4101 struct i40e_pf *pf = vsi->back; 4102 int base = vsi->base_vector; 4103 int rx_int_idx = 0; 4104 int tx_int_idx = 0; 4105 int vector, err; 4106 int irq_num; 4107 int cpu; 4108 4109 for (vector = 0; vector < q_vectors; vector++) { 4110 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 4111 4112 irq_num = pf->msix_entries[base + vector].vector; 4113 4114 if (q_vector->tx.ring && q_vector->rx.ring) { 4115 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4116 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 4117 tx_int_idx++; 4118 } else if (q_vector->rx.ring) { 4119 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4120 "%s-%s-%d", basename, "rx", rx_int_idx++); 4121 } else if (q_vector->tx.ring) { 4122 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 4123 "%s-%s-%d", basename, "tx", tx_int_idx++); 4124 } else { 4125 /* skip this unused q_vector */ 4126 continue; 4127 } 4128 err = request_irq(irq_num, 4129 vsi->irq_handler, 4130 0, 4131 q_vector->name, 4132 q_vector); 4133 if (err) { 4134 dev_info(&pf->pdev->dev, 4135 "MSIX request_irq failed, error: %d\n", err); 4136 goto free_queue_irqs; 4137 } 4138 4139 /* register for affinity change notifications */ 4140 q_vector->irq_num = irq_num; 4141 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 4142 q_vector->affinity_notify.release = i40e_irq_affinity_release; 4143 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 4144 /* Spread affinity hints out across online CPUs. 4145 * 4146 * get_cpu_mask returns a static constant mask with 4147 * a permanent lifetime so it's ok to pass to 4148 * irq_update_affinity_hint without making a copy. 4149 */ 4150 cpu = cpumask_local_spread(q_vector->v_idx, -1); 4151 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu)); 4152 } 4153 4154 vsi->irqs_ready = true; 4155 return 0; 4156 4157 free_queue_irqs: 4158 while (vector) { 4159 vector--; 4160 irq_num = pf->msix_entries[base + vector].vector; 4161 irq_set_affinity_notifier(irq_num, NULL); 4162 irq_update_affinity_hint(irq_num, NULL); 4163 free_irq(irq_num, &vsi->q_vectors[vector]); 4164 } 4165 return err; 4166 } 4167 4168 /** 4169 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 4170 * @vsi: the VSI being un-configured 4171 **/ 4172 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 4173 { 4174 struct i40e_pf *pf = vsi->back; 4175 struct i40e_hw *hw = &pf->hw; 4176 int base = vsi->base_vector; 4177 int i; 4178 4179 /* disable interrupt causation from each queue */ 4180 for (i = 0; i < vsi->num_queue_pairs; i++) { 4181 u32 val; 4182 4183 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 4184 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 4185 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 4186 4187 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 4188 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 4189 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 4190 4191 if (!i40e_enabled_xdp_vsi(vsi)) 4192 continue; 4193 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 4194 } 4195 4196 /* disable each interrupt */ 4197 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4198 for (i = vsi->base_vector; 4199 i < (vsi->num_q_vectors + vsi->base_vector); i++) 4200 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 4201 4202 i40e_flush(hw); 4203 for (i = 0; i < vsi->num_q_vectors; i++) 4204 synchronize_irq(pf->msix_entries[i + base].vector); 4205 } else { 4206 /* Legacy and MSI mode - this stops all interrupt handling */ 4207 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 4208 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 4209 i40e_flush(hw); 4210 synchronize_irq(pf->pdev->irq); 4211 } 4212 } 4213 4214 /** 4215 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 4216 * @vsi: the VSI being configured 4217 **/ 4218 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 4219 { 4220 struct i40e_pf *pf = vsi->back; 4221 int i; 4222 4223 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4224 for (i = 0; i < vsi->num_q_vectors; i++) 4225 i40e_irq_dynamic_enable(vsi, i); 4226 } else { 4227 i40e_irq_dynamic_enable_icr0(pf); 4228 } 4229 4230 i40e_flush(&pf->hw); 4231 return 0; 4232 } 4233 4234 /** 4235 * i40e_free_misc_vector - Free the vector that handles non-queue events 4236 * @pf: board private structure 4237 **/ 4238 static void i40e_free_misc_vector(struct i40e_pf *pf) 4239 { 4240 /* Disable ICR 0 */ 4241 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 4242 i40e_flush(&pf->hw); 4243 4244 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 4245 free_irq(pf->msix_entries[0].vector, pf); 4246 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 4247 } 4248 } 4249 4250 /** 4251 * i40e_intr - MSI/Legacy and non-queue interrupt handler 4252 * @irq: interrupt number 4253 * @data: pointer to a q_vector 4254 * 4255 * This is the handler used for all MSI/Legacy interrupts, and deals 4256 * with both queue and non-queue interrupts. This is also used in 4257 * MSIX mode to handle the non-queue interrupts. 4258 **/ 4259 static irqreturn_t i40e_intr(int irq, void *data) 4260 { 4261 struct i40e_pf *pf = (struct i40e_pf *)data; 4262 struct i40e_hw *hw = &pf->hw; 4263 irqreturn_t ret = IRQ_NONE; 4264 u32 icr0, icr0_remaining; 4265 u32 val, ena_mask; 4266 4267 icr0 = rd32(hw, I40E_PFINT_ICR0); 4268 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 4269 4270 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 4271 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 4272 goto enable_intr; 4273 4274 /* if interrupt but no bits showing, must be SWINT */ 4275 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 4276 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 4277 pf->sw_int_count++; 4278 4279 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 4280 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 4281 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 4282 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 4283 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 4284 } 4285 4286 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 4287 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 4288 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 4289 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 4290 4291 /* We do not have a way to disarm Queue causes while leaving 4292 * interrupt enabled for all other causes, ideally 4293 * interrupt should be disabled while we are in NAPI but 4294 * this is not a performance path and napi_schedule() 4295 * can deal with rescheduling. 4296 */ 4297 if (!test_bit(__I40E_DOWN, pf->state)) 4298 napi_schedule_irqoff(&q_vector->napi); 4299 } 4300 4301 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 4302 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 4303 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 4304 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 4305 } 4306 4307 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 4308 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 4309 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 4310 } 4311 4312 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 4313 /* disable any further VFLR event notifications */ 4314 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) { 4315 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 4316 4317 reg &= ~I40E_PFINT_ICR0_VFLR_MASK; 4318 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 4319 } else { 4320 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 4321 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 4322 } 4323 } 4324 4325 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 4326 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4327 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 4328 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 4329 val = rd32(hw, I40E_GLGEN_RSTAT); 4330 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 4331 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 4332 if (val == I40E_RESET_CORER) { 4333 pf->corer_count++; 4334 } else if (val == I40E_RESET_GLOBR) { 4335 pf->globr_count++; 4336 } else if (val == I40E_RESET_EMPR) { 4337 pf->empr_count++; 4338 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 4339 } 4340 } 4341 4342 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 4343 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 4344 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 4345 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 4346 rd32(hw, I40E_PFHMC_ERRORINFO), 4347 rd32(hw, I40E_PFHMC_ERRORDATA)); 4348 } 4349 4350 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 4351 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 4352 4353 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_EVENT0_MASK) 4354 schedule_work(&pf->ptp_extts0_work); 4355 4356 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) 4357 i40e_ptp_tx_hwtstamp(pf); 4358 4359 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4360 } 4361 4362 /* If a critical error is pending we have no choice but to reset the 4363 * device. 4364 * Report and mask out any remaining unexpected interrupts. 4365 */ 4366 icr0_remaining = icr0 & ena_mask; 4367 if (icr0_remaining) { 4368 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 4369 icr0_remaining); 4370 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 4371 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 4372 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 4373 dev_info(&pf->pdev->dev, "device will be reset\n"); 4374 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 4375 i40e_service_event_schedule(pf); 4376 } 4377 ena_mask &= ~icr0_remaining; 4378 } 4379 ret = IRQ_HANDLED; 4380 4381 enable_intr: 4382 /* re-enable interrupt causes */ 4383 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 4384 if (!test_bit(__I40E_DOWN, pf->state) || 4385 test_bit(__I40E_RECOVERY_MODE, pf->state)) { 4386 i40e_service_event_schedule(pf); 4387 i40e_irq_dynamic_enable_icr0(pf); 4388 } 4389 4390 return ret; 4391 } 4392 4393 /** 4394 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 4395 * @tx_ring: tx ring to clean 4396 * @budget: how many cleans we're allowed 4397 * 4398 * Returns true if there's any budget left (e.g. the clean is finished) 4399 **/ 4400 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 4401 { 4402 struct i40e_vsi *vsi = tx_ring->vsi; 4403 u16 i = tx_ring->next_to_clean; 4404 struct i40e_tx_buffer *tx_buf; 4405 struct i40e_tx_desc *tx_desc; 4406 4407 tx_buf = &tx_ring->tx_bi[i]; 4408 tx_desc = I40E_TX_DESC(tx_ring, i); 4409 i -= tx_ring->count; 4410 4411 do { 4412 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4413 4414 /* if next_to_watch is not set then there is no work pending */ 4415 if (!eop_desc) 4416 break; 4417 4418 /* prevent any other reads prior to eop_desc */ 4419 smp_rmb(); 4420 4421 /* if the descriptor isn't done, no work yet to do */ 4422 if (!(eop_desc->cmd_type_offset_bsz & 4423 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4424 break; 4425 4426 /* clear next_to_watch to prevent false hangs */ 4427 tx_buf->next_to_watch = NULL; 4428 4429 tx_desc->buffer_addr = 0; 4430 tx_desc->cmd_type_offset_bsz = 0; 4431 /* move past filter desc */ 4432 tx_buf++; 4433 tx_desc++; 4434 i++; 4435 if (unlikely(!i)) { 4436 i -= tx_ring->count; 4437 tx_buf = tx_ring->tx_bi; 4438 tx_desc = I40E_TX_DESC(tx_ring, 0); 4439 } 4440 /* unmap skb header data */ 4441 dma_unmap_single(tx_ring->dev, 4442 dma_unmap_addr(tx_buf, dma), 4443 dma_unmap_len(tx_buf, len), 4444 DMA_TO_DEVICE); 4445 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4446 kfree(tx_buf->raw_buf); 4447 4448 tx_buf->raw_buf = NULL; 4449 tx_buf->tx_flags = 0; 4450 tx_buf->next_to_watch = NULL; 4451 dma_unmap_len_set(tx_buf, len, 0); 4452 tx_desc->buffer_addr = 0; 4453 tx_desc->cmd_type_offset_bsz = 0; 4454 4455 /* move us past the eop_desc for start of next FD desc */ 4456 tx_buf++; 4457 tx_desc++; 4458 i++; 4459 if (unlikely(!i)) { 4460 i -= tx_ring->count; 4461 tx_buf = tx_ring->tx_bi; 4462 tx_desc = I40E_TX_DESC(tx_ring, 0); 4463 } 4464 4465 /* update budget accounting */ 4466 budget--; 4467 } while (likely(budget)); 4468 4469 i += tx_ring->count; 4470 tx_ring->next_to_clean = i; 4471 4472 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4473 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4474 4475 return budget > 0; 4476 } 4477 4478 /** 4479 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4480 * @irq: interrupt number 4481 * @data: pointer to a q_vector 4482 **/ 4483 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4484 { 4485 struct i40e_q_vector *q_vector = data; 4486 struct i40e_vsi *vsi; 4487 4488 if (!q_vector->tx.ring) 4489 return IRQ_HANDLED; 4490 4491 vsi = q_vector->tx.ring->vsi; 4492 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4493 4494 return IRQ_HANDLED; 4495 } 4496 4497 /** 4498 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4499 * @vsi: the VSI being configured 4500 * @v_idx: vector index 4501 * @qp_idx: queue pair index 4502 **/ 4503 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4504 { 4505 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4506 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4507 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4508 4509 tx_ring->q_vector = q_vector; 4510 tx_ring->next = q_vector->tx.ring; 4511 q_vector->tx.ring = tx_ring; 4512 q_vector->tx.count++; 4513 4514 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4515 if (i40e_enabled_xdp_vsi(vsi)) { 4516 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4517 4518 xdp_ring->q_vector = q_vector; 4519 xdp_ring->next = q_vector->tx.ring; 4520 q_vector->tx.ring = xdp_ring; 4521 q_vector->tx.count++; 4522 } 4523 4524 rx_ring->q_vector = q_vector; 4525 rx_ring->next = q_vector->rx.ring; 4526 q_vector->rx.ring = rx_ring; 4527 q_vector->rx.count++; 4528 } 4529 4530 /** 4531 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4532 * @vsi: the VSI being configured 4533 * 4534 * This function maps descriptor rings to the queue-specific vectors 4535 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4536 * one vector per queue pair, but on a constrained vector budget, we 4537 * group the queue pairs as "efficiently" as possible. 4538 **/ 4539 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4540 { 4541 int qp_remaining = vsi->num_queue_pairs; 4542 int q_vectors = vsi->num_q_vectors; 4543 int num_ringpairs; 4544 int v_start = 0; 4545 int qp_idx = 0; 4546 4547 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4548 * group them so there are multiple queues per vector. 4549 * It is also important to go through all the vectors available to be 4550 * sure that if we don't use all the vectors, that the remaining vectors 4551 * are cleared. This is especially important when decreasing the 4552 * number of queues in use. 4553 */ 4554 for (; v_start < q_vectors; v_start++) { 4555 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4556 4557 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4558 4559 q_vector->num_ringpairs = num_ringpairs; 4560 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4561 4562 q_vector->rx.count = 0; 4563 q_vector->tx.count = 0; 4564 q_vector->rx.ring = NULL; 4565 q_vector->tx.ring = NULL; 4566 4567 while (num_ringpairs--) { 4568 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4569 qp_idx++; 4570 qp_remaining--; 4571 } 4572 } 4573 } 4574 4575 /** 4576 * i40e_vsi_request_irq - Request IRQ from the OS 4577 * @vsi: the VSI being configured 4578 * @basename: name for the vector 4579 **/ 4580 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4581 { 4582 struct i40e_pf *pf = vsi->back; 4583 int err; 4584 4585 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4586 err = i40e_vsi_request_irq_msix(vsi, basename); 4587 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4588 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4589 pf->int_name, pf); 4590 else 4591 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4592 pf->int_name, pf); 4593 4594 if (err) 4595 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4596 4597 return err; 4598 } 4599 4600 #ifdef CONFIG_NET_POLL_CONTROLLER 4601 /** 4602 * i40e_netpoll - A Polling 'interrupt' handler 4603 * @netdev: network interface device structure 4604 * 4605 * This is used by netconsole to send skbs without having to re-enable 4606 * interrupts. It's not called while the normal interrupt routine is executing. 4607 **/ 4608 static void i40e_netpoll(struct net_device *netdev) 4609 { 4610 struct i40e_netdev_priv *np = netdev_priv(netdev); 4611 struct i40e_vsi *vsi = np->vsi; 4612 struct i40e_pf *pf = vsi->back; 4613 int i; 4614 4615 /* if interface is down do nothing */ 4616 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4617 return; 4618 4619 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4620 for (i = 0; i < vsi->num_q_vectors; i++) 4621 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4622 } else { 4623 i40e_intr(pf->pdev->irq, netdev); 4624 } 4625 } 4626 #endif 4627 4628 #define I40E_QTX_ENA_WAIT_COUNT 50 4629 4630 /** 4631 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4632 * @pf: the PF being configured 4633 * @pf_q: the PF queue 4634 * @enable: enable or disable state of the queue 4635 * 4636 * This routine will wait for the given Tx queue of the PF to reach the 4637 * enabled or disabled state. 4638 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4639 * multiple retries; else will return 0 in case of success. 4640 **/ 4641 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4642 { 4643 int i; 4644 u32 tx_reg; 4645 4646 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4647 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4648 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4649 break; 4650 4651 usleep_range(10, 20); 4652 } 4653 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4654 return -ETIMEDOUT; 4655 4656 return 0; 4657 } 4658 4659 /** 4660 * i40e_control_tx_q - Start or stop a particular Tx queue 4661 * @pf: the PF structure 4662 * @pf_q: the PF queue to configure 4663 * @enable: start or stop the queue 4664 * 4665 * This function enables or disables a single queue. Note that any delay 4666 * required after the operation is expected to be handled by the caller of 4667 * this function. 4668 **/ 4669 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4670 { 4671 struct i40e_hw *hw = &pf->hw; 4672 u32 tx_reg; 4673 int i; 4674 4675 /* warn the TX unit of coming changes */ 4676 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4677 if (!enable) 4678 usleep_range(10, 20); 4679 4680 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4681 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4682 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4683 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4684 break; 4685 usleep_range(1000, 2000); 4686 } 4687 4688 /* Skip if the queue is already in the requested state */ 4689 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4690 return; 4691 4692 /* turn on/off the queue */ 4693 if (enable) { 4694 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4695 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4696 } else { 4697 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4698 } 4699 4700 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4701 } 4702 4703 /** 4704 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4705 * @seid: VSI SEID 4706 * @pf: the PF structure 4707 * @pf_q: the PF queue to configure 4708 * @is_xdp: true if the queue is used for XDP 4709 * @enable: start or stop the queue 4710 **/ 4711 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4712 bool is_xdp, bool enable) 4713 { 4714 int ret; 4715 4716 i40e_control_tx_q(pf, pf_q, enable); 4717 4718 /* wait for the change to finish */ 4719 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4720 if (ret) { 4721 dev_info(&pf->pdev->dev, 4722 "VSI seid %d %sTx ring %d %sable timeout\n", 4723 seid, (is_xdp ? "XDP " : ""), pf_q, 4724 (enable ? "en" : "dis")); 4725 } 4726 4727 return ret; 4728 } 4729 4730 /** 4731 * i40e_vsi_enable_tx - Start a VSI's rings 4732 * @vsi: the VSI being configured 4733 **/ 4734 static int i40e_vsi_enable_tx(struct i40e_vsi *vsi) 4735 { 4736 struct i40e_pf *pf = vsi->back; 4737 int i, pf_q, ret = 0; 4738 4739 pf_q = vsi->base_queue; 4740 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4741 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4742 pf_q, 4743 false /*is xdp*/, true); 4744 if (ret) 4745 break; 4746 4747 if (!i40e_enabled_xdp_vsi(vsi)) 4748 continue; 4749 4750 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4751 pf_q + vsi->alloc_queue_pairs, 4752 true /*is xdp*/, true); 4753 if (ret) 4754 break; 4755 } 4756 return ret; 4757 } 4758 4759 /** 4760 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4761 * @pf: the PF being configured 4762 * @pf_q: the PF queue 4763 * @enable: enable or disable state of the queue 4764 * 4765 * This routine will wait for the given Rx queue of the PF to reach the 4766 * enabled or disabled state. 4767 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4768 * multiple retries; else will return 0 in case of success. 4769 **/ 4770 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4771 { 4772 int i; 4773 u32 rx_reg; 4774 4775 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4776 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4777 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4778 break; 4779 4780 usleep_range(10, 20); 4781 } 4782 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4783 return -ETIMEDOUT; 4784 4785 return 0; 4786 } 4787 4788 /** 4789 * i40e_control_rx_q - Start or stop a particular Rx queue 4790 * @pf: the PF structure 4791 * @pf_q: the PF queue to configure 4792 * @enable: start or stop the queue 4793 * 4794 * This function enables or disables a single queue. Note that 4795 * any delay required after the operation is expected to be 4796 * handled by the caller of this function. 4797 **/ 4798 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4799 { 4800 struct i40e_hw *hw = &pf->hw; 4801 u32 rx_reg; 4802 int i; 4803 4804 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4805 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4806 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4807 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4808 break; 4809 usleep_range(1000, 2000); 4810 } 4811 4812 /* Skip if the queue is already in the requested state */ 4813 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4814 return; 4815 4816 /* turn on/off the queue */ 4817 if (enable) 4818 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4819 else 4820 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4821 4822 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4823 } 4824 4825 /** 4826 * i40e_control_wait_rx_q 4827 * @pf: the PF structure 4828 * @pf_q: queue being configured 4829 * @enable: start or stop the rings 4830 * 4831 * This function enables or disables a single queue along with waiting 4832 * for the change to finish. The caller of this function should handle 4833 * the delays needed in the case of disabling queues. 4834 **/ 4835 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4836 { 4837 int ret = 0; 4838 4839 i40e_control_rx_q(pf, pf_q, enable); 4840 4841 /* wait for the change to finish */ 4842 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4843 if (ret) 4844 return ret; 4845 4846 return ret; 4847 } 4848 4849 /** 4850 * i40e_vsi_enable_rx - Start a VSI's rings 4851 * @vsi: the VSI being configured 4852 **/ 4853 static int i40e_vsi_enable_rx(struct i40e_vsi *vsi) 4854 { 4855 struct i40e_pf *pf = vsi->back; 4856 int i, pf_q, ret = 0; 4857 4858 pf_q = vsi->base_queue; 4859 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4860 ret = i40e_control_wait_rx_q(pf, pf_q, true); 4861 if (ret) { 4862 dev_info(&pf->pdev->dev, 4863 "VSI seid %d Rx ring %d enable timeout\n", 4864 vsi->seid, pf_q); 4865 break; 4866 } 4867 } 4868 4869 return ret; 4870 } 4871 4872 /** 4873 * i40e_vsi_start_rings - Start a VSI's rings 4874 * @vsi: the VSI being configured 4875 **/ 4876 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4877 { 4878 int ret = 0; 4879 4880 /* do rx first for enable and last for disable */ 4881 ret = i40e_vsi_enable_rx(vsi); 4882 if (ret) 4883 return ret; 4884 ret = i40e_vsi_enable_tx(vsi); 4885 4886 return ret; 4887 } 4888 4889 #define I40E_DISABLE_TX_GAP_MSEC 50 4890 4891 /** 4892 * i40e_vsi_stop_rings - Stop a VSI's rings 4893 * @vsi: the VSI being configured 4894 **/ 4895 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4896 { 4897 struct i40e_pf *pf = vsi->back; 4898 int pf_q, err, q_end; 4899 4900 /* When port TX is suspended, don't wait */ 4901 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4902 return i40e_vsi_stop_rings_no_wait(vsi); 4903 4904 q_end = vsi->base_queue + vsi->num_queue_pairs; 4905 for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) 4906 i40e_pre_tx_queue_cfg(&pf->hw, (u32)pf_q, false); 4907 4908 for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) { 4909 err = i40e_control_wait_rx_q(pf, pf_q, false); 4910 if (err) 4911 dev_info(&pf->pdev->dev, 4912 "VSI seid %d Rx ring %d disable timeout\n", 4913 vsi->seid, pf_q); 4914 } 4915 4916 msleep(I40E_DISABLE_TX_GAP_MSEC); 4917 pf_q = vsi->base_queue; 4918 for (pf_q = vsi->base_queue; pf_q < q_end; pf_q++) 4919 wr32(&pf->hw, I40E_QTX_ENA(pf_q), 0); 4920 4921 i40e_vsi_wait_queues_disabled(vsi); 4922 } 4923 4924 /** 4925 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4926 * @vsi: the VSI being shutdown 4927 * 4928 * This function stops all the rings for a VSI but does not delay to verify 4929 * that rings have been disabled. It is expected that the caller is shutting 4930 * down multiple VSIs at once and will delay together for all the VSIs after 4931 * initiating the shutdown. This is particularly useful for shutting down lots 4932 * of VFs together. Otherwise, a large delay can be incurred while configuring 4933 * each VSI in serial. 4934 **/ 4935 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4936 { 4937 struct i40e_pf *pf = vsi->back; 4938 int i, pf_q; 4939 4940 pf_q = vsi->base_queue; 4941 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4942 i40e_control_tx_q(pf, pf_q, false); 4943 i40e_control_rx_q(pf, pf_q, false); 4944 } 4945 } 4946 4947 /** 4948 * i40e_vsi_free_irq - Free the irq association with the OS 4949 * @vsi: the VSI being configured 4950 **/ 4951 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4952 { 4953 struct i40e_pf *pf = vsi->back; 4954 struct i40e_hw *hw = &pf->hw; 4955 int base = vsi->base_vector; 4956 u32 val, qp; 4957 int i; 4958 4959 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4960 if (!vsi->q_vectors) 4961 return; 4962 4963 if (!vsi->irqs_ready) 4964 return; 4965 4966 vsi->irqs_ready = false; 4967 for (i = 0; i < vsi->num_q_vectors; i++) { 4968 int irq_num; 4969 u16 vector; 4970 4971 vector = i + base; 4972 irq_num = pf->msix_entries[vector].vector; 4973 4974 /* free only the irqs that were actually requested */ 4975 if (!vsi->q_vectors[i] || 4976 !vsi->q_vectors[i]->num_ringpairs) 4977 continue; 4978 4979 /* clear the affinity notifier in the IRQ descriptor */ 4980 irq_set_affinity_notifier(irq_num, NULL); 4981 /* remove our suggested affinity mask for this IRQ */ 4982 irq_update_affinity_hint(irq_num, NULL); 4983 free_irq(irq_num, vsi->q_vectors[i]); 4984 4985 /* Tear down the interrupt queue link list 4986 * 4987 * We know that they come in pairs and always 4988 * the Rx first, then the Tx. To clear the 4989 * link list, stick the EOL value into the 4990 * next_q field of the registers. 4991 */ 4992 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4993 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4994 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4995 val |= I40E_QUEUE_END_OF_LIST 4996 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4997 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4998 4999 while (qp != I40E_QUEUE_END_OF_LIST) { 5000 u32 next; 5001 5002 val = rd32(hw, I40E_QINT_RQCTL(qp)); 5003 5004 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 5005 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 5006 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5007 I40E_QINT_RQCTL_INTEVENT_MASK); 5008 5009 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 5010 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 5011 5012 wr32(hw, I40E_QINT_RQCTL(qp), val); 5013 5014 val = rd32(hw, I40E_QINT_TQCTL(qp)); 5015 5016 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 5017 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 5018 5019 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 5020 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 5021 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 5022 I40E_QINT_TQCTL_INTEVENT_MASK); 5023 5024 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 5025 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 5026 5027 wr32(hw, I40E_QINT_TQCTL(qp), val); 5028 qp = next; 5029 } 5030 } 5031 } else { 5032 free_irq(pf->pdev->irq, pf); 5033 5034 val = rd32(hw, I40E_PFINT_LNKLST0); 5035 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 5036 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 5037 val |= I40E_QUEUE_END_OF_LIST 5038 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 5039 wr32(hw, I40E_PFINT_LNKLST0, val); 5040 5041 val = rd32(hw, I40E_QINT_RQCTL(qp)); 5042 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 5043 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 5044 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 5045 I40E_QINT_RQCTL_INTEVENT_MASK); 5046 5047 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 5048 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 5049 5050 wr32(hw, I40E_QINT_RQCTL(qp), val); 5051 5052 val = rd32(hw, I40E_QINT_TQCTL(qp)); 5053 5054 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 5055 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 5056 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 5057 I40E_QINT_TQCTL_INTEVENT_MASK); 5058 5059 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 5060 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 5061 5062 wr32(hw, I40E_QINT_TQCTL(qp), val); 5063 } 5064 } 5065 5066 /** 5067 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 5068 * @vsi: the VSI being configured 5069 * @v_idx: Index of vector to be freed 5070 * 5071 * This function frees the memory allocated to the q_vector. In addition if 5072 * NAPI is enabled it will delete any references to the NAPI struct prior 5073 * to freeing the q_vector. 5074 **/ 5075 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 5076 { 5077 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 5078 struct i40e_ring *ring; 5079 5080 if (!q_vector) 5081 return; 5082 5083 /* disassociate q_vector from rings */ 5084 i40e_for_each_ring(ring, q_vector->tx) 5085 ring->q_vector = NULL; 5086 5087 i40e_for_each_ring(ring, q_vector->rx) 5088 ring->q_vector = NULL; 5089 5090 /* only VSI w/ an associated netdev is set up w/ NAPI */ 5091 if (vsi->netdev) 5092 netif_napi_del(&q_vector->napi); 5093 5094 vsi->q_vectors[v_idx] = NULL; 5095 5096 kfree_rcu(q_vector, rcu); 5097 } 5098 5099 /** 5100 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 5101 * @vsi: the VSI being un-configured 5102 * 5103 * This frees the memory allocated to the q_vectors and 5104 * deletes references to the NAPI struct. 5105 **/ 5106 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 5107 { 5108 int v_idx; 5109 5110 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 5111 i40e_free_q_vector(vsi, v_idx); 5112 } 5113 5114 /** 5115 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 5116 * @pf: board private structure 5117 **/ 5118 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 5119 { 5120 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 5121 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 5122 pci_disable_msix(pf->pdev); 5123 kfree(pf->msix_entries); 5124 pf->msix_entries = NULL; 5125 kfree(pf->irq_pile); 5126 pf->irq_pile = NULL; 5127 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 5128 pci_disable_msi(pf->pdev); 5129 } 5130 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 5131 } 5132 5133 /** 5134 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 5135 * @pf: board private structure 5136 * 5137 * We go through and clear interrupt specific resources and reset the structure 5138 * to pre-load conditions 5139 **/ 5140 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 5141 { 5142 int i; 5143 5144 if (test_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) 5145 i40e_free_misc_vector(pf); 5146 5147 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 5148 I40E_IWARP_IRQ_PILE_ID); 5149 5150 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 5151 for (i = 0; i < pf->num_alloc_vsi; i++) 5152 if (pf->vsi[i]) 5153 i40e_vsi_free_q_vectors(pf->vsi[i]); 5154 i40e_reset_interrupt_capability(pf); 5155 } 5156 5157 /** 5158 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 5159 * @vsi: the VSI being configured 5160 **/ 5161 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 5162 { 5163 int q_idx; 5164 5165 if (!vsi->netdev) 5166 return; 5167 5168 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 5169 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 5170 5171 if (q_vector->rx.ring || q_vector->tx.ring) 5172 napi_enable(&q_vector->napi); 5173 } 5174 } 5175 5176 /** 5177 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 5178 * @vsi: the VSI being configured 5179 **/ 5180 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 5181 { 5182 int q_idx; 5183 5184 if (!vsi->netdev) 5185 return; 5186 5187 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 5188 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 5189 5190 if (q_vector->rx.ring || q_vector->tx.ring) 5191 napi_disable(&q_vector->napi); 5192 } 5193 } 5194 5195 /** 5196 * i40e_vsi_close - Shut down a VSI 5197 * @vsi: the vsi to be quelled 5198 **/ 5199 static void i40e_vsi_close(struct i40e_vsi *vsi) 5200 { 5201 struct i40e_pf *pf = vsi->back; 5202 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 5203 i40e_down(vsi); 5204 i40e_vsi_free_irq(vsi); 5205 i40e_vsi_free_tx_resources(vsi); 5206 i40e_vsi_free_rx_resources(vsi); 5207 vsi->current_netdev_flags = 0; 5208 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 5209 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 5210 set_bit(__I40E_CLIENT_RESET, pf->state); 5211 } 5212 5213 /** 5214 * i40e_quiesce_vsi - Pause a given VSI 5215 * @vsi: the VSI being paused 5216 **/ 5217 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 5218 { 5219 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 5220 return; 5221 5222 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 5223 if (vsi->netdev && netif_running(vsi->netdev)) 5224 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 5225 else 5226 i40e_vsi_close(vsi); 5227 } 5228 5229 /** 5230 * i40e_unquiesce_vsi - Resume a given VSI 5231 * @vsi: the VSI being resumed 5232 **/ 5233 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 5234 { 5235 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 5236 return; 5237 5238 if (vsi->netdev && netif_running(vsi->netdev)) 5239 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 5240 else 5241 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 5242 } 5243 5244 /** 5245 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 5246 * @pf: the PF 5247 **/ 5248 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 5249 { 5250 int v; 5251 5252 for (v = 0; v < pf->num_alloc_vsi; v++) { 5253 if (pf->vsi[v]) 5254 i40e_quiesce_vsi(pf->vsi[v]); 5255 } 5256 } 5257 5258 /** 5259 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 5260 * @pf: the PF 5261 **/ 5262 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 5263 { 5264 int v; 5265 5266 for (v = 0; v < pf->num_alloc_vsi; v++) { 5267 if (pf->vsi[v]) 5268 i40e_unquiesce_vsi(pf->vsi[v]); 5269 } 5270 } 5271 5272 /** 5273 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 5274 * @vsi: the VSI being configured 5275 * 5276 * Wait until all queues on a given VSI have been disabled. 5277 **/ 5278 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 5279 { 5280 struct i40e_pf *pf = vsi->back; 5281 int i, pf_q, ret; 5282 5283 pf_q = vsi->base_queue; 5284 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 5285 /* Check and wait for the Tx queue */ 5286 ret = i40e_pf_txq_wait(pf, pf_q, false); 5287 if (ret) { 5288 dev_info(&pf->pdev->dev, 5289 "VSI seid %d Tx ring %d disable timeout\n", 5290 vsi->seid, pf_q); 5291 return ret; 5292 } 5293 5294 if (!i40e_enabled_xdp_vsi(vsi)) 5295 goto wait_rx; 5296 5297 /* Check and wait for the XDP Tx queue */ 5298 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 5299 false); 5300 if (ret) { 5301 dev_info(&pf->pdev->dev, 5302 "VSI seid %d XDP Tx ring %d disable timeout\n", 5303 vsi->seid, pf_q); 5304 return ret; 5305 } 5306 wait_rx: 5307 /* Check and wait for the Rx queue */ 5308 ret = i40e_pf_rxq_wait(pf, pf_q, false); 5309 if (ret) { 5310 dev_info(&pf->pdev->dev, 5311 "VSI seid %d Rx ring %d disable timeout\n", 5312 vsi->seid, pf_q); 5313 return ret; 5314 } 5315 } 5316 5317 return 0; 5318 } 5319 5320 #ifdef CONFIG_I40E_DCB 5321 /** 5322 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 5323 * @pf: the PF 5324 * 5325 * This function waits for the queues to be in disabled state for all the 5326 * VSIs that are managed by this PF. 5327 **/ 5328 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 5329 { 5330 int v, ret = 0; 5331 5332 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 5333 if (pf->vsi[v]) { 5334 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 5335 if (ret) 5336 break; 5337 } 5338 } 5339 5340 return ret; 5341 } 5342 5343 #endif 5344 5345 /** 5346 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 5347 * @pf: pointer to PF 5348 * 5349 * Get TC map for ISCSI PF type that will include iSCSI TC 5350 * and LAN TC. 5351 **/ 5352 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 5353 { 5354 struct i40e_dcb_app_priority_table app; 5355 struct i40e_hw *hw = &pf->hw; 5356 u8 enabled_tc = 1; /* TC0 is always enabled */ 5357 u8 tc, i; 5358 /* Get the iSCSI APP TLV */ 5359 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5360 5361 for (i = 0; i < dcbcfg->numapps; i++) { 5362 app = dcbcfg->app[i]; 5363 if (app.selector == I40E_APP_SEL_TCPIP && 5364 app.protocolid == I40E_APP_PROTOID_ISCSI) { 5365 tc = dcbcfg->etscfg.prioritytable[app.priority]; 5366 enabled_tc |= BIT(tc); 5367 break; 5368 } 5369 } 5370 5371 return enabled_tc; 5372 } 5373 5374 /** 5375 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5376 * @dcbcfg: the corresponding DCBx configuration structure 5377 * 5378 * Return the number of TCs from given DCBx configuration 5379 **/ 5380 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5381 { 5382 int i, tc_unused = 0; 5383 u8 num_tc = 0; 5384 u8 ret = 0; 5385 5386 /* Scan the ETS Config Priority Table to find 5387 * traffic class enabled for a given priority 5388 * and create a bitmask of enabled TCs 5389 */ 5390 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5391 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5392 5393 /* Now scan the bitmask to check for 5394 * contiguous TCs starting with TC0 5395 */ 5396 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5397 if (num_tc & BIT(i)) { 5398 if (!tc_unused) { 5399 ret++; 5400 } else { 5401 pr_err("Non-contiguous TC - Disabling DCB\n"); 5402 return 1; 5403 } 5404 } else { 5405 tc_unused = 1; 5406 } 5407 } 5408 5409 /* There is always at least TC0 */ 5410 if (!ret) 5411 ret = 1; 5412 5413 return ret; 5414 } 5415 5416 /** 5417 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5418 * @dcbcfg: the corresponding DCBx configuration structure 5419 * 5420 * Query the current DCB configuration and return the number of 5421 * traffic classes enabled from the given DCBX config 5422 **/ 5423 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5424 { 5425 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5426 u8 enabled_tc = 1; 5427 u8 i; 5428 5429 for (i = 0; i < num_tc; i++) 5430 enabled_tc |= BIT(i); 5431 5432 return enabled_tc; 5433 } 5434 5435 /** 5436 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5437 * @pf: PF being queried 5438 * 5439 * Query the current MQPRIO configuration and return the number of 5440 * traffic classes enabled. 5441 **/ 5442 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5443 { 5444 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 5445 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5446 u8 enabled_tc = 1, i; 5447 5448 for (i = 1; i < num_tc; i++) 5449 enabled_tc |= BIT(i); 5450 return enabled_tc; 5451 } 5452 5453 /** 5454 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5455 * @pf: PF being queried 5456 * 5457 * Return number of traffic classes enabled for the given PF 5458 **/ 5459 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5460 { 5461 struct i40e_hw *hw = &pf->hw; 5462 u8 i, enabled_tc = 1; 5463 u8 num_tc = 0; 5464 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5465 5466 if (i40e_is_tc_mqprio_enabled(pf)) 5467 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5468 5469 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5470 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5471 return 1; 5472 5473 /* SFP mode will be enabled for all TCs on port */ 5474 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5475 return i40e_dcb_get_num_tc(dcbcfg); 5476 5477 /* MFP mode return count of enabled TCs for this PF */ 5478 if (pf->hw.func_caps.iscsi) 5479 enabled_tc = i40e_get_iscsi_tc_map(pf); 5480 else 5481 return 1; /* Only TC0 */ 5482 5483 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5484 if (enabled_tc & BIT(i)) 5485 num_tc++; 5486 } 5487 return num_tc; 5488 } 5489 5490 /** 5491 * i40e_pf_get_tc_map - Get bitmap for enabled traffic classes 5492 * @pf: PF being queried 5493 * 5494 * Return a bitmap for enabled traffic classes for this PF. 5495 **/ 5496 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5497 { 5498 if (i40e_is_tc_mqprio_enabled(pf)) 5499 return i40e_mqprio_get_enabled_tc(pf); 5500 5501 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5502 * default TC 5503 */ 5504 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5505 return I40E_DEFAULT_TRAFFIC_CLASS; 5506 5507 /* SFP mode we want PF to be enabled for all TCs */ 5508 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5509 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5510 5511 /* MFP enabled and iSCSI PF type */ 5512 if (pf->hw.func_caps.iscsi) 5513 return i40e_get_iscsi_tc_map(pf); 5514 else 5515 return I40E_DEFAULT_TRAFFIC_CLASS; 5516 } 5517 5518 /** 5519 * i40e_vsi_get_bw_info - Query VSI BW Information 5520 * @vsi: the VSI being queried 5521 * 5522 * Returns 0 on success, negative value on failure 5523 **/ 5524 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5525 { 5526 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5527 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5528 struct i40e_pf *pf = vsi->back; 5529 struct i40e_hw *hw = &pf->hw; 5530 u32 tc_bw_max; 5531 int ret; 5532 int i; 5533 5534 /* Get the VSI level BW configuration */ 5535 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5536 if (ret) { 5537 dev_info(&pf->pdev->dev, 5538 "couldn't get PF vsi bw config, err %pe aq_err %s\n", 5539 ERR_PTR(ret), 5540 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5541 return -EINVAL; 5542 } 5543 5544 /* Get the VSI level BW configuration per TC */ 5545 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5546 NULL); 5547 if (ret) { 5548 dev_info(&pf->pdev->dev, 5549 "couldn't get PF vsi ets bw config, err %pe aq_err %s\n", 5550 ERR_PTR(ret), 5551 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5552 return -EINVAL; 5553 } 5554 5555 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5556 dev_info(&pf->pdev->dev, 5557 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5558 bw_config.tc_valid_bits, 5559 bw_ets_config.tc_valid_bits); 5560 /* Still continuing */ 5561 } 5562 5563 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5564 vsi->bw_max_quanta = bw_config.max_bw; 5565 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5566 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5567 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5568 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5569 vsi->bw_ets_limit_credits[i] = 5570 le16_to_cpu(bw_ets_config.credits[i]); 5571 /* 3 bits out of 4 for each TC */ 5572 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5573 } 5574 5575 return 0; 5576 } 5577 5578 /** 5579 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5580 * @vsi: the VSI being configured 5581 * @enabled_tc: TC bitmap 5582 * @bw_share: BW shared credits per TC 5583 * 5584 * Returns 0 on success, negative value on failure 5585 **/ 5586 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5587 u8 *bw_share) 5588 { 5589 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5590 struct i40e_pf *pf = vsi->back; 5591 int ret; 5592 int i; 5593 5594 /* There is no need to reset BW when mqprio mode is on. */ 5595 if (i40e_is_tc_mqprio_enabled(pf)) 5596 return 0; 5597 if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5598 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5599 if (ret) 5600 dev_info(&pf->pdev->dev, 5601 "Failed to reset tx rate for vsi->seid %u\n", 5602 vsi->seid); 5603 return ret; 5604 } 5605 memset(&bw_data, 0, sizeof(bw_data)); 5606 bw_data.tc_valid_bits = enabled_tc; 5607 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5608 bw_data.tc_bw_credits[i] = bw_share[i]; 5609 5610 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5611 if (ret) { 5612 dev_info(&pf->pdev->dev, 5613 "AQ command Config VSI BW allocation per TC failed = %d\n", 5614 pf->hw.aq.asq_last_status); 5615 return -EINVAL; 5616 } 5617 5618 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5619 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5620 5621 return 0; 5622 } 5623 5624 /** 5625 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5626 * @vsi: the VSI being configured 5627 * @enabled_tc: TC map to be enabled 5628 * 5629 **/ 5630 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5631 { 5632 struct net_device *netdev = vsi->netdev; 5633 struct i40e_pf *pf = vsi->back; 5634 struct i40e_hw *hw = &pf->hw; 5635 u8 netdev_tc = 0; 5636 int i; 5637 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5638 5639 if (!netdev) 5640 return; 5641 5642 if (!enabled_tc) { 5643 netdev_reset_tc(netdev); 5644 return; 5645 } 5646 5647 /* Set up actual enabled TCs on the VSI */ 5648 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5649 return; 5650 5651 /* set per TC queues for the VSI */ 5652 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5653 /* Only set TC queues for enabled tcs 5654 * 5655 * e.g. For a VSI that has TC0 and TC3 enabled the 5656 * enabled_tc bitmap would be 0x00001001; the driver 5657 * will set the numtc for netdev as 2 that will be 5658 * referenced by the netdev layer as TC 0 and 1. 5659 */ 5660 if (vsi->tc_config.enabled_tc & BIT(i)) 5661 netdev_set_tc_queue(netdev, 5662 vsi->tc_config.tc_info[i].netdev_tc, 5663 vsi->tc_config.tc_info[i].qcount, 5664 vsi->tc_config.tc_info[i].qoffset); 5665 } 5666 5667 if (i40e_is_tc_mqprio_enabled(pf)) 5668 return; 5669 5670 /* Assign UP2TC map for the VSI */ 5671 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5672 /* Get the actual TC# for the UP */ 5673 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5674 /* Get the mapped netdev TC# for the UP */ 5675 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5676 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5677 } 5678 } 5679 5680 /** 5681 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5682 * @vsi: the VSI being configured 5683 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5684 **/ 5685 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5686 struct i40e_vsi_context *ctxt) 5687 { 5688 /* copy just the sections touched not the entire info 5689 * since not all sections are valid as returned by 5690 * update vsi params 5691 */ 5692 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5693 memcpy(&vsi->info.queue_mapping, 5694 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5695 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5696 sizeof(vsi->info.tc_mapping)); 5697 } 5698 5699 /** 5700 * i40e_update_adq_vsi_queues - update queue mapping for ADq VSI 5701 * @vsi: the VSI being reconfigured 5702 * @vsi_offset: offset from main VF VSI 5703 */ 5704 int i40e_update_adq_vsi_queues(struct i40e_vsi *vsi, int vsi_offset) 5705 { 5706 struct i40e_vsi_context ctxt = {}; 5707 struct i40e_pf *pf; 5708 struct i40e_hw *hw; 5709 int ret; 5710 5711 if (!vsi) 5712 return I40E_ERR_PARAM; 5713 pf = vsi->back; 5714 hw = &pf->hw; 5715 5716 ctxt.seid = vsi->seid; 5717 ctxt.pf_num = hw->pf_id; 5718 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id + vsi_offset; 5719 ctxt.uplink_seid = vsi->uplink_seid; 5720 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5721 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 5722 ctxt.info = vsi->info; 5723 5724 i40e_vsi_setup_queue_map(vsi, &ctxt, vsi->tc_config.enabled_tc, 5725 false); 5726 if (vsi->reconfig_rss) { 5727 vsi->rss_size = min_t(int, pf->alloc_rss_size, 5728 vsi->num_queue_pairs); 5729 ret = i40e_vsi_config_rss(vsi); 5730 if (ret) { 5731 dev_info(&pf->pdev->dev, "Failed to reconfig rss for num_queues\n"); 5732 return ret; 5733 } 5734 vsi->reconfig_rss = false; 5735 } 5736 5737 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5738 if (ret) { 5739 dev_info(&pf->pdev->dev, "Update vsi config failed, err %pe aq_err %s\n", 5740 ERR_PTR(ret), 5741 i40e_aq_str(hw, hw->aq.asq_last_status)); 5742 return ret; 5743 } 5744 /* update the local VSI info with updated queue map */ 5745 i40e_vsi_update_queue_map(vsi, &ctxt); 5746 vsi->info.valid_sections = 0; 5747 5748 return ret; 5749 } 5750 5751 /** 5752 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5753 * @vsi: VSI to be configured 5754 * @enabled_tc: TC bitmap 5755 * 5756 * This configures a particular VSI for TCs that are mapped to the 5757 * given TC bitmap. It uses default bandwidth share for TCs across 5758 * VSIs to configure TC for a particular VSI. 5759 * 5760 * NOTE: 5761 * It is expected that the VSI queues have been quisced before calling 5762 * this function. 5763 **/ 5764 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5765 { 5766 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5767 struct i40e_pf *pf = vsi->back; 5768 struct i40e_hw *hw = &pf->hw; 5769 struct i40e_vsi_context ctxt; 5770 int ret = 0; 5771 int i; 5772 5773 /* Check if enabled_tc is same as existing or new TCs */ 5774 if (vsi->tc_config.enabled_tc == enabled_tc && 5775 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5776 return ret; 5777 5778 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5779 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5780 if (enabled_tc & BIT(i)) 5781 bw_share[i] = 1; 5782 } 5783 5784 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5785 if (ret) { 5786 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5787 5788 dev_info(&pf->pdev->dev, 5789 "Failed configuring TC map %d for VSI %d\n", 5790 enabled_tc, vsi->seid); 5791 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5792 &bw_config, NULL); 5793 if (ret) { 5794 dev_info(&pf->pdev->dev, 5795 "Failed querying vsi bw info, err %pe aq_err %s\n", 5796 ERR_PTR(ret), 5797 i40e_aq_str(hw, hw->aq.asq_last_status)); 5798 goto out; 5799 } 5800 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5801 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5802 5803 if (!valid_tc) 5804 valid_tc = bw_config.tc_valid_bits; 5805 /* Always enable TC0, no matter what */ 5806 valid_tc |= 1; 5807 dev_info(&pf->pdev->dev, 5808 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5809 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5810 enabled_tc = valid_tc; 5811 } 5812 5813 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5814 if (ret) { 5815 dev_err(&pf->pdev->dev, 5816 "Unable to configure TC map %d for VSI %d\n", 5817 enabled_tc, vsi->seid); 5818 goto out; 5819 } 5820 } 5821 5822 /* Update Queue Pairs Mapping for currently enabled UPs */ 5823 ctxt.seid = vsi->seid; 5824 ctxt.pf_num = vsi->back->hw.pf_id; 5825 ctxt.vf_num = 0; 5826 ctxt.uplink_seid = vsi->uplink_seid; 5827 ctxt.info = vsi->info; 5828 if (i40e_is_tc_mqprio_enabled(pf)) { 5829 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5830 if (ret) 5831 goto out; 5832 } else { 5833 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5834 } 5835 5836 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5837 * queues changed. 5838 */ 5839 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5840 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5841 vsi->num_queue_pairs); 5842 ret = i40e_vsi_config_rss(vsi); 5843 if (ret) { 5844 dev_info(&vsi->back->pdev->dev, 5845 "Failed to reconfig rss for num_queues\n"); 5846 return ret; 5847 } 5848 vsi->reconfig_rss = false; 5849 } 5850 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5851 ctxt.info.valid_sections |= 5852 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5853 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5854 } 5855 5856 /* Update the VSI after updating the VSI queue-mapping 5857 * information 5858 */ 5859 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5860 if (ret) { 5861 dev_info(&pf->pdev->dev, 5862 "Update vsi tc config failed, err %pe aq_err %s\n", 5863 ERR_PTR(ret), 5864 i40e_aq_str(hw, hw->aq.asq_last_status)); 5865 goto out; 5866 } 5867 /* update the local VSI info with updated queue map */ 5868 i40e_vsi_update_queue_map(vsi, &ctxt); 5869 vsi->info.valid_sections = 0; 5870 5871 /* Update current VSI BW information */ 5872 ret = i40e_vsi_get_bw_info(vsi); 5873 if (ret) { 5874 dev_info(&pf->pdev->dev, 5875 "Failed updating vsi bw info, err %pe aq_err %s\n", 5876 ERR_PTR(ret), 5877 i40e_aq_str(hw, hw->aq.asq_last_status)); 5878 goto out; 5879 } 5880 5881 /* Update the netdev TC setup */ 5882 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5883 out: 5884 return ret; 5885 } 5886 5887 /** 5888 * i40e_get_link_speed - Returns link speed for the interface 5889 * @vsi: VSI to be configured 5890 * 5891 **/ 5892 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5893 { 5894 struct i40e_pf *pf = vsi->back; 5895 5896 switch (pf->hw.phy.link_info.link_speed) { 5897 case I40E_LINK_SPEED_40GB: 5898 return 40000; 5899 case I40E_LINK_SPEED_25GB: 5900 return 25000; 5901 case I40E_LINK_SPEED_20GB: 5902 return 20000; 5903 case I40E_LINK_SPEED_10GB: 5904 return 10000; 5905 case I40E_LINK_SPEED_1GB: 5906 return 1000; 5907 default: 5908 return -EINVAL; 5909 } 5910 } 5911 5912 /** 5913 * i40e_bw_bytes_to_mbits - Convert max_tx_rate from bytes to mbits 5914 * @vsi: Pointer to vsi structure 5915 * @max_tx_rate: max TX rate in bytes to be converted into Mbits 5916 * 5917 * Helper function to convert units before send to set BW limit 5918 **/ 5919 static u64 i40e_bw_bytes_to_mbits(struct i40e_vsi *vsi, u64 max_tx_rate) 5920 { 5921 if (max_tx_rate < I40E_BW_MBPS_DIVISOR) { 5922 dev_warn(&vsi->back->pdev->dev, 5923 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5924 max_tx_rate = I40E_BW_CREDIT_DIVISOR; 5925 } else { 5926 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 5927 } 5928 5929 return max_tx_rate; 5930 } 5931 5932 /** 5933 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5934 * @vsi: VSI to be configured 5935 * @seid: seid of the channel/VSI 5936 * @max_tx_rate: max TX rate to be configured as BW limit 5937 * 5938 * Helper function to set BW limit for a given VSI 5939 **/ 5940 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5941 { 5942 struct i40e_pf *pf = vsi->back; 5943 u64 credits = 0; 5944 int speed = 0; 5945 int ret = 0; 5946 5947 speed = i40e_get_link_speed(vsi); 5948 if (max_tx_rate > speed) { 5949 dev_err(&pf->pdev->dev, 5950 "Invalid max tx rate %llu specified for VSI seid %d.", 5951 max_tx_rate, seid); 5952 return -EINVAL; 5953 } 5954 if (max_tx_rate && max_tx_rate < I40E_BW_CREDIT_DIVISOR) { 5955 dev_warn(&pf->pdev->dev, 5956 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5957 max_tx_rate = I40E_BW_CREDIT_DIVISOR; 5958 } 5959 5960 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5961 credits = max_tx_rate; 5962 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5963 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5964 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5965 if (ret) 5966 dev_err(&pf->pdev->dev, 5967 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %pe aq_err %s\n", 5968 max_tx_rate, seid, ERR_PTR(ret), 5969 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5970 return ret; 5971 } 5972 5973 /** 5974 * i40e_remove_queue_channels - Remove queue channels for the TCs 5975 * @vsi: VSI to be configured 5976 * 5977 * Remove queue channels for the TCs 5978 **/ 5979 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5980 { 5981 enum i40e_admin_queue_err last_aq_status; 5982 struct i40e_cloud_filter *cfilter; 5983 struct i40e_channel *ch, *ch_tmp; 5984 struct i40e_pf *pf = vsi->back; 5985 struct hlist_node *node; 5986 int ret, i; 5987 5988 /* Reset rss size that was stored when reconfiguring rss for 5989 * channel VSIs with non-power-of-2 queue count. 5990 */ 5991 vsi->current_rss_size = 0; 5992 5993 /* perform cleanup for channels if they exist */ 5994 if (list_empty(&vsi->ch_list)) 5995 return; 5996 5997 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5998 struct i40e_vsi *p_vsi; 5999 6000 list_del(&ch->list); 6001 p_vsi = ch->parent_vsi; 6002 if (!p_vsi || !ch->initialized) { 6003 kfree(ch); 6004 continue; 6005 } 6006 /* Reset queue contexts */ 6007 for (i = 0; i < ch->num_queue_pairs; i++) { 6008 struct i40e_ring *tx_ring, *rx_ring; 6009 u16 pf_q; 6010 6011 pf_q = ch->base_queue + i; 6012 tx_ring = vsi->tx_rings[pf_q]; 6013 tx_ring->ch = NULL; 6014 6015 rx_ring = vsi->rx_rings[pf_q]; 6016 rx_ring->ch = NULL; 6017 } 6018 6019 /* Reset BW configured for this VSI via mqprio */ 6020 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 6021 if (ret) 6022 dev_info(&vsi->back->pdev->dev, 6023 "Failed to reset tx rate for ch->seid %u\n", 6024 ch->seid); 6025 6026 /* delete cloud filters associated with this channel */ 6027 hlist_for_each_entry_safe(cfilter, node, 6028 &pf->cloud_filter_list, cloud_node) { 6029 if (cfilter->seid != ch->seid) 6030 continue; 6031 6032 hash_del(&cfilter->cloud_node); 6033 if (cfilter->dst_port) 6034 ret = i40e_add_del_cloud_filter_big_buf(vsi, 6035 cfilter, 6036 false); 6037 else 6038 ret = i40e_add_del_cloud_filter(vsi, cfilter, 6039 false); 6040 last_aq_status = pf->hw.aq.asq_last_status; 6041 if (ret) 6042 dev_info(&pf->pdev->dev, 6043 "Failed to delete cloud filter, err %pe aq_err %s\n", 6044 ERR_PTR(ret), 6045 i40e_aq_str(&pf->hw, last_aq_status)); 6046 kfree(cfilter); 6047 } 6048 6049 /* delete VSI from FW */ 6050 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 6051 NULL); 6052 if (ret) 6053 dev_err(&vsi->back->pdev->dev, 6054 "unable to remove channel (%d) for parent VSI(%d)\n", 6055 ch->seid, p_vsi->seid); 6056 kfree(ch); 6057 } 6058 INIT_LIST_HEAD(&vsi->ch_list); 6059 } 6060 6061 /** 6062 * i40e_get_max_queues_for_channel 6063 * @vsi: ptr to VSI to which channels are associated with 6064 * 6065 * Helper function which returns max value among the queue counts set on the 6066 * channels/TCs created. 6067 **/ 6068 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 6069 { 6070 struct i40e_channel *ch, *ch_tmp; 6071 int max = 0; 6072 6073 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 6074 if (!ch->initialized) 6075 continue; 6076 if (ch->num_queue_pairs > max) 6077 max = ch->num_queue_pairs; 6078 } 6079 6080 return max; 6081 } 6082 6083 /** 6084 * i40e_validate_num_queues - validate num_queues w.r.t channel 6085 * @pf: ptr to PF device 6086 * @num_queues: number of queues 6087 * @vsi: the parent VSI 6088 * @reconfig_rss: indicates should the RSS be reconfigured or not 6089 * 6090 * This function validates number of queues in the context of new channel 6091 * which is being established and determines if RSS should be reconfigured 6092 * or not for parent VSI. 6093 **/ 6094 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 6095 struct i40e_vsi *vsi, bool *reconfig_rss) 6096 { 6097 int max_ch_queues; 6098 6099 if (!reconfig_rss) 6100 return -EINVAL; 6101 6102 *reconfig_rss = false; 6103 if (vsi->current_rss_size) { 6104 if (num_queues > vsi->current_rss_size) { 6105 dev_dbg(&pf->pdev->dev, 6106 "Error: num_queues (%d) > vsi's current_size(%d)\n", 6107 num_queues, vsi->current_rss_size); 6108 return -EINVAL; 6109 } else if ((num_queues < vsi->current_rss_size) && 6110 (!is_power_of_2(num_queues))) { 6111 dev_dbg(&pf->pdev->dev, 6112 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 6113 num_queues, vsi->current_rss_size); 6114 return -EINVAL; 6115 } 6116 } 6117 6118 if (!is_power_of_2(num_queues)) { 6119 /* Find the max num_queues configured for channel if channel 6120 * exist. 6121 * if channel exist, then enforce 'num_queues' to be more than 6122 * max ever queues configured for channel. 6123 */ 6124 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 6125 if (num_queues < max_ch_queues) { 6126 dev_dbg(&pf->pdev->dev, 6127 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 6128 num_queues, max_ch_queues); 6129 return -EINVAL; 6130 } 6131 *reconfig_rss = true; 6132 } 6133 6134 return 0; 6135 } 6136 6137 /** 6138 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 6139 * @vsi: the VSI being setup 6140 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 6141 * 6142 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 6143 **/ 6144 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 6145 { 6146 struct i40e_pf *pf = vsi->back; 6147 u8 seed[I40E_HKEY_ARRAY_SIZE]; 6148 struct i40e_hw *hw = &pf->hw; 6149 int local_rss_size; 6150 u8 *lut; 6151 int ret; 6152 6153 if (!vsi->rss_size) 6154 return -EINVAL; 6155 6156 if (rss_size > vsi->rss_size) 6157 return -EINVAL; 6158 6159 local_rss_size = min_t(int, vsi->rss_size, rss_size); 6160 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 6161 if (!lut) 6162 return -ENOMEM; 6163 6164 /* Ignoring user configured lut if there is one */ 6165 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 6166 6167 /* Use user configured hash key if there is one, otherwise 6168 * use default. 6169 */ 6170 if (vsi->rss_hkey_user) 6171 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 6172 else 6173 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 6174 6175 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 6176 if (ret) { 6177 dev_info(&pf->pdev->dev, 6178 "Cannot set RSS lut, err %pe aq_err %s\n", 6179 ERR_PTR(ret), 6180 i40e_aq_str(hw, hw->aq.asq_last_status)); 6181 kfree(lut); 6182 return ret; 6183 } 6184 kfree(lut); 6185 6186 /* Do the update w.r.t. storing rss_size */ 6187 if (!vsi->orig_rss_size) 6188 vsi->orig_rss_size = vsi->rss_size; 6189 vsi->current_rss_size = local_rss_size; 6190 6191 return ret; 6192 } 6193 6194 /** 6195 * i40e_channel_setup_queue_map - Setup a channel queue map 6196 * @pf: ptr to PF device 6197 * @ctxt: VSI context structure 6198 * @ch: ptr to channel structure 6199 * 6200 * Setup queue map for a specific channel 6201 **/ 6202 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 6203 struct i40e_vsi_context *ctxt, 6204 struct i40e_channel *ch) 6205 { 6206 u16 qcount, qmap, sections = 0; 6207 u8 offset = 0; 6208 int pow; 6209 6210 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 6211 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 6212 6213 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 6214 ch->num_queue_pairs = qcount; 6215 6216 /* find the next higher power-of-2 of num queue pairs */ 6217 pow = ilog2(qcount); 6218 if (!is_power_of_2(qcount)) 6219 pow++; 6220 6221 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 6222 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 6223 6224 /* Setup queue TC[0].qmap for given VSI context */ 6225 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 6226 6227 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 6228 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 6229 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 6230 ctxt->info.valid_sections |= cpu_to_le16(sections); 6231 } 6232 6233 /** 6234 * i40e_add_channel - add a channel by adding VSI 6235 * @pf: ptr to PF device 6236 * @uplink_seid: underlying HW switching element (VEB) ID 6237 * @ch: ptr to channel structure 6238 * 6239 * Add a channel (VSI) using add_vsi and queue_map 6240 **/ 6241 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 6242 struct i40e_channel *ch) 6243 { 6244 struct i40e_hw *hw = &pf->hw; 6245 struct i40e_vsi_context ctxt; 6246 u8 enabled_tc = 0x1; /* TC0 enabled */ 6247 int ret; 6248 6249 if (ch->type != I40E_VSI_VMDQ2) { 6250 dev_info(&pf->pdev->dev, 6251 "add new vsi failed, ch->type %d\n", ch->type); 6252 return -EINVAL; 6253 } 6254 6255 memset(&ctxt, 0, sizeof(ctxt)); 6256 ctxt.pf_num = hw->pf_id; 6257 ctxt.vf_num = 0; 6258 ctxt.uplink_seid = uplink_seid; 6259 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 6260 if (ch->type == I40E_VSI_VMDQ2) 6261 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 6262 6263 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 6264 ctxt.info.valid_sections |= 6265 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6266 ctxt.info.switch_id = 6267 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6268 } 6269 6270 /* Set queue map for a given VSI context */ 6271 i40e_channel_setup_queue_map(pf, &ctxt, ch); 6272 6273 /* Now time to create VSI */ 6274 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 6275 if (ret) { 6276 dev_info(&pf->pdev->dev, 6277 "add new vsi failed, err %pe aq_err %s\n", 6278 ERR_PTR(ret), 6279 i40e_aq_str(&pf->hw, 6280 pf->hw.aq.asq_last_status)); 6281 return -ENOENT; 6282 } 6283 6284 /* Success, update channel, set enabled_tc only if the channel 6285 * is not a macvlan 6286 */ 6287 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; 6288 ch->seid = ctxt.seid; 6289 ch->vsi_number = ctxt.vsi_number; 6290 ch->stat_counter_idx = le16_to_cpu(ctxt.info.stat_counter_idx); 6291 6292 /* copy just the sections touched not the entire info 6293 * since not all sections are valid as returned by 6294 * update vsi params 6295 */ 6296 ch->info.mapping_flags = ctxt.info.mapping_flags; 6297 memcpy(&ch->info.queue_mapping, 6298 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 6299 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 6300 sizeof(ctxt.info.tc_mapping)); 6301 6302 return 0; 6303 } 6304 6305 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 6306 u8 *bw_share) 6307 { 6308 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 6309 int ret; 6310 int i; 6311 6312 memset(&bw_data, 0, sizeof(bw_data)); 6313 bw_data.tc_valid_bits = ch->enabled_tc; 6314 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 6315 bw_data.tc_bw_credits[i] = bw_share[i]; 6316 6317 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 6318 &bw_data, NULL); 6319 if (ret) { 6320 dev_info(&vsi->back->pdev->dev, 6321 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 6322 vsi->back->hw.aq.asq_last_status, ch->seid); 6323 return -EINVAL; 6324 } 6325 6326 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 6327 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 6328 6329 return 0; 6330 } 6331 6332 /** 6333 * i40e_channel_config_tx_ring - config TX ring associated with new channel 6334 * @pf: ptr to PF device 6335 * @vsi: the VSI being setup 6336 * @ch: ptr to channel structure 6337 * 6338 * Configure TX rings associated with channel (VSI) since queues are being 6339 * from parent VSI. 6340 **/ 6341 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 6342 struct i40e_vsi *vsi, 6343 struct i40e_channel *ch) 6344 { 6345 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 6346 int ret; 6347 int i; 6348 6349 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 6350 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6351 if (ch->enabled_tc & BIT(i)) 6352 bw_share[i] = 1; 6353 } 6354 6355 /* configure BW for new VSI */ 6356 ret = i40e_channel_config_bw(vsi, ch, bw_share); 6357 if (ret) { 6358 dev_info(&vsi->back->pdev->dev, 6359 "Failed configuring TC map %d for channel (seid %u)\n", 6360 ch->enabled_tc, ch->seid); 6361 return ret; 6362 } 6363 6364 for (i = 0; i < ch->num_queue_pairs; i++) { 6365 struct i40e_ring *tx_ring, *rx_ring; 6366 u16 pf_q; 6367 6368 pf_q = ch->base_queue + i; 6369 6370 /* Get to TX ring ptr of main VSI, for re-setup TX queue 6371 * context 6372 */ 6373 tx_ring = vsi->tx_rings[pf_q]; 6374 tx_ring->ch = ch; 6375 6376 /* Get the RX ring ptr */ 6377 rx_ring = vsi->rx_rings[pf_q]; 6378 rx_ring->ch = ch; 6379 } 6380 6381 return 0; 6382 } 6383 6384 /** 6385 * i40e_setup_hw_channel - setup new channel 6386 * @pf: ptr to PF device 6387 * @vsi: the VSI being setup 6388 * @ch: ptr to channel structure 6389 * @uplink_seid: underlying HW switching element (VEB) ID 6390 * @type: type of channel to be created (VMDq2/VF) 6391 * 6392 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6393 * and configures TX rings accordingly 6394 **/ 6395 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 6396 struct i40e_vsi *vsi, 6397 struct i40e_channel *ch, 6398 u16 uplink_seid, u8 type) 6399 { 6400 int ret; 6401 6402 ch->initialized = false; 6403 ch->base_queue = vsi->next_base_queue; 6404 ch->type = type; 6405 6406 /* Proceed with creation of channel (VMDq2) VSI */ 6407 ret = i40e_add_channel(pf, uplink_seid, ch); 6408 if (ret) { 6409 dev_info(&pf->pdev->dev, 6410 "failed to add_channel using uplink_seid %u\n", 6411 uplink_seid); 6412 return ret; 6413 } 6414 6415 /* Mark the successful creation of channel */ 6416 ch->initialized = true; 6417 6418 /* Reconfigure TX queues using QTX_CTL register */ 6419 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 6420 if (ret) { 6421 dev_info(&pf->pdev->dev, 6422 "failed to configure TX rings for channel %u\n", 6423 ch->seid); 6424 return ret; 6425 } 6426 6427 /* update 'next_base_queue' */ 6428 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 6429 dev_dbg(&pf->pdev->dev, 6430 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 6431 ch->seid, ch->vsi_number, ch->stat_counter_idx, 6432 ch->num_queue_pairs, 6433 vsi->next_base_queue); 6434 return ret; 6435 } 6436 6437 /** 6438 * i40e_setup_channel - setup new channel using uplink element 6439 * @pf: ptr to PF device 6440 * @vsi: pointer to the VSI to set up the channel within 6441 * @ch: ptr to channel structure 6442 * 6443 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6444 * and uplink switching element (uplink_seid) 6445 **/ 6446 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 6447 struct i40e_channel *ch) 6448 { 6449 u8 vsi_type; 6450 u16 seid; 6451 int ret; 6452 6453 if (vsi->type == I40E_VSI_MAIN) { 6454 vsi_type = I40E_VSI_VMDQ2; 6455 } else { 6456 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6457 vsi->type); 6458 return false; 6459 } 6460 6461 /* underlying switching element */ 6462 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6463 6464 /* create channel (VSI), configure TX rings */ 6465 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6466 if (ret) { 6467 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6468 return false; 6469 } 6470 6471 return ch->initialized ? true : false; 6472 } 6473 6474 /** 6475 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6476 * @vsi: ptr to VSI which has PF backing 6477 * 6478 * Sets up switch mode correctly if it needs to be changed and perform 6479 * what are allowed modes. 6480 **/ 6481 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6482 { 6483 u8 mode; 6484 struct i40e_pf *pf = vsi->back; 6485 struct i40e_hw *hw = &pf->hw; 6486 int ret; 6487 6488 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6489 if (ret) 6490 return -EINVAL; 6491 6492 if (hw->dev_caps.switch_mode) { 6493 /* if switch mode is set, support mode2 (non-tunneled for 6494 * cloud filter) for now 6495 */ 6496 u32 switch_mode = hw->dev_caps.switch_mode & 6497 I40E_SWITCH_MODE_MASK; 6498 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6499 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6500 return 0; 6501 dev_err(&pf->pdev->dev, 6502 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6503 hw->dev_caps.switch_mode); 6504 return -EINVAL; 6505 } 6506 } 6507 6508 /* Set Bit 7 to be valid */ 6509 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6510 6511 /* Set L4type for TCP support */ 6512 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6513 6514 /* Set cloud filter mode */ 6515 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6516 6517 /* Prep mode field for set_switch_config */ 6518 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6519 pf->last_sw_conf_valid_flags, 6520 mode, NULL); 6521 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6522 dev_err(&pf->pdev->dev, 6523 "couldn't set switch config bits, err %pe aq_err %s\n", 6524 ERR_PTR(ret), 6525 i40e_aq_str(hw, 6526 hw->aq.asq_last_status)); 6527 6528 return ret; 6529 } 6530 6531 /** 6532 * i40e_create_queue_channel - function to create channel 6533 * @vsi: VSI to be configured 6534 * @ch: ptr to channel (it contains channel specific params) 6535 * 6536 * This function creates channel (VSI) using num_queues specified by user, 6537 * reconfigs RSS if needed. 6538 **/ 6539 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6540 struct i40e_channel *ch) 6541 { 6542 struct i40e_pf *pf = vsi->back; 6543 bool reconfig_rss; 6544 int err; 6545 6546 if (!ch) 6547 return -EINVAL; 6548 6549 if (!ch->num_queue_pairs) { 6550 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6551 ch->num_queue_pairs); 6552 return -EINVAL; 6553 } 6554 6555 /* validate user requested num_queues for channel */ 6556 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6557 &reconfig_rss); 6558 if (err) { 6559 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6560 ch->num_queue_pairs); 6561 return -EINVAL; 6562 } 6563 6564 /* By default we are in VEPA mode, if this is the first VF/VMDq 6565 * VSI to be added switch to VEB mode. 6566 */ 6567 6568 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6569 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6570 6571 if (vsi->type == I40E_VSI_MAIN) { 6572 if (i40e_is_tc_mqprio_enabled(pf)) 6573 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 6574 else 6575 i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG); 6576 } 6577 /* now onwards for main VSI, number of queues will be value 6578 * of TC0's queue count 6579 */ 6580 } 6581 6582 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6583 * it should be more than num_queues 6584 */ 6585 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6586 dev_dbg(&pf->pdev->dev, 6587 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6588 vsi->cnt_q_avail, ch->num_queue_pairs); 6589 return -EINVAL; 6590 } 6591 6592 /* reconfig_rss only if vsi type is MAIN_VSI */ 6593 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6594 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6595 if (err) { 6596 dev_info(&pf->pdev->dev, 6597 "Error: unable to reconfig rss for num_queues (%u)\n", 6598 ch->num_queue_pairs); 6599 return -EINVAL; 6600 } 6601 } 6602 6603 if (!i40e_setup_channel(pf, vsi, ch)) { 6604 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6605 return -EINVAL; 6606 } 6607 6608 dev_info(&pf->pdev->dev, 6609 "Setup channel (id:%u) utilizing num_queues %d\n", 6610 ch->seid, ch->num_queue_pairs); 6611 6612 /* configure VSI for BW limit */ 6613 if (ch->max_tx_rate) { 6614 u64 credits = ch->max_tx_rate; 6615 6616 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6617 return -EINVAL; 6618 6619 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6620 dev_dbg(&pf->pdev->dev, 6621 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6622 ch->max_tx_rate, 6623 credits, 6624 ch->seid); 6625 } 6626 6627 /* in case of VF, this will be main SRIOV VSI */ 6628 ch->parent_vsi = vsi; 6629 6630 /* and update main_vsi's count for queue_available to use */ 6631 vsi->cnt_q_avail -= ch->num_queue_pairs; 6632 6633 return 0; 6634 } 6635 6636 /** 6637 * i40e_configure_queue_channels - Add queue channel for the given TCs 6638 * @vsi: VSI to be configured 6639 * 6640 * Configures queue channel mapping to the given TCs 6641 **/ 6642 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6643 { 6644 struct i40e_channel *ch; 6645 u64 max_rate = 0; 6646 int ret = 0, i; 6647 6648 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6649 vsi->tc_seid_map[0] = vsi->seid; 6650 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6651 if (vsi->tc_config.enabled_tc & BIT(i)) { 6652 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6653 if (!ch) { 6654 ret = -ENOMEM; 6655 goto err_free; 6656 } 6657 6658 INIT_LIST_HEAD(&ch->list); 6659 ch->num_queue_pairs = 6660 vsi->tc_config.tc_info[i].qcount; 6661 ch->base_queue = 6662 vsi->tc_config.tc_info[i].qoffset; 6663 6664 /* Bandwidth limit through tc interface is in bytes/s, 6665 * change to Mbit/s 6666 */ 6667 max_rate = vsi->mqprio_qopt.max_rate[i]; 6668 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6669 ch->max_tx_rate = max_rate; 6670 6671 list_add_tail(&ch->list, &vsi->ch_list); 6672 6673 ret = i40e_create_queue_channel(vsi, ch); 6674 if (ret) { 6675 dev_err(&vsi->back->pdev->dev, 6676 "Failed creating queue channel with TC%d: queues %d\n", 6677 i, ch->num_queue_pairs); 6678 goto err_free; 6679 } 6680 vsi->tc_seid_map[i] = ch->seid; 6681 } 6682 } 6683 6684 /* reset to reconfigure TX queue contexts */ 6685 i40e_do_reset(vsi->back, I40E_PF_RESET_FLAG, true); 6686 return ret; 6687 6688 err_free: 6689 i40e_remove_queue_channels(vsi); 6690 return ret; 6691 } 6692 6693 /** 6694 * i40e_veb_config_tc - Configure TCs for given VEB 6695 * @veb: given VEB 6696 * @enabled_tc: TC bitmap 6697 * 6698 * Configures given TC bitmap for VEB (switching) element 6699 **/ 6700 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6701 { 6702 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6703 struct i40e_pf *pf = veb->pf; 6704 int ret = 0; 6705 int i; 6706 6707 /* No TCs or already enabled TCs just return */ 6708 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6709 return ret; 6710 6711 bw_data.tc_valid_bits = enabled_tc; 6712 /* bw_data.absolute_credits is not set (relative) */ 6713 6714 /* Enable ETS TCs with equal BW Share for now */ 6715 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6716 if (enabled_tc & BIT(i)) 6717 bw_data.tc_bw_share_credits[i] = 1; 6718 } 6719 6720 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6721 &bw_data, NULL); 6722 if (ret) { 6723 dev_info(&pf->pdev->dev, 6724 "VEB bw config failed, err %pe aq_err %s\n", 6725 ERR_PTR(ret), 6726 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6727 goto out; 6728 } 6729 6730 /* Update the BW information */ 6731 ret = i40e_veb_get_bw_info(veb); 6732 if (ret) { 6733 dev_info(&pf->pdev->dev, 6734 "Failed getting veb bw config, err %pe aq_err %s\n", 6735 ERR_PTR(ret), 6736 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6737 } 6738 6739 out: 6740 return ret; 6741 } 6742 6743 #ifdef CONFIG_I40E_DCB 6744 /** 6745 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6746 * @pf: PF struct 6747 * 6748 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6749 * the caller would've quiesce all the VSIs before calling 6750 * this function 6751 **/ 6752 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6753 { 6754 u8 tc_map = 0; 6755 int ret; 6756 u8 v; 6757 6758 /* Enable the TCs available on PF to all VEBs */ 6759 tc_map = i40e_pf_get_tc_map(pf); 6760 if (tc_map == I40E_DEFAULT_TRAFFIC_CLASS) 6761 return; 6762 6763 for (v = 0; v < I40E_MAX_VEB; v++) { 6764 if (!pf->veb[v]) 6765 continue; 6766 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6767 if (ret) { 6768 dev_info(&pf->pdev->dev, 6769 "Failed configuring TC for VEB seid=%d\n", 6770 pf->veb[v]->seid); 6771 /* Will try to configure as many components */ 6772 } 6773 } 6774 6775 /* Update each VSI */ 6776 for (v = 0; v < pf->num_alloc_vsi; v++) { 6777 if (!pf->vsi[v]) 6778 continue; 6779 6780 /* - Enable all TCs for the LAN VSI 6781 * - For all others keep them at TC0 for now 6782 */ 6783 if (v == pf->lan_vsi) 6784 tc_map = i40e_pf_get_tc_map(pf); 6785 else 6786 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6787 6788 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6789 if (ret) { 6790 dev_info(&pf->pdev->dev, 6791 "Failed configuring TC for VSI seid=%d\n", 6792 pf->vsi[v]->seid); 6793 /* Will try to configure as many components */ 6794 } else { 6795 /* Re-configure VSI vectors based on updated TC map */ 6796 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6797 if (pf->vsi[v]->netdev) 6798 i40e_dcbnl_set_all(pf->vsi[v]); 6799 } 6800 } 6801 } 6802 6803 /** 6804 * i40e_resume_port_tx - Resume port Tx 6805 * @pf: PF struct 6806 * 6807 * Resume a port's Tx and issue a PF reset in case of failure to 6808 * resume. 6809 **/ 6810 static int i40e_resume_port_tx(struct i40e_pf *pf) 6811 { 6812 struct i40e_hw *hw = &pf->hw; 6813 int ret; 6814 6815 ret = i40e_aq_resume_port_tx(hw, NULL); 6816 if (ret) { 6817 dev_info(&pf->pdev->dev, 6818 "Resume Port Tx failed, err %pe aq_err %s\n", 6819 ERR_PTR(ret), 6820 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6821 /* Schedule PF reset to recover */ 6822 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6823 i40e_service_event_schedule(pf); 6824 } 6825 6826 return ret; 6827 } 6828 6829 /** 6830 * i40e_suspend_port_tx - Suspend port Tx 6831 * @pf: PF struct 6832 * 6833 * Suspend a port's Tx and issue a PF reset in case of failure. 6834 **/ 6835 static int i40e_suspend_port_tx(struct i40e_pf *pf) 6836 { 6837 struct i40e_hw *hw = &pf->hw; 6838 int ret; 6839 6840 ret = i40e_aq_suspend_port_tx(hw, pf->mac_seid, NULL); 6841 if (ret) { 6842 dev_info(&pf->pdev->dev, 6843 "Suspend Port Tx failed, err %pe aq_err %s\n", 6844 ERR_PTR(ret), 6845 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6846 /* Schedule PF reset to recover */ 6847 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6848 i40e_service_event_schedule(pf); 6849 } 6850 6851 return ret; 6852 } 6853 6854 /** 6855 * i40e_hw_set_dcb_config - Program new DCBX settings into HW 6856 * @pf: PF being configured 6857 * @new_cfg: New DCBX configuration 6858 * 6859 * Program DCB settings into HW and reconfigure VEB/VSIs on 6860 * given PF. Uses "Set LLDP MIB" AQC to program the hardware. 6861 **/ 6862 static int i40e_hw_set_dcb_config(struct i40e_pf *pf, 6863 struct i40e_dcbx_config *new_cfg) 6864 { 6865 struct i40e_dcbx_config *old_cfg = &pf->hw.local_dcbx_config; 6866 int ret; 6867 6868 /* Check if need reconfiguration */ 6869 if (!memcmp(&new_cfg, &old_cfg, sizeof(new_cfg))) { 6870 dev_dbg(&pf->pdev->dev, "No Change in DCB Config required.\n"); 6871 return 0; 6872 } 6873 6874 /* Config change disable all VSIs */ 6875 i40e_pf_quiesce_all_vsi(pf); 6876 6877 /* Copy the new config to the current config */ 6878 *old_cfg = *new_cfg; 6879 old_cfg->etsrec = old_cfg->etscfg; 6880 ret = i40e_set_dcb_config(&pf->hw); 6881 if (ret) { 6882 dev_info(&pf->pdev->dev, 6883 "Set DCB Config failed, err %pe aq_err %s\n", 6884 ERR_PTR(ret), 6885 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6886 goto out; 6887 } 6888 6889 /* Changes in configuration update VEB/VSI */ 6890 i40e_dcb_reconfigure(pf); 6891 out: 6892 /* In case of reset do not try to resume anything */ 6893 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) { 6894 /* Re-start the VSIs if disabled */ 6895 ret = i40e_resume_port_tx(pf); 6896 /* In case of error no point in resuming VSIs */ 6897 if (ret) 6898 goto err; 6899 i40e_pf_unquiesce_all_vsi(pf); 6900 } 6901 err: 6902 return ret; 6903 } 6904 6905 /** 6906 * i40e_hw_dcb_config - Program new DCBX settings into HW 6907 * @pf: PF being configured 6908 * @new_cfg: New DCBX configuration 6909 * 6910 * Program DCB settings into HW and reconfigure VEB/VSIs on 6911 * given PF 6912 **/ 6913 int i40e_hw_dcb_config(struct i40e_pf *pf, struct i40e_dcbx_config *new_cfg) 6914 { 6915 struct i40e_aqc_configure_switching_comp_ets_data ets_data; 6916 u8 prio_type[I40E_MAX_TRAFFIC_CLASS] = {0}; 6917 u32 mfs_tc[I40E_MAX_TRAFFIC_CLASS]; 6918 struct i40e_dcbx_config *old_cfg; 6919 u8 mode[I40E_MAX_TRAFFIC_CLASS]; 6920 struct i40e_rx_pb_config pb_cfg; 6921 struct i40e_hw *hw = &pf->hw; 6922 u8 num_ports = hw->num_ports; 6923 bool need_reconfig; 6924 int ret = -EINVAL; 6925 u8 lltc_map = 0; 6926 u8 tc_map = 0; 6927 u8 new_numtc; 6928 u8 i; 6929 6930 dev_dbg(&pf->pdev->dev, "Configuring DCB registers directly\n"); 6931 /* Un-pack information to Program ETS HW via shared API 6932 * numtc, tcmap 6933 * LLTC map 6934 * ETS/NON-ETS arbiter mode 6935 * max exponent (credit refills) 6936 * Total number of ports 6937 * PFC priority bit-map 6938 * Priority Table 6939 * BW % per TC 6940 * Arbiter mode between UPs sharing same TC 6941 * TSA table (ETS or non-ETS) 6942 * EEE enabled or not 6943 * MFS TC table 6944 */ 6945 6946 new_numtc = i40e_dcb_get_num_tc(new_cfg); 6947 6948 memset(&ets_data, 0, sizeof(ets_data)); 6949 for (i = 0; i < new_numtc; i++) { 6950 tc_map |= BIT(i); 6951 switch (new_cfg->etscfg.tsatable[i]) { 6952 case I40E_IEEE_TSA_ETS: 6953 prio_type[i] = I40E_DCB_PRIO_TYPE_ETS; 6954 ets_data.tc_bw_share_credits[i] = 6955 new_cfg->etscfg.tcbwtable[i]; 6956 break; 6957 case I40E_IEEE_TSA_STRICT: 6958 prio_type[i] = I40E_DCB_PRIO_TYPE_STRICT; 6959 lltc_map |= BIT(i); 6960 ets_data.tc_bw_share_credits[i] = 6961 I40E_DCB_STRICT_PRIO_CREDITS; 6962 break; 6963 default: 6964 /* Invalid TSA type */ 6965 need_reconfig = false; 6966 goto out; 6967 } 6968 } 6969 6970 old_cfg = &hw->local_dcbx_config; 6971 /* Check if need reconfiguration */ 6972 need_reconfig = i40e_dcb_need_reconfig(pf, old_cfg, new_cfg); 6973 6974 /* If needed, enable/disable frame tagging, disable all VSIs 6975 * and suspend port tx 6976 */ 6977 if (need_reconfig) { 6978 /* Enable DCB tagging only when more than one TC */ 6979 if (new_numtc > 1) 6980 pf->flags |= I40E_FLAG_DCB_ENABLED; 6981 else 6982 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6983 6984 set_bit(__I40E_PORT_SUSPENDED, pf->state); 6985 /* Reconfiguration needed quiesce all VSIs */ 6986 i40e_pf_quiesce_all_vsi(pf); 6987 ret = i40e_suspend_port_tx(pf); 6988 if (ret) 6989 goto err; 6990 } 6991 6992 /* Configure Port ETS Tx Scheduler */ 6993 ets_data.tc_valid_bits = tc_map; 6994 ets_data.tc_strict_priority_flags = lltc_map; 6995 ret = i40e_aq_config_switch_comp_ets 6996 (hw, pf->mac_seid, &ets_data, 6997 i40e_aqc_opc_modify_switching_comp_ets, NULL); 6998 if (ret) { 6999 dev_info(&pf->pdev->dev, 7000 "Modify Port ETS failed, err %pe aq_err %s\n", 7001 ERR_PTR(ret), 7002 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7003 goto out; 7004 } 7005 7006 /* Configure Rx ETS HW */ 7007 memset(&mode, I40E_DCB_ARB_MODE_ROUND_ROBIN, sizeof(mode)); 7008 i40e_dcb_hw_set_num_tc(hw, new_numtc); 7009 i40e_dcb_hw_rx_fifo_config(hw, I40E_DCB_ARB_MODE_ROUND_ROBIN, 7010 I40E_DCB_ARB_MODE_STRICT_PRIORITY, 7011 I40E_DCB_DEFAULT_MAX_EXPONENT, 7012 lltc_map); 7013 i40e_dcb_hw_rx_cmd_monitor_config(hw, new_numtc, num_ports); 7014 i40e_dcb_hw_rx_ets_bw_config(hw, new_cfg->etscfg.tcbwtable, mode, 7015 prio_type); 7016 i40e_dcb_hw_pfc_config(hw, new_cfg->pfc.pfcenable, 7017 new_cfg->etscfg.prioritytable); 7018 i40e_dcb_hw_rx_up2tc_config(hw, new_cfg->etscfg.prioritytable); 7019 7020 /* Configure Rx Packet Buffers in HW */ 7021 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7022 mfs_tc[i] = pf->vsi[pf->lan_vsi]->netdev->mtu; 7023 mfs_tc[i] += I40E_PACKET_HDR_PAD; 7024 } 7025 7026 i40e_dcb_hw_calculate_pool_sizes(hw, num_ports, 7027 false, new_cfg->pfc.pfcenable, 7028 mfs_tc, &pb_cfg); 7029 i40e_dcb_hw_rx_pb_config(hw, &pf->pb_cfg, &pb_cfg); 7030 7031 /* Update the local Rx Packet buffer config */ 7032 pf->pb_cfg = pb_cfg; 7033 7034 /* Inform the FW about changes to DCB configuration */ 7035 ret = i40e_aq_dcb_updated(&pf->hw, NULL); 7036 if (ret) { 7037 dev_info(&pf->pdev->dev, 7038 "DCB Updated failed, err %pe aq_err %s\n", 7039 ERR_PTR(ret), 7040 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7041 goto out; 7042 } 7043 7044 /* Update the port DCBx configuration */ 7045 *old_cfg = *new_cfg; 7046 7047 /* Changes in configuration update VEB/VSI */ 7048 i40e_dcb_reconfigure(pf); 7049 out: 7050 /* Re-start the VSIs if disabled */ 7051 if (need_reconfig) { 7052 ret = i40e_resume_port_tx(pf); 7053 7054 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 7055 /* In case of error no point in resuming VSIs */ 7056 if (ret) 7057 goto err; 7058 7059 /* Wait for the PF's queues to be disabled */ 7060 ret = i40e_pf_wait_queues_disabled(pf); 7061 if (ret) { 7062 /* Schedule PF reset to recover */ 7063 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 7064 i40e_service_event_schedule(pf); 7065 goto err; 7066 } else { 7067 i40e_pf_unquiesce_all_vsi(pf); 7068 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 7069 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 7070 } 7071 /* registers are set, lets apply */ 7072 if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB) 7073 ret = i40e_hw_set_dcb_config(pf, new_cfg); 7074 } 7075 7076 err: 7077 return ret; 7078 } 7079 7080 /** 7081 * i40e_dcb_sw_default_config - Set default DCB configuration when DCB in SW 7082 * @pf: PF being queried 7083 * 7084 * Set default DCB configuration in case DCB is to be done in SW. 7085 **/ 7086 int i40e_dcb_sw_default_config(struct i40e_pf *pf) 7087 { 7088 struct i40e_dcbx_config *dcb_cfg = &pf->hw.local_dcbx_config; 7089 struct i40e_aqc_configure_switching_comp_ets_data ets_data; 7090 struct i40e_hw *hw = &pf->hw; 7091 int err; 7092 7093 if (pf->hw_features & I40E_HW_USE_SET_LLDP_MIB) { 7094 /* Update the local cached instance with TC0 ETS */ 7095 memset(&pf->tmp_cfg, 0, sizeof(struct i40e_dcbx_config)); 7096 pf->tmp_cfg.etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING; 7097 pf->tmp_cfg.etscfg.maxtcs = 0; 7098 pf->tmp_cfg.etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW; 7099 pf->tmp_cfg.etscfg.tsatable[0] = I40E_IEEE_TSA_ETS; 7100 pf->tmp_cfg.pfc.willing = I40E_IEEE_DEFAULT_PFC_WILLING; 7101 pf->tmp_cfg.pfc.pfccap = I40E_MAX_TRAFFIC_CLASS; 7102 /* FW needs one App to configure HW */ 7103 pf->tmp_cfg.numapps = I40E_IEEE_DEFAULT_NUM_APPS; 7104 pf->tmp_cfg.app[0].selector = I40E_APP_SEL_ETHTYPE; 7105 pf->tmp_cfg.app[0].priority = I40E_IEEE_DEFAULT_APP_PRIO; 7106 pf->tmp_cfg.app[0].protocolid = I40E_APP_PROTOID_FCOE; 7107 7108 return i40e_hw_set_dcb_config(pf, &pf->tmp_cfg); 7109 } 7110 7111 memset(&ets_data, 0, sizeof(ets_data)); 7112 ets_data.tc_valid_bits = I40E_DEFAULT_TRAFFIC_CLASS; /* TC0 only */ 7113 ets_data.tc_strict_priority_flags = 0; /* ETS */ 7114 ets_data.tc_bw_share_credits[0] = I40E_IEEE_DEFAULT_ETS_TCBW; /* 100% to TC0 */ 7115 7116 /* Enable ETS on the Physical port */ 7117 err = i40e_aq_config_switch_comp_ets 7118 (hw, pf->mac_seid, &ets_data, 7119 i40e_aqc_opc_enable_switching_comp_ets, NULL); 7120 if (err) { 7121 dev_info(&pf->pdev->dev, 7122 "Enable Port ETS failed, err %pe aq_err %s\n", 7123 ERR_PTR(err), 7124 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7125 err = -ENOENT; 7126 goto out; 7127 } 7128 7129 /* Update the local cached instance with TC0 ETS */ 7130 dcb_cfg->etscfg.willing = I40E_IEEE_DEFAULT_ETS_WILLING; 7131 dcb_cfg->etscfg.cbs = 0; 7132 dcb_cfg->etscfg.maxtcs = I40E_MAX_TRAFFIC_CLASS; 7133 dcb_cfg->etscfg.tcbwtable[0] = I40E_IEEE_DEFAULT_ETS_TCBW; 7134 7135 out: 7136 return err; 7137 } 7138 7139 /** 7140 * i40e_init_pf_dcb - Initialize DCB configuration 7141 * @pf: PF being configured 7142 * 7143 * Query the current DCB configuration and cache it 7144 * in the hardware structure 7145 **/ 7146 static int i40e_init_pf_dcb(struct i40e_pf *pf) 7147 { 7148 struct i40e_hw *hw = &pf->hw; 7149 int err; 7150 7151 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 7152 * Also do not enable DCBx if FW LLDP agent is disabled 7153 */ 7154 if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) { 7155 dev_info(&pf->pdev->dev, "DCB is not supported.\n"); 7156 err = I40E_NOT_SUPPORTED; 7157 goto out; 7158 } 7159 if (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) { 7160 dev_info(&pf->pdev->dev, "FW LLDP is disabled, attempting SW DCB\n"); 7161 err = i40e_dcb_sw_default_config(pf); 7162 if (err) { 7163 dev_info(&pf->pdev->dev, "Could not initialize SW DCB\n"); 7164 goto out; 7165 } 7166 dev_info(&pf->pdev->dev, "SW DCB initialization succeeded.\n"); 7167 pf->dcbx_cap = DCB_CAP_DCBX_HOST | 7168 DCB_CAP_DCBX_VER_IEEE; 7169 /* at init capable but disabled */ 7170 pf->flags |= I40E_FLAG_DCB_CAPABLE; 7171 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 7172 goto out; 7173 } 7174 err = i40e_init_dcb(hw, true); 7175 if (!err) { 7176 /* Device/Function is not DCBX capable */ 7177 if ((!hw->func_caps.dcb) || 7178 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 7179 dev_info(&pf->pdev->dev, 7180 "DCBX offload is not supported or is disabled for this PF.\n"); 7181 } else { 7182 /* When status is not DISABLED then DCBX in FW */ 7183 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 7184 DCB_CAP_DCBX_VER_IEEE; 7185 7186 pf->flags |= I40E_FLAG_DCB_CAPABLE; 7187 /* Enable DCB tagging only when more than one TC 7188 * or explicitly disable if only one TC 7189 */ 7190 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 7191 pf->flags |= I40E_FLAG_DCB_ENABLED; 7192 else 7193 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 7194 dev_dbg(&pf->pdev->dev, 7195 "DCBX offload is supported for this PF.\n"); 7196 } 7197 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 7198 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 7199 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP; 7200 } else { 7201 dev_info(&pf->pdev->dev, 7202 "Query for DCB configuration failed, err %pe aq_err %s\n", 7203 ERR_PTR(err), 7204 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7205 } 7206 7207 out: 7208 return err; 7209 } 7210 #endif /* CONFIG_I40E_DCB */ 7211 7212 /** 7213 * i40e_print_link_message - print link up or down 7214 * @vsi: the VSI for which link needs a message 7215 * @isup: true of link is up, false otherwise 7216 */ 7217 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 7218 { 7219 enum i40e_aq_link_speed new_speed; 7220 struct i40e_pf *pf = vsi->back; 7221 char *speed = "Unknown"; 7222 char *fc = "Unknown"; 7223 char *fec = ""; 7224 char *req_fec = ""; 7225 char *an = ""; 7226 7227 if (isup) 7228 new_speed = pf->hw.phy.link_info.link_speed; 7229 else 7230 new_speed = I40E_LINK_SPEED_UNKNOWN; 7231 7232 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 7233 return; 7234 vsi->current_isup = isup; 7235 vsi->current_speed = new_speed; 7236 if (!isup) { 7237 netdev_info(vsi->netdev, "NIC Link is Down\n"); 7238 return; 7239 } 7240 7241 /* Warn user if link speed on NPAR enabled partition is not at 7242 * least 10GB 7243 */ 7244 if (pf->hw.func_caps.npar_enable && 7245 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 7246 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 7247 netdev_warn(vsi->netdev, 7248 "The partition detected link speed that is less than 10Gbps\n"); 7249 7250 switch (pf->hw.phy.link_info.link_speed) { 7251 case I40E_LINK_SPEED_40GB: 7252 speed = "40 G"; 7253 break; 7254 case I40E_LINK_SPEED_20GB: 7255 speed = "20 G"; 7256 break; 7257 case I40E_LINK_SPEED_25GB: 7258 speed = "25 G"; 7259 break; 7260 case I40E_LINK_SPEED_10GB: 7261 speed = "10 G"; 7262 break; 7263 case I40E_LINK_SPEED_5GB: 7264 speed = "5 G"; 7265 break; 7266 case I40E_LINK_SPEED_2_5GB: 7267 speed = "2.5 G"; 7268 break; 7269 case I40E_LINK_SPEED_1GB: 7270 speed = "1000 M"; 7271 break; 7272 case I40E_LINK_SPEED_100MB: 7273 speed = "100 M"; 7274 break; 7275 default: 7276 break; 7277 } 7278 7279 switch (pf->hw.fc.current_mode) { 7280 case I40E_FC_FULL: 7281 fc = "RX/TX"; 7282 break; 7283 case I40E_FC_TX_PAUSE: 7284 fc = "TX"; 7285 break; 7286 case I40E_FC_RX_PAUSE: 7287 fc = "RX"; 7288 break; 7289 default: 7290 fc = "None"; 7291 break; 7292 } 7293 7294 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 7295 req_fec = "None"; 7296 fec = "None"; 7297 an = "False"; 7298 7299 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 7300 an = "True"; 7301 7302 if (pf->hw.phy.link_info.fec_info & 7303 I40E_AQ_CONFIG_FEC_KR_ENA) 7304 fec = "CL74 FC-FEC/BASE-R"; 7305 else if (pf->hw.phy.link_info.fec_info & 7306 I40E_AQ_CONFIG_FEC_RS_ENA) 7307 fec = "CL108 RS-FEC"; 7308 7309 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 7310 * both RS and FC are requested 7311 */ 7312 if (vsi->back->hw.phy.link_info.req_fec_info & 7313 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 7314 if (vsi->back->hw.phy.link_info.req_fec_info & 7315 I40E_AQ_REQUEST_FEC_RS) 7316 req_fec = "CL108 RS-FEC"; 7317 else 7318 req_fec = "CL74 FC-FEC/BASE-R"; 7319 } 7320 netdev_info(vsi->netdev, 7321 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 7322 speed, req_fec, fec, an, fc); 7323 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) { 7324 req_fec = "None"; 7325 fec = "None"; 7326 an = "False"; 7327 7328 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 7329 an = "True"; 7330 7331 if (pf->hw.phy.link_info.fec_info & 7332 I40E_AQ_CONFIG_FEC_KR_ENA) 7333 fec = "CL74 FC-FEC/BASE-R"; 7334 7335 if (pf->hw.phy.link_info.req_fec_info & 7336 I40E_AQ_REQUEST_FEC_KR) 7337 req_fec = "CL74 FC-FEC/BASE-R"; 7338 7339 netdev_info(vsi->netdev, 7340 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 7341 speed, req_fec, fec, an, fc); 7342 } else { 7343 netdev_info(vsi->netdev, 7344 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", 7345 speed, fc); 7346 } 7347 7348 } 7349 7350 /** 7351 * i40e_up_complete - Finish the last steps of bringing up a connection 7352 * @vsi: the VSI being configured 7353 **/ 7354 static int i40e_up_complete(struct i40e_vsi *vsi) 7355 { 7356 struct i40e_pf *pf = vsi->back; 7357 int err; 7358 7359 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7360 i40e_vsi_configure_msix(vsi); 7361 else 7362 i40e_configure_msi_and_legacy(vsi); 7363 7364 /* start rings */ 7365 err = i40e_vsi_start_rings(vsi); 7366 if (err) 7367 return err; 7368 7369 clear_bit(__I40E_VSI_DOWN, vsi->state); 7370 i40e_napi_enable_all(vsi); 7371 i40e_vsi_enable_irq(vsi); 7372 7373 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 7374 (vsi->netdev)) { 7375 i40e_print_link_message(vsi, true); 7376 netif_tx_start_all_queues(vsi->netdev); 7377 netif_carrier_on(vsi->netdev); 7378 } 7379 7380 /* replay FDIR SB filters */ 7381 if (vsi->type == I40E_VSI_FDIR) { 7382 /* reset fd counters */ 7383 pf->fd_add_err = 0; 7384 pf->fd_atr_cnt = 0; 7385 i40e_fdir_filter_restore(vsi); 7386 } 7387 7388 /* On the next run of the service_task, notify any clients of the new 7389 * opened netdev 7390 */ 7391 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 7392 i40e_service_event_schedule(pf); 7393 7394 return 0; 7395 } 7396 7397 /** 7398 * i40e_vsi_reinit_locked - Reset the VSI 7399 * @vsi: the VSI being configured 7400 * 7401 * Rebuild the ring structs after some configuration 7402 * has changed, e.g. MTU size. 7403 **/ 7404 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 7405 { 7406 struct i40e_pf *pf = vsi->back; 7407 7408 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 7409 usleep_range(1000, 2000); 7410 i40e_down(vsi); 7411 7412 i40e_up(vsi); 7413 clear_bit(__I40E_CONFIG_BUSY, pf->state); 7414 } 7415 7416 /** 7417 * i40e_force_link_state - Force the link status 7418 * @pf: board private structure 7419 * @is_up: whether the link state should be forced up or down 7420 **/ 7421 static int i40e_force_link_state(struct i40e_pf *pf, bool is_up) 7422 { 7423 struct i40e_aq_get_phy_abilities_resp abilities; 7424 struct i40e_aq_set_phy_config config = {0}; 7425 bool non_zero_phy_type = is_up; 7426 struct i40e_hw *hw = &pf->hw; 7427 u64 mask; 7428 u8 speed; 7429 int err; 7430 7431 /* Card might've been put in an unstable state by other drivers 7432 * and applications, which causes incorrect speed values being 7433 * set on startup. In order to clear speed registers, we call 7434 * get_phy_capabilities twice, once to get initial state of 7435 * available speeds, and once to get current PHY config. 7436 */ 7437 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 7438 NULL); 7439 if (err) { 7440 dev_err(&pf->pdev->dev, 7441 "failed to get phy cap., ret = %pe last_status = %s\n", 7442 ERR_PTR(err), 7443 i40e_aq_str(hw, hw->aq.asq_last_status)); 7444 return err; 7445 } 7446 speed = abilities.link_speed; 7447 7448 /* Get the current phy config */ 7449 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 7450 NULL); 7451 if (err) { 7452 dev_err(&pf->pdev->dev, 7453 "failed to get phy cap., ret = %pe last_status = %s\n", 7454 ERR_PTR(err), 7455 i40e_aq_str(hw, hw->aq.asq_last_status)); 7456 return err; 7457 } 7458 7459 /* If link needs to go up, but was not forced to go down, 7460 * and its speed values are OK, no need for a flap 7461 * if non_zero_phy_type was set, still need to force up 7462 */ 7463 if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) 7464 non_zero_phy_type = true; 7465 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 7466 return I40E_SUCCESS; 7467 7468 /* To force link we need to set bits for all supported PHY types, 7469 * but there are now more than 32, so we need to split the bitmap 7470 * across two fields. 7471 */ 7472 mask = I40E_PHY_TYPES_BITMASK; 7473 config.phy_type = 7474 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 7475 config.phy_type_ext = 7476 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0; 7477 /* Copy the old settings, except of phy_type */ 7478 config.abilities = abilities.abilities; 7479 if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) { 7480 if (is_up) 7481 config.abilities |= I40E_AQ_PHY_ENABLE_LINK; 7482 else 7483 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK); 7484 } 7485 if (abilities.link_speed != 0) 7486 config.link_speed = abilities.link_speed; 7487 else 7488 config.link_speed = speed; 7489 config.eee_capability = abilities.eee_capability; 7490 config.eeer = abilities.eeer_val; 7491 config.low_power_ctrl = abilities.d3_lpan; 7492 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 7493 I40E_AQ_PHY_FEC_CONFIG_MASK; 7494 err = i40e_aq_set_phy_config(hw, &config, NULL); 7495 7496 if (err) { 7497 dev_err(&pf->pdev->dev, 7498 "set phy config ret = %pe last_status = %s\n", 7499 ERR_PTR(err), 7500 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7501 return err; 7502 } 7503 7504 /* Update the link info */ 7505 err = i40e_update_link_info(hw); 7506 if (err) { 7507 /* Wait a little bit (on 40G cards it sometimes takes a really 7508 * long time for link to come back from the atomic reset) 7509 * and try once more 7510 */ 7511 msleep(1000); 7512 i40e_update_link_info(hw); 7513 } 7514 7515 i40e_aq_set_link_restart_an(hw, is_up, NULL); 7516 7517 return I40E_SUCCESS; 7518 } 7519 7520 /** 7521 * i40e_up - Bring the connection back up after being down 7522 * @vsi: the VSI being configured 7523 **/ 7524 int i40e_up(struct i40e_vsi *vsi) 7525 { 7526 int err; 7527 7528 if (vsi->type == I40E_VSI_MAIN && 7529 (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED || 7530 vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)) 7531 i40e_force_link_state(vsi->back, true); 7532 7533 err = i40e_vsi_configure(vsi); 7534 if (!err) 7535 err = i40e_up_complete(vsi); 7536 7537 return err; 7538 } 7539 7540 /** 7541 * i40e_down - Shutdown the connection processing 7542 * @vsi: the VSI being stopped 7543 **/ 7544 void i40e_down(struct i40e_vsi *vsi) 7545 { 7546 int i; 7547 7548 /* It is assumed that the caller of this function 7549 * sets the vsi->state __I40E_VSI_DOWN bit. 7550 */ 7551 if (vsi->netdev) { 7552 netif_carrier_off(vsi->netdev); 7553 netif_tx_disable(vsi->netdev); 7554 } 7555 i40e_vsi_disable_irq(vsi); 7556 i40e_vsi_stop_rings(vsi); 7557 if (vsi->type == I40E_VSI_MAIN && 7558 (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED || 7559 vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)) 7560 i40e_force_link_state(vsi->back, false); 7561 i40e_napi_disable_all(vsi); 7562 7563 for (i = 0; i < vsi->num_queue_pairs; i++) { 7564 i40e_clean_tx_ring(vsi->tx_rings[i]); 7565 if (i40e_enabled_xdp_vsi(vsi)) { 7566 /* Make sure that in-progress ndo_xdp_xmit and 7567 * ndo_xsk_wakeup calls are completed. 7568 */ 7569 synchronize_rcu(); 7570 i40e_clean_tx_ring(vsi->xdp_rings[i]); 7571 } 7572 i40e_clean_rx_ring(vsi->rx_rings[i]); 7573 } 7574 7575 } 7576 7577 /** 7578 * i40e_validate_mqprio_qopt- validate queue mapping info 7579 * @vsi: the VSI being configured 7580 * @mqprio_qopt: queue parametrs 7581 **/ 7582 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 7583 struct tc_mqprio_qopt_offload *mqprio_qopt) 7584 { 7585 u64 sum_max_rate = 0; 7586 u64 max_rate = 0; 7587 int i; 7588 7589 if (mqprio_qopt->qopt.offset[0] != 0 || 7590 mqprio_qopt->qopt.num_tc < 1 || 7591 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 7592 return -EINVAL; 7593 for (i = 0; ; i++) { 7594 if (!mqprio_qopt->qopt.count[i]) 7595 return -EINVAL; 7596 if (mqprio_qopt->min_rate[i]) { 7597 dev_err(&vsi->back->pdev->dev, 7598 "Invalid min tx rate (greater than 0) specified\n"); 7599 return -EINVAL; 7600 } 7601 max_rate = mqprio_qopt->max_rate[i]; 7602 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 7603 sum_max_rate += max_rate; 7604 7605 if (i >= mqprio_qopt->qopt.num_tc - 1) 7606 break; 7607 if (mqprio_qopt->qopt.offset[i + 1] != 7608 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 7609 return -EINVAL; 7610 } 7611 if (vsi->num_queue_pairs < 7612 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 7613 dev_err(&vsi->back->pdev->dev, 7614 "Failed to create traffic channel, insufficient number of queues.\n"); 7615 return -EINVAL; 7616 } 7617 if (sum_max_rate > i40e_get_link_speed(vsi)) { 7618 dev_err(&vsi->back->pdev->dev, 7619 "Invalid max tx rate specified\n"); 7620 return -EINVAL; 7621 } 7622 return 0; 7623 } 7624 7625 /** 7626 * i40e_vsi_set_default_tc_config - set default values for tc configuration 7627 * @vsi: the VSI being configured 7628 **/ 7629 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 7630 { 7631 u16 qcount; 7632 int i; 7633 7634 /* Only TC0 is enabled */ 7635 vsi->tc_config.numtc = 1; 7636 vsi->tc_config.enabled_tc = 1; 7637 qcount = min_t(int, vsi->alloc_queue_pairs, 7638 i40e_pf_get_max_q_per_tc(vsi->back)); 7639 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7640 /* For the TC that is not enabled set the offset to default 7641 * queue and allocate one queue for the given TC. 7642 */ 7643 vsi->tc_config.tc_info[i].qoffset = 0; 7644 if (i == 0) 7645 vsi->tc_config.tc_info[i].qcount = qcount; 7646 else 7647 vsi->tc_config.tc_info[i].qcount = 1; 7648 vsi->tc_config.tc_info[i].netdev_tc = 0; 7649 } 7650 } 7651 7652 /** 7653 * i40e_del_macvlan_filter 7654 * @hw: pointer to the HW structure 7655 * @seid: seid of the channel VSI 7656 * @macaddr: the mac address to apply as a filter 7657 * @aq_err: store the admin Q error 7658 * 7659 * This function deletes a mac filter on the channel VSI which serves as the 7660 * macvlan. Returns 0 on success. 7661 **/ 7662 static int i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid, 7663 const u8 *macaddr, int *aq_err) 7664 { 7665 struct i40e_aqc_remove_macvlan_element_data element; 7666 int status; 7667 7668 memset(&element, 0, sizeof(element)); 7669 ether_addr_copy(element.mac_addr, macaddr); 7670 element.vlan_tag = 0; 7671 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 7672 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL); 7673 *aq_err = hw->aq.asq_last_status; 7674 7675 return status; 7676 } 7677 7678 /** 7679 * i40e_add_macvlan_filter 7680 * @hw: pointer to the HW structure 7681 * @seid: seid of the channel VSI 7682 * @macaddr: the mac address to apply as a filter 7683 * @aq_err: store the admin Q error 7684 * 7685 * This function adds a mac filter on the channel VSI which serves as the 7686 * macvlan. Returns 0 on success. 7687 **/ 7688 static int i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid, 7689 const u8 *macaddr, int *aq_err) 7690 { 7691 struct i40e_aqc_add_macvlan_element_data element; 7692 u16 cmd_flags = 0; 7693 int status; 7694 7695 ether_addr_copy(element.mac_addr, macaddr); 7696 element.vlan_tag = 0; 7697 element.queue_number = 0; 7698 element.match_method = I40E_AQC_MM_ERR_NO_RES; 7699 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 7700 element.flags = cpu_to_le16(cmd_flags); 7701 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL); 7702 *aq_err = hw->aq.asq_last_status; 7703 7704 return status; 7705 } 7706 7707 /** 7708 * i40e_reset_ch_rings - Reset the queue contexts in a channel 7709 * @vsi: the VSI we want to access 7710 * @ch: the channel we want to access 7711 */ 7712 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch) 7713 { 7714 struct i40e_ring *tx_ring, *rx_ring; 7715 u16 pf_q; 7716 int i; 7717 7718 for (i = 0; i < ch->num_queue_pairs; i++) { 7719 pf_q = ch->base_queue + i; 7720 tx_ring = vsi->tx_rings[pf_q]; 7721 tx_ring->ch = NULL; 7722 rx_ring = vsi->rx_rings[pf_q]; 7723 rx_ring->ch = NULL; 7724 } 7725 } 7726 7727 /** 7728 * i40e_free_macvlan_channels 7729 * @vsi: the VSI we want to access 7730 * 7731 * This function frees the Qs of the channel VSI from 7732 * the stack and also deletes the channel VSIs which 7733 * serve as macvlans. 7734 */ 7735 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi) 7736 { 7737 struct i40e_channel *ch, *ch_tmp; 7738 int ret; 7739 7740 if (list_empty(&vsi->macvlan_list)) 7741 return; 7742 7743 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7744 struct i40e_vsi *parent_vsi; 7745 7746 if (i40e_is_channel_macvlan(ch)) { 7747 i40e_reset_ch_rings(vsi, ch); 7748 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7749 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev); 7750 netdev_set_sb_channel(ch->fwd->netdev, 0); 7751 kfree(ch->fwd); 7752 ch->fwd = NULL; 7753 } 7754 7755 list_del(&ch->list); 7756 parent_vsi = ch->parent_vsi; 7757 if (!parent_vsi || !ch->initialized) { 7758 kfree(ch); 7759 continue; 7760 } 7761 7762 /* remove the VSI */ 7763 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 7764 NULL); 7765 if (ret) 7766 dev_err(&vsi->back->pdev->dev, 7767 "unable to remove channel (%d) for parent VSI(%d)\n", 7768 ch->seid, parent_vsi->seid); 7769 kfree(ch); 7770 } 7771 vsi->macvlan_cnt = 0; 7772 } 7773 7774 /** 7775 * i40e_fwd_ring_up - bring the macvlan device up 7776 * @vsi: the VSI we want to access 7777 * @vdev: macvlan netdevice 7778 * @fwd: the private fwd structure 7779 */ 7780 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev, 7781 struct i40e_fwd_adapter *fwd) 7782 { 7783 struct i40e_channel *ch = NULL, *ch_tmp, *iter; 7784 int ret = 0, num_tc = 1, i, aq_err; 7785 struct i40e_pf *pf = vsi->back; 7786 struct i40e_hw *hw = &pf->hw; 7787 7788 /* Go through the list and find an available channel */ 7789 list_for_each_entry_safe(iter, ch_tmp, &vsi->macvlan_list, list) { 7790 if (!i40e_is_channel_macvlan(iter)) { 7791 iter->fwd = fwd; 7792 /* record configuration for macvlan interface in vdev */ 7793 for (i = 0; i < num_tc; i++) 7794 netdev_bind_sb_channel_queue(vsi->netdev, vdev, 7795 i, 7796 iter->num_queue_pairs, 7797 iter->base_queue); 7798 for (i = 0; i < iter->num_queue_pairs; i++) { 7799 struct i40e_ring *tx_ring, *rx_ring; 7800 u16 pf_q; 7801 7802 pf_q = iter->base_queue + i; 7803 7804 /* Get to TX ring ptr */ 7805 tx_ring = vsi->tx_rings[pf_q]; 7806 tx_ring->ch = iter; 7807 7808 /* Get the RX ring ptr */ 7809 rx_ring = vsi->rx_rings[pf_q]; 7810 rx_ring->ch = iter; 7811 } 7812 ch = iter; 7813 break; 7814 } 7815 } 7816 7817 if (!ch) 7818 return -EINVAL; 7819 7820 /* Guarantee all rings are updated before we update the 7821 * MAC address filter. 7822 */ 7823 wmb(); 7824 7825 /* Add a mac filter */ 7826 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err); 7827 if (ret) { 7828 /* if we cannot add the MAC rule then disable the offload */ 7829 macvlan_release_l2fw_offload(vdev); 7830 for (i = 0; i < ch->num_queue_pairs; i++) { 7831 struct i40e_ring *rx_ring; 7832 u16 pf_q; 7833 7834 pf_q = ch->base_queue + i; 7835 rx_ring = vsi->rx_rings[pf_q]; 7836 rx_ring->netdev = NULL; 7837 } 7838 dev_info(&pf->pdev->dev, 7839 "Error adding mac filter on macvlan err %pe, aq_err %s\n", 7840 ERR_PTR(ret), 7841 i40e_aq_str(hw, aq_err)); 7842 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n"); 7843 } 7844 7845 return ret; 7846 } 7847 7848 /** 7849 * i40e_setup_macvlans - create the channels which will be macvlans 7850 * @vsi: the VSI we want to access 7851 * @macvlan_cnt: no. of macvlans to be setup 7852 * @qcnt: no. of Qs per macvlan 7853 * @vdev: macvlan netdevice 7854 */ 7855 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt, 7856 struct net_device *vdev) 7857 { 7858 struct i40e_pf *pf = vsi->back; 7859 struct i40e_hw *hw = &pf->hw; 7860 struct i40e_vsi_context ctxt; 7861 u16 sections, qmap, num_qps; 7862 struct i40e_channel *ch; 7863 int i, pow, ret = 0; 7864 u8 offset = 0; 7865 7866 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt) 7867 return -EINVAL; 7868 7869 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt); 7870 7871 /* find the next higher power-of-2 of num queue pairs */ 7872 pow = fls(roundup_pow_of_two(num_qps) - 1); 7873 7874 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 7875 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 7876 7877 /* Setup context bits for the main VSI */ 7878 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 7879 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 7880 memset(&ctxt, 0, sizeof(ctxt)); 7881 ctxt.seid = vsi->seid; 7882 ctxt.pf_num = vsi->back->hw.pf_id; 7883 ctxt.vf_num = 0; 7884 ctxt.uplink_seid = vsi->uplink_seid; 7885 ctxt.info = vsi->info; 7886 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap); 7887 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 7888 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 7889 ctxt.info.valid_sections |= cpu_to_le16(sections); 7890 7891 /* Reconfigure RSS for main VSI with new max queue count */ 7892 vsi->rss_size = max_t(u16, num_qps, qcnt); 7893 ret = i40e_vsi_config_rss(vsi); 7894 if (ret) { 7895 dev_info(&pf->pdev->dev, 7896 "Failed to reconfig RSS for num_queues (%u)\n", 7897 vsi->rss_size); 7898 return ret; 7899 } 7900 vsi->reconfig_rss = true; 7901 dev_dbg(&vsi->back->pdev->dev, 7902 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size); 7903 vsi->next_base_queue = num_qps; 7904 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps; 7905 7906 /* Update the VSI after updating the VSI queue-mapping 7907 * information 7908 */ 7909 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7910 if (ret) { 7911 dev_info(&pf->pdev->dev, 7912 "Update vsi tc config failed, err %pe aq_err %s\n", 7913 ERR_PTR(ret), 7914 i40e_aq_str(hw, hw->aq.asq_last_status)); 7915 return ret; 7916 } 7917 /* update the local VSI info with updated queue map */ 7918 i40e_vsi_update_queue_map(vsi, &ctxt); 7919 vsi->info.valid_sections = 0; 7920 7921 /* Create channels for macvlans */ 7922 INIT_LIST_HEAD(&vsi->macvlan_list); 7923 for (i = 0; i < macvlan_cnt; i++) { 7924 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 7925 if (!ch) { 7926 ret = -ENOMEM; 7927 goto err_free; 7928 } 7929 INIT_LIST_HEAD(&ch->list); 7930 ch->num_queue_pairs = qcnt; 7931 if (!i40e_setup_channel(pf, vsi, ch)) { 7932 ret = -EINVAL; 7933 kfree(ch); 7934 goto err_free; 7935 } 7936 ch->parent_vsi = vsi; 7937 vsi->cnt_q_avail -= ch->num_queue_pairs; 7938 vsi->macvlan_cnt++; 7939 list_add_tail(&ch->list, &vsi->macvlan_list); 7940 } 7941 7942 return ret; 7943 7944 err_free: 7945 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n"); 7946 i40e_free_macvlan_channels(vsi); 7947 7948 return ret; 7949 } 7950 7951 /** 7952 * i40e_fwd_add - configure macvlans 7953 * @netdev: net device to configure 7954 * @vdev: macvlan netdevice 7955 **/ 7956 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) 7957 { 7958 struct i40e_netdev_priv *np = netdev_priv(netdev); 7959 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors; 7960 struct i40e_vsi *vsi = np->vsi; 7961 struct i40e_pf *pf = vsi->back; 7962 struct i40e_fwd_adapter *fwd; 7963 int avail_macvlan, ret; 7964 7965 if ((pf->flags & I40E_FLAG_DCB_ENABLED)) { 7966 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); 7967 return ERR_PTR(-EINVAL); 7968 } 7969 if (i40e_is_tc_mqprio_enabled(pf)) { 7970 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); 7971 return ERR_PTR(-EINVAL); 7972 } 7973 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) { 7974 netdev_info(netdev, "Not enough vectors available to support macvlans\n"); 7975 return ERR_PTR(-EINVAL); 7976 } 7977 7978 /* The macvlan device has to be a single Q device so that the 7979 * tc_to_txq field can be reused to pick the tx queue. 7980 */ 7981 if (netif_is_multiqueue(vdev)) 7982 return ERR_PTR(-ERANGE); 7983 7984 if (!vsi->macvlan_cnt) { 7985 /* reserve bit 0 for the pf device */ 7986 set_bit(0, vsi->fwd_bitmask); 7987 7988 /* Try to reserve as many queues as possible for macvlans. First 7989 * reserve 3/4th of max vectors, then half, then quarter and 7990 * calculate Qs per macvlan as you go 7991 */ 7992 vectors = pf->num_lan_msix; 7993 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) { 7994 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/ 7995 q_per_macvlan = 4; 7996 macvlan_cnt = (vectors - 32) / 4; 7997 } else if (vectors <= 64 && vectors > 32) { 7998 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/ 7999 q_per_macvlan = 2; 8000 macvlan_cnt = (vectors - 16) / 2; 8001 } else if (vectors <= 32 && vectors > 16) { 8002 /* allocate 1 Q per macvlan and 16 Qs to the PF*/ 8003 q_per_macvlan = 1; 8004 macvlan_cnt = vectors - 16; 8005 } else if (vectors <= 16 && vectors > 8) { 8006 /* allocate 1 Q per macvlan and 8 Qs to the PF */ 8007 q_per_macvlan = 1; 8008 macvlan_cnt = vectors - 8; 8009 } else { 8010 /* allocate 1 Q per macvlan and 1 Q to the PF */ 8011 q_per_macvlan = 1; 8012 macvlan_cnt = vectors - 1; 8013 } 8014 8015 if (macvlan_cnt == 0) 8016 return ERR_PTR(-EBUSY); 8017 8018 /* Quiesce VSI queues */ 8019 i40e_quiesce_vsi(vsi); 8020 8021 /* sets up the macvlans but does not "enable" them */ 8022 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan, 8023 vdev); 8024 if (ret) 8025 return ERR_PTR(ret); 8026 8027 /* Unquiesce VSI */ 8028 i40e_unquiesce_vsi(vsi); 8029 } 8030 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask, 8031 vsi->macvlan_cnt); 8032 if (avail_macvlan >= I40E_MAX_MACVLANS) 8033 return ERR_PTR(-EBUSY); 8034 8035 /* create the fwd struct */ 8036 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL); 8037 if (!fwd) 8038 return ERR_PTR(-ENOMEM); 8039 8040 set_bit(avail_macvlan, vsi->fwd_bitmask); 8041 fwd->bit_no = avail_macvlan; 8042 netdev_set_sb_channel(vdev, avail_macvlan); 8043 fwd->netdev = vdev; 8044 8045 if (!netif_running(netdev)) 8046 return fwd; 8047 8048 /* Set fwd ring up */ 8049 ret = i40e_fwd_ring_up(vsi, vdev, fwd); 8050 if (ret) { 8051 /* unbind the queues and drop the subordinate channel config */ 8052 netdev_unbind_sb_channel(netdev, vdev); 8053 netdev_set_sb_channel(vdev, 0); 8054 8055 kfree(fwd); 8056 return ERR_PTR(-EINVAL); 8057 } 8058 8059 return fwd; 8060 } 8061 8062 /** 8063 * i40e_del_all_macvlans - Delete all the mac filters on the channels 8064 * @vsi: the VSI we want to access 8065 */ 8066 static void i40e_del_all_macvlans(struct i40e_vsi *vsi) 8067 { 8068 struct i40e_channel *ch, *ch_tmp; 8069 struct i40e_pf *pf = vsi->back; 8070 struct i40e_hw *hw = &pf->hw; 8071 int aq_err, ret = 0; 8072 8073 if (list_empty(&vsi->macvlan_list)) 8074 return; 8075 8076 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 8077 if (i40e_is_channel_macvlan(ch)) { 8078 ret = i40e_del_macvlan_filter(hw, ch->seid, 8079 i40e_channel_mac(ch), 8080 &aq_err); 8081 if (!ret) { 8082 /* Reset queue contexts */ 8083 i40e_reset_ch_rings(vsi, ch); 8084 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 8085 netdev_unbind_sb_channel(vsi->netdev, 8086 ch->fwd->netdev); 8087 netdev_set_sb_channel(ch->fwd->netdev, 0); 8088 kfree(ch->fwd); 8089 ch->fwd = NULL; 8090 } 8091 } 8092 } 8093 } 8094 8095 /** 8096 * i40e_fwd_del - delete macvlan interfaces 8097 * @netdev: net device to configure 8098 * @vdev: macvlan netdevice 8099 */ 8100 static void i40e_fwd_del(struct net_device *netdev, void *vdev) 8101 { 8102 struct i40e_netdev_priv *np = netdev_priv(netdev); 8103 struct i40e_fwd_adapter *fwd = vdev; 8104 struct i40e_channel *ch, *ch_tmp; 8105 struct i40e_vsi *vsi = np->vsi; 8106 struct i40e_pf *pf = vsi->back; 8107 struct i40e_hw *hw = &pf->hw; 8108 int aq_err, ret = 0; 8109 8110 /* Find the channel associated with the macvlan and del mac filter */ 8111 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 8112 if (i40e_is_channel_macvlan(ch) && 8113 ether_addr_equal(i40e_channel_mac(ch), 8114 fwd->netdev->dev_addr)) { 8115 ret = i40e_del_macvlan_filter(hw, ch->seid, 8116 i40e_channel_mac(ch), 8117 &aq_err); 8118 if (!ret) { 8119 /* Reset queue contexts */ 8120 i40e_reset_ch_rings(vsi, ch); 8121 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 8122 netdev_unbind_sb_channel(netdev, fwd->netdev); 8123 netdev_set_sb_channel(fwd->netdev, 0); 8124 kfree(ch->fwd); 8125 ch->fwd = NULL; 8126 } else { 8127 dev_info(&pf->pdev->dev, 8128 "Error deleting mac filter on macvlan err %pe, aq_err %s\n", 8129 ERR_PTR(ret), 8130 i40e_aq_str(hw, aq_err)); 8131 } 8132 break; 8133 } 8134 } 8135 } 8136 8137 /** 8138 * i40e_setup_tc - configure multiple traffic classes 8139 * @netdev: net device to configure 8140 * @type_data: tc offload data 8141 **/ 8142 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 8143 { 8144 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 8145 struct i40e_netdev_priv *np = netdev_priv(netdev); 8146 struct i40e_vsi *vsi = np->vsi; 8147 struct i40e_pf *pf = vsi->back; 8148 u8 enabled_tc = 0, num_tc, hw; 8149 bool need_reset = false; 8150 int old_queue_pairs; 8151 int ret = -EINVAL; 8152 u16 mode; 8153 int i; 8154 8155 old_queue_pairs = vsi->num_queue_pairs; 8156 num_tc = mqprio_qopt->qopt.num_tc; 8157 hw = mqprio_qopt->qopt.hw; 8158 mode = mqprio_qopt->mode; 8159 if (!hw) { 8160 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 8161 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 8162 goto config_tc; 8163 } 8164 8165 /* Check if MFP enabled */ 8166 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 8167 netdev_info(netdev, 8168 "Configuring TC not supported in MFP mode\n"); 8169 return ret; 8170 } 8171 switch (mode) { 8172 case TC_MQPRIO_MODE_DCB: 8173 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 8174 8175 /* Check if DCB enabled to continue */ 8176 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 8177 netdev_info(netdev, 8178 "DCB is not enabled for adapter\n"); 8179 return ret; 8180 } 8181 8182 /* Check whether tc count is within enabled limit */ 8183 if (num_tc > i40e_pf_get_num_tc(pf)) { 8184 netdev_info(netdev, 8185 "TC count greater than enabled on link for adapter\n"); 8186 return ret; 8187 } 8188 break; 8189 case TC_MQPRIO_MODE_CHANNEL: 8190 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 8191 netdev_info(netdev, 8192 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 8193 return ret; 8194 } 8195 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 8196 return ret; 8197 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 8198 if (ret) 8199 return ret; 8200 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 8201 sizeof(*mqprio_qopt)); 8202 pf->flags |= I40E_FLAG_TC_MQPRIO; 8203 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 8204 break; 8205 default: 8206 return -EINVAL; 8207 } 8208 8209 config_tc: 8210 /* Generate TC map for number of tc requested */ 8211 for (i = 0; i < num_tc; i++) 8212 enabled_tc |= BIT(i); 8213 8214 /* Requesting same TC configuration as already enabled */ 8215 if (enabled_tc == vsi->tc_config.enabled_tc && 8216 mode != TC_MQPRIO_MODE_CHANNEL) 8217 return 0; 8218 8219 /* Quiesce VSI queues */ 8220 i40e_quiesce_vsi(vsi); 8221 8222 if (!hw && !i40e_is_tc_mqprio_enabled(pf)) 8223 i40e_remove_queue_channels(vsi); 8224 8225 /* Configure VSI for enabled TCs */ 8226 ret = i40e_vsi_config_tc(vsi, enabled_tc); 8227 if (ret) { 8228 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 8229 vsi->seid); 8230 need_reset = true; 8231 goto exit; 8232 } else if (enabled_tc && 8233 (!is_power_of_2(vsi->tc_config.tc_info[0].qcount))) { 8234 netdev_info(netdev, 8235 "Failed to create channel. Override queues (%u) not power of 2\n", 8236 vsi->tc_config.tc_info[0].qcount); 8237 ret = -EINVAL; 8238 need_reset = true; 8239 goto exit; 8240 } 8241 8242 dev_info(&vsi->back->pdev->dev, 8243 "Setup channel (id:%u) utilizing num_queues %d\n", 8244 vsi->seid, vsi->tc_config.tc_info[0].qcount); 8245 8246 if (i40e_is_tc_mqprio_enabled(pf)) { 8247 if (vsi->mqprio_qopt.max_rate[0]) { 8248 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, 8249 vsi->mqprio_qopt.max_rate[0]); 8250 8251 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 8252 if (!ret) { 8253 u64 credits = max_tx_rate; 8254 8255 do_div(credits, I40E_BW_CREDIT_DIVISOR); 8256 dev_dbg(&vsi->back->pdev->dev, 8257 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 8258 max_tx_rate, 8259 credits, 8260 vsi->seid); 8261 } else { 8262 need_reset = true; 8263 goto exit; 8264 } 8265 } 8266 ret = i40e_configure_queue_channels(vsi); 8267 if (ret) { 8268 vsi->num_queue_pairs = old_queue_pairs; 8269 netdev_info(netdev, 8270 "Failed configuring queue channels\n"); 8271 need_reset = true; 8272 goto exit; 8273 } 8274 } 8275 8276 exit: 8277 /* Reset the configuration data to defaults, only TC0 is enabled */ 8278 if (need_reset) { 8279 i40e_vsi_set_default_tc_config(vsi); 8280 need_reset = false; 8281 } 8282 8283 /* Unquiesce VSI */ 8284 i40e_unquiesce_vsi(vsi); 8285 return ret; 8286 } 8287 8288 /** 8289 * i40e_set_cld_element - sets cloud filter element data 8290 * @filter: cloud filter rule 8291 * @cld: ptr to cloud filter element data 8292 * 8293 * This is helper function to copy data into cloud filter element 8294 **/ 8295 static inline void 8296 i40e_set_cld_element(struct i40e_cloud_filter *filter, 8297 struct i40e_aqc_cloud_filters_element_data *cld) 8298 { 8299 u32 ipa; 8300 int i; 8301 8302 memset(cld, 0, sizeof(*cld)); 8303 ether_addr_copy(cld->outer_mac, filter->dst_mac); 8304 ether_addr_copy(cld->inner_mac, filter->src_mac); 8305 8306 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 8307 return; 8308 8309 if (filter->n_proto == ETH_P_IPV6) { 8310 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 8311 for (i = 0; i < ARRAY_SIZE(filter->dst_ipv6); i++) { 8312 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 8313 8314 *(__le32 *)&cld->ipaddr.raw_v6.data[i * 2] = cpu_to_le32(ipa); 8315 } 8316 } else { 8317 ipa = be32_to_cpu(filter->dst_ipv4); 8318 8319 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 8320 } 8321 8322 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 8323 8324 /* tenant_id is not supported by FW now, once the support is enabled 8325 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 8326 */ 8327 if (filter->tenant_id) 8328 return; 8329 } 8330 8331 /** 8332 * i40e_add_del_cloud_filter - Add/del cloud filter 8333 * @vsi: pointer to VSI 8334 * @filter: cloud filter rule 8335 * @add: if true, add, if false, delete 8336 * 8337 * Add or delete a cloud filter for a specific flow spec. 8338 * Returns 0 if the filter were successfully added. 8339 **/ 8340 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 8341 struct i40e_cloud_filter *filter, bool add) 8342 { 8343 struct i40e_aqc_cloud_filters_element_data cld_filter; 8344 struct i40e_pf *pf = vsi->back; 8345 int ret; 8346 static const u16 flag_table[128] = { 8347 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 8348 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 8349 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 8350 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 8351 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 8352 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 8353 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 8354 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 8355 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 8356 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 8357 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 8358 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 8359 [I40E_CLOUD_FILTER_FLAGS_IIP] = 8360 I40E_AQC_ADD_CLOUD_FILTER_IIP, 8361 }; 8362 8363 if (filter->flags >= ARRAY_SIZE(flag_table)) 8364 return I40E_ERR_CONFIG; 8365 8366 memset(&cld_filter, 0, sizeof(cld_filter)); 8367 8368 /* copy element needed to add cloud filter from filter */ 8369 i40e_set_cld_element(filter, &cld_filter); 8370 8371 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 8372 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 8373 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 8374 8375 if (filter->n_proto == ETH_P_IPV6) 8376 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 8377 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 8378 else 8379 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 8380 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 8381 8382 if (add) 8383 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 8384 &cld_filter, 1); 8385 else 8386 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 8387 &cld_filter, 1); 8388 if (ret) 8389 dev_dbg(&pf->pdev->dev, 8390 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 8391 add ? "add" : "delete", filter->dst_port, ret, 8392 pf->hw.aq.asq_last_status); 8393 else 8394 dev_info(&pf->pdev->dev, 8395 "%s cloud filter for VSI: %d\n", 8396 add ? "Added" : "Deleted", filter->seid); 8397 return ret; 8398 } 8399 8400 /** 8401 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 8402 * @vsi: pointer to VSI 8403 * @filter: cloud filter rule 8404 * @add: if true, add, if false, delete 8405 * 8406 * Add or delete a cloud filter for a specific flow spec using big buffer. 8407 * Returns 0 if the filter were successfully added. 8408 **/ 8409 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 8410 struct i40e_cloud_filter *filter, 8411 bool add) 8412 { 8413 struct i40e_aqc_cloud_filters_element_bb cld_filter; 8414 struct i40e_pf *pf = vsi->back; 8415 int ret; 8416 8417 /* Both (src/dst) valid mac_addr are not supported */ 8418 if ((is_valid_ether_addr(filter->dst_mac) && 8419 is_valid_ether_addr(filter->src_mac)) || 8420 (is_multicast_ether_addr(filter->dst_mac) && 8421 is_multicast_ether_addr(filter->src_mac))) 8422 return -EOPNOTSUPP; 8423 8424 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 8425 * ports are not supported via big buffer now. 8426 */ 8427 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 8428 return -EOPNOTSUPP; 8429 8430 /* adding filter using src_port/src_ip is not supported at this stage */ 8431 if (filter->src_port || 8432 (filter->src_ipv4 && filter->n_proto != ETH_P_IPV6) || 8433 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 8434 return -EOPNOTSUPP; 8435 8436 memset(&cld_filter, 0, sizeof(cld_filter)); 8437 8438 /* copy element needed to add cloud filter from filter */ 8439 i40e_set_cld_element(filter, &cld_filter.element); 8440 8441 if (is_valid_ether_addr(filter->dst_mac) || 8442 is_valid_ether_addr(filter->src_mac) || 8443 is_multicast_ether_addr(filter->dst_mac) || 8444 is_multicast_ether_addr(filter->src_mac)) { 8445 /* MAC + IP : unsupported mode */ 8446 if (filter->dst_ipv4) 8447 return -EOPNOTSUPP; 8448 8449 /* since we validated that L4 port must be valid before 8450 * we get here, start with respective "flags" value 8451 * and update if vlan is present or not 8452 */ 8453 cld_filter.element.flags = 8454 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 8455 8456 if (filter->vlan_id) { 8457 cld_filter.element.flags = 8458 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 8459 } 8460 8461 } else if ((filter->dst_ipv4 && filter->n_proto != ETH_P_IPV6) || 8462 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 8463 cld_filter.element.flags = 8464 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 8465 if (filter->n_proto == ETH_P_IPV6) 8466 cld_filter.element.flags |= 8467 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 8468 else 8469 cld_filter.element.flags |= 8470 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 8471 } else { 8472 dev_err(&pf->pdev->dev, 8473 "either mac or ip has to be valid for cloud filter\n"); 8474 return -EINVAL; 8475 } 8476 8477 /* Now copy L4 port in Byte 6..7 in general fields */ 8478 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 8479 be16_to_cpu(filter->dst_port); 8480 8481 if (add) { 8482 /* Validate current device switch mode, change if necessary */ 8483 ret = i40e_validate_and_set_switch_mode(vsi); 8484 if (ret) { 8485 dev_err(&pf->pdev->dev, 8486 "failed to set switch mode, ret %d\n", 8487 ret); 8488 return ret; 8489 } 8490 8491 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 8492 &cld_filter, 1); 8493 } else { 8494 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 8495 &cld_filter, 1); 8496 } 8497 8498 if (ret) 8499 dev_dbg(&pf->pdev->dev, 8500 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 8501 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 8502 else 8503 dev_info(&pf->pdev->dev, 8504 "%s cloud filter for VSI: %d, L4 port: %d\n", 8505 add ? "add" : "delete", filter->seid, 8506 ntohs(filter->dst_port)); 8507 return ret; 8508 } 8509 8510 /** 8511 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 8512 * @vsi: Pointer to VSI 8513 * @f: Pointer to struct flow_cls_offload 8514 * @filter: Pointer to cloud filter structure 8515 * 8516 **/ 8517 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 8518 struct flow_cls_offload *f, 8519 struct i40e_cloud_filter *filter) 8520 { 8521 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 8522 struct flow_dissector *dissector = rule->match.dissector; 8523 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 8524 struct i40e_pf *pf = vsi->back; 8525 u8 field_flags = 0; 8526 8527 if (dissector->used_keys & 8528 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 8529 BIT(FLOW_DISSECTOR_KEY_BASIC) | 8530 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 8531 BIT(FLOW_DISSECTOR_KEY_VLAN) | 8532 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 8533 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 8534 BIT(FLOW_DISSECTOR_KEY_PORTS) | 8535 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 8536 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 8537 dissector->used_keys); 8538 return -EOPNOTSUPP; 8539 } 8540 8541 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 8542 struct flow_match_enc_keyid match; 8543 8544 flow_rule_match_enc_keyid(rule, &match); 8545 if (match.mask->keyid != 0) 8546 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 8547 8548 filter->tenant_id = be32_to_cpu(match.key->keyid); 8549 } 8550 8551 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 8552 struct flow_match_basic match; 8553 8554 flow_rule_match_basic(rule, &match); 8555 n_proto_key = ntohs(match.key->n_proto); 8556 n_proto_mask = ntohs(match.mask->n_proto); 8557 8558 if (n_proto_key == ETH_P_ALL) { 8559 n_proto_key = 0; 8560 n_proto_mask = 0; 8561 } 8562 filter->n_proto = n_proto_key & n_proto_mask; 8563 filter->ip_proto = match.key->ip_proto; 8564 } 8565 8566 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 8567 struct flow_match_eth_addrs match; 8568 8569 flow_rule_match_eth_addrs(rule, &match); 8570 8571 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 8572 if (!is_zero_ether_addr(match.mask->dst)) { 8573 if (is_broadcast_ether_addr(match.mask->dst)) { 8574 field_flags |= I40E_CLOUD_FIELD_OMAC; 8575 } else { 8576 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 8577 match.mask->dst); 8578 return I40E_ERR_CONFIG; 8579 } 8580 } 8581 8582 if (!is_zero_ether_addr(match.mask->src)) { 8583 if (is_broadcast_ether_addr(match.mask->src)) { 8584 field_flags |= I40E_CLOUD_FIELD_IMAC; 8585 } else { 8586 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 8587 match.mask->src); 8588 return I40E_ERR_CONFIG; 8589 } 8590 } 8591 ether_addr_copy(filter->dst_mac, match.key->dst); 8592 ether_addr_copy(filter->src_mac, match.key->src); 8593 } 8594 8595 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 8596 struct flow_match_vlan match; 8597 8598 flow_rule_match_vlan(rule, &match); 8599 if (match.mask->vlan_id) { 8600 if (match.mask->vlan_id == VLAN_VID_MASK) { 8601 field_flags |= I40E_CLOUD_FIELD_IVLAN; 8602 8603 } else { 8604 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 8605 match.mask->vlan_id); 8606 return I40E_ERR_CONFIG; 8607 } 8608 } 8609 8610 filter->vlan_id = cpu_to_be16(match.key->vlan_id); 8611 } 8612 8613 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 8614 struct flow_match_control match; 8615 8616 flow_rule_match_control(rule, &match); 8617 addr_type = match.key->addr_type; 8618 } 8619 8620 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 8621 struct flow_match_ipv4_addrs match; 8622 8623 flow_rule_match_ipv4_addrs(rule, &match); 8624 if (match.mask->dst) { 8625 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 8626 field_flags |= I40E_CLOUD_FIELD_IIP; 8627 } else { 8628 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 8629 &match.mask->dst); 8630 return I40E_ERR_CONFIG; 8631 } 8632 } 8633 8634 if (match.mask->src) { 8635 if (match.mask->src == cpu_to_be32(0xffffffff)) { 8636 field_flags |= I40E_CLOUD_FIELD_IIP; 8637 } else { 8638 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 8639 &match.mask->src); 8640 return I40E_ERR_CONFIG; 8641 } 8642 } 8643 8644 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 8645 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 8646 return I40E_ERR_CONFIG; 8647 } 8648 filter->dst_ipv4 = match.key->dst; 8649 filter->src_ipv4 = match.key->src; 8650 } 8651 8652 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 8653 struct flow_match_ipv6_addrs match; 8654 8655 flow_rule_match_ipv6_addrs(rule, &match); 8656 8657 /* src and dest IPV6 address should not be LOOPBACK 8658 * (0:0:0:0:0:0:0:1), which can be represented as ::1 8659 */ 8660 if (ipv6_addr_loopback(&match.key->dst) || 8661 ipv6_addr_loopback(&match.key->src)) { 8662 dev_err(&pf->pdev->dev, 8663 "Bad ipv6, addr is LOOPBACK\n"); 8664 return I40E_ERR_CONFIG; 8665 } 8666 if (!ipv6_addr_any(&match.mask->dst) || 8667 !ipv6_addr_any(&match.mask->src)) 8668 field_flags |= I40E_CLOUD_FIELD_IIP; 8669 8670 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32, 8671 sizeof(filter->src_ipv6)); 8672 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32, 8673 sizeof(filter->dst_ipv6)); 8674 } 8675 8676 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 8677 struct flow_match_ports match; 8678 8679 flow_rule_match_ports(rule, &match); 8680 if (match.mask->src) { 8681 if (match.mask->src == cpu_to_be16(0xffff)) { 8682 field_flags |= I40E_CLOUD_FIELD_IIP; 8683 } else { 8684 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 8685 be16_to_cpu(match.mask->src)); 8686 return I40E_ERR_CONFIG; 8687 } 8688 } 8689 8690 if (match.mask->dst) { 8691 if (match.mask->dst == cpu_to_be16(0xffff)) { 8692 field_flags |= I40E_CLOUD_FIELD_IIP; 8693 } else { 8694 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 8695 be16_to_cpu(match.mask->dst)); 8696 return I40E_ERR_CONFIG; 8697 } 8698 } 8699 8700 filter->dst_port = match.key->dst; 8701 filter->src_port = match.key->src; 8702 8703 switch (filter->ip_proto) { 8704 case IPPROTO_TCP: 8705 case IPPROTO_UDP: 8706 break; 8707 default: 8708 dev_err(&pf->pdev->dev, 8709 "Only UDP and TCP transport are supported\n"); 8710 return -EINVAL; 8711 } 8712 } 8713 filter->flags = field_flags; 8714 return 0; 8715 } 8716 8717 /** 8718 * i40e_handle_tclass: Forward to a traffic class on the device 8719 * @vsi: Pointer to VSI 8720 * @tc: traffic class index on the device 8721 * @filter: Pointer to cloud filter structure 8722 * 8723 **/ 8724 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 8725 struct i40e_cloud_filter *filter) 8726 { 8727 struct i40e_channel *ch, *ch_tmp; 8728 8729 /* direct to a traffic class on the same device */ 8730 if (tc == 0) { 8731 filter->seid = vsi->seid; 8732 return 0; 8733 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 8734 if (!filter->dst_port) { 8735 dev_err(&vsi->back->pdev->dev, 8736 "Specify destination port to direct to traffic class that is not default\n"); 8737 return -EINVAL; 8738 } 8739 if (list_empty(&vsi->ch_list)) 8740 return -EINVAL; 8741 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 8742 list) { 8743 if (ch->seid == vsi->tc_seid_map[tc]) 8744 filter->seid = ch->seid; 8745 } 8746 return 0; 8747 } 8748 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 8749 return -EINVAL; 8750 } 8751 8752 /** 8753 * i40e_configure_clsflower - Configure tc flower filters 8754 * @vsi: Pointer to VSI 8755 * @cls_flower: Pointer to struct flow_cls_offload 8756 * 8757 **/ 8758 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 8759 struct flow_cls_offload *cls_flower) 8760 { 8761 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 8762 struct i40e_cloud_filter *filter = NULL; 8763 struct i40e_pf *pf = vsi->back; 8764 int err = 0; 8765 8766 if (tc < 0) { 8767 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 8768 return -EOPNOTSUPP; 8769 } 8770 8771 if (!tc) { 8772 dev_err(&pf->pdev->dev, "Unable to add filter because of invalid destination"); 8773 return -EINVAL; 8774 } 8775 8776 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 8777 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 8778 return -EBUSY; 8779 8780 if (pf->fdir_pf_active_filters || 8781 (!hlist_empty(&pf->fdir_filter_list))) { 8782 dev_err(&vsi->back->pdev->dev, 8783 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 8784 return -EINVAL; 8785 } 8786 8787 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 8788 dev_err(&vsi->back->pdev->dev, 8789 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 8790 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8791 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8792 } 8793 8794 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 8795 if (!filter) 8796 return -ENOMEM; 8797 8798 filter->cookie = cls_flower->cookie; 8799 8800 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 8801 if (err < 0) 8802 goto err; 8803 8804 err = i40e_handle_tclass(vsi, tc, filter); 8805 if (err < 0) 8806 goto err; 8807 8808 /* Add cloud filter */ 8809 if (filter->dst_port) 8810 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 8811 else 8812 err = i40e_add_del_cloud_filter(vsi, filter, true); 8813 8814 if (err) { 8815 dev_err(&pf->pdev->dev, "Failed to add cloud filter, err %d\n", 8816 err); 8817 goto err; 8818 } 8819 8820 /* add filter to the ordered list */ 8821 INIT_HLIST_NODE(&filter->cloud_node); 8822 8823 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 8824 8825 pf->num_cloud_filters++; 8826 8827 return err; 8828 err: 8829 kfree(filter); 8830 return err; 8831 } 8832 8833 /** 8834 * i40e_find_cloud_filter - Find the could filter in the list 8835 * @vsi: Pointer to VSI 8836 * @cookie: filter specific cookie 8837 * 8838 **/ 8839 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 8840 unsigned long *cookie) 8841 { 8842 struct i40e_cloud_filter *filter = NULL; 8843 struct hlist_node *node2; 8844 8845 hlist_for_each_entry_safe(filter, node2, 8846 &vsi->back->cloud_filter_list, cloud_node) 8847 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 8848 return filter; 8849 return NULL; 8850 } 8851 8852 /** 8853 * i40e_delete_clsflower - Remove tc flower filters 8854 * @vsi: Pointer to VSI 8855 * @cls_flower: Pointer to struct flow_cls_offload 8856 * 8857 **/ 8858 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 8859 struct flow_cls_offload *cls_flower) 8860 { 8861 struct i40e_cloud_filter *filter = NULL; 8862 struct i40e_pf *pf = vsi->back; 8863 int err = 0; 8864 8865 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 8866 8867 if (!filter) 8868 return -EINVAL; 8869 8870 hash_del(&filter->cloud_node); 8871 8872 if (filter->dst_port) 8873 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 8874 else 8875 err = i40e_add_del_cloud_filter(vsi, filter, false); 8876 8877 kfree(filter); 8878 if (err) { 8879 dev_err(&pf->pdev->dev, 8880 "Failed to delete cloud filter, err %pe\n", 8881 ERR_PTR(err)); 8882 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 8883 } 8884 8885 pf->num_cloud_filters--; 8886 if (!pf->num_cloud_filters) 8887 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8888 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8889 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8890 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8891 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8892 } 8893 return 0; 8894 } 8895 8896 /** 8897 * i40e_setup_tc_cls_flower - flower classifier offloads 8898 * @np: net device to configure 8899 * @cls_flower: offload data 8900 **/ 8901 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 8902 struct flow_cls_offload *cls_flower) 8903 { 8904 struct i40e_vsi *vsi = np->vsi; 8905 8906 switch (cls_flower->command) { 8907 case FLOW_CLS_REPLACE: 8908 return i40e_configure_clsflower(vsi, cls_flower); 8909 case FLOW_CLS_DESTROY: 8910 return i40e_delete_clsflower(vsi, cls_flower); 8911 case FLOW_CLS_STATS: 8912 return -EOPNOTSUPP; 8913 default: 8914 return -EOPNOTSUPP; 8915 } 8916 } 8917 8918 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8919 void *cb_priv) 8920 { 8921 struct i40e_netdev_priv *np = cb_priv; 8922 8923 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 8924 return -EOPNOTSUPP; 8925 8926 switch (type) { 8927 case TC_SETUP_CLSFLOWER: 8928 return i40e_setup_tc_cls_flower(np, type_data); 8929 8930 default: 8931 return -EOPNOTSUPP; 8932 } 8933 } 8934 8935 static LIST_HEAD(i40e_block_cb_list); 8936 8937 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8938 void *type_data) 8939 { 8940 struct i40e_netdev_priv *np = netdev_priv(netdev); 8941 8942 switch (type) { 8943 case TC_SETUP_QDISC_MQPRIO: 8944 return i40e_setup_tc(netdev, type_data); 8945 case TC_SETUP_BLOCK: 8946 return flow_block_cb_setup_simple(type_data, 8947 &i40e_block_cb_list, 8948 i40e_setup_tc_block_cb, 8949 np, np, true); 8950 default: 8951 return -EOPNOTSUPP; 8952 } 8953 } 8954 8955 /** 8956 * i40e_open - Called when a network interface is made active 8957 * @netdev: network interface device structure 8958 * 8959 * The open entry point is called when a network interface is made 8960 * active by the system (IFF_UP). At this point all resources needed 8961 * for transmit and receive operations are allocated, the interrupt 8962 * handler is registered with the OS, the netdev watchdog subtask is 8963 * enabled, and the stack is notified that the interface is ready. 8964 * 8965 * Returns 0 on success, negative value on failure 8966 **/ 8967 int i40e_open(struct net_device *netdev) 8968 { 8969 struct i40e_netdev_priv *np = netdev_priv(netdev); 8970 struct i40e_vsi *vsi = np->vsi; 8971 struct i40e_pf *pf = vsi->back; 8972 int err; 8973 8974 /* disallow open during test or if eeprom is broken */ 8975 if (test_bit(__I40E_TESTING, pf->state) || 8976 test_bit(__I40E_BAD_EEPROM, pf->state)) 8977 return -EBUSY; 8978 8979 netif_carrier_off(netdev); 8980 8981 if (i40e_force_link_state(pf, true)) 8982 return -EAGAIN; 8983 8984 err = i40e_vsi_open(vsi); 8985 if (err) 8986 return err; 8987 8988 /* configure global TSO hardware offload settings */ 8989 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 8990 TCP_FLAG_FIN) >> 16); 8991 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 8992 TCP_FLAG_FIN | 8993 TCP_FLAG_CWR) >> 16); 8994 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 8995 udp_tunnel_get_rx_info(netdev); 8996 8997 return 0; 8998 } 8999 9000 /** 9001 * i40e_netif_set_realnum_tx_rx_queues - Update number of tx/rx queues 9002 * @vsi: vsi structure 9003 * 9004 * This updates netdev's number of tx/rx queues 9005 * 9006 * Returns status of setting tx/rx queues 9007 **/ 9008 static int i40e_netif_set_realnum_tx_rx_queues(struct i40e_vsi *vsi) 9009 { 9010 int ret; 9011 9012 ret = netif_set_real_num_rx_queues(vsi->netdev, 9013 vsi->num_queue_pairs); 9014 if (ret) 9015 return ret; 9016 9017 return netif_set_real_num_tx_queues(vsi->netdev, 9018 vsi->num_queue_pairs); 9019 } 9020 9021 /** 9022 * i40e_vsi_open - 9023 * @vsi: the VSI to open 9024 * 9025 * Finish initialization of the VSI. 9026 * 9027 * Returns 0 on success, negative value on failure 9028 * 9029 * Note: expects to be called while under rtnl_lock() 9030 **/ 9031 int i40e_vsi_open(struct i40e_vsi *vsi) 9032 { 9033 struct i40e_pf *pf = vsi->back; 9034 char int_name[I40E_INT_NAME_STR_LEN]; 9035 int err; 9036 9037 /* allocate descriptors */ 9038 err = i40e_vsi_setup_tx_resources(vsi); 9039 if (err) 9040 goto err_setup_tx; 9041 err = i40e_vsi_setup_rx_resources(vsi); 9042 if (err) 9043 goto err_setup_rx; 9044 9045 err = i40e_vsi_configure(vsi); 9046 if (err) 9047 goto err_setup_rx; 9048 9049 if (vsi->netdev) { 9050 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 9051 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 9052 err = i40e_vsi_request_irq(vsi, int_name); 9053 if (err) 9054 goto err_setup_rx; 9055 9056 /* Notify the stack of the actual queue counts. */ 9057 err = i40e_netif_set_realnum_tx_rx_queues(vsi); 9058 if (err) 9059 goto err_set_queues; 9060 9061 } else if (vsi->type == I40E_VSI_FDIR) { 9062 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 9063 dev_driver_string(&pf->pdev->dev), 9064 dev_name(&pf->pdev->dev)); 9065 err = i40e_vsi_request_irq(vsi, int_name); 9066 if (err) 9067 goto err_setup_rx; 9068 9069 } else { 9070 err = -EINVAL; 9071 goto err_setup_rx; 9072 } 9073 9074 err = i40e_up_complete(vsi); 9075 if (err) 9076 goto err_up_complete; 9077 9078 return 0; 9079 9080 err_up_complete: 9081 i40e_down(vsi); 9082 err_set_queues: 9083 i40e_vsi_free_irq(vsi); 9084 err_setup_rx: 9085 i40e_vsi_free_rx_resources(vsi); 9086 err_setup_tx: 9087 i40e_vsi_free_tx_resources(vsi); 9088 if (vsi == pf->vsi[pf->lan_vsi]) 9089 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 9090 9091 return err; 9092 } 9093 9094 /** 9095 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 9096 * @pf: Pointer to PF 9097 * 9098 * This function destroys the hlist where all the Flow Director 9099 * filters were saved. 9100 **/ 9101 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 9102 { 9103 struct i40e_fdir_filter *filter; 9104 struct i40e_flex_pit *pit_entry, *tmp; 9105 struct hlist_node *node2; 9106 9107 hlist_for_each_entry_safe(filter, node2, 9108 &pf->fdir_filter_list, fdir_node) { 9109 hlist_del(&filter->fdir_node); 9110 kfree(filter); 9111 } 9112 9113 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 9114 list_del(&pit_entry->list); 9115 kfree(pit_entry); 9116 } 9117 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 9118 9119 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 9120 list_del(&pit_entry->list); 9121 kfree(pit_entry); 9122 } 9123 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 9124 9125 pf->fdir_pf_active_filters = 0; 9126 i40e_reset_fdir_filter_cnt(pf); 9127 9128 /* Reprogram the default input set for TCP/IPv4 */ 9129 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 9130 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9131 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9132 9133 /* Reprogram the default input set for TCP/IPv6 */ 9134 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_TCP, 9135 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9136 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9137 9138 /* Reprogram the default input set for UDP/IPv4 */ 9139 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 9140 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9141 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9142 9143 /* Reprogram the default input set for UDP/IPv6 */ 9144 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_UDP, 9145 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9146 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9147 9148 /* Reprogram the default input set for SCTP/IPv4 */ 9149 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 9150 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9151 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9152 9153 /* Reprogram the default input set for SCTP/IPv6 */ 9154 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_SCTP, 9155 I40E_L3_V6_SRC_MASK | I40E_L3_V6_DST_MASK | 9156 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9157 9158 /* Reprogram the default input set for Other/IPv4 */ 9159 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 9160 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9161 9162 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 9163 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9164 9165 /* Reprogram the default input set for Other/IPv6 */ 9166 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV6_OTHER, 9167 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9168 9169 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV6, 9170 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 9171 } 9172 9173 /** 9174 * i40e_cloud_filter_exit - Cleans up the cloud filters 9175 * @pf: Pointer to PF 9176 * 9177 * This function destroys the hlist where all the cloud filters 9178 * were saved. 9179 **/ 9180 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 9181 { 9182 struct i40e_cloud_filter *cfilter; 9183 struct hlist_node *node; 9184 9185 hlist_for_each_entry_safe(cfilter, node, 9186 &pf->cloud_filter_list, cloud_node) { 9187 hlist_del(&cfilter->cloud_node); 9188 kfree(cfilter); 9189 } 9190 pf->num_cloud_filters = 0; 9191 9192 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 9193 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 9194 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 9195 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 9196 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 9197 } 9198 } 9199 9200 /** 9201 * i40e_close - Disables a network interface 9202 * @netdev: network interface device structure 9203 * 9204 * The close entry point is called when an interface is de-activated 9205 * by the OS. The hardware is still under the driver's control, but 9206 * this netdev interface is disabled. 9207 * 9208 * Returns 0, this is not allowed to fail 9209 **/ 9210 int i40e_close(struct net_device *netdev) 9211 { 9212 struct i40e_netdev_priv *np = netdev_priv(netdev); 9213 struct i40e_vsi *vsi = np->vsi; 9214 9215 i40e_vsi_close(vsi); 9216 9217 return 0; 9218 } 9219 9220 /** 9221 * i40e_do_reset - Start a PF or Core Reset sequence 9222 * @pf: board private structure 9223 * @reset_flags: which reset is requested 9224 * @lock_acquired: indicates whether or not the lock has been acquired 9225 * before this function was called. 9226 * 9227 * The essential difference in resets is that the PF Reset 9228 * doesn't clear the packet buffers, doesn't reset the PE 9229 * firmware, and doesn't bother the other PFs on the chip. 9230 **/ 9231 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 9232 { 9233 u32 val; 9234 9235 /* do the biggest reset indicated */ 9236 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 9237 9238 /* Request a Global Reset 9239 * 9240 * This will start the chip's countdown to the actual full 9241 * chip reset event, and a warning interrupt to be sent 9242 * to all PFs, including the requestor. Our handler 9243 * for the warning interrupt will deal with the shutdown 9244 * and recovery of the switch setup. 9245 */ 9246 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 9247 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 9248 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 9249 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 9250 9251 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 9252 9253 /* Request a Core Reset 9254 * 9255 * Same as Global Reset, except does *not* include the MAC/PHY 9256 */ 9257 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 9258 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 9259 val |= I40E_GLGEN_RTRIG_CORER_MASK; 9260 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 9261 i40e_flush(&pf->hw); 9262 9263 } else if (reset_flags & I40E_PF_RESET_FLAG) { 9264 9265 /* Request a PF Reset 9266 * 9267 * Resets only the PF-specific registers 9268 * 9269 * This goes directly to the tear-down and rebuild of 9270 * the switch, since we need to do all the recovery as 9271 * for the Core Reset. 9272 */ 9273 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 9274 i40e_handle_reset_warning(pf, lock_acquired); 9275 9276 } else if (reset_flags & I40E_PF_RESET_AND_REBUILD_FLAG) { 9277 /* Request a PF Reset 9278 * 9279 * Resets PF and reinitializes PFs VSI. 9280 */ 9281 i40e_prep_for_reset(pf); 9282 i40e_reset_and_rebuild(pf, true, lock_acquired); 9283 dev_info(&pf->pdev->dev, 9284 pf->flags & I40E_FLAG_DISABLE_FW_LLDP ? 9285 "FW LLDP is disabled\n" : 9286 "FW LLDP is enabled\n"); 9287 9288 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 9289 int v; 9290 9291 /* Find the VSI(s) that requested a re-init */ 9292 dev_info(&pf->pdev->dev, 9293 "VSI reinit requested\n"); 9294 for (v = 0; v < pf->num_alloc_vsi; v++) { 9295 struct i40e_vsi *vsi = pf->vsi[v]; 9296 9297 if (vsi != NULL && 9298 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 9299 vsi->state)) 9300 i40e_vsi_reinit_locked(pf->vsi[v]); 9301 } 9302 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 9303 int v; 9304 9305 /* Find the VSI(s) that needs to be brought down */ 9306 dev_info(&pf->pdev->dev, "VSI down requested\n"); 9307 for (v = 0; v < pf->num_alloc_vsi; v++) { 9308 struct i40e_vsi *vsi = pf->vsi[v]; 9309 9310 if (vsi != NULL && 9311 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 9312 vsi->state)) { 9313 set_bit(__I40E_VSI_DOWN, vsi->state); 9314 i40e_down(vsi); 9315 } 9316 } 9317 } else { 9318 dev_info(&pf->pdev->dev, 9319 "bad reset request 0x%08x\n", reset_flags); 9320 } 9321 } 9322 9323 #ifdef CONFIG_I40E_DCB 9324 /** 9325 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 9326 * @pf: board private structure 9327 * @old_cfg: current DCB config 9328 * @new_cfg: new DCB config 9329 **/ 9330 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 9331 struct i40e_dcbx_config *old_cfg, 9332 struct i40e_dcbx_config *new_cfg) 9333 { 9334 bool need_reconfig = false; 9335 9336 /* Check if ETS configuration has changed */ 9337 if (memcmp(&new_cfg->etscfg, 9338 &old_cfg->etscfg, 9339 sizeof(new_cfg->etscfg))) { 9340 /* If Priority Table has changed reconfig is needed */ 9341 if (memcmp(&new_cfg->etscfg.prioritytable, 9342 &old_cfg->etscfg.prioritytable, 9343 sizeof(new_cfg->etscfg.prioritytable))) { 9344 need_reconfig = true; 9345 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 9346 } 9347 9348 if (memcmp(&new_cfg->etscfg.tcbwtable, 9349 &old_cfg->etscfg.tcbwtable, 9350 sizeof(new_cfg->etscfg.tcbwtable))) 9351 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 9352 9353 if (memcmp(&new_cfg->etscfg.tsatable, 9354 &old_cfg->etscfg.tsatable, 9355 sizeof(new_cfg->etscfg.tsatable))) 9356 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 9357 } 9358 9359 /* Check if PFC configuration has changed */ 9360 if (memcmp(&new_cfg->pfc, 9361 &old_cfg->pfc, 9362 sizeof(new_cfg->pfc))) { 9363 need_reconfig = true; 9364 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 9365 } 9366 9367 /* Check if APP Table has changed */ 9368 if (memcmp(&new_cfg->app, 9369 &old_cfg->app, 9370 sizeof(new_cfg->app))) { 9371 need_reconfig = true; 9372 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 9373 } 9374 9375 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 9376 return need_reconfig; 9377 } 9378 9379 /** 9380 * i40e_handle_lldp_event - Handle LLDP Change MIB event 9381 * @pf: board private structure 9382 * @e: event info posted on ARQ 9383 **/ 9384 static int i40e_handle_lldp_event(struct i40e_pf *pf, 9385 struct i40e_arq_event_info *e) 9386 { 9387 struct i40e_aqc_lldp_get_mib *mib = 9388 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 9389 struct i40e_hw *hw = &pf->hw; 9390 struct i40e_dcbx_config tmp_dcbx_cfg; 9391 bool need_reconfig = false; 9392 int ret = 0; 9393 u8 type; 9394 9395 /* X710-T*L 2.5G and 5G speeds don't support DCB */ 9396 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 9397 (hw->phy.link_info.link_speed & 9398 ~(I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB)) && 9399 !(pf->flags & I40E_FLAG_DCB_CAPABLE)) 9400 /* let firmware decide if the DCB should be disabled */ 9401 pf->flags |= I40E_FLAG_DCB_CAPABLE; 9402 9403 /* Not DCB capable or capability disabled */ 9404 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 9405 return ret; 9406 9407 /* Ignore if event is not for Nearest Bridge */ 9408 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 9409 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 9410 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 9411 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 9412 return ret; 9413 9414 /* Check MIB Type and return if event for Remote MIB update */ 9415 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 9416 dev_dbg(&pf->pdev->dev, 9417 "LLDP event mib type %s\n", type ? "remote" : "local"); 9418 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 9419 /* Update the remote cached instance and return */ 9420 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 9421 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 9422 &hw->remote_dcbx_config); 9423 goto exit; 9424 } 9425 9426 /* Store the old configuration */ 9427 tmp_dcbx_cfg = hw->local_dcbx_config; 9428 9429 /* Reset the old DCBx configuration data */ 9430 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 9431 /* Get updated DCBX data from firmware */ 9432 ret = i40e_get_dcb_config(&pf->hw); 9433 if (ret) { 9434 /* X710-T*L 2.5G and 5G speeds don't support DCB */ 9435 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 9436 (hw->phy.link_info.link_speed & 9437 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) { 9438 dev_warn(&pf->pdev->dev, 9439 "DCB is not supported for X710-T*L 2.5/5G speeds\n"); 9440 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 9441 } else { 9442 dev_info(&pf->pdev->dev, 9443 "Failed querying DCB configuration data from firmware, err %pe aq_err %s\n", 9444 ERR_PTR(ret), 9445 i40e_aq_str(&pf->hw, 9446 pf->hw.aq.asq_last_status)); 9447 } 9448 goto exit; 9449 } 9450 9451 /* No change detected in DCBX configs */ 9452 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 9453 sizeof(tmp_dcbx_cfg))) { 9454 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 9455 goto exit; 9456 } 9457 9458 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 9459 &hw->local_dcbx_config); 9460 9461 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 9462 9463 if (!need_reconfig) 9464 goto exit; 9465 9466 /* Enable DCB tagging only when more than one TC */ 9467 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 9468 pf->flags |= I40E_FLAG_DCB_ENABLED; 9469 else 9470 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 9471 9472 set_bit(__I40E_PORT_SUSPENDED, pf->state); 9473 /* Reconfiguration needed quiesce all VSIs */ 9474 i40e_pf_quiesce_all_vsi(pf); 9475 9476 /* Changes in configuration update VEB/VSI */ 9477 i40e_dcb_reconfigure(pf); 9478 9479 ret = i40e_resume_port_tx(pf); 9480 9481 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 9482 /* In case of error no point in resuming VSIs */ 9483 if (ret) 9484 goto exit; 9485 9486 /* Wait for the PF's queues to be disabled */ 9487 ret = i40e_pf_wait_queues_disabled(pf); 9488 if (ret) { 9489 /* Schedule PF reset to recover */ 9490 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9491 i40e_service_event_schedule(pf); 9492 } else { 9493 i40e_pf_unquiesce_all_vsi(pf); 9494 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 9495 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 9496 } 9497 9498 exit: 9499 return ret; 9500 } 9501 #endif /* CONFIG_I40E_DCB */ 9502 9503 /** 9504 * i40e_do_reset_safe - Protected reset path for userland calls. 9505 * @pf: board private structure 9506 * @reset_flags: which reset is requested 9507 * 9508 **/ 9509 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 9510 { 9511 rtnl_lock(); 9512 i40e_do_reset(pf, reset_flags, true); 9513 rtnl_unlock(); 9514 } 9515 9516 /** 9517 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 9518 * @pf: board private structure 9519 * @e: event info posted on ARQ 9520 * 9521 * Handler for LAN Queue Overflow Event generated by the firmware for PF 9522 * and VF queues 9523 **/ 9524 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 9525 struct i40e_arq_event_info *e) 9526 { 9527 struct i40e_aqc_lan_overflow *data = 9528 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 9529 u32 queue = le32_to_cpu(data->prtdcb_rupto); 9530 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 9531 struct i40e_hw *hw = &pf->hw; 9532 struct i40e_vf *vf; 9533 u16 vf_id; 9534 9535 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 9536 queue, qtx_ctl); 9537 9538 /* Queue belongs to VF, find the VF and issue VF reset */ 9539 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 9540 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 9541 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 9542 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 9543 vf_id -= hw->func_caps.vf_base_id; 9544 vf = &pf->vf[vf_id]; 9545 i40e_vc_notify_vf_reset(vf); 9546 /* Allow VF to process pending reset notification */ 9547 msleep(20); 9548 i40e_reset_vf(vf, false); 9549 } 9550 } 9551 9552 /** 9553 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 9554 * @pf: board private structure 9555 **/ 9556 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 9557 { 9558 u32 val, fcnt_prog; 9559 9560 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 9561 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 9562 return fcnt_prog; 9563 } 9564 9565 /** 9566 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 9567 * @pf: board private structure 9568 **/ 9569 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 9570 { 9571 u32 val, fcnt_prog; 9572 9573 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 9574 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 9575 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 9576 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 9577 return fcnt_prog; 9578 } 9579 9580 /** 9581 * i40e_get_global_fd_count - Get total FD filters programmed on device 9582 * @pf: board private structure 9583 **/ 9584 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 9585 { 9586 u32 val, fcnt_prog; 9587 9588 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 9589 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 9590 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 9591 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 9592 return fcnt_prog; 9593 } 9594 9595 /** 9596 * i40e_reenable_fdir_sb - Restore FDir SB capability 9597 * @pf: board private structure 9598 **/ 9599 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 9600 { 9601 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 9602 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 9603 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9604 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 9605 } 9606 9607 /** 9608 * i40e_reenable_fdir_atr - Restore FDir ATR capability 9609 * @pf: board private structure 9610 **/ 9611 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 9612 { 9613 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 9614 /* ATR uses the same filtering logic as SB rules. It only 9615 * functions properly if the input set mask is at the default 9616 * settings. It is safe to restore the default input set 9617 * because there are no active TCPv4 filter rules. 9618 */ 9619 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 9620 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 9621 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 9622 9623 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 9624 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9625 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 9626 } 9627 } 9628 9629 /** 9630 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 9631 * @pf: board private structure 9632 * @filter: FDir filter to remove 9633 */ 9634 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 9635 struct i40e_fdir_filter *filter) 9636 { 9637 /* Update counters */ 9638 pf->fdir_pf_active_filters--; 9639 pf->fd_inv = 0; 9640 9641 switch (filter->flow_type) { 9642 case TCP_V4_FLOW: 9643 pf->fd_tcp4_filter_cnt--; 9644 break; 9645 case UDP_V4_FLOW: 9646 pf->fd_udp4_filter_cnt--; 9647 break; 9648 case SCTP_V4_FLOW: 9649 pf->fd_sctp4_filter_cnt--; 9650 break; 9651 case TCP_V6_FLOW: 9652 pf->fd_tcp6_filter_cnt--; 9653 break; 9654 case UDP_V6_FLOW: 9655 pf->fd_udp6_filter_cnt--; 9656 break; 9657 case SCTP_V6_FLOW: 9658 pf->fd_udp6_filter_cnt--; 9659 break; 9660 case IP_USER_FLOW: 9661 switch (filter->ipl4_proto) { 9662 case IPPROTO_TCP: 9663 pf->fd_tcp4_filter_cnt--; 9664 break; 9665 case IPPROTO_UDP: 9666 pf->fd_udp4_filter_cnt--; 9667 break; 9668 case IPPROTO_SCTP: 9669 pf->fd_sctp4_filter_cnt--; 9670 break; 9671 case IPPROTO_IP: 9672 pf->fd_ip4_filter_cnt--; 9673 break; 9674 } 9675 break; 9676 case IPV6_USER_FLOW: 9677 switch (filter->ipl4_proto) { 9678 case IPPROTO_TCP: 9679 pf->fd_tcp6_filter_cnt--; 9680 break; 9681 case IPPROTO_UDP: 9682 pf->fd_udp6_filter_cnt--; 9683 break; 9684 case IPPROTO_SCTP: 9685 pf->fd_sctp6_filter_cnt--; 9686 break; 9687 case IPPROTO_IP: 9688 pf->fd_ip6_filter_cnt--; 9689 break; 9690 } 9691 break; 9692 } 9693 9694 /* Remove the filter from the list and free memory */ 9695 hlist_del(&filter->fdir_node); 9696 kfree(filter); 9697 } 9698 9699 /** 9700 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 9701 * @pf: board private structure 9702 **/ 9703 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 9704 { 9705 struct i40e_fdir_filter *filter; 9706 u32 fcnt_prog, fcnt_avail; 9707 struct hlist_node *node; 9708 9709 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9710 return; 9711 9712 /* Check if we have enough room to re-enable FDir SB capability. */ 9713 fcnt_prog = i40e_get_global_fd_count(pf); 9714 fcnt_avail = pf->fdir_pf_filter_count; 9715 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 9716 (pf->fd_add_err == 0) || 9717 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 9718 i40e_reenable_fdir_sb(pf); 9719 9720 /* We should wait for even more space before re-enabling ATR. 9721 * Additionally, we cannot enable ATR as long as we still have TCP SB 9722 * rules active. 9723 */ 9724 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 9725 pf->fd_tcp4_filter_cnt == 0 && pf->fd_tcp6_filter_cnt == 0) 9726 i40e_reenable_fdir_atr(pf); 9727 9728 /* if hw had a problem adding a filter, delete it */ 9729 if (pf->fd_inv > 0) { 9730 hlist_for_each_entry_safe(filter, node, 9731 &pf->fdir_filter_list, fdir_node) 9732 if (filter->fd_id == pf->fd_inv) 9733 i40e_delete_invalid_filter(pf, filter); 9734 } 9735 } 9736 9737 #define I40E_MIN_FD_FLUSH_INTERVAL 10 9738 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 9739 /** 9740 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 9741 * @pf: board private structure 9742 **/ 9743 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 9744 { 9745 unsigned long min_flush_time; 9746 int flush_wait_retry = 50; 9747 bool disable_atr = false; 9748 int fd_room; 9749 int reg; 9750 9751 if (!time_after(jiffies, pf->fd_flush_timestamp + 9752 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 9753 return; 9754 9755 /* If the flush is happening too quick and we have mostly SB rules we 9756 * should not re-enable ATR for some time. 9757 */ 9758 min_flush_time = pf->fd_flush_timestamp + 9759 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 9760 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 9761 9762 if (!(time_after(jiffies, min_flush_time)) && 9763 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 9764 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9765 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 9766 disable_atr = true; 9767 } 9768 9769 pf->fd_flush_timestamp = jiffies; 9770 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 9771 /* flush all filters */ 9772 wr32(&pf->hw, I40E_PFQF_CTL_1, 9773 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 9774 i40e_flush(&pf->hw); 9775 pf->fd_flush_cnt++; 9776 pf->fd_add_err = 0; 9777 do { 9778 /* Check FD flush status every 5-6msec */ 9779 usleep_range(5000, 6000); 9780 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 9781 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 9782 break; 9783 } while (flush_wait_retry--); 9784 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 9785 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 9786 } else { 9787 /* replay sideband filters */ 9788 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 9789 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 9790 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 9791 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 9792 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9793 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 9794 } 9795 } 9796 9797 /** 9798 * i40e_get_current_atr_cnt - Get the count of total FD ATR filters programmed 9799 * @pf: board private structure 9800 **/ 9801 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 9802 { 9803 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 9804 } 9805 9806 /** 9807 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 9808 * @pf: board private structure 9809 **/ 9810 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 9811 { 9812 9813 /* if interface is down do nothing */ 9814 if (test_bit(__I40E_DOWN, pf->state)) 9815 return; 9816 9817 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9818 i40e_fdir_flush_and_replay(pf); 9819 9820 i40e_fdir_check_and_reenable(pf); 9821 9822 } 9823 9824 /** 9825 * i40e_vsi_link_event - notify VSI of a link event 9826 * @vsi: vsi to be notified 9827 * @link_up: link up or down 9828 **/ 9829 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 9830 { 9831 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 9832 return; 9833 9834 switch (vsi->type) { 9835 case I40E_VSI_MAIN: 9836 if (!vsi->netdev || !vsi->netdev_registered) 9837 break; 9838 9839 if (link_up) { 9840 netif_carrier_on(vsi->netdev); 9841 netif_tx_wake_all_queues(vsi->netdev); 9842 } else { 9843 netif_carrier_off(vsi->netdev); 9844 netif_tx_stop_all_queues(vsi->netdev); 9845 } 9846 break; 9847 9848 case I40E_VSI_SRIOV: 9849 case I40E_VSI_VMDQ2: 9850 case I40E_VSI_CTRL: 9851 case I40E_VSI_IWARP: 9852 case I40E_VSI_MIRROR: 9853 default: 9854 /* there is no notification for other VSIs */ 9855 break; 9856 } 9857 } 9858 9859 /** 9860 * i40e_veb_link_event - notify elements on the veb of a link event 9861 * @veb: veb to be notified 9862 * @link_up: link up or down 9863 **/ 9864 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 9865 { 9866 struct i40e_pf *pf; 9867 int i; 9868 9869 if (!veb || !veb->pf) 9870 return; 9871 pf = veb->pf; 9872 9873 /* depth first... */ 9874 for (i = 0; i < I40E_MAX_VEB; i++) 9875 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 9876 i40e_veb_link_event(pf->veb[i], link_up); 9877 9878 /* ... now the local VSIs */ 9879 for (i = 0; i < pf->num_alloc_vsi; i++) 9880 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 9881 i40e_vsi_link_event(pf->vsi[i], link_up); 9882 } 9883 9884 /** 9885 * i40e_link_event - Update netif_carrier status 9886 * @pf: board private structure 9887 **/ 9888 static void i40e_link_event(struct i40e_pf *pf) 9889 { 9890 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9891 u8 new_link_speed, old_link_speed; 9892 bool new_link, old_link; 9893 int status; 9894 #ifdef CONFIG_I40E_DCB 9895 int err; 9896 #endif /* CONFIG_I40E_DCB */ 9897 9898 /* set this to force the get_link_status call to refresh state */ 9899 pf->hw.phy.get_link_info = true; 9900 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 9901 status = i40e_get_link_status(&pf->hw, &new_link); 9902 9903 /* On success, disable temp link polling */ 9904 if (status == I40E_SUCCESS) { 9905 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9906 } else { 9907 /* Enable link polling temporarily until i40e_get_link_status 9908 * returns I40E_SUCCESS 9909 */ 9910 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9911 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 9912 status); 9913 return; 9914 } 9915 9916 old_link_speed = pf->hw.phy.link_info_old.link_speed; 9917 new_link_speed = pf->hw.phy.link_info.link_speed; 9918 9919 if (new_link == old_link && 9920 new_link_speed == old_link_speed && 9921 (test_bit(__I40E_VSI_DOWN, vsi->state) || 9922 new_link == netif_carrier_ok(vsi->netdev))) 9923 return; 9924 9925 i40e_print_link_message(vsi, new_link); 9926 9927 /* Notify the base of the switch tree connected to 9928 * the link. Floating VEBs are not notified. 9929 */ 9930 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 9931 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 9932 else 9933 i40e_vsi_link_event(vsi, new_link); 9934 9935 if (pf->vf) 9936 i40e_vc_notify_link_state(pf); 9937 9938 if (pf->flags & I40E_FLAG_PTP) 9939 i40e_ptp_set_increment(pf); 9940 #ifdef CONFIG_I40E_DCB 9941 if (new_link == old_link) 9942 return; 9943 /* Not SW DCB so firmware will take care of default settings */ 9944 if (pf->dcbx_cap & DCB_CAP_DCBX_LLD_MANAGED) 9945 return; 9946 9947 /* We cover here only link down, as after link up in case of SW DCB 9948 * SW LLDP agent will take care of setting it up 9949 */ 9950 if (!new_link) { 9951 dev_dbg(&pf->pdev->dev, "Reconfig DCB to single TC as result of Link Down\n"); 9952 memset(&pf->tmp_cfg, 0, sizeof(pf->tmp_cfg)); 9953 err = i40e_dcb_sw_default_config(pf); 9954 if (err) { 9955 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 9956 I40E_FLAG_DCB_ENABLED); 9957 } else { 9958 pf->dcbx_cap = DCB_CAP_DCBX_HOST | 9959 DCB_CAP_DCBX_VER_IEEE; 9960 pf->flags |= I40E_FLAG_DCB_CAPABLE; 9961 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 9962 } 9963 } 9964 #endif /* CONFIG_I40E_DCB */ 9965 } 9966 9967 /** 9968 * i40e_watchdog_subtask - periodic checks not using event driven response 9969 * @pf: board private structure 9970 **/ 9971 static void i40e_watchdog_subtask(struct i40e_pf *pf) 9972 { 9973 int i; 9974 9975 /* if interface is down do nothing */ 9976 if (test_bit(__I40E_DOWN, pf->state) || 9977 test_bit(__I40E_CONFIG_BUSY, pf->state)) 9978 return; 9979 9980 /* make sure we don't do these things too often */ 9981 if (time_before(jiffies, (pf->service_timer_previous + 9982 pf->service_timer_period))) 9983 return; 9984 pf->service_timer_previous = jiffies; 9985 9986 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 9987 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 9988 i40e_link_event(pf); 9989 9990 /* Update the stats for active netdevs so the network stack 9991 * can look at updated numbers whenever it cares to 9992 */ 9993 for (i = 0; i < pf->num_alloc_vsi; i++) 9994 if (pf->vsi[i] && pf->vsi[i]->netdev) 9995 i40e_update_stats(pf->vsi[i]); 9996 9997 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 9998 /* Update the stats for the active switching components */ 9999 for (i = 0; i < I40E_MAX_VEB; i++) 10000 if (pf->veb[i]) 10001 i40e_update_veb_stats(pf->veb[i]); 10002 } 10003 10004 i40e_ptp_rx_hang(pf); 10005 i40e_ptp_tx_hang(pf); 10006 } 10007 10008 /** 10009 * i40e_reset_subtask - Set up for resetting the device and driver 10010 * @pf: board private structure 10011 **/ 10012 static void i40e_reset_subtask(struct i40e_pf *pf) 10013 { 10014 u32 reset_flags = 0; 10015 10016 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 10017 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 10018 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 10019 } 10020 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 10021 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 10022 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 10023 } 10024 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 10025 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 10026 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 10027 } 10028 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 10029 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 10030 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 10031 } 10032 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 10033 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 10034 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 10035 } 10036 10037 /* If there's a recovery already waiting, it takes 10038 * precedence before starting a new reset sequence. 10039 */ 10040 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 10041 i40e_prep_for_reset(pf); 10042 i40e_reset(pf); 10043 i40e_rebuild(pf, false, false); 10044 } 10045 10046 /* If we're already down or resetting, just bail */ 10047 if (reset_flags && 10048 !test_bit(__I40E_DOWN, pf->state) && 10049 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 10050 i40e_do_reset(pf, reset_flags, false); 10051 } 10052 } 10053 10054 /** 10055 * i40e_handle_link_event - Handle link event 10056 * @pf: board private structure 10057 * @e: event info posted on ARQ 10058 **/ 10059 static void i40e_handle_link_event(struct i40e_pf *pf, 10060 struct i40e_arq_event_info *e) 10061 { 10062 struct i40e_aqc_get_link_status *status = 10063 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 10064 10065 /* Do a new status request to re-enable LSE reporting 10066 * and load new status information into the hw struct 10067 * This completely ignores any state information 10068 * in the ARQ event info, instead choosing to always 10069 * issue the AQ update link status command. 10070 */ 10071 i40e_link_event(pf); 10072 10073 /* Check if module meets thermal requirements */ 10074 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 10075 dev_err(&pf->pdev->dev, 10076 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 10077 dev_err(&pf->pdev->dev, 10078 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 10079 } else { 10080 /* check for unqualified module, if link is down, suppress 10081 * the message if link was forced to be down. 10082 */ 10083 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 10084 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 10085 (!(status->link_info & I40E_AQ_LINK_UP)) && 10086 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 10087 dev_err(&pf->pdev->dev, 10088 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 10089 dev_err(&pf->pdev->dev, 10090 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 10091 } 10092 } 10093 } 10094 10095 /** 10096 * i40e_clean_adminq_subtask - Clean the AdminQ rings 10097 * @pf: board private structure 10098 **/ 10099 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 10100 { 10101 struct i40e_arq_event_info event; 10102 struct i40e_hw *hw = &pf->hw; 10103 u16 pending, i = 0; 10104 u16 opcode; 10105 u32 oldval; 10106 int ret; 10107 u32 val; 10108 10109 /* Do not run clean AQ when PF reset fails */ 10110 if (test_bit(__I40E_RESET_FAILED, pf->state)) 10111 return; 10112 10113 /* check for error indications */ 10114 val = rd32(&pf->hw, pf->hw.aq.arq.len); 10115 oldval = val; 10116 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 10117 if (hw->debug_mask & I40E_DEBUG_AQ) 10118 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 10119 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 10120 } 10121 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 10122 if (hw->debug_mask & I40E_DEBUG_AQ) 10123 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 10124 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 10125 pf->arq_overflows++; 10126 } 10127 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 10128 if (hw->debug_mask & I40E_DEBUG_AQ) 10129 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 10130 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 10131 } 10132 if (oldval != val) 10133 wr32(&pf->hw, pf->hw.aq.arq.len, val); 10134 10135 val = rd32(&pf->hw, pf->hw.aq.asq.len); 10136 oldval = val; 10137 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 10138 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10139 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 10140 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 10141 } 10142 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 10143 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10144 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 10145 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 10146 } 10147 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 10148 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 10149 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 10150 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 10151 } 10152 if (oldval != val) 10153 wr32(&pf->hw, pf->hw.aq.asq.len, val); 10154 10155 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 10156 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 10157 if (!event.msg_buf) 10158 return; 10159 10160 do { 10161 ret = i40e_clean_arq_element(hw, &event, &pending); 10162 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 10163 break; 10164 else if (ret) { 10165 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 10166 break; 10167 } 10168 10169 opcode = le16_to_cpu(event.desc.opcode); 10170 switch (opcode) { 10171 10172 case i40e_aqc_opc_get_link_status: 10173 rtnl_lock(); 10174 i40e_handle_link_event(pf, &event); 10175 rtnl_unlock(); 10176 break; 10177 case i40e_aqc_opc_send_msg_to_pf: 10178 ret = i40e_vc_process_vf_msg(pf, 10179 le16_to_cpu(event.desc.retval), 10180 le32_to_cpu(event.desc.cookie_high), 10181 le32_to_cpu(event.desc.cookie_low), 10182 event.msg_buf, 10183 event.msg_len); 10184 break; 10185 case i40e_aqc_opc_lldp_update_mib: 10186 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 10187 #ifdef CONFIG_I40E_DCB 10188 rtnl_lock(); 10189 i40e_handle_lldp_event(pf, &event); 10190 rtnl_unlock(); 10191 #endif /* CONFIG_I40E_DCB */ 10192 break; 10193 case i40e_aqc_opc_event_lan_overflow: 10194 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 10195 i40e_handle_lan_overflow_event(pf, &event); 10196 break; 10197 case i40e_aqc_opc_send_msg_to_peer: 10198 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 10199 break; 10200 case i40e_aqc_opc_nvm_erase: 10201 case i40e_aqc_opc_nvm_update: 10202 case i40e_aqc_opc_oem_post_update: 10203 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 10204 "ARQ NVM operation 0x%04x completed\n", 10205 opcode); 10206 break; 10207 default: 10208 dev_info(&pf->pdev->dev, 10209 "ARQ: Unknown event 0x%04x ignored\n", 10210 opcode); 10211 break; 10212 } 10213 } while (i++ < pf->adminq_work_limit); 10214 10215 if (i < pf->adminq_work_limit) 10216 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 10217 10218 /* re-enable Admin queue interrupt cause */ 10219 val = rd32(hw, I40E_PFINT_ICR0_ENA); 10220 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 10221 wr32(hw, I40E_PFINT_ICR0_ENA, val); 10222 i40e_flush(hw); 10223 10224 kfree(event.msg_buf); 10225 } 10226 10227 /** 10228 * i40e_verify_eeprom - make sure eeprom is good to use 10229 * @pf: board private structure 10230 **/ 10231 static void i40e_verify_eeprom(struct i40e_pf *pf) 10232 { 10233 int err; 10234 10235 err = i40e_diag_eeprom_test(&pf->hw); 10236 if (err) { 10237 /* retry in case of garbage read */ 10238 err = i40e_diag_eeprom_test(&pf->hw); 10239 if (err) { 10240 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 10241 err); 10242 set_bit(__I40E_BAD_EEPROM, pf->state); 10243 } 10244 } 10245 10246 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 10247 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 10248 clear_bit(__I40E_BAD_EEPROM, pf->state); 10249 } 10250 } 10251 10252 /** 10253 * i40e_enable_pf_switch_lb 10254 * @pf: pointer to the PF structure 10255 * 10256 * enable switch loop back or die - no point in a return value 10257 **/ 10258 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 10259 { 10260 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10261 struct i40e_vsi_context ctxt; 10262 int ret; 10263 10264 ctxt.seid = pf->main_vsi_seid; 10265 ctxt.pf_num = pf->hw.pf_id; 10266 ctxt.vf_num = 0; 10267 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 10268 if (ret) { 10269 dev_info(&pf->pdev->dev, 10270 "couldn't get PF vsi config, err %pe aq_err %s\n", 10271 ERR_PTR(ret), 10272 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10273 return; 10274 } 10275 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 10276 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 10277 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10278 10279 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 10280 if (ret) { 10281 dev_info(&pf->pdev->dev, 10282 "update vsi switch failed, err %pe aq_err %s\n", 10283 ERR_PTR(ret), 10284 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10285 } 10286 } 10287 10288 /** 10289 * i40e_disable_pf_switch_lb 10290 * @pf: pointer to the PF structure 10291 * 10292 * disable switch loop back or die - no point in a return value 10293 **/ 10294 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 10295 { 10296 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10297 struct i40e_vsi_context ctxt; 10298 int ret; 10299 10300 ctxt.seid = pf->main_vsi_seid; 10301 ctxt.pf_num = pf->hw.pf_id; 10302 ctxt.vf_num = 0; 10303 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 10304 if (ret) { 10305 dev_info(&pf->pdev->dev, 10306 "couldn't get PF vsi config, err %pe aq_err %s\n", 10307 ERR_PTR(ret), 10308 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10309 return; 10310 } 10311 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 10312 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 10313 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10314 10315 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 10316 if (ret) { 10317 dev_info(&pf->pdev->dev, 10318 "update vsi switch failed, err %pe aq_err %s\n", 10319 ERR_PTR(ret), 10320 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10321 } 10322 } 10323 10324 /** 10325 * i40e_config_bridge_mode - Configure the HW bridge mode 10326 * @veb: pointer to the bridge instance 10327 * 10328 * Configure the loop back mode for the LAN VSI that is downlink to the 10329 * specified HW bridge instance. It is expected this function is called 10330 * when a new HW bridge is instantiated. 10331 **/ 10332 static void i40e_config_bridge_mode(struct i40e_veb *veb) 10333 { 10334 struct i40e_pf *pf = veb->pf; 10335 10336 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 10337 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 10338 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 10339 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 10340 i40e_disable_pf_switch_lb(pf); 10341 else 10342 i40e_enable_pf_switch_lb(pf); 10343 } 10344 10345 /** 10346 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 10347 * @veb: pointer to the VEB instance 10348 * 10349 * This is a recursive function that first builds the attached VSIs then 10350 * recurses in to build the next layer of VEB. We track the connections 10351 * through our own index numbers because the seid's from the HW could 10352 * change across the reset. 10353 **/ 10354 static int i40e_reconstitute_veb(struct i40e_veb *veb) 10355 { 10356 struct i40e_vsi *ctl_vsi = NULL; 10357 struct i40e_pf *pf = veb->pf; 10358 int v, veb_idx; 10359 int ret; 10360 10361 /* build VSI that owns this VEB, temporarily attached to base VEB */ 10362 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 10363 if (pf->vsi[v] && 10364 pf->vsi[v]->veb_idx == veb->idx && 10365 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 10366 ctl_vsi = pf->vsi[v]; 10367 break; 10368 } 10369 } 10370 if (!ctl_vsi) { 10371 dev_info(&pf->pdev->dev, 10372 "missing owner VSI for veb_idx %d\n", veb->idx); 10373 ret = -ENOENT; 10374 goto end_reconstitute; 10375 } 10376 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 10377 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 10378 ret = i40e_add_vsi(ctl_vsi); 10379 if (ret) { 10380 dev_info(&pf->pdev->dev, 10381 "rebuild of veb_idx %d owner VSI failed: %d\n", 10382 veb->idx, ret); 10383 goto end_reconstitute; 10384 } 10385 i40e_vsi_reset_stats(ctl_vsi); 10386 10387 /* create the VEB in the switch and move the VSI onto the VEB */ 10388 ret = i40e_add_veb(veb, ctl_vsi); 10389 if (ret) 10390 goto end_reconstitute; 10391 10392 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 10393 veb->bridge_mode = BRIDGE_MODE_VEB; 10394 else 10395 veb->bridge_mode = BRIDGE_MODE_VEPA; 10396 i40e_config_bridge_mode(veb); 10397 10398 /* create the remaining VSIs attached to this VEB */ 10399 for (v = 0; v < pf->num_alloc_vsi; v++) { 10400 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 10401 continue; 10402 10403 if (pf->vsi[v]->veb_idx == veb->idx) { 10404 struct i40e_vsi *vsi = pf->vsi[v]; 10405 10406 vsi->uplink_seid = veb->seid; 10407 ret = i40e_add_vsi(vsi); 10408 if (ret) { 10409 dev_info(&pf->pdev->dev, 10410 "rebuild of vsi_idx %d failed: %d\n", 10411 v, ret); 10412 goto end_reconstitute; 10413 } 10414 i40e_vsi_reset_stats(vsi); 10415 } 10416 } 10417 10418 /* create any VEBs attached to this VEB - RECURSION */ 10419 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 10420 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 10421 pf->veb[veb_idx]->uplink_seid = veb->seid; 10422 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 10423 if (ret) 10424 break; 10425 } 10426 } 10427 10428 end_reconstitute: 10429 return ret; 10430 } 10431 10432 /** 10433 * i40e_get_capabilities - get info about the HW 10434 * @pf: the PF struct 10435 * @list_type: AQ capability to be queried 10436 **/ 10437 static int i40e_get_capabilities(struct i40e_pf *pf, 10438 enum i40e_admin_queue_opc list_type) 10439 { 10440 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 10441 u16 data_size; 10442 int buf_len; 10443 int err; 10444 10445 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 10446 do { 10447 cap_buf = kzalloc(buf_len, GFP_KERNEL); 10448 if (!cap_buf) 10449 return -ENOMEM; 10450 10451 /* this loads the data into the hw struct for us */ 10452 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 10453 &data_size, list_type, 10454 NULL); 10455 /* data loaded, buffer no longer needed */ 10456 kfree(cap_buf); 10457 10458 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 10459 /* retry with a larger buffer */ 10460 buf_len = data_size; 10461 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK || err) { 10462 dev_info(&pf->pdev->dev, 10463 "capability discovery failed, err %pe aq_err %s\n", 10464 ERR_PTR(err), 10465 i40e_aq_str(&pf->hw, 10466 pf->hw.aq.asq_last_status)); 10467 return -ENODEV; 10468 } 10469 } while (err); 10470 10471 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 10472 if (list_type == i40e_aqc_opc_list_func_capabilities) { 10473 dev_info(&pf->pdev->dev, 10474 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 10475 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 10476 pf->hw.func_caps.num_msix_vectors, 10477 pf->hw.func_caps.num_msix_vectors_vf, 10478 pf->hw.func_caps.fd_filters_guaranteed, 10479 pf->hw.func_caps.fd_filters_best_effort, 10480 pf->hw.func_caps.num_tx_qp, 10481 pf->hw.func_caps.num_vsis); 10482 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 10483 dev_info(&pf->pdev->dev, 10484 "switch_mode=0x%04x, function_valid=0x%08x\n", 10485 pf->hw.dev_caps.switch_mode, 10486 pf->hw.dev_caps.valid_functions); 10487 dev_info(&pf->pdev->dev, 10488 "SR-IOV=%d, num_vfs for all function=%u\n", 10489 pf->hw.dev_caps.sr_iov_1_1, 10490 pf->hw.dev_caps.num_vfs); 10491 dev_info(&pf->pdev->dev, 10492 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 10493 pf->hw.dev_caps.num_vsis, 10494 pf->hw.dev_caps.num_rx_qp, 10495 pf->hw.dev_caps.num_tx_qp); 10496 } 10497 } 10498 if (list_type == i40e_aqc_opc_list_func_capabilities) { 10499 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 10500 + pf->hw.func_caps.num_vfs) 10501 if (pf->hw.revision_id == 0 && 10502 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 10503 dev_info(&pf->pdev->dev, 10504 "got num_vsis %d, setting num_vsis to %d\n", 10505 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 10506 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 10507 } 10508 } 10509 return 0; 10510 } 10511 10512 static int i40e_vsi_clear(struct i40e_vsi *vsi); 10513 10514 /** 10515 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 10516 * @pf: board private structure 10517 **/ 10518 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 10519 { 10520 struct i40e_vsi *vsi; 10521 10522 /* quick workaround for an NVM issue that leaves a critical register 10523 * uninitialized 10524 */ 10525 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 10526 static const u32 hkey[] = { 10527 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 10528 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 10529 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 10530 0x95b3a76d}; 10531 int i; 10532 10533 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 10534 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 10535 } 10536 10537 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 10538 return; 10539 10540 /* find existing VSI and see if it needs configuring */ 10541 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 10542 10543 /* create a new VSI if none exists */ 10544 if (!vsi) { 10545 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 10546 pf->vsi[pf->lan_vsi]->seid, 0); 10547 if (!vsi) { 10548 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 10549 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10550 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10551 return; 10552 } 10553 } 10554 10555 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 10556 } 10557 10558 /** 10559 * i40e_fdir_teardown - release the Flow Director resources 10560 * @pf: board private structure 10561 **/ 10562 static void i40e_fdir_teardown(struct i40e_pf *pf) 10563 { 10564 struct i40e_vsi *vsi; 10565 10566 i40e_fdir_filter_exit(pf); 10567 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 10568 if (vsi) 10569 i40e_vsi_release(vsi); 10570 } 10571 10572 /** 10573 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 10574 * @vsi: PF main vsi 10575 * @seid: seid of main or channel VSIs 10576 * 10577 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 10578 * existed before reset 10579 **/ 10580 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 10581 { 10582 struct i40e_cloud_filter *cfilter; 10583 struct i40e_pf *pf = vsi->back; 10584 struct hlist_node *node; 10585 int ret; 10586 10587 /* Add cloud filters back if they exist */ 10588 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 10589 cloud_node) { 10590 if (cfilter->seid != seid) 10591 continue; 10592 10593 if (cfilter->dst_port) 10594 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 10595 true); 10596 else 10597 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 10598 10599 if (ret) { 10600 dev_dbg(&pf->pdev->dev, 10601 "Failed to rebuild cloud filter, err %pe aq_err %s\n", 10602 ERR_PTR(ret), 10603 i40e_aq_str(&pf->hw, 10604 pf->hw.aq.asq_last_status)); 10605 return ret; 10606 } 10607 } 10608 return 0; 10609 } 10610 10611 /** 10612 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 10613 * @vsi: PF main vsi 10614 * 10615 * Rebuilds channel VSIs if they existed before reset 10616 **/ 10617 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 10618 { 10619 struct i40e_channel *ch, *ch_tmp; 10620 int ret; 10621 10622 if (list_empty(&vsi->ch_list)) 10623 return 0; 10624 10625 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 10626 if (!ch->initialized) 10627 break; 10628 /* Proceed with creation of channel (VMDq2) VSI */ 10629 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 10630 if (ret) { 10631 dev_info(&vsi->back->pdev->dev, 10632 "failed to rebuild channels using uplink_seid %u\n", 10633 vsi->uplink_seid); 10634 return ret; 10635 } 10636 /* Reconfigure TX queues using QTX_CTL register */ 10637 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 10638 if (ret) { 10639 dev_info(&vsi->back->pdev->dev, 10640 "failed to configure TX rings for channel %u\n", 10641 ch->seid); 10642 return ret; 10643 } 10644 /* update 'next_base_queue' */ 10645 vsi->next_base_queue = vsi->next_base_queue + 10646 ch->num_queue_pairs; 10647 if (ch->max_tx_rate) { 10648 u64 credits = ch->max_tx_rate; 10649 10650 if (i40e_set_bw_limit(vsi, ch->seid, 10651 ch->max_tx_rate)) 10652 return -EINVAL; 10653 10654 do_div(credits, I40E_BW_CREDIT_DIVISOR); 10655 dev_dbg(&vsi->back->pdev->dev, 10656 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 10657 ch->max_tx_rate, 10658 credits, 10659 ch->seid); 10660 } 10661 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 10662 if (ret) { 10663 dev_dbg(&vsi->back->pdev->dev, 10664 "Failed to rebuild cloud filters for channel VSI %u\n", 10665 ch->seid); 10666 return ret; 10667 } 10668 } 10669 return 0; 10670 } 10671 10672 /** 10673 * i40e_clean_xps_state - clean xps state for every tx_ring 10674 * @vsi: ptr to the VSI 10675 **/ 10676 static void i40e_clean_xps_state(struct i40e_vsi *vsi) 10677 { 10678 int i; 10679 10680 if (vsi->tx_rings) 10681 for (i = 0; i < vsi->num_queue_pairs; i++) 10682 if (vsi->tx_rings[i]) 10683 clear_bit(__I40E_TX_XPS_INIT_DONE, 10684 vsi->tx_rings[i]->state); 10685 } 10686 10687 /** 10688 * i40e_prep_for_reset - prep for the core to reset 10689 * @pf: board private structure 10690 * 10691 * Close up the VFs and other things in prep for PF Reset. 10692 **/ 10693 static void i40e_prep_for_reset(struct i40e_pf *pf) 10694 { 10695 struct i40e_hw *hw = &pf->hw; 10696 int ret = 0; 10697 u32 v; 10698 10699 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 10700 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 10701 return; 10702 if (i40e_check_asq_alive(&pf->hw)) 10703 i40e_vc_notify_reset(pf); 10704 10705 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 10706 10707 /* quiesce the VSIs and their queues that are not already DOWN */ 10708 i40e_pf_quiesce_all_vsi(pf); 10709 10710 for (v = 0; v < pf->num_alloc_vsi; v++) { 10711 if (pf->vsi[v]) { 10712 i40e_clean_xps_state(pf->vsi[v]); 10713 pf->vsi[v]->seid = 0; 10714 } 10715 } 10716 10717 i40e_shutdown_adminq(&pf->hw); 10718 10719 /* call shutdown HMC */ 10720 if (hw->hmc.hmc_obj) { 10721 ret = i40e_shutdown_lan_hmc(hw); 10722 if (ret) 10723 dev_warn(&pf->pdev->dev, 10724 "shutdown_lan_hmc failed: %d\n", ret); 10725 } 10726 10727 /* Save the current PTP time so that we can restore the time after the 10728 * reset completes. 10729 */ 10730 i40e_ptp_save_hw_time(pf); 10731 } 10732 10733 /** 10734 * i40e_send_version - update firmware with driver version 10735 * @pf: PF struct 10736 */ 10737 static void i40e_send_version(struct i40e_pf *pf) 10738 { 10739 struct i40e_driver_version dv; 10740 10741 dv.major_version = 0xff; 10742 dv.minor_version = 0xff; 10743 dv.build_version = 0xff; 10744 dv.subbuild_version = 0; 10745 strscpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); 10746 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 10747 } 10748 10749 /** 10750 * i40e_get_oem_version - get OEM specific version information 10751 * @hw: pointer to the hardware structure 10752 **/ 10753 static void i40e_get_oem_version(struct i40e_hw *hw) 10754 { 10755 u16 block_offset = 0xffff; 10756 u16 block_length = 0; 10757 u16 capabilities = 0; 10758 u16 gen_snap = 0; 10759 u16 release = 0; 10760 10761 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 10762 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 10763 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 10764 #define I40E_NVM_OEM_GEN_OFFSET 0x02 10765 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 10766 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 10767 #define I40E_NVM_OEM_LENGTH 3 10768 10769 /* Check if pointer to OEM version block is valid. */ 10770 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 10771 if (block_offset == 0xffff) 10772 return; 10773 10774 /* Check if OEM version block has correct length. */ 10775 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 10776 &block_length); 10777 if (block_length < I40E_NVM_OEM_LENGTH) 10778 return; 10779 10780 /* Check if OEM version format is as expected. */ 10781 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 10782 &capabilities); 10783 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 10784 return; 10785 10786 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 10787 &gen_snap); 10788 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 10789 &release); 10790 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 10791 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 10792 } 10793 10794 /** 10795 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 10796 * @pf: board private structure 10797 **/ 10798 static int i40e_reset(struct i40e_pf *pf) 10799 { 10800 struct i40e_hw *hw = &pf->hw; 10801 int ret; 10802 10803 ret = i40e_pf_reset(hw); 10804 if (ret) { 10805 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 10806 set_bit(__I40E_RESET_FAILED, pf->state); 10807 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 10808 } else { 10809 pf->pfr_count++; 10810 } 10811 return ret; 10812 } 10813 10814 /** 10815 * i40e_rebuild - rebuild using a saved config 10816 * @pf: board private structure 10817 * @reinit: if the Main VSI needs to re-initialized. 10818 * @lock_acquired: indicates whether or not the lock has been acquired 10819 * before this function was called. 10820 **/ 10821 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 10822 { 10823 const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf); 10824 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10825 struct i40e_hw *hw = &pf->hw; 10826 int ret; 10827 u32 val; 10828 int v; 10829 10830 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 10831 is_recovery_mode_reported) 10832 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev); 10833 10834 if (test_bit(__I40E_DOWN, pf->state) && 10835 !test_bit(__I40E_RECOVERY_MODE, pf->state)) 10836 goto clear_recovery; 10837 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 10838 10839 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 10840 ret = i40e_init_adminq(&pf->hw); 10841 if (ret) { 10842 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %pe aq_err %s\n", 10843 ERR_PTR(ret), 10844 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10845 goto clear_recovery; 10846 } 10847 i40e_get_oem_version(&pf->hw); 10848 10849 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) { 10850 /* The following delay is necessary for firmware update. */ 10851 mdelay(1000); 10852 } 10853 10854 /* re-verify the eeprom if we just had an EMP reset */ 10855 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 10856 i40e_verify_eeprom(pf); 10857 10858 /* if we are going out of or into recovery mode we have to act 10859 * accordingly with regard to resources initialization 10860 * and deinitialization 10861 */ 10862 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10863 if (i40e_get_capabilities(pf, 10864 i40e_aqc_opc_list_func_capabilities)) 10865 goto end_unlock; 10866 10867 if (is_recovery_mode_reported) { 10868 /* we're staying in recovery mode so we'll reinitialize 10869 * misc vector here 10870 */ 10871 if (i40e_setup_misc_vector_for_recovery_mode(pf)) 10872 goto end_unlock; 10873 } else { 10874 if (!lock_acquired) 10875 rtnl_lock(); 10876 /* we're going out of recovery mode so we'll free 10877 * the IRQ allocated specifically for recovery mode 10878 * and restore the interrupt scheme 10879 */ 10880 free_irq(pf->pdev->irq, pf); 10881 i40e_clear_interrupt_scheme(pf); 10882 if (i40e_restore_interrupt_scheme(pf)) 10883 goto end_unlock; 10884 } 10885 10886 /* tell the firmware that we're starting */ 10887 i40e_send_version(pf); 10888 10889 /* bail out in case recovery mode was detected, as there is 10890 * no need for further configuration. 10891 */ 10892 goto end_unlock; 10893 } 10894 10895 i40e_clear_pxe_mode(hw); 10896 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 10897 if (ret) 10898 goto end_core_reset; 10899 10900 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10901 hw->func_caps.num_rx_qp, 0, 0); 10902 if (ret) { 10903 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 10904 goto end_core_reset; 10905 } 10906 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10907 if (ret) { 10908 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 10909 goto end_core_reset; 10910 } 10911 10912 #ifdef CONFIG_I40E_DCB 10913 /* Enable FW to write a default DCB config on link-up 10914 * unless I40E_FLAG_TC_MQPRIO was enabled or DCB 10915 * is not supported with new link speed 10916 */ 10917 if (i40e_is_tc_mqprio_enabled(pf)) { 10918 i40e_aq_set_dcb_parameters(hw, false, NULL); 10919 } else { 10920 if (I40E_IS_X710TL_DEVICE(hw->device_id) && 10921 (hw->phy.link_info.link_speed & 10922 (I40E_LINK_SPEED_2_5GB | I40E_LINK_SPEED_5GB))) { 10923 i40e_aq_set_dcb_parameters(hw, false, NULL); 10924 dev_warn(&pf->pdev->dev, 10925 "DCB is not supported for X710-T*L 2.5/5G speeds\n"); 10926 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 10927 } else { 10928 i40e_aq_set_dcb_parameters(hw, true, NULL); 10929 ret = i40e_init_pf_dcb(pf); 10930 if (ret) { 10931 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", 10932 ret); 10933 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 10934 /* Continue without DCB enabled */ 10935 } 10936 } 10937 } 10938 10939 #endif /* CONFIG_I40E_DCB */ 10940 if (!lock_acquired) 10941 rtnl_lock(); 10942 ret = i40e_setup_pf_switch(pf, reinit, true); 10943 if (ret) 10944 goto end_unlock; 10945 10946 /* The driver only wants link up/down and module qualification 10947 * reports from firmware. Note the negative logic. 10948 */ 10949 ret = i40e_aq_set_phy_int_mask(&pf->hw, 10950 ~(I40E_AQ_EVENT_LINK_UPDOWN | 10951 I40E_AQ_EVENT_MEDIA_NA | 10952 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 10953 if (ret) 10954 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n", 10955 ERR_PTR(ret), 10956 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10957 10958 /* Rebuild the VSIs and VEBs that existed before reset. 10959 * They are still in our local switch element arrays, so only 10960 * need to rebuild the switch model in the HW. 10961 * 10962 * If there were VEBs but the reconstitution failed, we'll try 10963 * to recover minimal use by getting the basic PF VSI working. 10964 */ 10965 if (vsi->uplink_seid != pf->mac_seid) { 10966 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 10967 /* find the one VEB connected to the MAC, and find orphans */ 10968 for (v = 0; v < I40E_MAX_VEB; v++) { 10969 if (!pf->veb[v]) 10970 continue; 10971 10972 if (pf->veb[v]->uplink_seid == pf->mac_seid || 10973 pf->veb[v]->uplink_seid == 0) { 10974 ret = i40e_reconstitute_veb(pf->veb[v]); 10975 10976 if (!ret) 10977 continue; 10978 10979 /* If Main VEB failed, we're in deep doodoo, 10980 * so give up rebuilding the switch and set up 10981 * for minimal rebuild of PF VSI. 10982 * If orphan failed, we'll report the error 10983 * but try to keep going. 10984 */ 10985 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 10986 dev_info(&pf->pdev->dev, 10987 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 10988 ret); 10989 vsi->uplink_seid = pf->mac_seid; 10990 break; 10991 } else if (pf->veb[v]->uplink_seid == 0) { 10992 dev_info(&pf->pdev->dev, 10993 "rebuild of orphan VEB failed: %d\n", 10994 ret); 10995 } 10996 } 10997 } 10998 } 10999 11000 if (vsi->uplink_seid == pf->mac_seid) { 11001 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 11002 /* no VEB, so rebuild only the Main VSI */ 11003 ret = i40e_add_vsi(vsi); 11004 if (ret) { 11005 dev_info(&pf->pdev->dev, 11006 "rebuild of Main VSI failed: %d\n", ret); 11007 goto end_unlock; 11008 } 11009 } 11010 11011 if (vsi->mqprio_qopt.max_rate[0]) { 11012 u64 max_tx_rate = i40e_bw_bytes_to_mbits(vsi, 11013 vsi->mqprio_qopt.max_rate[0]); 11014 u64 credits = 0; 11015 11016 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 11017 if (ret) 11018 goto end_unlock; 11019 11020 credits = max_tx_rate; 11021 do_div(credits, I40E_BW_CREDIT_DIVISOR); 11022 dev_dbg(&vsi->back->pdev->dev, 11023 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 11024 max_tx_rate, 11025 credits, 11026 vsi->seid); 11027 } 11028 11029 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 11030 if (ret) 11031 goto end_unlock; 11032 11033 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 11034 * for this main VSI if they exist 11035 */ 11036 ret = i40e_rebuild_channels(vsi); 11037 if (ret) 11038 goto end_unlock; 11039 11040 /* Reconfigure hardware for allowing smaller MSS in the case 11041 * of TSO, so that we avoid the MDD being fired and causing 11042 * a reset in the case of small MSS+TSO. 11043 */ 11044 #define I40E_REG_MSS 0x000E64DC 11045 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 11046 #define I40E_64BYTE_MSS 0x400000 11047 val = rd32(hw, I40E_REG_MSS); 11048 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11049 val &= ~I40E_REG_MSS_MIN_MASK; 11050 val |= I40E_64BYTE_MSS; 11051 wr32(hw, I40E_REG_MSS, val); 11052 } 11053 11054 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 11055 msleep(75); 11056 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11057 if (ret) 11058 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n", 11059 ERR_PTR(ret), 11060 i40e_aq_str(&pf->hw, 11061 pf->hw.aq.asq_last_status)); 11062 } 11063 /* reinit the misc interrupt */ 11064 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11065 ret = i40e_setup_misc_vector(pf); 11066 if (ret) 11067 goto end_unlock; 11068 } 11069 11070 /* Add a filter to drop all Flow control frames from any VSI from being 11071 * transmitted. By doing so we stop a malicious VF from sending out 11072 * PAUSE or PFC frames and potentially controlling traffic for other 11073 * PF/VF VSIs. 11074 * The FW can still send Flow control frames if enabled. 11075 */ 11076 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11077 pf->main_vsi_seid); 11078 11079 /* restart the VSIs that were rebuilt and running before the reset */ 11080 i40e_pf_unquiesce_all_vsi(pf); 11081 11082 /* Release the RTNL lock before we start resetting VFs */ 11083 if (!lock_acquired) 11084 rtnl_unlock(); 11085 11086 /* Restore promiscuous settings */ 11087 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 11088 if (ret) 11089 dev_warn(&pf->pdev->dev, 11090 "Failed to restore promiscuous setting: %s, err %pe aq_err %s\n", 11091 pf->cur_promisc ? "on" : "off", 11092 ERR_PTR(ret), 11093 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11094 11095 i40e_reset_all_vfs(pf, true); 11096 11097 /* tell the firmware that we're starting */ 11098 i40e_send_version(pf); 11099 11100 /* We've already released the lock, so don't do it again */ 11101 goto end_core_reset; 11102 11103 end_unlock: 11104 if (!lock_acquired) 11105 rtnl_unlock(); 11106 end_core_reset: 11107 clear_bit(__I40E_RESET_FAILED, pf->state); 11108 clear_recovery: 11109 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 11110 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state); 11111 } 11112 11113 /** 11114 * i40e_reset_and_rebuild - reset and rebuild using a saved config 11115 * @pf: board private structure 11116 * @reinit: if the Main VSI needs to re-initialized. 11117 * @lock_acquired: indicates whether or not the lock has been acquired 11118 * before this function was called. 11119 **/ 11120 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 11121 bool lock_acquired) 11122 { 11123 int ret; 11124 11125 if (test_bit(__I40E_IN_REMOVE, pf->state)) 11126 return; 11127 /* Now we wait for GRST to settle out. 11128 * We don't have to delete the VEBs or VSIs from the hw switch 11129 * because the reset will make them disappear. 11130 */ 11131 ret = i40e_reset(pf); 11132 if (!ret) 11133 i40e_rebuild(pf, reinit, lock_acquired); 11134 } 11135 11136 /** 11137 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 11138 * @pf: board private structure 11139 * 11140 * Close up the VFs and other things in prep for a Core Reset, 11141 * then get ready to rebuild the world. 11142 * @lock_acquired: indicates whether or not the lock has been acquired 11143 * before this function was called. 11144 **/ 11145 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 11146 { 11147 i40e_prep_for_reset(pf); 11148 i40e_reset_and_rebuild(pf, false, lock_acquired); 11149 } 11150 11151 /** 11152 * i40e_handle_mdd_event 11153 * @pf: pointer to the PF structure 11154 * 11155 * Called from the MDD irq handler to identify possibly malicious vfs 11156 **/ 11157 static void i40e_handle_mdd_event(struct i40e_pf *pf) 11158 { 11159 struct i40e_hw *hw = &pf->hw; 11160 bool mdd_detected = false; 11161 struct i40e_vf *vf; 11162 u32 reg; 11163 int i; 11164 11165 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 11166 return; 11167 11168 /* find what triggered the MDD event */ 11169 reg = rd32(hw, I40E_GL_MDET_TX); 11170 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 11171 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 11172 I40E_GL_MDET_TX_PF_NUM_SHIFT; 11173 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 11174 I40E_GL_MDET_TX_VF_NUM_SHIFT; 11175 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 11176 I40E_GL_MDET_TX_EVENT_SHIFT; 11177 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 11178 I40E_GL_MDET_TX_QUEUE_SHIFT) - 11179 pf->hw.func_caps.base_queue; 11180 if (netif_msg_tx_err(pf)) 11181 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 11182 event, queue, pf_num, vf_num); 11183 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 11184 mdd_detected = true; 11185 } 11186 reg = rd32(hw, I40E_GL_MDET_RX); 11187 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 11188 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 11189 I40E_GL_MDET_RX_FUNCTION_SHIFT; 11190 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 11191 I40E_GL_MDET_RX_EVENT_SHIFT; 11192 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 11193 I40E_GL_MDET_RX_QUEUE_SHIFT) - 11194 pf->hw.func_caps.base_queue; 11195 if (netif_msg_rx_err(pf)) 11196 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 11197 event, queue, func); 11198 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 11199 mdd_detected = true; 11200 } 11201 11202 if (mdd_detected) { 11203 reg = rd32(hw, I40E_PF_MDET_TX); 11204 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 11205 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 11206 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n"); 11207 } 11208 reg = rd32(hw, I40E_PF_MDET_RX); 11209 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 11210 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 11211 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n"); 11212 } 11213 } 11214 11215 /* see if one of the VFs needs its hand slapped */ 11216 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 11217 vf = &(pf->vf[i]); 11218 reg = rd32(hw, I40E_VP_MDET_TX(i)); 11219 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 11220 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 11221 vf->num_mdd_events++; 11222 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 11223 i); 11224 dev_info(&pf->pdev->dev, 11225 "Use PF Control I/F to re-enable the VF\n"); 11226 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 11227 } 11228 11229 reg = rd32(hw, I40E_VP_MDET_RX(i)); 11230 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 11231 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 11232 vf->num_mdd_events++; 11233 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 11234 i); 11235 dev_info(&pf->pdev->dev, 11236 "Use PF Control I/F to re-enable the VF\n"); 11237 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 11238 } 11239 } 11240 11241 /* re-enable mdd interrupt cause */ 11242 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 11243 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 11244 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 11245 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 11246 i40e_flush(hw); 11247 } 11248 11249 /** 11250 * i40e_service_task - Run the driver's async subtasks 11251 * @work: pointer to work_struct containing our data 11252 **/ 11253 static void i40e_service_task(struct work_struct *work) 11254 { 11255 struct i40e_pf *pf = container_of(work, 11256 struct i40e_pf, 11257 service_task); 11258 unsigned long start_time = jiffies; 11259 11260 /* don't bother with service tasks if a reset is in progress */ 11261 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 11262 test_bit(__I40E_SUSPENDED, pf->state)) 11263 return; 11264 11265 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 11266 return; 11267 11268 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) { 11269 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]); 11270 i40e_sync_filters_subtask(pf); 11271 i40e_reset_subtask(pf); 11272 i40e_handle_mdd_event(pf); 11273 i40e_vc_process_vflr_event(pf); 11274 i40e_watchdog_subtask(pf); 11275 i40e_fdir_reinit_subtask(pf); 11276 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 11277 /* Client subtask will reopen next time through. */ 11278 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], 11279 true); 11280 } else { 11281 i40e_client_subtask(pf); 11282 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 11283 pf->state)) 11284 i40e_notify_client_of_l2_param_changes( 11285 pf->vsi[pf->lan_vsi]); 11286 } 11287 i40e_sync_filters_subtask(pf); 11288 } else { 11289 i40e_reset_subtask(pf); 11290 } 11291 11292 i40e_clean_adminq_subtask(pf); 11293 11294 /* flush memory to make sure state is correct before next watchdog */ 11295 smp_mb__before_atomic(); 11296 clear_bit(__I40E_SERVICE_SCHED, pf->state); 11297 11298 /* If the tasks have taken longer than one timer cycle or there 11299 * is more work to be done, reschedule the service task now 11300 * rather than wait for the timer to tick again. 11301 */ 11302 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 11303 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 11304 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 11305 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 11306 i40e_service_event_schedule(pf); 11307 } 11308 11309 /** 11310 * i40e_service_timer - timer callback 11311 * @t: timer list pointer 11312 **/ 11313 static void i40e_service_timer(struct timer_list *t) 11314 { 11315 struct i40e_pf *pf = from_timer(pf, t, service_timer); 11316 11317 mod_timer(&pf->service_timer, 11318 round_jiffies(jiffies + pf->service_timer_period)); 11319 i40e_service_event_schedule(pf); 11320 } 11321 11322 /** 11323 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 11324 * @vsi: the VSI being configured 11325 **/ 11326 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 11327 { 11328 struct i40e_pf *pf = vsi->back; 11329 11330 switch (vsi->type) { 11331 case I40E_VSI_MAIN: 11332 vsi->alloc_queue_pairs = pf->num_lan_qps; 11333 if (!vsi->num_tx_desc) 11334 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11335 I40E_REQ_DESCRIPTOR_MULTIPLE); 11336 if (!vsi->num_rx_desc) 11337 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11338 I40E_REQ_DESCRIPTOR_MULTIPLE); 11339 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 11340 vsi->num_q_vectors = pf->num_lan_msix; 11341 else 11342 vsi->num_q_vectors = 1; 11343 11344 break; 11345 11346 case I40E_VSI_FDIR: 11347 vsi->alloc_queue_pairs = 1; 11348 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT, 11349 I40E_REQ_DESCRIPTOR_MULTIPLE); 11350 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT, 11351 I40E_REQ_DESCRIPTOR_MULTIPLE); 11352 vsi->num_q_vectors = pf->num_fdsb_msix; 11353 break; 11354 11355 case I40E_VSI_VMDQ2: 11356 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 11357 if (!vsi->num_tx_desc) 11358 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11359 I40E_REQ_DESCRIPTOR_MULTIPLE); 11360 if (!vsi->num_rx_desc) 11361 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11362 I40E_REQ_DESCRIPTOR_MULTIPLE); 11363 vsi->num_q_vectors = pf->num_vmdq_msix; 11364 break; 11365 11366 case I40E_VSI_SRIOV: 11367 vsi->alloc_queue_pairs = pf->num_vf_qps; 11368 if (!vsi->num_tx_desc) 11369 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11370 I40E_REQ_DESCRIPTOR_MULTIPLE); 11371 if (!vsi->num_rx_desc) 11372 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 11373 I40E_REQ_DESCRIPTOR_MULTIPLE); 11374 break; 11375 11376 default: 11377 WARN_ON(1); 11378 return -ENODATA; 11379 } 11380 11381 if (is_kdump_kernel()) { 11382 vsi->num_tx_desc = I40E_MIN_NUM_DESCRIPTORS; 11383 vsi->num_rx_desc = I40E_MIN_NUM_DESCRIPTORS; 11384 } 11385 11386 return 0; 11387 } 11388 11389 /** 11390 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 11391 * @vsi: VSI pointer 11392 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 11393 * 11394 * On error: returns error code (negative) 11395 * On success: returns 0 11396 **/ 11397 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 11398 { 11399 struct i40e_ring **next_rings; 11400 int size; 11401 int ret = 0; 11402 11403 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 11404 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 11405 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 11406 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 11407 if (!vsi->tx_rings) 11408 return -ENOMEM; 11409 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 11410 if (i40e_enabled_xdp_vsi(vsi)) { 11411 vsi->xdp_rings = next_rings; 11412 next_rings += vsi->alloc_queue_pairs; 11413 } 11414 vsi->rx_rings = next_rings; 11415 11416 if (alloc_qvectors) { 11417 /* allocate memory for q_vector pointers */ 11418 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 11419 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 11420 if (!vsi->q_vectors) { 11421 ret = -ENOMEM; 11422 goto err_vectors; 11423 } 11424 } 11425 return ret; 11426 11427 err_vectors: 11428 kfree(vsi->tx_rings); 11429 return ret; 11430 } 11431 11432 /** 11433 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 11434 * @pf: board private structure 11435 * @type: type of VSI 11436 * 11437 * On error: returns error code (negative) 11438 * On success: returns vsi index in PF (positive) 11439 **/ 11440 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 11441 { 11442 int ret = -ENODEV; 11443 struct i40e_vsi *vsi; 11444 int vsi_idx; 11445 int i; 11446 11447 /* Need to protect the allocation of the VSIs at the PF level */ 11448 mutex_lock(&pf->switch_mutex); 11449 11450 /* VSI list may be fragmented if VSI creation/destruction has 11451 * been happening. We can afford to do a quick scan to look 11452 * for any free VSIs in the list. 11453 * 11454 * find next empty vsi slot, looping back around if necessary 11455 */ 11456 i = pf->next_vsi; 11457 while (i < pf->num_alloc_vsi && pf->vsi[i]) 11458 i++; 11459 if (i >= pf->num_alloc_vsi) { 11460 i = 0; 11461 while (i < pf->next_vsi && pf->vsi[i]) 11462 i++; 11463 } 11464 11465 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 11466 vsi_idx = i; /* Found one! */ 11467 } else { 11468 ret = -ENODEV; 11469 goto unlock_pf; /* out of VSI slots! */ 11470 } 11471 pf->next_vsi = ++i; 11472 11473 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 11474 if (!vsi) { 11475 ret = -ENOMEM; 11476 goto unlock_pf; 11477 } 11478 vsi->type = type; 11479 vsi->back = pf; 11480 set_bit(__I40E_VSI_DOWN, vsi->state); 11481 vsi->flags = 0; 11482 vsi->idx = vsi_idx; 11483 vsi->int_rate_limit = 0; 11484 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 11485 pf->rss_table_size : 64; 11486 vsi->netdev_registered = false; 11487 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 11488 hash_init(vsi->mac_filter_hash); 11489 vsi->irqs_ready = false; 11490 11491 if (type == I40E_VSI_MAIN) { 11492 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL); 11493 if (!vsi->af_xdp_zc_qps) 11494 goto err_rings; 11495 } 11496 11497 ret = i40e_set_num_rings_in_vsi(vsi); 11498 if (ret) 11499 goto err_rings; 11500 11501 ret = i40e_vsi_alloc_arrays(vsi, true); 11502 if (ret) 11503 goto err_rings; 11504 11505 /* Setup default MSIX irq handler for VSI */ 11506 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 11507 11508 /* Initialize VSI lock */ 11509 spin_lock_init(&vsi->mac_filter_hash_lock); 11510 pf->vsi[vsi_idx] = vsi; 11511 ret = vsi_idx; 11512 goto unlock_pf; 11513 11514 err_rings: 11515 bitmap_free(vsi->af_xdp_zc_qps); 11516 pf->next_vsi = i - 1; 11517 kfree(vsi); 11518 unlock_pf: 11519 mutex_unlock(&pf->switch_mutex); 11520 return ret; 11521 } 11522 11523 /** 11524 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 11525 * @vsi: VSI pointer 11526 * @free_qvectors: a bool to specify if q_vectors need to be freed. 11527 * 11528 * On error: returns error code (negative) 11529 * On success: returns 0 11530 **/ 11531 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 11532 { 11533 /* free the ring and vector containers */ 11534 if (free_qvectors) { 11535 kfree(vsi->q_vectors); 11536 vsi->q_vectors = NULL; 11537 } 11538 kfree(vsi->tx_rings); 11539 vsi->tx_rings = NULL; 11540 vsi->rx_rings = NULL; 11541 vsi->xdp_rings = NULL; 11542 } 11543 11544 /** 11545 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 11546 * and lookup table 11547 * @vsi: Pointer to VSI structure 11548 */ 11549 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 11550 { 11551 if (!vsi) 11552 return; 11553 11554 kfree(vsi->rss_hkey_user); 11555 vsi->rss_hkey_user = NULL; 11556 11557 kfree(vsi->rss_lut_user); 11558 vsi->rss_lut_user = NULL; 11559 } 11560 11561 /** 11562 * i40e_vsi_clear - Deallocate the VSI provided 11563 * @vsi: the VSI being un-configured 11564 **/ 11565 static int i40e_vsi_clear(struct i40e_vsi *vsi) 11566 { 11567 struct i40e_pf *pf; 11568 11569 if (!vsi) 11570 return 0; 11571 11572 if (!vsi->back) 11573 goto free_vsi; 11574 pf = vsi->back; 11575 11576 mutex_lock(&pf->switch_mutex); 11577 if (!pf->vsi[vsi->idx]) { 11578 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 11579 vsi->idx, vsi->idx, vsi->type); 11580 goto unlock_vsi; 11581 } 11582 11583 if (pf->vsi[vsi->idx] != vsi) { 11584 dev_err(&pf->pdev->dev, 11585 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 11586 pf->vsi[vsi->idx]->idx, 11587 pf->vsi[vsi->idx]->type, 11588 vsi->idx, vsi->type); 11589 goto unlock_vsi; 11590 } 11591 11592 /* updates the PF for this cleared vsi */ 11593 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 11594 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 11595 11596 bitmap_free(vsi->af_xdp_zc_qps); 11597 i40e_vsi_free_arrays(vsi, true); 11598 i40e_clear_rss_config_user(vsi); 11599 11600 pf->vsi[vsi->idx] = NULL; 11601 if (vsi->idx < pf->next_vsi) 11602 pf->next_vsi = vsi->idx; 11603 11604 unlock_vsi: 11605 mutex_unlock(&pf->switch_mutex); 11606 free_vsi: 11607 kfree(vsi); 11608 11609 return 0; 11610 } 11611 11612 /** 11613 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 11614 * @vsi: the VSI being cleaned 11615 **/ 11616 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 11617 { 11618 int i; 11619 11620 if (vsi->tx_rings && vsi->tx_rings[0]) { 11621 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 11622 kfree_rcu(vsi->tx_rings[i], rcu); 11623 WRITE_ONCE(vsi->tx_rings[i], NULL); 11624 WRITE_ONCE(vsi->rx_rings[i], NULL); 11625 if (vsi->xdp_rings) 11626 WRITE_ONCE(vsi->xdp_rings[i], NULL); 11627 } 11628 } 11629 } 11630 11631 /** 11632 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 11633 * @vsi: the VSI being configured 11634 **/ 11635 static int i40e_alloc_rings(struct i40e_vsi *vsi) 11636 { 11637 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 11638 struct i40e_pf *pf = vsi->back; 11639 struct i40e_ring *ring; 11640 11641 /* Set basic values in the rings to be used later during open() */ 11642 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 11643 /* allocate space for both Tx and Rx in one shot */ 11644 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 11645 if (!ring) 11646 goto err_out; 11647 11648 ring->queue_index = i; 11649 ring->reg_idx = vsi->base_queue + i; 11650 ring->ring_active = false; 11651 ring->vsi = vsi; 11652 ring->netdev = vsi->netdev; 11653 ring->dev = &pf->pdev->dev; 11654 ring->count = vsi->num_tx_desc; 11655 ring->size = 0; 11656 ring->dcb_tc = 0; 11657 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 11658 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 11659 ring->itr_setting = pf->tx_itr_default; 11660 WRITE_ONCE(vsi->tx_rings[i], ring++); 11661 11662 if (!i40e_enabled_xdp_vsi(vsi)) 11663 goto setup_rx; 11664 11665 ring->queue_index = vsi->alloc_queue_pairs + i; 11666 ring->reg_idx = vsi->base_queue + ring->queue_index; 11667 ring->ring_active = false; 11668 ring->vsi = vsi; 11669 ring->netdev = NULL; 11670 ring->dev = &pf->pdev->dev; 11671 ring->count = vsi->num_tx_desc; 11672 ring->size = 0; 11673 ring->dcb_tc = 0; 11674 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 11675 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 11676 set_ring_xdp(ring); 11677 ring->itr_setting = pf->tx_itr_default; 11678 WRITE_ONCE(vsi->xdp_rings[i], ring++); 11679 11680 setup_rx: 11681 ring->queue_index = i; 11682 ring->reg_idx = vsi->base_queue + i; 11683 ring->ring_active = false; 11684 ring->vsi = vsi; 11685 ring->netdev = vsi->netdev; 11686 ring->dev = &pf->pdev->dev; 11687 ring->count = vsi->num_rx_desc; 11688 ring->size = 0; 11689 ring->dcb_tc = 0; 11690 ring->itr_setting = pf->rx_itr_default; 11691 WRITE_ONCE(vsi->rx_rings[i], ring); 11692 } 11693 11694 return 0; 11695 11696 err_out: 11697 i40e_vsi_clear_rings(vsi); 11698 return -ENOMEM; 11699 } 11700 11701 /** 11702 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 11703 * @pf: board private structure 11704 * @vectors: the number of MSI-X vectors to request 11705 * 11706 * Returns the number of vectors reserved, or error 11707 **/ 11708 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 11709 { 11710 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 11711 I40E_MIN_MSIX, vectors); 11712 if (vectors < 0) { 11713 dev_info(&pf->pdev->dev, 11714 "MSI-X vector reservation failed: %d\n", vectors); 11715 vectors = 0; 11716 } 11717 11718 return vectors; 11719 } 11720 11721 /** 11722 * i40e_init_msix - Setup the MSIX capability 11723 * @pf: board private structure 11724 * 11725 * Work with the OS to set up the MSIX vectors needed. 11726 * 11727 * Returns the number of vectors reserved or negative on failure 11728 **/ 11729 static int i40e_init_msix(struct i40e_pf *pf) 11730 { 11731 struct i40e_hw *hw = &pf->hw; 11732 int cpus, extra_vectors; 11733 int vectors_left; 11734 int v_budget, i; 11735 int v_actual; 11736 int iwarp_requested = 0; 11737 11738 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 11739 return -ENODEV; 11740 11741 /* The number of vectors we'll request will be comprised of: 11742 * - Add 1 for "other" cause for Admin Queue events, etc. 11743 * - The number of LAN queue pairs 11744 * - Queues being used for RSS. 11745 * We don't need as many as max_rss_size vectors. 11746 * use rss_size instead in the calculation since that 11747 * is governed by number of cpus in the system. 11748 * - assumes symmetric Tx/Rx pairing 11749 * - The number of VMDq pairs 11750 * - The CPU count within the NUMA node if iWARP is enabled 11751 * Once we count this up, try the request. 11752 * 11753 * If we can't get what we want, we'll simplify to nearly nothing 11754 * and try again. If that still fails, we punt. 11755 */ 11756 vectors_left = hw->func_caps.num_msix_vectors; 11757 v_budget = 0; 11758 11759 /* reserve one vector for miscellaneous handler */ 11760 if (vectors_left) { 11761 v_budget++; 11762 vectors_left--; 11763 } 11764 11765 /* reserve some vectors for the main PF traffic queues. Initially we 11766 * only reserve at most 50% of the available vectors, in the case that 11767 * the number of online CPUs is large. This ensures that we can enable 11768 * extra features as well. Once we've enabled the other features, we 11769 * will use any remaining vectors to reach as close as we can to the 11770 * number of online CPUs. 11771 */ 11772 cpus = num_online_cpus(); 11773 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 11774 vectors_left -= pf->num_lan_msix; 11775 11776 /* reserve one vector for sideband flow director */ 11777 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11778 if (vectors_left) { 11779 pf->num_fdsb_msix = 1; 11780 v_budget++; 11781 vectors_left--; 11782 } else { 11783 pf->num_fdsb_msix = 0; 11784 } 11785 } 11786 11787 /* can we reserve enough for iWARP? */ 11788 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11789 iwarp_requested = pf->num_iwarp_msix; 11790 11791 if (!vectors_left) 11792 pf->num_iwarp_msix = 0; 11793 else if (vectors_left < pf->num_iwarp_msix) 11794 pf->num_iwarp_msix = 1; 11795 v_budget += pf->num_iwarp_msix; 11796 vectors_left -= pf->num_iwarp_msix; 11797 } 11798 11799 /* any vectors left over go for VMDq support */ 11800 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 11801 if (!vectors_left) { 11802 pf->num_vmdq_msix = 0; 11803 pf->num_vmdq_qps = 0; 11804 } else { 11805 int vmdq_vecs_wanted = 11806 pf->num_vmdq_vsis * pf->num_vmdq_qps; 11807 int vmdq_vecs = 11808 min_t(int, vectors_left, vmdq_vecs_wanted); 11809 11810 /* if we're short on vectors for what's desired, we limit 11811 * the queues per vmdq. If this is still more than are 11812 * available, the user will need to change the number of 11813 * queues/vectors used by the PF later with the ethtool 11814 * channels command 11815 */ 11816 if (vectors_left < vmdq_vecs_wanted) { 11817 pf->num_vmdq_qps = 1; 11818 vmdq_vecs_wanted = pf->num_vmdq_vsis; 11819 vmdq_vecs = min_t(int, 11820 vectors_left, 11821 vmdq_vecs_wanted); 11822 } 11823 pf->num_vmdq_msix = pf->num_vmdq_qps; 11824 11825 v_budget += vmdq_vecs; 11826 vectors_left -= vmdq_vecs; 11827 } 11828 } 11829 11830 /* On systems with a large number of SMP cores, we previously limited 11831 * the number of vectors for num_lan_msix to be at most 50% of the 11832 * available vectors, to allow for other features. Now, we add back 11833 * the remaining vectors. However, we ensure that the total 11834 * num_lan_msix will not exceed num_online_cpus(). To do this, we 11835 * calculate the number of vectors we can add without going over the 11836 * cap of CPUs. For systems with a small number of CPUs this will be 11837 * zero. 11838 */ 11839 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 11840 pf->num_lan_msix += extra_vectors; 11841 vectors_left -= extra_vectors; 11842 11843 WARN(vectors_left < 0, 11844 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 11845 11846 v_budget += pf->num_lan_msix; 11847 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 11848 GFP_KERNEL); 11849 if (!pf->msix_entries) 11850 return -ENOMEM; 11851 11852 for (i = 0; i < v_budget; i++) 11853 pf->msix_entries[i].entry = i; 11854 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 11855 11856 if (v_actual < I40E_MIN_MSIX) { 11857 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 11858 kfree(pf->msix_entries); 11859 pf->msix_entries = NULL; 11860 pci_disable_msix(pf->pdev); 11861 return -ENODEV; 11862 11863 } else if (v_actual == I40E_MIN_MSIX) { 11864 /* Adjust for minimal MSIX use */ 11865 pf->num_vmdq_vsis = 0; 11866 pf->num_vmdq_qps = 0; 11867 pf->num_lan_qps = 1; 11868 pf->num_lan_msix = 1; 11869 11870 } else if (v_actual != v_budget) { 11871 /* If we have limited resources, we will start with no vectors 11872 * for the special features and then allocate vectors to some 11873 * of these features based on the policy and at the end disable 11874 * the features that did not get any vectors. 11875 */ 11876 int vec; 11877 11878 dev_info(&pf->pdev->dev, 11879 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 11880 v_actual, v_budget); 11881 /* reserve the misc vector */ 11882 vec = v_actual - 1; 11883 11884 /* Scale vector usage down */ 11885 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 11886 pf->num_vmdq_vsis = 1; 11887 pf->num_vmdq_qps = 1; 11888 11889 /* partition out the remaining vectors */ 11890 switch (vec) { 11891 case 2: 11892 pf->num_lan_msix = 1; 11893 break; 11894 case 3: 11895 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11896 pf->num_lan_msix = 1; 11897 pf->num_iwarp_msix = 1; 11898 } else { 11899 pf->num_lan_msix = 2; 11900 } 11901 break; 11902 default: 11903 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11904 pf->num_iwarp_msix = min_t(int, (vec / 3), 11905 iwarp_requested); 11906 pf->num_vmdq_vsis = min_t(int, (vec / 3), 11907 I40E_DEFAULT_NUM_VMDQ_VSI); 11908 } else { 11909 pf->num_vmdq_vsis = min_t(int, (vec / 2), 11910 I40E_DEFAULT_NUM_VMDQ_VSI); 11911 } 11912 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11913 pf->num_fdsb_msix = 1; 11914 vec--; 11915 } 11916 pf->num_lan_msix = min_t(int, 11917 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 11918 pf->num_lan_msix); 11919 pf->num_lan_qps = pf->num_lan_msix; 11920 break; 11921 } 11922 } 11923 11924 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 11925 (pf->num_fdsb_msix == 0)) { 11926 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 11927 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 11928 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11929 } 11930 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 11931 (pf->num_vmdq_msix == 0)) { 11932 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 11933 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 11934 } 11935 11936 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 11937 (pf->num_iwarp_msix == 0)) { 11938 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 11939 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11940 } 11941 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 11942 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 11943 pf->num_lan_msix, 11944 pf->num_vmdq_msix * pf->num_vmdq_vsis, 11945 pf->num_fdsb_msix, 11946 pf->num_iwarp_msix); 11947 11948 return v_actual; 11949 } 11950 11951 /** 11952 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 11953 * @vsi: the VSI being configured 11954 * @v_idx: index of the vector in the vsi struct 11955 * 11956 * We allocate one q_vector. If allocation fails we return -ENOMEM. 11957 **/ 11958 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx) 11959 { 11960 struct i40e_q_vector *q_vector; 11961 11962 /* allocate q_vector */ 11963 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 11964 if (!q_vector) 11965 return -ENOMEM; 11966 11967 q_vector->vsi = vsi; 11968 q_vector->v_idx = v_idx; 11969 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 11970 11971 if (vsi->netdev) 11972 netif_napi_add(vsi->netdev, &q_vector->napi, i40e_napi_poll); 11973 11974 /* tie q_vector and vsi together */ 11975 vsi->q_vectors[v_idx] = q_vector; 11976 11977 return 0; 11978 } 11979 11980 /** 11981 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 11982 * @vsi: the VSI being configured 11983 * 11984 * We allocate one q_vector per queue interrupt. If allocation fails we 11985 * return -ENOMEM. 11986 **/ 11987 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 11988 { 11989 struct i40e_pf *pf = vsi->back; 11990 int err, v_idx, num_q_vectors; 11991 11992 /* if not MSIX, give the one vector only to the LAN VSI */ 11993 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 11994 num_q_vectors = vsi->num_q_vectors; 11995 else if (vsi == pf->vsi[pf->lan_vsi]) 11996 num_q_vectors = 1; 11997 else 11998 return -EINVAL; 11999 12000 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 12001 err = i40e_vsi_alloc_q_vector(vsi, v_idx); 12002 if (err) 12003 goto err_out; 12004 } 12005 12006 return 0; 12007 12008 err_out: 12009 while (v_idx--) 12010 i40e_free_q_vector(vsi, v_idx); 12011 12012 return err; 12013 } 12014 12015 /** 12016 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 12017 * @pf: board private structure to initialize 12018 **/ 12019 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 12020 { 12021 int vectors = 0; 12022 ssize_t size; 12023 12024 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 12025 vectors = i40e_init_msix(pf); 12026 if (vectors < 0) { 12027 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 12028 I40E_FLAG_IWARP_ENABLED | 12029 I40E_FLAG_RSS_ENABLED | 12030 I40E_FLAG_DCB_CAPABLE | 12031 I40E_FLAG_DCB_ENABLED | 12032 I40E_FLAG_SRIOV_ENABLED | 12033 I40E_FLAG_FD_SB_ENABLED | 12034 I40E_FLAG_FD_ATR_ENABLED | 12035 I40E_FLAG_VMDQ_ENABLED); 12036 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 12037 12038 /* rework the queue expectations without MSIX */ 12039 i40e_determine_queue_usage(pf); 12040 } 12041 } 12042 12043 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 12044 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 12045 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 12046 vectors = pci_enable_msi(pf->pdev); 12047 if (vectors < 0) { 12048 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 12049 vectors); 12050 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 12051 } 12052 vectors = 1; /* one MSI or Legacy vector */ 12053 } 12054 12055 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 12056 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 12057 12058 /* set up vector assignment tracking */ 12059 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 12060 pf->irq_pile = kzalloc(size, GFP_KERNEL); 12061 if (!pf->irq_pile) 12062 return -ENOMEM; 12063 12064 pf->irq_pile->num_entries = vectors; 12065 12066 /* track first vector for misc interrupts, ignore return */ 12067 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 12068 12069 return 0; 12070 } 12071 12072 /** 12073 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 12074 * @pf: private board data structure 12075 * 12076 * Restore the interrupt scheme that was cleared when we suspended the 12077 * device. This should be called during resume to re-allocate the q_vectors 12078 * and reacquire IRQs. 12079 */ 12080 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 12081 { 12082 int err, i; 12083 12084 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 12085 * scheme. We need to re-enabled them here in order to attempt to 12086 * re-acquire the MSI or MSI-X vectors 12087 */ 12088 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 12089 12090 err = i40e_init_interrupt_scheme(pf); 12091 if (err) 12092 return err; 12093 12094 /* Now that we've re-acquired IRQs, we need to remap the vectors and 12095 * rings together again. 12096 */ 12097 for (i = 0; i < pf->num_alloc_vsi; i++) { 12098 if (pf->vsi[i]) { 12099 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 12100 if (err) 12101 goto err_unwind; 12102 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 12103 } 12104 } 12105 12106 err = i40e_setup_misc_vector(pf); 12107 if (err) 12108 goto err_unwind; 12109 12110 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 12111 i40e_client_update_msix_info(pf); 12112 12113 return 0; 12114 12115 err_unwind: 12116 while (i--) { 12117 if (pf->vsi[i]) 12118 i40e_vsi_free_q_vectors(pf->vsi[i]); 12119 } 12120 12121 return err; 12122 } 12123 12124 /** 12125 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle 12126 * non queue events in recovery mode 12127 * @pf: board private structure 12128 * 12129 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage 12130 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode. 12131 * This is handled differently than in recovery mode since no Tx/Rx resources 12132 * are being allocated. 12133 **/ 12134 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf) 12135 { 12136 int err; 12137 12138 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 12139 err = i40e_setup_misc_vector(pf); 12140 12141 if (err) { 12142 dev_info(&pf->pdev->dev, 12143 "MSI-X misc vector request failed, error %d\n", 12144 err); 12145 return err; 12146 } 12147 } else { 12148 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED; 12149 12150 err = request_irq(pf->pdev->irq, i40e_intr, flags, 12151 pf->int_name, pf); 12152 12153 if (err) { 12154 dev_info(&pf->pdev->dev, 12155 "MSI/legacy misc vector request failed, error %d\n", 12156 err); 12157 return err; 12158 } 12159 i40e_enable_misc_int_causes(pf); 12160 i40e_irq_dynamic_enable_icr0(pf); 12161 } 12162 12163 return 0; 12164 } 12165 12166 /** 12167 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 12168 * @pf: board private structure 12169 * 12170 * This sets up the handler for MSIX 0, which is used to manage the 12171 * non-queue interrupts, e.g. AdminQ and errors. This is not used 12172 * when in MSI or Legacy interrupt mode. 12173 **/ 12174 static int i40e_setup_misc_vector(struct i40e_pf *pf) 12175 { 12176 struct i40e_hw *hw = &pf->hw; 12177 int err = 0; 12178 12179 /* Only request the IRQ once, the first time through. */ 12180 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 12181 err = request_irq(pf->msix_entries[0].vector, 12182 i40e_intr, 0, pf->int_name, pf); 12183 if (err) { 12184 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 12185 dev_info(&pf->pdev->dev, 12186 "request_irq for %s failed: %d\n", 12187 pf->int_name, err); 12188 return -EFAULT; 12189 } 12190 } 12191 12192 i40e_enable_misc_int_causes(pf); 12193 12194 /* associate no queues to the misc vector */ 12195 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 12196 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1); 12197 12198 i40e_flush(hw); 12199 12200 i40e_irq_dynamic_enable_icr0(pf); 12201 12202 return err; 12203 } 12204 12205 /** 12206 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 12207 * @vsi: Pointer to vsi structure 12208 * @seed: Buffter to store the hash keys 12209 * @lut: Buffer to store the lookup table entries 12210 * @lut_size: Size of buffer to store the lookup table entries 12211 * 12212 * Return 0 on success, negative on failure 12213 */ 12214 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 12215 u8 *lut, u16 lut_size) 12216 { 12217 struct i40e_pf *pf = vsi->back; 12218 struct i40e_hw *hw = &pf->hw; 12219 int ret = 0; 12220 12221 if (seed) { 12222 ret = i40e_aq_get_rss_key(hw, vsi->id, 12223 (struct i40e_aqc_get_set_rss_key_data *)seed); 12224 if (ret) { 12225 dev_info(&pf->pdev->dev, 12226 "Cannot get RSS key, err %pe aq_err %s\n", 12227 ERR_PTR(ret), 12228 i40e_aq_str(&pf->hw, 12229 pf->hw.aq.asq_last_status)); 12230 return ret; 12231 } 12232 } 12233 12234 if (lut) { 12235 bool pf_lut = vsi->type == I40E_VSI_MAIN; 12236 12237 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 12238 if (ret) { 12239 dev_info(&pf->pdev->dev, 12240 "Cannot get RSS lut, err %pe aq_err %s\n", 12241 ERR_PTR(ret), 12242 i40e_aq_str(&pf->hw, 12243 pf->hw.aq.asq_last_status)); 12244 return ret; 12245 } 12246 } 12247 12248 return ret; 12249 } 12250 12251 /** 12252 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 12253 * @vsi: Pointer to vsi structure 12254 * @seed: RSS hash seed 12255 * @lut: Lookup table 12256 * @lut_size: Lookup table size 12257 * 12258 * Returns 0 on success, negative on failure 12259 **/ 12260 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 12261 const u8 *lut, u16 lut_size) 12262 { 12263 struct i40e_pf *pf = vsi->back; 12264 struct i40e_hw *hw = &pf->hw; 12265 u16 vf_id = vsi->vf_id; 12266 u8 i; 12267 12268 /* Fill out hash function seed */ 12269 if (seed) { 12270 u32 *seed_dw = (u32 *)seed; 12271 12272 if (vsi->type == I40E_VSI_MAIN) { 12273 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 12274 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 12275 } else if (vsi->type == I40E_VSI_SRIOV) { 12276 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 12277 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 12278 } else { 12279 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 12280 } 12281 } 12282 12283 if (lut) { 12284 u32 *lut_dw = (u32 *)lut; 12285 12286 if (vsi->type == I40E_VSI_MAIN) { 12287 if (lut_size != I40E_HLUT_ARRAY_SIZE) 12288 return -EINVAL; 12289 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12290 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 12291 } else if (vsi->type == I40E_VSI_SRIOV) { 12292 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 12293 return -EINVAL; 12294 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12295 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 12296 } else { 12297 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12298 } 12299 } 12300 i40e_flush(hw); 12301 12302 return 0; 12303 } 12304 12305 /** 12306 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 12307 * @vsi: Pointer to VSI structure 12308 * @seed: Buffer to store the keys 12309 * @lut: Buffer to store the lookup table entries 12310 * @lut_size: Size of buffer to store the lookup table entries 12311 * 12312 * Returns 0 on success, negative on failure 12313 */ 12314 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 12315 u8 *lut, u16 lut_size) 12316 { 12317 struct i40e_pf *pf = vsi->back; 12318 struct i40e_hw *hw = &pf->hw; 12319 u16 i; 12320 12321 if (seed) { 12322 u32 *seed_dw = (u32 *)seed; 12323 12324 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 12325 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 12326 } 12327 if (lut) { 12328 u32 *lut_dw = (u32 *)lut; 12329 12330 if (lut_size != I40E_HLUT_ARRAY_SIZE) 12331 return -EINVAL; 12332 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12333 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 12334 } 12335 12336 return 0; 12337 } 12338 12339 /** 12340 * i40e_config_rss - Configure RSS keys and lut 12341 * @vsi: Pointer to VSI structure 12342 * @seed: RSS hash seed 12343 * @lut: Lookup table 12344 * @lut_size: Lookup table size 12345 * 12346 * Returns 0 on success, negative on failure 12347 */ 12348 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 12349 { 12350 struct i40e_pf *pf = vsi->back; 12351 12352 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 12353 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 12354 else 12355 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 12356 } 12357 12358 /** 12359 * i40e_get_rss - Get RSS keys and lut 12360 * @vsi: Pointer to VSI structure 12361 * @seed: Buffer to store the keys 12362 * @lut: Buffer to store the lookup table entries 12363 * @lut_size: Size of buffer to store the lookup table entries 12364 * 12365 * Returns 0 on success, negative on failure 12366 */ 12367 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 12368 { 12369 struct i40e_pf *pf = vsi->back; 12370 12371 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 12372 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 12373 else 12374 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 12375 } 12376 12377 /** 12378 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 12379 * @pf: Pointer to board private structure 12380 * @lut: Lookup table 12381 * @rss_table_size: Lookup table size 12382 * @rss_size: Range of queue number for hashing 12383 */ 12384 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 12385 u16 rss_table_size, u16 rss_size) 12386 { 12387 u16 i; 12388 12389 for (i = 0; i < rss_table_size; i++) 12390 lut[i] = i % rss_size; 12391 } 12392 12393 /** 12394 * i40e_pf_config_rss - Prepare for RSS if used 12395 * @pf: board private structure 12396 **/ 12397 static int i40e_pf_config_rss(struct i40e_pf *pf) 12398 { 12399 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 12400 u8 seed[I40E_HKEY_ARRAY_SIZE]; 12401 u8 *lut; 12402 struct i40e_hw *hw = &pf->hw; 12403 u32 reg_val; 12404 u64 hena; 12405 int ret; 12406 12407 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 12408 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 12409 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 12410 hena |= i40e_pf_get_default_rss_hena(pf); 12411 12412 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 12413 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 12414 12415 /* Determine the RSS table size based on the hardware capabilities */ 12416 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 12417 reg_val = (pf->rss_table_size == 512) ? 12418 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 12419 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 12420 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 12421 12422 /* Determine the RSS size of the VSI */ 12423 if (!vsi->rss_size) { 12424 u16 qcount; 12425 /* If the firmware does something weird during VSI init, we 12426 * could end up with zero TCs. Check for that to avoid 12427 * divide-by-zero. It probably won't pass traffic, but it also 12428 * won't panic. 12429 */ 12430 qcount = vsi->num_queue_pairs / 12431 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 12432 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 12433 } 12434 if (!vsi->rss_size) 12435 return -EINVAL; 12436 12437 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 12438 if (!lut) 12439 return -ENOMEM; 12440 12441 /* Use user configured lut if there is one, otherwise use default */ 12442 if (vsi->rss_lut_user) 12443 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 12444 else 12445 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 12446 12447 /* Use user configured hash key if there is one, otherwise 12448 * use default. 12449 */ 12450 if (vsi->rss_hkey_user) 12451 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 12452 else 12453 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 12454 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 12455 kfree(lut); 12456 12457 return ret; 12458 } 12459 12460 /** 12461 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 12462 * @pf: board private structure 12463 * @queue_count: the requested queue count for rss. 12464 * 12465 * returns 0 if rss is not enabled, if enabled returns the final rss queue 12466 * count which may be different from the requested queue count. 12467 * Note: expects to be called while under rtnl_lock() 12468 **/ 12469 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 12470 { 12471 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 12472 int new_rss_size; 12473 12474 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 12475 return 0; 12476 12477 queue_count = min_t(int, queue_count, num_online_cpus()); 12478 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 12479 12480 if (queue_count != vsi->num_queue_pairs) { 12481 u16 qcount; 12482 12483 vsi->req_queue_pairs = queue_count; 12484 i40e_prep_for_reset(pf); 12485 if (test_bit(__I40E_IN_REMOVE, pf->state)) 12486 return pf->alloc_rss_size; 12487 12488 pf->alloc_rss_size = new_rss_size; 12489 12490 i40e_reset_and_rebuild(pf, true, true); 12491 12492 /* Discard the user configured hash keys and lut, if less 12493 * queues are enabled. 12494 */ 12495 if (queue_count < vsi->rss_size) { 12496 i40e_clear_rss_config_user(vsi); 12497 dev_dbg(&pf->pdev->dev, 12498 "discard user configured hash keys and lut\n"); 12499 } 12500 12501 /* Reset vsi->rss_size, as number of enabled queues changed */ 12502 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 12503 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 12504 12505 i40e_pf_config_rss(pf); 12506 } 12507 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 12508 vsi->req_queue_pairs, pf->rss_size_max); 12509 return pf->alloc_rss_size; 12510 } 12511 12512 /** 12513 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 12514 * @pf: board private structure 12515 **/ 12516 int i40e_get_partition_bw_setting(struct i40e_pf *pf) 12517 { 12518 bool min_valid, max_valid; 12519 u32 max_bw, min_bw; 12520 int status; 12521 12522 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 12523 &min_valid, &max_valid); 12524 12525 if (!status) { 12526 if (min_valid) 12527 pf->min_bw = min_bw; 12528 if (max_valid) 12529 pf->max_bw = max_bw; 12530 } 12531 12532 return status; 12533 } 12534 12535 /** 12536 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 12537 * @pf: board private structure 12538 **/ 12539 int i40e_set_partition_bw_setting(struct i40e_pf *pf) 12540 { 12541 struct i40e_aqc_configure_partition_bw_data bw_data; 12542 int status; 12543 12544 memset(&bw_data, 0, sizeof(bw_data)); 12545 12546 /* Set the valid bit for this PF */ 12547 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 12548 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 12549 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 12550 12551 /* Set the new bandwidths */ 12552 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 12553 12554 return status; 12555 } 12556 12557 /** 12558 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 12559 * @pf: board private structure 12560 **/ 12561 int i40e_commit_partition_bw_setting(struct i40e_pf *pf) 12562 { 12563 /* Commit temporary BW setting to permanent NVM image */ 12564 enum i40e_admin_queue_err last_aq_status; 12565 u16 nvm_word; 12566 int ret; 12567 12568 if (pf->hw.partition_id != 1) { 12569 dev_info(&pf->pdev->dev, 12570 "Commit BW only works on partition 1! This is partition %d", 12571 pf->hw.partition_id); 12572 ret = I40E_NOT_SUPPORTED; 12573 goto bw_commit_out; 12574 } 12575 12576 /* Acquire NVM for read access */ 12577 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 12578 last_aq_status = pf->hw.aq.asq_last_status; 12579 if (ret) { 12580 dev_info(&pf->pdev->dev, 12581 "Cannot acquire NVM for read access, err %pe aq_err %s\n", 12582 ERR_PTR(ret), 12583 i40e_aq_str(&pf->hw, last_aq_status)); 12584 goto bw_commit_out; 12585 } 12586 12587 /* Read word 0x10 of NVM - SW compatibility word 1 */ 12588 ret = i40e_aq_read_nvm(&pf->hw, 12589 I40E_SR_NVM_CONTROL_WORD, 12590 0x10, sizeof(nvm_word), &nvm_word, 12591 false, NULL); 12592 /* Save off last admin queue command status before releasing 12593 * the NVM 12594 */ 12595 last_aq_status = pf->hw.aq.asq_last_status; 12596 i40e_release_nvm(&pf->hw); 12597 if (ret) { 12598 dev_info(&pf->pdev->dev, "NVM read error, err %pe aq_err %s\n", 12599 ERR_PTR(ret), 12600 i40e_aq_str(&pf->hw, last_aq_status)); 12601 goto bw_commit_out; 12602 } 12603 12604 /* Wait a bit for NVM release to complete */ 12605 msleep(50); 12606 12607 /* Acquire NVM for write access */ 12608 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 12609 last_aq_status = pf->hw.aq.asq_last_status; 12610 if (ret) { 12611 dev_info(&pf->pdev->dev, 12612 "Cannot acquire NVM for write access, err %pe aq_err %s\n", 12613 ERR_PTR(ret), 12614 i40e_aq_str(&pf->hw, last_aq_status)); 12615 goto bw_commit_out; 12616 } 12617 /* Write it back out unchanged to initiate update NVM, 12618 * which will force a write of the shadow (alt) RAM to 12619 * the NVM - thus storing the bandwidth values permanently. 12620 */ 12621 ret = i40e_aq_update_nvm(&pf->hw, 12622 I40E_SR_NVM_CONTROL_WORD, 12623 0x10, sizeof(nvm_word), 12624 &nvm_word, true, 0, NULL); 12625 /* Save off last admin queue command status before releasing 12626 * the NVM 12627 */ 12628 last_aq_status = pf->hw.aq.asq_last_status; 12629 i40e_release_nvm(&pf->hw); 12630 if (ret) 12631 dev_info(&pf->pdev->dev, 12632 "BW settings NOT SAVED, err %pe aq_err %s\n", 12633 ERR_PTR(ret), 12634 i40e_aq_str(&pf->hw, last_aq_status)); 12635 bw_commit_out: 12636 12637 return ret; 12638 } 12639 12640 /** 12641 * i40e_is_total_port_shutdown_enabled - read NVM and return value 12642 * if total port shutdown feature is enabled for this PF 12643 * @pf: board private structure 12644 **/ 12645 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf) 12646 { 12647 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4) 12648 #define I40E_FEATURES_ENABLE_PTR 0x2A 12649 #define I40E_CURRENT_SETTING_PTR 0x2B 12650 #define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D 12651 #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1 12652 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0) 12653 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4 12654 int read_status = I40E_SUCCESS; 12655 u16 sr_emp_sr_settings_ptr = 0; 12656 u16 features_enable = 0; 12657 u16 link_behavior = 0; 12658 bool ret = false; 12659 12660 read_status = i40e_read_nvm_word(&pf->hw, 12661 I40E_SR_EMP_SR_SETTINGS_PTR, 12662 &sr_emp_sr_settings_ptr); 12663 if (read_status) 12664 goto err_nvm; 12665 read_status = i40e_read_nvm_word(&pf->hw, 12666 sr_emp_sr_settings_ptr + 12667 I40E_FEATURES_ENABLE_PTR, 12668 &features_enable); 12669 if (read_status) 12670 goto err_nvm; 12671 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) { 12672 read_status = i40e_read_nvm_module_data(&pf->hw, 12673 I40E_SR_EMP_SR_SETTINGS_PTR, 12674 I40E_CURRENT_SETTING_PTR, 12675 I40E_LINK_BEHAVIOR_WORD_OFFSET, 12676 I40E_LINK_BEHAVIOR_WORD_LENGTH, 12677 &link_behavior); 12678 if (read_status) 12679 goto err_nvm; 12680 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH); 12681 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior; 12682 } 12683 return ret; 12684 12685 err_nvm: 12686 dev_warn(&pf->pdev->dev, 12687 "total-port-shutdown feature is off due to read nvm error: %pe\n", 12688 ERR_PTR(read_status)); 12689 return ret; 12690 } 12691 12692 /** 12693 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 12694 * @pf: board private structure to initialize 12695 * 12696 * i40e_sw_init initializes the Adapter private data structure. 12697 * Fields are initialized based on PCI device information and 12698 * OS network device settings (MTU size). 12699 **/ 12700 static int i40e_sw_init(struct i40e_pf *pf) 12701 { 12702 int err = 0; 12703 int size; 12704 u16 pow; 12705 12706 /* Set default capability flags */ 12707 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 12708 I40E_FLAG_MSI_ENABLED | 12709 I40E_FLAG_MSIX_ENABLED; 12710 12711 /* Set default ITR */ 12712 pf->rx_itr_default = I40E_ITR_RX_DEF; 12713 pf->tx_itr_default = I40E_ITR_TX_DEF; 12714 12715 /* Depending on PF configurations, it is possible that the RSS 12716 * maximum might end up larger than the available queues 12717 */ 12718 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 12719 pf->alloc_rss_size = 1; 12720 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 12721 pf->rss_size_max = min_t(int, pf->rss_size_max, 12722 pf->hw.func_caps.num_tx_qp); 12723 12724 /* find the next higher power-of-2 of num cpus */ 12725 pow = roundup_pow_of_two(num_online_cpus()); 12726 pf->rss_size_max = min_t(int, pf->rss_size_max, pow); 12727 12728 if (pf->hw.func_caps.rss) { 12729 pf->flags |= I40E_FLAG_RSS_ENABLED; 12730 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 12731 num_online_cpus()); 12732 } 12733 12734 /* MFP mode enabled */ 12735 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 12736 pf->flags |= I40E_FLAG_MFP_ENABLED; 12737 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 12738 if (i40e_get_partition_bw_setting(pf)) { 12739 dev_warn(&pf->pdev->dev, 12740 "Could not get partition bw settings\n"); 12741 } else { 12742 dev_info(&pf->pdev->dev, 12743 "Partition BW Min = %8.8x, Max = %8.8x\n", 12744 pf->min_bw, pf->max_bw); 12745 12746 /* nudge the Tx scheduler */ 12747 i40e_set_partition_bw_setting(pf); 12748 } 12749 } 12750 12751 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 12752 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 12753 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 12754 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 12755 if (pf->flags & I40E_FLAG_MFP_ENABLED && 12756 pf->hw.num_partitions > 1) 12757 dev_info(&pf->pdev->dev, 12758 "Flow Director Sideband mode Disabled in MFP mode\n"); 12759 else 12760 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 12761 pf->fdir_pf_filter_count = 12762 pf->hw.func_caps.fd_filters_guaranteed; 12763 pf->hw.fdir_shared_filter_count = 12764 pf->hw.func_caps.fd_filters_best_effort; 12765 } 12766 12767 if (pf->hw.mac.type == I40E_MAC_X722) { 12768 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 12769 I40E_HW_128_QP_RSS_CAPABLE | 12770 I40E_HW_ATR_EVICT_CAPABLE | 12771 I40E_HW_WB_ON_ITR_CAPABLE | 12772 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 12773 I40E_HW_NO_PCI_LINK_CHECK | 12774 I40E_HW_USE_SET_LLDP_MIB | 12775 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 12776 I40E_HW_PTP_L4_CAPABLE | 12777 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 12778 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 12779 12780 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 12781 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 12782 I40E_FDEVICT_PCTYPE_DEFAULT) { 12783 dev_warn(&pf->pdev->dev, 12784 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 12785 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 12786 } 12787 } else if ((pf->hw.aq.api_maj_ver > 1) || 12788 ((pf->hw.aq.api_maj_ver == 1) && 12789 (pf->hw.aq.api_min_ver > 4))) { 12790 /* Supported in FW API version higher than 1.4 */ 12791 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 12792 } 12793 12794 /* Enable HW ATR eviction if possible */ 12795 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 12796 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 12797 12798 if ((pf->hw.mac.type == I40E_MAC_XL710) && 12799 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 12800 (pf->hw.aq.fw_maj_ver < 4))) { 12801 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 12802 /* No DCB support for FW < v4.33 */ 12803 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 12804 } 12805 12806 /* Disable FW LLDP if FW < v4.3 */ 12807 if ((pf->hw.mac.type == I40E_MAC_XL710) && 12808 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 12809 (pf->hw.aq.fw_maj_ver < 4))) 12810 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 12811 12812 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 12813 if ((pf->hw.mac.type == I40E_MAC_XL710) && 12814 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 12815 (pf->hw.aq.fw_maj_ver >= 5))) 12816 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 12817 12818 /* Enable PTP L4 if FW > v6.0 */ 12819 if (pf->hw.mac.type == I40E_MAC_XL710 && 12820 pf->hw.aq.fw_maj_ver >= 6) 12821 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 12822 12823 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 12824 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 12825 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 12826 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 12827 } 12828 12829 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 12830 pf->flags |= I40E_FLAG_IWARP_ENABLED; 12831 /* IWARP needs one extra vector for CQP just like MISC.*/ 12832 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 12833 } 12834 /* Stopping FW LLDP engine is supported on XL710 and X722 12835 * starting from FW versions determined in i40e_init_adminq. 12836 * Stopping the FW LLDP engine is not supported on XL710 12837 * if NPAR is functioning so unset this hw flag in this case. 12838 */ 12839 if (pf->hw.mac.type == I40E_MAC_XL710 && 12840 pf->hw.func_caps.npar_enable && 12841 (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE)) 12842 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE; 12843 12844 #ifdef CONFIG_PCI_IOV 12845 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 12846 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 12847 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 12848 pf->num_req_vfs = min_t(int, 12849 pf->hw.func_caps.num_vfs, 12850 I40E_MAX_VF_COUNT); 12851 } 12852 #endif /* CONFIG_PCI_IOV */ 12853 pf->eeprom_version = 0xDEAD; 12854 pf->lan_veb = I40E_NO_VEB; 12855 pf->lan_vsi = I40E_NO_VSI; 12856 12857 /* By default FW has this off for performance reasons */ 12858 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 12859 12860 /* set up queue assignment tracking */ 12861 size = sizeof(struct i40e_lump_tracking) 12862 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 12863 pf->qp_pile = kzalloc(size, GFP_KERNEL); 12864 if (!pf->qp_pile) { 12865 err = -ENOMEM; 12866 goto sw_init_done; 12867 } 12868 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 12869 12870 pf->tx_timeout_recovery_level = 1; 12871 12872 if (pf->hw.mac.type != I40E_MAC_X722 && 12873 i40e_is_total_port_shutdown_enabled(pf)) { 12874 /* Link down on close must be on when total port shutdown 12875 * is enabled for a given port 12876 */ 12877 pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED | 12878 I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED); 12879 dev_info(&pf->pdev->dev, 12880 "total-port-shutdown was enabled, link-down-on-close is forced on\n"); 12881 } 12882 mutex_init(&pf->switch_mutex); 12883 12884 sw_init_done: 12885 return err; 12886 } 12887 12888 /** 12889 * i40e_set_ntuple - set the ntuple feature flag and take action 12890 * @pf: board private structure to initialize 12891 * @features: the feature set that the stack is suggesting 12892 * 12893 * returns a bool to indicate if reset needs to happen 12894 **/ 12895 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 12896 { 12897 bool need_reset = false; 12898 12899 /* Check if Flow Director n-tuple support was enabled or disabled. If 12900 * the state changed, we need to reset. 12901 */ 12902 if (features & NETIF_F_NTUPLE) { 12903 /* Enable filters and mark for reset */ 12904 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 12905 need_reset = true; 12906 /* enable FD_SB only if there is MSI-X vector and no cloud 12907 * filters exist 12908 */ 12909 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 12910 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 12911 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 12912 } 12913 } else { 12914 /* turn off filters, mark for reset and clear SW filter list */ 12915 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 12916 need_reset = true; 12917 i40e_fdir_filter_exit(pf); 12918 } 12919 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 12920 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 12921 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 12922 12923 /* reset fd counters */ 12924 pf->fd_add_err = 0; 12925 pf->fd_atr_cnt = 0; 12926 /* if ATR was auto disabled it can be re-enabled. */ 12927 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 12928 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 12929 (I40E_DEBUG_FD & pf->hw.debug_mask)) 12930 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 12931 } 12932 return need_reset; 12933 } 12934 12935 /** 12936 * i40e_clear_rss_lut - clear the rx hash lookup table 12937 * @vsi: the VSI being configured 12938 **/ 12939 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 12940 { 12941 struct i40e_pf *pf = vsi->back; 12942 struct i40e_hw *hw = &pf->hw; 12943 u16 vf_id = vsi->vf_id; 12944 u8 i; 12945 12946 if (vsi->type == I40E_VSI_MAIN) { 12947 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12948 wr32(hw, I40E_PFQF_HLUT(i), 0); 12949 } else if (vsi->type == I40E_VSI_SRIOV) { 12950 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12951 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 12952 } else { 12953 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12954 } 12955 } 12956 12957 /** 12958 * i40e_set_loopback - turn on/off loopback mode on underlying PF 12959 * @vsi: ptr to VSI 12960 * @ena: flag to indicate the on/off setting 12961 */ 12962 static int i40e_set_loopback(struct i40e_vsi *vsi, bool ena) 12963 { 12964 bool if_running = netif_running(vsi->netdev) && 12965 !test_and_set_bit(__I40E_VSI_DOWN, vsi->state); 12966 int ret; 12967 12968 if (if_running) 12969 i40e_down(vsi); 12970 12971 ret = i40e_aq_set_mac_loopback(&vsi->back->hw, ena, NULL); 12972 if (ret) 12973 netdev_err(vsi->netdev, "Failed to toggle loopback state\n"); 12974 if (if_running) 12975 i40e_up(vsi); 12976 12977 return ret; 12978 } 12979 12980 /** 12981 * i40e_set_features - set the netdev feature flags 12982 * @netdev: ptr to the netdev being adjusted 12983 * @features: the feature set that the stack is suggesting 12984 * Note: expects to be called while under rtnl_lock() 12985 **/ 12986 static int i40e_set_features(struct net_device *netdev, 12987 netdev_features_t features) 12988 { 12989 struct i40e_netdev_priv *np = netdev_priv(netdev); 12990 struct i40e_vsi *vsi = np->vsi; 12991 struct i40e_pf *pf = vsi->back; 12992 bool need_reset; 12993 12994 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 12995 i40e_pf_config_rss(pf); 12996 else if (!(features & NETIF_F_RXHASH) && 12997 netdev->features & NETIF_F_RXHASH) 12998 i40e_clear_rss_lut(vsi); 12999 13000 if (features & NETIF_F_HW_VLAN_CTAG_RX) 13001 i40e_vlan_stripping_enable(vsi); 13002 else 13003 i40e_vlan_stripping_disable(vsi); 13004 13005 if (!(features & NETIF_F_HW_TC) && 13006 (netdev->features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 13007 dev_err(&pf->pdev->dev, 13008 "Offloaded tc filters active, can't turn hw_tc_offload off"); 13009 return -EINVAL; 13010 } 13011 13012 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt) 13013 i40e_del_all_macvlans(vsi); 13014 13015 need_reset = i40e_set_ntuple(pf, features); 13016 13017 if (need_reset) 13018 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 13019 13020 if ((features ^ netdev->features) & NETIF_F_LOOPBACK) 13021 return i40e_set_loopback(vsi, !!(features & NETIF_F_LOOPBACK)); 13022 13023 return 0; 13024 } 13025 13026 static int i40e_udp_tunnel_set_port(struct net_device *netdev, 13027 unsigned int table, unsigned int idx, 13028 struct udp_tunnel_info *ti) 13029 { 13030 struct i40e_netdev_priv *np = netdev_priv(netdev); 13031 struct i40e_hw *hw = &np->vsi->back->hw; 13032 u8 type, filter_index; 13033 int ret; 13034 13035 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN : 13036 I40E_AQC_TUNNEL_TYPE_NGE; 13037 13038 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index, 13039 NULL); 13040 if (ret) { 13041 netdev_info(netdev, "add UDP port failed, err %pe aq_err %s\n", 13042 ERR_PTR(ret), 13043 i40e_aq_str(hw, hw->aq.asq_last_status)); 13044 return -EIO; 13045 } 13046 13047 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index); 13048 return 0; 13049 } 13050 13051 static int i40e_udp_tunnel_unset_port(struct net_device *netdev, 13052 unsigned int table, unsigned int idx, 13053 struct udp_tunnel_info *ti) 13054 { 13055 struct i40e_netdev_priv *np = netdev_priv(netdev); 13056 struct i40e_hw *hw = &np->vsi->back->hw; 13057 int ret; 13058 13059 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL); 13060 if (ret) { 13061 netdev_info(netdev, "delete UDP port failed, err %pe aq_err %s\n", 13062 ERR_PTR(ret), 13063 i40e_aq_str(hw, hw->aq.asq_last_status)); 13064 return -EIO; 13065 } 13066 13067 return 0; 13068 } 13069 13070 static int i40e_get_phys_port_id(struct net_device *netdev, 13071 struct netdev_phys_item_id *ppid) 13072 { 13073 struct i40e_netdev_priv *np = netdev_priv(netdev); 13074 struct i40e_pf *pf = np->vsi->back; 13075 struct i40e_hw *hw = &pf->hw; 13076 13077 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 13078 return -EOPNOTSUPP; 13079 13080 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 13081 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 13082 13083 return 0; 13084 } 13085 13086 /** 13087 * i40e_ndo_fdb_add - add an entry to the hardware database 13088 * @ndm: the input from the stack 13089 * @tb: pointer to array of nladdr (unused) 13090 * @dev: the net device pointer 13091 * @addr: the MAC address entry being added 13092 * @vid: VLAN ID 13093 * @flags: instructions from stack about fdb operation 13094 * @extack: netlink extended ack, unused currently 13095 */ 13096 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 13097 struct net_device *dev, 13098 const unsigned char *addr, u16 vid, 13099 u16 flags, 13100 struct netlink_ext_ack *extack) 13101 { 13102 struct i40e_netdev_priv *np = netdev_priv(dev); 13103 struct i40e_pf *pf = np->vsi->back; 13104 int err = 0; 13105 13106 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 13107 return -EOPNOTSUPP; 13108 13109 if (vid) { 13110 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 13111 return -EINVAL; 13112 } 13113 13114 /* Hardware does not support aging addresses so if a 13115 * ndm_state is given only allow permanent addresses 13116 */ 13117 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 13118 netdev_info(dev, "FDB only supports static addresses\n"); 13119 return -EINVAL; 13120 } 13121 13122 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 13123 err = dev_uc_add_excl(dev, addr); 13124 else if (is_multicast_ether_addr(addr)) 13125 err = dev_mc_add_excl(dev, addr); 13126 else 13127 err = -EINVAL; 13128 13129 /* Only return duplicate errors if NLM_F_EXCL is set */ 13130 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 13131 err = 0; 13132 13133 return err; 13134 } 13135 13136 /** 13137 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 13138 * @dev: the netdev being configured 13139 * @nlh: RTNL message 13140 * @flags: bridge flags 13141 * @extack: netlink extended ack 13142 * 13143 * Inserts a new hardware bridge if not already created and 13144 * enables the bridging mode requested (VEB or VEPA). If the 13145 * hardware bridge has already been inserted and the request 13146 * is to change the mode then that requires a PF reset to 13147 * allow rebuild of the components with required hardware 13148 * bridge mode enabled. 13149 * 13150 * Note: expects to be called while under rtnl_lock() 13151 **/ 13152 static int i40e_ndo_bridge_setlink(struct net_device *dev, 13153 struct nlmsghdr *nlh, 13154 u16 flags, 13155 struct netlink_ext_ack *extack) 13156 { 13157 struct i40e_netdev_priv *np = netdev_priv(dev); 13158 struct i40e_vsi *vsi = np->vsi; 13159 struct i40e_pf *pf = vsi->back; 13160 struct i40e_veb *veb = NULL; 13161 struct nlattr *attr, *br_spec; 13162 int i, rem; 13163 13164 /* Only for PF VSI for now */ 13165 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 13166 return -EOPNOTSUPP; 13167 13168 /* Find the HW bridge for PF VSI */ 13169 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 13170 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 13171 veb = pf->veb[i]; 13172 } 13173 13174 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 13175 if (!br_spec) 13176 return -EINVAL; 13177 13178 nla_for_each_nested(attr, br_spec, rem) { 13179 __u16 mode; 13180 13181 if (nla_type(attr) != IFLA_BRIDGE_MODE) 13182 continue; 13183 13184 mode = nla_get_u16(attr); 13185 if ((mode != BRIDGE_MODE_VEPA) && 13186 (mode != BRIDGE_MODE_VEB)) 13187 return -EINVAL; 13188 13189 /* Insert a new HW bridge */ 13190 if (!veb) { 13191 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 13192 vsi->tc_config.enabled_tc); 13193 if (veb) { 13194 veb->bridge_mode = mode; 13195 i40e_config_bridge_mode(veb); 13196 } else { 13197 /* No Bridge HW offload available */ 13198 return -ENOENT; 13199 } 13200 break; 13201 } else if (mode != veb->bridge_mode) { 13202 /* Existing HW bridge but different mode needs reset */ 13203 veb->bridge_mode = mode; 13204 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 13205 if (mode == BRIDGE_MODE_VEB) 13206 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 13207 else 13208 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 13209 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 13210 break; 13211 } 13212 } 13213 13214 return 0; 13215 } 13216 13217 /** 13218 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 13219 * @skb: skb buff 13220 * @pid: process id 13221 * @seq: RTNL message seq # 13222 * @dev: the netdev being configured 13223 * @filter_mask: unused 13224 * @nlflags: netlink flags passed in 13225 * 13226 * Return the mode in which the hardware bridge is operating in 13227 * i.e VEB or VEPA. 13228 **/ 13229 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 13230 struct net_device *dev, 13231 u32 __always_unused filter_mask, 13232 int nlflags) 13233 { 13234 struct i40e_netdev_priv *np = netdev_priv(dev); 13235 struct i40e_vsi *vsi = np->vsi; 13236 struct i40e_pf *pf = vsi->back; 13237 struct i40e_veb *veb = NULL; 13238 int i; 13239 13240 /* Only for PF VSI for now */ 13241 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 13242 return -EOPNOTSUPP; 13243 13244 /* Find the HW bridge for the PF VSI */ 13245 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 13246 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 13247 veb = pf->veb[i]; 13248 } 13249 13250 if (!veb) 13251 return 0; 13252 13253 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 13254 0, 0, nlflags, filter_mask, NULL); 13255 } 13256 13257 /** 13258 * i40e_features_check - Validate encapsulated packet conforms to limits 13259 * @skb: skb buff 13260 * @dev: This physical port's netdev 13261 * @features: Offload features that the stack believes apply 13262 **/ 13263 static netdev_features_t i40e_features_check(struct sk_buff *skb, 13264 struct net_device *dev, 13265 netdev_features_t features) 13266 { 13267 size_t len; 13268 13269 /* No point in doing any of this if neither checksum nor GSO are 13270 * being requested for this frame. We can rule out both by just 13271 * checking for CHECKSUM_PARTIAL 13272 */ 13273 if (skb->ip_summed != CHECKSUM_PARTIAL) 13274 return features; 13275 13276 /* We cannot support GSO if the MSS is going to be less than 13277 * 64 bytes. If it is then we need to drop support for GSO. 13278 */ 13279 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 13280 features &= ~NETIF_F_GSO_MASK; 13281 13282 /* MACLEN can support at most 63 words */ 13283 len = skb_network_header(skb) - skb->data; 13284 if (len & ~(63 * 2)) 13285 goto out_err; 13286 13287 /* IPLEN and EIPLEN can support at most 127 dwords */ 13288 len = skb_transport_header(skb) - skb_network_header(skb); 13289 if (len & ~(127 * 4)) 13290 goto out_err; 13291 13292 if (skb->encapsulation) { 13293 /* L4TUNLEN can support 127 words */ 13294 len = skb_inner_network_header(skb) - skb_transport_header(skb); 13295 if (len & ~(127 * 2)) 13296 goto out_err; 13297 13298 /* IPLEN can support at most 127 dwords */ 13299 len = skb_inner_transport_header(skb) - 13300 skb_inner_network_header(skb); 13301 if (len & ~(127 * 4)) 13302 goto out_err; 13303 } 13304 13305 /* No need to validate L4LEN as TCP is the only protocol with a 13306 * flexible value and we support all possible values supported 13307 * by TCP, which is at most 15 dwords 13308 */ 13309 13310 return features; 13311 out_err: 13312 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 13313 } 13314 13315 /** 13316 * i40e_xdp_setup - add/remove an XDP program 13317 * @vsi: VSI to changed 13318 * @prog: XDP program 13319 * @extack: netlink extended ack 13320 **/ 13321 static int i40e_xdp_setup(struct i40e_vsi *vsi, struct bpf_prog *prog, 13322 struct netlink_ext_ack *extack) 13323 { 13324 int frame_size = i40e_max_vsi_frame_size(vsi, prog); 13325 struct i40e_pf *pf = vsi->back; 13326 struct bpf_prog *old_prog; 13327 bool need_reset; 13328 int i; 13329 13330 /* Don't allow frames that span over multiple buffers */ 13331 if (vsi->netdev->mtu > frame_size - I40E_PACKET_HDR_PAD) { 13332 NL_SET_ERR_MSG_MOD(extack, "MTU too large for linear frames and XDP prog does not support frags"); 13333 return -EINVAL; 13334 } 13335 13336 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 13337 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 13338 13339 if (need_reset) 13340 i40e_prep_for_reset(pf); 13341 13342 /* VSI shall be deleted in a moment, just return EINVAL */ 13343 if (test_bit(__I40E_IN_REMOVE, pf->state)) 13344 return -EINVAL; 13345 13346 old_prog = xchg(&vsi->xdp_prog, prog); 13347 13348 if (need_reset) { 13349 if (!prog) { 13350 xdp_features_clear_redirect_target(vsi->netdev); 13351 /* Wait until ndo_xsk_wakeup completes. */ 13352 synchronize_rcu(); 13353 } 13354 i40e_reset_and_rebuild(pf, true, true); 13355 } 13356 13357 if (!i40e_enabled_xdp_vsi(vsi) && prog) { 13358 if (i40e_realloc_rx_bi_zc(vsi, true)) 13359 return -ENOMEM; 13360 } else if (i40e_enabled_xdp_vsi(vsi) && !prog) { 13361 if (i40e_realloc_rx_bi_zc(vsi, false)) 13362 return -ENOMEM; 13363 } 13364 13365 for (i = 0; i < vsi->num_queue_pairs; i++) 13366 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 13367 13368 if (old_prog) 13369 bpf_prog_put(old_prog); 13370 13371 /* Kick start the NAPI context if there is an AF_XDP socket open 13372 * on that queue id. This so that receiving will start. 13373 */ 13374 if (need_reset && prog) { 13375 for (i = 0; i < vsi->num_queue_pairs; i++) 13376 if (vsi->xdp_rings[i]->xsk_pool) 13377 (void)i40e_xsk_wakeup(vsi->netdev, i, 13378 XDP_WAKEUP_RX); 13379 xdp_features_set_redirect_target(vsi->netdev, true); 13380 } 13381 13382 return 0; 13383 } 13384 13385 /** 13386 * i40e_enter_busy_conf - Enters busy config state 13387 * @vsi: vsi 13388 * 13389 * Returns 0 on success, <0 for failure. 13390 **/ 13391 static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 13392 { 13393 struct i40e_pf *pf = vsi->back; 13394 int timeout = 50; 13395 13396 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 13397 timeout--; 13398 if (!timeout) 13399 return -EBUSY; 13400 usleep_range(1000, 2000); 13401 } 13402 13403 return 0; 13404 } 13405 13406 /** 13407 * i40e_exit_busy_conf - Exits busy config state 13408 * @vsi: vsi 13409 **/ 13410 static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 13411 { 13412 struct i40e_pf *pf = vsi->back; 13413 13414 clear_bit(__I40E_CONFIG_BUSY, pf->state); 13415 } 13416 13417 /** 13418 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 13419 * @vsi: vsi 13420 * @queue_pair: queue pair 13421 **/ 13422 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 13423 { 13424 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 13425 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 13426 memset(&vsi->tx_rings[queue_pair]->stats, 0, 13427 sizeof(vsi->tx_rings[queue_pair]->stats)); 13428 if (i40e_enabled_xdp_vsi(vsi)) { 13429 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 13430 sizeof(vsi->xdp_rings[queue_pair]->stats)); 13431 } 13432 } 13433 13434 /** 13435 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 13436 * @vsi: vsi 13437 * @queue_pair: queue pair 13438 **/ 13439 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 13440 { 13441 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 13442 if (i40e_enabled_xdp_vsi(vsi)) { 13443 /* Make sure that in-progress ndo_xdp_xmit calls are 13444 * completed. 13445 */ 13446 synchronize_rcu(); 13447 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 13448 } 13449 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 13450 } 13451 13452 /** 13453 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 13454 * @vsi: vsi 13455 * @queue_pair: queue pair 13456 * @enable: true for enable, false for disable 13457 **/ 13458 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 13459 bool enable) 13460 { 13461 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13462 struct i40e_q_vector *q_vector = rxr->q_vector; 13463 13464 if (!vsi->netdev) 13465 return; 13466 13467 /* All rings in a qp belong to the same qvector. */ 13468 if (q_vector->rx.ring || q_vector->tx.ring) { 13469 if (enable) 13470 napi_enable(&q_vector->napi); 13471 else 13472 napi_disable(&q_vector->napi); 13473 } 13474 } 13475 13476 /** 13477 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 13478 * @vsi: vsi 13479 * @queue_pair: queue pair 13480 * @enable: true for enable, false for disable 13481 * 13482 * Returns 0 on success, <0 on failure. 13483 **/ 13484 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 13485 bool enable) 13486 { 13487 struct i40e_pf *pf = vsi->back; 13488 int pf_q, ret = 0; 13489 13490 pf_q = vsi->base_queue + queue_pair; 13491 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 13492 false /*is xdp*/, enable); 13493 if (ret) { 13494 dev_info(&pf->pdev->dev, 13495 "VSI seid %d Tx ring %d %sable timeout\n", 13496 vsi->seid, pf_q, (enable ? "en" : "dis")); 13497 return ret; 13498 } 13499 13500 i40e_control_rx_q(pf, pf_q, enable); 13501 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 13502 if (ret) { 13503 dev_info(&pf->pdev->dev, 13504 "VSI seid %d Rx ring %d %sable timeout\n", 13505 vsi->seid, pf_q, (enable ? "en" : "dis")); 13506 return ret; 13507 } 13508 13509 /* Due to HW errata, on Rx disable only, the register can 13510 * indicate done before it really is. Needs 50ms to be sure 13511 */ 13512 if (!enable) 13513 mdelay(50); 13514 13515 if (!i40e_enabled_xdp_vsi(vsi)) 13516 return ret; 13517 13518 ret = i40e_control_wait_tx_q(vsi->seid, pf, 13519 pf_q + vsi->alloc_queue_pairs, 13520 true /*is xdp*/, enable); 13521 if (ret) { 13522 dev_info(&pf->pdev->dev, 13523 "VSI seid %d XDP Tx ring %d %sable timeout\n", 13524 vsi->seid, pf_q, (enable ? "en" : "dis")); 13525 } 13526 13527 return ret; 13528 } 13529 13530 /** 13531 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 13532 * @vsi: vsi 13533 * @queue_pair: queue_pair 13534 **/ 13535 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 13536 { 13537 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13538 struct i40e_pf *pf = vsi->back; 13539 struct i40e_hw *hw = &pf->hw; 13540 13541 /* All rings in a qp belong to the same qvector. */ 13542 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 13543 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 13544 else 13545 i40e_irq_dynamic_enable_icr0(pf); 13546 13547 i40e_flush(hw); 13548 } 13549 13550 /** 13551 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 13552 * @vsi: vsi 13553 * @queue_pair: queue_pair 13554 **/ 13555 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 13556 { 13557 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 13558 struct i40e_pf *pf = vsi->back; 13559 struct i40e_hw *hw = &pf->hw; 13560 13561 /* For simplicity, instead of removing the qp interrupt causes 13562 * from the interrupt linked list, we simply disable the interrupt, and 13563 * leave the list intact. 13564 * 13565 * All rings in a qp belong to the same qvector. 13566 */ 13567 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 13568 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 13569 13570 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 13571 i40e_flush(hw); 13572 synchronize_irq(pf->msix_entries[intpf].vector); 13573 } else { 13574 /* Legacy and MSI mode - this stops all interrupt handling */ 13575 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 13576 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 13577 i40e_flush(hw); 13578 synchronize_irq(pf->pdev->irq); 13579 } 13580 } 13581 13582 /** 13583 * i40e_queue_pair_disable - Disables a queue pair 13584 * @vsi: vsi 13585 * @queue_pair: queue pair 13586 * 13587 * Returns 0 on success, <0 on failure. 13588 **/ 13589 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 13590 { 13591 int err; 13592 13593 err = i40e_enter_busy_conf(vsi); 13594 if (err) 13595 return err; 13596 13597 i40e_queue_pair_disable_irq(vsi, queue_pair); 13598 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 13599 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 13600 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 13601 i40e_queue_pair_clean_rings(vsi, queue_pair); 13602 i40e_queue_pair_reset_stats(vsi, queue_pair); 13603 13604 return err; 13605 } 13606 13607 /** 13608 * i40e_queue_pair_enable - Enables a queue pair 13609 * @vsi: vsi 13610 * @queue_pair: queue pair 13611 * 13612 * Returns 0 on success, <0 on failure. 13613 **/ 13614 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 13615 { 13616 int err; 13617 13618 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 13619 if (err) 13620 return err; 13621 13622 if (i40e_enabled_xdp_vsi(vsi)) { 13623 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 13624 if (err) 13625 return err; 13626 } 13627 13628 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 13629 if (err) 13630 return err; 13631 13632 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 13633 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 13634 i40e_queue_pair_enable_irq(vsi, queue_pair); 13635 13636 i40e_exit_busy_conf(vsi); 13637 13638 return err; 13639 } 13640 13641 /** 13642 * i40e_xdp - implements ndo_bpf for i40e 13643 * @dev: netdevice 13644 * @xdp: XDP command 13645 **/ 13646 static int i40e_xdp(struct net_device *dev, 13647 struct netdev_bpf *xdp) 13648 { 13649 struct i40e_netdev_priv *np = netdev_priv(dev); 13650 struct i40e_vsi *vsi = np->vsi; 13651 13652 if (vsi->type != I40E_VSI_MAIN) 13653 return -EINVAL; 13654 13655 switch (xdp->command) { 13656 case XDP_SETUP_PROG: 13657 return i40e_xdp_setup(vsi, xdp->prog, xdp->extack); 13658 case XDP_SETUP_XSK_POOL: 13659 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool, 13660 xdp->xsk.queue_id); 13661 default: 13662 return -EINVAL; 13663 } 13664 } 13665 13666 static const struct net_device_ops i40e_netdev_ops = { 13667 .ndo_open = i40e_open, 13668 .ndo_stop = i40e_close, 13669 .ndo_start_xmit = i40e_lan_xmit_frame, 13670 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 13671 .ndo_set_rx_mode = i40e_set_rx_mode, 13672 .ndo_validate_addr = eth_validate_addr, 13673 .ndo_set_mac_address = i40e_set_mac, 13674 .ndo_change_mtu = i40e_change_mtu, 13675 .ndo_eth_ioctl = i40e_ioctl, 13676 .ndo_tx_timeout = i40e_tx_timeout, 13677 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 13678 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 13679 #ifdef CONFIG_NET_POLL_CONTROLLER 13680 .ndo_poll_controller = i40e_netpoll, 13681 #endif 13682 .ndo_setup_tc = __i40e_setup_tc, 13683 .ndo_select_queue = i40e_lan_select_queue, 13684 .ndo_set_features = i40e_set_features, 13685 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 13686 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 13687 .ndo_get_vf_stats = i40e_get_vf_stats, 13688 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 13689 .ndo_get_vf_config = i40e_ndo_get_vf_config, 13690 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 13691 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 13692 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 13693 .ndo_get_phys_port_id = i40e_get_phys_port_id, 13694 .ndo_fdb_add = i40e_ndo_fdb_add, 13695 .ndo_features_check = i40e_features_check, 13696 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 13697 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 13698 .ndo_bpf = i40e_xdp, 13699 .ndo_xdp_xmit = i40e_xdp_xmit, 13700 .ndo_xsk_wakeup = i40e_xsk_wakeup, 13701 .ndo_dfwd_add_station = i40e_fwd_add, 13702 .ndo_dfwd_del_station = i40e_fwd_del, 13703 }; 13704 13705 /** 13706 * i40e_config_netdev - Setup the netdev flags 13707 * @vsi: the VSI being configured 13708 * 13709 * Returns 0 on success, negative value on failure 13710 **/ 13711 static int i40e_config_netdev(struct i40e_vsi *vsi) 13712 { 13713 struct i40e_pf *pf = vsi->back; 13714 struct i40e_hw *hw = &pf->hw; 13715 struct i40e_netdev_priv *np; 13716 struct net_device *netdev; 13717 u8 broadcast[ETH_ALEN]; 13718 u8 mac_addr[ETH_ALEN]; 13719 int etherdev_size; 13720 netdev_features_t hw_enc_features; 13721 netdev_features_t hw_features; 13722 13723 etherdev_size = sizeof(struct i40e_netdev_priv); 13724 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 13725 if (!netdev) 13726 return -ENOMEM; 13727 13728 vsi->netdev = netdev; 13729 np = netdev_priv(netdev); 13730 np->vsi = vsi; 13731 13732 hw_enc_features = NETIF_F_SG | 13733 NETIF_F_HW_CSUM | 13734 NETIF_F_HIGHDMA | 13735 NETIF_F_SOFT_FEATURES | 13736 NETIF_F_TSO | 13737 NETIF_F_TSO_ECN | 13738 NETIF_F_TSO6 | 13739 NETIF_F_GSO_GRE | 13740 NETIF_F_GSO_GRE_CSUM | 13741 NETIF_F_GSO_PARTIAL | 13742 NETIF_F_GSO_IPXIP4 | 13743 NETIF_F_GSO_IPXIP6 | 13744 NETIF_F_GSO_UDP_TUNNEL | 13745 NETIF_F_GSO_UDP_TUNNEL_CSUM | 13746 NETIF_F_GSO_UDP_L4 | 13747 NETIF_F_SCTP_CRC | 13748 NETIF_F_RXHASH | 13749 NETIF_F_RXCSUM | 13750 0; 13751 13752 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 13753 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 13754 13755 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic; 13756 13757 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 13758 13759 netdev->hw_enc_features |= hw_enc_features; 13760 13761 /* record features VLANs can make use of */ 13762 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 13763 13764 #define I40E_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ 13765 NETIF_F_GSO_GRE_CSUM | \ 13766 NETIF_F_GSO_IPXIP4 | \ 13767 NETIF_F_GSO_IPXIP6 | \ 13768 NETIF_F_GSO_UDP_TUNNEL | \ 13769 NETIF_F_GSO_UDP_TUNNEL_CSUM) 13770 13771 netdev->gso_partial_features = I40E_GSO_PARTIAL_FEATURES; 13772 netdev->features |= NETIF_F_GSO_PARTIAL | 13773 I40E_GSO_PARTIAL_FEATURES; 13774 13775 netdev->mpls_features |= NETIF_F_SG; 13776 netdev->mpls_features |= NETIF_F_HW_CSUM; 13777 netdev->mpls_features |= NETIF_F_TSO; 13778 netdev->mpls_features |= NETIF_F_TSO6; 13779 netdev->mpls_features |= I40E_GSO_PARTIAL_FEATURES; 13780 13781 /* enable macvlan offloads */ 13782 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD; 13783 13784 hw_features = hw_enc_features | 13785 NETIF_F_HW_VLAN_CTAG_TX | 13786 NETIF_F_HW_VLAN_CTAG_RX; 13787 13788 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 13789 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 13790 13791 netdev->hw_features |= hw_features | NETIF_F_LOOPBACK; 13792 13793 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 13794 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 13795 13796 netdev->features &= ~NETIF_F_HW_TC; 13797 13798 if (vsi->type == I40E_VSI_MAIN) { 13799 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 13800 ether_addr_copy(mac_addr, hw->mac.perm_addr); 13801 /* The following steps are necessary for two reasons. First, 13802 * some older NVM configurations load a default MAC-VLAN 13803 * filter that will accept any tagged packet, and we want to 13804 * replace this with a normal filter. Additionally, it is 13805 * possible our MAC address was provided by the platform using 13806 * Open Firmware or similar. 13807 * 13808 * Thus, we need to remove the default filter and install one 13809 * specific to the MAC address. 13810 */ 13811 i40e_rm_default_mac_filter(vsi, mac_addr); 13812 spin_lock_bh(&vsi->mac_filter_hash_lock); 13813 i40e_add_mac_filter(vsi, mac_addr); 13814 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13815 13816 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | 13817 NETDEV_XDP_ACT_REDIRECT | 13818 NETDEV_XDP_ACT_XSK_ZEROCOPY | 13819 NETDEV_XDP_ACT_RX_SG; 13820 netdev->xdp_zc_max_segs = I40E_MAX_BUFFER_TXD; 13821 } else { 13822 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 13823 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 13824 * the end, which is 4 bytes long, so force truncation of the 13825 * original name by IFNAMSIZ - 4 13826 */ 13827 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 13828 IFNAMSIZ - 4, 13829 pf->vsi[pf->lan_vsi]->netdev->name); 13830 eth_random_addr(mac_addr); 13831 13832 spin_lock_bh(&vsi->mac_filter_hash_lock); 13833 i40e_add_mac_filter(vsi, mac_addr); 13834 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13835 } 13836 13837 /* Add the broadcast filter so that we initially will receive 13838 * broadcast packets. Note that when a new VLAN is first added the 13839 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 13840 * specific filters as part of transitioning into "vlan" operation. 13841 * When more VLANs are added, the driver will copy each existing MAC 13842 * filter and add it for the new VLAN. 13843 * 13844 * Broadcast filters are handled specially by 13845 * i40e_sync_filters_subtask, as the driver must to set the broadcast 13846 * promiscuous bit instead of adding this directly as a MAC/VLAN 13847 * filter. The subtask will update the correct broadcast promiscuous 13848 * bits as VLANs become active or inactive. 13849 */ 13850 eth_broadcast_addr(broadcast); 13851 spin_lock_bh(&vsi->mac_filter_hash_lock); 13852 i40e_add_mac_filter(vsi, broadcast); 13853 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13854 13855 eth_hw_addr_set(netdev, mac_addr); 13856 ether_addr_copy(netdev->perm_addr, mac_addr); 13857 13858 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */ 13859 netdev->neigh_priv_len = sizeof(u32) * 4; 13860 13861 netdev->priv_flags |= IFF_UNICAST_FLT; 13862 netdev->priv_flags |= IFF_SUPP_NOFCS; 13863 /* Setup netdev TC information */ 13864 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 13865 13866 netdev->netdev_ops = &i40e_netdev_ops; 13867 netdev->watchdog_timeo = 5 * HZ; 13868 i40e_set_ethtool_ops(netdev); 13869 13870 /* MTU range: 68 - 9706 */ 13871 netdev->min_mtu = ETH_MIN_MTU; 13872 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 13873 13874 return 0; 13875 } 13876 13877 /** 13878 * i40e_vsi_delete - Delete a VSI from the switch 13879 * @vsi: the VSI being removed 13880 * 13881 * Returns 0 on success, negative value on failure 13882 **/ 13883 static void i40e_vsi_delete(struct i40e_vsi *vsi) 13884 { 13885 /* remove default VSI is not allowed */ 13886 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 13887 return; 13888 13889 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 13890 } 13891 13892 /** 13893 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 13894 * @vsi: the VSI being queried 13895 * 13896 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 13897 **/ 13898 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 13899 { 13900 struct i40e_veb *veb; 13901 struct i40e_pf *pf = vsi->back; 13902 13903 /* Uplink is not a bridge so default to VEB */ 13904 if (vsi->veb_idx >= I40E_MAX_VEB) 13905 return 1; 13906 13907 veb = pf->veb[vsi->veb_idx]; 13908 if (!veb) { 13909 dev_info(&pf->pdev->dev, 13910 "There is no veb associated with the bridge\n"); 13911 return -ENOENT; 13912 } 13913 13914 /* Uplink is a bridge in VEPA mode */ 13915 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 13916 return 0; 13917 } else { 13918 /* Uplink is a bridge in VEB mode */ 13919 return 1; 13920 } 13921 13922 /* VEPA is now default bridge, so return 0 */ 13923 return 0; 13924 } 13925 13926 /** 13927 * i40e_add_vsi - Add a VSI to the switch 13928 * @vsi: the VSI being configured 13929 * 13930 * This initializes a VSI context depending on the VSI type to be added and 13931 * passes it down to the add_vsi aq command. 13932 **/ 13933 static int i40e_add_vsi(struct i40e_vsi *vsi) 13934 { 13935 int ret = -ENODEV; 13936 struct i40e_pf *pf = vsi->back; 13937 struct i40e_hw *hw = &pf->hw; 13938 struct i40e_vsi_context ctxt; 13939 struct i40e_mac_filter *f; 13940 struct hlist_node *h; 13941 int bkt; 13942 13943 u8 enabled_tc = 0x1; /* TC0 enabled */ 13944 int f_count = 0; 13945 13946 memset(&ctxt, 0, sizeof(ctxt)); 13947 switch (vsi->type) { 13948 case I40E_VSI_MAIN: 13949 /* The PF's main VSI is already setup as part of the 13950 * device initialization, so we'll not bother with 13951 * the add_vsi call, but we will retrieve the current 13952 * VSI context. 13953 */ 13954 ctxt.seid = pf->main_vsi_seid; 13955 ctxt.pf_num = pf->hw.pf_id; 13956 ctxt.vf_num = 0; 13957 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 13958 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13959 if (ret) { 13960 dev_info(&pf->pdev->dev, 13961 "couldn't get PF vsi config, err %pe aq_err %s\n", 13962 ERR_PTR(ret), 13963 i40e_aq_str(&pf->hw, 13964 pf->hw.aq.asq_last_status)); 13965 return -ENOENT; 13966 } 13967 vsi->info = ctxt.info; 13968 vsi->info.valid_sections = 0; 13969 13970 vsi->seid = ctxt.seid; 13971 vsi->id = ctxt.vsi_number; 13972 13973 enabled_tc = i40e_pf_get_tc_map(pf); 13974 13975 /* Source pruning is enabled by default, so the flag is 13976 * negative logic - if it's set, we need to fiddle with 13977 * the VSI to disable source pruning. 13978 */ 13979 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 13980 memset(&ctxt, 0, sizeof(ctxt)); 13981 ctxt.seid = pf->main_vsi_seid; 13982 ctxt.pf_num = pf->hw.pf_id; 13983 ctxt.vf_num = 0; 13984 ctxt.info.valid_sections |= 13985 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13986 ctxt.info.switch_id = 13987 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 13988 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13989 if (ret) { 13990 dev_info(&pf->pdev->dev, 13991 "update vsi failed, err %d aq_err %s\n", 13992 ret, 13993 i40e_aq_str(&pf->hw, 13994 pf->hw.aq.asq_last_status)); 13995 ret = -ENOENT; 13996 goto err; 13997 } 13998 } 13999 14000 /* MFP mode setup queue map and update VSI */ 14001 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 14002 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 14003 memset(&ctxt, 0, sizeof(ctxt)); 14004 ctxt.seid = pf->main_vsi_seid; 14005 ctxt.pf_num = pf->hw.pf_id; 14006 ctxt.vf_num = 0; 14007 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 14008 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 14009 if (ret) { 14010 dev_info(&pf->pdev->dev, 14011 "update vsi failed, err %pe aq_err %s\n", 14012 ERR_PTR(ret), 14013 i40e_aq_str(&pf->hw, 14014 pf->hw.aq.asq_last_status)); 14015 ret = -ENOENT; 14016 goto err; 14017 } 14018 /* update the local VSI info queue map */ 14019 i40e_vsi_update_queue_map(vsi, &ctxt); 14020 vsi->info.valid_sections = 0; 14021 } else { 14022 /* Default/Main VSI is only enabled for TC0 14023 * reconfigure it to enable all TCs that are 14024 * available on the port in SFP mode. 14025 * For MFP case the iSCSI PF would use this 14026 * flow to enable LAN+iSCSI TC. 14027 */ 14028 ret = i40e_vsi_config_tc(vsi, enabled_tc); 14029 if (ret) { 14030 /* Single TC condition is not fatal, 14031 * message and continue 14032 */ 14033 dev_info(&pf->pdev->dev, 14034 "failed to configure TCs for main VSI tc_map 0x%08x, err %pe aq_err %s\n", 14035 enabled_tc, 14036 ERR_PTR(ret), 14037 i40e_aq_str(&pf->hw, 14038 pf->hw.aq.asq_last_status)); 14039 } 14040 } 14041 break; 14042 14043 case I40E_VSI_FDIR: 14044 ctxt.pf_num = hw->pf_id; 14045 ctxt.vf_num = 0; 14046 ctxt.uplink_seid = vsi->uplink_seid; 14047 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 14048 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 14049 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 14050 (i40e_is_vsi_uplink_mode_veb(vsi))) { 14051 ctxt.info.valid_sections |= 14052 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 14053 ctxt.info.switch_id = 14054 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 14055 } 14056 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 14057 break; 14058 14059 case I40E_VSI_VMDQ2: 14060 ctxt.pf_num = hw->pf_id; 14061 ctxt.vf_num = 0; 14062 ctxt.uplink_seid = vsi->uplink_seid; 14063 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 14064 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 14065 14066 /* This VSI is connected to VEB so the switch_id 14067 * should be set to zero by default. 14068 */ 14069 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 14070 ctxt.info.valid_sections |= 14071 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 14072 ctxt.info.switch_id = 14073 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 14074 } 14075 14076 /* Setup the VSI tx/rx queue map for TC0 only for now */ 14077 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 14078 break; 14079 14080 case I40E_VSI_SRIOV: 14081 ctxt.pf_num = hw->pf_id; 14082 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 14083 ctxt.uplink_seid = vsi->uplink_seid; 14084 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 14085 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 14086 14087 /* This VSI is connected to VEB so the switch_id 14088 * should be set to zero by default. 14089 */ 14090 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 14091 ctxt.info.valid_sections |= 14092 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 14093 ctxt.info.switch_id = 14094 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 14095 } 14096 14097 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 14098 ctxt.info.valid_sections |= 14099 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 14100 ctxt.info.queueing_opt_flags |= 14101 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 14102 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 14103 } 14104 14105 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 14106 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 14107 if (pf->vf[vsi->vf_id].spoofchk) { 14108 ctxt.info.valid_sections |= 14109 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 14110 ctxt.info.sec_flags |= 14111 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 14112 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 14113 } 14114 /* Setup the VSI tx/rx queue map for TC0 only for now */ 14115 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 14116 break; 14117 14118 case I40E_VSI_IWARP: 14119 /* send down message to iWARP */ 14120 break; 14121 14122 default: 14123 return -ENODEV; 14124 } 14125 14126 if (vsi->type != I40E_VSI_MAIN) { 14127 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 14128 if (ret) { 14129 dev_info(&vsi->back->pdev->dev, 14130 "add vsi failed, err %pe aq_err %s\n", 14131 ERR_PTR(ret), 14132 i40e_aq_str(&pf->hw, 14133 pf->hw.aq.asq_last_status)); 14134 ret = -ENOENT; 14135 goto err; 14136 } 14137 vsi->info = ctxt.info; 14138 vsi->info.valid_sections = 0; 14139 vsi->seid = ctxt.seid; 14140 vsi->id = ctxt.vsi_number; 14141 } 14142 14143 spin_lock_bh(&vsi->mac_filter_hash_lock); 14144 vsi->active_filters = 0; 14145 /* If macvlan filters already exist, force them to get loaded */ 14146 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 14147 f->state = I40E_FILTER_NEW; 14148 f_count++; 14149 } 14150 spin_unlock_bh(&vsi->mac_filter_hash_lock); 14151 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 14152 14153 if (f_count) { 14154 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 14155 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 14156 } 14157 14158 /* Update VSI BW information */ 14159 ret = i40e_vsi_get_bw_info(vsi); 14160 if (ret) { 14161 dev_info(&pf->pdev->dev, 14162 "couldn't get vsi bw info, err %pe aq_err %s\n", 14163 ERR_PTR(ret), 14164 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14165 /* VSI is already added so not tearing that up */ 14166 ret = 0; 14167 } 14168 14169 err: 14170 return ret; 14171 } 14172 14173 /** 14174 * i40e_vsi_release - Delete a VSI and free its resources 14175 * @vsi: the VSI being removed 14176 * 14177 * Returns 0 on success or < 0 on error 14178 **/ 14179 int i40e_vsi_release(struct i40e_vsi *vsi) 14180 { 14181 struct i40e_mac_filter *f; 14182 struct hlist_node *h; 14183 struct i40e_veb *veb = NULL; 14184 struct i40e_pf *pf; 14185 u16 uplink_seid; 14186 int i, n, bkt; 14187 14188 pf = vsi->back; 14189 14190 /* release of a VEB-owner or last VSI is not allowed */ 14191 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 14192 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 14193 vsi->seid, vsi->uplink_seid); 14194 return -ENODEV; 14195 } 14196 if (vsi == pf->vsi[pf->lan_vsi] && 14197 !test_bit(__I40E_DOWN, pf->state)) { 14198 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 14199 return -ENODEV; 14200 } 14201 set_bit(__I40E_VSI_RELEASING, vsi->state); 14202 uplink_seid = vsi->uplink_seid; 14203 if (vsi->type != I40E_VSI_SRIOV) { 14204 if (vsi->netdev_registered) { 14205 vsi->netdev_registered = false; 14206 if (vsi->netdev) { 14207 /* results in a call to i40e_close() */ 14208 unregister_netdev(vsi->netdev); 14209 } 14210 } else { 14211 i40e_vsi_close(vsi); 14212 } 14213 i40e_vsi_disable_irq(vsi); 14214 } 14215 14216 spin_lock_bh(&vsi->mac_filter_hash_lock); 14217 14218 /* clear the sync flag on all filters */ 14219 if (vsi->netdev) { 14220 __dev_uc_unsync(vsi->netdev, NULL); 14221 __dev_mc_unsync(vsi->netdev, NULL); 14222 } 14223 14224 /* make sure any remaining filters are marked for deletion */ 14225 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 14226 __i40e_del_filter(vsi, f); 14227 14228 spin_unlock_bh(&vsi->mac_filter_hash_lock); 14229 14230 i40e_sync_vsi_filters(vsi); 14231 14232 i40e_vsi_delete(vsi); 14233 i40e_vsi_free_q_vectors(vsi); 14234 if (vsi->netdev) { 14235 free_netdev(vsi->netdev); 14236 vsi->netdev = NULL; 14237 } 14238 i40e_vsi_clear_rings(vsi); 14239 i40e_vsi_clear(vsi); 14240 14241 /* If this was the last thing on the VEB, except for the 14242 * controlling VSI, remove the VEB, which puts the controlling 14243 * VSI onto the next level down in the switch. 14244 * 14245 * Well, okay, there's one more exception here: don't remove 14246 * the orphan VEBs yet. We'll wait for an explicit remove request 14247 * from up the network stack. 14248 */ 14249 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 14250 if (pf->vsi[i] && 14251 pf->vsi[i]->uplink_seid == uplink_seid && 14252 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 14253 n++; /* count the VSIs */ 14254 } 14255 } 14256 for (i = 0; i < I40E_MAX_VEB; i++) { 14257 if (!pf->veb[i]) 14258 continue; 14259 if (pf->veb[i]->uplink_seid == uplink_seid) 14260 n++; /* count the VEBs */ 14261 if (pf->veb[i]->seid == uplink_seid) 14262 veb = pf->veb[i]; 14263 } 14264 if (n == 0 && veb && veb->uplink_seid != 0) 14265 i40e_veb_release(veb); 14266 14267 return 0; 14268 } 14269 14270 /** 14271 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 14272 * @vsi: ptr to the VSI 14273 * 14274 * This should only be called after i40e_vsi_mem_alloc() which allocates the 14275 * corresponding SW VSI structure and initializes num_queue_pairs for the 14276 * newly allocated VSI. 14277 * 14278 * Returns 0 on success or negative on failure 14279 **/ 14280 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 14281 { 14282 int ret = -ENOENT; 14283 struct i40e_pf *pf = vsi->back; 14284 14285 if (vsi->q_vectors[0]) { 14286 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 14287 vsi->seid); 14288 return -EEXIST; 14289 } 14290 14291 if (vsi->base_vector) { 14292 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 14293 vsi->seid, vsi->base_vector); 14294 return -EEXIST; 14295 } 14296 14297 ret = i40e_vsi_alloc_q_vectors(vsi); 14298 if (ret) { 14299 dev_info(&pf->pdev->dev, 14300 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 14301 vsi->num_q_vectors, vsi->seid, ret); 14302 vsi->num_q_vectors = 0; 14303 goto vector_setup_out; 14304 } 14305 14306 /* In Legacy mode, we do not have to get any other vector since we 14307 * piggyback on the misc/ICR0 for queue interrupts. 14308 */ 14309 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 14310 return ret; 14311 if (vsi->num_q_vectors) 14312 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 14313 vsi->num_q_vectors, vsi->idx); 14314 if (vsi->base_vector < 0) { 14315 dev_info(&pf->pdev->dev, 14316 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 14317 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 14318 i40e_vsi_free_q_vectors(vsi); 14319 ret = -ENOENT; 14320 goto vector_setup_out; 14321 } 14322 14323 vector_setup_out: 14324 return ret; 14325 } 14326 14327 /** 14328 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 14329 * @vsi: pointer to the vsi. 14330 * 14331 * This re-allocates a vsi's queue resources. 14332 * 14333 * Returns pointer to the successfully allocated and configured VSI sw struct 14334 * on success, otherwise returns NULL on failure. 14335 **/ 14336 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 14337 { 14338 u16 alloc_queue_pairs; 14339 struct i40e_pf *pf; 14340 u8 enabled_tc; 14341 int ret; 14342 14343 if (!vsi) 14344 return NULL; 14345 14346 pf = vsi->back; 14347 14348 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 14349 i40e_vsi_clear_rings(vsi); 14350 14351 i40e_vsi_free_arrays(vsi, false); 14352 i40e_set_num_rings_in_vsi(vsi); 14353 ret = i40e_vsi_alloc_arrays(vsi, false); 14354 if (ret) 14355 goto err_vsi; 14356 14357 alloc_queue_pairs = vsi->alloc_queue_pairs * 14358 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 14359 14360 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 14361 if (ret < 0) { 14362 dev_info(&pf->pdev->dev, 14363 "failed to get tracking for %d queues for VSI %d err %d\n", 14364 alloc_queue_pairs, vsi->seid, ret); 14365 goto err_vsi; 14366 } 14367 vsi->base_queue = ret; 14368 14369 /* Update the FW view of the VSI. Force a reset of TC and queue 14370 * layout configurations. 14371 */ 14372 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 14373 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 14374 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 14375 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 14376 if (vsi->type == I40E_VSI_MAIN) 14377 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 14378 14379 /* assign it some queues */ 14380 ret = i40e_alloc_rings(vsi); 14381 if (ret) 14382 goto err_rings; 14383 14384 /* map all of the rings to the q_vectors */ 14385 i40e_vsi_map_rings_to_vectors(vsi); 14386 return vsi; 14387 14388 err_rings: 14389 i40e_vsi_free_q_vectors(vsi); 14390 if (vsi->netdev_registered) { 14391 vsi->netdev_registered = false; 14392 unregister_netdev(vsi->netdev); 14393 free_netdev(vsi->netdev); 14394 vsi->netdev = NULL; 14395 } 14396 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 14397 err_vsi: 14398 i40e_vsi_clear(vsi); 14399 return NULL; 14400 } 14401 14402 /** 14403 * i40e_vsi_setup - Set up a VSI by a given type 14404 * @pf: board private structure 14405 * @type: VSI type 14406 * @uplink_seid: the switch element to link to 14407 * @param1: usage depends upon VSI type. For VF types, indicates VF id 14408 * 14409 * This allocates the sw VSI structure and its queue resources, then add a VSI 14410 * to the identified VEB. 14411 * 14412 * Returns pointer to the successfully allocated and configure VSI sw struct on 14413 * success, otherwise returns NULL on failure. 14414 **/ 14415 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 14416 u16 uplink_seid, u32 param1) 14417 { 14418 struct i40e_vsi *vsi = NULL; 14419 struct i40e_veb *veb = NULL; 14420 u16 alloc_queue_pairs; 14421 int ret, i; 14422 int v_idx; 14423 14424 /* The requested uplink_seid must be either 14425 * - the PF's port seid 14426 * no VEB is needed because this is the PF 14427 * or this is a Flow Director special case VSI 14428 * - seid of an existing VEB 14429 * - seid of a VSI that owns an existing VEB 14430 * - seid of a VSI that doesn't own a VEB 14431 * a new VEB is created and the VSI becomes the owner 14432 * - seid of the PF VSI, which is what creates the first VEB 14433 * this is a special case of the previous 14434 * 14435 * Find which uplink_seid we were given and create a new VEB if needed 14436 */ 14437 for (i = 0; i < I40E_MAX_VEB; i++) { 14438 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 14439 veb = pf->veb[i]; 14440 break; 14441 } 14442 } 14443 14444 if (!veb && uplink_seid != pf->mac_seid) { 14445 14446 for (i = 0; i < pf->num_alloc_vsi; i++) { 14447 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 14448 vsi = pf->vsi[i]; 14449 break; 14450 } 14451 } 14452 if (!vsi) { 14453 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 14454 uplink_seid); 14455 return NULL; 14456 } 14457 14458 if (vsi->uplink_seid == pf->mac_seid) 14459 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 14460 vsi->tc_config.enabled_tc); 14461 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 14462 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 14463 vsi->tc_config.enabled_tc); 14464 if (veb) { 14465 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 14466 dev_info(&vsi->back->pdev->dev, 14467 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 14468 return NULL; 14469 } 14470 /* We come up by default in VEPA mode if SRIOV is not 14471 * already enabled, in which case we can't force VEPA 14472 * mode. 14473 */ 14474 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 14475 veb->bridge_mode = BRIDGE_MODE_VEPA; 14476 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 14477 } 14478 i40e_config_bridge_mode(veb); 14479 } 14480 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 14481 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 14482 veb = pf->veb[i]; 14483 } 14484 if (!veb) { 14485 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 14486 return NULL; 14487 } 14488 14489 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14490 uplink_seid = veb->seid; 14491 } 14492 14493 /* get vsi sw struct */ 14494 v_idx = i40e_vsi_mem_alloc(pf, type); 14495 if (v_idx < 0) 14496 goto err_alloc; 14497 vsi = pf->vsi[v_idx]; 14498 if (!vsi) 14499 goto err_alloc; 14500 vsi->type = type; 14501 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 14502 14503 if (type == I40E_VSI_MAIN) 14504 pf->lan_vsi = v_idx; 14505 else if (type == I40E_VSI_SRIOV) 14506 vsi->vf_id = param1; 14507 /* assign it some queues */ 14508 alloc_queue_pairs = vsi->alloc_queue_pairs * 14509 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 14510 14511 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 14512 if (ret < 0) { 14513 dev_info(&pf->pdev->dev, 14514 "failed to get tracking for %d queues for VSI %d err=%d\n", 14515 alloc_queue_pairs, vsi->seid, ret); 14516 goto err_vsi; 14517 } 14518 vsi->base_queue = ret; 14519 14520 /* get a VSI from the hardware */ 14521 vsi->uplink_seid = uplink_seid; 14522 ret = i40e_add_vsi(vsi); 14523 if (ret) 14524 goto err_vsi; 14525 14526 switch (vsi->type) { 14527 /* setup the netdev if needed */ 14528 case I40E_VSI_MAIN: 14529 case I40E_VSI_VMDQ2: 14530 ret = i40e_config_netdev(vsi); 14531 if (ret) 14532 goto err_netdev; 14533 ret = i40e_netif_set_realnum_tx_rx_queues(vsi); 14534 if (ret) 14535 goto err_netdev; 14536 ret = register_netdev(vsi->netdev); 14537 if (ret) 14538 goto err_netdev; 14539 vsi->netdev_registered = true; 14540 netif_carrier_off(vsi->netdev); 14541 #ifdef CONFIG_I40E_DCB 14542 /* Setup DCB netlink interface */ 14543 i40e_dcbnl_setup(vsi); 14544 #endif /* CONFIG_I40E_DCB */ 14545 fallthrough; 14546 case I40E_VSI_FDIR: 14547 /* set up vectors and rings if needed */ 14548 ret = i40e_vsi_setup_vectors(vsi); 14549 if (ret) 14550 goto err_msix; 14551 14552 ret = i40e_alloc_rings(vsi); 14553 if (ret) 14554 goto err_rings; 14555 14556 /* map all of the rings to the q_vectors */ 14557 i40e_vsi_map_rings_to_vectors(vsi); 14558 14559 i40e_vsi_reset_stats(vsi); 14560 break; 14561 default: 14562 /* no netdev or rings for the other VSI types */ 14563 break; 14564 } 14565 14566 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 14567 (vsi->type == I40E_VSI_VMDQ2)) { 14568 ret = i40e_vsi_config_rss(vsi); 14569 } 14570 return vsi; 14571 14572 err_rings: 14573 i40e_vsi_free_q_vectors(vsi); 14574 err_msix: 14575 if (vsi->netdev_registered) { 14576 vsi->netdev_registered = false; 14577 unregister_netdev(vsi->netdev); 14578 free_netdev(vsi->netdev); 14579 vsi->netdev = NULL; 14580 } 14581 err_netdev: 14582 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 14583 err_vsi: 14584 i40e_vsi_clear(vsi); 14585 err_alloc: 14586 return NULL; 14587 } 14588 14589 /** 14590 * i40e_veb_get_bw_info - Query VEB BW information 14591 * @veb: the veb to query 14592 * 14593 * Query the Tx scheduler BW configuration data for given VEB 14594 **/ 14595 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 14596 { 14597 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 14598 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 14599 struct i40e_pf *pf = veb->pf; 14600 struct i40e_hw *hw = &pf->hw; 14601 u32 tc_bw_max; 14602 int ret = 0; 14603 int i; 14604 14605 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 14606 &bw_data, NULL); 14607 if (ret) { 14608 dev_info(&pf->pdev->dev, 14609 "query veb bw config failed, err %pe aq_err %s\n", 14610 ERR_PTR(ret), 14611 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 14612 goto out; 14613 } 14614 14615 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 14616 &ets_data, NULL); 14617 if (ret) { 14618 dev_info(&pf->pdev->dev, 14619 "query veb bw ets config failed, err %pe aq_err %s\n", 14620 ERR_PTR(ret), 14621 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 14622 goto out; 14623 } 14624 14625 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 14626 veb->bw_max_quanta = ets_data.tc_bw_max; 14627 veb->is_abs_credits = bw_data.absolute_credits_enable; 14628 veb->enabled_tc = ets_data.tc_valid_bits; 14629 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 14630 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 14631 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 14632 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 14633 veb->bw_tc_limit_credits[i] = 14634 le16_to_cpu(bw_data.tc_bw_limits[i]); 14635 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 14636 } 14637 14638 out: 14639 return ret; 14640 } 14641 14642 /** 14643 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 14644 * @pf: board private structure 14645 * 14646 * On error: returns error code (negative) 14647 * On success: returns vsi index in PF (positive) 14648 **/ 14649 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 14650 { 14651 int ret = -ENOENT; 14652 struct i40e_veb *veb; 14653 int i; 14654 14655 /* Need to protect the allocation of switch elements at the PF level */ 14656 mutex_lock(&pf->switch_mutex); 14657 14658 /* VEB list may be fragmented if VEB creation/destruction has 14659 * been happening. We can afford to do a quick scan to look 14660 * for any free slots in the list. 14661 * 14662 * find next empty veb slot, looping back around if necessary 14663 */ 14664 i = 0; 14665 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 14666 i++; 14667 if (i >= I40E_MAX_VEB) { 14668 ret = -ENOMEM; 14669 goto err_alloc_veb; /* out of VEB slots! */ 14670 } 14671 14672 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 14673 if (!veb) { 14674 ret = -ENOMEM; 14675 goto err_alloc_veb; 14676 } 14677 veb->pf = pf; 14678 veb->idx = i; 14679 veb->enabled_tc = 1; 14680 14681 pf->veb[i] = veb; 14682 ret = i; 14683 err_alloc_veb: 14684 mutex_unlock(&pf->switch_mutex); 14685 return ret; 14686 } 14687 14688 /** 14689 * i40e_switch_branch_release - Delete a branch of the switch tree 14690 * @branch: where to start deleting 14691 * 14692 * This uses recursion to find the tips of the branch to be 14693 * removed, deleting until we get back to and can delete this VEB. 14694 **/ 14695 static void i40e_switch_branch_release(struct i40e_veb *branch) 14696 { 14697 struct i40e_pf *pf = branch->pf; 14698 u16 branch_seid = branch->seid; 14699 u16 veb_idx = branch->idx; 14700 int i; 14701 14702 /* release any VEBs on this VEB - RECURSION */ 14703 for (i = 0; i < I40E_MAX_VEB; i++) { 14704 if (!pf->veb[i]) 14705 continue; 14706 if (pf->veb[i]->uplink_seid == branch->seid) 14707 i40e_switch_branch_release(pf->veb[i]); 14708 } 14709 14710 /* Release the VSIs on this VEB, but not the owner VSI. 14711 * 14712 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 14713 * the VEB itself, so don't use (*branch) after this loop. 14714 */ 14715 for (i = 0; i < pf->num_alloc_vsi; i++) { 14716 if (!pf->vsi[i]) 14717 continue; 14718 if (pf->vsi[i]->uplink_seid == branch_seid && 14719 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 14720 i40e_vsi_release(pf->vsi[i]); 14721 } 14722 } 14723 14724 /* There's one corner case where the VEB might not have been 14725 * removed, so double check it here and remove it if needed. 14726 * This case happens if the veb was created from the debugfs 14727 * commands and no VSIs were added to it. 14728 */ 14729 if (pf->veb[veb_idx]) 14730 i40e_veb_release(pf->veb[veb_idx]); 14731 } 14732 14733 /** 14734 * i40e_veb_clear - remove veb struct 14735 * @veb: the veb to remove 14736 **/ 14737 static void i40e_veb_clear(struct i40e_veb *veb) 14738 { 14739 if (!veb) 14740 return; 14741 14742 if (veb->pf) { 14743 struct i40e_pf *pf = veb->pf; 14744 14745 mutex_lock(&pf->switch_mutex); 14746 if (pf->veb[veb->idx] == veb) 14747 pf->veb[veb->idx] = NULL; 14748 mutex_unlock(&pf->switch_mutex); 14749 } 14750 14751 kfree(veb); 14752 } 14753 14754 /** 14755 * i40e_veb_release - Delete a VEB and free its resources 14756 * @veb: the VEB being removed 14757 **/ 14758 void i40e_veb_release(struct i40e_veb *veb) 14759 { 14760 struct i40e_vsi *vsi = NULL; 14761 struct i40e_pf *pf; 14762 int i, n = 0; 14763 14764 pf = veb->pf; 14765 14766 /* find the remaining VSI and check for extras */ 14767 for (i = 0; i < pf->num_alloc_vsi; i++) { 14768 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 14769 n++; 14770 vsi = pf->vsi[i]; 14771 } 14772 } 14773 if (n != 1) { 14774 dev_info(&pf->pdev->dev, 14775 "can't remove VEB %d with %d VSIs left\n", 14776 veb->seid, n); 14777 return; 14778 } 14779 14780 /* move the remaining VSI to uplink veb */ 14781 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 14782 if (veb->uplink_seid) { 14783 vsi->uplink_seid = veb->uplink_seid; 14784 if (veb->uplink_seid == pf->mac_seid) 14785 vsi->veb_idx = I40E_NO_VEB; 14786 else 14787 vsi->veb_idx = veb->veb_idx; 14788 } else { 14789 /* floating VEB */ 14790 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 14791 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 14792 } 14793 14794 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14795 i40e_veb_clear(veb); 14796 } 14797 14798 /** 14799 * i40e_add_veb - create the VEB in the switch 14800 * @veb: the VEB to be instantiated 14801 * @vsi: the controlling VSI 14802 **/ 14803 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 14804 { 14805 struct i40e_pf *pf = veb->pf; 14806 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 14807 int ret; 14808 14809 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 14810 veb->enabled_tc, false, 14811 &veb->seid, enable_stats, NULL); 14812 14813 /* get a VEB from the hardware */ 14814 if (ret) { 14815 dev_info(&pf->pdev->dev, 14816 "couldn't add VEB, err %pe aq_err %s\n", 14817 ERR_PTR(ret), 14818 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14819 return -EPERM; 14820 } 14821 14822 /* get statistics counter */ 14823 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 14824 &veb->stats_idx, NULL, NULL, NULL); 14825 if (ret) { 14826 dev_info(&pf->pdev->dev, 14827 "couldn't get VEB statistics idx, err %pe aq_err %s\n", 14828 ERR_PTR(ret), 14829 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14830 return -EPERM; 14831 } 14832 ret = i40e_veb_get_bw_info(veb); 14833 if (ret) { 14834 dev_info(&pf->pdev->dev, 14835 "couldn't get VEB bw info, err %pe aq_err %s\n", 14836 ERR_PTR(ret), 14837 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14838 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14839 return -ENOENT; 14840 } 14841 14842 vsi->uplink_seid = veb->seid; 14843 vsi->veb_idx = veb->idx; 14844 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14845 14846 return 0; 14847 } 14848 14849 /** 14850 * i40e_veb_setup - Set up a VEB 14851 * @pf: board private structure 14852 * @flags: VEB setup flags 14853 * @uplink_seid: the switch element to link to 14854 * @vsi_seid: the initial VSI seid 14855 * @enabled_tc: Enabled TC bit-map 14856 * 14857 * This allocates the sw VEB structure and links it into the switch 14858 * It is possible and legal for this to be a duplicate of an already 14859 * existing VEB. It is also possible for both uplink and vsi seids 14860 * to be zero, in order to create a floating VEB. 14861 * 14862 * Returns pointer to the successfully allocated VEB sw struct on 14863 * success, otherwise returns NULL on failure. 14864 **/ 14865 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 14866 u16 uplink_seid, u16 vsi_seid, 14867 u8 enabled_tc) 14868 { 14869 struct i40e_veb *veb, *uplink_veb = NULL; 14870 int vsi_idx, veb_idx; 14871 int ret; 14872 14873 /* if one seid is 0, the other must be 0 to create a floating relay */ 14874 if ((uplink_seid == 0 || vsi_seid == 0) && 14875 (uplink_seid + vsi_seid != 0)) { 14876 dev_info(&pf->pdev->dev, 14877 "one, not both seid's are 0: uplink=%d vsi=%d\n", 14878 uplink_seid, vsi_seid); 14879 return NULL; 14880 } 14881 14882 /* make sure there is such a vsi and uplink */ 14883 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 14884 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 14885 break; 14886 if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) { 14887 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 14888 vsi_seid); 14889 return NULL; 14890 } 14891 14892 if (uplink_seid && uplink_seid != pf->mac_seid) { 14893 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 14894 if (pf->veb[veb_idx] && 14895 pf->veb[veb_idx]->seid == uplink_seid) { 14896 uplink_veb = pf->veb[veb_idx]; 14897 break; 14898 } 14899 } 14900 if (!uplink_veb) { 14901 dev_info(&pf->pdev->dev, 14902 "uplink seid %d not found\n", uplink_seid); 14903 return NULL; 14904 } 14905 } 14906 14907 /* get veb sw struct */ 14908 veb_idx = i40e_veb_mem_alloc(pf); 14909 if (veb_idx < 0) 14910 goto err_alloc; 14911 veb = pf->veb[veb_idx]; 14912 veb->flags = flags; 14913 veb->uplink_seid = uplink_seid; 14914 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 14915 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 14916 14917 /* create the VEB in the switch */ 14918 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 14919 if (ret) 14920 goto err_veb; 14921 if (vsi_idx == pf->lan_vsi) 14922 pf->lan_veb = veb->idx; 14923 14924 return veb; 14925 14926 err_veb: 14927 i40e_veb_clear(veb); 14928 err_alloc: 14929 return NULL; 14930 } 14931 14932 /** 14933 * i40e_setup_pf_switch_element - set PF vars based on switch type 14934 * @pf: board private structure 14935 * @ele: element we are building info from 14936 * @num_reported: total number of elements 14937 * @printconfig: should we print the contents 14938 * 14939 * helper function to assist in extracting a few useful SEID values. 14940 **/ 14941 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 14942 struct i40e_aqc_switch_config_element_resp *ele, 14943 u16 num_reported, bool printconfig) 14944 { 14945 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 14946 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 14947 u8 element_type = ele->element_type; 14948 u16 seid = le16_to_cpu(ele->seid); 14949 14950 if (printconfig) 14951 dev_info(&pf->pdev->dev, 14952 "type=%d seid=%d uplink=%d downlink=%d\n", 14953 element_type, seid, uplink_seid, downlink_seid); 14954 14955 switch (element_type) { 14956 case I40E_SWITCH_ELEMENT_TYPE_MAC: 14957 pf->mac_seid = seid; 14958 break; 14959 case I40E_SWITCH_ELEMENT_TYPE_VEB: 14960 /* Main VEB? */ 14961 if (uplink_seid != pf->mac_seid) 14962 break; 14963 if (pf->lan_veb >= I40E_MAX_VEB) { 14964 int v; 14965 14966 /* find existing or else empty VEB */ 14967 for (v = 0; v < I40E_MAX_VEB; v++) { 14968 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 14969 pf->lan_veb = v; 14970 break; 14971 } 14972 } 14973 if (pf->lan_veb >= I40E_MAX_VEB) { 14974 v = i40e_veb_mem_alloc(pf); 14975 if (v < 0) 14976 break; 14977 pf->lan_veb = v; 14978 } 14979 } 14980 if (pf->lan_veb >= I40E_MAX_VEB) 14981 break; 14982 14983 pf->veb[pf->lan_veb]->seid = seid; 14984 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 14985 pf->veb[pf->lan_veb]->pf = pf; 14986 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 14987 break; 14988 case I40E_SWITCH_ELEMENT_TYPE_VSI: 14989 if (num_reported != 1) 14990 break; 14991 /* This is immediately after a reset so we can assume this is 14992 * the PF's VSI 14993 */ 14994 pf->mac_seid = uplink_seid; 14995 pf->pf_seid = downlink_seid; 14996 pf->main_vsi_seid = seid; 14997 if (printconfig) 14998 dev_info(&pf->pdev->dev, 14999 "pf_seid=%d main_vsi_seid=%d\n", 15000 pf->pf_seid, pf->main_vsi_seid); 15001 break; 15002 case I40E_SWITCH_ELEMENT_TYPE_PF: 15003 case I40E_SWITCH_ELEMENT_TYPE_VF: 15004 case I40E_SWITCH_ELEMENT_TYPE_EMP: 15005 case I40E_SWITCH_ELEMENT_TYPE_BMC: 15006 case I40E_SWITCH_ELEMENT_TYPE_PE: 15007 case I40E_SWITCH_ELEMENT_TYPE_PA: 15008 /* ignore these for now */ 15009 break; 15010 default: 15011 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 15012 element_type, seid); 15013 break; 15014 } 15015 } 15016 15017 /** 15018 * i40e_fetch_switch_configuration - Get switch config from firmware 15019 * @pf: board private structure 15020 * @printconfig: should we print the contents 15021 * 15022 * Get the current switch configuration from the device and 15023 * extract a few useful SEID values. 15024 **/ 15025 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 15026 { 15027 struct i40e_aqc_get_switch_config_resp *sw_config; 15028 u16 next_seid = 0; 15029 int ret = 0; 15030 u8 *aq_buf; 15031 int i; 15032 15033 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 15034 if (!aq_buf) 15035 return -ENOMEM; 15036 15037 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 15038 do { 15039 u16 num_reported, num_total; 15040 15041 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 15042 I40E_AQ_LARGE_BUF, 15043 &next_seid, NULL); 15044 if (ret) { 15045 dev_info(&pf->pdev->dev, 15046 "get switch config failed err %d aq_err %s\n", 15047 ret, 15048 i40e_aq_str(&pf->hw, 15049 pf->hw.aq.asq_last_status)); 15050 kfree(aq_buf); 15051 return -ENOENT; 15052 } 15053 15054 num_reported = le16_to_cpu(sw_config->header.num_reported); 15055 num_total = le16_to_cpu(sw_config->header.num_total); 15056 15057 if (printconfig) 15058 dev_info(&pf->pdev->dev, 15059 "header: %d reported %d total\n", 15060 num_reported, num_total); 15061 15062 for (i = 0; i < num_reported; i++) { 15063 struct i40e_aqc_switch_config_element_resp *ele = 15064 &sw_config->element[i]; 15065 15066 i40e_setup_pf_switch_element(pf, ele, num_reported, 15067 printconfig); 15068 } 15069 } while (next_seid != 0); 15070 15071 kfree(aq_buf); 15072 return ret; 15073 } 15074 15075 /** 15076 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 15077 * @pf: board private structure 15078 * @reinit: if the Main VSI needs to re-initialized. 15079 * @lock_acquired: indicates whether or not the lock has been acquired 15080 * 15081 * Returns 0 on success, negative value on failure 15082 **/ 15083 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit, bool lock_acquired) 15084 { 15085 u16 flags = 0; 15086 int ret; 15087 15088 /* find out what's out there already */ 15089 ret = i40e_fetch_switch_configuration(pf, false); 15090 if (ret) { 15091 dev_info(&pf->pdev->dev, 15092 "couldn't fetch switch config, err %pe aq_err %s\n", 15093 ERR_PTR(ret), 15094 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15095 return ret; 15096 } 15097 i40e_pf_reset_stats(pf); 15098 15099 /* set the switch config bit for the whole device to 15100 * support limited promisc or true promisc 15101 * when user requests promisc. The default is limited 15102 * promisc. 15103 */ 15104 15105 if ((pf->hw.pf_id == 0) && 15106 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 15107 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 15108 pf->last_sw_conf_flags = flags; 15109 } 15110 15111 if (pf->hw.pf_id == 0) { 15112 u16 valid_flags; 15113 15114 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 15115 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 15116 NULL); 15117 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 15118 dev_info(&pf->pdev->dev, 15119 "couldn't set switch config bits, err %pe aq_err %s\n", 15120 ERR_PTR(ret), 15121 i40e_aq_str(&pf->hw, 15122 pf->hw.aq.asq_last_status)); 15123 /* not a fatal problem, just keep going */ 15124 } 15125 pf->last_sw_conf_valid_flags = valid_flags; 15126 } 15127 15128 /* first time setup */ 15129 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 15130 struct i40e_vsi *vsi = NULL; 15131 u16 uplink_seid; 15132 15133 /* Set up the PF VSI associated with the PF's main VSI 15134 * that is already in the HW switch 15135 */ 15136 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 15137 uplink_seid = pf->veb[pf->lan_veb]->seid; 15138 else 15139 uplink_seid = pf->mac_seid; 15140 if (pf->lan_vsi == I40E_NO_VSI) 15141 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 15142 else if (reinit) 15143 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 15144 if (!vsi) { 15145 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 15146 i40e_cloud_filter_exit(pf); 15147 i40e_fdir_teardown(pf); 15148 return -EAGAIN; 15149 } 15150 } else { 15151 /* force a reset of TC and queue layout configurations */ 15152 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 15153 15154 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 15155 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 15156 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 15157 } 15158 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 15159 15160 i40e_fdir_sb_setup(pf); 15161 15162 /* Setup static PF queue filter control settings */ 15163 ret = i40e_setup_pf_filter_control(pf); 15164 if (ret) { 15165 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 15166 ret); 15167 /* Failure here should not stop continuing other steps */ 15168 } 15169 15170 /* enable RSS in the HW, even for only one queue, as the stack can use 15171 * the hash 15172 */ 15173 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 15174 i40e_pf_config_rss(pf); 15175 15176 /* fill in link information and enable LSE reporting */ 15177 i40e_link_event(pf); 15178 15179 /* Initialize user-specific link properties */ 15180 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 15181 I40E_AQ_AN_COMPLETED) ? true : false); 15182 15183 i40e_ptp_init(pf); 15184 15185 if (!lock_acquired) 15186 rtnl_lock(); 15187 15188 /* repopulate tunnel port filters */ 15189 udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev); 15190 15191 if (!lock_acquired) 15192 rtnl_unlock(); 15193 15194 return ret; 15195 } 15196 15197 /** 15198 * i40e_determine_queue_usage - Work out queue distribution 15199 * @pf: board private structure 15200 **/ 15201 static void i40e_determine_queue_usage(struct i40e_pf *pf) 15202 { 15203 int queues_left; 15204 int q_max; 15205 15206 pf->num_lan_qps = 0; 15207 15208 /* Find the max queues to be put into basic use. We'll always be 15209 * using TC0, whether or not DCB is running, and TC0 will get the 15210 * big RSS set. 15211 */ 15212 queues_left = pf->hw.func_caps.num_tx_qp; 15213 15214 if ((queues_left == 1) || 15215 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 15216 /* one qp for PF, no queues for anything else */ 15217 queues_left = 0; 15218 pf->alloc_rss_size = pf->num_lan_qps = 1; 15219 15220 /* make sure all the fancies are disabled */ 15221 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 15222 I40E_FLAG_IWARP_ENABLED | 15223 I40E_FLAG_FD_SB_ENABLED | 15224 I40E_FLAG_FD_ATR_ENABLED | 15225 I40E_FLAG_DCB_CAPABLE | 15226 I40E_FLAG_DCB_ENABLED | 15227 I40E_FLAG_SRIOV_ENABLED | 15228 I40E_FLAG_VMDQ_ENABLED); 15229 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 15230 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 15231 I40E_FLAG_FD_SB_ENABLED | 15232 I40E_FLAG_FD_ATR_ENABLED | 15233 I40E_FLAG_DCB_CAPABLE))) { 15234 /* one qp for PF */ 15235 pf->alloc_rss_size = pf->num_lan_qps = 1; 15236 queues_left -= pf->num_lan_qps; 15237 15238 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 15239 I40E_FLAG_IWARP_ENABLED | 15240 I40E_FLAG_FD_SB_ENABLED | 15241 I40E_FLAG_FD_ATR_ENABLED | 15242 I40E_FLAG_DCB_ENABLED | 15243 I40E_FLAG_VMDQ_ENABLED); 15244 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 15245 } else { 15246 /* Not enough queues for all TCs */ 15247 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 15248 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 15249 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 15250 I40E_FLAG_DCB_ENABLED); 15251 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 15252 } 15253 15254 /* limit lan qps to the smaller of qps, cpus or msix */ 15255 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 15256 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 15257 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 15258 pf->num_lan_qps = q_max; 15259 15260 queues_left -= pf->num_lan_qps; 15261 } 15262 15263 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 15264 if (queues_left > 1) { 15265 queues_left -= 1; /* save 1 queue for FD */ 15266 } else { 15267 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 15268 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 15269 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 15270 } 15271 } 15272 15273 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15274 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 15275 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 15276 (queues_left / pf->num_vf_qps)); 15277 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 15278 } 15279 15280 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 15281 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 15282 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 15283 (queues_left / pf->num_vmdq_qps)); 15284 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 15285 } 15286 15287 pf->queues_left = queues_left; 15288 dev_dbg(&pf->pdev->dev, 15289 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 15290 pf->hw.func_caps.num_tx_qp, 15291 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 15292 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 15293 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 15294 queues_left); 15295 } 15296 15297 /** 15298 * i40e_setup_pf_filter_control - Setup PF static filter control 15299 * @pf: PF to be setup 15300 * 15301 * i40e_setup_pf_filter_control sets up a PF's initial filter control 15302 * settings. If PE/FCoE are enabled then it will also set the per PF 15303 * based filter sizes required for them. It also enables Flow director, 15304 * ethertype and macvlan type filter settings for the pf. 15305 * 15306 * Returns 0 on success, negative on failure 15307 **/ 15308 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 15309 { 15310 struct i40e_filter_control_settings *settings = &pf->filter_settings; 15311 15312 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 15313 15314 /* Flow Director is enabled */ 15315 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 15316 settings->enable_fdir = true; 15317 15318 /* Ethtype and MACVLAN filters enabled for PF */ 15319 settings->enable_ethtype = true; 15320 settings->enable_macvlan = true; 15321 15322 if (i40e_set_filter_control(&pf->hw, settings)) 15323 return -ENOENT; 15324 15325 return 0; 15326 } 15327 15328 #define INFO_STRING_LEN 255 15329 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 15330 static void i40e_print_features(struct i40e_pf *pf) 15331 { 15332 struct i40e_hw *hw = &pf->hw; 15333 char *buf; 15334 int i; 15335 15336 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 15337 if (!buf) 15338 return; 15339 15340 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 15341 #ifdef CONFIG_PCI_IOV 15342 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 15343 #endif 15344 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 15345 pf->hw.func_caps.num_vsis, 15346 pf->vsi[pf->lan_vsi]->num_queue_pairs); 15347 if (pf->flags & I40E_FLAG_RSS_ENABLED) 15348 i += scnprintf(&buf[i], REMAIN(i), " RSS"); 15349 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 15350 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); 15351 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 15352 i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); 15353 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); 15354 } 15355 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 15356 i += scnprintf(&buf[i], REMAIN(i), " DCB"); 15357 i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); 15358 i += scnprintf(&buf[i], REMAIN(i), " Geneve"); 15359 if (pf->flags & I40E_FLAG_PTP) 15360 i += scnprintf(&buf[i], REMAIN(i), " PTP"); 15361 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 15362 i += scnprintf(&buf[i], REMAIN(i), " VEB"); 15363 else 15364 i += scnprintf(&buf[i], REMAIN(i), " VEPA"); 15365 15366 dev_info(&pf->pdev->dev, "%s\n", buf); 15367 kfree(buf); 15368 WARN_ON(i > INFO_STRING_LEN); 15369 } 15370 15371 /** 15372 * i40e_get_platform_mac_addr - get platform-specific MAC address 15373 * @pdev: PCI device information struct 15374 * @pf: board private structure 15375 * 15376 * Look up the MAC address for the device. First we'll try 15377 * eth_platform_get_mac_address, which will check Open Firmware, or arch 15378 * specific fallback. Otherwise, we'll default to the stored value in 15379 * firmware. 15380 **/ 15381 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 15382 { 15383 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 15384 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 15385 } 15386 15387 /** 15388 * i40e_set_fec_in_flags - helper function for setting FEC options in flags 15389 * @fec_cfg: FEC option to set in flags 15390 * @flags: ptr to flags in which we set FEC option 15391 **/ 15392 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags) 15393 { 15394 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) 15395 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC; 15396 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) || 15397 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) { 15398 *flags |= I40E_FLAG_RS_FEC; 15399 *flags &= ~I40E_FLAG_BASE_R_FEC; 15400 } 15401 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) || 15402 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) { 15403 *flags |= I40E_FLAG_BASE_R_FEC; 15404 *flags &= ~I40E_FLAG_RS_FEC; 15405 } 15406 if (fec_cfg == 0) 15407 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC); 15408 } 15409 15410 /** 15411 * i40e_check_recovery_mode - check if we are running transition firmware 15412 * @pf: board private structure 15413 * 15414 * Check registers indicating the firmware runs in recovery mode. Sets the 15415 * appropriate driver state. 15416 * 15417 * Returns true if the recovery mode was detected, false otherwise 15418 **/ 15419 static bool i40e_check_recovery_mode(struct i40e_pf *pf) 15420 { 15421 u32 val = rd32(&pf->hw, I40E_GL_FWSTS); 15422 15423 if (val & I40E_GL_FWSTS_FWS1B_MASK) { 15424 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n"); 15425 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 15426 set_bit(__I40E_RECOVERY_MODE, pf->state); 15427 15428 return true; 15429 } 15430 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15431 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n"); 15432 15433 return false; 15434 } 15435 15436 /** 15437 * i40e_pf_loop_reset - perform reset in a loop. 15438 * @pf: board private structure 15439 * 15440 * This function is useful when a NIC is about to enter recovery mode. 15441 * When a NIC's internal data structures are corrupted the NIC's 15442 * firmware is going to enter recovery mode. 15443 * Right after a POR it takes about 7 minutes for firmware to enter 15444 * recovery mode. Until that time a NIC is in some kind of intermediate 15445 * state. After that time period the NIC almost surely enters 15446 * recovery mode. The only way for a driver to detect intermediate 15447 * state is to issue a series of pf-resets and check a return value. 15448 * If a PF reset returns success then the firmware could be in recovery 15449 * mode so the caller of this code needs to check for recovery mode 15450 * if this function returns success. There is a little chance that 15451 * firmware will hang in intermediate state forever. 15452 * Since waiting 7 minutes is quite a lot of time this function waits 15453 * 10 seconds and then gives up by returning an error. 15454 * 15455 * Return 0 on success, negative on failure. 15456 **/ 15457 static int i40e_pf_loop_reset(struct i40e_pf *pf) 15458 { 15459 /* wait max 10 seconds for PF reset to succeed */ 15460 const unsigned long time_end = jiffies + 10 * HZ; 15461 struct i40e_hw *hw = &pf->hw; 15462 int ret; 15463 15464 ret = i40e_pf_reset(hw); 15465 while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) { 15466 usleep_range(10000, 20000); 15467 ret = i40e_pf_reset(hw); 15468 } 15469 15470 if (ret == I40E_SUCCESS) 15471 pf->pfr_count++; 15472 else 15473 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); 15474 15475 return ret; 15476 } 15477 15478 /** 15479 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset 15480 * @pf: board private structure 15481 * 15482 * Check FW registers to determine if FW issued unexpected EMP Reset. 15483 * Every time when unexpected EMP Reset occurs the FW increments 15484 * a counter of unexpected EMP Resets. When the counter reaches 10 15485 * the FW should enter the Recovery mode 15486 * 15487 * Returns true if FW issued unexpected EMP Reset 15488 **/ 15489 static bool i40e_check_fw_empr(struct i40e_pf *pf) 15490 { 15491 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) & 15492 I40E_GL_FWSTS_FWS1B_MASK; 15493 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) && 15494 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10); 15495 } 15496 15497 /** 15498 * i40e_handle_resets - handle EMP resets and PF resets 15499 * @pf: board private structure 15500 * 15501 * Handle both EMP resets and PF resets and conclude whether there are 15502 * any issues regarding these resets. If there are any issues then 15503 * generate log entry. 15504 * 15505 * Return 0 if NIC is healthy or negative value when there are issues 15506 * with resets 15507 **/ 15508 static int i40e_handle_resets(struct i40e_pf *pf) 15509 { 15510 const int pfr = i40e_pf_loop_reset(pf); 15511 const bool is_empr = i40e_check_fw_empr(pf); 15512 15513 if (is_empr || pfr != I40E_SUCCESS) 15514 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n"); 15515 15516 return is_empr ? I40E_ERR_RESET_FAILED : pfr; 15517 } 15518 15519 /** 15520 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode 15521 * @pf: board private structure 15522 * @hw: ptr to the hardware info 15523 * 15524 * This function does a minimal setup of all subsystems needed for running 15525 * recovery mode. 15526 * 15527 * Returns 0 on success, negative on failure 15528 **/ 15529 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw) 15530 { 15531 struct i40e_vsi *vsi; 15532 int err; 15533 int v_idx; 15534 15535 pci_set_drvdata(pf->pdev, pf); 15536 pci_save_state(pf->pdev); 15537 15538 /* set up periodic task facility */ 15539 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15540 pf->service_timer_period = HZ; 15541 15542 INIT_WORK(&pf->service_task, i40e_service_task); 15543 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15544 15545 err = i40e_init_interrupt_scheme(pf); 15546 if (err) 15547 goto err_switch_setup; 15548 15549 /* The number of VSIs reported by the FW is the minimum guaranteed 15550 * to us; HW supports far more and we share the remaining pool with 15551 * the other PFs. We allocate space for more than the guarantee with 15552 * the understanding that we might not get them all later. 15553 */ 15554 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15555 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15556 else 15557 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15558 15559 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */ 15560 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15561 GFP_KERNEL); 15562 if (!pf->vsi) { 15563 err = -ENOMEM; 15564 goto err_switch_setup; 15565 } 15566 15567 /* We allocate one VSI which is needed as absolute minimum 15568 * in order to register the netdev 15569 */ 15570 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN); 15571 if (v_idx < 0) { 15572 err = v_idx; 15573 goto err_switch_setup; 15574 } 15575 pf->lan_vsi = v_idx; 15576 vsi = pf->vsi[v_idx]; 15577 if (!vsi) { 15578 err = -EFAULT; 15579 goto err_switch_setup; 15580 } 15581 vsi->alloc_queue_pairs = 1; 15582 err = i40e_config_netdev(vsi); 15583 if (err) 15584 goto err_switch_setup; 15585 err = register_netdev(vsi->netdev); 15586 if (err) 15587 goto err_switch_setup; 15588 vsi->netdev_registered = true; 15589 i40e_dbg_pf_init(pf); 15590 15591 err = i40e_setup_misc_vector_for_recovery_mode(pf); 15592 if (err) 15593 goto err_switch_setup; 15594 15595 /* tell the firmware that we're starting */ 15596 i40e_send_version(pf); 15597 15598 /* since everything's happy, start the service_task timer */ 15599 mod_timer(&pf->service_timer, 15600 round_jiffies(jiffies + pf->service_timer_period)); 15601 15602 return 0; 15603 15604 err_switch_setup: 15605 i40e_reset_interrupt_capability(pf); 15606 timer_shutdown_sync(&pf->service_timer); 15607 i40e_shutdown_adminq(hw); 15608 iounmap(hw->hw_addr); 15609 pci_release_mem_regions(pf->pdev); 15610 pci_disable_device(pf->pdev); 15611 kfree(pf); 15612 15613 return err; 15614 } 15615 15616 /** 15617 * i40e_set_subsystem_device_id - set subsystem device id 15618 * @hw: pointer to the hardware info 15619 * 15620 * Set PCI subsystem device id either from a pci_dev structure or 15621 * a specific FW register. 15622 **/ 15623 static inline void i40e_set_subsystem_device_id(struct i40e_hw *hw) 15624 { 15625 struct pci_dev *pdev = ((struct i40e_pf *)hw->back)->pdev; 15626 15627 hw->subsystem_device_id = pdev->subsystem_device ? 15628 pdev->subsystem_device : 15629 (ushort)(rd32(hw, I40E_PFPCI_SUBSYSID) & USHRT_MAX); 15630 } 15631 15632 /** 15633 * i40e_probe - Device initialization routine 15634 * @pdev: PCI device information struct 15635 * @ent: entry in i40e_pci_tbl 15636 * 15637 * i40e_probe initializes a PF identified by a pci_dev structure. 15638 * The OS initialization, configuring of the PF private structure, 15639 * and a hardware reset occur. 15640 * 15641 * Returns 0 on success, negative on failure 15642 **/ 15643 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 15644 { 15645 struct i40e_aq_get_phy_abilities_resp abilities; 15646 #ifdef CONFIG_I40E_DCB 15647 enum i40e_get_fw_lldp_status_resp lldp_status; 15648 #endif /* CONFIG_I40E_DCB */ 15649 struct i40e_pf *pf; 15650 struct i40e_hw *hw; 15651 static u16 pfs_found; 15652 u16 wol_nvm_bits; 15653 u16 link_status; 15654 #ifdef CONFIG_I40E_DCB 15655 int status; 15656 #endif /* CONFIG_I40E_DCB */ 15657 int err; 15658 u32 val; 15659 u32 i; 15660 15661 err = pci_enable_device_mem(pdev); 15662 if (err) 15663 return err; 15664 15665 /* set up for high or low dma */ 15666 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 15667 if (err) { 15668 dev_err(&pdev->dev, 15669 "DMA configuration failed: 0x%x\n", err); 15670 goto err_dma; 15671 } 15672 15673 /* set up pci connections */ 15674 err = pci_request_mem_regions(pdev, i40e_driver_name); 15675 if (err) { 15676 dev_info(&pdev->dev, 15677 "pci_request_selected_regions failed %d\n", err); 15678 goto err_pci_reg; 15679 } 15680 15681 pci_set_master(pdev); 15682 15683 /* Now that we have a PCI connection, we need to do the 15684 * low level device setup. This is primarily setting up 15685 * the Admin Queue structures and then querying for the 15686 * device's current profile information. 15687 */ 15688 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 15689 if (!pf) { 15690 err = -ENOMEM; 15691 goto err_pf_alloc; 15692 } 15693 pf->next_vsi = 0; 15694 pf->pdev = pdev; 15695 set_bit(__I40E_DOWN, pf->state); 15696 15697 hw = &pf->hw; 15698 hw->back = pf; 15699 15700 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 15701 I40E_MAX_CSR_SPACE); 15702 /* We believe that the highest register to read is 15703 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size 15704 * is not less than that before mapping to prevent a 15705 * kernel panic. 15706 */ 15707 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) { 15708 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n", 15709 pf->ioremap_len); 15710 err = -ENOMEM; 15711 goto err_ioremap; 15712 } 15713 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 15714 if (!hw->hw_addr) { 15715 err = -EIO; 15716 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 15717 (unsigned int)pci_resource_start(pdev, 0), 15718 pf->ioremap_len, err); 15719 goto err_ioremap; 15720 } 15721 hw->vendor_id = pdev->vendor; 15722 hw->device_id = pdev->device; 15723 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 15724 hw->subsystem_vendor_id = pdev->subsystem_vendor; 15725 i40e_set_subsystem_device_id(hw); 15726 hw->bus.device = PCI_SLOT(pdev->devfn); 15727 hw->bus.func = PCI_FUNC(pdev->devfn); 15728 hw->bus.bus_id = pdev->bus->number; 15729 pf->instance = pfs_found; 15730 15731 /* Select something other than the 802.1ad ethertype for the 15732 * switch to use internally and drop on ingress. 15733 */ 15734 hw->switch_tag = 0xffff; 15735 hw->first_tag = ETH_P_8021AD; 15736 hw->second_tag = ETH_P_8021Q; 15737 15738 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 15739 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 15740 INIT_LIST_HEAD(&pf->ddp_old_prof); 15741 15742 /* set up the locks for the AQ, do this only once in probe 15743 * and destroy them only once in remove 15744 */ 15745 mutex_init(&hw->aq.asq_mutex); 15746 mutex_init(&hw->aq.arq_mutex); 15747 15748 pf->msg_enable = netif_msg_init(debug, 15749 NETIF_MSG_DRV | 15750 NETIF_MSG_PROBE | 15751 NETIF_MSG_LINK); 15752 if (debug < -1) 15753 pf->hw.debug_mask = debug; 15754 15755 /* do a special CORER for clearing PXE mode once at init */ 15756 if (hw->revision_id == 0 && 15757 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 15758 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 15759 i40e_flush(hw); 15760 msleep(200); 15761 pf->corer_count++; 15762 15763 i40e_clear_pxe_mode(hw); 15764 } 15765 15766 /* Reset here to make sure all is clean and to define PF 'n' */ 15767 i40e_clear_hw(hw); 15768 15769 err = i40e_set_mac_type(hw); 15770 if (err) { 15771 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15772 err); 15773 goto err_pf_reset; 15774 } 15775 15776 err = i40e_handle_resets(pf); 15777 if (err) 15778 goto err_pf_reset; 15779 15780 i40e_check_recovery_mode(pf); 15781 15782 if (is_kdump_kernel()) { 15783 hw->aq.num_arq_entries = I40E_MIN_ARQ_LEN; 15784 hw->aq.num_asq_entries = I40E_MIN_ASQ_LEN; 15785 } else { 15786 hw->aq.num_arq_entries = I40E_AQ_LEN; 15787 hw->aq.num_asq_entries = I40E_AQ_LEN; 15788 } 15789 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 15790 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 15791 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 15792 15793 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 15794 "%s-%s:misc", 15795 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 15796 15797 err = i40e_init_shared_code(hw); 15798 if (err) { 15799 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15800 err); 15801 goto err_pf_reset; 15802 } 15803 15804 /* set up a default setting for link flow control */ 15805 pf->hw.fc.requested_mode = I40E_FC_NONE; 15806 15807 err = i40e_init_adminq(hw); 15808 if (err) { 15809 if (err == I40E_ERR_FIRMWARE_API_VERSION) 15810 dev_info(&pdev->dev, 15811 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", 15812 hw->aq.api_maj_ver, 15813 hw->aq.api_min_ver, 15814 I40E_FW_API_VERSION_MAJOR, 15815 I40E_FW_MINOR_VERSION(hw)); 15816 else 15817 dev_info(&pdev->dev, 15818 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 15819 15820 goto err_pf_reset; 15821 } 15822 i40e_get_oem_version(hw); 15823 15824 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */ 15825 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n", 15826 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 15827 hw->aq.api_maj_ver, hw->aq.api_min_ver, 15828 i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id, 15829 hw->subsystem_vendor_id, hw->subsystem_device_id); 15830 15831 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 15832 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 15833 dev_dbg(&pdev->dev, 15834 "The driver for the device detected a newer version of the NVM image v%u.%u than v%u.%u.\n", 15835 hw->aq.api_maj_ver, 15836 hw->aq.api_min_ver, 15837 I40E_FW_API_VERSION_MAJOR, 15838 I40E_FW_MINOR_VERSION(hw)); 15839 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 15840 dev_info(&pdev->dev, 15841 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n", 15842 hw->aq.api_maj_ver, 15843 hw->aq.api_min_ver, 15844 I40E_FW_API_VERSION_MAJOR, 15845 I40E_FW_MINOR_VERSION(hw)); 15846 15847 i40e_verify_eeprom(pf); 15848 15849 /* Rev 0 hardware was never productized */ 15850 if (hw->revision_id < 1) 15851 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 15852 15853 i40e_clear_pxe_mode(hw); 15854 15855 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 15856 if (err) 15857 goto err_adminq_setup; 15858 15859 err = i40e_sw_init(pf); 15860 if (err) { 15861 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 15862 goto err_sw_init; 15863 } 15864 15865 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15866 return i40e_init_recovery_mode(pf, hw); 15867 15868 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 15869 hw->func_caps.num_rx_qp, 0, 0); 15870 if (err) { 15871 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 15872 goto err_init_lan_hmc; 15873 } 15874 15875 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 15876 if (err) { 15877 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 15878 err = -ENOENT; 15879 goto err_configure_lan_hmc; 15880 } 15881 15882 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 15883 * Ignore error return codes because if it was already disabled via 15884 * hardware settings this will fail 15885 */ 15886 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 15887 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 15888 i40e_aq_stop_lldp(hw, true, false, NULL); 15889 } 15890 15891 /* allow a platform config to override the HW addr */ 15892 i40e_get_platform_mac_addr(pdev, pf); 15893 15894 if (!is_valid_ether_addr(hw->mac.addr)) { 15895 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 15896 err = -EIO; 15897 goto err_mac_addr; 15898 } 15899 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 15900 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 15901 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 15902 if (is_valid_ether_addr(hw->mac.port_addr)) 15903 pf->hw_features |= I40E_HW_PORT_ID_VALID; 15904 15905 i40e_ptp_alloc_pins(pf); 15906 pci_set_drvdata(pdev, pf); 15907 pci_save_state(pdev); 15908 15909 #ifdef CONFIG_I40E_DCB 15910 status = i40e_get_fw_lldp_status(&pf->hw, &lldp_status); 15911 (!status && 15912 lldp_status == I40E_GET_FW_LLDP_STATUS_ENABLED) ? 15913 (pf->flags &= ~I40E_FLAG_DISABLE_FW_LLDP) : 15914 (pf->flags |= I40E_FLAG_DISABLE_FW_LLDP); 15915 dev_info(&pdev->dev, 15916 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ? 15917 "FW LLDP is disabled\n" : 15918 "FW LLDP is enabled\n"); 15919 15920 /* Enable FW to write default DCB config on link-up */ 15921 i40e_aq_set_dcb_parameters(hw, true, NULL); 15922 15923 err = i40e_init_pf_dcb(pf); 15924 if (err) { 15925 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 15926 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 15927 /* Continue without DCB enabled */ 15928 } 15929 #endif /* CONFIG_I40E_DCB */ 15930 15931 /* set up periodic task facility */ 15932 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15933 pf->service_timer_period = HZ; 15934 15935 INIT_WORK(&pf->service_task, i40e_service_task); 15936 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15937 15938 /* NVM bit on means WoL disabled for the port */ 15939 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 15940 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 15941 pf->wol_en = false; 15942 else 15943 pf->wol_en = true; 15944 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 15945 15946 /* set up the main switch operations */ 15947 i40e_determine_queue_usage(pf); 15948 err = i40e_init_interrupt_scheme(pf); 15949 if (err) 15950 goto err_switch_setup; 15951 15952 /* Reduce Tx and Rx pairs for kdump 15953 * When MSI-X is enabled, it's not allowed to use more TC queue 15954 * pairs than MSI-X vectors (pf->num_lan_msix) exist. Thus 15955 * vsi->num_queue_pairs will be equal to pf->num_lan_msix, i.e., 1. 15956 */ 15957 if (is_kdump_kernel()) 15958 pf->num_lan_msix = 1; 15959 15960 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port; 15961 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port; 15962 pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP; 15963 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared; 15964 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS; 15965 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN | 15966 UDP_TUNNEL_TYPE_GENEVE; 15967 15968 /* The number of VSIs reported by the FW is the minimum guaranteed 15969 * to us; HW supports far more and we share the remaining pool with 15970 * the other PFs. We allocate space for more than the guarantee with 15971 * the understanding that we might not get them all later. 15972 */ 15973 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15974 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15975 else 15976 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15977 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 15978 dev_warn(&pf->pdev->dev, 15979 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 15980 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 15981 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 15982 } 15983 15984 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 15985 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15986 GFP_KERNEL); 15987 if (!pf->vsi) { 15988 err = -ENOMEM; 15989 goto err_switch_setup; 15990 } 15991 15992 #ifdef CONFIG_PCI_IOV 15993 /* prep for VF support */ 15994 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15995 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15996 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15997 if (pci_num_vf(pdev)) 15998 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 15999 } 16000 #endif 16001 err = i40e_setup_pf_switch(pf, false, false); 16002 if (err) { 16003 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 16004 goto err_vsis; 16005 } 16006 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 16007 16008 /* if FDIR VSI was set up, start it now */ 16009 for (i = 0; i < pf->num_alloc_vsi; i++) { 16010 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 16011 i40e_vsi_open(pf->vsi[i]); 16012 break; 16013 } 16014 } 16015 16016 /* The driver only wants link up/down and module qualification 16017 * reports from firmware. Note the negative logic. 16018 */ 16019 err = i40e_aq_set_phy_int_mask(&pf->hw, 16020 ~(I40E_AQ_EVENT_LINK_UPDOWN | 16021 I40E_AQ_EVENT_MEDIA_NA | 16022 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 16023 if (err) 16024 dev_info(&pf->pdev->dev, "set phy mask fail, err %pe aq_err %s\n", 16025 ERR_PTR(err), 16026 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 16027 16028 /* Reconfigure hardware for allowing smaller MSS in the case 16029 * of TSO, so that we avoid the MDD being fired and causing 16030 * a reset in the case of small MSS+TSO. 16031 */ 16032 val = rd32(hw, I40E_REG_MSS); 16033 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 16034 val &= ~I40E_REG_MSS_MIN_MASK; 16035 val |= I40E_64BYTE_MSS; 16036 wr32(hw, I40E_REG_MSS, val); 16037 } 16038 16039 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 16040 msleep(75); 16041 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 16042 if (err) 16043 dev_info(&pf->pdev->dev, "link restart failed, err %pe aq_err %s\n", 16044 ERR_PTR(err), 16045 i40e_aq_str(&pf->hw, 16046 pf->hw.aq.asq_last_status)); 16047 } 16048 /* The main driver is (mostly) up and happy. We need to set this state 16049 * before setting up the misc vector or we get a race and the vector 16050 * ends up disabled forever. 16051 */ 16052 clear_bit(__I40E_DOWN, pf->state); 16053 16054 /* In case of MSIX we are going to setup the misc vector right here 16055 * to handle admin queue events etc. In case of legacy and MSI 16056 * the misc functionality and queue processing is combined in 16057 * the same vector and that gets setup at open. 16058 */ 16059 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 16060 err = i40e_setup_misc_vector(pf); 16061 if (err) { 16062 dev_info(&pdev->dev, 16063 "setup of misc vector failed: %d\n", err); 16064 i40e_cloud_filter_exit(pf); 16065 i40e_fdir_teardown(pf); 16066 goto err_vsis; 16067 } 16068 } 16069 16070 #ifdef CONFIG_PCI_IOV 16071 /* prep for VF support */ 16072 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 16073 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 16074 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 16075 /* disable link interrupts for VFs */ 16076 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 16077 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 16078 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 16079 i40e_flush(hw); 16080 16081 if (pci_num_vf(pdev)) { 16082 dev_info(&pdev->dev, 16083 "Active VFs found, allocating resources.\n"); 16084 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 16085 if (err) 16086 dev_info(&pdev->dev, 16087 "Error %d allocating resources for existing VFs\n", 16088 err); 16089 } 16090 } 16091 #endif /* CONFIG_PCI_IOV */ 16092 16093 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 16094 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 16095 pf->num_iwarp_msix, 16096 I40E_IWARP_IRQ_PILE_ID); 16097 if (pf->iwarp_base_vector < 0) { 16098 dev_info(&pdev->dev, 16099 "failed to get tracking for %d vectors for IWARP err=%d\n", 16100 pf->num_iwarp_msix, pf->iwarp_base_vector); 16101 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 16102 } 16103 } 16104 16105 i40e_dbg_pf_init(pf); 16106 16107 /* tell the firmware that we're starting */ 16108 i40e_send_version(pf); 16109 16110 /* since everything's happy, start the service_task timer */ 16111 mod_timer(&pf->service_timer, 16112 round_jiffies(jiffies + pf->service_timer_period)); 16113 16114 /* add this PF to client device list and launch a client service task */ 16115 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 16116 err = i40e_lan_add_device(pf); 16117 if (err) 16118 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 16119 err); 16120 } 16121 16122 #define PCI_SPEED_SIZE 8 16123 #define PCI_WIDTH_SIZE 8 16124 /* Devices on the IOSF bus do not have this information 16125 * and will report PCI Gen 1 x 1 by default so don't bother 16126 * checking them. 16127 */ 16128 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 16129 char speed[PCI_SPEED_SIZE] = "Unknown"; 16130 char width[PCI_WIDTH_SIZE] = "Unknown"; 16131 16132 /* Get the negotiated link width and speed from PCI config 16133 * space 16134 */ 16135 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 16136 &link_status); 16137 16138 i40e_set_pci_config_data(hw, link_status); 16139 16140 switch (hw->bus.speed) { 16141 case i40e_bus_speed_8000: 16142 strscpy(speed, "8.0", PCI_SPEED_SIZE); break; 16143 case i40e_bus_speed_5000: 16144 strscpy(speed, "5.0", PCI_SPEED_SIZE); break; 16145 case i40e_bus_speed_2500: 16146 strscpy(speed, "2.5", PCI_SPEED_SIZE); break; 16147 default: 16148 break; 16149 } 16150 switch (hw->bus.width) { 16151 case i40e_bus_width_pcie_x8: 16152 strscpy(width, "8", PCI_WIDTH_SIZE); break; 16153 case i40e_bus_width_pcie_x4: 16154 strscpy(width, "4", PCI_WIDTH_SIZE); break; 16155 case i40e_bus_width_pcie_x2: 16156 strscpy(width, "2", PCI_WIDTH_SIZE); break; 16157 case i40e_bus_width_pcie_x1: 16158 strscpy(width, "1", PCI_WIDTH_SIZE); break; 16159 default: 16160 break; 16161 } 16162 16163 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 16164 speed, width); 16165 16166 if (hw->bus.width < i40e_bus_width_pcie_x8 || 16167 hw->bus.speed < i40e_bus_speed_8000) { 16168 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 16169 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 16170 } 16171 } 16172 16173 /* get the requested speeds from the fw */ 16174 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 16175 if (err) 16176 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %pe last_status = %s\n", 16177 ERR_PTR(err), 16178 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 16179 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 16180 16181 /* set the FEC config due to the board capabilities */ 16182 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags); 16183 16184 /* get the supported phy types from the fw */ 16185 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 16186 if (err) 16187 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %pe last_status = %s\n", 16188 ERR_PTR(err), 16189 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 16190 16191 /* make sure the MFS hasn't been set lower than the default */ 16192 #define MAX_FRAME_SIZE_DEFAULT 0x2600 16193 val = (rd32(&pf->hw, I40E_PRTGL_SAH) & 16194 I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT; 16195 if (val < MAX_FRAME_SIZE_DEFAULT) 16196 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n", 16197 i, val); 16198 16199 /* Add a filter to drop all Flow control frames from any VSI from being 16200 * transmitted. By doing so we stop a malicious VF from sending out 16201 * PAUSE or PFC frames and potentially controlling traffic for other 16202 * PF/VF VSIs. 16203 * The FW can still send Flow control frames if enabled. 16204 */ 16205 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 16206 pf->main_vsi_seid); 16207 16208 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 16209 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 16210 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 16211 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 16212 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 16213 /* print a string summarizing features */ 16214 i40e_print_features(pf); 16215 16216 return 0; 16217 16218 /* Unwind what we've done if something failed in the setup */ 16219 err_vsis: 16220 set_bit(__I40E_DOWN, pf->state); 16221 i40e_clear_interrupt_scheme(pf); 16222 kfree(pf->vsi); 16223 err_switch_setup: 16224 i40e_reset_interrupt_capability(pf); 16225 timer_shutdown_sync(&pf->service_timer); 16226 err_mac_addr: 16227 err_configure_lan_hmc: 16228 (void)i40e_shutdown_lan_hmc(hw); 16229 err_init_lan_hmc: 16230 kfree(pf->qp_pile); 16231 err_sw_init: 16232 err_adminq_setup: 16233 err_pf_reset: 16234 iounmap(hw->hw_addr); 16235 err_ioremap: 16236 kfree(pf); 16237 err_pf_alloc: 16238 pci_release_mem_regions(pdev); 16239 err_pci_reg: 16240 err_dma: 16241 pci_disable_device(pdev); 16242 return err; 16243 } 16244 16245 /** 16246 * i40e_remove - Device removal routine 16247 * @pdev: PCI device information struct 16248 * 16249 * i40e_remove is called by the PCI subsystem to alert the driver 16250 * that is should release a PCI device. This could be caused by a 16251 * Hot-Plug event, or because the driver is going to be removed from 16252 * memory. 16253 **/ 16254 static void i40e_remove(struct pci_dev *pdev) 16255 { 16256 struct i40e_pf *pf = pci_get_drvdata(pdev); 16257 struct i40e_hw *hw = &pf->hw; 16258 int ret_code; 16259 int i; 16260 16261 i40e_dbg_pf_exit(pf); 16262 16263 i40e_ptp_stop(pf); 16264 16265 /* Disable RSS in hw */ 16266 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 16267 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 16268 16269 /* Grab __I40E_RESET_RECOVERY_PENDING and set __I40E_IN_REMOVE 16270 * flags, once they are set, i40e_rebuild should not be called as 16271 * i40e_prep_for_reset always returns early. 16272 */ 16273 while (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 16274 usleep_range(1000, 2000); 16275 set_bit(__I40E_IN_REMOVE, pf->state); 16276 16277 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 16278 set_bit(__I40E_VF_RESETS_DISABLED, pf->state); 16279 i40e_free_vfs(pf); 16280 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 16281 } 16282 /* no more scheduling of any task */ 16283 set_bit(__I40E_SUSPENDED, pf->state); 16284 set_bit(__I40E_DOWN, pf->state); 16285 if (pf->service_timer.function) 16286 timer_shutdown_sync(&pf->service_timer); 16287 if (pf->service_task.func) 16288 cancel_work_sync(&pf->service_task); 16289 16290 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 16291 struct i40e_vsi *vsi = pf->vsi[0]; 16292 16293 /* We know that we have allocated only one vsi for this PF, 16294 * it was just for registering netdevice, so the interface 16295 * could be visible in the 'ifconfig' output 16296 */ 16297 unregister_netdev(vsi->netdev); 16298 free_netdev(vsi->netdev); 16299 16300 goto unmap; 16301 } 16302 16303 /* Client close must be called explicitly here because the timer 16304 * has been stopped. 16305 */ 16306 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 16307 16308 i40e_fdir_teardown(pf); 16309 16310 /* If there is a switch structure or any orphans, remove them. 16311 * This will leave only the PF's VSI remaining. 16312 */ 16313 for (i = 0; i < I40E_MAX_VEB; i++) { 16314 if (!pf->veb[i]) 16315 continue; 16316 16317 if (pf->veb[i]->uplink_seid == pf->mac_seid || 16318 pf->veb[i]->uplink_seid == 0) 16319 i40e_switch_branch_release(pf->veb[i]); 16320 } 16321 16322 /* Now we can shutdown the PF's VSI, just before we kill 16323 * adminq and hmc. 16324 */ 16325 if (pf->vsi[pf->lan_vsi]) 16326 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 16327 16328 i40e_cloud_filter_exit(pf); 16329 16330 /* remove attached clients */ 16331 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 16332 ret_code = i40e_lan_del_device(pf); 16333 if (ret_code) 16334 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 16335 ret_code); 16336 } 16337 16338 /* shutdown and destroy the HMC */ 16339 if (hw->hmc.hmc_obj) { 16340 ret_code = i40e_shutdown_lan_hmc(hw); 16341 if (ret_code) 16342 dev_warn(&pdev->dev, 16343 "Failed to destroy the HMC resources: %d\n", 16344 ret_code); 16345 } 16346 16347 unmap: 16348 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 16349 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 16350 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 16351 free_irq(pf->pdev->irq, pf); 16352 16353 /* shutdown the adminq */ 16354 i40e_shutdown_adminq(hw); 16355 16356 /* destroy the locks only once, here */ 16357 mutex_destroy(&hw->aq.arq_mutex); 16358 mutex_destroy(&hw->aq.asq_mutex); 16359 16360 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 16361 rtnl_lock(); 16362 i40e_clear_interrupt_scheme(pf); 16363 for (i = 0; i < pf->num_alloc_vsi; i++) { 16364 if (pf->vsi[i]) { 16365 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) 16366 i40e_vsi_clear_rings(pf->vsi[i]); 16367 i40e_vsi_clear(pf->vsi[i]); 16368 pf->vsi[i] = NULL; 16369 } 16370 } 16371 rtnl_unlock(); 16372 16373 for (i = 0; i < I40E_MAX_VEB; i++) { 16374 kfree(pf->veb[i]); 16375 pf->veb[i] = NULL; 16376 } 16377 16378 kfree(pf->qp_pile); 16379 kfree(pf->vsi); 16380 16381 iounmap(hw->hw_addr); 16382 kfree(pf); 16383 pci_release_mem_regions(pdev); 16384 16385 pci_disable_device(pdev); 16386 } 16387 16388 /** 16389 * i40e_pci_error_detected - warning that something funky happened in PCI land 16390 * @pdev: PCI device information struct 16391 * @error: the type of PCI error 16392 * 16393 * Called to warn that something happened and the error handling steps 16394 * are in progress. Allows the driver to quiesce things, be ready for 16395 * remediation. 16396 **/ 16397 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 16398 pci_channel_state_t error) 16399 { 16400 struct i40e_pf *pf = pci_get_drvdata(pdev); 16401 16402 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 16403 16404 if (!pf) { 16405 dev_info(&pdev->dev, 16406 "Cannot recover - error happened during device probe\n"); 16407 return PCI_ERS_RESULT_DISCONNECT; 16408 } 16409 16410 /* shutdown all operations */ 16411 if (!test_bit(__I40E_SUSPENDED, pf->state)) 16412 i40e_prep_for_reset(pf); 16413 16414 /* Request a slot reset */ 16415 return PCI_ERS_RESULT_NEED_RESET; 16416 } 16417 16418 /** 16419 * i40e_pci_error_slot_reset - a PCI slot reset just happened 16420 * @pdev: PCI device information struct 16421 * 16422 * Called to find if the driver can work with the device now that 16423 * the pci slot has been reset. If a basic connection seems good 16424 * (registers are readable and have sane content) then return a 16425 * happy little PCI_ERS_RESULT_xxx. 16426 **/ 16427 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 16428 { 16429 struct i40e_pf *pf = pci_get_drvdata(pdev); 16430 pci_ers_result_t result; 16431 u32 reg; 16432 16433 dev_dbg(&pdev->dev, "%s\n", __func__); 16434 if (pci_enable_device_mem(pdev)) { 16435 dev_info(&pdev->dev, 16436 "Cannot re-enable PCI device after reset.\n"); 16437 result = PCI_ERS_RESULT_DISCONNECT; 16438 } else { 16439 pci_set_master(pdev); 16440 pci_restore_state(pdev); 16441 pci_save_state(pdev); 16442 pci_wake_from_d3(pdev, false); 16443 16444 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 16445 if (reg == 0) 16446 result = PCI_ERS_RESULT_RECOVERED; 16447 else 16448 result = PCI_ERS_RESULT_DISCONNECT; 16449 } 16450 16451 return result; 16452 } 16453 16454 /** 16455 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 16456 * @pdev: PCI device information struct 16457 */ 16458 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 16459 { 16460 struct i40e_pf *pf = pci_get_drvdata(pdev); 16461 16462 i40e_prep_for_reset(pf); 16463 } 16464 16465 /** 16466 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 16467 * @pdev: PCI device information struct 16468 */ 16469 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 16470 { 16471 struct i40e_pf *pf = pci_get_drvdata(pdev); 16472 16473 if (test_bit(__I40E_IN_REMOVE, pf->state)) 16474 return; 16475 16476 i40e_reset_and_rebuild(pf, false, false); 16477 } 16478 16479 /** 16480 * i40e_pci_error_resume - restart operations after PCI error recovery 16481 * @pdev: PCI device information struct 16482 * 16483 * Called to allow the driver to bring things back up after PCI error 16484 * and/or reset recovery has finished. 16485 **/ 16486 static void i40e_pci_error_resume(struct pci_dev *pdev) 16487 { 16488 struct i40e_pf *pf = pci_get_drvdata(pdev); 16489 16490 dev_dbg(&pdev->dev, "%s\n", __func__); 16491 if (test_bit(__I40E_SUSPENDED, pf->state)) 16492 return; 16493 16494 i40e_handle_reset_warning(pf, false); 16495 } 16496 16497 /** 16498 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 16499 * using the mac_address_write admin q function 16500 * @pf: pointer to i40e_pf struct 16501 **/ 16502 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 16503 { 16504 struct i40e_hw *hw = &pf->hw; 16505 u8 mac_addr[6]; 16506 u16 flags = 0; 16507 int ret; 16508 16509 /* Get current MAC address in case it's an LAA */ 16510 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 16511 ether_addr_copy(mac_addr, 16512 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 16513 } else { 16514 dev_err(&pf->pdev->dev, 16515 "Failed to retrieve MAC address; using default\n"); 16516 ether_addr_copy(mac_addr, hw->mac.addr); 16517 } 16518 16519 /* The FW expects the mac address write cmd to first be called with 16520 * one of these flags before calling it again with the multicast 16521 * enable flags. 16522 */ 16523 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 16524 16525 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 16526 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 16527 16528 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 16529 if (ret) { 16530 dev_err(&pf->pdev->dev, 16531 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 16532 return; 16533 } 16534 16535 flags = I40E_AQC_MC_MAG_EN 16536 | I40E_AQC_WOL_PRESERVE_ON_PFR 16537 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 16538 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 16539 if (ret) 16540 dev_err(&pf->pdev->dev, 16541 "Failed to enable Multicast Magic Packet wake up\n"); 16542 } 16543 16544 /** 16545 * i40e_shutdown - PCI callback for shutting down 16546 * @pdev: PCI device information struct 16547 **/ 16548 static void i40e_shutdown(struct pci_dev *pdev) 16549 { 16550 struct i40e_pf *pf = pci_get_drvdata(pdev); 16551 struct i40e_hw *hw = &pf->hw; 16552 16553 set_bit(__I40E_SUSPENDED, pf->state); 16554 set_bit(__I40E_DOWN, pf->state); 16555 16556 del_timer_sync(&pf->service_timer); 16557 cancel_work_sync(&pf->service_task); 16558 i40e_cloud_filter_exit(pf); 16559 i40e_fdir_teardown(pf); 16560 16561 /* Client close must be called explicitly here because the timer 16562 * has been stopped. 16563 */ 16564 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 16565 16566 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 16567 i40e_enable_mc_magic_wake(pf); 16568 16569 i40e_prep_for_reset(pf); 16570 16571 wr32(hw, I40E_PFPM_APM, 16572 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 16573 wr32(hw, I40E_PFPM_WUFC, 16574 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 16575 16576 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 16577 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 16578 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 16579 free_irq(pf->pdev->irq, pf); 16580 16581 /* Since we're going to destroy queues during the 16582 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 16583 * whole section 16584 */ 16585 rtnl_lock(); 16586 i40e_clear_interrupt_scheme(pf); 16587 rtnl_unlock(); 16588 16589 if (system_state == SYSTEM_POWER_OFF) { 16590 pci_wake_from_d3(pdev, pf->wol_en); 16591 pci_set_power_state(pdev, PCI_D3hot); 16592 } 16593 } 16594 16595 /** 16596 * i40e_suspend - PM callback for moving to D3 16597 * @dev: generic device information structure 16598 **/ 16599 static int __maybe_unused i40e_suspend(struct device *dev) 16600 { 16601 struct i40e_pf *pf = dev_get_drvdata(dev); 16602 struct i40e_hw *hw = &pf->hw; 16603 16604 /* If we're already suspended, then there is nothing to do */ 16605 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 16606 return 0; 16607 16608 set_bit(__I40E_DOWN, pf->state); 16609 16610 /* Ensure service task will not be running */ 16611 del_timer_sync(&pf->service_timer); 16612 cancel_work_sync(&pf->service_task); 16613 16614 /* Client close must be called explicitly here because the timer 16615 * has been stopped. 16616 */ 16617 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 16618 16619 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 16620 i40e_enable_mc_magic_wake(pf); 16621 16622 /* Since we're going to destroy queues during the 16623 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 16624 * whole section 16625 */ 16626 rtnl_lock(); 16627 16628 i40e_prep_for_reset(pf); 16629 16630 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 16631 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 16632 16633 /* Clear the interrupt scheme and release our IRQs so that the system 16634 * can safely hibernate even when there are a large number of CPUs. 16635 * Otherwise hibernation might fail when mapping all the vectors back 16636 * to CPU0. 16637 */ 16638 i40e_clear_interrupt_scheme(pf); 16639 16640 rtnl_unlock(); 16641 16642 return 0; 16643 } 16644 16645 /** 16646 * i40e_resume - PM callback for waking up from D3 16647 * @dev: generic device information structure 16648 **/ 16649 static int __maybe_unused i40e_resume(struct device *dev) 16650 { 16651 struct i40e_pf *pf = dev_get_drvdata(dev); 16652 int err; 16653 16654 /* If we're not suspended, then there is nothing to do */ 16655 if (!test_bit(__I40E_SUSPENDED, pf->state)) 16656 return 0; 16657 16658 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 16659 * since we're going to be restoring queues 16660 */ 16661 rtnl_lock(); 16662 16663 /* We cleared the interrupt scheme when we suspended, so we need to 16664 * restore it now to resume device functionality. 16665 */ 16666 err = i40e_restore_interrupt_scheme(pf); 16667 if (err) { 16668 dev_err(dev, "Cannot restore interrupt scheme: %d\n", 16669 err); 16670 } 16671 16672 clear_bit(__I40E_DOWN, pf->state); 16673 i40e_reset_and_rebuild(pf, false, true); 16674 16675 rtnl_unlock(); 16676 16677 /* Clear suspended state last after everything is recovered */ 16678 clear_bit(__I40E_SUSPENDED, pf->state); 16679 16680 /* Restart the service task */ 16681 mod_timer(&pf->service_timer, 16682 round_jiffies(jiffies + pf->service_timer_period)); 16683 16684 return 0; 16685 } 16686 16687 static const struct pci_error_handlers i40e_err_handler = { 16688 .error_detected = i40e_pci_error_detected, 16689 .slot_reset = i40e_pci_error_slot_reset, 16690 .reset_prepare = i40e_pci_error_reset_prepare, 16691 .reset_done = i40e_pci_error_reset_done, 16692 .resume = i40e_pci_error_resume, 16693 }; 16694 16695 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 16696 16697 static struct pci_driver i40e_driver = { 16698 .name = i40e_driver_name, 16699 .id_table = i40e_pci_tbl, 16700 .probe = i40e_probe, 16701 .remove = i40e_remove, 16702 .driver = { 16703 .pm = &i40e_pm_ops, 16704 }, 16705 .shutdown = i40e_shutdown, 16706 .err_handler = &i40e_err_handler, 16707 .sriov_configure = i40e_pci_sriov_configure, 16708 }; 16709 16710 /** 16711 * i40e_init_module - Driver registration routine 16712 * 16713 * i40e_init_module is the first routine called when the driver is 16714 * loaded. All it does is register with the PCI subsystem. 16715 **/ 16716 static int __init i40e_init_module(void) 16717 { 16718 int err; 16719 16720 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string); 16721 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 16722 16723 /* There is no need to throttle the number of active tasks because 16724 * each device limits its own task using a state bit for scheduling 16725 * the service task, and the device tasks do not interfere with each 16726 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 16727 * since we need to be able to guarantee forward progress even under 16728 * memory pressure. 16729 */ 16730 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 16731 if (!i40e_wq) { 16732 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 16733 return -ENOMEM; 16734 } 16735 16736 i40e_dbg_init(); 16737 err = pci_register_driver(&i40e_driver); 16738 if (err) { 16739 destroy_workqueue(i40e_wq); 16740 i40e_dbg_exit(); 16741 return err; 16742 } 16743 16744 return 0; 16745 } 16746 module_init(i40e_init_module); 16747 16748 /** 16749 * i40e_exit_module - Driver exit cleanup routine 16750 * 16751 * i40e_exit_module is called just before the driver is removed 16752 * from memory. 16753 **/ 16754 static void __exit i40e_exit_module(void) 16755 { 16756 pci_unregister_driver(&i40e_driver); 16757 destroy_workqueue(i40e_wq); 16758 ida_destroy(&i40e_client_ida); 16759 i40e_dbg_exit(); 16760 } 16761 module_exit(i40e_exit_module); 16762