1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/of_net.h> 6 #include <linux/pci.h> 7 #include <linux/bpf.h> 8 #include <generated/utsrelease.h> 9 10 /* Local includes */ 11 #include "i40e.h" 12 #include "i40e_diag.h" 13 #include "i40e_xsk.h" 14 #include <net/udp_tunnel.h> 15 #include <net/xdp_sock_drv.h> 16 /* All i40e tracepoints are defined by the include below, which 17 * must be included exactly once across the whole kernel with 18 * CREATE_TRACE_POINTS defined 19 */ 20 #define CREATE_TRACE_POINTS 21 #include "i40e_trace.h" 22 23 const char i40e_driver_name[] = "i40e"; 24 static const char i40e_driver_string[] = 25 "Intel(R) Ethernet Connection XL710 Network Driver"; 26 27 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation."; 28 29 /* a bit of forward declarations */ 30 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 31 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 32 static int i40e_add_vsi(struct i40e_vsi *vsi); 33 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 34 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 35 static int i40e_setup_misc_vector(struct i40e_pf *pf); 36 static void i40e_determine_queue_usage(struct i40e_pf *pf); 37 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 38 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 39 static int i40e_reset(struct i40e_pf *pf); 40 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 41 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf); 42 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf); 43 static bool i40e_check_recovery_mode(struct i40e_pf *pf); 44 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw); 45 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 46 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 47 static int i40e_get_capabilities(struct i40e_pf *pf, 48 enum i40e_admin_queue_opc list_type); 49 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf); 50 51 /* i40e_pci_tbl - PCI Device ID Table 52 * 53 * Last entry must be all 0s 54 * 55 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 56 * Class, Class Mask, private data (not used) } 57 */ 58 static const struct pci_device_id i40e_pci_tbl[] = { 59 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 60 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 61 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 62 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 63 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 64 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 83 /* required last entry */ 84 {0, } 85 }; 86 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 87 88 #define I40E_MAX_VF_COUNT 128 89 static int debug = -1; 90 module_param(debug, uint, 0); 91 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 92 93 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 94 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 95 MODULE_LICENSE("GPL v2"); 96 97 static struct workqueue_struct *i40e_wq; 98 99 /** 100 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 101 * @hw: pointer to the HW structure 102 * @mem: ptr to mem struct to fill out 103 * @size: size of memory requested 104 * @alignment: what to align the allocation to 105 **/ 106 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 107 u64 size, u32 alignment) 108 { 109 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 110 111 mem->size = ALIGN(size, alignment); 112 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa, 113 GFP_KERNEL); 114 if (!mem->va) 115 return -ENOMEM; 116 117 return 0; 118 } 119 120 /** 121 * i40e_free_dma_mem_d - OS specific memory free for shared code 122 * @hw: pointer to the HW structure 123 * @mem: ptr to mem struct to free 124 **/ 125 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 126 { 127 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 128 129 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 130 mem->va = NULL; 131 mem->pa = 0; 132 mem->size = 0; 133 134 return 0; 135 } 136 137 /** 138 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 139 * @hw: pointer to the HW structure 140 * @mem: ptr to mem struct to fill out 141 * @size: size of memory requested 142 **/ 143 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 144 u32 size) 145 { 146 mem->size = size; 147 mem->va = kzalloc(size, GFP_KERNEL); 148 149 if (!mem->va) 150 return -ENOMEM; 151 152 return 0; 153 } 154 155 /** 156 * i40e_free_virt_mem_d - OS specific memory free for shared code 157 * @hw: pointer to the HW structure 158 * @mem: ptr to mem struct to free 159 **/ 160 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 161 { 162 /* it's ok to kfree a NULL pointer */ 163 kfree(mem->va); 164 mem->va = NULL; 165 mem->size = 0; 166 167 return 0; 168 } 169 170 /** 171 * i40e_get_lump - find a lump of free generic resource 172 * @pf: board private structure 173 * @pile: the pile of resource to search 174 * @needed: the number of items needed 175 * @id: an owner id to stick on the items assigned 176 * 177 * Returns the base item index of the lump, or negative for error 178 * 179 * The search_hint trick and lack of advanced fit-finding only work 180 * because we're highly likely to have all the same size lump requests. 181 * Linear search time and any fragmentation should be minimal. 182 **/ 183 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 184 u16 needed, u16 id) 185 { 186 int ret = -ENOMEM; 187 int i, j; 188 189 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 190 dev_info(&pf->pdev->dev, 191 "param err: pile=%s needed=%d id=0x%04x\n", 192 pile ? "<valid>" : "<null>", needed, id); 193 return -EINVAL; 194 } 195 196 /* start the linear search with an imperfect hint */ 197 i = pile->search_hint; 198 while (i < pile->num_entries) { 199 /* skip already allocated entries */ 200 if (pile->list[i] & I40E_PILE_VALID_BIT) { 201 i++; 202 continue; 203 } 204 205 /* do we have enough in this lump? */ 206 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 207 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 208 break; 209 } 210 211 if (j == needed) { 212 /* there was enough, so assign it to the requestor */ 213 for (j = 0; j < needed; j++) 214 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 215 ret = i; 216 pile->search_hint = i + j; 217 break; 218 } 219 220 /* not enough, so skip over it and continue looking */ 221 i += j; 222 } 223 224 return ret; 225 } 226 227 /** 228 * i40e_put_lump - return a lump of generic resource 229 * @pile: the pile of resource to search 230 * @index: the base item index 231 * @id: the owner id of the items assigned 232 * 233 * Returns the count of items in the lump 234 **/ 235 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 236 { 237 int valid_id = (id | I40E_PILE_VALID_BIT); 238 int count = 0; 239 int i; 240 241 if (!pile || index >= pile->num_entries) 242 return -EINVAL; 243 244 for (i = index; 245 i < pile->num_entries && pile->list[i] == valid_id; 246 i++) { 247 pile->list[i] = 0; 248 count++; 249 } 250 251 if (count && index < pile->search_hint) 252 pile->search_hint = index; 253 254 return count; 255 } 256 257 /** 258 * i40e_find_vsi_from_id - searches for the vsi with the given id 259 * @pf: the pf structure to search for the vsi 260 * @id: id of the vsi it is searching for 261 **/ 262 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 263 { 264 int i; 265 266 for (i = 0; i < pf->num_alloc_vsi; i++) 267 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 268 return pf->vsi[i]; 269 270 return NULL; 271 } 272 273 /** 274 * i40e_service_event_schedule - Schedule the service task to wake up 275 * @pf: board private structure 276 * 277 * If not already scheduled, this puts the task into the work queue 278 **/ 279 void i40e_service_event_schedule(struct i40e_pf *pf) 280 { 281 if ((!test_bit(__I40E_DOWN, pf->state) && 282 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) || 283 test_bit(__I40E_RECOVERY_MODE, pf->state)) 284 queue_work(i40e_wq, &pf->service_task); 285 } 286 287 /** 288 * i40e_tx_timeout - Respond to a Tx Hang 289 * @netdev: network interface device structure 290 * 291 * If any port has noticed a Tx timeout, it is likely that the whole 292 * device is munged, not just the one netdev port, so go for the full 293 * reset. 294 **/ 295 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue) 296 { 297 struct i40e_netdev_priv *np = netdev_priv(netdev); 298 struct i40e_vsi *vsi = np->vsi; 299 struct i40e_pf *pf = vsi->back; 300 struct i40e_ring *tx_ring = NULL; 301 unsigned int i; 302 u32 head, val; 303 304 pf->tx_timeout_count++; 305 306 /* with txqueue index, find the tx_ring struct */ 307 for (i = 0; i < vsi->num_queue_pairs; i++) { 308 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 309 if (txqueue == 310 vsi->tx_rings[i]->queue_index) { 311 tx_ring = vsi->tx_rings[i]; 312 break; 313 } 314 } 315 } 316 317 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 318 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 319 else if (time_before(jiffies, 320 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 321 return; /* don't do any new action before the next timeout */ 322 323 /* don't kick off another recovery if one is already pending */ 324 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state)) 325 return; 326 327 if (tx_ring) { 328 head = i40e_get_head(tx_ring); 329 /* Read interrupt register */ 330 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 331 val = rd32(&pf->hw, 332 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 333 tx_ring->vsi->base_vector - 1)); 334 else 335 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 336 337 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 338 vsi->seid, txqueue, tx_ring->next_to_clean, 339 head, tx_ring->next_to_use, 340 readl(tx_ring->tail), val); 341 } 342 343 pf->tx_timeout_last_recovery = jiffies; 344 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n", 345 pf->tx_timeout_recovery_level, txqueue); 346 347 switch (pf->tx_timeout_recovery_level) { 348 case 1: 349 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 350 break; 351 case 2: 352 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 353 break; 354 case 3: 355 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 356 break; 357 default: 358 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 359 break; 360 } 361 362 i40e_service_event_schedule(pf); 363 pf->tx_timeout_recovery_level++; 364 } 365 366 /** 367 * i40e_get_vsi_stats_struct - Get System Network Statistics 368 * @vsi: the VSI we care about 369 * 370 * Returns the address of the device statistics structure. 371 * The statistics are actually updated from the service task. 372 **/ 373 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 374 { 375 return &vsi->net_stats; 376 } 377 378 /** 379 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 380 * @ring: Tx ring to get statistics from 381 * @stats: statistics entry to be updated 382 **/ 383 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 384 struct rtnl_link_stats64 *stats) 385 { 386 u64 bytes, packets; 387 unsigned int start; 388 389 do { 390 start = u64_stats_fetch_begin_irq(&ring->syncp); 391 packets = ring->stats.packets; 392 bytes = ring->stats.bytes; 393 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 394 395 stats->tx_packets += packets; 396 stats->tx_bytes += bytes; 397 } 398 399 /** 400 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 401 * @netdev: network interface device structure 402 * @stats: data structure to store statistics 403 * 404 * Returns the address of the device statistics structure. 405 * The statistics are actually updated from the service task. 406 **/ 407 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 408 struct rtnl_link_stats64 *stats) 409 { 410 struct i40e_netdev_priv *np = netdev_priv(netdev); 411 struct i40e_vsi *vsi = np->vsi; 412 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 413 struct i40e_ring *ring; 414 int i; 415 416 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 417 return; 418 419 if (!vsi->tx_rings) 420 return; 421 422 rcu_read_lock(); 423 for (i = 0; i < vsi->num_queue_pairs; i++) { 424 u64 bytes, packets; 425 unsigned int start; 426 427 ring = READ_ONCE(vsi->tx_rings[i]); 428 if (!ring) 429 continue; 430 i40e_get_netdev_stats_struct_tx(ring, stats); 431 432 if (i40e_enabled_xdp_vsi(vsi)) { 433 ring = READ_ONCE(vsi->xdp_rings[i]); 434 if (!ring) 435 continue; 436 i40e_get_netdev_stats_struct_tx(ring, stats); 437 } 438 439 ring = READ_ONCE(vsi->rx_rings[i]); 440 if (!ring) 441 continue; 442 do { 443 start = u64_stats_fetch_begin_irq(&ring->syncp); 444 packets = ring->stats.packets; 445 bytes = ring->stats.bytes; 446 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 447 448 stats->rx_packets += packets; 449 stats->rx_bytes += bytes; 450 451 } 452 rcu_read_unlock(); 453 454 /* following stats updated by i40e_watchdog_subtask() */ 455 stats->multicast = vsi_stats->multicast; 456 stats->tx_errors = vsi_stats->tx_errors; 457 stats->tx_dropped = vsi_stats->tx_dropped; 458 stats->rx_errors = vsi_stats->rx_errors; 459 stats->rx_dropped = vsi_stats->rx_dropped; 460 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 461 stats->rx_length_errors = vsi_stats->rx_length_errors; 462 } 463 464 /** 465 * i40e_vsi_reset_stats - Resets all stats of the given vsi 466 * @vsi: the VSI to have its stats reset 467 **/ 468 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 469 { 470 struct rtnl_link_stats64 *ns; 471 int i; 472 473 if (!vsi) 474 return; 475 476 ns = i40e_get_vsi_stats_struct(vsi); 477 memset(ns, 0, sizeof(*ns)); 478 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 479 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 480 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 481 if (vsi->rx_rings && vsi->rx_rings[0]) { 482 for (i = 0; i < vsi->num_queue_pairs; i++) { 483 memset(&vsi->rx_rings[i]->stats, 0, 484 sizeof(vsi->rx_rings[i]->stats)); 485 memset(&vsi->rx_rings[i]->rx_stats, 0, 486 sizeof(vsi->rx_rings[i]->rx_stats)); 487 memset(&vsi->tx_rings[i]->stats, 0, 488 sizeof(vsi->tx_rings[i]->stats)); 489 memset(&vsi->tx_rings[i]->tx_stats, 0, 490 sizeof(vsi->tx_rings[i]->tx_stats)); 491 } 492 } 493 vsi->stat_offsets_loaded = false; 494 } 495 496 /** 497 * i40e_pf_reset_stats - Reset all of the stats for the given PF 498 * @pf: the PF to be reset 499 **/ 500 void i40e_pf_reset_stats(struct i40e_pf *pf) 501 { 502 int i; 503 504 memset(&pf->stats, 0, sizeof(pf->stats)); 505 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 506 pf->stat_offsets_loaded = false; 507 508 for (i = 0; i < I40E_MAX_VEB; i++) { 509 if (pf->veb[i]) { 510 memset(&pf->veb[i]->stats, 0, 511 sizeof(pf->veb[i]->stats)); 512 memset(&pf->veb[i]->stats_offsets, 0, 513 sizeof(pf->veb[i]->stats_offsets)); 514 memset(&pf->veb[i]->tc_stats, 0, 515 sizeof(pf->veb[i]->tc_stats)); 516 memset(&pf->veb[i]->tc_stats_offsets, 0, 517 sizeof(pf->veb[i]->tc_stats_offsets)); 518 pf->veb[i]->stat_offsets_loaded = false; 519 } 520 } 521 pf->hw_csum_rx_error = 0; 522 } 523 524 /** 525 * i40e_stat_update48 - read and update a 48 bit stat from the chip 526 * @hw: ptr to the hardware info 527 * @hireg: the high 32 bit reg to read 528 * @loreg: the low 32 bit reg to read 529 * @offset_loaded: has the initial offset been loaded yet 530 * @offset: ptr to current offset value 531 * @stat: ptr to the stat 532 * 533 * Since the device stats are not reset at PFReset, they likely will not 534 * be zeroed when the driver starts. We'll save the first values read 535 * and use them as offsets to be subtracted from the raw values in order 536 * to report stats that count from zero. In the process, we also manage 537 * the potential roll-over. 538 **/ 539 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 540 bool offset_loaded, u64 *offset, u64 *stat) 541 { 542 u64 new_data; 543 544 if (hw->device_id == I40E_DEV_ID_QEMU) { 545 new_data = rd32(hw, loreg); 546 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 547 } else { 548 new_data = rd64(hw, loreg); 549 } 550 if (!offset_loaded) 551 *offset = new_data; 552 if (likely(new_data >= *offset)) 553 *stat = new_data - *offset; 554 else 555 *stat = (new_data + BIT_ULL(48)) - *offset; 556 *stat &= 0xFFFFFFFFFFFFULL; 557 } 558 559 /** 560 * i40e_stat_update32 - read and update a 32 bit stat from the chip 561 * @hw: ptr to the hardware info 562 * @reg: the hw reg to read 563 * @offset_loaded: has the initial offset been loaded yet 564 * @offset: ptr to current offset value 565 * @stat: ptr to the stat 566 **/ 567 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 568 bool offset_loaded, u64 *offset, u64 *stat) 569 { 570 u32 new_data; 571 572 new_data = rd32(hw, reg); 573 if (!offset_loaded) 574 *offset = new_data; 575 if (likely(new_data >= *offset)) 576 *stat = (u32)(new_data - *offset); 577 else 578 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 579 } 580 581 /** 582 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 583 * @hw: ptr to the hardware info 584 * @reg: the hw reg to read and clear 585 * @stat: ptr to the stat 586 **/ 587 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 588 { 589 u32 new_data = rd32(hw, reg); 590 591 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 592 *stat += new_data; 593 } 594 595 /** 596 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 597 * @vsi: the VSI to be updated 598 **/ 599 void i40e_update_eth_stats(struct i40e_vsi *vsi) 600 { 601 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 602 struct i40e_pf *pf = vsi->back; 603 struct i40e_hw *hw = &pf->hw; 604 struct i40e_eth_stats *oes; 605 struct i40e_eth_stats *es; /* device's eth stats */ 606 607 es = &vsi->eth_stats; 608 oes = &vsi->eth_stats_offsets; 609 610 /* Gather up the stats that the hw collects */ 611 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 612 vsi->stat_offsets_loaded, 613 &oes->tx_errors, &es->tx_errors); 614 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 615 vsi->stat_offsets_loaded, 616 &oes->rx_discards, &es->rx_discards); 617 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 618 vsi->stat_offsets_loaded, 619 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 620 621 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 622 I40E_GLV_GORCL(stat_idx), 623 vsi->stat_offsets_loaded, 624 &oes->rx_bytes, &es->rx_bytes); 625 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 626 I40E_GLV_UPRCL(stat_idx), 627 vsi->stat_offsets_loaded, 628 &oes->rx_unicast, &es->rx_unicast); 629 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 630 I40E_GLV_MPRCL(stat_idx), 631 vsi->stat_offsets_loaded, 632 &oes->rx_multicast, &es->rx_multicast); 633 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 634 I40E_GLV_BPRCL(stat_idx), 635 vsi->stat_offsets_loaded, 636 &oes->rx_broadcast, &es->rx_broadcast); 637 638 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 639 I40E_GLV_GOTCL(stat_idx), 640 vsi->stat_offsets_loaded, 641 &oes->tx_bytes, &es->tx_bytes); 642 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 643 I40E_GLV_UPTCL(stat_idx), 644 vsi->stat_offsets_loaded, 645 &oes->tx_unicast, &es->tx_unicast); 646 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 647 I40E_GLV_MPTCL(stat_idx), 648 vsi->stat_offsets_loaded, 649 &oes->tx_multicast, &es->tx_multicast); 650 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 651 I40E_GLV_BPTCL(stat_idx), 652 vsi->stat_offsets_loaded, 653 &oes->tx_broadcast, &es->tx_broadcast); 654 vsi->stat_offsets_loaded = true; 655 } 656 657 /** 658 * i40e_update_veb_stats - Update Switch component statistics 659 * @veb: the VEB being updated 660 **/ 661 void i40e_update_veb_stats(struct i40e_veb *veb) 662 { 663 struct i40e_pf *pf = veb->pf; 664 struct i40e_hw *hw = &pf->hw; 665 struct i40e_eth_stats *oes; 666 struct i40e_eth_stats *es; /* device's eth stats */ 667 struct i40e_veb_tc_stats *veb_oes; 668 struct i40e_veb_tc_stats *veb_es; 669 int i, idx = 0; 670 671 idx = veb->stats_idx; 672 es = &veb->stats; 673 oes = &veb->stats_offsets; 674 veb_es = &veb->tc_stats; 675 veb_oes = &veb->tc_stats_offsets; 676 677 /* Gather up the stats that the hw collects */ 678 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 679 veb->stat_offsets_loaded, 680 &oes->tx_discards, &es->tx_discards); 681 if (hw->revision_id > 0) 682 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 683 veb->stat_offsets_loaded, 684 &oes->rx_unknown_protocol, 685 &es->rx_unknown_protocol); 686 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 687 veb->stat_offsets_loaded, 688 &oes->rx_bytes, &es->rx_bytes); 689 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 690 veb->stat_offsets_loaded, 691 &oes->rx_unicast, &es->rx_unicast); 692 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 693 veb->stat_offsets_loaded, 694 &oes->rx_multicast, &es->rx_multicast); 695 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 696 veb->stat_offsets_loaded, 697 &oes->rx_broadcast, &es->rx_broadcast); 698 699 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 700 veb->stat_offsets_loaded, 701 &oes->tx_bytes, &es->tx_bytes); 702 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 703 veb->stat_offsets_loaded, 704 &oes->tx_unicast, &es->tx_unicast); 705 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 706 veb->stat_offsets_loaded, 707 &oes->tx_multicast, &es->tx_multicast); 708 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 709 veb->stat_offsets_loaded, 710 &oes->tx_broadcast, &es->tx_broadcast); 711 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 712 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 713 I40E_GLVEBTC_RPCL(i, idx), 714 veb->stat_offsets_loaded, 715 &veb_oes->tc_rx_packets[i], 716 &veb_es->tc_rx_packets[i]); 717 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 718 I40E_GLVEBTC_RBCL(i, idx), 719 veb->stat_offsets_loaded, 720 &veb_oes->tc_rx_bytes[i], 721 &veb_es->tc_rx_bytes[i]); 722 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 723 I40E_GLVEBTC_TPCL(i, idx), 724 veb->stat_offsets_loaded, 725 &veb_oes->tc_tx_packets[i], 726 &veb_es->tc_tx_packets[i]); 727 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 728 I40E_GLVEBTC_TBCL(i, idx), 729 veb->stat_offsets_loaded, 730 &veb_oes->tc_tx_bytes[i], 731 &veb_es->tc_tx_bytes[i]); 732 } 733 veb->stat_offsets_loaded = true; 734 } 735 736 /** 737 * i40e_update_vsi_stats - Update the vsi statistics counters. 738 * @vsi: the VSI to be updated 739 * 740 * There are a few instances where we store the same stat in a 741 * couple of different structs. This is partly because we have 742 * the netdev stats that need to be filled out, which is slightly 743 * different from the "eth_stats" defined by the chip and used in 744 * VF communications. We sort it out here. 745 **/ 746 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 747 { 748 struct i40e_pf *pf = vsi->back; 749 struct rtnl_link_stats64 *ons; 750 struct rtnl_link_stats64 *ns; /* netdev stats */ 751 struct i40e_eth_stats *oes; 752 struct i40e_eth_stats *es; /* device's eth stats */ 753 u32 tx_restart, tx_busy; 754 struct i40e_ring *p; 755 u32 rx_page, rx_buf; 756 u64 bytes, packets; 757 unsigned int start; 758 u64 tx_linearize; 759 u64 tx_force_wb; 760 u64 rx_p, rx_b; 761 u64 tx_p, tx_b; 762 u16 q; 763 764 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 765 test_bit(__I40E_CONFIG_BUSY, pf->state)) 766 return; 767 768 ns = i40e_get_vsi_stats_struct(vsi); 769 ons = &vsi->net_stats_offsets; 770 es = &vsi->eth_stats; 771 oes = &vsi->eth_stats_offsets; 772 773 /* Gather up the netdev and vsi stats that the driver collects 774 * on the fly during packet processing 775 */ 776 rx_b = rx_p = 0; 777 tx_b = tx_p = 0; 778 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 779 rx_page = 0; 780 rx_buf = 0; 781 rcu_read_lock(); 782 for (q = 0; q < vsi->num_queue_pairs; q++) { 783 /* locate Tx ring */ 784 p = READ_ONCE(vsi->tx_rings[q]); 785 if (!p) 786 continue; 787 788 do { 789 start = u64_stats_fetch_begin_irq(&p->syncp); 790 packets = p->stats.packets; 791 bytes = p->stats.bytes; 792 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 793 tx_b += bytes; 794 tx_p += packets; 795 tx_restart += p->tx_stats.restart_queue; 796 tx_busy += p->tx_stats.tx_busy; 797 tx_linearize += p->tx_stats.tx_linearize; 798 tx_force_wb += p->tx_stats.tx_force_wb; 799 800 /* locate Rx ring */ 801 p = READ_ONCE(vsi->rx_rings[q]); 802 if (!p) 803 continue; 804 805 do { 806 start = u64_stats_fetch_begin_irq(&p->syncp); 807 packets = p->stats.packets; 808 bytes = p->stats.bytes; 809 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 810 rx_b += bytes; 811 rx_p += packets; 812 rx_buf += p->rx_stats.alloc_buff_failed; 813 rx_page += p->rx_stats.alloc_page_failed; 814 815 if (i40e_enabled_xdp_vsi(vsi)) { 816 /* locate XDP ring */ 817 p = READ_ONCE(vsi->xdp_rings[q]); 818 if (!p) 819 continue; 820 821 do { 822 start = u64_stats_fetch_begin_irq(&p->syncp); 823 packets = p->stats.packets; 824 bytes = p->stats.bytes; 825 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 826 tx_b += bytes; 827 tx_p += packets; 828 tx_restart += p->tx_stats.restart_queue; 829 tx_busy += p->tx_stats.tx_busy; 830 tx_linearize += p->tx_stats.tx_linearize; 831 tx_force_wb += p->tx_stats.tx_force_wb; 832 } 833 } 834 rcu_read_unlock(); 835 vsi->tx_restart = tx_restart; 836 vsi->tx_busy = tx_busy; 837 vsi->tx_linearize = tx_linearize; 838 vsi->tx_force_wb = tx_force_wb; 839 vsi->rx_page_failed = rx_page; 840 vsi->rx_buf_failed = rx_buf; 841 842 ns->rx_packets = rx_p; 843 ns->rx_bytes = rx_b; 844 ns->tx_packets = tx_p; 845 ns->tx_bytes = tx_b; 846 847 /* update netdev stats from eth stats */ 848 i40e_update_eth_stats(vsi); 849 ons->tx_errors = oes->tx_errors; 850 ns->tx_errors = es->tx_errors; 851 ons->multicast = oes->rx_multicast; 852 ns->multicast = es->rx_multicast; 853 ons->rx_dropped = oes->rx_discards; 854 ns->rx_dropped = es->rx_discards; 855 ons->tx_dropped = oes->tx_discards; 856 ns->tx_dropped = es->tx_discards; 857 858 /* pull in a couple PF stats if this is the main vsi */ 859 if (vsi == pf->vsi[pf->lan_vsi]) { 860 ns->rx_crc_errors = pf->stats.crc_errors; 861 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 862 ns->rx_length_errors = pf->stats.rx_length_errors; 863 } 864 } 865 866 /** 867 * i40e_update_pf_stats - Update the PF statistics counters. 868 * @pf: the PF to be updated 869 **/ 870 static void i40e_update_pf_stats(struct i40e_pf *pf) 871 { 872 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 873 struct i40e_hw_port_stats *nsd = &pf->stats; 874 struct i40e_hw *hw = &pf->hw; 875 u32 val; 876 int i; 877 878 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 879 I40E_GLPRT_GORCL(hw->port), 880 pf->stat_offsets_loaded, 881 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 882 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 883 I40E_GLPRT_GOTCL(hw->port), 884 pf->stat_offsets_loaded, 885 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 886 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 887 pf->stat_offsets_loaded, 888 &osd->eth.rx_discards, 889 &nsd->eth.rx_discards); 890 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 891 I40E_GLPRT_UPRCL(hw->port), 892 pf->stat_offsets_loaded, 893 &osd->eth.rx_unicast, 894 &nsd->eth.rx_unicast); 895 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 896 I40E_GLPRT_MPRCL(hw->port), 897 pf->stat_offsets_loaded, 898 &osd->eth.rx_multicast, 899 &nsd->eth.rx_multicast); 900 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 901 I40E_GLPRT_BPRCL(hw->port), 902 pf->stat_offsets_loaded, 903 &osd->eth.rx_broadcast, 904 &nsd->eth.rx_broadcast); 905 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 906 I40E_GLPRT_UPTCL(hw->port), 907 pf->stat_offsets_loaded, 908 &osd->eth.tx_unicast, 909 &nsd->eth.tx_unicast); 910 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 911 I40E_GLPRT_MPTCL(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->eth.tx_multicast, 914 &nsd->eth.tx_multicast); 915 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 916 I40E_GLPRT_BPTCL(hw->port), 917 pf->stat_offsets_loaded, 918 &osd->eth.tx_broadcast, 919 &nsd->eth.tx_broadcast); 920 921 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 922 pf->stat_offsets_loaded, 923 &osd->tx_dropped_link_down, 924 &nsd->tx_dropped_link_down); 925 926 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 927 pf->stat_offsets_loaded, 928 &osd->crc_errors, &nsd->crc_errors); 929 930 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 931 pf->stat_offsets_loaded, 932 &osd->illegal_bytes, &nsd->illegal_bytes); 933 934 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 935 pf->stat_offsets_loaded, 936 &osd->mac_local_faults, 937 &nsd->mac_local_faults); 938 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 939 pf->stat_offsets_loaded, 940 &osd->mac_remote_faults, 941 &nsd->mac_remote_faults); 942 943 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 944 pf->stat_offsets_loaded, 945 &osd->rx_length_errors, 946 &nsd->rx_length_errors); 947 948 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 949 pf->stat_offsets_loaded, 950 &osd->link_xon_rx, &nsd->link_xon_rx); 951 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 952 pf->stat_offsets_loaded, 953 &osd->link_xon_tx, &nsd->link_xon_tx); 954 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 955 pf->stat_offsets_loaded, 956 &osd->link_xoff_rx, &nsd->link_xoff_rx); 957 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 958 pf->stat_offsets_loaded, 959 &osd->link_xoff_tx, &nsd->link_xoff_tx); 960 961 for (i = 0; i < 8; i++) { 962 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 963 pf->stat_offsets_loaded, 964 &osd->priority_xoff_rx[i], 965 &nsd->priority_xoff_rx[i]); 966 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 967 pf->stat_offsets_loaded, 968 &osd->priority_xon_rx[i], 969 &nsd->priority_xon_rx[i]); 970 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 971 pf->stat_offsets_loaded, 972 &osd->priority_xon_tx[i], 973 &nsd->priority_xon_tx[i]); 974 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 975 pf->stat_offsets_loaded, 976 &osd->priority_xoff_tx[i], 977 &nsd->priority_xoff_tx[i]); 978 i40e_stat_update32(hw, 979 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 980 pf->stat_offsets_loaded, 981 &osd->priority_xon_2_xoff[i], 982 &nsd->priority_xon_2_xoff[i]); 983 } 984 985 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 986 I40E_GLPRT_PRC64L(hw->port), 987 pf->stat_offsets_loaded, 988 &osd->rx_size_64, &nsd->rx_size_64); 989 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 990 I40E_GLPRT_PRC127L(hw->port), 991 pf->stat_offsets_loaded, 992 &osd->rx_size_127, &nsd->rx_size_127); 993 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 994 I40E_GLPRT_PRC255L(hw->port), 995 pf->stat_offsets_loaded, 996 &osd->rx_size_255, &nsd->rx_size_255); 997 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 998 I40E_GLPRT_PRC511L(hw->port), 999 pf->stat_offsets_loaded, 1000 &osd->rx_size_511, &nsd->rx_size_511); 1001 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1002 I40E_GLPRT_PRC1023L(hw->port), 1003 pf->stat_offsets_loaded, 1004 &osd->rx_size_1023, &nsd->rx_size_1023); 1005 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1006 I40E_GLPRT_PRC1522L(hw->port), 1007 pf->stat_offsets_loaded, 1008 &osd->rx_size_1522, &nsd->rx_size_1522); 1009 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1010 I40E_GLPRT_PRC9522L(hw->port), 1011 pf->stat_offsets_loaded, 1012 &osd->rx_size_big, &nsd->rx_size_big); 1013 1014 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1015 I40E_GLPRT_PTC64L(hw->port), 1016 pf->stat_offsets_loaded, 1017 &osd->tx_size_64, &nsd->tx_size_64); 1018 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1019 I40E_GLPRT_PTC127L(hw->port), 1020 pf->stat_offsets_loaded, 1021 &osd->tx_size_127, &nsd->tx_size_127); 1022 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1023 I40E_GLPRT_PTC255L(hw->port), 1024 pf->stat_offsets_loaded, 1025 &osd->tx_size_255, &nsd->tx_size_255); 1026 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1027 I40E_GLPRT_PTC511L(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->tx_size_511, &nsd->tx_size_511); 1030 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1031 I40E_GLPRT_PTC1023L(hw->port), 1032 pf->stat_offsets_loaded, 1033 &osd->tx_size_1023, &nsd->tx_size_1023); 1034 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1035 I40E_GLPRT_PTC1522L(hw->port), 1036 pf->stat_offsets_loaded, 1037 &osd->tx_size_1522, &nsd->tx_size_1522); 1038 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1039 I40E_GLPRT_PTC9522L(hw->port), 1040 pf->stat_offsets_loaded, 1041 &osd->tx_size_big, &nsd->tx_size_big); 1042 1043 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1044 pf->stat_offsets_loaded, 1045 &osd->rx_undersize, &nsd->rx_undersize); 1046 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1047 pf->stat_offsets_loaded, 1048 &osd->rx_fragments, &nsd->rx_fragments); 1049 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1050 pf->stat_offsets_loaded, 1051 &osd->rx_oversize, &nsd->rx_oversize); 1052 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1053 pf->stat_offsets_loaded, 1054 &osd->rx_jabber, &nsd->rx_jabber); 1055 1056 /* FDIR stats */ 1057 i40e_stat_update_and_clear32(hw, 1058 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1059 &nsd->fd_atr_match); 1060 i40e_stat_update_and_clear32(hw, 1061 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1062 &nsd->fd_sb_match); 1063 i40e_stat_update_and_clear32(hw, 1064 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1065 &nsd->fd_atr_tunnel_match); 1066 1067 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1068 nsd->tx_lpi_status = 1069 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1070 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1071 nsd->rx_lpi_status = 1072 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1073 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1074 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1075 pf->stat_offsets_loaded, 1076 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1077 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1078 pf->stat_offsets_loaded, 1079 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1080 1081 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1082 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1083 nsd->fd_sb_status = true; 1084 else 1085 nsd->fd_sb_status = false; 1086 1087 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1088 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1089 nsd->fd_atr_status = true; 1090 else 1091 nsd->fd_atr_status = false; 1092 1093 pf->stat_offsets_loaded = true; 1094 } 1095 1096 /** 1097 * i40e_update_stats - Update the various statistics counters. 1098 * @vsi: the VSI to be updated 1099 * 1100 * Update the various stats for this VSI and its related entities. 1101 **/ 1102 void i40e_update_stats(struct i40e_vsi *vsi) 1103 { 1104 struct i40e_pf *pf = vsi->back; 1105 1106 if (vsi == pf->vsi[pf->lan_vsi]) 1107 i40e_update_pf_stats(pf); 1108 1109 i40e_update_vsi_stats(vsi); 1110 } 1111 1112 /** 1113 * i40e_count_filters - counts VSI mac filters 1114 * @vsi: the VSI to be searched 1115 * 1116 * Returns count of mac filters 1117 **/ 1118 int i40e_count_filters(struct i40e_vsi *vsi) 1119 { 1120 struct i40e_mac_filter *f; 1121 struct hlist_node *h; 1122 int bkt; 1123 int cnt = 0; 1124 1125 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 1126 ++cnt; 1127 1128 return cnt; 1129 } 1130 1131 /** 1132 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1133 * @vsi: the VSI to be searched 1134 * @macaddr: the MAC address 1135 * @vlan: the vlan 1136 * 1137 * Returns ptr to the filter object or NULL 1138 **/ 1139 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1140 const u8 *macaddr, s16 vlan) 1141 { 1142 struct i40e_mac_filter *f; 1143 u64 key; 1144 1145 if (!vsi || !macaddr) 1146 return NULL; 1147 1148 key = i40e_addr_to_hkey(macaddr); 1149 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1150 if ((ether_addr_equal(macaddr, f->macaddr)) && 1151 (vlan == f->vlan)) 1152 return f; 1153 } 1154 return NULL; 1155 } 1156 1157 /** 1158 * i40e_find_mac - Find a mac addr in the macvlan filters list 1159 * @vsi: the VSI to be searched 1160 * @macaddr: the MAC address we are searching for 1161 * 1162 * Returns the first filter with the provided MAC address or NULL if 1163 * MAC address was not found 1164 **/ 1165 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1166 { 1167 struct i40e_mac_filter *f; 1168 u64 key; 1169 1170 if (!vsi || !macaddr) 1171 return NULL; 1172 1173 key = i40e_addr_to_hkey(macaddr); 1174 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1175 if ((ether_addr_equal(macaddr, f->macaddr))) 1176 return f; 1177 } 1178 return NULL; 1179 } 1180 1181 /** 1182 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1183 * @vsi: the VSI to be searched 1184 * 1185 * Returns true if VSI is in vlan mode or false otherwise 1186 **/ 1187 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1188 { 1189 /* If we have a PVID, always operate in VLAN mode */ 1190 if (vsi->info.pvid) 1191 return true; 1192 1193 /* We need to operate in VLAN mode whenever we have any filters with 1194 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1195 * time, incurring search cost repeatedly. However, we can notice two 1196 * things: 1197 * 1198 * 1) the only place where we can gain a VLAN filter is in 1199 * i40e_add_filter. 1200 * 1201 * 2) the only place where filters are actually removed is in 1202 * i40e_sync_filters_subtask. 1203 * 1204 * Thus, we can simply use a boolean value, has_vlan_filters which we 1205 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1206 * we have to perform the full search after deleting filters in 1207 * i40e_sync_filters_subtask, but we already have to search 1208 * filters here and can perform the check at the same time. This 1209 * results in avoiding embedding a loop for VLAN mode inside another 1210 * loop over all the filters, and should maintain correctness as noted 1211 * above. 1212 */ 1213 return vsi->has_vlan_filter; 1214 } 1215 1216 /** 1217 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1218 * @vsi: the VSI to configure 1219 * @tmp_add_list: list of filters ready to be added 1220 * @tmp_del_list: list of filters ready to be deleted 1221 * @vlan_filters: the number of active VLAN filters 1222 * 1223 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1224 * behave as expected. If we have any active VLAN filters remaining or about 1225 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1226 * so that they only match against untagged traffic. If we no longer have any 1227 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1228 * so that they match against both tagged and untagged traffic. In this way, 1229 * we ensure that we correctly receive the desired traffic. This ensures that 1230 * when we have an active VLAN we will receive only untagged traffic and 1231 * traffic matching active VLANs. If we have no active VLANs then we will 1232 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1233 * 1234 * Finally, in a similar fashion, this function also corrects filters when 1235 * there is an active PVID assigned to this VSI. 1236 * 1237 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1238 * 1239 * This function is only expected to be called from within 1240 * i40e_sync_vsi_filters. 1241 * 1242 * NOTE: This function expects to be called while under the 1243 * mac_filter_hash_lock 1244 */ 1245 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1246 struct hlist_head *tmp_add_list, 1247 struct hlist_head *tmp_del_list, 1248 int vlan_filters) 1249 { 1250 s16 pvid = le16_to_cpu(vsi->info.pvid); 1251 struct i40e_mac_filter *f, *add_head; 1252 struct i40e_new_mac_filter *new; 1253 struct hlist_node *h; 1254 int bkt, new_vlan; 1255 1256 /* To determine if a particular filter needs to be replaced we 1257 * have the three following conditions: 1258 * 1259 * a) if we have a PVID assigned, then all filters which are 1260 * not marked as VLAN=PVID must be replaced with filters that 1261 * are. 1262 * b) otherwise, if we have any active VLANS, all filters 1263 * which are marked as VLAN=-1 must be replaced with 1264 * filters marked as VLAN=0 1265 * c) finally, if we do not have any active VLANS, all filters 1266 * which are marked as VLAN=0 must be replaced with filters 1267 * marked as VLAN=-1 1268 */ 1269 1270 /* Update the filters about to be added in place */ 1271 hlist_for_each_entry(new, tmp_add_list, hlist) { 1272 if (pvid && new->f->vlan != pvid) 1273 new->f->vlan = pvid; 1274 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1275 new->f->vlan = 0; 1276 else if (!vlan_filters && new->f->vlan == 0) 1277 new->f->vlan = I40E_VLAN_ANY; 1278 } 1279 1280 /* Update the remaining active filters */ 1281 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1282 /* Combine the checks for whether a filter needs to be changed 1283 * and then determine the new VLAN inside the if block, in 1284 * order to avoid duplicating code for adding the new filter 1285 * then deleting the old filter. 1286 */ 1287 if ((pvid && f->vlan != pvid) || 1288 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1289 (!vlan_filters && f->vlan == 0)) { 1290 /* Determine the new vlan we will be adding */ 1291 if (pvid) 1292 new_vlan = pvid; 1293 else if (vlan_filters) 1294 new_vlan = 0; 1295 else 1296 new_vlan = I40E_VLAN_ANY; 1297 1298 /* Create the new filter */ 1299 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1300 if (!add_head) 1301 return -ENOMEM; 1302 1303 /* Create a temporary i40e_new_mac_filter */ 1304 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1305 if (!new) 1306 return -ENOMEM; 1307 1308 new->f = add_head; 1309 new->state = add_head->state; 1310 1311 /* Add the new filter to the tmp list */ 1312 hlist_add_head(&new->hlist, tmp_add_list); 1313 1314 /* Put the original filter into the delete list */ 1315 f->state = I40E_FILTER_REMOVE; 1316 hash_del(&f->hlist); 1317 hlist_add_head(&f->hlist, tmp_del_list); 1318 } 1319 } 1320 1321 vsi->has_vlan_filter = !!vlan_filters; 1322 1323 return 0; 1324 } 1325 1326 /** 1327 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1328 * @vsi: the PF Main VSI - inappropriate for any other VSI 1329 * @macaddr: the MAC address 1330 * 1331 * Remove whatever filter the firmware set up so the driver can manage 1332 * its own filtering intelligently. 1333 **/ 1334 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1335 { 1336 struct i40e_aqc_remove_macvlan_element_data element; 1337 struct i40e_pf *pf = vsi->back; 1338 1339 /* Only appropriate for the PF main VSI */ 1340 if (vsi->type != I40E_VSI_MAIN) 1341 return; 1342 1343 memset(&element, 0, sizeof(element)); 1344 ether_addr_copy(element.mac_addr, macaddr); 1345 element.vlan_tag = 0; 1346 /* Ignore error returns, some firmware does it this way... */ 1347 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1348 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1349 1350 memset(&element, 0, sizeof(element)); 1351 ether_addr_copy(element.mac_addr, macaddr); 1352 element.vlan_tag = 0; 1353 /* ...and some firmware does it this way. */ 1354 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1355 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1356 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1357 } 1358 1359 /** 1360 * i40e_add_filter - Add a mac/vlan filter to the VSI 1361 * @vsi: the VSI to be searched 1362 * @macaddr: the MAC address 1363 * @vlan: the vlan 1364 * 1365 * Returns ptr to the filter object or NULL when no memory available. 1366 * 1367 * NOTE: This function is expected to be called with mac_filter_hash_lock 1368 * being held. 1369 **/ 1370 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1371 const u8 *macaddr, s16 vlan) 1372 { 1373 struct i40e_mac_filter *f; 1374 u64 key; 1375 1376 if (!vsi || !macaddr) 1377 return NULL; 1378 1379 f = i40e_find_filter(vsi, macaddr, vlan); 1380 if (!f) { 1381 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1382 if (!f) 1383 return NULL; 1384 1385 /* Update the boolean indicating if we need to function in 1386 * VLAN mode. 1387 */ 1388 if (vlan >= 0) 1389 vsi->has_vlan_filter = true; 1390 1391 ether_addr_copy(f->macaddr, macaddr); 1392 f->vlan = vlan; 1393 f->state = I40E_FILTER_NEW; 1394 INIT_HLIST_NODE(&f->hlist); 1395 1396 key = i40e_addr_to_hkey(macaddr); 1397 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1398 1399 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1400 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1401 } 1402 1403 /* If we're asked to add a filter that has been marked for removal, it 1404 * is safe to simply restore it to active state. __i40e_del_filter 1405 * will have simply deleted any filters which were previously marked 1406 * NEW or FAILED, so if it is currently marked REMOVE it must have 1407 * previously been ACTIVE. Since we haven't yet run the sync filters 1408 * task, just restore this filter to the ACTIVE state so that the 1409 * sync task leaves it in place 1410 */ 1411 if (f->state == I40E_FILTER_REMOVE) 1412 f->state = I40E_FILTER_ACTIVE; 1413 1414 return f; 1415 } 1416 1417 /** 1418 * __i40e_del_filter - Remove a specific filter from the VSI 1419 * @vsi: VSI to remove from 1420 * @f: the filter to remove from the list 1421 * 1422 * This function should be called instead of i40e_del_filter only if you know 1423 * the exact filter you will remove already, such as via i40e_find_filter or 1424 * i40e_find_mac. 1425 * 1426 * NOTE: This function is expected to be called with mac_filter_hash_lock 1427 * being held. 1428 * ANOTHER NOTE: This function MUST be called from within the context of 1429 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1430 * instead of list_for_each_entry(). 1431 **/ 1432 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1433 { 1434 if (!f) 1435 return; 1436 1437 /* If the filter was never added to firmware then we can just delete it 1438 * directly and we don't want to set the status to remove or else an 1439 * admin queue command will unnecessarily fire. 1440 */ 1441 if ((f->state == I40E_FILTER_FAILED) || 1442 (f->state == I40E_FILTER_NEW)) { 1443 hash_del(&f->hlist); 1444 kfree(f); 1445 } else { 1446 f->state = I40E_FILTER_REMOVE; 1447 } 1448 1449 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1450 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1451 } 1452 1453 /** 1454 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1455 * @vsi: the VSI to be searched 1456 * @macaddr: the MAC address 1457 * @vlan: the VLAN 1458 * 1459 * NOTE: This function is expected to be called with mac_filter_hash_lock 1460 * being held. 1461 * ANOTHER NOTE: This function MUST be called from within the context of 1462 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1463 * instead of list_for_each_entry(). 1464 **/ 1465 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1466 { 1467 struct i40e_mac_filter *f; 1468 1469 if (!vsi || !macaddr) 1470 return; 1471 1472 f = i40e_find_filter(vsi, macaddr, vlan); 1473 __i40e_del_filter(vsi, f); 1474 } 1475 1476 /** 1477 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1478 * @vsi: the VSI to be searched 1479 * @macaddr: the mac address to be filtered 1480 * 1481 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1482 * go through all the macvlan filters and add a macvlan filter for each 1483 * unique vlan that already exists. If a PVID has been assigned, instead only 1484 * add the macaddr to that VLAN. 1485 * 1486 * Returns last filter added on success, else NULL 1487 **/ 1488 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1489 const u8 *macaddr) 1490 { 1491 struct i40e_mac_filter *f, *add = NULL; 1492 struct hlist_node *h; 1493 int bkt; 1494 1495 if (vsi->info.pvid) 1496 return i40e_add_filter(vsi, macaddr, 1497 le16_to_cpu(vsi->info.pvid)); 1498 1499 if (!i40e_is_vsi_in_vlan(vsi)) 1500 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1501 1502 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1503 if (f->state == I40E_FILTER_REMOVE) 1504 continue; 1505 add = i40e_add_filter(vsi, macaddr, f->vlan); 1506 if (!add) 1507 return NULL; 1508 } 1509 1510 return add; 1511 } 1512 1513 /** 1514 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1515 * @vsi: the VSI to be searched 1516 * @macaddr: the mac address to be removed 1517 * 1518 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1519 * associated with. 1520 * 1521 * Returns 0 for success, or error 1522 **/ 1523 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1524 { 1525 struct i40e_mac_filter *f; 1526 struct hlist_node *h; 1527 bool found = false; 1528 int bkt; 1529 1530 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1531 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1532 if (ether_addr_equal(macaddr, f->macaddr)) { 1533 __i40e_del_filter(vsi, f); 1534 found = true; 1535 } 1536 } 1537 1538 if (found) 1539 return 0; 1540 else 1541 return -ENOENT; 1542 } 1543 1544 /** 1545 * i40e_set_mac - NDO callback to set mac address 1546 * @netdev: network interface device structure 1547 * @p: pointer to an address structure 1548 * 1549 * Returns 0 on success, negative on failure 1550 **/ 1551 static int i40e_set_mac(struct net_device *netdev, void *p) 1552 { 1553 struct i40e_netdev_priv *np = netdev_priv(netdev); 1554 struct i40e_vsi *vsi = np->vsi; 1555 struct i40e_pf *pf = vsi->back; 1556 struct i40e_hw *hw = &pf->hw; 1557 struct sockaddr *addr = p; 1558 1559 if (!is_valid_ether_addr(addr->sa_data)) 1560 return -EADDRNOTAVAIL; 1561 1562 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1563 netdev_info(netdev, "already using mac address %pM\n", 1564 addr->sa_data); 1565 return 0; 1566 } 1567 1568 if (test_bit(__I40E_DOWN, pf->state) || 1569 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 1570 return -EADDRNOTAVAIL; 1571 1572 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1573 netdev_info(netdev, "returning to hw mac address %pM\n", 1574 hw->mac.addr); 1575 else 1576 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1577 1578 /* Copy the address first, so that we avoid a possible race with 1579 * .set_rx_mode(). 1580 * - Remove old address from MAC filter 1581 * - Copy new address 1582 * - Add new address to MAC filter 1583 */ 1584 spin_lock_bh(&vsi->mac_filter_hash_lock); 1585 i40e_del_mac_filter(vsi, netdev->dev_addr); 1586 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1587 i40e_add_mac_filter(vsi, netdev->dev_addr); 1588 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1589 1590 if (vsi->type == I40E_VSI_MAIN) { 1591 i40e_status ret; 1592 1593 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, 1594 addr->sa_data, NULL); 1595 if (ret) 1596 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1597 i40e_stat_str(hw, ret), 1598 i40e_aq_str(hw, hw->aq.asq_last_status)); 1599 } 1600 1601 /* schedule our worker thread which will take care of 1602 * applying the new filter changes 1603 */ 1604 i40e_service_event_schedule(pf); 1605 return 0; 1606 } 1607 1608 /** 1609 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1610 * @vsi: vsi structure 1611 * @seed: RSS hash seed 1612 **/ 1613 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1614 u8 *lut, u16 lut_size) 1615 { 1616 struct i40e_pf *pf = vsi->back; 1617 struct i40e_hw *hw = &pf->hw; 1618 int ret = 0; 1619 1620 if (seed) { 1621 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1622 (struct i40e_aqc_get_set_rss_key_data *)seed; 1623 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1624 if (ret) { 1625 dev_info(&pf->pdev->dev, 1626 "Cannot set RSS key, err %s aq_err %s\n", 1627 i40e_stat_str(hw, ret), 1628 i40e_aq_str(hw, hw->aq.asq_last_status)); 1629 return ret; 1630 } 1631 } 1632 if (lut) { 1633 bool pf_lut = vsi->type == I40E_VSI_MAIN; 1634 1635 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1636 if (ret) { 1637 dev_info(&pf->pdev->dev, 1638 "Cannot set RSS lut, err %s aq_err %s\n", 1639 i40e_stat_str(hw, ret), 1640 i40e_aq_str(hw, hw->aq.asq_last_status)); 1641 return ret; 1642 } 1643 } 1644 return ret; 1645 } 1646 1647 /** 1648 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1649 * @vsi: VSI structure 1650 **/ 1651 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1652 { 1653 struct i40e_pf *pf = vsi->back; 1654 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1655 u8 *lut; 1656 int ret; 1657 1658 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1659 return 0; 1660 if (!vsi->rss_size) 1661 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1662 vsi->num_queue_pairs); 1663 if (!vsi->rss_size) 1664 return -EINVAL; 1665 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1666 if (!lut) 1667 return -ENOMEM; 1668 1669 /* Use the user configured hash keys and lookup table if there is one, 1670 * otherwise use default 1671 */ 1672 if (vsi->rss_lut_user) 1673 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1674 else 1675 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1676 if (vsi->rss_hkey_user) 1677 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1678 else 1679 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1680 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1681 kfree(lut); 1682 return ret; 1683 } 1684 1685 /** 1686 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1687 * @vsi: the VSI being configured, 1688 * @ctxt: VSI context structure 1689 * @enabled_tc: number of traffic classes to enable 1690 * 1691 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1692 **/ 1693 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1694 struct i40e_vsi_context *ctxt, 1695 u8 enabled_tc) 1696 { 1697 u16 qcount = 0, max_qcount, qmap, sections = 0; 1698 int i, override_q, pow, num_qps, ret; 1699 u8 netdev_tc = 0, offset = 0; 1700 1701 if (vsi->type != I40E_VSI_MAIN) 1702 return -EINVAL; 1703 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1704 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1705 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1706 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1707 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1708 1709 /* find the next higher power-of-2 of num queue pairs */ 1710 pow = ilog2(num_qps); 1711 if (!is_power_of_2(num_qps)) 1712 pow++; 1713 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1714 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1715 1716 /* Setup queue offset/count for all TCs for given VSI */ 1717 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1718 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1719 /* See if the given TC is enabled for the given VSI */ 1720 if (vsi->tc_config.enabled_tc & BIT(i)) { 1721 offset = vsi->mqprio_qopt.qopt.offset[i]; 1722 qcount = vsi->mqprio_qopt.qopt.count[i]; 1723 if (qcount > max_qcount) 1724 max_qcount = qcount; 1725 vsi->tc_config.tc_info[i].qoffset = offset; 1726 vsi->tc_config.tc_info[i].qcount = qcount; 1727 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1728 } else { 1729 /* TC is not enabled so set the offset to 1730 * default queue and allocate one queue 1731 * for the given TC. 1732 */ 1733 vsi->tc_config.tc_info[i].qoffset = 0; 1734 vsi->tc_config.tc_info[i].qcount = 1; 1735 vsi->tc_config.tc_info[i].netdev_tc = 0; 1736 } 1737 } 1738 1739 /* Set actual Tx/Rx queue pairs */ 1740 vsi->num_queue_pairs = offset + qcount; 1741 1742 /* Setup queue TC[0].qmap for given VSI context */ 1743 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1744 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1745 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1746 ctxt->info.valid_sections |= cpu_to_le16(sections); 1747 1748 /* Reconfigure RSS for main VSI with max queue count */ 1749 vsi->rss_size = max_qcount; 1750 ret = i40e_vsi_config_rss(vsi); 1751 if (ret) { 1752 dev_info(&vsi->back->pdev->dev, 1753 "Failed to reconfig rss for num_queues (%u)\n", 1754 max_qcount); 1755 return ret; 1756 } 1757 vsi->reconfig_rss = true; 1758 dev_dbg(&vsi->back->pdev->dev, 1759 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1760 1761 /* Find queue count available for channel VSIs and starting offset 1762 * for channel VSIs 1763 */ 1764 override_q = vsi->mqprio_qopt.qopt.count[0]; 1765 if (override_q && override_q < vsi->num_queue_pairs) { 1766 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1767 vsi->next_base_queue = override_q; 1768 } 1769 return 0; 1770 } 1771 1772 /** 1773 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1774 * @vsi: the VSI being setup 1775 * @ctxt: VSI context structure 1776 * @enabled_tc: Enabled TCs bitmap 1777 * @is_add: True if called before Add VSI 1778 * 1779 * Setup VSI queue mapping for enabled traffic classes. 1780 **/ 1781 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1782 struct i40e_vsi_context *ctxt, 1783 u8 enabled_tc, 1784 bool is_add) 1785 { 1786 struct i40e_pf *pf = vsi->back; 1787 u16 sections = 0; 1788 u8 netdev_tc = 0; 1789 u16 numtc = 1; 1790 u16 qcount; 1791 u8 offset; 1792 u16 qmap; 1793 int i; 1794 u16 num_tc_qps = 0; 1795 1796 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1797 offset = 0; 1798 1799 /* Number of queues per enabled TC */ 1800 num_tc_qps = vsi->alloc_queue_pairs; 1801 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1802 /* Find numtc from enabled TC bitmap */ 1803 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1804 if (enabled_tc & BIT(i)) /* TC is enabled */ 1805 numtc++; 1806 } 1807 if (!numtc) { 1808 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1809 numtc = 1; 1810 } 1811 num_tc_qps = num_tc_qps / numtc; 1812 num_tc_qps = min_t(int, num_tc_qps, 1813 i40e_pf_get_max_q_per_tc(pf)); 1814 } 1815 1816 vsi->tc_config.numtc = numtc; 1817 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1818 1819 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 1820 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1821 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 1822 1823 /* Setup queue offset/count for all TCs for given VSI */ 1824 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1825 /* See if the given TC is enabled for the given VSI */ 1826 if (vsi->tc_config.enabled_tc & BIT(i)) { 1827 /* TC is enabled */ 1828 int pow, num_qps; 1829 1830 switch (vsi->type) { 1831 case I40E_VSI_MAIN: 1832 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | 1833 I40E_FLAG_FD_ATR_ENABLED)) || 1834 vsi->tc_config.enabled_tc != 1) { 1835 qcount = min_t(int, pf->alloc_rss_size, 1836 num_tc_qps); 1837 break; 1838 } 1839 fallthrough; 1840 case I40E_VSI_FDIR: 1841 case I40E_VSI_SRIOV: 1842 case I40E_VSI_VMDQ2: 1843 default: 1844 qcount = num_tc_qps; 1845 WARN_ON(i != 0); 1846 break; 1847 } 1848 vsi->tc_config.tc_info[i].qoffset = offset; 1849 vsi->tc_config.tc_info[i].qcount = qcount; 1850 1851 /* find the next higher power-of-2 of num queue pairs */ 1852 num_qps = qcount; 1853 pow = 0; 1854 while (num_qps && (BIT_ULL(pow) < qcount)) { 1855 pow++; 1856 num_qps >>= 1; 1857 } 1858 1859 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1860 qmap = 1861 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1862 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1863 1864 offset += qcount; 1865 } else { 1866 /* TC is not enabled so set the offset to 1867 * default queue and allocate one queue 1868 * for the given TC. 1869 */ 1870 vsi->tc_config.tc_info[i].qoffset = 0; 1871 vsi->tc_config.tc_info[i].qcount = 1; 1872 vsi->tc_config.tc_info[i].netdev_tc = 0; 1873 1874 qmap = 0; 1875 } 1876 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1877 } 1878 1879 /* Set actual Tx/Rx queue pairs */ 1880 vsi->num_queue_pairs = offset; 1881 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1882 if (vsi->req_queue_pairs > 0) 1883 vsi->num_queue_pairs = vsi->req_queue_pairs; 1884 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1885 vsi->num_queue_pairs = pf->num_lan_msix; 1886 } 1887 1888 /* Scheduler section valid can only be set for ADD VSI */ 1889 if (is_add) { 1890 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1891 1892 ctxt->info.up_enable_bits = enabled_tc; 1893 } 1894 if (vsi->type == I40E_VSI_SRIOV) { 1895 ctxt->info.mapping_flags |= 1896 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1897 for (i = 0; i < vsi->num_queue_pairs; i++) 1898 ctxt->info.queue_mapping[i] = 1899 cpu_to_le16(vsi->base_queue + i); 1900 } else { 1901 ctxt->info.mapping_flags |= 1902 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1903 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1904 } 1905 ctxt->info.valid_sections |= cpu_to_le16(sections); 1906 } 1907 1908 /** 1909 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1910 * @netdev: the netdevice 1911 * @addr: address to add 1912 * 1913 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1914 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1915 */ 1916 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1917 { 1918 struct i40e_netdev_priv *np = netdev_priv(netdev); 1919 struct i40e_vsi *vsi = np->vsi; 1920 1921 if (i40e_add_mac_filter(vsi, addr)) 1922 return 0; 1923 else 1924 return -ENOMEM; 1925 } 1926 1927 /** 1928 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1929 * @netdev: the netdevice 1930 * @addr: address to add 1931 * 1932 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1933 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1934 */ 1935 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1936 { 1937 struct i40e_netdev_priv *np = netdev_priv(netdev); 1938 struct i40e_vsi *vsi = np->vsi; 1939 1940 /* Under some circumstances, we might receive a request to delete 1941 * our own device address from our uc list. Because we store the 1942 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1943 * such requests and not delete our device address from this list. 1944 */ 1945 if (ether_addr_equal(addr, netdev->dev_addr)) 1946 return 0; 1947 1948 i40e_del_mac_filter(vsi, addr); 1949 1950 return 0; 1951 } 1952 1953 /** 1954 * i40e_set_rx_mode - NDO callback to set the netdev filters 1955 * @netdev: network interface device structure 1956 **/ 1957 static void i40e_set_rx_mode(struct net_device *netdev) 1958 { 1959 struct i40e_netdev_priv *np = netdev_priv(netdev); 1960 struct i40e_vsi *vsi = np->vsi; 1961 1962 spin_lock_bh(&vsi->mac_filter_hash_lock); 1963 1964 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1965 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1966 1967 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1968 1969 /* check for other flag changes */ 1970 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1971 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1972 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1973 } 1974 } 1975 1976 /** 1977 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1978 * @vsi: Pointer to VSI struct 1979 * @from: Pointer to list which contains MAC filter entries - changes to 1980 * those entries needs to be undone. 1981 * 1982 * MAC filter entries from this list were slated for deletion. 1983 **/ 1984 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1985 struct hlist_head *from) 1986 { 1987 struct i40e_mac_filter *f; 1988 struct hlist_node *h; 1989 1990 hlist_for_each_entry_safe(f, h, from, hlist) { 1991 u64 key = i40e_addr_to_hkey(f->macaddr); 1992 1993 /* Move the element back into MAC filter list*/ 1994 hlist_del(&f->hlist); 1995 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1996 } 1997 } 1998 1999 /** 2000 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 2001 * @vsi: Pointer to vsi struct 2002 * @from: Pointer to list which contains MAC filter entries - changes to 2003 * those entries needs to be undone. 2004 * 2005 * MAC filter entries from this list were slated for addition. 2006 **/ 2007 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 2008 struct hlist_head *from) 2009 { 2010 struct i40e_new_mac_filter *new; 2011 struct hlist_node *h; 2012 2013 hlist_for_each_entry_safe(new, h, from, hlist) { 2014 /* We can simply free the wrapper structure */ 2015 hlist_del(&new->hlist); 2016 kfree(new); 2017 } 2018 } 2019 2020 /** 2021 * i40e_next_entry - Get the next non-broadcast filter from a list 2022 * @next: pointer to filter in list 2023 * 2024 * Returns the next non-broadcast filter in the list. Required so that we 2025 * ignore broadcast filters within the list, since these are not handled via 2026 * the normal firmware update path. 2027 */ 2028 static 2029 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2030 { 2031 hlist_for_each_entry_continue(next, hlist) { 2032 if (!is_broadcast_ether_addr(next->f->macaddr)) 2033 return next; 2034 } 2035 2036 return NULL; 2037 } 2038 2039 /** 2040 * i40e_update_filter_state - Update filter state based on return data 2041 * from firmware 2042 * @count: Number of filters added 2043 * @add_list: return data from fw 2044 * @add_head: pointer to first filter in current batch 2045 * 2046 * MAC filter entries from list were slated to be added to device. Returns 2047 * number of successful filters. Note that 0 does NOT mean success! 2048 **/ 2049 static int 2050 i40e_update_filter_state(int count, 2051 struct i40e_aqc_add_macvlan_element_data *add_list, 2052 struct i40e_new_mac_filter *add_head) 2053 { 2054 int retval = 0; 2055 int i; 2056 2057 for (i = 0; i < count; i++) { 2058 /* Always check status of each filter. We don't need to check 2059 * the firmware return status because we pre-set the filter 2060 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2061 * request to the adminq. Thus, if it no longer matches then 2062 * we know the filter is active. 2063 */ 2064 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2065 add_head->state = I40E_FILTER_FAILED; 2066 } else { 2067 add_head->state = I40E_FILTER_ACTIVE; 2068 retval++; 2069 } 2070 2071 add_head = i40e_next_filter(add_head); 2072 if (!add_head) 2073 break; 2074 } 2075 2076 return retval; 2077 } 2078 2079 /** 2080 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2081 * @vsi: ptr to the VSI 2082 * @vsi_name: name to display in messages 2083 * @list: the list of filters to send to firmware 2084 * @num_del: the number of filters to delete 2085 * @retval: Set to -EIO on failure to delete 2086 * 2087 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2088 * *retval instead of a return value so that success does not force ret_val to 2089 * be set to 0. This ensures that a sequence of calls to this function 2090 * preserve the previous value of *retval on successful delete. 2091 */ 2092 static 2093 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2094 struct i40e_aqc_remove_macvlan_element_data *list, 2095 int num_del, int *retval) 2096 { 2097 struct i40e_hw *hw = &vsi->back->hw; 2098 i40e_status aq_ret; 2099 int aq_err; 2100 2101 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 2102 aq_err = hw->aq.asq_last_status; 2103 2104 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2105 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 2106 *retval = -EIO; 2107 dev_info(&vsi->back->pdev->dev, 2108 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 2109 vsi_name, i40e_stat_str(hw, aq_ret), 2110 i40e_aq_str(hw, aq_err)); 2111 } 2112 } 2113 2114 /** 2115 * i40e_aqc_add_filters - Request firmware to add a set of filters 2116 * @vsi: ptr to the VSI 2117 * @vsi_name: name to display in messages 2118 * @list: the list of filters to send to firmware 2119 * @add_head: Position in the add hlist 2120 * @num_add: the number of filters to add 2121 * 2122 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2123 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2124 * space for more filters. 2125 */ 2126 static 2127 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2128 struct i40e_aqc_add_macvlan_element_data *list, 2129 struct i40e_new_mac_filter *add_head, 2130 int num_add) 2131 { 2132 struct i40e_hw *hw = &vsi->back->hw; 2133 int aq_err, fcnt; 2134 2135 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 2136 aq_err = hw->aq.asq_last_status; 2137 fcnt = i40e_update_filter_state(num_add, list, add_head); 2138 2139 if (fcnt != num_add) { 2140 if (vsi->type == I40E_VSI_MAIN) { 2141 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2142 dev_warn(&vsi->back->pdev->dev, 2143 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2144 i40e_aq_str(hw, aq_err), vsi_name); 2145 } else if (vsi->type == I40E_VSI_SRIOV || 2146 vsi->type == I40E_VSI_VMDQ1 || 2147 vsi->type == I40E_VSI_VMDQ2) { 2148 dev_warn(&vsi->back->pdev->dev, 2149 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n", 2150 i40e_aq_str(hw, aq_err), vsi_name, vsi_name); 2151 } else { 2152 dev_warn(&vsi->back->pdev->dev, 2153 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n", 2154 i40e_aq_str(hw, aq_err), vsi_name, vsi->type); 2155 } 2156 } 2157 } 2158 2159 /** 2160 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2161 * @vsi: pointer to the VSI 2162 * @vsi_name: the VSI name 2163 * @f: filter data 2164 * 2165 * This function sets or clears the promiscuous broadcast flags for VLAN 2166 * filters in order to properly receive broadcast frames. Assumes that only 2167 * broadcast filters are passed. 2168 * 2169 * Returns status indicating success or failure; 2170 **/ 2171 static i40e_status 2172 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2173 struct i40e_mac_filter *f) 2174 { 2175 bool enable = f->state == I40E_FILTER_NEW; 2176 struct i40e_hw *hw = &vsi->back->hw; 2177 i40e_status aq_ret; 2178 2179 if (f->vlan == I40E_VLAN_ANY) { 2180 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2181 vsi->seid, 2182 enable, 2183 NULL); 2184 } else { 2185 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2186 vsi->seid, 2187 enable, 2188 f->vlan, 2189 NULL); 2190 } 2191 2192 if (aq_ret) { 2193 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2194 dev_warn(&vsi->back->pdev->dev, 2195 "Error %s, forcing overflow promiscuous on %s\n", 2196 i40e_aq_str(hw, hw->aq.asq_last_status), 2197 vsi_name); 2198 } 2199 2200 return aq_ret; 2201 } 2202 2203 /** 2204 * i40e_set_promiscuous - set promiscuous mode 2205 * @pf: board private structure 2206 * @promisc: promisc on or off 2207 * 2208 * There are different ways of setting promiscuous mode on a PF depending on 2209 * what state/environment we're in. This identifies and sets it appropriately. 2210 * Returns 0 on success. 2211 **/ 2212 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2213 { 2214 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2215 struct i40e_hw *hw = &pf->hw; 2216 i40e_status aq_ret; 2217 2218 if (vsi->type == I40E_VSI_MAIN && 2219 pf->lan_veb != I40E_NO_VEB && 2220 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2221 /* set defport ON for Main VSI instead of true promisc 2222 * this way we will get all unicast/multicast and VLAN 2223 * promisc behavior but will not get VF or VMDq traffic 2224 * replicated on the Main VSI. 2225 */ 2226 if (promisc) 2227 aq_ret = i40e_aq_set_default_vsi(hw, 2228 vsi->seid, 2229 NULL); 2230 else 2231 aq_ret = i40e_aq_clear_default_vsi(hw, 2232 vsi->seid, 2233 NULL); 2234 if (aq_ret) { 2235 dev_info(&pf->pdev->dev, 2236 "Set default VSI failed, err %s, aq_err %s\n", 2237 i40e_stat_str(hw, aq_ret), 2238 i40e_aq_str(hw, hw->aq.asq_last_status)); 2239 } 2240 } else { 2241 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2242 hw, 2243 vsi->seid, 2244 promisc, NULL, 2245 true); 2246 if (aq_ret) { 2247 dev_info(&pf->pdev->dev, 2248 "set unicast promisc failed, err %s, aq_err %s\n", 2249 i40e_stat_str(hw, aq_ret), 2250 i40e_aq_str(hw, hw->aq.asq_last_status)); 2251 } 2252 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2253 hw, 2254 vsi->seid, 2255 promisc, NULL); 2256 if (aq_ret) { 2257 dev_info(&pf->pdev->dev, 2258 "set multicast promisc failed, err %s, aq_err %s\n", 2259 i40e_stat_str(hw, aq_ret), 2260 i40e_aq_str(hw, hw->aq.asq_last_status)); 2261 } 2262 } 2263 2264 if (!aq_ret) 2265 pf->cur_promisc = promisc; 2266 2267 return aq_ret; 2268 } 2269 2270 /** 2271 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2272 * @vsi: ptr to the VSI 2273 * 2274 * Push any outstanding VSI filter changes through the AdminQ. 2275 * 2276 * Returns 0 or error value 2277 **/ 2278 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2279 { 2280 struct hlist_head tmp_add_list, tmp_del_list; 2281 struct i40e_mac_filter *f; 2282 struct i40e_new_mac_filter *new, *add_head = NULL; 2283 struct i40e_hw *hw = &vsi->back->hw; 2284 bool old_overflow, new_overflow; 2285 unsigned int failed_filters = 0; 2286 unsigned int vlan_filters = 0; 2287 char vsi_name[16] = "PF"; 2288 int filter_list_len = 0; 2289 i40e_status aq_ret = 0; 2290 u32 changed_flags = 0; 2291 struct hlist_node *h; 2292 struct i40e_pf *pf; 2293 int num_add = 0; 2294 int num_del = 0; 2295 int retval = 0; 2296 u16 cmd_flags; 2297 int list_size; 2298 int bkt; 2299 2300 /* empty array typed pointers, kcalloc later */ 2301 struct i40e_aqc_add_macvlan_element_data *add_list; 2302 struct i40e_aqc_remove_macvlan_element_data *del_list; 2303 2304 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2305 usleep_range(1000, 2000); 2306 pf = vsi->back; 2307 2308 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2309 2310 if (vsi->netdev) { 2311 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2312 vsi->current_netdev_flags = vsi->netdev->flags; 2313 } 2314 2315 INIT_HLIST_HEAD(&tmp_add_list); 2316 INIT_HLIST_HEAD(&tmp_del_list); 2317 2318 if (vsi->type == I40E_VSI_SRIOV) 2319 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2320 else if (vsi->type != I40E_VSI_MAIN) 2321 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2322 2323 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2324 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2325 2326 spin_lock_bh(&vsi->mac_filter_hash_lock); 2327 /* Create a list of filters to delete. */ 2328 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2329 if (f->state == I40E_FILTER_REMOVE) { 2330 /* Move the element into temporary del_list */ 2331 hash_del(&f->hlist); 2332 hlist_add_head(&f->hlist, &tmp_del_list); 2333 2334 /* Avoid counting removed filters */ 2335 continue; 2336 } 2337 if (f->state == I40E_FILTER_NEW) { 2338 /* Create a temporary i40e_new_mac_filter */ 2339 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2340 if (!new) 2341 goto err_no_memory_locked; 2342 2343 /* Store pointer to the real filter */ 2344 new->f = f; 2345 new->state = f->state; 2346 2347 /* Add it to the hash list */ 2348 hlist_add_head(&new->hlist, &tmp_add_list); 2349 } 2350 2351 /* Count the number of active (current and new) VLAN 2352 * filters we have now. Does not count filters which 2353 * are marked for deletion. 2354 */ 2355 if (f->vlan > 0) 2356 vlan_filters++; 2357 } 2358 2359 retval = i40e_correct_mac_vlan_filters(vsi, 2360 &tmp_add_list, 2361 &tmp_del_list, 2362 vlan_filters); 2363 if (retval) 2364 goto err_no_memory_locked; 2365 2366 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2367 } 2368 2369 /* Now process 'del_list' outside the lock */ 2370 if (!hlist_empty(&tmp_del_list)) { 2371 filter_list_len = hw->aq.asq_buf_size / 2372 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2373 list_size = filter_list_len * 2374 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2375 del_list = kzalloc(list_size, GFP_ATOMIC); 2376 if (!del_list) 2377 goto err_no_memory; 2378 2379 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2380 cmd_flags = 0; 2381 2382 /* handle broadcast filters by updating the broadcast 2383 * promiscuous flag and release filter list. 2384 */ 2385 if (is_broadcast_ether_addr(f->macaddr)) { 2386 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2387 2388 hlist_del(&f->hlist); 2389 kfree(f); 2390 continue; 2391 } 2392 2393 /* add to delete list */ 2394 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2395 if (f->vlan == I40E_VLAN_ANY) { 2396 del_list[num_del].vlan_tag = 0; 2397 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2398 } else { 2399 del_list[num_del].vlan_tag = 2400 cpu_to_le16((u16)(f->vlan)); 2401 } 2402 2403 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2404 del_list[num_del].flags = cmd_flags; 2405 num_del++; 2406 2407 /* flush a full buffer */ 2408 if (num_del == filter_list_len) { 2409 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2410 num_del, &retval); 2411 memset(del_list, 0, list_size); 2412 num_del = 0; 2413 } 2414 /* Release memory for MAC filter entries which were 2415 * synced up with HW. 2416 */ 2417 hlist_del(&f->hlist); 2418 kfree(f); 2419 } 2420 2421 if (num_del) { 2422 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2423 num_del, &retval); 2424 } 2425 2426 kfree(del_list); 2427 del_list = NULL; 2428 } 2429 2430 if (!hlist_empty(&tmp_add_list)) { 2431 /* Do all the adds now. */ 2432 filter_list_len = hw->aq.asq_buf_size / 2433 sizeof(struct i40e_aqc_add_macvlan_element_data); 2434 list_size = filter_list_len * 2435 sizeof(struct i40e_aqc_add_macvlan_element_data); 2436 add_list = kzalloc(list_size, GFP_ATOMIC); 2437 if (!add_list) 2438 goto err_no_memory; 2439 2440 num_add = 0; 2441 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2442 /* handle broadcast filters by updating the broadcast 2443 * promiscuous flag instead of adding a MAC filter. 2444 */ 2445 if (is_broadcast_ether_addr(new->f->macaddr)) { 2446 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2447 new->f)) 2448 new->state = I40E_FILTER_FAILED; 2449 else 2450 new->state = I40E_FILTER_ACTIVE; 2451 continue; 2452 } 2453 2454 /* add to add array */ 2455 if (num_add == 0) 2456 add_head = new; 2457 cmd_flags = 0; 2458 ether_addr_copy(add_list[num_add].mac_addr, 2459 new->f->macaddr); 2460 if (new->f->vlan == I40E_VLAN_ANY) { 2461 add_list[num_add].vlan_tag = 0; 2462 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2463 } else { 2464 add_list[num_add].vlan_tag = 2465 cpu_to_le16((u16)(new->f->vlan)); 2466 } 2467 add_list[num_add].queue_number = 0; 2468 /* set invalid match method for later detection */ 2469 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2470 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2471 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2472 num_add++; 2473 2474 /* flush a full buffer */ 2475 if (num_add == filter_list_len) { 2476 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2477 add_head, num_add); 2478 memset(add_list, 0, list_size); 2479 num_add = 0; 2480 } 2481 } 2482 if (num_add) { 2483 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2484 num_add); 2485 } 2486 /* Now move all of the filters from the temp add list back to 2487 * the VSI's list. 2488 */ 2489 spin_lock_bh(&vsi->mac_filter_hash_lock); 2490 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2491 /* Only update the state if we're still NEW */ 2492 if (new->f->state == I40E_FILTER_NEW) 2493 new->f->state = new->state; 2494 hlist_del(&new->hlist); 2495 kfree(new); 2496 } 2497 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2498 kfree(add_list); 2499 add_list = NULL; 2500 } 2501 2502 /* Determine the number of active and failed filters. */ 2503 spin_lock_bh(&vsi->mac_filter_hash_lock); 2504 vsi->active_filters = 0; 2505 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2506 if (f->state == I40E_FILTER_ACTIVE) 2507 vsi->active_filters++; 2508 else if (f->state == I40E_FILTER_FAILED) 2509 failed_filters++; 2510 } 2511 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2512 2513 /* Check if we are able to exit overflow promiscuous mode. We can 2514 * safely exit if we didn't just enter, we no longer have any failed 2515 * filters, and we have reduced filters below the threshold value. 2516 */ 2517 if (old_overflow && !failed_filters && 2518 vsi->active_filters < vsi->promisc_threshold) { 2519 dev_info(&pf->pdev->dev, 2520 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2521 vsi_name); 2522 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2523 vsi->promisc_threshold = 0; 2524 } 2525 2526 /* if the VF is not trusted do not do promisc */ 2527 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2528 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2529 goto out; 2530 } 2531 2532 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2533 2534 /* If we are entering overflow promiscuous, we need to calculate a new 2535 * threshold for when we are safe to exit 2536 */ 2537 if (!old_overflow && new_overflow) 2538 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2539 2540 /* check for changes in promiscuous modes */ 2541 if (changed_flags & IFF_ALLMULTI) { 2542 bool cur_multipromisc; 2543 2544 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2545 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2546 vsi->seid, 2547 cur_multipromisc, 2548 NULL); 2549 if (aq_ret) { 2550 retval = i40e_aq_rc_to_posix(aq_ret, 2551 hw->aq.asq_last_status); 2552 dev_info(&pf->pdev->dev, 2553 "set multi promisc failed on %s, err %s aq_err %s\n", 2554 vsi_name, 2555 i40e_stat_str(hw, aq_ret), 2556 i40e_aq_str(hw, hw->aq.asq_last_status)); 2557 } else { 2558 dev_info(&pf->pdev->dev, "%s is %s allmulti mode.\n", 2559 vsi->netdev->name, 2560 cur_multipromisc ? "entering" : "leaving"); 2561 } 2562 } 2563 2564 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2565 bool cur_promisc; 2566 2567 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2568 new_overflow); 2569 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2570 if (aq_ret) { 2571 retval = i40e_aq_rc_to_posix(aq_ret, 2572 hw->aq.asq_last_status); 2573 dev_info(&pf->pdev->dev, 2574 "Setting promiscuous %s failed on %s, err %s aq_err %s\n", 2575 cur_promisc ? "on" : "off", 2576 vsi_name, 2577 i40e_stat_str(hw, aq_ret), 2578 i40e_aq_str(hw, hw->aq.asq_last_status)); 2579 } 2580 } 2581 out: 2582 /* if something went wrong then set the changed flag so we try again */ 2583 if (retval) 2584 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2585 2586 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2587 return retval; 2588 2589 err_no_memory: 2590 /* Restore elements on the temporary add and delete lists */ 2591 spin_lock_bh(&vsi->mac_filter_hash_lock); 2592 err_no_memory_locked: 2593 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2594 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2595 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2596 2597 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2598 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2599 return -ENOMEM; 2600 } 2601 2602 /** 2603 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2604 * @pf: board private structure 2605 **/ 2606 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2607 { 2608 int v; 2609 2610 if (!pf) 2611 return; 2612 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2613 return; 2614 if (test_and_set_bit(__I40E_VF_DISABLE, pf->state)) { 2615 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 2616 return; 2617 } 2618 2619 for (v = 0; v < pf->num_alloc_vsi; v++) { 2620 if (pf->vsi[v] && 2621 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2622 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2623 2624 if (ret) { 2625 /* come back and try again later */ 2626 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2627 pf->state); 2628 break; 2629 } 2630 } 2631 } 2632 clear_bit(__I40E_VF_DISABLE, pf->state); 2633 } 2634 2635 /** 2636 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2637 * @vsi: the vsi 2638 **/ 2639 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2640 { 2641 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2642 return I40E_RXBUFFER_2048; 2643 else 2644 return I40E_RXBUFFER_3072; 2645 } 2646 2647 /** 2648 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2649 * @netdev: network interface device structure 2650 * @new_mtu: new value for maximum frame size 2651 * 2652 * Returns 0 on success, negative on failure 2653 **/ 2654 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2655 { 2656 struct i40e_netdev_priv *np = netdev_priv(netdev); 2657 struct i40e_vsi *vsi = np->vsi; 2658 struct i40e_pf *pf = vsi->back; 2659 2660 if (i40e_enabled_xdp_vsi(vsi)) { 2661 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2662 2663 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2664 return -EINVAL; 2665 } 2666 2667 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2668 netdev->mtu, new_mtu); 2669 netdev->mtu = new_mtu; 2670 if (netif_running(netdev)) 2671 i40e_vsi_reinit_locked(vsi); 2672 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2673 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2674 return 0; 2675 } 2676 2677 /** 2678 * i40e_ioctl - Access the hwtstamp interface 2679 * @netdev: network interface device structure 2680 * @ifr: interface request data 2681 * @cmd: ioctl command 2682 **/ 2683 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2684 { 2685 struct i40e_netdev_priv *np = netdev_priv(netdev); 2686 struct i40e_pf *pf = np->vsi->back; 2687 2688 switch (cmd) { 2689 case SIOCGHWTSTAMP: 2690 return i40e_ptp_get_ts_config(pf, ifr); 2691 case SIOCSHWTSTAMP: 2692 return i40e_ptp_set_ts_config(pf, ifr); 2693 default: 2694 return -EOPNOTSUPP; 2695 } 2696 } 2697 2698 /** 2699 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2700 * @vsi: the vsi being adjusted 2701 **/ 2702 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2703 { 2704 struct i40e_vsi_context ctxt; 2705 i40e_status ret; 2706 2707 /* Don't modify stripping options if a port VLAN is active */ 2708 if (vsi->info.pvid) 2709 return; 2710 2711 if ((vsi->info.valid_sections & 2712 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2713 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2714 return; /* already enabled */ 2715 2716 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2717 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2718 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2719 2720 ctxt.seid = vsi->seid; 2721 ctxt.info = vsi->info; 2722 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2723 if (ret) { 2724 dev_info(&vsi->back->pdev->dev, 2725 "update vlan stripping failed, err %s aq_err %s\n", 2726 i40e_stat_str(&vsi->back->hw, ret), 2727 i40e_aq_str(&vsi->back->hw, 2728 vsi->back->hw.aq.asq_last_status)); 2729 } 2730 } 2731 2732 /** 2733 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2734 * @vsi: the vsi being adjusted 2735 **/ 2736 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2737 { 2738 struct i40e_vsi_context ctxt; 2739 i40e_status ret; 2740 2741 /* Don't modify stripping options if a port VLAN is active */ 2742 if (vsi->info.pvid) 2743 return; 2744 2745 if ((vsi->info.valid_sections & 2746 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2747 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2748 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2749 return; /* already disabled */ 2750 2751 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2752 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2753 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2754 2755 ctxt.seid = vsi->seid; 2756 ctxt.info = vsi->info; 2757 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2758 if (ret) { 2759 dev_info(&vsi->back->pdev->dev, 2760 "update vlan stripping failed, err %s aq_err %s\n", 2761 i40e_stat_str(&vsi->back->hw, ret), 2762 i40e_aq_str(&vsi->back->hw, 2763 vsi->back->hw.aq.asq_last_status)); 2764 } 2765 } 2766 2767 /** 2768 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2769 * @vsi: the vsi being configured 2770 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2771 * 2772 * This is a helper function for adding a new MAC/VLAN filter with the 2773 * specified VLAN for each existing MAC address already in the hash table. 2774 * This function does *not* perform any accounting to update filters based on 2775 * VLAN mode. 2776 * 2777 * NOTE: this function expects to be called while under the 2778 * mac_filter_hash_lock 2779 **/ 2780 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2781 { 2782 struct i40e_mac_filter *f, *add_f; 2783 struct hlist_node *h; 2784 int bkt; 2785 2786 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2787 if (f->state == I40E_FILTER_REMOVE) 2788 continue; 2789 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2790 if (!add_f) { 2791 dev_info(&vsi->back->pdev->dev, 2792 "Could not add vlan filter %d for %pM\n", 2793 vid, f->macaddr); 2794 return -ENOMEM; 2795 } 2796 } 2797 2798 return 0; 2799 } 2800 2801 /** 2802 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2803 * @vsi: the VSI being configured 2804 * @vid: VLAN id to be added 2805 **/ 2806 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2807 { 2808 int err; 2809 2810 if (vsi->info.pvid) 2811 return -EINVAL; 2812 2813 /* The network stack will attempt to add VID=0, with the intention to 2814 * receive priority tagged packets with a VLAN of 0. Our HW receives 2815 * these packets by default when configured to receive untagged 2816 * packets, so we don't need to add a filter for this case. 2817 * Additionally, HW interprets adding a VID=0 filter as meaning to 2818 * receive *only* tagged traffic and stops receiving untagged traffic. 2819 * Thus, we do not want to actually add a filter for VID=0 2820 */ 2821 if (!vid) 2822 return 0; 2823 2824 /* Locked once because all functions invoked below iterates list*/ 2825 spin_lock_bh(&vsi->mac_filter_hash_lock); 2826 err = i40e_add_vlan_all_mac(vsi, vid); 2827 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2828 if (err) 2829 return err; 2830 2831 /* schedule our worker thread which will take care of 2832 * applying the new filter changes 2833 */ 2834 i40e_service_event_schedule(vsi->back); 2835 return 0; 2836 } 2837 2838 /** 2839 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2840 * @vsi: the vsi being configured 2841 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2842 * 2843 * This function should be used to remove all VLAN filters which match the 2844 * given VID. It does not schedule the service event and does not take the 2845 * mac_filter_hash_lock so it may be combined with other operations under 2846 * a single invocation of the mac_filter_hash_lock. 2847 * 2848 * NOTE: this function expects to be called while under the 2849 * mac_filter_hash_lock 2850 */ 2851 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2852 { 2853 struct i40e_mac_filter *f; 2854 struct hlist_node *h; 2855 int bkt; 2856 2857 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2858 if (f->vlan == vid) 2859 __i40e_del_filter(vsi, f); 2860 } 2861 } 2862 2863 /** 2864 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2865 * @vsi: the VSI being configured 2866 * @vid: VLAN id to be removed 2867 **/ 2868 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2869 { 2870 if (!vid || vsi->info.pvid) 2871 return; 2872 2873 spin_lock_bh(&vsi->mac_filter_hash_lock); 2874 i40e_rm_vlan_all_mac(vsi, vid); 2875 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2876 2877 /* schedule our worker thread which will take care of 2878 * applying the new filter changes 2879 */ 2880 i40e_service_event_schedule(vsi->back); 2881 } 2882 2883 /** 2884 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2885 * @netdev: network interface to be adjusted 2886 * @proto: unused protocol value 2887 * @vid: vlan id to be added 2888 * 2889 * net_device_ops implementation for adding vlan ids 2890 **/ 2891 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2892 __always_unused __be16 proto, u16 vid) 2893 { 2894 struct i40e_netdev_priv *np = netdev_priv(netdev); 2895 struct i40e_vsi *vsi = np->vsi; 2896 int ret = 0; 2897 2898 if (vid >= VLAN_N_VID) 2899 return -EINVAL; 2900 2901 ret = i40e_vsi_add_vlan(vsi, vid); 2902 if (!ret) 2903 set_bit(vid, vsi->active_vlans); 2904 2905 return ret; 2906 } 2907 2908 /** 2909 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 2910 * @netdev: network interface to be adjusted 2911 * @proto: unused protocol value 2912 * @vid: vlan id to be added 2913 **/ 2914 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 2915 __always_unused __be16 proto, u16 vid) 2916 { 2917 struct i40e_netdev_priv *np = netdev_priv(netdev); 2918 struct i40e_vsi *vsi = np->vsi; 2919 2920 if (vid >= VLAN_N_VID) 2921 return; 2922 set_bit(vid, vsi->active_vlans); 2923 } 2924 2925 /** 2926 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2927 * @netdev: network interface to be adjusted 2928 * @proto: unused protocol value 2929 * @vid: vlan id to be removed 2930 * 2931 * net_device_ops implementation for removing vlan ids 2932 **/ 2933 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2934 __always_unused __be16 proto, u16 vid) 2935 { 2936 struct i40e_netdev_priv *np = netdev_priv(netdev); 2937 struct i40e_vsi *vsi = np->vsi; 2938 2939 /* return code is ignored as there is nothing a user 2940 * can do about failure to remove and a log message was 2941 * already printed from the other function 2942 */ 2943 i40e_vsi_kill_vlan(vsi, vid); 2944 2945 clear_bit(vid, vsi->active_vlans); 2946 2947 return 0; 2948 } 2949 2950 /** 2951 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2952 * @vsi: the vsi being brought back up 2953 **/ 2954 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2955 { 2956 u16 vid; 2957 2958 if (!vsi->netdev) 2959 return; 2960 2961 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2962 i40e_vlan_stripping_enable(vsi); 2963 else 2964 i40e_vlan_stripping_disable(vsi); 2965 2966 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2967 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 2968 vid); 2969 } 2970 2971 /** 2972 * i40e_vsi_add_pvid - Add pvid for the VSI 2973 * @vsi: the vsi being adjusted 2974 * @vid: the vlan id to set as a PVID 2975 **/ 2976 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2977 { 2978 struct i40e_vsi_context ctxt; 2979 i40e_status ret; 2980 2981 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2982 vsi->info.pvid = cpu_to_le16(vid); 2983 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2984 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2985 I40E_AQ_VSI_PVLAN_EMOD_STR; 2986 2987 ctxt.seid = vsi->seid; 2988 ctxt.info = vsi->info; 2989 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2990 if (ret) { 2991 dev_info(&vsi->back->pdev->dev, 2992 "add pvid failed, err %s aq_err %s\n", 2993 i40e_stat_str(&vsi->back->hw, ret), 2994 i40e_aq_str(&vsi->back->hw, 2995 vsi->back->hw.aq.asq_last_status)); 2996 return -ENOENT; 2997 } 2998 2999 return 0; 3000 } 3001 3002 /** 3003 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 3004 * @vsi: the vsi being adjusted 3005 * 3006 * Just use the vlan_rx_register() service to put it back to normal 3007 **/ 3008 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 3009 { 3010 vsi->info.pvid = 0; 3011 3012 i40e_vlan_stripping_disable(vsi); 3013 } 3014 3015 /** 3016 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 3017 * @vsi: ptr to the VSI 3018 * 3019 * If this function returns with an error, then it's possible one or 3020 * more of the rings is populated (while the rest are not). It is the 3021 * callers duty to clean those orphaned rings. 3022 * 3023 * Return 0 on success, negative on failure 3024 **/ 3025 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 3026 { 3027 int i, err = 0; 3028 3029 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3030 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 3031 3032 if (!i40e_enabled_xdp_vsi(vsi)) 3033 return err; 3034 3035 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3036 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3037 3038 return err; 3039 } 3040 3041 /** 3042 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3043 * @vsi: ptr to the VSI 3044 * 3045 * Free VSI's transmit software resources 3046 **/ 3047 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3048 { 3049 int i; 3050 3051 if (vsi->tx_rings) { 3052 for (i = 0; i < vsi->num_queue_pairs; i++) 3053 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3054 i40e_free_tx_resources(vsi->tx_rings[i]); 3055 } 3056 3057 if (vsi->xdp_rings) { 3058 for (i = 0; i < vsi->num_queue_pairs; i++) 3059 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3060 i40e_free_tx_resources(vsi->xdp_rings[i]); 3061 } 3062 } 3063 3064 /** 3065 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3066 * @vsi: ptr to the VSI 3067 * 3068 * If this function returns with an error, then it's possible one or 3069 * more of the rings is populated (while the rest are not). It is the 3070 * callers duty to clean those orphaned rings. 3071 * 3072 * Return 0 on success, negative on failure 3073 **/ 3074 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3075 { 3076 int i, err = 0; 3077 3078 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3079 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3080 return err; 3081 } 3082 3083 /** 3084 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3085 * @vsi: ptr to the VSI 3086 * 3087 * Free all receive software resources 3088 **/ 3089 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3090 { 3091 int i; 3092 3093 if (!vsi->rx_rings) 3094 return; 3095 3096 for (i = 0; i < vsi->num_queue_pairs; i++) 3097 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3098 i40e_free_rx_resources(vsi->rx_rings[i]); 3099 } 3100 3101 /** 3102 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3103 * @ring: The Tx ring to configure 3104 * 3105 * This enables/disables XPS for a given Tx descriptor ring 3106 * based on the TCs enabled for the VSI that ring belongs to. 3107 **/ 3108 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3109 { 3110 int cpu; 3111 3112 if (!ring->q_vector || !ring->netdev || ring->ch) 3113 return; 3114 3115 /* We only initialize XPS once, so as not to overwrite user settings */ 3116 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3117 return; 3118 3119 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3120 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3121 ring->queue_index); 3122 } 3123 3124 /** 3125 * i40e_xsk_umem - Retrieve the AF_XDP ZC if XDP and ZC is enabled 3126 * @ring: The Tx or Rx ring 3127 * 3128 * Returns the UMEM or NULL. 3129 **/ 3130 static struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring) 3131 { 3132 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi); 3133 int qid = ring->queue_index; 3134 3135 if (ring_is_xdp(ring)) 3136 qid -= ring->vsi->alloc_queue_pairs; 3137 3138 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps)) 3139 return NULL; 3140 3141 return xdp_get_umem_from_qid(ring->vsi->netdev, qid); 3142 } 3143 3144 /** 3145 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3146 * @ring: The Tx ring to configure 3147 * 3148 * Configure the Tx descriptor ring in the HMC context. 3149 **/ 3150 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3151 { 3152 struct i40e_vsi *vsi = ring->vsi; 3153 u16 pf_q = vsi->base_queue + ring->queue_index; 3154 struct i40e_hw *hw = &vsi->back->hw; 3155 struct i40e_hmc_obj_txq tx_ctx; 3156 i40e_status err = 0; 3157 u32 qtx_ctl = 0; 3158 3159 if (ring_is_xdp(ring)) 3160 ring->xsk_umem = i40e_xsk_umem(ring); 3161 3162 /* some ATR related tx ring init */ 3163 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3164 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3165 ring->atr_count = 0; 3166 } else { 3167 ring->atr_sample_rate = 0; 3168 } 3169 3170 /* configure XPS */ 3171 i40e_config_xps_tx_ring(ring); 3172 3173 /* clear the context structure first */ 3174 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3175 3176 tx_ctx.new_context = 1; 3177 tx_ctx.base = (ring->dma / 128); 3178 tx_ctx.qlen = ring->count; 3179 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3180 I40E_FLAG_FD_ATR_ENABLED)); 3181 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3182 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3183 if (vsi->type != I40E_VSI_FDIR) 3184 tx_ctx.head_wb_ena = 1; 3185 tx_ctx.head_wb_addr = ring->dma + 3186 (ring->count * sizeof(struct i40e_tx_desc)); 3187 3188 /* As part of VSI creation/update, FW allocates certain 3189 * Tx arbitration queue sets for each TC enabled for 3190 * the VSI. The FW returns the handles to these queue 3191 * sets as part of the response buffer to Add VSI, 3192 * Update VSI, etc. AQ commands. It is expected that 3193 * these queue set handles be associated with the Tx 3194 * queues by the driver as part of the TX queue context 3195 * initialization. This has to be done regardless of 3196 * DCB as by default everything is mapped to TC0. 3197 */ 3198 3199 if (ring->ch) 3200 tx_ctx.rdylist = 3201 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3202 3203 else 3204 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3205 3206 tx_ctx.rdylist_act = 0; 3207 3208 /* clear the context in the HMC */ 3209 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3210 if (err) { 3211 dev_info(&vsi->back->pdev->dev, 3212 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3213 ring->queue_index, pf_q, err); 3214 return -ENOMEM; 3215 } 3216 3217 /* set the context in the HMC */ 3218 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3219 if (err) { 3220 dev_info(&vsi->back->pdev->dev, 3221 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3222 ring->queue_index, pf_q, err); 3223 return -ENOMEM; 3224 } 3225 3226 /* Now associate this queue with this PCI function */ 3227 if (ring->ch) { 3228 if (ring->ch->type == I40E_VSI_VMDQ2) 3229 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3230 else 3231 return -EINVAL; 3232 3233 qtx_ctl |= (ring->ch->vsi_number << 3234 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3235 I40E_QTX_CTL_VFVM_INDX_MASK; 3236 } else { 3237 if (vsi->type == I40E_VSI_VMDQ2) { 3238 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3239 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3240 I40E_QTX_CTL_VFVM_INDX_MASK; 3241 } else { 3242 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3243 } 3244 } 3245 3246 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3247 I40E_QTX_CTL_PF_INDX_MASK); 3248 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3249 i40e_flush(hw); 3250 3251 /* cache tail off for easier writes later */ 3252 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3253 3254 return 0; 3255 } 3256 3257 /** 3258 * i40e_configure_rx_ring - Configure a receive ring context 3259 * @ring: The Rx ring to configure 3260 * 3261 * Configure the Rx descriptor ring in the HMC context. 3262 **/ 3263 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3264 { 3265 struct i40e_vsi *vsi = ring->vsi; 3266 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3267 u16 pf_q = vsi->base_queue + ring->queue_index; 3268 struct i40e_hw *hw = &vsi->back->hw; 3269 struct i40e_hmc_obj_rxq rx_ctx; 3270 i40e_status err = 0; 3271 bool ok; 3272 int ret; 3273 3274 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3275 3276 /* clear the context structure first */ 3277 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3278 3279 if (ring->vsi->type == I40E_VSI_MAIN) 3280 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 3281 3282 kfree(ring->rx_bi); 3283 ring->xsk_umem = i40e_xsk_umem(ring); 3284 if (ring->xsk_umem) { 3285 ret = i40e_alloc_rx_bi_zc(ring); 3286 if (ret) 3287 return ret; 3288 ring->rx_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem); 3289 /* For AF_XDP ZC, we disallow packets to span on 3290 * multiple buffers, thus letting us skip that 3291 * handling in the fast-path. 3292 */ 3293 chain_len = 1; 3294 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3295 MEM_TYPE_XSK_BUFF_POOL, 3296 NULL); 3297 if (ret) 3298 return ret; 3299 dev_info(&vsi->back->pdev->dev, 3300 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 3301 ring->queue_index); 3302 3303 } else { 3304 ret = i40e_alloc_rx_bi(ring); 3305 if (ret) 3306 return ret; 3307 ring->rx_buf_len = vsi->rx_buf_len; 3308 if (ring->vsi->type == I40E_VSI_MAIN) { 3309 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3310 MEM_TYPE_PAGE_SHARED, 3311 NULL); 3312 if (ret) 3313 return ret; 3314 } 3315 } 3316 3317 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3318 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3319 3320 rx_ctx.base = (ring->dma / 128); 3321 rx_ctx.qlen = ring->count; 3322 3323 /* use 32 byte descriptors */ 3324 rx_ctx.dsize = 1; 3325 3326 /* descriptor type is always zero 3327 * rx_ctx.dtype = 0; 3328 */ 3329 rx_ctx.hsplit_0 = 0; 3330 3331 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3332 if (hw->revision_id == 0) 3333 rx_ctx.lrxqthresh = 0; 3334 else 3335 rx_ctx.lrxqthresh = 1; 3336 rx_ctx.crcstrip = 1; 3337 rx_ctx.l2tsel = 1; 3338 /* this controls whether VLAN is stripped from inner headers */ 3339 rx_ctx.showiv = 0; 3340 /* set the prefena field to 1 because the manual says to */ 3341 rx_ctx.prefena = 1; 3342 3343 /* clear the context in the HMC */ 3344 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3345 if (err) { 3346 dev_info(&vsi->back->pdev->dev, 3347 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3348 ring->queue_index, pf_q, err); 3349 return -ENOMEM; 3350 } 3351 3352 /* set the context in the HMC */ 3353 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3354 if (err) { 3355 dev_info(&vsi->back->pdev->dev, 3356 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3357 ring->queue_index, pf_q, err); 3358 return -ENOMEM; 3359 } 3360 3361 /* configure Rx buffer alignment */ 3362 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3363 clear_ring_build_skb_enabled(ring); 3364 else 3365 set_ring_build_skb_enabled(ring); 3366 3367 /* cache tail for quicker writes, and clear the reg before use */ 3368 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3369 writel(0, ring->tail); 3370 3371 if (ring->xsk_umem) { 3372 xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq); 3373 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); 3374 } else { 3375 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3376 } 3377 if (!ok) { 3378 /* Log this in case the user has forgotten to give the kernel 3379 * any buffers, even later in the application. 3380 */ 3381 dev_info(&vsi->back->pdev->dev, 3382 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n", 3383 ring->xsk_umem ? "UMEM enabled " : "", 3384 ring->queue_index, pf_q); 3385 } 3386 3387 return 0; 3388 } 3389 3390 /** 3391 * i40e_vsi_configure_tx - Configure the VSI for Tx 3392 * @vsi: VSI structure describing this set of rings and resources 3393 * 3394 * Configure the Tx VSI for operation. 3395 **/ 3396 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3397 { 3398 int err = 0; 3399 u16 i; 3400 3401 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3402 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3403 3404 if (err || !i40e_enabled_xdp_vsi(vsi)) 3405 return err; 3406 3407 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3408 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3409 3410 return err; 3411 } 3412 3413 /** 3414 * i40e_vsi_configure_rx - Configure the VSI for Rx 3415 * @vsi: the VSI being configured 3416 * 3417 * Configure the Rx VSI for operation. 3418 **/ 3419 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3420 { 3421 int err = 0; 3422 u16 i; 3423 3424 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3425 vsi->max_frame = I40E_MAX_RXBUFFER; 3426 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3427 #if (PAGE_SIZE < 8192) 3428 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3429 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3430 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3431 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3432 #endif 3433 } else { 3434 vsi->max_frame = I40E_MAX_RXBUFFER; 3435 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3436 I40E_RXBUFFER_2048; 3437 } 3438 3439 /* set up individual rings */ 3440 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3441 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3442 3443 return err; 3444 } 3445 3446 /** 3447 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3448 * @vsi: ptr to the VSI 3449 **/ 3450 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3451 { 3452 struct i40e_ring *tx_ring, *rx_ring; 3453 u16 qoffset, qcount; 3454 int i, n; 3455 3456 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3457 /* Reset the TC information */ 3458 for (i = 0; i < vsi->num_queue_pairs; i++) { 3459 rx_ring = vsi->rx_rings[i]; 3460 tx_ring = vsi->tx_rings[i]; 3461 rx_ring->dcb_tc = 0; 3462 tx_ring->dcb_tc = 0; 3463 } 3464 return; 3465 } 3466 3467 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3468 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3469 continue; 3470 3471 qoffset = vsi->tc_config.tc_info[n].qoffset; 3472 qcount = vsi->tc_config.tc_info[n].qcount; 3473 for (i = qoffset; i < (qoffset + qcount); i++) { 3474 rx_ring = vsi->rx_rings[i]; 3475 tx_ring = vsi->tx_rings[i]; 3476 rx_ring->dcb_tc = n; 3477 tx_ring->dcb_tc = n; 3478 } 3479 } 3480 } 3481 3482 /** 3483 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3484 * @vsi: ptr to the VSI 3485 **/ 3486 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3487 { 3488 if (vsi->netdev) 3489 i40e_set_rx_mode(vsi->netdev); 3490 } 3491 3492 /** 3493 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3494 * @vsi: Pointer to the targeted VSI 3495 * 3496 * This function replays the hlist on the hw where all the SB Flow Director 3497 * filters were saved. 3498 **/ 3499 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3500 { 3501 struct i40e_fdir_filter *filter; 3502 struct i40e_pf *pf = vsi->back; 3503 struct hlist_node *node; 3504 3505 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3506 return; 3507 3508 /* Reset FDir counters as we're replaying all existing filters */ 3509 pf->fd_tcp4_filter_cnt = 0; 3510 pf->fd_udp4_filter_cnt = 0; 3511 pf->fd_sctp4_filter_cnt = 0; 3512 pf->fd_ip4_filter_cnt = 0; 3513 3514 hlist_for_each_entry_safe(filter, node, 3515 &pf->fdir_filter_list, fdir_node) { 3516 i40e_add_del_fdir(vsi, filter, true); 3517 } 3518 } 3519 3520 /** 3521 * i40e_vsi_configure - Set up the VSI for action 3522 * @vsi: the VSI being configured 3523 **/ 3524 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3525 { 3526 int err; 3527 3528 i40e_set_vsi_rx_mode(vsi); 3529 i40e_restore_vlan(vsi); 3530 i40e_vsi_config_dcb_rings(vsi); 3531 err = i40e_vsi_configure_tx(vsi); 3532 if (!err) 3533 err = i40e_vsi_configure_rx(vsi); 3534 3535 return err; 3536 } 3537 3538 /** 3539 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3540 * @vsi: the VSI being configured 3541 **/ 3542 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3543 { 3544 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3545 struct i40e_pf *pf = vsi->back; 3546 struct i40e_hw *hw = &pf->hw; 3547 u16 vector; 3548 int i, q; 3549 u32 qp; 3550 3551 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3552 * and PFINT_LNKLSTn registers, e.g.: 3553 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3554 */ 3555 qp = vsi->base_queue; 3556 vector = vsi->base_vector; 3557 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3558 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3559 3560 q_vector->rx.next_update = jiffies + 1; 3561 q_vector->rx.target_itr = 3562 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3563 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3564 q_vector->rx.target_itr >> 1); 3565 q_vector->rx.current_itr = q_vector->rx.target_itr; 3566 3567 q_vector->tx.next_update = jiffies + 1; 3568 q_vector->tx.target_itr = 3569 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3570 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3571 q_vector->tx.target_itr >> 1); 3572 q_vector->tx.current_itr = q_vector->tx.target_itr; 3573 3574 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3575 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3576 3577 /* Linked list for the queuepairs assigned to this vector */ 3578 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3579 for (q = 0; q < q_vector->num_ringpairs; q++) { 3580 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3581 u32 val; 3582 3583 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3584 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3585 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3586 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3587 (I40E_QUEUE_TYPE_TX << 3588 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3589 3590 wr32(hw, I40E_QINT_RQCTL(qp), val); 3591 3592 if (has_xdp) { 3593 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3594 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3595 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3596 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3597 (I40E_QUEUE_TYPE_TX << 3598 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3599 3600 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3601 } 3602 3603 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3604 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3605 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3606 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3607 (I40E_QUEUE_TYPE_RX << 3608 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3609 3610 /* Terminate the linked list */ 3611 if (q == (q_vector->num_ringpairs - 1)) 3612 val |= (I40E_QUEUE_END_OF_LIST << 3613 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3614 3615 wr32(hw, I40E_QINT_TQCTL(qp), val); 3616 qp++; 3617 } 3618 } 3619 3620 i40e_flush(hw); 3621 } 3622 3623 /** 3624 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3625 * @pf: pointer to private device data structure 3626 **/ 3627 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3628 { 3629 struct i40e_hw *hw = &pf->hw; 3630 u32 val; 3631 3632 /* clear things first */ 3633 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3634 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3635 3636 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3637 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3638 I40E_PFINT_ICR0_ENA_GRST_MASK | 3639 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3640 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3641 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3642 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3643 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3644 3645 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3646 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3647 3648 if (pf->flags & I40E_FLAG_PTP) 3649 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3650 3651 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3652 3653 /* SW_ITR_IDX = 0, but don't change INTENA */ 3654 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3655 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3656 3657 /* OTHER_ITR_IDX = 0 */ 3658 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3659 } 3660 3661 /** 3662 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3663 * @vsi: the VSI being configured 3664 **/ 3665 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3666 { 3667 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3668 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3669 struct i40e_pf *pf = vsi->back; 3670 struct i40e_hw *hw = &pf->hw; 3671 u32 val; 3672 3673 /* set the ITR configuration */ 3674 q_vector->rx.next_update = jiffies + 1; 3675 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 3676 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1); 3677 q_vector->rx.current_itr = q_vector->rx.target_itr; 3678 q_vector->tx.next_update = jiffies + 1; 3679 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 3680 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1); 3681 q_vector->tx.current_itr = q_vector->tx.target_itr; 3682 3683 i40e_enable_misc_int_causes(pf); 3684 3685 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3686 wr32(hw, I40E_PFINT_LNKLST0, 0); 3687 3688 /* Associate the queue pair to the vector and enable the queue int */ 3689 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3690 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3691 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3692 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3693 3694 wr32(hw, I40E_QINT_RQCTL(0), val); 3695 3696 if (i40e_enabled_xdp_vsi(vsi)) { 3697 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3698 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3699 (I40E_QUEUE_TYPE_TX 3700 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3701 3702 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3703 } 3704 3705 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3706 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3707 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3708 3709 wr32(hw, I40E_QINT_TQCTL(0), val); 3710 i40e_flush(hw); 3711 } 3712 3713 /** 3714 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3715 * @pf: board private structure 3716 **/ 3717 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3718 { 3719 struct i40e_hw *hw = &pf->hw; 3720 3721 wr32(hw, I40E_PFINT_DYN_CTL0, 3722 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3723 i40e_flush(hw); 3724 } 3725 3726 /** 3727 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3728 * @pf: board private structure 3729 **/ 3730 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 3731 { 3732 struct i40e_hw *hw = &pf->hw; 3733 u32 val; 3734 3735 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3736 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3737 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3738 3739 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3740 i40e_flush(hw); 3741 } 3742 3743 /** 3744 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3745 * @irq: interrupt number 3746 * @data: pointer to a q_vector 3747 **/ 3748 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3749 { 3750 struct i40e_q_vector *q_vector = data; 3751 3752 if (!q_vector->tx.ring && !q_vector->rx.ring) 3753 return IRQ_HANDLED; 3754 3755 napi_schedule_irqoff(&q_vector->napi); 3756 3757 return IRQ_HANDLED; 3758 } 3759 3760 /** 3761 * i40e_irq_affinity_notify - Callback for affinity changes 3762 * @notify: context as to what irq was changed 3763 * @mask: the new affinity mask 3764 * 3765 * This is a callback function used by the irq_set_affinity_notifier function 3766 * so that we may register to receive changes to the irq affinity masks. 3767 **/ 3768 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3769 const cpumask_t *mask) 3770 { 3771 struct i40e_q_vector *q_vector = 3772 container_of(notify, struct i40e_q_vector, affinity_notify); 3773 3774 cpumask_copy(&q_vector->affinity_mask, mask); 3775 } 3776 3777 /** 3778 * i40e_irq_affinity_release - Callback for affinity notifier release 3779 * @ref: internal core kernel usage 3780 * 3781 * This is a callback function used by the irq_set_affinity_notifier function 3782 * to inform the current notification subscriber that they will no longer 3783 * receive notifications. 3784 **/ 3785 static void i40e_irq_affinity_release(struct kref *ref) {} 3786 3787 /** 3788 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3789 * @vsi: the VSI being configured 3790 * @basename: name for the vector 3791 * 3792 * Allocates MSI-X vectors and requests interrupts from the kernel. 3793 **/ 3794 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3795 { 3796 int q_vectors = vsi->num_q_vectors; 3797 struct i40e_pf *pf = vsi->back; 3798 int base = vsi->base_vector; 3799 int rx_int_idx = 0; 3800 int tx_int_idx = 0; 3801 int vector, err; 3802 int irq_num; 3803 int cpu; 3804 3805 for (vector = 0; vector < q_vectors; vector++) { 3806 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3807 3808 irq_num = pf->msix_entries[base + vector].vector; 3809 3810 if (q_vector->tx.ring && q_vector->rx.ring) { 3811 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3812 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3813 tx_int_idx++; 3814 } else if (q_vector->rx.ring) { 3815 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3816 "%s-%s-%d", basename, "rx", rx_int_idx++); 3817 } else if (q_vector->tx.ring) { 3818 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3819 "%s-%s-%d", basename, "tx", tx_int_idx++); 3820 } else { 3821 /* skip this unused q_vector */ 3822 continue; 3823 } 3824 err = request_irq(irq_num, 3825 vsi->irq_handler, 3826 0, 3827 q_vector->name, 3828 q_vector); 3829 if (err) { 3830 dev_info(&pf->pdev->dev, 3831 "MSIX request_irq failed, error: %d\n", err); 3832 goto free_queue_irqs; 3833 } 3834 3835 /* register for affinity change notifications */ 3836 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3837 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3838 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3839 /* Spread affinity hints out across online CPUs. 3840 * 3841 * get_cpu_mask returns a static constant mask with 3842 * a permanent lifetime so it's ok to pass to 3843 * irq_set_affinity_hint without making a copy. 3844 */ 3845 cpu = cpumask_local_spread(q_vector->v_idx, -1); 3846 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 3847 } 3848 3849 vsi->irqs_ready = true; 3850 return 0; 3851 3852 free_queue_irqs: 3853 while (vector) { 3854 vector--; 3855 irq_num = pf->msix_entries[base + vector].vector; 3856 irq_set_affinity_notifier(irq_num, NULL); 3857 irq_set_affinity_hint(irq_num, NULL); 3858 free_irq(irq_num, &vsi->q_vectors[vector]); 3859 } 3860 return err; 3861 } 3862 3863 /** 3864 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3865 * @vsi: the VSI being un-configured 3866 **/ 3867 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3868 { 3869 struct i40e_pf *pf = vsi->back; 3870 struct i40e_hw *hw = &pf->hw; 3871 int base = vsi->base_vector; 3872 int i; 3873 3874 /* disable interrupt causation from each queue */ 3875 for (i = 0; i < vsi->num_queue_pairs; i++) { 3876 u32 val; 3877 3878 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3879 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3880 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3881 3882 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3883 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3884 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3885 3886 if (!i40e_enabled_xdp_vsi(vsi)) 3887 continue; 3888 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3889 } 3890 3891 /* disable each interrupt */ 3892 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3893 for (i = vsi->base_vector; 3894 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3895 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3896 3897 i40e_flush(hw); 3898 for (i = 0; i < vsi->num_q_vectors; i++) 3899 synchronize_irq(pf->msix_entries[i + base].vector); 3900 } else { 3901 /* Legacy and MSI mode - this stops all interrupt handling */ 3902 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3903 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3904 i40e_flush(hw); 3905 synchronize_irq(pf->pdev->irq); 3906 } 3907 } 3908 3909 /** 3910 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3911 * @vsi: the VSI being configured 3912 **/ 3913 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3914 { 3915 struct i40e_pf *pf = vsi->back; 3916 int i; 3917 3918 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3919 for (i = 0; i < vsi->num_q_vectors; i++) 3920 i40e_irq_dynamic_enable(vsi, i); 3921 } else { 3922 i40e_irq_dynamic_enable_icr0(pf); 3923 } 3924 3925 i40e_flush(&pf->hw); 3926 return 0; 3927 } 3928 3929 /** 3930 * i40e_free_misc_vector - Free the vector that handles non-queue events 3931 * @pf: board private structure 3932 **/ 3933 static void i40e_free_misc_vector(struct i40e_pf *pf) 3934 { 3935 /* Disable ICR 0 */ 3936 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3937 i40e_flush(&pf->hw); 3938 3939 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3940 synchronize_irq(pf->msix_entries[0].vector); 3941 free_irq(pf->msix_entries[0].vector, pf); 3942 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3943 } 3944 } 3945 3946 /** 3947 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3948 * @irq: interrupt number 3949 * @data: pointer to a q_vector 3950 * 3951 * This is the handler used for all MSI/Legacy interrupts, and deals 3952 * with both queue and non-queue interrupts. This is also used in 3953 * MSIX mode to handle the non-queue interrupts. 3954 **/ 3955 static irqreturn_t i40e_intr(int irq, void *data) 3956 { 3957 struct i40e_pf *pf = (struct i40e_pf *)data; 3958 struct i40e_hw *hw = &pf->hw; 3959 irqreturn_t ret = IRQ_NONE; 3960 u32 icr0, icr0_remaining; 3961 u32 val, ena_mask; 3962 3963 icr0 = rd32(hw, I40E_PFINT_ICR0); 3964 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3965 3966 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3967 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3968 goto enable_intr; 3969 3970 /* if interrupt but no bits showing, must be SWINT */ 3971 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3972 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3973 pf->sw_int_count++; 3974 3975 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3976 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3977 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3978 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3979 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3980 } 3981 3982 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3983 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3984 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3985 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3986 3987 /* We do not have a way to disarm Queue causes while leaving 3988 * interrupt enabled for all other causes, ideally 3989 * interrupt should be disabled while we are in NAPI but 3990 * this is not a performance path and napi_schedule() 3991 * can deal with rescheduling. 3992 */ 3993 if (!test_bit(__I40E_DOWN, pf->state)) 3994 napi_schedule_irqoff(&q_vector->napi); 3995 } 3996 3997 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3998 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3999 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 4000 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 4001 } 4002 4003 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 4004 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 4005 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 4006 } 4007 4008 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 4009 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 4010 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 4011 } 4012 4013 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 4014 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4015 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 4016 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 4017 val = rd32(hw, I40E_GLGEN_RSTAT); 4018 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 4019 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 4020 if (val == I40E_RESET_CORER) { 4021 pf->corer_count++; 4022 } else if (val == I40E_RESET_GLOBR) { 4023 pf->globr_count++; 4024 } else if (val == I40E_RESET_EMPR) { 4025 pf->empr_count++; 4026 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 4027 } 4028 } 4029 4030 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 4031 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 4032 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 4033 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 4034 rd32(hw, I40E_PFHMC_ERRORINFO), 4035 rd32(hw, I40E_PFHMC_ERRORDATA)); 4036 } 4037 4038 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 4039 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 4040 4041 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 4042 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4043 i40e_ptp_tx_hwtstamp(pf); 4044 } 4045 } 4046 4047 /* If a critical error is pending we have no choice but to reset the 4048 * device. 4049 * Report and mask out any remaining unexpected interrupts. 4050 */ 4051 icr0_remaining = icr0 & ena_mask; 4052 if (icr0_remaining) { 4053 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 4054 icr0_remaining); 4055 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 4056 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 4057 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 4058 dev_info(&pf->pdev->dev, "device will be reset\n"); 4059 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 4060 i40e_service_event_schedule(pf); 4061 } 4062 ena_mask &= ~icr0_remaining; 4063 } 4064 ret = IRQ_HANDLED; 4065 4066 enable_intr: 4067 /* re-enable interrupt causes */ 4068 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 4069 if (!test_bit(__I40E_DOWN, pf->state) || 4070 test_bit(__I40E_RECOVERY_MODE, pf->state)) { 4071 i40e_service_event_schedule(pf); 4072 i40e_irq_dynamic_enable_icr0(pf); 4073 } 4074 4075 return ret; 4076 } 4077 4078 /** 4079 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 4080 * @tx_ring: tx ring to clean 4081 * @budget: how many cleans we're allowed 4082 * 4083 * Returns true if there's any budget left (e.g. the clean is finished) 4084 **/ 4085 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 4086 { 4087 struct i40e_vsi *vsi = tx_ring->vsi; 4088 u16 i = tx_ring->next_to_clean; 4089 struct i40e_tx_buffer *tx_buf; 4090 struct i40e_tx_desc *tx_desc; 4091 4092 tx_buf = &tx_ring->tx_bi[i]; 4093 tx_desc = I40E_TX_DESC(tx_ring, i); 4094 i -= tx_ring->count; 4095 4096 do { 4097 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4098 4099 /* if next_to_watch is not set then there is no work pending */ 4100 if (!eop_desc) 4101 break; 4102 4103 /* prevent any other reads prior to eop_desc */ 4104 smp_rmb(); 4105 4106 /* if the descriptor isn't done, no work yet to do */ 4107 if (!(eop_desc->cmd_type_offset_bsz & 4108 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4109 break; 4110 4111 /* clear next_to_watch to prevent false hangs */ 4112 tx_buf->next_to_watch = NULL; 4113 4114 tx_desc->buffer_addr = 0; 4115 tx_desc->cmd_type_offset_bsz = 0; 4116 /* move past filter desc */ 4117 tx_buf++; 4118 tx_desc++; 4119 i++; 4120 if (unlikely(!i)) { 4121 i -= tx_ring->count; 4122 tx_buf = tx_ring->tx_bi; 4123 tx_desc = I40E_TX_DESC(tx_ring, 0); 4124 } 4125 /* unmap skb header data */ 4126 dma_unmap_single(tx_ring->dev, 4127 dma_unmap_addr(tx_buf, dma), 4128 dma_unmap_len(tx_buf, len), 4129 DMA_TO_DEVICE); 4130 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4131 kfree(tx_buf->raw_buf); 4132 4133 tx_buf->raw_buf = NULL; 4134 tx_buf->tx_flags = 0; 4135 tx_buf->next_to_watch = NULL; 4136 dma_unmap_len_set(tx_buf, len, 0); 4137 tx_desc->buffer_addr = 0; 4138 tx_desc->cmd_type_offset_bsz = 0; 4139 4140 /* move us past the eop_desc for start of next FD desc */ 4141 tx_buf++; 4142 tx_desc++; 4143 i++; 4144 if (unlikely(!i)) { 4145 i -= tx_ring->count; 4146 tx_buf = tx_ring->tx_bi; 4147 tx_desc = I40E_TX_DESC(tx_ring, 0); 4148 } 4149 4150 /* update budget accounting */ 4151 budget--; 4152 } while (likely(budget)); 4153 4154 i += tx_ring->count; 4155 tx_ring->next_to_clean = i; 4156 4157 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4158 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4159 4160 return budget > 0; 4161 } 4162 4163 /** 4164 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4165 * @irq: interrupt number 4166 * @data: pointer to a q_vector 4167 **/ 4168 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4169 { 4170 struct i40e_q_vector *q_vector = data; 4171 struct i40e_vsi *vsi; 4172 4173 if (!q_vector->tx.ring) 4174 return IRQ_HANDLED; 4175 4176 vsi = q_vector->tx.ring->vsi; 4177 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4178 4179 return IRQ_HANDLED; 4180 } 4181 4182 /** 4183 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4184 * @vsi: the VSI being configured 4185 * @v_idx: vector index 4186 * @qp_idx: queue pair index 4187 **/ 4188 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4189 { 4190 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4191 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4192 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4193 4194 tx_ring->q_vector = q_vector; 4195 tx_ring->next = q_vector->tx.ring; 4196 q_vector->tx.ring = tx_ring; 4197 q_vector->tx.count++; 4198 4199 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4200 if (i40e_enabled_xdp_vsi(vsi)) { 4201 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4202 4203 xdp_ring->q_vector = q_vector; 4204 xdp_ring->next = q_vector->tx.ring; 4205 q_vector->tx.ring = xdp_ring; 4206 q_vector->tx.count++; 4207 } 4208 4209 rx_ring->q_vector = q_vector; 4210 rx_ring->next = q_vector->rx.ring; 4211 q_vector->rx.ring = rx_ring; 4212 q_vector->rx.count++; 4213 } 4214 4215 /** 4216 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4217 * @vsi: the VSI being configured 4218 * 4219 * This function maps descriptor rings to the queue-specific vectors 4220 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4221 * one vector per queue pair, but on a constrained vector budget, we 4222 * group the queue pairs as "efficiently" as possible. 4223 **/ 4224 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4225 { 4226 int qp_remaining = vsi->num_queue_pairs; 4227 int q_vectors = vsi->num_q_vectors; 4228 int num_ringpairs; 4229 int v_start = 0; 4230 int qp_idx = 0; 4231 4232 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4233 * group them so there are multiple queues per vector. 4234 * It is also important to go through all the vectors available to be 4235 * sure that if we don't use all the vectors, that the remaining vectors 4236 * are cleared. This is especially important when decreasing the 4237 * number of queues in use. 4238 */ 4239 for (; v_start < q_vectors; v_start++) { 4240 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4241 4242 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4243 4244 q_vector->num_ringpairs = num_ringpairs; 4245 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4246 4247 q_vector->rx.count = 0; 4248 q_vector->tx.count = 0; 4249 q_vector->rx.ring = NULL; 4250 q_vector->tx.ring = NULL; 4251 4252 while (num_ringpairs--) { 4253 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4254 qp_idx++; 4255 qp_remaining--; 4256 } 4257 } 4258 } 4259 4260 /** 4261 * i40e_vsi_request_irq - Request IRQ from the OS 4262 * @vsi: the VSI being configured 4263 * @basename: name for the vector 4264 **/ 4265 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4266 { 4267 struct i40e_pf *pf = vsi->back; 4268 int err; 4269 4270 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4271 err = i40e_vsi_request_irq_msix(vsi, basename); 4272 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4273 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4274 pf->int_name, pf); 4275 else 4276 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4277 pf->int_name, pf); 4278 4279 if (err) 4280 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4281 4282 return err; 4283 } 4284 4285 #ifdef CONFIG_NET_POLL_CONTROLLER 4286 /** 4287 * i40e_netpoll - A Polling 'interrupt' handler 4288 * @netdev: network interface device structure 4289 * 4290 * This is used by netconsole to send skbs without having to re-enable 4291 * interrupts. It's not called while the normal interrupt routine is executing. 4292 **/ 4293 static void i40e_netpoll(struct net_device *netdev) 4294 { 4295 struct i40e_netdev_priv *np = netdev_priv(netdev); 4296 struct i40e_vsi *vsi = np->vsi; 4297 struct i40e_pf *pf = vsi->back; 4298 int i; 4299 4300 /* if interface is down do nothing */ 4301 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4302 return; 4303 4304 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4305 for (i = 0; i < vsi->num_q_vectors; i++) 4306 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4307 } else { 4308 i40e_intr(pf->pdev->irq, netdev); 4309 } 4310 } 4311 #endif 4312 4313 #define I40E_QTX_ENA_WAIT_COUNT 50 4314 4315 /** 4316 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4317 * @pf: the PF being configured 4318 * @pf_q: the PF queue 4319 * @enable: enable or disable state of the queue 4320 * 4321 * This routine will wait for the given Tx queue of the PF to reach the 4322 * enabled or disabled state. 4323 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4324 * multiple retries; else will return 0 in case of success. 4325 **/ 4326 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4327 { 4328 int i; 4329 u32 tx_reg; 4330 4331 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4332 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4333 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4334 break; 4335 4336 usleep_range(10, 20); 4337 } 4338 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4339 return -ETIMEDOUT; 4340 4341 return 0; 4342 } 4343 4344 /** 4345 * i40e_control_tx_q - Start or stop a particular Tx queue 4346 * @pf: the PF structure 4347 * @pf_q: the PF queue to configure 4348 * @enable: start or stop the queue 4349 * 4350 * This function enables or disables a single queue. Note that any delay 4351 * required after the operation is expected to be handled by the caller of 4352 * this function. 4353 **/ 4354 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4355 { 4356 struct i40e_hw *hw = &pf->hw; 4357 u32 tx_reg; 4358 int i; 4359 4360 /* warn the TX unit of coming changes */ 4361 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4362 if (!enable) 4363 usleep_range(10, 20); 4364 4365 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4366 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4367 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4368 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4369 break; 4370 usleep_range(1000, 2000); 4371 } 4372 4373 /* Skip if the queue is already in the requested state */ 4374 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4375 return; 4376 4377 /* turn on/off the queue */ 4378 if (enable) { 4379 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4380 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4381 } else { 4382 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4383 } 4384 4385 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4386 } 4387 4388 /** 4389 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4390 * @seid: VSI SEID 4391 * @pf: the PF structure 4392 * @pf_q: the PF queue to configure 4393 * @is_xdp: true if the queue is used for XDP 4394 * @enable: start or stop the queue 4395 **/ 4396 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4397 bool is_xdp, bool enable) 4398 { 4399 int ret; 4400 4401 i40e_control_tx_q(pf, pf_q, enable); 4402 4403 /* wait for the change to finish */ 4404 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4405 if (ret) { 4406 dev_info(&pf->pdev->dev, 4407 "VSI seid %d %sTx ring %d %sable timeout\n", 4408 seid, (is_xdp ? "XDP " : ""), pf_q, 4409 (enable ? "en" : "dis")); 4410 } 4411 4412 return ret; 4413 } 4414 4415 /** 4416 * i40e_vsi_control_tx - Start or stop a VSI's rings 4417 * @vsi: the VSI being configured 4418 * @enable: start or stop the rings 4419 **/ 4420 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4421 { 4422 struct i40e_pf *pf = vsi->back; 4423 int i, pf_q, ret = 0; 4424 4425 pf_q = vsi->base_queue; 4426 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4427 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4428 pf_q, 4429 false /*is xdp*/, enable); 4430 if (ret) 4431 break; 4432 4433 if (!i40e_enabled_xdp_vsi(vsi)) 4434 continue; 4435 4436 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4437 pf_q + vsi->alloc_queue_pairs, 4438 true /*is xdp*/, enable); 4439 if (ret) 4440 break; 4441 } 4442 return ret; 4443 } 4444 4445 /** 4446 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4447 * @pf: the PF being configured 4448 * @pf_q: the PF queue 4449 * @enable: enable or disable state of the queue 4450 * 4451 * This routine will wait for the given Rx queue of the PF to reach the 4452 * enabled or disabled state. 4453 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4454 * multiple retries; else will return 0 in case of success. 4455 **/ 4456 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4457 { 4458 int i; 4459 u32 rx_reg; 4460 4461 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4462 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4463 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4464 break; 4465 4466 usleep_range(10, 20); 4467 } 4468 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4469 return -ETIMEDOUT; 4470 4471 return 0; 4472 } 4473 4474 /** 4475 * i40e_control_rx_q - Start or stop a particular Rx queue 4476 * @pf: the PF structure 4477 * @pf_q: the PF queue to configure 4478 * @enable: start or stop the queue 4479 * 4480 * This function enables or disables a single queue. Note that 4481 * any delay required after the operation is expected to be 4482 * handled by the caller of this function. 4483 **/ 4484 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4485 { 4486 struct i40e_hw *hw = &pf->hw; 4487 u32 rx_reg; 4488 int i; 4489 4490 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4491 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4492 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4493 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4494 break; 4495 usleep_range(1000, 2000); 4496 } 4497 4498 /* Skip if the queue is already in the requested state */ 4499 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4500 return; 4501 4502 /* turn on/off the queue */ 4503 if (enable) 4504 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4505 else 4506 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4507 4508 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4509 } 4510 4511 /** 4512 * i40e_control_wait_rx_q 4513 * @pf: the PF structure 4514 * @pf_q: queue being configured 4515 * @enable: start or stop the rings 4516 * 4517 * This function enables or disables a single queue along with waiting 4518 * for the change to finish. The caller of this function should handle 4519 * the delays needed in the case of disabling queues. 4520 **/ 4521 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4522 { 4523 int ret = 0; 4524 4525 i40e_control_rx_q(pf, pf_q, enable); 4526 4527 /* wait for the change to finish */ 4528 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4529 if (ret) 4530 return ret; 4531 4532 return ret; 4533 } 4534 4535 /** 4536 * i40e_vsi_control_rx - Start or stop a VSI's rings 4537 * @vsi: the VSI being configured 4538 * @enable: start or stop the rings 4539 **/ 4540 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4541 { 4542 struct i40e_pf *pf = vsi->back; 4543 int i, pf_q, ret = 0; 4544 4545 pf_q = vsi->base_queue; 4546 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4547 ret = i40e_control_wait_rx_q(pf, pf_q, enable); 4548 if (ret) { 4549 dev_info(&pf->pdev->dev, 4550 "VSI seid %d Rx ring %d %sable timeout\n", 4551 vsi->seid, pf_q, (enable ? "en" : "dis")); 4552 break; 4553 } 4554 } 4555 4556 /* Due to HW errata, on Rx disable only, the register can indicate done 4557 * before it really is. Needs 50ms to be sure 4558 */ 4559 if (!enable) 4560 mdelay(50); 4561 4562 return ret; 4563 } 4564 4565 /** 4566 * i40e_vsi_start_rings - Start a VSI's rings 4567 * @vsi: the VSI being configured 4568 **/ 4569 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4570 { 4571 int ret = 0; 4572 4573 /* do rx first for enable and last for disable */ 4574 ret = i40e_vsi_control_rx(vsi, true); 4575 if (ret) 4576 return ret; 4577 ret = i40e_vsi_control_tx(vsi, true); 4578 4579 return ret; 4580 } 4581 4582 /** 4583 * i40e_vsi_stop_rings - Stop a VSI's rings 4584 * @vsi: the VSI being configured 4585 **/ 4586 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4587 { 4588 /* When port TX is suspended, don't wait */ 4589 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4590 return i40e_vsi_stop_rings_no_wait(vsi); 4591 4592 /* do rx first for enable and last for disable 4593 * Ignore return value, we need to shutdown whatever we can 4594 */ 4595 i40e_vsi_control_tx(vsi, false); 4596 i40e_vsi_control_rx(vsi, false); 4597 } 4598 4599 /** 4600 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4601 * @vsi: the VSI being shutdown 4602 * 4603 * This function stops all the rings for a VSI but does not delay to verify 4604 * that rings have been disabled. It is expected that the caller is shutting 4605 * down multiple VSIs at once and will delay together for all the VSIs after 4606 * initiating the shutdown. This is particularly useful for shutting down lots 4607 * of VFs together. Otherwise, a large delay can be incurred while configuring 4608 * each VSI in serial. 4609 **/ 4610 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4611 { 4612 struct i40e_pf *pf = vsi->back; 4613 int i, pf_q; 4614 4615 pf_q = vsi->base_queue; 4616 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4617 i40e_control_tx_q(pf, pf_q, false); 4618 i40e_control_rx_q(pf, pf_q, false); 4619 } 4620 } 4621 4622 /** 4623 * i40e_vsi_free_irq - Free the irq association with the OS 4624 * @vsi: the VSI being configured 4625 **/ 4626 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4627 { 4628 struct i40e_pf *pf = vsi->back; 4629 struct i40e_hw *hw = &pf->hw; 4630 int base = vsi->base_vector; 4631 u32 val, qp; 4632 int i; 4633 4634 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4635 if (!vsi->q_vectors) 4636 return; 4637 4638 if (!vsi->irqs_ready) 4639 return; 4640 4641 vsi->irqs_ready = false; 4642 for (i = 0; i < vsi->num_q_vectors; i++) { 4643 int irq_num; 4644 u16 vector; 4645 4646 vector = i + base; 4647 irq_num = pf->msix_entries[vector].vector; 4648 4649 /* free only the irqs that were actually requested */ 4650 if (!vsi->q_vectors[i] || 4651 !vsi->q_vectors[i]->num_ringpairs) 4652 continue; 4653 4654 /* clear the affinity notifier in the IRQ descriptor */ 4655 irq_set_affinity_notifier(irq_num, NULL); 4656 /* remove our suggested affinity mask for this IRQ */ 4657 irq_set_affinity_hint(irq_num, NULL); 4658 synchronize_irq(irq_num); 4659 free_irq(irq_num, vsi->q_vectors[i]); 4660 4661 /* Tear down the interrupt queue link list 4662 * 4663 * We know that they come in pairs and always 4664 * the Rx first, then the Tx. To clear the 4665 * link list, stick the EOL value into the 4666 * next_q field of the registers. 4667 */ 4668 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4669 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4670 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4671 val |= I40E_QUEUE_END_OF_LIST 4672 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4673 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4674 4675 while (qp != I40E_QUEUE_END_OF_LIST) { 4676 u32 next; 4677 4678 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4679 4680 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4681 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4682 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4683 I40E_QINT_RQCTL_INTEVENT_MASK); 4684 4685 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4686 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4687 4688 wr32(hw, I40E_QINT_RQCTL(qp), val); 4689 4690 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4691 4692 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4693 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4694 4695 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4696 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4697 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4698 I40E_QINT_TQCTL_INTEVENT_MASK); 4699 4700 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4701 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4702 4703 wr32(hw, I40E_QINT_TQCTL(qp), val); 4704 qp = next; 4705 } 4706 } 4707 } else { 4708 free_irq(pf->pdev->irq, pf); 4709 4710 val = rd32(hw, I40E_PFINT_LNKLST0); 4711 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4712 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4713 val |= I40E_QUEUE_END_OF_LIST 4714 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4715 wr32(hw, I40E_PFINT_LNKLST0, val); 4716 4717 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4718 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4719 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4720 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4721 I40E_QINT_RQCTL_INTEVENT_MASK); 4722 4723 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4724 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4725 4726 wr32(hw, I40E_QINT_RQCTL(qp), val); 4727 4728 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4729 4730 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4731 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4732 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4733 I40E_QINT_TQCTL_INTEVENT_MASK); 4734 4735 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4736 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4737 4738 wr32(hw, I40E_QINT_TQCTL(qp), val); 4739 } 4740 } 4741 4742 /** 4743 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4744 * @vsi: the VSI being configured 4745 * @v_idx: Index of vector to be freed 4746 * 4747 * This function frees the memory allocated to the q_vector. In addition if 4748 * NAPI is enabled it will delete any references to the NAPI struct prior 4749 * to freeing the q_vector. 4750 **/ 4751 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4752 { 4753 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4754 struct i40e_ring *ring; 4755 4756 if (!q_vector) 4757 return; 4758 4759 /* disassociate q_vector from rings */ 4760 i40e_for_each_ring(ring, q_vector->tx) 4761 ring->q_vector = NULL; 4762 4763 i40e_for_each_ring(ring, q_vector->rx) 4764 ring->q_vector = NULL; 4765 4766 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4767 if (vsi->netdev) 4768 netif_napi_del(&q_vector->napi); 4769 4770 vsi->q_vectors[v_idx] = NULL; 4771 4772 kfree_rcu(q_vector, rcu); 4773 } 4774 4775 /** 4776 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4777 * @vsi: the VSI being un-configured 4778 * 4779 * This frees the memory allocated to the q_vectors and 4780 * deletes references to the NAPI struct. 4781 **/ 4782 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4783 { 4784 int v_idx; 4785 4786 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4787 i40e_free_q_vector(vsi, v_idx); 4788 } 4789 4790 /** 4791 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4792 * @pf: board private structure 4793 **/ 4794 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4795 { 4796 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4797 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4798 pci_disable_msix(pf->pdev); 4799 kfree(pf->msix_entries); 4800 pf->msix_entries = NULL; 4801 kfree(pf->irq_pile); 4802 pf->irq_pile = NULL; 4803 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4804 pci_disable_msi(pf->pdev); 4805 } 4806 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4807 } 4808 4809 /** 4810 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4811 * @pf: board private structure 4812 * 4813 * We go through and clear interrupt specific resources and reset the structure 4814 * to pre-load conditions 4815 **/ 4816 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4817 { 4818 int i; 4819 4820 i40e_free_misc_vector(pf); 4821 4822 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4823 I40E_IWARP_IRQ_PILE_ID); 4824 4825 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4826 for (i = 0; i < pf->num_alloc_vsi; i++) 4827 if (pf->vsi[i]) 4828 i40e_vsi_free_q_vectors(pf->vsi[i]); 4829 i40e_reset_interrupt_capability(pf); 4830 } 4831 4832 /** 4833 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4834 * @vsi: the VSI being configured 4835 **/ 4836 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4837 { 4838 int q_idx; 4839 4840 if (!vsi->netdev) 4841 return; 4842 4843 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4844 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4845 4846 if (q_vector->rx.ring || q_vector->tx.ring) 4847 napi_enable(&q_vector->napi); 4848 } 4849 } 4850 4851 /** 4852 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4853 * @vsi: the VSI being configured 4854 **/ 4855 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4856 { 4857 int q_idx; 4858 4859 if (!vsi->netdev) 4860 return; 4861 4862 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4863 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4864 4865 if (q_vector->rx.ring || q_vector->tx.ring) 4866 napi_disable(&q_vector->napi); 4867 } 4868 } 4869 4870 /** 4871 * i40e_vsi_close - Shut down a VSI 4872 * @vsi: the vsi to be quelled 4873 **/ 4874 static void i40e_vsi_close(struct i40e_vsi *vsi) 4875 { 4876 struct i40e_pf *pf = vsi->back; 4877 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4878 i40e_down(vsi); 4879 i40e_vsi_free_irq(vsi); 4880 i40e_vsi_free_tx_resources(vsi); 4881 i40e_vsi_free_rx_resources(vsi); 4882 vsi->current_netdev_flags = 0; 4883 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 4884 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4885 set_bit(__I40E_CLIENT_RESET, pf->state); 4886 } 4887 4888 /** 4889 * i40e_quiesce_vsi - Pause a given VSI 4890 * @vsi: the VSI being paused 4891 **/ 4892 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4893 { 4894 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4895 return; 4896 4897 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4898 if (vsi->netdev && netif_running(vsi->netdev)) 4899 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4900 else 4901 i40e_vsi_close(vsi); 4902 } 4903 4904 /** 4905 * i40e_unquiesce_vsi - Resume a given VSI 4906 * @vsi: the VSI being resumed 4907 **/ 4908 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4909 { 4910 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4911 return; 4912 4913 if (vsi->netdev && netif_running(vsi->netdev)) 4914 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4915 else 4916 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4917 } 4918 4919 /** 4920 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4921 * @pf: the PF 4922 **/ 4923 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4924 { 4925 int v; 4926 4927 for (v = 0; v < pf->num_alloc_vsi; v++) { 4928 if (pf->vsi[v]) 4929 i40e_quiesce_vsi(pf->vsi[v]); 4930 } 4931 } 4932 4933 /** 4934 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4935 * @pf: the PF 4936 **/ 4937 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4938 { 4939 int v; 4940 4941 for (v = 0; v < pf->num_alloc_vsi; v++) { 4942 if (pf->vsi[v]) 4943 i40e_unquiesce_vsi(pf->vsi[v]); 4944 } 4945 } 4946 4947 /** 4948 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4949 * @vsi: the VSI being configured 4950 * 4951 * Wait until all queues on a given VSI have been disabled. 4952 **/ 4953 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4954 { 4955 struct i40e_pf *pf = vsi->back; 4956 int i, pf_q, ret; 4957 4958 pf_q = vsi->base_queue; 4959 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4960 /* Check and wait for the Tx queue */ 4961 ret = i40e_pf_txq_wait(pf, pf_q, false); 4962 if (ret) { 4963 dev_info(&pf->pdev->dev, 4964 "VSI seid %d Tx ring %d disable timeout\n", 4965 vsi->seid, pf_q); 4966 return ret; 4967 } 4968 4969 if (!i40e_enabled_xdp_vsi(vsi)) 4970 goto wait_rx; 4971 4972 /* Check and wait for the XDP Tx queue */ 4973 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4974 false); 4975 if (ret) { 4976 dev_info(&pf->pdev->dev, 4977 "VSI seid %d XDP Tx ring %d disable timeout\n", 4978 vsi->seid, pf_q); 4979 return ret; 4980 } 4981 wait_rx: 4982 /* Check and wait for the Rx queue */ 4983 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4984 if (ret) { 4985 dev_info(&pf->pdev->dev, 4986 "VSI seid %d Rx ring %d disable timeout\n", 4987 vsi->seid, pf_q); 4988 return ret; 4989 } 4990 } 4991 4992 return 0; 4993 } 4994 4995 #ifdef CONFIG_I40E_DCB 4996 /** 4997 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4998 * @pf: the PF 4999 * 5000 * This function waits for the queues to be in disabled state for all the 5001 * VSIs that are managed by this PF. 5002 **/ 5003 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 5004 { 5005 int v, ret = 0; 5006 5007 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 5008 if (pf->vsi[v]) { 5009 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 5010 if (ret) 5011 break; 5012 } 5013 } 5014 5015 return ret; 5016 } 5017 5018 #endif 5019 5020 /** 5021 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 5022 * @pf: pointer to PF 5023 * 5024 * Get TC map for ISCSI PF type that will include iSCSI TC 5025 * and LAN TC. 5026 **/ 5027 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 5028 { 5029 struct i40e_dcb_app_priority_table app; 5030 struct i40e_hw *hw = &pf->hw; 5031 u8 enabled_tc = 1; /* TC0 is always enabled */ 5032 u8 tc, i; 5033 /* Get the iSCSI APP TLV */ 5034 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5035 5036 for (i = 0; i < dcbcfg->numapps; i++) { 5037 app = dcbcfg->app[i]; 5038 if (app.selector == I40E_APP_SEL_TCPIP && 5039 app.protocolid == I40E_APP_PROTOID_ISCSI) { 5040 tc = dcbcfg->etscfg.prioritytable[app.priority]; 5041 enabled_tc |= BIT(tc); 5042 break; 5043 } 5044 } 5045 5046 return enabled_tc; 5047 } 5048 5049 /** 5050 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5051 * @dcbcfg: the corresponding DCBx configuration structure 5052 * 5053 * Return the number of TCs from given DCBx configuration 5054 **/ 5055 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5056 { 5057 int i, tc_unused = 0; 5058 u8 num_tc = 0; 5059 u8 ret = 0; 5060 5061 /* Scan the ETS Config Priority Table to find 5062 * traffic class enabled for a given priority 5063 * and create a bitmask of enabled TCs 5064 */ 5065 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5066 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5067 5068 /* Now scan the bitmask to check for 5069 * contiguous TCs starting with TC0 5070 */ 5071 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5072 if (num_tc & BIT(i)) { 5073 if (!tc_unused) { 5074 ret++; 5075 } else { 5076 pr_err("Non-contiguous TC - Disabling DCB\n"); 5077 return 1; 5078 } 5079 } else { 5080 tc_unused = 1; 5081 } 5082 } 5083 5084 /* There is always at least TC0 */ 5085 if (!ret) 5086 ret = 1; 5087 5088 return ret; 5089 } 5090 5091 /** 5092 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5093 * @dcbcfg: the corresponding DCBx configuration structure 5094 * 5095 * Query the current DCB configuration and return the number of 5096 * traffic classes enabled from the given DCBX config 5097 **/ 5098 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5099 { 5100 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5101 u8 enabled_tc = 1; 5102 u8 i; 5103 5104 for (i = 0; i < num_tc; i++) 5105 enabled_tc |= BIT(i); 5106 5107 return enabled_tc; 5108 } 5109 5110 /** 5111 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5112 * @pf: PF being queried 5113 * 5114 * Query the current MQPRIO configuration and return the number of 5115 * traffic classes enabled. 5116 **/ 5117 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5118 { 5119 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 5120 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5121 u8 enabled_tc = 1, i; 5122 5123 for (i = 1; i < num_tc; i++) 5124 enabled_tc |= BIT(i); 5125 return enabled_tc; 5126 } 5127 5128 /** 5129 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5130 * @pf: PF being queried 5131 * 5132 * Return number of traffic classes enabled for the given PF 5133 **/ 5134 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5135 { 5136 struct i40e_hw *hw = &pf->hw; 5137 u8 i, enabled_tc = 1; 5138 u8 num_tc = 0; 5139 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5140 5141 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5142 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5143 5144 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5145 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5146 return 1; 5147 5148 /* SFP mode will be enabled for all TCs on port */ 5149 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5150 return i40e_dcb_get_num_tc(dcbcfg); 5151 5152 /* MFP mode return count of enabled TCs for this PF */ 5153 if (pf->hw.func_caps.iscsi) 5154 enabled_tc = i40e_get_iscsi_tc_map(pf); 5155 else 5156 return 1; /* Only TC0 */ 5157 5158 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5159 if (enabled_tc & BIT(i)) 5160 num_tc++; 5161 } 5162 return num_tc; 5163 } 5164 5165 /** 5166 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 5167 * @pf: PF being queried 5168 * 5169 * Return a bitmap for enabled traffic classes for this PF. 5170 **/ 5171 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5172 { 5173 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5174 return i40e_mqprio_get_enabled_tc(pf); 5175 5176 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5177 * default TC 5178 */ 5179 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5180 return I40E_DEFAULT_TRAFFIC_CLASS; 5181 5182 /* SFP mode we want PF to be enabled for all TCs */ 5183 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5184 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5185 5186 /* MFP enabled and iSCSI PF type */ 5187 if (pf->hw.func_caps.iscsi) 5188 return i40e_get_iscsi_tc_map(pf); 5189 else 5190 return I40E_DEFAULT_TRAFFIC_CLASS; 5191 } 5192 5193 /** 5194 * i40e_vsi_get_bw_info - Query VSI BW Information 5195 * @vsi: the VSI being queried 5196 * 5197 * Returns 0 on success, negative value on failure 5198 **/ 5199 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5200 { 5201 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5202 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5203 struct i40e_pf *pf = vsi->back; 5204 struct i40e_hw *hw = &pf->hw; 5205 i40e_status ret; 5206 u32 tc_bw_max; 5207 int i; 5208 5209 /* Get the VSI level BW configuration */ 5210 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5211 if (ret) { 5212 dev_info(&pf->pdev->dev, 5213 "couldn't get PF vsi bw config, err %s aq_err %s\n", 5214 i40e_stat_str(&pf->hw, ret), 5215 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5216 return -EINVAL; 5217 } 5218 5219 /* Get the VSI level BW configuration per TC */ 5220 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5221 NULL); 5222 if (ret) { 5223 dev_info(&pf->pdev->dev, 5224 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 5225 i40e_stat_str(&pf->hw, ret), 5226 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5227 return -EINVAL; 5228 } 5229 5230 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5231 dev_info(&pf->pdev->dev, 5232 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5233 bw_config.tc_valid_bits, 5234 bw_ets_config.tc_valid_bits); 5235 /* Still continuing */ 5236 } 5237 5238 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5239 vsi->bw_max_quanta = bw_config.max_bw; 5240 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5241 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5242 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5243 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5244 vsi->bw_ets_limit_credits[i] = 5245 le16_to_cpu(bw_ets_config.credits[i]); 5246 /* 3 bits out of 4 for each TC */ 5247 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5248 } 5249 5250 return 0; 5251 } 5252 5253 /** 5254 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5255 * @vsi: the VSI being configured 5256 * @enabled_tc: TC bitmap 5257 * @bw_share: BW shared credits per TC 5258 * 5259 * Returns 0 on success, negative value on failure 5260 **/ 5261 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5262 u8 *bw_share) 5263 { 5264 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5265 struct i40e_pf *pf = vsi->back; 5266 i40e_status ret; 5267 int i; 5268 5269 /* There is no need to reset BW when mqprio mode is on. */ 5270 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5271 return 0; 5272 if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5273 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5274 if (ret) 5275 dev_info(&pf->pdev->dev, 5276 "Failed to reset tx rate for vsi->seid %u\n", 5277 vsi->seid); 5278 return ret; 5279 } 5280 bw_data.tc_valid_bits = enabled_tc; 5281 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5282 bw_data.tc_bw_credits[i] = bw_share[i]; 5283 5284 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5285 if (ret) { 5286 dev_info(&pf->pdev->dev, 5287 "AQ command Config VSI BW allocation per TC failed = %d\n", 5288 pf->hw.aq.asq_last_status); 5289 return -EINVAL; 5290 } 5291 5292 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5293 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5294 5295 return 0; 5296 } 5297 5298 /** 5299 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5300 * @vsi: the VSI being configured 5301 * @enabled_tc: TC map to be enabled 5302 * 5303 **/ 5304 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5305 { 5306 struct net_device *netdev = vsi->netdev; 5307 struct i40e_pf *pf = vsi->back; 5308 struct i40e_hw *hw = &pf->hw; 5309 u8 netdev_tc = 0; 5310 int i; 5311 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5312 5313 if (!netdev) 5314 return; 5315 5316 if (!enabled_tc) { 5317 netdev_reset_tc(netdev); 5318 return; 5319 } 5320 5321 /* Set up actual enabled TCs on the VSI */ 5322 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5323 return; 5324 5325 /* set per TC queues for the VSI */ 5326 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5327 /* Only set TC queues for enabled tcs 5328 * 5329 * e.g. For a VSI that has TC0 and TC3 enabled the 5330 * enabled_tc bitmap would be 0x00001001; the driver 5331 * will set the numtc for netdev as 2 that will be 5332 * referenced by the netdev layer as TC 0 and 1. 5333 */ 5334 if (vsi->tc_config.enabled_tc & BIT(i)) 5335 netdev_set_tc_queue(netdev, 5336 vsi->tc_config.tc_info[i].netdev_tc, 5337 vsi->tc_config.tc_info[i].qcount, 5338 vsi->tc_config.tc_info[i].qoffset); 5339 } 5340 5341 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5342 return; 5343 5344 /* Assign UP2TC map for the VSI */ 5345 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5346 /* Get the actual TC# for the UP */ 5347 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5348 /* Get the mapped netdev TC# for the UP */ 5349 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5350 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5351 } 5352 } 5353 5354 /** 5355 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5356 * @vsi: the VSI being configured 5357 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5358 **/ 5359 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5360 struct i40e_vsi_context *ctxt) 5361 { 5362 /* copy just the sections touched not the entire info 5363 * since not all sections are valid as returned by 5364 * update vsi params 5365 */ 5366 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5367 memcpy(&vsi->info.queue_mapping, 5368 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5369 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5370 sizeof(vsi->info.tc_mapping)); 5371 } 5372 5373 /** 5374 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5375 * @vsi: VSI to be configured 5376 * @enabled_tc: TC bitmap 5377 * 5378 * This configures a particular VSI for TCs that are mapped to the 5379 * given TC bitmap. It uses default bandwidth share for TCs across 5380 * VSIs to configure TC for a particular VSI. 5381 * 5382 * NOTE: 5383 * It is expected that the VSI queues have been quisced before calling 5384 * this function. 5385 **/ 5386 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5387 { 5388 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5389 struct i40e_pf *pf = vsi->back; 5390 struct i40e_hw *hw = &pf->hw; 5391 struct i40e_vsi_context ctxt; 5392 int ret = 0; 5393 int i; 5394 5395 /* Check if enabled_tc is same as existing or new TCs */ 5396 if (vsi->tc_config.enabled_tc == enabled_tc && 5397 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5398 return ret; 5399 5400 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5401 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5402 if (enabled_tc & BIT(i)) 5403 bw_share[i] = 1; 5404 } 5405 5406 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5407 if (ret) { 5408 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5409 5410 dev_info(&pf->pdev->dev, 5411 "Failed configuring TC map %d for VSI %d\n", 5412 enabled_tc, vsi->seid); 5413 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5414 &bw_config, NULL); 5415 if (ret) { 5416 dev_info(&pf->pdev->dev, 5417 "Failed querying vsi bw info, err %s aq_err %s\n", 5418 i40e_stat_str(hw, ret), 5419 i40e_aq_str(hw, hw->aq.asq_last_status)); 5420 goto out; 5421 } 5422 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5423 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5424 5425 if (!valid_tc) 5426 valid_tc = bw_config.tc_valid_bits; 5427 /* Always enable TC0, no matter what */ 5428 valid_tc |= 1; 5429 dev_info(&pf->pdev->dev, 5430 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5431 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5432 enabled_tc = valid_tc; 5433 } 5434 5435 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5436 if (ret) { 5437 dev_err(&pf->pdev->dev, 5438 "Unable to configure TC map %d for VSI %d\n", 5439 enabled_tc, vsi->seid); 5440 goto out; 5441 } 5442 } 5443 5444 /* Update Queue Pairs Mapping for currently enabled UPs */ 5445 ctxt.seid = vsi->seid; 5446 ctxt.pf_num = vsi->back->hw.pf_id; 5447 ctxt.vf_num = 0; 5448 ctxt.uplink_seid = vsi->uplink_seid; 5449 ctxt.info = vsi->info; 5450 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { 5451 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5452 if (ret) 5453 goto out; 5454 } else { 5455 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5456 } 5457 5458 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5459 * queues changed. 5460 */ 5461 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5462 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5463 vsi->num_queue_pairs); 5464 ret = i40e_vsi_config_rss(vsi); 5465 if (ret) { 5466 dev_info(&vsi->back->pdev->dev, 5467 "Failed to reconfig rss for num_queues\n"); 5468 return ret; 5469 } 5470 vsi->reconfig_rss = false; 5471 } 5472 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5473 ctxt.info.valid_sections |= 5474 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5475 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5476 } 5477 5478 /* Update the VSI after updating the VSI queue-mapping 5479 * information 5480 */ 5481 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5482 if (ret) { 5483 dev_info(&pf->pdev->dev, 5484 "Update vsi tc config failed, err %s aq_err %s\n", 5485 i40e_stat_str(hw, ret), 5486 i40e_aq_str(hw, hw->aq.asq_last_status)); 5487 goto out; 5488 } 5489 /* update the local VSI info with updated queue map */ 5490 i40e_vsi_update_queue_map(vsi, &ctxt); 5491 vsi->info.valid_sections = 0; 5492 5493 /* Update current VSI BW information */ 5494 ret = i40e_vsi_get_bw_info(vsi); 5495 if (ret) { 5496 dev_info(&pf->pdev->dev, 5497 "Failed updating vsi bw info, err %s aq_err %s\n", 5498 i40e_stat_str(hw, ret), 5499 i40e_aq_str(hw, hw->aq.asq_last_status)); 5500 goto out; 5501 } 5502 5503 /* Update the netdev TC setup */ 5504 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5505 out: 5506 return ret; 5507 } 5508 5509 /** 5510 * i40e_get_link_speed - Returns link speed for the interface 5511 * @vsi: VSI to be configured 5512 * 5513 **/ 5514 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5515 { 5516 struct i40e_pf *pf = vsi->back; 5517 5518 switch (pf->hw.phy.link_info.link_speed) { 5519 case I40E_LINK_SPEED_40GB: 5520 return 40000; 5521 case I40E_LINK_SPEED_25GB: 5522 return 25000; 5523 case I40E_LINK_SPEED_20GB: 5524 return 20000; 5525 case I40E_LINK_SPEED_10GB: 5526 return 10000; 5527 case I40E_LINK_SPEED_1GB: 5528 return 1000; 5529 default: 5530 return -EINVAL; 5531 } 5532 } 5533 5534 /** 5535 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5536 * @vsi: VSI to be configured 5537 * @seid: seid of the channel/VSI 5538 * @max_tx_rate: max TX rate to be configured as BW limit 5539 * 5540 * Helper function to set BW limit for a given VSI 5541 **/ 5542 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5543 { 5544 struct i40e_pf *pf = vsi->back; 5545 u64 credits = 0; 5546 int speed = 0; 5547 int ret = 0; 5548 5549 speed = i40e_get_link_speed(vsi); 5550 if (max_tx_rate > speed) { 5551 dev_err(&pf->pdev->dev, 5552 "Invalid max tx rate %llu specified for VSI seid %d.", 5553 max_tx_rate, seid); 5554 return -EINVAL; 5555 } 5556 if (max_tx_rate && max_tx_rate < 50) { 5557 dev_warn(&pf->pdev->dev, 5558 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5559 max_tx_rate = 50; 5560 } 5561 5562 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5563 credits = max_tx_rate; 5564 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5565 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5566 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5567 if (ret) 5568 dev_err(&pf->pdev->dev, 5569 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n", 5570 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret), 5571 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5572 return ret; 5573 } 5574 5575 /** 5576 * i40e_remove_queue_channels - Remove queue channels for the TCs 5577 * @vsi: VSI to be configured 5578 * 5579 * Remove queue channels for the TCs 5580 **/ 5581 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5582 { 5583 enum i40e_admin_queue_err last_aq_status; 5584 struct i40e_cloud_filter *cfilter; 5585 struct i40e_channel *ch, *ch_tmp; 5586 struct i40e_pf *pf = vsi->back; 5587 struct hlist_node *node; 5588 int ret, i; 5589 5590 /* Reset rss size that was stored when reconfiguring rss for 5591 * channel VSIs with non-power-of-2 queue count. 5592 */ 5593 vsi->current_rss_size = 0; 5594 5595 /* perform cleanup for channels if they exist */ 5596 if (list_empty(&vsi->ch_list)) 5597 return; 5598 5599 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5600 struct i40e_vsi *p_vsi; 5601 5602 list_del(&ch->list); 5603 p_vsi = ch->parent_vsi; 5604 if (!p_vsi || !ch->initialized) { 5605 kfree(ch); 5606 continue; 5607 } 5608 /* Reset queue contexts */ 5609 for (i = 0; i < ch->num_queue_pairs; i++) { 5610 struct i40e_ring *tx_ring, *rx_ring; 5611 u16 pf_q; 5612 5613 pf_q = ch->base_queue + i; 5614 tx_ring = vsi->tx_rings[pf_q]; 5615 tx_ring->ch = NULL; 5616 5617 rx_ring = vsi->rx_rings[pf_q]; 5618 rx_ring->ch = NULL; 5619 } 5620 5621 /* Reset BW configured for this VSI via mqprio */ 5622 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 5623 if (ret) 5624 dev_info(&vsi->back->pdev->dev, 5625 "Failed to reset tx rate for ch->seid %u\n", 5626 ch->seid); 5627 5628 /* delete cloud filters associated with this channel */ 5629 hlist_for_each_entry_safe(cfilter, node, 5630 &pf->cloud_filter_list, cloud_node) { 5631 if (cfilter->seid != ch->seid) 5632 continue; 5633 5634 hash_del(&cfilter->cloud_node); 5635 if (cfilter->dst_port) 5636 ret = i40e_add_del_cloud_filter_big_buf(vsi, 5637 cfilter, 5638 false); 5639 else 5640 ret = i40e_add_del_cloud_filter(vsi, cfilter, 5641 false); 5642 last_aq_status = pf->hw.aq.asq_last_status; 5643 if (ret) 5644 dev_info(&pf->pdev->dev, 5645 "Failed to delete cloud filter, err %s aq_err %s\n", 5646 i40e_stat_str(&pf->hw, ret), 5647 i40e_aq_str(&pf->hw, last_aq_status)); 5648 kfree(cfilter); 5649 } 5650 5651 /* delete VSI from FW */ 5652 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 5653 NULL); 5654 if (ret) 5655 dev_err(&vsi->back->pdev->dev, 5656 "unable to remove channel (%d) for parent VSI(%d)\n", 5657 ch->seid, p_vsi->seid); 5658 kfree(ch); 5659 } 5660 INIT_LIST_HEAD(&vsi->ch_list); 5661 } 5662 5663 /** 5664 * i40e_is_any_channel - channel exist or not 5665 * @vsi: ptr to VSI to which channels are associated with 5666 * 5667 * Returns true or false if channel(s) exist for associated VSI or not 5668 **/ 5669 static bool i40e_is_any_channel(struct i40e_vsi *vsi) 5670 { 5671 struct i40e_channel *ch, *ch_tmp; 5672 5673 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5674 if (ch->initialized) 5675 return true; 5676 } 5677 5678 return false; 5679 } 5680 5681 /** 5682 * i40e_get_max_queues_for_channel 5683 * @vsi: ptr to VSI to which channels are associated with 5684 * 5685 * Helper function which returns max value among the queue counts set on the 5686 * channels/TCs created. 5687 **/ 5688 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 5689 { 5690 struct i40e_channel *ch, *ch_tmp; 5691 int max = 0; 5692 5693 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5694 if (!ch->initialized) 5695 continue; 5696 if (ch->num_queue_pairs > max) 5697 max = ch->num_queue_pairs; 5698 } 5699 5700 return max; 5701 } 5702 5703 /** 5704 * i40e_validate_num_queues - validate num_queues w.r.t channel 5705 * @pf: ptr to PF device 5706 * @num_queues: number of queues 5707 * @vsi: the parent VSI 5708 * @reconfig_rss: indicates should the RSS be reconfigured or not 5709 * 5710 * This function validates number of queues in the context of new channel 5711 * which is being established and determines if RSS should be reconfigured 5712 * or not for parent VSI. 5713 **/ 5714 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 5715 struct i40e_vsi *vsi, bool *reconfig_rss) 5716 { 5717 int max_ch_queues; 5718 5719 if (!reconfig_rss) 5720 return -EINVAL; 5721 5722 *reconfig_rss = false; 5723 if (vsi->current_rss_size) { 5724 if (num_queues > vsi->current_rss_size) { 5725 dev_dbg(&pf->pdev->dev, 5726 "Error: num_queues (%d) > vsi's current_size(%d)\n", 5727 num_queues, vsi->current_rss_size); 5728 return -EINVAL; 5729 } else if ((num_queues < vsi->current_rss_size) && 5730 (!is_power_of_2(num_queues))) { 5731 dev_dbg(&pf->pdev->dev, 5732 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 5733 num_queues, vsi->current_rss_size); 5734 return -EINVAL; 5735 } 5736 } 5737 5738 if (!is_power_of_2(num_queues)) { 5739 /* Find the max num_queues configured for channel if channel 5740 * exist. 5741 * if channel exist, then enforce 'num_queues' to be more than 5742 * max ever queues configured for channel. 5743 */ 5744 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 5745 if (num_queues < max_ch_queues) { 5746 dev_dbg(&pf->pdev->dev, 5747 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 5748 num_queues, max_ch_queues); 5749 return -EINVAL; 5750 } 5751 *reconfig_rss = true; 5752 } 5753 5754 return 0; 5755 } 5756 5757 /** 5758 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 5759 * @vsi: the VSI being setup 5760 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 5761 * 5762 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 5763 **/ 5764 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 5765 { 5766 struct i40e_pf *pf = vsi->back; 5767 u8 seed[I40E_HKEY_ARRAY_SIZE]; 5768 struct i40e_hw *hw = &pf->hw; 5769 int local_rss_size; 5770 u8 *lut; 5771 int ret; 5772 5773 if (!vsi->rss_size) 5774 return -EINVAL; 5775 5776 if (rss_size > vsi->rss_size) 5777 return -EINVAL; 5778 5779 local_rss_size = min_t(int, vsi->rss_size, rss_size); 5780 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 5781 if (!lut) 5782 return -ENOMEM; 5783 5784 /* Ignoring user configured lut if there is one */ 5785 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 5786 5787 /* Use user configured hash key if there is one, otherwise 5788 * use default. 5789 */ 5790 if (vsi->rss_hkey_user) 5791 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 5792 else 5793 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 5794 5795 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 5796 if (ret) { 5797 dev_info(&pf->pdev->dev, 5798 "Cannot set RSS lut, err %s aq_err %s\n", 5799 i40e_stat_str(hw, ret), 5800 i40e_aq_str(hw, hw->aq.asq_last_status)); 5801 kfree(lut); 5802 return ret; 5803 } 5804 kfree(lut); 5805 5806 /* Do the update w.r.t. storing rss_size */ 5807 if (!vsi->orig_rss_size) 5808 vsi->orig_rss_size = vsi->rss_size; 5809 vsi->current_rss_size = local_rss_size; 5810 5811 return ret; 5812 } 5813 5814 /** 5815 * i40e_channel_setup_queue_map - Setup a channel queue map 5816 * @pf: ptr to PF device 5817 * @vsi: the VSI being setup 5818 * @ctxt: VSI context structure 5819 * @ch: ptr to channel structure 5820 * 5821 * Setup queue map for a specific channel 5822 **/ 5823 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 5824 struct i40e_vsi_context *ctxt, 5825 struct i40e_channel *ch) 5826 { 5827 u16 qcount, qmap, sections = 0; 5828 u8 offset = 0; 5829 int pow; 5830 5831 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 5832 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 5833 5834 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 5835 ch->num_queue_pairs = qcount; 5836 5837 /* find the next higher power-of-2 of num queue pairs */ 5838 pow = ilog2(qcount); 5839 if (!is_power_of_2(qcount)) 5840 pow++; 5841 5842 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5843 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 5844 5845 /* Setup queue TC[0].qmap for given VSI context */ 5846 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 5847 5848 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 5849 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 5850 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 5851 ctxt->info.valid_sections |= cpu_to_le16(sections); 5852 } 5853 5854 /** 5855 * i40e_add_channel - add a channel by adding VSI 5856 * @pf: ptr to PF device 5857 * @uplink_seid: underlying HW switching element (VEB) ID 5858 * @ch: ptr to channel structure 5859 * 5860 * Add a channel (VSI) using add_vsi and queue_map 5861 **/ 5862 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 5863 struct i40e_channel *ch) 5864 { 5865 struct i40e_hw *hw = &pf->hw; 5866 struct i40e_vsi_context ctxt; 5867 u8 enabled_tc = 0x1; /* TC0 enabled */ 5868 int ret; 5869 5870 if (ch->type != I40E_VSI_VMDQ2) { 5871 dev_info(&pf->pdev->dev, 5872 "add new vsi failed, ch->type %d\n", ch->type); 5873 return -EINVAL; 5874 } 5875 5876 memset(&ctxt, 0, sizeof(ctxt)); 5877 ctxt.pf_num = hw->pf_id; 5878 ctxt.vf_num = 0; 5879 ctxt.uplink_seid = uplink_seid; 5880 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5881 if (ch->type == I40E_VSI_VMDQ2) 5882 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 5883 5884 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 5885 ctxt.info.valid_sections |= 5886 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5887 ctxt.info.switch_id = 5888 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 5889 } 5890 5891 /* Set queue map for a given VSI context */ 5892 i40e_channel_setup_queue_map(pf, &ctxt, ch); 5893 5894 /* Now time to create VSI */ 5895 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 5896 if (ret) { 5897 dev_info(&pf->pdev->dev, 5898 "add new vsi failed, err %s aq_err %s\n", 5899 i40e_stat_str(&pf->hw, ret), 5900 i40e_aq_str(&pf->hw, 5901 pf->hw.aq.asq_last_status)); 5902 return -ENOENT; 5903 } 5904 5905 /* Success, update channel, set enabled_tc only if the channel 5906 * is not a macvlan 5907 */ 5908 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; 5909 ch->seid = ctxt.seid; 5910 ch->vsi_number = ctxt.vsi_number; 5911 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); 5912 5913 /* copy just the sections touched not the entire info 5914 * since not all sections are valid as returned by 5915 * update vsi params 5916 */ 5917 ch->info.mapping_flags = ctxt.info.mapping_flags; 5918 memcpy(&ch->info.queue_mapping, 5919 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 5920 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 5921 sizeof(ctxt.info.tc_mapping)); 5922 5923 return 0; 5924 } 5925 5926 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 5927 u8 *bw_share) 5928 { 5929 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5930 i40e_status ret; 5931 int i; 5932 5933 bw_data.tc_valid_bits = ch->enabled_tc; 5934 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5935 bw_data.tc_bw_credits[i] = bw_share[i]; 5936 5937 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 5938 &bw_data, NULL); 5939 if (ret) { 5940 dev_info(&vsi->back->pdev->dev, 5941 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 5942 vsi->back->hw.aq.asq_last_status, ch->seid); 5943 return -EINVAL; 5944 } 5945 5946 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5947 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 5948 5949 return 0; 5950 } 5951 5952 /** 5953 * i40e_channel_config_tx_ring - config TX ring associated with new channel 5954 * @pf: ptr to PF device 5955 * @vsi: the VSI being setup 5956 * @ch: ptr to channel structure 5957 * 5958 * Configure TX rings associated with channel (VSI) since queues are being 5959 * from parent VSI. 5960 **/ 5961 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 5962 struct i40e_vsi *vsi, 5963 struct i40e_channel *ch) 5964 { 5965 i40e_status ret; 5966 int i; 5967 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5968 5969 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5970 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5971 if (ch->enabled_tc & BIT(i)) 5972 bw_share[i] = 1; 5973 } 5974 5975 /* configure BW for new VSI */ 5976 ret = i40e_channel_config_bw(vsi, ch, bw_share); 5977 if (ret) { 5978 dev_info(&vsi->back->pdev->dev, 5979 "Failed configuring TC map %d for channel (seid %u)\n", 5980 ch->enabled_tc, ch->seid); 5981 return ret; 5982 } 5983 5984 for (i = 0; i < ch->num_queue_pairs; i++) { 5985 struct i40e_ring *tx_ring, *rx_ring; 5986 u16 pf_q; 5987 5988 pf_q = ch->base_queue + i; 5989 5990 /* Get to TX ring ptr of main VSI, for re-setup TX queue 5991 * context 5992 */ 5993 tx_ring = vsi->tx_rings[pf_q]; 5994 tx_ring->ch = ch; 5995 5996 /* Get the RX ring ptr */ 5997 rx_ring = vsi->rx_rings[pf_q]; 5998 rx_ring->ch = ch; 5999 } 6000 6001 return 0; 6002 } 6003 6004 /** 6005 * i40e_setup_hw_channel - setup new channel 6006 * @pf: ptr to PF device 6007 * @vsi: the VSI being setup 6008 * @ch: ptr to channel structure 6009 * @uplink_seid: underlying HW switching element (VEB) ID 6010 * @type: type of channel to be created (VMDq2/VF) 6011 * 6012 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6013 * and configures TX rings accordingly 6014 **/ 6015 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 6016 struct i40e_vsi *vsi, 6017 struct i40e_channel *ch, 6018 u16 uplink_seid, u8 type) 6019 { 6020 int ret; 6021 6022 ch->initialized = false; 6023 ch->base_queue = vsi->next_base_queue; 6024 ch->type = type; 6025 6026 /* Proceed with creation of channel (VMDq2) VSI */ 6027 ret = i40e_add_channel(pf, uplink_seid, ch); 6028 if (ret) { 6029 dev_info(&pf->pdev->dev, 6030 "failed to add_channel using uplink_seid %u\n", 6031 uplink_seid); 6032 return ret; 6033 } 6034 6035 /* Mark the successful creation of channel */ 6036 ch->initialized = true; 6037 6038 /* Reconfigure TX queues using QTX_CTL register */ 6039 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 6040 if (ret) { 6041 dev_info(&pf->pdev->dev, 6042 "failed to configure TX rings for channel %u\n", 6043 ch->seid); 6044 return ret; 6045 } 6046 6047 /* update 'next_base_queue' */ 6048 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 6049 dev_dbg(&pf->pdev->dev, 6050 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 6051 ch->seid, ch->vsi_number, ch->stat_counter_idx, 6052 ch->num_queue_pairs, 6053 vsi->next_base_queue); 6054 return ret; 6055 } 6056 6057 /** 6058 * i40e_setup_channel - setup new channel using uplink element 6059 * @pf: ptr to PF device 6060 * @type: type of channel to be created (VMDq2/VF) 6061 * @uplink_seid: underlying HW switching element (VEB) ID 6062 * @ch: ptr to channel structure 6063 * 6064 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6065 * and uplink switching element (uplink_seid) 6066 **/ 6067 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 6068 struct i40e_channel *ch) 6069 { 6070 u8 vsi_type; 6071 u16 seid; 6072 int ret; 6073 6074 if (vsi->type == I40E_VSI_MAIN) { 6075 vsi_type = I40E_VSI_VMDQ2; 6076 } else { 6077 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6078 vsi->type); 6079 return false; 6080 } 6081 6082 /* underlying switching element */ 6083 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6084 6085 /* create channel (VSI), configure TX rings */ 6086 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6087 if (ret) { 6088 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6089 return false; 6090 } 6091 6092 return ch->initialized ? true : false; 6093 } 6094 6095 /** 6096 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6097 * @vsi: ptr to VSI which has PF backing 6098 * 6099 * Sets up switch mode correctly if it needs to be changed and perform 6100 * what are allowed modes. 6101 **/ 6102 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6103 { 6104 u8 mode; 6105 struct i40e_pf *pf = vsi->back; 6106 struct i40e_hw *hw = &pf->hw; 6107 int ret; 6108 6109 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6110 if (ret) 6111 return -EINVAL; 6112 6113 if (hw->dev_caps.switch_mode) { 6114 /* if switch mode is set, support mode2 (non-tunneled for 6115 * cloud filter) for now 6116 */ 6117 u32 switch_mode = hw->dev_caps.switch_mode & 6118 I40E_SWITCH_MODE_MASK; 6119 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6120 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6121 return 0; 6122 dev_err(&pf->pdev->dev, 6123 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6124 hw->dev_caps.switch_mode); 6125 return -EINVAL; 6126 } 6127 } 6128 6129 /* Set Bit 7 to be valid */ 6130 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6131 6132 /* Set L4type for TCP support */ 6133 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6134 6135 /* Set cloud filter mode */ 6136 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6137 6138 /* Prep mode field for set_switch_config */ 6139 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6140 pf->last_sw_conf_valid_flags, 6141 mode, NULL); 6142 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6143 dev_err(&pf->pdev->dev, 6144 "couldn't set switch config bits, err %s aq_err %s\n", 6145 i40e_stat_str(hw, ret), 6146 i40e_aq_str(hw, 6147 hw->aq.asq_last_status)); 6148 6149 return ret; 6150 } 6151 6152 /** 6153 * i40e_create_queue_channel - function to create channel 6154 * @vsi: VSI to be configured 6155 * @ch: ptr to channel (it contains channel specific params) 6156 * 6157 * This function creates channel (VSI) using num_queues specified by user, 6158 * reconfigs RSS if needed. 6159 **/ 6160 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6161 struct i40e_channel *ch) 6162 { 6163 struct i40e_pf *pf = vsi->back; 6164 bool reconfig_rss; 6165 int err; 6166 6167 if (!ch) 6168 return -EINVAL; 6169 6170 if (!ch->num_queue_pairs) { 6171 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6172 ch->num_queue_pairs); 6173 return -EINVAL; 6174 } 6175 6176 /* validate user requested num_queues for channel */ 6177 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6178 &reconfig_rss); 6179 if (err) { 6180 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6181 ch->num_queue_pairs); 6182 return -EINVAL; 6183 } 6184 6185 /* By default we are in VEPA mode, if this is the first VF/VMDq 6186 * VSI to be added switch to VEB mode. 6187 */ 6188 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) || 6189 (!i40e_is_any_channel(vsi))) { 6190 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) { 6191 dev_dbg(&pf->pdev->dev, 6192 "Failed to create channel. Override queues (%u) not power of 2\n", 6193 vsi->tc_config.tc_info[0].qcount); 6194 return -EINVAL; 6195 } 6196 6197 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6198 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6199 6200 if (vsi->type == I40E_VSI_MAIN) { 6201 if (pf->flags & I40E_FLAG_TC_MQPRIO) 6202 i40e_do_reset(pf, I40E_PF_RESET_FLAG, 6203 true); 6204 else 6205 i40e_do_reset_safe(pf, 6206 I40E_PF_RESET_FLAG); 6207 } 6208 } 6209 /* now onwards for main VSI, number of queues will be value 6210 * of TC0's queue count 6211 */ 6212 } 6213 6214 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6215 * it should be more than num_queues 6216 */ 6217 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6218 dev_dbg(&pf->pdev->dev, 6219 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6220 vsi->cnt_q_avail, ch->num_queue_pairs); 6221 return -EINVAL; 6222 } 6223 6224 /* reconfig_rss only if vsi type is MAIN_VSI */ 6225 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6226 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6227 if (err) { 6228 dev_info(&pf->pdev->dev, 6229 "Error: unable to reconfig rss for num_queues (%u)\n", 6230 ch->num_queue_pairs); 6231 return -EINVAL; 6232 } 6233 } 6234 6235 if (!i40e_setup_channel(pf, vsi, ch)) { 6236 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6237 return -EINVAL; 6238 } 6239 6240 dev_info(&pf->pdev->dev, 6241 "Setup channel (id:%u) utilizing num_queues %d\n", 6242 ch->seid, ch->num_queue_pairs); 6243 6244 /* configure VSI for BW limit */ 6245 if (ch->max_tx_rate) { 6246 u64 credits = ch->max_tx_rate; 6247 6248 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6249 return -EINVAL; 6250 6251 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6252 dev_dbg(&pf->pdev->dev, 6253 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6254 ch->max_tx_rate, 6255 credits, 6256 ch->seid); 6257 } 6258 6259 /* in case of VF, this will be main SRIOV VSI */ 6260 ch->parent_vsi = vsi; 6261 6262 /* and update main_vsi's count for queue_available to use */ 6263 vsi->cnt_q_avail -= ch->num_queue_pairs; 6264 6265 return 0; 6266 } 6267 6268 /** 6269 * i40e_configure_queue_channels - Add queue channel for the given TCs 6270 * @vsi: VSI to be configured 6271 * 6272 * Configures queue channel mapping to the given TCs 6273 **/ 6274 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6275 { 6276 struct i40e_channel *ch; 6277 u64 max_rate = 0; 6278 int ret = 0, i; 6279 6280 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6281 vsi->tc_seid_map[0] = vsi->seid; 6282 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6283 if (vsi->tc_config.enabled_tc & BIT(i)) { 6284 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6285 if (!ch) { 6286 ret = -ENOMEM; 6287 goto err_free; 6288 } 6289 6290 INIT_LIST_HEAD(&ch->list); 6291 ch->num_queue_pairs = 6292 vsi->tc_config.tc_info[i].qcount; 6293 ch->base_queue = 6294 vsi->tc_config.tc_info[i].qoffset; 6295 6296 /* Bandwidth limit through tc interface is in bytes/s, 6297 * change to Mbit/s 6298 */ 6299 max_rate = vsi->mqprio_qopt.max_rate[i]; 6300 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6301 ch->max_tx_rate = max_rate; 6302 6303 list_add_tail(&ch->list, &vsi->ch_list); 6304 6305 ret = i40e_create_queue_channel(vsi, ch); 6306 if (ret) { 6307 dev_err(&vsi->back->pdev->dev, 6308 "Failed creating queue channel with TC%d: queues %d\n", 6309 i, ch->num_queue_pairs); 6310 goto err_free; 6311 } 6312 vsi->tc_seid_map[i] = ch->seid; 6313 } 6314 } 6315 return ret; 6316 6317 err_free: 6318 i40e_remove_queue_channels(vsi); 6319 return ret; 6320 } 6321 6322 /** 6323 * i40e_veb_config_tc - Configure TCs for given VEB 6324 * @veb: given VEB 6325 * @enabled_tc: TC bitmap 6326 * 6327 * Configures given TC bitmap for VEB (switching) element 6328 **/ 6329 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6330 { 6331 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6332 struct i40e_pf *pf = veb->pf; 6333 int ret = 0; 6334 int i; 6335 6336 /* No TCs or already enabled TCs just return */ 6337 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6338 return ret; 6339 6340 bw_data.tc_valid_bits = enabled_tc; 6341 /* bw_data.absolute_credits is not set (relative) */ 6342 6343 /* Enable ETS TCs with equal BW Share for now */ 6344 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6345 if (enabled_tc & BIT(i)) 6346 bw_data.tc_bw_share_credits[i] = 1; 6347 } 6348 6349 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6350 &bw_data, NULL); 6351 if (ret) { 6352 dev_info(&pf->pdev->dev, 6353 "VEB bw config failed, err %s aq_err %s\n", 6354 i40e_stat_str(&pf->hw, ret), 6355 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6356 goto out; 6357 } 6358 6359 /* Update the BW information */ 6360 ret = i40e_veb_get_bw_info(veb); 6361 if (ret) { 6362 dev_info(&pf->pdev->dev, 6363 "Failed getting veb bw config, err %s aq_err %s\n", 6364 i40e_stat_str(&pf->hw, ret), 6365 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6366 } 6367 6368 out: 6369 return ret; 6370 } 6371 6372 #ifdef CONFIG_I40E_DCB 6373 /** 6374 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6375 * @pf: PF struct 6376 * 6377 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6378 * the caller would've quiesce all the VSIs before calling 6379 * this function 6380 **/ 6381 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6382 { 6383 u8 tc_map = 0; 6384 int ret; 6385 u8 v; 6386 6387 /* Enable the TCs available on PF to all VEBs */ 6388 tc_map = i40e_pf_get_tc_map(pf); 6389 for (v = 0; v < I40E_MAX_VEB; v++) { 6390 if (!pf->veb[v]) 6391 continue; 6392 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6393 if (ret) { 6394 dev_info(&pf->pdev->dev, 6395 "Failed configuring TC for VEB seid=%d\n", 6396 pf->veb[v]->seid); 6397 /* Will try to configure as many components */ 6398 } 6399 } 6400 6401 /* Update each VSI */ 6402 for (v = 0; v < pf->num_alloc_vsi; v++) { 6403 if (!pf->vsi[v]) 6404 continue; 6405 6406 /* - Enable all TCs for the LAN VSI 6407 * - For all others keep them at TC0 for now 6408 */ 6409 if (v == pf->lan_vsi) 6410 tc_map = i40e_pf_get_tc_map(pf); 6411 else 6412 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6413 6414 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6415 if (ret) { 6416 dev_info(&pf->pdev->dev, 6417 "Failed configuring TC for VSI seid=%d\n", 6418 pf->vsi[v]->seid); 6419 /* Will try to configure as many components */ 6420 } else { 6421 /* Re-configure VSI vectors based on updated TC map */ 6422 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6423 if (pf->vsi[v]->netdev) 6424 i40e_dcbnl_set_all(pf->vsi[v]); 6425 } 6426 } 6427 } 6428 6429 /** 6430 * i40e_resume_port_tx - Resume port Tx 6431 * @pf: PF struct 6432 * 6433 * Resume a port's Tx and issue a PF reset in case of failure to 6434 * resume. 6435 **/ 6436 static int i40e_resume_port_tx(struct i40e_pf *pf) 6437 { 6438 struct i40e_hw *hw = &pf->hw; 6439 int ret; 6440 6441 ret = i40e_aq_resume_port_tx(hw, NULL); 6442 if (ret) { 6443 dev_info(&pf->pdev->dev, 6444 "Resume Port Tx failed, err %s aq_err %s\n", 6445 i40e_stat_str(&pf->hw, ret), 6446 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6447 /* Schedule PF reset to recover */ 6448 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6449 i40e_service_event_schedule(pf); 6450 } 6451 6452 return ret; 6453 } 6454 6455 /** 6456 * i40e_init_pf_dcb - Initialize DCB configuration 6457 * @pf: PF being configured 6458 * 6459 * Query the current DCB configuration and cache it 6460 * in the hardware structure 6461 **/ 6462 static int i40e_init_pf_dcb(struct i40e_pf *pf) 6463 { 6464 struct i40e_hw *hw = &pf->hw; 6465 int err = 0; 6466 6467 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 6468 * Also do not enable DCBx if FW LLDP agent is disabled 6469 */ 6470 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) || 6471 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) { 6472 dev_info(&pf->pdev->dev, "DCB is not supported or FW LLDP is disabled\n"); 6473 err = I40E_NOT_SUPPORTED; 6474 goto out; 6475 } 6476 6477 err = i40e_init_dcb(hw, true); 6478 if (!err) { 6479 /* Device/Function is not DCBX capable */ 6480 if ((!hw->func_caps.dcb) || 6481 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 6482 dev_info(&pf->pdev->dev, 6483 "DCBX offload is not supported or is disabled for this PF.\n"); 6484 } else { 6485 /* When status is not DISABLED then DCBX in FW */ 6486 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 6487 DCB_CAP_DCBX_VER_IEEE; 6488 6489 pf->flags |= I40E_FLAG_DCB_CAPABLE; 6490 /* Enable DCB tagging only when more than one TC 6491 * or explicitly disable if only one TC 6492 */ 6493 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6494 pf->flags |= I40E_FLAG_DCB_ENABLED; 6495 else 6496 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6497 dev_dbg(&pf->pdev->dev, 6498 "DCBX offload is supported for this PF.\n"); 6499 } 6500 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 6501 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 6502 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP; 6503 } else { 6504 dev_info(&pf->pdev->dev, 6505 "Query for DCB configuration failed, err %s aq_err %s\n", 6506 i40e_stat_str(&pf->hw, err), 6507 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6508 } 6509 6510 out: 6511 return err; 6512 } 6513 #endif /* CONFIG_I40E_DCB */ 6514 6515 /** 6516 * i40e_print_link_message - print link up or down 6517 * @vsi: the VSI for which link needs a message 6518 * @isup: true of link is up, false otherwise 6519 */ 6520 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 6521 { 6522 enum i40e_aq_link_speed new_speed; 6523 struct i40e_pf *pf = vsi->back; 6524 char *speed = "Unknown"; 6525 char *fc = "Unknown"; 6526 char *fec = ""; 6527 char *req_fec = ""; 6528 char *an = ""; 6529 6530 if (isup) 6531 new_speed = pf->hw.phy.link_info.link_speed; 6532 else 6533 new_speed = I40E_LINK_SPEED_UNKNOWN; 6534 6535 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 6536 return; 6537 vsi->current_isup = isup; 6538 vsi->current_speed = new_speed; 6539 if (!isup) { 6540 netdev_info(vsi->netdev, "NIC Link is Down\n"); 6541 return; 6542 } 6543 6544 /* Warn user if link speed on NPAR enabled partition is not at 6545 * least 10GB 6546 */ 6547 if (pf->hw.func_caps.npar_enable && 6548 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 6549 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 6550 netdev_warn(vsi->netdev, 6551 "The partition detected link speed that is less than 10Gbps\n"); 6552 6553 switch (pf->hw.phy.link_info.link_speed) { 6554 case I40E_LINK_SPEED_40GB: 6555 speed = "40 G"; 6556 break; 6557 case I40E_LINK_SPEED_20GB: 6558 speed = "20 G"; 6559 break; 6560 case I40E_LINK_SPEED_25GB: 6561 speed = "25 G"; 6562 break; 6563 case I40E_LINK_SPEED_10GB: 6564 speed = "10 G"; 6565 break; 6566 case I40E_LINK_SPEED_5GB: 6567 speed = "5 G"; 6568 break; 6569 case I40E_LINK_SPEED_2_5GB: 6570 speed = "2.5 G"; 6571 break; 6572 case I40E_LINK_SPEED_1GB: 6573 speed = "1000 M"; 6574 break; 6575 case I40E_LINK_SPEED_100MB: 6576 speed = "100 M"; 6577 break; 6578 default: 6579 break; 6580 } 6581 6582 switch (pf->hw.fc.current_mode) { 6583 case I40E_FC_FULL: 6584 fc = "RX/TX"; 6585 break; 6586 case I40E_FC_TX_PAUSE: 6587 fc = "TX"; 6588 break; 6589 case I40E_FC_RX_PAUSE: 6590 fc = "RX"; 6591 break; 6592 default: 6593 fc = "None"; 6594 break; 6595 } 6596 6597 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 6598 req_fec = "None"; 6599 fec = "None"; 6600 an = "False"; 6601 6602 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6603 an = "True"; 6604 6605 if (pf->hw.phy.link_info.fec_info & 6606 I40E_AQ_CONFIG_FEC_KR_ENA) 6607 fec = "CL74 FC-FEC/BASE-R"; 6608 else if (pf->hw.phy.link_info.fec_info & 6609 I40E_AQ_CONFIG_FEC_RS_ENA) 6610 fec = "CL108 RS-FEC"; 6611 6612 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 6613 * both RS and FC are requested 6614 */ 6615 if (vsi->back->hw.phy.link_info.req_fec_info & 6616 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 6617 if (vsi->back->hw.phy.link_info.req_fec_info & 6618 I40E_AQ_REQUEST_FEC_RS) 6619 req_fec = "CL108 RS-FEC"; 6620 else 6621 req_fec = "CL74 FC-FEC/BASE-R"; 6622 } 6623 netdev_info(vsi->netdev, 6624 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 6625 speed, req_fec, fec, an, fc); 6626 } else { 6627 netdev_info(vsi->netdev, 6628 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", 6629 speed, fc); 6630 } 6631 6632 } 6633 6634 /** 6635 * i40e_up_complete - Finish the last steps of bringing up a connection 6636 * @vsi: the VSI being configured 6637 **/ 6638 static int i40e_up_complete(struct i40e_vsi *vsi) 6639 { 6640 struct i40e_pf *pf = vsi->back; 6641 int err; 6642 6643 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6644 i40e_vsi_configure_msix(vsi); 6645 else 6646 i40e_configure_msi_and_legacy(vsi); 6647 6648 /* start rings */ 6649 err = i40e_vsi_start_rings(vsi); 6650 if (err) 6651 return err; 6652 6653 clear_bit(__I40E_VSI_DOWN, vsi->state); 6654 i40e_napi_enable_all(vsi); 6655 i40e_vsi_enable_irq(vsi); 6656 6657 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 6658 (vsi->netdev)) { 6659 i40e_print_link_message(vsi, true); 6660 netif_tx_start_all_queues(vsi->netdev); 6661 netif_carrier_on(vsi->netdev); 6662 } 6663 6664 /* replay FDIR SB filters */ 6665 if (vsi->type == I40E_VSI_FDIR) { 6666 /* reset fd counters */ 6667 pf->fd_add_err = 0; 6668 pf->fd_atr_cnt = 0; 6669 i40e_fdir_filter_restore(vsi); 6670 } 6671 6672 /* On the next run of the service_task, notify any clients of the new 6673 * opened netdev 6674 */ 6675 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 6676 i40e_service_event_schedule(pf); 6677 6678 return 0; 6679 } 6680 6681 /** 6682 * i40e_vsi_reinit_locked - Reset the VSI 6683 * @vsi: the VSI being configured 6684 * 6685 * Rebuild the ring structs after some configuration 6686 * has changed, e.g. MTU size. 6687 **/ 6688 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 6689 { 6690 struct i40e_pf *pf = vsi->back; 6691 6692 WARN_ON(in_interrupt()); 6693 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 6694 usleep_range(1000, 2000); 6695 i40e_down(vsi); 6696 6697 i40e_up(vsi); 6698 clear_bit(__I40E_CONFIG_BUSY, pf->state); 6699 } 6700 6701 /** 6702 * i40e_force_link_state - Force the link status 6703 * @pf: board private structure 6704 * @is_up: whether the link state should be forced up or down 6705 **/ 6706 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up) 6707 { 6708 struct i40e_aq_get_phy_abilities_resp abilities; 6709 struct i40e_aq_set_phy_config config = {0}; 6710 bool non_zero_phy_type = is_up; 6711 struct i40e_hw *hw = &pf->hw; 6712 i40e_status err; 6713 u64 mask; 6714 u8 speed; 6715 6716 /* Card might've been put in an unstable state by other drivers 6717 * and applications, which causes incorrect speed values being 6718 * set on startup. In order to clear speed registers, we call 6719 * get_phy_capabilities twice, once to get initial state of 6720 * available speeds, and once to get current PHY config. 6721 */ 6722 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 6723 NULL); 6724 if (err) { 6725 dev_err(&pf->pdev->dev, 6726 "failed to get phy cap., ret = %s last_status = %s\n", 6727 i40e_stat_str(hw, err), 6728 i40e_aq_str(hw, hw->aq.asq_last_status)); 6729 return err; 6730 } 6731 speed = abilities.link_speed; 6732 6733 /* Get the current phy config */ 6734 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 6735 NULL); 6736 if (err) { 6737 dev_err(&pf->pdev->dev, 6738 "failed to get phy cap., ret = %s last_status = %s\n", 6739 i40e_stat_str(hw, err), 6740 i40e_aq_str(hw, hw->aq.asq_last_status)); 6741 return err; 6742 } 6743 6744 /* If link needs to go up, but was not forced to go down, 6745 * and its speed values are OK, no need for a flap 6746 * if non_zero_phy_type was set, still need to force up 6747 */ 6748 if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) 6749 non_zero_phy_type = true; 6750 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 6751 return I40E_SUCCESS; 6752 6753 /* To force link we need to set bits for all supported PHY types, 6754 * but there are now more than 32, so we need to split the bitmap 6755 * across two fields. 6756 */ 6757 mask = I40E_PHY_TYPES_BITMASK; 6758 config.phy_type = 6759 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 6760 config.phy_type_ext = 6761 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0; 6762 /* Copy the old settings, except of phy_type */ 6763 config.abilities = abilities.abilities; 6764 if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) { 6765 if (is_up) 6766 config.abilities |= I40E_AQ_PHY_ENABLE_LINK; 6767 else 6768 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK); 6769 } 6770 if (abilities.link_speed != 0) 6771 config.link_speed = abilities.link_speed; 6772 else 6773 config.link_speed = speed; 6774 config.eee_capability = abilities.eee_capability; 6775 config.eeer = abilities.eeer_val; 6776 config.low_power_ctrl = abilities.d3_lpan; 6777 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 6778 I40E_AQ_PHY_FEC_CONFIG_MASK; 6779 err = i40e_aq_set_phy_config(hw, &config, NULL); 6780 6781 if (err) { 6782 dev_err(&pf->pdev->dev, 6783 "set phy config ret = %s last_status = %s\n", 6784 i40e_stat_str(&pf->hw, err), 6785 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6786 return err; 6787 } 6788 6789 /* Update the link info */ 6790 err = i40e_update_link_info(hw); 6791 if (err) { 6792 /* Wait a little bit (on 40G cards it sometimes takes a really 6793 * long time for link to come back from the atomic reset) 6794 * and try once more 6795 */ 6796 msleep(1000); 6797 i40e_update_link_info(hw); 6798 } 6799 6800 i40e_aq_set_link_restart_an(hw, is_up, NULL); 6801 6802 return I40E_SUCCESS; 6803 } 6804 6805 /** 6806 * i40e_up - Bring the connection back up after being down 6807 * @vsi: the VSI being configured 6808 **/ 6809 int i40e_up(struct i40e_vsi *vsi) 6810 { 6811 int err; 6812 6813 if (vsi->type == I40E_VSI_MAIN && 6814 (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED || 6815 vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)) 6816 i40e_force_link_state(vsi->back, true); 6817 6818 err = i40e_vsi_configure(vsi); 6819 if (!err) 6820 err = i40e_up_complete(vsi); 6821 6822 return err; 6823 } 6824 6825 /** 6826 * i40e_down - Shutdown the connection processing 6827 * @vsi: the VSI being stopped 6828 **/ 6829 void i40e_down(struct i40e_vsi *vsi) 6830 { 6831 int i; 6832 6833 /* It is assumed that the caller of this function 6834 * sets the vsi->state __I40E_VSI_DOWN bit. 6835 */ 6836 if (vsi->netdev) { 6837 netif_carrier_off(vsi->netdev); 6838 netif_tx_disable(vsi->netdev); 6839 } 6840 i40e_vsi_disable_irq(vsi); 6841 i40e_vsi_stop_rings(vsi); 6842 if (vsi->type == I40E_VSI_MAIN && 6843 (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED || 6844 vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)) 6845 i40e_force_link_state(vsi->back, false); 6846 i40e_napi_disable_all(vsi); 6847 6848 for (i = 0; i < vsi->num_queue_pairs; i++) { 6849 i40e_clean_tx_ring(vsi->tx_rings[i]); 6850 if (i40e_enabled_xdp_vsi(vsi)) { 6851 /* Make sure that in-progress ndo_xdp_xmit and 6852 * ndo_xsk_wakeup calls are completed. 6853 */ 6854 synchronize_rcu(); 6855 i40e_clean_tx_ring(vsi->xdp_rings[i]); 6856 } 6857 i40e_clean_rx_ring(vsi->rx_rings[i]); 6858 } 6859 6860 } 6861 6862 /** 6863 * i40e_validate_mqprio_qopt- validate queue mapping info 6864 * @vsi: the VSI being configured 6865 * @mqprio_qopt: queue parametrs 6866 **/ 6867 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 6868 struct tc_mqprio_qopt_offload *mqprio_qopt) 6869 { 6870 u64 sum_max_rate = 0; 6871 u64 max_rate = 0; 6872 int i; 6873 6874 if (mqprio_qopt->qopt.offset[0] != 0 || 6875 mqprio_qopt->qopt.num_tc < 1 || 6876 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 6877 return -EINVAL; 6878 for (i = 0; ; i++) { 6879 if (!mqprio_qopt->qopt.count[i]) 6880 return -EINVAL; 6881 if (mqprio_qopt->min_rate[i]) { 6882 dev_err(&vsi->back->pdev->dev, 6883 "Invalid min tx rate (greater than 0) specified\n"); 6884 return -EINVAL; 6885 } 6886 max_rate = mqprio_qopt->max_rate[i]; 6887 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6888 sum_max_rate += max_rate; 6889 6890 if (i >= mqprio_qopt->qopt.num_tc - 1) 6891 break; 6892 if (mqprio_qopt->qopt.offset[i + 1] != 6893 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 6894 return -EINVAL; 6895 } 6896 if (vsi->num_queue_pairs < 6897 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 6898 return -EINVAL; 6899 } 6900 if (sum_max_rate > i40e_get_link_speed(vsi)) { 6901 dev_err(&vsi->back->pdev->dev, 6902 "Invalid max tx rate specified\n"); 6903 return -EINVAL; 6904 } 6905 return 0; 6906 } 6907 6908 /** 6909 * i40e_vsi_set_default_tc_config - set default values for tc configuration 6910 * @vsi: the VSI being configured 6911 **/ 6912 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 6913 { 6914 u16 qcount; 6915 int i; 6916 6917 /* Only TC0 is enabled */ 6918 vsi->tc_config.numtc = 1; 6919 vsi->tc_config.enabled_tc = 1; 6920 qcount = min_t(int, vsi->alloc_queue_pairs, 6921 i40e_pf_get_max_q_per_tc(vsi->back)); 6922 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6923 /* For the TC that is not enabled set the offset to to default 6924 * queue and allocate one queue for the given TC. 6925 */ 6926 vsi->tc_config.tc_info[i].qoffset = 0; 6927 if (i == 0) 6928 vsi->tc_config.tc_info[i].qcount = qcount; 6929 else 6930 vsi->tc_config.tc_info[i].qcount = 1; 6931 vsi->tc_config.tc_info[i].netdev_tc = 0; 6932 } 6933 } 6934 6935 /** 6936 * i40e_del_macvlan_filter 6937 * @hw: pointer to the HW structure 6938 * @seid: seid of the channel VSI 6939 * @macaddr: the mac address to apply as a filter 6940 * @aq_err: store the admin Q error 6941 * 6942 * This function deletes a mac filter on the channel VSI which serves as the 6943 * macvlan. Returns 0 on success. 6944 **/ 6945 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid, 6946 const u8 *macaddr, int *aq_err) 6947 { 6948 struct i40e_aqc_remove_macvlan_element_data element; 6949 i40e_status status; 6950 6951 memset(&element, 0, sizeof(element)); 6952 ether_addr_copy(element.mac_addr, macaddr); 6953 element.vlan_tag = 0; 6954 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 6955 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL); 6956 *aq_err = hw->aq.asq_last_status; 6957 6958 return status; 6959 } 6960 6961 /** 6962 * i40e_add_macvlan_filter 6963 * @hw: pointer to the HW structure 6964 * @seid: seid of the channel VSI 6965 * @macaddr: the mac address to apply as a filter 6966 * @aq_err: store the admin Q error 6967 * 6968 * This function adds a mac filter on the channel VSI which serves as the 6969 * macvlan. Returns 0 on success. 6970 **/ 6971 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid, 6972 const u8 *macaddr, int *aq_err) 6973 { 6974 struct i40e_aqc_add_macvlan_element_data element; 6975 i40e_status status; 6976 u16 cmd_flags = 0; 6977 6978 ether_addr_copy(element.mac_addr, macaddr); 6979 element.vlan_tag = 0; 6980 element.queue_number = 0; 6981 element.match_method = I40E_AQC_MM_ERR_NO_RES; 6982 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 6983 element.flags = cpu_to_le16(cmd_flags); 6984 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL); 6985 *aq_err = hw->aq.asq_last_status; 6986 6987 return status; 6988 } 6989 6990 /** 6991 * i40e_reset_ch_rings - Reset the queue contexts in a channel 6992 * @vsi: the VSI we want to access 6993 * @ch: the channel we want to access 6994 */ 6995 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch) 6996 { 6997 struct i40e_ring *tx_ring, *rx_ring; 6998 u16 pf_q; 6999 int i; 7000 7001 for (i = 0; i < ch->num_queue_pairs; i++) { 7002 pf_q = ch->base_queue + i; 7003 tx_ring = vsi->tx_rings[pf_q]; 7004 tx_ring->ch = NULL; 7005 rx_ring = vsi->rx_rings[pf_q]; 7006 rx_ring->ch = NULL; 7007 } 7008 } 7009 7010 /** 7011 * i40e_free_macvlan_channels 7012 * @vsi: the VSI we want to access 7013 * 7014 * This function frees the Qs of the channel VSI from 7015 * the stack and also deletes the channel VSIs which 7016 * serve as macvlans. 7017 */ 7018 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi) 7019 { 7020 struct i40e_channel *ch, *ch_tmp; 7021 int ret; 7022 7023 if (list_empty(&vsi->macvlan_list)) 7024 return; 7025 7026 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7027 struct i40e_vsi *parent_vsi; 7028 7029 if (i40e_is_channel_macvlan(ch)) { 7030 i40e_reset_ch_rings(vsi, ch); 7031 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7032 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev); 7033 netdev_set_sb_channel(ch->fwd->netdev, 0); 7034 kfree(ch->fwd); 7035 ch->fwd = NULL; 7036 } 7037 7038 list_del(&ch->list); 7039 parent_vsi = ch->parent_vsi; 7040 if (!parent_vsi || !ch->initialized) { 7041 kfree(ch); 7042 continue; 7043 } 7044 7045 /* remove the VSI */ 7046 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 7047 NULL); 7048 if (ret) 7049 dev_err(&vsi->back->pdev->dev, 7050 "unable to remove channel (%d) for parent VSI(%d)\n", 7051 ch->seid, parent_vsi->seid); 7052 kfree(ch); 7053 } 7054 vsi->macvlan_cnt = 0; 7055 } 7056 7057 /** 7058 * i40e_fwd_ring_up - bring the macvlan device up 7059 * @vsi: the VSI we want to access 7060 * @vdev: macvlan netdevice 7061 * @fwd: the private fwd structure 7062 */ 7063 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev, 7064 struct i40e_fwd_adapter *fwd) 7065 { 7066 int ret = 0, num_tc = 1, i, aq_err; 7067 struct i40e_channel *ch, *ch_tmp; 7068 struct i40e_pf *pf = vsi->back; 7069 struct i40e_hw *hw = &pf->hw; 7070 7071 if (list_empty(&vsi->macvlan_list)) 7072 return -EINVAL; 7073 7074 /* Go through the list and find an available channel */ 7075 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7076 if (!i40e_is_channel_macvlan(ch)) { 7077 ch->fwd = fwd; 7078 /* record configuration for macvlan interface in vdev */ 7079 for (i = 0; i < num_tc; i++) 7080 netdev_bind_sb_channel_queue(vsi->netdev, vdev, 7081 i, 7082 ch->num_queue_pairs, 7083 ch->base_queue); 7084 for (i = 0; i < ch->num_queue_pairs; i++) { 7085 struct i40e_ring *tx_ring, *rx_ring; 7086 u16 pf_q; 7087 7088 pf_q = ch->base_queue + i; 7089 7090 /* Get to TX ring ptr */ 7091 tx_ring = vsi->tx_rings[pf_q]; 7092 tx_ring->ch = ch; 7093 7094 /* Get the RX ring ptr */ 7095 rx_ring = vsi->rx_rings[pf_q]; 7096 rx_ring->ch = ch; 7097 } 7098 break; 7099 } 7100 } 7101 7102 /* Guarantee all rings are updated before we update the 7103 * MAC address filter. 7104 */ 7105 wmb(); 7106 7107 /* Add a mac filter */ 7108 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err); 7109 if (ret) { 7110 /* if we cannot add the MAC rule then disable the offload */ 7111 macvlan_release_l2fw_offload(vdev); 7112 for (i = 0; i < ch->num_queue_pairs; i++) { 7113 struct i40e_ring *rx_ring; 7114 u16 pf_q; 7115 7116 pf_q = ch->base_queue + i; 7117 rx_ring = vsi->rx_rings[pf_q]; 7118 rx_ring->netdev = NULL; 7119 } 7120 dev_info(&pf->pdev->dev, 7121 "Error adding mac filter on macvlan err %s, aq_err %s\n", 7122 i40e_stat_str(hw, ret), 7123 i40e_aq_str(hw, aq_err)); 7124 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n"); 7125 } 7126 7127 return ret; 7128 } 7129 7130 /** 7131 * i40e_setup_macvlans - create the channels which will be macvlans 7132 * @vsi: the VSI we want to access 7133 * @macvlan_cnt: no. of macvlans to be setup 7134 * @qcnt: no. of Qs per macvlan 7135 * @vdev: macvlan netdevice 7136 */ 7137 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt, 7138 struct net_device *vdev) 7139 { 7140 struct i40e_pf *pf = vsi->back; 7141 struct i40e_hw *hw = &pf->hw; 7142 struct i40e_vsi_context ctxt; 7143 u16 sections, qmap, num_qps; 7144 struct i40e_channel *ch; 7145 int i, pow, ret = 0; 7146 u8 offset = 0; 7147 7148 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt) 7149 return -EINVAL; 7150 7151 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt); 7152 7153 /* find the next higher power-of-2 of num queue pairs */ 7154 pow = fls(roundup_pow_of_two(num_qps) - 1); 7155 7156 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 7157 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 7158 7159 /* Setup context bits for the main VSI */ 7160 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 7161 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 7162 memset(&ctxt, 0, sizeof(ctxt)); 7163 ctxt.seid = vsi->seid; 7164 ctxt.pf_num = vsi->back->hw.pf_id; 7165 ctxt.vf_num = 0; 7166 ctxt.uplink_seid = vsi->uplink_seid; 7167 ctxt.info = vsi->info; 7168 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap); 7169 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 7170 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 7171 ctxt.info.valid_sections |= cpu_to_le16(sections); 7172 7173 /* Reconfigure RSS for main VSI with new max queue count */ 7174 vsi->rss_size = max_t(u16, num_qps, qcnt); 7175 ret = i40e_vsi_config_rss(vsi); 7176 if (ret) { 7177 dev_info(&pf->pdev->dev, 7178 "Failed to reconfig RSS for num_queues (%u)\n", 7179 vsi->rss_size); 7180 return ret; 7181 } 7182 vsi->reconfig_rss = true; 7183 dev_dbg(&vsi->back->pdev->dev, 7184 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size); 7185 vsi->next_base_queue = num_qps; 7186 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps; 7187 7188 /* Update the VSI after updating the VSI queue-mapping 7189 * information 7190 */ 7191 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7192 if (ret) { 7193 dev_info(&pf->pdev->dev, 7194 "Update vsi tc config failed, err %s aq_err %s\n", 7195 i40e_stat_str(hw, ret), 7196 i40e_aq_str(hw, hw->aq.asq_last_status)); 7197 return ret; 7198 } 7199 /* update the local VSI info with updated queue map */ 7200 i40e_vsi_update_queue_map(vsi, &ctxt); 7201 vsi->info.valid_sections = 0; 7202 7203 /* Create channels for macvlans */ 7204 INIT_LIST_HEAD(&vsi->macvlan_list); 7205 for (i = 0; i < macvlan_cnt; i++) { 7206 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 7207 if (!ch) { 7208 ret = -ENOMEM; 7209 goto err_free; 7210 } 7211 INIT_LIST_HEAD(&ch->list); 7212 ch->num_queue_pairs = qcnt; 7213 if (!i40e_setup_channel(pf, vsi, ch)) { 7214 ret = -EINVAL; 7215 kfree(ch); 7216 goto err_free; 7217 } 7218 ch->parent_vsi = vsi; 7219 vsi->cnt_q_avail -= ch->num_queue_pairs; 7220 vsi->macvlan_cnt++; 7221 list_add_tail(&ch->list, &vsi->macvlan_list); 7222 } 7223 7224 return ret; 7225 7226 err_free: 7227 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n"); 7228 i40e_free_macvlan_channels(vsi); 7229 7230 return ret; 7231 } 7232 7233 /** 7234 * i40e_fwd_add - configure macvlans 7235 * @netdev: net device to configure 7236 * @vdev: macvlan netdevice 7237 **/ 7238 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) 7239 { 7240 struct i40e_netdev_priv *np = netdev_priv(netdev); 7241 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors; 7242 struct i40e_vsi *vsi = np->vsi; 7243 struct i40e_pf *pf = vsi->back; 7244 struct i40e_fwd_adapter *fwd; 7245 int avail_macvlan, ret; 7246 7247 if ((pf->flags & I40E_FLAG_DCB_ENABLED)) { 7248 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); 7249 return ERR_PTR(-EINVAL); 7250 } 7251 if ((pf->flags & I40E_FLAG_TC_MQPRIO)) { 7252 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); 7253 return ERR_PTR(-EINVAL); 7254 } 7255 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) { 7256 netdev_info(netdev, "Not enough vectors available to support macvlans\n"); 7257 return ERR_PTR(-EINVAL); 7258 } 7259 7260 /* The macvlan device has to be a single Q device so that the 7261 * tc_to_txq field can be reused to pick the tx queue. 7262 */ 7263 if (netif_is_multiqueue(vdev)) 7264 return ERR_PTR(-ERANGE); 7265 7266 if (!vsi->macvlan_cnt) { 7267 /* reserve bit 0 for the pf device */ 7268 set_bit(0, vsi->fwd_bitmask); 7269 7270 /* Try to reserve as many queues as possible for macvlans. First 7271 * reserve 3/4th of max vectors, then half, then quarter and 7272 * calculate Qs per macvlan as you go 7273 */ 7274 vectors = pf->num_lan_msix; 7275 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) { 7276 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/ 7277 q_per_macvlan = 4; 7278 macvlan_cnt = (vectors - 32) / 4; 7279 } else if (vectors <= 64 && vectors > 32) { 7280 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/ 7281 q_per_macvlan = 2; 7282 macvlan_cnt = (vectors - 16) / 2; 7283 } else if (vectors <= 32 && vectors > 16) { 7284 /* allocate 1 Q per macvlan and 16 Qs to the PF*/ 7285 q_per_macvlan = 1; 7286 macvlan_cnt = vectors - 16; 7287 } else if (vectors <= 16 && vectors > 8) { 7288 /* allocate 1 Q per macvlan and 8 Qs to the PF */ 7289 q_per_macvlan = 1; 7290 macvlan_cnt = vectors - 8; 7291 } else { 7292 /* allocate 1 Q per macvlan and 1 Q to the PF */ 7293 q_per_macvlan = 1; 7294 macvlan_cnt = vectors - 1; 7295 } 7296 7297 if (macvlan_cnt == 0) 7298 return ERR_PTR(-EBUSY); 7299 7300 /* Quiesce VSI queues */ 7301 i40e_quiesce_vsi(vsi); 7302 7303 /* sets up the macvlans but does not "enable" them */ 7304 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan, 7305 vdev); 7306 if (ret) 7307 return ERR_PTR(ret); 7308 7309 /* Unquiesce VSI */ 7310 i40e_unquiesce_vsi(vsi); 7311 } 7312 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask, 7313 vsi->macvlan_cnt); 7314 if (avail_macvlan >= I40E_MAX_MACVLANS) 7315 return ERR_PTR(-EBUSY); 7316 7317 /* create the fwd struct */ 7318 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL); 7319 if (!fwd) 7320 return ERR_PTR(-ENOMEM); 7321 7322 set_bit(avail_macvlan, vsi->fwd_bitmask); 7323 fwd->bit_no = avail_macvlan; 7324 netdev_set_sb_channel(vdev, avail_macvlan); 7325 fwd->netdev = vdev; 7326 7327 if (!netif_running(netdev)) 7328 return fwd; 7329 7330 /* Set fwd ring up */ 7331 ret = i40e_fwd_ring_up(vsi, vdev, fwd); 7332 if (ret) { 7333 /* unbind the queues and drop the subordinate channel config */ 7334 netdev_unbind_sb_channel(netdev, vdev); 7335 netdev_set_sb_channel(vdev, 0); 7336 7337 kfree(fwd); 7338 return ERR_PTR(-EINVAL); 7339 } 7340 7341 return fwd; 7342 } 7343 7344 /** 7345 * i40e_del_all_macvlans - Delete all the mac filters on the channels 7346 * @vsi: the VSI we want to access 7347 */ 7348 static void i40e_del_all_macvlans(struct i40e_vsi *vsi) 7349 { 7350 struct i40e_channel *ch, *ch_tmp; 7351 struct i40e_pf *pf = vsi->back; 7352 struct i40e_hw *hw = &pf->hw; 7353 int aq_err, ret = 0; 7354 7355 if (list_empty(&vsi->macvlan_list)) 7356 return; 7357 7358 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7359 if (i40e_is_channel_macvlan(ch)) { 7360 ret = i40e_del_macvlan_filter(hw, ch->seid, 7361 i40e_channel_mac(ch), 7362 &aq_err); 7363 if (!ret) { 7364 /* Reset queue contexts */ 7365 i40e_reset_ch_rings(vsi, ch); 7366 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7367 netdev_unbind_sb_channel(vsi->netdev, 7368 ch->fwd->netdev); 7369 netdev_set_sb_channel(ch->fwd->netdev, 0); 7370 kfree(ch->fwd); 7371 ch->fwd = NULL; 7372 } 7373 } 7374 } 7375 } 7376 7377 /** 7378 * i40e_fwd_del - delete macvlan interfaces 7379 * @netdev: net device to configure 7380 * @vdev: macvlan netdevice 7381 */ 7382 static void i40e_fwd_del(struct net_device *netdev, void *vdev) 7383 { 7384 struct i40e_netdev_priv *np = netdev_priv(netdev); 7385 struct i40e_fwd_adapter *fwd = vdev; 7386 struct i40e_channel *ch, *ch_tmp; 7387 struct i40e_vsi *vsi = np->vsi; 7388 struct i40e_pf *pf = vsi->back; 7389 struct i40e_hw *hw = &pf->hw; 7390 int aq_err, ret = 0; 7391 7392 /* Find the channel associated with the macvlan and del mac filter */ 7393 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7394 if (i40e_is_channel_macvlan(ch) && 7395 ether_addr_equal(i40e_channel_mac(ch), 7396 fwd->netdev->dev_addr)) { 7397 ret = i40e_del_macvlan_filter(hw, ch->seid, 7398 i40e_channel_mac(ch), 7399 &aq_err); 7400 if (!ret) { 7401 /* Reset queue contexts */ 7402 i40e_reset_ch_rings(vsi, ch); 7403 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7404 netdev_unbind_sb_channel(netdev, fwd->netdev); 7405 netdev_set_sb_channel(fwd->netdev, 0); 7406 kfree(ch->fwd); 7407 ch->fwd = NULL; 7408 } else { 7409 dev_info(&pf->pdev->dev, 7410 "Error deleting mac filter on macvlan err %s, aq_err %s\n", 7411 i40e_stat_str(hw, ret), 7412 i40e_aq_str(hw, aq_err)); 7413 } 7414 break; 7415 } 7416 } 7417 } 7418 7419 /** 7420 * i40e_setup_tc - configure multiple traffic classes 7421 * @netdev: net device to configure 7422 * @type_data: tc offload data 7423 **/ 7424 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 7425 { 7426 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 7427 struct i40e_netdev_priv *np = netdev_priv(netdev); 7428 struct i40e_vsi *vsi = np->vsi; 7429 struct i40e_pf *pf = vsi->back; 7430 u8 enabled_tc = 0, num_tc, hw; 7431 bool need_reset = false; 7432 int old_queue_pairs; 7433 int ret = -EINVAL; 7434 u16 mode; 7435 int i; 7436 7437 old_queue_pairs = vsi->num_queue_pairs; 7438 num_tc = mqprio_qopt->qopt.num_tc; 7439 hw = mqprio_qopt->qopt.hw; 7440 mode = mqprio_qopt->mode; 7441 if (!hw) { 7442 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 7443 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 7444 goto config_tc; 7445 } 7446 7447 /* Check if MFP enabled */ 7448 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 7449 netdev_info(netdev, 7450 "Configuring TC not supported in MFP mode\n"); 7451 return ret; 7452 } 7453 switch (mode) { 7454 case TC_MQPRIO_MODE_DCB: 7455 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 7456 7457 /* Check if DCB enabled to continue */ 7458 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 7459 netdev_info(netdev, 7460 "DCB is not enabled for adapter\n"); 7461 return ret; 7462 } 7463 7464 /* Check whether tc count is within enabled limit */ 7465 if (num_tc > i40e_pf_get_num_tc(pf)) { 7466 netdev_info(netdev, 7467 "TC count greater than enabled on link for adapter\n"); 7468 return ret; 7469 } 7470 break; 7471 case TC_MQPRIO_MODE_CHANNEL: 7472 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 7473 netdev_info(netdev, 7474 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 7475 return ret; 7476 } 7477 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 7478 return ret; 7479 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 7480 if (ret) 7481 return ret; 7482 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 7483 sizeof(*mqprio_qopt)); 7484 pf->flags |= I40E_FLAG_TC_MQPRIO; 7485 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 7486 break; 7487 default: 7488 return -EINVAL; 7489 } 7490 7491 config_tc: 7492 /* Generate TC map for number of tc requested */ 7493 for (i = 0; i < num_tc; i++) 7494 enabled_tc |= BIT(i); 7495 7496 /* Requesting same TC configuration as already enabled */ 7497 if (enabled_tc == vsi->tc_config.enabled_tc && 7498 mode != TC_MQPRIO_MODE_CHANNEL) 7499 return 0; 7500 7501 /* Quiesce VSI queues */ 7502 i40e_quiesce_vsi(vsi); 7503 7504 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) 7505 i40e_remove_queue_channels(vsi); 7506 7507 /* Configure VSI for enabled TCs */ 7508 ret = i40e_vsi_config_tc(vsi, enabled_tc); 7509 if (ret) { 7510 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 7511 vsi->seid); 7512 need_reset = true; 7513 goto exit; 7514 } else { 7515 dev_info(&vsi->back->pdev->dev, 7516 "Setup channel (id:%u) utilizing num_queues %d\n", 7517 vsi->seid, vsi->tc_config.tc_info[0].qcount); 7518 } 7519 7520 if (pf->flags & I40E_FLAG_TC_MQPRIO) { 7521 if (vsi->mqprio_qopt.max_rate[0]) { 7522 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 7523 7524 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 7525 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 7526 if (!ret) { 7527 u64 credits = max_tx_rate; 7528 7529 do_div(credits, I40E_BW_CREDIT_DIVISOR); 7530 dev_dbg(&vsi->back->pdev->dev, 7531 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 7532 max_tx_rate, 7533 credits, 7534 vsi->seid); 7535 } else { 7536 need_reset = true; 7537 goto exit; 7538 } 7539 } 7540 ret = i40e_configure_queue_channels(vsi); 7541 if (ret) { 7542 vsi->num_queue_pairs = old_queue_pairs; 7543 netdev_info(netdev, 7544 "Failed configuring queue channels\n"); 7545 need_reset = true; 7546 goto exit; 7547 } 7548 } 7549 7550 exit: 7551 /* Reset the configuration data to defaults, only TC0 is enabled */ 7552 if (need_reset) { 7553 i40e_vsi_set_default_tc_config(vsi); 7554 need_reset = false; 7555 } 7556 7557 /* Unquiesce VSI */ 7558 i40e_unquiesce_vsi(vsi); 7559 return ret; 7560 } 7561 7562 /** 7563 * i40e_set_cld_element - sets cloud filter element data 7564 * @filter: cloud filter rule 7565 * @cld: ptr to cloud filter element data 7566 * 7567 * This is helper function to copy data into cloud filter element 7568 **/ 7569 static inline void 7570 i40e_set_cld_element(struct i40e_cloud_filter *filter, 7571 struct i40e_aqc_cloud_filters_element_data *cld) 7572 { 7573 int i, j; 7574 u32 ipa; 7575 7576 memset(cld, 0, sizeof(*cld)); 7577 ether_addr_copy(cld->outer_mac, filter->dst_mac); 7578 ether_addr_copy(cld->inner_mac, filter->src_mac); 7579 7580 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 7581 return; 7582 7583 if (filter->n_proto == ETH_P_IPV6) { 7584 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 7585 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); 7586 i++, j += 2) { 7587 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 7588 ipa = cpu_to_le32(ipa); 7589 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); 7590 } 7591 } else { 7592 ipa = be32_to_cpu(filter->dst_ipv4); 7593 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 7594 } 7595 7596 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 7597 7598 /* tenant_id is not supported by FW now, once the support is enabled 7599 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 7600 */ 7601 if (filter->tenant_id) 7602 return; 7603 } 7604 7605 /** 7606 * i40e_add_del_cloud_filter - Add/del cloud filter 7607 * @vsi: pointer to VSI 7608 * @filter: cloud filter rule 7609 * @add: if true, add, if false, delete 7610 * 7611 * Add or delete a cloud filter for a specific flow spec. 7612 * Returns 0 if the filter were successfully added. 7613 **/ 7614 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 7615 struct i40e_cloud_filter *filter, bool add) 7616 { 7617 struct i40e_aqc_cloud_filters_element_data cld_filter; 7618 struct i40e_pf *pf = vsi->back; 7619 int ret; 7620 static const u16 flag_table[128] = { 7621 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 7622 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 7623 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 7624 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 7625 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 7626 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 7627 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 7628 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 7629 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 7630 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 7631 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 7632 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 7633 [I40E_CLOUD_FILTER_FLAGS_IIP] = 7634 I40E_AQC_ADD_CLOUD_FILTER_IIP, 7635 }; 7636 7637 if (filter->flags >= ARRAY_SIZE(flag_table)) 7638 return I40E_ERR_CONFIG; 7639 7640 /* copy element needed to add cloud filter from filter */ 7641 i40e_set_cld_element(filter, &cld_filter); 7642 7643 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 7644 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 7645 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 7646 7647 if (filter->n_proto == ETH_P_IPV6) 7648 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7649 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7650 else 7651 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7652 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7653 7654 if (add) 7655 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 7656 &cld_filter, 1); 7657 else 7658 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 7659 &cld_filter, 1); 7660 if (ret) 7661 dev_dbg(&pf->pdev->dev, 7662 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 7663 add ? "add" : "delete", filter->dst_port, ret, 7664 pf->hw.aq.asq_last_status); 7665 else 7666 dev_info(&pf->pdev->dev, 7667 "%s cloud filter for VSI: %d\n", 7668 add ? "Added" : "Deleted", filter->seid); 7669 return ret; 7670 } 7671 7672 /** 7673 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 7674 * @vsi: pointer to VSI 7675 * @filter: cloud filter rule 7676 * @add: if true, add, if false, delete 7677 * 7678 * Add or delete a cloud filter for a specific flow spec using big buffer. 7679 * Returns 0 if the filter were successfully added. 7680 **/ 7681 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 7682 struct i40e_cloud_filter *filter, 7683 bool add) 7684 { 7685 struct i40e_aqc_cloud_filters_element_bb cld_filter; 7686 struct i40e_pf *pf = vsi->back; 7687 int ret; 7688 7689 /* Both (src/dst) valid mac_addr are not supported */ 7690 if ((is_valid_ether_addr(filter->dst_mac) && 7691 is_valid_ether_addr(filter->src_mac)) || 7692 (is_multicast_ether_addr(filter->dst_mac) && 7693 is_multicast_ether_addr(filter->src_mac))) 7694 return -EOPNOTSUPP; 7695 7696 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 7697 * ports are not supported via big buffer now. 7698 */ 7699 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 7700 return -EOPNOTSUPP; 7701 7702 /* adding filter using src_port/src_ip is not supported at this stage */ 7703 if (filter->src_port || filter->src_ipv4 || 7704 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 7705 return -EOPNOTSUPP; 7706 7707 /* copy element needed to add cloud filter from filter */ 7708 i40e_set_cld_element(filter, &cld_filter.element); 7709 7710 if (is_valid_ether_addr(filter->dst_mac) || 7711 is_valid_ether_addr(filter->src_mac) || 7712 is_multicast_ether_addr(filter->dst_mac) || 7713 is_multicast_ether_addr(filter->src_mac)) { 7714 /* MAC + IP : unsupported mode */ 7715 if (filter->dst_ipv4) 7716 return -EOPNOTSUPP; 7717 7718 /* since we validated that L4 port must be valid before 7719 * we get here, start with respective "flags" value 7720 * and update if vlan is present or not 7721 */ 7722 cld_filter.element.flags = 7723 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 7724 7725 if (filter->vlan_id) { 7726 cld_filter.element.flags = 7727 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 7728 } 7729 7730 } else if (filter->dst_ipv4 || 7731 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 7732 cld_filter.element.flags = 7733 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 7734 if (filter->n_proto == ETH_P_IPV6) 7735 cld_filter.element.flags |= 7736 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7737 else 7738 cld_filter.element.flags |= 7739 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7740 } else { 7741 dev_err(&pf->pdev->dev, 7742 "either mac or ip has to be valid for cloud filter\n"); 7743 return -EINVAL; 7744 } 7745 7746 /* Now copy L4 port in Byte 6..7 in general fields */ 7747 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 7748 be16_to_cpu(filter->dst_port); 7749 7750 if (add) { 7751 /* Validate current device switch mode, change if necessary */ 7752 ret = i40e_validate_and_set_switch_mode(vsi); 7753 if (ret) { 7754 dev_err(&pf->pdev->dev, 7755 "failed to set switch mode, ret %d\n", 7756 ret); 7757 return ret; 7758 } 7759 7760 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 7761 &cld_filter, 1); 7762 } else { 7763 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 7764 &cld_filter, 1); 7765 } 7766 7767 if (ret) 7768 dev_dbg(&pf->pdev->dev, 7769 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 7770 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 7771 else 7772 dev_info(&pf->pdev->dev, 7773 "%s cloud filter for VSI: %d, L4 port: %d\n", 7774 add ? "add" : "delete", filter->seid, 7775 ntohs(filter->dst_port)); 7776 return ret; 7777 } 7778 7779 /** 7780 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 7781 * @vsi: Pointer to VSI 7782 * @cls_flower: Pointer to struct flow_cls_offload 7783 * @filter: Pointer to cloud filter structure 7784 * 7785 **/ 7786 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 7787 struct flow_cls_offload *f, 7788 struct i40e_cloud_filter *filter) 7789 { 7790 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 7791 struct flow_dissector *dissector = rule->match.dissector; 7792 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 7793 struct i40e_pf *pf = vsi->back; 7794 u8 field_flags = 0; 7795 7796 if (dissector->used_keys & 7797 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 7798 BIT(FLOW_DISSECTOR_KEY_BASIC) | 7799 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 7800 BIT(FLOW_DISSECTOR_KEY_VLAN) | 7801 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 7802 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 7803 BIT(FLOW_DISSECTOR_KEY_PORTS) | 7804 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 7805 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 7806 dissector->used_keys); 7807 return -EOPNOTSUPP; 7808 } 7809 7810 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 7811 struct flow_match_enc_keyid match; 7812 7813 flow_rule_match_enc_keyid(rule, &match); 7814 if (match.mask->keyid != 0) 7815 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 7816 7817 filter->tenant_id = be32_to_cpu(match.key->keyid); 7818 } 7819 7820 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 7821 struct flow_match_basic match; 7822 7823 flow_rule_match_basic(rule, &match); 7824 n_proto_key = ntohs(match.key->n_proto); 7825 n_proto_mask = ntohs(match.mask->n_proto); 7826 7827 if (n_proto_key == ETH_P_ALL) { 7828 n_proto_key = 0; 7829 n_proto_mask = 0; 7830 } 7831 filter->n_proto = n_proto_key & n_proto_mask; 7832 filter->ip_proto = match.key->ip_proto; 7833 } 7834 7835 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 7836 struct flow_match_eth_addrs match; 7837 7838 flow_rule_match_eth_addrs(rule, &match); 7839 7840 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 7841 if (!is_zero_ether_addr(match.mask->dst)) { 7842 if (is_broadcast_ether_addr(match.mask->dst)) { 7843 field_flags |= I40E_CLOUD_FIELD_OMAC; 7844 } else { 7845 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 7846 match.mask->dst); 7847 return I40E_ERR_CONFIG; 7848 } 7849 } 7850 7851 if (!is_zero_ether_addr(match.mask->src)) { 7852 if (is_broadcast_ether_addr(match.mask->src)) { 7853 field_flags |= I40E_CLOUD_FIELD_IMAC; 7854 } else { 7855 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 7856 match.mask->src); 7857 return I40E_ERR_CONFIG; 7858 } 7859 } 7860 ether_addr_copy(filter->dst_mac, match.key->dst); 7861 ether_addr_copy(filter->src_mac, match.key->src); 7862 } 7863 7864 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 7865 struct flow_match_vlan match; 7866 7867 flow_rule_match_vlan(rule, &match); 7868 if (match.mask->vlan_id) { 7869 if (match.mask->vlan_id == VLAN_VID_MASK) { 7870 field_flags |= I40E_CLOUD_FIELD_IVLAN; 7871 7872 } else { 7873 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 7874 match.mask->vlan_id); 7875 return I40E_ERR_CONFIG; 7876 } 7877 } 7878 7879 filter->vlan_id = cpu_to_be16(match.key->vlan_id); 7880 } 7881 7882 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 7883 struct flow_match_control match; 7884 7885 flow_rule_match_control(rule, &match); 7886 addr_type = match.key->addr_type; 7887 } 7888 7889 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 7890 struct flow_match_ipv4_addrs match; 7891 7892 flow_rule_match_ipv4_addrs(rule, &match); 7893 if (match.mask->dst) { 7894 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 7895 field_flags |= I40E_CLOUD_FIELD_IIP; 7896 } else { 7897 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 7898 &match.mask->dst); 7899 return I40E_ERR_CONFIG; 7900 } 7901 } 7902 7903 if (match.mask->src) { 7904 if (match.mask->src == cpu_to_be32(0xffffffff)) { 7905 field_flags |= I40E_CLOUD_FIELD_IIP; 7906 } else { 7907 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 7908 &match.mask->src); 7909 return I40E_ERR_CONFIG; 7910 } 7911 } 7912 7913 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 7914 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 7915 return I40E_ERR_CONFIG; 7916 } 7917 filter->dst_ipv4 = match.key->dst; 7918 filter->src_ipv4 = match.key->src; 7919 } 7920 7921 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 7922 struct flow_match_ipv6_addrs match; 7923 7924 flow_rule_match_ipv6_addrs(rule, &match); 7925 7926 /* src and dest IPV6 address should not be LOOPBACK 7927 * (0:0:0:0:0:0:0:1), which can be represented as ::1 7928 */ 7929 if (ipv6_addr_loopback(&match.key->dst) || 7930 ipv6_addr_loopback(&match.key->src)) { 7931 dev_err(&pf->pdev->dev, 7932 "Bad ipv6, addr is LOOPBACK\n"); 7933 return I40E_ERR_CONFIG; 7934 } 7935 if (!ipv6_addr_any(&match.mask->dst) || 7936 !ipv6_addr_any(&match.mask->src)) 7937 field_flags |= I40E_CLOUD_FIELD_IIP; 7938 7939 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32, 7940 sizeof(filter->src_ipv6)); 7941 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32, 7942 sizeof(filter->dst_ipv6)); 7943 } 7944 7945 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 7946 struct flow_match_ports match; 7947 7948 flow_rule_match_ports(rule, &match); 7949 if (match.mask->src) { 7950 if (match.mask->src == cpu_to_be16(0xffff)) { 7951 field_flags |= I40E_CLOUD_FIELD_IIP; 7952 } else { 7953 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 7954 be16_to_cpu(match.mask->src)); 7955 return I40E_ERR_CONFIG; 7956 } 7957 } 7958 7959 if (match.mask->dst) { 7960 if (match.mask->dst == cpu_to_be16(0xffff)) { 7961 field_flags |= I40E_CLOUD_FIELD_IIP; 7962 } else { 7963 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 7964 be16_to_cpu(match.mask->dst)); 7965 return I40E_ERR_CONFIG; 7966 } 7967 } 7968 7969 filter->dst_port = match.key->dst; 7970 filter->src_port = match.key->src; 7971 7972 switch (filter->ip_proto) { 7973 case IPPROTO_TCP: 7974 case IPPROTO_UDP: 7975 break; 7976 default: 7977 dev_err(&pf->pdev->dev, 7978 "Only UDP and TCP transport are supported\n"); 7979 return -EINVAL; 7980 } 7981 } 7982 filter->flags = field_flags; 7983 return 0; 7984 } 7985 7986 /** 7987 * i40e_handle_tclass: Forward to a traffic class on the device 7988 * @vsi: Pointer to VSI 7989 * @tc: traffic class index on the device 7990 * @filter: Pointer to cloud filter structure 7991 * 7992 **/ 7993 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 7994 struct i40e_cloud_filter *filter) 7995 { 7996 struct i40e_channel *ch, *ch_tmp; 7997 7998 /* direct to a traffic class on the same device */ 7999 if (tc == 0) { 8000 filter->seid = vsi->seid; 8001 return 0; 8002 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 8003 if (!filter->dst_port) { 8004 dev_err(&vsi->back->pdev->dev, 8005 "Specify destination port to direct to traffic class that is not default\n"); 8006 return -EINVAL; 8007 } 8008 if (list_empty(&vsi->ch_list)) 8009 return -EINVAL; 8010 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 8011 list) { 8012 if (ch->seid == vsi->tc_seid_map[tc]) 8013 filter->seid = ch->seid; 8014 } 8015 return 0; 8016 } 8017 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 8018 return -EINVAL; 8019 } 8020 8021 /** 8022 * i40e_configure_clsflower - Configure tc flower filters 8023 * @vsi: Pointer to VSI 8024 * @cls_flower: Pointer to struct flow_cls_offload 8025 * 8026 **/ 8027 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 8028 struct flow_cls_offload *cls_flower) 8029 { 8030 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 8031 struct i40e_cloud_filter *filter = NULL; 8032 struct i40e_pf *pf = vsi->back; 8033 int err = 0; 8034 8035 if (tc < 0) { 8036 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 8037 return -EOPNOTSUPP; 8038 } 8039 8040 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 8041 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 8042 return -EBUSY; 8043 8044 if (pf->fdir_pf_active_filters || 8045 (!hlist_empty(&pf->fdir_filter_list))) { 8046 dev_err(&vsi->back->pdev->dev, 8047 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 8048 return -EINVAL; 8049 } 8050 8051 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 8052 dev_err(&vsi->back->pdev->dev, 8053 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 8054 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8055 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8056 } 8057 8058 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 8059 if (!filter) 8060 return -ENOMEM; 8061 8062 filter->cookie = cls_flower->cookie; 8063 8064 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 8065 if (err < 0) 8066 goto err; 8067 8068 err = i40e_handle_tclass(vsi, tc, filter); 8069 if (err < 0) 8070 goto err; 8071 8072 /* Add cloud filter */ 8073 if (filter->dst_port) 8074 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 8075 else 8076 err = i40e_add_del_cloud_filter(vsi, filter, true); 8077 8078 if (err) { 8079 dev_err(&pf->pdev->dev, 8080 "Failed to add cloud filter, err %s\n", 8081 i40e_stat_str(&pf->hw, err)); 8082 goto err; 8083 } 8084 8085 /* add filter to the ordered list */ 8086 INIT_HLIST_NODE(&filter->cloud_node); 8087 8088 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 8089 8090 pf->num_cloud_filters++; 8091 8092 return err; 8093 err: 8094 kfree(filter); 8095 return err; 8096 } 8097 8098 /** 8099 * i40e_find_cloud_filter - Find the could filter in the list 8100 * @vsi: Pointer to VSI 8101 * @cookie: filter specific cookie 8102 * 8103 **/ 8104 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 8105 unsigned long *cookie) 8106 { 8107 struct i40e_cloud_filter *filter = NULL; 8108 struct hlist_node *node2; 8109 8110 hlist_for_each_entry_safe(filter, node2, 8111 &vsi->back->cloud_filter_list, cloud_node) 8112 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 8113 return filter; 8114 return NULL; 8115 } 8116 8117 /** 8118 * i40e_delete_clsflower - Remove tc flower filters 8119 * @vsi: Pointer to VSI 8120 * @cls_flower: Pointer to struct flow_cls_offload 8121 * 8122 **/ 8123 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 8124 struct flow_cls_offload *cls_flower) 8125 { 8126 struct i40e_cloud_filter *filter = NULL; 8127 struct i40e_pf *pf = vsi->back; 8128 int err = 0; 8129 8130 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 8131 8132 if (!filter) 8133 return -EINVAL; 8134 8135 hash_del(&filter->cloud_node); 8136 8137 if (filter->dst_port) 8138 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 8139 else 8140 err = i40e_add_del_cloud_filter(vsi, filter, false); 8141 8142 kfree(filter); 8143 if (err) { 8144 dev_err(&pf->pdev->dev, 8145 "Failed to delete cloud filter, err %s\n", 8146 i40e_stat_str(&pf->hw, err)); 8147 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 8148 } 8149 8150 pf->num_cloud_filters--; 8151 if (!pf->num_cloud_filters) 8152 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8153 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8154 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8155 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8156 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8157 } 8158 return 0; 8159 } 8160 8161 /** 8162 * i40e_setup_tc_cls_flower - flower classifier offloads 8163 * @netdev: net device to configure 8164 * @type_data: offload data 8165 **/ 8166 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 8167 struct flow_cls_offload *cls_flower) 8168 { 8169 struct i40e_vsi *vsi = np->vsi; 8170 8171 switch (cls_flower->command) { 8172 case FLOW_CLS_REPLACE: 8173 return i40e_configure_clsflower(vsi, cls_flower); 8174 case FLOW_CLS_DESTROY: 8175 return i40e_delete_clsflower(vsi, cls_flower); 8176 case FLOW_CLS_STATS: 8177 return -EOPNOTSUPP; 8178 default: 8179 return -EOPNOTSUPP; 8180 } 8181 } 8182 8183 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8184 void *cb_priv) 8185 { 8186 struct i40e_netdev_priv *np = cb_priv; 8187 8188 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 8189 return -EOPNOTSUPP; 8190 8191 switch (type) { 8192 case TC_SETUP_CLSFLOWER: 8193 return i40e_setup_tc_cls_flower(np, type_data); 8194 8195 default: 8196 return -EOPNOTSUPP; 8197 } 8198 } 8199 8200 static LIST_HEAD(i40e_block_cb_list); 8201 8202 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8203 void *type_data) 8204 { 8205 struct i40e_netdev_priv *np = netdev_priv(netdev); 8206 8207 switch (type) { 8208 case TC_SETUP_QDISC_MQPRIO: 8209 return i40e_setup_tc(netdev, type_data); 8210 case TC_SETUP_BLOCK: 8211 return flow_block_cb_setup_simple(type_data, 8212 &i40e_block_cb_list, 8213 i40e_setup_tc_block_cb, 8214 np, np, true); 8215 default: 8216 return -EOPNOTSUPP; 8217 } 8218 } 8219 8220 /** 8221 * i40e_open - Called when a network interface is made active 8222 * @netdev: network interface device structure 8223 * 8224 * The open entry point is called when a network interface is made 8225 * active by the system (IFF_UP). At this point all resources needed 8226 * for transmit and receive operations are allocated, the interrupt 8227 * handler is registered with the OS, the netdev watchdog subtask is 8228 * enabled, and the stack is notified that the interface is ready. 8229 * 8230 * Returns 0 on success, negative value on failure 8231 **/ 8232 int i40e_open(struct net_device *netdev) 8233 { 8234 struct i40e_netdev_priv *np = netdev_priv(netdev); 8235 struct i40e_vsi *vsi = np->vsi; 8236 struct i40e_pf *pf = vsi->back; 8237 int err; 8238 8239 /* disallow open during test or if eeprom is broken */ 8240 if (test_bit(__I40E_TESTING, pf->state) || 8241 test_bit(__I40E_BAD_EEPROM, pf->state)) 8242 return -EBUSY; 8243 8244 netif_carrier_off(netdev); 8245 8246 if (i40e_force_link_state(pf, true)) 8247 return -EAGAIN; 8248 8249 err = i40e_vsi_open(vsi); 8250 if (err) 8251 return err; 8252 8253 /* configure global TSO hardware offload settings */ 8254 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 8255 TCP_FLAG_FIN) >> 16); 8256 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 8257 TCP_FLAG_FIN | 8258 TCP_FLAG_CWR) >> 16); 8259 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 8260 8261 udp_tunnel_get_rx_info(netdev); 8262 8263 return 0; 8264 } 8265 8266 /** 8267 * i40e_vsi_open - 8268 * @vsi: the VSI to open 8269 * 8270 * Finish initialization of the VSI. 8271 * 8272 * Returns 0 on success, negative value on failure 8273 * 8274 * Note: expects to be called while under rtnl_lock() 8275 **/ 8276 int i40e_vsi_open(struct i40e_vsi *vsi) 8277 { 8278 struct i40e_pf *pf = vsi->back; 8279 char int_name[I40E_INT_NAME_STR_LEN]; 8280 int err; 8281 8282 /* allocate descriptors */ 8283 err = i40e_vsi_setup_tx_resources(vsi); 8284 if (err) 8285 goto err_setup_tx; 8286 err = i40e_vsi_setup_rx_resources(vsi); 8287 if (err) 8288 goto err_setup_rx; 8289 8290 err = i40e_vsi_configure(vsi); 8291 if (err) 8292 goto err_setup_rx; 8293 8294 if (vsi->netdev) { 8295 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 8296 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 8297 err = i40e_vsi_request_irq(vsi, int_name); 8298 if (err) 8299 goto err_setup_rx; 8300 8301 /* Notify the stack of the actual queue counts. */ 8302 err = netif_set_real_num_tx_queues(vsi->netdev, 8303 vsi->num_queue_pairs); 8304 if (err) 8305 goto err_set_queues; 8306 8307 err = netif_set_real_num_rx_queues(vsi->netdev, 8308 vsi->num_queue_pairs); 8309 if (err) 8310 goto err_set_queues; 8311 8312 } else if (vsi->type == I40E_VSI_FDIR) { 8313 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 8314 dev_driver_string(&pf->pdev->dev), 8315 dev_name(&pf->pdev->dev)); 8316 err = i40e_vsi_request_irq(vsi, int_name); 8317 8318 } else { 8319 err = -EINVAL; 8320 goto err_setup_rx; 8321 } 8322 8323 err = i40e_up_complete(vsi); 8324 if (err) 8325 goto err_up_complete; 8326 8327 return 0; 8328 8329 err_up_complete: 8330 i40e_down(vsi); 8331 err_set_queues: 8332 i40e_vsi_free_irq(vsi); 8333 err_setup_rx: 8334 i40e_vsi_free_rx_resources(vsi); 8335 err_setup_tx: 8336 i40e_vsi_free_tx_resources(vsi); 8337 if (vsi == pf->vsi[pf->lan_vsi]) 8338 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 8339 8340 return err; 8341 } 8342 8343 /** 8344 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 8345 * @pf: Pointer to PF 8346 * 8347 * This function destroys the hlist where all the Flow Director 8348 * filters were saved. 8349 **/ 8350 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 8351 { 8352 struct i40e_fdir_filter *filter; 8353 struct i40e_flex_pit *pit_entry, *tmp; 8354 struct hlist_node *node2; 8355 8356 hlist_for_each_entry_safe(filter, node2, 8357 &pf->fdir_filter_list, fdir_node) { 8358 hlist_del(&filter->fdir_node); 8359 kfree(filter); 8360 } 8361 8362 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 8363 list_del(&pit_entry->list); 8364 kfree(pit_entry); 8365 } 8366 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 8367 8368 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 8369 list_del(&pit_entry->list); 8370 kfree(pit_entry); 8371 } 8372 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 8373 8374 pf->fdir_pf_active_filters = 0; 8375 pf->fd_tcp4_filter_cnt = 0; 8376 pf->fd_udp4_filter_cnt = 0; 8377 pf->fd_sctp4_filter_cnt = 0; 8378 pf->fd_ip4_filter_cnt = 0; 8379 8380 /* Reprogram the default input set for TCP/IPv4 */ 8381 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8382 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8383 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8384 8385 /* Reprogram the default input set for UDP/IPv4 */ 8386 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 8387 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8388 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8389 8390 /* Reprogram the default input set for SCTP/IPv4 */ 8391 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 8392 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8393 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8394 8395 /* Reprogram the default input set for Other/IPv4 */ 8396 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 8397 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 8398 8399 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 8400 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 8401 } 8402 8403 /** 8404 * i40e_cloud_filter_exit - Cleans up the cloud filters 8405 * @pf: Pointer to PF 8406 * 8407 * This function destroys the hlist where all the cloud filters 8408 * were saved. 8409 **/ 8410 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 8411 { 8412 struct i40e_cloud_filter *cfilter; 8413 struct hlist_node *node; 8414 8415 hlist_for_each_entry_safe(cfilter, node, 8416 &pf->cloud_filter_list, cloud_node) { 8417 hlist_del(&cfilter->cloud_node); 8418 kfree(cfilter); 8419 } 8420 pf->num_cloud_filters = 0; 8421 8422 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8423 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8424 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8425 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8426 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8427 } 8428 } 8429 8430 /** 8431 * i40e_close - Disables a network interface 8432 * @netdev: network interface device structure 8433 * 8434 * The close entry point is called when an interface is de-activated 8435 * by the OS. The hardware is still under the driver's control, but 8436 * this netdev interface is disabled. 8437 * 8438 * Returns 0, this is not allowed to fail 8439 **/ 8440 int i40e_close(struct net_device *netdev) 8441 { 8442 struct i40e_netdev_priv *np = netdev_priv(netdev); 8443 struct i40e_vsi *vsi = np->vsi; 8444 8445 i40e_vsi_close(vsi); 8446 8447 return 0; 8448 } 8449 8450 /** 8451 * i40e_do_reset - Start a PF or Core Reset sequence 8452 * @pf: board private structure 8453 * @reset_flags: which reset is requested 8454 * @lock_acquired: indicates whether or not the lock has been acquired 8455 * before this function was called. 8456 * 8457 * The essential difference in resets is that the PF Reset 8458 * doesn't clear the packet buffers, doesn't reset the PE 8459 * firmware, and doesn't bother the other PFs on the chip. 8460 **/ 8461 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 8462 { 8463 u32 val; 8464 8465 WARN_ON(in_interrupt()); 8466 8467 8468 /* do the biggest reset indicated */ 8469 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 8470 8471 /* Request a Global Reset 8472 * 8473 * This will start the chip's countdown to the actual full 8474 * chip reset event, and a warning interrupt to be sent 8475 * to all PFs, including the requestor. Our handler 8476 * for the warning interrupt will deal with the shutdown 8477 * and recovery of the switch setup. 8478 */ 8479 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 8480 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8481 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 8482 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 8483 8484 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 8485 8486 /* Request a Core Reset 8487 * 8488 * Same as Global Reset, except does *not* include the MAC/PHY 8489 */ 8490 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 8491 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8492 val |= I40E_GLGEN_RTRIG_CORER_MASK; 8493 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 8494 i40e_flush(&pf->hw); 8495 8496 } else if (reset_flags & I40E_PF_RESET_FLAG) { 8497 8498 /* Request a PF Reset 8499 * 8500 * Resets only the PF-specific registers 8501 * 8502 * This goes directly to the tear-down and rebuild of 8503 * the switch, since we need to do all the recovery as 8504 * for the Core Reset. 8505 */ 8506 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 8507 i40e_handle_reset_warning(pf, lock_acquired); 8508 8509 dev_info(&pf->pdev->dev, 8510 pf->flags & I40E_FLAG_DISABLE_FW_LLDP ? 8511 "FW LLDP is disabled\n" : 8512 "FW LLDP is enabled\n"); 8513 8514 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 8515 int v; 8516 8517 /* Find the VSI(s) that requested a re-init */ 8518 dev_info(&pf->pdev->dev, 8519 "VSI reinit requested\n"); 8520 for (v = 0; v < pf->num_alloc_vsi; v++) { 8521 struct i40e_vsi *vsi = pf->vsi[v]; 8522 8523 if (vsi != NULL && 8524 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 8525 vsi->state)) 8526 i40e_vsi_reinit_locked(pf->vsi[v]); 8527 } 8528 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 8529 int v; 8530 8531 /* Find the VSI(s) that needs to be brought down */ 8532 dev_info(&pf->pdev->dev, "VSI down requested\n"); 8533 for (v = 0; v < pf->num_alloc_vsi; v++) { 8534 struct i40e_vsi *vsi = pf->vsi[v]; 8535 8536 if (vsi != NULL && 8537 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 8538 vsi->state)) { 8539 set_bit(__I40E_VSI_DOWN, vsi->state); 8540 i40e_down(vsi); 8541 } 8542 } 8543 } else { 8544 dev_info(&pf->pdev->dev, 8545 "bad reset request 0x%08x\n", reset_flags); 8546 } 8547 } 8548 8549 #ifdef CONFIG_I40E_DCB 8550 /** 8551 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 8552 * @pf: board private structure 8553 * @old_cfg: current DCB config 8554 * @new_cfg: new DCB config 8555 **/ 8556 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 8557 struct i40e_dcbx_config *old_cfg, 8558 struct i40e_dcbx_config *new_cfg) 8559 { 8560 bool need_reconfig = false; 8561 8562 /* Check if ETS configuration has changed */ 8563 if (memcmp(&new_cfg->etscfg, 8564 &old_cfg->etscfg, 8565 sizeof(new_cfg->etscfg))) { 8566 /* If Priority Table has changed reconfig is needed */ 8567 if (memcmp(&new_cfg->etscfg.prioritytable, 8568 &old_cfg->etscfg.prioritytable, 8569 sizeof(new_cfg->etscfg.prioritytable))) { 8570 need_reconfig = true; 8571 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 8572 } 8573 8574 if (memcmp(&new_cfg->etscfg.tcbwtable, 8575 &old_cfg->etscfg.tcbwtable, 8576 sizeof(new_cfg->etscfg.tcbwtable))) 8577 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 8578 8579 if (memcmp(&new_cfg->etscfg.tsatable, 8580 &old_cfg->etscfg.tsatable, 8581 sizeof(new_cfg->etscfg.tsatable))) 8582 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 8583 } 8584 8585 /* Check if PFC configuration has changed */ 8586 if (memcmp(&new_cfg->pfc, 8587 &old_cfg->pfc, 8588 sizeof(new_cfg->pfc))) { 8589 need_reconfig = true; 8590 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 8591 } 8592 8593 /* Check if APP Table has changed */ 8594 if (memcmp(&new_cfg->app, 8595 &old_cfg->app, 8596 sizeof(new_cfg->app))) { 8597 need_reconfig = true; 8598 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 8599 } 8600 8601 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 8602 return need_reconfig; 8603 } 8604 8605 /** 8606 * i40e_handle_lldp_event - Handle LLDP Change MIB event 8607 * @pf: board private structure 8608 * @e: event info posted on ARQ 8609 **/ 8610 static int i40e_handle_lldp_event(struct i40e_pf *pf, 8611 struct i40e_arq_event_info *e) 8612 { 8613 struct i40e_aqc_lldp_get_mib *mib = 8614 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 8615 struct i40e_hw *hw = &pf->hw; 8616 struct i40e_dcbx_config tmp_dcbx_cfg; 8617 bool need_reconfig = false; 8618 int ret = 0; 8619 u8 type; 8620 8621 /* Not DCB capable or capability disabled */ 8622 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 8623 return ret; 8624 8625 /* Ignore if event is not for Nearest Bridge */ 8626 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 8627 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 8628 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 8629 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 8630 return ret; 8631 8632 /* Check MIB Type and return if event for Remote MIB update */ 8633 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 8634 dev_dbg(&pf->pdev->dev, 8635 "LLDP event mib type %s\n", type ? "remote" : "local"); 8636 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 8637 /* Update the remote cached instance and return */ 8638 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 8639 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 8640 &hw->remote_dcbx_config); 8641 goto exit; 8642 } 8643 8644 /* Store the old configuration */ 8645 tmp_dcbx_cfg = hw->local_dcbx_config; 8646 8647 /* Reset the old DCBx configuration data */ 8648 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 8649 /* Get updated DCBX data from firmware */ 8650 ret = i40e_get_dcb_config(&pf->hw); 8651 if (ret) { 8652 dev_info(&pf->pdev->dev, 8653 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 8654 i40e_stat_str(&pf->hw, ret), 8655 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8656 goto exit; 8657 } 8658 8659 /* No change detected in DCBX configs */ 8660 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 8661 sizeof(tmp_dcbx_cfg))) { 8662 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 8663 goto exit; 8664 } 8665 8666 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 8667 &hw->local_dcbx_config); 8668 8669 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 8670 8671 if (!need_reconfig) 8672 goto exit; 8673 8674 /* Enable DCB tagging only when more than one TC */ 8675 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 8676 pf->flags |= I40E_FLAG_DCB_ENABLED; 8677 else 8678 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 8679 8680 set_bit(__I40E_PORT_SUSPENDED, pf->state); 8681 /* Reconfiguration needed quiesce all VSIs */ 8682 i40e_pf_quiesce_all_vsi(pf); 8683 8684 /* Changes in configuration update VEB/VSI */ 8685 i40e_dcb_reconfigure(pf); 8686 8687 ret = i40e_resume_port_tx(pf); 8688 8689 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 8690 /* In case of error no point in resuming VSIs */ 8691 if (ret) 8692 goto exit; 8693 8694 /* Wait for the PF's queues to be disabled */ 8695 ret = i40e_pf_wait_queues_disabled(pf); 8696 if (ret) { 8697 /* Schedule PF reset to recover */ 8698 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8699 i40e_service_event_schedule(pf); 8700 } else { 8701 i40e_pf_unquiesce_all_vsi(pf); 8702 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 8703 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 8704 } 8705 8706 exit: 8707 return ret; 8708 } 8709 #endif /* CONFIG_I40E_DCB */ 8710 8711 /** 8712 * i40e_do_reset_safe - Protected reset path for userland calls. 8713 * @pf: board private structure 8714 * @reset_flags: which reset is requested 8715 * 8716 **/ 8717 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 8718 { 8719 rtnl_lock(); 8720 i40e_do_reset(pf, reset_flags, true); 8721 rtnl_unlock(); 8722 } 8723 8724 /** 8725 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 8726 * @pf: board private structure 8727 * @e: event info posted on ARQ 8728 * 8729 * Handler for LAN Queue Overflow Event generated by the firmware for PF 8730 * and VF queues 8731 **/ 8732 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 8733 struct i40e_arq_event_info *e) 8734 { 8735 struct i40e_aqc_lan_overflow *data = 8736 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 8737 u32 queue = le32_to_cpu(data->prtdcb_rupto); 8738 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 8739 struct i40e_hw *hw = &pf->hw; 8740 struct i40e_vf *vf; 8741 u16 vf_id; 8742 8743 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 8744 queue, qtx_ctl); 8745 8746 /* Queue belongs to VF, find the VF and issue VF reset */ 8747 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 8748 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 8749 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 8750 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 8751 vf_id -= hw->func_caps.vf_base_id; 8752 vf = &pf->vf[vf_id]; 8753 i40e_vc_notify_vf_reset(vf); 8754 /* Allow VF to process pending reset notification */ 8755 msleep(20); 8756 i40e_reset_vf(vf, false); 8757 } 8758 } 8759 8760 /** 8761 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 8762 * @pf: board private structure 8763 **/ 8764 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 8765 { 8766 u32 val, fcnt_prog; 8767 8768 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8769 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 8770 return fcnt_prog; 8771 } 8772 8773 /** 8774 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 8775 * @pf: board private structure 8776 **/ 8777 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 8778 { 8779 u32 val, fcnt_prog; 8780 8781 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8782 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 8783 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 8784 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 8785 return fcnt_prog; 8786 } 8787 8788 /** 8789 * i40e_get_global_fd_count - Get total FD filters programmed on device 8790 * @pf: board private structure 8791 **/ 8792 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 8793 { 8794 u32 val, fcnt_prog; 8795 8796 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 8797 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 8798 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 8799 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 8800 return fcnt_prog; 8801 } 8802 8803 /** 8804 * i40e_reenable_fdir_sb - Restore FDir SB capability 8805 * @pf: board private structure 8806 **/ 8807 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 8808 { 8809 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 8810 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8811 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8812 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 8813 } 8814 8815 /** 8816 * i40e_reenable_fdir_atr - Restore FDir ATR capability 8817 * @pf: board private structure 8818 **/ 8819 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 8820 { 8821 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 8822 /* ATR uses the same filtering logic as SB rules. It only 8823 * functions properly if the input set mask is at the default 8824 * settings. It is safe to restore the default input set 8825 * because there are no active TCPv4 filter rules. 8826 */ 8827 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8828 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8829 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8830 8831 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8832 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8833 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 8834 } 8835 } 8836 8837 /** 8838 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 8839 * @pf: board private structure 8840 * @filter: FDir filter to remove 8841 */ 8842 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 8843 struct i40e_fdir_filter *filter) 8844 { 8845 /* Update counters */ 8846 pf->fdir_pf_active_filters--; 8847 pf->fd_inv = 0; 8848 8849 switch (filter->flow_type) { 8850 case TCP_V4_FLOW: 8851 pf->fd_tcp4_filter_cnt--; 8852 break; 8853 case UDP_V4_FLOW: 8854 pf->fd_udp4_filter_cnt--; 8855 break; 8856 case SCTP_V4_FLOW: 8857 pf->fd_sctp4_filter_cnt--; 8858 break; 8859 case IP_USER_FLOW: 8860 switch (filter->ip4_proto) { 8861 case IPPROTO_TCP: 8862 pf->fd_tcp4_filter_cnt--; 8863 break; 8864 case IPPROTO_UDP: 8865 pf->fd_udp4_filter_cnt--; 8866 break; 8867 case IPPROTO_SCTP: 8868 pf->fd_sctp4_filter_cnt--; 8869 break; 8870 case IPPROTO_IP: 8871 pf->fd_ip4_filter_cnt--; 8872 break; 8873 } 8874 break; 8875 } 8876 8877 /* Remove the filter from the list and free memory */ 8878 hlist_del(&filter->fdir_node); 8879 kfree(filter); 8880 } 8881 8882 /** 8883 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 8884 * @pf: board private structure 8885 **/ 8886 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 8887 { 8888 struct i40e_fdir_filter *filter; 8889 u32 fcnt_prog, fcnt_avail; 8890 struct hlist_node *node; 8891 8892 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8893 return; 8894 8895 /* Check if we have enough room to re-enable FDir SB capability. */ 8896 fcnt_prog = i40e_get_global_fd_count(pf); 8897 fcnt_avail = pf->fdir_pf_filter_count; 8898 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 8899 (pf->fd_add_err == 0) || 8900 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 8901 i40e_reenable_fdir_sb(pf); 8902 8903 /* We should wait for even more space before re-enabling ATR. 8904 * Additionally, we cannot enable ATR as long as we still have TCP SB 8905 * rules active. 8906 */ 8907 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 8908 (pf->fd_tcp4_filter_cnt == 0)) 8909 i40e_reenable_fdir_atr(pf); 8910 8911 /* if hw had a problem adding a filter, delete it */ 8912 if (pf->fd_inv > 0) { 8913 hlist_for_each_entry_safe(filter, node, 8914 &pf->fdir_filter_list, fdir_node) 8915 if (filter->fd_id == pf->fd_inv) 8916 i40e_delete_invalid_filter(pf, filter); 8917 } 8918 } 8919 8920 #define I40E_MIN_FD_FLUSH_INTERVAL 10 8921 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 8922 /** 8923 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 8924 * @pf: board private structure 8925 **/ 8926 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 8927 { 8928 unsigned long min_flush_time; 8929 int flush_wait_retry = 50; 8930 bool disable_atr = false; 8931 int fd_room; 8932 int reg; 8933 8934 if (!time_after(jiffies, pf->fd_flush_timestamp + 8935 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 8936 return; 8937 8938 /* If the flush is happening too quick and we have mostly SB rules we 8939 * should not re-enable ATR for some time. 8940 */ 8941 min_flush_time = pf->fd_flush_timestamp + 8942 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 8943 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 8944 8945 if (!(time_after(jiffies, min_flush_time)) && 8946 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 8947 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8948 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 8949 disable_atr = true; 8950 } 8951 8952 pf->fd_flush_timestamp = jiffies; 8953 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8954 /* flush all filters */ 8955 wr32(&pf->hw, I40E_PFQF_CTL_1, 8956 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 8957 i40e_flush(&pf->hw); 8958 pf->fd_flush_cnt++; 8959 pf->fd_add_err = 0; 8960 do { 8961 /* Check FD flush status every 5-6msec */ 8962 usleep_range(5000, 6000); 8963 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 8964 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 8965 break; 8966 } while (flush_wait_retry--); 8967 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 8968 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 8969 } else { 8970 /* replay sideband filters */ 8971 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 8972 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 8973 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8974 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 8975 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8976 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 8977 } 8978 } 8979 8980 /** 8981 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 8982 * @pf: board private structure 8983 **/ 8984 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 8985 { 8986 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 8987 } 8988 8989 /** 8990 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 8991 * @pf: board private structure 8992 **/ 8993 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 8994 { 8995 8996 /* if interface is down do nothing */ 8997 if (test_bit(__I40E_DOWN, pf->state)) 8998 return; 8999 9000 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9001 i40e_fdir_flush_and_replay(pf); 9002 9003 i40e_fdir_check_and_reenable(pf); 9004 9005 } 9006 9007 /** 9008 * i40e_vsi_link_event - notify VSI of a link event 9009 * @vsi: vsi to be notified 9010 * @link_up: link up or down 9011 **/ 9012 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 9013 { 9014 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 9015 return; 9016 9017 switch (vsi->type) { 9018 case I40E_VSI_MAIN: 9019 if (!vsi->netdev || !vsi->netdev_registered) 9020 break; 9021 9022 if (link_up) { 9023 netif_carrier_on(vsi->netdev); 9024 netif_tx_wake_all_queues(vsi->netdev); 9025 } else { 9026 netif_carrier_off(vsi->netdev); 9027 netif_tx_stop_all_queues(vsi->netdev); 9028 } 9029 break; 9030 9031 case I40E_VSI_SRIOV: 9032 case I40E_VSI_VMDQ2: 9033 case I40E_VSI_CTRL: 9034 case I40E_VSI_IWARP: 9035 case I40E_VSI_MIRROR: 9036 default: 9037 /* there is no notification for other VSIs */ 9038 break; 9039 } 9040 } 9041 9042 /** 9043 * i40e_veb_link_event - notify elements on the veb of a link event 9044 * @veb: veb to be notified 9045 * @link_up: link up or down 9046 **/ 9047 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 9048 { 9049 struct i40e_pf *pf; 9050 int i; 9051 9052 if (!veb || !veb->pf) 9053 return; 9054 pf = veb->pf; 9055 9056 /* depth first... */ 9057 for (i = 0; i < I40E_MAX_VEB; i++) 9058 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 9059 i40e_veb_link_event(pf->veb[i], link_up); 9060 9061 /* ... now the local VSIs */ 9062 for (i = 0; i < pf->num_alloc_vsi; i++) 9063 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 9064 i40e_vsi_link_event(pf->vsi[i], link_up); 9065 } 9066 9067 /** 9068 * i40e_link_event - Update netif_carrier status 9069 * @pf: board private structure 9070 **/ 9071 static void i40e_link_event(struct i40e_pf *pf) 9072 { 9073 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9074 u8 new_link_speed, old_link_speed; 9075 i40e_status status; 9076 bool new_link, old_link; 9077 9078 /* set this to force the get_link_status call to refresh state */ 9079 pf->hw.phy.get_link_info = true; 9080 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 9081 status = i40e_get_link_status(&pf->hw, &new_link); 9082 9083 /* On success, disable temp link polling */ 9084 if (status == I40E_SUCCESS) { 9085 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9086 } else { 9087 /* Enable link polling temporarily until i40e_get_link_status 9088 * returns I40E_SUCCESS 9089 */ 9090 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9091 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 9092 status); 9093 return; 9094 } 9095 9096 old_link_speed = pf->hw.phy.link_info_old.link_speed; 9097 new_link_speed = pf->hw.phy.link_info.link_speed; 9098 9099 if (new_link == old_link && 9100 new_link_speed == old_link_speed && 9101 (test_bit(__I40E_VSI_DOWN, vsi->state) || 9102 new_link == netif_carrier_ok(vsi->netdev))) 9103 return; 9104 9105 i40e_print_link_message(vsi, new_link); 9106 9107 /* Notify the base of the switch tree connected to 9108 * the link. Floating VEBs are not notified. 9109 */ 9110 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 9111 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 9112 else 9113 i40e_vsi_link_event(vsi, new_link); 9114 9115 if (pf->vf) 9116 i40e_vc_notify_link_state(pf); 9117 9118 if (pf->flags & I40E_FLAG_PTP) 9119 i40e_ptp_set_increment(pf); 9120 } 9121 9122 /** 9123 * i40e_watchdog_subtask - periodic checks not using event driven response 9124 * @pf: board private structure 9125 **/ 9126 static void i40e_watchdog_subtask(struct i40e_pf *pf) 9127 { 9128 int i; 9129 9130 /* if interface is down do nothing */ 9131 if (test_bit(__I40E_DOWN, pf->state) || 9132 test_bit(__I40E_CONFIG_BUSY, pf->state)) 9133 return; 9134 9135 /* make sure we don't do these things too often */ 9136 if (time_before(jiffies, (pf->service_timer_previous + 9137 pf->service_timer_period))) 9138 return; 9139 pf->service_timer_previous = jiffies; 9140 9141 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 9142 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 9143 i40e_link_event(pf); 9144 9145 /* Update the stats for active netdevs so the network stack 9146 * can look at updated numbers whenever it cares to 9147 */ 9148 for (i = 0; i < pf->num_alloc_vsi; i++) 9149 if (pf->vsi[i] && pf->vsi[i]->netdev) 9150 i40e_update_stats(pf->vsi[i]); 9151 9152 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 9153 /* Update the stats for the active switching components */ 9154 for (i = 0; i < I40E_MAX_VEB; i++) 9155 if (pf->veb[i]) 9156 i40e_update_veb_stats(pf->veb[i]); 9157 } 9158 9159 i40e_ptp_rx_hang(pf); 9160 i40e_ptp_tx_hang(pf); 9161 } 9162 9163 /** 9164 * i40e_reset_subtask - Set up for resetting the device and driver 9165 * @pf: board private structure 9166 **/ 9167 static void i40e_reset_subtask(struct i40e_pf *pf) 9168 { 9169 u32 reset_flags = 0; 9170 9171 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 9172 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 9173 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 9174 } 9175 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 9176 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 9177 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9178 } 9179 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 9180 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 9181 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 9182 } 9183 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 9184 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 9185 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 9186 } 9187 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 9188 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 9189 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 9190 } 9191 9192 /* If there's a recovery already waiting, it takes 9193 * precedence before starting a new reset sequence. 9194 */ 9195 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 9196 i40e_prep_for_reset(pf, false); 9197 i40e_reset(pf); 9198 i40e_rebuild(pf, false, false); 9199 } 9200 9201 /* If we're already down or resetting, just bail */ 9202 if (reset_flags && 9203 !test_bit(__I40E_DOWN, pf->state) && 9204 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 9205 i40e_do_reset(pf, reset_flags, false); 9206 } 9207 } 9208 9209 /** 9210 * i40e_handle_link_event - Handle link event 9211 * @pf: board private structure 9212 * @e: event info posted on ARQ 9213 **/ 9214 static void i40e_handle_link_event(struct i40e_pf *pf, 9215 struct i40e_arq_event_info *e) 9216 { 9217 struct i40e_aqc_get_link_status *status = 9218 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 9219 9220 /* Do a new status request to re-enable LSE reporting 9221 * and load new status information into the hw struct 9222 * This completely ignores any state information 9223 * in the ARQ event info, instead choosing to always 9224 * issue the AQ update link status command. 9225 */ 9226 i40e_link_event(pf); 9227 9228 /* Check if module meets thermal requirements */ 9229 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 9230 dev_err(&pf->pdev->dev, 9231 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 9232 dev_err(&pf->pdev->dev, 9233 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 9234 } else { 9235 /* check for unqualified module, if link is down, suppress 9236 * the message if link was forced to be down. 9237 */ 9238 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 9239 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 9240 (!(status->link_info & I40E_AQ_LINK_UP)) && 9241 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 9242 dev_err(&pf->pdev->dev, 9243 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 9244 dev_err(&pf->pdev->dev, 9245 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 9246 } 9247 } 9248 } 9249 9250 /** 9251 * i40e_clean_adminq_subtask - Clean the AdminQ rings 9252 * @pf: board private structure 9253 **/ 9254 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 9255 { 9256 struct i40e_arq_event_info event; 9257 struct i40e_hw *hw = &pf->hw; 9258 u16 pending, i = 0; 9259 i40e_status ret; 9260 u16 opcode; 9261 u32 oldval; 9262 u32 val; 9263 9264 /* Do not run clean AQ when PF reset fails */ 9265 if (test_bit(__I40E_RESET_FAILED, pf->state)) 9266 return; 9267 9268 /* check for error indications */ 9269 val = rd32(&pf->hw, pf->hw.aq.arq.len); 9270 oldval = val; 9271 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 9272 if (hw->debug_mask & I40E_DEBUG_AQ) 9273 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 9274 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 9275 } 9276 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 9277 if (hw->debug_mask & I40E_DEBUG_AQ) 9278 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 9279 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 9280 pf->arq_overflows++; 9281 } 9282 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 9283 if (hw->debug_mask & I40E_DEBUG_AQ) 9284 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 9285 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 9286 } 9287 if (oldval != val) 9288 wr32(&pf->hw, pf->hw.aq.arq.len, val); 9289 9290 val = rd32(&pf->hw, pf->hw.aq.asq.len); 9291 oldval = val; 9292 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 9293 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9294 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 9295 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 9296 } 9297 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 9298 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9299 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 9300 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 9301 } 9302 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 9303 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9304 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 9305 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 9306 } 9307 if (oldval != val) 9308 wr32(&pf->hw, pf->hw.aq.asq.len, val); 9309 9310 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 9311 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 9312 if (!event.msg_buf) 9313 return; 9314 9315 do { 9316 ret = i40e_clean_arq_element(hw, &event, &pending); 9317 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 9318 break; 9319 else if (ret) { 9320 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 9321 break; 9322 } 9323 9324 opcode = le16_to_cpu(event.desc.opcode); 9325 switch (opcode) { 9326 9327 case i40e_aqc_opc_get_link_status: 9328 i40e_handle_link_event(pf, &event); 9329 break; 9330 case i40e_aqc_opc_send_msg_to_pf: 9331 ret = i40e_vc_process_vf_msg(pf, 9332 le16_to_cpu(event.desc.retval), 9333 le32_to_cpu(event.desc.cookie_high), 9334 le32_to_cpu(event.desc.cookie_low), 9335 event.msg_buf, 9336 event.msg_len); 9337 break; 9338 case i40e_aqc_opc_lldp_update_mib: 9339 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 9340 #ifdef CONFIG_I40E_DCB 9341 rtnl_lock(); 9342 ret = i40e_handle_lldp_event(pf, &event); 9343 rtnl_unlock(); 9344 #endif /* CONFIG_I40E_DCB */ 9345 break; 9346 case i40e_aqc_opc_event_lan_overflow: 9347 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 9348 i40e_handle_lan_overflow_event(pf, &event); 9349 break; 9350 case i40e_aqc_opc_send_msg_to_peer: 9351 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 9352 break; 9353 case i40e_aqc_opc_nvm_erase: 9354 case i40e_aqc_opc_nvm_update: 9355 case i40e_aqc_opc_oem_post_update: 9356 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 9357 "ARQ NVM operation 0x%04x completed\n", 9358 opcode); 9359 break; 9360 default: 9361 dev_info(&pf->pdev->dev, 9362 "ARQ: Unknown event 0x%04x ignored\n", 9363 opcode); 9364 break; 9365 } 9366 } while (i++ < pf->adminq_work_limit); 9367 9368 if (i < pf->adminq_work_limit) 9369 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 9370 9371 /* re-enable Admin queue interrupt cause */ 9372 val = rd32(hw, I40E_PFINT_ICR0_ENA); 9373 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 9374 wr32(hw, I40E_PFINT_ICR0_ENA, val); 9375 i40e_flush(hw); 9376 9377 kfree(event.msg_buf); 9378 } 9379 9380 /** 9381 * i40e_verify_eeprom - make sure eeprom is good to use 9382 * @pf: board private structure 9383 **/ 9384 static void i40e_verify_eeprom(struct i40e_pf *pf) 9385 { 9386 int err; 9387 9388 err = i40e_diag_eeprom_test(&pf->hw); 9389 if (err) { 9390 /* retry in case of garbage read */ 9391 err = i40e_diag_eeprom_test(&pf->hw); 9392 if (err) { 9393 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 9394 err); 9395 set_bit(__I40E_BAD_EEPROM, pf->state); 9396 } 9397 } 9398 9399 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 9400 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 9401 clear_bit(__I40E_BAD_EEPROM, pf->state); 9402 } 9403 } 9404 9405 /** 9406 * i40e_enable_pf_switch_lb 9407 * @pf: pointer to the PF structure 9408 * 9409 * enable switch loop back or die - no point in a return value 9410 **/ 9411 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 9412 { 9413 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9414 struct i40e_vsi_context ctxt; 9415 int ret; 9416 9417 ctxt.seid = pf->main_vsi_seid; 9418 ctxt.pf_num = pf->hw.pf_id; 9419 ctxt.vf_num = 0; 9420 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9421 if (ret) { 9422 dev_info(&pf->pdev->dev, 9423 "couldn't get PF vsi config, err %s aq_err %s\n", 9424 i40e_stat_str(&pf->hw, ret), 9425 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9426 return; 9427 } 9428 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9429 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9430 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9431 9432 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 9433 if (ret) { 9434 dev_info(&pf->pdev->dev, 9435 "update vsi switch failed, err %s aq_err %s\n", 9436 i40e_stat_str(&pf->hw, ret), 9437 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9438 } 9439 } 9440 9441 /** 9442 * i40e_disable_pf_switch_lb 9443 * @pf: pointer to the PF structure 9444 * 9445 * disable switch loop back or die - no point in a return value 9446 **/ 9447 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 9448 { 9449 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9450 struct i40e_vsi_context ctxt; 9451 int ret; 9452 9453 ctxt.seid = pf->main_vsi_seid; 9454 ctxt.pf_num = pf->hw.pf_id; 9455 ctxt.vf_num = 0; 9456 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9457 if (ret) { 9458 dev_info(&pf->pdev->dev, 9459 "couldn't get PF vsi config, err %s aq_err %s\n", 9460 i40e_stat_str(&pf->hw, ret), 9461 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9462 return; 9463 } 9464 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9465 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9466 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9467 9468 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 9469 if (ret) { 9470 dev_info(&pf->pdev->dev, 9471 "update vsi switch failed, err %s aq_err %s\n", 9472 i40e_stat_str(&pf->hw, ret), 9473 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9474 } 9475 } 9476 9477 /** 9478 * i40e_config_bridge_mode - Configure the HW bridge mode 9479 * @veb: pointer to the bridge instance 9480 * 9481 * Configure the loop back mode for the LAN VSI that is downlink to the 9482 * specified HW bridge instance. It is expected this function is called 9483 * when a new HW bridge is instantiated. 9484 **/ 9485 static void i40e_config_bridge_mode(struct i40e_veb *veb) 9486 { 9487 struct i40e_pf *pf = veb->pf; 9488 9489 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 9490 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 9491 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 9492 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 9493 i40e_disable_pf_switch_lb(pf); 9494 else 9495 i40e_enable_pf_switch_lb(pf); 9496 } 9497 9498 /** 9499 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 9500 * @veb: pointer to the VEB instance 9501 * 9502 * This is a recursive function that first builds the attached VSIs then 9503 * recurses in to build the next layer of VEB. We track the connections 9504 * through our own index numbers because the seid's from the HW could 9505 * change across the reset. 9506 **/ 9507 static int i40e_reconstitute_veb(struct i40e_veb *veb) 9508 { 9509 struct i40e_vsi *ctl_vsi = NULL; 9510 struct i40e_pf *pf = veb->pf; 9511 int v, veb_idx; 9512 int ret; 9513 9514 /* build VSI that owns this VEB, temporarily attached to base VEB */ 9515 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 9516 if (pf->vsi[v] && 9517 pf->vsi[v]->veb_idx == veb->idx && 9518 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 9519 ctl_vsi = pf->vsi[v]; 9520 break; 9521 } 9522 } 9523 if (!ctl_vsi) { 9524 dev_info(&pf->pdev->dev, 9525 "missing owner VSI for veb_idx %d\n", veb->idx); 9526 ret = -ENOENT; 9527 goto end_reconstitute; 9528 } 9529 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 9530 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 9531 ret = i40e_add_vsi(ctl_vsi); 9532 if (ret) { 9533 dev_info(&pf->pdev->dev, 9534 "rebuild of veb_idx %d owner VSI failed: %d\n", 9535 veb->idx, ret); 9536 goto end_reconstitute; 9537 } 9538 i40e_vsi_reset_stats(ctl_vsi); 9539 9540 /* create the VEB in the switch and move the VSI onto the VEB */ 9541 ret = i40e_add_veb(veb, ctl_vsi); 9542 if (ret) 9543 goto end_reconstitute; 9544 9545 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 9546 veb->bridge_mode = BRIDGE_MODE_VEB; 9547 else 9548 veb->bridge_mode = BRIDGE_MODE_VEPA; 9549 i40e_config_bridge_mode(veb); 9550 9551 /* create the remaining VSIs attached to this VEB */ 9552 for (v = 0; v < pf->num_alloc_vsi; v++) { 9553 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 9554 continue; 9555 9556 if (pf->vsi[v]->veb_idx == veb->idx) { 9557 struct i40e_vsi *vsi = pf->vsi[v]; 9558 9559 vsi->uplink_seid = veb->seid; 9560 ret = i40e_add_vsi(vsi); 9561 if (ret) { 9562 dev_info(&pf->pdev->dev, 9563 "rebuild of vsi_idx %d failed: %d\n", 9564 v, ret); 9565 goto end_reconstitute; 9566 } 9567 i40e_vsi_reset_stats(vsi); 9568 } 9569 } 9570 9571 /* create any VEBs attached to this VEB - RECURSION */ 9572 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 9573 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 9574 pf->veb[veb_idx]->uplink_seid = veb->seid; 9575 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 9576 if (ret) 9577 break; 9578 } 9579 } 9580 9581 end_reconstitute: 9582 return ret; 9583 } 9584 9585 /** 9586 * i40e_get_capabilities - get info about the HW 9587 * @pf: the PF struct 9588 **/ 9589 static int i40e_get_capabilities(struct i40e_pf *pf, 9590 enum i40e_admin_queue_opc list_type) 9591 { 9592 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 9593 u16 data_size; 9594 int buf_len; 9595 int err; 9596 9597 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 9598 do { 9599 cap_buf = kzalloc(buf_len, GFP_KERNEL); 9600 if (!cap_buf) 9601 return -ENOMEM; 9602 9603 /* this loads the data into the hw struct for us */ 9604 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 9605 &data_size, list_type, 9606 NULL); 9607 /* data loaded, buffer no longer needed */ 9608 kfree(cap_buf); 9609 9610 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 9611 /* retry with a larger buffer */ 9612 buf_len = data_size; 9613 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 9614 dev_info(&pf->pdev->dev, 9615 "capability discovery failed, err %s aq_err %s\n", 9616 i40e_stat_str(&pf->hw, err), 9617 i40e_aq_str(&pf->hw, 9618 pf->hw.aq.asq_last_status)); 9619 return -ENODEV; 9620 } 9621 } while (err); 9622 9623 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 9624 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9625 dev_info(&pf->pdev->dev, 9626 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 9627 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 9628 pf->hw.func_caps.num_msix_vectors, 9629 pf->hw.func_caps.num_msix_vectors_vf, 9630 pf->hw.func_caps.fd_filters_guaranteed, 9631 pf->hw.func_caps.fd_filters_best_effort, 9632 pf->hw.func_caps.num_tx_qp, 9633 pf->hw.func_caps.num_vsis); 9634 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 9635 dev_info(&pf->pdev->dev, 9636 "switch_mode=0x%04x, function_valid=0x%08x\n", 9637 pf->hw.dev_caps.switch_mode, 9638 pf->hw.dev_caps.valid_functions); 9639 dev_info(&pf->pdev->dev, 9640 "SR-IOV=%d, num_vfs for all function=%u\n", 9641 pf->hw.dev_caps.sr_iov_1_1, 9642 pf->hw.dev_caps.num_vfs); 9643 dev_info(&pf->pdev->dev, 9644 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 9645 pf->hw.dev_caps.num_vsis, 9646 pf->hw.dev_caps.num_rx_qp, 9647 pf->hw.dev_caps.num_tx_qp); 9648 } 9649 } 9650 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9651 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 9652 + pf->hw.func_caps.num_vfs) 9653 if (pf->hw.revision_id == 0 && 9654 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 9655 dev_info(&pf->pdev->dev, 9656 "got num_vsis %d, setting num_vsis to %d\n", 9657 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 9658 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 9659 } 9660 } 9661 return 0; 9662 } 9663 9664 static int i40e_vsi_clear(struct i40e_vsi *vsi); 9665 9666 /** 9667 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 9668 * @pf: board private structure 9669 **/ 9670 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 9671 { 9672 struct i40e_vsi *vsi; 9673 9674 /* quick workaround for an NVM issue that leaves a critical register 9675 * uninitialized 9676 */ 9677 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 9678 static const u32 hkey[] = { 9679 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 9680 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 9681 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 9682 0x95b3a76d}; 9683 int i; 9684 9685 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 9686 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 9687 } 9688 9689 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 9690 return; 9691 9692 /* find existing VSI and see if it needs configuring */ 9693 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9694 9695 /* create a new VSI if none exists */ 9696 if (!vsi) { 9697 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 9698 pf->vsi[pf->lan_vsi]->seid, 0); 9699 if (!vsi) { 9700 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 9701 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 9702 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 9703 return; 9704 } 9705 } 9706 9707 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 9708 } 9709 9710 /** 9711 * i40e_fdir_teardown - release the Flow Director resources 9712 * @pf: board private structure 9713 **/ 9714 static void i40e_fdir_teardown(struct i40e_pf *pf) 9715 { 9716 struct i40e_vsi *vsi; 9717 9718 i40e_fdir_filter_exit(pf); 9719 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9720 if (vsi) 9721 i40e_vsi_release(vsi); 9722 } 9723 9724 /** 9725 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 9726 * @vsi: PF main vsi 9727 * @seid: seid of main or channel VSIs 9728 * 9729 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 9730 * existed before reset 9731 **/ 9732 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 9733 { 9734 struct i40e_cloud_filter *cfilter; 9735 struct i40e_pf *pf = vsi->back; 9736 struct hlist_node *node; 9737 i40e_status ret; 9738 9739 /* Add cloud filters back if they exist */ 9740 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 9741 cloud_node) { 9742 if (cfilter->seid != seid) 9743 continue; 9744 9745 if (cfilter->dst_port) 9746 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 9747 true); 9748 else 9749 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 9750 9751 if (ret) { 9752 dev_dbg(&pf->pdev->dev, 9753 "Failed to rebuild cloud filter, err %s aq_err %s\n", 9754 i40e_stat_str(&pf->hw, ret), 9755 i40e_aq_str(&pf->hw, 9756 pf->hw.aq.asq_last_status)); 9757 return ret; 9758 } 9759 } 9760 return 0; 9761 } 9762 9763 /** 9764 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 9765 * @vsi: PF main vsi 9766 * 9767 * Rebuilds channel VSIs if they existed before reset 9768 **/ 9769 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 9770 { 9771 struct i40e_channel *ch, *ch_tmp; 9772 i40e_status ret; 9773 9774 if (list_empty(&vsi->ch_list)) 9775 return 0; 9776 9777 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 9778 if (!ch->initialized) 9779 break; 9780 /* Proceed with creation of channel (VMDq2) VSI */ 9781 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 9782 if (ret) { 9783 dev_info(&vsi->back->pdev->dev, 9784 "failed to rebuild channels using uplink_seid %u\n", 9785 vsi->uplink_seid); 9786 return ret; 9787 } 9788 /* Reconfigure TX queues using QTX_CTL register */ 9789 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 9790 if (ret) { 9791 dev_info(&vsi->back->pdev->dev, 9792 "failed to configure TX rings for channel %u\n", 9793 ch->seid); 9794 return ret; 9795 } 9796 /* update 'next_base_queue' */ 9797 vsi->next_base_queue = vsi->next_base_queue + 9798 ch->num_queue_pairs; 9799 if (ch->max_tx_rate) { 9800 u64 credits = ch->max_tx_rate; 9801 9802 if (i40e_set_bw_limit(vsi, ch->seid, 9803 ch->max_tx_rate)) 9804 return -EINVAL; 9805 9806 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9807 dev_dbg(&vsi->back->pdev->dev, 9808 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9809 ch->max_tx_rate, 9810 credits, 9811 ch->seid); 9812 } 9813 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 9814 if (ret) { 9815 dev_dbg(&vsi->back->pdev->dev, 9816 "Failed to rebuild cloud filters for channel VSI %u\n", 9817 ch->seid); 9818 return ret; 9819 } 9820 } 9821 return 0; 9822 } 9823 9824 /** 9825 * i40e_prep_for_reset - prep for the core to reset 9826 * @pf: board private structure 9827 * @lock_acquired: indicates whether or not the lock has been acquired 9828 * before this function was called. 9829 * 9830 * Close up the VFs and other things in prep for PF Reset. 9831 **/ 9832 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 9833 { 9834 struct i40e_hw *hw = &pf->hw; 9835 i40e_status ret = 0; 9836 u32 v; 9837 9838 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 9839 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9840 return; 9841 if (i40e_check_asq_alive(&pf->hw)) 9842 i40e_vc_notify_reset(pf); 9843 9844 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 9845 9846 /* quiesce the VSIs and their queues that are not already DOWN */ 9847 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 9848 if (!lock_acquired) 9849 rtnl_lock(); 9850 i40e_pf_quiesce_all_vsi(pf); 9851 if (!lock_acquired) 9852 rtnl_unlock(); 9853 9854 for (v = 0; v < pf->num_alloc_vsi; v++) { 9855 if (pf->vsi[v]) 9856 pf->vsi[v]->seid = 0; 9857 } 9858 9859 i40e_shutdown_adminq(&pf->hw); 9860 9861 /* call shutdown HMC */ 9862 if (hw->hmc.hmc_obj) { 9863 ret = i40e_shutdown_lan_hmc(hw); 9864 if (ret) 9865 dev_warn(&pf->pdev->dev, 9866 "shutdown_lan_hmc failed: %d\n", ret); 9867 } 9868 9869 /* Save the current PTP time so that we can restore the time after the 9870 * reset completes. 9871 */ 9872 i40e_ptp_save_hw_time(pf); 9873 } 9874 9875 /** 9876 * i40e_send_version - update firmware with driver version 9877 * @pf: PF struct 9878 */ 9879 static void i40e_send_version(struct i40e_pf *pf) 9880 { 9881 struct i40e_driver_version dv; 9882 9883 dv.major_version = 0xff; 9884 dv.minor_version = 0xff; 9885 dv.build_version = 0xff; 9886 dv.subbuild_version = 0; 9887 strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); 9888 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 9889 } 9890 9891 /** 9892 * i40e_get_oem_version - get OEM specific version information 9893 * @hw: pointer to the hardware structure 9894 **/ 9895 static void i40e_get_oem_version(struct i40e_hw *hw) 9896 { 9897 u16 block_offset = 0xffff; 9898 u16 block_length = 0; 9899 u16 capabilities = 0; 9900 u16 gen_snap = 0; 9901 u16 release = 0; 9902 9903 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 9904 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 9905 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 9906 #define I40E_NVM_OEM_GEN_OFFSET 0x02 9907 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 9908 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 9909 #define I40E_NVM_OEM_LENGTH 3 9910 9911 /* Check if pointer to OEM version block is valid. */ 9912 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 9913 if (block_offset == 0xffff) 9914 return; 9915 9916 /* Check if OEM version block has correct length. */ 9917 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 9918 &block_length); 9919 if (block_length < I40E_NVM_OEM_LENGTH) 9920 return; 9921 9922 /* Check if OEM version format is as expected. */ 9923 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 9924 &capabilities); 9925 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 9926 return; 9927 9928 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 9929 &gen_snap); 9930 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 9931 &release); 9932 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 9933 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 9934 } 9935 9936 /** 9937 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 9938 * @pf: board private structure 9939 **/ 9940 static int i40e_reset(struct i40e_pf *pf) 9941 { 9942 struct i40e_hw *hw = &pf->hw; 9943 i40e_status ret; 9944 9945 ret = i40e_pf_reset(hw); 9946 if (ret) { 9947 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 9948 set_bit(__I40E_RESET_FAILED, pf->state); 9949 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9950 } else { 9951 pf->pfr_count++; 9952 } 9953 return ret; 9954 } 9955 9956 /** 9957 * i40e_rebuild - rebuild using a saved config 9958 * @pf: board private structure 9959 * @reinit: if the Main VSI needs to re-initialized. 9960 * @lock_acquired: indicates whether or not the lock has been acquired 9961 * before this function was called. 9962 **/ 9963 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 9964 { 9965 int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state); 9966 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9967 struct i40e_hw *hw = &pf->hw; 9968 u8 set_fc_aq_fail = 0; 9969 i40e_status ret; 9970 u32 val; 9971 int v; 9972 9973 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 9974 i40e_check_recovery_mode(pf)) { 9975 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev); 9976 } 9977 9978 if (test_bit(__I40E_DOWN, pf->state) && 9979 !test_bit(__I40E_RECOVERY_MODE, pf->state) && 9980 !old_recovery_mode_bit) 9981 goto clear_recovery; 9982 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 9983 9984 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 9985 ret = i40e_init_adminq(&pf->hw); 9986 if (ret) { 9987 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 9988 i40e_stat_str(&pf->hw, ret), 9989 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9990 goto clear_recovery; 9991 } 9992 i40e_get_oem_version(&pf->hw); 9993 9994 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 9995 ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) || 9996 hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) { 9997 /* The following delay is necessary for 4.33 firmware and older 9998 * to recover after EMP reset. 200 ms should suffice but we 9999 * put here 300 ms to be sure that FW is ready to operate 10000 * after reset. 10001 */ 10002 mdelay(300); 10003 } 10004 10005 /* re-verify the eeprom if we just had an EMP reset */ 10006 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 10007 i40e_verify_eeprom(pf); 10008 10009 /* if we are going out of or into recovery mode we have to act 10010 * accordingly with regard to resources initialization 10011 * and deinitialization 10012 */ 10013 if (test_bit(__I40E_RECOVERY_MODE, pf->state) || 10014 old_recovery_mode_bit) { 10015 if (i40e_get_capabilities(pf, 10016 i40e_aqc_opc_list_func_capabilities)) 10017 goto end_unlock; 10018 10019 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10020 /* we're staying in recovery mode so we'll reinitialize 10021 * misc vector here 10022 */ 10023 if (i40e_setup_misc_vector_for_recovery_mode(pf)) 10024 goto end_unlock; 10025 } else { 10026 if (!lock_acquired) 10027 rtnl_lock(); 10028 /* we're going out of recovery mode so we'll free 10029 * the IRQ allocated specifically for recovery mode 10030 * and restore the interrupt scheme 10031 */ 10032 free_irq(pf->pdev->irq, pf); 10033 i40e_clear_interrupt_scheme(pf); 10034 if (i40e_restore_interrupt_scheme(pf)) 10035 goto end_unlock; 10036 } 10037 10038 /* tell the firmware that we're starting */ 10039 i40e_send_version(pf); 10040 10041 /* bail out in case recovery mode was detected, as there is 10042 * no need for further configuration. 10043 */ 10044 goto end_unlock; 10045 } 10046 10047 i40e_clear_pxe_mode(hw); 10048 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 10049 if (ret) 10050 goto end_core_reset; 10051 10052 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10053 hw->func_caps.num_rx_qp, 0, 0); 10054 if (ret) { 10055 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 10056 goto end_core_reset; 10057 } 10058 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10059 if (ret) { 10060 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 10061 goto end_core_reset; 10062 } 10063 10064 /* Enable FW to write a default DCB config on link-up */ 10065 i40e_aq_set_dcb_parameters(hw, true, NULL); 10066 10067 #ifdef CONFIG_I40E_DCB 10068 ret = i40e_init_pf_dcb(pf); 10069 if (ret) { 10070 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 10071 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 10072 /* Continue without DCB enabled */ 10073 } 10074 #endif /* CONFIG_I40E_DCB */ 10075 /* do basic switch setup */ 10076 if (!lock_acquired) 10077 rtnl_lock(); 10078 ret = i40e_setup_pf_switch(pf, reinit); 10079 if (ret) 10080 goto end_unlock; 10081 10082 /* The driver only wants link up/down and module qualification 10083 * reports from firmware. Note the negative logic. 10084 */ 10085 ret = i40e_aq_set_phy_int_mask(&pf->hw, 10086 ~(I40E_AQ_EVENT_LINK_UPDOWN | 10087 I40E_AQ_EVENT_MEDIA_NA | 10088 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 10089 if (ret) 10090 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 10091 i40e_stat_str(&pf->hw, ret), 10092 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10093 10094 /* make sure our flow control settings are restored */ 10095 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 10096 if (ret) 10097 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 10098 i40e_stat_str(&pf->hw, ret), 10099 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10100 10101 /* Rebuild the VSIs and VEBs that existed before reset. 10102 * They are still in our local switch element arrays, so only 10103 * need to rebuild the switch model in the HW. 10104 * 10105 * If there were VEBs but the reconstitution failed, we'll try 10106 * try to recover minimal use by getting the basic PF VSI working. 10107 */ 10108 if (vsi->uplink_seid != pf->mac_seid) { 10109 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 10110 /* find the one VEB connected to the MAC, and find orphans */ 10111 for (v = 0; v < I40E_MAX_VEB; v++) { 10112 if (!pf->veb[v]) 10113 continue; 10114 10115 if (pf->veb[v]->uplink_seid == pf->mac_seid || 10116 pf->veb[v]->uplink_seid == 0) { 10117 ret = i40e_reconstitute_veb(pf->veb[v]); 10118 10119 if (!ret) 10120 continue; 10121 10122 /* If Main VEB failed, we're in deep doodoo, 10123 * so give up rebuilding the switch and set up 10124 * for minimal rebuild of PF VSI. 10125 * If orphan failed, we'll report the error 10126 * but try to keep going. 10127 */ 10128 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 10129 dev_info(&pf->pdev->dev, 10130 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 10131 ret); 10132 vsi->uplink_seid = pf->mac_seid; 10133 break; 10134 } else if (pf->veb[v]->uplink_seid == 0) { 10135 dev_info(&pf->pdev->dev, 10136 "rebuild of orphan VEB failed: %d\n", 10137 ret); 10138 } 10139 } 10140 } 10141 } 10142 10143 if (vsi->uplink_seid == pf->mac_seid) { 10144 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 10145 /* no VEB, so rebuild only the Main VSI */ 10146 ret = i40e_add_vsi(vsi); 10147 if (ret) { 10148 dev_info(&pf->pdev->dev, 10149 "rebuild of Main VSI failed: %d\n", ret); 10150 goto end_unlock; 10151 } 10152 } 10153 10154 if (vsi->mqprio_qopt.max_rate[0]) { 10155 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 10156 u64 credits = 0; 10157 10158 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 10159 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 10160 if (ret) 10161 goto end_unlock; 10162 10163 credits = max_tx_rate; 10164 do_div(credits, I40E_BW_CREDIT_DIVISOR); 10165 dev_dbg(&vsi->back->pdev->dev, 10166 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 10167 max_tx_rate, 10168 credits, 10169 vsi->seid); 10170 } 10171 10172 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 10173 if (ret) 10174 goto end_unlock; 10175 10176 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 10177 * for this main VSI if they exist 10178 */ 10179 ret = i40e_rebuild_channels(vsi); 10180 if (ret) 10181 goto end_unlock; 10182 10183 /* Reconfigure hardware for allowing smaller MSS in the case 10184 * of TSO, so that we avoid the MDD being fired and causing 10185 * a reset in the case of small MSS+TSO. 10186 */ 10187 #define I40E_REG_MSS 0x000E64DC 10188 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 10189 #define I40E_64BYTE_MSS 0x400000 10190 val = rd32(hw, I40E_REG_MSS); 10191 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 10192 val &= ~I40E_REG_MSS_MIN_MASK; 10193 val |= I40E_64BYTE_MSS; 10194 wr32(hw, I40E_REG_MSS, val); 10195 } 10196 10197 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 10198 msleep(75); 10199 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 10200 if (ret) 10201 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 10202 i40e_stat_str(&pf->hw, ret), 10203 i40e_aq_str(&pf->hw, 10204 pf->hw.aq.asq_last_status)); 10205 } 10206 /* reinit the misc interrupt */ 10207 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10208 ret = i40e_setup_misc_vector(pf); 10209 10210 /* Add a filter to drop all Flow control frames from any VSI from being 10211 * transmitted. By doing so we stop a malicious VF from sending out 10212 * PAUSE or PFC frames and potentially controlling traffic for other 10213 * PF/VF VSIs. 10214 * The FW can still send Flow control frames if enabled. 10215 */ 10216 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 10217 pf->main_vsi_seid); 10218 10219 /* restart the VSIs that were rebuilt and running before the reset */ 10220 i40e_pf_unquiesce_all_vsi(pf); 10221 10222 /* Release the RTNL lock before we start resetting VFs */ 10223 if (!lock_acquired) 10224 rtnl_unlock(); 10225 10226 /* Restore promiscuous settings */ 10227 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 10228 if (ret) 10229 dev_warn(&pf->pdev->dev, 10230 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n", 10231 pf->cur_promisc ? "on" : "off", 10232 i40e_stat_str(&pf->hw, ret), 10233 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10234 10235 i40e_reset_all_vfs(pf, true); 10236 10237 /* tell the firmware that we're starting */ 10238 i40e_send_version(pf); 10239 10240 /* We've already released the lock, so don't do it again */ 10241 goto end_core_reset; 10242 10243 end_unlock: 10244 if (!lock_acquired) 10245 rtnl_unlock(); 10246 end_core_reset: 10247 clear_bit(__I40E_RESET_FAILED, pf->state); 10248 clear_recovery: 10249 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 10250 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state); 10251 } 10252 10253 /** 10254 * i40e_reset_and_rebuild - reset and rebuild using a saved config 10255 * @pf: board private structure 10256 * @reinit: if the Main VSI needs to re-initialized. 10257 * @lock_acquired: indicates whether or not the lock has been acquired 10258 * before this function was called. 10259 **/ 10260 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 10261 bool lock_acquired) 10262 { 10263 int ret; 10264 /* Now we wait for GRST to settle out. 10265 * We don't have to delete the VEBs or VSIs from the hw switch 10266 * because the reset will make them disappear. 10267 */ 10268 ret = i40e_reset(pf); 10269 if (!ret) 10270 i40e_rebuild(pf, reinit, lock_acquired); 10271 } 10272 10273 /** 10274 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 10275 * @pf: board private structure 10276 * 10277 * Close up the VFs and other things in prep for a Core Reset, 10278 * then get ready to rebuild the world. 10279 * @lock_acquired: indicates whether or not the lock has been acquired 10280 * before this function was called. 10281 **/ 10282 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 10283 { 10284 i40e_prep_for_reset(pf, lock_acquired); 10285 i40e_reset_and_rebuild(pf, false, lock_acquired); 10286 } 10287 10288 /** 10289 * i40e_handle_mdd_event 10290 * @pf: pointer to the PF structure 10291 * 10292 * Called from the MDD irq handler to identify possibly malicious vfs 10293 **/ 10294 static void i40e_handle_mdd_event(struct i40e_pf *pf) 10295 { 10296 struct i40e_hw *hw = &pf->hw; 10297 bool mdd_detected = false; 10298 struct i40e_vf *vf; 10299 u32 reg; 10300 int i; 10301 10302 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 10303 return; 10304 10305 /* find what triggered the MDD event */ 10306 reg = rd32(hw, I40E_GL_MDET_TX); 10307 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 10308 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 10309 I40E_GL_MDET_TX_PF_NUM_SHIFT; 10310 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 10311 I40E_GL_MDET_TX_VF_NUM_SHIFT; 10312 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 10313 I40E_GL_MDET_TX_EVENT_SHIFT; 10314 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 10315 I40E_GL_MDET_TX_QUEUE_SHIFT) - 10316 pf->hw.func_caps.base_queue; 10317 if (netif_msg_tx_err(pf)) 10318 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 10319 event, queue, pf_num, vf_num); 10320 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 10321 mdd_detected = true; 10322 } 10323 reg = rd32(hw, I40E_GL_MDET_RX); 10324 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 10325 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 10326 I40E_GL_MDET_RX_FUNCTION_SHIFT; 10327 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 10328 I40E_GL_MDET_RX_EVENT_SHIFT; 10329 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 10330 I40E_GL_MDET_RX_QUEUE_SHIFT) - 10331 pf->hw.func_caps.base_queue; 10332 if (netif_msg_rx_err(pf)) 10333 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 10334 event, queue, func); 10335 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 10336 mdd_detected = true; 10337 } 10338 10339 if (mdd_detected) { 10340 reg = rd32(hw, I40E_PF_MDET_TX); 10341 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 10342 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 10343 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n"); 10344 } 10345 reg = rd32(hw, I40E_PF_MDET_RX); 10346 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 10347 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 10348 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n"); 10349 } 10350 } 10351 10352 /* see if one of the VFs needs its hand slapped */ 10353 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 10354 vf = &(pf->vf[i]); 10355 reg = rd32(hw, I40E_VP_MDET_TX(i)); 10356 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 10357 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 10358 vf->num_mdd_events++; 10359 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 10360 i); 10361 dev_info(&pf->pdev->dev, 10362 "Use PF Control I/F to re-enable the VF\n"); 10363 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 10364 } 10365 10366 reg = rd32(hw, I40E_VP_MDET_RX(i)); 10367 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 10368 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 10369 vf->num_mdd_events++; 10370 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 10371 i); 10372 dev_info(&pf->pdev->dev, 10373 "Use PF Control I/F to re-enable the VF\n"); 10374 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 10375 } 10376 } 10377 10378 /* re-enable mdd interrupt cause */ 10379 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 10380 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 10381 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 10382 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 10383 i40e_flush(hw); 10384 } 10385 10386 static const char *i40e_tunnel_name(u8 type) 10387 { 10388 switch (type) { 10389 case UDP_TUNNEL_TYPE_VXLAN: 10390 return "vxlan"; 10391 case UDP_TUNNEL_TYPE_GENEVE: 10392 return "geneve"; 10393 default: 10394 return "unknown"; 10395 } 10396 } 10397 10398 /** 10399 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 10400 * @pf: board private structure 10401 **/ 10402 static void i40e_sync_udp_filters(struct i40e_pf *pf) 10403 { 10404 int i; 10405 10406 /* loop through and set pending bit for all active UDP filters */ 10407 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 10408 if (pf->udp_ports[i].port) 10409 pf->pending_udp_bitmap |= BIT_ULL(i); 10410 } 10411 10412 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 10413 } 10414 10415 /** 10416 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 10417 * @pf: board private structure 10418 **/ 10419 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 10420 { 10421 struct i40e_hw *hw = &pf->hw; 10422 u8 filter_index, type; 10423 u16 port; 10424 int i; 10425 10426 if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state)) 10427 return; 10428 10429 /* acquire RTNL to maintain state of flags and port requests */ 10430 rtnl_lock(); 10431 10432 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 10433 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 10434 struct i40e_udp_port_config *udp_port; 10435 i40e_status ret = 0; 10436 10437 udp_port = &pf->udp_ports[i]; 10438 pf->pending_udp_bitmap &= ~BIT_ULL(i); 10439 10440 port = READ_ONCE(udp_port->port); 10441 type = READ_ONCE(udp_port->type); 10442 filter_index = READ_ONCE(udp_port->filter_index); 10443 10444 /* release RTNL while we wait on AQ command */ 10445 rtnl_unlock(); 10446 10447 if (port) 10448 ret = i40e_aq_add_udp_tunnel(hw, port, 10449 type, 10450 &filter_index, 10451 NULL); 10452 else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED) 10453 ret = i40e_aq_del_udp_tunnel(hw, filter_index, 10454 NULL); 10455 10456 /* reacquire RTNL so we can update filter_index */ 10457 rtnl_lock(); 10458 10459 if (ret) { 10460 dev_info(&pf->pdev->dev, 10461 "%s %s port %d, index %d failed, err %s aq_err %s\n", 10462 i40e_tunnel_name(type), 10463 port ? "add" : "delete", 10464 port, 10465 filter_index, 10466 i40e_stat_str(&pf->hw, ret), 10467 i40e_aq_str(&pf->hw, 10468 pf->hw.aq.asq_last_status)); 10469 if (port) { 10470 /* failed to add, just reset port, 10471 * drop pending bit for any deletion 10472 */ 10473 udp_port->port = 0; 10474 pf->pending_udp_bitmap &= ~BIT_ULL(i); 10475 } 10476 } else if (port) { 10477 /* record filter index on success */ 10478 udp_port->filter_index = filter_index; 10479 } 10480 } 10481 } 10482 10483 rtnl_unlock(); 10484 } 10485 10486 /** 10487 * i40e_service_task - Run the driver's async subtasks 10488 * @work: pointer to work_struct containing our data 10489 **/ 10490 static void i40e_service_task(struct work_struct *work) 10491 { 10492 struct i40e_pf *pf = container_of(work, 10493 struct i40e_pf, 10494 service_task); 10495 unsigned long start_time = jiffies; 10496 10497 /* don't bother with service tasks if a reset is in progress */ 10498 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 10499 test_bit(__I40E_SUSPENDED, pf->state)) 10500 return; 10501 10502 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 10503 return; 10504 10505 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10506 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]); 10507 i40e_sync_filters_subtask(pf); 10508 i40e_reset_subtask(pf); 10509 i40e_handle_mdd_event(pf); 10510 i40e_vc_process_vflr_event(pf); 10511 i40e_watchdog_subtask(pf); 10512 i40e_fdir_reinit_subtask(pf); 10513 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 10514 /* Client subtask will reopen next time through. */ 10515 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], 10516 true); 10517 } else { 10518 i40e_client_subtask(pf); 10519 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 10520 pf->state)) 10521 i40e_notify_client_of_l2_param_changes( 10522 pf->vsi[pf->lan_vsi]); 10523 } 10524 i40e_sync_filters_subtask(pf); 10525 i40e_sync_udp_filters_subtask(pf); 10526 } else { 10527 i40e_reset_subtask(pf); 10528 } 10529 10530 i40e_clean_adminq_subtask(pf); 10531 10532 /* flush memory to make sure state is correct before next watchdog */ 10533 smp_mb__before_atomic(); 10534 clear_bit(__I40E_SERVICE_SCHED, pf->state); 10535 10536 /* If the tasks have taken longer than one timer cycle or there 10537 * is more work to be done, reschedule the service task now 10538 * rather than wait for the timer to tick again. 10539 */ 10540 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 10541 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 10542 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 10543 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 10544 i40e_service_event_schedule(pf); 10545 } 10546 10547 /** 10548 * i40e_service_timer - timer callback 10549 * @data: pointer to PF struct 10550 **/ 10551 static void i40e_service_timer(struct timer_list *t) 10552 { 10553 struct i40e_pf *pf = from_timer(pf, t, service_timer); 10554 10555 mod_timer(&pf->service_timer, 10556 round_jiffies(jiffies + pf->service_timer_period)); 10557 i40e_service_event_schedule(pf); 10558 } 10559 10560 /** 10561 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 10562 * @vsi: the VSI being configured 10563 **/ 10564 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 10565 { 10566 struct i40e_pf *pf = vsi->back; 10567 10568 switch (vsi->type) { 10569 case I40E_VSI_MAIN: 10570 vsi->alloc_queue_pairs = pf->num_lan_qps; 10571 if (!vsi->num_tx_desc) 10572 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10573 I40E_REQ_DESCRIPTOR_MULTIPLE); 10574 if (!vsi->num_rx_desc) 10575 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10576 I40E_REQ_DESCRIPTOR_MULTIPLE); 10577 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10578 vsi->num_q_vectors = pf->num_lan_msix; 10579 else 10580 vsi->num_q_vectors = 1; 10581 10582 break; 10583 10584 case I40E_VSI_FDIR: 10585 vsi->alloc_queue_pairs = 1; 10586 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT, 10587 I40E_REQ_DESCRIPTOR_MULTIPLE); 10588 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT, 10589 I40E_REQ_DESCRIPTOR_MULTIPLE); 10590 vsi->num_q_vectors = pf->num_fdsb_msix; 10591 break; 10592 10593 case I40E_VSI_VMDQ2: 10594 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 10595 if (!vsi->num_tx_desc) 10596 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10597 I40E_REQ_DESCRIPTOR_MULTIPLE); 10598 if (!vsi->num_rx_desc) 10599 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10600 I40E_REQ_DESCRIPTOR_MULTIPLE); 10601 vsi->num_q_vectors = pf->num_vmdq_msix; 10602 break; 10603 10604 case I40E_VSI_SRIOV: 10605 vsi->alloc_queue_pairs = pf->num_vf_qps; 10606 if (!vsi->num_tx_desc) 10607 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10608 I40E_REQ_DESCRIPTOR_MULTIPLE); 10609 if (!vsi->num_rx_desc) 10610 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10611 I40E_REQ_DESCRIPTOR_MULTIPLE); 10612 break; 10613 10614 default: 10615 WARN_ON(1); 10616 return -ENODATA; 10617 } 10618 10619 return 0; 10620 } 10621 10622 /** 10623 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 10624 * @vsi: VSI pointer 10625 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 10626 * 10627 * On error: returns error code (negative) 10628 * On success: returns 0 10629 **/ 10630 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 10631 { 10632 struct i40e_ring **next_rings; 10633 int size; 10634 int ret = 0; 10635 10636 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 10637 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 10638 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 10639 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 10640 if (!vsi->tx_rings) 10641 return -ENOMEM; 10642 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 10643 if (i40e_enabled_xdp_vsi(vsi)) { 10644 vsi->xdp_rings = next_rings; 10645 next_rings += vsi->alloc_queue_pairs; 10646 } 10647 vsi->rx_rings = next_rings; 10648 10649 if (alloc_qvectors) { 10650 /* allocate memory for q_vector pointers */ 10651 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 10652 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 10653 if (!vsi->q_vectors) { 10654 ret = -ENOMEM; 10655 goto err_vectors; 10656 } 10657 } 10658 return ret; 10659 10660 err_vectors: 10661 kfree(vsi->tx_rings); 10662 return ret; 10663 } 10664 10665 /** 10666 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 10667 * @pf: board private structure 10668 * @type: type of VSI 10669 * 10670 * On error: returns error code (negative) 10671 * On success: returns vsi index in PF (positive) 10672 **/ 10673 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 10674 { 10675 int ret = -ENODEV; 10676 struct i40e_vsi *vsi; 10677 int vsi_idx; 10678 int i; 10679 10680 /* Need to protect the allocation of the VSIs at the PF level */ 10681 mutex_lock(&pf->switch_mutex); 10682 10683 /* VSI list may be fragmented if VSI creation/destruction has 10684 * been happening. We can afford to do a quick scan to look 10685 * for any free VSIs in the list. 10686 * 10687 * find next empty vsi slot, looping back around if necessary 10688 */ 10689 i = pf->next_vsi; 10690 while (i < pf->num_alloc_vsi && pf->vsi[i]) 10691 i++; 10692 if (i >= pf->num_alloc_vsi) { 10693 i = 0; 10694 while (i < pf->next_vsi && pf->vsi[i]) 10695 i++; 10696 } 10697 10698 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 10699 vsi_idx = i; /* Found one! */ 10700 } else { 10701 ret = -ENODEV; 10702 goto unlock_pf; /* out of VSI slots! */ 10703 } 10704 pf->next_vsi = ++i; 10705 10706 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 10707 if (!vsi) { 10708 ret = -ENOMEM; 10709 goto unlock_pf; 10710 } 10711 vsi->type = type; 10712 vsi->back = pf; 10713 set_bit(__I40E_VSI_DOWN, vsi->state); 10714 vsi->flags = 0; 10715 vsi->idx = vsi_idx; 10716 vsi->int_rate_limit = 0; 10717 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 10718 pf->rss_table_size : 64; 10719 vsi->netdev_registered = false; 10720 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 10721 hash_init(vsi->mac_filter_hash); 10722 vsi->irqs_ready = false; 10723 10724 if (type == I40E_VSI_MAIN) { 10725 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL); 10726 if (!vsi->af_xdp_zc_qps) 10727 goto err_rings; 10728 } 10729 10730 ret = i40e_set_num_rings_in_vsi(vsi); 10731 if (ret) 10732 goto err_rings; 10733 10734 ret = i40e_vsi_alloc_arrays(vsi, true); 10735 if (ret) 10736 goto err_rings; 10737 10738 /* Setup default MSIX irq handler for VSI */ 10739 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 10740 10741 /* Initialize VSI lock */ 10742 spin_lock_init(&vsi->mac_filter_hash_lock); 10743 pf->vsi[vsi_idx] = vsi; 10744 ret = vsi_idx; 10745 goto unlock_pf; 10746 10747 err_rings: 10748 bitmap_free(vsi->af_xdp_zc_qps); 10749 pf->next_vsi = i - 1; 10750 kfree(vsi); 10751 unlock_pf: 10752 mutex_unlock(&pf->switch_mutex); 10753 return ret; 10754 } 10755 10756 /** 10757 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 10758 * @vsi: VSI pointer 10759 * @free_qvectors: a bool to specify if q_vectors need to be freed. 10760 * 10761 * On error: returns error code (negative) 10762 * On success: returns 0 10763 **/ 10764 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 10765 { 10766 /* free the ring and vector containers */ 10767 if (free_qvectors) { 10768 kfree(vsi->q_vectors); 10769 vsi->q_vectors = NULL; 10770 } 10771 kfree(vsi->tx_rings); 10772 vsi->tx_rings = NULL; 10773 vsi->rx_rings = NULL; 10774 vsi->xdp_rings = NULL; 10775 } 10776 10777 /** 10778 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 10779 * and lookup table 10780 * @vsi: Pointer to VSI structure 10781 */ 10782 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 10783 { 10784 if (!vsi) 10785 return; 10786 10787 kfree(vsi->rss_hkey_user); 10788 vsi->rss_hkey_user = NULL; 10789 10790 kfree(vsi->rss_lut_user); 10791 vsi->rss_lut_user = NULL; 10792 } 10793 10794 /** 10795 * i40e_vsi_clear - Deallocate the VSI provided 10796 * @vsi: the VSI being un-configured 10797 **/ 10798 static int i40e_vsi_clear(struct i40e_vsi *vsi) 10799 { 10800 struct i40e_pf *pf; 10801 10802 if (!vsi) 10803 return 0; 10804 10805 if (!vsi->back) 10806 goto free_vsi; 10807 pf = vsi->back; 10808 10809 mutex_lock(&pf->switch_mutex); 10810 if (!pf->vsi[vsi->idx]) { 10811 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 10812 vsi->idx, vsi->idx, vsi->type); 10813 goto unlock_vsi; 10814 } 10815 10816 if (pf->vsi[vsi->idx] != vsi) { 10817 dev_err(&pf->pdev->dev, 10818 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 10819 pf->vsi[vsi->idx]->idx, 10820 pf->vsi[vsi->idx]->type, 10821 vsi->idx, vsi->type); 10822 goto unlock_vsi; 10823 } 10824 10825 /* updates the PF for this cleared vsi */ 10826 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 10827 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 10828 10829 bitmap_free(vsi->af_xdp_zc_qps); 10830 i40e_vsi_free_arrays(vsi, true); 10831 i40e_clear_rss_config_user(vsi); 10832 10833 pf->vsi[vsi->idx] = NULL; 10834 if (vsi->idx < pf->next_vsi) 10835 pf->next_vsi = vsi->idx; 10836 10837 unlock_vsi: 10838 mutex_unlock(&pf->switch_mutex); 10839 free_vsi: 10840 kfree(vsi); 10841 10842 return 0; 10843 } 10844 10845 /** 10846 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 10847 * @vsi: the VSI being cleaned 10848 **/ 10849 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 10850 { 10851 int i; 10852 10853 if (vsi->tx_rings && vsi->tx_rings[0]) { 10854 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10855 kfree_rcu(vsi->tx_rings[i], rcu); 10856 WRITE_ONCE(vsi->tx_rings[i], NULL); 10857 WRITE_ONCE(vsi->rx_rings[i], NULL); 10858 if (vsi->xdp_rings) 10859 WRITE_ONCE(vsi->xdp_rings[i], NULL); 10860 } 10861 } 10862 } 10863 10864 /** 10865 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 10866 * @vsi: the VSI being configured 10867 **/ 10868 static int i40e_alloc_rings(struct i40e_vsi *vsi) 10869 { 10870 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 10871 struct i40e_pf *pf = vsi->back; 10872 struct i40e_ring *ring; 10873 10874 /* Set basic values in the rings to be used later during open() */ 10875 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10876 /* allocate space for both Tx and Rx in one shot */ 10877 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 10878 if (!ring) 10879 goto err_out; 10880 10881 ring->queue_index = i; 10882 ring->reg_idx = vsi->base_queue + i; 10883 ring->ring_active = false; 10884 ring->vsi = vsi; 10885 ring->netdev = vsi->netdev; 10886 ring->dev = &pf->pdev->dev; 10887 ring->count = vsi->num_tx_desc; 10888 ring->size = 0; 10889 ring->dcb_tc = 0; 10890 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10891 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10892 ring->itr_setting = pf->tx_itr_default; 10893 WRITE_ONCE(vsi->tx_rings[i], ring++); 10894 10895 if (!i40e_enabled_xdp_vsi(vsi)) 10896 goto setup_rx; 10897 10898 ring->queue_index = vsi->alloc_queue_pairs + i; 10899 ring->reg_idx = vsi->base_queue + ring->queue_index; 10900 ring->ring_active = false; 10901 ring->vsi = vsi; 10902 ring->netdev = NULL; 10903 ring->dev = &pf->pdev->dev; 10904 ring->count = vsi->num_tx_desc; 10905 ring->size = 0; 10906 ring->dcb_tc = 0; 10907 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10908 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10909 set_ring_xdp(ring); 10910 ring->itr_setting = pf->tx_itr_default; 10911 WRITE_ONCE(vsi->xdp_rings[i], ring++); 10912 10913 setup_rx: 10914 ring->queue_index = i; 10915 ring->reg_idx = vsi->base_queue + i; 10916 ring->ring_active = false; 10917 ring->vsi = vsi; 10918 ring->netdev = vsi->netdev; 10919 ring->dev = &pf->pdev->dev; 10920 ring->count = vsi->num_rx_desc; 10921 ring->size = 0; 10922 ring->dcb_tc = 0; 10923 ring->itr_setting = pf->rx_itr_default; 10924 WRITE_ONCE(vsi->rx_rings[i], ring); 10925 } 10926 10927 return 0; 10928 10929 err_out: 10930 i40e_vsi_clear_rings(vsi); 10931 return -ENOMEM; 10932 } 10933 10934 /** 10935 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 10936 * @pf: board private structure 10937 * @vectors: the number of MSI-X vectors to request 10938 * 10939 * Returns the number of vectors reserved, or error 10940 **/ 10941 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 10942 { 10943 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 10944 I40E_MIN_MSIX, vectors); 10945 if (vectors < 0) { 10946 dev_info(&pf->pdev->dev, 10947 "MSI-X vector reservation failed: %d\n", vectors); 10948 vectors = 0; 10949 } 10950 10951 return vectors; 10952 } 10953 10954 /** 10955 * i40e_init_msix - Setup the MSIX capability 10956 * @pf: board private structure 10957 * 10958 * Work with the OS to set up the MSIX vectors needed. 10959 * 10960 * Returns the number of vectors reserved or negative on failure 10961 **/ 10962 static int i40e_init_msix(struct i40e_pf *pf) 10963 { 10964 struct i40e_hw *hw = &pf->hw; 10965 int cpus, extra_vectors; 10966 int vectors_left; 10967 int v_budget, i; 10968 int v_actual; 10969 int iwarp_requested = 0; 10970 10971 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10972 return -ENODEV; 10973 10974 /* The number of vectors we'll request will be comprised of: 10975 * - Add 1 for "other" cause for Admin Queue events, etc. 10976 * - The number of LAN queue pairs 10977 * - Queues being used for RSS. 10978 * We don't need as many as max_rss_size vectors. 10979 * use rss_size instead in the calculation since that 10980 * is governed by number of cpus in the system. 10981 * - assumes symmetric Tx/Rx pairing 10982 * - The number of VMDq pairs 10983 * - The CPU count within the NUMA node if iWARP is enabled 10984 * Once we count this up, try the request. 10985 * 10986 * If we can't get what we want, we'll simplify to nearly nothing 10987 * and try again. If that still fails, we punt. 10988 */ 10989 vectors_left = hw->func_caps.num_msix_vectors; 10990 v_budget = 0; 10991 10992 /* reserve one vector for miscellaneous handler */ 10993 if (vectors_left) { 10994 v_budget++; 10995 vectors_left--; 10996 } 10997 10998 /* reserve some vectors for the main PF traffic queues. Initially we 10999 * only reserve at most 50% of the available vectors, in the case that 11000 * the number of online CPUs is large. This ensures that we can enable 11001 * extra features as well. Once we've enabled the other features, we 11002 * will use any remaining vectors to reach as close as we can to the 11003 * number of online CPUs. 11004 */ 11005 cpus = num_online_cpus(); 11006 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 11007 vectors_left -= pf->num_lan_msix; 11008 11009 /* reserve one vector for sideband flow director */ 11010 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11011 if (vectors_left) { 11012 pf->num_fdsb_msix = 1; 11013 v_budget++; 11014 vectors_left--; 11015 } else { 11016 pf->num_fdsb_msix = 0; 11017 } 11018 } 11019 11020 /* can we reserve enough for iWARP? */ 11021 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11022 iwarp_requested = pf->num_iwarp_msix; 11023 11024 if (!vectors_left) 11025 pf->num_iwarp_msix = 0; 11026 else if (vectors_left < pf->num_iwarp_msix) 11027 pf->num_iwarp_msix = 1; 11028 v_budget += pf->num_iwarp_msix; 11029 vectors_left -= pf->num_iwarp_msix; 11030 } 11031 11032 /* any vectors left over go for VMDq support */ 11033 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 11034 if (!vectors_left) { 11035 pf->num_vmdq_msix = 0; 11036 pf->num_vmdq_qps = 0; 11037 } else { 11038 int vmdq_vecs_wanted = 11039 pf->num_vmdq_vsis * pf->num_vmdq_qps; 11040 int vmdq_vecs = 11041 min_t(int, vectors_left, vmdq_vecs_wanted); 11042 11043 /* if we're short on vectors for what's desired, we limit 11044 * the queues per vmdq. If this is still more than are 11045 * available, the user will need to change the number of 11046 * queues/vectors used by the PF later with the ethtool 11047 * channels command 11048 */ 11049 if (vectors_left < vmdq_vecs_wanted) { 11050 pf->num_vmdq_qps = 1; 11051 vmdq_vecs_wanted = pf->num_vmdq_vsis; 11052 vmdq_vecs = min_t(int, 11053 vectors_left, 11054 vmdq_vecs_wanted); 11055 } 11056 pf->num_vmdq_msix = pf->num_vmdq_qps; 11057 11058 v_budget += vmdq_vecs; 11059 vectors_left -= vmdq_vecs; 11060 } 11061 } 11062 11063 /* On systems with a large number of SMP cores, we previously limited 11064 * the number of vectors for num_lan_msix to be at most 50% of the 11065 * available vectors, to allow for other features. Now, we add back 11066 * the remaining vectors. However, we ensure that the total 11067 * num_lan_msix will not exceed num_online_cpus(). To do this, we 11068 * calculate the number of vectors we can add without going over the 11069 * cap of CPUs. For systems with a small number of CPUs this will be 11070 * zero. 11071 */ 11072 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 11073 pf->num_lan_msix += extra_vectors; 11074 vectors_left -= extra_vectors; 11075 11076 WARN(vectors_left < 0, 11077 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 11078 11079 v_budget += pf->num_lan_msix; 11080 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 11081 GFP_KERNEL); 11082 if (!pf->msix_entries) 11083 return -ENOMEM; 11084 11085 for (i = 0; i < v_budget; i++) 11086 pf->msix_entries[i].entry = i; 11087 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 11088 11089 if (v_actual < I40E_MIN_MSIX) { 11090 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 11091 kfree(pf->msix_entries); 11092 pf->msix_entries = NULL; 11093 pci_disable_msix(pf->pdev); 11094 return -ENODEV; 11095 11096 } else if (v_actual == I40E_MIN_MSIX) { 11097 /* Adjust for minimal MSIX use */ 11098 pf->num_vmdq_vsis = 0; 11099 pf->num_vmdq_qps = 0; 11100 pf->num_lan_qps = 1; 11101 pf->num_lan_msix = 1; 11102 11103 } else if (v_actual != v_budget) { 11104 /* If we have limited resources, we will start with no vectors 11105 * for the special features and then allocate vectors to some 11106 * of these features based on the policy and at the end disable 11107 * the features that did not get any vectors. 11108 */ 11109 int vec; 11110 11111 dev_info(&pf->pdev->dev, 11112 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 11113 v_actual, v_budget); 11114 /* reserve the misc vector */ 11115 vec = v_actual - 1; 11116 11117 /* Scale vector usage down */ 11118 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 11119 pf->num_vmdq_vsis = 1; 11120 pf->num_vmdq_qps = 1; 11121 11122 /* partition out the remaining vectors */ 11123 switch (vec) { 11124 case 2: 11125 pf->num_lan_msix = 1; 11126 break; 11127 case 3: 11128 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11129 pf->num_lan_msix = 1; 11130 pf->num_iwarp_msix = 1; 11131 } else { 11132 pf->num_lan_msix = 2; 11133 } 11134 break; 11135 default: 11136 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11137 pf->num_iwarp_msix = min_t(int, (vec / 3), 11138 iwarp_requested); 11139 pf->num_vmdq_vsis = min_t(int, (vec / 3), 11140 I40E_DEFAULT_NUM_VMDQ_VSI); 11141 } else { 11142 pf->num_vmdq_vsis = min_t(int, (vec / 2), 11143 I40E_DEFAULT_NUM_VMDQ_VSI); 11144 } 11145 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11146 pf->num_fdsb_msix = 1; 11147 vec--; 11148 } 11149 pf->num_lan_msix = min_t(int, 11150 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 11151 pf->num_lan_msix); 11152 pf->num_lan_qps = pf->num_lan_msix; 11153 break; 11154 } 11155 } 11156 11157 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 11158 (pf->num_fdsb_msix == 0)) { 11159 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 11160 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 11161 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11162 } 11163 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 11164 (pf->num_vmdq_msix == 0)) { 11165 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 11166 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 11167 } 11168 11169 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 11170 (pf->num_iwarp_msix == 0)) { 11171 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 11172 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11173 } 11174 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 11175 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 11176 pf->num_lan_msix, 11177 pf->num_vmdq_msix * pf->num_vmdq_vsis, 11178 pf->num_fdsb_msix, 11179 pf->num_iwarp_msix); 11180 11181 return v_actual; 11182 } 11183 11184 /** 11185 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 11186 * @vsi: the VSI being configured 11187 * @v_idx: index of the vector in the vsi struct 11188 * @cpu: cpu to be used on affinity_mask 11189 * 11190 * We allocate one q_vector. If allocation fails we return -ENOMEM. 11191 **/ 11192 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 11193 { 11194 struct i40e_q_vector *q_vector; 11195 11196 /* allocate q_vector */ 11197 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 11198 if (!q_vector) 11199 return -ENOMEM; 11200 11201 q_vector->vsi = vsi; 11202 q_vector->v_idx = v_idx; 11203 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 11204 11205 if (vsi->netdev) 11206 netif_napi_add(vsi->netdev, &q_vector->napi, 11207 i40e_napi_poll, NAPI_POLL_WEIGHT); 11208 11209 /* tie q_vector and vsi together */ 11210 vsi->q_vectors[v_idx] = q_vector; 11211 11212 return 0; 11213 } 11214 11215 /** 11216 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 11217 * @vsi: the VSI being configured 11218 * 11219 * We allocate one q_vector per queue interrupt. If allocation fails we 11220 * return -ENOMEM. 11221 **/ 11222 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 11223 { 11224 struct i40e_pf *pf = vsi->back; 11225 int err, v_idx, num_q_vectors, current_cpu; 11226 11227 /* if not MSIX, give the one vector only to the LAN VSI */ 11228 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 11229 num_q_vectors = vsi->num_q_vectors; 11230 else if (vsi == pf->vsi[pf->lan_vsi]) 11231 num_q_vectors = 1; 11232 else 11233 return -EINVAL; 11234 11235 current_cpu = cpumask_first(cpu_online_mask); 11236 11237 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 11238 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 11239 if (err) 11240 goto err_out; 11241 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 11242 if (unlikely(current_cpu >= nr_cpu_ids)) 11243 current_cpu = cpumask_first(cpu_online_mask); 11244 } 11245 11246 return 0; 11247 11248 err_out: 11249 while (v_idx--) 11250 i40e_free_q_vector(vsi, v_idx); 11251 11252 return err; 11253 } 11254 11255 /** 11256 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 11257 * @pf: board private structure to initialize 11258 **/ 11259 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 11260 { 11261 int vectors = 0; 11262 ssize_t size; 11263 11264 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11265 vectors = i40e_init_msix(pf); 11266 if (vectors < 0) { 11267 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 11268 I40E_FLAG_IWARP_ENABLED | 11269 I40E_FLAG_RSS_ENABLED | 11270 I40E_FLAG_DCB_CAPABLE | 11271 I40E_FLAG_DCB_ENABLED | 11272 I40E_FLAG_SRIOV_ENABLED | 11273 I40E_FLAG_FD_SB_ENABLED | 11274 I40E_FLAG_FD_ATR_ENABLED | 11275 I40E_FLAG_VMDQ_ENABLED); 11276 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11277 11278 /* rework the queue expectations without MSIX */ 11279 i40e_determine_queue_usage(pf); 11280 } 11281 } 11282 11283 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 11284 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 11285 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 11286 vectors = pci_enable_msi(pf->pdev); 11287 if (vectors < 0) { 11288 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 11289 vectors); 11290 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 11291 } 11292 vectors = 1; /* one MSI or Legacy vector */ 11293 } 11294 11295 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 11296 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 11297 11298 /* set up vector assignment tracking */ 11299 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 11300 pf->irq_pile = kzalloc(size, GFP_KERNEL); 11301 if (!pf->irq_pile) 11302 return -ENOMEM; 11303 11304 pf->irq_pile->num_entries = vectors; 11305 pf->irq_pile->search_hint = 0; 11306 11307 /* track first vector for misc interrupts, ignore return */ 11308 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 11309 11310 return 0; 11311 } 11312 11313 /** 11314 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 11315 * @pf: private board data structure 11316 * 11317 * Restore the interrupt scheme that was cleared when we suspended the 11318 * device. This should be called during resume to re-allocate the q_vectors 11319 * and reacquire IRQs. 11320 */ 11321 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 11322 { 11323 int err, i; 11324 11325 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 11326 * scheme. We need to re-enabled them here in order to attempt to 11327 * re-acquire the MSI or MSI-X vectors 11328 */ 11329 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 11330 11331 err = i40e_init_interrupt_scheme(pf); 11332 if (err) 11333 return err; 11334 11335 /* Now that we've re-acquired IRQs, we need to remap the vectors and 11336 * rings together again. 11337 */ 11338 for (i = 0; i < pf->num_alloc_vsi; i++) { 11339 if (pf->vsi[i]) { 11340 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 11341 if (err) 11342 goto err_unwind; 11343 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 11344 } 11345 } 11346 11347 err = i40e_setup_misc_vector(pf); 11348 if (err) 11349 goto err_unwind; 11350 11351 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 11352 i40e_client_update_msix_info(pf); 11353 11354 return 0; 11355 11356 err_unwind: 11357 while (i--) { 11358 if (pf->vsi[i]) 11359 i40e_vsi_free_q_vectors(pf->vsi[i]); 11360 } 11361 11362 return err; 11363 } 11364 11365 /** 11366 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle 11367 * non queue events in recovery mode 11368 * @pf: board private structure 11369 * 11370 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage 11371 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode. 11372 * This is handled differently than in recovery mode since no Tx/Rx resources 11373 * are being allocated. 11374 **/ 11375 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf) 11376 { 11377 int err; 11378 11379 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11380 err = i40e_setup_misc_vector(pf); 11381 11382 if (err) { 11383 dev_info(&pf->pdev->dev, 11384 "MSI-X misc vector request failed, error %d\n", 11385 err); 11386 return err; 11387 } 11388 } else { 11389 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED; 11390 11391 err = request_irq(pf->pdev->irq, i40e_intr, flags, 11392 pf->int_name, pf); 11393 11394 if (err) { 11395 dev_info(&pf->pdev->dev, 11396 "MSI/legacy misc vector request failed, error %d\n", 11397 err); 11398 return err; 11399 } 11400 i40e_enable_misc_int_causes(pf); 11401 i40e_irq_dynamic_enable_icr0(pf); 11402 } 11403 11404 return 0; 11405 } 11406 11407 /** 11408 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 11409 * @pf: board private structure 11410 * 11411 * This sets up the handler for MSIX 0, which is used to manage the 11412 * non-queue interrupts, e.g. AdminQ and errors. This is not used 11413 * when in MSI or Legacy interrupt mode. 11414 **/ 11415 static int i40e_setup_misc_vector(struct i40e_pf *pf) 11416 { 11417 struct i40e_hw *hw = &pf->hw; 11418 int err = 0; 11419 11420 /* Only request the IRQ once, the first time through. */ 11421 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 11422 err = request_irq(pf->msix_entries[0].vector, 11423 i40e_intr, 0, pf->int_name, pf); 11424 if (err) { 11425 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 11426 dev_info(&pf->pdev->dev, 11427 "request_irq for %s failed: %d\n", 11428 pf->int_name, err); 11429 return -EFAULT; 11430 } 11431 } 11432 11433 i40e_enable_misc_int_causes(pf); 11434 11435 /* associate no queues to the misc vector */ 11436 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 11437 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1); 11438 11439 i40e_flush(hw); 11440 11441 i40e_irq_dynamic_enable_icr0(pf); 11442 11443 return err; 11444 } 11445 11446 /** 11447 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 11448 * @vsi: Pointer to vsi structure 11449 * @seed: Buffter to store the hash keys 11450 * @lut: Buffer to store the lookup table entries 11451 * @lut_size: Size of buffer to store the lookup table entries 11452 * 11453 * Return 0 on success, negative on failure 11454 */ 11455 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 11456 u8 *lut, u16 lut_size) 11457 { 11458 struct i40e_pf *pf = vsi->back; 11459 struct i40e_hw *hw = &pf->hw; 11460 int ret = 0; 11461 11462 if (seed) { 11463 ret = i40e_aq_get_rss_key(hw, vsi->id, 11464 (struct i40e_aqc_get_set_rss_key_data *)seed); 11465 if (ret) { 11466 dev_info(&pf->pdev->dev, 11467 "Cannot get RSS key, err %s aq_err %s\n", 11468 i40e_stat_str(&pf->hw, ret), 11469 i40e_aq_str(&pf->hw, 11470 pf->hw.aq.asq_last_status)); 11471 return ret; 11472 } 11473 } 11474 11475 if (lut) { 11476 bool pf_lut = vsi->type == I40E_VSI_MAIN; 11477 11478 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 11479 if (ret) { 11480 dev_info(&pf->pdev->dev, 11481 "Cannot get RSS lut, err %s aq_err %s\n", 11482 i40e_stat_str(&pf->hw, ret), 11483 i40e_aq_str(&pf->hw, 11484 pf->hw.aq.asq_last_status)); 11485 return ret; 11486 } 11487 } 11488 11489 return ret; 11490 } 11491 11492 /** 11493 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 11494 * @vsi: Pointer to vsi structure 11495 * @seed: RSS hash seed 11496 * @lut: Lookup table 11497 * @lut_size: Lookup table size 11498 * 11499 * Returns 0 on success, negative on failure 11500 **/ 11501 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 11502 const u8 *lut, u16 lut_size) 11503 { 11504 struct i40e_pf *pf = vsi->back; 11505 struct i40e_hw *hw = &pf->hw; 11506 u16 vf_id = vsi->vf_id; 11507 u8 i; 11508 11509 /* Fill out hash function seed */ 11510 if (seed) { 11511 u32 *seed_dw = (u32 *)seed; 11512 11513 if (vsi->type == I40E_VSI_MAIN) { 11514 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 11515 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 11516 } else if (vsi->type == I40E_VSI_SRIOV) { 11517 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 11518 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 11519 } else { 11520 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 11521 } 11522 } 11523 11524 if (lut) { 11525 u32 *lut_dw = (u32 *)lut; 11526 11527 if (vsi->type == I40E_VSI_MAIN) { 11528 if (lut_size != I40E_HLUT_ARRAY_SIZE) 11529 return -EINVAL; 11530 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11531 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 11532 } else if (vsi->type == I40E_VSI_SRIOV) { 11533 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 11534 return -EINVAL; 11535 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 11536 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 11537 } else { 11538 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 11539 } 11540 } 11541 i40e_flush(hw); 11542 11543 return 0; 11544 } 11545 11546 /** 11547 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 11548 * @vsi: Pointer to VSI structure 11549 * @seed: Buffer to store the keys 11550 * @lut: Buffer to store the lookup table entries 11551 * @lut_size: Size of buffer to store the lookup table entries 11552 * 11553 * Returns 0 on success, negative on failure 11554 */ 11555 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 11556 u8 *lut, u16 lut_size) 11557 { 11558 struct i40e_pf *pf = vsi->back; 11559 struct i40e_hw *hw = &pf->hw; 11560 u16 i; 11561 11562 if (seed) { 11563 u32 *seed_dw = (u32 *)seed; 11564 11565 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 11566 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 11567 } 11568 if (lut) { 11569 u32 *lut_dw = (u32 *)lut; 11570 11571 if (lut_size != I40E_HLUT_ARRAY_SIZE) 11572 return -EINVAL; 11573 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11574 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 11575 } 11576 11577 return 0; 11578 } 11579 11580 /** 11581 * i40e_config_rss - Configure RSS keys and lut 11582 * @vsi: Pointer to VSI structure 11583 * @seed: RSS hash seed 11584 * @lut: Lookup table 11585 * @lut_size: Lookup table size 11586 * 11587 * Returns 0 on success, negative on failure 11588 */ 11589 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 11590 { 11591 struct i40e_pf *pf = vsi->back; 11592 11593 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 11594 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 11595 else 11596 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 11597 } 11598 11599 /** 11600 * i40e_get_rss - Get RSS keys and lut 11601 * @vsi: Pointer to VSI structure 11602 * @seed: Buffer to store the keys 11603 * @lut: Buffer to store the lookup table entries 11604 * @lut_size: Size of buffer to store the lookup table entries 11605 * 11606 * Returns 0 on success, negative on failure 11607 */ 11608 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 11609 { 11610 struct i40e_pf *pf = vsi->back; 11611 11612 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 11613 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 11614 else 11615 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 11616 } 11617 11618 /** 11619 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 11620 * @pf: Pointer to board private structure 11621 * @lut: Lookup table 11622 * @rss_table_size: Lookup table size 11623 * @rss_size: Range of queue number for hashing 11624 */ 11625 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 11626 u16 rss_table_size, u16 rss_size) 11627 { 11628 u16 i; 11629 11630 for (i = 0; i < rss_table_size; i++) 11631 lut[i] = i % rss_size; 11632 } 11633 11634 /** 11635 * i40e_pf_config_rss - Prepare for RSS if used 11636 * @pf: board private structure 11637 **/ 11638 static int i40e_pf_config_rss(struct i40e_pf *pf) 11639 { 11640 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11641 u8 seed[I40E_HKEY_ARRAY_SIZE]; 11642 u8 *lut; 11643 struct i40e_hw *hw = &pf->hw; 11644 u32 reg_val; 11645 u64 hena; 11646 int ret; 11647 11648 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 11649 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 11650 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 11651 hena |= i40e_pf_get_default_rss_hena(pf); 11652 11653 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 11654 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 11655 11656 /* Determine the RSS table size based on the hardware capabilities */ 11657 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 11658 reg_val = (pf->rss_table_size == 512) ? 11659 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 11660 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 11661 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 11662 11663 /* Determine the RSS size of the VSI */ 11664 if (!vsi->rss_size) { 11665 u16 qcount; 11666 /* If the firmware does something weird during VSI init, we 11667 * could end up with zero TCs. Check for that to avoid 11668 * divide-by-zero. It probably won't pass traffic, but it also 11669 * won't panic. 11670 */ 11671 qcount = vsi->num_queue_pairs / 11672 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 11673 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11674 } 11675 if (!vsi->rss_size) 11676 return -EINVAL; 11677 11678 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 11679 if (!lut) 11680 return -ENOMEM; 11681 11682 /* Use user configured lut if there is one, otherwise use default */ 11683 if (vsi->rss_lut_user) 11684 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 11685 else 11686 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 11687 11688 /* Use user configured hash key if there is one, otherwise 11689 * use default. 11690 */ 11691 if (vsi->rss_hkey_user) 11692 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 11693 else 11694 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 11695 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 11696 kfree(lut); 11697 11698 return ret; 11699 } 11700 11701 /** 11702 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 11703 * @pf: board private structure 11704 * @queue_count: the requested queue count for rss. 11705 * 11706 * returns 0 if rss is not enabled, if enabled returns the final rss queue 11707 * count which may be different from the requested queue count. 11708 * Note: expects to be called while under rtnl_lock() 11709 **/ 11710 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 11711 { 11712 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11713 int new_rss_size; 11714 11715 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 11716 return 0; 11717 11718 queue_count = min_t(int, queue_count, num_online_cpus()); 11719 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 11720 11721 if (queue_count != vsi->num_queue_pairs) { 11722 u16 qcount; 11723 11724 vsi->req_queue_pairs = queue_count; 11725 i40e_prep_for_reset(pf, true); 11726 11727 pf->alloc_rss_size = new_rss_size; 11728 11729 i40e_reset_and_rebuild(pf, true, true); 11730 11731 /* Discard the user configured hash keys and lut, if less 11732 * queues are enabled. 11733 */ 11734 if (queue_count < vsi->rss_size) { 11735 i40e_clear_rss_config_user(vsi); 11736 dev_dbg(&pf->pdev->dev, 11737 "discard user configured hash keys and lut\n"); 11738 } 11739 11740 /* Reset vsi->rss_size, as number of enabled queues changed */ 11741 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 11742 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11743 11744 i40e_pf_config_rss(pf); 11745 } 11746 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 11747 vsi->req_queue_pairs, pf->rss_size_max); 11748 return pf->alloc_rss_size; 11749 } 11750 11751 /** 11752 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 11753 * @pf: board private structure 11754 **/ 11755 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 11756 { 11757 i40e_status status; 11758 bool min_valid, max_valid; 11759 u32 max_bw, min_bw; 11760 11761 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 11762 &min_valid, &max_valid); 11763 11764 if (!status) { 11765 if (min_valid) 11766 pf->min_bw = min_bw; 11767 if (max_valid) 11768 pf->max_bw = max_bw; 11769 } 11770 11771 return status; 11772 } 11773 11774 /** 11775 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 11776 * @pf: board private structure 11777 **/ 11778 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 11779 { 11780 struct i40e_aqc_configure_partition_bw_data bw_data; 11781 i40e_status status; 11782 11783 /* Set the valid bit for this PF */ 11784 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 11785 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 11786 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 11787 11788 /* Set the new bandwidths */ 11789 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 11790 11791 return status; 11792 } 11793 11794 /** 11795 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 11796 * @pf: board private structure 11797 **/ 11798 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 11799 { 11800 /* Commit temporary BW setting to permanent NVM image */ 11801 enum i40e_admin_queue_err last_aq_status; 11802 i40e_status ret; 11803 u16 nvm_word; 11804 11805 if (pf->hw.partition_id != 1) { 11806 dev_info(&pf->pdev->dev, 11807 "Commit BW only works on partition 1! This is partition %d", 11808 pf->hw.partition_id); 11809 ret = I40E_NOT_SUPPORTED; 11810 goto bw_commit_out; 11811 } 11812 11813 /* Acquire NVM for read access */ 11814 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 11815 last_aq_status = pf->hw.aq.asq_last_status; 11816 if (ret) { 11817 dev_info(&pf->pdev->dev, 11818 "Cannot acquire NVM for read access, err %s aq_err %s\n", 11819 i40e_stat_str(&pf->hw, ret), 11820 i40e_aq_str(&pf->hw, last_aq_status)); 11821 goto bw_commit_out; 11822 } 11823 11824 /* Read word 0x10 of NVM - SW compatibility word 1 */ 11825 ret = i40e_aq_read_nvm(&pf->hw, 11826 I40E_SR_NVM_CONTROL_WORD, 11827 0x10, sizeof(nvm_word), &nvm_word, 11828 false, NULL); 11829 /* Save off last admin queue command status before releasing 11830 * the NVM 11831 */ 11832 last_aq_status = pf->hw.aq.asq_last_status; 11833 i40e_release_nvm(&pf->hw); 11834 if (ret) { 11835 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 11836 i40e_stat_str(&pf->hw, ret), 11837 i40e_aq_str(&pf->hw, last_aq_status)); 11838 goto bw_commit_out; 11839 } 11840 11841 /* Wait a bit for NVM release to complete */ 11842 msleep(50); 11843 11844 /* Acquire NVM for write access */ 11845 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 11846 last_aq_status = pf->hw.aq.asq_last_status; 11847 if (ret) { 11848 dev_info(&pf->pdev->dev, 11849 "Cannot acquire NVM for write access, err %s aq_err %s\n", 11850 i40e_stat_str(&pf->hw, ret), 11851 i40e_aq_str(&pf->hw, last_aq_status)); 11852 goto bw_commit_out; 11853 } 11854 /* Write it back out unchanged to initiate update NVM, 11855 * which will force a write of the shadow (alt) RAM to 11856 * the NVM - thus storing the bandwidth values permanently. 11857 */ 11858 ret = i40e_aq_update_nvm(&pf->hw, 11859 I40E_SR_NVM_CONTROL_WORD, 11860 0x10, sizeof(nvm_word), 11861 &nvm_word, true, 0, NULL); 11862 /* Save off last admin queue command status before releasing 11863 * the NVM 11864 */ 11865 last_aq_status = pf->hw.aq.asq_last_status; 11866 i40e_release_nvm(&pf->hw); 11867 if (ret) 11868 dev_info(&pf->pdev->dev, 11869 "BW settings NOT SAVED, err %s aq_err %s\n", 11870 i40e_stat_str(&pf->hw, ret), 11871 i40e_aq_str(&pf->hw, last_aq_status)); 11872 bw_commit_out: 11873 11874 return ret; 11875 } 11876 11877 /** 11878 * i40e_is_total_port_shutdown_enabled - read NVM and return value 11879 * if total port shutdown feature is enabled for this PF 11880 * @pf: board private structure 11881 **/ 11882 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf) 11883 { 11884 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4) 11885 #define I40E_FEATURES_ENABLE_PTR 0x2A 11886 #define I40E_CURRENT_SETTING_PTR 0x2B 11887 #define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D 11888 #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1 11889 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0) 11890 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4 11891 i40e_status read_status = I40E_SUCCESS; 11892 u16 sr_emp_sr_settings_ptr = 0; 11893 u16 features_enable = 0; 11894 u16 link_behavior = 0; 11895 bool ret = false; 11896 11897 read_status = i40e_read_nvm_word(&pf->hw, 11898 I40E_SR_EMP_SR_SETTINGS_PTR, 11899 &sr_emp_sr_settings_ptr); 11900 if (read_status) 11901 goto err_nvm; 11902 read_status = i40e_read_nvm_word(&pf->hw, 11903 sr_emp_sr_settings_ptr + 11904 I40E_FEATURES_ENABLE_PTR, 11905 &features_enable); 11906 if (read_status) 11907 goto err_nvm; 11908 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) { 11909 read_status = i40e_read_nvm_module_data(&pf->hw, 11910 I40E_SR_EMP_SR_SETTINGS_PTR, 11911 I40E_CURRENT_SETTING_PTR, 11912 I40E_LINK_BEHAVIOR_WORD_OFFSET, 11913 I40E_LINK_BEHAVIOR_WORD_LENGTH, 11914 &link_behavior); 11915 if (read_status) 11916 goto err_nvm; 11917 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH); 11918 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior; 11919 } 11920 return ret; 11921 11922 err_nvm: 11923 dev_warn(&pf->pdev->dev, 11924 "total-port-shutdown feature is off due to read nvm error: %s\n", 11925 i40e_stat_str(&pf->hw, read_status)); 11926 return ret; 11927 } 11928 11929 /** 11930 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 11931 * @pf: board private structure to initialize 11932 * 11933 * i40e_sw_init initializes the Adapter private data structure. 11934 * Fields are initialized based on PCI device information and 11935 * OS network device settings (MTU size). 11936 **/ 11937 static int i40e_sw_init(struct i40e_pf *pf) 11938 { 11939 int err = 0; 11940 int size; 11941 11942 /* Set default capability flags */ 11943 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 11944 I40E_FLAG_MSI_ENABLED | 11945 I40E_FLAG_MSIX_ENABLED; 11946 11947 /* Set default ITR */ 11948 pf->rx_itr_default = I40E_ITR_RX_DEF; 11949 pf->tx_itr_default = I40E_ITR_TX_DEF; 11950 11951 /* Depending on PF configurations, it is possible that the RSS 11952 * maximum might end up larger than the available queues 11953 */ 11954 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 11955 pf->alloc_rss_size = 1; 11956 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 11957 pf->rss_size_max = min_t(int, pf->rss_size_max, 11958 pf->hw.func_caps.num_tx_qp); 11959 if (pf->hw.func_caps.rss) { 11960 pf->flags |= I40E_FLAG_RSS_ENABLED; 11961 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 11962 num_online_cpus()); 11963 } 11964 11965 /* MFP mode enabled */ 11966 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 11967 pf->flags |= I40E_FLAG_MFP_ENABLED; 11968 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 11969 if (i40e_get_partition_bw_setting(pf)) { 11970 dev_warn(&pf->pdev->dev, 11971 "Could not get partition bw settings\n"); 11972 } else { 11973 dev_info(&pf->pdev->dev, 11974 "Partition BW Min = %8.8x, Max = %8.8x\n", 11975 pf->min_bw, pf->max_bw); 11976 11977 /* nudge the Tx scheduler */ 11978 i40e_set_partition_bw_setting(pf); 11979 } 11980 } 11981 11982 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 11983 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 11984 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 11985 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 11986 if (pf->flags & I40E_FLAG_MFP_ENABLED && 11987 pf->hw.num_partitions > 1) 11988 dev_info(&pf->pdev->dev, 11989 "Flow Director Sideband mode Disabled in MFP mode\n"); 11990 else 11991 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11992 pf->fdir_pf_filter_count = 11993 pf->hw.func_caps.fd_filters_guaranteed; 11994 pf->hw.fdir_shared_filter_count = 11995 pf->hw.func_caps.fd_filters_best_effort; 11996 } 11997 11998 if (pf->hw.mac.type == I40E_MAC_X722) { 11999 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 12000 I40E_HW_128_QP_RSS_CAPABLE | 12001 I40E_HW_ATR_EVICT_CAPABLE | 12002 I40E_HW_WB_ON_ITR_CAPABLE | 12003 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 12004 I40E_HW_NO_PCI_LINK_CHECK | 12005 I40E_HW_USE_SET_LLDP_MIB | 12006 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 12007 I40E_HW_PTP_L4_CAPABLE | 12008 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 12009 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 12010 12011 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 12012 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 12013 I40E_FDEVICT_PCTYPE_DEFAULT) { 12014 dev_warn(&pf->pdev->dev, 12015 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 12016 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 12017 } 12018 } else if ((pf->hw.aq.api_maj_ver > 1) || 12019 ((pf->hw.aq.api_maj_ver == 1) && 12020 (pf->hw.aq.api_min_ver > 4))) { 12021 /* Supported in FW API version higher than 1.4 */ 12022 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 12023 } 12024 12025 /* Enable HW ATR eviction if possible */ 12026 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 12027 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 12028 12029 if ((pf->hw.mac.type == I40E_MAC_XL710) && 12030 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 12031 (pf->hw.aq.fw_maj_ver < 4))) { 12032 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 12033 /* No DCB support for FW < v4.33 */ 12034 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 12035 } 12036 12037 /* Disable FW LLDP if FW < v4.3 */ 12038 if ((pf->hw.mac.type == I40E_MAC_XL710) && 12039 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 12040 (pf->hw.aq.fw_maj_ver < 4))) 12041 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 12042 12043 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 12044 if ((pf->hw.mac.type == I40E_MAC_XL710) && 12045 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 12046 (pf->hw.aq.fw_maj_ver >= 5))) 12047 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 12048 12049 /* Enable PTP L4 if FW > v6.0 */ 12050 if (pf->hw.mac.type == I40E_MAC_XL710 && 12051 pf->hw.aq.fw_maj_ver >= 6) 12052 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 12053 12054 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 12055 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 12056 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 12057 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 12058 } 12059 12060 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 12061 pf->flags |= I40E_FLAG_IWARP_ENABLED; 12062 /* IWARP needs one extra vector for CQP just like MISC.*/ 12063 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 12064 } 12065 /* Stopping FW LLDP engine is supported on XL710 and X722 12066 * starting from FW versions determined in i40e_init_adminq. 12067 * Stopping the FW LLDP engine is not supported on XL710 12068 * if NPAR is functioning so unset this hw flag in this case. 12069 */ 12070 if (pf->hw.mac.type == I40E_MAC_XL710 && 12071 pf->hw.func_caps.npar_enable && 12072 (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE)) 12073 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE; 12074 12075 #ifdef CONFIG_PCI_IOV 12076 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 12077 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 12078 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 12079 pf->num_req_vfs = min_t(int, 12080 pf->hw.func_caps.num_vfs, 12081 I40E_MAX_VF_COUNT); 12082 } 12083 #endif /* CONFIG_PCI_IOV */ 12084 pf->eeprom_version = 0xDEAD; 12085 pf->lan_veb = I40E_NO_VEB; 12086 pf->lan_vsi = I40E_NO_VSI; 12087 12088 /* By default FW has this off for performance reasons */ 12089 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 12090 12091 /* set up queue assignment tracking */ 12092 size = sizeof(struct i40e_lump_tracking) 12093 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 12094 pf->qp_pile = kzalloc(size, GFP_KERNEL); 12095 if (!pf->qp_pile) { 12096 err = -ENOMEM; 12097 goto sw_init_done; 12098 } 12099 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 12100 pf->qp_pile->search_hint = 0; 12101 12102 pf->tx_timeout_recovery_level = 1; 12103 12104 if (pf->hw.mac.type != I40E_MAC_X722 && 12105 i40e_is_total_port_shutdown_enabled(pf)) { 12106 /* Link down on close must be on when total port shutdown 12107 * is enabled for a given port 12108 */ 12109 pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED | 12110 I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED); 12111 dev_info(&pf->pdev->dev, 12112 "total-port-shutdown was enabled, link-down-on-close is forced on\n"); 12113 } 12114 mutex_init(&pf->switch_mutex); 12115 12116 sw_init_done: 12117 return err; 12118 } 12119 12120 /** 12121 * i40e_set_ntuple - set the ntuple feature flag and take action 12122 * @pf: board private structure to initialize 12123 * @features: the feature set that the stack is suggesting 12124 * 12125 * returns a bool to indicate if reset needs to happen 12126 **/ 12127 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 12128 { 12129 bool need_reset = false; 12130 12131 /* Check if Flow Director n-tuple support was enabled or disabled. If 12132 * the state changed, we need to reset. 12133 */ 12134 if (features & NETIF_F_NTUPLE) { 12135 /* Enable filters and mark for reset */ 12136 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 12137 need_reset = true; 12138 /* enable FD_SB only if there is MSI-X vector and no cloud 12139 * filters exist 12140 */ 12141 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 12142 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 12143 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 12144 } 12145 } else { 12146 /* turn off filters, mark for reset and clear SW filter list */ 12147 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 12148 need_reset = true; 12149 i40e_fdir_filter_exit(pf); 12150 } 12151 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 12152 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 12153 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 12154 12155 /* reset fd counters */ 12156 pf->fd_add_err = 0; 12157 pf->fd_atr_cnt = 0; 12158 /* if ATR was auto disabled it can be re-enabled. */ 12159 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 12160 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 12161 (I40E_DEBUG_FD & pf->hw.debug_mask)) 12162 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 12163 } 12164 return need_reset; 12165 } 12166 12167 /** 12168 * i40e_clear_rss_lut - clear the rx hash lookup table 12169 * @vsi: the VSI being configured 12170 **/ 12171 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 12172 { 12173 struct i40e_pf *pf = vsi->back; 12174 struct i40e_hw *hw = &pf->hw; 12175 u16 vf_id = vsi->vf_id; 12176 u8 i; 12177 12178 if (vsi->type == I40E_VSI_MAIN) { 12179 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12180 wr32(hw, I40E_PFQF_HLUT(i), 0); 12181 } else if (vsi->type == I40E_VSI_SRIOV) { 12182 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12183 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 12184 } else { 12185 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12186 } 12187 } 12188 12189 /** 12190 * i40e_set_features - set the netdev feature flags 12191 * @netdev: ptr to the netdev being adjusted 12192 * @features: the feature set that the stack is suggesting 12193 * Note: expects to be called while under rtnl_lock() 12194 **/ 12195 static int i40e_set_features(struct net_device *netdev, 12196 netdev_features_t features) 12197 { 12198 struct i40e_netdev_priv *np = netdev_priv(netdev); 12199 struct i40e_vsi *vsi = np->vsi; 12200 struct i40e_pf *pf = vsi->back; 12201 bool need_reset; 12202 12203 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 12204 i40e_pf_config_rss(pf); 12205 else if (!(features & NETIF_F_RXHASH) && 12206 netdev->features & NETIF_F_RXHASH) 12207 i40e_clear_rss_lut(vsi); 12208 12209 if (features & NETIF_F_HW_VLAN_CTAG_RX) 12210 i40e_vlan_stripping_enable(vsi); 12211 else 12212 i40e_vlan_stripping_disable(vsi); 12213 12214 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 12215 dev_err(&pf->pdev->dev, 12216 "Offloaded tc filters active, can't turn hw_tc_offload off"); 12217 return -EINVAL; 12218 } 12219 12220 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt) 12221 i40e_del_all_macvlans(vsi); 12222 12223 need_reset = i40e_set_ntuple(pf, features); 12224 12225 if (need_reset) 12226 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12227 12228 return 0; 12229 } 12230 12231 /** 12232 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 12233 * @pf: board private structure 12234 * @port: The UDP port to look up 12235 * 12236 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 12237 **/ 12238 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 12239 { 12240 u8 i; 12241 12242 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 12243 /* Do not report ports with pending deletions as 12244 * being available. 12245 */ 12246 if (!port && (pf->pending_udp_bitmap & BIT_ULL(i))) 12247 continue; 12248 if (pf->udp_ports[i].port == port) 12249 return i; 12250 } 12251 12252 return i; 12253 } 12254 12255 /** 12256 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 12257 * @netdev: This physical port's netdev 12258 * @ti: Tunnel endpoint information 12259 **/ 12260 static void i40e_udp_tunnel_add(struct net_device *netdev, 12261 struct udp_tunnel_info *ti) 12262 { 12263 struct i40e_netdev_priv *np = netdev_priv(netdev); 12264 struct i40e_vsi *vsi = np->vsi; 12265 struct i40e_pf *pf = vsi->back; 12266 u16 port = ntohs(ti->port); 12267 u8 next_idx; 12268 u8 idx; 12269 12270 idx = i40e_get_udp_port_idx(pf, port); 12271 12272 /* Check if port already exists */ 12273 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 12274 netdev_info(netdev, "port %d already offloaded\n", port); 12275 return; 12276 } 12277 12278 /* Now check if there is space to add the new port */ 12279 next_idx = i40e_get_udp_port_idx(pf, 0); 12280 12281 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 12282 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 12283 port); 12284 return; 12285 } 12286 12287 switch (ti->type) { 12288 case UDP_TUNNEL_TYPE_VXLAN: 12289 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 12290 break; 12291 case UDP_TUNNEL_TYPE_GENEVE: 12292 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) 12293 return; 12294 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 12295 break; 12296 default: 12297 return; 12298 } 12299 12300 /* New port: add it and mark its index in the bitmap */ 12301 pf->udp_ports[next_idx].port = port; 12302 pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED; 12303 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 12304 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 12305 } 12306 12307 /** 12308 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 12309 * @netdev: This physical port's netdev 12310 * @ti: Tunnel endpoint information 12311 **/ 12312 static void i40e_udp_tunnel_del(struct net_device *netdev, 12313 struct udp_tunnel_info *ti) 12314 { 12315 struct i40e_netdev_priv *np = netdev_priv(netdev); 12316 struct i40e_vsi *vsi = np->vsi; 12317 struct i40e_pf *pf = vsi->back; 12318 u16 port = ntohs(ti->port); 12319 u8 idx; 12320 12321 idx = i40e_get_udp_port_idx(pf, port); 12322 12323 /* Check if port already exists */ 12324 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 12325 goto not_found; 12326 12327 switch (ti->type) { 12328 case UDP_TUNNEL_TYPE_VXLAN: 12329 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 12330 goto not_found; 12331 break; 12332 case UDP_TUNNEL_TYPE_GENEVE: 12333 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 12334 goto not_found; 12335 break; 12336 default: 12337 goto not_found; 12338 } 12339 12340 /* if port exists, set it to 0 (mark for deletion) 12341 * and make it pending 12342 */ 12343 pf->udp_ports[idx].port = 0; 12344 12345 /* Toggle pending bit instead of setting it. This way if we are 12346 * deleting a port that has yet to be added we just clear the pending 12347 * bit and don't have to worry about it. 12348 */ 12349 pf->pending_udp_bitmap ^= BIT_ULL(idx); 12350 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 12351 12352 return; 12353 not_found: 12354 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 12355 port); 12356 } 12357 12358 static int i40e_get_phys_port_id(struct net_device *netdev, 12359 struct netdev_phys_item_id *ppid) 12360 { 12361 struct i40e_netdev_priv *np = netdev_priv(netdev); 12362 struct i40e_pf *pf = np->vsi->back; 12363 struct i40e_hw *hw = &pf->hw; 12364 12365 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 12366 return -EOPNOTSUPP; 12367 12368 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 12369 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 12370 12371 return 0; 12372 } 12373 12374 /** 12375 * i40e_ndo_fdb_add - add an entry to the hardware database 12376 * @ndm: the input from the stack 12377 * @tb: pointer to array of nladdr (unused) 12378 * @dev: the net device pointer 12379 * @addr: the MAC address entry being added 12380 * @vid: VLAN ID 12381 * @flags: instructions from stack about fdb operation 12382 */ 12383 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 12384 struct net_device *dev, 12385 const unsigned char *addr, u16 vid, 12386 u16 flags, 12387 struct netlink_ext_ack *extack) 12388 { 12389 struct i40e_netdev_priv *np = netdev_priv(dev); 12390 struct i40e_pf *pf = np->vsi->back; 12391 int err = 0; 12392 12393 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 12394 return -EOPNOTSUPP; 12395 12396 if (vid) { 12397 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 12398 return -EINVAL; 12399 } 12400 12401 /* Hardware does not support aging addresses so if a 12402 * ndm_state is given only allow permanent addresses 12403 */ 12404 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 12405 netdev_info(dev, "FDB only supports static addresses\n"); 12406 return -EINVAL; 12407 } 12408 12409 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 12410 err = dev_uc_add_excl(dev, addr); 12411 else if (is_multicast_ether_addr(addr)) 12412 err = dev_mc_add_excl(dev, addr); 12413 else 12414 err = -EINVAL; 12415 12416 /* Only return duplicate errors if NLM_F_EXCL is set */ 12417 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 12418 err = 0; 12419 12420 return err; 12421 } 12422 12423 /** 12424 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 12425 * @dev: the netdev being configured 12426 * @nlh: RTNL message 12427 * @flags: bridge flags 12428 * @extack: netlink extended ack 12429 * 12430 * Inserts a new hardware bridge if not already created and 12431 * enables the bridging mode requested (VEB or VEPA). If the 12432 * hardware bridge has already been inserted and the request 12433 * is to change the mode then that requires a PF reset to 12434 * allow rebuild of the components with required hardware 12435 * bridge mode enabled. 12436 * 12437 * Note: expects to be called while under rtnl_lock() 12438 **/ 12439 static int i40e_ndo_bridge_setlink(struct net_device *dev, 12440 struct nlmsghdr *nlh, 12441 u16 flags, 12442 struct netlink_ext_ack *extack) 12443 { 12444 struct i40e_netdev_priv *np = netdev_priv(dev); 12445 struct i40e_vsi *vsi = np->vsi; 12446 struct i40e_pf *pf = vsi->back; 12447 struct i40e_veb *veb = NULL; 12448 struct nlattr *attr, *br_spec; 12449 int i, rem; 12450 12451 /* Only for PF VSI for now */ 12452 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 12453 return -EOPNOTSUPP; 12454 12455 /* Find the HW bridge for PF VSI */ 12456 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12457 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12458 veb = pf->veb[i]; 12459 } 12460 12461 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12462 12463 nla_for_each_nested(attr, br_spec, rem) { 12464 __u16 mode; 12465 12466 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12467 continue; 12468 12469 mode = nla_get_u16(attr); 12470 if ((mode != BRIDGE_MODE_VEPA) && 12471 (mode != BRIDGE_MODE_VEB)) 12472 return -EINVAL; 12473 12474 /* Insert a new HW bridge */ 12475 if (!veb) { 12476 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 12477 vsi->tc_config.enabled_tc); 12478 if (veb) { 12479 veb->bridge_mode = mode; 12480 i40e_config_bridge_mode(veb); 12481 } else { 12482 /* No Bridge HW offload available */ 12483 return -ENOENT; 12484 } 12485 break; 12486 } else if (mode != veb->bridge_mode) { 12487 /* Existing HW bridge but different mode needs reset */ 12488 veb->bridge_mode = mode; 12489 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 12490 if (mode == BRIDGE_MODE_VEB) 12491 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 12492 else 12493 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 12494 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12495 break; 12496 } 12497 } 12498 12499 return 0; 12500 } 12501 12502 /** 12503 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 12504 * @skb: skb buff 12505 * @pid: process id 12506 * @seq: RTNL message seq # 12507 * @dev: the netdev being configured 12508 * @filter_mask: unused 12509 * @nlflags: netlink flags passed in 12510 * 12511 * Return the mode in which the hardware bridge is operating in 12512 * i.e VEB or VEPA. 12513 **/ 12514 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12515 struct net_device *dev, 12516 u32 __always_unused filter_mask, 12517 int nlflags) 12518 { 12519 struct i40e_netdev_priv *np = netdev_priv(dev); 12520 struct i40e_vsi *vsi = np->vsi; 12521 struct i40e_pf *pf = vsi->back; 12522 struct i40e_veb *veb = NULL; 12523 int i; 12524 12525 /* Only for PF VSI for now */ 12526 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 12527 return -EOPNOTSUPP; 12528 12529 /* Find the HW bridge for the PF VSI */ 12530 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12531 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12532 veb = pf->veb[i]; 12533 } 12534 12535 if (!veb) 12536 return 0; 12537 12538 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 12539 0, 0, nlflags, filter_mask, NULL); 12540 } 12541 12542 /** 12543 * i40e_features_check - Validate encapsulated packet conforms to limits 12544 * @skb: skb buff 12545 * @dev: This physical port's netdev 12546 * @features: Offload features that the stack believes apply 12547 **/ 12548 static netdev_features_t i40e_features_check(struct sk_buff *skb, 12549 struct net_device *dev, 12550 netdev_features_t features) 12551 { 12552 size_t len; 12553 12554 /* No point in doing any of this if neither checksum nor GSO are 12555 * being requested for this frame. We can rule out both by just 12556 * checking for CHECKSUM_PARTIAL 12557 */ 12558 if (skb->ip_summed != CHECKSUM_PARTIAL) 12559 return features; 12560 12561 /* We cannot support GSO if the MSS is going to be less than 12562 * 64 bytes. If it is then we need to drop support for GSO. 12563 */ 12564 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 12565 features &= ~NETIF_F_GSO_MASK; 12566 12567 /* MACLEN can support at most 63 words */ 12568 len = skb_network_header(skb) - skb->data; 12569 if (len & ~(63 * 2)) 12570 goto out_err; 12571 12572 /* IPLEN and EIPLEN can support at most 127 dwords */ 12573 len = skb_transport_header(skb) - skb_network_header(skb); 12574 if (len & ~(127 * 4)) 12575 goto out_err; 12576 12577 if (skb->encapsulation) { 12578 /* L4TUNLEN can support 127 words */ 12579 len = skb_inner_network_header(skb) - skb_transport_header(skb); 12580 if (len & ~(127 * 2)) 12581 goto out_err; 12582 12583 /* IPLEN can support at most 127 dwords */ 12584 len = skb_inner_transport_header(skb) - 12585 skb_inner_network_header(skb); 12586 if (len & ~(127 * 4)) 12587 goto out_err; 12588 } 12589 12590 /* No need to validate L4LEN as TCP is the only protocol with a 12591 * a flexible value and we support all possible values supported 12592 * by TCP, which is at most 15 dwords 12593 */ 12594 12595 return features; 12596 out_err: 12597 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 12598 } 12599 12600 /** 12601 * i40e_xdp_setup - add/remove an XDP program 12602 * @vsi: VSI to changed 12603 * @prog: XDP program 12604 **/ 12605 static int i40e_xdp_setup(struct i40e_vsi *vsi, 12606 struct bpf_prog *prog) 12607 { 12608 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 12609 struct i40e_pf *pf = vsi->back; 12610 struct bpf_prog *old_prog; 12611 bool need_reset; 12612 int i; 12613 12614 /* Don't allow frames that span over multiple buffers */ 12615 if (frame_size > vsi->rx_buf_len) 12616 return -EINVAL; 12617 12618 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 12619 return 0; 12620 12621 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 12622 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 12623 12624 if (need_reset) 12625 i40e_prep_for_reset(pf, true); 12626 12627 old_prog = xchg(&vsi->xdp_prog, prog); 12628 12629 if (need_reset) { 12630 if (!prog) 12631 /* Wait until ndo_xsk_wakeup completes. */ 12632 synchronize_rcu(); 12633 i40e_reset_and_rebuild(pf, true, true); 12634 } 12635 12636 for (i = 0; i < vsi->num_queue_pairs; i++) 12637 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 12638 12639 if (old_prog) 12640 bpf_prog_put(old_prog); 12641 12642 /* Kick start the NAPI context if there is an AF_XDP socket open 12643 * on that queue id. This so that receiving will start. 12644 */ 12645 if (need_reset && prog) 12646 for (i = 0; i < vsi->num_queue_pairs; i++) 12647 if (vsi->xdp_rings[i]->xsk_umem) 12648 (void)i40e_xsk_wakeup(vsi->netdev, i, 12649 XDP_WAKEUP_RX); 12650 12651 return 0; 12652 } 12653 12654 /** 12655 * i40e_enter_busy_conf - Enters busy config state 12656 * @vsi: vsi 12657 * 12658 * Returns 0 on success, <0 for failure. 12659 **/ 12660 static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 12661 { 12662 struct i40e_pf *pf = vsi->back; 12663 int timeout = 50; 12664 12665 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 12666 timeout--; 12667 if (!timeout) 12668 return -EBUSY; 12669 usleep_range(1000, 2000); 12670 } 12671 12672 return 0; 12673 } 12674 12675 /** 12676 * i40e_exit_busy_conf - Exits busy config state 12677 * @vsi: vsi 12678 **/ 12679 static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 12680 { 12681 struct i40e_pf *pf = vsi->back; 12682 12683 clear_bit(__I40E_CONFIG_BUSY, pf->state); 12684 } 12685 12686 /** 12687 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 12688 * @vsi: vsi 12689 * @queue_pair: queue pair 12690 **/ 12691 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 12692 { 12693 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 12694 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 12695 memset(&vsi->tx_rings[queue_pair]->stats, 0, 12696 sizeof(vsi->tx_rings[queue_pair]->stats)); 12697 if (i40e_enabled_xdp_vsi(vsi)) { 12698 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 12699 sizeof(vsi->xdp_rings[queue_pair]->stats)); 12700 } 12701 } 12702 12703 /** 12704 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 12705 * @vsi: vsi 12706 * @queue_pair: queue pair 12707 **/ 12708 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 12709 { 12710 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 12711 if (i40e_enabled_xdp_vsi(vsi)) { 12712 /* Make sure that in-progress ndo_xdp_xmit calls are 12713 * completed. 12714 */ 12715 synchronize_rcu(); 12716 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 12717 } 12718 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 12719 } 12720 12721 /** 12722 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 12723 * @vsi: vsi 12724 * @queue_pair: queue pair 12725 * @enable: true for enable, false for disable 12726 **/ 12727 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 12728 bool enable) 12729 { 12730 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12731 struct i40e_q_vector *q_vector = rxr->q_vector; 12732 12733 if (!vsi->netdev) 12734 return; 12735 12736 /* All rings in a qp belong to the same qvector. */ 12737 if (q_vector->rx.ring || q_vector->tx.ring) { 12738 if (enable) 12739 napi_enable(&q_vector->napi); 12740 else 12741 napi_disable(&q_vector->napi); 12742 } 12743 } 12744 12745 /** 12746 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 12747 * @vsi: vsi 12748 * @queue_pair: queue pair 12749 * @enable: true for enable, false for disable 12750 * 12751 * Returns 0 on success, <0 on failure. 12752 **/ 12753 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 12754 bool enable) 12755 { 12756 struct i40e_pf *pf = vsi->back; 12757 int pf_q, ret = 0; 12758 12759 pf_q = vsi->base_queue + queue_pair; 12760 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 12761 false /*is xdp*/, enable); 12762 if (ret) { 12763 dev_info(&pf->pdev->dev, 12764 "VSI seid %d Tx ring %d %sable timeout\n", 12765 vsi->seid, pf_q, (enable ? "en" : "dis")); 12766 return ret; 12767 } 12768 12769 i40e_control_rx_q(pf, pf_q, enable); 12770 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 12771 if (ret) { 12772 dev_info(&pf->pdev->dev, 12773 "VSI seid %d Rx ring %d %sable timeout\n", 12774 vsi->seid, pf_q, (enable ? "en" : "dis")); 12775 return ret; 12776 } 12777 12778 /* Due to HW errata, on Rx disable only, the register can 12779 * indicate done before it really is. Needs 50ms to be sure 12780 */ 12781 if (!enable) 12782 mdelay(50); 12783 12784 if (!i40e_enabled_xdp_vsi(vsi)) 12785 return ret; 12786 12787 ret = i40e_control_wait_tx_q(vsi->seid, pf, 12788 pf_q + vsi->alloc_queue_pairs, 12789 true /*is xdp*/, enable); 12790 if (ret) { 12791 dev_info(&pf->pdev->dev, 12792 "VSI seid %d XDP Tx ring %d %sable timeout\n", 12793 vsi->seid, pf_q, (enable ? "en" : "dis")); 12794 } 12795 12796 return ret; 12797 } 12798 12799 /** 12800 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 12801 * @vsi: vsi 12802 * @queue_pair: queue_pair 12803 **/ 12804 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 12805 { 12806 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12807 struct i40e_pf *pf = vsi->back; 12808 struct i40e_hw *hw = &pf->hw; 12809 12810 /* All rings in a qp belong to the same qvector. */ 12811 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 12812 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 12813 else 12814 i40e_irq_dynamic_enable_icr0(pf); 12815 12816 i40e_flush(hw); 12817 } 12818 12819 /** 12820 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 12821 * @vsi: vsi 12822 * @queue_pair: queue_pair 12823 **/ 12824 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 12825 { 12826 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12827 struct i40e_pf *pf = vsi->back; 12828 struct i40e_hw *hw = &pf->hw; 12829 12830 /* For simplicity, instead of removing the qp interrupt causes 12831 * from the interrupt linked list, we simply disable the interrupt, and 12832 * leave the list intact. 12833 * 12834 * All rings in a qp belong to the same qvector. 12835 */ 12836 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 12837 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 12838 12839 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 12840 i40e_flush(hw); 12841 synchronize_irq(pf->msix_entries[intpf].vector); 12842 } else { 12843 /* Legacy and MSI mode - this stops all interrupt handling */ 12844 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 12845 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 12846 i40e_flush(hw); 12847 synchronize_irq(pf->pdev->irq); 12848 } 12849 } 12850 12851 /** 12852 * i40e_queue_pair_disable - Disables a queue pair 12853 * @vsi: vsi 12854 * @queue_pair: queue pair 12855 * 12856 * Returns 0 on success, <0 on failure. 12857 **/ 12858 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 12859 { 12860 int err; 12861 12862 err = i40e_enter_busy_conf(vsi); 12863 if (err) 12864 return err; 12865 12866 i40e_queue_pair_disable_irq(vsi, queue_pair); 12867 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 12868 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 12869 i40e_queue_pair_clean_rings(vsi, queue_pair); 12870 i40e_queue_pair_reset_stats(vsi, queue_pair); 12871 12872 return err; 12873 } 12874 12875 /** 12876 * i40e_queue_pair_enable - Enables a queue pair 12877 * @vsi: vsi 12878 * @queue_pair: queue pair 12879 * 12880 * Returns 0 on success, <0 on failure. 12881 **/ 12882 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 12883 { 12884 int err; 12885 12886 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 12887 if (err) 12888 return err; 12889 12890 if (i40e_enabled_xdp_vsi(vsi)) { 12891 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 12892 if (err) 12893 return err; 12894 } 12895 12896 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 12897 if (err) 12898 return err; 12899 12900 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 12901 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 12902 i40e_queue_pair_enable_irq(vsi, queue_pair); 12903 12904 i40e_exit_busy_conf(vsi); 12905 12906 return err; 12907 } 12908 12909 /** 12910 * i40e_xdp - implements ndo_bpf for i40e 12911 * @dev: netdevice 12912 * @xdp: XDP command 12913 **/ 12914 static int i40e_xdp(struct net_device *dev, 12915 struct netdev_bpf *xdp) 12916 { 12917 struct i40e_netdev_priv *np = netdev_priv(dev); 12918 struct i40e_vsi *vsi = np->vsi; 12919 12920 if (vsi->type != I40E_VSI_MAIN) 12921 return -EINVAL; 12922 12923 switch (xdp->command) { 12924 case XDP_SETUP_PROG: 12925 return i40e_xdp_setup(vsi, xdp->prog); 12926 case XDP_SETUP_XSK_UMEM: 12927 return i40e_xsk_umem_setup(vsi, xdp->xsk.umem, 12928 xdp->xsk.queue_id); 12929 default: 12930 return -EINVAL; 12931 } 12932 } 12933 12934 static const struct net_device_ops i40e_netdev_ops = { 12935 .ndo_open = i40e_open, 12936 .ndo_stop = i40e_close, 12937 .ndo_start_xmit = i40e_lan_xmit_frame, 12938 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 12939 .ndo_set_rx_mode = i40e_set_rx_mode, 12940 .ndo_validate_addr = eth_validate_addr, 12941 .ndo_set_mac_address = i40e_set_mac, 12942 .ndo_change_mtu = i40e_change_mtu, 12943 .ndo_do_ioctl = i40e_ioctl, 12944 .ndo_tx_timeout = i40e_tx_timeout, 12945 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 12946 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 12947 #ifdef CONFIG_NET_POLL_CONTROLLER 12948 .ndo_poll_controller = i40e_netpoll, 12949 #endif 12950 .ndo_setup_tc = __i40e_setup_tc, 12951 .ndo_set_features = i40e_set_features, 12952 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 12953 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 12954 .ndo_get_vf_stats = i40e_get_vf_stats, 12955 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 12956 .ndo_get_vf_config = i40e_ndo_get_vf_config, 12957 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 12958 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 12959 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 12960 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 12961 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 12962 .ndo_get_phys_port_id = i40e_get_phys_port_id, 12963 .ndo_fdb_add = i40e_ndo_fdb_add, 12964 .ndo_features_check = i40e_features_check, 12965 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 12966 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 12967 .ndo_bpf = i40e_xdp, 12968 .ndo_xdp_xmit = i40e_xdp_xmit, 12969 .ndo_xsk_wakeup = i40e_xsk_wakeup, 12970 .ndo_dfwd_add_station = i40e_fwd_add, 12971 .ndo_dfwd_del_station = i40e_fwd_del, 12972 }; 12973 12974 /** 12975 * i40e_config_netdev - Setup the netdev flags 12976 * @vsi: the VSI being configured 12977 * 12978 * Returns 0 on success, negative value on failure 12979 **/ 12980 static int i40e_config_netdev(struct i40e_vsi *vsi) 12981 { 12982 struct i40e_pf *pf = vsi->back; 12983 struct i40e_hw *hw = &pf->hw; 12984 struct i40e_netdev_priv *np; 12985 struct net_device *netdev; 12986 u8 broadcast[ETH_ALEN]; 12987 u8 mac_addr[ETH_ALEN]; 12988 int etherdev_size; 12989 netdev_features_t hw_enc_features; 12990 netdev_features_t hw_features; 12991 12992 etherdev_size = sizeof(struct i40e_netdev_priv); 12993 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 12994 if (!netdev) 12995 return -ENOMEM; 12996 12997 vsi->netdev = netdev; 12998 np = netdev_priv(netdev); 12999 np->vsi = vsi; 13000 13001 hw_enc_features = NETIF_F_SG | 13002 NETIF_F_IP_CSUM | 13003 NETIF_F_IPV6_CSUM | 13004 NETIF_F_HIGHDMA | 13005 NETIF_F_SOFT_FEATURES | 13006 NETIF_F_TSO | 13007 NETIF_F_TSO_ECN | 13008 NETIF_F_TSO6 | 13009 NETIF_F_GSO_GRE | 13010 NETIF_F_GSO_GRE_CSUM | 13011 NETIF_F_GSO_PARTIAL | 13012 NETIF_F_GSO_IPXIP4 | 13013 NETIF_F_GSO_IPXIP6 | 13014 NETIF_F_GSO_UDP_TUNNEL | 13015 NETIF_F_GSO_UDP_TUNNEL_CSUM | 13016 NETIF_F_GSO_UDP_L4 | 13017 NETIF_F_SCTP_CRC | 13018 NETIF_F_RXHASH | 13019 NETIF_F_RXCSUM | 13020 0; 13021 13022 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 13023 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 13024 13025 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 13026 13027 netdev->hw_enc_features |= hw_enc_features; 13028 13029 /* record features VLANs can make use of */ 13030 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 13031 13032 /* enable macvlan offloads */ 13033 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD; 13034 13035 hw_features = hw_enc_features | 13036 NETIF_F_HW_VLAN_CTAG_TX | 13037 NETIF_F_HW_VLAN_CTAG_RX; 13038 13039 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 13040 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 13041 13042 netdev->hw_features |= hw_features; 13043 13044 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 13045 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 13046 13047 if (vsi->type == I40E_VSI_MAIN) { 13048 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 13049 ether_addr_copy(mac_addr, hw->mac.perm_addr); 13050 /* The following steps are necessary for two reasons. First, 13051 * some older NVM configurations load a default MAC-VLAN 13052 * filter that will accept any tagged packet, and we want to 13053 * replace this with a normal filter. Additionally, it is 13054 * possible our MAC address was provided by the platform using 13055 * Open Firmware or similar. 13056 * 13057 * Thus, we need to remove the default filter and install one 13058 * specific to the MAC address. 13059 */ 13060 i40e_rm_default_mac_filter(vsi, mac_addr); 13061 spin_lock_bh(&vsi->mac_filter_hash_lock); 13062 i40e_add_mac_filter(vsi, mac_addr); 13063 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13064 } else { 13065 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 13066 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 13067 * the end, which is 4 bytes long, so force truncation of the 13068 * original name by IFNAMSIZ - 4 13069 */ 13070 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 13071 IFNAMSIZ - 4, 13072 pf->vsi[pf->lan_vsi]->netdev->name); 13073 eth_random_addr(mac_addr); 13074 13075 spin_lock_bh(&vsi->mac_filter_hash_lock); 13076 i40e_add_mac_filter(vsi, mac_addr); 13077 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13078 } 13079 13080 /* Add the broadcast filter so that we initially will receive 13081 * broadcast packets. Note that when a new VLAN is first added the 13082 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 13083 * specific filters as part of transitioning into "vlan" operation. 13084 * When more VLANs are added, the driver will copy each existing MAC 13085 * filter and add it for the new VLAN. 13086 * 13087 * Broadcast filters are handled specially by 13088 * i40e_sync_filters_subtask, as the driver must to set the broadcast 13089 * promiscuous bit instead of adding this directly as a MAC/VLAN 13090 * filter. The subtask will update the correct broadcast promiscuous 13091 * bits as VLANs become active or inactive. 13092 */ 13093 eth_broadcast_addr(broadcast); 13094 spin_lock_bh(&vsi->mac_filter_hash_lock); 13095 i40e_add_mac_filter(vsi, broadcast); 13096 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13097 13098 ether_addr_copy(netdev->dev_addr, mac_addr); 13099 ether_addr_copy(netdev->perm_addr, mac_addr); 13100 13101 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */ 13102 netdev->neigh_priv_len = sizeof(u32) * 4; 13103 13104 netdev->priv_flags |= IFF_UNICAST_FLT; 13105 netdev->priv_flags |= IFF_SUPP_NOFCS; 13106 /* Setup netdev TC information */ 13107 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 13108 13109 netdev->netdev_ops = &i40e_netdev_ops; 13110 netdev->watchdog_timeo = 5 * HZ; 13111 i40e_set_ethtool_ops(netdev); 13112 13113 /* MTU range: 68 - 9706 */ 13114 netdev->min_mtu = ETH_MIN_MTU; 13115 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 13116 13117 return 0; 13118 } 13119 13120 /** 13121 * i40e_vsi_delete - Delete a VSI from the switch 13122 * @vsi: the VSI being removed 13123 * 13124 * Returns 0 on success, negative value on failure 13125 **/ 13126 static void i40e_vsi_delete(struct i40e_vsi *vsi) 13127 { 13128 /* remove default VSI is not allowed */ 13129 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 13130 return; 13131 13132 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 13133 } 13134 13135 /** 13136 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 13137 * @vsi: the VSI being queried 13138 * 13139 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 13140 **/ 13141 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 13142 { 13143 struct i40e_veb *veb; 13144 struct i40e_pf *pf = vsi->back; 13145 13146 /* Uplink is not a bridge so default to VEB */ 13147 if (vsi->veb_idx >= I40E_MAX_VEB) 13148 return 1; 13149 13150 veb = pf->veb[vsi->veb_idx]; 13151 if (!veb) { 13152 dev_info(&pf->pdev->dev, 13153 "There is no veb associated with the bridge\n"); 13154 return -ENOENT; 13155 } 13156 13157 /* Uplink is a bridge in VEPA mode */ 13158 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 13159 return 0; 13160 } else { 13161 /* Uplink is a bridge in VEB mode */ 13162 return 1; 13163 } 13164 13165 /* VEPA is now default bridge, so return 0 */ 13166 return 0; 13167 } 13168 13169 /** 13170 * i40e_add_vsi - Add a VSI to the switch 13171 * @vsi: the VSI being configured 13172 * 13173 * This initializes a VSI context depending on the VSI type to be added and 13174 * passes it down to the add_vsi aq command. 13175 **/ 13176 static int i40e_add_vsi(struct i40e_vsi *vsi) 13177 { 13178 int ret = -ENODEV; 13179 struct i40e_pf *pf = vsi->back; 13180 struct i40e_hw *hw = &pf->hw; 13181 struct i40e_vsi_context ctxt; 13182 struct i40e_mac_filter *f; 13183 struct hlist_node *h; 13184 int bkt; 13185 13186 u8 enabled_tc = 0x1; /* TC0 enabled */ 13187 int f_count = 0; 13188 13189 memset(&ctxt, 0, sizeof(ctxt)); 13190 switch (vsi->type) { 13191 case I40E_VSI_MAIN: 13192 /* The PF's main VSI is already setup as part of the 13193 * device initialization, so we'll not bother with 13194 * the add_vsi call, but we will retrieve the current 13195 * VSI context. 13196 */ 13197 ctxt.seid = pf->main_vsi_seid; 13198 ctxt.pf_num = pf->hw.pf_id; 13199 ctxt.vf_num = 0; 13200 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 13201 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13202 if (ret) { 13203 dev_info(&pf->pdev->dev, 13204 "couldn't get PF vsi config, err %s aq_err %s\n", 13205 i40e_stat_str(&pf->hw, ret), 13206 i40e_aq_str(&pf->hw, 13207 pf->hw.aq.asq_last_status)); 13208 return -ENOENT; 13209 } 13210 vsi->info = ctxt.info; 13211 vsi->info.valid_sections = 0; 13212 13213 vsi->seid = ctxt.seid; 13214 vsi->id = ctxt.vsi_number; 13215 13216 enabled_tc = i40e_pf_get_tc_map(pf); 13217 13218 /* Source pruning is enabled by default, so the flag is 13219 * negative logic - if it's set, we need to fiddle with 13220 * the VSI to disable source pruning. 13221 */ 13222 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 13223 memset(&ctxt, 0, sizeof(ctxt)); 13224 ctxt.seid = pf->main_vsi_seid; 13225 ctxt.pf_num = pf->hw.pf_id; 13226 ctxt.vf_num = 0; 13227 ctxt.info.valid_sections |= 13228 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13229 ctxt.info.switch_id = 13230 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 13231 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13232 if (ret) { 13233 dev_info(&pf->pdev->dev, 13234 "update vsi failed, err %s aq_err %s\n", 13235 i40e_stat_str(&pf->hw, ret), 13236 i40e_aq_str(&pf->hw, 13237 pf->hw.aq.asq_last_status)); 13238 ret = -ENOENT; 13239 goto err; 13240 } 13241 } 13242 13243 /* MFP mode setup queue map and update VSI */ 13244 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 13245 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 13246 memset(&ctxt, 0, sizeof(ctxt)); 13247 ctxt.seid = pf->main_vsi_seid; 13248 ctxt.pf_num = pf->hw.pf_id; 13249 ctxt.vf_num = 0; 13250 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 13251 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13252 if (ret) { 13253 dev_info(&pf->pdev->dev, 13254 "update vsi failed, err %s aq_err %s\n", 13255 i40e_stat_str(&pf->hw, ret), 13256 i40e_aq_str(&pf->hw, 13257 pf->hw.aq.asq_last_status)); 13258 ret = -ENOENT; 13259 goto err; 13260 } 13261 /* update the local VSI info queue map */ 13262 i40e_vsi_update_queue_map(vsi, &ctxt); 13263 vsi->info.valid_sections = 0; 13264 } else { 13265 /* Default/Main VSI is only enabled for TC0 13266 * reconfigure it to enable all TCs that are 13267 * available on the port in SFP mode. 13268 * For MFP case the iSCSI PF would use this 13269 * flow to enable LAN+iSCSI TC. 13270 */ 13271 ret = i40e_vsi_config_tc(vsi, enabled_tc); 13272 if (ret) { 13273 /* Single TC condition is not fatal, 13274 * message and continue 13275 */ 13276 dev_info(&pf->pdev->dev, 13277 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 13278 enabled_tc, 13279 i40e_stat_str(&pf->hw, ret), 13280 i40e_aq_str(&pf->hw, 13281 pf->hw.aq.asq_last_status)); 13282 } 13283 } 13284 break; 13285 13286 case I40E_VSI_FDIR: 13287 ctxt.pf_num = hw->pf_id; 13288 ctxt.vf_num = 0; 13289 ctxt.uplink_seid = vsi->uplink_seid; 13290 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13291 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13292 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 13293 (i40e_is_vsi_uplink_mode_veb(vsi))) { 13294 ctxt.info.valid_sections |= 13295 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13296 ctxt.info.switch_id = 13297 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13298 } 13299 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13300 break; 13301 13302 case I40E_VSI_VMDQ2: 13303 ctxt.pf_num = hw->pf_id; 13304 ctxt.vf_num = 0; 13305 ctxt.uplink_seid = vsi->uplink_seid; 13306 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13307 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 13308 13309 /* This VSI is connected to VEB so the switch_id 13310 * should be set to zero by default. 13311 */ 13312 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13313 ctxt.info.valid_sections |= 13314 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13315 ctxt.info.switch_id = 13316 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13317 } 13318 13319 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13320 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13321 break; 13322 13323 case I40E_VSI_SRIOV: 13324 ctxt.pf_num = hw->pf_id; 13325 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 13326 ctxt.uplink_seid = vsi->uplink_seid; 13327 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13328 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 13329 13330 /* This VSI is connected to VEB so the switch_id 13331 * should be set to zero by default. 13332 */ 13333 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13334 ctxt.info.valid_sections |= 13335 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13336 ctxt.info.switch_id = 13337 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13338 } 13339 13340 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 13341 ctxt.info.valid_sections |= 13342 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 13343 ctxt.info.queueing_opt_flags |= 13344 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 13345 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 13346 } 13347 13348 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 13349 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 13350 if (pf->vf[vsi->vf_id].spoofchk) { 13351 ctxt.info.valid_sections |= 13352 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 13353 ctxt.info.sec_flags |= 13354 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 13355 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 13356 } 13357 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13358 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13359 break; 13360 13361 case I40E_VSI_IWARP: 13362 /* send down message to iWARP */ 13363 break; 13364 13365 default: 13366 return -ENODEV; 13367 } 13368 13369 if (vsi->type != I40E_VSI_MAIN) { 13370 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 13371 if (ret) { 13372 dev_info(&vsi->back->pdev->dev, 13373 "add vsi failed, err %s aq_err %s\n", 13374 i40e_stat_str(&pf->hw, ret), 13375 i40e_aq_str(&pf->hw, 13376 pf->hw.aq.asq_last_status)); 13377 ret = -ENOENT; 13378 goto err; 13379 } 13380 vsi->info = ctxt.info; 13381 vsi->info.valid_sections = 0; 13382 vsi->seid = ctxt.seid; 13383 vsi->id = ctxt.vsi_number; 13384 } 13385 13386 vsi->active_filters = 0; 13387 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 13388 spin_lock_bh(&vsi->mac_filter_hash_lock); 13389 /* If macvlan filters already exist, force them to get loaded */ 13390 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 13391 f->state = I40E_FILTER_NEW; 13392 f_count++; 13393 } 13394 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13395 13396 if (f_count) { 13397 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 13398 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 13399 } 13400 13401 /* Update VSI BW information */ 13402 ret = i40e_vsi_get_bw_info(vsi); 13403 if (ret) { 13404 dev_info(&pf->pdev->dev, 13405 "couldn't get vsi bw info, err %s aq_err %s\n", 13406 i40e_stat_str(&pf->hw, ret), 13407 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13408 /* VSI is already added so not tearing that up */ 13409 ret = 0; 13410 } 13411 13412 err: 13413 return ret; 13414 } 13415 13416 /** 13417 * i40e_vsi_release - Delete a VSI and free its resources 13418 * @vsi: the VSI being removed 13419 * 13420 * Returns 0 on success or < 0 on error 13421 **/ 13422 int i40e_vsi_release(struct i40e_vsi *vsi) 13423 { 13424 struct i40e_mac_filter *f; 13425 struct hlist_node *h; 13426 struct i40e_veb *veb = NULL; 13427 struct i40e_pf *pf; 13428 u16 uplink_seid; 13429 int i, n, bkt; 13430 13431 pf = vsi->back; 13432 13433 /* release of a VEB-owner or last VSI is not allowed */ 13434 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 13435 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 13436 vsi->seid, vsi->uplink_seid); 13437 return -ENODEV; 13438 } 13439 if (vsi == pf->vsi[pf->lan_vsi] && 13440 !test_bit(__I40E_DOWN, pf->state)) { 13441 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 13442 return -ENODEV; 13443 } 13444 13445 uplink_seid = vsi->uplink_seid; 13446 if (vsi->type != I40E_VSI_SRIOV) { 13447 if (vsi->netdev_registered) { 13448 vsi->netdev_registered = false; 13449 if (vsi->netdev) { 13450 /* results in a call to i40e_close() */ 13451 unregister_netdev(vsi->netdev); 13452 } 13453 } else { 13454 i40e_vsi_close(vsi); 13455 } 13456 i40e_vsi_disable_irq(vsi); 13457 } 13458 13459 spin_lock_bh(&vsi->mac_filter_hash_lock); 13460 13461 /* clear the sync flag on all filters */ 13462 if (vsi->netdev) { 13463 __dev_uc_unsync(vsi->netdev, NULL); 13464 __dev_mc_unsync(vsi->netdev, NULL); 13465 } 13466 13467 /* make sure any remaining filters are marked for deletion */ 13468 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 13469 __i40e_del_filter(vsi, f); 13470 13471 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13472 13473 i40e_sync_vsi_filters(vsi); 13474 13475 i40e_vsi_delete(vsi); 13476 i40e_vsi_free_q_vectors(vsi); 13477 if (vsi->netdev) { 13478 free_netdev(vsi->netdev); 13479 vsi->netdev = NULL; 13480 } 13481 i40e_vsi_clear_rings(vsi); 13482 i40e_vsi_clear(vsi); 13483 13484 /* If this was the last thing on the VEB, except for the 13485 * controlling VSI, remove the VEB, which puts the controlling 13486 * VSI onto the next level down in the switch. 13487 * 13488 * Well, okay, there's one more exception here: don't remove 13489 * the orphan VEBs yet. We'll wait for an explicit remove request 13490 * from up the network stack. 13491 */ 13492 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 13493 if (pf->vsi[i] && 13494 pf->vsi[i]->uplink_seid == uplink_seid && 13495 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13496 n++; /* count the VSIs */ 13497 } 13498 } 13499 for (i = 0; i < I40E_MAX_VEB; i++) { 13500 if (!pf->veb[i]) 13501 continue; 13502 if (pf->veb[i]->uplink_seid == uplink_seid) 13503 n++; /* count the VEBs */ 13504 if (pf->veb[i]->seid == uplink_seid) 13505 veb = pf->veb[i]; 13506 } 13507 if (n == 0 && veb && veb->uplink_seid != 0) 13508 i40e_veb_release(veb); 13509 13510 return 0; 13511 } 13512 13513 /** 13514 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 13515 * @vsi: ptr to the VSI 13516 * 13517 * This should only be called after i40e_vsi_mem_alloc() which allocates the 13518 * corresponding SW VSI structure and initializes num_queue_pairs for the 13519 * newly allocated VSI. 13520 * 13521 * Returns 0 on success or negative on failure 13522 **/ 13523 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 13524 { 13525 int ret = -ENOENT; 13526 struct i40e_pf *pf = vsi->back; 13527 13528 if (vsi->q_vectors[0]) { 13529 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 13530 vsi->seid); 13531 return -EEXIST; 13532 } 13533 13534 if (vsi->base_vector) { 13535 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 13536 vsi->seid, vsi->base_vector); 13537 return -EEXIST; 13538 } 13539 13540 ret = i40e_vsi_alloc_q_vectors(vsi); 13541 if (ret) { 13542 dev_info(&pf->pdev->dev, 13543 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 13544 vsi->num_q_vectors, vsi->seid, ret); 13545 vsi->num_q_vectors = 0; 13546 goto vector_setup_out; 13547 } 13548 13549 /* In Legacy mode, we do not have to get any other vector since we 13550 * piggyback on the misc/ICR0 for queue interrupts. 13551 */ 13552 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 13553 return ret; 13554 if (vsi->num_q_vectors) 13555 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 13556 vsi->num_q_vectors, vsi->idx); 13557 if (vsi->base_vector < 0) { 13558 dev_info(&pf->pdev->dev, 13559 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 13560 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 13561 i40e_vsi_free_q_vectors(vsi); 13562 ret = -ENOENT; 13563 goto vector_setup_out; 13564 } 13565 13566 vector_setup_out: 13567 return ret; 13568 } 13569 13570 /** 13571 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 13572 * @vsi: pointer to the vsi. 13573 * 13574 * This re-allocates a vsi's queue resources. 13575 * 13576 * Returns pointer to the successfully allocated and configured VSI sw struct 13577 * on success, otherwise returns NULL on failure. 13578 **/ 13579 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 13580 { 13581 u16 alloc_queue_pairs; 13582 struct i40e_pf *pf; 13583 u8 enabled_tc; 13584 int ret; 13585 13586 if (!vsi) 13587 return NULL; 13588 13589 pf = vsi->back; 13590 13591 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 13592 i40e_vsi_clear_rings(vsi); 13593 13594 i40e_vsi_free_arrays(vsi, false); 13595 i40e_set_num_rings_in_vsi(vsi); 13596 ret = i40e_vsi_alloc_arrays(vsi, false); 13597 if (ret) 13598 goto err_vsi; 13599 13600 alloc_queue_pairs = vsi->alloc_queue_pairs * 13601 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 13602 13603 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 13604 if (ret < 0) { 13605 dev_info(&pf->pdev->dev, 13606 "failed to get tracking for %d queues for VSI %d err %d\n", 13607 alloc_queue_pairs, vsi->seid, ret); 13608 goto err_vsi; 13609 } 13610 vsi->base_queue = ret; 13611 13612 /* Update the FW view of the VSI. Force a reset of TC and queue 13613 * layout configurations. 13614 */ 13615 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 13616 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 13617 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 13618 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 13619 if (vsi->type == I40E_VSI_MAIN) 13620 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 13621 13622 /* assign it some queues */ 13623 ret = i40e_alloc_rings(vsi); 13624 if (ret) 13625 goto err_rings; 13626 13627 /* map all of the rings to the q_vectors */ 13628 i40e_vsi_map_rings_to_vectors(vsi); 13629 return vsi; 13630 13631 err_rings: 13632 i40e_vsi_free_q_vectors(vsi); 13633 if (vsi->netdev_registered) { 13634 vsi->netdev_registered = false; 13635 unregister_netdev(vsi->netdev); 13636 free_netdev(vsi->netdev); 13637 vsi->netdev = NULL; 13638 } 13639 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13640 err_vsi: 13641 i40e_vsi_clear(vsi); 13642 return NULL; 13643 } 13644 13645 /** 13646 * i40e_vsi_setup - Set up a VSI by a given type 13647 * @pf: board private structure 13648 * @type: VSI type 13649 * @uplink_seid: the switch element to link to 13650 * @param1: usage depends upon VSI type. For VF types, indicates VF id 13651 * 13652 * This allocates the sw VSI structure and its queue resources, then add a VSI 13653 * to the identified VEB. 13654 * 13655 * Returns pointer to the successfully allocated and configure VSI sw struct on 13656 * success, otherwise returns NULL on failure. 13657 **/ 13658 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 13659 u16 uplink_seid, u32 param1) 13660 { 13661 struct i40e_vsi *vsi = NULL; 13662 struct i40e_veb *veb = NULL; 13663 u16 alloc_queue_pairs; 13664 int ret, i; 13665 int v_idx; 13666 13667 /* The requested uplink_seid must be either 13668 * - the PF's port seid 13669 * no VEB is needed because this is the PF 13670 * or this is a Flow Director special case VSI 13671 * - seid of an existing VEB 13672 * - seid of a VSI that owns an existing VEB 13673 * - seid of a VSI that doesn't own a VEB 13674 * a new VEB is created and the VSI becomes the owner 13675 * - seid of the PF VSI, which is what creates the first VEB 13676 * this is a special case of the previous 13677 * 13678 * Find which uplink_seid we were given and create a new VEB if needed 13679 */ 13680 for (i = 0; i < I40E_MAX_VEB; i++) { 13681 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 13682 veb = pf->veb[i]; 13683 break; 13684 } 13685 } 13686 13687 if (!veb && uplink_seid != pf->mac_seid) { 13688 13689 for (i = 0; i < pf->num_alloc_vsi; i++) { 13690 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 13691 vsi = pf->vsi[i]; 13692 break; 13693 } 13694 } 13695 if (!vsi) { 13696 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 13697 uplink_seid); 13698 return NULL; 13699 } 13700 13701 if (vsi->uplink_seid == pf->mac_seid) 13702 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 13703 vsi->tc_config.enabled_tc); 13704 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 13705 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 13706 vsi->tc_config.enabled_tc); 13707 if (veb) { 13708 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 13709 dev_info(&vsi->back->pdev->dev, 13710 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 13711 return NULL; 13712 } 13713 /* We come up by default in VEPA mode if SRIOV is not 13714 * already enabled, in which case we can't force VEPA 13715 * mode. 13716 */ 13717 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 13718 veb->bridge_mode = BRIDGE_MODE_VEPA; 13719 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 13720 } 13721 i40e_config_bridge_mode(veb); 13722 } 13723 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 13724 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 13725 veb = pf->veb[i]; 13726 } 13727 if (!veb) { 13728 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 13729 return NULL; 13730 } 13731 13732 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 13733 uplink_seid = veb->seid; 13734 } 13735 13736 /* get vsi sw struct */ 13737 v_idx = i40e_vsi_mem_alloc(pf, type); 13738 if (v_idx < 0) 13739 goto err_alloc; 13740 vsi = pf->vsi[v_idx]; 13741 if (!vsi) 13742 goto err_alloc; 13743 vsi->type = type; 13744 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 13745 13746 if (type == I40E_VSI_MAIN) 13747 pf->lan_vsi = v_idx; 13748 else if (type == I40E_VSI_SRIOV) 13749 vsi->vf_id = param1; 13750 /* assign it some queues */ 13751 alloc_queue_pairs = vsi->alloc_queue_pairs * 13752 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 13753 13754 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 13755 if (ret < 0) { 13756 dev_info(&pf->pdev->dev, 13757 "failed to get tracking for %d queues for VSI %d err=%d\n", 13758 alloc_queue_pairs, vsi->seid, ret); 13759 goto err_vsi; 13760 } 13761 vsi->base_queue = ret; 13762 13763 /* get a VSI from the hardware */ 13764 vsi->uplink_seid = uplink_seid; 13765 ret = i40e_add_vsi(vsi); 13766 if (ret) 13767 goto err_vsi; 13768 13769 switch (vsi->type) { 13770 /* setup the netdev if needed */ 13771 case I40E_VSI_MAIN: 13772 case I40E_VSI_VMDQ2: 13773 ret = i40e_config_netdev(vsi); 13774 if (ret) 13775 goto err_netdev; 13776 ret = register_netdev(vsi->netdev); 13777 if (ret) 13778 goto err_netdev; 13779 vsi->netdev_registered = true; 13780 netif_carrier_off(vsi->netdev); 13781 #ifdef CONFIG_I40E_DCB 13782 /* Setup DCB netlink interface */ 13783 i40e_dcbnl_setup(vsi); 13784 #endif /* CONFIG_I40E_DCB */ 13785 fallthrough; 13786 case I40E_VSI_FDIR: 13787 /* set up vectors and rings if needed */ 13788 ret = i40e_vsi_setup_vectors(vsi); 13789 if (ret) 13790 goto err_msix; 13791 13792 ret = i40e_alloc_rings(vsi); 13793 if (ret) 13794 goto err_rings; 13795 13796 /* map all of the rings to the q_vectors */ 13797 i40e_vsi_map_rings_to_vectors(vsi); 13798 13799 i40e_vsi_reset_stats(vsi); 13800 break; 13801 default: 13802 /* no netdev or rings for the other VSI types */ 13803 break; 13804 } 13805 13806 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 13807 (vsi->type == I40E_VSI_VMDQ2)) { 13808 ret = i40e_vsi_config_rss(vsi); 13809 } 13810 return vsi; 13811 13812 err_rings: 13813 i40e_vsi_free_q_vectors(vsi); 13814 err_msix: 13815 if (vsi->netdev_registered) { 13816 vsi->netdev_registered = false; 13817 unregister_netdev(vsi->netdev); 13818 free_netdev(vsi->netdev); 13819 vsi->netdev = NULL; 13820 } 13821 err_netdev: 13822 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13823 err_vsi: 13824 i40e_vsi_clear(vsi); 13825 err_alloc: 13826 return NULL; 13827 } 13828 13829 /** 13830 * i40e_veb_get_bw_info - Query VEB BW information 13831 * @veb: the veb to query 13832 * 13833 * Query the Tx scheduler BW configuration data for given VEB 13834 **/ 13835 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 13836 { 13837 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 13838 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 13839 struct i40e_pf *pf = veb->pf; 13840 struct i40e_hw *hw = &pf->hw; 13841 u32 tc_bw_max; 13842 int ret = 0; 13843 int i; 13844 13845 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 13846 &bw_data, NULL); 13847 if (ret) { 13848 dev_info(&pf->pdev->dev, 13849 "query veb bw config failed, err %s aq_err %s\n", 13850 i40e_stat_str(&pf->hw, ret), 13851 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13852 goto out; 13853 } 13854 13855 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 13856 &ets_data, NULL); 13857 if (ret) { 13858 dev_info(&pf->pdev->dev, 13859 "query veb bw ets config failed, err %s aq_err %s\n", 13860 i40e_stat_str(&pf->hw, ret), 13861 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13862 goto out; 13863 } 13864 13865 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 13866 veb->bw_max_quanta = ets_data.tc_bw_max; 13867 veb->is_abs_credits = bw_data.absolute_credits_enable; 13868 veb->enabled_tc = ets_data.tc_valid_bits; 13869 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 13870 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 13871 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 13872 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 13873 veb->bw_tc_limit_credits[i] = 13874 le16_to_cpu(bw_data.tc_bw_limits[i]); 13875 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 13876 } 13877 13878 out: 13879 return ret; 13880 } 13881 13882 /** 13883 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 13884 * @pf: board private structure 13885 * 13886 * On error: returns error code (negative) 13887 * On success: returns vsi index in PF (positive) 13888 **/ 13889 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 13890 { 13891 int ret = -ENOENT; 13892 struct i40e_veb *veb; 13893 int i; 13894 13895 /* Need to protect the allocation of switch elements at the PF level */ 13896 mutex_lock(&pf->switch_mutex); 13897 13898 /* VEB list may be fragmented if VEB creation/destruction has 13899 * been happening. We can afford to do a quick scan to look 13900 * for any free slots in the list. 13901 * 13902 * find next empty veb slot, looping back around if necessary 13903 */ 13904 i = 0; 13905 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 13906 i++; 13907 if (i >= I40E_MAX_VEB) { 13908 ret = -ENOMEM; 13909 goto err_alloc_veb; /* out of VEB slots! */ 13910 } 13911 13912 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 13913 if (!veb) { 13914 ret = -ENOMEM; 13915 goto err_alloc_veb; 13916 } 13917 veb->pf = pf; 13918 veb->idx = i; 13919 veb->enabled_tc = 1; 13920 13921 pf->veb[i] = veb; 13922 ret = i; 13923 err_alloc_veb: 13924 mutex_unlock(&pf->switch_mutex); 13925 return ret; 13926 } 13927 13928 /** 13929 * i40e_switch_branch_release - Delete a branch of the switch tree 13930 * @branch: where to start deleting 13931 * 13932 * This uses recursion to find the tips of the branch to be 13933 * removed, deleting until we get back to and can delete this VEB. 13934 **/ 13935 static void i40e_switch_branch_release(struct i40e_veb *branch) 13936 { 13937 struct i40e_pf *pf = branch->pf; 13938 u16 branch_seid = branch->seid; 13939 u16 veb_idx = branch->idx; 13940 int i; 13941 13942 /* release any VEBs on this VEB - RECURSION */ 13943 for (i = 0; i < I40E_MAX_VEB; i++) { 13944 if (!pf->veb[i]) 13945 continue; 13946 if (pf->veb[i]->uplink_seid == branch->seid) 13947 i40e_switch_branch_release(pf->veb[i]); 13948 } 13949 13950 /* Release the VSIs on this VEB, but not the owner VSI. 13951 * 13952 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 13953 * the VEB itself, so don't use (*branch) after this loop. 13954 */ 13955 for (i = 0; i < pf->num_alloc_vsi; i++) { 13956 if (!pf->vsi[i]) 13957 continue; 13958 if (pf->vsi[i]->uplink_seid == branch_seid && 13959 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13960 i40e_vsi_release(pf->vsi[i]); 13961 } 13962 } 13963 13964 /* There's one corner case where the VEB might not have been 13965 * removed, so double check it here and remove it if needed. 13966 * This case happens if the veb was created from the debugfs 13967 * commands and no VSIs were added to it. 13968 */ 13969 if (pf->veb[veb_idx]) 13970 i40e_veb_release(pf->veb[veb_idx]); 13971 } 13972 13973 /** 13974 * i40e_veb_clear - remove veb struct 13975 * @veb: the veb to remove 13976 **/ 13977 static void i40e_veb_clear(struct i40e_veb *veb) 13978 { 13979 if (!veb) 13980 return; 13981 13982 if (veb->pf) { 13983 struct i40e_pf *pf = veb->pf; 13984 13985 mutex_lock(&pf->switch_mutex); 13986 if (pf->veb[veb->idx] == veb) 13987 pf->veb[veb->idx] = NULL; 13988 mutex_unlock(&pf->switch_mutex); 13989 } 13990 13991 kfree(veb); 13992 } 13993 13994 /** 13995 * i40e_veb_release - Delete a VEB and free its resources 13996 * @veb: the VEB being removed 13997 **/ 13998 void i40e_veb_release(struct i40e_veb *veb) 13999 { 14000 struct i40e_vsi *vsi = NULL; 14001 struct i40e_pf *pf; 14002 int i, n = 0; 14003 14004 pf = veb->pf; 14005 14006 /* find the remaining VSI and check for extras */ 14007 for (i = 0; i < pf->num_alloc_vsi; i++) { 14008 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 14009 n++; 14010 vsi = pf->vsi[i]; 14011 } 14012 } 14013 if (n != 1) { 14014 dev_info(&pf->pdev->dev, 14015 "can't remove VEB %d with %d VSIs left\n", 14016 veb->seid, n); 14017 return; 14018 } 14019 14020 /* move the remaining VSI to uplink veb */ 14021 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 14022 if (veb->uplink_seid) { 14023 vsi->uplink_seid = veb->uplink_seid; 14024 if (veb->uplink_seid == pf->mac_seid) 14025 vsi->veb_idx = I40E_NO_VEB; 14026 else 14027 vsi->veb_idx = veb->veb_idx; 14028 } else { 14029 /* floating VEB */ 14030 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 14031 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 14032 } 14033 14034 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14035 i40e_veb_clear(veb); 14036 } 14037 14038 /** 14039 * i40e_add_veb - create the VEB in the switch 14040 * @veb: the VEB to be instantiated 14041 * @vsi: the controlling VSI 14042 **/ 14043 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 14044 { 14045 struct i40e_pf *pf = veb->pf; 14046 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 14047 int ret; 14048 14049 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 14050 veb->enabled_tc, false, 14051 &veb->seid, enable_stats, NULL); 14052 14053 /* get a VEB from the hardware */ 14054 if (ret) { 14055 dev_info(&pf->pdev->dev, 14056 "couldn't add VEB, err %s aq_err %s\n", 14057 i40e_stat_str(&pf->hw, ret), 14058 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14059 return -EPERM; 14060 } 14061 14062 /* get statistics counter */ 14063 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 14064 &veb->stats_idx, NULL, NULL, NULL); 14065 if (ret) { 14066 dev_info(&pf->pdev->dev, 14067 "couldn't get VEB statistics idx, err %s aq_err %s\n", 14068 i40e_stat_str(&pf->hw, ret), 14069 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14070 return -EPERM; 14071 } 14072 ret = i40e_veb_get_bw_info(veb); 14073 if (ret) { 14074 dev_info(&pf->pdev->dev, 14075 "couldn't get VEB bw info, err %s aq_err %s\n", 14076 i40e_stat_str(&pf->hw, ret), 14077 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14078 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 14079 return -ENOENT; 14080 } 14081 14082 vsi->uplink_seid = veb->seid; 14083 vsi->veb_idx = veb->idx; 14084 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 14085 14086 return 0; 14087 } 14088 14089 /** 14090 * i40e_veb_setup - Set up a VEB 14091 * @pf: board private structure 14092 * @flags: VEB setup flags 14093 * @uplink_seid: the switch element to link to 14094 * @vsi_seid: the initial VSI seid 14095 * @enabled_tc: Enabled TC bit-map 14096 * 14097 * This allocates the sw VEB structure and links it into the switch 14098 * It is possible and legal for this to be a duplicate of an already 14099 * existing VEB. It is also possible for both uplink and vsi seids 14100 * to be zero, in order to create a floating VEB. 14101 * 14102 * Returns pointer to the successfully allocated VEB sw struct on 14103 * success, otherwise returns NULL on failure. 14104 **/ 14105 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 14106 u16 uplink_seid, u16 vsi_seid, 14107 u8 enabled_tc) 14108 { 14109 struct i40e_veb *veb, *uplink_veb = NULL; 14110 int vsi_idx, veb_idx; 14111 int ret; 14112 14113 /* if one seid is 0, the other must be 0 to create a floating relay */ 14114 if ((uplink_seid == 0 || vsi_seid == 0) && 14115 (uplink_seid + vsi_seid != 0)) { 14116 dev_info(&pf->pdev->dev, 14117 "one, not both seid's are 0: uplink=%d vsi=%d\n", 14118 uplink_seid, vsi_seid); 14119 return NULL; 14120 } 14121 14122 /* make sure there is such a vsi and uplink */ 14123 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 14124 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 14125 break; 14126 if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) { 14127 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 14128 vsi_seid); 14129 return NULL; 14130 } 14131 14132 if (uplink_seid && uplink_seid != pf->mac_seid) { 14133 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 14134 if (pf->veb[veb_idx] && 14135 pf->veb[veb_idx]->seid == uplink_seid) { 14136 uplink_veb = pf->veb[veb_idx]; 14137 break; 14138 } 14139 } 14140 if (!uplink_veb) { 14141 dev_info(&pf->pdev->dev, 14142 "uplink seid %d not found\n", uplink_seid); 14143 return NULL; 14144 } 14145 } 14146 14147 /* get veb sw struct */ 14148 veb_idx = i40e_veb_mem_alloc(pf); 14149 if (veb_idx < 0) 14150 goto err_alloc; 14151 veb = pf->veb[veb_idx]; 14152 veb->flags = flags; 14153 veb->uplink_seid = uplink_seid; 14154 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 14155 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 14156 14157 /* create the VEB in the switch */ 14158 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 14159 if (ret) 14160 goto err_veb; 14161 if (vsi_idx == pf->lan_vsi) 14162 pf->lan_veb = veb->idx; 14163 14164 return veb; 14165 14166 err_veb: 14167 i40e_veb_clear(veb); 14168 err_alloc: 14169 return NULL; 14170 } 14171 14172 /** 14173 * i40e_setup_pf_switch_element - set PF vars based on switch type 14174 * @pf: board private structure 14175 * @ele: element we are building info from 14176 * @num_reported: total number of elements 14177 * @printconfig: should we print the contents 14178 * 14179 * helper function to assist in extracting a few useful SEID values. 14180 **/ 14181 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 14182 struct i40e_aqc_switch_config_element_resp *ele, 14183 u16 num_reported, bool printconfig) 14184 { 14185 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 14186 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 14187 u8 element_type = ele->element_type; 14188 u16 seid = le16_to_cpu(ele->seid); 14189 14190 if (printconfig) 14191 dev_info(&pf->pdev->dev, 14192 "type=%d seid=%d uplink=%d downlink=%d\n", 14193 element_type, seid, uplink_seid, downlink_seid); 14194 14195 switch (element_type) { 14196 case I40E_SWITCH_ELEMENT_TYPE_MAC: 14197 pf->mac_seid = seid; 14198 break; 14199 case I40E_SWITCH_ELEMENT_TYPE_VEB: 14200 /* Main VEB? */ 14201 if (uplink_seid != pf->mac_seid) 14202 break; 14203 if (pf->lan_veb >= I40E_MAX_VEB) { 14204 int v; 14205 14206 /* find existing or else empty VEB */ 14207 for (v = 0; v < I40E_MAX_VEB; v++) { 14208 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 14209 pf->lan_veb = v; 14210 break; 14211 } 14212 } 14213 if (pf->lan_veb >= I40E_MAX_VEB) { 14214 v = i40e_veb_mem_alloc(pf); 14215 if (v < 0) 14216 break; 14217 pf->lan_veb = v; 14218 } 14219 } 14220 if (pf->lan_veb >= I40E_MAX_VEB) 14221 break; 14222 14223 pf->veb[pf->lan_veb]->seid = seid; 14224 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 14225 pf->veb[pf->lan_veb]->pf = pf; 14226 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 14227 break; 14228 case I40E_SWITCH_ELEMENT_TYPE_VSI: 14229 if (num_reported != 1) 14230 break; 14231 /* This is immediately after a reset so we can assume this is 14232 * the PF's VSI 14233 */ 14234 pf->mac_seid = uplink_seid; 14235 pf->pf_seid = downlink_seid; 14236 pf->main_vsi_seid = seid; 14237 if (printconfig) 14238 dev_info(&pf->pdev->dev, 14239 "pf_seid=%d main_vsi_seid=%d\n", 14240 pf->pf_seid, pf->main_vsi_seid); 14241 break; 14242 case I40E_SWITCH_ELEMENT_TYPE_PF: 14243 case I40E_SWITCH_ELEMENT_TYPE_VF: 14244 case I40E_SWITCH_ELEMENT_TYPE_EMP: 14245 case I40E_SWITCH_ELEMENT_TYPE_BMC: 14246 case I40E_SWITCH_ELEMENT_TYPE_PE: 14247 case I40E_SWITCH_ELEMENT_TYPE_PA: 14248 /* ignore these for now */ 14249 break; 14250 default: 14251 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 14252 element_type, seid); 14253 break; 14254 } 14255 } 14256 14257 /** 14258 * i40e_fetch_switch_configuration - Get switch config from firmware 14259 * @pf: board private structure 14260 * @printconfig: should we print the contents 14261 * 14262 * Get the current switch configuration from the device and 14263 * extract a few useful SEID values. 14264 **/ 14265 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 14266 { 14267 struct i40e_aqc_get_switch_config_resp *sw_config; 14268 u16 next_seid = 0; 14269 int ret = 0; 14270 u8 *aq_buf; 14271 int i; 14272 14273 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 14274 if (!aq_buf) 14275 return -ENOMEM; 14276 14277 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 14278 do { 14279 u16 num_reported, num_total; 14280 14281 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 14282 I40E_AQ_LARGE_BUF, 14283 &next_seid, NULL); 14284 if (ret) { 14285 dev_info(&pf->pdev->dev, 14286 "get switch config failed err %s aq_err %s\n", 14287 i40e_stat_str(&pf->hw, ret), 14288 i40e_aq_str(&pf->hw, 14289 pf->hw.aq.asq_last_status)); 14290 kfree(aq_buf); 14291 return -ENOENT; 14292 } 14293 14294 num_reported = le16_to_cpu(sw_config->header.num_reported); 14295 num_total = le16_to_cpu(sw_config->header.num_total); 14296 14297 if (printconfig) 14298 dev_info(&pf->pdev->dev, 14299 "header: %d reported %d total\n", 14300 num_reported, num_total); 14301 14302 for (i = 0; i < num_reported; i++) { 14303 struct i40e_aqc_switch_config_element_resp *ele = 14304 &sw_config->element[i]; 14305 14306 i40e_setup_pf_switch_element(pf, ele, num_reported, 14307 printconfig); 14308 } 14309 } while (next_seid != 0); 14310 14311 kfree(aq_buf); 14312 return ret; 14313 } 14314 14315 /** 14316 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 14317 * @pf: board private structure 14318 * @reinit: if the Main VSI needs to re-initialized. 14319 * 14320 * Returns 0 on success, negative value on failure 14321 **/ 14322 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 14323 { 14324 u16 flags = 0; 14325 int ret; 14326 14327 /* find out what's out there already */ 14328 ret = i40e_fetch_switch_configuration(pf, false); 14329 if (ret) { 14330 dev_info(&pf->pdev->dev, 14331 "couldn't fetch switch config, err %s aq_err %s\n", 14332 i40e_stat_str(&pf->hw, ret), 14333 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14334 return ret; 14335 } 14336 i40e_pf_reset_stats(pf); 14337 14338 /* set the switch config bit for the whole device to 14339 * support limited promisc or true promisc 14340 * when user requests promisc. The default is limited 14341 * promisc. 14342 */ 14343 14344 if ((pf->hw.pf_id == 0) && 14345 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 14346 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14347 pf->last_sw_conf_flags = flags; 14348 } 14349 14350 if (pf->hw.pf_id == 0) { 14351 u16 valid_flags; 14352 14353 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14354 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 14355 NULL); 14356 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 14357 dev_info(&pf->pdev->dev, 14358 "couldn't set switch config bits, err %s aq_err %s\n", 14359 i40e_stat_str(&pf->hw, ret), 14360 i40e_aq_str(&pf->hw, 14361 pf->hw.aq.asq_last_status)); 14362 /* not a fatal problem, just keep going */ 14363 } 14364 pf->last_sw_conf_valid_flags = valid_flags; 14365 } 14366 14367 /* first time setup */ 14368 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 14369 struct i40e_vsi *vsi = NULL; 14370 u16 uplink_seid; 14371 14372 /* Set up the PF VSI associated with the PF's main VSI 14373 * that is already in the HW switch 14374 */ 14375 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 14376 uplink_seid = pf->veb[pf->lan_veb]->seid; 14377 else 14378 uplink_seid = pf->mac_seid; 14379 if (pf->lan_vsi == I40E_NO_VSI) 14380 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 14381 else if (reinit) 14382 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 14383 if (!vsi) { 14384 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 14385 i40e_cloud_filter_exit(pf); 14386 i40e_fdir_teardown(pf); 14387 return -EAGAIN; 14388 } 14389 } else { 14390 /* force a reset of TC and queue layout configurations */ 14391 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 14392 14393 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 14394 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 14395 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 14396 } 14397 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 14398 14399 i40e_fdir_sb_setup(pf); 14400 14401 /* Setup static PF queue filter control settings */ 14402 ret = i40e_setup_pf_filter_control(pf); 14403 if (ret) { 14404 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 14405 ret); 14406 /* Failure here should not stop continuing other steps */ 14407 } 14408 14409 /* enable RSS in the HW, even for only one queue, as the stack can use 14410 * the hash 14411 */ 14412 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 14413 i40e_pf_config_rss(pf); 14414 14415 /* fill in link information and enable LSE reporting */ 14416 i40e_link_event(pf); 14417 14418 /* Initialize user-specific link properties */ 14419 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 14420 I40E_AQ_AN_COMPLETED) ? true : false); 14421 14422 i40e_ptp_init(pf); 14423 14424 /* repopulate tunnel port filters */ 14425 i40e_sync_udp_filters(pf); 14426 14427 return ret; 14428 } 14429 14430 /** 14431 * i40e_determine_queue_usage - Work out queue distribution 14432 * @pf: board private structure 14433 **/ 14434 static void i40e_determine_queue_usage(struct i40e_pf *pf) 14435 { 14436 int queues_left; 14437 int q_max; 14438 14439 pf->num_lan_qps = 0; 14440 14441 /* Find the max queues to be put into basic use. We'll always be 14442 * using TC0, whether or not DCB is running, and TC0 will get the 14443 * big RSS set. 14444 */ 14445 queues_left = pf->hw.func_caps.num_tx_qp; 14446 14447 if ((queues_left == 1) || 14448 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 14449 /* one qp for PF, no queues for anything else */ 14450 queues_left = 0; 14451 pf->alloc_rss_size = pf->num_lan_qps = 1; 14452 14453 /* make sure all the fancies are disabled */ 14454 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 14455 I40E_FLAG_IWARP_ENABLED | 14456 I40E_FLAG_FD_SB_ENABLED | 14457 I40E_FLAG_FD_ATR_ENABLED | 14458 I40E_FLAG_DCB_CAPABLE | 14459 I40E_FLAG_DCB_ENABLED | 14460 I40E_FLAG_SRIOV_ENABLED | 14461 I40E_FLAG_VMDQ_ENABLED); 14462 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14463 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 14464 I40E_FLAG_FD_SB_ENABLED | 14465 I40E_FLAG_FD_ATR_ENABLED | 14466 I40E_FLAG_DCB_CAPABLE))) { 14467 /* one qp for PF */ 14468 pf->alloc_rss_size = pf->num_lan_qps = 1; 14469 queues_left -= pf->num_lan_qps; 14470 14471 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 14472 I40E_FLAG_IWARP_ENABLED | 14473 I40E_FLAG_FD_SB_ENABLED | 14474 I40E_FLAG_FD_ATR_ENABLED | 14475 I40E_FLAG_DCB_ENABLED | 14476 I40E_FLAG_VMDQ_ENABLED); 14477 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14478 } else { 14479 /* Not enough queues for all TCs */ 14480 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 14481 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 14482 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 14483 I40E_FLAG_DCB_ENABLED); 14484 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 14485 } 14486 14487 /* limit lan qps to the smaller of qps, cpus or msix */ 14488 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 14489 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 14490 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 14491 pf->num_lan_qps = q_max; 14492 14493 queues_left -= pf->num_lan_qps; 14494 } 14495 14496 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 14497 if (queues_left > 1) { 14498 queues_left -= 1; /* save 1 queue for FD */ 14499 } else { 14500 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 14501 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14502 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 14503 } 14504 } 14505 14506 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 14507 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 14508 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 14509 (queues_left / pf->num_vf_qps)); 14510 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 14511 } 14512 14513 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 14514 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 14515 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 14516 (queues_left / pf->num_vmdq_qps)); 14517 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 14518 } 14519 14520 pf->queues_left = queues_left; 14521 dev_dbg(&pf->pdev->dev, 14522 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 14523 pf->hw.func_caps.num_tx_qp, 14524 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 14525 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 14526 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 14527 queues_left); 14528 } 14529 14530 /** 14531 * i40e_setup_pf_filter_control - Setup PF static filter control 14532 * @pf: PF to be setup 14533 * 14534 * i40e_setup_pf_filter_control sets up a PF's initial filter control 14535 * settings. If PE/FCoE are enabled then it will also set the per PF 14536 * based filter sizes required for them. It also enables Flow director, 14537 * ethertype and macvlan type filter settings for the pf. 14538 * 14539 * Returns 0 on success, negative on failure 14540 **/ 14541 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 14542 { 14543 struct i40e_filter_control_settings *settings = &pf->filter_settings; 14544 14545 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 14546 14547 /* Flow Director is enabled */ 14548 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 14549 settings->enable_fdir = true; 14550 14551 /* Ethtype and MACVLAN filters enabled for PF */ 14552 settings->enable_ethtype = true; 14553 settings->enable_macvlan = true; 14554 14555 if (i40e_set_filter_control(&pf->hw, settings)) 14556 return -ENOENT; 14557 14558 return 0; 14559 } 14560 14561 #define INFO_STRING_LEN 255 14562 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 14563 static void i40e_print_features(struct i40e_pf *pf) 14564 { 14565 struct i40e_hw *hw = &pf->hw; 14566 char *buf; 14567 int i; 14568 14569 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 14570 if (!buf) 14571 return; 14572 14573 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 14574 #ifdef CONFIG_PCI_IOV 14575 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 14576 #endif 14577 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 14578 pf->hw.func_caps.num_vsis, 14579 pf->vsi[pf->lan_vsi]->num_queue_pairs); 14580 if (pf->flags & I40E_FLAG_RSS_ENABLED) 14581 i += scnprintf(&buf[i], REMAIN(i), " RSS"); 14582 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 14583 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); 14584 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 14585 i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); 14586 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); 14587 } 14588 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 14589 i += scnprintf(&buf[i], REMAIN(i), " DCB"); 14590 i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); 14591 i += scnprintf(&buf[i], REMAIN(i), " Geneve"); 14592 if (pf->flags & I40E_FLAG_PTP) 14593 i += scnprintf(&buf[i], REMAIN(i), " PTP"); 14594 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 14595 i += scnprintf(&buf[i], REMAIN(i), " VEB"); 14596 else 14597 i += scnprintf(&buf[i], REMAIN(i), " VEPA"); 14598 14599 dev_info(&pf->pdev->dev, "%s\n", buf); 14600 kfree(buf); 14601 WARN_ON(i > INFO_STRING_LEN); 14602 } 14603 14604 /** 14605 * i40e_get_platform_mac_addr - get platform-specific MAC address 14606 * @pdev: PCI device information struct 14607 * @pf: board private structure 14608 * 14609 * Look up the MAC address for the device. First we'll try 14610 * eth_platform_get_mac_address, which will check Open Firmware, or arch 14611 * specific fallback. Otherwise, we'll default to the stored value in 14612 * firmware. 14613 **/ 14614 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 14615 { 14616 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 14617 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 14618 } 14619 14620 /** 14621 * i40e_set_fec_in_flags - helper function for setting FEC options in flags 14622 * @fec_cfg: FEC option to set in flags 14623 * @flags: ptr to flags in which we set FEC option 14624 **/ 14625 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags) 14626 { 14627 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) 14628 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC; 14629 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) || 14630 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) { 14631 *flags |= I40E_FLAG_RS_FEC; 14632 *flags &= ~I40E_FLAG_BASE_R_FEC; 14633 } 14634 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) || 14635 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) { 14636 *flags |= I40E_FLAG_BASE_R_FEC; 14637 *flags &= ~I40E_FLAG_RS_FEC; 14638 } 14639 if (fec_cfg == 0) 14640 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC); 14641 } 14642 14643 /** 14644 * i40e_check_recovery_mode - check if we are running transition firmware 14645 * @pf: board private structure 14646 * 14647 * Check registers indicating the firmware runs in recovery mode. Sets the 14648 * appropriate driver state. 14649 * 14650 * Returns true if the recovery mode was detected, false otherwise 14651 **/ 14652 static bool i40e_check_recovery_mode(struct i40e_pf *pf) 14653 { 14654 u32 val = rd32(&pf->hw, I40E_GL_FWSTS); 14655 14656 if (val & I40E_GL_FWSTS_FWS1B_MASK) { 14657 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n"); 14658 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 14659 set_bit(__I40E_RECOVERY_MODE, pf->state); 14660 14661 return true; 14662 } 14663 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 14664 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n"); 14665 14666 return false; 14667 } 14668 14669 /** 14670 * i40e_pf_loop_reset - perform reset in a loop. 14671 * @pf: board private structure 14672 * 14673 * This function is useful when a NIC is about to enter recovery mode. 14674 * When a NIC's internal data structures are corrupted the NIC's 14675 * firmware is going to enter recovery mode. 14676 * Right after a POR it takes about 7 minutes for firmware to enter 14677 * recovery mode. Until that time a NIC is in some kind of intermediate 14678 * state. After that time period the NIC almost surely enters 14679 * recovery mode. The only way for a driver to detect intermediate 14680 * state is to issue a series of pf-resets and check a return value. 14681 * If a PF reset returns success then the firmware could be in recovery 14682 * mode so the caller of this code needs to check for recovery mode 14683 * if this function returns success. There is a little chance that 14684 * firmware will hang in intermediate state forever. 14685 * Since waiting 7 minutes is quite a lot of time this function waits 14686 * 10 seconds and then gives up by returning an error. 14687 * 14688 * Return 0 on success, negative on failure. 14689 **/ 14690 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf) 14691 { 14692 /* wait max 10 seconds for PF reset to succeed */ 14693 const unsigned long time_end = jiffies + 10 * HZ; 14694 14695 struct i40e_hw *hw = &pf->hw; 14696 i40e_status ret; 14697 14698 ret = i40e_pf_reset(hw); 14699 while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) { 14700 usleep_range(10000, 20000); 14701 ret = i40e_pf_reset(hw); 14702 } 14703 14704 if (ret == I40E_SUCCESS) 14705 pf->pfr_count++; 14706 else 14707 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); 14708 14709 return ret; 14710 } 14711 14712 /** 14713 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset 14714 * @pf: board private structure 14715 * 14716 * Check FW registers to determine if FW issued unexpected EMP Reset. 14717 * Every time when unexpected EMP Reset occurs the FW increments 14718 * a counter of unexpected EMP Resets. When the counter reaches 10 14719 * the FW should enter the Recovery mode 14720 * 14721 * Returns true if FW issued unexpected EMP Reset 14722 **/ 14723 static bool i40e_check_fw_empr(struct i40e_pf *pf) 14724 { 14725 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) & 14726 I40E_GL_FWSTS_FWS1B_MASK; 14727 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) && 14728 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10); 14729 } 14730 14731 /** 14732 * i40e_handle_resets - handle EMP resets and PF resets 14733 * @pf: board private structure 14734 * 14735 * Handle both EMP resets and PF resets and conclude whether there are 14736 * any issues regarding these resets. If there are any issues then 14737 * generate log entry. 14738 * 14739 * Return 0 if NIC is healthy or negative value when there are issues 14740 * with resets 14741 **/ 14742 static i40e_status i40e_handle_resets(struct i40e_pf *pf) 14743 { 14744 const i40e_status pfr = i40e_pf_loop_reset(pf); 14745 const bool is_empr = i40e_check_fw_empr(pf); 14746 14747 if (is_empr || pfr != I40E_SUCCESS) 14748 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n"); 14749 14750 return is_empr ? I40E_ERR_RESET_FAILED : pfr; 14751 } 14752 14753 /** 14754 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode 14755 * @pf: board private structure 14756 * @hw: ptr to the hardware info 14757 * 14758 * This function does a minimal setup of all subsystems needed for running 14759 * recovery mode. 14760 * 14761 * Returns 0 on success, negative on failure 14762 **/ 14763 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw) 14764 { 14765 struct i40e_vsi *vsi; 14766 int err; 14767 int v_idx; 14768 14769 pci_save_state(pf->pdev); 14770 14771 /* set up periodic task facility */ 14772 timer_setup(&pf->service_timer, i40e_service_timer, 0); 14773 pf->service_timer_period = HZ; 14774 14775 INIT_WORK(&pf->service_task, i40e_service_task); 14776 clear_bit(__I40E_SERVICE_SCHED, pf->state); 14777 14778 err = i40e_init_interrupt_scheme(pf); 14779 if (err) 14780 goto err_switch_setup; 14781 14782 /* The number of VSIs reported by the FW is the minimum guaranteed 14783 * to us; HW supports far more and we share the remaining pool with 14784 * the other PFs. We allocate space for more than the guarantee with 14785 * the understanding that we might not get them all later. 14786 */ 14787 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 14788 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 14789 else 14790 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 14791 14792 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */ 14793 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 14794 GFP_KERNEL); 14795 if (!pf->vsi) { 14796 err = -ENOMEM; 14797 goto err_switch_setup; 14798 } 14799 14800 /* We allocate one VSI which is needed as absolute minimum 14801 * in order to register the netdev 14802 */ 14803 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN); 14804 if (v_idx < 0) 14805 goto err_switch_setup; 14806 pf->lan_vsi = v_idx; 14807 vsi = pf->vsi[v_idx]; 14808 if (!vsi) 14809 goto err_switch_setup; 14810 vsi->alloc_queue_pairs = 1; 14811 err = i40e_config_netdev(vsi); 14812 if (err) 14813 goto err_switch_setup; 14814 err = register_netdev(vsi->netdev); 14815 if (err) 14816 goto err_switch_setup; 14817 vsi->netdev_registered = true; 14818 i40e_dbg_pf_init(pf); 14819 14820 err = i40e_setup_misc_vector_for_recovery_mode(pf); 14821 if (err) 14822 goto err_switch_setup; 14823 14824 /* tell the firmware that we're starting */ 14825 i40e_send_version(pf); 14826 14827 /* since everything's happy, start the service_task timer */ 14828 mod_timer(&pf->service_timer, 14829 round_jiffies(jiffies + pf->service_timer_period)); 14830 14831 return 0; 14832 14833 err_switch_setup: 14834 i40e_reset_interrupt_capability(pf); 14835 del_timer_sync(&pf->service_timer); 14836 i40e_shutdown_adminq(hw); 14837 iounmap(hw->hw_addr); 14838 pci_disable_pcie_error_reporting(pf->pdev); 14839 pci_release_mem_regions(pf->pdev); 14840 pci_disable_device(pf->pdev); 14841 kfree(pf); 14842 14843 return err; 14844 } 14845 14846 /** 14847 * i40e_probe - Device initialization routine 14848 * @pdev: PCI device information struct 14849 * @ent: entry in i40e_pci_tbl 14850 * 14851 * i40e_probe initializes a PF identified by a pci_dev structure. 14852 * The OS initialization, configuring of the PF private structure, 14853 * and a hardware reset occur. 14854 * 14855 * Returns 0 on success, negative on failure 14856 **/ 14857 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 14858 { 14859 struct i40e_aq_get_phy_abilities_resp abilities; 14860 struct i40e_pf *pf; 14861 struct i40e_hw *hw; 14862 static u16 pfs_found; 14863 u16 wol_nvm_bits; 14864 u16 link_status; 14865 int err; 14866 u32 val; 14867 u32 i; 14868 u8 set_fc_aq_fail; 14869 14870 err = pci_enable_device_mem(pdev); 14871 if (err) 14872 return err; 14873 14874 /* set up for high or low dma */ 14875 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 14876 if (err) { 14877 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 14878 if (err) { 14879 dev_err(&pdev->dev, 14880 "DMA configuration failed: 0x%x\n", err); 14881 goto err_dma; 14882 } 14883 } 14884 14885 /* set up pci connections */ 14886 err = pci_request_mem_regions(pdev, i40e_driver_name); 14887 if (err) { 14888 dev_info(&pdev->dev, 14889 "pci_request_selected_regions failed %d\n", err); 14890 goto err_pci_reg; 14891 } 14892 14893 pci_enable_pcie_error_reporting(pdev); 14894 pci_set_master(pdev); 14895 14896 /* Now that we have a PCI connection, we need to do the 14897 * low level device setup. This is primarily setting up 14898 * the Admin Queue structures and then querying for the 14899 * device's current profile information. 14900 */ 14901 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 14902 if (!pf) { 14903 err = -ENOMEM; 14904 goto err_pf_alloc; 14905 } 14906 pf->next_vsi = 0; 14907 pf->pdev = pdev; 14908 set_bit(__I40E_DOWN, pf->state); 14909 14910 hw = &pf->hw; 14911 hw->back = pf; 14912 14913 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 14914 I40E_MAX_CSR_SPACE); 14915 /* We believe that the highest register to read is 14916 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size 14917 * is not less than that before mapping to prevent a 14918 * kernel panic. 14919 */ 14920 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) { 14921 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n", 14922 pf->ioremap_len); 14923 err = -ENOMEM; 14924 goto err_ioremap; 14925 } 14926 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 14927 if (!hw->hw_addr) { 14928 err = -EIO; 14929 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 14930 (unsigned int)pci_resource_start(pdev, 0), 14931 pf->ioremap_len, err); 14932 goto err_ioremap; 14933 } 14934 hw->vendor_id = pdev->vendor; 14935 hw->device_id = pdev->device; 14936 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 14937 hw->subsystem_vendor_id = pdev->subsystem_vendor; 14938 hw->subsystem_device_id = pdev->subsystem_device; 14939 hw->bus.device = PCI_SLOT(pdev->devfn); 14940 hw->bus.func = PCI_FUNC(pdev->devfn); 14941 hw->bus.bus_id = pdev->bus->number; 14942 pf->instance = pfs_found; 14943 14944 /* Select something other than the 802.1ad ethertype for the 14945 * switch to use internally and drop on ingress. 14946 */ 14947 hw->switch_tag = 0xffff; 14948 hw->first_tag = ETH_P_8021AD; 14949 hw->second_tag = ETH_P_8021Q; 14950 14951 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 14952 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 14953 INIT_LIST_HEAD(&pf->ddp_old_prof); 14954 14955 /* set up the locks for the AQ, do this only once in probe 14956 * and destroy them only once in remove 14957 */ 14958 mutex_init(&hw->aq.asq_mutex); 14959 mutex_init(&hw->aq.arq_mutex); 14960 14961 pf->msg_enable = netif_msg_init(debug, 14962 NETIF_MSG_DRV | 14963 NETIF_MSG_PROBE | 14964 NETIF_MSG_LINK); 14965 if (debug < -1) 14966 pf->hw.debug_mask = debug; 14967 14968 /* do a special CORER for clearing PXE mode once at init */ 14969 if (hw->revision_id == 0 && 14970 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 14971 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 14972 i40e_flush(hw); 14973 msleep(200); 14974 pf->corer_count++; 14975 14976 i40e_clear_pxe_mode(hw); 14977 } 14978 14979 /* Reset here to make sure all is clean and to define PF 'n' */ 14980 i40e_clear_hw(hw); 14981 14982 err = i40e_set_mac_type(hw); 14983 if (err) { 14984 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 14985 err); 14986 goto err_pf_reset; 14987 } 14988 14989 err = i40e_handle_resets(pf); 14990 if (err) 14991 goto err_pf_reset; 14992 14993 i40e_check_recovery_mode(pf); 14994 14995 hw->aq.num_arq_entries = I40E_AQ_LEN; 14996 hw->aq.num_asq_entries = I40E_AQ_LEN; 14997 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 14998 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 14999 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 15000 15001 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 15002 "%s-%s:misc", 15003 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 15004 15005 err = i40e_init_shared_code(hw); 15006 if (err) { 15007 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 15008 err); 15009 goto err_pf_reset; 15010 } 15011 15012 /* set up a default setting for link flow control */ 15013 pf->hw.fc.requested_mode = I40E_FC_NONE; 15014 15015 err = i40e_init_adminq(hw); 15016 if (err) { 15017 if (err == I40E_ERR_FIRMWARE_API_VERSION) 15018 dev_info(&pdev->dev, 15019 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", 15020 hw->aq.api_maj_ver, 15021 hw->aq.api_min_ver, 15022 I40E_FW_API_VERSION_MAJOR, 15023 I40E_FW_MINOR_VERSION(hw)); 15024 else 15025 dev_info(&pdev->dev, 15026 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 15027 15028 goto err_pf_reset; 15029 } 15030 i40e_get_oem_version(hw); 15031 15032 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */ 15033 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n", 15034 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 15035 hw->aq.api_maj_ver, hw->aq.api_min_ver, 15036 i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id, 15037 hw->subsystem_vendor_id, hw->subsystem_device_id); 15038 15039 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 15040 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 15041 dev_info(&pdev->dev, 15042 "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n", 15043 hw->aq.api_maj_ver, 15044 hw->aq.api_min_ver, 15045 I40E_FW_API_VERSION_MAJOR, 15046 I40E_FW_MINOR_VERSION(hw)); 15047 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 15048 dev_info(&pdev->dev, 15049 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n", 15050 hw->aq.api_maj_ver, 15051 hw->aq.api_min_ver, 15052 I40E_FW_API_VERSION_MAJOR, 15053 I40E_FW_MINOR_VERSION(hw)); 15054 15055 i40e_verify_eeprom(pf); 15056 15057 /* Rev 0 hardware was never productized */ 15058 if (hw->revision_id < 1) 15059 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 15060 15061 i40e_clear_pxe_mode(hw); 15062 15063 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 15064 if (err) 15065 goto err_adminq_setup; 15066 15067 err = i40e_sw_init(pf); 15068 if (err) { 15069 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 15070 goto err_sw_init; 15071 } 15072 15073 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 15074 return i40e_init_recovery_mode(pf, hw); 15075 15076 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 15077 hw->func_caps.num_rx_qp, 0, 0); 15078 if (err) { 15079 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 15080 goto err_init_lan_hmc; 15081 } 15082 15083 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 15084 if (err) { 15085 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 15086 err = -ENOENT; 15087 goto err_configure_lan_hmc; 15088 } 15089 15090 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 15091 * Ignore error return codes because if it was already disabled via 15092 * hardware settings this will fail 15093 */ 15094 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 15095 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 15096 i40e_aq_stop_lldp(hw, true, false, NULL); 15097 } 15098 15099 /* allow a platform config to override the HW addr */ 15100 i40e_get_platform_mac_addr(pdev, pf); 15101 15102 if (!is_valid_ether_addr(hw->mac.addr)) { 15103 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 15104 err = -EIO; 15105 goto err_mac_addr; 15106 } 15107 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 15108 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 15109 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 15110 if (is_valid_ether_addr(hw->mac.port_addr)) 15111 pf->hw_features |= I40E_HW_PORT_ID_VALID; 15112 15113 pci_set_drvdata(pdev, pf); 15114 pci_save_state(pdev); 15115 15116 dev_info(&pdev->dev, 15117 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ? 15118 "FW LLDP is disabled\n" : 15119 "FW LLDP is enabled\n"); 15120 15121 /* Enable FW to write default DCB config on link-up */ 15122 i40e_aq_set_dcb_parameters(hw, true, NULL); 15123 15124 #ifdef CONFIG_I40E_DCB 15125 err = i40e_init_pf_dcb(pf); 15126 if (err) { 15127 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 15128 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 15129 /* Continue without DCB enabled */ 15130 } 15131 #endif /* CONFIG_I40E_DCB */ 15132 15133 /* set up periodic task facility */ 15134 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15135 pf->service_timer_period = HZ; 15136 15137 INIT_WORK(&pf->service_task, i40e_service_task); 15138 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15139 15140 /* NVM bit on means WoL disabled for the port */ 15141 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 15142 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 15143 pf->wol_en = false; 15144 else 15145 pf->wol_en = true; 15146 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 15147 15148 /* set up the main switch operations */ 15149 i40e_determine_queue_usage(pf); 15150 err = i40e_init_interrupt_scheme(pf); 15151 if (err) 15152 goto err_switch_setup; 15153 15154 /* The number of VSIs reported by the FW is the minimum guaranteed 15155 * to us; HW supports far more and we share the remaining pool with 15156 * the other PFs. We allocate space for more than the guarantee with 15157 * the understanding that we might not get them all later. 15158 */ 15159 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15160 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15161 else 15162 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15163 15164 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 15165 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15166 GFP_KERNEL); 15167 if (!pf->vsi) { 15168 err = -ENOMEM; 15169 goto err_switch_setup; 15170 } 15171 15172 #ifdef CONFIG_PCI_IOV 15173 /* prep for VF support */ 15174 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15175 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15176 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15177 if (pci_num_vf(pdev)) 15178 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 15179 } 15180 #endif 15181 err = i40e_setup_pf_switch(pf, false); 15182 if (err) { 15183 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 15184 goto err_vsis; 15185 } 15186 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 15187 15188 /* Make sure flow control is set according to current settings */ 15189 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 15190 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 15191 dev_dbg(&pf->pdev->dev, 15192 "Set fc with err %s aq_err %s on get_phy_cap\n", 15193 i40e_stat_str(hw, err), 15194 i40e_aq_str(hw, hw->aq.asq_last_status)); 15195 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 15196 dev_dbg(&pf->pdev->dev, 15197 "Set fc with err %s aq_err %s on set_phy_config\n", 15198 i40e_stat_str(hw, err), 15199 i40e_aq_str(hw, hw->aq.asq_last_status)); 15200 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 15201 dev_dbg(&pf->pdev->dev, 15202 "Set fc with err %s aq_err %s on get_link_info\n", 15203 i40e_stat_str(hw, err), 15204 i40e_aq_str(hw, hw->aq.asq_last_status)); 15205 15206 /* if FDIR VSI was set up, start it now */ 15207 for (i = 0; i < pf->num_alloc_vsi; i++) { 15208 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 15209 i40e_vsi_open(pf->vsi[i]); 15210 break; 15211 } 15212 } 15213 15214 /* The driver only wants link up/down and module qualification 15215 * reports from firmware. Note the negative logic. 15216 */ 15217 err = i40e_aq_set_phy_int_mask(&pf->hw, 15218 ~(I40E_AQ_EVENT_LINK_UPDOWN | 15219 I40E_AQ_EVENT_MEDIA_NA | 15220 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 15221 if (err) 15222 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 15223 i40e_stat_str(&pf->hw, err), 15224 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15225 15226 /* Reconfigure hardware for allowing smaller MSS in the case 15227 * of TSO, so that we avoid the MDD being fired and causing 15228 * a reset in the case of small MSS+TSO. 15229 */ 15230 val = rd32(hw, I40E_REG_MSS); 15231 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 15232 val &= ~I40E_REG_MSS_MIN_MASK; 15233 val |= I40E_64BYTE_MSS; 15234 wr32(hw, I40E_REG_MSS, val); 15235 } 15236 15237 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 15238 msleep(75); 15239 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 15240 if (err) 15241 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 15242 i40e_stat_str(&pf->hw, err), 15243 i40e_aq_str(&pf->hw, 15244 pf->hw.aq.asq_last_status)); 15245 } 15246 /* The main driver is (mostly) up and happy. We need to set this state 15247 * before setting up the misc vector or we get a race and the vector 15248 * ends up disabled forever. 15249 */ 15250 clear_bit(__I40E_DOWN, pf->state); 15251 15252 /* In case of MSIX we are going to setup the misc vector right here 15253 * to handle admin queue events etc. In case of legacy and MSI 15254 * the misc functionality and queue processing is combined in 15255 * the same vector and that gets setup at open. 15256 */ 15257 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 15258 err = i40e_setup_misc_vector(pf); 15259 if (err) { 15260 dev_info(&pdev->dev, 15261 "setup of misc vector failed: %d\n", err); 15262 goto err_vsis; 15263 } 15264 } 15265 15266 #ifdef CONFIG_PCI_IOV 15267 /* prep for VF support */ 15268 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15269 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15270 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15271 /* disable link interrupts for VFs */ 15272 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 15273 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 15274 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 15275 i40e_flush(hw); 15276 15277 if (pci_num_vf(pdev)) { 15278 dev_info(&pdev->dev, 15279 "Active VFs found, allocating resources.\n"); 15280 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 15281 if (err) 15282 dev_info(&pdev->dev, 15283 "Error %d allocating resources for existing VFs\n", 15284 err); 15285 } 15286 } 15287 #endif /* CONFIG_PCI_IOV */ 15288 15289 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15290 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 15291 pf->num_iwarp_msix, 15292 I40E_IWARP_IRQ_PILE_ID); 15293 if (pf->iwarp_base_vector < 0) { 15294 dev_info(&pdev->dev, 15295 "failed to get tracking for %d vectors for IWARP err=%d\n", 15296 pf->num_iwarp_msix, pf->iwarp_base_vector); 15297 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 15298 } 15299 } 15300 15301 i40e_dbg_pf_init(pf); 15302 15303 /* tell the firmware that we're starting */ 15304 i40e_send_version(pf); 15305 15306 /* since everything's happy, start the service_task timer */ 15307 mod_timer(&pf->service_timer, 15308 round_jiffies(jiffies + pf->service_timer_period)); 15309 15310 /* add this PF to client device list and launch a client service task */ 15311 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15312 err = i40e_lan_add_device(pf); 15313 if (err) 15314 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 15315 err); 15316 } 15317 15318 #define PCI_SPEED_SIZE 8 15319 #define PCI_WIDTH_SIZE 8 15320 /* Devices on the IOSF bus do not have this information 15321 * and will report PCI Gen 1 x 1 by default so don't bother 15322 * checking them. 15323 */ 15324 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 15325 char speed[PCI_SPEED_SIZE] = "Unknown"; 15326 char width[PCI_WIDTH_SIZE] = "Unknown"; 15327 15328 /* Get the negotiated link width and speed from PCI config 15329 * space 15330 */ 15331 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 15332 &link_status); 15333 15334 i40e_set_pci_config_data(hw, link_status); 15335 15336 switch (hw->bus.speed) { 15337 case i40e_bus_speed_8000: 15338 strlcpy(speed, "8.0", PCI_SPEED_SIZE); break; 15339 case i40e_bus_speed_5000: 15340 strlcpy(speed, "5.0", PCI_SPEED_SIZE); break; 15341 case i40e_bus_speed_2500: 15342 strlcpy(speed, "2.5", PCI_SPEED_SIZE); break; 15343 default: 15344 break; 15345 } 15346 switch (hw->bus.width) { 15347 case i40e_bus_width_pcie_x8: 15348 strlcpy(width, "8", PCI_WIDTH_SIZE); break; 15349 case i40e_bus_width_pcie_x4: 15350 strlcpy(width, "4", PCI_WIDTH_SIZE); break; 15351 case i40e_bus_width_pcie_x2: 15352 strlcpy(width, "2", PCI_WIDTH_SIZE); break; 15353 case i40e_bus_width_pcie_x1: 15354 strlcpy(width, "1", PCI_WIDTH_SIZE); break; 15355 default: 15356 break; 15357 } 15358 15359 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 15360 speed, width); 15361 15362 if (hw->bus.width < i40e_bus_width_pcie_x8 || 15363 hw->bus.speed < i40e_bus_speed_8000) { 15364 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 15365 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 15366 } 15367 } 15368 15369 /* get the requested speeds from the fw */ 15370 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 15371 if (err) 15372 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 15373 i40e_stat_str(&pf->hw, err), 15374 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15375 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 15376 15377 /* set the FEC config due to the board capabilities */ 15378 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags); 15379 15380 /* get the supported phy types from the fw */ 15381 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 15382 if (err) 15383 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 15384 i40e_stat_str(&pf->hw, err), 15385 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15386 15387 /* make sure the MFS hasn't been set lower than the default */ 15388 #define MAX_FRAME_SIZE_DEFAULT 0x2600 15389 val = (rd32(&pf->hw, I40E_PRTGL_SAH) & 15390 I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT; 15391 if (val < MAX_FRAME_SIZE_DEFAULT) 15392 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n", 15393 i, val); 15394 15395 /* Add a filter to drop all Flow control frames from any VSI from being 15396 * transmitted. By doing so we stop a malicious VF from sending out 15397 * PAUSE or PFC frames and potentially controlling traffic for other 15398 * PF/VF VSIs. 15399 * The FW can still send Flow control frames if enabled. 15400 */ 15401 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 15402 pf->main_vsi_seid); 15403 15404 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 15405 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 15406 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 15407 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 15408 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 15409 /* print a string summarizing features */ 15410 i40e_print_features(pf); 15411 15412 return 0; 15413 15414 /* Unwind what we've done if something failed in the setup */ 15415 err_vsis: 15416 set_bit(__I40E_DOWN, pf->state); 15417 i40e_clear_interrupt_scheme(pf); 15418 kfree(pf->vsi); 15419 err_switch_setup: 15420 i40e_reset_interrupt_capability(pf); 15421 del_timer_sync(&pf->service_timer); 15422 err_mac_addr: 15423 err_configure_lan_hmc: 15424 (void)i40e_shutdown_lan_hmc(hw); 15425 err_init_lan_hmc: 15426 kfree(pf->qp_pile); 15427 err_sw_init: 15428 err_adminq_setup: 15429 err_pf_reset: 15430 iounmap(hw->hw_addr); 15431 err_ioremap: 15432 kfree(pf); 15433 err_pf_alloc: 15434 pci_disable_pcie_error_reporting(pdev); 15435 pci_release_mem_regions(pdev); 15436 err_pci_reg: 15437 err_dma: 15438 pci_disable_device(pdev); 15439 return err; 15440 } 15441 15442 /** 15443 * i40e_remove - Device removal routine 15444 * @pdev: PCI device information struct 15445 * 15446 * i40e_remove is called by the PCI subsystem to alert the driver 15447 * that is should release a PCI device. This could be caused by a 15448 * Hot-Plug event, or because the driver is going to be removed from 15449 * memory. 15450 **/ 15451 static void i40e_remove(struct pci_dev *pdev) 15452 { 15453 struct i40e_pf *pf = pci_get_drvdata(pdev); 15454 struct i40e_hw *hw = &pf->hw; 15455 i40e_status ret_code; 15456 int i; 15457 15458 i40e_dbg_pf_exit(pf); 15459 15460 i40e_ptp_stop(pf); 15461 15462 /* Disable RSS in hw */ 15463 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 15464 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 15465 15466 while (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 15467 usleep_range(1000, 2000); 15468 15469 /* no more scheduling of any task */ 15470 set_bit(__I40E_SUSPENDED, pf->state); 15471 set_bit(__I40E_DOWN, pf->state); 15472 if (pf->service_timer.function) 15473 del_timer_sync(&pf->service_timer); 15474 if (pf->service_task.func) 15475 cancel_work_sync(&pf->service_task); 15476 15477 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 15478 struct i40e_vsi *vsi = pf->vsi[0]; 15479 15480 /* We know that we have allocated only one vsi for this PF, 15481 * it was just for registering netdevice, so the interface 15482 * could be visible in the 'ifconfig' output 15483 */ 15484 unregister_netdev(vsi->netdev); 15485 free_netdev(vsi->netdev); 15486 15487 goto unmap; 15488 } 15489 15490 /* Client close must be called explicitly here because the timer 15491 * has been stopped. 15492 */ 15493 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15494 15495 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 15496 i40e_free_vfs(pf); 15497 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 15498 } 15499 15500 i40e_fdir_teardown(pf); 15501 15502 /* If there is a switch structure or any orphans, remove them. 15503 * This will leave only the PF's VSI remaining. 15504 */ 15505 for (i = 0; i < I40E_MAX_VEB; i++) { 15506 if (!pf->veb[i]) 15507 continue; 15508 15509 if (pf->veb[i]->uplink_seid == pf->mac_seid || 15510 pf->veb[i]->uplink_seid == 0) 15511 i40e_switch_branch_release(pf->veb[i]); 15512 } 15513 15514 /* Now we can shutdown the PF's VSI, just before we kill 15515 * adminq and hmc. 15516 */ 15517 if (pf->vsi[pf->lan_vsi]) 15518 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 15519 15520 i40e_cloud_filter_exit(pf); 15521 15522 /* remove attached clients */ 15523 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15524 ret_code = i40e_lan_del_device(pf); 15525 if (ret_code) 15526 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 15527 ret_code); 15528 } 15529 15530 /* shutdown and destroy the HMC */ 15531 if (hw->hmc.hmc_obj) { 15532 ret_code = i40e_shutdown_lan_hmc(hw); 15533 if (ret_code) 15534 dev_warn(&pdev->dev, 15535 "Failed to destroy the HMC resources: %d\n", 15536 ret_code); 15537 } 15538 15539 unmap: 15540 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 15541 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 15542 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 15543 free_irq(pf->pdev->irq, pf); 15544 15545 /* shutdown the adminq */ 15546 i40e_shutdown_adminq(hw); 15547 15548 /* destroy the locks only once, here */ 15549 mutex_destroy(&hw->aq.arq_mutex); 15550 mutex_destroy(&hw->aq.asq_mutex); 15551 15552 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 15553 rtnl_lock(); 15554 i40e_clear_interrupt_scheme(pf); 15555 for (i = 0; i < pf->num_alloc_vsi; i++) { 15556 if (pf->vsi[i]) { 15557 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) 15558 i40e_vsi_clear_rings(pf->vsi[i]); 15559 i40e_vsi_clear(pf->vsi[i]); 15560 pf->vsi[i] = NULL; 15561 } 15562 } 15563 rtnl_unlock(); 15564 15565 for (i = 0; i < I40E_MAX_VEB; i++) { 15566 kfree(pf->veb[i]); 15567 pf->veb[i] = NULL; 15568 } 15569 15570 kfree(pf->qp_pile); 15571 kfree(pf->vsi); 15572 15573 iounmap(hw->hw_addr); 15574 kfree(pf); 15575 pci_release_mem_regions(pdev); 15576 15577 pci_disable_pcie_error_reporting(pdev); 15578 pci_disable_device(pdev); 15579 } 15580 15581 /** 15582 * i40e_pci_error_detected - warning that something funky happened in PCI land 15583 * @pdev: PCI device information struct 15584 * @error: the type of PCI error 15585 * 15586 * Called to warn that something happened and the error handling steps 15587 * are in progress. Allows the driver to quiesce things, be ready for 15588 * remediation. 15589 **/ 15590 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 15591 pci_channel_state_t error) 15592 { 15593 struct i40e_pf *pf = pci_get_drvdata(pdev); 15594 15595 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 15596 15597 if (!pf) { 15598 dev_info(&pdev->dev, 15599 "Cannot recover - error happened during device probe\n"); 15600 return PCI_ERS_RESULT_DISCONNECT; 15601 } 15602 15603 /* shutdown all operations */ 15604 if (!test_bit(__I40E_SUSPENDED, pf->state)) 15605 i40e_prep_for_reset(pf, false); 15606 15607 /* Request a slot reset */ 15608 return PCI_ERS_RESULT_NEED_RESET; 15609 } 15610 15611 /** 15612 * i40e_pci_error_slot_reset - a PCI slot reset just happened 15613 * @pdev: PCI device information struct 15614 * 15615 * Called to find if the driver can work with the device now that 15616 * the pci slot has been reset. If a basic connection seems good 15617 * (registers are readable and have sane content) then return a 15618 * happy little PCI_ERS_RESULT_xxx. 15619 **/ 15620 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 15621 { 15622 struct i40e_pf *pf = pci_get_drvdata(pdev); 15623 pci_ers_result_t result; 15624 u32 reg; 15625 15626 dev_dbg(&pdev->dev, "%s\n", __func__); 15627 if (pci_enable_device_mem(pdev)) { 15628 dev_info(&pdev->dev, 15629 "Cannot re-enable PCI device after reset.\n"); 15630 result = PCI_ERS_RESULT_DISCONNECT; 15631 } else { 15632 pci_set_master(pdev); 15633 pci_restore_state(pdev); 15634 pci_save_state(pdev); 15635 pci_wake_from_d3(pdev, false); 15636 15637 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 15638 if (reg == 0) 15639 result = PCI_ERS_RESULT_RECOVERED; 15640 else 15641 result = PCI_ERS_RESULT_DISCONNECT; 15642 } 15643 15644 return result; 15645 } 15646 15647 /** 15648 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 15649 * @pdev: PCI device information struct 15650 */ 15651 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 15652 { 15653 struct i40e_pf *pf = pci_get_drvdata(pdev); 15654 15655 i40e_prep_for_reset(pf, false); 15656 } 15657 15658 /** 15659 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 15660 * @pdev: PCI device information struct 15661 */ 15662 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 15663 { 15664 struct i40e_pf *pf = pci_get_drvdata(pdev); 15665 15666 i40e_reset_and_rebuild(pf, false, false); 15667 } 15668 15669 /** 15670 * i40e_pci_error_resume - restart operations after PCI error recovery 15671 * @pdev: PCI device information struct 15672 * 15673 * Called to allow the driver to bring things back up after PCI error 15674 * and/or reset recovery has finished. 15675 **/ 15676 static void i40e_pci_error_resume(struct pci_dev *pdev) 15677 { 15678 struct i40e_pf *pf = pci_get_drvdata(pdev); 15679 15680 dev_dbg(&pdev->dev, "%s\n", __func__); 15681 if (test_bit(__I40E_SUSPENDED, pf->state)) 15682 return; 15683 15684 i40e_handle_reset_warning(pf, false); 15685 } 15686 15687 /** 15688 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 15689 * using the mac_address_write admin q function 15690 * @pf: pointer to i40e_pf struct 15691 **/ 15692 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 15693 { 15694 struct i40e_hw *hw = &pf->hw; 15695 i40e_status ret; 15696 u8 mac_addr[6]; 15697 u16 flags = 0; 15698 15699 /* Get current MAC address in case it's an LAA */ 15700 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 15701 ether_addr_copy(mac_addr, 15702 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 15703 } else { 15704 dev_err(&pf->pdev->dev, 15705 "Failed to retrieve MAC address; using default\n"); 15706 ether_addr_copy(mac_addr, hw->mac.addr); 15707 } 15708 15709 /* The FW expects the mac address write cmd to first be called with 15710 * one of these flags before calling it again with the multicast 15711 * enable flags. 15712 */ 15713 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 15714 15715 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 15716 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 15717 15718 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 15719 if (ret) { 15720 dev_err(&pf->pdev->dev, 15721 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 15722 return; 15723 } 15724 15725 flags = I40E_AQC_MC_MAG_EN 15726 | I40E_AQC_WOL_PRESERVE_ON_PFR 15727 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 15728 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 15729 if (ret) 15730 dev_err(&pf->pdev->dev, 15731 "Failed to enable Multicast Magic Packet wake up\n"); 15732 } 15733 15734 /** 15735 * i40e_shutdown - PCI callback for shutting down 15736 * @pdev: PCI device information struct 15737 **/ 15738 static void i40e_shutdown(struct pci_dev *pdev) 15739 { 15740 struct i40e_pf *pf = pci_get_drvdata(pdev); 15741 struct i40e_hw *hw = &pf->hw; 15742 15743 set_bit(__I40E_SUSPENDED, pf->state); 15744 set_bit(__I40E_DOWN, pf->state); 15745 15746 del_timer_sync(&pf->service_timer); 15747 cancel_work_sync(&pf->service_task); 15748 i40e_cloud_filter_exit(pf); 15749 i40e_fdir_teardown(pf); 15750 15751 /* Client close must be called explicitly here because the timer 15752 * has been stopped. 15753 */ 15754 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15755 15756 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 15757 i40e_enable_mc_magic_wake(pf); 15758 15759 i40e_prep_for_reset(pf, false); 15760 15761 wr32(hw, I40E_PFPM_APM, 15762 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 15763 wr32(hw, I40E_PFPM_WUFC, 15764 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 15765 15766 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 15767 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 15768 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 15769 free_irq(pf->pdev->irq, pf); 15770 15771 /* Since we're going to destroy queues during the 15772 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 15773 * whole section 15774 */ 15775 rtnl_lock(); 15776 i40e_clear_interrupt_scheme(pf); 15777 rtnl_unlock(); 15778 15779 if (system_state == SYSTEM_POWER_OFF) { 15780 pci_wake_from_d3(pdev, pf->wol_en); 15781 pci_set_power_state(pdev, PCI_D3hot); 15782 } 15783 } 15784 15785 /** 15786 * i40e_suspend - PM callback for moving to D3 15787 * @dev: generic device information structure 15788 **/ 15789 static int __maybe_unused i40e_suspend(struct device *dev) 15790 { 15791 struct i40e_pf *pf = dev_get_drvdata(dev); 15792 struct i40e_hw *hw = &pf->hw; 15793 15794 /* If we're already suspended, then there is nothing to do */ 15795 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 15796 return 0; 15797 15798 set_bit(__I40E_DOWN, pf->state); 15799 15800 /* Ensure service task will not be running */ 15801 del_timer_sync(&pf->service_timer); 15802 cancel_work_sync(&pf->service_task); 15803 15804 /* Client close must be called explicitly here because the timer 15805 * has been stopped. 15806 */ 15807 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15808 15809 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 15810 i40e_enable_mc_magic_wake(pf); 15811 15812 /* Since we're going to destroy queues during the 15813 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 15814 * whole section 15815 */ 15816 rtnl_lock(); 15817 15818 i40e_prep_for_reset(pf, true); 15819 15820 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 15821 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 15822 15823 /* Clear the interrupt scheme and release our IRQs so that the system 15824 * can safely hibernate even when there are a large number of CPUs. 15825 * Otherwise hibernation might fail when mapping all the vectors back 15826 * to CPU0. 15827 */ 15828 i40e_clear_interrupt_scheme(pf); 15829 15830 rtnl_unlock(); 15831 15832 return 0; 15833 } 15834 15835 /** 15836 * i40e_resume - PM callback for waking up from D3 15837 * @dev: generic device information structure 15838 **/ 15839 static int __maybe_unused i40e_resume(struct device *dev) 15840 { 15841 struct i40e_pf *pf = dev_get_drvdata(dev); 15842 int err; 15843 15844 /* If we're not suspended, then there is nothing to do */ 15845 if (!test_bit(__I40E_SUSPENDED, pf->state)) 15846 return 0; 15847 15848 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 15849 * since we're going to be restoring queues 15850 */ 15851 rtnl_lock(); 15852 15853 /* We cleared the interrupt scheme when we suspended, so we need to 15854 * restore it now to resume device functionality. 15855 */ 15856 err = i40e_restore_interrupt_scheme(pf); 15857 if (err) { 15858 dev_err(dev, "Cannot restore interrupt scheme: %d\n", 15859 err); 15860 } 15861 15862 clear_bit(__I40E_DOWN, pf->state); 15863 i40e_reset_and_rebuild(pf, false, true); 15864 15865 rtnl_unlock(); 15866 15867 /* Clear suspended state last after everything is recovered */ 15868 clear_bit(__I40E_SUSPENDED, pf->state); 15869 15870 /* Restart the service task */ 15871 mod_timer(&pf->service_timer, 15872 round_jiffies(jiffies + pf->service_timer_period)); 15873 15874 return 0; 15875 } 15876 15877 static const struct pci_error_handlers i40e_err_handler = { 15878 .error_detected = i40e_pci_error_detected, 15879 .slot_reset = i40e_pci_error_slot_reset, 15880 .reset_prepare = i40e_pci_error_reset_prepare, 15881 .reset_done = i40e_pci_error_reset_done, 15882 .resume = i40e_pci_error_resume, 15883 }; 15884 15885 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 15886 15887 static struct pci_driver i40e_driver = { 15888 .name = i40e_driver_name, 15889 .id_table = i40e_pci_tbl, 15890 .probe = i40e_probe, 15891 .remove = i40e_remove, 15892 .driver = { 15893 .pm = &i40e_pm_ops, 15894 }, 15895 .shutdown = i40e_shutdown, 15896 .err_handler = &i40e_err_handler, 15897 .sriov_configure = i40e_pci_sriov_configure, 15898 }; 15899 15900 /** 15901 * i40e_init_module - Driver registration routine 15902 * 15903 * i40e_init_module is the first routine called when the driver is 15904 * loaded. All it does is register with the PCI subsystem. 15905 **/ 15906 static int __init i40e_init_module(void) 15907 { 15908 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string); 15909 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 15910 15911 /* There is no need to throttle the number of active tasks because 15912 * each device limits its own task using a state bit for scheduling 15913 * the service task, and the device tasks do not interfere with each 15914 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 15915 * since we need to be able to guarantee forward progress even under 15916 * memory pressure. 15917 */ 15918 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 15919 if (!i40e_wq) { 15920 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 15921 return -ENOMEM; 15922 } 15923 15924 i40e_dbg_init(); 15925 return pci_register_driver(&i40e_driver); 15926 } 15927 module_init(i40e_init_module); 15928 15929 /** 15930 * i40e_exit_module - Driver exit cleanup routine 15931 * 15932 * i40e_exit_module is called just before the driver is removed 15933 * from memory. 15934 **/ 15935 static void __exit i40e_exit_module(void) 15936 { 15937 pci_unregister_driver(&i40e_driver); 15938 destroy_workqueue(i40e_wq); 15939 i40e_dbg_exit(); 15940 } 15941 module_exit(i40e_exit_module); 15942