1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/of_net.h> 6 #include <linux/pci.h> 7 #include <linux/bpf.h> 8 #include <generated/utsrelease.h> 9 10 /* Local includes */ 11 #include "i40e.h" 12 #include "i40e_diag.h" 13 #include "i40e_xsk.h" 14 #include <net/udp_tunnel.h> 15 #include <net/xdp_sock_drv.h> 16 /* All i40e tracepoints are defined by the include below, which 17 * must be included exactly once across the whole kernel with 18 * CREATE_TRACE_POINTS defined 19 */ 20 #define CREATE_TRACE_POINTS 21 #include "i40e_trace.h" 22 23 const char i40e_driver_name[] = "i40e"; 24 static const char i40e_driver_string[] = 25 "Intel(R) Ethernet Connection XL710 Network Driver"; 26 27 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation."; 28 29 /* a bit of forward declarations */ 30 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 31 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 32 static int i40e_add_vsi(struct i40e_vsi *vsi); 33 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 34 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 35 static int i40e_setup_misc_vector(struct i40e_pf *pf); 36 static void i40e_determine_queue_usage(struct i40e_pf *pf); 37 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 38 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 39 static int i40e_reset(struct i40e_pf *pf); 40 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 41 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf); 42 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf); 43 static bool i40e_check_recovery_mode(struct i40e_pf *pf); 44 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw); 45 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 46 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 47 static int i40e_get_capabilities(struct i40e_pf *pf, 48 enum i40e_admin_queue_opc list_type); 49 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf); 50 51 /* i40e_pci_tbl - PCI Device ID Table 52 * 53 * Last entry must be all 0s 54 * 55 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 56 * Class, Class Mask, private data (not used) } 57 */ 58 static const struct pci_device_id i40e_pci_tbl[] = { 59 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 60 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 61 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 62 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 63 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 64 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 83 /* required last entry */ 84 {0, } 85 }; 86 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 87 88 #define I40E_MAX_VF_COUNT 128 89 static int debug = -1; 90 module_param(debug, uint, 0); 91 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 92 93 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 94 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 95 MODULE_LICENSE("GPL v2"); 96 97 static struct workqueue_struct *i40e_wq; 98 99 /** 100 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 101 * @hw: pointer to the HW structure 102 * @mem: ptr to mem struct to fill out 103 * @size: size of memory requested 104 * @alignment: what to align the allocation to 105 **/ 106 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 107 u64 size, u32 alignment) 108 { 109 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 110 111 mem->size = ALIGN(size, alignment); 112 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa, 113 GFP_KERNEL); 114 if (!mem->va) 115 return -ENOMEM; 116 117 return 0; 118 } 119 120 /** 121 * i40e_free_dma_mem_d - OS specific memory free for shared code 122 * @hw: pointer to the HW structure 123 * @mem: ptr to mem struct to free 124 **/ 125 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 126 { 127 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 128 129 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 130 mem->va = NULL; 131 mem->pa = 0; 132 mem->size = 0; 133 134 return 0; 135 } 136 137 /** 138 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 139 * @hw: pointer to the HW structure 140 * @mem: ptr to mem struct to fill out 141 * @size: size of memory requested 142 **/ 143 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 144 u32 size) 145 { 146 mem->size = size; 147 mem->va = kzalloc(size, GFP_KERNEL); 148 149 if (!mem->va) 150 return -ENOMEM; 151 152 return 0; 153 } 154 155 /** 156 * i40e_free_virt_mem_d - OS specific memory free for shared code 157 * @hw: pointer to the HW structure 158 * @mem: ptr to mem struct to free 159 **/ 160 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 161 { 162 /* it's ok to kfree a NULL pointer */ 163 kfree(mem->va); 164 mem->va = NULL; 165 mem->size = 0; 166 167 return 0; 168 } 169 170 /** 171 * i40e_get_lump - find a lump of free generic resource 172 * @pf: board private structure 173 * @pile: the pile of resource to search 174 * @needed: the number of items needed 175 * @id: an owner id to stick on the items assigned 176 * 177 * Returns the base item index of the lump, or negative for error 178 * 179 * The search_hint trick and lack of advanced fit-finding only work 180 * because we're highly likely to have all the same size lump requests. 181 * Linear search time and any fragmentation should be minimal. 182 **/ 183 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 184 u16 needed, u16 id) 185 { 186 int ret = -ENOMEM; 187 int i, j; 188 189 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 190 dev_info(&pf->pdev->dev, 191 "param err: pile=%s needed=%d id=0x%04x\n", 192 pile ? "<valid>" : "<null>", needed, id); 193 return -EINVAL; 194 } 195 196 /* start the linear search with an imperfect hint */ 197 i = pile->search_hint; 198 while (i < pile->num_entries) { 199 /* skip already allocated entries */ 200 if (pile->list[i] & I40E_PILE_VALID_BIT) { 201 i++; 202 continue; 203 } 204 205 /* do we have enough in this lump? */ 206 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 207 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 208 break; 209 } 210 211 if (j == needed) { 212 /* there was enough, so assign it to the requestor */ 213 for (j = 0; j < needed; j++) 214 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 215 ret = i; 216 pile->search_hint = i + j; 217 break; 218 } 219 220 /* not enough, so skip over it and continue looking */ 221 i += j; 222 } 223 224 return ret; 225 } 226 227 /** 228 * i40e_put_lump - return a lump of generic resource 229 * @pile: the pile of resource to search 230 * @index: the base item index 231 * @id: the owner id of the items assigned 232 * 233 * Returns the count of items in the lump 234 **/ 235 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 236 { 237 int valid_id = (id | I40E_PILE_VALID_BIT); 238 int count = 0; 239 int i; 240 241 if (!pile || index >= pile->num_entries) 242 return -EINVAL; 243 244 for (i = index; 245 i < pile->num_entries && pile->list[i] == valid_id; 246 i++) { 247 pile->list[i] = 0; 248 count++; 249 } 250 251 if (count && index < pile->search_hint) 252 pile->search_hint = index; 253 254 return count; 255 } 256 257 /** 258 * i40e_find_vsi_from_id - searches for the vsi with the given id 259 * @pf: the pf structure to search for the vsi 260 * @id: id of the vsi it is searching for 261 **/ 262 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 263 { 264 int i; 265 266 for (i = 0; i < pf->num_alloc_vsi; i++) 267 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 268 return pf->vsi[i]; 269 270 return NULL; 271 } 272 273 /** 274 * i40e_service_event_schedule - Schedule the service task to wake up 275 * @pf: board private structure 276 * 277 * If not already scheduled, this puts the task into the work queue 278 **/ 279 void i40e_service_event_schedule(struct i40e_pf *pf) 280 { 281 if ((!test_bit(__I40E_DOWN, pf->state) && 282 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) || 283 test_bit(__I40E_RECOVERY_MODE, pf->state)) 284 queue_work(i40e_wq, &pf->service_task); 285 } 286 287 /** 288 * i40e_tx_timeout - Respond to a Tx Hang 289 * @netdev: network interface device structure 290 * @txqueue: queue number timing out 291 * 292 * If any port has noticed a Tx timeout, it is likely that the whole 293 * device is munged, not just the one netdev port, so go for the full 294 * reset. 295 **/ 296 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue) 297 { 298 struct i40e_netdev_priv *np = netdev_priv(netdev); 299 struct i40e_vsi *vsi = np->vsi; 300 struct i40e_pf *pf = vsi->back; 301 struct i40e_ring *tx_ring = NULL; 302 unsigned int i; 303 u32 head, val; 304 305 pf->tx_timeout_count++; 306 307 /* with txqueue index, find the tx_ring struct */ 308 for (i = 0; i < vsi->num_queue_pairs; i++) { 309 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 310 if (txqueue == 311 vsi->tx_rings[i]->queue_index) { 312 tx_ring = vsi->tx_rings[i]; 313 break; 314 } 315 } 316 } 317 318 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 319 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 320 else if (time_before(jiffies, 321 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 322 return; /* don't do any new action before the next timeout */ 323 324 /* don't kick off another recovery if one is already pending */ 325 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state)) 326 return; 327 328 if (tx_ring) { 329 head = i40e_get_head(tx_ring); 330 /* Read interrupt register */ 331 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 332 val = rd32(&pf->hw, 333 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 334 tx_ring->vsi->base_vector - 1)); 335 else 336 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 337 338 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 339 vsi->seid, txqueue, tx_ring->next_to_clean, 340 head, tx_ring->next_to_use, 341 readl(tx_ring->tail), val); 342 } 343 344 pf->tx_timeout_last_recovery = jiffies; 345 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n", 346 pf->tx_timeout_recovery_level, txqueue); 347 348 switch (pf->tx_timeout_recovery_level) { 349 case 1: 350 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 351 break; 352 case 2: 353 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 354 break; 355 case 3: 356 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 357 break; 358 default: 359 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 360 break; 361 } 362 363 i40e_service_event_schedule(pf); 364 pf->tx_timeout_recovery_level++; 365 } 366 367 /** 368 * i40e_get_vsi_stats_struct - Get System Network Statistics 369 * @vsi: the VSI we care about 370 * 371 * Returns the address of the device statistics structure. 372 * The statistics are actually updated from the service task. 373 **/ 374 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 375 { 376 return &vsi->net_stats; 377 } 378 379 /** 380 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 381 * @ring: Tx ring to get statistics from 382 * @stats: statistics entry to be updated 383 **/ 384 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 385 struct rtnl_link_stats64 *stats) 386 { 387 u64 bytes, packets; 388 unsigned int start; 389 390 do { 391 start = u64_stats_fetch_begin_irq(&ring->syncp); 392 packets = ring->stats.packets; 393 bytes = ring->stats.bytes; 394 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 395 396 stats->tx_packets += packets; 397 stats->tx_bytes += bytes; 398 } 399 400 /** 401 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 402 * @netdev: network interface device structure 403 * @stats: data structure to store statistics 404 * 405 * Returns the address of the device statistics structure. 406 * The statistics are actually updated from the service task. 407 **/ 408 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 409 struct rtnl_link_stats64 *stats) 410 { 411 struct i40e_netdev_priv *np = netdev_priv(netdev); 412 struct i40e_vsi *vsi = np->vsi; 413 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 414 struct i40e_ring *ring; 415 int i; 416 417 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 418 return; 419 420 if (!vsi->tx_rings) 421 return; 422 423 rcu_read_lock(); 424 for (i = 0; i < vsi->num_queue_pairs; i++) { 425 u64 bytes, packets; 426 unsigned int start; 427 428 ring = READ_ONCE(vsi->tx_rings[i]); 429 if (!ring) 430 continue; 431 i40e_get_netdev_stats_struct_tx(ring, stats); 432 433 if (i40e_enabled_xdp_vsi(vsi)) { 434 ring = READ_ONCE(vsi->xdp_rings[i]); 435 if (!ring) 436 continue; 437 i40e_get_netdev_stats_struct_tx(ring, stats); 438 } 439 440 ring = READ_ONCE(vsi->rx_rings[i]); 441 if (!ring) 442 continue; 443 do { 444 start = u64_stats_fetch_begin_irq(&ring->syncp); 445 packets = ring->stats.packets; 446 bytes = ring->stats.bytes; 447 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 448 449 stats->rx_packets += packets; 450 stats->rx_bytes += bytes; 451 452 } 453 rcu_read_unlock(); 454 455 /* following stats updated by i40e_watchdog_subtask() */ 456 stats->multicast = vsi_stats->multicast; 457 stats->tx_errors = vsi_stats->tx_errors; 458 stats->tx_dropped = vsi_stats->tx_dropped; 459 stats->rx_errors = vsi_stats->rx_errors; 460 stats->rx_dropped = vsi_stats->rx_dropped; 461 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 462 stats->rx_length_errors = vsi_stats->rx_length_errors; 463 } 464 465 /** 466 * i40e_vsi_reset_stats - Resets all stats of the given vsi 467 * @vsi: the VSI to have its stats reset 468 **/ 469 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 470 { 471 struct rtnl_link_stats64 *ns; 472 int i; 473 474 if (!vsi) 475 return; 476 477 ns = i40e_get_vsi_stats_struct(vsi); 478 memset(ns, 0, sizeof(*ns)); 479 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 480 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 481 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 482 if (vsi->rx_rings && vsi->rx_rings[0]) { 483 for (i = 0; i < vsi->num_queue_pairs; i++) { 484 memset(&vsi->rx_rings[i]->stats, 0, 485 sizeof(vsi->rx_rings[i]->stats)); 486 memset(&vsi->rx_rings[i]->rx_stats, 0, 487 sizeof(vsi->rx_rings[i]->rx_stats)); 488 memset(&vsi->tx_rings[i]->stats, 0, 489 sizeof(vsi->tx_rings[i]->stats)); 490 memset(&vsi->tx_rings[i]->tx_stats, 0, 491 sizeof(vsi->tx_rings[i]->tx_stats)); 492 } 493 } 494 vsi->stat_offsets_loaded = false; 495 } 496 497 /** 498 * i40e_pf_reset_stats - Reset all of the stats for the given PF 499 * @pf: the PF to be reset 500 **/ 501 void i40e_pf_reset_stats(struct i40e_pf *pf) 502 { 503 int i; 504 505 memset(&pf->stats, 0, sizeof(pf->stats)); 506 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 507 pf->stat_offsets_loaded = false; 508 509 for (i = 0; i < I40E_MAX_VEB; i++) { 510 if (pf->veb[i]) { 511 memset(&pf->veb[i]->stats, 0, 512 sizeof(pf->veb[i]->stats)); 513 memset(&pf->veb[i]->stats_offsets, 0, 514 sizeof(pf->veb[i]->stats_offsets)); 515 memset(&pf->veb[i]->tc_stats, 0, 516 sizeof(pf->veb[i]->tc_stats)); 517 memset(&pf->veb[i]->tc_stats_offsets, 0, 518 sizeof(pf->veb[i]->tc_stats_offsets)); 519 pf->veb[i]->stat_offsets_loaded = false; 520 } 521 } 522 pf->hw_csum_rx_error = 0; 523 } 524 525 /** 526 * i40e_stat_update48 - read and update a 48 bit stat from the chip 527 * @hw: ptr to the hardware info 528 * @hireg: the high 32 bit reg to read 529 * @loreg: the low 32 bit reg to read 530 * @offset_loaded: has the initial offset been loaded yet 531 * @offset: ptr to current offset value 532 * @stat: ptr to the stat 533 * 534 * Since the device stats are not reset at PFReset, they likely will not 535 * be zeroed when the driver starts. We'll save the first values read 536 * and use them as offsets to be subtracted from the raw values in order 537 * to report stats that count from zero. In the process, we also manage 538 * the potential roll-over. 539 **/ 540 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 541 bool offset_loaded, u64 *offset, u64 *stat) 542 { 543 u64 new_data; 544 545 if (hw->device_id == I40E_DEV_ID_QEMU) { 546 new_data = rd32(hw, loreg); 547 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 548 } else { 549 new_data = rd64(hw, loreg); 550 } 551 if (!offset_loaded) 552 *offset = new_data; 553 if (likely(new_data >= *offset)) 554 *stat = new_data - *offset; 555 else 556 *stat = (new_data + BIT_ULL(48)) - *offset; 557 *stat &= 0xFFFFFFFFFFFFULL; 558 } 559 560 /** 561 * i40e_stat_update32 - read and update a 32 bit stat from the chip 562 * @hw: ptr to the hardware info 563 * @reg: the hw reg to read 564 * @offset_loaded: has the initial offset been loaded yet 565 * @offset: ptr to current offset value 566 * @stat: ptr to the stat 567 **/ 568 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 569 bool offset_loaded, u64 *offset, u64 *stat) 570 { 571 u32 new_data; 572 573 new_data = rd32(hw, reg); 574 if (!offset_loaded) 575 *offset = new_data; 576 if (likely(new_data >= *offset)) 577 *stat = (u32)(new_data - *offset); 578 else 579 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 580 } 581 582 /** 583 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 584 * @hw: ptr to the hardware info 585 * @reg: the hw reg to read and clear 586 * @stat: ptr to the stat 587 **/ 588 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 589 { 590 u32 new_data = rd32(hw, reg); 591 592 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 593 *stat += new_data; 594 } 595 596 /** 597 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 598 * @vsi: the VSI to be updated 599 **/ 600 void i40e_update_eth_stats(struct i40e_vsi *vsi) 601 { 602 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 603 struct i40e_pf *pf = vsi->back; 604 struct i40e_hw *hw = &pf->hw; 605 struct i40e_eth_stats *oes; 606 struct i40e_eth_stats *es; /* device's eth stats */ 607 608 es = &vsi->eth_stats; 609 oes = &vsi->eth_stats_offsets; 610 611 /* Gather up the stats that the hw collects */ 612 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 613 vsi->stat_offsets_loaded, 614 &oes->tx_errors, &es->tx_errors); 615 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 616 vsi->stat_offsets_loaded, 617 &oes->rx_discards, &es->rx_discards); 618 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 619 vsi->stat_offsets_loaded, 620 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 621 622 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 623 I40E_GLV_GORCL(stat_idx), 624 vsi->stat_offsets_loaded, 625 &oes->rx_bytes, &es->rx_bytes); 626 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 627 I40E_GLV_UPRCL(stat_idx), 628 vsi->stat_offsets_loaded, 629 &oes->rx_unicast, &es->rx_unicast); 630 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 631 I40E_GLV_MPRCL(stat_idx), 632 vsi->stat_offsets_loaded, 633 &oes->rx_multicast, &es->rx_multicast); 634 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 635 I40E_GLV_BPRCL(stat_idx), 636 vsi->stat_offsets_loaded, 637 &oes->rx_broadcast, &es->rx_broadcast); 638 639 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 640 I40E_GLV_GOTCL(stat_idx), 641 vsi->stat_offsets_loaded, 642 &oes->tx_bytes, &es->tx_bytes); 643 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 644 I40E_GLV_UPTCL(stat_idx), 645 vsi->stat_offsets_loaded, 646 &oes->tx_unicast, &es->tx_unicast); 647 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 648 I40E_GLV_MPTCL(stat_idx), 649 vsi->stat_offsets_loaded, 650 &oes->tx_multicast, &es->tx_multicast); 651 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 652 I40E_GLV_BPTCL(stat_idx), 653 vsi->stat_offsets_loaded, 654 &oes->tx_broadcast, &es->tx_broadcast); 655 vsi->stat_offsets_loaded = true; 656 } 657 658 /** 659 * i40e_update_veb_stats - Update Switch component statistics 660 * @veb: the VEB being updated 661 **/ 662 void i40e_update_veb_stats(struct i40e_veb *veb) 663 { 664 struct i40e_pf *pf = veb->pf; 665 struct i40e_hw *hw = &pf->hw; 666 struct i40e_eth_stats *oes; 667 struct i40e_eth_stats *es; /* device's eth stats */ 668 struct i40e_veb_tc_stats *veb_oes; 669 struct i40e_veb_tc_stats *veb_es; 670 int i, idx = 0; 671 672 idx = veb->stats_idx; 673 es = &veb->stats; 674 oes = &veb->stats_offsets; 675 veb_es = &veb->tc_stats; 676 veb_oes = &veb->tc_stats_offsets; 677 678 /* Gather up the stats that the hw collects */ 679 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 680 veb->stat_offsets_loaded, 681 &oes->tx_discards, &es->tx_discards); 682 if (hw->revision_id > 0) 683 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 684 veb->stat_offsets_loaded, 685 &oes->rx_unknown_protocol, 686 &es->rx_unknown_protocol); 687 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 688 veb->stat_offsets_loaded, 689 &oes->rx_bytes, &es->rx_bytes); 690 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 691 veb->stat_offsets_loaded, 692 &oes->rx_unicast, &es->rx_unicast); 693 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 694 veb->stat_offsets_loaded, 695 &oes->rx_multicast, &es->rx_multicast); 696 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 697 veb->stat_offsets_loaded, 698 &oes->rx_broadcast, &es->rx_broadcast); 699 700 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 701 veb->stat_offsets_loaded, 702 &oes->tx_bytes, &es->tx_bytes); 703 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 704 veb->stat_offsets_loaded, 705 &oes->tx_unicast, &es->tx_unicast); 706 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 707 veb->stat_offsets_loaded, 708 &oes->tx_multicast, &es->tx_multicast); 709 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 710 veb->stat_offsets_loaded, 711 &oes->tx_broadcast, &es->tx_broadcast); 712 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 713 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 714 I40E_GLVEBTC_RPCL(i, idx), 715 veb->stat_offsets_loaded, 716 &veb_oes->tc_rx_packets[i], 717 &veb_es->tc_rx_packets[i]); 718 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 719 I40E_GLVEBTC_RBCL(i, idx), 720 veb->stat_offsets_loaded, 721 &veb_oes->tc_rx_bytes[i], 722 &veb_es->tc_rx_bytes[i]); 723 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 724 I40E_GLVEBTC_TPCL(i, idx), 725 veb->stat_offsets_loaded, 726 &veb_oes->tc_tx_packets[i], 727 &veb_es->tc_tx_packets[i]); 728 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 729 I40E_GLVEBTC_TBCL(i, idx), 730 veb->stat_offsets_loaded, 731 &veb_oes->tc_tx_bytes[i], 732 &veb_es->tc_tx_bytes[i]); 733 } 734 veb->stat_offsets_loaded = true; 735 } 736 737 /** 738 * i40e_update_vsi_stats - Update the vsi statistics counters. 739 * @vsi: the VSI to be updated 740 * 741 * There are a few instances where we store the same stat in a 742 * couple of different structs. This is partly because we have 743 * the netdev stats that need to be filled out, which is slightly 744 * different from the "eth_stats" defined by the chip and used in 745 * VF communications. We sort it out here. 746 **/ 747 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 748 { 749 struct i40e_pf *pf = vsi->back; 750 struct rtnl_link_stats64 *ons; 751 struct rtnl_link_stats64 *ns; /* netdev stats */ 752 struct i40e_eth_stats *oes; 753 struct i40e_eth_stats *es; /* device's eth stats */ 754 u32 tx_restart, tx_busy; 755 struct i40e_ring *p; 756 u32 rx_page, rx_buf; 757 u64 bytes, packets; 758 unsigned int start; 759 u64 tx_linearize; 760 u64 tx_force_wb; 761 u64 rx_p, rx_b; 762 u64 tx_p, tx_b; 763 u16 q; 764 765 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 766 test_bit(__I40E_CONFIG_BUSY, pf->state)) 767 return; 768 769 ns = i40e_get_vsi_stats_struct(vsi); 770 ons = &vsi->net_stats_offsets; 771 es = &vsi->eth_stats; 772 oes = &vsi->eth_stats_offsets; 773 774 /* Gather up the netdev and vsi stats that the driver collects 775 * on the fly during packet processing 776 */ 777 rx_b = rx_p = 0; 778 tx_b = tx_p = 0; 779 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 780 rx_page = 0; 781 rx_buf = 0; 782 rcu_read_lock(); 783 for (q = 0; q < vsi->num_queue_pairs; q++) { 784 /* locate Tx ring */ 785 p = READ_ONCE(vsi->tx_rings[q]); 786 if (!p) 787 continue; 788 789 do { 790 start = u64_stats_fetch_begin_irq(&p->syncp); 791 packets = p->stats.packets; 792 bytes = p->stats.bytes; 793 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 794 tx_b += bytes; 795 tx_p += packets; 796 tx_restart += p->tx_stats.restart_queue; 797 tx_busy += p->tx_stats.tx_busy; 798 tx_linearize += p->tx_stats.tx_linearize; 799 tx_force_wb += p->tx_stats.tx_force_wb; 800 801 /* locate Rx ring */ 802 p = READ_ONCE(vsi->rx_rings[q]); 803 if (!p) 804 continue; 805 806 do { 807 start = u64_stats_fetch_begin_irq(&p->syncp); 808 packets = p->stats.packets; 809 bytes = p->stats.bytes; 810 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 811 rx_b += bytes; 812 rx_p += packets; 813 rx_buf += p->rx_stats.alloc_buff_failed; 814 rx_page += p->rx_stats.alloc_page_failed; 815 816 if (i40e_enabled_xdp_vsi(vsi)) { 817 /* locate XDP ring */ 818 p = READ_ONCE(vsi->xdp_rings[q]); 819 if (!p) 820 continue; 821 822 do { 823 start = u64_stats_fetch_begin_irq(&p->syncp); 824 packets = p->stats.packets; 825 bytes = p->stats.bytes; 826 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 827 tx_b += bytes; 828 tx_p += packets; 829 tx_restart += p->tx_stats.restart_queue; 830 tx_busy += p->tx_stats.tx_busy; 831 tx_linearize += p->tx_stats.tx_linearize; 832 tx_force_wb += p->tx_stats.tx_force_wb; 833 } 834 } 835 rcu_read_unlock(); 836 vsi->tx_restart = tx_restart; 837 vsi->tx_busy = tx_busy; 838 vsi->tx_linearize = tx_linearize; 839 vsi->tx_force_wb = tx_force_wb; 840 vsi->rx_page_failed = rx_page; 841 vsi->rx_buf_failed = rx_buf; 842 843 ns->rx_packets = rx_p; 844 ns->rx_bytes = rx_b; 845 ns->tx_packets = tx_p; 846 ns->tx_bytes = tx_b; 847 848 /* update netdev stats from eth stats */ 849 i40e_update_eth_stats(vsi); 850 ons->tx_errors = oes->tx_errors; 851 ns->tx_errors = es->tx_errors; 852 ons->multicast = oes->rx_multicast; 853 ns->multicast = es->rx_multicast; 854 ons->rx_dropped = oes->rx_discards; 855 ns->rx_dropped = es->rx_discards; 856 ons->tx_dropped = oes->tx_discards; 857 ns->tx_dropped = es->tx_discards; 858 859 /* pull in a couple PF stats if this is the main vsi */ 860 if (vsi == pf->vsi[pf->lan_vsi]) { 861 ns->rx_crc_errors = pf->stats.crc_errors; 862 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 863 ns->rx_length_errors = pf->stats.rx_length_errors; 864 } 865 } 866 867 /** 868 * i40e_update_pf_stats - Update the PF statistics counters. 869 * @pf: the PF to be updated 870 **/ 871 static void i40e_update_pf_stats(struct i40e_pf *pf) 872 { 873 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 874 struct i40e_hw_port_stats *nsd = &pf->stats; 875 struct i40e_hw *hw = &pf->hw; 876 u32 val; 877 int i; 878 879 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 880 I40E_GLPRT_GORCL(hw->port), 881 pf->stat_offsets_loaded, 882 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 883 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 884 I40E_GLPRT_GOTCL(hw->port), 885 pf->stat_offsets_loaded, 886 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 887 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 888 pf->stat_offsets_loaded, 889 &osd->eth.rx_discards, 890 &nsd->eth.rx_discards); 891 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 892 I40E_GLPRT_UPRCL(hw->port), 893 pf->stat_offsets_loaded, 894 &osd->eth.rx_unicast, 895 &nsd->eth.rx_unicast); 896 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 897 I40E_GLPRT_MPRCL(hw->port), 898 pf->stat_offsets_loaded, 899 &osd->eth.rx_multicast, 900 &nsd->eth.rx_multicast); 901 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 902 I40E_GLPRT_BPRCL(hw->port), 903 pf->stat_offsets_loaded, 904 &osd->eth.rx_broadcast, 905 &nsd->eth.rx_broadcast); 906 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 907 I40E_GLPRT_UPTCL(hw->port), 908 pf->stat_offsets_loaded, 909 &osd->eth.tx_unicast, 910 &nsd->eth.tx_unicast); 911 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 912 I40E_GLPRT_MPTCL(hw->port), 913 pf->stat_offsets_loaded, 914 &osd->eth.tx_multicast, 915 &nsd->eth.tx_multicast); 916 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 917 I40E_GLPRT_BPTCL(hw->port), 918 pf->stat_offsets_loaded, 919 &osd->eth.tx_broadcast, 920 &nsd->eth.tx_broadcast); 921 922 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 923 pf->stat_offsets_loaded, 924 &osd->tx_dropped_link_down, 925 &nsd->tx_dropped_link_down); 926 927 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 928 pf->stat_offsets_loaded, 929 &osd->crc_errors, &nsd->crc_errors); 930 931 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 932 pf->stat_offsets_loaded, 933 &osd->illegal_bytes, &nsd->illegal_bytes); 934 935 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 936 pf->stat_offsets_loaded, 937 &osd->mac_local_faults, 938 &nsd->mac_local_faults); 939 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 940 pf->stat_offsets_loaded, 941 &osd->mac_remote_faults, 942 &nsd->mac_remote_faults); 943 944 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 945 pf->stat_offsets_loaded, 946 &osd->rx_length_errors, 947 &nsd->rx_length_errors); 948 949 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 950 pf->stat_offsets_loaded, 951 &osd->link_xon_rx, &nsd->link_xon_rx); 952 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 953 pf->stat_offsets_loaded, 954 &osd->link_xon_tx, &nsd->link_xon_tx); 955 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 956 pf->stat_offsets_loaded, 957 &osd->link_xoff_rx, &nsd->link_xoff_rx); 958 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 959 pf->stat_offsets_loaded, 960 &osd->link_xoff_tx, &nsd->link_xoff_tx); 961 962 for (i = 0; i < 8; i++) { 963 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 964 pf->stat_offsets_loaded, 965 &osd->priority_xoff_rx[i], 966 &nsd->priority_xoff_rx[i]); 967 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 968 pf->stat_offsets_loaded, 969 &osd->priority_xon_rx[i], 970 &nsd->priority_xon_rx[i]); 971 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 972 pf->stat_offsets_loaded, 973 &osd->priority_xon_tx[i], 974 &nsd->priority_xon_tx[i]); 975 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 976 pf->stat_offsets_loaded, 977 &osd->priority_xoff_tx[i], 978 &nsd->priority_xoff_tx[i]); 979 i40e_stat_update32(hw, 980 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 981 pf->stat_offsets_loaded, 982 &osd->priority_xon_2_xoff[i], 983 &nsd->priority_xon_2_xoff[i]); 984 } 985 986 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 987 I40E_GLPRT_PRC64L(hw->port), 988 pf->stat_offsets_loaded, 989 &osd->rx_size_64, &nsd->rx_size_64); 990 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 991 I40E_GLPRT_PRC127L(hw->port), 992 pf->stat_offsets_loaded, 993 &osd->rx_size_127, &nsd->rx_size_127); 994 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 995 I40E_GLPRT_PRC255L(hw->port), 996 pf->stat_offsets_loaded, 997 &osd->rx_size_255, &nsd->rx_size_255); 998 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 999 I40E_GLPRT_PRC511L(hw->port), 1000 pf->stat_offsets_loaded, 1001 &osd->rx_size_511, &nsd->rx_size_511); 1002 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1003 I40E_GLPRT_PRC1023L(hw->port), 1004 pf->stat_offsets_loaded, 1005 &osd->rx_size_1023, &nsd->rx_size_1023); 1006 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1007 I40E_GLPRT_PRC1522L(hw->port), 1008 pf->stat_offsets_loaded, 1009 &osd->rx_size_1522, &nsd->rx_size_1522); 1010 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1011 I40E_GLPRT_PRC9522L(hw->port), 1012 pf->stat_offsets_loaded, 1013 &osd->rx_size_big, &nsd->rx_size_big); 1014 1015 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1016 I40E_GLPRT_PTC64L(hw->port), 1017 pf->stat_offsets_loaded, 1018 &osd->tx_size_64, &nsd->tx_size_64); 1019 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1020 I40E_GLPRT_PTC127L(hw->port), 1021 pf->stat_offsets_loaded, 1022 &osd->tx_size_127, &nsd->tx_size_127); 1023 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1024 I40E_GLPRT_PTC255L(hw->port), 1025 pf->stat_offsets_loaded, 1026 &osd->tx_size_255, &nsd->tx_size_255); 1027 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1028 I40E_GLPRT_PTC511L(hw->port), 1029 pf->stat_offsets_loaded, 1030 &osd->tx_size_511, &nsd->tx_size_511); 1031 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1032 I40E_GLPRT_PTC1023L(hw->port), 1033 pf->stat_offsets_loaded, 1034 &osd->tx_size_1023, &nsd->tx_size_1023); 1035 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1036 I40E_GLPRT_PTC1522L(hw->port), 1037 pf->stat_offsets_loaded, 1038 &osd->tx_size_1522, &nsd->tx_size_1522); 1039 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1040 I40E_GLPRT_PTC9522L(hw->port), 1041 pf->stat_offsets_loaded, 1042 &osd->tx_size_big, &nsd->tx_size_big); 1043 1044 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1045 pf->stat_offsets_loaded, 1046 &osd->rx_undersize, &nsd->rx_undersize); 1047 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1048 pf->stat_offsets_loaded, 1049 &osd->rx_fragments, &nsd->rx_fragments); 1050 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1051 pf->stat_offsets_loaded, 1052 &osd->rx_oversize, &nsd->rx_oversize); 1053 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1054 pf->stat_offsets_loaded, 1055 &osd->rx_jabber, &nsd->rx_jabber); 1056 1057 /* FDIR stats */ 1058 i40e_stat_update_and_clear32(hw, 1059 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1060 &nsd->fd_atr_match); 1061 i40e_stat_update_and_clear32(hw, 1062 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1063 &nsd->fd_sb_match); 1064 i40e_stat_update_and_clear32(hw, 1065 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1066 &nsd->fd_atr_tunnel_match); 1067 1068 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1069 nsd->tx_lpi_status = 1070 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1071 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1072 nsd->rx_lpi_status = 1073 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1074 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1075 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1076 pf->stat_offsets_loaded, 1077 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1078 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1079 pf->stat_offsets_loaded, 1080 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1081 1082 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1083 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1084 nsd->fd_sb_status = true; 1085 else 1086 nsd->fd_sb_status = false; 1087 1088 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1089 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1090 nsd->fd_atr_status = true; 1091 else 1092 nsd->fd_atr_status = false; 1093 1094 pf->stat_offsets_loaded = true; 1095 } 1096 1097 /** 1098 * i40e_update_stats - Update the various statistics counters. 1099 * @vsi: the VSI to be updated 1100 * 1101 * Update the various stats for this VSI and its related entities. 1102 **/ 1103 void i40e_update_stats(struct i40e_vsi *vsi) 1104 { 1105 struct i40e_pf *pf = vsi->back; 1106 1107 if (vsi == pf->vsi[pf->lan_vsi]) 1108 i40e_update_pf_stats(pf); 1109 1110 i40e_update_vsi_stats(vsi); 1111 } 1112 1113 /** 1114 * i40e_count_filters - counts VSI mac filters 1115 * @vsi: the VSI to be searched 1116 * 1117 * Returns count of mac filters 1118 **/ 1119 int i40e_count_filters(struct i40e_vsi *vsi) 1120 { 1121 struct i40e_mac_filter *f; 1122 struct hlist_node *h; 1123 int bkt; 1124 int cnt = 0; 1125 1126 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 1127 ++cnt; 1128 1129 return cnt; 1130 } 1131 1132 /** 1133 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1134 * @vsi: the VSI to be searched 1135 * @macaddr: the MAC address 1136 * @vlan: the vlan 1137 * 1138 * Returns ptr to the filter object or NULL 1139 **/ 1140 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1141 const u8 *macaddr, s16 vlan) 1142 { 1143 struct i40e_mac_filter *f; 1144 u64 key; 1145 1146 if (!vsi || !macaddr) 1147 return NULL; 1148 1149 key = i40e_addr_to_hkey(macaddr); 1150 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1151 if ((ether_addr_equal(macaddr, f->macaddr)) && 1152 (vlan == f->vlan)) 1153 return f; 1154 } 1155 return NULL; 1156 } 1157 1158 /** 1159 * i40e_find_mac - Find a mac addr in the macvlan filters list 1160 * @vsi: the VSI to be searched 1161 * @macaddr: the MAC address we are searching for 1162 * 1163 * Returns the first filter with the provided MAC address or NULL if 1164 * MAC address was not found 1165 **/ 1166 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1167 { 1168 struct i40e_mac_filter *f; 1169 u64 key; 1170 1171 if (!vsi || !macaddr) 1172 return NULL; 1173 1174 key = i40e_addr_to_hkey(macaddr); 1175 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1176 if ((ether_addr_equal(macaddr, f->macaddr))) 1177 return f; 1178 } 1179 return NULL; 1180 } 1181 1182 /** 1183 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1184 * @vsi: the VSI to be searched 1185 * 1186 * Returns true if VSI is in vlan mode or false otherwise 1187 **/ 1188 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1189 { 1190 /* If we have a PVID, always operate in VLAN mode */ 1191 if (vsi->info.pvid) 1192 return true; 1193 1194 /* We need to operate in VLAN mode whenever we have any filters with 1195 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1196 * time, incurring search cost repeatedly. However, we can notice two 1197 * things: 1198 * 1199 * 1) the only place where we can gain a VLAN filter is in 1200 * i40e_add_filter. 1201 * 1202 * 2) the only place where filters are actually removed is in 1203 * i40e_sync_filters_subtask. 1204 * 1205 * Thus, we can simply use a boolean value, has_vlan_filters which we 1206 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1207 * we have to perform the full search after deleting filters in 1208 * i40e_sync_filters_subtask, but we already have to search 1209 * filters here and can perform the check at the same time. This 1210 * results in avoiding embedding a loop for VLAN mode inside another 1211 * loop over all the filters, and should maintain correctness as noted 1212 * above. 1213 */ 1214 return vsi->has_vlan_filter; 1215 } 1216 1217 /** 1218 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1219 * @vsi: the VSI to configure 1220 * @tmp_add_list: list of filters ready to be added 1221 * @tmp_del_list: list of filters ready to be deleted 1222 * @vlan_filters: the number of active VLAN filters 1223 * 1224 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1225 * behave as expected. If we have any active VLAN filters remaining or about 1226 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1227 * so that they only match against untagged traffic. If we no longer have any 1228 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1229 * so that they match against both tagged and untagged traffic. In this way, 1230 * we ensure that we correctly receive the desired traffic. This ensures that 1231 * when we have an active VLAN we will receive only untagged traffic and 1232 * traffic matching active VLANs. If we have no active VLANs then we will 1233 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1234 * 1235 * Finally, in a similar fashion, this function also corrects filters when 1236 * there is an active PVID assigned to this VSI. 1237 * 1238 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1239 * 1240 * This function is only expected to be called from within 1241 * i40e_sync_vsi_filters. 1242 * 1243 * NOTE: This function expects to be called while under the 1244 * mac_filter_hash_lock 1245 */ 1246 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1247 struct hlist_head *tmp_add_list, 1248 struct hlist_head *tmp_del_list, 1249 int vlan_filters) 1250 { 1251 s16 pvid = le16_to_cpu(vsi->info.pvid); 1252 struct i40e_mac_filter *f, *add_head; 1253 struct i40e_new_mac_filter *new; 1254 struct hlist_node *h; 1255 int bkt, new_vlan; 1256 1257 /* To determine if a particular filter needs to be replaced we 1258 * have the three following conditions: 1259 * 1260 * a) if we have a PVID assigned, then all filters which are 1261 * not marked as VLAN=PVID must be replaced with filters that 1262 * are. 1263 * b) otherwise, if we have any active VLANS, all filters 1264 * which are marked as VLAN=-1 must be replaced with 1265 * filters marked as VLAN=0 1266 * c) finally, if we do not have any active VLANS, all filters 1267 * which are marked as VLAN=0 must be replaced with filters 1268 * marked as VLAN=-1 1269 */ 1270 1271 /* Update the filters about to be added in place */ 1272 hlist_for_each_entry(new, tmp_add_list, hlist) { 1273 if (pvid && new->f->vlan != pvid) 1274 new->f->vlan = pvid; 1275 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1276 new->f->vlan = 0; 1277 else if (!vlan_filters && new->f->vlan == 0) 1278 new->f->vlan = I40E_VLAN_ANY; 1279 } 1280 1281 /* Update the remaining active filters */ 1282 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1283 /* Combine the checks for whether a filter needs to be changed 1284 * and then determine the new VLAN inside the if block, in 1285 * order to avoid duplicating code for adding the new filter 1286 * then deleting the old filter. 1287 */ 1288 if ((pvid && f->vlan != pvid) || 1289 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1290 (!vlan_filters && f->vlan == 0)) { 1291 /* Determine the new vlan we will be adding */ 1292 if (pvid) 1293 new_vlan = pvid; 1294 else if (vlan_filters) 1295 new_vlan = 0; 1296 else 1297 new_vlan = I40E_VLAN_ANY; 1298 1299 /* Create the new filter */ 1300 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1301 if (!add_head) 1302 return -ENOMEM; 1303 1304 /* Create a temporary i40e_new_mac_filter */ 1305 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1306 if (!new) 1307 return -ENOMEM; 1308 1309 new->f = add_head; 1310 new->state = add_head->state; 1311 1312 /* Add the new filter to the tmp list */ 1313 hlist_add_head(&new->hlist, tmp_add_list); 1314 1315 /* Put the original filter into the delete list */ 1316 f->state = I40E_FILTER_REMOVE; 1317 hash_del(&f->hlist); 1318 hlist_add_head(&f->hlist, tmp_del_list); 1319 } 1320 } 1321 1322 vsi->has_vlan_filter = !!vlan_filters; 1323 1324 return 0; 1325 } 1326 1327 /** 1328 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1329 * @vsi: the PF Main VSI - inappropriate for any other VSI 1330 * @macaddr: the MAC address 1331 * 1332 * Remove whatever filter the firmware set up so the driver can manage 1333 * its own filtering intelligently. 1334 **/ 1335 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1336 { 1337 struct i40e_aqc_remove_macvlan_element_data element; 1338 struct i40e_pf *pf = vsi->back; 1339 1340 /* Only appropriate for the PF main VSI */ 1341 if (vsi->type != I40E_VSI_MAIN) 1342 return; 1343 1344 memset(&element, 0, sizeof(element)); 1345 ether_addr_copy(element.mac_addr, macaddr); 1346 element.vlan_tag = 0; 1347 /* Ignore error returns, some firmware does it this way... */ 1348 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1349 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1350 1351 memset(&element, 0, sizeof(element)); 1352 ether_addr_copy(element.mac_addr, macaddr); 1353 element.vlan_tag = 0; 1354 /* ...and some firmware does it this way. */ 1355 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1356 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1357 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1358 } 1359 1360 /** 1361 * i40e_add_filter - Add a mac/vlan filter to the VSI 1362 * @vsi: the VSI to be searched 1363 * @macaddr: the MAC address 1364 * @vlan: the vlan 1365 * 1366 * Returns ptr to the filter object or NULL when no memory available. 1367 * 1368 * NOTE: This function is expected to be called with mac_filter_hash_lock 1369 * being held. 1370 **/ 1371 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1372 const u8 *macaddr, s16 vlan) 1373 { 1374 struct i40e_mac_filter *f; 1375 u64 key; 1376 1377 if (!vsi || !macaddr) 1378 return NULL; 1379 1380 f = i40e_find_filter(vsi, macaddr, vlan); 1381 if (!f) { 1382 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1383 if (!f) 1384 return NULL; 1385 1386 /* Update the boolean indicating if we need to function in 1387 * VLAN mode. 1388 */ 1389 if (vlan >= 0) 1390 vsi->has_vlan_filter = true; 1391 1392 ether_addr_copy(f->macaddr, macaddr); 1393 f->vlan = vlan; 1394 f->state = I40E_FILTER_NEW; 1395 INIT_HLIST_NODE(&f->hlist); 1396 1397 key = i40e_addr_to_hkey(macaddr); 1398 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1399 1400 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1401 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1402 } 1403 1404 /* If we're asked to add a filter that has been marked for removal, it 1405 * is safe to simply restore it to active state. __i40e_del_filter 1406 * will have simply deleted any filters which were previously marked 1407 * NEW or FAILED, so if it is currently marked REMOVE it must have 1408 * previously been ACTIVE. Since we haven't yet run the sync filters 1409 * task, just restore this filter to the ACTIVE state so that the 1410 * sync task leaves it in place 1411 */ 1412 if (f->state == I40E_FILTER_REMOVE) 1413 f->state = I40E_FILTER_ACTIVE; 1414 1415 return f; 1416 } 1417 1418 /** 1419 * __i40e_del_filter - Remove a specific filter from the VSI 1420 * @vsi: VSI to remove from 1421 * @f: the filter to remove from the list 1422 * 1423 * This function should be called instead of i40e_del_filter only if you know 1424 * the exact filter you will remove already, such as via i40e_find_filter or 1425 * i40e_find_mac. 1426 * 1427 * NOTE: This function is expected to be called with mac_filter_hash_lock 1428 * being held. 1429 * ANOTHER NOTE: This function MUST be called from within the context of 1430 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1431 * instead of list_for_each_entry(). 1432 **/ 1433 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1434 { 1435 if (!f) 1436 return; 1437 1438 /* If the filter was never added to firmware then we can just delete it 1439 * directly and we don't want to set the status to remove or else an 1440 * admin queue command will unnecessarily fire. 1441 */ 1442 if ((f->state == I40E_FILTER_FAILED) || 1443 (f->state == I40E_FILTER_NEW)) { 1444 hash_del(&f->hlist); 1445 kfree(f); 1446 } else { 1447 f->state = I40E_FILTER_REMOVE; 1448 } 1449 1450 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1451 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1452 } 1453 1454 /** 1455 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1456 * @vsi: the VSI to be searched 1457 * @macaddr: the MAC address 1458 * @vlan: the VLAN 1459 * 1460 * NOTE: This function is expected to be called with mac_filter_hash_lock 1461 * being held. 1462 * ANOTHER NOTE: This function MUST be called from within the context of 1463 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1464 * instead of list_for_each_entry(). 1465 **/ 1466 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1467 { 1468 struct i40e_mac_filter *f; 1469 1470 if (!vsi || !macaddr) 1471 return; 1472 1473 f = i40e_find_filter(vsi, macaddr, vlan); 1474 __i40e_del_filter(vsi, f); 1475 } 1476 1477 /** 1478 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1479 * @vsi: the VSI to be searched 1480 * @macaddr: the mac address to be filtered 1481 * 1482 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1483 * go through all the macvlan filters and add a macvlan filter for each 1484 * unique vlan that already exists. If a PVID has been assigned, instead only 1485 * add the macaddr to that VLAN. 1486 * 1487 * Returns last filter added on success, else NULL 1488 **/ 1489 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1490 const u8 *macaddr) 1491 { 1492 struct i40e_mac_filter *f, *add = NULL; 1493 struct hlist_node *h; 1494 int bkt; 1495 1496 if (vsi->info.pvid) 1497 return i40e_add_filter(vsi, macaddr, 1498 le16_to_cpu(vsi->info.pvid)); 1499 1500 if (!i40e_is_vsi_in_vlan(vsi)) 1501 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1502 1503 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1504 if (f->state == I40E_FILTER_REMOVE) 1505 continue; 1506 add = i40e_add_filter(vsi, macaddr, f->vlan); 1507 if (!add) 1508 return NULL; 1509 } 1510 1511 return add; 1512 } 1513 1514 /** 1515 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1516 * @vsi: the VSI to be searched 1517 * @macaddr: the mac address to be removed 1518 * 1519 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1520 * associated with. 1521 * 1522 * Returns 0 for success, or error 1523 **/ 1524 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1525 { 1526 struct i40e_mac_filter *f; 1527 struct hlist_node *h; 1528 bool found = false; 1529 int bkt; 1530 1531 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1532 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1533 if (ether_addr_equal(macaddr, f->macaddr)) { 1534 __i40e_del_filter(vsi, f); 1535 found = true; 1536 } 1537 } 1538 1539 if (found) 1540 return 0; 1541 else 1542 return -ENOENT; 1543 } 1544 1545 /** 1546 * i40e_set_mac - NDO callback to set mac address 1547 * @netdev: network interface device structure 1548 * @p: pointer to an address structure 1549 * 1550 * Returns 0 on success, negative on failure 1551 **/ 1552 static int i40e_set_mac(struct net_device *netdev, void *p) 1553 { 1554 struct i40e_netdev_priv *np = netdev_priv(netdev); 1555 struct i40e_vsi *vsi = np->vsi; 1556 struct i40e_pf *pf = vsi->back; 1557 struct i40e_hw *hw = &pf->hw; 1558 struct sockaddr *addr = p; 1559 1560 if (!is_valid_ether_addr(addr->sa_data)) 1561 return -EADDRNOTAVAIL; 1562 1563 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1564 netdev_info(netdev, "already using mac address %pM\n", 1565 addr->sa_data); 1566 return 0; 1567 } 1568 1569 if (test_bit(__I40E_DOWN, pf->state) || 1570 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 1571 return -EADDRNOTAVAIL; 1572 1573 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1574 netdev_info(netdev, "returning to hw mac address %pM\n", 1575 hw->mac.addr); 1576 else 1577 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1578 1579 /* Copy the address first, so that we avoid a possible race with 1580 * .set_rx_mode(). 1581 * - Remove old address from MAC filter 1582 * - Copy new address 1583 * - Add new address to MAC filter 1584 */ 1585 spin_lock_bh(&vsi->mac_filter_hash_lock); 1586 i40e_del_mac_filter(vsi, netdev->dev_addr); 1587 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1588 i40e_add_mac_filter(vsi, netdev->dev_addr); 1589 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1590 1591 if (vsi->type == I40E_VSI_MAIN) { 1592 i40e_status ret; 1593 1594 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, 1595 addr->sa_data, NULL); 1596 if (ret) 1597 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1598 i40e_stat_str(hw, ret), 1599 i40e_aq_str(hw, hw->aq.asq_last_status)); 1600 } 1601 1602 /* schedule our worker thread which will take care of 1603 * applying the new filter changes 1604 */ 1605 i40e_service_event_schedule(pf); 1606 return 0; 1607 } 1608 1609 /** 1610 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1611 * @vsi: vsi structure 1612 * @seed: RSS hash seed 1613 * @lut: pointer to lookup table of lut_size 1614 * @lut_size: size of the lookup table 1615 **/ 1616 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1617 u8 *lut, u16 lut_size) 1618 { 1619 struct i40e_pf *pf = vsi->back; 1620 struct i40e_hw *hw = &pf->hw; 1621 int ret = 0; 1622 1623 if (seed) { 1624 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1625 (struct i40e_aqc_get_set_rss_key_data *)seed; 1626 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1627 if (ret) { 1628 dev_info(&pf->pdev->dev, 1629 "Cannot set RSS key, err %s aq_err %s\n", 1630 i40e_stat_str(hw, ret), 1631 i40e_aq_str(hw, hw->aq.asq_last_status)); 1632 return ret; 1633 } 1634 } 1635 if (lut) { 1636 bool pf_lut = vsi->type == I40E_VSI_MAIN; 1637 1638 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1639 if (ret) { 1640 dev_info(&pf->pdev->dev, 1641 "Cannot set RSS lut, err %s aq_err %s\n", 1642 i40e_stat_str(hw, ret), 1643 i40e_aq_str(hw, hw->aq.asq_last_status)); 1644 return ret; 1645 } 1646 } 1647 return ret; 1648 } 1649 1650 /** 1651 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1652 * @vsi: VSI structure 1653 **/ 1654 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1655 { 1656 struct i40e_pf *pf = vsi->back; 1657 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1658 u8 *lut; 1659 int ret; 1660 1661 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1662 return 0; 1663 if (!vsi->rss_size) 1664 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1665 vsi->num_queue_pairs); 1666 if (!vsi->rss_size) 1667 return -EINVAL; 1668 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1669 if (!lut) 1670 return -ENOMEM; 1671 1672 /* Use the user configured hash keys and lookup table if there is one, 1673 * otherwise use default 1674 */ 1675 if (vsi->rss_lut_user) 1676 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1677 else 1678 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1679 if (vsi->rss_hkey_user) 1680 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1681 else 1682 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1683 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1684 kfree(lut); 1685 return ret; 1686 } 1687 1688 /** 1689 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1690 * @vsi: the VSI being configured, 1691 * @ctxt: VSI context structure 1692 * @enabled_tc: number of traffic classes to enable 1693 * 1694 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1695 **/ 1696 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1697 struct i40e_vsi_context *ctxt, 1698 u8 enabled_tc) 1699 { 1700 u16 qcount = 0, max_qcount, qmap, sections = 0; 1701 int i, override_q, pow, num_qps, ret; 1702 u8 netdev_tc = 0, offset = 0; 1703 1704 if (vsi->type != I40E_VSI_MAIN) 1705 return -EINVAL; 1706 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1707 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1708 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1709 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1710 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1711 1712 /* find the next higher power-of-2 of num queue pairs */ 1713 pow = ilog2(num_qps); 1714 if (!is_power_of_2(num_qps)) 1715 pow++; 1716 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1717 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1718 1719 /* Setup queue offset/count for all TCs for given VSI */ 1720 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1721 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1722 /* See if the given TC is enabled for the given VSI */ 1723 if (vsi->tc_config.enabled_tc & BIT(i)) { 1724 offset = vsi->mqprio_qopt.qopt.offset[i]; 1725 qcount = vsi->mqprio_qopt.qopt.count[i]; 1726 if (qcount > max_qcount) 1727 max_qcount = qcount; 1728 vsi->tc_config.tc_info[i].qoffset = offset; 1729 vsi->tc_config.tc_info[i].qcount = qcount; 1730 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1731 } else { 1732 /* TC is not enabled so set the offset to 1733 * default queue and allocate one queue 1734 * for the given TC. 1735 */ 1736 vsi->tc_config.tc_info[i].qoffset = 0; 1737 vsi->tc_config.tc_info[i].qcount = 1; 1738 vsi->tc_config.tc_info[i].netdev_tc = 0; 1739 } 1740 } 1741 1742 /* Set actual Tx/Rx queue pairs */ 1743 vsi->num_queue_pairs = offset + qcount; 1744 1745 /* Setup queue TC[0].qmap for given VSI context */ 1746 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1747 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1748 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1749 ctxt->info.valid_sections |= cpu_to_le16(sections); 1750 1751 /* Reconfigure RSS for main VSI with max queue count */ 1752 vsi->rss_size = max_qcount; 1753 ret = i40e_vsi_config_rss(vsi); 1754 if (ret) { 1755 dev_info(&vsi->back->pdev->dev, 1756 "Failed to reconfig rss for num_queues (%u)\n", 1757 max_qcount); 1758 return ret; 1759 } 1760 vsi->reconfig_rss = true; 1761 dev_dbg(&vsi->back->pdev->dev, 1762 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1763 1764 /* Find queue count available for channel VSIs and starting offset 1765 * for channel VSIs 1766 */ 1767 override_q = vsi->mqprio_qopt.qopt.count[0]; 1768 if (override_q && override_q < vsi->num_queue_pairs) { 1769 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1770 vsi->next_base_queue = override_q; 1771 } 1772 return 0; 1773 } 1774 1775 /** 1776 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1777 * @vsi: the VSI being setup 1778 * @ctxt: VSI context structure 1779 * @enabled_tc: Enabled TCs bitmap 1780 * @is_add: True if called before Add VSI 1781 * 1782 * Setup VSI queue mapping for enabled traffic classes. 1783 **/ 1784 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1785 struct i40e_vsi_context *ctxt, 1786 u8 enabled_tc, 1787 bool is_add) 1788 { 1789 struct i40e_pf *pf = vsi->back; 1790 u16 sections = 0; 1791 u8 netdev_tc = 0; 1792 u16 numtc = 1; 1793 u16 qcount; 1794 u8 offset; 1795 u16 qmap; 1796 int i; 1797 u16 num_tc_qps = 0; 1798 1799 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1800 offset = 0; 1801 1802 /* Number of queues per enabled TC */ 1803 num_tc_qps = vsi->alloc_queue_pairs; 1804 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1805 /* Find numtc from enabled TC bitmap */ 1806 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1807 if (enabled_tc & BIT(i)) /* TC is enabled */ 1808 numtc++; 1809 } 1810 if (!numtc) { 1811 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1812 numtc = 1; 1813 } 1814 num_tc_qps = num_tc_qps / numtc; 1815 num_tc_qps = min_t(int, num_tc_qps, 1816 i40e_pf_get_max_q_per_tc(pf)); 1817 } 1818 1819 vsi->tc_config.numtc = numtc; 1820 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1821 1822 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 1823 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1824 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 1825 1826 /* Setup queue offset/count for all TCs for given VSI */ 1827 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1828 /* See if the given TC is enabled for the given VSI */ 1829 if (vsi->tc_config.enabled_tc & BIT(i)) { 1830 /* TC is enabled */ 1831 int pow, num_qps; 1832 1833 switch (vsi->type) { 1834 case I40E_VSI_MAIN: 1835 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | 1836 I40E_FLAG_FD_ATR_ENABLED)) || 1837 vsi->tc_config.enabled_tc != 1) { 1838 qcount = min_t(int, pf->alloc_rss_size, 1839 num_tc_qps); 1840 break; 1841 } 1842 fallthrough; 1843 case I40E_VSI_FDIR: 1844 case I40E_VSI_SRIOV: 1845 case I40E_VSI_VMDQ2: 1846 default: 1847 qcount = num_tc_qps; 1848 WARN_ON(i != 0); 1849 break; 1850 } 1851 vsi->tc_config.tc_info[i].qoffset = offset; 1852 vsi->tc_config.tc_info[i].qcount = qcount; 1853 1854 /* find the next higher power-of-2 of num queue pairs */ 1855 num_qps = qcount; 1856 pow = 0; 1857 while (num_qps && (BIT_ULL(pow) < qcount)) { 1858 pow++; 1859 num_qps >>= 1; 1860 } 1861 1862 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1863 qmap = 1864 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1865 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1866 1867 offset += qcount; 1868 } else { 1869 /* TC is not enabled so set the offset to 1870 * default queue and allocate one queue 1871 * for the given TC. 1872 */ 1873 vsi->tc_config.tc_info[i].qoffset = 0; 1874 vsi->tc_config.tc_info[i].qcount = 1; 1875 vsi->tc_config.tc_info[i].netdev_tc = 0; 1876 1877 qmap = 0; 1878 } 1879 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1880 } 1881 1882 /* Set actual Tx/Rx queue pairs */ 1883 vsi->num_queue_pairs = offset; 1884 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1885 if (vsi->req_queue_pairs > 0) 1886 vsi->num_queue_pairs = vsi->req_queue_pairs; 1887 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1888 vsi->num_queue_pairs = pf->num_lan_msix; 1889 } 1890 1891 /* Scheduler section valid can only be set for ADD VSI */ 1892 if (is_add) { 1893 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1894 1895 ctxt->info.up_enable_bits = enabled_tc; 1896 } 1897 if (vsi->type == I40E_VSI_SRIOV) { 1898 ctxt->info.mapping_flags |= 1899 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1900 for (i = 0; i < vsi->num_queue_pairs; i++) 1901 ctxt->info.queue_mapping[i] = 1902 cpu_to_le16(vsi->base_queue + i); 1903 } else { 1904 ctxt->info.mapping_flags |= 1905 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1906 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1907 } 1908 ctxt->info.valid_sections |= cpu_to_le16(sections); 1909 } 1910 1911 /** 1912 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1913 * @netdev: the netdevice 1914 * @addr: address to add 1915 * 1916 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1917 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1918 */ 1919 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1920 { 1921 struct i40e_netdev_priv *np = netdev_priv(netdev); 1922 struct i40e_vsi *vsi = np->vsi; 1923 1924 if (i40e_add_mac_filter(vsi, addr)) 1925 return 0; 1926 else 1927 return -ENOMEM; 1928 } 1929 1930 /** 1931 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1932 * @netdev: the netdevice 1933 * @addr: address to add 1934 * 1935 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1936 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1937 */ 1938 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1939 { 1940 struct i40e_netdev_priv *np = netdev_priv(netdev); 1941 struct i40e_vsi *vsi = np->vsi; 1942 1943 /* Under some circumstances, we might receive a request to delete 1944 * our own device address from our uc list. Because we store the 1945 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1946 * such requests and not delete our device address from this list. 1947 */ 1948 if (ether_addr_equal(addr, netdev->dev_addr)) 1949 return 0; 1950 1951 i40e_del_mac_filter(vsi, addr); 1952 1953 return 0; 1954 } 1955 1956 /** 1957 * i40e_set_rx_mode - NDO callback to set the netdev filters 1958 * @netdev: network interface device structure 1959 **/ 1960 static void i40e_set_rx_mode(struct net_device *netdev) 1961 { 1962 struct i40e_netdev_priv *np = netdev_priv(netdev); 1963 struct i40e_vsi *vsi = np->vsi; 1964 1965 spin_lock_bh(&vsi->mac_filter_hash_lock); 1966 1967 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1968 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1969 1970 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1971 1972 /* check for other flag changes */ 1973 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1974 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1975 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1976 } 1977 } 1978 1979 /** 1980 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1981 * @vsi: Pointer to VSI struct 1982 * @from: Pointer to list which contains MAC filter entries - changes to 1983 * those entries needs to be undone. 1984 * 1985 * MAC filter entries from this list were slated for deletion. 1986 **/ 1987 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1988 struct hlist_head *from) 1989 { 1990 struct i40e_mac_filter *f; 1991 struct hlist_node *h; 1992 1993 hlist_for_each_entry_safe(f, h, from, hlist) { 1994 u64 key = i40e_addr_to_hkey(f->macaddr); 1995 1996 /* Move the element back into MAC filter list*/ 1997 hlist_del(&f->hlist); 1998 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1999 } 2000 } 2001 2002 /** 2003 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 2004 * @vsi: Pointer to vsi struct 2005 * @from: Pointer to list which contains MAC filter entries - changes to 2006 * those entries needs to be undone. 2007 * 2008 * MAC filter entries from this list were slated for addition. 2009 **/ 2010 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 2011 struct hlist_head *from) 2012 { 2013 struct i40e_new_mac_filter *new; 2014 struct hlist_node *h; 2015 2016 hlist_for_each_entry_safe(new, h, from, hlist) { 2017 /* We can simply free the wrapper structure */ 2018 hlist_del(&new->hlist); 2019 kfree(new); 2020 } 2021 } 2022 2023 /** 2024 * i40e_next_entry - Get the next non-broadcast filter from a list 2025 * @next: pointer to filter in list 2026 * 2027 * Returns the next non-broadcast filter in the list. Required so that we 2028 * ignore broadcast filters within the list, since these are not handled via 2029 * the normal firmware update path. 2030 */ 2031 static 2032 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2033 { 2034 hlist_for_each_entry_continue(next, hlist) { 2035 if (!is_broadcast_ether_addr(next->f->macaddr)) 2036 return next; 2037 } 2038 2039 return NULL; 2040 } 2041 2042 /** 2043 * i40e_update_filter_state - Update filter state based on return data 2044 * from firmware 2045 * @count: Number of filters added 2046 * @add_list: return data from fw 2047 * @add_head: pointer to first filter in current batch 2048 * 2049 * MAC filter entries from list were slated to be added to device. Returns 2050 * number of successful filters. Note that 0 does NOT mean success! 2051 **/ 2052 static int 2053 i40e_update_filter_state(int count, 2054 struct i40e_aqc_add_macvlan_element_data *add_list, 2055 struct i40e_new_mac_filter *add_head) 2056 { 2057 int retval = 0; 2058 int i; 2059 2060 for (i = 0; i < count; i++) { 2061 /* Always check status of each filter. We don't need to check 2062 * the firmware return status because we pre-set the filter 2063 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2064 * request to the adminq. Thus, if it no longer matches then 2065 * we know the filter is active. 2066 */ 2067 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2068 add_head->state = I40E_FILTER_FAILED; 2069 } else { 2070 add_head->state = I40E_FILTER_ACTIVE; 2071 retval++; 2072 } 2073 2074 add_head = i40e_next_filter(add_head); 2075 if (!add_head) 2076 break; 2077 } 2078 2079 return retval; 2080 } 2081 2082 /** 2083 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2084 * @vsi: ptr to the VSI 2085 * @vsi_name: name to display in messages 2086 * @list: the list of filters to send to firmware 2087 * @num_del: the number of filters to delete 2088 * @retval: Set to -EIO on failure to delete 2089 * 2090 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2091 * *retval instead of a return value so that success does not force ret_val to 2092 * be set to 0. This ensures that a sequence of calls to this function 2093 * preserve the previous value of *retval on successful delete. 2094 */ 2095 static 2096 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2097 struct i40e_aqc_remove_macvlan_element_data *list, 2098 int num_del, int *retval) 2099 { 2100 struct i40e_hw *hw = &vsi->back->hw; 2101 i40e_status aq_ret; 2102 int aq_err; 2103 2104 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 2105 aq_err = hw->aq.asq_last_status; 2106 2107 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2108 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 2109 *retval = -EIO; 2110 dev_info(&vsi->back->pdev->dev, 2111 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 2112 vsi_name, i40e_stat_str(hw, aq_ret), 2113 i40e_aq_str(hw, aq_err)); 2114 } 2115 } 2116 2117 /** 2118 * i40e_aqc_add_filters - Request firmware to add a set of filters 2119 * @vsi: ptr to the VSI 2120 * @vsi_name: name to display in messages 2121 * @list: the list of filters to send to firmware 2122 * @add_head: Position in the add hlist 2123 * @num_add: the number of filters to add 2124 * 2125 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2126 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2127 * space for more filters. 2128 */ 2129 static 2130 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2131 struct i40e_aqc_add_macvlan_element_data *list, 2132 struct i40e_new_mac_filter *add_head, 2133 int num_add) 2134 { 2135 struct i40e_hw *hw = &vsi->back->hw; 2136 int aq_err, fcnt; 2137 2138 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 2139 aq_err = hw->aq.asq_last_status; 2140 fcnt = i40e_update_filter_state(num_add, list, add_head); 2141 2142 if (fcnt != num_add) { 2143 if (vsi->type == I40E_VSI_MAIN) { 2144 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2145 dev_warn(&vsi->back->pdev->dev, 2146 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2147 i40e_aq_str(hw, aq_err), vsi_name); 2148 } else if (vsi->type == I40E_VSI_SRIOV || 2149 vsi->type == I40E_VSI_VMDQ1 || 2150 vsi->type == I40E_VSI_VMDQ2) { 2151 dev_warn(&vsi->back->pdev->dev, 2152 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n", 2153 i40e_aq_str(hw, aq_err), vsi_name, vsi_name); 2154 } else { 2155 dev_warn(&vsi->back->pdev->dev, 2156 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n", 2157 i40e_aq_str(hw, aq_err), vsi_name, vsi->type); 2158 } 2159 } 2160 } 2161 2162 /** 2163 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2164 * @vsi: pointer to the VSI 2165 * @vsi_name: the VSI name 2166 * @f: filter data 2167 * 2168 * This function sets or clears the promiscuous broadcast flags for VLAN 2169 * filters in order to properly receive broadcast frames. Assumes that only 2170 * broadcast filters are passed. 2171 * 2172 * Returns status indicating success or failure; 2173 **/ 2174 static i40e_status 2175 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2176 struct i40e_mac_filter *f) 2177 { 2178 bool enable = f->state == I40E_FILTER_NEW; 2179 struct i40e_hw *hw = &vsi->back->hw; 2180 i40e_status aq_ret; 2181 2182 if (f->vlan == I40E_VLAN_ANY) { 2183 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2184 vsi->seid, 2185 enable, 2186 NULL); 2187 } else { 2188 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2189 vsi->seid, 2190 enable, 2191 f->vlan, 2192 NULL); 2193 } 2194 2195 if (aq_ret) { 2196 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2197 dev_warn(&vsi->back->pdev->dev, 2198 "Error %s, forcing overflow promiscuous on %s\n", 2199 i40e_aq_str(hw, hw->aq.asq_last_status), 2200 vsi_name); 2201 } 2202 2203 return aq_ret; 2204 } 2205 2206 /** 2207 * i40e_set_promiscuous - set promiscuous mode 2208 * @pf: board private structure 2209 * @promisc: promisc on or off 2210 * 2211 * There are different ways of setting promiscuous mode on a PF depending on 2212 * what state/environment we're in. This identifies and sets it appropriately. 2213 * Returns 0 on success. 2214 **/ 2215 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2216 { 2217 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2218 struct i40e_hw *hw = &pf->hw; 2219 i40e_status aq_ret; 2220 2221 if (vsi->type == I40E_VSI_MAIN && 2222 pf->lan_veb != I40E_NO_VEB && 2223 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2224 /* set defport ON for Main VSI instead of true promisc 2225 * this way we will get all unicast/multicast and VLAN 2226 * promisc behavior but will not get VF or VMDq traffic 2227 * replicated on the Main VSI. 2228 */ 2229 if (promisc) 2230 aq_ret = i40e_aq_set_default_vsi(hw, 2231 vsi->seid, 2232 NULL); 2233 else 2234 aq_ret = i40e_aq_clear_default_vsi(hw, 2235 vsi->seid, 2236 NULL); 2237 if (aq_ret) { 2238 dev_info(&pf->pdev->dev, 2239 "Set default VSI failed, err %s, aq_err %s\n", 2240 i40e_stat_str(hw, aq_ret), 2241 i40e_aq_str(hw, hw->aq.asq_last_status)); 2242 } 2243 } else { 2244 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2245 hw, 2246 vsi->seid, 2247 promisc, NULL, 2248 true); 2249 if (aq_ret) { 2250 dev_info(&pf->pdev->dev, 2251 "set unicast promisc failed, err %s, aq_err %s\n", 2252 i40e_stat_str(hw, aq_ret), 2253 i40e_aq_str(hw, hw->aq.asq_last_status)); 2254 } 2255 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2256 hw, 2257 vsi->seid, 2258 promisc, NULL); 2259 if (aq_ret) { 2260 dev_info(&pf->pdev->dev, 2261 "set multicast promisc failed, err %s, aq_err %s\n", 2262 i40e_stat_str(hw, aq_ret), 2263 i40e_aq_str(hw, hw->aq.asq_last_status)); 2264 } 2265 } 2266 2267 if (!aq_ret) 2268 pf->cur_promisc = promisc; 2269 2270 return aq_ret; 2271 } 2272 2273 /** 2274 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2275 * @vsi: ptr to the VSI 2276 * 2277 * Push any outstanding VSI filter changes through the AdminQ. 2278 * 2279 * Returns 0 or error value 2280 **/ 2281 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2282 { 2283 struct hlist_head tmp_add_list, tmp_del_list; 2284 struct i40e_mac_filter *f; 2285 struct i40e_new_mac_filter *new, *add_head = NULL; 2286 struct i40e_hw *hw = &vsi->back->hw; 2287 bool old_overflow, new_overflow; 2288 unsigned int failed_filters = 0; 2289 unsigned int vlan_filters = 0; 2290 char vsi_name[16] = "PF"; 2291 int filter_list_len = 0; 2292 i40e_status aq_ret = 0; 2293 u32 changed_flags = 0; 2294 struct hlist_node *h; 2295 struct i40e_pf *pf; 2296 int num_add = 0; 2297 int num_del = 0; 2298 int retval = 0; 2299 u16 cmd_flags; 2300 int list_size; 2301 int bkt; 2302 2303 /* empty array typed pointers, kcalloc later */ 2304 struct i40e_aqc_add_macvlan_element_data *add_list; 2305 struct i40e_aqc_remove_macvlan_element_data *del_list; 2306 2307 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2308 usleep_range(1000, 2000); 2309 pf = vsi->back; 2310 2311 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2312 2313 if (vsi->netdev) { 2314 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2315 vsi->current_netdev_flags = vsi->netdev->flags; 2316 } 2317 2318 INIT_HLIST_HEAD(&tmp_add_list); 2319 INIT_HLIST_HEAD(&tmp_del_list); 2320 2321 if (vsi->type == I40E_VSI_SRIOV) 2322 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2323 else if (vsi->type != I40E_VSI_MAIN) 2324 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2325 2326 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2327 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2328 2329 spin_lock_bh(&vsi->mac_filter_hash_lock); 2330 /* Create a list of filters to delete. */ 2331 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2332 if (f->state == I40E_FILTER_REMOVE) { 2333 /* Move the element into temporary del_list */ 2334 hash_del(&f->hlist); 2335 hlist_add_head(&f->hlist, &tmp_del_list); 2336 2337 /* Avoid counting removed filters */ 2338 continue; 2339 } 2340 if (f->state == I40E_FILTER_NEW) { 2341 /* Create a temporary i40e_new_mac_filter */ 2342 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2343 if (!new) 2344 goto err_no_memory_locked; 2345 2346 /* Store pointer to the real filter */ 2347 new->f = f; 2348 new->state = f->state; 2349 2350 /* Add it to the hash list */ 2351 hlist_add_head(&new->hlist, &tmp_add_list); 2352 } 2353 2354 /* Count the number of active (current and new) VLAN 2355 * filters we have now. Does not count filters which 2356 * are marked for deletion. 2357 */ 2358 if (f->vlan > 0) 2359 vlan_filters++; 2360 } 2361 2362 retval = i40e_correct_mac_vlan_filters(vsi, 2363 &tmp_add_list, 2364 &tmp_del_list, 2365 vlan_filters); 2366 if (retval) 2367 goto err_no_memory_locked; 2368 2369 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2370 } 2371 2372 /* Now process 'del_list' outside the lock */ 2373 if (!hlist_empty(&tmp_del_list)) { 2374 filter_list_len = hw->aq.asq_buf_size / 2375 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2376 list_size = filter_list_len * 2377 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2378 del_list = kzalloc(list_size, GFP_ATOMIC); 2379 if (!del_list) 2380 goto err_no_memory; 2381 2382 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2383 cmd_flags = 0; 2384 2385 /* handle broadcast filters by updating the broadcast 2386 * promiscuous flag and release filter list. 2387 */ 2388 if (is_broadcast_ether_addr(f->macaddr)) { 2389 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2390 2391 hlist_del(&f->hlist); 2392 kfree(f); 2393 continue; 2394 } 2395 2396 /* add to delete list */ 2397 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2398 if (f->vlan == I40E_VLAN_ANY) { 2399 del_list[num_del].vlan_tag = 0; 2400 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2401 } else { 2402 del_list[num_del].vlan_tag = 2403 cpu_to_le16((u16)(f->vlan)); 2404 } 2405 2406 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2407 del_list[num_del].flags = cmd_flags; 2408 num_del++; 2409 2410 /* flush a full buffer */ 2411 if (num_del == filter_list_len) { 2412 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2413 num_del, &retval); 2414 memset(del_list, 0, list_size); 2415 num_del = 0; 2416 } 2417 /* Release memory for MAC filter entries which were 2418 * synced up with HW. 2419 */ 2420 hlist_del(&f->hlist); 2421 kfree(f); 2422 } 2423 2424 if (num_del) { 2425 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2426 num_del, &retval); 2427 } 2428 2429 kfree(del_list); 2430 del_list = NULL; 2431 } 2432 2433 if (!hlist_empty(&tmp_add_list)) { 2434 /* Do all the adds now. */ 2435 filter_list_len = hw->aq.asq_buf_size / 2436 sizeof(struct i40e_aqc_add_macvlan_element_data); 2437 list_size = filter_list_len * 2438 sizeof(struct i40e_aqc_add_macvlan_element_data); 2439 add_list = kzalloc(list_size, GFP_ATOMIC); 2440 if (!add_list) 2441 goto err_no_memory; 2442 2443 num_add = 0; 2444 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2445 /* handle broadcast filters by updating the broadcast 2446 * promiscuous flag instead of adding a MAC filter. 2447 */ 2448 if (is_broadcast_ether_addr(new->f->macaddr)) { 2449 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2450 new->f)) 2451 new->state = I40E_FILTER_FAILED; 2452 else 2453 new->state = I40E_FILTER_ACTIVE; 2454 continue; 2455 } 2456 2457 /* add to add array */ 2458 if (num_add == 0) 2459 add_head = new; 2460 cmd_flags = 0; 2461 ether_addr_copy(add_list[num_add].mac_addr, 2462 new->f->macaddr); 2463 if (new->f->vlan == I40E_VLAN_ANY) { 2464 add_list[num_add].vlan_tag = 0; 2465 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2466 } else { 2467 add_list[num_add].vlan_tag = 2468 cpu_to_le16((u16)(new->f->vlan)); 2469 } 2470 add_list[num_add].queue_number = 0; 2471 /* set invalid match method for later detection */ 2472 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2473 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2474 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2475 num_add++; 2476 2477 /* flush a full buffer */ 2478 if (num_add == filter_list_len) { 2479 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2480 add_head, num_add); 2481 memset(add_list, 0, list_size); 2482 num_add = 0; 2483 } 2484 } 2485 if (num_add) { 2486 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2487 num_add); 2488 } 2489 /* Now move all of the filters from the temp add list back to 2490 * the VSI's list. 2491 */ 2492 spin_lock_bh(&vsi->mac_filter_hash_lock); 2493 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2494 /* Only update the state if we're still NEW */ 2495 if (new->f->state == I40E_FILTER_NEW) 2496 new->f->state = new->state; 2497 hlist_del(&new->hlist); 2498 kfree(new); 2499 } 2500 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2501 kfree(add_list); 2502 add_list = NULL; 2503 } 2504 2505 /* Determine the number of active and failed filters. */ 2506 spin_lock_bh(&vsi->mac_filter_hash_lock); 2507 vsi->active_filters = 0; 2508 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2509 if (f->state == I40E_FILTER_ACTIVE) 2510 vsi->active_filters++; 2511 else if (f->state == I40E_FILTER_FAILED) 2512 failed_filters++; 2513 } 2514 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2515 2516 /* Check if we are able to exit overflow promiscuous mode. We can 2517 * safely exit if we didn't just enter, we no longer have any failed 2518 * filters, and we have reduced filters below the threshold value. 2519 */ 2520 if (old_overflow && !failed_filters && 2521 vsi->active_filters < vsi->promisc_threshold) { 2522 dev_info(&pf->pdev->dev, 2523 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2524 vsi_name); 2525 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2526 vsi->promisc_threshold = 0; 2527 } 2528 2529 /* if the VF is not trusted do not do promisc */ 2530 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2531 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2532 goto out; 2533 } 2534 2535 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2536 2537 /* If we are entering overflow promiscuous, we need to calculate a new 2538 * threshold for when we are safe to exit 2539 */ 2540 if (!old_overflow && new_overflow) 2541 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2542 2543 /* check for changes in promiscuous modes */ 2544 if (changed_flags & IFF_ALLMULTI) { 2545 bool cur_multipromisc; 2546 2547 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2548 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2549 vsi->seid, 2550 cur_multipromisc, 2551 NULL); 2552 if (aq_ret) { 2553 retval = i40e_aq_rc_to_posix(aq_ret, 2554 hw->aq.asq_last_status); 2555 dev_info(&pf->pdev->dev, 2556 "set multi promisc failed on %s, err %s aq_err %s\n", 2557 vsi_name, 2558 i40e_stat_str(hw, aq_ret), 2559 i40e_aq_str(hw, hw->aq.asq_last_status)); 2560 } else { 2561 dev_info(&pf->pdev->dev, "%s is %s allmulti mode.\n", 2562 vsi->netdev->name, 2563 cur_multipromisc ? "entering" : "leaving"); 2564 } 2565 } 2566 2567 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2568 bool cur_promisc; 2569 2570 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2571 new_overflow); 2572 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2573 if (aq_ret) { 2574 retval = i40e_aq_rc_to_posix(aq_ret, 2575 hw->aq.asq_last_status); 2576 dev_info(&pf->pdev->dev, 2577 "Setting promiscuous %s failed on %s, err %s aq_err %s\n", 2578 cur_promisc ? "on" : "off", 2579 vsi_name, 2580 i40e_stat_str(hw, aq_ret), 2581 i40e_aq_str(hw, hw->aq.asq_last_status)); 2582 } 2583 } 2584 out: 2585 /* if something went wrong then set the changed flag so we try again */ 2586 if (retval) 2587 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2588 2589 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2590 return retval; 2591 2592 err_no_memory: 2593 /* Restore elements on the temporary add and delete lists */ 2594 spin_lock_bh(&vsi->mac_filter_hash_lock); 2595 err_no_memory_locked: 2596 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2597 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2598 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2599 2600 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2601 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2602 return -ENOMEM; 2603 } 2604 2605 /** 2606 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2607 * @pf: board private structure 2608 **/ 2609 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2610 { 2611 int v; 2612 2613 if (!pf) 2614 return; 2615 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2616 return; 2617 if (test_and_set_bit(__I40E_VF_DISABLE, pf->state)) { 2618 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 2619 return; 2620 } 2621 2622 for (v = 0; v < pf->num_alloc_vsi; v++) { 2623 if (pf->vsi[v] && 2624 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2625 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2626 2627 if (ret) { 2628 /* come back and try again later */ 2629 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2630 pf->state); 2631 break; 2632 } 2633 } 2634 } 2635 clear_bit(__I40E_VF_DISABLE, pf->state); 2636 } 2637 2638 /** 2639 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2640 * @vsi: the vsi 2641 **/ 2642 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2643 { 2644 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2645 return I40E_RXBUFFER_2048; 2646 else 2647 return I40E_RXBUFFER_3072; 2648 } 2649 2650 /** 2651 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2652 * @netdev: network interface device structure 2653 * @new_mtu: new value for maximum frame size 2654 * 2655 * Returns 0 on success, negative on failure 2656 **/ 2657 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2658 { 2659 struct i40e_netdev_priv *np = netdev_priv(netdev); 2660 struct i40e_vsi *vsi = np->vsi; 2661 struct i40e_pf *pf = vsi->back; 2662 2663 if (i40e_enabled_xdp_vsi(vsi)) { 2664 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2665 2666 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2667 return -EINVAL; 2668 } 2669 2670 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2671 netdev->mtu, new_mtu); 2672 netdev->mtu = new_mtu; 2673 if (netif_running(netdev)) 2674 i40e_vsi_reinit_locked(vsi); 2675 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2676 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2677 return 0; 2678 } 2679 2680 /** 2681 * i40e_ioctl - Access the hwtstamp interface 2682 * @netdev: network interface device structure 2683 * @ifr: interface request data 2684 * @cmd: ioctl command 2685 **/ 2686 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2687 { 2688 struct i40e_netdev_priv *np = netdev_priv(netdev); 2689 struct i40e_pf *pf = np->vsi->back; 2690 2691 switch (cmd) { 2692 case SIOCGHWTSTAMP: 2693 return i40e_ptp_get_ts_config(pf, ifr); 2694 case SIOCSHWTSTAMP: 2695 return i40e_ptp_set_ts_config(pf, ifr); 2696 default: 2697 return -EOPNOTSUPP; 2698 } 2699 } 2700 2701 /** 2702 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2703 * @vsi: the vsi being adjusted 2704 **/ 2705 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2706 { 2707 struct i40e_vsi_context ctxt; 2708 i40e_status ret; 2709 2710 /* Don't modify stripping options if a port VLAN is active */ 2711 if (vsi->info.pvid) 2712 return; 2713 2714 if ((vsi->info.valid_sections & 2715 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2716 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2717 return; /* already enabled */ 2718 2719 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2720 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2721 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2722 2723 ctxt.seid = vsi->seid; 2724 ctxt.info = vsi->info; 2725 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2726 if (ret) { 2727 dev_info(&vsi->back->pdev->dev, 2728 "update vlan stripping failed, err %s aq_err %s\n", 2729 i40e_stat_str(&vsi->back->hw, ret), 2730 i40e_aq_str(&vsi->back->hw, 2731 vsi->back->hw.aq.asq_last_status)); 2732 } 2733 } 2734 2735 /** 2736 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2737 * @vsi: the vsi being adjusted 2738 **/ 2739 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2740 { 2741 struct i40e_vsi_context ctxt; 2742 i40e_status ret; 2743 2744 /* Don't modify stripping options if a port VLAN is active */ 2745 if (vsi->info.pvid) 2746 return; 2747 2748 if ((vsi->info.valid_sections & 2749 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2750 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2751 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2752 return; /* already disabled */ 2753 2754 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2755 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2756 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2757 2758 ctxt.seid = vsi->seid; 2759 ctxt.info = vsi->info; 2760 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2761 if (ret) { 2762 dev_info(&vsi->back->pdev->dev, 2763 "update vlan stripping failed, err %s aq_err %s\n", 2764 i40e_stat_str(&vsi->back->hw, ret), 2765 i40e_aq_str(&vsi->back->hw, 2766 vsi->back->hw.aq.asq_last_status)); 2767 } 2768 } 2769 2770 /** 2771 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2772 * @vsi: the vsi being configured 2773 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2774 * 2775 * This is a helper function for adding a new MAC/VLAN filter with the 2776 * specified VLAN for each existing MAC address already in the hash table. 2777 * This function does *not* perform any accounting to update filters based on 2778 * VLAN mode. 2779 * 2780 * NOTE: this function expects to be called while under the 2781 * mac_filter_hash_lock 2782 **/ 2783 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2784 { 2785 struct i40e_mac_filter *f, *add_f; 2786 struct hlist_node *h; 2787 int bkt; 2788 2789 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2790 if (f->state == I40E_FILTER_REMOVE) 2791 continue; 2792 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2793 if (!add_f) { 2794 dev_info(&vsi->back->pdev->dev, 2795 "Could not add vlan filter %d for %pM\n", 2796 vid, f->macaddr); 2797 return -ENOMEM; 2798 } 2799 } 2800 2801 return 0; 2802 } 2803 2804 /** 2805 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2806 * @vsi: the VSI being configured 2807 * @vid: VLAN id to be added 2808 **/ 2809 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2810 { 2811 int err; 2812 2813 if (vsi->info.pvid) 2814 return -EINVAL; 2815 2816 /* The network stack will attempt to add VID=0, with the intention to 2817 * receive priority tagged packets with a VLAN of 0. Our HW receives 2818 * these packets by default when configured to receive untagged 2819 * packets, so we don't need to add a filter for this case. 2820 * Additionally, HW interprets adding a VID=0 filter as meaning to 2821 * receive *only* tagged traffic and stops receiving untagged traffic. 2822 * Thus, we do not want to actually add a filter for VID=0 2823 */ 2824 if (!vid) 2825 return 0; 2826 2827 /* Locked once because all functions invoked below iterates list*/ 2828 spin_lock_bh(&vsi->mac_filter_hash_lock); 2829 err = i40e_add_vlan_all_mac(vsi, vid); 2830 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2831 if (err) 2832 return err; 2833 2834 /* schedule our worker thread which will take care of 2835 * applying the new filter changes 2836 */ 2837 i40e_service_event_schedule(vsi->back); 2838 return 0; 2839 } 2840 2841 /** 2842 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2843 * @vsi: the vsi being configured 2844 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2845 * 2846 * This function should be used to remove all VLAN filters which match the 2847 * given VID. It does not schedule the service event and does not take the 2848 * mac_filter_hash_lock so it may be combined with other operations under 2849 * a single invocation of the mac_filter_hash_lock. 2850 * 2851 * NOTE: this function expects to be called while under the 2852 * mac_filter_hash_lock 2853 */ 2854 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2855 { 2856 struct i40e_mac_filter *f; 2857 struct hlist_node *h; 2858 int bkt; 2859 2860 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2861 if (f->vlan == vid) 2862 __i40e_del_filter(vsi, f); 2863 } 2864 } 2865 2866 /** 2867 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2868 * @vsi: the VSI being configured 2869 * @vid: VLAN id to be removed 2870 **/ 2871 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2872 { 2873 if (!vid || vsi->info.pvid) 2874 return; 2875 2876 spin_lock_bh(&vsi->mac_filter_hash_lock); 2877 i40e_rm_vlan_all_mac(vsi, vid); 2878 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2879 2880 /* schedule our worker thread which will take care of 2881 * applying the new filter changes 2882 */ 2883 i40e_service_event_schedule(vsi->back); 2884 } 2885 2886 /** 2887 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2888 * @netdev: network interface to be adjusted 2889 * @proto: unused protocol value 2890 * @vid: vlan id to be added 2891 * 2892 * net_device_ops implementation for adding vlan ids 2893 **/ 2894 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2895 __always_unused __be16 proto, u16 vid) 2896 { 2897 struct i40e_netdev_priv *np = netdev_priv(netdev); 2898 struct i40e_vsi *vsi = np->vsi; 2899 int ret = 0; 2900 2901 if (vid >= VLAN_N_VID) 2902 return -EINVAL; 2903 2904 ret = i40e_vsi_add_vlan(vsi, vid); 2905 if (!ret) 2906 set_bit(vid, vsi->active_vlans); 2907 2908 return ret; 2909 } 2910 2911 /** 2912 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 2913 * @netdev: network interface to be adjusted 2914 * @proto: unused protocol value 2915 * @vid: vlan id to be added 2916 **/ 2917 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 2918 __always_unused __be16 proto, u16 vid) 2919 { 2920 struct i40e_netdev_priv *np = netdev_priv(netdev); 2921 struct i40e_vsi *vsi = np->vsi; 2922 2923 if (vid >= VLAN_N_VID) 2924 return; 2925 set_bit(vid, vsi->active_vlans); 2926 } 2927 2928 /** 2929 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2930 * @netdev: network interface to be adjusted 2931 * @proto: unused protocol value 2932 * @vid: vlan id to be removed 2933 * 2934 * net_device_ops implementation for removing vlan ids 2935 **/ 2936 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2937 __always_unused __be16 proto, u16 vid) 2938 { 2939 struct i40e_netdev_priv *np = netdev_priv(netdev); 2940 struct i40e_vsi *vsi = np->vsi; 2941 2942 /* return code is ignored as there is nothing a user 2943 * can do about failure to remove and a log message was 2944 * already printed from the other function 2945 */ 2946 i40e_vsi_kill_vlan(vsi, vid); 2947 2948 clear_bit(vid, vsi->active_vlans); 2949 2950 return 0; 2951 } 2952 2953 /** 2954 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2955 * @vsi: the vsi being brought back up 2956 **/ 2957 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2958 { 2959 u16 vid; 2960 2961 if (!vsi->netdev) 2962 return; 2963 2964 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2965 i40e_vlan_stripping_enable(vsi); 2966 else 2967 i40e_vlan_stripping_disable(vsi); 2968 2969 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2970 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 2971 vid); 2972 } 2973 2974 /** 2975 * i40e_vsi_add_pvid - Add pvid for the VSI 2976 * @vsi: the vsi being adjusted 2977 * @vid: the vlan id to set as a PVID 2978 **/ 2979 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2980 { 2981 struct i40e_vsi_context ctxt; 2982 i40e_status ret; 2983 2984 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2985 vsi->info.pvid = cpu_to_le16(vid); 2986 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2987 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2988 I40E_AQ_VSI_PVLAN_EMOD_STR; 2989 2990 ctxt.seid = vsi->seid; 2991 ctxt.info = vsi->info; 2992 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2993 if (ret) { 2994 dev_info(&vsi->back->pdev->dev, 2995 "add pvid failed, err %s aq_err %s\n", 2996 i40e_stat_str(&vsi->back->hw, ret), 2997 i40e_aq_str(&vsi->back->hw, 2998 vsi->back->hw.aq.asq_last_status)); 2999 return -ENOENT; 3000 } 3001 3002 return 0; 3003 } 3004 3005 /** 3006 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 3007 * @vsi: the vsi being adjusted 3008 * 3009 * Just use the vlan_rx_register() service to put it back to normal 3010 **/ 3011 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 3012 { 3013 vsi->info.pvid = 0; 3014 3015 i40e_vlan_stripping_disable(vsi); 3016 } 3017 3018 /** 3019 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 3020 * @vsi: ptr to the VSI 3021 * 3022 * If this function returns with an error, then it's possible one or 3023 * more of the rings is populated (while the rest are not). It is the 3024 * callers duty to clean those orphaned rings. 3025 * 3026 * Return 0 on success, negative on failure 3027 **/ 3028 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 3029 { 3030 int i, err = 0; 3031 3032 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3033 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 3034 3035 if (!i40e_enabled_xdp_vsi(vsi)) 3036 return err; 3037 3038 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3039 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3040 3041 return err; 3042 } 3043 3044 /** 3045 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3046 * @vsi: ptr to the VSI 3047 * 3048 * Free VSI's transmit software resources 3049 **/ 3050 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3051 { 3052 int i; 3053 3054 if (vsi->tx_rings) { 3055 for (i = 0; i < vsi->num_queue_pairs; i++) 3056 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3057 i40e_free_tx_resources(vsi->tx_rings[i]); 3058 } 3059 3060 if (vsi->xdp_rings) { 3061 for (i = 0; i < vsi->num_queue_pairs; i++) 3062 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3063 i40e_free_tx_resources(vsi->xdp_rings[i]); 3064 } 3065 } 3066 3067 /** 3068 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3069 * @vsi: ptr to the VSI 3070 * 3071 * If this function returns with an error, then it's possible one or 3072 * more of the rings is populated (while the rest are not). It is the 3073 * callers duty to clean those orphaned rings. 3074 * 3075 * Return 0 on success, negative on failure 3076 **/ 3077 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3078 { 3079 int i, err = 0; 3080 3081 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3082 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3083 return err; 3084 } 3085 3086 /** 3087 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3088 * @vsi: ptr to the VSI 3089 * 3090 * Free all receive software resources 3091 **/ 3092 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3093 { 3094 int i; 3095 3096 if (!vsi->rx_rings) 3097 return; 3098 3099 for (i = 0; i < vsi->num_queue_pairs; i++) 3100 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3101 i40e_free_rx_resources(vsi->rx_rings[i]); 3102 } 3103 3104 /** 3105 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3106 * @ring: The Tx ring to configure 3107 * 3108 * This enables/disables XPS for a given Tx descriptor ring 3109 * based on the TCs enabled for the VSI that ring belongs to. 3110 **/ 3111 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3112 { 3113 int cpu; 3114 3115 if (!ring->q_vector || !ring->netdev || ring->ch) 3116 return; 3117 3118 /* We only initialize XPS once, so as not to overwrite user settings */ 3119 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3120 return; 3121 3122 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3123 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3124 ring->queue_index); 3125 } 3126 3127 /** 3128 * i40e_xsk_pool - Retrieve the AF_XDP buffer pool if XDP and ZC is enabled 3129 * @ring: The Tx or Rx ring 3130 * 3131 * Returns the AF_XDP buffer pool or NULL. 3132 **/ 3133 static struct xsk_buff_pool *i40e_xsk_pool(struct i40e_ring *ring) 3134 { 3135 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi); 3136 int qid = ring->queue_index; 3137 3138 if (ring_is_xdp(ring)) 3139 qid -= ring->vsi->alloc_queue_pairs; 3140 3141 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps)) 3142 return NULL; 3143 3144 return xsk_get_pool_from_qid(ring->vsi->netdev, qid); 3145 } 3146 3147 /** 3148 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3149 * @ring: The Tx ring to configure 3150 * 3151 * Configure the Tx descriptor ring in the HMC context. 3152 **/ 3153 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3154 { 3155 struct i40e_vsi *vsi = ring->vsi; 3156 u16 pf_q = vsi->base_queue + ring->queue_index; 3157 struct i40e_hw *hw = &vsi->back->hw; 3158 struct i40e_hmc_obj_txq tx_ctx; 3159 i40e_status err = 0; 3160 u32 qtx_ctl = 0; 3161 3162 if (ring_is_xdp(ring)) 3163 ring->xsk_pool = i40e_xsk_pool(ring); 3164 3165 /* some ATR related tx ring init */ 3166 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3167 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3168 ring->atr_count = 0; 3169 } else { 3170 ring->atr_sample_rate = 0; 3171 } 3172 3173 /* configure XPS */ 3174 i40e_config_xps_tx_ring(ring); 3175 3176 /* clear the context structure first */ 3177 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3178 3179 tx_ctx.new_context = 1; 3180 tx_ctx.base = (ring->dma / 128); 3181 tx_ctx.qlen = ring->count; 3182 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3183 I40E_FLAG_FD_ATR_ENABLED)); 3184 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3185 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3186 if (vsi->type != I40E_VSI_FDIR) 3187 tx_ctx.head_wb_ena = 1; 3188 tx_ctx.head_wb_addr = ring->dma + 3189 (ring->count * sizeof(struct i40e_tx_desc)); 3190 3191 /* As part of VSI creation/update, FW allocates certain 3192 * Tx arbitration queue sets for each TC enabled for 3193 * the VSI. The FW returns the handles to these queue 3194 * sets as part of the response buffer to Add VSI, 3195 * Update VSI, etc. AQ commands. It is expected that 3196 * these queue set handles be associated with the Tx 3197 * queues by the driver as part of the TX queue context 3198 * initialization. This has to be done regardless of 3199 * DCB as by default everything is mapped to TC0. 3200 */ 3201 3202 if (ring->ch) 3203 tx_ctx.rdylist = 3204 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3205 3206 else 3207 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3208 3209 tx_ctx.rdylist_act = 0; 3210 3211 /* clear the context in the HMC */ 3212 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3213 if (err) { 3214 dev_info(&vsi->back->pdev->dev, 3215 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3216 ring->queue_index, pf_q, err); 3217 return -ENOMEM; 3218 } 3219 3220 /* set the context in the HMC */ 3221 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3222 if (err) { 3223 dev_info(&vsi->back->pdev->dev, 3224 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3225 ring->queue_index, pf_q, err); 3226 return -ENOMEM; 3227 } 3228 3229 /* Now associate this queue with this PCI function */ 3230 if (ring->ch) { 3231 if (ring->ch->type == I40E_VSI_VMDQ2) 3232 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3233 else 3234 return -EINVAL; 3235 3236 qtx_ctl |= (ring->ch->vsi_number << 3237 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3238 I40E_QTX_CTL_VFVM_INDX_MASK; 3239 } else { 3240 if (vsi->type == I40E_VSI_VMDQ2) { 3241 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3242 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3243 I40E_QTX_CTL_VFVM_INDX_MASK; 3244 } else { 3245 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3246 } 3247 } 3248 3249 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3250 I40E_QTX_CTL_PF_INDX_MASK); 3251 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3252 i40e_flush(hw); 3253 3254 /* cache tail off for easier writes later */ 3255 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3256 3257 return 0; 3258 } 3259 3260 /** 3261 * i40e_configure_rx_ring - Configure a receive ring context 3262 * @ring: The Rx ring to configure 3263 * 3264 * Configure the Rx descriptor ring in the HMC context. 3265 **/ 3266 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3267 { 3268 struct i40e_vsi *vsi = ring->vsi; 3269 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3270 u16 pf_q = vsi->base_queue + ring->queue_index; 3271 struct i40e_hw *hw = &vsi->back->hw; 3272 struct i40e_hmc_obj_rxq rx_ctx; 3273 i40e_status err = 0; 3274 bool ok; 3275 int ret; 3276 3277 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3278 3279 /* clear the context structure first */ 3280 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3281 3282 if (ring->vsi->type == I40E_VSI_MAIN) 3283 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 3284 3285 kfree(ring->rx_bi); 3286 ring->xsk_pool = i40e_xsk_pool(ring); 3287 if (ring->xsk_pool) { 3288 ret = i40e_alloc_rx_bi_zc(ring); 3289 if (ret) 3290 return ret; 3291 ring->rx_buf_len = 3292 xsk_pool_get_rx_frame_size(ring->xsk_pool); 3293 /* For AF_XDP ZC, we disallow packets to span on 3294 * multiple buffers, thus letting us skip that 3295 * handling in the fast-path. 3296 */ 3297 chain_len = 1; 3298 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3299 MEM_TYPE_XSK_BUFF_POOL, 3300 NULL); 3301 if (ret) 3302 return ret; 3303 dev_info(&vsi->back->pdev->dev, 3304 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 3305 ring->queue_index); 3306 3307 } else { 3308 ret = i40e_alloc_rx_bi(ring); 3309 if (ret) 3310 return ret; 3311 ring->rx_buf_len = vsi->rx_buf_len; 3312 if (ring->vsi->type == I40E_VSI_MAIN) { 3313 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3314 MEM_TYPE_PAGE_SHARED, 3315 NULL); 3316 if (ret) 3317 return ret; 3318 } 3319 } 3320 3321 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3322 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3323 3324 rx_ctx.base = (ring->dma / 128); 3325 rx_ctx.qlen = ring->count; 3326 3327 /* use 16 byte descriptors */ 3328 rx_ctx.dsize = 0; 3329 3330 /* descriptor type is always zero 3331 * rx_ctx.dtype = 0; 3332 */ 3333 rx_ctx.hsplit_0 = 0; 3334 3335 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3336 if (hw->revision_id == 0) 3337 rx_ctx.lrxqthresh = 0; 3338 else 3339 rx_ctx.lrxqthresh = 1; 3340 rx_ctx.crcstrip = 1; 3341 rx_ctx.l2tsel = 1; 3342 /* this controls whether VLAN is stripped from inner headers */ 3343 rx_ctx.showiv = 0; 3344 /* set the prefena field to 1 because the manual says to */ 3345 rx_ctx.prefena = 1; 3346 3347 /* clear the context in the HMC */ 3348 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3349 if (err) { 3350 dev_info(&vsi->back->pdev->dev, 3351 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3352 ring->queue_index, pf_q, err); 3353 return -ENOMEM; 3354 } 3355 3356 /* set the context in the HMC */ 3357 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3358 if (err) { 3359 dev_info(&vsi->back->pdev->dev, 3360 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3361 ring->queue_index, pf_q, err); 3362 return -ENOMEM; 3363 } 3364 3365 /* configure Rx buffer alignment */ 3366 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3367 clear_ring_build_skb_enabled(ring); 3368 else 3369 set_ring_build_skb_enabled(ring); 3370 3371 /* cache tail for quicker writes, and clear the reg before use */ 3372 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3373 writel(0, ring->tail); 3374 3375 if (ring->xsk_pool) { 3376 xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); 3377 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); 3378 } else { 3379 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3380 } 3381 if (!ok) { 3382 /* Log this in case the user has forgotten to give the kernel 3383 * any buffers, even later in the application. 3384 */ 3385 dev_info(&vsi->back->pdev->dev, 3386 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n", 3387 ring->xsk_pool ? "AF_XDP ZC enabled " : "", 3388 ring->queue_index, pf_q); 3389 } 3390 3391 return 0; 3392 } 3393 3394 /** 3395 * i40e_vsi_configure_tx - Configure the VSI for Tx 3396 * @vsi: VSI structure describing this set of rings and resources 3397 * 3398 * Configure the Tx VSI for operation. 3399 **/ 3400 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3401 { 3402 int err = 0; 3403 u16 i; 3404 3405 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3406 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3407 3408 if (err || !i40e_enabled_xdp_vsi(vsi)) 3409 return err; 3410 3411 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3412 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3413 3414 return err; 3415 } 3416 3417 /** 3418 * i40e_vsi_configure_rx - Configure the VSI for Rx 3419 * @vsi: the VSI being configured 3420 * 3421 * Configure the Rx VSI for operation. 3422 **/ 3423 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3424 { 3425 int err = 0; 3426 u16 i; 3427 3428 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3429 vsi->max_frame = I40E_MAX_RXBUFFER; 3430 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3431 #if (PAGE_SIZE < 8192) 3432 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3433 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3434 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3435 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3436 #endif 3437 } else { 3438 vsi->max_frame = I40E_MAX_RXBUFFER; 3439 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3440 I40E_RXBUFFER_2048; 3441 } 3442 3443 /* set up individual rings */ 3444 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3445 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3446 3447 return err; 3448 } 3449 3450 /** 3451 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3452 * @vsi: ptr to the VSI 3453 **/ 3454 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3455 { 3456 struct i40e_ring *tx_ring, *rx_ring; 3457 u16 qoffset, qcount; 3458 int i, n; 3459 3460 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3461 /* Reset the TC information */ 3462 for (i = 0; i < vsi->num_queue_pairs; i++) { 3463 rx_ring = vsi->rx_rings[i]; 3464 tx_ring = vsi->tx_rings[i]; 3465 rx_ring->dcb_tc = 0; 3466 tx_ring->dcb_tc = 0; 3467 } 3468 return; 3469 } 3470 3471 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3472 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3473 continue; 3474 3475 qoffset = vsi->tc_config.tc_info[n].qoffset; 3476 qcount = vsi->tc_config.tc_info[n].qcount; 3477 for (i = qoffset; i < (qoffset + qcount); i++) { 3478 rx_ring = vsi->rx_rings[i]; 3479 tx_ring = vsi->tx_rings[i]; 3480 rx_ring->dcb_tc = n; 3481 tx_ring->dcb_tc = n; 3482 } 3483 } 3484 } 3485 3486 /** 3487 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3488 * @vsi: ptr to the VSI 3489 **/ 3490 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3491 { 3492 if (vsi->netdev) 3493 i40e_set_rx_mode(vsi->netdev); 3494 } 3495 3496 /** 3497 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3498 * @vsi: Pointer to the targeted VSI 3499 * 3500 * This function replays the hlist on the hw where all the SB Flow Director 3501 * filters were saved. 3502 **/ 3503 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3504 { 3505 struct i40e_fdir_filter *filter; 3506 struct i40e_pf *pf = vsi->back; 3507 struct hlist_node *node; 3508 3509 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3510 return; 3511 3512 /* Reset FDir counters as we're replaying all existing filters */ 3513 pf->fd_tcp4_filter_cnt = 0; 3514 pf->fd_udp4_filter_cnt = 0; 3515 pf->fd_sctp4_filter_cnt = 0; 3516 pf->fd_ip4_filter_cnt = 0; 3517 3518 hlist_for_each_entry_safe(filter, node, 3519 &pf->fdir_filter_list, fdir_node) { 3520 i40e_add_del_fdir(vsi, filter, true); 3521 } 3522 } 3523 3524 /** 3525 * i40e_vsi_configure - Set up the VSI for action 3526 * @vsi: the VSI being configured 3527 **/ 3528 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3529 { 3530 int err; 3531 3532 i40e_set_vsi_rx_mode(vsi); 3533 i40e_restore_vlan(vsi); 3534 i40e_vsi_config_dcb_rings(vsi); 3535 err = i40e_vsi_configure_tx(vsi); 3536 if (!err) 3537 err = i40e_vsi_configure_rx(vsi); 3538 3539 return err; 3540 } 3541 3542 /** 3543 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3544 * @vsi: the VSI being configured 3545 **/ 3546 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3547 { 3548 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3549 struct i40e_pf *pf = vsi->back; 3550 struct i40e_hw *hw = &pf->hw; 3551 u16 vector; 3552 int i, q; 3553 u32 qp; 3554 3555 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3556 * and PFINT_LNKLSTn registers, e.g.: 3557 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3558 */ 3559 qp = vsi->base_queue; 3560 vector = vsi->base_vector; 3561 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3562 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3563 3564 q_vector->rx.next_update = jiffies + 1; 3565 q_vector->rx.target_itr = 3566 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3567 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3568 q_vector->rx.target_itr >> 1); 3569 q_vector->rx.current_itr = q_vector->rx.target_itr; 3570 3571 q_vector->tx.next_update = jiffies + 1; 3572 q_vector->tx.target_itr = 3573 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3574 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3575 q_vector->tx.target_itr >> 1); 3576 q_vector->tx.current_itr = q_vector->tx.target_itr; 3577 3578 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3579 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3580 3581 /* Linked list for the queuepairs assigned to this vector */ 3582 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3583 for (q = 0; q < q_vector->num_ringpairs; q++) { 3584 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3585 u32 val; 3586 3587 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3588 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3589 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3590 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3591 (I40E_QUEUE_TYPE_TX << 3592 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3593 3594 wr32(hw, I40E_QINT_RQCTL(qp), val); 3595 3596 if (has_xdp) { 3597 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3598 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3599 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3600 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3601 (I40E_QUEUE_TYPE_TX << 3602 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3603 3604 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3605 } 3606 3607 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3608 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3609 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3610 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3611 (I40E_QUEUE_TYPE_RX << 3612 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3613 3614 /* Terminate the linked list */ 3615 if (q == (q_vector->num_ringpairs - 1)) 3616 val |= (I40E_QUEUE_END_OF_LIST << 3617 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3618 3619 wr32(hw, I40E_QINT_TQCTL(qp), val); 3620 qp++; 3621 } 3622 } 3623 3624 i40e_flush(hw); 3625 } 3626 3627 /** 3628 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3629 * @pf: pointer to private device data structure 3630 **/ 3631 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3632 { 3633 struct i40e_hw *hw = &pf->hw; 3634 u32 val; 3635 3636 /* clear things first */ 3637 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3638 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3639 3640 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3641 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3642 I40E_PFINT_ICR0_ENA_GRST_MASK | 3643 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3644 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3645 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3646 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3647 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3648 3649 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3650 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3651 3652 if (pf->flags & I40E_FLAG_PTP) 3653 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3654 3655 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3656 3657 /* SW_ITR_IDX = 0, but don't change INTENA */ 3658 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3659 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3660 3661 /* OTHER_ITR_IDX = 0 */ 3662 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3663 } 3664 3665 /** 3666 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3667 * @vsi: the VSI being configured 3668 **/ 3669 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3670 { 3671 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3672 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3673 struct i40e_pf *pf = vsi->back; 3674 struct i40e_hw *hw = &pf->hw; 3675 u32 val; 3676 3677 /* set the ITR configuration */ 3678 q_vector->rx.next_update = jiffies + 1; 3679 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 3680 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1); 3681 q_vector->rx.current_itr = q_vector->rx.target_itr; 3682 q_vector->tx.next_update = jiffies + 1; 3683 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 3684 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1); 3685 q_vector->tx.current_itr = q_vector->tx.target_itr; 3686 3687 i40e_enable_misc_int_causes(pf); 3688 3689 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3690 wr32(hw, I40E_PFINT_LNKLST0, 0); 3691 3692 /* Associate the queue pair to the vector and enable the queue int */ 3693 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3694 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3695 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3696 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3697 3698 wr32(hw, I40E_QINT_RQCTL(0), val); 3699 3700 if (i40e_enabled_xdp_vsi(vsi)) { 3701 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3702 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3703 (I40E_QUEUE_TYPE_TX 3704 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3705 3706 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3707 } 3708 3709 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3710 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3711 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3712 3713 wr32(hw, I40E_QINT_TQCTL(0), val); 3714 i40e_flush(hw); 3715 } 3716 3717 /** 3718 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3719 * @pf: board private structure 3720 **/ 3721 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3722 { 3723 struct i40e_hw *hw = &pf->hw; 3724 3725 wr32(hw, I40E_PFINT_DYN_CTL0, 3726 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3727 i40e_flush(hw); 3728 } 3729 3730 /** 3731 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3732 * @pf: board private structure 3733 **/ 3734 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 3735 { 3736 struct i40e_hw *hw = &pf->hw; 3737 u32 val; 3738 3739 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3740 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3741 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3742 3743 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3744 i40e_flush(hw); 3745 } 3746 3747 /** 3748 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3749 * @irq: interrupt number 3750 * @data: pointer to a q_vector 3751 **/ 3752 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3753 { 3754 struct i40e_q_vector *q_vector = data; 3755 3756 if (!q_vector->tx.ring && !q_vector->rx.ring) 3757 return IRQ_HANDLED; 3758 3759 napi_schedule_irqoff(&q_vector->napi); 3760 3761 return IRQ_HANDLED; 3762 } 3763 3764 /** 3765 * i40e_irq_affinity_notify - Callback for affinity changes 3766 * @notify: context as to what irq was changed 3767 * @mask: the new affinity mask 3768 * 3769 * This is a callback function used by the irq_set_affinity_notifier function 3770 * so that we may register to receive changes to the irq affinity masks. 3771 **/ 3772 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3773 const cpumask_t *mask) 3774 { 3775 struct i40e_q_vector *q_vector = 3776 container_of(notify, struct i40e_q_vector, affinity_notify); 3777 3778 cpumask_copy(&q_vector->affinity_mask, mask); 3779 } 3780 3781 /** 3782 * i40e_irq_affinity_release - Callback for affinity notifier release 3783 * @ref: internal core kernel usage 3784 * 3785 * This is a callback function used by the irq_set_affinity_notifier function 3786 * to inform the current notification subscriber that they will no longer 3787 * receive notifications. 3788 **/ 3789 static void i40e_irq_affinity_release(struct kref *ref) {} 3790 3791 /** 3792 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3793 * @vsi: the VSI being configured 3794 * @basename: name for the vector 3795 * 3796 * Allocates MSI-X vectors and requests interrupts from the kernel. 3797 **/ 3798 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3799 { 3800 int q_vectors = vsi->num_q_vectors; 3801 struct i40e_pf *pf = vsi->back; 3802 int base = vsi->base_vector; 3803 int rx_int_idx = 0; 3804 int tx_int_idx = 0; 3805 int vector, err; 3806 int irq_num; 3807 int cpu; 3808 3809 for (vector = 0; vector < q_vectors; vector++) { 3810 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3811 3812 irq_num = pf->msix_entries[base + vector].vector; 3813 3814 if (q_vector->tx.ring && q_vector->rx.ring) { 3815 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3816 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3817 tx_int_idx++; 3818 } else if (q_vector->rx.ring) { 3819 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3820 "%s-%s-%d", basename, "rx", rx_int_idx++); 3821 } else if (q_vector->tx.ring) { 3822 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3823 "%s-%s-%d", basename, "tx", tx_int_idx++); 3824 } else { 3825 /* skip this unused q_vector */ 3826 continue; 3827 } 3828 err = request_irq(irq_num, 3829 vsi->irq_handler, 3830 0, 3831 q_vector->name, 3832 q_vector); 3833 if (err) { 3834 dev_info(&pf->pdev->dev, 3835 "MSIX request_irq failed, error: %d\n", err); 3836 goto free_queue_irqs; 3837 } 3838 3839 /* register for affinity change notifications */ 3840 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3841 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3842 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3843 /* Spread affinity hints out across online CPUs. 3844 * 3845 * get_cpu_mask returns a static constant mask with 3846 * a permanent lifetime so it's ok to pass to 3847 * irq_set_affinity_hint without making a copy. 3848 */ 3849 cpu = cpumask_local_spread(q_vector->v_idx, -1); 3850 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 3851 } 3852 3853 vsi->irqs_ready = true; 3854 return 0; 3855 3856 free_queue_irqs: 3857 while (vector) { 3858 vector--; 3859 irq_num = pf->msix_entries[base + vector].vector; 3860 irq_set_affinity_notifier(irq_num, NULL); 3861 irq_set_affinity_hint(irq_num, NULL); 3862 free_irq(irq_num, &vsi->q_vectors[vector]); 3863 } 3864 return err; 3865 } 3866 3867 /** 3868 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3869 * @vsi: the VSI being un-configured 3870 **/ 3871 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3872 { 3873 struct i40e_pf *pf = vsi->back; 3874 struct i40e_hw *hw = &pf->hw; 3875 int base = vsi->base_vector; 3876 int i; 3877 3878 /* disable interrupt causation from each queue */ 3879 for (i = 0; i < vsi->num_queue_pairs; i++) { 3880 u32 val; 3881 3882 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3883 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3884 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3885 3886 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3887 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3888 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3889 3890 if (!i40e_enabled_xdp_vsi(vsi)) 3891 continue; 3892 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3893 } 3894 3895 /* disable each interrupt */ 3896 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3897 for (i = vsi->base_vector; 3898 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3899 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3900 3901 i40e_flush(hw); 3902 for (i = 0; i < vsi->num_q_vectors; i++) 3903 synchronize_irq(pf->msix_entries[i + base].vector); 3904 } else { 3905 /* Legacy and MSI mode - this stops all interrupt handling */ 3906 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3907 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3908 i40e_flush(hw); 3909 synchronize_irq(pf->pdev->irq); 3910 } 3911 } 3912 3913 /** 3914 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3915 * @vsi: the VSI being configured 3916 **/ 3917 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3918 { 3919 struct i40e_pf *pf = vsi->back; 3920 int i; 3921 3922 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3923 for (i = 0; i < vsi->num_q_vectors; i++) 3924 i40e_irq_dynamic_enable(vsi, i); 3925 } else { 3926 i40e_irq_dynamic_enable_icr0(pf); 3927 } 3928 3929 i40e_flush(&pf->hw); 3930 return 0; 3931 } 3932 3933 /** 3934 * i40e_free_misc_vector - Free the vector that handles non-queue events 3935 * @pf: board private structure 3936 **/ 3937 static void i40e_free_misc_vector(struct i40e_pf *pf) 3938 { 3939 /* Disable ICR 0 */ 3940 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3941 i40e_flush(&pf->hw); 3942 3943 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3944 synchronize_irq(pf->msix_entries[0].vector); 3945 free_irq(pf->msix_entries[0].vector, pf); 3946 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3947 } 3948 } 3949 3950 /** 3951 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3952 * @irq: interrupt number 3953 * @data: pointer to a q_vector 3954 * 3955 * This is the handler used for all MSI/Legacy interrupts, and deals 3956 * with both queue and non-queue interrupts. This is also used in 3957 * MSIX mode to handle the non-queue interrupts. 3958 **/ 3959 static irqreturn_t i40e_intr(int irq, void *data) 3960 { 3961 struct i40e_pf *pf = (struct i40e_pf *)data; 3962 struct i40e_hw *hw = &pf->hw; 3963 irqreturn_t ret = IRQ_NONE; 3964 u32 icr0, icr0_remaining; 3965 u32 val, ena_mask; 3966 3967 icr0 = rd32(hw, I40E_PFINT_ICR0); 3968 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3969 3970 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3971 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3972 goto enable_intr; 3973 3974 /* if interrupt but no bits showing, must be SWINT */ 3975 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3976 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3977 pf->sw_int_count++; 3978 3979 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3980 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3981 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3982 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3983 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3984 } 3985 3986 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3987 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3988 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3989 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3990 3991 /* We do not have a way to disarm Queue causes while leaving 3992 * interrupt enabled for all other causes, ideally 3993 * interrupt should be disabled while we are in NAPI but 3994 * this is not a performance path and napi_schedule() 3995 * can deal with rescheduling. 3996 */ 3997 if (!test_bit(__I40E_DOWN, pf->state)) 3998 napi_schedule_irqoff(&q_vector->napi); 3999 } 4000 4001 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 4002 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 4003 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 4004 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 4005 } 4006 4007 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 4008 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 4009 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 4010 } 4011 4012 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 4013 /* disable any further VFLR event notifications */ 4014 if (test_bit(__I40E_VF_RESETS_DISABLED, pf->state)) { 4015 u32 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 4016 4017 reg &= ~I40E_PFINT_ICR0_VFLR_MASK; 4018 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 4019 } else { 4020 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 4021 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 4022 } 4023 } 4024 4025 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 4026 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4027 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 4028 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 4029 val = rd32(hw, I40E_GLGEN_RSTAT); 4030 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 4031 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 4032 if (val == I40E_RESET_CORER) { 4033 pf->corer_count++; 4034 } else if (val == I40E_RESET_GLOBR) { 4035 pf->globr_count++; 4036 } else if (val == I40E_RESET_EMPR) { 4037 pf->empr_count++; 4038 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 4039 } 4040 } 4041 4042 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 4043 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 4044 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 4045 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 4046 rd32(hw, I40E_PFHMC_ERRORINFO), 4047 rd32(hw, I40E_PFHMC_ERRORDATA)); 4048 } 4049 4050 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 4051 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 4052 4053 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 4054 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4055 i40e_ptp_tx_hwtstamp(pf); 4056 } 4057 } 4058 4059 /* If a critical error is pending we have no choice but to reset the 4060 * device. 4061 * Report and mask out any remaining unexpected interrupts. 4062 */ 4063 icr0_remaining = icr0 & ena_mask; 4064 if (icr0_remaining) { 4065 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 4066 icr0_remaining); 4067 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 4068 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 4069 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 4070 dev_info(&pf->pdev->dev, "device will be reset\n"); 4071 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 4072 i40e_service_event_schedule(pf); 4073 } 4074 ena_mask &= ~icr0_remaining; 4075 } 4076 ret = IRQ_HANDLED; 4077 4078 enable_intr: 4079 /* re-enable interrupt causes */ 4080 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 4081 if (!test_bit(__I40E_DOWN, pf->state) || 4082 test_bit(__I40E_RECOVERY_MODE, pf->state)) { 4083 i40e_service_event_schedule(pf); 4084 i40e_irq_dynamic_enable_icr0(pf); 4085 } 4086 4087 return ret; 4088 } 4089 4090 /** 4091 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 4092 * @tx_ring: tx ring to clean 4093 * @budget: how many cleans we're allowed 4094 * 4095 * Returns true if there's any budget left (e.g. the clean is finished) 4096 **/ 4097 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 4098 { 4099 struct i40e_vsi *vsi = tx_ring->vsi; 4100 u16 i = tx_ring->next_to_clean; 4101 struct i40e_tx_buffer *tx_buf; 4102 struct i40e_tx_desc *tx_desc; 4103 4104 tx_buf = &tx_ring->tx_bi[i]; 4105 tx_desc = I40E_TX_DESC(tx_ring, i); 4106 i -= tx_ring->count; 4107 4108 do { 4109 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4110 4111 /* if next_to_watch is not set then there is no work pending */ 4112 if (!eop_desc) 4113 break; 4114 4115 /* prevent any other reads prior to eop_desc */ 4116 smp_rmb(); 4117 4118 /* if the descriptor isn't done, no work yet to do */ 4119 if (!(eop_desc->cmd_type_offset_bsz & 4120 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4121 break; 4122 4123 /* clear next_to_watch to prevent false hangs */ 4124 tx_buf->next_to_watch = NULL; 4125 4126 tx_desc->buffer_addr = 0; 4127 tx_desc->cmd_type_offset_bsz = 0; 4128 /* move past filter desc */ 4129 tx_buf++; 4130 tx_desc++; 4131 i++; 4132 if (unlikely(!i)) { 4133 i -= tx_ring->count; 4134 tx_buf = tx_ring->tx_bi; 4135 tx_desc = I40E_TX_DESC(tx_ring, 0); 4136 } 4137 /* unmap skb header data */ 4138 dma_unmap_single(tx_ring->dev, 4139 dma_unmap_addr(tx_buf, dma), 4140 dma_unmap_len(tx_buf, len), 4141 DMA_TO_DEVICE); 4142 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4143 kfree(tx_buf->raw_buf); 4144 4145 tx_buf->raw_buf = NULL; 4146 tx_buf->tx_flags = 0; 4147 tx_buf->next_to_watch = NULL; 4148 dma_unmap_len_set(tx_buf, len, 0); 4149 tx_desc->buffer_addr = 0; 4150 tx_desc->cmd_type_offset_bsz = 0; 4151 4152 /* move us past the eop_desc for start of next FD desc */ 4153 tx_buf++; 4154 tx_desc++; 4155 i++; 4156 if (unlikely(!i)) { 4157 i -= tx_ring->count; 4158 tx_buf = tx_ring->tx_bi; 4159 tx_desc = I40E_TX_DESC(tx_ring, 0); 4160 } 4161 4162 /* update budget accounting */ 4163 budget--; 4164 } while (likely(budget)); 4165 4166 i += tx_ring->count; 4167 tx_ring->next_to_clean = i; 4168 4169 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4170 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4171 4172 return budget > 0; 4173 } 4174 4175 /** 4176 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4177 * @irq: interrupt number 4178 * @data: pointer to a q_vector 4179 **/ 4180 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4181 { 4182 struct i40e_q_vector *q_vector = data; 4183 struct i40e_vsi *vsi; 4184 4185 if (!q_vector->tx.ring) 4186 return IRQ_HANDLED; 4187 4188 vsi = q_vector->tx.ring->vsi; 4189 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4190 4191 return IRQ_HANDLED; 4192 } 4193 4194 /** 4195 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4196 * @vsi: the VSI being configured 4197 * @v_idx: vector index 4198 * @qp_idx: queue pair index 4199 **/ 4200 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4201 { 4202 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4203 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4204 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4205 4206 tx_ring->q_vector = q_vector; 4207 tx_ring->next = q_vector->tx.ring; 4208 q_vector->tx.ring = tx_ring; 4209 q_vector->tx.count++; 4210 4211 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4212 if (i40e_enabled_xdp_vsi(vsi)) { 4213 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4214 4215 xdp_ring->q_vector = q_vector; 4216 xdp_ring->next = q_vector->tx.ring; 4217 q_vector->tx.ring = xdp_ring; 4218 q_vector->tx.count++; 4219 } 4220 4221 rx_ring->q_vector = q_vector; 4222 rx_ring->next = q_vector->rx.ring; 4223 q_vector->rx.ring = rx_ring; 4224 q_vector->rx.count++; 4225 } 4226 4227 /** 4228 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4229 * @vsi: the VSI being configured 4230 * 4231 * This function maps descriptor rings to the queue-specific vectors 4232 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4233 * one vector per queue pair, but on a constrained vector budget, we 4234 * group the queue pairs as "efficiently" as possible. 4235 **/ 4236 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4237 { 4238 int qp_remaining = vsi->num_queue_pairs; 4239 int q_vectors = vsi->num_q_vectors; 4240 int num_ringpairs; 4241 int v_start = 0; 4242 int qp_idx = 0; 4243 4244 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4245 * group them so there are multiple queues per vector. 4246 * It is also important to go through all the vectors available to be 4247 * sure that if we don't use all the vectors, that the remaining vectors 4248 * are cleared. This is especially important when decreasing the 4249 * number of queues in use. 4250 */ 4251 for (; v_start < q_vectors; v_start++) { 4252 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4253 4254 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4255 4256 q_vector->num_ringpairs = num_ringpairs; 4257 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4258 4259 q_vector->rx.count = 0; 4260 q_vector->tx.count = 0; 4261 q_vector->rx.ring = NULL; 4262 q_vector->tx.ring = NULL; 4263 4264 while (num_ringpairs--) { 4265 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4266 qp_idx++; 4267 qp_remaining--; 4268 } 4269 } 4270 } 4271 4272 /** 4273 * i40e_vsi_request_irq - Request IRQ from the OS 4274 * @vsi: the VSI being configured 4275 * @basename: name for the vector 4276 **/ 4277 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4278 { 4279 struct i40e_pf *pf = vsi->back; 4280 int err; 4281 4282 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4283 err = i40e_vsi_request_irq_msix(vsi, basename); 4284 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4285 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4286 pf->int_name, pf); 4287 else 4288 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4289 pf->int_name, pf); 4290 4291 if (err) 4292 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4293 4294 return err; 4295 } 4296 4297 #ifdef CONFIG_NET_POLL_CONTROLLER 4298 /** 4299 * i40e_netpoll - A Polling 'interrupt' handler 4300 * @netdev: network interface device structure 4301 * 4302 * This is used by netconsole to send skbs without having to re-enable 4303 * interrupts. It's not called while the normal interrupt routine is executing. 4304 **/ 4305 static void i40e_netpoll(struct net_device *netdev) 4306 { 4307 struct i40e_netdev_priv *np = netdev_priv(netdev); 4308 struct i40e_vsi *vsi = np->vsi; 4309 struct i40e_pf *pf = vsi->back; 4310 int i; 4311 4312 /* if interface is down do nothing */ 4313 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4314 return; 4315 4316 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4317 for (i = 0; i < vsi->num_q_vectors; i++) 4318 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4319 } else { 4320 i40e_intr(pf->pdev->irq, netdev); 4321 } 4322 } 4323 #endif 4324 4325 #define I40E_QTX_ENA_WAIT_COUNT 50 4326 4327 /** 4328 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4329 * @pf: the PF being configured 4330 * @pf_q: the PF queue 4331 * @enable: enable or disable state of the queue 4332 * 4333 * This routine will wait for the given Tx queue of the PF to reach the 4334 * enabled or disabled state. 4335 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4336 * multiple retries; else will return 0 in case of success. 4337 **/ 4338 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4339 { 4340 int i; 4341 u32 tx_reg; 4342 4343 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4344 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4345 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4346 break; 4347 4348 usleep_range(10, 20); 4349 } 4350 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4351 return -ETIMEDOUT; 4352 4353 return 0; 4354 } 4355 4356 /** 4357 * i40e_control_tx_q - Start or stop a particular Tx queue 4358 * @pf: the PF structure 4359 * @pf_q: the PF queue to configure 4360 * @enable: start or stop the queue 4361 * 4362 * This function enables or disables a single queue. Note that any delay 4363 * required after the operation is expected to be handled by the caller of 4364 * this function. 4365 **/ 4366 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4367 { 4368 struct i40e_hw *hw = &pf->hw; 4369 u32 tx_reg; 4370 int i; 4371 4372 /* warn the TX unit of coming changes */ 4373 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4374 if (!enable) 4375 usleep_range(10, 20); 4376 4377 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4378 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4379 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4380 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4381 break; 4382 usleep_range(1000, 2000); 4383 } 4384 4385 /* Skip if the queue is already in the requested state */ 4386 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4387 return; 4388 4389 /* turn on/off the queue */ 4390 if (enable) { 4391 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4392 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4393 } else { 4394 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4395 } 4396 4397 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4398 } 4399 4400 /** 4401 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4402 * @seid: VSI SEID 4403 * @pf: the PF structure 4404 * @pf_q: the PF queue to configure 4405 * @is_xdp: true if the queue is used for XDP 4406 * @enable: start or stop the queue 4407 **/ 4408 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4409 bool is_xdp, bool enable) 4410 { 4411 int ret; 4412 4413 i40e_control_tx_q(pf, pf_q, enable); 4414 4415 /* wait for the change to finish */ 4416 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4417 if (ret) { 4418 dev_info(&pf->pdev->dev, 4419 "VSI seid %d %sTx ring %d %sable timeout\n", 4420 seid, (is_xdp ? "XDP " : ""), pf_q, 4421 (enable ? "en" : "dis")); 4422 } 4423 4424 return ret; 4425 } 4426 4427 /** 4428 * i40e_vsi_control_tx - Start or stop a VSI's rings 4429 * @vsi: the VSI being configured 4430 * @enable: start or stop the rings 4431 **/ 4432 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4433 { 4434 struct i40e_pf *pf = vsi->back; 4435 int i, pf_q, ret = 0; 4436 4437 pf_q = vsi->base_queue; 4438 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4439 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4440 pf_q, 4441 false /*is xdp*/, enable); 4442 if (ret) 4443 break; 4444 4445 if (!i40e_enabled_xdp_vsi(vsi)) 4446 continue; 4447 4448 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4449 pf_q + vsi->alloc_queue_pairs, 4450 true /*is xdp*/, enable); 4451 if (ret) 4452 break; 4453 } 4454 return ret; 4455 } 4456 4457 /** 4458 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4459 * @pf: the PF being configured 4460 * @pf_q: the PF queue 4461 * @enable: enable or disable state of the queue 4462 * 4463 * This routine will wait for the given Rx queue of the PF to reach the 4464 * enabled or disabled state. 4465 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4466 * multiple retries; else will return 0 in case of success. 4467 **/ 4468 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4469 { 4470 int i; 4471 u32 rx_reg; 4472 4473 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4474 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4475 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4476 break; 4477 4478 usleep_range(10, 20); 4479 } 4480 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4481 return -ETIMEDOUT; 4482 4483 return 0; 4484 } 4485 4486 /** 4487 * i40e_control_rx_q - Start or stop a particular Rx queue 4488 * @pf: the PF structure 4489 * @pf_q: the PF queue to configure 4490 * @enable: start or stop the queue 4491 * 4492 * This function enables or disables a single queue. Note that 4493 * any delay required after the operation is expected to be 4494 * handled by the caller of this function. 4495 **/ 4496 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4497 { 4498 struct i40e_hw *hw = &pf->hw; 4499 u32 rx_reg; 4500 int i; 4501 4502 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4503 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4504 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4505 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4506 break; 4507 usleep_range(1000, 2000); 4508 } 4509 4510 /* Skip if the queue is already in the requested state */ 4511 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4512 return; 4513 4514 /* turn on/off the queue */ 4515 if (enable) 4516 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4517 else 4518 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4519 4520 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4521 } 4522 4523 /** 4524 * i40e_control_wait_rx_q 4525 * @pf: the PF structure 4526 * @pf_q: queue being configured 4527 * @enable: start or stop the rings 4528 * 4529 * This function enables or disables a single queue along with waiting 4530 * for the change to finish. The caller of this function should handle 4531 * the delays needed in the case of disabling queues. 4532 **/ 4533 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4534 { 4535 int ret = 0; 4536 4537 i40e_control_rx_q(pf, pf_q, enable); 4538 4539 /* wait for the change to finish */ 4540 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4541 if (ret) 4542 return ret; 4543 4544 return ret; 4545 } 4546 4547 /** 4548 * i40e_vsi_control_rx - Start or stop a VSI's rings 4549 * @vsi: the VSI being configured 4550 * @enable: start or stop the rings 4551 **/ 4552 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4553 { 4554 struct i40e_pf *pf = vsi->back; 4555 int i, pf_q, ret = 0; 4556 4557 pf_q = vsi->base_queue; 4558 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4559 ret = i40e_control_wait_rx_q(pf, pf_q, enable); 4560 if (ret) { 4561 dev_info(&pf->pdev->dev, 4562 "VSI seid %d Rx ring %d %sable timeout\n", 4563 vsi->seid, pf_q, (enable ? "en" : "dis")); 4564 break; 4565 } 4566 } 4567 4568 /* Due to HW errata, on Rx disable only, the register can indicate done 4569 * before it really is. Needs 50ms to be sure 4570 */ 4571 if (!enable) 4572 mdelay(50); 4573 4574 return ret; 4575 } 4576 4577 /** 4578 * i40e_vsi_start_rings - Start a VSI's rings 4579 * @vsi: the VSI being configured 4580 **/ 4581 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4582 { 4583 int ret = 0; 4584 4585 /* do rx first for enable and last for disable */ 4586 ret = i40e_vsi_control_rx(vsi, true); 4587 if (ret) 4588 return ret; 4589 ret = i40e_vsi_control_tx(vsi, true); 4590 4591 return ret; 4592 } 4593 4594 /** 4595 * i40e_vsi_stop_rings - Stop a VSI's rings 4596 * @vsi: the VSI being configured 4597 **/ 4598 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4599 { 4600 /* When port TX is suspended, don't wait */ 4601 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4602 return i40e_vsi_stop_rings_no_wait(vsi); 4603 4604 /* do rx first for enable and last for disable 4605 * Ignore return value, we need to shutdown whatever we can 4606 */ 4607 i40e_vsi_control_tx(vsi, false); 4608 i40e_vsi_control_rx(vsi, false); 4609 } 4610 4611 /** 4612 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4613 * @vsi: the VSI being shutdown 4614 * 4615 * This function stops all the rings for a VSI but does not delay to verify 4616 * that rings have been disabled. It is expected that the caller is shutting 4617 * down multiple VSIs at once and will delay together for all the VSIs after 4618 * initiating the shutdown. This is particularly useful for shutting down lots 4619 * of VFs together. Otherwise, a large delay can be incurred while configuring 4620 * each VSI in serial. 4621 **/ 4622 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4623 { 4624 struct i40e_pf *pf = vsi->back; 4625 int i, pf_q; 4626 4627 pf_q = vsi->base_queue; 4628 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4629 i40e_control_tx_q(pf, pf_q, false); 4630 i40e_control_rx_q(pf, pf_q, false); 4631 } 4632 } 4633 4634 /** 4635 * i40e_vsi_free_irq - Free the irq association with the OS 4636 * @vsi: the VSI being configured 4637 **/ 4638 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4639 { 4640 struct i40e_pf *pf = vsi->back; 4641 struct i40e_hw *hw = &pf->hw; 4642 int base = vsi->base_vector; 4643 u32 val, qp; 4644 int i; 4645 4646 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4647 if (!vsi->q_vectors) 4648 return; 4649 4650 if (!vsi->irqs_ready) 4651 return; 4652 4653 vsi->irqs_ready = false; 4654 for (i = 0; i < vsi->num_q_vectors; i++) { 4655 int irq_num; 4656 u16 vector; 4657 4658 vector = i + base; 4659 irq_num = pf->msix_entries[vector].vector; 4660 4661 /* free only the irqs that were actually requested */ 4662 if (!vsi->q_vectors[i] || 4663 !vsi->q_vectors[i]->num_ringpairs) 4664 continue; 4665 4666 /* clear the affinity notifier in the IRQ descriptor */ 4667 irq_set_affinity_notifier(irq_num, NULL); 4668 /* remove our suggested affinity mask for this IRQ */ 4669 irq_set_affinity_hint(irq_num, NULL); 4670 synchronize_irq(irq_num); 4671 free_irq(irq_num, vsi->q_vectors[i]); 4672 4673 /* Tear down the interrupt queue link list 4674 * 4675 * We know that they come in pairs and always 4676 * the Rx first, then the Tx. To clear the 4677 * link list, stick the EOL value into the 4678 * next_q field of the registers. 4679 */ 4680 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4681 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4682 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4683 val |= I40E_QUEUE_END_OF_LIST 4684 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4685 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4686 4687 while (qp != I40E_QUEUE_END_OF_LIST) { 4688 u32 next; 4689 4690 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4691 4692 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4693 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4694 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4695 I40E_QINT_RQCTL_INTEVENT_MASK); 4696 4697 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4698 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4699 4700 wr32(hw, I40E_QINT_RQCTL(qp), val); 4701 4702 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4703 4704 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4705 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4706 4707 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4708 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4709 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4710 I40E_QINT_TQCTL_INTEVENT_MASK); 4711 4712 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4713 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4714 4715 wr32(hw, I40E_QINT_TQCTL(qp), val); 4716 qp = next; 4717 } 4718 } 4719 } else { 4720 free_irq(pf->pdev->irq, pf); 4721 4722 val = rd32(hw, I40E_PFINT_LNKLST0); 4723 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4724 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4725 val |= I40E_QUEUE_END_OF_LIST 4726 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4727 wr32(hw, I40E_PFINT_LNKLST0, val); 4728 4729 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4730 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4731 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4732 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4733 I40E_QINT_RQCTL_INTEVENT_MASK); 4734 4735 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4736 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4737 4738 wr32(hw, I40E_QINT_RQCTL(qp), val); 4739 4740 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4741 4742 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4743 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4744 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4745 I40E_QINT_TQCTL_INTEVENT_MASK); 4746 4747 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4748 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4749 4750 wr32(hw, I40E_QINT_TQCTL(qp), val); 4751 } 4752 } 4753 4754 /** 4755 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4756 * @vsi: the VSI being configured 4757 * @v_idx: Index of vector to be freed 4758 * 4759 * This function frees the memory allocated to the q_vector. In addition if 4760 * NAPI is enabled it will delete any references to the NAPI struct prior 4761 * to freeing the q_vector. 4762 **/ 4763 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4764 { 4765 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4766 struct i40e_ring *ring; 4767 4768 if (!q_vector) 4769 return; 4770 4771 /* disassociate q_vector from rings */ 4772 i40e_for_each_ring(ring, q_vector->tx) 4773 ring->q_vector = NULL; 4774 4775 i40e_for_each_ring(ring, q_vector->rx) 4776 ring->q_vector = NULL; 4777 4778 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4779 if (vsi->netdev) 4780 netif_napi_del(&q_vector->napi); 4781 4782 vsi->q_vectors[v_idx] = NULL; 4783 4784 kfree_rcu(q_vector, rcu); 4785 } 4786 4787 /** 4788 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4789 * @vsi: the VSI being un-configured 4790 * 4791 * This frees the memory allocated to the q_vectors and 4792 * deletes references to the NAPI struct. 4793 **/ 4794 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4795 { 4796 int v_idx; 4797 4798 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4799 i40e_free_q_vector(vsi, v_idx); 4800 } 4801 4802 /** 4803 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4804 * @pf: board private structure 4805 **/ 4806 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4807 { 4808 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4809 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4810 pci_disable_msix(pf->pdev); 4811 kfree(pf->msix_entries); 4812 pf->msix_entries = NULL; 4813 kfree(pf->irq_pile); 4814 pf->irq_pile = NULL; 4815 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4816 pci_disable_msi(pf->pdev); 4817 } 4818 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4819 } 4820 4821 /** 4822 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4823 * @pf: board private structure 4824 * 4825 * We go through and clear interrupt specific resources and reset the structure 4826 * to pre-load conditions 4827 **/ 4828 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4829 { 4830 int i; 4831 4832 i40e_free_misc_vector(pf); 4833 4834 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4835 I40E_IWARP_IRQ_PILE_ID); 4836 4837 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4838 for (i = 0; i < pf->num_alloc_vsi; i++) 4839 if (pf->vsi[i]) 4840 i40e_vsi_free_q_vectors(pf->vsi[i]); 4841 i40e_reset_interrupt_capability(pf); 4842 } 4843 4844 /** 4845 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4846 * @vsi: the VSI being configured 4847 **/ 4848 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4849 { 4850 int q_idx; 4851 4852 if (!vsi->netdev) 4853 return; 4854 4855 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4856 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4857 4858 if (q_vector->rx.ring || q_vector->tx.ring) 4859 napi_enable(&q_vector->napi); 4860 } 4861 } 4862 4863 /** 4864 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4865 * @vsi: the VSI being configured 4866 **/ 4867 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4868 { 4869 int q_idx; 4870 4871 if (!vsi->netdev) 4872 return; 4873 4874 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4875 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4876 4877 if (q_vector->rx.ring || q_vector->tx.ring) 4878 napi_disable(&q_vector->napi); 4879 } 4880 } 4881 4882 /** 4883 * i40e_vsi_close - Shut down a VSI 4884 * @vsi: the vsi to be quelled 4885 **/ 4886 static void i40e_vsi_close(struct i40e_vsi *vsi) 4887 { 4888 struct i40e_pf *pf = vsi->back; 4889 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4890 i40e_down(vsi); 4891 i40e_vsi_free_irq(vsi); 4892 i40e_vsi_free_tx_resources(vsi); 4893 i40e_vsi_free_rx_resources(vsi); 4894 vsi->current_netdev_flags = 0; 4895 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 4896 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4897 set_bit(__I40E_CLIENT_RESET, pf->state); 4898 } 4899 4900 /** 4901 * i40e_quiesce_vsi - Pause a given VSI 4902 * @vsi: the VSI being paused 4903 **/ 4904 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4905 { 4906 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4907 return; 4908 4909 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4910 if (vsi->netdev && netif_running(vsi->netdev)) 4911 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4912 else 4913 i40e_vsi_close(vsi); 4914 } 4915 4916 /** 4917 * i40e_unquiesce_vsi - Resume a given VSI 4918 * @vsi: the VSI being resumed 4919 **/ 4920 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4921 { 4922 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4923 return; 4924 4925 if (vsi->netdev && netif_running(vsi->netdev)) 4926 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4927 else 4928 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4929 } 4930 4931 /** 4932 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4933 * @pf: the PF 4934 **/ 4935 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4936 { 4937 int v; 4938 4939 for (v = 0; v < pf->num_alloc_vsi; v++) { 4940 if (pf->vsi[v]) 4941 i40e_quiesce_vsi(pf->vsi[v]); 4942 } 4943 } 4944 4945 /** 4946 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4947 * @pf: the PF 4948 **/ 4949 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4950 { 4951 int v; 4952 4953 for (v = 0; v < pf->num_alloc_vsi; v++) { 4954 if (pf->vsi[v]) 4955 i40e_unquiesce_vsi(pf->vsi[v]); 4956 } 4957 } 4958 4959 /** 4960 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4961 * @vsi: the VSI being configured 4962 * 4963 * Wait until all queues on a given VSI have been disabled. 4964 **/ 4965 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4966 { 4967 struct i40e_pf *pf = vsi->back; 4968 int i, pf_q, ret; 4969 4970 pf_q = vsi->base_queue; 4971 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4972 /* Check and wait for the Tx queue */ 4973 ret = i40e_pf_txq_wait(pf, pf_q, false); 4974 if (ret) { 4975 dev_info(&pf->pdev->dev, 4976 "VSI seid %d Tx ring %d disable timeout\n", 4977 vsi->seid, pf_q); 4978 return ret; 4979 } 4980 4981 if (!i40e_enabled_xdp_vsi(vsi)) 4982 goto wait_rx; 4983 4984 /* Check and wait for the XDP Tx queue */ 4985 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4986 false); 4987 if (ret) { 4988 dev_info(&pf->pdev->dev, 4989 "VSI seid %d XDP Tx ring %d disable timeout\n", 4990 vsi->seid, pf_q); 4991 return ret; 4992 } 4993 wait_rx: 4994 /* Check and wait for the Rx queue */ 4995 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4996 if (ret) { 4997 dev_info(&pf->pdev->dev, 4998 "VSI seid %d Rx ring %d disable timeout\n", 4999 vsi->seid, pf_q); 5000 return ret; 5001 } 5002 } 5003 5004 return 0; 5005 } 5006 5007 #ifdef CONFIG_I40E_DCB 5008 /** 5009 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 5010 * @pf: the PF 5011 * 5012 * This function waits for the queues to be in disabled state for all the 5013 * VSIs that are managed by this PF. 5014 **/ 5015 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 5016 { 5017 int v, ret = 0; 5018 5019 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 5020 if (pf->vsi[v]) { 5021 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 5022 if (ret) 5023 break; 5024 } 5025 } 5026 5027 return ret; 5028 } 5029 5030 #endif 5031 5032 /** 5033 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 5034 * @pf: pointer to PF 5035 * 5036 * Get TC map for ISCSI PF type that will include iSCSI TC 5037 * and LAN TC. 5038 **/ 5039 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 5040 { 5041 struct i40e_dcb_app_priority_table app; 5042 struct i40e_hw *hw = &pf->hw; 5043 u8 enabled_tc = 1; /* TC0 is always enabled */ 5044 u8 tc, i; 5045 /* Get the iSCSI APP TLV */ 5046 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5047 5048 for (i = 0; i < dcbcfg->numapps; i++) { 5049 app = dcbcfg->app[i]; 5050 if (app.selector == I40E_APP_SEL_TCPIP && 5051 app.protocolid == I40E_APP_PROTOID_ISCSI) { 5052 tc = dcbcfg->etscfg.prioritytable[app.priority]; 5053 enabled_tc |= BIT(tc); 5054 break; 5055 } 5056 } 5057 5058 return enabled_tc; 5059 } 5060 5061 /** 5062 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5063 * @dcbcfg: the corresponding DCBx configuration structure 5064 * 5065 * Return the number of TCs from given DCBx configuration 5066 **/ 5067 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5068 { 5069 int i, tc_unused = 0; 5070 u8 num_tc = 0; 5071 u8 ret = 0; 5072 5073 /* Scan the ETS Config Priority Table to find 5074 * traffic class enabled for a given priority 5075 * and create a bitmask of enabled TCs 5076 */ 5077 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5078 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5079 5080 /* Now scan the bitmask to check for 5081 * contiguous TCs starting with TC0 5082 */ 5083 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5084 if (num_tc & BIT(i)) { 5085 if (!tc_unused) { 5086 ret++; 5087 } else { 5088 pr_err("Non-contiguous TC - Disabling DCB\n"); 5089 return 1; 5090 } 5091 } else { 5092 tc_unused = 1; 5093 } 5094 } 5095 5096 /* There is always at least TC0 */ 5097 if (!ret) 5098 ret = 1; 5099 5100 return ret; 5101 } 5102 5103 /** 5104 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5105 * @dcbcfg: the corresponding DCBx configuration structure 5106 * 5107 * Query the current DCB configuration and return the number of 5108 * traffic classes enabled from the given DCBX config 5109 **/ 5110 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5111 { 5112 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5113 u8 enabled_tc = 1; 5114 u8 i; 5115 5116 for (i = 0; i < num_tc; i++) 5117 enabled_tc |= BIT(i); 5118 5119 return enabled_tc; 5120 } 5121 5122 /** 5123 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5124 * @pf: PF being queried 5125 * 5126 * Query the current MQPRIO configuration and return the number of 5127 * traffic classes enabled. 5128 **/ 5129 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5130 { 5131 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 5132 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5133 u8 enabled_tc = 1, i; 5134 5135 for (i = 1; i < num_tc; i++) 5136 enabled_tc |= BIT(i); 5137 return enabled_tc; 5138 } 5139 5140 /** 5141 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5142 * @pf: PF being queried 5143 * 5144 * Return number of traffic classes enabled for the given PF 5145 **/ 5146 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5147 { 5148 struct i40e_hw *hw = &pf->hw; 5149 u8 i, enabled_tc = 1; 5150 u8 num_tc = 0; 5151 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5152 5153 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5154 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5155 5156 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5157 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5158 return 1; 5159 5160 /* SFP mode will be enabled for all TCs on port */ 5161 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5162 return i40e_dcb_get_num_tc(dcbcfg); 5163 5164 /* MFP mode return count of enabled TCs for this PF */ 5165 if (pf->hw.func_caps.iscsi) 5166 enabled_tc = i40e_get_iscsi_tc_map(pf); 5167 else 5168 return 1; /* Only TC0 */ 5169 5170 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5171 if (enabled_tc & BIT(i)) 5172 num_tc++; 5173 } 5174 return num_tc; 5175 } 5176 5177 /** 5178 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 5179 * @pf: PF being queried 5180 * 5181 * Return a bitmap for enabled traffic classes for this PF. 5182 **/ 5183 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5184 { 5185 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5186 return i40e_mqprio_get_enabled_tc(pf); 5187 5188 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5189 * default TC 5190 */ 5191 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5192 return I40E_DEFAULT_TRAFFIC_CLASS; 5193 5194 /* SFP mode we want PF to be enabled for all TCs */ 5195 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5196 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5197 5198 /* MFP enabled and iSCSI PF type */ 5199 if (pf->hw.func_caps.iscsi) 5200 return i40e_get_iscsi_tc_map(pf); 5201 else 5202 return I40E_DEFAULT_TRAFFIC_CLASS; 5203 } 5204 5205 /** 5206 * i40e_vsi_get_bw_info - Query VSI BW Information 5207 * @vsi: the VSI being queried 5208 * 5209 * Returns 0 on success, negative value on failure 5210 **/ 5211 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5212 { 5213 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5214 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5215 struct i40e_pf *pf = vsi->back; 5216 struct i40e_hw *hw = &pf->hw; 5217 i40e_status ret; 5218 u32 tc_bw_max; 5219 int i; 5220 5221 /* Get the VSI level BW configuration */ 5222 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5223 if (ret) { 5224 dev_info(&pf->pdev->dev, 5225 "couldn't get PF vsi bw config, err %s aq_err %s\n", 5226 i40e_stat_str(&pf->hw, ret), 5227 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5228 return -EINVAL; 5229 } 5230 5231 /* Get the VSI level BW configuration per TC */ 5232 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5233 NULL); 5234 if (ret) { 5235 dev_info(&pf->pdev->dev, 5236 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 5237 i40e_stat_str(&pf->hw, ret), 5238 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5239 return -EINVAL; 5240 } 5241 5242 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5243 dev_info(&pf->pdev->dev, 5244 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5245 bw_config.tc_valid_bits, 5246 bw_ets_config.tc_valid_bits); 5247 /* Still continuing */ 5248 } 5249 5250 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5251 vsi->bw_max_quanta = bw_config.max_bw; 5252 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5253 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5254 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5255 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5256 vsi->bw_ets_limit_credits[i] = 5257 le16_to_cpu(bw_ets_config.credits[i]); 5258 /* 3 bits out of 4 for each TC */ 5259 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5260 } 5261 5262 return 0; 5263 } 5264 5265 /** 5266 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5267 * @vsi: the VSI being configured 5268 * @enabled_tc: TC bitmap 5269 * @bw_share: BW shared credits per TC 5270 * 5271 * Returns 0 on success, negative value on failure 5272 **/ 5273 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5274 u8 *bw_share) 5275 { 5276 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5277 struct i40e_pf *pf = vsi->back; 5278 i40e_status ret; 5279 int i; 5280 5281 /* There is no need to reset BW when mqprio mode is on. */ 5282 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5283 return 0; 5284 if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5285 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5286 if (ret) 5287 dev_info(&pf->pdev->dev, 5288 "Failed to reset tx rate for vsi->seid %u\n", 5289 vsi->seid); 5290 return ret; 5291 } 5292 bw_data.tc_valid_bits = enabled_tc; 5293 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5294 bw_data.tc_bw_credits[i] = bw_share[i]; 5295 5296 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5297 if (ret) { 5298 dev_info(&pf->pdev->dev, 5299 "AQ command Config VSI BW allocation per TC failed = %d\n", 5300 pf->hw.aq.asq_last_status); 5301 return -EINVAL; 5302 } 5303 5304 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5305 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5306 5307 return 0; 5308 } 5309 5310 /** 5311 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5312 * @vsi: the VSI being configured 5313 * @enabled_tc: TC map to be enabled 5314 * 5315 **/ 5316 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5317 { 5318 struct net_device *netdev = vsi->netdev; 5319 struct i40e_pf *pf = vsi->back; 5320 struct i40e_hw *hw = &pf->hw; 5321 u8 netdev_tc = 0; 5322 int i; 5323 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5324 5325 if (!netdev) 5326 return; 5327 5328 if (!enabled_tc) { 5329 netdev_reset_tc(netdev); 5330 return; 5331 } 5332 5333 /* Set up actual enabled TCs on the VSI */ 5334 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5335 return; 5336 5337 /* set per TC queues for the VSI */ 5338 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5339 /* Only set TC queues for enabled tcs 5340 * 5341 * e.g. For a VSI that has TC0 and TC3 enabled the 5342 * enabled_tc bitmap would be 0x00001001; the driver 5343 * will set the numtc for netdev as 2 that will be 5344 * referenced by the netdev layer as TC 0 and 1. 5345 */ 5346 if (vsi->tc_config.enabled_tc & BIT(i)) 5347 netdev_set_tc_queue(netdev, 5348 vsi->tc_config.tc_info[i].netdev_tc, 5349 vsi->tc_config.tc_info[i].qcount, 5350 vsi->tc_config.tc_info[i].qoffset); 5351 } 5352 5353 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5354 return; 5355 5356 /* Assign UP2TC map for the VSI */ 5357 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5358 /* Get the actual TC# for the UP */ 5359 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5360 /* Get the mapped netdev TC# for the UP */ 5361 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5362 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5363 } 5364 } 5365 5366 /** 5367 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5368 * @vsi: the VSI being configured 5369 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5370 **/ 5371 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5372 struct i40e_vsi_context *ctxt) 5373 { 5374 /* copy just the sections touched not the entire info 5375 * since not all sections are valid as returned by 5376 * update vsi params 5377 */ 5378 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5379 memcpy(&vsi->info.queue_mapping, 5380 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5381 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5382 sizeof(vsi->info.tc_mapping)); 5383 } 5384 5385 /** 5386 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5387 * @vsi: VSI to be configured 5388 * @enabled_tc: TC bitmap 5389 * 5390 * This configures a particular VSI for TCs that are mapped to the 5391 * given TC bitmap. It uses default bandwidth share for TCs across 5392 * VSIs to configure TC for a particular VSI. 5393 * 5394 * NOTE: 5395 * It is expected that the VSI queues have been quisced before calling 5396 * this function. 5397 **/ 5398 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5399 { 5400 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5401 struct i40e_pf *pf = vsi->back; 5402 struct i40e_hw *hw = &pf->hw; 5403 struct i40e_vsi_context ctxt; 5404 int ret = 0; 5405 int i; 5406 5407 /* Check if enabled_tc is same as existing or new TCs */ 5408 if (vsi->tc_config.enabled_tc == enabled_tc && 5409 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5410 return ret; 5411 5412 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5413 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5414 if (enabled_tc & BIT(i)) 5415 bw_share[i] = 1; 5416 } 5417 5418 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5419 if (ret) { 5420 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5421 5422 dev_info(&pf->pdev->dev, 5423 "Failed configuring TC map %d for VSI %d\n", 5424 enabled_tc, vsi->seid); 5425 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5426 &bw_config, NULL); 5427 if (ret) { 5428 dev_info(&pf->pdev->dev, 5429 "Failed querying vsi bw info, err %s aq_err %s\n", 5430 i40e_stat_str(hw, ret), 5431 i40e_aq_str(hw, hw->aq.asq_last_status)); 5432 goto out; 5433 } 5434 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5435 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5436 5437 if (!valid_tc) 5438 valid_tc = bw_config.tc_valid_bits; 5439 /* Always enable TC0, no matter what */ 5440 valid_tc |= 1; 5441 dev_info(&pf->pdev->dev, 5442 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5443 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5444 enabled_tc = valid_tc; 5445 } 5446 5447 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5448 if (ret) { 5449 dev_err(&pf->pdev->dev, 5450 "Unable to configure TC map %d for VSI %d\n", 5451 enabled_tc, vsi->seid); 5452 goto out; 5453 } 5454 } 5455 5456 /* Update Queue Pairs Mapping for currently enabled UPs */ 5457 ctxt.seid = vsi->seid; 5458 ctxt.pf_num = vsi->back->hw.pf_id; 5459 ctxt.vf_num = 0; 5460 ctxt.uplink_seid = vsi->uplink_seid; 5461 ctxt.info = vsi->info; 5462 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { 5463 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5464 if (ret) 5465 goto out; 5466 } else { 5467 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5468 } 5469 5470 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5471 * queues changed. 5472 */ 5473 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5474 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5475 vsi->num_queue_pairs); 5476 ret = i40e_vsi_config_rss(vsi); 5477 if (ret) { 5478 dev_info(&vsi->back->pdev->dev, 5479 "Failed to reconfig rss for num_queues\n"); 5480 return ret; 5481 } 5482 vsi->reconfig_rss = false; 5483 } 5484 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5485 ctxt.info.valid_sections |= 5486 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5487 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5488 } 5489 5490 /* Update the VSI after updating the VSI queue-mapping 5491 * information 5492 */ 5493 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5494 if (ret) { 5495 dev_info(&pf->pdev->dev, 5496 "Update vsi tc config failed, err %s aq_err %s\n", 5497 i40e_stat_str(hw, ret), 5498 i40e_aq_str(hw, hw->aq.asq_last_status)); 5499 goto out; 5500 } 5501 /* update the local VSI info with updated queue map */ 5502 i40e_vsi_update_queue_map(vsi, &ctxt); 5503 vsi->info.valid_sections = 0; 5504 5505 /* Update current VSI BW information */ 5506 ret = i40e_vsi_get_bw_info(vsi); 5507 if (ret) { 5508 dev_info(&pf->pdev->dev, 5509 "Failed updating vsi bw info, err %s aq_err %s\n", 5510 i40e_stat_str(hw, ret), 5511 i40e_aq_str(hw, hw->aq.asq_last_status)); 5512 goto out; 5513 } 5514 5515 /* Update the netdev TC setup */ 5516 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5517 out: 5518 return ret; 5519 } 5520 5521 /** 5522 * i40e_get_link_speed - Returns link speed for the interface 5523 * @vsi: VSI to be configured 5524 * 5525 **/ 5526 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5527 { 5528 struct i40e_pf *pf = vsi->back; 5529 5530 switch (pf->hw.phy.link_info.link_speed) { 5531 case I40E_LINK_SPEED_40GB: 5532 return 40000; 5533 case I40E_LINK_SPEED_25GB: 5534 return 25000; 5535 case I40E_LINK_SPEED_20GB: 5536 return 20000; 5537 case I40E_LINK_SPEED_10GB: 5538 return 10000; 5539 case I40E_LINK_SPEED_1GB: 5540 return 1000; 5541 default: 5542 return -EINVAL; 5543 } 5544 } 5545 5546 /** 5547 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5548 * @vsi: VSI to be configured 5549 * @seid: seid of the channel/VSI 5550 * @max_tx_rate: max TX rate to be configured as BW limit 5551 * 5552 * Helper function to set BW limit for a given VSI 5553 **/ 5554 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5555 { 5556 struct i40e_pf *pf = vsi->back; 5557 u64 credits = 0; 5558 int speed = 0; 5559 int ret = 0; 5560 5561 speed = i40e_get_link_speed(vsi); 5562 if (max_tx_rate > speed) { 5563 dev_err(&pf->pdev->dev, 5564 "Invalid max tx rate %llu specified for VSI seid %d.", 5565 max_tx_rate, seid); 5566 return -EINVAL; 5567 } 5568 if (max_tx_rate && max_tx_rate < 50) { 5569 dev_warn(&pf->pdev->dev, 5570 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5571 max_tx_rate = 50; 5572 } 5573 5574 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5575 credits = max_tx_rate; 5576 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5577 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5578 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5579 if (ret) 5580 dev_err(&pf->pdev->dev, 5581 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n", 5582 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret), 5583 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5584 return ret; 5585 } 5586 5587 /** 5588 * i40e_remove_queue_channels - Remove queue channels for the TCs 5589 * @vsi: VSI to be configured 5590 * 5591 * Remove queue channels for the TCs 5592 **/ 5593 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5594 { 5595 enum i40e_admin_queue_err last_aq_status; 5596 struct i40e_cloud_filter *cfilter; 5597 struct i40e_channel *ch, *ch_tmp; 5598 struct i40e_pf *pf = vsi->back; 5599 struct hlist_node *node; 5600 int ret, i; 5601 5602 /* Reset rss size that was stored when reconfiguring rss for 5603 * channel VSIs with non-power-of-2 queue count. 5604 */ 5605 vsi->current_rss_size = 0; 5606 5607 /* perform cleanup for channels if they exist */ 5608 if (list_empty(&vsi->ch_list)) 5609 return; 5610 5611 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5612 struct i40e_vsi *p_vsi; 5613 5614 list_del(&ch->list); 5615 p_vsi = ch->parent_vsi; 5616 if (!p_vsi || !ch->initialized) { 5617 kfree(ch); 5618 continue; 5619 } 5620 /* Reset queue contexts */ 5621 for (i = 0; i < ch->num_queue_pairs; i++) { 5622 struct i40e_ring *tx_ring, *rx_ring; 5623 u16 pf_q; 5624 5625 pf_q = ch->base_queue + i; 5626 tx_ring = vsi->tx_rings[pf_q]; 5627 tx_ring->ch = NULL; 5628 5629 rx_ring = vsi->rx_rings[pf_q]; 5630 rx_ring->ch = NULL; 5631 } 5632 5633 /* Reset BW configured for this VSI via mqprio */ 5634 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 5635 if (ret) 5636 dev_info(&vsi->back->pdev->dev, 5637 "Failed to reset tx rate for ch->seid %u\n", 5638 ch->seid); 5639 5640 /* delete cloud filters associated with this channel */ 5641 hlist_for_each_entry_safe(cfilter, node, 5642 &pf->cloud_filter_list, cloud_node) { 5643 if (cfilter->seid != ch->seid) 5644 continue; 5645 5646 hash_del(&cfilter->cloud_node); 5647 if (cfilter->dst_port) 5648 ret = i40e_add_del_cloud_filter_big_buf(vsi, 5649 cfilter, 5650 false); 5651 else 5652 ret = i40e_add_del_cloud_filter(vsi, cfilter, 5653 false); 5654 last_aq_status = pf->hw.aq.asq_last_status; 5655 if (ret) 5656 dev_info(&pf->pdev->dev, 5657 "Failed to delete cloud filter, err %s aq_err %s\n", 5658 i40e_stat_str(&pf->hw, ret), 5659 i40e_aq_str(&pf->hw, last_aq_status)); 5660 kfree(cfilter); 5661 } 5662 5663 /* delete VSI from FW */ 5664 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 5665 NULL); 5666 if (ret) 5667 dev_err(&vsi->back->pdev->dev, 5668 "unable to remove channel (%d) for parent VSI(%d)\n", 5669 ch->seid, p_vsi->seid); 5670 kfree(ch); 5671 } 5672 INIT_LIST_HEAD(&vsi->ch_list); 5673 } 5674 5675 /** 5676 * i40e_is_any_channel - channel exist or not 5677 * @vsi: ptr to VSI to which channels are associated with 5678 * 5679 * Returns true or false if channel(s) exist for associated VSI or not 5680 **/ 5681 static bool i40e_is_any_channel(struct i40e_vsi *vsi) 5682 { 5683 struct i40e_channel *ch, *ch_tmp; 5684 5685 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5686 if (ch->initialized) 5687 return true; 5688 } 5689 5690 return false; 5691 } 5692 5693 /** 5694 * i40e_get_max_queues_for_channel 5695 * @vsi: ptr to VSI to which channels are associated with 5696 * 5697 * Helper function which returns max value among the queue counts set on the 5698 * channels/TCs created. 5699 **/ 5700 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 5701 { 5702 struct i40e_channel *ch, *ch_tmp; 5703 int max = 0; 5704 5705 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5706 if (!ch->initialized) 5707 continue; 5708 if (ch->num_queue_pairs > max) 5709 max = ch->num_queue_pairs; 5710 } 5711 5712 return max; 5713 } 5714 5715 /** 5716 * i40e_validate_num_queues - validate num_queues w.r.t channel 5717 * @pf: ptr to PF device 5718 * @num_queues: number of queues 5719 * @vsi: the parent VSI 5720 * @reconfig_rss: indicates should the RSS be reconfigured or not 5721 * 5722 * This function validates number of queues in the context of new channel 5723 * which is being established and determines if RSS should be reconfigured 5724 * or not for parent VSI. 5725 **/ 5726 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 5727 struct i40e_vsi *vsi, bool *reconfig_rss) 5728 { 5729 int max_ch_queues; 5730 5731 if (!reconfig_rss) 5732 return -EINVAL; 5733 5734 *reconfig_rss = false; 5735 if (vsi->current_rss_size) { 5736 if (num_queues > vsi->current_rss_size) { 5737 dev_dbg(&pf->pdev->dev, 5738 "Error: num_queues (%d) > vsi's current_size(%d)\n", 5739 num_queues, vsi->current_rss_size); 5740 return -EINVAL; 5741 } else if ((num_queues < vsi->current_rss_size) && 5742 (!is_power_of_2(num_queues))) { 5743 dev_dbg(&pf->pdev->dev, 5744 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 5745 num_queues, vsi->current_rss_size); 5746 return -EINVAL; 5747 } 5748 } 5749 5750 if (!is_power_of_2(num_queues)) { 5751 /* Find the max num_queues configured for channel if channel 5752 * exist. 5753 * if channel exist, then enforce 'num_queues' to be more than 5754 * max ever queues configured for channel. 5755 */ 5756 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 5757 if (num_queues < max_ch_queues) { 5758 dev_dbg(&pf->pdev->dev, 5759 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 5760 num_queues, max_ch_queues); 5761 return -EINVAL; 5762 } 5763 *reconfig_rss = true; 5764 } 5765 5766 return 0; 5767 } 5768 5769 /** 5770 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 5771 * @vsi: the VSI being setup 5772 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 5773 * 5774 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 5775 **/ 5776 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 5777 { 5778 struct i40e_pf *pf = vsi->back; 5779 u8 seed[I40E_HKEY_ARRAY_SIZE]; 5780 struct i40e_hw *hw = &pf->hw; 5781 int local_rss_size; 5782 u8 *lut; 5783 int ret; 5784 5785 if (!vsi->rss_size) 5786 return -EINVAL; 5787 5788 if (rss_size > vsi->rss_size) 5789 return -EINVAL; 5790 5791 local_rss_size = min_t(int, vsi->rss_size, rss_size); 5792 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 5793 if (!lut) 5794 return -ENOMEM; 5795 5796 /* Ignoring user configured lut if there is one */ 5797 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 5798 5799 /* Use user configured hash key if there is one, otherwise 5800 * use default. 5801 */ 5802 if (vsi->rss_hkey_user) 5803 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 5804 else 5805 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 5806 5807 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 5808 if (ret) { 5809 dev_info(&pf->pdev->dev, 5810 "Cannot set RSS lut, err %s aq_err %s\n", 5811 i40e_stat_str(hw, ret), 5812 i40e_aq_str(hw, hw->aq.asq_last_status)); 5813 kfree(lut); 5814 return ret; 5815 } 5816 kfree(lut); 5817 5818 /* Do the update w.r.t. storing rss_size */ 5819 if (!vsi->orig_rss_size) 5820 vsi->orig_rss_size = vsi->rss_size; 5821 vsi->current_rss_size = local_rss_size; 5822 5823 return ret; 5824 } 5825 5826 /** 5827 * i40e_channel_setup_queue_map - Setup a channel queue map 5828 * @pf: ptr to PF device 5829 * @ctxt: VSI context structure 5830 * @ch: ptr to channel structure 5831 * 5832 * Setup queue map for a specific channel 5833 **/ 5834 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 5835 struct i40e_vsi_context *ctxt, 5836 struct i40e_channel *ch) 5837 { 5838 u16 qcount, qmap, sections = 0; 5839 u8 offset = 0; 5840 int pow; 5841 5842 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 5843 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 5844 5845 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 5846 ch->num_queue_pairs = qcount; 5847 5848 /* find the next higher power-of-2 of num queue pairs */ 5849 pow = ilog2(qcount); 5850 if (!is_power_of_2(qcount)) 5851 pow++; 5852 5853 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5854 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 5855 5856 /* Setup queue TC[0].qmap for given VSI context */ 5857 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 5858 5859 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 5860 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 5861 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 5862 ctxt->info.valid_sections |= cpu_to_le16(sections); 5863 } 5864 5865 /** 5866 * i40e_add_channel - add a channel by adding VSI 5867 * @pf: ptr to PF device 5868 * @uplink_seid: underlying HW switching element (VEB) ID 5869 * @ch: ptr to channel structure 5870 * 5871 * Add a channel (VSI) using add_vsi and queue_map 5872 **/ 5873 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 5874 struct i40e_channel *ch) 5875 { 5876 struct i40e_hw *hw = &pf->hw; 5877 struct i40e_vsi_context ctxt; 5878 u8 enabled_tc = 0x1; /* TC0 enabled */ 5879 int ret; 5880 5881 if (ch->type != I40E_VSI_VMDQ2) { 5882 dev_info(&pf->pdev->dev, 5883 "add new vsi failed, ch->type %d\n", ch->type); 5884 return -EINVAL; 5885 } 5886 5887 memset(&ctxt, 0, sizeof(ctxt)); 5888 ctxt.pf_num = hw->pf_id; 5889 ctxt.vf_num = 0; 5890 ctxt.uplink_seid = uplink_seid; 5891 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5892 if (ch->type == I40E_VSI_VMDQ2) 5893 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 5894 5895 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 5896 ctxt.info.valid_sections |= 5897 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5898 ctxt.info.switch_id = 5899 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 5900 } 5901 5902 /* Set queue map for a given VSI context */ 5903 i40e_channel_setup_queue_map(pf, &ctxt, ch); 5904 5905 /* Now time to create VSI */ 5906 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 5907 if (ret) { 5908 dev_info(&pf->pdev->dev, 5909 "add new vsi failed, err %s aq_err %s\n", 5910 i40e_stat_str(&pf->hw, ret), 5911 i40e_aq_str(&pf->hw, 5912 pf->hw.aq.asq_last_status)); 5913 return -ENOENT; 5914 } 5915 5916 /* Success, update channel, set enabled_tc only if the channel 5917 * is not a macvlan 5918 */ 5919 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; 5920 ch->seid = ctxt.seid; 5921 ch->vsi_number = ctxt.vsi_number; 5922 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); 5923 5924 /* copy just the sections touched not the entire info 5925 * since not all sections are valid as returned by 5926 * update vsi params 5927 */ 5928 ch->info.mapping_flags = ctxt.info.mapping_flags; 5929 memcpy(&ch->info.queue_mapping, 5930 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 5931 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 5932 sizeof(ctxt.info.tc_mapping)); 5933 5934 return 0; 5935 } 5936 5937 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 5938 u8 *bw_share) 5939 { 5940 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5941 i40e_status ret; 5942 int i; 5943 5944 bw_data.tc_valid_bits = ch->enabled_tc; 5945 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5946 bw_data.tc_bw_credits[i] = bw_share[i]; 5947 5948 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 5949 &bw_data, NULL); 5950 if (ret) { 5951 dev_info(&vsi->back->pdev->dev, 5952 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 5953 vsi->back->hw.aq.asq_last_status, ch->seid); 5954 return -EINVAL; 5955 } 5956 5957 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5958 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 5959 5960 return 0; 5961 } 5962 5963 /** 5964 * i40e_channel_config_tx_ring - config TX ring associated with new channel 5965 * @pf: ptr to PF device 5966 * @vsi: the VSI being setup 5967 * @ch: ptr to channel structure 5968 * 5969 * Configure TX rings associated with channel (VSI) since queues are being 5970 * from parent VSI. 5971 **/ 5972 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 5973 struct i40e_vsi *vsi, 5974 struct i40e_channel *ch) 5975 { 5976 i40e_status ret; 5977 int i; 5978 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5979 5980 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5981 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5982 if (ch->enabled_tc & BIT(i)) 5983 bw_share[i] = 1; 5984 } 5985 5986 /* configure BW for new VSI */ 5987 ret = i40e_channel_config_bw(vsi, ch, bw_share); 5988 if (ret) { 5989 dev_info(&vsi->back->pdev->dev, 5990 "Failed configuring TC map %d for channel (seid %u)\n", 5991 ch->enabled_tc, ch->seid); 5992 return ret; 5993 } 5994 5995 for (i = 0; i < ch->num_queue_pairs; i++) { 5996 struct i40e_ring *tx_ring, *rx_ring; 5997 u16 pf_q; 5998 5999 pf_q = ch->base_queue + i; 6000 6001 /* Get to TX ring ptr of main VSI, for re-setup TX queue 6002 * context 6003 */ 6004 tx_ring = vsi->tx_rings[pf_q]; 6005 tx_ring->ch = ch; 6006 6007 /* Get the RX ring ptr */ 6008 rx_ring = vsi->rx_rings[pf_q]; 6009 rx_ring->ch = ch; 6010 } 6011 6012 return 0; 6013 } 6014 6015 /** 6016 * i40e_setup_hw_channel - setup new channel 6017 * @pf: ptr to PF device 6018 * @vsi: the VSI being setup 6019 * @ch: ptr to channel structure 6020 * @uplink_seid: underlying HW switching element (VEB) ID 6021 * @type: type of channel to be created (VMDq2/VF) 6022 * 6023 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6024 * and configures TX rings accordingly 6025 **/ 6026 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 6027 struct i40e_vsi *vsi, 6028 struct i40e_channel *ch, 6029 u16 uplink_seid, u8 type) 6030 { 6031 int ret; 6032 6033 ch->initialized = false; 6034 ch->base_queue = vsi->next_base_queue; 6035 ch->type = type; 6036 6037 /* Proceed with creation of channel (VMDq2) VSI */ 6038 ret = i40e_add_channel(pf, uplink_seid, ch); 6039 if (ret) { 6040 dev_info(&pf->pdev->dev, 6041 "failed to add_channel using uplink_seid %u\n", 6042 uplink_seid); 6043 return ret; 6044 } 6045 6046 /* Mark the successful creation of channel */ 6047 ch->initialized = true; 6048 6049 /* Reconfigure TX queues using QTX_CTL register */ 6050 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 6051 if (ret) { 6052 dev_info(&pf->pdev->dev, 6053 "failed to configure TX rings for channel %u\n", 6054 ch->seid); 6055 return ret; 6056 } 6057 6058 /* update 'next_base_queue' */ 6059 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 6060 dev_dbg(&pf->pdev->dev, 6061 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 6062 ch->seid, ch->vsi_number, ch->stat_counter_idx, 6063 ch->num_queue_pairs, 6064 vsi->next_base_queue); 6065 return ret; 6066 } 6067 6068 /** 6069 * i40e_setup_channel - setup new channel using uplink element 6070 * @pf: ptr to PF device 6071 * @vsi: pointer to the VSI to set up the channel within 6072 * @ch: ptr to channel structure 6073 * 6074 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6075 * and uplink switching element (uplink_seid) 6076 **/ 6077 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 6078 struct i40e_channel *ch) 6079 { 6080 u8 vsi_type; 6081 u16 seid; 6082 int ret; 6083 6084 if (vsi->type == I40E_VSI_MAIN) { 6085 vsi_type = I40E_VSI_VMDQ2; 6086 } else { 6087 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6088 vsi->type); 6089 return false; 6090 } 6091 6092 /* underlying switching element */ 6093 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6094 6095 /* create channel (VSI), configure TX rings */ 6096 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6097 if (ret) { 6098 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6099 return false; 6100 } 6101 6102 return ch->initialized ? true : false; 6103 } 6104 6105 /** 6106 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6107 * @vsi: ptr to VSI which has PF backing 6108 * 6109 * Sets up switch mode correctly if it needs to be changed and perform 6110 * what are allowed modes. 6111 **/ 6112 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6113 { 6114 u8 mode; 6115 struct i40e_pf *pf = vsi->back; 6116 struct i40e_hw *hw = &pf->hw; 6117 int ret; 6118 6119 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6120 if (ret) 6121 return -EINVAL; 6122 6123 if (hw->dev_caps.switch_mode) { 6124 /* if switch mode is set, support mode2 (non-tunneled for 6125 * cloud filter) for now 6126 */ 6127 u32 switch_mode = hw->dev_caps.switch_mode & 6128 I40E_SWITCH_MODE_MASK; 6129 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6130 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6131 return 0; 6132 dev_err(&pf->pdev->dev, 6133 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6134 hw->dev_caps.switch_mode); 6135 return -EINVAL; 6136 } 6137 } 6138 6139 /* Set Bit 7 to be valid */ 6140 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6141 6142 /* Set L4type for TCP support */ 6143 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6144 6145 /* Set cloud filter mode */ 6146 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6147 6148 /* Prep mode field for set_switch_config */ 6149 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6150 pf->last_sw_conf_valid_flags, 6151 mode, NULL); 6152 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6153 dev_err(&pf->pdev->dev, 6154 "couldn't set switch config bits, err %s aq_err %s\n", 6155 i40e_stat_str(hw, ret), 6156 i40e_aq_str(hw, 6157 hw->aq.asq_last_status)); 6158 6159 return ret; 6160 } 6161 6162 /** 6163 * i40e_create_queue_channel - function to create channel 6164 * @vsi: VSI to be configured 6165 * @ch: ptr to channel (it contains channel specific params) 6166 * 6167 * This function creates channel (VSI) using num_queues specified by user, 6168 * reconfigs RSS if needed. 6169 **/ 6170 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6171 struct i40e_channel *ch) 6172 { 6173 struct i40e_pf *pf = vsi->back; 6174 bool reconfig_rss; 6175 int err; 6176 6177 if (!ch) 6178 return -EINVAL; 6179 6180 if (!ch->num_queue_pairs) { 6181 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6182 ch->num_queue_pairs); 6183 return -EINVAL; 6184 } 6185 6186 /* validate user requested num_queues for channel */ 6187 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6188 &reconfig_rss); 6189 if (err) { 6190 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6191 ch->num_queue_pairs); 6192 return -EINVAL; 6193 } 6194 6195 /* By default we are in VEPA mode, if this is the first VF/VMDq 6196 * VSI to be added switch to VEB mode. 6197 */ 6198 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) || 6199 (!i40e_is_any_channel(vsi))) { 6200 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) { 6201 dev_dbg(&pf->pdev->dev, 6202 "Failed to create channel. Override queues (%u) not power of 2\n", 6203 vsi->tc_config.tc_info[0].qcount); 6204 return -EINVAL; 6205 } 6206 6207 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6208 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6209 6210 if (vsi->type == I40E_VSI_MAIN) { 6211 if (pf->flags & I40E_FLAG_TC_MQPRIO) 6212 i40e_do_reset(pf, I40E_PF_RESET_FLAG, 6213 true); 6214 else 6215 i40e_do_reset_safe(pf, 6216 I40E_PF_RESET_FLAG); 6217 } 6218 } 6219 /* now onwards for main VSI, number of queues will be value 6220 * of TC0's queue count 6221 */ 6222 } 6223 6224 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6225 * it should be more than num_queues 6226 */ 6227 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6228 dev_dbg(&pf->pdev->dev, 6229 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6230 vsi->cnt_q_avail, ch->num_queue_pairs); 6231 return -EINVAL; 6232 } 6233 6234 /* reconfig_rss only if vsi type is MAIN_VSI */ 6235 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6236 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6237 if (err) { 6238 dev_info(&pf->pdev->dev, 6239 "Error: unable to reconfig rss for num_queues (%u)\n", 6240 ch->num_queue_pairs); 6241 return -EINVAL; 6242 } 6243 } 6244 6245 if (!i40e_setup_channel(pf, vsi, ch)) { 6246 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6247 return -EINVAL; 6248 } 6249 6250 dev_info(&pf->pdev->dev, 6251 "Setup channel (id:%u) utilizing num_queues %d\n", 6252 ch->seid, ch->num_queue_pairs); 6253 6254 /* configure VSI for BW limit */ 6255 if (ch->max_tx_rate) { 6256 u64 credits = ch->max_tx_rate; 6257 6258 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6259 return -EINVAL; 6260 6261 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6262 dev_dbg(&pf->pdev->dev, 6263 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6264 ch->max_tx_rate, 6265 credits, 6266 ch->seid); 6267 } 6268 6269 /* in case of VF, this will be main SRIOV VSI */ 6270 ch->parent_vsi = vsi; 6271 6272 /* and update main_vsi's count for queue_available to use */ 6273 vsi->cnt_q_avail -= ch->num_queue_pairs; 6274 6275 return 0; 6276 } 6277 6278 /** 6279 * i40e_configure_queue_channels - Add queue channel for the given TCs 6280 * @vsi: VSI to be configured 6281 * 6282 * Configures queue channel mapping to the given TCs 6283 **/ 6284 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6285 { 6286 struct i40e_channel *ch; 6287 u64 max_rate = 0; 6288 int ret = 0, i; 6289 6290 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6291 vsi->tc_seid_map[0] = vsi->seid; 6292 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6293 if (vsi->tc_config.enabled_tc & BIT(i)) { 6294 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6295 if (!ch) { 6296 ret = -ENOMEM; 6297 goto err_free; 6298 } 6299 6300 INIT_LIST_HEAD(&ch->list); 6301 ch->num_queue_pairs = 6302 vsi->tc_config.tc_info[i].qcount; 6303 ch->base_queue = 6304 vsi->tc_config.tc_info[i].qoffset; 6305 6306 /* Bandwidth limit through tc interface is in bytes/s, 6307 * change to Mbit/s 6308 */ 6309 max_rate = vsi->mqprio_qopt.max_rate[i]; 6310 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6311 ch->max_tx_rate = max_rate; 6312 6313 list_add_tail(&ch->list, &vsi->ch_list); 6314 6315 ret = i40e_create_queue_channel(vsi, ch); 6316 if (ret) { 6317 dev_err(&vsi->back->pdev->dev, 6318 "Failed creating queue channel with TC%d: queues %d\n", 6319 i, ch->num_queue_pairs); 6320 goto err_free; 6321 } 6322 vsi->tc_seid_map[i] = ch->seid; 6323 } 6324 } 6325 return ret; 6326 6327 err_free: 6328 i40e_remove_queue_channels(vsi); 6329 return ret; 6330 } 6331 6332 /** 6333 * i40e_veb_config_tc - Configure TCs for given VEB 6334 * @veb: given VEB 6335 * @enabled_tc: TC bitmap 6336 * 6337 * Configures given TC bitmap for VEB (switching) element 6338 **/ 6339 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6340 { 6341 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6342 struct i40e_pf *pf = veb->pf; 6343 int ret = 0; 6344 int i; 6345 6346 /* No TCs or already enabled TCs just return */ 6347 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6348 return ret; 6349 6350 bw_data.tc_valid_bits = enabled_tc; 6351 /* bw_data.absolute_credits is not set (relative) */ 6352 6353 /* Enable ETS TCs with equal BW Share for now */ 6354 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6355 if (enabled_tc & BIT(i)) 6356 bw_data.tc_bw_share_credits[i] = 1; 6357 } 6358 6359 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6360 &bw_data, NULL); 6361 if (ret) { 6362 dev_info(&pf->pdev->dev, 6363 "VEB bw config failed, err %s aq_err %s\n", 6364 i40e_stat_str(&pf->hw, ret), 6365 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6366 goto out; 6367 } 6368 6369 /* Update the BW information */ 6370 ret = i40e_veb_get_bw_info(veb); 6371 if (ret) { 6372 dev_info(&pf->pdev->dev, 6373 "Failed getting veb bw config, err %s aq_err %s\n", 6374 i40e_stat_str(&pf->hw, ret), 6375 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6376 } 6377 6378 out: 6379 return ret; 6380 } 6381 6382 #ifdef CONFIG_I40E_DCB 6383 /** 6384 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6385 * @pf: PF struct 6386 * 6387 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6388 * the caller would've quiesce all the VSIs before calling 6389 * this function 6390 **/ 6391 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6392 { 6393 u8 tc_map = 0; 6394 int ret; 6395 u8 v; 6396 6397 /* Enable the TCs available on PF to all VEBs */ 6398 tc_map = i40e_pf_get_tc_map(pf); 6399 for (v = 0; v < I40E_MAX_VEB; v++) { 6400 if (!pf->veb[v]) 6401 continue; 6402 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6403 if (ret) { 6404 dev_info(&pf->pdev->dev, 6405 "Failed configuring TC for VEB seid=%d\n", 6406 pf->veb[v]->seid); 6407 /* Will try to configure as many components */ 6408 } 6409 } 6410 6411 /* Update each VSI */ 6412 for (v = 0; v < pf->num_alloc_vsi; v++) { 6413 if (!pf->vsi[v]) 6414 continue; 6415 6416 /* - Enable all TCs for the LAN VSI 6417 * - For all others keep them at TC0 for now 6418 */ 6419 if (v == pf->lan_vsi) 6420 tc_map = i40e_pf_get_tc_map(pf); 6421 else 6422 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6423 6424 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6425 if (ret) { 6426 dev_info(&pf->pdev->dev, 6427 "Failed configuring TC for VSI seid=%d\n", 6428 pf->vsi[v]->seid); 6429 /* Will try to configure as many components */ 6430 } else { 6431 /* Re-configure VSI vectors based on updated TC map */ 6432 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6433 if (pf->vsi[v]->netdev) 6434 i40e_dcbnl_set_all(pf->vsi[v]); 6435 } 6436 } 6437 } 6438 6439 /** 6440 * i40e_resume_port_tx - Resume port Tx 6441 * @pf: PF struct 6442 * 6443 * Resume a port's Tx and issue a PF reset in case of failure to 6444 * resume. 6445 **/ 6446 static int i40e_resume_port_tx(struct i40e_pf *pf) 6447 { 6448 struct i40e_hw *hw = &pf->hw; 6449 int ret; 6450 6451 ret = i40e_aq_resume_port_tx(hw, NULL); 6452 if (ret) { 6453 dev_info(&pf->pdev->dev, 6454 "Resume Port Tx failed, err %s aq_err %s\n", 6455 i40e_stat_str(&pf->hw, ret), 6456 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6457 /* Schedule PF reset to recover */ 6458 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6459 i40e_service_event_schedule(pf); 6460 } 6461 6462 return ret; 6463 } 6464 6465 /** 6466 * i40e_init_pf_dcb - Initialize DCB configuration 6467 * @pf: PF being configured 6468 * 6469 * Query the current DCB configuration and cache it 6470 * in the hardware structure 6471 **/ 6472 static int i40e_init_pf_dcb(struct i40e_pf *pf) 6473 { 6474 struct i40e_hw *hw = &pf->hw; 6475 int err = 0; 6476 6477 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 6478 * Also do not enable DCBx if FW LLDP agent is disabled 6479 */ 6480 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) || 6481 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) { 6482 dev_info(&pf->pdev->dev, "DCB is not supported or FW LLDP is disabled\n"); 6483 err = I40E_NOT_SUPPORTED; 6484 goto out; 6485 } 6486 6487 err = i40e_init_dcb(hw, true); 6488 if (!err) { 6489 /* Device/Function is not DCBX capable */ 6490 if ((!hw->func_caps.dcb) || 6491 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 6492 dev_info(&pf->pdev->dev, 6493 "DCBX offload is not supported or is disabled for this PF.\n"); 6494 } else { 6495 /* When status is not DISABLED then DCBX in FW */ 6496 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 6497 DCB_CAP_DCBX_VER_IEEE; 6498 6499 pf->flags |= I40E_FLAG_DCB_CAPABLE; 6500 /* Enable DCB tagging only when more than one TC 6501 * or explicitly disable if only one TC 6502 */ 6503 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6504 pf->flags |= I40E_FLAG_DCB_ENABLED; 6505 else 6506 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6507 dev_dbg(&pf->pdev->dev, 6508 "DCBX offload is supported for this PF.\n"); 6509 } 6510 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 6511 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 6512 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP; 6513 } else { 6514 dev_info(&pf->pdev->dev, 6515 "Query for DCB configuration failed, err %s aq_err %s\n", 6516 i40e_stat_str(&pf->hw, err), 6517 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6518 } 6519 6520 out: 6521 return err; 6522 } 6523 #endif /* CONFIG_I40E_DCB */ 6524 6525 /** 6526 * i40e_print_link_message - print link up or down 6527 * @vsi: the VSI for which link needs a message 6528 * @isup: true of link is up, false otherwise 6529 */ 6530 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 6531 { 6532 enum i40e_aq_link_speed new_speed; 6533 struct i40e_pf *pf = vsi->back; 6534 char *speed = "Unknown"; 6535 char *fc = "Unknown"; 6536 char *fec = ""; 6537 char *req_fec = ""; 6538 char *an = ""; 6539 6540 if (isup) 6541 new_speed = pf->hw.phy.link_info.link_speed; 6542 else 6543 new_speed = I40E_LINK_SPEED_UNKNOWN; 6544 6545 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 6546 return; 6547 vsi->current_isup = isup; 6548 vsi->current_speed = new_speed; 6549 if (!isup) { 6550 netdev_info(vsi->netdev, "NIC Link is Down\n"); 6551 return; 6552 } 6553 6554 /* Warn user if link speed on NPAR enabled partition is not at 6555 * least 10GB 6556 */ 6557 if (pf->hw.func_caps.npar_enable && 6558 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 6559 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 6560 netdev_warn(vsi->netdev, 6561 "The partition detected link speed that is less than 10Gbps\n"); 6562 6563 switch (pf->hw.phy.link_info.link_speed) { 6564 case I40E_LINK_SPEED_40GB: 6565 speed = "40 G"; 6566 break; 6567 case I40E_LINK_SPEED_20GB: 6568 speed = "20 G"; 6569 break; 6570 case I40E_LINK_SPEED_25GB: 6571 speed = "25 G"; 6572 break; 6573 case I40E_LINK_SPEED_10GB: 6574 speed = "10 G"; 6575 break; 6576 case I40E_LINK_SPEED_5GB: 6577 speed = "5 G"; 6578 break; 6579 case I40E_LINK_SPEED_2_5GB: 6580 speed = "2.5 G"; 6581 break; 6582 case I40E_LINK_SPEED_1GB: 6583 speed = "1000 M"; 6584 break; 6585 case I40E_LINK_SPEED_100MB: 6586 speed = "100 M"; 6587 break; 6588 default: 6589 break; 6590 } 6591 6592 switch (pf->hw.fc.current_mode) { 6593 case I40E_FC_FULL: 6594 fc = "RX/TX"; 6595 break; 6596 case I40E_FC_TX_PAUSE: 6597 fc = "TX"; 6598 break; 6599 case I40E_FC_RX_PAUSE: 6600 fc = "RX"; 6601 break; 6602 default: 6603 fc = "None"; 6604 break; 6605 } 6606 6607 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 6608 req_fec = "None"; 6609 fec = "None"; 6610 an = "False"; 6611 6612 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6613 an = "True"; 6614 6615 if (pf->hw.phy.link_info.fec_info & 6616 I40E_AQ_CONFIG_FEC_KR_ENA) 6617 fec = "CL74 FC-FEC/BASE-R"; 6618 else if (pf->hw.phy.link_info.fec_info & 6619 I40E_AQ_CONFIG_FEC_RS_ENA) 6620 fec = "CL108 RS-FEC"; 6621 6622 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 6623 * both RS and FC are requested 6624 */ 6625 if (vsi->back->hw.phy.link_info.req_fec_info & 6626 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 6627 if (vsi->back->hw.phy.link_info.req_fec_info & 6628 I40E_AQ_REQUEST_FEC_RS) 6629 req_fec = "CL108 RS-FEC"; 6630 else 6631 req_fec = "CL74 FC-FEC/BASE-R"; 6632 } 6633 netdev_info(vsi->netdev, 6634 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 6635 speed, req_fec, fec, an, fc); 6636 } else if (pf->hw.device_id == I40E_DEV_ID_KX_X722) { 6637 req_fec = "None"; 6638 fec = "None"; 6639 an = "False"; 6640 6641 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6642 an = "True"; 6643 6644 if (pf->hw.phy.link_info.fec_info & 6645 I40E_AQ_CONFIG_FEC_KR_ENA) 6646 fec = "CL74 FC-FEC/BASE-R"; 6647 6648 if (pf->hw.phy.link_info.req_fec_info & 6649 I40E_AQ_REQUEST_FEC_KR) 6650 req_fec = "CL74 FC-FEC/BASE-R"; 6651 6652 netdev_info(vsi->netdev, 6653 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 6654 speed, req_fec, fec, an, fc); 6655 } else { 6656 netdev_info(vsi->netdev, 6657 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", 6658 speed, fc); 6659 } 6660 6661 } 6662 6663 /** 6664 * i40e_up_complete - Finish the last steps of bringing up a connection 6665 * @vsi: the VSI being configured 6666 **/ 6667 static int i40e_up_complete(struct i40e_vsi *vsi) 6668 { 6669 struct i40e_pf *pf = vsi->back; 6670 int err; 6671 6672 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6673 i40e_vsi_configure_msix(vsi); 6674 else 6675 i40e_configure_msi_and_legacy(vsi); 6676 6677 /* start rings */ 6678 err = i40e_vsi_start_rings(vsi); 6679 if (err) 6680 return err; 6681 6682 clear_bit(__I40E_VSI_DOWN, vsi->state); 6683 i40e_napi_enable_all(vsi); 6684 i40e_vsi_enable_irq(vsi); 6685 6686 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 6687 (vsi->netdev)) { 6688 i40e_print_link_message(vsi, true); 6689 netif_tx_start_all_queues(vsi->netdev); 6690 netif_carrier_on(vsi->netdev); 6691 } 6692 6693 /* replay FDIR SB filters */ 6694 if (vsi->type == I40E_VSI_FDIR) { 6695 /* reset fd counters */ 6696 pf->fd_add_err = 0; 6697 pf->fd_atr_cnt = 0; 6698 i40e_fdir_filter_restore(vsi); 6699 } 6700 6701 /* On the next run of the service_task, notify any clients of the new 6702 * opened netdev 6703 */ 6704 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 6705 i40e_service_event_schedule(pf); 6706 6707 return 0; 6708 } 6709 6710 /** 6711 * i40e_vsi_reinit_locked - Reset the VSI 6712 * @vsi: the VSI being configured 6713 * 6714 * Rebuild the ring structs after some configuration 6715 * has changed, e.g. MTU size. 6716 **/ 6717 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 6718 { 6719 struct i40e_pf *pf = vsi->back; 6720 6721 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 6722 usleep_range(1000, 2000); 6723 i40e_down(vsi); 6724 6725 i40e_up(vsi); 6726 clear_bit(__I40E_CONFIG_BUSY, pf->state); 6727 } 6728 6729 /** 6730 * i40e_force_link_state - Force the link status 6731 * @pf: board private structure 6732 * @is_up: whether the link state should be forced up or down 6733 **/ 6734 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up) 6735 { 6736 struct i40e_aq_get_phy_abilities_resp abilities; 6737 struct i40e_aq_set_phy_config config = {0}; 6738 bool non_zero_phy_type = is_up; 6739 struct i40e_hw *hw = &pf->hw; 6740 i40e_status err; 6741 u64 mask; 6742 u8 speed; 6743 6744 /* Card might've been put in an unstable state by other drivers 6745 * and applications, which causes incorrect speed values being 6746 * set on startup. In order to clear speed registers, we call 6747 * get_phy_capabilities twice, once to get initial state of 6748 * available speeds, and once to get current PHY config. 6749 */ 6750 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 6751 NULL); 6752 if (err) { 6753 dev_err(&pf->pdev->dev, 6754 "failed to get phy cap., ret = %s last_status = %s\n", 6755 i40e_stat_str(hw, err), 6756 i40e_aq_str(hw, hw->aq.asq_last_status)); 6757 return err; 6758 } 6759 speed = abilities.link_speed; 6760 6761 /* Get the current phy config */ 6762 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 6763 NULL); 6764 if (err) { 6765 dev_err(&pf->pdev->dev, 6766 "failed to get phy cap., ret = %s last_status = %s\n", 6767 i40e_stat_str(hw, err), 6768 i40e_aq_str(hw, hw->aq.asq_last_status)); 6769 return err; 6770 } 6771 6772 /* If link needs to go up, but was not forced to go down, 6773 * and its speed values are OK, no need for a flap 6774 * if non_zero_phy_type was set, still need to force up 6775 */ 6776 if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) 6777 non_zero_phy_type = true; 6778 else if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 6779 return I40E_SUCCESS; 6780 6781 /* To force link we need to set bits for all supported PHY types, 6782 * but there are now more than 32, so we need to split the bitmap 6783 * across two fields. 6784 */ 6785 mask = I40E_PHY_TYPES_BITMASK; 6786 config.phy_type = 6787 non_zero_phy_type ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 6788 config.phy_type_ext = 6789 non_zero_phy_type ? (u8)((mask >> 32) & 0xff) : 0; 6790 /* Copy the old settings, except of phy_type */ 6791 config.abilities = abilities.abilities; 6792 if (pf->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED) { 6793 if (is_up) 6794 config.abilities |= I40E_AQ_PHY_ENABLE_LINK; 6795 else 6796 config.abilities &= ~(I40E_AQ_PHY_ENABLE_LINK); 6797 } 6798 if (abilities.link_speed != 0) 6799 config.link_speed = abilities.link_speed; 6800 else 6801 config.link_speed = speed; 6802 config.eee_capability = abilities.eee_capability; 6803 config.eeer = abilities.eeer_val; 6804 config.low_power_ctrl = abilities.d3_lpan; 6805 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 6806 I40E_AQ_PHY_FEC_CONFIG_MASK; 6807 err = i40e_aq_set_phy_config(hw, &config, NULL); 6808 6809 if (err) { 6810 dev_err(&pf->pdev->dev, 6811 "set phy config ret = %s last_status = %s\n", 6812 i40e_stat_str(&pf->hw, err), 6813 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6814 return err; 6815 } 6816 6817 /* Update the link info */ 6818 err = i40e_update_link_info(hw); 6819 if (err) { 6820 /* Wait a little bit (on 40G cards it sometimes takes a really 6821 * long time for link to come back from the atomic reset) 6822 * and try once more 6823 */ 6824 msleep(1000); 6825 i40e_update_link_info(hw); 6826 } 6827 6828 i40e_aq_set_link_restart_an(hw, is_up, NULL); 6829 6830 return I40E_SUCCESS; 6831 } 6832 6833 /** 6834 * i40e_up - Bring the connection back up after being down 6835 * @vsi: the VSI being configured 6836 **/ 6837 int i40e_up(struct i40e_vsi *vsi) 6838 { 6839 int err; 6840 6841 if (vsi->type == I40E_VSI_MAIN && 6842 (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED || 6843 vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)) 6844 i40e_force_link_state(vsi->back, true); 6845 6846 err = i40e_vsi_configure(vsi); 6847 if (!err) 6848 err = i40e_up_complete(vsi); 6849 6850 return err; 6851 } 6852 6853 /** 6854 * i40e_down - Shutdown the connection processing 6855 * @vsi: the VSI being stopped 6856 **/ 6857 void i40e_down(struct i40e_vsi *vsi) 6858 { 6859 int i; 6860 6861 /* It is assumed that the caller of this function 6862 * sets the vsi->state __I40E_VSI_DOWN bit. 6863 */ 6864 if (vsi->netdev) { 6865 netif_carrier_off(vsi->netdev); 6866 netif_tx_disable(vsi->netdev); 6867 } 6868 i40e_vsi_disable_irq(vsi); 6869 i40e_vsi_stop_rings(vsi); 6870 if (vsi->type == I40E_VSI_MAIN && 6871 (vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED || 6872 vsi->back->flags & I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED)) 6873 i40e_force_link_state(vsi->back, false); 6874 i40e_napi_disable_all(vsi); 6875 6876 for (i = 0; i < vsi->num_queue_pairs; i++) { 6877 i40e_clean_tx_ring(vsi->tx_rings[i]); 6878 if (i40e_enabled_xdp_vsi(vsi)) { 6879 /* Make sure that in-progress ndo_xdp_xmit and 6880 * ndo_xsk_wakeup calls are completed. 6881 */ 6882 synchronize_rcu(); 6883 i40e_clean_tx_ring(vsi->xdp_rings[i]); 6884 } 6885 i40e_clean_rx_ring(vsi->rx_rings[i]); 6886 } 6887 6888 } 6889 6890 /** 6891 * i40e_validate_mqprio_qopt- validate queue mapping info 6892 * @vsi: the VSI being configured 6893 * @mqprio_qopt: queue parametrs 6894 **/ 6895 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 6896 struct tc_mqprio_qopt_offload *mqprio_qopt) 6897 { 6898 u64 sum_max_rate = 0; 6899 u64 max_rate = 0; 6900 int i; 6901 6902 if (mqprio_qopt->qopt.offset[0] != 0 || 6903 mqprio_qopt->qopt.num_tc < 1 || 6904 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 6905 return -EINVAL; 6906 for (i = 0; ; i++) { 6907 if (!mqprio_qopt->qopt.count[i]) 6908 return -EINVAL; 6909 if (mqprio_qopt->min_rate[i]) { 6910 dev_err(&vsi->back->pdev->dev, 6911 "Invalid min tx rate (greater than 0) specified\n"); 6912 return -EINVAL; 6913 } 6914 max_rate = mqprio_qopt->max_rate[i]; 6915 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6916 sum_max_rate += max_rate; 6917 6918 if (i >= mqprio_qopt->qopt.num_tc - 1) 6919 break; 6920 if (mqprio_qopt->qopt.offset[i + 1] != 6921 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 6922 return -EINVAL; 6923 } 6924 if (vsi->num_queue_pairs < 6925 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 6926 return -EINVAL; 6927 } 6928 if (sum_max_rate > i40e_get_link_speed(vsi)) { 6929 dev_err(&vsi->back->pdev->dev, 6930 "Invalid max tx rate specified\n"); 6931 return -EINVAL; 6932 } 6933 return 0; 6934 } 6935 6936 /** 6937 * i40e_vsi_set_default_tc_config - set default values for tc configuration 6938 * @vsi: the VSI being configured 6939 **/ 6940 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 6941 { 6942 u16 qcount; 6943 int i; 6944 6945 /* Only TC0 is enabled */ 6946 vsi->tc_config.numtc = 1; 6947 vsi->tc_config.enabled_tc = 1; 6948 qcount = min_t(int, vsi->alloc_queue_pairs, 6949 i40e_pf_get_max_q_per_tc(vsi->back)); 6950 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6951 /* For the TC that is not enabled set the offset to to default 6952 * queue and allocate one queue for the given TC. 6953 */ 6954 vsi->tc_config.tc_info[i].qoffset = 0; 6955 if (i == 0) 6956 vsi->tc_config.tc_info[i].qcount = qcount; 6957 else 6958 vsi->tc_config.tc_info[i].qcount = 1; 6959 vsi->tc_config.tc_info[i].netdev_tc = 0; 6960 } 6961 } 6962 6963 /** 6964 * i40e_del_macvlan_filter 6965 * @hw: pointer to the HW structure 6966 * @seid: seid of the channel VSI 6967 * @macaddr: the mac address to apply as a filter 6968 * @aq_err: store the admin Q error 6969 * 6970 * This function deletes a mac filter on the channel VSI which serves as the 6971 * macvlan. Returns 0 on success. 6972 **/ 6973 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid, 6974 const u8 *macaddr, int *aq_err) 6975 { 6976 struct i40e_aqc_remove_macvlan_element_data element; 6977 i40e_status status; 6978 6979 memset(&element, 0, sizeof(element)); 6980 ether_addr_copy(element.mac_addr, macaddr); 6981 element.vlan_tag = 0; 6982 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 6983 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL); 6984 *aq_err = hw->aq.asq_last_status; 6985 6986 return status; 6987 } 6988 6989 /** 6990 * i40e_add_macvlan_filter 6991 * @hw: pointer to the HW structure 6992 * @seid: seid of the channel VSI 6993 * @macaddr: the mac address to apply as a filter 6994 * @aq_err: store the admin Q error 6995 * 6996 * This function adds a mac filter on the channel VSI which serves as the 6997 * macvlan. Returns 0 on success. 6998 **/ 6999 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid, 7000 const u8 *macaddr, int *aq_err) 7001 { 7002 struct i40e_aqc_add_macvlan_element_data element; 7003 i40e_status status; 7004 u16 cmd_flags = 0; 7005 7006 ether_addr_copy(element.mac_addr, macaddr); 7007 element.vlan_tag = 0; 7008 element.queue_number = 0; 7009 element.match_method = I40E_AQC_MM_ERR_NO_RES; 7010 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 7011 element.flags = cpu_to_le16(cmd_flags); 7012 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL); 7013 *aq_err = hw->aq.asq_last_status; 7014 7015 return status; 7016 } 7017 7018 /** 7019 * i40e_reset_ch_rings - Reset the queue contexts in a channel 7020 * @vsi: the VSI we want to access 7021 * @ch: the channel we want to access 7022 */ 7023 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch) 7024 { 7025 struct i40e_ring *tx_ring, *rx_ring; 7026 u16 pf_q; 7027 int i; 7028 7029 for (i = 0; i < ch->num_queue_pairs; i++) { 7030 pf_q = ch->base_queue + i; 7031 tx_ring = vsi->tx_rings[pf_q]; 7032 tx_ring->ch = NULL; 7033 rx_ring = vsi->rx_rings[pf_q]; 7034 rx_ring->ch = NULL; 7035 } 7036 } 7037 7038 /** 7039 * i40e_free_macvlan_channels 7040 * @vsi: the VSI we want to access 7041 * 7042 * This function frees the Qs of the channel VSI from 7043 * the stack and also deletes the channel VSIs which 7044 * serve as macvlans. 7045 */ 7046 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi) 7047 { 7048 struct i40e_channel *ch, *ch_tmp; 7049 int ret; 7050 7051 if (list_empty(&vsi->macvlan_list)) 7052 return; 7053 7054 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7055 struct i40e_vsi *parent_vsi; 7056 7057 if (i40e_is_channel_macvlan(ch)) { 7058 i40e_reset_ch_rings(vsi, ch); 7059 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7060 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev); 7061 netdev_set_sb_channel(ch->fwd->netdev, 0); 7062 kfree(ch->fwd); 7063 ch->fwd = NULL; 7064 } 7065 7066 list_del(&ch->list); 7067 parent_vsi = ch->parent_vsi; 7068 if (!parent_vsi || !ch->initialized) { 7069 kfree(ch); 7070 continue; 7071 } 7072 7073 /* remove the VSI */ 7074 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 7075 NULL); 7076 if (ret) 7077 dev_err(&vsi->back->pdev->dev, 7078 "unable to remove channel (%d) for parent VSI(%d)\n", 7079 ch->seid, parent_vsi->seid); 7080 kfree(ch); 7081 } 7082 vsi->macvlan_cnt = 0; 7083 } 7084 7085 /** 7086 * i40e_fwd_ring_up - bring the macvlan device up 7087 * @vsi: the VSI we want to access 7088 * @vdev: macvlan netdevice 7089 * @fwd: the private fwd structure 7090 */ 7091 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev, 7092 struct i40e_fwd_adapter *fwd) 7093 { 7094 int ret = 0, num_tc = 1, i, aq_err; 7095 struct i40e_channel *ch, *ch_tmp; 7096 struct i40e_pf *pf = vsi->back; 7097 struct i40e_hw *hw = &pf->hw; 7098 7099 if (list_empty(&vsi->macvlan_list)) 7100 return -EINVAL; 7101 7102 /* Go through the list and find an available channel */ 7103 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7104 if (!i40e_is_channel_macvlan(ch)) { 7105 ch->fwd = fwd; 7106 /* record configuration for macvlan interface in vdev */ 7107 for (i = 0; i < num_tc; i++) 7108 netdev_bind_sb_channel_queue(vsi->netdev, vdev, 7109 i, 7110 ch->num_queue_pairs, 7111 ch->base_queue); 7112 for (i = 0; i < ch->num_queue_pairs; i++) { 7113 struct i40e_ring *tx_ring, *rx_ring; 7114 u16 pf_q; 7115 7116 pf_q = ch->base_queue + i; 7117 7118 /* Get to TX ring ptr */ 7119 tx_ring = vsi->tx_rings[pf_q]; 7120 tx_ring->ch = ch; 7121 7122 /* Get the RX ring ptr */ 7123 rx_ring = vsi->rx_rings[pf_q]; 7124 rx_ring->ch = ch; 7125 } 7126 break; 7127 } 7128 } 7129 7130 /* Guarantee all rings are updated before we update the 7131 * MAC address filter. 7132 */ 7133 wmb(); 7134 7135 /* Add a mac filter */ 7136 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err); 7137 if (ret) { 7138 /* if we cannot add the MAC rule then disable the offload */ 7139 macvlan_release_l2fw_offload(vdev); 7140 for (i = 0; i < ch->num_queue_pairs; i++) { 7141 struct i40e_ring *rx_ring; 7142 u16 pf_q; 7143 7144 pf_q = ch->base_queue + i; 7145 rx_ring = vsi->rx_rings[pf_q]; 7146 rx_ring->netdev = NULL; 7147 } 7148 dev_info(&pf->pdev->dev, 7149 "Error adding mac filter on macvlan err %s, aq_err %s\n", 7150 i40e_stat_str(hw, ret), 7151 i40e_aq_str(hw, aq_err)); 7152 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n"); 7153 } 7154 7155 return ret; 7156 } 7157 7158 /** 7159 * i40e_setup_macvlans - create the channels which will be macvlans 7160 * @vsi: the VSI we want to access 7161 * @macvlan_cnt: no. of macvlans to be setup 7162 * @qcnt: no. of Qs per macvlan 7163 * @vdev: macvlan netdevice 7164 */ 7165 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt, 7166 struct net_device *vdev) 7167 { 7168 struct i40e_pf *pf = vsi->back; 7169 struct i40e_hw *hw = &pf->hw; 7170 struct i40e_vsi_context ctxt; 7171 u16 sections, qmap, num_qps; 7172 struct i40e_channel *ch; 7173 int i, pow, ret = 0; 7174 u8 offset = 0; 7175 7176 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt) 7177 return -EINVAL; 7178 7179 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt); 7180 7181 /* find the next higher power-of-2 of num queue pairs */ 7182 pow = fls(roundup_pow_of_two(num_qps) - 1); 7183 7184 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 7185 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 7186 7187 /* Setup context bits for the main VSI */ 7188 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 7189 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 7190 memset(&ctxt, 0, sizeof(ctxt)); 7191 ctxt.seid = vsi->seid; 7192 ctxt.pf_num = vsi->back->hw.pf_id; 7193 ctxt.vf_num = 0; 7194 ctxt.uplink_seid = vsi->uplink_seid; 7195 ctxt.info = vsi->info; 7196 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap); 7197 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 7198 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 7199 ctxt.info.valid_sections |= cpu_to_le16(sections); 7200 7201 /* Reconfigure RSS for main VSI with new max queue count */ 7202 vsi->rss_size = max_t(u16, num_qps, qcnt); 7203 ret = i40e_vsi_config_rss(vsi); 7204 if (ret) { 7205 dev_info(&pf->pdev->dev, 7206 "Failed to reconfig RSS for num_queues (%u)\n", 7207 vsi->rss_size); 7208 return ret; 7209 } 7210 vsi->reconfig_rss = true; 7211 dev_dbg(&vsi->back->pdev->dev, 7212 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size); 7213 vsi->next_base_queue = num_qps; 7214 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps; 7215 7216 /* Update the VSI after updating the VSI queue-mapping 7217 * information 7218 */ 7219 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7220 if (ret) { 7221 dev_info(&pf->pdev->dev, 7222 "Update vsi tc config failed, err %s aq_err %s\n", 7223 i40e_stat_str(hw, ret), 7224 i40e_aq_str(hw, hw->aq.asq_last_status)); 7225 return ret; 7226 } 7227 /* update the local VSI info with updated queue map */ 7228 i40e_vsi_update_queue_map(vsi, &ctxt); 7229 vsi->info.valid_sections = 0; 7230 7231 /* Create channels for macvlans */ 7232 INIT_LIST_HEAD(&vsi->macvlan_list); 7233 for (i = 0; i < macvlan_cnt; i++) { 7234 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 7235 if (!ch) { 7236 ret = -ENOMEM; 7237 goto err_free; 7238 } 7239 INIT_LIST_HEAD(&ch->list); 7240 ch->num_queue_pairs = qcnt; 7241 if (!i40e_setup_channel(pf, vsi, ch)) { 7242 ret = -EINVAL; 7243 kfree(ch); 7244 goto err_free; 7245 } 7246 ch->parent_vsi = vsi; 7247 vsi->cnt_q_avail -= ch->num_queue_pairs; 7248 vsi->macvlan_cnt++; 7249 list_add_tail(&ch->list, &vsi->macvlan_list); 7250 } 7251 7252 return ret; 7253 7254 err_free: 7255 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n"); 7256 i40e_free_macvlan_channels(vsi); 7257 7258 return ret; 7259 } 7260 7261 /** 7262 * i40e_fwd_add - configure macvlans 7263 * @netdev: net device to configure 7264 * @vdev: macvlan netdevice 7265 **/ 7266 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) 7267 { 7268 struct i40e_netdev_priv *np = netdev_priv(netdev); 7269 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors; 7270 struct i40e_vsi *vsi = np->vsi; 7271 struct i40e_pf *pf = vsi->back; 7272 struct i40e_fwd_adapter *fwd; 7273 int avail_macvlan, ret; 7274 7275 if ((pf->flags & I40E_FLAG_DCB_ENABLED)) { 7276 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); 7277 return ERR_PTR(-EINVAL); 7278 } 7279 if ((pf->flags & I40E_FLAG_TC_MQPRIO)) { 7280 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); 7281 return ERR_PTR(-EINVAL); 7282 } 7283 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) { 7284 netdev_info(netdev, "Not enough vectors available to support macvlans\n"); 7285 return ERR_PTR(-EINVAL); 7286 } 7287 7288 /* The macvlan device has to be a single Q device so that the 7289 * tc_to_txq field can be reused to pick the tx queue. 7290 */ 7291 if (netif_is_multiqueue(vdev)) 7292 return ERR_PTR(-ERANGE); 7293 7294 if (!vsi->macvlan_cnt) { 7295 /* reserve bit 0 for the pf device */ 7296 set_bit(0, vsi->fwd_bitmask); 7297 7298 /* Try to reserve as many queues as possible for macvlans. First 7299 * reserve 3/4th of max vectors, then half, then quarter and 7300 * calculate Qs per macvlan as you go 7301 */ 7302 vectors = pf->num_lan_msix; 7303 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) { 7304 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/ 7305 q_per_macvlan = 4; 7306 macvlan_cnt = (vectors - 32) / 4; 7307 } else if (vectors <= 64 && vectors > 32) { 7308 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/ 7309 q_per_macvlan = 2; 7310 macvlan_cnt = (vectors - 16) / 2; 7311 } else if (vectors <= 32 && vectors > 16) { 7312 /* allocate 1 Q per macvlan and 16 Qs to the PF*/ 7313 q_per_macvlan = 1; 7314 macvlan_cnt = vectors - 16; 7315 } else if (vectors <= 16 && vectors > 8) { 7316 /* allocate 1 Q per macvlan and 8 Qs to the PF */ 7317 q_per_macvlan = 1; 7318 macvlan_cnt = vectors - 8; 7319 } else { 7320 /* allocate 1 Q per macvlan and 1 Q to the PF */ 7321 q_per_macvlan = 1; 7322 macvlan_cnt = vectors - 1; 7323 } 7324 7325 if (macvlan_cnt == 0) 7326 return ERR_PTR(-EBUSY); 7327 7328 /* Quiesce VSI queues */ 7329 i40e_quiesce_vsi(vsi); 7330 7331 /* sets up the macvlans but does not "enable" them */ 7332 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan, 7333 vdev); 7334 if (ret) 7335 return ERR_PTR(ret); 7336 7337 /* Unquiesce VSI */ 7338 i40e_unquiesce_vsi(vsi); 7339 } 7340 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask, 7341 vsi->macvlan_cnt); 7342 if (avail_macvlan >= I40E_MAX_MACVLANS) 7343 return ERR_PTR(-EBUSY); 7344 7345 /* create the fwd struct */ 7346 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL); 7347 if (!fwd) 7348 return ERR_PTR(-ENOMEM); 7349 7350 set_bit(avail_macvlan, vsi->fwd_bitmask); 7351 fwd->bit_no = avail_macvlan; 7352 netdev_set_sb_channel(vdev, avail_macvlan); 7353 fwd->netdev = vdev; 7354 7355 if (!netif_running(netdev)) 7356 return fwd; 7357 7358 /* Set fwd ring up */ 7359 ret = i40e_fwd_ring_up(vsi, vdev, fwd); 7360 if (ret) { 7361 /* unbind the queues and drop the subordinate channel config */ 7362 netdev_unbind_sb_channel(netdev, vdev); 7363 netdev_set_sb_channel(vdev, 0); 7364 7365 kfree(fwd); 7366 return ERR_PTR(-EINVAL); 7367 } 7368 7369 return fwd; 7370 } 7371 7372 /** 7373 * i40e_del_all_macvlans - Delete all the mac filters on the channels 7374 * @vsi: the VSI we want to access 7375 */ 7376 static void i40e_del_all_macvlans(struct i40e_vsi *vsi) 7377 { 7378 struct i40e_channel *ch, *ch_tmp; 7379 struct i40e_pf *pf = vsi->back; 7380 struct i40e_hw *hw = &pf->hw; 7381 int aq_err, ret = 0; 7382 7383 if (list_empty(&vsi->macvlan_list)) 7384 return; 7385 7386 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7387 if (i40e_is_channel_macvlan(ch)) { 7388 ret = i40e_del_macvlan_filter(hw, ch->seid, 7389 i40e_channel_mac(ch), 7390 &aq_err); 7391 if (!ret) { 7392 /* Reset queue contexts */ 7393 i40e_reset_ch_rings(vsi, ch); 7394 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7395 netdev_unbind_sb_channel(vsi->netdev, 7396 ch->fwd->netdev); 7397 netdev_set_sb_channel(ch->fwd->netdev, 0); 7398 kfree(ch->fwd); 7399 ch->fwd = NULL; 7400 } 7401 } 7402 } 7403 } 7404 7405 /** 7406 * i40e_fwd_del - delete macvlan interfaces 7407 * @netdev: net device to configure 7408 * @vdev: macvlan netdevice 7409 */ 7410 static void i40e_fwd_del(struct net_device *netdev, void *vdev) 7411 { 7412 struct i40e_netdev_priv *np = netdev_priv(netdev); 7413 struct i40e_fwd_adapter *fwd = vdev; 7414 struct i40e_channel *ch, *ch_tmp; 7415 struct i40e_vsi *vsi = np->vsi; 7416 struct i40e_pf *pf = vsi->back; 7417 struct i40e_hw *hw = &pf->hw; 7418 int aq_err, ret = 0; 7419 7420 /* Find the channel associated with the macvlan and del mac filter */ 7421 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7422 if (i40e_is_channel_macvlan(ch) && 7423 ether_addr_equal(i40e_channel_mac(ch), 7424 fwd->netdev->dev_addr)) { 7425 ret = i40e_del_macvlan_filter(hw, ch->seid, 7426 i40e_channel_mac(ch), 7427 &aq_err); 7428 if (!ret) { 7429 /* Reset queue contexts */ 7430 i40e_reset_ch_rings(vsi, ch); 7431 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7432 netdev_unbind_sb_channel(netdev, fwd->netdev); 7433 netdev_set_sb_channel(fwd->netdev, 0); 7434 kfree(ch->fwd); 7435 ch->fwd = NULL; 7436 } else { 7437 dev_info(&pf->pdev->dev, 7438 "Error deleting mac filter on macvlan err %s, aq_err %s\n", 7439 i40e_stat_str(hw, ret), 7440 i40e_aq_str(hw, aq_err)); 7441 } 7442 break; 7443 } 7444 } 7445 } 7446 7447 /** 7448 * i40e_setup_tc - configure multiple traffic classes 7449 * @netdev: net device to configure 7450 * @type_data: tc offload data 7451 **/ 7452 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 7453 { 7454 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 7455 struct i40e_netdev_priv *np = netdev_priv(netdev); 7456 struct i40e_vsi *vsi = np->vsi; 7457 struct i40e_pf *pf = vsi->back; 7458 u8 enabled_tc = 0, num_tc, hw; 7459 bool need_reset = false; 7460 int old_queue_pairs; 7461 int ret = -EINVAL; 7462 u16 mode; 7463 int i; 7464 7465 old_queue_pairs = vsi->num_queue_pairs; 7466 num_tc = mqprio_qopt->qopt.num_tc; 7467 hw = mqprio_qopt->qopt.hw; 7468 mode = mqprio_qopt->mode; 7469 if (!hw) { 7470 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 7471 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 7472 goto config_tc; 7473 } 7474 7475 /* Check if MFP enabled */ 7476 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 7477 netdev_info(netdev, 7478 "Configuring TC not supported in MFP mode\n"); 7479 return ret; 7480 } 7481 switch (mode) { 7482 case TC_MQPRIO_MODE_DCB: 7483 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 7484 7485 /* Check if DCB enabled to continue */ 7486 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 7487 netdev_info(netdev, 7488 "DCB is not enabled for adapter\n"); 7489 return ret; 7490 } 7491 7492 /* Check whether tc count is within enabled limit */ 7493 if (num_tc > i40e_pf_get_num_tc(pf)) { 7494 netdev_info(netdev, 7495 "TC count greater than enabled on link for adapter\n"); 7496 return ret; 7497 } 7498 break; 7499 case TC_MQPRIO_MODE_CHANNEL: 7500 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 7501 netdev_info(netdev, 7502 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 7503 return ret; 7504 } 7505 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 7506 return ret; 7507 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 7508 if (ret) 7509 return ret; 7510 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 7511 sizeof(*mqprio_qopt)); 7512 pf->flags |= I40E_FLAG_TC_MQPRIO; 7513 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 7514 break; 7515 default: 7516 return -EINVAL; 7517 } 7518 7519 config_tc: 7520 /* Generate TC map for number of tc requested */ 7521 for (i = 0; i < num_tc; i++) 7522 enabled_tc |= BIT(i); 7523 7524 /* Requesting same TC configuration as already enabled */ 7525 if (enabled_tc == vsi->tc_config.enabled_tc && 7526 mode != TC_MQPRIO_MODE_CHANNEL) 7527 return 0; 7528 7529 /* Quiesce VSI queues */ 7530 i40e_quiesce_vsi(vsi); 7531 7532 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) 7533 i40e_remove_queue_channels(vsi); 7534 7535 /* Configure VSI for enabled TCs */ 7536 ret = i40e_vsi_config_tc(vsi, enabled_tc); 7537 if (ret) { 7538 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 7539 vsi->seid); 7540 need_reset = true; 7541 goto exit; 7542 } else { 7543 dev_info(&vsi->back->pdev->dev, 7544 "Setup channel (id:%u) utilizing num_queues %d\n", 7545 vsi->seid, vsi->tc_config.tc_info[0].qcount); 7546 } 7547 7548 if (pf->flags & I40E_FLAG_TC_MQPRIO) { 7549 if (vsi->mqprio_qopt.max_rate[0]) { 7550 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 7551 7552 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 7553 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 7554 if (!ret) { 7555 u64 credits = max_tx_rate; 7556 7557 do_div(credits, I40E_BW_CREDIT_DIVISOR); 7558 dev_dbg(&vsi->back->pdev->dev, 7559 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 7560 max_tx_rate, 7561 credits, 7562 vsi->seid); 7563 } else { 7564 need_reset = true; 7565 goto exit; 7566 } 7567 } 7568 ret = i40e_configure_queue_channels(vsi); 7569 if (ret) { 7570 vsi->num_queue_pairs = old_queue_pairs; 7571 netdev_info(netdev, 7572 "Failed configuring queue channels\n"); 7573 need_reset = true; 7574 goto exit; 7575 } 7576 } 7577 7578 exit: 7579 /* Reset the configuration data to defaults, only TC0 is enabled */ 7580 if (need_reset) { 7581 i40e_vsi_set_default_tc_config(vsi); 7582 need_reset = false; 7583 } 7584 7585 /* Unquiesce VSI */ 7586 i40e_unquiesce_vsi(vsi); 7587 return ret; 7588 } 7589 7590 /** 7591 * i40e_set_cld_element - sets cloud filter element data 7592 * @filter: cloud filter rule 7593 * @cld: ptr to cloud filter element data 7594 * 7595 * This is helper function to copy data into cloud filter element 7596 **/ 7597 static inline void 7598 i40e_set_cld_element(struct i40e_cloud_filter *filter, 7599 struct i40e_aqc_cloud_filters_element_data *cld) 7600 { 7601 int i, j; 7602 u32 ipa; 7603 7604 memset(cld, 0, sizeof(*cld)); 7605 ether_addr_copy(cld->outer_mac, filter->dst_mac); 7606 ether_addr_copy(cld->inner_mac, filter->src_mac); 7607 7608 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 7609 return; 7610 7611 if (filter->n_proto == ETH_P_IPV6) { 7612 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 7613 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); 7614 i++, j += 2) { 7615 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 7616 ipa = cpu_to_le32(ipa); 7617 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); 7618 } 7619 } else { 7620 ipa = be32_to_cpu(filter->dst_ipv4); 7621 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 7622 } 7623 7624 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 7625 7626 /* tenant_id is not supported by FW now, once the support is enabled 7627 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 7628 */ 7629 if (filter->tenant_id) 7630 return; 7631 } 7632 7633 /** 7634 * i40e_add_del_cloud_filter - Add/del cloud filter 7635 * @vsi: pointer to VSI 7636 * @filter: cloud filter rule 7637 * @add: if true, add, if false, delete 7638 * 7639 * Add or delete a cloud filter for a specific flow spec. 7640 * Returns 0 if the filter were successfully added. 7641 **/ 7642 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 7643 struct i40e_cloud_filter *filter, bool add) 7644 { 7645 struct i40e_aqc_cloud_filters_element_data cld_filter; 7646 struct i40e_pf *pf = vsi->back; 7647 int ret; 7648 static const u16 flag_table[128] = { 7649 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 7650 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 7651 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 7652 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 7653 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 7654 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 7655 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 7656 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 7657 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 7658 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 7659 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 7660 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 7661 [I40E_CLOUD_FILTER_FLAGS_IIP] = 7662 I40E_AQC_ADD_CLOUD_FILTER_IIP, 7663 }; 7664 7665 if (filter->flags >= ARRAY_SIZE(flag_table)) 7666 return I40E_ERR_CONFIG; 7667 7668 /* copy element needed to add cloud filter from filter */ 7669 i40e_set_cld_element(filter, &cld_filter); 7670 7671 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 7672 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 7673 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 7674 7675 if (filter->n_proto == ETH_P_IPV6) 7676 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7677 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7678 else 7679 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7680 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7681 7682 if (add) 7683 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 7684 &cld_filter, 1); 7685 else 7686 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 7687 &cld_filter, 1); 7688 if (ret) 7689 dev_dbg(&pf->pdev->dev, 7690 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 7691 add ? "add" : "delete", filter->dst_port, ret, 7692 pf->hw.aq.asq_last_status); 7693 else 7694 dev_info(&pf->pdev->dev, 7695 "%s cloud filter for VSI: %d\n", 7696 add ? "Added" : "Deleted", filter->seid); 7697 return ret; 7698 } 7699 7700 /** 7701 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 7702 * @vsi: pointer to VSI 7703 * @filter: cloud filter rule 7704 * @add: if true, add, if false, delete 7705 * 7706 * Add or delete a cloud filter for a specific flow spec using big buffer. 7707 * Returns 0 if the filter were successfully added. 7708 **/ 7709 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 7710 struct i40e_cloud_filter *filter, 7711 bool add) 7712 { 7713 struct i40e_aqc_cloud_filters_element_bb cld_filter; 7714 struct i40e_pf *pf = vsi->back; 7715 int ret; 7716 7717 /* Both (src/dst) valid mac_addr are not supported */ 7718 if ((is_valid_ether_addr(filter->dst_mac) && 7719 is_valid_ether_addr(filter->src_mac)) || 7720 (is_multicast_ether_addr(filter->dst_mac) && 7721 is_multicast_ether_addr(filter->src_mac))) 7722 return -EOPNOTSUPP; 7723 7724 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 7725 * ports are not supported via big buffer now. 7726 */ 7727 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 7728 return -EOPNOTSUPP; 7729 7730 /* adding filter using src_port/src_ip is not supported at this stage */ 7731 if (filter->src_port || filter->src_ipv4 || 7732 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 7733 return -EOPNOTSUPP; 7734 7735 /* copy element needed to add cloud filter from filter */ 7736 i40e_set_cld_element(filter, &cld_filter.element); 7737 7738 if (is_valid_ether_addr(filter->dst_mac) || 7739 is_valid_ether_addr(filter->src_mac) || 7740 is_multicast_ether_addr(filter->dst_mac) || 7741 is_multicast_ether_addr(filter->src_mac)) { 7742 /* MAC + IP : unsupported mode */ 7743 if (filter->dst_ipv4) 7744 return -EOPNOTSUPP; 7745 7746 /* since we validated that L4 port must be valid before 7747 * we get here, start with respective "flags" value 7748 * and update if vlan is present or not 7749 */ 7750 cld_filter.element.flags = 7751 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 7752 7753 if (filter->vlan_id) { 7754 cld_filter.element.flags = 7755 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 7756 } 7757 7758 } else if (filter->dst_ipv4 || 7759 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 7760 cld_filter.element.flags = 7761 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 7762 if (filter->n_proto == ETH_P_IPV6) 7763 cld_filter.element.flags |= 7764 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7765 else 7766 cld_filter.element.flags |= 7767 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7768 } else { 7769 dev_err(&pf->pdev->dev, 7770 "either mac or ip has to be valid for cloud filter\n"); 7771 return -EINVAL; 7772 } 7773 7774 /* Now copy L4 port in Byte 6..7 in general fields */ 7775 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 7776 be16_to_cpu(filter->dst_port); 7777 7778 if (add) { 7779 /* Validate current device switch mode, change if necessary */ 7780 ret = i40e_validate_and_set_switch_mode(vsi); 7781 if (ret) { 7782 dev_err(&pf->pdev->dev, 7783 "failed to set switch mode, ret %d\n", 7784 ret); 7785 return ret; 7786 } 7787 7788 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 7789 &cld_filter, 1); 7790 } else { 7791 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 7792 &cld_filter, 1); 7793 } 7794 7795 if (ret) 7796 dev_dbg(&pf->pdev->dev, 7797 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 7798 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 7799 else 7800 dev_info(&pf->pdev->dev, 7801 "%s cloud filter for VSI: %d, L4 port: %d\n", 7802 add ? "add" : "delete", filter->seid, 7803 ntohs(filter->dst_port)); 7804 return ret; 7805 } 7806 7807 /** 7808 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 7809 * @vsi: Pointer to VSI 7810 * @f: Pointer to struct flow_cls_offload 7811 * @filter: Pointer to cloud filter structure 7812 * 7813 **/ 7814 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 7815 struct flow_cls_offload *f, 7816 struct i40e_cloud_filter *filter) 7817 { 7818 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 7819 struct flow_dissector *dissector = rule->match.dissector; 7820 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 7821 struct i40e_pf *pf = vsi->back; 7822 u8 field_flags = 0; 7823 7824 if (dissector->used_keys & 7825 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 7826 BIT(FLOW_DISSECTOR_KEY_BASIC) | 7827 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 7828 BIT(FLOW_DISSECTOR_KEY_VLAN) | 7829 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 7830 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 7831 BIT(FLOW_DISSECTOR_KEY_PORTS) | 7832 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 7833 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 7834 dissector->used_keys); 7835 return -EOPNOTSUPP; 7836 } 7837 7838 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 7839 struct flow_match_enc_keyid match; 7840 7841 flow_rule_match_enc_keyid(rule, &match); 7842 if (match.mask->keyid != 0) 7843 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 7844 7845 filter->tenant_id = be32_to_cpu(match.key->keyid); 7846 } 7847 7848 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 7849 struct flow_match_basic match; 7850 7851 flow_rule_match_basic(rule, &match); 7852 n_proto_key = ntohs(match.key->n_proto); 7853 n_proto_mask = ntohs(match.mask->n_proto); 7854 7855 if (n_proto_key == ETH_P_ALL) { 7856 n_proto_key = 0; 7857 n_proto_mask = 0; 7858 } 7859 filter->n_proto = n_proto_key & n_proto_mask; 7860 filter->ip_proto = match.key->ip_proto; 7861 } 7862 7863 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 7864 struct flow_match_eth_addrs match; 7865 7866 flow_rule_match_eth_addrs(rule, &match); 7867 7868 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 7869 if (!is_zero_ether_addr(match.mask->dst)) { 7870 if (is_broadcast_ether_addr(match.mask->dst)) { 7871 field_flags |= I40E_CLOUD_FIELD_OMAC; 7872 } else { 7873 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 7874 match.mask->dst); 7875 return I40E_ERR_CONFIG; 7876 } 7877 } 7878 7879 if (!is_zero_ether_addr(match.mask->src)) { 7880 if (is_broadcast_ether_addr(match.mask->src)) { 7881 field_flags |= I40E_CLOUD_FIELD_IMAC; 7882 } else { 7883 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 7884 match.mask->src); 7885 return I40E_ERR_CONFIG; 7886 } 7887 } 7888 ether_addr_copy(filter->dst_mac, match.key->dst); 7889 ether_addr_copy(filter->src_mac, match.key->src); 7890 } 7891 7892 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 7893 struct flow_match_vlan match; 7894 7895 flow_rule_match_vlan(rule, &match); 7896 if (match.mask->vlan_id) { 7897 if (match.mask->vlan_id == VLAN_VID_MASK) { 7898 field_flags |= I40E_CLOUD_FIELD_IVLAN; 7899 7900 } else { 7901 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 7902 match.mask->vlan_id); 7903 return I40E_ERR_CONFIG; 7904 } 7905 } 7906 7907 filter->vlan_id = cpu_to_be16(match.key->vlan_id); 7908 } 7909 7910 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 7911 struct flow_match_control match; 7912 7913 flow_rule_match_control(rule, &match); 7914 addr_type = match.key->addr_type; 7915 } 7916 7917 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 7918 struct flow_match_ipv4_addrs match; 7919 7920 flow_rule_match_ipv4_addrs(rule, &match); 7921 if (match.mask->dst) { 7922 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 7923 field_flags |= I40E_CLOUD_FIELD_IIP; 7924 } else { 7925 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 7926 &match.mask->dst); 7927 return I40E_ERR_CONFIG; 7928 } 7929 } 7930 7931 if (match.mask->src) { 7932 if (match.mask->src == cpu_to_be32(0xffffffff)) { 7933 field_flags |= I40E_CLOUD_FIELD_IIP; 7934 } else { 7935 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 7936 &match.mask->src); 7937 return I40E_ERR_CONFIG; 7938 } 7939 } 7940 7941 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 7942 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 7943 return I40E_ERR_CONFIG; 7944 } 7945 filter->dst_ipv4 = match.key->dst; 7946 filter->src_ipv4 = match.key->src; 7947 } 7948 7949 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 7950 struct flow_match_ipv6_addrs match; 7951 7952 flow_rule_match_ipv6_addrs(rule, &match); 7953 7954 /* src and dest IPV6 address should not be LOOPBACK 7955 * (0:0:0:0:0:0:0:1), which can be represented as ::1 7956 */ 7957 if (ipv6_addr_loopback(&match.key->dst) || 7958 ipv6_addr_loopback(&match.key->src)) { 7959 dev_err(&pf->pdev->dev, 7960 "Bad ipv6, addr is LOOPBACK\n"); 7961 return I40E_ERR_CONFIG; 7962 } 7963 if (!ipv6_addr_any(&match.mask->dst) || 7964 !ipv6_addr_any(&match.mask->src)) 7965 field_flags |= I40E_CLOUD_FIELD_IIP; 7966 7967 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32, 7968 sizeof(filter->src_ipv6)); 7969 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32, 7970 sizeof(filter->dst_ipv6)); 7971 } 7972 7973 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 7974 struct flow_match_ports match; 7975 7976 flow_rule_match_ports(rule, &match); 7977 if (match.mask->src) { 7978 if (match.mask->src == cpu_to_be16(0xffff)) { 7979 field_flags |= I40E_CLOUD_FIELD_IIP; 7980 } else { 7981 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 7982 be16_to_cpu(match.mask->src)); 7983 return I40E_ERR_CONFIG; 7984 } 7985 } 7986 7987 if (match.mask->dst) { 7988 if (match.mask->dst == cpu_to_be16(0xffff)) { 7989 field_flags |= I40E_CLOUD_FIELD_IIP; 7990 } else { 7991 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 7992 be16_to_cpu(match.mask->dst)); 7993 return I40E_ERR_CONFIG; 7994 } 7995 } 7996 7997 filter->dst_port = match.key->dst; 7998 filter->src_port = match.key->src; 7999 8000 switch (filter->ip_proto) { 8001 case IPPROTO_TCP: 8002 case IPPROTO_UDP: 8003 break; 8004 default: 8005 dev_err(&pf->pdev->dev, 8006 "Only UDP and TCP transport are supported\n"); 8007 return -EINVAL; 8008 } 8009 } 8010 filter->flags = field_flags; 8011 return 0; 8012 } 8013 8014 /** 8015 * i40e_handle_tclass: Forward to a traffic class on the device 8016 * @vsi: Pointer to VSI 8017 * @tc: traffic class index on the device 8018 * @filter: Pointer to cloud filter structure 8019 * 8020 **/ 8021 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 8022 struct i40e_cloud_filter *filter) 8023 { 8024 struct i40e_channel *ch, *ch_tmp; 8025 8026 /* direct to a traffic class on the same device */ 8027 if (tc == 0) { 8028 filter->seid = vsi->seid; 8029 return 0; 8030 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 8031 if (!filter->dst_port) { 8032 dev_err(&vsi->back->pdev->dev, 8033 "Specify destination port to direct to traffic class that is not default\n"); 8034 return -EINVAL; 8035 } 8036 if (list_empty(&vsi->ch_list)) 8037 return -EINVAL; 8038 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 8039 list) { 8040 if (ch->seid == vsi->tc_seid_map[tc]) 8041 filter->seid = ch->seid; 8042 } 8043 return 0; 8044 } 8045 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 8046 return -EINVAL; 8047 } 8048 8049 /** 8050 * i40e_configure_clsflower - Configure tc flower filters 8051 * @vsi: Pointer to VSI 8052 * @cls_flower: Pointer to struct flow_cls_offload 8053 * 8054 **/ 8055 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 8056 struct flow_cls_offload *cls_flower) 8057 { 8058 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 8059 struct i40e_cloud_filter *filter = NULL; 8060 struct i40e_pf *pf = vsi->back; 8061 int err = 0; 8062 8063 if (tc < 0) { 8064 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 8065 return -EOPNOTSUPP; 8066 } 8067 8068 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 8069 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 8070 return -EBUSY; 8071 8072 if (pf->fdir_pf_active_filters || 8073 (!hlist_empty(&pf->fdir_filter_list))) { 8074 dev_err(&vsi->back->pdev->dev, 8075 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 8076 return -EINVAL; 8077 } 8078 8079 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 8080 dev_err(&vsi->back->pdev->dev, 8081 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 8082 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8083 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8084 } 8085 8086 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 8087 if (!filter) 8088 return -ENOMEM; 8089 8090 filter->cookie = cls_flower->cookie; 8091 8092 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 8093 if (err < 0) 8094 goto err; 8095 8096 err = i40e_handle_tclass(vsi, tc, filter); 8097 if (err < 0) 8098 goto err; 8099 8100 /* Add cloud filter */ 8101 if (filter->dst_port) 8102 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 8103 else 8104 err = i40e_add_del_cloud_filter(vsi, filter, true); 8105 8106 if (err) { 8107 dev_err(&pf->pdev->dev, 8108 "Failed to add cloud filter, err %s\n", 8109 i40e_stat_str(&pf->hw, err)); 8110 goto err; 8111 } 8112 8113 /* add filter to the ordered list */ 8114 INIT_HLIST_NODE(&filter->cloud_node); 8115 8116 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 8117 8118 pf->num_cloud_filters++; 8119 8120 return err; 8121 err: 8122 kfree(filter); 8123 return err; 8124 } 8125 8126 /** 8127 * i40e_find_cloud_filter - Find the could filter in the list 8128 * @vsi: Pointer to VSI 8129 * @cookie: filter specific cookie 8130 * 8131 **/ 8132 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 8133 unsigned long *cookie) 8134 { 8135 struct i40e_cloud_filter *filter = NULL; 8136 struct hlist_node *node2; 8137 8138 hlist_for_each_entry_safe(filter, node2, 8139 &vsi->back->cloud_filter_list, cloud_node) 8140 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 8141 return filter; 8142 return NULL; 8143 } 8144 8145 /** 8146 * i40e_delete_clsflower - Remove tc flower filters 8147 * @vsi: Pointer to VSI 8148 * @cls_flower: Pointer to struct flow_cls_offload 8149 * 8150 **/ 8151 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 8152 struct flow_cls_offload *cls_flower) 8153 { 8154 struct i40e_cloud_filter *filter = NULL; 8155 struct i40e_pf *pf = vsi->back; 8156 int err = 0; 8157 8158 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 8159 8160 if (!filter) 8161 return -EINVAL; 8162 8163 hash_del(&filter->cloud_node); 8164 8165 if (filter->dst_port) 8166 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 8167 else 8168 err = i40e_add_del_cloud_filter(vsi, filter, false); 8169 8170 kfree(filter); 8171 if (err) { 8172 dev_err(&pf->pdev->dev, 8173 "Failed to delete cloud filter, err %s\n", 8174 i40e_stat_str(&pf->hw, err)); 8175 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 8176 } 8177 8178 pf->num_cloud_filters--; 8179 if (!pf->num_cloud_filters) 8180 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8181 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8182 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8183 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8184 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8185 } 8186 return 0; 8187 } 8188 8189 /** 8190 * i40e_setup_tc_cls_flower - flower classifier offloads 8191 * @np: net device to configure 8192 * @cls_flower: offload data 8193 **/ 8194 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 8195 struct flow_cls_offload *cls_flower) 8196 { 8197 struct i40e_vsi *vsi = np->vsi; 8198 8199 switch (cls_flower->command) { 8200 case FLOW_CLS_REPLACE: 8201 return i40e_configure_clsflower(vsi, cls_flower); 8202 case FLOW_CLS_DESTROY: 8203 return i40e_delete_clsflower(vsi, cls_flower); 8204 case FLOW_CLS_STATS: 8205 return -EOPNOTSUPP; 8206 default: 8207 return -EOPNOTSUPP; 8208 } 8209 } 8210 8211 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8212 void *cb_priv) 8213 { 8214 struct i40e_netdev_priv *np = cb_priv; 8215 8216 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 8217 return -EOPNOTSUPP; 8218 8219 switch (type) { 8220 case TC_SETUP_CLSFLOWER: 8221 return i40e_setup_tc_cls_flower(np, type_data); 8222 8223 default: 8224 return -EOPNOTSUPP; 8225 } 8226 } 8227 8228 static LIST_HEAD(i40e_block_cb_list); 8229 8230 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8231 void *type_data) 8232 { 8233 struct i40e_netdev_priv *np = netdev_priv(netdev); 8234 8235 switch (type) { 8236 case TC_SETUP_QDISC_MQPRIO: 8237 return i40e_setup_tc(netdev, type_data); 8238 case TC_SETUP_BLOCK: 8239 return flow_block_cb_setup_simple(type_data, 8240 &i40e_block_cb_list, 8241 i40e_setup_tc_block_cb, 8242 np, np, true); 8243 default: 8244 return -EOPNOTSUPP; 8245 } 8246 } 8247 8248 /** 8249 * i40e_open - Called when a network interface is made active 8250 * @netdev: network interface device structure 8251 * 8252 * The open entry point is called when a network interface is made 8253 * active by the system (IFF_UP). At this point all resources needed 8254 * for transmit and receive operations are allocated, the interrupt 8255 * handler is registered with the OS, the netdev watchdog subtask is 8256 * enabled, and the stack is notified that the interface is ready. 8257 * 8258 * Returns 0 on success, negative value on failure 8259 **/ 8260 int i40e_open(struct net_device *netdev) 8261 { 8262 struct i40e_netdev_priv *np = netdev_priv(netdev); 8263 struct i40e_vsi *vsi = np->vsi; 8264 struct i40e_pf *pf = vsi->back; 8265 int err; 8266 8267 /* disallow open during test or if eeprom is broken */ 8268 if (test_bit(__I40E_TESTING, pf->state) || 8269 test_bit(__I40E_BAD_EEPROM, pf->state)) 8270 return -EBUSY; 8271 8272 netif_carrier_off(netdev); 8273 8274 if (i40e_force_link_state(pf, true)) 8275 return -EAGAIN; 8276 8277 err = i40e_vsi_open(vsi); 8278 if (err) 8279 return err; 8280 8281 /* configure global TSO hardware offload settings */ 8282 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 8283 TCP_FLAG_FIN) >> 16); 8284 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 8285 TCP_FLAG_FIN | 8286 TCP_FLAG_CWR) >> 16); 8287 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 8288 8289 udp_tunnel_get_rx_info(netdev); 8290 8291 return 0; 8292 } 8293 8294 /** 8295 * i40e_vsi_open - 8296 * @vsi: the VSI to open 8297 * 8298 * Finish initialization of the VSI. 8299 * 8300 * Returns 0 on success, negative value on failure 8301 * 8302 * Note: expects to be called while under rtnl_lock() 8303 **/ 8304 int i40e_vsi_open(struct i40e_vsi *vsi) 8305 { 8306 struct i40e_pf *pf = vsi->back; 8307 char int_name[I40E_INT_NAME_STR_LEN]; 8308 int err; 8309 8310 /* allocate descriptors */ 8311 err = i40e_vsi_setup_tx_resources(vsi); 8312 if (err) 8313 goto err_setup_tx; 8314 err = i40e_vsi_setup_rx_resources(vsi); 8315 if (err) 8316 goto err_setup_rx; 8317 8318 err = i40e_vsi_configure(vsi); 8319 if (err) 8320 goto err_setup_rx; 8321 8322 if (vsi->netdev) { 8323 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 8324 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 8325 err = i40e_vsi_request_irq(vsi, int_name); 8326 if (err) 8327 goto err_setup_rx; 8328 8329 /* Notify the stack of the actual queue counts. */ 8330 err = netif_set_real_num_tx_queues(vsi->netdev, 8331 vsi->num_queue_pairs); 8332 if (err) 8333 goto err_set_queues; 8334 8335 err = netif_set_real_num_rx_queues(vsi->netdev, 8336 vsi->num_queue_pairs); 8337 if (err) 8338 goto err_set_queues; 8339 8340 } else if (vsi->type == I40E_VSI_FDIR) { 8341 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 8342 dev_driver_string(&pf->pdev->dev), 8343 dev_name(&pf->pdev->dev)); 8344 err = i40e_vsi_request_irq(vsi, int_name); 8345 8346 } else { 8347 err = -EINVAL; 8348 goto err_setup_rx; 8349 } 8350 8351 err = i40e_up_complete(vsi); 8352 if (err) 8353 goto err_up_complete; 8354 8355 return 0; 8356 8357 err_up_complete: 8358 i40e_down(vsi); 8359 err_set_queues: 8360 i40e_vsi_free_irq(vsi); 8361 err_setup_rx: 8362 i40e_vsi_free_rx_resources(vsi); 8363 err_setup_tx: 8364 i40e_vsi_free_tx_resources(vsi); 8365 if (vsi == pf->vsi[pf->lan_vsi]) 8366 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 8367 8368 return err; 8369 } 8370 8371 /** 8372 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 8373 * @pf: Pointer to PF 8374 * 8375 * This function destroys the hlist where all the Flow Director 8376 * filters were saved. 8377 **/ 8378 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 8379 { 8380 struct i40e_fdir_filter *filter; 8381 struct i40e_flex_pit *pit_entry, *tmp; 8382 struct hlist_node *node2; 8383 8384 hlist_for_each_entry_safe(filter, node2, 8385 &pf->fdir_filter_list, fdir_node) { 8386 hlist_del(&filter->fdir_node); 8387 kfree(filter); 8388 } 8389 8390 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 8391 list_del(&pit_entry->list); 8392 kfree(pit_entry); 8393 } 8394 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 8395 8396 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 8397 list_del(&pit_entry->list); 8398 kfree(pit_entry); 8399 } 8400 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 8401 8402 pf->fdir_pf_active_filters = 0; 8403 pf->fd_tcp4_filter_cnt = 0; 8404 pf->fd_udp4_filter_cnt = 0; 8405 pf->fd_sctp4_filter_cnt = 0; 8406 pf->fd_ip4_filter_cnt = 0; 8407 8408 /* Reprogram the default input set for TCP/IPv4 */ 8409 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8410 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8411 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8412 8413 /* Reprogram the default input set for UDP/IPv4 */ 8414 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 8415 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8416 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8417 8418 /* Reprogram the default input set for SCTP/IPv4 */ 8419 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 8420 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8421 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8422 8423 /* Reprogram the default input set for Other/IPv4 */ 8424 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 8425 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 8426 8427 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 8428 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 8429 } 8430 8431 /** 8432 * i40e_cloud_filter_exit - Cleans up the cloud filters 8433 * @pf: Pointer to PF 8434 * 8435 * This function destroys the hlist where all the cloud filters 8436 * were saved. 8437 **/ 8438 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 8439 { 8440 struct i40e_cloud_filter *cfilter; 8441 struct hlist_node *node; 8442 8443 hlist_for_each_entry_safe(cfilter, node, 8444 &pf->cloud_filter_list, cloud_node) { 8445 hlist_del(&cfilter->cloud_node); 8446 kfree(cfilter); 8447 } 8448 pf->num_cloud_filters = 0; 8449 8450 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8451 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8452 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8453 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8454 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8455 } 8456 } 8457 8458 /** 8459 * i40e_close - Disables a network interface 8460 * @netdev: network interface device structure 8461 * 8462 * The close entry point is called when an interface is de-activated 8463 * by the OS. The hardware is still under the driver's control, but 8464 * this netdev interface is disabled. 8465 * 8466 * Returns 0, this is not allowed to fail 8467 **/ 8468 int i40e_close(struct net_device *netdev) 8469 { 8470 struct i40e_netdev_priv *np = netdev_priv(netdev); 8471 struct i40e_vsi *vsi = np->vsi; 8472 8473 i40e_vsi_close(vsi); 8474 8475 return 0; 8476 } 8477 8478 /** 8479 * i40e_do_reset - Start a PF or Core Reset sequence 8480 * @pf: board private structure 8481 * @reset_flags: which reset is requested 8482 * @lock_acquired: indicates whether or not the lock has been acquired 8483 * before this function was called. 8484 * 8485 * The essential difference in resets is that the PF Reset 8486 * doesn't clear the packet buffers, doesn't reset the PE 8487 * firmware, and doesn't bother the other PFs on the chip. 8488 **/ 8489 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 8490 { 8491 u32 val; 8492 8493 /* do the biggest reset indicated */ 8494 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 8495 8496 /* Request a Global Reset 8497 * 8498 * This will start the chip's countdown to the actual full 8499 * chip reset event, and a warning interrupt to be sent 8500 * to all PFs, including the requestor. Our handler 8501 * for the warning interrupt will deal with the shutdown 8502 * and recovery of the switch setup. 8503 */ 8504 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 8505 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8506 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 8507 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 8508 8509 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 8510 8511 /* Request a Core Reset 8512 * 8513 * Same as Global Reset, except does *not* include the MAC/PHY 8514 */ 8515 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 8516 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8517 val |= I40E_GLGEN_RTRIG_CORER_MASK; 8518 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 8519 i40e_flush(&pf->hw); 8520 8521 } else if (reset_flags & I40E_PF_RESET_FLAG) { 8522 8523 /* Request a PF Reset 8524 * 8525 * Resets only the PF-specific registers 8526 * 8527 * This goes directly to the tear-down and rebuild of 8528 * the switch, since we need to do all the recovery as 8529 * for the Core Reset. 8530 */ 8531 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 8532 i40e_handle_reset_warning(pf, lock_acquired); 8533 8534 dev_info(&pf->pdev->dev, 8535 pf->flags & I40E_FLAG_DISABLE_FW_LLDP ? 8536 "FW LLDP is disabled\n" : 8537 "FW LLDP is enabled\n"); 8538 8539 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 8540 int v; 8541 8542 /* Find the VSI(s) that requested a re-init */ 8543 dev_info(&pf->pdev->dev, 8544 "VSI reinit requested\n"); 8545 for (v = 0; v < pf->num_alloc_vsi; v++) { 8546 struct i40e_vsi *vsi = pf->vsi[v]; 8547 8548 if (vsi != NULL && 8549 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 8550 vsi->state)) 8551 i40e_vsi_reinit_locked(pf->vsi[v]); 8552 } 8553 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 8554 int v; 8555 8556 /* Find the VSI(s) that needs to be brought down */ 8557 dev_info(&pf->pdev->dev, "VSI down requested\n"); 8558 for (v = 0; v < pf->num_alloc_vsi; v++) { 8559 struct i40e_vsi *vsi = pf->vsi[v]; 8560 8561 if (vsi != NULL && 8562 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 8563 vsi->state)) { 8564 set_bit(__I40E_VSI_DOWN, vsi->state); 8565 i40e_down(vsi); 8566 } 8567 } 8568 } else { 8569 dev_info(&pf->pdev->dev, 8570 "bad reset request 0x%08x\n", reset_flags); 8571 } 8572 } 8573 8574 #ifdef CONFIG_I40E_DCB 8575 /** 8576 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 8577 * @pf: board private structure 8578 * @old_cfg: current DCB config 8579 * @new_cfg: new DCB config 8580 **/ 8581 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 8582 struct i40e_dcbx_config *old_cfg, 8583 struct i40e_dcbx_config *new_cfg) 8584 { 8585 bool need_reconfig = false; 8586 8587 /* Check if ETS configuration has changed */ 8588 if (memcmp(&new_cfg->etscfg, 8589 &old_cfg->etscfg, 8590 sizeof(new_cfg->etscfg))) { 8591 /* If Priority Table has changed reconfig is needed */ 8592 if (memcmp(&new_cfg->etscfg.prioritytable, 8593 &old_cfg->etscfg.prioritytable, 8594 sizeof(new_cfg->etscfg.prioritytable))) { 8595 need_reconfig = true; 8596 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 8597 } 8598 8599 if (memcmp(&new_cfg->etscfg.tcbwtable, 8600 &old_cfg->etscfg.tcbwtable, 8601 sizeof(new_cfg->etscfg.tcbwtable))) 8602 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 8603 8604 if (memcmp(&new_cfg->etscfg.tsatable, 8605 &old_cfg->etscfg.tsatable, 8606 sizeof(new_cfg->etscfg.tsatable))) 8607 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 8608 } 8609 8610 /* Check if PFC configuration has changed */ 8611 if (memcmp(&new_cfg->pfc, 8612 &old_cfg->pfc, 8613 sizeof(new_cfg->pfc))) { 8614 need_reconfig = true; 8615 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 8616 } 8617 8618 /* Check if APP Table has changed */ 8619 if (memcmp(&new_cfg->app, 8620 &old_cfg->app, 8621 sizeof(new_cfg->app))) { 8622 need_reconfig = true; 8623 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 8624 } 8625 8626 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 8627 return need_reconfig; 8628 } 8629 8630 /** 8631 * i40e_handle_lldp_event - Handle LLDP Change MIB event 8632 * @pf: board private structure 8633 * @e: event info posted on ARQ 8634 **/ 8635 static int i40e_handle_lldp_event(struct i40e_pf *pf, 8636 struct i40e_arq_event_info *e) 8637 { 8638 struct i40e_aqc_lldp_get_mib *mib = 8639 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 8640 struct i40e_hw *hw = &pf->hw; 8641 struct i40e_dcbx_config tmp_dcbx_cfg; 8642 bool need_reconfig = false; 8643 int ret = 0; 8644 u8 type; 8645 8646 /* Not DCB capable or capability disabled */ 8647 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 8648 return ret; 8649 8650 /* Ignore if event is not for Nearest Bridge */ 8651 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 8652 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 8653 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 8654 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 8655 return ret; 8656 8657 /* Check MIB Type and return if event for Remote MIB update */ 8658 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 8659 dev_dbg(&pf->pdev->dev, 8660 "LLDP event mib type %s\n", type ? "remote" : "local"); 8661 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 8662 /* Update the remote cached instance and return */ 8663 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 8664 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 8665 &hw->remote_dcbx_config); 8666 goto exit; 8667 } 8668 8669 /* Store the old configuration */ 8670 tmp_dcbx_cfg = hw->local_dcbx_config; 8671 8672 /* Reset the old DCBx configuration data */ 8673 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 8674 /* Get updated DCBX data from firmware */ 8675 ret = i40e_get_dcb_config(&pf->hw); 8676 if (ret) { 8677 dev_info(&pf->pdev->dev, 8678 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 8679 i40e_stat_str(&pf->hw, ret), 8680 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8681 goto exit; 8682 } 8683 8684 /* No change detected in DCBX configs */ 8685 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 8686 sizeof(tmp_dcbx_cfg))) { 8687 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 8688 goto exit; 8689 } 8690 8691 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 8692 &hw->local_dcbx_config); 8693 8694 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 8695 8696 if (!need_reconfig) 8697 goto exit; 8698 8699 /* Enable DCB tagging only when more than one TC */ 8700 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 8701 pf->flags |= I40E_FLAG_DCB_ENABLED; 8702 else 8703 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 8704 8705 set_bit(__I40E_PORT_SUSPENDED, pf->state); 8706 /* Reconfiguration needed quiesce all VSIs */ 8707 i40e_pf_quiesce_all_vsi(pf); 8708 8709 /* Changes in configuration update VEB/VSI */ 8710 i40e_dcb_reconfigure(pf); 8711 8712 ret = i40e_resume_port_tx(pf); 8713 8714 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 8715 /* In case of error no point in resuming VSIs */ 8716 if (ret) 8717 goto exit; 8718 8719 /* Wait for the PF's queues to be disabled */ 8720 ret = i40e_pf_wait_queues_disabled(pf); 8721 if (ret) { 8722 /* Schedule PF reset to recover */ 8723 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8724 i40e_service_event_schedule(pf); 8725 } else { 8726 i40e_pf_unquiesce_all_vsi(pf); 8727 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 8728 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 8729 } 8730 8731 exit: 8732 return ret; 8733 } 8734 #endif /* CONFIG_I40E_DCB */ 8735 8736 /** 8737 * i40e_do_reset_safe - Protected reset path for userland calls. 8738 * @pf: board private structure 8739 * @reset_flags: which reset is requested 8740 * 8741 **/ 8742 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 8743 { 8744 rtnl_lock(); 8745 i40e_do_reset(pf, reset_flags, true); 8746 rtnl_unlock(); 8747 } 8748 8749 /** 8750 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 8751 * @pf: board private structure 8752 * @e: event info posted on ARQ 8753 * 8754 * Handler for LAN Queue Overflow Event generated by the firmware for PF 8755 * and VF queues 8756 **/ 8757 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 8758 struct i40e_arq_event_info *e) 8759 { 8760 struct i40e_aqc_lan_overflow *data = 8761 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 8762 u32 queue = le32_to_cpu(data->prtdcb_rupto); 8763 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 8764 struct i40e_hw *hw = &pf->hw; 8765 struct i40e_vf *vf; 8766 u16 vf_id; 8767 8768 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 8769 queue, qtx_ctl); 8770 8771 /* Queue belongs to VF, find the VF and issue VF reset */ 8772 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 8773 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 8774 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 8775 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 8776 vf_id -= hw->func_caps.vf_base_id; 8777 vf = &pf->vf[vf_id]; 8778 i40e_vc_notify_vf_reset(vf); 8779 /* Allow VF to process pending reset notification */ 8780 msleep(20); 8781 i40e_reset_vf(vf, false); 8782 } 8783 } 8784 8785 /** 8786 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 8787 * @pf: board private structure 8788 **/ 8789 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 8790 { 8791 u32 val, fcnt_prog; 8792 8793 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8794 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 8795 return fcnt_prog; 8796 } 8797 8798 /** 8799 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 8800 * @pf: board private structure 8801 **/ 8802 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 8803 { 8804 u32 val, fcnt_prog; 8805 8806 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8807 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 8808 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 8809 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 8810 return fcnt_prog; 8811 } 8812 8813 /** 8814 * i40e_get_global_fd_count - Get total FD filters programmed on device 8815 * @pf: board private structure 8816 **/ 8817 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 8818 { 8819 u32 val, fcnt_prog; 8820 8821 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 8822 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 8823 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 8824 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 8825 return fcnt_prog; 8826 } 8827 8828 /** 8829 * i40e_reenable_fdir_sb - Restore FDir SB capability 8830 * @pf: board private structure 8831 **/ 8832 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 8833 { 8834 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 8835 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8836 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8837 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 8838 } 8839 8840 /** 8841 * i40e_reenable_fdir_atr - Restore FDir ATR capability 8842 * @pf: board private structure 8843 **/ 8844 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 8845 { 8846 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 8847 /* ATR uses the same filtering logic as SB rules. It only 8848 * functions properly if the input set mask is at the default 8849 * settings. It is safe to restore the default input set 8850 * because there are no active TCPv4 filter rules. 8851 */ 8852 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8853 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8854 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8855 8856 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8857 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8858 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 8859 } 8860 } 8861 8862 /** 8863 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 8864 * @pf: board private structure 8865 * @filter: FDir filter to remove 8866 */ 8867 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 8868 struct i40e_fdir_filter *filter) 8869 { 8870 /* Update counters */ 8871 pf->fdir_pf_active_filters--; 8872 pf->fd_inv = 0; 8873 8874 switch (filter->flow_type) { 8875 case TCP_V4_FLOW: 8876 pf->fd_tcp4_filter_cnt--; 8877 break; 8878 case UDP_V4_FLOW: 8879 pf->fd_udp4_filter_cnt--; 8880 break; 8881 case SCTP_V4_FLOW: 8882 pf->fd_sctp4_filter_cnt--; 8883 break; 8884 case IP_USER_FLOW: 8885 switch (filter->ip4_proto) { 8886 case IPPROTO_TCP: 8887 pf->fd_tcp4_filter_cnt--; 8888 break; 8889 case IPPROTO_UDP: 8890 pf->fd_udp4_filter_cnt--; 8891 break; 8892 case IPPROTO_SCTP: 8893 pf->fd_sctp4_filter_cnt--; 8894 break; 8895 case IPPROTO_IP: 8896 pf->fd_ip4_filter_cnt--; 8897 break; 8898 } 8899 break; 8900 } 8901 8902 /* Remove the filter from the list and free memory */ 8903 hlist_del(&filter->fdir_node); 8904 kfree(filter); 8905 } 8906 8907 /** 8908 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 8909 * @pf: board private structure 8910 **/ 8911 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 8912 { 8913 struct i40e_fdir_filter *filter; 8914 u32 fcnt_prog, fcnt_avail; 8915 struct hlist_node *node; 8916 8917 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8918 return; 8919 8920 /* Check if we have enough room to re-enable FDir SB capability. */ 8921 fcnt_prog = i40e_get_global_fd_count(pf); 8922 fcnt_avail = pf->fdir_pf_filter_count; 8923 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 8924 (pf->fd_add_err == 0) || 8925 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 8926 i40e_reenable_fdir_sb(pf); 8927 8928 /* We should wait for even more space before re-enabling ATR. 8929 * Additionally, we cannot enable ATR as long as we still have TCP SB 8930 * rules active. 8931 */ 8932 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 8933 (pf->fd_tcp4_filter_cnt == 0)) 8934 i40e_reenable_fdir_atr(pf); 8935 8936 /* if hw had a problem adding a filter, delete it */ 8937 if (pf->fd_inv > 0) { 8938 hlist_for_each_entry_safe(filter, node, 8939 &pf->fdir_filter_list, fdir_node) 8940 if (filter->fd_id == pf->fd_inv) 8941 i40e_delete_invalid_filter(pf, filter); 8942 } 8943 } 8944 8945 #define I40E_MIN_FD_FLUSH_INTERVAL 10 8946 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 8947 /** 8948 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 8949 * @pf: board private structure 8950 **/ 8951 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 8952 { 8953 unsigned long min_flush_time; 8954 int flush_wait_retry = 50; 8955 bool disable_atr = false; 8956 int fd_room; 8957 int reg; 8958 8959 if (!time_after(jiffies, pf->fd_flush_timestamp + 8960 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 8961 return; 8962 8963 /* If the flush is happening too quick and we have mostly SB rules we 8964 * should not re-enable ATR for some time. 8965 */ 8966 min_flush_time = pf->fd_flush_timestamp + 8967 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 8968 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 8969 8970 if (!(time_after(jiffies, min_flush_time)) && 8971 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 8972 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8973 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 8974 disable_atr = true; 8975 } 8976 8977 pf->fd_flush_timestamp = jiffies; 8978 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8979 /* flush all filters */ 8980 wr32(&pf->hw, I40E_PFQF_CTL_1, 8981 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 8982 i40e_flush(&pf->hw); 8983 pf->fd_flush_cnt++; 8984 pf->fd_add_err = 0; 8985 do { 8986 /* Check FD flush status every 5-6msec */ 8987 usleep_range(5000, 6000); 8988 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 8989 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 8990 break; 8991 } while (flush_wait_retry--); 8992 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 8993 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 8994 } else { 8995 /* replay sideband filters */ 8996 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 8997 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 8998 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8999 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 9000 if (I40E_DEBUG_FD & pf->hw.debug_mask) 9001 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 9002 } 9003 } 9004 9005 /** 9006 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 9007 * @pf: board private structure 9008 **/ 9009 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 9010 { 9011 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 9012 } 9013 9014 /** 9015 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 9016 * @pf: board private structure 9017 **/ 9018 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 9019 { 9020 9021 /* if interface is down do nothing */ 9022 if (test_bit(__I40E_DOWN, pf->state)) 9023 return; 9024 9025 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 9026 i40e_fdir_flush_and_replay(pf); 9027 9028 i40e_fdir_check_and_reenable(pf); 9029 9030 } 9031 9032 /** 9033 * i40e_vsi_link_event - notify VSI of a link event 9034 * @vsi: vsi to be notified 9035 * @link_up: link up or down 9036 **/ 9037 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 9038 { 9039 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 9040 return; 9041 9042 switch (vsi->type) { 9043 case I40E_VSI_MAIN: 9044 if (!vsi->netdev || !vsi->netdev_registered) 9045 break; 9046 9047 if (link_up) { 9048 netif_carrier_on(vsi->netdev); 9049 netif_tx_wake_all_queues(vsi->netdev); 9050 } else { 9051 netif_carrier_off(vsi->netdev); 9052 netif_tx_stop_all_queues(vsi->netdev); 9053 } 9054 break; 9055 9056 case I40E_VSI_SRIOV: 9057 case I40E_VSI_VMDQ2: 9058 case I40E_VSI_CTRL: 9059 case I40E_VSI_IWARP: 9060 case I40E_VSI_MIRROR: 9061 default: 9062 /* there is no notification for other VSIs */ 9063 break; 9064 } 9065 } 9066 9067 /** 9068 * i40e_veb_link_event - notify elements on the veb of a link event 9069 * @veb: veb to be notified 9070 * @link_up: link up or down 9071 **/ 9072 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 9073 { 9074 struct i40e_pf *pf; 9075 int i; 9076 9077 if (!veb || !veb->pf) 9078 return; 9079 pf = veb->pf; 9080 9081 /* depth first... */ 9082 for (i = 0; i < I40E_MAX_VEB; i++) 9083 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 9084 i40e_veb_link_event(pf->veb[i], link_up); 9085 9086 /* ... now the local VSIs */ 9087 for (i = 0; i < pf->num_alloc_vsi; i++) 9088 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 9089 i40e_vsi_link_event(pf->vsi[i], link_up); 9090 } 9091 9092 /** 9093 * i40e_link_event - Update netif_carrier status 9094 * @pf: board private structure 9095 **/ 9096 static void i40e_link_event(struct i40e_pf *pf) 9097 { 9098 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9099 u8 new_link_speed, old_link_speed; 9100 i40e_status status; 9101 bool new_link, old_link; 9102 9103 /* set this to force the get_link_status call to refresh state */ 9104 pf->hw.phy.get_link_info = true; 9105 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 9106 status = i40e_get_link_status(&pf->hw, &new_link); 9107 9108 /* On success, disable temp link polling */ 9109 if (status == I40E_SUCCESS) { 9110 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9111 } else { 9112 /* Enable link polling temporarily until i40e_get_link_status 9113 * returns I40E_SUCCESS 9114 */ 9115 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9116 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 9117 status); 9118 return; 9119 } 9120 9121 old_link_speed = pf->hw.phy.link_info_old.link_speed; 9122 new_link_speed = pf->hw.phy.link_info.link_speed; 9123 9124 if (new_link == old_link && 9125 new_link_speed == old_link_speed && 9126 (test_bit(__I40E_VSI_DOWN, vsi->state) || 9127 new_link == netif_carrier_ok(vsi->netdev))) 9128 return; 9129 9130 i40e_print_link_message(vsi, new_link); 9131 9132 /* Notify the base of the switch tree connected to 9133 * the link. Floating VEBs are not notified. 9134 */ 9135 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 9136 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 9137 else 9138 i40e_vsi_link_event(vsi, new_link); 9139 9140 if (pf->vf) 9141 i40e_vc_notify_link_state(pf); 9142 9143 if (pf->flags & I40E_FLAG_PTP) 9144 i40e_ptp_set_increment(pf); 9145 } 9146 9147 /** 9148 * i40e_watchdog_subtask - periodic checks not using event driven response 9149 * @pf: board private structure 9150 **/ 9151 static void i40e_watchdog_subtask(struct i40e_pf *pf) 9152 { 9153 int i; 9154 9155 /* if interface is down do nothing */ 9156 if (test_bit(__I40E_DOWN, pf->state) || 9157 test_bit(__I40E_CONFIG_BUSY, pf->state)) 9158 return; 9159 9160 /* make sure we don't do these things too often */ 9161 if (time_before(jiffies, (pf->service_timer_previous + 9162 pf->service_timer_period))) 9163 return; 9164 pf->service_timer_previous = jiffies; 9165 9166 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 9167 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 9168 i40e_link_event(pf); 9169 9170 /* Update the stats for active netdevs so the network stack 9171 * can look at updated numbers whenever it cares to 9172 */ 9173 for (i = 0; i < pf->num_alloc_vsi; i++) 9174 if (pf->vsi[i] && pf->vsi[i]->netdev) 9175 i40e_update_stats(pf->vsi[i]); 9176 9177 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 9178 /* Update the stats for the active switching components */ 9179 for (i = 0; i < I40E_MAX_VEB; i++) 9180 if (pf->veb[i]) 9181 i40e_update_veb_stats(pf->veb[i]); 9182 } 9183 9184 i40e_ptp_rx_hang(pf); 9185 i40e_ptp_tx_hang(pf); 9186 } 9187 9188 /** 9189 * i40e_reset_subtask - Set up for resetting the device and driver 9190 * @pf: board private structure 9191 **/ 9192 static void i40e_reset_subtask(struct i40e_pf *pf) 9193 { 9194 u32 reset_flags = 0; 9195 9196 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 9197 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 9198 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 9199 } 9200 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 9201 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 9202 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9203 } 9204 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 9205 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 9206 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 9207 } 9208 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 9209 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 9210 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 9211 } 9212 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 9213 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 9214 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 9215 } 9216 9217 /* If there's a recovery already waiting, it takes 9218 * precedence before starting a new reset sequence. 9219 */ 9220 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 9221 i40e_prep_for_reset(pf, false); 9222 i40e_reset(pf); 9223 i40e_rebuild(pf, false, false); 9224 } 9225 9226 /* If we're already down or resetting, just bail */ 9227 if (reset_flags && 9228 !test_bit(__I40E_DOWN, pf->state) && 9229 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 9230 i40e_do_reset(pf, reset_flags, false); 9231 } 9232 } 9233 9234 /** 9235 * i40e_handle_link_event - Handle link event 9236 * @pf: board private structure 9237 * @e: event info posted on ARQ 9238 **/ 9239 static void i40e_handle_link_event(struct i40e_pf *pf, 9240 struct i40e_arq_event_info *e) 9241 { 9242 struct i40e_aqc_get_link_status *status = 9243 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 9244 9245 /* Do a new status request to re-enable LSE reporting 9246 * and load new status information into the hw struct 9247 * This completely ignores any state information 9248 * in the ARQ event info, instead choosing to always 9249 * issue the AQ update link status command. 9250 */ 9251 i40e_link_event(pf); 9252 9253 /* Check if module meets thermal requirements */ 9254 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 9255 dev_err(&pf->pdev->dev, 9256 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 9257 dev_err(&pf->pdev->dev, 9258 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 9259 } else { 9260 /* check for unqualified module, if link is down, suppress 9261 * the message if link was forced to be down. 9262 */ 9263 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 9264 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 9265 (!(status->link_info & I40E_AQ_LINK_UP)) && 9266 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 9267 dev_err(&pf->pdev->dev, 9268 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 9269 dev_err(&pf->pdev->dev, 9270 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 9271 } 9272 } 9273 } 9274 9275 /** 9276 * i40e_clean_adminq_subtask - Clean the AdminQ rings 9277 * @pf: board private structure 9278 **/ 9279 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 9280 { 9281 struct i40e_arq_event_info event; 9282 struct i40e_hw *hw = &pf->hw; 9283 u16 pending, i = 0; 9284 i40e_status ret; 9285 u16 opcode; 9286 u32 oldval; 9287 u32 val; 9288 9289 /* Do not run clean AQ when PF reset fails */ 9290 if (test_bit(__I40E_RESET_FAILED, pf->state)) 9291 return; 9292 9293 /* check for error indications */ 9294 val = rd32(&pf->hw, pf->hw.aq.arq.len); 9295 oldval = val; 9296 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 9297 if (hw->debug_mask & I40E_DEBUG_AQ) 9298 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 9299 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 9300 } 9301 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 9302 if (hw->debug_mask & I40E_DEBUG_AQ) 9303 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 9304 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 9305 pf->arq_overflows++; 9306 } 9307 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 9308 if (hw->debug_mask & I40E_DEBUG_AQ) 9309 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 9310 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 9311 } 9312 if (oldval != val) 9313 wr32(&pf->hw, pf->hw.aq.arq.len, val); 9314 9315 val = rd32(&pf->hw, pf->hw.aq.asq.len); 9316 oldval = val; 9317 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 9318 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9319 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 9320 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 9321 } 9322 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 9323 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9324 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 9325 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 9326 } 9327 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 9328 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9329 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 9330 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 9331 } 9332 if (oldval != val) 9333 wr32(&pf->hw, pf->hw.aq.asq.len, val); 9334 9335 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 9336 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 9337 if (!event.msg_buf) 9338 return; 9339 9340 do { 9341 ret = i40e_clean_arq_element(hw, &event, &pending); 9342 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 9343 break; 9344 else if (ret) { 9345 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 9346 break; 9347 } 9348 9349 opcode = le16_to_cpu(event.desc.opcode); 9350 switch (opcode) { 9351 9352 case i40e_aqc_opc_get_link_status: 9353 i40e_handle_link_event(pf, &event); 9354 break; 9355 case i40e_aqc_opc_send_msg_to_pf: 9356 ret = i40e_vc_process_vf_msg(pf, 9357 le16_to_cpu(event.desc.retval), 9358 le32_to_cpu(event.desc.cookie_high), 9359 le32_to_cpu(event.desc.cookie_low), 9360 event.msg_buf, 9361 event.msg_len); 9362 break; 9363 case i40e_aqc_opc_lldp_update_mib: 9364 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 9365 #ifdef CONFIG_I40E_DCB 9366 rtnl_lock(); 9367 ret = i40e_handle_lldp_event(pf, &event); 9368 rtnl_unlock(); 9369 #endif /* CONFIG_I40E_DCB */ 9370 break; 9371 case i40e_aqc_opc_event_lan_overflow: 9372 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 9373 i40e_handle_lan_overflow_event(pf, &event); 9374 break; 9375 case i40e_aqc_opc_send_msg_to_peer: 9376 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 9377 break; 9378 case i40e_aqc_opc_nvm_erase: 9379 case i40e_aqc_opc_nvm_update: 9380 case i40e_aqc_opc_oem_post_update: 9381 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 9382 "ARQ NVM operation 0x%04x completed\n", 9383 opcode); 9384 break; 9385 default: 9386 dev_info(&pf->pdev->dev, 9387 "ARQ: Unknown event 0x%04x ignored\n", 9388 opcode); 9389 break; 9390 } 9391 } while (i++ < pf->adminq_work_limit); 9392 9393 if (i < pf->adminq_work_limit) 9394 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 9395 9396 /* re-enable Admin queue interrupt cause */ 9397 val = rd32(hw, I40E_PFINT_ICR0_ENA); 9398 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 9399 wr32(hw, I40E_PFINT_ICR0_ENA, val); 9400 i40e_flush(hw); 9401 9402 kfree(event.msg_buf); 9403 } 9404 9405 /** 9406 * i40e_verify_eeprom - make sure eeprom is good to use 9407 * @pf: board private structure 9408 **/ 9409 static void i40e_verify_eeprom(struct i40e_pf *pf) 9410 { 9411 int err; 9412 9413 err = i40e_diag_eeprom_test(&pf->hw); 9414 if (err) { 9415 /* retry in case of garbage read */ 9416 err = i40e_diag_eeprom_test(&pf->hw); 9417 if (err) { 9418 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 9419 err); 9420 set_bit(__I40E_BAD_EEPROM, pf->state); 9421 } 9422 } 9423 9424 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 9425 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 9426 clear_bit(__I40E_BAD_EEPROM, pf->state); 9427 } 9428 } 9429 9430 /** 9431 * i40e_enable_pf_switch_lb 9432 * @pf: pointer to the PF structure 9433 * 9434 * enable switch loop back or die - no point in a return value 9435 **/ 9436 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 9437 { 9438 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9439 struct i40e_vsi_context ctxt; 9440 int ret; 9441 9442 ctxt.seid = pf->main_vsi_seid; 9443 ctxt.pf_num = pf->hw.pf_id; 9444 ctxt.vf_num = 0; 9445 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9446 if (ret) { 9447 dev_info(&pf->pdev->dev, 9448 "couldn't get PF vsi config, err %s aq_err %s\n", 9449 i40e_stat_str(&pf->hw, ret), 9450 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9451 return; 9452 } 9453 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9454 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9455 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9456 9457 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 9458 if (ret) { 9459 dev_info(&pf->pdev->dev, 9460 "update vsi switch failed, err %s aq_err %s\n", 9461 i40e_stat_str(&pf->hw, ret), 9462 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9463 } 9464 } 9465 9466 /** 9467 * i40e_disable_pf_switch_lb 9468 * @pf: pointer to the PF structure 9469 * 9470 * disable switch loop back or die - no point in a return value 9471 **/ 9472 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 9473 { 9474 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9475 struct i40e_vsi_context ctxt; 9476 int ret; 9477 9478 ctxt.seid = pf->main_vsi_seid; 9479 ctxt.pf_num = pf->hw.pf_id; 9480 ctxt.vf_num = 0; 9481 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9482 if (ret) { 9483 dev_info(&pf->pdev->dev, 9484 "couldn't get PF vsi config, err %s aq_err %s\n", 9485 i40e_stat_str(&pf->hw, ret), 9486 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9487 return; 9488 } 9489 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9490 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9491 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9492 9493 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 9494 if (ret) { 9495 dev_info(&pf->pdev->dev, 9496 "update vsi switch failed, err %s aq_err %s\n", 9497 i40e_stat_str(&pf->hw, ret), 9498 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9499 } 9500 } 9501 9502 /** 9503 * i40e_config_bridge_mode - Configure the HW bridge mode 9504 * @veb: pointer to the bridge instance 9505 * 9506 * Configure the loop back mode for the LAN VSI that is downlink to the 9507 * specified HW bridge instance. It is expected this function is called 9508 * when a new HW bridge is instantiated. 9509 **/ 9510 static void i40e_config_bridge_mode(struct i40e_veb *veb) 9511 { 9512 struct i40e_pf *pf = veb->pf; 9513 9514 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 9515 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 9516 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 9517 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 9518 i40e_disable_pf_switch_lb(pf); 9519 else 9520 i40e_enable_pf_switch_lb(pf); 9521 } 9522 9523 /** 9524 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 9525 * @veb: pointer to the VEB instance 9526 * 9527 * This is a recursive function that first builds the attached VSIs then 9528 * recurses in to build the next layer of VEB. We track the connections 9529 * through our own index numbers because the seid's from the HW could 9530 * change across the reset. 9531 **/ 9532 static int i40e_reconstitute_veb(struct i40e_veb *veb) 9533 { 9534 struct i40e_vsi *ctl_vsi = NULL; 9535 struct i40e_pf *pf = veb->pf; 9536 int v, veb_idx; 9537 int ret; 9538 9539 /* build VSI that owns this VEB, temporarily attached to base VEB */ 9540 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 9541 if (pf->vsi[v] && 9542 pf->vsi[v]->veb_idx == veb->idx && 9543 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 9544 ctl_vsi = pf->vsi[v]; 9545 break; 9546 } 9547 } 9548 if (!ctl_vsi) { 9549 dev_info(&pf->pdev->dev, 9550 "missing owner VSI for veb_idx %d\n", veb->idx); 9551 ret = -ENOENT; 9552 goto end_reconstitute; 9553 } 9554 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 9555 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 9556 ret = i40e_add_vsi(ctl_vsi); 9557 if (ret) { 9558 dev_info(&pf->pdev->dev, 9559 "rebuild of veb_idx %d owner VSI failed: %d\n", 9560 veb->idx, ret); 9561 goto end_reconstitute; 9562 } 9563 i40e_vsi_reset_stats(ctl_vsi); 9564 9565 /* create the VEB in the switch and move the VSI onto the VEB */ 9566 ret = i40e_add_veb(veb, ctl_vsi); 9567 if (ret) 9568 goto end_reconstitute; 9569 9570 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 9571 veb->bridge_mode = BRIDGE_MODE_VEB; 9572 else 9573 veb->bridge_mode = BRIDGE_MODE_VEPA; 9574 i40e_config_bridge_mode(veb); 9575 9576 /* create the remaining VSIs attached to this VEB */ 9577 for (v = 0; v < pf->num_alloc_vsi; v++) { 9578 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 9579 continue; 9580 9581 if (pf->vsi[v]->veb_idx == veb->idx) { 9582 struct i40e_vsi *vsi = pf->vsi[v]; 9583 9584 vsi->uplink_seid = veb->seid; 9585 ret = i40e_add_vsi(vsi); 9586 if (ret) { 9587 dev_info(&pf->pdev->dev, 9588 "rebuild of vsi_idx %d failed: %d\n", 9589 v, ret); 9590 goto end_reconstitute; 9591 } 9592 i40e_vsi_reset_stats(vsi); 9593 } 9594 } 9595 9596 /* create any VEBs attached to this VEB - RECURSION */ 9597 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 9598 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 9599 pf->veb[veb_idx]->uplink_seid = veb->seid; 9600 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 9601 if (ret) 9602 break; 9603 } 9604 } 9605 9606 end_reconstitute: 9607 return ret; 9608 } 9609 9610 /** 9611 * i40e_get_capabilities - get info about the HW 9612 * @pf: the PF struct 9613 * @list_type: AQ capability to be queried 9614 **/ 9615 static int i40e_get_capabilities(struct i40e_pf *pf, 9616 enum i40e_admin_queue_opc list_type) 9617 { 9618 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 9619 u16 data_size; 9620 int buf_len; 9621 int err; 9622 9623 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 9624 do { 9625 cap_buf = kzalloc(buf_len, GFP_KERNEL); 9626 if (!cap_buf) 9627 return -ENOMEM; 9628 9629 /* this loads the data into the hw struct for us */ 9630 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 9631 &data_size, list_type, 9632 NULL); 9633 /* data loaded, buffer no longer needed */ 9634 kfree(cap_buf); 9635 9636 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 9637 /* retry with a larger buffer */ 9638 buf_len = data_size; 9639 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 9640 dev_info(&pf->pdev->dev, 9641 "capability discovery failed, err %s aq_err %s\n", 9642 i40e_stat_str(&pf->hw, err), 9643 i40e_aq_str(&pf->hw, 9644 pf->hw.aq.asq_last_status)); 9645 return -ENODEV; 9646 } 9647 } while (err); 9648 9649 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 9650 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9651 dev_info(&pf->pdev->dev, 9652 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 9653 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 9654 pf->hw.func_caps.num_msix_vectors, 9655 pf->hw.func_caps.num_msix_vectors_vf, 9656 pf->hw.func_caps.fd_filters_guaranteed, 9657 pf->hw.func_caps.fd_filters_best_effort, 9658 pf->hw.func_caps.num_tx_qp, 9659 pf->hw.func_caps.num_vsis); 9660 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 9661 dev_info(&pf->pdev->dev, 9662 "switch_mode=0x%04x, function_valid=0x%08x\n", 9663 pf->hw.dev_caps.switch_mode, 9664 pf->hw.dev_caps.valid_functions); 9665 dev_info(&pf->pdev->dev, 9666 "SR-IOV=%d, num_vfs for all function=%u\n", 9667 pf->hw.dev_caps.sr_iov_1_1, 9668 pf->hw.dev_caps.num_vfs); 9669 dev_info(&pf->pdev->dev, 9670 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 9671 pf->hw.dev_caps.num_vsis, 9672 pf->hw.dev_caps.num_rx_qp, 9673 pf->hw.dev_caps.num_tx_qp); 9674 } 9675 } 9676 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9677 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 9678 + pf->hw.func_caps.num_vfs) 9679 if (pf->hw.revision_id == 0 && 9680 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 9681 dev_info(&pf->pdev->dev, 9682 "got num_vsis %d, setting num_vsis to %d\n", 9683 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 9684 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 9685 } 9686 } 9687 return 0; 9688 } 9689 9690 static int i40e_vsi_clear(struct i40e_vsi *vsi); 9691 9692 /** 9693 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 9694 * @pf: board private structure 9695 **/ 9696 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 9697 { 9698 struct i40e_vsi *vsi; 9699 9700 /* quick workaround for an NVM issue that leaves a critical register 9701 * uninitialized 9702 */ 9703 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 9704 static const u32 hkey[] = { 9705 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 9706 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 9707 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 9708 0x95b3a76d}; 9709 int i; 9710 9711 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 9712 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 9713 } 9714 9715 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 9716 return; 9717 9718 /* find existing VSI and see if it needs configuring */ 9719 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9720 9721 /* create a new VSI if none exists */ 9722 if (!vsi) { 9723 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 9724 pf->vsi[pf->lan_vsi]->seid, 0); 9725 if (!vsi) { 9726 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 9727 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 9728 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 9729 return; 9730 } 9731 } 9732 9733 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 9734 } 9735 9736 /** 9737 * i40e_fdir_teardown - release the Flow Director resources 9738 * @pf: board private structure 9739 **/ 9740 static void i40e_fdir_teardown(struct i40e_pf *pf) 9741 { 9742 struct i40e_vsi *vsi; 9743 9744 i40e_fdir_filter_exit(pf); 9745 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9746 if (vsi) 9747 i40e_vsi_release(vsi); 9748 } 9749 9750 /** 9751 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 9752 * @vsi: PF main vsi 9753 * @seid: seid of main or channel VSIs 9754 * 9755 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 9756 * existed before reset 9757 **/ 9758 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 9759 { 9760 struct i40e_cloud_filter *cfilter; 9761 struct i40e_pf *pf = vsi->back; 9762 struct hlist_node *node; 9763 i40e_status ret; 9764 9765 /* Add cloud filters back if they exist */ 9766 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 9767 cloud_node) { 9768 if (cfilter->seid != seid) 9769 continue; 9770 9771 if (cfilter->dst_port) 9772 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 9773 true); 9774 else 9775 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 9776 9777 if (ret) { 9778 dev_dbg(&pf->pdev->dev, 9779 "Failed to rebuild cloud filter, err %s aq_err %s\n", 9780 i40e_stat_str(&pf->hw, ret), 9781 i40e_aq_str(&pf->hw, 9782 pf->hw.aq.asq_last_status)); 9783 return ret; 9784 } 9785 } 9786 return 0; 9787 } 9788 9789 /** 9790 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 9791 * @vsi: PF main vsi 9792 * 9793 * Rebuilds channel VSIs if they existed before reset 9794 **/ 9795 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 9796 { 9797 struct i40e_channel *ch, *ch_tmp; 9798 i40e_status ret; 9799 9800 if (list_empty(&vsi->ch_list)) 9801 return 0; 9802 9803 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 9804 if (!ch->initialized) 9805 break; 9806 /* Proceed with creation of channel (VMDq2) VSI */ 9807 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 9808 if (ret) { 9809 dev_info(&vsi->back->pdev->dev, 9810 "failed to rebuild channels using uplink_seid %u\n", 9811 vsi->uplink_seid); 9812 return ret; 9813 } 9814 /* Reconfigure TX queues using QTX_CTL register */ 9815 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 9816 if (ret) { 9817 dev_info(&vsi->back->pdev->dev, 9818 "failed to configure TX rings for channel %u\n", 9819 ch->seid); 9820 return ret; 9821 } 9822 /* update 'next_base_queue' */ 9823 vsi->next_base_queue = vsi->next_base_queue + 9824 ch->num_queue_pairs; 9825 if (ch->max_tx_rate) { 9826 u64 credits = ch->max_tx_rate; 9827 9828 if (i40e_set_bw_limit(vsi, ch->seid, 9829 ch->max_tx_rate)) 9830 return -EINVAL; 9831 9832 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9833 dev_dbg(&vsi->back->pdev->dev, 9834 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9835 ch->max_tx_rate, 9836 credits, 9837 ch->seid); 9838 } 9839 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 9840 if (ret) { 9841 dev_dbg(&vsi->back->pdev->dev, 9842 "Failed to rebuild cloud filters for channel VSI %u\n", 9843 ch->seid); 9844 return ret; 9845 } 9846 } 9847 return 0; 9848 } 9849 9850 /** 9851 * i40e_prep_for_reset - prep for the core to reset 9852 * @pf: board private structure 9853 * @lock_acquired: indicates whether or not the lock has been acquired 9854 * before this function was called. 9855 * 9856 * Close up the VFs and other things in prep for PF Reset. 9857 **/ 9858 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 9859 { 9860 struct i40e_hw *hw = &pf->hw; 9861 i40e_status ret = 0; 9862 u32 v; 9863 9864 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 9865 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9866 return; 9867 if (i40e_check_asq_alive(&pf->hw)) 9868 i40e_vc_notify_reset(pf); 9869 9870 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 9871 9872 /* quiesce the VSIs and their queues that are not already DOWN */ 9873 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 9874 if (!lock_acquired) 9875 rtnl_lock(); 9876 i40e_pf_quiesce_all_vsi(pf); 9877 if (!lock_acquired) 9878 rtnl_unlock(); 9879 9880 for (v = 0; v < pf->num_alloc_vsi; v++) { 9881 if (pf->vsi[v]) 9882 pf->vsi[v]->seid = 0; 9883 } 9884 9885 i40e_shutdown_adminq(&pf->hw); 9886 9887 /* call shutdown HMC */ 9888 if (hw->hmc.hmc_obj) { 9889 ret = i40e_shutdown_lan_hmc(hw); 9890 if (ret) 9891 dev_warn(&pf->pdev->dev, 9892 "shutdown_lan_hmc failed: %d\n", ret); 9893 } 9894 9895 /* Save the current PTP time so that we can restore the time after the 9896 * reset completes. 9897 */ 9898 i40e_ptp_save_hw_time(pf); 9899 } 9900 9901 /** 9902 * i40e_send_version - update firmware with driver version 9903 * @pf: PF struct 9904 */ 9905 static void i40e_send_version(struct i40e_pf *pf) 9906 { 9907 struct i40e_driver_version dv; 9908 9909 dv.major_version = 0xff; 9910 dv.minor_version = 0xff; 9911 dv.build_version = 0xff; 9912 dv.subbuild_version = 0; 9913 strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); 9914 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 9915 } 9916 9917 /** 9918 * i40e_get_oem_version - get OEM specific version information 9919 * @hw: pointer to the hardware structure 9920 **/ 9921 static void i40e_get_oem_version(struct i40e_hw *hw) 9922 { 9923 u16 block_offset = 0xffff; 9924 u16 block_length = 0; 9925 u16 capabilities = 0; 9926 u16 gen_snap = 0; 9927 u16 release = 0; 9928 9929 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 9930 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 9931 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 9932 #define I40E_NVM_OEM_GEN_OFFSET 0x02 9933 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 9934 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 9935 #define I40E_NVM_OEM_LENGTH 3 9936 9937 /* Check if pointer to OEM version block is valid. */ 9938 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 9939 if (block_offset == 0xffff) 9940 return; 9941 9942 /* Check if OEM version block has correct length. */ 9943 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 9944 &block_length); 9945 if (block_length < I40E_NVM_OEM_LENGTH) 9946 return; 9947 9948 /* Check if OEM version format is as expected. */ 9949 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 9950 &capabilities); 9951 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 9952 return; 9953 9954 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 9955 &gen_snap); 9956 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 9957 &release); 9958 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 9959 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 9960 } 9961 9962 /** 9963 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 9964 * @pf: board private structure 9965 **/ 9966 static int i40e_reset(struct i40e_pf *pf) 9967 { 9968 struct i40e_hw *hw = &pf->hw; 9969 i40e_status ret; 9970 9971 ret = i40e_pf_reset(hw); 9972 if (ret) { 9973 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 9974 set_bit(__I40E_RESET_FAILED, pf->state); 9975 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9976 } else { 9977 pf->pfr_count++; 9978 } 9979 return ret; 9980 } 9981 9982 /** 9983 * i40e_rebuild - rebuild using a saved config 9984 * @pf: board private structure 9985 * @reinit: if the Main VSI needs to re-initialized. 9986 * @lock_acquired: indicates whether or not the lock has been acquired 9987 * before this function was called. 9988 **/ 9989 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 9990 { 9991 int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state); 9992 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9993 struct i40e_hw *hw = &pf->hw; 9994 u8 set_fc_aq_fail = 0; 9995 i40e_status ret; 9996 u32 val; 9997 int v; 9998 9999 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 10000 i40e_check_recovery_mode(pf)) { 10001 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev); 10002 } 10003 10004 if (test_bit(__I40E_DOWN, pf->state) && 10005 !test_bit(__I40E_RECOVERY_MODE, pf->state) && 10006 !old_recovery_mode_bit) 10007 goto clear_recovery; 10008 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 10009 10010 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 10011 ret = i40e_init_adminq(&pf->hw); 10012 if (ret) { 10013 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 10014 i40e_stat_str(&pf->hw, ret), 10015 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10016 goto clear_recovery; 10017 } 10018 i40e_get_oem_version(&pf->hw); 10019 10020 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 10021 ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) || 10022 hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) { 10023 /* The following delay is necessary for 4.33 firmware and older 10024 * to recover after EMP reset. 200 ms should suffice but we 10025 * put here 300 ms to be sure that FW is ready to operate 10026 * after reset. 10027 */ 10028 mdelay(300); 10029 } 10030 10031 /* re-verify the eeprom if we just had an EMP reset */ 10032 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 10033 i40e_verify_eeprom(pf); 10034 10035 /* if we are going out of or into recovery mode we have to act 10036 * accordingly with regard to resources initialization 10037 * and deinitialization 10038 */ 10039 if (test_bit(__I40E_RECOVERY_MODE, pf->state) || 10040 old_recovery_mode_bit) { 10041 if (i40e_get_capabilities(pf, 10042 i40e_aqc_opc_list_func_capabilities)) 10043 goto end_unlock; 10044 10045 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10046 /* we're staying in recovery mode so we'll reinitialize 10047 * misc vector here 10048 */ 10049 if (i40e_setup_misc_vector_for_recovery_mode(pf)) 10050 goto end_unlock; 10051 } else { 10052 if (!lock_acquired) 10053 rtnl_lock(); 10054 /* we're going out of recovery mode so we'll free 10055 * the IRQ allocated specifically for recovery mode 10056 * and restore the interrupt scheme 10057 */ 10058 free_irq(pf->pdev->irq, pf); 10059 i40e_clear_interrupt_scheme(pf); 10060 if (i40e_restore_interrupt_scheme(pf)) 10061 goto end_unlock; 10062 } 10063 10064 /* tell the firmware that we're starting */ 10065 i40e_send_version(pf); 10066 10067 /* bail out in case recovery mode was detected, as there is 10068 * no need for further configuration. 10069 */ 10070 goto end_unlock; 10071 } 10072 10073 i40e_clear_pxe_mode(hw); 10074 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 10075 if (ret) 10076 goto end_core_reset; 10077 10078 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10079 hw->func_caps.num_rx_qp, 0, 0); 10080 if (ret) { 10081 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 10082 goto end_core_reset; 10083 } 10084 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10085 if (ret) { 10086 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 10087 goto end_core_reset; 10088 } 10089 10090 /* Enable FW to write a default DCB config on link-up */ 10091 i40e_aq_set_dcb_parameters(hw, true, NULL); 10092 10093 #ifdef CONFIG_I40E_DCB 10094 ret = i40e_init_pf_dcb(pf); 10095 if (ret) { 10096 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 10097 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 10098 /* Continue without DCB enabled */ 10099 } 10100 #endif /* CONFIG_I40E_DCB */ 10101 /* do basic switch setup */ 10102 if (!lock_acquired) 10103 rtnl_lock(); 10104 ret = i40e_setup_pf_switch(pf, reinit); 10105 if (ret) 10106 goto end_unlock; 10107 10108 /* The driver only wants link up/down and module qualification 10109 * reports from firmware. Note the negative logic. 10110 */ 10111 ret = i40e_aq_set_phy_int_mask(&pf->hw, 10112 ~(I40E_AQ_EVENT_LINK_UPDOWN | 10113 I40E_AQ_EVENT_MEDIA_NA | 10114 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 10115 if (ret) 10116 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 10117 i40e_stat_str(&pf->hw, ret), 10118 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10119 10120 /* make sure our flow control settings are restored */ 10121 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 10122 if (ret) 10123 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 10124 i40e_stat_str(&pf->hw, ret), 10125 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10126 10127 /* Rebuild the VSIs and VEBs that existed before reset. 10128 * They are still in our local switch element arrays, so only 10129 * need to rebuild the switch model in the HW. 10130 * 10131 * If there were VEBs but the reconstitution failed, we'll try 10132 * try to recover minimal use by getting the basic PF VSI working. 10133 */ 10134 if (vsi->uplink_seid != pf->mac_seid) { 10135 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 10136 /* find the one VEB connected to the MAC, and find orphans */ 10137 for (v = 0; v < I40E_MAX_VEB; v++) { 10138 if (!pf->veb[v]) 10139 continue; 10140 10141 if (pf->veb[v]->uplink_seid == pf->mac_seid || 10142 pf->veb[v]->uplink_seid == 0) { 10143 ret = i40e_reconstitute_veb(pf->veb[v]); 10144 10145 if (!ret) 10146 continue; 10147 10148 /* If Main VEB failed, we're in deep doodoo, 10149 * so give up rebuilding the switch and set up 10150 * for minimal rebuild of PF VSI. 10151 * If orphan failed, we'll report the error 10152 * but try to keep going. 10153 */ 10154 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 10155 dev_info(&pf->pdev->dev, 10156 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 10157 ret); 10158 vsi->uplink_seid = pf->mac_seid; 10159 break; 10160 } else if (pf->veb[v]->uplink_seid == 0) { 10161 dev_info(&pf->pdev->dev, 10162 "rebuild of orphan VEB failed: %d\n", 10163 ret); 10164 } 10165 } 10166 } 10167 } 10168 10169 if (vsi->uplink_seid == pf->mac_seid) { 10170 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 10171 /* no VEB, so rebuild only the Main VSI */ 10172 ret = i40e_add_vsi(vsi); 10173 if (ret) { 10174 dev_info(&pf->pdev->dev, 10175 "rebuild of Main VSI failed: %d\n", ret); 10176 goto end_unlock; 10177 } 10178 } 10179 10180 if (vsi->mqprio_qopt.max_rate[0]) { 10181 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 10182 u64 credits = 0; 10183 10184 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 10185 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 10186 if (ret) 10187 goto end_unlock; 10188 10189 credits = max_tx_rate; 10190 do_div(credits, I40E_BW_CREDIT_DIVISOR); 10191 dev_dbg(&vsi->back->pdev->dev, 10192 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 10193 max_tx_rate, 10194 credits, 10195 vsi->seid); 10196 } 10197 10198 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 10199 if (ret) 10200 goto end_unlock; 10201 10202 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 10203 * for this main VSI if they exist 10204 */ 10205 ret = i40e_rebuild_channels(vsi); 10206 if (ret) 10207 goto end_unlock; 10208 10209 /* Reconfigure hardware for allowing smaller MSS in the case 10210 * of TSO, so that we avoid the MDD being fired and causing 10211 * a reset in the case of small MSS+TSO. 10212 */ 10213 #define I40E_REG_MSS 0x000E64DC 10214 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 10215 #define I40E_64BYTE_MSS 0x400000 10216 val = rd32(hw, I40E_REG_MSS); 10217 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 10218 val &= ~I40E_REG_MSS_MIN_MASK; 10219 val |= I40E_64BYTE_MSS; 10220 wr32(hw, I40E_REG_MSS, val); 10221 } 10222 10223 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 10224 msleep(75); 10225 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 10226 if (ret) 10227 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 10228 i40e_stat_str(&pf->hw, ret), 10229 i40e_aq_str(&pf->hw, 10230 pf->hw.aq.asq_last_status)); 10231 } 10232 /* reinit the misc interrupt */ 10233 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10234 ret = i40e_setup_misc_vector(pf); 10235 10236 /* Add a filter to drop all Flow control frames from any VSI from being 10237 * transmitted. By doing so we stop a malicious VF from sending out 10238 * PAUSE or PFC frames and potentially controlling traffic for other 10239 * PF/VF VSIs. 10240 * The FW can still send Flow control frames if enabled. 10241 */ 10242 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 10243 pf->main_vsi_seid); 10244 10245 /* restart the VSIs that were rebuilt and running before the reset */ 10246 i40e_pf_unquiesce_all_vsi(pf); 10247 10248 /* Release the RTNL lock before we start resetting VFs */ 10249 if (!lock_acquired) 10250 rtnl_unlock(); 10251 10252 /* Restore promiscuous settings */ 10253 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 10254 if (ret) 10255 dev_warn(&pf->pdev->dev, 10256 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n", 10257 pf->cur_promisc ? "on" : "off", 10258 i40e_stat_str(&pf->hw, ret), 10259 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10260 10261 i40e_reset_all_vfs(pf, true); 10262 10263 /* tell the firmware that we're starting */ 10264 i40e_send_version(pf); 10265 10266 /* We've already released the lock, so don't do it again */ 10267 goto end_core_reset; 10268 10269 end_unlock: 10270 if (!lock_acquired) 10271 rtnl_unlock(); 10272 end_core_reset: 10273 clear_bit(__I40E_RESET_FAILED, pf->state); 10274 clear_recovery: 10275 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 10276 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state); 10277 } 10278 10279 /** 10280 * i40e_reset_and_rebuild - reset and rebuild using a saved config 10281 * @pf: board private structure 10282 * @reinit: if the Main VSI needs to re-initialized. 10283 * @lock_acquired: indicates whether or not the lock has been acquired 10284 * before this function was called. 10285 **/ 10286 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 10287 bool lock_acquired) 10288 { 10289 int ret; 10290 /* Now we wait for GRST to settle out. 10291 * We don't have to delete the VEBs or VSIs from the hw switch 10292 * because the reset will make them disappear. 10293 */ 10294 ret = i40e_reset(pf); 10295 if (!ret) 10296 i40e_rebuild(pf, reinit, lock_acquired); 10297 } 10298 10299 /** 10300 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 10301 * @pf: board private structure 10302 * 10303 * Close up the VFs and other things in prep for a Core Reset, 10304 * then get ready to rebuild the world. 10305 * @lock_acquired: indicates whether or not the lock has been acquired 10306 * before this function was called. 10307 **/ 10308 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 10309 { 10310 i40e_prep_for_reset(pf, lock_acquired); 10311 i40e_reset_and_rebuild(pf, false, lock_acquired); 10312 } 10313 10314 /** 10315 * i40e_handle_mdd_event 10316 * @pf: pointer to the PF structure 10317 * 10318 * Called from the MDD irq handler to identify possibly malicious vfs 10319 **/ 10320 static void i40e_handle_mdd_event(struct i40e_pf *pf) 10321 { 10322 struct i40e_hw *hw = &pf->hw; 10323 bool mdd_detected = false; 10324 struct i40e_vf *vf; 10325 u32 reg; 10326 int i; 10327 10328 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 10329 return; 10330 10331 /* find what triggered the MDD event */ 10332 reg = rd32(hw, I40E_GL_MDET_TX); 10333 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 10334 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 10335 I40E_GL_MDET_TX_PF_NUM_SHIFT; 10336 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 10337 I40E_GL_MDET_TX_VF_NUM_SHIFT; 10338 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 10339 I40E_GL_MDET_TX_EVENT_SHIFT; 10340 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 10341 I40E_GL_MDET_TX_QUEUE_SHIFT) - 10342 pf->hw.func_caps.base_queue; 10343 if (netif_msg_tx_err(pf)) 10344 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 10345 event, queue, pf_num, vf_num); 10346 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 10347 mdd_detected = true; 10348 } 10349 reg = rd32(hw, I40E_GL_MDET_RX); 10350 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 10351 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 10352 I40E_GL_MDET_RX_FUNCTION_SHIFT; 10353 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 10354 I40E_GL_MDET_RX_EVENT_SHIFT; 10355 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 10356 I40E_GL_MDET_RX_QUEUE_SHIFT) - 10357 pf->hw.func_caps.base_queue; 10358 if (netif_msg_rx_err(pf)) 10359 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 10360 event, queue, func); 10361 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 10362 mdd_detected = true; 10363 } 10364 10365 if (mdd_detected) { 10366 reg = rd32(hw, I40E_PF_MDET_TX); 10367 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 10368 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 10369 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n"); 10370 } 10371 reg = rd32(hw, I40E_PF_MDET_RX); 10372 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 10373 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 10374 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n"); 10375 } 10376 } 10377 10378 /* see if one of the VFs needs its hand slapped */ 10379 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 10380 vf = &(pf->vf[i]); 10381 reg = rd32(hw, I40E_VP_MDET_TX(i)); 10382 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 10383 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 10384 vf->num_mdd_events++; 10385 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 10386 i); 10387 dev_info(&pf->pdev->dev, 10388 "Use PF Control I/F to re-enable the VF\n"); 10389 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 10390 } 10391 10392 reg = rd32(hw, I40E_VP_MDET_RX(i)); 10393 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 10394 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 10395 vf->num_mdd_events++; 10396 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 10397 i); 10398 dev_info(&pf->pdev->dev, 10399 "Use PF Control I/F to re-enable the VF\n"); 10400 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 10401 } 10402 } 10403 10404 /* re-enable mdd interrupt cause */ 10405 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 10406 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 10407 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 10408 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 10409 i40e_flush(hw); 10410 } 10411 10412 /** 10413 * i40e_service_task - Run the driver's async subtasks 10414 * @work: pointer to work_struct containing our data 10415 **/ 10416 static void i40e_service_task(struct work_struct *work) 10417 { 10418 struct i40e_pf *pf = container_of(work, 10419 struct i40e_pf, 10420 service_task); 10421 unsigned long start_time = jiffies; 10422 10423 /* don't bother with service tasks if a reset is in progress */ 10424 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 10425 test_bit(__I40E_SUSPENDED, pf->state)) 10426 return; 10427 10428 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 10429 return; 10430 10431 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10432 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]); 10433 i40e_sync_filters_subtask(pf); 10434 i40e_reset_subtask(pf); 10435 i40e_handle_mdd_event(pf); 10436 i40e_vc_process_vflr_event(pf); 10437 i40e_watchdog_subtask(pf); 10438 i40e_fdir_reinit_subtask(pf); 10439 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 10440 /* Client subtask will reopen next time through. */ 10441 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], 10442 true); 10443 } else { 10444 i40e_client_subtask(pf); 10445 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 10446 pf->state)) 10447 i40e_notify_client_of_l2_param_changes( 10448 pf->vsi[pf->lan_vsi]); 10449 } 10450 i40e_sync_filters_subtask(pf); 10451 } else { 10452 i40e_reset_subtask(pf); 10453 } 10454 10455 i40e_clean_adminq_subtask(pf); 10456 10457 /* flush memory to make sure state is correct before next watchdog */ 10458 smp_mb__before_atomic(); 10459 clear_bit(__I40E_SERVICE_SCHED, pf->state); 10460 10461 /* If the tasks have taken longer than one timer cycle or there 10462 * is more work to be done, reschedule the service task now 10463 * rather than wait for the timer to tick again. 10464 */ 10465 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 10466 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 10467 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 10468 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 10469 i40e_service_event_schedule(pf); 10470 } 10471 10472 /** 10473 * i40e_service_timer - timer callback 10474 * @t: timer list pointer 10475 **/ 10476 static void i40e_service_timer(struct timer_list *t) 10477 { 10478 struct i40e_pf *pf = from_timer(pf, t, service_timer); 10479 10480 mod_timer(&pf->service_timer, 10481 round_jiffies(jiffies + pf->service_timer_period)); 10482 i40e_service_event_schedule(pf); 10483 } 10484 10485 /** 10486 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 10487 * @vsi: the VSI being configured 10488 **/ 10489 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 10490 { 10491 struct i40e_pf *pf = vsi->back; 10492 10493 switch (vsi->type) { 10494 case I40E_VSI_MAIN: 10495 vsi->alloc_queue_pairs = pf->num_lan_qps; 10496 if (!vsi->num_tx_desc) 10497 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10498 I40E_REQ_DESCRIPTOR_MULTIPLE); 10499 if (!vsi->num_rx_desc) 10500 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10501 I40E_REQ_DESCRIPTOR_MULTIPLE); 10502 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10503 vsi->num_q_vectors = pf->num_lan_msix; 10504 else 10505 vsi->num_q_vectors = 1; 10506 10507 break; 10508 10509 case I40E_VSI_FDIR: 10510 vsi->alloc_queue_pairs = 1; 10511 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT, 10512 I40E_REQ_DESCRIPTOR_MULTIPLE); 10513 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT, 10514 I40E_REQ_DESCRIPTOR_MULTIPLE); 10515 vsi->num_q_vectors = pf->num_fdsb_msix; 10516 break; 10517 10518 case I40E_VSI_VMDQ2: 10519 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 10520 if (!vsi->num_tx_desc) 10521 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10522 I40E_REQ_DESCRIPTOR_MULTIPLE); 10523 if (!vsi->num_rx_desc) 10524 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10525 I40E_REQ_DESCRIPTOR_MULTIPLE); 10526 vsi->num_q_vectors = pf->num_vmdq_msix; 10527 break; 10528 10529 case I40E_VSI_SRIOV: 10530 vsi->alloc_queue_pairs = pf->num_vf_qps; 10531 if (!vsi->num_tx_desc) 10532 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10533 I40E_REQ_DESCRIPTOR_MULTIPLE); 10534 if (!vsi->num_rx_desc) 10535 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10536 I40E_REQ_DESCRIPTOR_MULTIPLE); 10537 break; 10538 10539 default: 10540 WARN_ON(1); 10541 return -ENODATA; 10542 } 10543 10544 return 0; 10545 } 10546 10547 /** 10548 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 10549 * @vsi: VSI pointer 10550 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 10551 * 10552 * On error: returns error code (negative) 10553 * On success: returns 0 10554 **/ 10555 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 10556 { 10557 struct i40e_ring **next_rings; 10558 int size; 10559 int ret = 0; 10560 10561 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 10562 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 10563 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 10564 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 10565 if (!vsi->tx_rings) 10566 return -ENOMEM; 10567 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 10568 if (i40e_enabled_xdp_vsi(vsi)) { 10569 vsi->xdp_rings = next_rings; 10570 next_rings += vsi->alloc_queue_pairs; 10571 } 10572 vsi->rx_rings = next_rings; 10573 10574 if (alloc_qvectors) { 10575 /* allocate memory for q_vector pointers */ 10576 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 10577 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 10578 if (!vsi->q_vectors) { 10579 ret = -ENOMEM; 10580 goto err_vectors; 10581 } 10582 } 10583 return ret; 10584 10585 err_vectors: 10586 kfree(vsi->tx_rings); 10587 return ret; 10588 } 10589 10590 /** 10591 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 10592 * @pf: board private structure 10593 * @type: type of VSI 10594 * 10595 * On error: returns error code (negative) 10596 * On success: returns vsi index in PF (positive) 10597 **/ 10598 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 10599 { 10600 int ret = -ENODEV; 10601 struct i40e_vsi *vsi; 10602 int vsi_idx; 10603 int i; 10604 10605 /* Need to protect the allocation of the VSIs at the PF level */ 10606 mutex_lock(&pf->switch_mutex); 10607 10608 /* VSI list may be fragmented if VSI creation/destruction has 10609 * been happening. We can afford to do a quick scan to look 10610 * for any free VSIs in the list. 10611 * 10612 * find next empty vsi slot, looping back around if necessary 10613 */ 10614 i = pf->next_vsi; 10615 while (i < pf->num_alloc_vsi && pf->vsi[i]) 10616 i++; 10617 if (i >= pf->num_alloc_vsi) { 10618 i = 0; 10619 while (i < pf->next_vsi && pf->vsi[i]) 10620 i++; 10621 } 10622 10623 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 10624 vsi_idx = i; /* Found one! */ 10625 } else { 10626 ret = -ENODEV; 10627 goto unlock_pf; /* out of VSI slots! */ 10628 } 10629 pf->next_vsi = ++i; 10630 10631 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 10632 if (!vsi) { 10633 ret = -ENOMEM; 10634 goto unlock_pf; 10635 } 10636 vsi->type = type; 10637 vsi->back = pf; 10638 set_bit(__I40E_VSI_DOWN, vsi->state); 10639 vsi->flags = 0; 10640 vsi->idx = vsi_idx; 10641 vsi->int_rate_limit = 0; 10642 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 10643 pf->rss_table_size : 64; 10644 vsi->netdev_registered = false; 10645 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 10646 hash_init(vsi->mac_filter_hash); 10647 vsi->irqs_ready = false; 10648 10649 if (type == I40E_VSI_MAIN) { 10650 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL); 10651 if (!vsi->af_xdp_zc_qps) 10652 goto err_rings; 10653 } 10654 10655 ret = i40e_set_num_rings_in_vsi(vsi); 10656 if (ret) 10657 goto err_rings; 10658 10659 ret = i40e_vsi_alloc_arrays(vsi, true); 10660 if (ret) 10661 goto err_rings; 10662 10663 /* Setup default MSIX irq handler for VSI */ 10664 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 10665 10666 /* Initialize VSI lock */ 10667 spin_lock_init(&vsi->mac_filter_hash_lock); 10668 pf->vsi[vsi_idx] = vsi; 10669 ret = vsi_idx; 10670 goto unlock_pf; 10671 10672 err_rings: 10673 bitmap_free(vsi->af_xdp_zc_qps); 10674 pf->next_vsi = i - 1; 10675 kfree(vsi); 10676 unlock_pf: 10677 mutex_unlock(&pf->switch_mutex); 10678 return ret; 10679 } 10680 10681 /** 10682 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 10683 * @vsi: VSI pointer 10684 * @free_qvectors: a bool to specify if q_vectors need to be freed. 10685 * 10686 * On error: returns error code (negative) 10687 * On success: returns 0 10688 **/ 10689 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 10690 { 10691 /* free the ring and vector containers */ 10692 if (free_qvectors) { 10693 kfree(vsi->q_vectors); 10694 vsi->q_vectors = NULL; 10695 } 10696 kfree(vsi->tx_rings); 10697 vsi->tx_rings = NULL; 10698 vsi->rx_rings = NULL; 10699 vsi->xdp_rings = NULL; 10700 } 10701 10702 /** 10703 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 10704 * and lookup table 10705 * @vsi: Pointer to VSI structure 10706 */ 10707 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 10708 { 10709 if (!vsi) 10710 return; 10711 10712 kfree(vsi->rss_hkey_user); 10713 vsi->rss_hkey_user = NULL; 10714 10715 kfree(vsi->rss_lut_user); 10716 vsi->rss_lut_user = NULL; 10717 } 10718 10719 /** 10720 * i40e_vsi_clear - Deallocate the VSI provided 10721 * @vsi: the VSI being un-configured 10722 **/ 10723 static int i40e_vsi_clear(struct i40e_vsi *vsi) 10724 { 10725 struct i40e_pf *pf; 10726 10727 if (!vsi) 10728 return 0; 10729 10730 if (!vsi->back) 10731 goto free_vsi; 10732 pf = vsi->back; 10733 10734 mutex_lock(&pf->switch_mutex); 10735 if (!pf->vsi[vsi->idx]) { 10736 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 10737 vsi->idx, vsi->idx, vsi->type); 10738 goto unlock_vsi; 10739 } 10740 10741 if (pf->vsi[vsi->idx] != vsi) { 10742 dev_err(&pf->pdev->dev, 10743 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 10744 pf->vsi[vsi->idx]->idx, 10745 pf->vsi[vsi->idx]->type, 10746 vsi->idx, vsi->type); 10747 goto unlock_vsi; 10748 } 10749 10750 /* updates the PF for this cleared vsi */ 10751 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 10752 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 10753 10754 bitmap_free(vsi->af_xdp_zc_qps); 10755 i40e_vsi_free_arrays(vsi, true); 10756 i40e_clear_rss_config_user(vsi); 10757 10758 pf->vsi[vsi->idx] = NULL; 10759 if (vsi->idx < pf->next_vsi) 10760 pf->next_vsi = vsi->idx; 10761 10762 unlock_vsi: 10763 mutex_unlock(&pf->switch_mutex); 10764 free_vsi: 10765 kfree(vsi); 10766 10767 return 0; 10768 } 10769 10770 /** 10771 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 10772 * @vsi: the VSI being cleaned 10773 **/ 10774 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 10775 { 10776 int i; 10777 10778 if (vsi->tx_rings && vsi->tx_rings[0]) { 10779 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10780 kfree_rcu(vsi->tx_rings[i], rcu); 10781 WRITE_ONCE(vsi->tx_rings[i], NULL); 10782 WRITE_ONCE(vsi->rx_rings[i], NULL); 10783 if (vsi->xdp_rings) 10784 WRITE_ONCE(vsi->xdp_rings[i], NULL); 10785 } 10786 } 10787 } 10788 10789 /** 10790 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 10791 * @vsi: the VSI being configured 10792 **/ 10793 static int i40e_alloc_rings(struct i40e_vsi *vsi) 10794 { 10795 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 10796 struct i40e_pf *pf = vsi->back; 10797 struct i40e_ring *ring; 10798 10799 /* Set basic values in the rings to be used later during open() */ 10800 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10801 /* allocate space for both Tx and Rx in one shot */ 10802 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 10803 if (!ring) 10804 goto err_out; 10805 10806 ring->queue_index = i; 10807 ring->reg_idx = vsi->base_queue + i; 10808 ring->ring_active = false; 10809 ring->vsi = vsi; 10810 ring->netdev = vsi->netdev; 10811 ring->dev = &pf->pdev->dev; 10812 ring->count = vsi->num_tx_desc; 10813 ring->size = 0; 10814 ring->dcb_tc = 0; 10815 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10816 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10817 ring->itr_setting = pf->tx_itr_default; 10818 WRITE_ONCE(vsi->tx_rings[i], ring++); 10819 10820 if (!i40e_enabled_xdp_vsi(vsi)) 10821 goto setup_rx; 10822 10823 ring->queue_index = vsi->alloc_queue_pairs + i; 10824 ring->reg_idx = vsi->base_queue + ring->queue_index; 10825 ring->ring_active = false; 10826 ring->vsi = vsi; 10827 ring->netdev = NULL; 10828 ring->dev = &pf->pdev->dev; 10829 ring->count = vsi->num_tx_desc; 10830 ring->size = 0; 10831 ring->dcb_tc = 0; 10832 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10833 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10834 set_ring_xdp(ring); 10835 ring->itr_setting = pf->tx_itr_default; 10836 WRITE_ONCE(vsi->xdp_rings[i], ring++); 10837 10838 setup_rx: 10839 ring->queue_index = i; 10840 ring->reg_idx = vsi->base_queue + i; 10841 ring->ring_active = false; 10842 ring->vsi = vsi; 10843 ring->netdev = vsi->netdev; 10844 ring->dev = &pf->pdev->dev; 10845 ring->count = vsi->num_rx_desc; 10846 ring->size = 0; 10847 ring->dcb_tc = 0; 10848 ring->itr_setting = pf->rx_itr_default; 10849 WRITE_ONCE(vsi->rx_rings[i], ring); 10850 } 10851 10852 return 0; 10853 10854 err_out: 10855 i40e_vsi_clear_rings(vsi); 10856 return -ENOMEM; 10857 } 10858 10859 /** 10860 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 10861 * @pf: board private structure 10862 * @vectors: the number of MSI-X vectors to request 10863 * 10864 * Returns the number of vectors reserved, or error 10865 **/ 10866 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 10867 { 10868 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 10869 I40E_MIN_MSIX, vectors); 10870 if (vectors < 0) { 10871 dev_info(&pf->pdev->dev, 10872 "MSI-X vector reservation failed: %d\n", vectors); 10873 vectors = 0; 10874 } 10875 10876 return vectors; 10877 } 10878 10879 /** 10880 * i40e_init_msix - Setup the MSIX capability 10881 * @pf: board private structure 10882 * 10883 * Work with the OS to set up the MSIX vectors needed. 10884 * 10885 * Returns the number of vectors reserved or negative on failure 10886 **/ 10887 static int i40e_init_msix(struct i40e_pf *pf) 10888 { 10889 struct i40e_hw *hw = &pf->hw; 10890 int cpus, extra_vectors; 10891 int vectors_left; 10892 int v_budget, i; 10893 int v_actual; 10894 int iwarp_requested = 0; 10895 10896 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10897 return -ENODEV; 10898 10899 /* The number of vectors we'll request will be comprised of: 10900 * - Add 1 for "other" cause for Admin Queue events, etc. 10901 * - The number of LAN queue pairs 10902 * - Queues being used for RSS. 10903 * We don't need as many as max_rss_size vectors. 10904 * use rss_size instead in the calculation since that 10905 * is governed by number of cpus in the system. 10906 * - assumes symmetric Tx/Rx pairing 10907 * - The number of VMDq pairs 10908 * - The CPU count within the NUMA node if iWARP is enabled 10909 * Once we count this up, try the request. 10910 * 10911 * If we can't get what we want, we'll simplify to nearly nothing 10912 * and try again. If that still fails, we punt. 10913 */ 10914 vectors_left = hw->func_caps.num_msix_vectors; 10915 v_budget = 0; 10916 10917 /* reserve one vector for miscellaneous handler */ 10918 if (vectors_left) { 10919 v_budget++; 10920 vectors_left--; 10921 } 10922 10923 /* reserve some vectors for the main PF traffic queues. Initially we 10924 * only reserve at most 50% of the available vectors, in the case that 10925 * the number of online CPUs is large. This ensures that we can enable 10926 * extra features as well. Once we've enabled the other features, we 10927 * will use any remaining vectors to reach as close as we can to the 10928 * number of online CPUs. 10929 */ 10930 cpus = num_online_cpus(); 10931 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 10932 vectors_left -= pf->num_lan_msix; 10933 10934 /* reserve one vector for sideband flow director */ 10935 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10936 if (vectors_left) { 10937 pf->num_fdsb_msix = 1; 10938 v_budget++; 10939 vectors_left--; 10940 } else { 10941 pf->num_fdsb_msix = 0; 10942 } 10943 } 10944 10945 /* can we reserve enough for iWARP? */ 10946 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10947 iwarp_requested = pf->num_iwarp_msix; 10948 10949 if (!vectors_left) 10950 pf->num_iwarp_msix = 0; 10951 else if (vectors_left < pf->num_iwarp_msix) 10952 pf->num_iwarp_msix = 1; 10953 v_budget += pf->num_iwarp_msix; 10954 vectors_left -= pf->num_iwarp_msix; 10955 } 10956 10957 /* any vectors left over go for VMDq support */ 10958 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 10959 if (!vectors_left) { 10960 pf->num_vmdq_msix = 0; 10961 pf->num_vmdq_qps = 0; 10962 } else { 10963 int vmdq_vecs_wanted = 10964 pf->num_vmdq_vsis * pf->num_vmdq_qps; 10965 int vmdq_vecs = 10966 min_t(int, vectors_left, vmdq_vecs_wanted); 10967 10968 /* if we're short on vectors for what's desired, we limit 10969 * the queues per vmdq. If this is still more than are 10970 * available, the user will need to change the number of 10971 * queues/vectors used by the PF later with the ethtool 10972 * channels command 10973 */ 10974 if (vectors_left < vmdq_vecs_wanted) { 10975 pf->num_vmdq_qps = 1; 10976 vmdq_vecs_wanted = pf->num_vmdq_vsis; 10977 vmdq_vecs = min_t(int, 10978 vectors_left, 10979 vmdq_vecs_wanted); 10980 } 10981 pf->num_vmdq_msix = pf->num_vmdq_qps; 10982 10983 v_budget += vmdq_vecs; 10984 vectors_left -= vmdq_vecs; 10985 } 10986 } 10987 10988 /* On systems with a large number of SMP cores, we previously limited 10989 * the number of vectors for num_lan_msix to be at most 50% of the 10990 * available vectors, to allow for other features. Now, we add back 10991 * the remaining vectors. However, we ensure that the total 10992 * num_lan_msix will not exceed num_online_cpus(). To do this, we 10993 * calculate the number of vectors we can add without going over the 10994 * cap of CPUs. For systems with a small number of CPUs this will be 10995 * zero. 10996 */ 10997 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 10998 pf->num_lan_msix += extra_vectors; 10999 vectors_left -= extra_vectors; 11000 11001 WARN(vectors_left < 0, 11002 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 11003 11004 v_budget += pf->num_lan_msix; 11005 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 11006 GFP_KERNEL); 11007 if (!pf->msix_entries) 11008 return -ENOMEM; 11009 11010 for (i = 0; i < v_budget; i++) 11011 pf->msix_entries[i].entry = i; 11012 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 11013 11014 if (v_actual < I40E_MIN_MSIX) { 11015 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 11016 kfree(pf->msix_entries); 11017 pf->msix_entries = NULL; 11018 pci_disable_msix(pf->pdev); 11019 return -ENODEV; 11020 11021 } else if (v_actual == I40E_MIN_MSIX) { 11022 /* Adjust for minimal MSIX use */ 11023 pf->num_vmdq_vsis = 0; 11024 pf->num_vmdq_qps = 0; 11025 pf->num_lan_qps = 1; 11026 pf->num_lan_msix = 1; 11027 11028 } else if (v_actual != v_budget) { 11029 /* If we have limited resources, we will start with no vectors 11030 * for the special features and then allocate vectors to some 11031 * of these features based on the policy and at the end disable 11032 * the features that did not get any vectors. 11033 */ 11034 int vec; 11035 11036 dev_info(&pf->pdev->dev, 11037 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 11038 v_actual, v_budget); 11039 /* reserve the misc vector */ 11040 vec = v_actual - 1; 11041 11042 /* Scale vector usage down */ 11043 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 11044 pf->num_vmdq_vsis = 1; 11045 pf->num_vmdq_qps = 1; 11046 11047 /* partition out the remaining vectors */ 11048 switch (vec) { 11049 case 2: 11050 pf->num_lan_msix = 1; 11051 break; 11052 case 3: 11053 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11054 pf->num_lan_msix = 1; 11055 pf->num_iwarp_msix = 1; 11056 } else { 11057 pf->num_lan_msix = 2; 11058 } 11059 break; 11060 default: 11061 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11062 pf->num_iwarp_msix = min_t(int, (vec / 3), 11063 iwarp_requested); 11064 pf->num_vmdq_vsis = min_t(int, (vec / 3), 11065 I40E_DEFAULT_NUM_VMDQ_VSI); 11066 } else { 11067 pf->num_vmdq_vsis = min_t(int, (vec / 2), 11068 I40E_DEFAULT_NUM_VMDQ_VSI); 11069 } 11070 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11071 pf->num_fdsb_msix = 1; 11072 vec--; 11073 } 11074 pf->num_lan_msix = min_t(int, 11075 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 11076 pf->num_lan_msix); 11077 pf->num_lan_qps = pf->num_lan_msix; 11078 break; 11079 } 11080 } 11081 11082 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 11083 (pf->num_fdsb_msix == 0)) { 11084 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 11085 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 11086 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11087 } 11088 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 11089 (pf->num_vmdq_msix == 0)) { 11090 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 11091 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 11092 } 11093 11094 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 11095 (pf->num_iwarp_msix == 0)) { 11096 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 11097 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11098 } 11099 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 11100 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 11101 pf->num_lan_msix, 11102 pf->num_vmdq_msix * pf->num_vmdq_vsis, 11103 pf->num_fdsb_msix, 11104 pf->num_iwarp_msix); 11105 11106 return v_actual; 11107 } 11108 11109 /** 11110 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 11111 * @vsi: the VSI being configured 11112 * @v_idx: index of the vector in the vsi struct 11113 * 11114 * We allocate one q_vector. If allocation fails we return -ENOMEM. 11115 **/ 11116 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx) 11117 { 11118 struct i40e_q_vector *q_vector; 11119 11120 /* allocate q_vector */ 11121 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 11122 if (!q_vector) 11123 return -ENOMEM; 11124 11125 q_vector->vsi = vsi; 11126 q_vector->v_idx = v_idx; 11127 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 11128 11129 if (vsi->netdev) 11130 netif_napi_add(vsi->netdev, &q_vector->napi, 11131 i40e_napi_poll, NAPI_POLL_WEIGHT); 11132 11133 /* tie q_vector and vsi together */ 11134 vsi->q_vectors[v_idx] = q_vector; 11135 11136 return 0; 11137 } 11138 11139 /** 11140 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 11141 * @vsi: the VSI being configured 11142 * 11143 * We allocate one q_vector per queue interrupt. If allocation fails we 11144 * return -ENOMEM. 11145 **/ 11146 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 11147 { 11148 struct i40e_pf *pf = vsi->back; 11149 int err, v_idx, num_q_vectors; 11150 11151 /* if not MSIX, give the one vector only to the LAN VSI */ 11152 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 11153 num_q_vectors = vsi->num_q_vectors; 11154 else if (vsi == pf->vsi[pf->lan_vsi]) 11155 num_q_vectors = 1; 11156 else 11157 return -EINVAL; 11158 11159 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 11160 err = i40e_vsi_alloc_q_vector(vsi, v_idx); 11161 if (err) 11162 goto err_out; 11163 } 11164 11165 return 0; 11166 11167 err_out: 11168 while (v_idx--) 11169 i40e_free_q_vector(vsi, v_idx); 11170 11171 return err; 11172 } 11173 11174 /** 11175 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 11176 * @pf: board private structure to initialize 11177 **/ 11178 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 11179 { 11180 int vectors = 0; 11181 ssize_t size; 11182 11183 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11184 vectors = i40e_init_msix(pf); 11185 if (vectors < 0) { 11186 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 11187 I40E_FLAG_IWARP_ENABLED | 11188 I40E_FLAG_RSS_ENABLED | 11189 I40E_FLAG_DCB_CAPABLE | 11190 I40E_FLAG_DCB_ENABLED | 11191 I40E_FLAG_SRIOV_ENABLED | 11192 I40E_FLAG_FD_SB_ENABLED | 11193 I40E_FLAG_FD_ATR_ENABLED | 11194 I40E_FLAG_VMDQ_ENABLED); 11195 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11196 11197 /* rework the queue expectations without MSIX */ 11198 i40e_determine_queue_usage(pf); 11199 } 11200 } 11201 11202 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 11203 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 11204 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 11205 vectors = pci_enable_msi(pf->pdev); 11206 if (vectors < 0) { 11207 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 11208 vectors); 11209 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 11210 } 11211 vectors = 1; /* one MSI or Legacy vector */ 11212 } 11213 11214 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 11215 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 11216 11217 /* set up vector assignment tracking */ 11218 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 11219 pf->irq_pile = kzalloc(size, GFP_KERNEL); 11220 if (!pf->irq_pile) 11221 return -ENOMEM; 11222 11223 pf->irq_pile->num_entries = vectors; 11224 pf->irq_pile->search_hint = 0; 11225 11226 /* track first vector for misc interrupts, ignore return */ 11227 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 11228 11229 return 0; 11230 } 11231 11232 /** 11233 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 11234 * @pf: private board data structure 11235 * 11236 * Restore the interrupt scheme that was cleared when we suspended the 11237 * device. This should be called during resume to re-allocate the q_vectors 11238 * and reacquire IRQs. 11239 */ 11240 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 11241 { 11242 int err, i; 11243 11244 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 11245 * scheme. We need to re-enabled them here in order to attempt to 11246 * re-acquire the MSI or MSI-X vectors 11247 */ 11248 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 11249 11250 err = i40e_init_interrupt_scheme(pf); 11251 if (err) 11252 return err; 11253 11254 /* Now that we've re-acquired IRQs, we need to remap the vectors and 11255 * rings together again. 11256 */ 11257 for (i = 0; i < pf->num_alloc_vsi; i++) { 11258 if (pf->vsi[i]) { 11259 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 11260 if (err) 11261 goto err_unwind; 11262 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 11263 } 11264 } 11265 11266 err = i40e_setup_misc_vector(pf); 11267 if (err) 11268 goto err_unwind; 11269 11270 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 11271 i40e_client_update_msix_info(pf); 11272 11273 return 0; 11274 11275 err_unwind: 11276 while (i--) { 11277 if (pf->vsi[i]) 11278 i40e_vsi_free_q_vectors(pf->vsi[i]); 11279 } 11280 11281 return err; 11282 } 11283 11284 /** 11285 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle 11286 * non queue events in recovery mode 11287 * @pf: board private structure 11288 * 11289 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage 11290 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode. 11291 * This is handled differently than in recovery mode since no Tx/Rx resources 11292 * are being allocated. 11293 **/ 11294 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf) 11295 { 11296 int err; 11297 11298 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11299 err = i40e_setup_misc_vector(pf); 11300 11301 if (err) { 11302 dev_info(&pf->pdev->dev, 11303 "MSI-X misc vector request failed, error %d\n", 11304 err); 11305 return err; 11306 } 11307 } else { 11308 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED; 11309 11310 err = request_irq(pf->pdev->irq, i40e_intr, flags, 11311 pf->int_name, pf); 11312 11313 if (err) { 11314 dev_info(&pf->pdev->dev, 11315 "MSI/legacy misc vector request failed, error %d\n", 11316 err); 11317 return err; 11318 } 11319 i40e_enable_misc_int_causes(pf); 11320 i40e_irq_dynamic_enable_icr0(pf); 11321 } 11322 11323 return 0; 11324 } 11325 11326 /** 11327 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 11328 * @pf: board private structure 11329 * 11330 * This sets up the handler for MSIX 0, which is used to manage the 11331 * non-queue interrupts, e.g. AdminQ and errors. This is not used 11332 * when in MSI or Legacy interrupt mode. 11333 **/ 11334 static int i40e_setup_misc_vector(struct i40e_pf *pf) 11335 { 11336 struct i40e_hw *hw = &pf->hw; 11337 int err = 0; 11338 11339 /* Only request the IRQ once, the first time through. */ 11340 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 11341 err = request_irq(pf->msix_entries[0].vector, 11342 i40e_intr, 0, pf->int_name, pf); 11343 if (err) { 11344 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 11345 dev_info(&pf->pdev->dev, 11346 "request_irq for %s failed: %d\n", 11347 pf->int_name, err); 11348 return -EFAULT; 11349 } 11350 } 11351 11352 i40e_enable_misc_int_causes(pf); 11353 11354 /* associate no queues to the misc vector */ 11355 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 11356 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1); 11357 11358 i40e_flush(hw); 11359 11360 i40e_irq_dynamic_enable_icr0(pf); 11361 11362 return err; 11363 } 11364 11365 /** 11366 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 11367 * @vsi: Pointer to vsi structure 11368 * @seed: Buffter to store the hash keys 11369 * @lut: Buffer to store the lookup table entries 11370 * @lut_size: Size of buffer to store the lookup table entries 11371 * 11372 * Return 0 on success, negative on failure 11373 */ 11374 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 11375 u8 *lut, u16 lut_size) 11376 { 11377 struct i40e_pf *pf = vsi->back; 11378 struct i40e_hw *hw = &pf->hw; 11379 int ret = 0; 11380 11381 if (seed) { 11382 ret = i40e_aq_get_rss_key(hw, vsi->id, 11383 (struct i40e_aqc_get_set_rss_key_data *)seed); 11384 if (ret) { 11385 dev_info(&pf->pdev->dev, 11386 "Cannot get RSS key, err %s aq_err %s\n", 11387 i40e_stat_str(&pf->hw, ret), 11388 i40e_aq_str(&pf->hw, 11389 pf->hw.aq.asq_last_status)); 11390 return ret; 11391 } 11392 } 11393 11394 if (lut) { 11395 bool pf_lut = vsi->type == I40E_VSI_MAIN; 11396 11397 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 11398 if (ret) { 11399 dev_info(&pf->pdev->dev, 11400 "Cannot get RSS lut, err %s aq_err %s\n", 11401 i40e_stat_str(&pf->hw, ret), 11402 i40e_aq_str(&pf->hw, 11403 pf->hw.aq.asq_last_status)); 11404 return ret; 11405 } 11406 } 11407 11408 return ret; 11409 } 11410 11411 /** 11412 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 11413 * @vsi: Pointer to vsi structure 11414 * @seed: RSS hash seed 11415 * @lut: Lookup table 11416 * @lut_size: Lookup table size 11417 * 11418 * Returns 0 on success, negative on failure 11419 **/ 11420 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 11421 const u8 *lut, u16 lut_size) 11422 { 11423 struct i40e_pf *pf = vsi->back; 11424 struct i40e_hw *hw = &pf->hw; 11425 u16 vf_id = vsi->vf_id; 11426 u8 i; 11427 11428 /* Fill out hash function seed */ 11429 if (seed) { 11430 u32 *seed_dw = (u32 *)seed; 11431 11432 if (vsi->type == I40E_VSI_MAIN) { 11433 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 11434 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 11435 } else if (vsi->type == I40E_VSI_SRIOV) { 11436 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 11437 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 11438 } else { 11439 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 11440 } 11441 } 11442 11443 if (lut) { 11444 u32 *lut_dw = (u32 *)lut; 11445 11446 if (vsi->type == I40E_VSI_MAIN) { 11447 if (lut_size != I40E_HLUT_ARRAY_SIZE) 11448 return -EINVAL; 11449 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11450 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 11451 } else if (vsi->type == I40E_VSI_SRIOV) { 11452 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 11453 return -EINVAL; 11454 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 11455 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 11456 } else { 11457 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 11458 } 11459 } 11460 i40e_flush(hw); 11461 11462 return 0; 11463 } 11464 11465 /** 11466 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 11467 * @vsi: Pointer to VSI structure 11468 * @seed: Buffer to store the keys 11469 * @lut: Buffer to store the lookup table entries 11470 * @lut_size: Size of buffer to store the lookup table entries 11471 * 11472 * Returns 0 on success, negative on failure 11473 */ 11474 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 11475 u8 *lut, u16 lut_size) 11476 { 11477 struct i40e_pf *pf = vsi->back; 11478 struct i40e_hw *hw = &pf->hw; 11479 u16 i; 11480 11481 if (seed) { 11482 u32 *seed_dw = (u32 *)seed; 11483 11484 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 11485 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 11486 } 11487 if (lut) { 11488 u32 *lut_dw = (u32 *)lut; 11489 11490 if (lut_size != I40E_HLUT_ARRAY_SIZE) 11491 return -EINVAL; 11492 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11493 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 11494 } 11495 11496 return 0; 11497 } 11498 11499 /** 11500 * i40e_config_rss - Configure RSS keys and lut 11501 * @vsi: Pointer to VSI structure 11502 * @seed: RSS hash seed 11503 * @lut: Lookup table 11504 * @lut_size: Lookup table size 11505 * 11506 * Returns 0 on success, negative on failure 11507 */ 11508 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 11509 { 11510 struct i40e_pf *pf = vsi->back; 11511 11512 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 11513 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 11514 else 11515 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 11516 } 11517 11518 /** 11519 * i40e_get_rss - Get RSS keys and lut 11520 * @vsi: Pointer to VSI structure 11521 * @seed: Buffer to store the keys 11522 * @lut: Buffer to store the lookup table entries 11523 * @lut_size: Size of buffer to store the lookup table entries 11524 * 11525 * Returns 0 on success, negative on failure 11526 */ 11527 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 11528 { 11529 struct i40e_pf *pf = vsi->back; 11530 11531 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 11532 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 11533 else 11534 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 11535 } 11536 11537 /** 11538 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 11539 * @pf: Pointer to board private structure 11540 * @lut: Lookup table 11541 * @rss_table_size: Lookup table size 11542 * @rss_size: Range of queue number for hashing 11543 */ 11544 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 11545 u16 rss_table_size, u16 rss_size) 11546 { 11547 u16 i; 11548 11549 for (i = 0; i < rss_table_size; i++) 11550 lut[i] = i % rss_size; 11551 } 11552 11553 /** 11554 * i40e_pf_config_rss - Prepare for RSS if used 11555 * @pf: board private structure 11556 **/ 11557 static int i40e_pf_config_rss(struct i40e_pf *pf) 11558 { 11559 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11560 u8 seed[I40E_HKEY_ARRAY_SIZE]; 11561 u8 *lut; 11562 struct i40e_hw *hw = &pf->hw; 11563 u32 reg_val; 11564 u64 hena; 11565 int ret; 11566 11567 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 11568 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 11569 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 11570 hena |= i40e_pf_get_default_rss_hena(pf); 11571 11572 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 11573 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 11574 11575 /* Determine the RSS table size based on the hardware capabilities */ 11576 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 11577 reg_val = (pf->rss_table_size == 512) ? 11578 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 11579 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 11580 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 11581 11582 /* Determine the RSS size of the VSI */ 11583 if (!vsi->rss_size) { 11584 u16 qcount; 11585 /* If the firmware does something weird during VSI init, we 11586 * could end up with zero TCs. Check for that to avoid 11587 * divide-by-zero. It probably won't pass traffic, but it also 11588 * won't panic. 11589 */ 11590 qcount = vsi->num_queue_pairs / 11591 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 11592 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11593 } 11594 if (!vsi->rss_size) 11595 return -EINVAL; 11596 11597 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 11598 if (!lut) 11599 return -ENOMEM; 11600 11601 /* Use user configured lut if there is one, otherwise use default */ 11602 if (vsi->rss_lut_user) 11603 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 11604 else 11605 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 11606 11607 /* Use user configured hash key if there is one, otherwise 11608 * use default. 11609 */ 11610 if (vsi->rss_hkey_user) 11611 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 11612 else 11613 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 11614 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 11615 kfree(lut); 11616 11617 return ret; 11618 } 11619 11620 /** 11621 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 11622 * @pf: board private structure 11623 * @queue_count: the requested queue count for rss. 11624 * 11625 * returns 0 if rss is not enabled, if enabled returns the final rss queue 11626 * count which may be different from the requested queue count. 11627 * Note: expects to be called while under rtnl_lock() 11628 **/ 11629 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 11630 { 11631 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11632 int new_rss_size; 11633 11634 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 11635 return 0; 11636 11637 queue_count = min_t(int, queue_count, num_online_cpus()); 11638 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 11639 11640 if (queue_count != vsi->num_queue_pairs) { 11641 u16 qcount; 11642 11643 vsi->req_queue_pairs = queue_count; 11644 i40e_prep_for_reset(pf, true); 11645 11646 pf->alloc_rss_size = new_rss_size; 11647 11648 i40e_reset_and_rebuild(pf, true, true); 11649 11650 /* Discard the user configured hash keys and lut, if less 11651 * queues are enabled. 11652 */ 11653 if (queue_count < vsi->rss_size) { 11654 i40e_clear_rss_config_user(vsi); 11655 dev_dbg(&pf->pdev->dev, 11656 "discard user configured hash keys and lut\n"); 11657 } 11658 11659 /* Reset vsi->rss_size, as number of enabled queues changed */ 11660 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 11661 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11662 11663 i40e_pf_config_rss(pf); 11664 } 11665 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 11666 vsi->req_queue_pairs, pf->rss_size_max); 11667 return pf->alloc_rss_size; 11668 } 11669 11670 /** 11671 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 11672 * @pf: board private structure 11673 **/ 11674 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 11675 { 11676 i40e_status status; 11677 bool min_valid, max_valid; 11678 u32 max_bw, min_bw; 11679 11680 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 11681 &min_valid, &max_valid); 11682 11683 if (!status) { 11684 if (min_valid) 11685 pf->min_bw = min_bw; 11686 if (max_valid) 11687 pf->max_bw = max_bw; 11688 } 11689 11690 return status; 11691 } 11692 11693 /** 11694 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 11695 * @pf: board private structure 11696 **/ 11697 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 11698 { 11699 struct i40e_aqc_configure_partition_bw_data bw_data; 11700 i40e_status status; 11701 11702 /* Set the valid bit for this PF */ 11703 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 11704 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 11705 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 11706 11707 /* Set the new bandwidths */ 11708 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 11709 11710 return status; 11711 } 11712 11713 /** 11714 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 11715 * @pf: board private structure 11716 **/ 11717 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 11718 { 11719 /* Commit temporary BW setting to permanent NVM image */ 11720 enum i40e_admin_queue_err last_aq_status; 11721 i40e_status ret; 11722 u16 nvm_word; 11723 11724 if (pf->hw.partition_id != 1) { 11725 dev_info(&pf->pdev->dev, 11726 "Commit BW only works on partition 1! This is partition %d", 11727 pf->hw.partition_id); 11728 ret = I40E_NOT_SUPPORTED; 11729 goto bw_commit_out; 11730 } 11731 11732 /* Acquire NVM for read access */ 11733 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 11734 last_aq_status = pf->hw.aq.asq_last_status; 11735 if (ret) { 11736 dev_info(&pf->pdev->dev, 11737 "Cannot acquire NVM for read access, err %s aq_err %s\n", 11738 i40e_stat_str(&pf->hw, ret), 11739 i40e_aq_str(&pf->hw, last_aq_status)); 11740 goto bw_commit_out; 11741 } 11742 11743 /* Read word 0x10 of NVM - SW compatibility word 1 */ 11744 ret = i40e_aq_read_nvm(&pf->hw, 11745 I40E_SR_NVM_CONTROL_WORD, 11746 0x10, sizeof(nvm_word), &nvm_word, 11747 false, NULL); 11748 /* Save off last admin queue command status before releasing 11749 * the NVM 11750 */ 11751 last_aq_status = pf->hw.aq.asq_last_status; 11752 i40e_release_nvm(&pf->hw); 11753 if (ret) { 11754 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 11755 i40e_stat_str(&pf->hw, ret), 11756 i40e_aq_str(&pf->hw, last_aq_status)); 11757 goto bw_commit_out; 11758 } 11759 11760 /* Wait a bit for NVM release to complete */ 11761 msleep(50); 11762 11763 /* Acquire NVM for write access */ 11764 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 11765 last_aq_status = pf->hw.aq.asq_last_status; 11766 if (ret) { 11767 dev_info(&pf->pdev->dev, 11768 "Cannot acquire NVM for write access, err %s aq_err %s\n", 11769 i40e_stat_str(&pf->hw, ret), 11770 i40e_aq_str(&pf->hw, last_aq_status)); 11771 goto bw_commit_out; 11772 } 11773 /* Write it back out unchanged to initiate update NVM, 11774 * which will force a write of the shadow (alt) RAM to 11775 * the NVM - thus storing the bandwidth values permanently. 11776 */ 11777 ret = i40e_aq_update_nvm(&pf->hw, 11778 I40E_SR_NVM_CONTROL_WORD, 11779 0x10, sizeof(nvm_word), 11780 &nvm_word, true, 0, NULL); 11781 /* Save off last admin queue command status before releasing 11782 * the NVM 11783 */ 11784 last_aq_status = pf->hw.aq.asq_last_status; 11785 i40e_release_nvm(&pf->hw); 11786 if (ret) 11787 dev_info(&pf->pdev->dev, 11788 "BW settings NOT SAVED, err %s aq_err %s\n", 11789 i40e_stat_str(&pf->hw, ret), 11790 i40e_aq_str(&pf->hw, last_aq_status)); 11791 bw_commit_out: 11792 11793 return ret; 11794 } 11795 11796 /** 11797 * i40e_is_total_port_shutdown_enabled - read NVM and return value 11798 * if total port shutdown feature is enabled for this PF 11799 * @pf: board private structure 11800 **/ 11801 static bool i40e_is_total_port_shutdown_enabled(struct i40e_pf *pf) 11802 { 11803 #define I40E_TOTAL_PORT_SHUTDOWN_ENABLED BIT(4) 11804 #define I40E_FEATURES_ENABLE_PTR 0x2A 11805 #define I40E_CURRENT_SETTING_PTR 0x2B 11806 #define I40E_LINK_BEHAVIOR_WORD_OFFSET 0x2D 11807 #define I40E_LINK_BEHAVIOR_WORD_LENGTH 0x1 11808 #define I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED BIT(0) 11809 #define I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH 4 11810 i40e_status read_status = I40E_SUCCESS; 11811 u16 sr_emp_sr_settings_ptr = 0; 11812 u16 features_enable = 0; 11813 u16 link_behavior = 0; 11814 bool ret = false; 11815 11816 read_status = i40e_read_nvm_word(&pf->hw, 11817 I40E_SR_EMP_SR_SETTINGS_PTR, 11818 &sr_emp_sr_settings_ptr); 11819 if (read_status) 11820 goto err_nvm; 11821 read_status = i40e_read_nvm_word(&pf->hw, 11822 sr_emp_sr_settings_ptr + 11823 I40E_FEATURES_ENABLE_PTR, 11824 &features_enable); 11825 if (read_status) 11826 goto err_nvm; 11827 if (I40E_TOTAL_PORT_SHUTDOWN_ENABLED & features_enable) { 11828 read_status = i40e_read_nvm_module_data(&pf->hw, 11829 I40E_SR_EMP_SR_SETTINGS_PTR, 11830 I40E_CURRENT_SETTING_PTR, 11831 I40E_LINK_BEHAVIOR_WORD_OFFSET, 11832 I40E_LINK_BEHAVIOR_WORD_LENGTH, 11833 &link_behavior); 11834 if (read_status) 11835 goto err_nvm; 11836 link_behavior >>= (pf->hw.port * I40E_LINK_BEHAVIOR_PORT_BIT_LENGTH); 11837 ret = I40E_LINK_BEHAVIOR_OS_FORCED_ENABLED & link_behavior; 11838 } 11839 return ret; 11840 11841 err_nvm: 11842 dev_warn(&pf->pdev->dev, 11843 "total-port-shutdown feature is off due to read nvm error: %s\n", 11844 i40e_stat_str(&pf->hw, read_status)); 11845 return ret; 11846 } 11847 11848 /** 11849 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 11850 * @pf: board private structure to initialize 11851 * 11852 * i40e_sw_init initializes the Adapter private data structure. 11853 * Fields are initialized based on PCI device information and 11854 * OS network device settings (MTU size). 11855 **/ 11856 static int i40e_sw_init(struct i40e_pf *pf) 11857 { 11858 int err = 0; 11859 int size; 11860 11861 /* Set default capability flags */ 11862 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 11863 I40E_FLAG_MSI_ENABLED | 11864 I40E_FLAG_MSIX_ENABLED; 11865 11866 /* Set default ITR */ 11867 pf->rx_itr_default = I40E_ITR_RX_DEF; 11868 pf->tx_itr_default = I40E_ITR_TX_DEF; 11869 11870 /* Depending on PF configurations, it is possible that the RSS 11871 * maximum might end up larger than the available queues 11872 */ 11873 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 11874 pf->alloc_rss_size = 1; 11875 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 11876 pf->rss_size_max = min_t(int, pf->rss_size_max, 11877 pf->hw.func_caps.num_tx_qp); 11878 if (pf->hw.func_caps.rss) { 11879 pf->flags |= I40E_FLAG_RSS_ENABLED; 11880 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 11881 num_online_cpus()); 11882 } 11883 11884 /* MFP mode enabled */ 11885 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 11886 pf->flags |= I40E_FLAG_MFP_ENABLED; 11887 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 11888 if (i40e_get_partition_bw_setting(pf)) { 11889 dev_warn(&pf->pdev->dev, 11890 "Could not get partition bw settings\n"); 11891 } else { 11892 dev_info(&pf->pdev->dev, 11893 "Partition BW Min = %8.8x, Max = %8.8x\n", 11894 pf->min_bw, pf->max_bw); 11895 11896 /* nudge the Tx scheduler */ 11897 i40e_set_partition_bw_setting(pf); 11898 } 11899 } 11900 11901 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 11902 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 11903 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 11904 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 11905 if (pf->flags & I40E_FLAG_MFP_ENABLED && 11906 pf->hw.num_partitions > 1) 11907 dev_info(&pf->pdev->dev, 11908 "Flow Director Sideband mode Disabled in MFP mode\n"); 11909 else 11910 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11911 pf->fdir_pf_filter_count = 11912 pf->hw.func_caps.fd_filters_guaranteed; 11913 pf->hw.fdir_shared_filter_count = 11914 pf->hw.func_caps.fd_filters_best_effort; 11915 } 11916 11917 if (pf->hw.mac.type == I40E_MAC_X722) { 11918 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 11919 I40E_HW_128_QP_RSS_CAPABLE | 11920 I40E_HW_ATR_EVICT_CAPABLE | 11921 I40E_HW_WB_ON_ITR_CAPABLE | 11922 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 11923 I40E_HW_NO_PCI_LINK_CHECK | 11924 I40E_HW_USE_SET_LLDP_MIB | 11925 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 11926 I40E_HW_PTP_L4_CAPABLE | 11927 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 11928 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 11929 11930 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 11931 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 11932 I40E_FDEVICT_PCTYPE_DEFAULT) { 11933 dev_warn(&pf->pdev->dev, 11934 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 11935 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 11936 } 11937 } else if ((pf->hw.aq.api_maj_ver > 1) || 11938 ((pf->hw.aq.api_maj_ver == 1) && 11939 (pf->hw.aq.api_min_ver > 4))) { 11940 /* Supported in FW API version higher than 1.4 */ 11941 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 11942 } 11943 11944 /* Enable HW ATR eviction if possible */ 11945 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 11946 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 11947 11948 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11949 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 11950 (pf->hw.aq.fw_maj_ver < 4))) { 11951 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 11952 /* No DCB support for FW < v4.33 */ 11953 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 11954 } 11955 11956 /* Disable FW LLDP if FW < v4.3 */ 11957 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11958 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 11959 (pf->hw.aq.fw_maj_ver < 4))) 11960 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 11961 11962 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 11963 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11964 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 11965 (pf->hw.aq.fw_maj_ver >= 5))) 11966 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 11967 11968 /* Enable PTP L4 if FW > v6.0 */ 11969 if (pf->hw.mac.type == I40E_MAC_XL710 && 11970 pf->hw.aq.fw_maj_ver >= 6) 11971 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 11972 11973 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 11974 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 11975 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 11976 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 11977 } 11978 11979 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 11980 pf->flags |= I40E_FLAG_IWARP_ENABLED; 11981 /* IWARP needs one extra vector for CQP just like MISC.*/ 11982 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 11983 } 11984 /* Stopping FW LLDP engine is supported on XL710 and X722 11985 * starting from FW versions determined in i40e_init_adminq. 11986 * Stopping the FW LLDP engine is not supported on XL710 11987 * if NPAR is functioning so unset this hw flag in this case. 11988 */ 11989 if (pf->hw.mac.type == I40E_MAC_XL710 && 11990 pf->hw.func_caps.npar_enable && 11991 (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE)) 11992 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE; 11993 11994 #ifdef CONFIG_PCI_IOV 11995 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 11996 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 11997 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 11998 pf->num_req_vfs = min_t(int, 11999 pf->hw.func_caps.num_vfs, 12000 I40E_MAX_VF_COUNT); 12001 } 12002 #endif /* CONFIG_PCI_IOV */ 12003 pf->eeprom_version = 0xDEAD; 12004 pf->lan_veb = I40E_NO_VEB; 12005 pf->lan_vsi = I40E_NO_VSI; 12006 12007 /* By default FW has this off for performance reasons */ 12008 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 12009 12010 /* set up queue assignment tracking */ 12011 size = sizeof(struct i40e_lump_tracking) 12012 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 12013 pf->qp_pile = kzalloc(size, GFP_KERNEL); 12014 if (!pf->qp_pile) { 12015 err = -ENOMEM; 12016 goto sw_init_done; 12017 } 12018 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 12019 pf->qp_pile->search_hint = 0; 12020 12021 pf->tx_timeout_recovery_level = 1; 12022 12023 if (pf->hw.mac.type != I40E_MAC_X722 && 12024 i40e_is_total_port_shutdown_enabled(pf)) { 12025 /* Link down on close must be on when total port shutdown 12026 * is enabled for a given port 12027 */ 12028 pf->flags |= (I40E_FLAG_TOTAL_PORT_SHUTDOWN_ENABLED | 12029 I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED); 12030 dev_info(&pf->pdev->dev, 12031 "total-port-shutdown was enabled, link-down-on-close is forced on\n"); 12032 } 12033 mutex_init(&pf->switch_mutex); 12034 12035 sw_init_done: 12036 return err; 12037 } 12038 12039 /** 12040 * i40e_set_ntuple - set the ntuple feature flag and take action 12041 * @pf: board private structure to initialize 12042 * @features: the feature set that the stack is suggesting 12043 * 12044 * returns a bool to indicate if reset needs to happen 12045 **/ 12046 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 12047 { 12048 bool need_reset = false; 12049 12050 /* Check if Flow Director n-tuple support was enabled or disabled. If 12051 * the state changed, we need to reset. 12052 */ 12053 if (features & NETIF_F_NTUPLE) { 12054 /* Enable filters and mark for reset */ 12055 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 12056 need_reset = true; 12057 /* enable FD_SB only if there is MSI-X vector and no cloud 12058 * filters exist 12059 */ 12060 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 12061 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 12062 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 12063 } 12064 } else { 12065 /* turn off filters, mark for reset and clear SW filter list */ 12066 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 12067 need_reset = true; 12068 i40e_fdir_filter_exit(pf); 12069 } 12070 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 12071 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 12072 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 12073 12074 /* reset fd counters */ 12075 pf->fd_add_err = 0; 12076 pf->fd_atr_cnt = 0; 12077 /* if ATR was auto disabled it can be re-enabled. */ 12078 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 12079 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 12080 (I40E_DEBUG_FD & pf->hw.debug_mask)) 12081 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 12082 } 12083 return need_reset; 12084 } 12085 12086 /** 12087 * i40e_clear_rss_lut - clear the rx hash lookup table 12088 * @vsi: the VSI being configured 12089 **/ 12090 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 12091 { 12092 struct i40e_pf *pf = vsi->back; 12093 struct i40e_hw *hw = &pf->hw; 12094 u16 vf_id = vsi->vf_id; 12095 u8 i; 12096 12097 if (vsi->type == I40E_VSI_MAIN) { 12098 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12099 wr32(hw, I40E_PFQF_HLUT(i), 0); 12100 } else if (vsi->type == I40E_VSI_SRIOV) { 12101 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12102 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 12103 } else { 12104 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12105 } 12106 } 12107 12108 /** 12109 * i40e_set_features - set the netdev feature flags 12110 * @netdev: ptr to the netdev being adjusted 12111 * @features: the feature set that the stack is suggesting 12112 * Note: expects to be called while under rtnl_lock() 12113 **/ 12114 static int i40e_set_features(struct net_device *netdev, 12115 netdev_features_t features) 12116 { 12117 struct i40e_netdev_priv *np = netdev_priv(netdev); 12118 struct i40e_vsi *vsi = np->vsi; 12119 struct i40e_pf *pf = vsi->back; 12120 bool need_reset; 12121 12122 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 12123 i40e_pf_config_rss(pf); 12124 else if (!(features & NETIF_F_RXHASH) && 12125 netdev->features & NETIF_F_RXHASH) 12126 i40e_clear_rss_lut(vsi); 12127 12128 if (features & NETIF_F_HW_VLAN_CTAG_RX) 12129 i40e_vlan_stripping_enable(vsi); 12130 else 12131 i40e_vlan_stripping_disable(vsi); 12132 12133 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 12134 dev_err(&pf->pdev->dev, 12135 "Offloaded tc filters active, can't turn hw_tc_offload off"); 12136 return -EINVAL; 12137 } 12138 12139 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt) 12140 i40e_del_all_macvlans(vsi); 12141 12142 need_reset = i40e_set_ntuple(pf, features); 12143 12144 if (need_reset) 12145 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12146 12147 return 0; 12148 } 12149 12150 static int i40e_udp_tunnel_set_port(struct net_device *netdev, 12151 unsigned int table, unsigned int idx, 12152 struct udp_tunnel_info *ti) 12153 { 12154 struct i40e_netdev_priv *np = netdev_priv(netdev); 12155 struct i40e_hw *hw = &np->vsi->back->hw; 12156 u8 type, filter_index; 12157 i40e_status ret; 12158 12159 type = ti->type == UDP_TUNNEL_TYPE_VXLAN ? I40E_AQC_TUNNEL_TYPE_VXLAN : 12160 I40E_AQC_TUNNEL_TYPE_NGE; 12161 12162 ret = i40e_aq_add_udp_tunnel(hw, ntohs(ti->port), type, &filter_index, 12163 NULL); 12164 if (ret) { 12165 netdev_info(netdev, "add UDP port failed, err %s aq_err %s\n", 12166 i40e_stat_str(hw, ret), 12167 i40e_aq_str(hw, hw->aq.asq_last_status)); 12168 return -EIO; 12169 } 12170 12171 udp_tunnel_nic_set_port_priv(netdev, table, idx, filter_index); 12172 return 0; 12173 } 12174 12175 static int i40e_udp_tunnel_unset_port(struct net_device *netdev, 12176 unsigned int table, unsigned int idx, 12177 struct udp_tunnel_info *ti) 12178 { 12179 struct i40e_netdev_priv *np = netdev_priv(netdev); 12180 struct i40e_hw *hw = &np->vsi->back->hw; 12181 i40e_status ret; 12182 12183 ret = i40e_aq_del_udp_tunnel(hw, ti->hw_priv, NULL); 12184 if (ret) { 12185 netdev_info(netdev, "delete UDP port failed, err %s aq_err %s\n", 12186 i40e_stat_str(hw, ret), 12187 i40e_aq_str(hw, hw->aq.asq_last_status)); 12188 return -EIO; 12189 } 12190 12191 return 0; 12192 } 12193 12194 static int i40e_get_phys_port_id(struct net_device *netdev, 12195 struct netdev_phys_item_id *ppid) 12196 { 12197 struct i40e_netdev_priv *np = netdev_priv(netdev); 12198 struct i40e_pf *pf = np->vsi->back; 12199 struct i40e_hw *hw = &pf->hw; 12200 12201 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 12202 return -EOPNOTSUPP; 12203 12204 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 12205 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 12206 12207 return 0; 12208 } 12209 12210 /** 12211 * i40e_ndo_fdb_add - add an entry to the hardware database 12212 * @ndm: the input from the stack 12213 * @tb: pointer to array of nladdr (unused) 12214 * @dev: the net device pointer 12215 * @addr: the MAC address entry being added 12216 * @vid: VLAN ID 12217 * @flags: instructions from stack about fdb operation 12218 * @extack: netlink extended ack, unused currently 12219 */ 12220 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 12221 struct net_device *dev, 12222 const unsigned char *addr, u16 vid, 12223 u16 flags, 12224 struct netlink_ext_ack *extack) 12225 { 12226 struct i40e_netdev_priv *np = netdev_priv(dev); 12227 struct i40e_pf *pf = np->vsi->back; 12228 int err = 0; 12229 12230 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 12231 return -EOPNOTSUPP; 12232 12233 if (vid) { 12234 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 12235 return -EINVAL; 12236 } 12237 12238 /* Hardware does not support aging addresses so if a 12239 * ndm_state is given only allow permanent addresses 12240 */ 12241 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 12242 netdev_info(dev, "FDB only supports static addresses\n"); 12243 return -EINVAL; 12244 } 12245 12246 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 12247 err = dev_uc_add_excl(dev, addr); 12248 else if (is_multicast_ether_addr(addr)) 12249 err = dev_mc_add_excl(dev, addr); 12250 else 12251 err = -EINVAL; 12252 12253 /* Only return duplicate errors if NLM_F_EXCL is set */ 12254 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 12255 err = 0; 12256 12257 return err; 12258 } 12259 12260 /** 12261 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 12262 * @dev: the netdev being configured 12263 * @nlh: RTNL message 12264 * @flags: bridge flags 12265 * @extack: netlink extended ack 12266 * 12267 * Inserts a new hardware bridge if not already created and 12268 * enables the bridging mode requested (VEB or VEPA). If the 12269 * hardware bridge has already been inserted and the request 12270 * is to change the mode then that requires a PF reset to 12271 * allow rebuild of the components with required hardware 12272 * bridge mode enabled. 12273 * 12274 * Note: expects to be called while under rtnl_lock() 12275 **/ 12276 static int i40e_ndo_bridge_setlink(struct net_device *dev, 12277 struct nlmsghdr *nlh, 12278 u16 flags, 12279 struct netlink_ext_ack *extack) 12280 { 12281 struct i40e_netdev_priv *np = netdev_priv(dev); 12282 struct i40e_vsi *vsi = np->vsi; 12283 struct i40e_pf *pf = vsi->back; 12284 struct i40e_veb *veb = NULL; 12285 struct nlattr *attr, *br_spec; 12286 int i, rem; 12287 12288 /* Only for PF VSI for now */ 12289 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 12290 return -EOPNOTSUPP; 12291 12292 /* Find the HW bridge for PF VSI */ 12293 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12294 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12295 veb = pf->veb[i]; 12296 } 12297 12298 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12299 12300 nla_for_each_nested(attr, br_spec, rem) { 12301 __u16 mode; 12302 12303 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12304 continue; 12305 12306 mode = nla_get_u16(attr); 12307 if ((mode != BRIDGE_MODE_VEPA) && 12308 (mode != BRIDGE_MODE_VEB)) 12309 return -EINVAL; 12310 12311 /* Insert a new HW bridge */ 12312 if (!veb) { 12313 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 12314 vsi->tc_config.enabled_tc); 12315 if (veb) { 12316 veb->bridge_mode = mode; 12317 i40e_config_bridge_mode(veb); 12318 } else { 12319 /* No Bridge HW offload available */ 12320 return -ENOENT; 12321 } 12322 break; 12323 } else if (mode != veb->bridge_mode) { 12324 /* Existing HW bridge but different mode needs reset */ 12325 veb->bridge_mode = mode; 12326 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 12327 if (mode == BRIDGE_MODE_VEB) 12328 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 12329 else 12330 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 12331 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12332 break; 12333 } 12334 } 12335 12336 return 0; 12337 } 12338 12339 /** 12340 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 12341 * @skb: skb buff 12342 * @pid: process id 12343 * @seq: RTNL message seq # 12344 * @dev: the netdev being configured 12345 * @filter_mask: unused 12346 * @nlflags: netlink flags passed in 12347 * 12348 * Return the mode in which the hardware bridge is operating in 12349 * i.e VEB or VEPA. 12350 **/ 12351 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12352 struct net_device *dev, 12353 u32 __always_unused filter_mask, 12354 int nlflags) 12355 { 12356 struct i40e_netdev_priv *np = netdev_priv(dev); 12357 struct i40e_vsi *vsi = np->vsi; 12358 struct i40e_pf *pf = vsi->back; 12359 struct i40e_veb *veb = NULL; 12360 int i; 12361 12362 /* Only for PF VSI for now */ 12363 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 12364 return -EOPNOTSUPP; 12365 12366 /* Find the HW bridge for the PF VSI */ 12367 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12368 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12369 veb = pf->veb[i]; 12370 } 12371 12372 if (!veb) 12373 return 0; 12374 12375 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 12376 0, 0, nlflags, filter_mask, NULL); 12377 } 12378 12379 /** 12380 * i40e_features_check - Validate encapsulated packet conforms to limits 12381 * @skb: skb buff 12382 * @dev: This physical port's netdev 12383 * @features: Offload features that the stack believes apply 12384 **/ 12385 static netdev_features_t i40e_features_check(struct sk_buff *skb, 12386 struct net_device *dev, 12387 netdev_features_t features) 12388 { 12389 size_t len; 12390 12391 /* No point in doing any of this if neither checksum nor GSO are 12392 * being requested for this frame. We can rule out both by just 12393 * checking for CHECKSUM_PARTIAL 12394 */ 12395 if (skb->ip_summed != CHECKSUM_PARTIAL) 12396 return features; 12397 12398 /* We cannot support GSO if the MSS is going to be less than 12399 * 64 bytes. If it is then we need to drop support for GSO. 12400 */ 12401 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 12402 features &= ~NETIF_F_GSO_MASK; 12403 12404 /* MACLEN can support at most 63 words */ 12405 len = skb_network_header(skb) - skb->data; 12406 if (len & ~(63 * 2)) 12407 goto out_err; 12408 12409 /* IPLEN and EIPLEN can support at most 127 dwords */ 12410 len = skb_transport_header(skb) - skb_network_header(skb); 12411 if (len & ~(127 * 4)) 12412 goto out_err; 12413 12414 if (skb->encapsulation) { 12415 /* L4TUNLEN can support 127 words */ 12416 len = skb_inner_network_header(skb) - skb_transport_header(skb); 12417 if (len & ~(127 * 2)) 12418 goto out_err; 12419 12420 /* IPLEN can support at most 127 dwords */ 12421 len = skb_inner_transport_header(skb) - 12422 skb_inner_network_header(skb); 12423 if (len & ~(127 * 4)) 12424 goto out_err; 12425 } 12426 12427 /* No need to validate L4LEN as TCP is the only protocol with a 12428 * a flexible value and we support all possible values supported 12429 * by TCP, which is at most 15 dwords 12430 */ 12431 12432 return features; 12433 out_err: 12434 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 12435 } 12436 12437 /** 12438 * i40e_xdp_setup - add/remove an XDP program 12439 * @vsi: VSI to changed 12440 * @prog: XDP program 12441 **/ 12442 static int i40e_xdp_setup(struct i40e_vsi *vsi, 12443 struct bpf_prog *prog) 12444 { 12445 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 12446 struct i40e_pf *pf = vsi->back; 12447 struct bpf_prog *old_prog; 12448 bool need_reset; 12449 int i; 12450 12451 /* Don't allow frames that span over multiple buffers */ 12452 if (frame_size > vsi->rx_buf_len) 12453 return -EINVAL; 12454 12455 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 12456 return 0; 12457 12458 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 12459 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 12460 12461 if (need_reset) 12462 i40e_prep_for_reset(pf, true); 12463 12464 old_prog = xchg(&vsi->xdp_prog, prog); 12465 12466 if (need_reset) { 12467 if (!prog) 12468 /* Wait until ndo_xsk_wakeup completes. */ 12469 synchronize_rcu(); 12470 i40e_reset_and_rebuild(pf, true, true); 12471 } 12472 12473 for (i = 0; i < vsi->num_queue_pairs; i++) 12474 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 12475 12476 if (old_prog) 12477 bpf_prog_put(old_prog); 12478 12479 /* Kick start the NAPI context if there is an AF_XDP socket open 12480 * on that queue id. This so that receiving will start. 12481 */ 12482 if (need_reset && prog) 12483 for (i = 0; i < vsi->num_queue_pairs; i++) 12484 if (vsi->xdp_rings[i]->xsk_pool) 12485 (void)i40e_xsk_wakeup(vsi->netdev, i, 12486 XDP_WAKEUP_RX); 12487 12488 return 0; 12489 } 12490 12491 /** 12492 * i40e_enter_busy_conf - Enters busy config state 12493 * @vsi: vsi 12494 * 12495 * Returns 0 on success, <0 for failure. 12496 **/ 12497 static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 12498 { 12499 struct i40e_pf *pf = vsi->back; 12500 int timeout = 50; 12501 12502 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 12503 timeout--; 12504 if (!timeout) 12505 return -EBUSY; 12506 usleep_range(1000, 2000); 12507 } 12508 12509 return 0; 12510 } 12511 12512 /** 12513 * i40e_exit_busy_conf - Exits busy config state 12514 * @vsi: vsi 12515 **/ 12516 static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 12517 { 12518 struct i40e_pf *pf = vsi->back; 12519 12520 clear_bit(__I40E_CONFIG_BUSY, pf->state); 12521 } 12522 12523 /** 12524 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 12525 * @vsi: vsi 12526 * @queue_pair: queue pair 12527 **/ 12528 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 12529 { 12530 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 12531 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 12532 memset(&vsi->tx_rings[queue_pair]->stats, 0, 12533 sizeof(vsi->tx_rings[queue_pair]->stats)); 12534 if (i40e_enabled_xdp_vsi(vsi)) { 12535 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 12536 sizeof(vsi->xdp_rings[queue_pair]->stats)); 12537 } 12538 } 12539 12540 /** 12541 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 12542 * @vsi: vsi 12543 * @queue_pair: queue pair 12544 **/ 12545 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 12546 { 12547 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 12548 if (i40e_enabled_xdp_vsi(vsi)) { 12549 /* Make sure that in-progress ndo_xdp_xmit calls are 12550 * completed. 12551 */ 12552 synchronize_rcu(); 12553 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 12554 } 12555 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 12556 } 12557 12558 /** 12559 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 12560 * @vsi: vsi 12561 * @queue_pair: queue pair 12562 * @enable: true for enable, false for disable 12563 **/ 12564 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 12565 bool enable) 12566 { 12567 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12568 struct i40e_q_vector *q_vector = rxr->q_vector; 12569 12570 if (!vsi->netdev) 12571 return; 12572 12573 /* All rings in a qp belong to the same qvector. */ 12574 if (q_vector->rx.ring || q_vector->tx.ring) { 12575 if (enable) 12576 napi_enable(&q_vector->napi); 12577 else 12578 napi_disable(&q_vector->napi); 12579 } 12580 } 12581 12582 /** 12583 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 12584 * @vsi: vsi 12585 * @queue_pair: queue pair 12586 * @enable: true for enable, false for disable 12587 * 12588 * Returns 0 on success, <0 on failure. 12589 **/ 12590 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 12591 bool enable) 12592 { 12593 struct i40e_pf *pf = vsi->back; 12594 int pf_q, ret = 0; 12595 12596 pf_q = vsi->base_queue + queue_pair; 12597 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 12598 false /*is xdp*/, enable); 12599 if (ret) { 12600 dev_info(&pf->pdev->dev, 12601 "VSI seid %d Tx ring %d %sable timeout\n", 12602 vsi->seid, pf_q, (enable ? "en" : "dis")); 12603 return ret; 12604 } 12605 12606 i40e_control_rx_q(pf, pf_q, enable); 12607 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 12608 if (ret) { 12609 dev_info(&pf->pdev->dev, 12610 "VSI seid %d Rx ring %d %sable timeout\n", 12611 vsi->seid, pf_q, (enable ? "en" : "dis")); 12612 return ret; 12613 } 12614 12615 /* Due to HW errata, on Rx disable only, the register can 12616 * indicate done before it really is. Needs 50ms to be sure 12617 */ 12618 if (!enable) 12619 mdelay(50); 12620 12621 if (!i40e_enabled_xdp_vsi(vsi)) 12622 return ret; 12623 12624 ret = i40e_control_wait_tx_q(vsi->seid, pf, 12625 pf_q + vsi->alloc_queue_pairs, 12626 true /*is xdp*/, enable); 12627 if (ret) { 12628 dev_info(&pf->pdev->dev, 12629 "VSI seid %d XDP Tx ring %d %sable timeout\n", 12630 vsi->seid, pf_q, (enable ? "en" : "dis")); 12631 } 12632 12633 return ret; 12634 } 12635 12636 /** 12637 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 12638 * @vsi: vsi 12639 * @queue_pair: queue_pair 12640 **/ 12641 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 12642 { 12643 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12644 struct i40e_pf *pf = vsi->back; 12645 struct i40e_hw *hw = &pf->hw; 12646 12647 /* All rings in a qp belong to the same qvector. */ 12648 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 12649 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 12650 else 12651 i40e_irq_dynamic_enable_icr0(pf); 12652 12653 i40e_flush(hw); 12654 } 12655 12656 /** 12657 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 12658 * @vsi: vsi 12659 * @queue_pair: queue_pair 12660 **/ 12661 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 12662 { 12663 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12664 struct i40e_pf *pf = vsi->back; 12665 struct i40e_hw *hw = &pf->hw; 12666 12667 /* For simplicity, instead of removing the qp interrupt causes 12668 * from the interrupt linked list, we simply disable the interrupt, and 12669 * leave the list intact. 12670 * 12671 * All rings in a qp belong to the same qvector. 12672 */ 12673 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 12674 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 12675 12676 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 12677 i40e_flush(hw); 12678 synchronize_irq(pf->msix_entries[intpf].vector); 12679 } else { 12680 /* Legacy and MSI mode - this stops all interrupt handling */ 12681 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 12682 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 12683 i40e_flush(hw); 12684 synchronize_irq(pf->pdev->irq); 12685 } 12686 } 12687 12688 /** 12689 * i40e_queue_pair_disable - Disables a queue pair 12690 * @vsi: vsi 12691 * @queue_pair: queue pair 12692 * 12693 * Returns 0 on success, <0 on failure. 12694 **/ 12695 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 12696 { 12697 int err; 12698 12699 err = i40e_enter_busy_conf(vsi); 12700 if (err) 12701 return err; 12702 12703 i40e_queue_pair_disable_irq(vsi, queue_pair); 12704 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 12705 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 12706 i40e_queue_pair_clean_rings(vsi, queue_pair); 12707 i40e_queue_pair_reset_stats(vsi, queue_pair); 12708 12709 return err; 12710 } 12711 12712 /** 12713 * i40e_queue_pair_enable - Enables a queue pair 12714 * @vsi: vsi 12715 * @queue_pair: queue pair 12716 * 12717 * Returns 0 on success, <0 on failure. 12718 **/ 12719 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 12720 { 12721 int err; 12722 12723 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 12724 if (err) 12725 return err; 12726 12727 if (i40e_enabled_xdp_vsi(vsi)) { 12728 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 12729 if (err) 12730 return err; 12731 } 12732 12733 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 12734 if (err) 12735 return err; 12736 12737 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 12738 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 12739 i40e_queue_pair_enable_irq(vsi, queue_pair); 12740 12741 i40e_exit_busy_conf(vsi); 12742 12743 return err; 12744 } 12745 12746 /** 12747 * i40e_xdp - implements ndo_bpf for i40e 12748 * @dev: netdevice 12749 * @xdp: XDP command 12750 **/ 12751 static int i40e_xdp(struct net_device *dev, 12752 struct netdev_bpf *xdp) 12753 { 12754 struct i40e_netdev_priv *np = netdev_priv(dev); 12755 struct i40e_vsi *vsi = np->vsi; 12756 12757 if (vsi->type != I40E_VSI_MAIN) 12758 return -EINVAL; 12759 12760 switch (xdp->command) { 12761 case XDP_SETUP_PROG: 12762 return i40e_xdp_setup(vsi, xdp->prog); 12763 case XDP_SETUP_XSK_POOL: 12764 return i40e_xsk_pool_setup(vsi, xdp->xsk.pool, 12765 xdp->xsk.queue_id); 12766 default: 12767 return -EINVAL; 12768 } 12769 } 12770 12771 static const struct net_device_ops i40e_netdev_ops = { 12772 .ndo_open = i40e_open, 12773 .ndo_stop = i40e_close, 12774 .ndo_start_xmit = i40e_lan_xmit_frame, 12775 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 12776 .ndo_set_rx_mode = i40e_set_rx_mode, 12777 .ndo_validate_addr = eth_validate_addr, 12778 .ndo_set_mac_address = i40e_set_mac, 12779 .ndo_change_mtu = i40e_change_mtu, 12780 .ndo_do_ioctl = i40e_ioctl, 12781 .ndo_tx_timeout = i40e_tx_timeout, 12782 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 12783 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 12784 #ifdef CONFIG_NET_POLL_CONTROLLER 12785 .ndo_poll_controller = i40e_netpoll, 12786 #endif 12787 .ndo_setup_tc = __i40e_setup_tc, 12788 .ndo_set_features = i40e_set_features, 12789 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 12790 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 12791 .ndo_get_vf_stats = i40e_get_vf_stats, 12792 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 12793 .ndo_get_vf_config = i40e_ndo_get_vf_config, 12794 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 12795 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 12796 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 12797 .ndo_udp_tunnel_add = udp_tunnel_nic_add_port, 12798 .ndo_udp_tunnel_del = udp_tunnel_nic_del_port, 12799 .ndo_get_phys_port_id = i40e_get_phys_port_id, 12800 .ndo_fdb_add = i40e_ndo_fdb_add, 12801 .ndo_features_check = i40e_features_check, 12802 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 12803 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 12804 .ndo_bpf = i40e_xdp, 12805 .ndo_xdp_xmit = i40e_xdp_xmit, 12806 .ndo_xsk_wakeup = i40e_xsk_wakeup, 12807 .ndo_dfwd_add_station = i40e_fwd_add, 12808 .ndo_dfwd_del_station = i40e_fwd_del, 12809 }; 12810 12811 /** 12812 * i40e_config_netdev - Setup the netdev flags 12813 * @vsi: the VSI being configured 12814 * 12815 * Returns 0 on success, negative value on failure 12816 **/ 12817 static int i40e_config_netdev(struct i40e_vsi *vsi) 12818 { 12819 struct i40e_pf *pf = vsi->back; 12820 struct i40e_hw *hw = &pf->hw; 12821 struct i40e_netdev_priv *np; 12822 struct net_device *netdev; 12823 u8 broadcast[ETH_ALEN]; 12824 u8 mac_addr[ETH_ALEN]; 12825 int etherdev_size; 12826 netdev_features_t hw_enc_features; 12827 netdev_features_t hw_features; 12828 12829 etherdev_size = sizeof(struct i40e_netdev_priv); 12830 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 12831 if (!netdev) 12832 return -ENOMEM; 12833 12834 vsi->netdev = netdev; 12835 np = netdev_priv(netdev); 12836 np->vsi = vsi; 12837 12838 hw_enc_features = NETIF_F_SG | 12839 NETIF_F_IP_CSUM | 12840 NETIF_F_IPV6_CSUM | 12841 NETIF_F_HIGHDMA | 12842 NETIF_F_SOFT_FEATURES | 12843 NETIF_F_TSO | 12844 NETIF_F_TSO_ECN | 12845 NETIF_F_TSO6 | 12846 NETIF_F_GSO_GRE | 12847 NETIF_F_GSO_GRE_CSUM | 12848 NETIF_F_GSO_PARTIAL | 12849 NETIF_F_GSO_IPXIP4 | 12850 NETIF_F_GSO_IPXIP6 | 12851 NETIF_F_GSO_UDP_TUNNEL | 12852 NETIF_F_GSO_UDP_TUNNEL_CSUM | 12853 NETIF_F_GSO_UDP_L4 | 12854 NETIF_F_SCTP_CRC | 12855 NETIF_F_RXHASH | 12856 NETIF_F_RXCSUM | 12857 0; 12858 12859 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 12860 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 12861 12862 netdev->udp_tunnel_nic_info = &pf->udp_tunnel_nic; 12863 12864 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 12865 12866 netdev->hw_enc_features |= hw_enc_features; 12867 12868 /* record features VLANs can make use of */ 12869 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 12870 12871 /* enable macvlan offloads */ 12872 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD; 12873 12874 hw_features = hw_enc_features | 12875 NETIF_F_HW_VLAN_CTAG_TX | 12876 NETIF_F_HW_VLAN_CTAG_RX; 12877 12878 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 12879 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 12880 12881 netdev->hw_features |= hw_features; 12882 12883 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 12884 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 12885 12886 if (vsi->type == I40E_VSI_MAIN) { 12887 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 12888 ether_addr_copy(mac_addr, hw->mac.perm_addr); 12889 /* The following steps are necessary for two reasons. First, 12890 * some older NVM configurations load a default MAC-VLAN 12891 * filter that will accept any tagged packet, and we want to 12892 * replace this with a normal filter. Additionally, it is 12893 * possible our MAC address was provided by the platform using 12894 * Open Firmware or similar. 12895 * 12896 * Thus, we need to remove the default filter and install one 12897 * specific to the MAC address. 12898 */ 12899 i40e_rm_default_mac_filter(vsi, mac_addr); 12900 spin_lock_bh(&vsi->mac_filter_hash_lock); 12901 i40e_add_mac_filter(vsi, mac_addr); 12902 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12903 } else { 12904 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 12905 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 12906 * the end, which is 4 bytes long, so force truncation of the 12907 * original name by IFNAMSIZ - 4 12908 */ 12909 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 12910 IFNAMSIZ - 4, 12911 pf->vsi[pf->lan_vsi]->netdev->name); 12912 eth_random_addr(mac_addr); 12913 12914 spin_lock_bh(&vsi->mac_filter_hash_lock); 12915 i40e_add_mac_filter(vsi, mac_addr); 12916 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12917 } 12918 12919 /* Add the broadcast filter so that we initially will receive 12920 * broadcast packets. Note that when a new VLAN is first added the 12921 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 12922 * specific filters as part of transitioning into "vlan" operation. 12923 * When more VLANs are added, the driver will copy each existing MAC 12924 * filter and add it for the new VLAN. 12925 * 12926 * Broadcast filters are handled specially by 12927 * i40e_sync_filters_subtask, as the driver must to set the broadcast 12928 * promiscuous bit instead of adding this directly as a MAC/VLAN 12929 * filter. The subtask will update the correct broadcast promiscuous 12930 * bits as VLANs become active or inactive. 12931 */ 12932 eth_broadcast_addr(broadcast); 12933 spin_lock_bh(&vsi->mac_filter_hash_lock); 12934 i40e_add_mac_filter(vsi, broadcast); 12935 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12936 12937 ether_addr_copy(netdev->dev_addr, mac_addr); 12938 ether_addr_copy(netdev->perm_addr, mac_addr); 12939 12940 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */ 12941 netdev->neigh_priv_len = sizeof(u32) * 4; 12942 12943 netdev->priv_flags |= IFF_UNICAST_FLT; 12944 netdev->priv_flags |= IFF_SUPP_NOFCS; 12945 /* Setup netdev TC information */ 12946 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 12947 12948 netdev->netdev_ops = &i40e_netdev_ops; 12949 netdev->watchdog_timeo = 5 * HZ; 12950 i40e_set_ethtool_ops(netdev); 12951 12952 /* MTU range: 68 - 9706 */ 12953 netdev->min_mtu = ETH_MIN_MTU; 12954 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 12955 12956 return 0; 12957 } 12958 12959 /** 12960 * i40e_vsi_delete - Delete a VSI from the switch 12961 * @vsi: the VSI being removed 12962 * 12963 * Returns 0 on success, negative value on failure 12964 **/ 12965 static void i40e_vsi_delete(struct i40e_vsi *vsi) 12966 { 12967 /* remove default VSI is not allowed */ 12968 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 12969 return; 12970 12971 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 12972 } 12973 12974 /** 12975 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 12976 * @vsi: the VSI being queried 12977 * 12978 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 12979 **/ 12980 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 12981 { 12982 struct i40e_veb *veb; 12983 struct i40e_pf *pf = vsi->back; 12984 12985 /* Uplink is not a bridge so default to VEB */ 12986 if (vsi->veb_idx >= I40E_MAX_VEB) 12987 return 1; 12988 12989 veb = pf->veb[vsi->veb_idx]; 12990 if (!veb) { 12991 dev_info(&pf->pdev->dev, 12992 "There is no veb associated with the bridge\n"); 12993 return -ENOENT; 12994 } 12995 12996 /* Uplink is a bridge in VEPA mode */ 12997 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 12998 return 0; 12999 } else { 13000 /* Uplink is a bridge in VEB mode */ 13001 return 1; 13002 } 13003 13004 /* VEPA is now default bridge, so return 0 */ 13005 return 0; 13006 } 13007 13008 /** 13009 * i40e_add_vsi - Add a VSI to the switch 13010 * @vsi: the VSI being configured 13011 * 13012 * This initializes a VSI context depending on the VSI type to be added and 13013 * passes it down to the add_vsi aq command. 13014 **/ 13015 static int i40e_add_vsi(struct i40e_vsi *vsi) 13016 { 13017 int ret = -ENODEV; 13018 struct i40e_pf *pf = vsi->back; 13019 struct i40e_hw *hw = &pf->hw; 13020 struct i40e_vsi_context ctxt; 13021 struct i40e_mac_filter *f; 13022 struct hlist_node *h; 13023 int bkt; 13024 13025 u8 enabled_tc = 0x1; /* TC0 enabled */ 13026 int f_count = 0; 13027 13028 memset(&ctxt, 0, sizeof(ctxt)); 13029 switch (vsi->type) { 13030 case I40E_VSI_MAIN: 13031 /* The PF's main VSI is already setup as part of the 13032 * device initialization, so we'll not bother with 13033 * the add_vsi call, but we will retrieve the current 13034 * VSI context. 13035 */ 13036 ctxt.seid = pf->main_vsi_seid; 13037 ctxt.pf_num = pf->hw.pf_id; 13038 ctxt.vf_num = 0; 13039 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 13040 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13041 if (ret) { 13042 dev_info(&pf->pdev->dev, 13043 "couldn't get PF vsi config, err %s aq_err %s\n", 13044 i40e_stat_str(&pf->hw, ret), 13045 i40e_aq_str(&pf->hw, 13046 pf->hw.aq.asq_last_status)); 13047 return -ENOENT; 13048 } 13049 vsi->info = ctxt.info; 13050 vsi->info.valid_sections = 0; 13051 13052 vsi->seid = ctxt.seid; 13053 vsi->id = ctxt.vsi_number; 13054 13055 enabled_tc = i40e_pf_get_tc_map(pf); 13056 13057 /* Source pruning is enabled by default, so the flag is 13058 * negative logic - if it's set, we need to fiddle with 13059 * the VSI to disable source pruning. 13060 */ 13061 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 13062 memset(&ctxt, 0, sizeof(ctxt)); 13063 ctxt.seid = pf->main_vsi_seid; 13064 ctxt.pf_num = pf->hw.pf_id; 13065 ctxt.vf_num = 0; 13066 ctxt.info.valid_sections |= 13067 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13068 ctxt.info.switch_id = 13069 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 13070 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13071 if (ret) { 13072 dev_info(&pf->pdev->dev, 13073 "update vsi failed, err %s aq_err %s\n", 13074 i40e_stat_str(&pf->hw, ret), 13075 i40e_aq_str(&pf->hw, 13076 pf->hw.aq.asq_last_status)); 13077 ret = -ENOENT; 13078 goto err; 13079 } 13080 } 13081 13082 /* MFP mode setup queue map and update VSI */ 13083 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 13084 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 13085 memset(&ctxt, 0, sizeof(ctxt)); 13086 ctxt.seid = pf->main_vsi_seid; 13087 ctxt.pf_num = pf->hw.pf_id; 13088 ctxt.vf_num = 0; 13089 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 13090 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13091 if (ret) { 13092 dev_info(&pf->pdev->dev, 13093 "update vsi failed, err %s aq_err %s\n", 13094 i40e_stat_str(&pf->hw, ret), 13095 i40e_aq_str(&pf->hw, 13096 pf->hw.aq.asq_last_status)); 13097 ret = -ENOENT; 13098 goto err; 13099 } 13100 /* update the local VSI info queue map */ 13101 i40e_vsi_update_queue_map(vsi, &ctxt); 13102 vsi->info.valid_sections = 0; 13103 } else { 13104 /* Default/Main VSI is only enabled for TC0 13105 * reconfigure it to enable all TCs that are 13106 * available on the port in SFP mode. 13107 * For MFP case the iSCSI PF would use this 13108 * flow to enable LAN+iSCSI TC. 13109 */ 13110 ret = i40e_vsi_config_tc(vsi, enabled_tc); 13111 if (ret) { 13112 /* Single TC condition is not fatal, 13113 * message and continue 13114 */ 13115 dev_info(&pf->pdev->dev, 13116 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 13117 enabled_tc, 13118 i40e_stat_str(&pf->hw, ret), 13119 i40e_aq_str(&pf->hw, 13120 pf->hw.aq.asq_last_status)); 13121 } 13122 } 13123 break; 13124 13125 case I40E_VSI_FDIR: 13126 ctxt.pf_num = hw->pf_id; 13127 ctxt.vf_num = 0; 13128 ctxt.uplink_seid = vsi->uplink_seid; 13129 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13130 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13131 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 13132 (i40e_is_vsi_uplink_mode_veb(vsi))) { 13133 ctxt.info.valid_sections |= 13134 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13135 ctxt.info.switch_id = 13136 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13137 } 13138 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13139 break; 13140 13141 case I40E_VSI_VMDQ2: 13142 ctxt.pf_num = hw->pf_id; 13143 ctxt.vf_num = 0; 13144 ctxt.uplink_seid = vsi->uplink_seid; 13145 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13146 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 13147 13148 /* This VSI is connected to VEB so the switch_id 13149 * should be set to zero by default. 13150 */ 13151 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13152 ctxt.info.valid_sections |= 13153 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13154 ctxt.info.switch_id = 13155 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13156 } 13157 13158 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13159 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13160 break; 13161 13162 case I40E_VSI_SRIOV: 13163 ctxt.pf_num = hw->pf_id; 13164 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 13165 ctxt.uplink_seid = vsi->uplink_seid; 13166 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13167 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 13168 13169 /* This VSI is connected to VEB so the switch_id 13170 * should be set to zero by default. 13171 */ 13172 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13173 ctxt.info.valid_sections |= 13174 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13175 ctxt.info.switch_id = 13176 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13177 } 13178 13179 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 13180 ctxt.info.valid_sections |= 13181 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 13182 ctxt.info.queueing_opt_flags |= 13183 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 13184 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 13185 } 13186 13187 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 13188 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 13189 if (pf->vf[vsi->vf_id].spoofchk) { 13190 ctxt.info.valid_sections |= 13191 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 13192 ctxt.info.sec_flags |= 13193 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 13194 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 13195 } 13196 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13197 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13198 break; 13199 13200 case I40E_VSI_IWARP: 13201 /* send down message to iWARP */ 13202 break; 13203 13204 default: 13205 return -ENODEV; 13206 } 13207 13208 if (vsi->type != I40E_VSI_MAIN) { 13209 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 13210 if (ret) { 13211 dev_info(&vsi->back->pdev->dev, 13212 "add vsi failed, err %s aq_err %s\n", 13213 i40e_stat_str(&pf->hw, ret), 13214 i40e_aq_str(&pf->hw, 13215 pf->hw.aq.asq_last_status)); 13216 ret = -ENOENT; 13217 goto err; 13218 } 13219 vsi->info = ctxt.info; 13220 vsi->info.valid_sections = 0; 13221 vsi->seid = ctxt.seid; 13222 vsi->id = ctxt.vsi_number; 13223 } 13224 13225 vsi->active_filters = 0; 13226 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 13227 spin_lock_bh(&vsi->mac_filter_hash_lock); 13228 /* If macvlan filters already exist, force them to get loaded */ 13229 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 13230 f->state = I40E_FILTER_NEW; 13231 f_count++; 13232 } 13233 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13234 13235 if (f_count) { 13236 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 13237 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 13238 } 13239 13240 /* Update VSI BW information */ 13241 ret = i40e_vsi_get_bw_info(vsi); 13242 if (ret) { 13243 dev_info(&pf->pdev->dev, 13244 "couldn't get vsi bw info, err %s aq_err %s\n", 13245 i40e_stat_str(&pf->hw, ret), 13246 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13247 /* VSI is already added so not tearing that up */ 13248 ret = 0; 13249 } 13250 13251 err: 13252 return ret; 13253 } 13254 13255 /** 13256 * i40e_vsi_release - Delete a VSI and free its resources 13257 * @vsi: the VSI being removed 13258 * 13259 * Returns 0 on success or < 0 on error 13260 **/ 13261 int i40e_vsi_release(struct i40e_vsi *vsi) 13262 { 13263 struct i40e_mac_filter *f; 13264 struct hlist_node *h; 13265 struct i40e_veb *veb = NULL; 13266 struct i40e_pf *pf; 13267 u16 uplink_seid; 13268 int i, n, bkt; 13269 13270 pf = vsi->back; 13271 13272 /* release of a VEB-owner or last VSI is not allowed */ 13273 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 13274 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 13275 vsi->seid, vsi->uplink_seid); 13276 return -ENODEV; 13277 } 13278 if (vsi == pf->vsi[pf->lan_vsi] && 13279 !test_bit(__I40E_DOWN, pf->state)) { 13280 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 13281 return -ENODEV; 13282 } 13283 13284 uplink_seid = vsi->uplink_seid; 13285 if (vsi->type != I40E_VSI_SRIOV) { 13286 if (vsi->netdev_registered) { 13287 vsi->netdev_registered = false; 13288 if (vsi->netdev) { 13289 /* results in a call to i40e_close() */ 13290 unregister_netdev(vsi->netdev); 13291 } 13292 } else { 13293 i40e_vsi_close(vsi); 13294 } 13295 i40e_vsi_disable_irq(vsi); 13296 } 13297 13298 spin_lock_bh(&vsi->mac_filter_hash_lock); 13299 13300 /* clear the sync flag on all filters */ 13301 if (vsi->netdev) { 13302 __dev_uc_unsync(vsi->netdev, NULL); 13303 __dev_mc_unsync(vsi->netdev, NULL); 13304 } 13305 13306 /* make sure any remaining filters are marked for deletion */ 13307 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 13308 __i40e_del_filter(vsi, f); 13309 13310 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13311 13312 i40e_sync_vsi_filters(vsi); 13313 13314 i40e_vsi_delete(vsi); 13315 i40e_vsi_free_q_vectors(vsi); 13316 if (vsi->netdev) { 13317 free_netdev(vsi->netdev); 13318 vsi->netdev = NULL; 13319 } 13320 i40e_vsi_clear_rings(vsi); 13321 i40e_vsi_clear(vsi); 13322 13323 /* If this was the last thing on the VEB, except for the 13324 * controlling VSI, remove the VEB, which puts the controlling 13325 * VSI onto the next level down in the switch. 13326 * 13327 * Well, okay, there's one more exception here: don't remove 13328 * the orphan VEBs yet. We'll wait for an explicit remove request 13329 * from up the network stack. 13330 */ 13331 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 13332 if (pf->vsi[i] && 13333 pf->vsi[i]->uplink_seid == uplink_seid && 13334 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13335 n++; /* count the VSIs */ 13336 } 13337 } 13338 for (i = 0; i < I40E_MAX_VEB; i++) { 13339 if (!pf->veb[i]) 13340 continue; 13341 if (pf->veb[i]->uplink_seid == uplink_seid) 13342 n++; /* count the VEBs */ 13343 if (pf->veb[i]->seid == uplink_seid) 13344 veb = pf->veb[i]; 13345 } 13346 if (n == 0 && veb && veb->uplink_seid != 0) 13347 i40e_veb_release(veb); 13348 13349 return 0; 13350 } 13351 13352 /** 13353 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 13354 * @vsi: ptr to the VSI 13355 * 13356 * This should only be called after i40e_vsi_mem_alloc() which allocates the 13357 * corresponding SW VSI structure and initializes num_queue_pairs for the 13358 * newly allocated VSI. 13359 * 13360 * Returns 0 on success or negative on failure 13361 **/ 13362 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 13363 { 13364 int ret = -ENOENT; 13365 struct i40e_pf *pf = vsi->back; 13366 13367 if (vsi->q_vectors[0]) { 13368 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 13369 vsi->seid); 13370 return -EEXIST; 13371 } 13372 13373 if (vsi->base_vector) { 13374 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 13375 vsi->seid, vsi->base_vector); 13376 return -EEXIST; 13377 } 13378 13379 ret = i40e_vsi_alloc_q_vectors(vsi); 13380 if (ret) { 13381 dev_info(&pf->pdev->dev, 13382 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 13383 vsi->num_q_vectors, vsi->seid, ret); 13384 vsi->num_q_vectors = 0; 13385 goto vector_setup_out; 13386 } 13387 13388 /* In Legacy mode, we do not have to get any other vector since we 13389 * piggyback on the misc/ICR0 for queue interrupts. 13390 */ 13391 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 13392 return ret; 13393 if (vsi->num_q_vectors) 13394 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 13395 vsi->num_q_vectors, vsi->idx); 13396 if (vsi->base_vector < 0) { 13397 dev_info(&pf->pdev->dev, 13398 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 13399 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 13400 i40e_vsi_free_q_vectors(vsi); 13401 ret = -ENOENT; 13402 goto vector_setup_out; 13403 } 13404 13405 vector_setup_out: 13406 return ret; 13407 } 13408 13409 /** 13410 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 13411 * @vsi: pointer to the vsi. 13412 * 13413 * This re-allocates a vsi's queue resources. 13414 * 13415 * Returns pointer to the successfully allocated and configured VSI sw struct 13416 * on success, otherwise returns NULL on failure. 13417 **/ 13418 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 13419 { 13420 u16 alloc_queue_pairs; 13421 struct i40e_pf *pf; 13422 u8 enabled_tc; 13423 int ret; 13424 13425 if (!vsi) 13426 return NULL; 13427 13428 pf = vsi->back; 13429 13430 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 13431 i40e_vsi_clear_rings(vsi); 13432 13433 i40e_vsi_free_arrays(vsi, false); 13434 i40e_set_num_rings_in_vsi(vsi); 13435 ret = i40e_vsi_alloc_arrays(vsi, false); 13436 if (ret) 13437 goto err_vsi; 13438 13439 alloc_queue_pairs = vsi->alloc_queue_pairs * 13440 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 13441 13442 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 13443 if (ret < 0) { 13444 dev_info(&pf->pdev->dev, 13445 "failed to get tracking for %d queues for VSI %d err %d\n", 13446 alloc_queue_pairs, vsi->seid, ret); 13447 goto err_vsi; 13448 } 13449 vsi->base_queue = ret; 13450 13451 /* Update the FW view of the VSI. Force a reset of TC and queue 13452 * layout configurations. 13453 */ 13454 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 13455 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 13456 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 13457 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 13458 if (vsi->type == I40E_VSI_MAIN) 13459 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 13460 13461 /* assign it some queues */ 13462 ret = i40e_alloc_rings(vsi); 13463 if (ret) 13464 goto err_rings; 13465 13466 /* map all of the rings to the q_vectors */ 13467 i40e_vsi_map_rings_to_vectors(vsi); 13468 return vsi; 13469 13470 err_rings: 13471 i40e_vsi_free_q_vectors(vsi); 13472 if (vsi->netdev_registered) { 13473 vsi->netdev_registered = false; 13474 unregister_netdev(vsi->netdev); 13475 free_netdev(vsi->netdev); 13476 vsi->netdev = NULL; 13477 } 13478 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13479 err_vsi: 13480 i40e_vsi_clear(vsi); 13481 return NULL; 13482 } 13483 13484 /** 13485 * i40e_vsi_setup - Set up a VSI by a given type 13486 * @pf: board private structure 13487 * @type: VSI type 13488 * @uplink_seid: the switch element to link to 13489 * @param1: usage depends upon VSI type. For VF types, indicates VF id 13490 * 13491 * This allocates the sw VSI structure and its queue resources, then add a VSI 13492 * to the identified VEB. 13493 * 13494 * Returns pointer to the successfully allocated and configure VSI sw struct on 13495 * success, otherwise returns NULL on failure. 13496 **/ 13497 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 13498 u16 uplink_seid, u32 param1) 13499 { 13500 struct i40e_vsi *vsi = NULL; 13501 struct i40e_veb *veb = NULL; 13502 u16 alloc_queue_pairs; 13503 int ret, i; 13504 int v_idx; 13505 13506 /* The requested uplink_seid must be either 13507 * - the PF's port seid 13508 * no VEB is needed because this is the PF 13509 * or this is a Flow Director special case VSI 13510 * - seid of an existing VEB 13511 * - seid of a VSI that owns an existing VEB 13512 * - seid of a VSI that doesn't own a VEB 13513 * a new VEB is created and the VSI becomes the owner 13514 * - seid of the PF VSI, which is what creates the first VEB 13515 * this is a special case of the previous 13516 * 13517 * Find which uplink_seid we were given and create a new VEB if needed 13518 */ 13519 for (i = 0; i < I40E_MAX_VEB; i++) { 13520 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 13521 veb = pf->veb[i]; 13522 break; 13523 } 13524 } 13525 13526 if (!veb && uplink_seid != pf->mac_seid) { 13527 13528 for (i = 0; i < pf->num_alloc_vsi; i++) { 13529 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 13530 vsi = pf->vsi[i]; 13531 break; 13532 } 13533 } 13534 if (!vsi) { 13535 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 13536 uplink_seid); 13537 return NULL; 13538 } 13539 13540 if (vsi->uplink_seid == pf->mac_seid) 13541 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 13542 vsi->tc_config.enabled_tc); 13543 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 13544 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 13545 vsi->tc_config.enabled_tc); 13546 if (veb) { 13547 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 13548 dev_info(&vsi->back->pdev->dev, 13549 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 13550 return NULL; 13551 } 13552 /* We come up by default in VEPA mode if SRIOV is not 13553 * already enabled, in which case we can't force VEPA 13554 * mode. 13555 */ 13556 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 13557 veb->bridge_mode = BRIDGE_MODE_VEPA; 13558 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 13559 } 13560 i40e_config_bridge_mode(veb); 13561 } 13562 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 13563 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 13564 veb = pf->veb[i]; 13565 } 13566 if (!veb) { 13567 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 13568 return NULL; 13569 } 13570 13571 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 13572 uplink_seid = veb->seid; 13573 } 13574 13575 /* get vsi sw struct */ 13576 v_idx = i40e_vsi_mem_alloc(pf, type); 13577 if (v_idx < 0) 13578 goto err_alloc; 13579 vsi = pf->vsi[v_idx]; 13580 if (!vsi) 13581 goto err_alloc; 13582 vsi->type = type; 13583 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 13584 13585 if (type == I40E_VSI_MAIN) 13586 pf->lan_vsi = v_idx; 13587 else if (type == I40E_VSI_SRIOV) 13588 vsi->vf_id = param1; 13589 /* assign it some queues */ 13590 alloc_queue_pairs = vsi->alloc_queue_pairs * 13591 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 13592 13593 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 13594 if (ret < 0) { 13595 dev_info(&pf->pdev->dev, 13596 "failed to get tracking for %d queues for VSI %d err=%d\n", 13597 alloc_queue_pairs, vsi->seid, ret); 13598 goto err_vsi; 13599 } 13600 vsi->base_queue = ret; 13601 13602 /* get a VSI from the hardware */ 13603 vsi->uplink_seid = uplink_seid; 13604 ret = i40e_add_vsi(vsi); 13605 if (ret) 13606 goto err_vsi; 13607 13608 switch (vsi->type) { 13609 /* setup the netdev if needed */ 13610 case I40E_VSI_MAIN: 13611 case I40E_VSI_VMDQ2: 13612 ret = i40e_config_netdev(vsi); 13613 if (ret) 13614 goto err_netdev; 13615 ret = register_netdev(vsi->netdev); 13616 if (ret) 13617 goto err_netdev; 13618 vsi->netdev_registered = true; 13619 netif_carrier_off(vsi->netdev); 13620 #ifdef CONFIG_I40E_DCB 13621 /* Setup DCB netlink interface */ 13622 i40e_dcbnl_setup(vsi); 13623 #endif /* CONFIG_I40E_DCB */ 13624 fallthrough; 13625 case I40E_VSI_FDIR: 13626 /* set up vectors and rings if needed */ 13627 ret = i40e_vsi_setup_vectors(vsi); 13628 if (ret) 13629 goto err_msix; 13630 13631 ret = i40e_alloc_rings(vsi); 13632 if (ret) 13633 goto err_rings; 13634 13635 /* map all of the rings to the q_vectors */ 13636 i40e_vsi_map_rings_to_vectors(vsi); 13637 13638 i40e_vsi_reset_stats(vsi); 13639 break; 13640 default: 13641 /* no netdev or rings for the other VSI types */ 13642 break; 13643 } 13644 13645 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 13646 (vsi->type == I40E_VSI_VMDQ2)) { 13647 ret = i40e_vsi_config_rss(vsi); 13648 } 13649 return vsi; 13650 13651 err_rings: 13652 i40e_vsi_free_q_vectors(vsi); 13653 err_msix: 13654 if (vsi->netdev_registered) { 13655 vsi->netdev_registered = false; 13656 unregister_netdev(vsi->netdev); 13657 free_netdev(vsi->netdev); 13658 vsi->netdev = NULL; 13659 } 13660 err_netdev: 13661 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13662 err_vsi: 13663 i40e_vsi_clear(vsi); 13664 err_alloc: 13665 return NULL; 13666 } 13667 13668 /** 13669 * i40e_veb_get_bw_info - Query VEB BW information 13670 * @veb: the veb to query 13671 * 13672 * Query the Tx scheduler BW configuration data for given VEB 13673 **/ 13674 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 13675 { 13676 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 13677 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 13678 struct i40e_pf *pf = veb->pf; 13679 struct i40e_hw *hw = &pf->hw; 13680 u32 tc_bw_max; 13681 int ret = 0; 13682 int i; 13683 13684 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 13685 &bw_data, NULL); 13686 if (ret) { 13687 dev_info(&pf->pdev->dev, 13688 "query veb bw config failed, err %s aq_err %s\n", 13689 i40e_stat_str(&pf->hw, ret), 13690 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13691 goto out; 13692 } 13693 13694 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 13695 &ets_data, NULL); 13696 if (ret) { 13697 dev_info(&pf->pdev->dev, 13698 "query veb bw ets config failed, err %s aq_err %s\n", 13699 i40e_stat_str(&pf->hw, ret), 13700 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13701 goto out; 13702 } 13703 13704 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 13705 veb->bw_max_quanta = ets_data.tc_bw_max; 13706 veb->is_abs_credits = bw_data.absolute_credits_enable; 13707 veb->enabled_tc = ets_data.tc_valid_bits; 13708 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 13709 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 13710 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 13711 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 13712 veb->bw_tc_limit_credits[i] = 13713 le16_to_cpu(bw_data.tc_bw_limits[i]); 13714 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 13715 } 13716 13717 out: 13718 return ret; 13719 } 13720 13721 /** 13722 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 13723 * @pf: board private structure 13724 * 13725 * On error: returns error code (negative) 13726 * On success: returns vsi index in PF (positive) 13727 **/ 13728 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 13729 { 13730 int ret = -ENOENT; 13731 struct i40e_veb *veb; 13732 int i; 13733 13734 /* Need to protect the allocation of switch elements at the PF level */ 13735 mutex_lock(&pf->switch_mutex); 13736 13737 /* VEB list may be fragmented if VEB creation/destruction has 13738 * been happening. We can afford to do a quick scan to look 13739 * for any free slots in the list. 13740 * 13741 * find next empty veb slot, looping back around if necessary 13742 */ 13743 i = 0; 13744 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 13745 i++; 13746 if (i >= I40E_MAX_VEB) { 13747 ret = -ENOMEM; 13748 goto err_alloc_veb; /* out of VEB slots! */ 13749 } 13750 13751 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 13752 if (!veb) { 13753 ret = -ENOMEM; 13754 goto err_alloc_veb; 13755 } 13756 veb->pf = pf; 13757 veb->idx = i; 13758 veb->enabled_tc = 1; 13759 13760 pf->veb[i] = veb; 13761 ret = i; 13762 err_alloc_veb: 13763 mutex_unlock(&pf->switch_mutex); 13764 return ret; 13765 } 13766 13767 /** 13768 * i40e_switch_branch_release - Delete a branch of the switch tree 13769 * @branch: where to start deleting 13770 * 13771 * This uses recursion to find the tips of the branch to be 13772 * removed, deleting until we get back to and can delete this VEB. 13773 **/ 13774 static void i40e_switch_branch_release(struct i40e_veb *branch) 13775 { 13776 struct i40e_pf *pf = branch->pf; 13777 u16 branch_seid = branch->seid; 13778 u16 veb_idx = branch->idx; 13779 int i; 13780 13781 /* release any VEBs on this VEB - RECURSION */ 13782 for (i = 0; i < I40E_MAX_VEB; i++) { 13783 if (!pf->veb[i]) 13784 continue; 13785 if (pf->veb[i]->uplink_seid == branch->seid) 13786 i40e_switch_branch_release(pf->veb[i]); 13787 } 13788 13789 /* Release the VSIs on this VEB, but not the owner VSI. 13790 * 13791 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 13792 * the VEB itself, so don't use (*branch) after this loop. 13793 */ 13794 for (i = 0; i < pf->num_alloc_vsi; i++) { 13795 if (!pf->vsi[i]) 13796 continue; 13797 if (pf->vsi[i]->uplink_seid == branch_seid && 13798 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13799 i40e_vsi_release(pf->vsi[i]); 13800 } 13801 } 13802 13803 /* There's one corner case where the VEB might not have been 13804 * removed, so double check it here and remove it if needed. 13805 * This case happens if the veb was created from the debugfs 13806 * commands and no VSIs were added to it. 13807 */ 13808 if (pf->veb[veb_idx]) 13809 i40e_veb_release(pf->veb[veb_idx]); 13810 } 13811 13812 /** 13813 * i40e_veb_clear - remove veb struct 13814 * @veb: the veb to remove 13815 **/ 13816 static void i40e_veb_clear(struct i40e_veb *veb) 13817 { 13818 if (!veb) 13819 return; 13820 13821 if (veb->pf) { 13822 struct i40e_pf *pf = veb->pf; 13823 13824 mutex_lock(&pf->switch_mutex); 13825 if (pf->veb[veb->idx] == veb) 13826 pf->veb[veb->idx] = NULL; 13827 mutex_unlock(&pf->switch_mutex); 13828 } 13829 13830 kfree(veb); 13831 } 13832 13833 /** 13834 * i40e_veb_release - Delete a VEB and free its resources 13835 * @veb: the VEB being removed 13836 **/ 13837 void i40e_veb_release(struct i40e_veb *veb) 13838 { 13839 struct i40e_vsi *vsi = NULL; 13840 struct i40e_pf *pf; 13841 int i, n = 0; 13842 13843 pf = veb->pf; 13844 13845 /* find the remaining VSI and check for extras */ 13846 for (i = 0; i < pf->num_alloc_vsi; i++) { 13847 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 13848 n++; 13849 vsi = pf->vsi[i]; 13850 } 13851 } 13852 if (n != 1) { 13853 dev_info(&pf->pdev->dev, 13854 "can't remove VEB %d with %d VSIs left\n", 13855 veb->seid, n); 13856 return; 13857 } 13858 13859 /* move the remaining VSI to uplink veb */ 13860 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 13861 if (veb->uplink_seid) { 13862 vsi->uplink_seid = veb->uplink_seid; 13863 if (veb->uplink_seid == pf->mac_seid) 13864 vsi->veb_idx = I40E_NO_VEB; 13865 else 13866 vsi->veb_idx = veb->veb_idx; 13867 } else { 13868 /* floating VEB */ 13869 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 13870 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 13871 } 13872 13873 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 13874 i40e_veb_clear(veb); 13875 } 13876 13877 /** 13878 * i40e_add_veb - create the VEB in the switch 13879 * @veb: the VEB to be instantiated 13880 * @vsi: the controlling VSI 13881 **/ 13882 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 13883 { 13884 struct i40e_pf *pf = veb->pf; 13885 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 13886 int ret; 13887 13888 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 13889 veb->enabled_tc, false, 13890 &veb->seid, enable_stats, NULL); 13891 13892 /* get a VEB from the hardware */ 13893 if (ret) { 13894 dev_info(&pf->pdev->dev, 13895 "couldn't add VEB, err %s aq_err %s\n", 13896 i40e_stat_str(&pf->hw, ret), 13897 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13898 return -EPERM; 13899 } 13900 13901 /* get statistics counter */ 13902 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 13903 &veb->stats_idx, NULL, NULL, NULL); 13904 if (ret) { 13905 dev_info(&pf->pdev->dev, 13906 "couldn't get VEB statistics idx, err %s aq_err %s\n", 13907 i40e_stat_str(&pf->hw, ret), 13908 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13909 return -EPERM; 13910 } 13911 ret = i40e_veb_get_bw_info(veb); 13912 if (ret) { 13913 dev_info(&pf->pdev->dev, 13914 "couldn't get VEB bw info, err %s aq_err %s\n", 13915 i40e_stat_str(&pf->hw, ret), 13916 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13917 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 13918 return -ENOENT; 13919 } 13920 13921 vsi->uplink_seid = veb->seid; 13922 vsi->veb_idx = veb->idx; 13923 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 13924 13925 return 0; 13926 } 13927 13928 /** 13929 * i40e_veb_setup - Set up a VEB 13930 * @pf: board private structure 13931 * @flags: VEB setup flags 13932 * @uplink_seid: the switch element to link to 13933 * @vsi_seid: the initial VSI seid 13934 * @enabled_tc: Enabled TC bit-map 13935 * 13936 * This allocates the sw VEB structure and links it into the switch 13937 * It is possible and legal for this to be a duplicate of an already 13938 * existing VEB. It is also possible for both uplink and vsi seids 13939 * to be zero, in order to create a floating VEB. 13940 * 13941 * Returns pointer to the successfully allocated VEB sw struct on 13942 * success, otherwise returns NULL on failure. 13943 **/ 13944 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 13945 u16 uplink_seid, u16 vsi_seid, 13946 u8 enabled_tc) 13947 { 13948 struct i40e_veb *veb, *uplink_veb = NULL; 13949 int vsi_idx, veb_idx; 13950 int ret; 13951 13952 /* if one seid is 0, the other must be 0 to create a floating relay */ 13953 if ((uplink_seid == 0 || vsi_seid == 0) && 13954 (uplink_seid + vsi_seid != 0)) { 13955 dev_info(&pf->pdev->dev, 13956 "one, not both seid's are 0: uplink=%d vsi=%d\n", 13957 uplink_seid, vsi_seid); 13958 return NULL; 13959 } 13960 13961 /* make sure there is such a vsi and uplink */ 13962 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 13963 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 13964 break; 13965 if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) { 13966 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 13967 vsi_seid); 13968 return NULL; 13969 } 13970 13971 if (uplink_seid && uplink_seid != pf->mac_seid) { 13972 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 13973 if (pf->veb[veb_idx] && 13974 pf->veb[veb_idx]->seid == uplink_seid) { 13975 uplink_veb = pf->veb[veb_idx]; 13976 break; 13977 } 13978 } 13979 if (!uplink_veb) { 13980 dev_info(&pf->pdev->dev, 13981 "uplink seid %d not found\n", uplink_seid); 13982 return NULL; 13983 } 13984 } 13985 13986 /* get veb sw struct */ 13987 veb_idx = i40e_veb_mem_alloc(pf); 13988 if (veb_idx < 0) 13989 goto err_alloc; 13990 veb = pf->veb[veb_idx]; 13991 veb->flags = flags; 13992 veb->uplink_seid = uplink_seid; 13993 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 13994 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 13995 13996 /* create the VEB in the switch */ 13997 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 13998 if (ret) 13999 goto err_veb; 14000 if (vsi_idx == pf->lan_vsi) 14001 pf->lan_veb = veb->idx; 14002 14003 return veb; 14004 14005 err_veb: 14006 i40e_veb_clear(veb); 14007 err_alloc: 14008 return NULL; 14009 } 14010 14011 /** 14012 * i40e_setup_pf_switch_element - set PF vars based on switch type 14013 * @pf: board private structure 14014 * @ele: element we are building info from 14015 * @num_reported: total number of elements 14016 * @printconfig: should we print the contents 14017 * 14018 * helper function to assist in extracting a few useful SEID values. 14019 **/ 14020 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 14021 struct i40e_aqc_switch_config_element_resp *ele, 14022 u16 num_reported, bool printconfig) 14023 { 14024 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 14025 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 14026 u8 element_type = ele->element_type; 14027 u16 seid = le16_to_cpu(ele->seid); 14028 14029 if (printconfig) 14030 dev_info(&pf->pdev->dev, 14031 "type=%d seid=%d uplink=%d downlink=%d\n", 14032 element_type, seid, uplink_seid, downlink_seid); 14033 14034 switch (element_type) { 14035 case I40E_SWITCH_ELEMENT_TYPE_MAC: 14036 pf->mac_seid = seid; 14037 break; 14038 case I40E_SWITCH_ELEMENT_TYPE_VEB: 14039 /* Main VEB? */ 14040 if (uplink_seid != pf->mac_seid) 14041 break; 14042 if (pf->lan_veb >= I40E_MAX_VEB) { 14043 int v; 14044 14045 /* find existing or else empty VEB */ 14046 for (v = 0; v < I40E_MAX_VEB; v++) { 14047 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 14048 pf->lan_veb = v; 14049 break; 14050 } 14051 } 14052 if (pf->lan_veb >= I40E_MAX_VEB) { 14053 v = i40e_veb_mem_alloc(pf); 14054 if (v < 0) 14055 break; 14056 pf->lan_veb = v; 14057 } 14058 } 14059 if (pf->lan_veb >= I40E_MAX_VEB) 14060 break; 14061 14062 pf->veb[pf->lan_veb]->seid = seid; 14063 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 14064 pf->veb[pf->lan_veb]->pf = pf; 14065 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 14066 break; 14067 case I40E_SWITCH_ELEMENT_TYPE_VSI: 14068 if (num_reported != 1) 14069 break; 14070 /* This is immediately after a reset so we can assume this is 14071 * the PF's VSI 14072 */ 14073 pf->mac_seid = uplink_seid; 14074 pf->pf_seid = downlink_seid; 14075 pf->main_vsi_seid = seid; 14076 if (printconfig) 14077 dev_info(&pf->pdev->dev, 14078 "pf_seid=%d main_vsi_seid=%d\n", 14079 pf->pf_seid, pf->main_vsi_seid); 14080 break; 14081 case I40E_SWITCH_ELEMENT_TYPE_PF: 14082 case I40E_SWITCH_ELEMENT_TYPE_VF: 14083 case I40E_SWITCH_ELEMENT_TYPE_EMP: 14084 case I40E_SWITCH_ELEMENT_TYPE_BMC: 14085 case I40E_SWITCH_ELEMENT_TYPE_PE: 14086 case I40E_SWITCH_ELEMENT_TYPE_PA: 14087 /* ignore these for now */ 14088 break; 14089 default: 14090 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 14091 element_type, seid); 14092 break; 14093 } 14094 } 14095 14096 /** 14097 * i40e_fetch_switch_configuration - Get switch config from firmware 14098 * @pf: board private structure 14099 * @printconfig: should we print the contents 14100 * 14101 * Get the current switch configuration from the device and 14102 * extract a few useful SEID values. 14103 **/ 14104 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 14105 { 14106 struct i40e_aqc_get_switch_config_resp *sw_config; 14107 u16 next_seid = 0; 14108 int ret = 0; 14109 u8 *aq_buf; 14110 int i; 14111 14112 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 14113 if (!aq_buf) 14114 return -ENOMEM; 14115 14116 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 14117 do { 14118 u16 num_reported, num_total; 14119 14120 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 14121 I40E_AQ_LARGE_BUF, 14122 &next_seid, NULL); 14123 if (ret) { 14124 dev_info(&pf->pdev->dev, 14125 "get switch config failed err %s aq_err %s\n", 14126 i40e_stat_str(&pf->hw, ret), 14127 i40e_aq_str(&pf->hw, 14128 pf->hw.aq.asq_last_status)); 14129 kfree(aq_buf); 14130 return -ENOENT; 14131 } 14132 14133 num_reported = le16_to_cpu(sw_config->header.num_reported); 14134 num_total = le16_to_cpu(sw_config->header.num_total); 14135 14136 if (printconfig) 14137 dev_info(&pf->pdev->dev, 14138 "header: %d reported %d total\n", 14139 num_reported, num_total); 14140 14141 for (i = 0; i < num_reported; i++) { 14142 struct i40e_aqc_switch_config_element_resp *ele = 14143 &sw_config->element[i]; 14144 14145 i40e_setup_pf_switch_element(pf, ele, num_reported, 14146 printconfig); 14147 } 14148 } while (next_seid != 0); 14149 14150 kfree(aq_buf); 14151 return ret; 14152 } 14153 14154 /** 14155 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 14156 * @pf: board private structure 14157 * @reinit: if the Main VSI needs to re-initialized. 14158 * 14159 * Returns 0 on success, negative value on failure 14160 **/ 14161 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 14162 { 14163 u16 flags = 0; 14164 int ret; 14165 14166 /* find out what's out there already */ 14167 ret = i40e_fetch_switch_configuration(pf, false); 14168 if (ret) { 14169 dev_info(&pf->pdev->dev, 14170 "couldn't fetch switch config, err %s aq_err %s\n", 14171 i40e_stat_str(&pf->hw, ret), 14172 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14173 return ret; 14174 } 14175 i40e_pf_reset_stats(pf); 14176 14177 /* set the switch config bit for the whole device to 14178 * support limited promisc or true promisc 14179 * when user requests promisc. The default is limited 14180 * promisc. 14181 */ 14182 14183 if ((pf->hw.pf_id == 0) && 14184 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 14185 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14186 pf->last_sw_conf_flags = flags; 14187 } 14188 14189 if (pf->hw.pf_id == 0) { 14190 u16 valid_flags; 14191 14192 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14193 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 14194 NULL); 14195 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 14196 dev_info(&pf->pdev->dev, 14197 "couldn't set switch config bits, err %s aq_err %s\n", 14198 i40e_stat_str(&pf->hw, ret), 14199 i40e_aq_str(&pf->hw, 14200 pf->hw.aq.asq_last_status)); 14201 /* not a fatal problem, just keep going */ 14202 } 14203 pf->last_sw_conf_valid_flags = valid_flags; 14204 } 14205 14206 /* first time setup */ 14207 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 14208 struct i40e_vsi *vsi = NULL; 14209 u16 uplink_seid; 14210 14211 /* Set up the PF VSI associated with the PF's main VSI 14212 * that is already in the HW switch 14213 */ 14214 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 14215 uplink_seid = pf->veb[pf->lan_veb]->seid; 14216 else 14217 uplink_seid = pf->mac_seid; 14218 if (pf->lan_vsi == I40E_NO_VSI) 14219 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 14220 else if (reinit) 14221 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 14222 if (!vsi) { 14223 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 14224 i40e_cloud_filter_exit(pf); 14225 i40e_fdir_teardown(pf); 14226 return -EAGAIN; 14227 } 14228 } else { 14229 /* force a reset of TC and queue layout configurations */ 14230 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 14231 14232 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 14233 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 14234 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 14235 } 14236 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 14237 14238 i40e_fdir_sb_setup(pf); 14239 14240 /* Setup static PF queue filter control settings */ 14241 ret = i40e_setup_pf_filter_control(pf); 14242 if (ret) { 14243 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 14244 ret); 14245 /* Failure here should not stop continuing other steps */ 14246 } 14247 14248 /* enable RSS in the HW, even for only one queue, as the stack can use 14249 * the hash 14250 */ 14251 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 14252 i40e_pf_config_rss(pf); 14253 14254 /* fill in link information and enable LSE reporting */ 14255 i40e_link_event(pf); 14256 14257 /* Initialize user-specific link properties */ 14258 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 14259 I40E_AQ_AN_COMPLETED) ? true : false); 14260 14261 i40e_ptp_init(pf); 14262 14263 /* repopulate tunnel port filters */ 14264 udp_tunnel_nic_reset_ntf(pf->vsi[pf->lan_vsi]->netdev); 14265 14266 return ret; 14267 } 14268 14269 /** 14270 * i40e_determine_queue_usage - Work out queue distribution 14271 * @pf: board private structure 14272 **/ 14273 static void i40e_determine_queue_usage(struct i40e_pf *pf) 14274 { 14275 int queues_left; 14276 int q_max; 14277 14278 pf->num_lan_qps = 0; 14279 14280 /* Find the max queues to be put into basic use. We'll always be 14281 * using TC0, whether or not DCB is running, and TC0 will get the 14282 * big RSS set. 14283 */ 14284 queues_left = pf->hw.func_caps.num_tx_qp; 14285 14286 if ((queues_left == 1) || 14287 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 14288 /* one qp for PF, no queues for anything else */ 14289 queues_left = 0; 14290 pf->alloc_rss_size = pf->num_lan_qps = 1; 14291 14292 /* make sure all the fancies are disabled */ 14293 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 14294 I40E_FLAG_IWARP_ENABLED | 14295 I40E_FLAG_FD_SB_ENABLED | 14296 I40E_FLAG_FD_ATR_ENABLED | 14297 I40E_FLAG_DCB_CAPABLE | 14298 I40E_FLAG_DCB_ENABLED | 14299 I40E_FLAG_SRIOV_ENABLED | 14300 I40E_FLAG_VMDQ_ENABLED); 14301 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14302 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 14303 I40E_FLAG_FD_SB_ENABLED | 14304 I40E_FLAG_FD_ATR_ENABLED | 14305 I40E_FLAG_DCB_CAPABLE))) { 14306 /* one qp for PF */ 14307 pf->alloc_rss_size = pf->num_lan_qps = 1; 14308 queues_left -= pf->num_lan_qps; 14309 14310 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 14311 I40E_FLAG_IWARP_ENABLED | 14312 I40E_FLAG_FD_SB_ENABLED | 14313 I40E_FLAG_FD_ATR_ENABLED | 14314 I40E_FLAG_DCB_ENABLED | 14315 I40E_FLAG_VMDQ_ENABLED); 14316 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14317 } else { 14318 /* Not enough queues for all TCs */ 14319 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 14320 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 14321 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 14322 I40E_FLAG_DCB_ENABLED); 14323 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 14324 } 14325 14326 /* limit lan qps to the smaller of qps, cpus or msix */ 14327 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 14328 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 14329 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 14330 pf->num_lan_qps = q_max; 14331 14332 queues_left -= pf->num_lan_qps; 14333 } 14334 14335 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 14336 if (queues_left > 1) { 14337 queues_left -= 1; /* save 1 queue for FD */ 14338 } else { 14339 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 14340 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14341 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 14342 } 14343 } 14344 14345 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 14346 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 14347 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 14348 (queues_left / pf->num_vf_qps)); 14349 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 14350 } 14351 14352 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 14353 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 14354 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 14355 (queues_left / pf->num_vmdq_qps)); 14356 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 14357 } 14358 14359 pf->queues_left = queues_left; 14360 dev_dbg(&pf->pdev->dev, 14361 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 14362 pf->hw.func_caps.num_tx_qp, 14363 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 14364 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 14365 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 14366 queues_left); 14367 } 14368 14369 /** 14370 * i40e_setup_pf_filter_control - Setup PF static filter control 14371 * @pf: PF to be setup 14372 * 14373 * i40e_setup_pf_filter_control sets up a PF's initial filter control 14374 * settings. If PE/FCoE are enabled then it will also set the per PF 14375 * based filter sizes required for them. It also enables Flow director, 14376 * ethertype and macvlan type filter settings for the pf. 14377 * 14378 * Returns 0 on success, negative on failure 14379 **/ 14380 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 14381 { 14382 struct i40e_filter_control_settings *settings = &pf->filter_settings; 14383 14384 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 14385 14386 /* Flow Director is enabled */ 14387 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 14388 settings->enable_fdir = true; 14389 14390 /* Ethtype and MACVLAN filters enabled for PF */ 14391 settings->enable_ethtype = true; 14392 settings->enable_macvlan = true; 14393 14394 if (i40e_set_filter_control(&pf->hw, settings)) 14395 return -ENOENT; 14396 14397 return 0; 14398 } 14399 14400 #define INFO_STRING_LEN 255 14401 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 14402 static void i40e_print_features(struct i40e_pf *pf) 14403 { 14404 struct i40e_hw *hw = &pf->hw; 14405 char *buf; 14406 int i; 14407 14408 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 14409 if (!buf) 14410 return; 14411 14412 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 14413 #ifdef CONFIG_PCI_IOV 14414 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 14415 #endif 14416 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 14417 pf->hw.func_caps.num_vsis, 14418 pf->vsi[pf->lan_vsi]->num_queue_pairs); 14419 if (pf->flags & I40E_FLAG_RSS_ENABLED) 14420 i += scnprintf(&buf[i], REMAIN(i), " RSS"); 14421 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 14422 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); 14423 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 14424 i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); 14425 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); 14426 } 14427 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 14428 i += scnprintf(&buf[i], REMAIN(i), " DCB"); 14429 i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); 14430 i += scnprintf(&buf[i], REMAIN(i), " Geneve"); 14431 if (pf->flags & I40E_FLAG_PTP) 14432 i += scnprintf(&buf[i], REMAIN(i), " PTP"); 14433 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 14434 i += scnprintf(&buf[i], REMAIN(i), " VEB"); 14435 else 14436 i += scnprintf(&buf[i], REMAIN(i), " VEPA"); 14437 14438 dev_info(&pf->pdev->dev, "%s\n", buf); 14439 kfree(buf); 14440 WARN_ON(i > INFO_STRING_LEN); 14441 } 14442 14443 /** 14444 * i40e_get_platform_mac_addr - get platform-specific MAC address 14445 * @pdev: PCI device information struct 14446 * @pf: board private structure 14447 * 14448 * Look up the MAC address for the device. First we'll try 14449 * eth_platform_get_mac_address, which will check Open Firmware, or arch 14450 * specific fallback. Otherwise, we'll default to the stored value in 14451 * firmware. 14452 **/ 14453 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 14454 { 14455 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 14456 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 14457 } 14458 14459 /** 14460 * i40e_set_fec_in_flags - helper function for setting FEC options in flags 14461 * @fec_cfg: FEC option to set in flags 14462 * @flags: ptr to flags in which we set FEC option 14463 **/ 14464 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags) 14465 { 14466 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) 14467 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC; 14468 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) || 14469 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) { 14470 *flags |= I40E_FLAG_RS_FEC; 14471 *flags &= ~I40E_FLAG_BASE_R_FEC; 14472 } 14473 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) || 14474 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) { 14475 *flags |= I40E_FLAG_BASE_R_FEC; 14476 *flags &= ~I40E_FLAG_RS_FEC; 14477 } 14478 if (fec_cfg == 0) 14479 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC); 14480 } 14481 14482 /** 14483 * i40e_check_recovery_mode - check if we are running transition firmware 14484 * @pf: board private structure 14485 * 14486 * Check registers indicating the firmware runs in recovery mode. Sets the 14487 * appropriate driver state. 14488 * 14489 * Returns true if the recovery mode was detected, false otherwise 14490 **/ 14491 static bool i40e_check_recovery_mode(struct i40e_pf *pf) 14492 { 14493 u32 val = rd32(&pf->hw, I40E_GL_FWSTS); 14494 14495 if (val & I40E_GL_FWSTS_FWS1B_MASK) { 14496 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n"); 14497 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 14498 set_bit(__I40E_RECOVERY_MODE, pf->state); 14499 14500 return true; 14501 } 14502 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 14503 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n"); 14504 14505 return false; 14506 } 14507 14508 /** 14509 * i40e_pf_loop_reset - perform reset in a loop. 14510 * @pf: board private structure 14511 * 14512 * This function is useful when a NIC is about to enter recovery mode. 14513 * When a NIC's internal data structures are corrupted the NIC's 14514 * firmware is going to enter recovery mode. 14515 * Right after a POR it takes about 7 minutes for firmware to enter 14516 * recovery mode. Until that time a NIC is in some kind of intermediate 14517 * state. After that time period the NIC almost surely enters 14518 * recovery mode. The only way for a driver to detect intermediate 14519 * state is to issue a series of pf-resets and check a return value. 14520 * If a PF reset returns success then the firmware could be in recovery 14521 * mode so the caller of this code needs to check for recovery mode 14522 * if this function returns success. There is a little chance that 14523 * firmware will hang in intermediate state forever. 14524 * Since waiting 7 minutes is quite a lot of time this function waits 14525 * 10 seconds and then gives up by returning an error. 14526 * 14527 * Return 0 on success, negative on failure. 14528 **/ 14529 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf) 14530 { 14531 /* wait max 10 seconds for PF reset to succeed */ 14532 const unsigned long time_end = jiffies + 10 * HZ; 14533 14534 struct i40e_hw *hw = &pf->hw; 14535 i40e_status ret; 14536 14537 ret = i40e_pf_reset(hw); 14538 while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) { 14539 usleep_range(10000, 20000); 14540 ret = i40e_pf_reset(hw); 14541 } 14542 14543 if (ret == I40E_SUCCESS) 14544 pf->pfr_count++; 14545 else 14546 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); 14547 14548 return ret; 14549 } 14550 14551 /** 14552 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset 14553 * @pf: board private structure 14554 * 14555 * Check FW registers to determine if FW issued unexpected EMP Reset. 14556 * Every time when unexpected EMP Reset occurs the FW increments 14557 * a counter of unexpected EMP Resets. When the counter reaches 10 14558 * the FW should enter the Recovery mode 14559 * 14560 * Returns true if FW issued unexpected EMP Reset 14561 **/ 14562 static bool i40e_check_fw_empr(struct i40e_pf *pf) 14563 { 14564 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) & 14565 I40E_GL_FWSTS_FWS1B_MASK; 14566 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) && 14567 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10); 14568 } 14569 14570 /** 14571 * i40e_handle_resets - handle EMP resets and PF resets 14572 * @pf: board private structure 14573 * 14574 * Handle both EMP resets and PF resets and conclude whether there are 14575 * any issues regarding these resets. If there are any issues then 14576 * generate log entry. 14577 * 14578 * Return 0 if NIC is healthy or negative value when there are issues 14579 * with resets 14580 **/ 14581 static i40e_status i40e_handle_resets(struct i40e_pf *pf) 14582 { 14583 const i40e_status pfr = i40e_pf_loop_reset(pf); 14584 const bool is_empr = i40e_check_fw_empr(pf); 14585 14586 if (is_empr || pfr != I40E_SUCCESS) 14587 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n"); 14588 14589 return is_empr ? I40E_ERR_RESET_FAILED : pfr; 14590 } 14591 14592 /** 14593 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode 14594 * @pf: board private structure 14595 * @hw: ptr to the hardware info 14596 * 14597 * This function does a minimal setup of all subsystems needed for running 14598 * recovery mode. 14599 * 14600 * Returns 0 on success, negative on failure 14601 **/ 14602 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw) 14603 { 14604 struct i40e_vsi *vsi; 14605 int err; 14606 int v_idx; 14607 14608 pci_save_state(pf->pdev); 14609 14610 /* set up periodic task facility */ 14611 timer_setup(&pf->service_timer, i40e_service_timer, 0); 14612 pf->service_timer_period = HZ; 14613 14614 INIT_WORK(&pf->service_task, i40e_service_task); 14615 clear_bit(__I40E_SERVICE_SCHED, pf->state); 14616 14617 err = i40e_init_interrupt_scheme(pf); 14618 if (err) 14619 goto err_switch_setup; 14620 14621 /* The number of VSIs reported by the FW is the minimum guaranteed 14622 * to us; HW supports far more and we share the remaining pool with 14623 * the other PFs. We allocate space for more than the guarantee with 14624 * the understanding that we might not get them all later. 14625 */ 14626 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 14627 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 14628 else 14629 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 14630 14631 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */ 14632 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 14633 GFP_KERNEL); 14634 if (!pf->vsi) { 14635 err = -ENOMEM; 14636 goto err_switch_setup; 14637 } 14638 14639 /* We allocate one VSI which is needed as absolute minimum 14640 * in order to register the netdev 14641 */ 14642 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN); 14643 if (v_idx < 0) 14644 goto err_switch_setup; 14645 pf->lan_vsi = v_idx; 14646 vsi = pf->vsi[v_idx]; 14647 if (!vsi) 14648 goto err_switch_setup; 14649 vsi->alloc_queue_pairs = 1; 14650 err = i40e_config_netdev(vsi); 14651 if (err) 14652 goto err_switch_setup; 14653 err = register_netdev(vsi->netdev); 14654 if (err) 14655 goto err_switch_setup; 14656 vsi->netdev_registered = true; 14657 i40e_dbg_pf_init(pf); 14658 14659 err = i40e_setup_misc_vector_for_recovery_mode(pf); 14660 if (err) 14661 goto err_switch_setup; 14662 14663 /* tell the firmware that we're starting */ 14664 i40e_send_version(pf); 14665 14666 /* since everything's happy, start the service_task timer */ 14667 mod_timer(&pf->service_timer, 14668 round_jiffies(jiffies + pf->service_timer_period)); 14669 14670 return 0; 14671 14672 err_switch_setup: 14673 i40e_reset_interrupt_capability(pf); 14674 del_timer_sync(&pf->service_timer); 14675 i40e_shutdown_adminq(hw); 14676 iounmap(hw->hw_addr); 14677 pci_disable_pcie_error_reporting(pf->pdev); 14678 pci_release_mem_regions(pf->pdev); 14679 pci_disable_device(pf->pdev); 14680 kfree(pf); 14681 14682 return err; 14683 } 14684 14685 /** 14686 * i40e_probe - Device initialization routine 14687 * @pdev: PCI device information struct 14688 * @ent: entry in i40e_pci_tbl 14689 * 14690 * i40e_probe initializes a PF identified by a pci_dev structure. 14691 * The OS initialization, configuring of the PF private structure, 14692 * and a hardware reset occur. 14693 * 14694 * Returns 0 on success, negative on failure 14695 **/ 14696 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 14697 { 14698 struct i40e_aq_get_phy_abilities_resp abilities; 14699 struct i40e_pf *pf; 14700 struct i40e_hw *hw; 14701 static u16 pfs_found; 14702 u16 wol_nvm_bits; 14703 u16 link_status; 14704 int err; 14705 u32 val; 14706 u32 i; 14707 u8 set_fc_aq_fail; 14708 14709 err = pci_enable_device_mem(pdev); 14710 if (err) 14711 return err; 14712 14713 /* set up for high or low dma */ 14714 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 14715 if (err) { 14716 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 14717 if (err) { 14718 dev_err(&pdev->dev, 14719 "DMA configuration failed: 0x%x\n", err); 14720 goto err_dma; 14721 } 14722 } 14723 14724 /* set up pci connections */ 14725 err = pci_request_mem_regions(pdev, i40e_driver_name); 14726 if (err) { 14727 dev_info(&pdev->dev, 14728 "pci_request_selected_regions failed %d\n", err); 14729 goto err_pci_reg; 14730 } 14731 14732 pci_enable_pcie_error_reporting(pdev); 14733 pci_set_master(pdev); 14734 14735 /* Now that we have a PCI connection, we need to do the 14736 * low level device setup. This is primarily setting up 14737 * the Admin Queue structures and then querying for the 14738 * device's current profile information. 14739 */ 14740 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 14741 if (!pf) { 14742 err = -ENOMEM; 14743 goto err_pf_alloc; 14744 } 14745 pf->next_vsi = 0; 14746 pf->pdev = pdev; 14747 set_bit(__I40E_DOWN, pf->state); 14748 14749 hw = &pf->hw; 14750 hw->back = pf; 14751 14752 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 14753 I40E_MAX_CSR_SPACE); 14754 /* We believe that the highest register to read is 14755 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size 14756 * is not less than that before mapping to prevent a 14757 * kernel panic. 14758 */ 14759 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) { 14760 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n", 14761 pf->ioremap_len); 14762 err = -ENOMEM; 14763 goto err_ioremap; 14764 } 14765 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 14766 if (!hw->hw_addr) { 14767 err = -EIO; 14768 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 14769 (unsigned int)pci_resource_start(pdev, 0), 14770 pf->ioremap_len, err); 14771 goto err_ioremap; 14772 } 14773 hw->vendor_id = pdev->vendor; 14774 hw->device_id = pdev->device; 14775 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 14776 hw->subsystem_vendor_id = pdev->subsystem_vendor; 14777 hw->subsystem_device_id = pdev->subsystem_device; 14778 hw->bus.device = PCI_SLOT(pdev->devfn); 14779 hw->bus.func = PCI_FUNC(pdev->devfn); 14780 hw->bus.bus_id = pdev->bus->number; 14781 pf->instance = pfs_found; 14782 14783 /* Select something other than the 802.1ad ethertype for the 14784 * switch to use internally and drop on ingress. 14785 */ 14786 hw->switch_tag = 0xffff; 14787 hw->first_tag = ETH_P_8021AD; 14788 hw->second_tag = ETH_P_8021Q; 14789 14790 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 14791 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 14792 INIT_LIST_HEAD(&pf->ddp_old_prof); 14793 14794 /* set up the locks for the AQ, do this only once in probe 14795 * and destroy them only once in remove 14796 */ 14797 mutex_init(&hw->aq.asq_mutex); 14798 mutex_init(&hw->aq.arq_mutex); 14799 14800 pf->msg_enable = netif_msg_init(debug, 14801 NETIF_MSG_DRV | 14802 NETIF_MSG_PROBE | 14803 NETIF_MSG_LINK); 14804 if (debug < -1) 14805 pf->hw.debug_mask = debug; 14806 14807 /* do a special CORER for clearing PXE mode once at init */ 14808 if (hw->revision_id == 0 && 14809 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 14810 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 14811 i40e_flush(hw); 14812 msleep(200); 14813 pf->corer_count++; 14814 14815 i40e_clear_pxe_mode(hw); 14816 } 14817 14818 /* Reset here to make sure all is clean and to define PF 'n' */ 14819 i40e_clear_hw(hw); 14820 14821 err = i40e_set_mac_type(hw); 14822 if (err) { 14823 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 14824 err); 14825 goto err_pf_reset; 14826 } 14827 14828 err = i40e_handle_resets(pf); 14829 if (err) 14830 goto err_pf_reset; 14831 14832 i40e_check_recovery_mode(pf); 14833 14834 hw->aq.num_arq_entries = I40E_AQ_LEN; 14835 hw->aq.num_asq_entries = I40E_AQ_LEN; 14836 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 14837 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 14838 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 14839 14840 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 14841 "%s-%s:misc", 14842 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 14843 14844 err = i40e_init_shared_code(hw); 14845 if (err) { 14846 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 14847 err); 14848 goto err_pf_reset; 14849 } 14850 14851 /* set up a default setting for link flow control */ 14852 pf->hw.fc.requested_mode = I40E_FC_NONE; 14853 14854 err = i40e_init_adminq(hw); 14855 if (err) { 14856 if (err == I40E_ERR_FIRMWARE_API_VERSION) 14857 dev_info(&pdev->dev, 14858 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", 14859 hw->aq.api_maj_ver, 14860 hw->aq.api_min_ver, 14861 I40E_FW_API_VERSION_MAJOR, 14862 I40E_FW_MINOR_VERSION(hw)); 14863 else 14864 dev_info(&pdev->dev, 14865 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 14866 14867 goto err_pf_reset; 14868 } 14869 i40e_get_oem_version(hw); 14870 14871 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */ 14872 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n", 14873 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 14874 hw->aq.api_maj_ver, hw->aq.api_min_ver, 14875 i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id, 14876 hw->subsystem_vendor_id, hw->subsystem_device_id); 14877 14878 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 14879 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 14880 dev_info(&pdev->dev, 14881 "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n", 14882 hw->aq.api_maj_ver, 14883 hw->aq.api_min_ver, 14884 I40E_FW_API_VERSION_MAJOR, 14885 I40E_FW_MINOR_VERSION(hw)); 14886 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 14887 dev_info(&pdev->dev, 14888 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n", 14889 hw->aq.api_maj_ver, 14890 hw->aq.api_min_ver, 14891 I40E_FW_API_VERSION_MAJOR, 14892 I40E_FW_MINOR_VERSION(hw)); 14893 14894 i40e_verify_eeprom(pf); 14895 14896 /* Rev 0 hardware was never productized */ 14897 if (hw->revision_id < 1) 14898 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 14899 14900 i40e_clear_pxe_mode(hw); 14901 14902 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 14903 if (err) 14904 goto err_adminq_setup; 14905 14906 err = i40e_sw_init(pf); 14907 if (err) { 14908 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 14909 goto err_sw_init; 14910 } 14911 14912 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 14913 return i40e_init_recovery_mode(pf, hw); 14914 14915 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 14916 hw->func_caps.num_rx_qp, 0, 0); 14917 if (err) { 14918 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 14919 goto err_init_lan_hmc; 14920 } 14921 14922 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 14923 if (err) { 14924 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 14925 err = -ENOENT; 14926 goto err_configure_lan_hmc; 14927 } 14928 14929 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 14930 * Ignore error return codes because if it was already disabled via 14931 * hardware settings this will fail 14932 */ 14933 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 14934 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 14935 i40e_aq_stop_lldp(hw, true, false, NULL); 14936 } 14937 14938 /* allow a platform config to override the HW addr */ 14939 i40e_get_platform_mac_addr(pdev, pf); 14940 14941 if (!is_valid_ether_addr(hw->mac.addr)) { 14942 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 14943 err = -EIO; 14944 goto err_mac_addr; 14945 } 14946 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 14947 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 14948 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 14949 if (is_valid_ether_addr(hw->mac.port_addr)) 14950 pf->hw_features |= I40E_HW_PORT_ID_VALID; 14951 14952 pci_set_drvdata(pdev, pf); 14953 pci_save_state(pdev); 14954 14955 dev_info(&pdev->dev, 14956 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ? 14957 "FW LLDP is disabled\n" : 14958 "FW LLDP is enabled\n"); 14959 14960 /* Enable FW to write default DCB config on link-up */ 14961 i40e_aq_set_dcb_parameters(hw, true, NULL); 14962 14963 #ifdef CONFIG_I40E_DCB 14964 err = i40e_init_pf_dcb(pf); 14965 if (err) { 14966 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 14967 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 14968 /* Continue without DCB enabled */ 14969 } 14970 #endif /* CONFIG_I40E_DCB */ 14971 14972 /* set up periodic task facility */ 14973 timer_setup(&pf->service_timer, i40e_service_timer, 0); 14974 pf->service_timer_period = HZ; 14975 14976 INIT_WORK(&pf->service_task, i40e_service_task); 14977 clear_bit(__I40E_SERVICE_SCHED, pf->state); 14978 14979 /* NVM bit on means WoL disabled for the port */ 14980 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 14981 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 14982 pf->wol_en = false; 14983 else 14984 pf->wol_en = true; 14985 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 14986 14987 /* set up the main switch operations */ 14988 i40e_determine_queue_usage(pf); 14989 err = i40e_init_interrupt_scheme(pf); 14990 if (err) 14991 goto err_switch_setup; 14992 14993 pf->udp_tunnel_nic.set_port = i40e_udp_tunnel_set_port; 14994 pf->udp_tunnel_nic.unset_port = i40e_udp_tunnel_unset_port; 14995 pf->udp_tunnel_nic.flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP; 14996 pf->udp_tunnel_nic.shared = &pf->udp_tunnel_shared; 14997 pf->udp_tunnel_nic.tables[0].n_entries = I40E_MAX_PF_UDP_OFFLOAD_PORTS; 14998 pf->udp_tunnel_nic.tables[0].tunnel_types = UDP_TUNNEL_TYPE_VXLAN | 14999 UDP_TUNNEL_TYPE_GENEVE; 15000 15001 /* The number of VSIs reported by the FW is the minimum guaranteed 15002 * to us; HW supports far more and we share the remaining pool with 15003 * the other PFs. We allocate space for more than the guarantee with 15004 * the understanding that we might not get them all later. 15005 */ 15006 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15007 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15008 else 15009 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15010 if (pf->num_alloc_vsi > UDP_TUNNEL_NIC_MAX_SHARING_DEVICES) { 15011 dev_warn(&pf->pdev->dev, 15012 "limiting the VSI count due to UDP tunnel limitation %d > %d\n", 15013 pf->num_alloc_vsi, UDP_TUNNEL_NIC_MAX_SHARING_DEVICES); 15014 pf->num_alloc_vsi = UDP_TUNNEL_NIC_MAX_SHARING_DEVICES; 15015 } 15016 15017 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 15018 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15019 GFP_KERNEL); 15020 if (!pf->vsi) { 15021 err = -ENOMEM; 15022 goto err_switch_setup; 15023 } 15024 15025 #ifdef CONFIG_PCI_IOV 15026 /* prep for VF support */ 15027 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15028 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15029 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15030 if (pci_num_vf(pdev)) 15031 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 15032 } 15033 #endif 15034 err = i40e_setup_pf_switch(pf, false); 15035 if (err) { 15036 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 15037 goto err_vsis; 15038 } 15039 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 15040 15041 /* Make sure flow control is set according to current settings */ 15042 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 15043 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 15044 dev_dbg(&pf->pdev->dev, 15045 "Set fc with err %s aq_err %s on get_phy_cap\n", 15046 i40e_stat_str(hw, err), 15047 i40e_aq_str(hw, hw->aq.asq_last_status)); 15048 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 15049 dev_dbg(&pf->pdev->dev, 15050 "Set fc with err %s aq_err %s on set_phy_config\n", 15051 i40e_stat_str(hw, err), 15052 i40e_aq_str(hw, hw->aq.asq_last_status)); 15053 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 15054 dev_dbg(&pf->pdev->dev, 15055 "Set fc with err %s aq_err %s on get_link_info\n", 15056 i40e_stat_str(hw, err), 15057 i40e_aq_str(hw, hw->aq.asq_last_status)); 15058 15059 /* if FDIR VSI was set up, start it now */ 15060 for (i = 0; i < pf->num_alloc_vsi; i++) { 15061 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 15062 i40e_vsi_open(pf->vsi[i]); 15063 break; 15064 } 15065 } 15066 15067 /* The driver only wants link up/down and module qualification 15068 * reports from firmware. Note the negative logic. 15069 */ 15070 err = i40e_aq_set_phy_int_mask(&pf->hw, 15071 ~(I40E_AQ_EVENT_LINK_UPDOWN | 15072 I40E_AQ_EVENT_MEDIA_NA | 15073 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 15074 if (err) 15075 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 15076 i40e_stat_str(&pf->hw, err), 15077 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15078 15079 /* Reconfigure hardware for allowing smaller MSS in the case 15080 * of TSO, so that we avoid the MDD being fired and causing 15081 * a reset in the case of small MSS+TSO. 15082 */ 15083 val = rd32(hw, I40E_REG_MSS); 15084 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 15085 val &= ~I40E_REG_MSS_MIN_MASK; 15086 val |= I40E_64BYTE_MSS; 15087 wr32(hw, I40E_REG_MSS, val); 15088 } 15089 15090 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 15091 msleep(75); 15092 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 15093 if (err) 15094 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 15095 i40e_stat_str(&pf->hw, err), 15096 i40e_aq_str(&pf->hw, 15097 pf->hw.aq.asq_last_status)); 15098 } 15099 /* The main driver is (mostly) up and happy. We need to set this state 15100 * before setting up the misc vector or we get a race and the vector 15101 * ends up disabled forever. 15102 */ 15103 clear_bit(__I40E_DOWN, pf->state); 15104 15105 /* In case of MSIX we are going to setup the misc vector right here 15106 * to handle admin queue events etc. In case of legacy and MSI 15107 * the misc functionality and queue processing is combined in 15108 * the same vector and that gets setup at open. 15109 */ 15110 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 15111 err = i40e_setup_misc_vector(pf); 15112 if (err) { 15113 dev_info(&pdev->dev, 15114 "setup of misc vector failed: %d\n", err); 15115 goto err_vsis; 15116 } 15117 } 15118 15119 #ifdef CONFIG_PCI_IOV 15120 /* prep for VF support */ 15121 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15122 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15123 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15124 /* disable link interrupts for VFs */ 15125 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 15126 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 15127 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 15128 i40e_flush(hw); 15129 15130 if (pci_num_vf(pdev)) { 15131 dev_info(&pdev->dev, 15132 "Active VFs found, allocating resources.\n"); 15133 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 15134 if (err) 15135 dev_info(&pdev->dev, 15136 "Error %d allocating resources for existing VFs\n", 15137 err); 15138 } 15139 } 15140 #endif /* CONFIG_PCI_IOV */ 15141 15142 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15143 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 15144 pf->num_iwarp_msix, 15145 I40E_IWARP_IRQ_PILE_ID); 15146 if (pf->iwarp_base_vector < 0) { 15147 dev_info(&pdev->dev, 15148 "failed to get tracking for %d vectors for IWARP err=%d\n", 15149 pf->num_iwarp_msix, pf->iwarp_base_vector); 15150 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 15151 } 15152 } 15153 15154 i40e_dbg_pf_init(pf); 15155 15156 /* tell the firmware that we're starting */ 15157 i40e_send_version(pf); 15158 15159 /* since everything's happy, start the service_task timer */ 15160 mod_timer(&pf->service_timer, 15161 round_jiffies(jiffies + pf->service_timer_period)); 15162 15163 /* add this PF to client device list and launch a client service task */ 15164 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15165 err = i40e_lan_add_device(pf); 15166 if (err) 15167 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 15168 err); 15169 } 15170 15171 #define PCI_SPEED_SIZE 8 15172 #define PCI_WIDTH_SIZE 8 15173 /* Devices on the IOSF bus do not have this information 15174 * and will report PCI Gen 1 x 1 by default so don't bother 15175 * checking them. 15176 */ 15177 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 15178 char speed[PCI_SPEED_SIZE] = "Unknown"; 15179 char width[PCI_WIDTH_SIZE] = "Unknown"; 15180 15181 /* Get the negotiated link width and speed from PCI config 15182 * space 15183 */ 15184 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 15185 &link_status); 15186 15187 i40e_set_pci_config_data(hw, link_status); 15188 15189 switch (hw->bus.speed) { 15190 case i40e_bus_speed_8000: 15191 strlcpy(speed, "8.0", PCI_SPEED_SIZE); break; 15192 case i40e_bus_speed_5000: 15193 strlcpy(speed, "5.0", PCI_SPEED_SIZE); break; 15194 case i40e_bus_speed_2500: 15195 strlcpy(speed, "2.5", PCI_SPEED_SIZE); break; 15196 default: 15197 break; 15198 } 15199 switch (hw->bus.width) { 15200 case i40e_bus_width_pcie_x8: 15201 strlcpy(width, "8", PCI_WIDTH_SIZE); break; 15202 case i40e_bus_width_pcie_x4: 15203 strlcpy(width, "4", PCI_WIDTH_SIZE); break; 15204 case i40e_bus_width_pcie_x2: 15205 strlcpy(width, "2", PCI_WIDTH_SIZE); break; 15206 case i40e_bus_width_pcie_x1: 15207 strlcpy(width, "1", PCI_WIDTH_SIZE); break; 15208 default: 15209 break; 15210 } 15211 15212 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 15213 speed, width); 15214 15215 if (hw->bus.width < i40e_bus_width_pcie_x8 || 15216 hw->bus.speed < i40e_bus_speed_8000) { 15217 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 15218 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 15219 } 15220 } 15221 15222 /* get the requested speeds from the fw */ 15223 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 15224 if (err) 15225 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 15226 i40e_stat_str(&pf->hw, err), 15227 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15228 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 15229 15230 /* set the FEC config due to the board capabilities */ 15231 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags); 15232 15233 /* get the supported phy types from the fw */ 15234 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 15235 if (err) 15236 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 15237 i40e_stat_str(&pf->hw, err), 15238 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15239 15240 /* make sure the MFS hasn't been set lower than the default */ 15241 #define MAX_FRAME_SIZE_DEFAULT 0x2600 15242 val = (rd32(&pf->hw, I40E_PRTGL_SAH) & 15243 I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT; 15244 if (val < MAX_FRAME_SIZE_DEFAULT) 15245 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n", 15246 i, val); 15247 15248 /* Add a filter to drop all Flow control frames from any VSI from being 15249 * transmitted. By doing so we stop a malicious VF from sending out 15250 * PAUSE or PFC frames and potentially controlling traffic for other 15251 * PF/VF VSIs. 15252 * The FW can still send Flow control frames if enabled. 15253 */ 15254 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 15255 pf->main_vsi_seid); 15256 15257 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 15258 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 15259 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 15260 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 15261 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 15262 /* print a string summarizing features */ 15263 i40e_print_features(pf); 15264 15265 return 0; 15266 15267 /* Unwind what we've done if something failed in the setup */ 15268 err_vsis: 15269 set_bit(__I40E_DOWN, pf->state); 15270 i40e_clear_interrupt_scheme(pf); 15271 kfree(pf->vsi); 15272 err_switch_setup: 15273 i40e_reset_interrupt_capability(pf); 15274 del_timer_sync(&pf->service_timer); 15275 err_mac_addr: 15276 err_configure_lan_hmc: 15277 (void)i40e_shutdown_lan_hmc(hw); 15278 err_init_lan_hmc: 15279 kfree(pf->qp_pile); 15280 err_sw_init: 15281 err_adminq_setup: 15282 err_pf_reset: 15283 iounmap(hw->hw_addr); 15284 err_ioremap: 15285 kfree(pf); 15286 err_pf_alloc: 15287 pci_disable_pcie_error_reporting(pdev); 15288 pci_release_mem_regions(pdev); 15289 err_pci_reg: 15290 err_dma: 15291 pci_disable_device(pdev); 15292 return err; 15293 } 15294 15295 /** 15296 * i40e_remove - Device removal routine 15297 * @pdev: PCI device information struct 15298 * 15299 * i40e_remove is called by the PCI subsystem to alert the driver 15300 * that is should release a PCI device. This could be caused by a 15301 * Hot-Plug event, or because the driver is going to be removed from 15302 * memory. 15303 **/ 15304 static void i40e_remove(struct pci_dev *pdev) 15305 { 15306 struct i40e_pf *pf = pci_get_drvdata(pdev); 15307 struct i40e_hw *hw = &pf->hw; 15308 i40e_status ret_code; 15309 int i; 15310 15311 i40e_dbg_pf_exit(pf); 15312 15313 i40e_ptp_stop(pf); 15314 15315 /* Disable RSS in hw */ 15316 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 15317 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 15318 15319 while (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 15320 usleep_range(1000, 2000); 15321 15322 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 15323 set_bit(__I40E_VF_RESETS_DISABLED, pf->state); 15324 i40e_free_vfs(pf); 15325 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 15326 } 15327 /* no more scheduling of any task */ 15328 set_bit(__I40E_SUSPENDED, pf->state); 15329 set_bit(__I40E_DOWN, pf->state); 15330 if (pf->service_timer.function) 15331 del_timer_sync(&pf->service_timer); 15332 if (pf->service_task.func) 15333 cancel_work_sync(&pf->service_task); 15334 15335 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 15336 struct i40e_vsi *vsi = pf->vsi[0]; 15337 15338 /* We know that we have allocated only one vsi for this PF, 15339 * it was just for registering netdevice, so the interface 15340 * could be visible in the 'ifconfig' output 15341 */ 15342 unregister_netdev(vsi->netdev); 15343 free_netdev(vsi->netdev); 15344 15345 goto unmap; 15346 } 15347 15348 /* Client close must be called explicitly here because the timer 15349 * has been stopped. 15350 */ 15351 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15352 15353 i40e_fdir_teardown(pf); 15354 15355 /* If there is a switch structure or any orphans, remove them. 15356 * This will leave only the PF's VSI remaining. 15357 */ 15358 for (i = 0; i < I40E_MAX_VEB; i++) { 15359 if (!pf->veb[i]) 15360 continue; 15361 15362 if (pf->veb[i]->uplink_seid == pf->mac_seid || 15363 pf->veb[i]->uplink_seid == 0) 15364 i40e_switch_branch_release(pf->veb[i]); 15365 } 15366 15367 /* Now we can shutdown the PF's VSI, just before we kill 15368 * adminq and hmc. 15369 */ 15370 if (pf->vsi[pf->lan_vsi]) 15371 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 15372 15373 i40e_cloud_filter_exit(pf); 15374 15375 /* remove attached clients */ 15376 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15377 ret_code = i40e_lan_del_device(pf); 15378 if (ret_code) 15379 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 15380 ret_code); 15381 } 15382 15383 /* shutdown and destroy the HMC */ 15384 if (hw->hmc.hmc_obj) { 15385 ret_code = i40e_shutdown_lan_hmc(hw); 15386 if (ret_code) 15387 dev_warn(&pdev->dev, 15388 "Failed to destroy the HMC resources: %d\n", 15389 ret_code); 15390 } 15391 15392 unmap: 15393 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 15394 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 15395 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 15396 free_irq(pf->pdev->irq, pf); 15397 15398 /* shutdown the adminq */ 15399 i40e_shutdown_adminq(hw); 15400 15401 /* destroy the locks only once, here */ 15402 mutex_destroy(&hw->aq.arq_mutex); 15403 mutex_destroy(&hw->aq.asq_mutex); 15404 15405 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 15406 rtnl_lock(); 15407 i40e_clear_interrupt_scheme(pf); 15408 for (i = 0; i < pf->num_alloc_vsi; i++) { 15409 if (pf->vsi[i]) { 15410 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) 15411 i40e_vsi_clear_rings(pf->vsi[i]); 15412 i40e_vsi_clear(pf->vsi[i]); 15413 pf->vsi[i] = NULL; 15414 } 15415 } 15416 rtnl_unlock(); 15417 15418 for (i = 0; i < I40E_MAX_VEB; i++) { 15419 kfree(pf->veb[i]); 15420 pf->veb[i] = NULL; 15421 } 15422 15423 kfree(pf->qp_pile); 15424 kfree(pf->vsi); 15425 15426 iounmap(hw->hw_addr); 15427 kfree(pf); 15428 pci_release_mem_regions(pdev); 15429 15430 pci_disable_pcie_error_reporting(pdev); 15431 pci_disable_device(pdev); 15432 } 15433 15434 /** 15435 * i40e_pci_error_detected - warning that something funky happened in PCI land 15436 * @pdev: PCI device information struct 15437 * @error: the type of PCI error 15438 * 15439 * Called to warn that something happened and the error handling steps 15440 * are in progress. Allows the driver to quiesce things, be ready for 15441 * remediation. 15442 **/ 15443 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 15444 pci_channel_state_t error) 15445 { 15446 struct i40e_pf *pf = pci_get_drvdata(pdev); 15447 15448 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 15449 15450 if (!pf) { 15451 dev_info(&pdev->dev, 15452 "Cannot recover - error happened during device probe\n"); 15453 return PCI_ERS_RESULT_DISCONNECT; 15454 } 15455 15456 /* shutdown all operations */ 15457 if (!test_bit(__I40E_SUSPENDED, pf->state)) 15458 i40e_prep_for_reset(pf, false); 15459 15460 /* Request a slot reset */ 15461 return PCI_ERS_RESULT_NEED_RESET; 15462 } 15463 15464 /** 15465 * i40e_pci_error_slot_reset - a PCI slot reset just happened 15466 * @pdev: PCI device information struct 15467 * 15468 * Called to find if the driver can work with the device now that 15469 * the pci slot has been reset. If a basic connection seems good 15470 * (registers are readable and have sane content) then return a 15471 * happy little PCI_ERS_RESULT_xxx. 15472 **/ 15473 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 15474 { 15475 struct i40e_pf *pf = pci_get_drvdata(pdev); 15476 pci_ers_result_t result; 15477 u32 reg; 15478 15479 dev_dbg(&pdev->dev, "%s\n", __func__); 15480 if (pci_enable_device_mem(pdev)) { 15481 dev_info(&pdev->dev, 15482 "Cannot re-enable PCI device after reset.\n"); 15483 result = PCI_ERS_RESULT_DISCONNECT; 15484 } else { 15485 pci_set_master(pdev); 15486 pci_restore_state(pdev); 15487 pci_save_state(pdev); 15488 pci_wake_from_d3(pdev, false); 15489 15490 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 15491 if (reg == 0) 15492 result = PCI_ERS_RESULT_RECOVERED; 15493 else 15494 result = PCI_ERS_RESULT_DISCONNECT; 15495 } 15496 15497 return result; 15498 } 15499 15500 /** 15501 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 15502 * @pdev: PCI device information struct 15503 */ 15504 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 15505 { 15506 struct i40e_pf *pf = pci_get_drvdata(pdev); 15507 15508 i40e_prep_for_reset(pf, false); 15509 } 15510 15511 /** 15512 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 15513 * @pdev: PCI device information struct 15514 */ 15515 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 15516 { 15517 struct i40e_pf *pf = pci_get_drvdata(pdev); 15518 15519 i40e_reset_and_rebuild(pf, false, false); 15520 } 15521 15522 /** 15523 * i40e_pci_error_resume - restart operations after PCI error recovery 15524 * @pdev: PCI device information struct 15525 * 15526 * Called to allow the driver to bring things back up after PCI error 15527 * and/or reset recovery has finished. 15528 **/ 15529 static void i40e_pci_error_resume(struct pci_dev *pdev) 15530 { 15531 struct i40e_pf *pf = pci_get_drvdata(pdev); 15532 15533 dev_dbg(&pdev->dev, "%s\n", __func__); 15534 if (test_bit(__I40E_SUSPENDED, pf->state)) 15535 return; 15536 15537 i40e_handle_reset_warning(pf, false); 15538 } 15539 15540 /** 15541 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 15542 * using the mac_address_write admin q function 15543 * @pf: pointer to i40e_pf struct 15544 **/ 15545 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 15546 { 15547 struct i40e_hw *hw = &pf->hw; 15548 i40e_status ret; 15549 u8 mac_addr[6]; 15550 u16 flags = 0; 15551 15552 /* Get current MAC address in case it's an LAA */ 15553 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 15554 ether_addr_copy(mac_addr, 15555 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 15556 } else { 15557 dev_err(&pf->pdev->dev, 15558 "Failed to retrieve MAC address; using default\n"); 15559 ether_addr_copy(mac_addr, hw->mac.addr); 15560 } 15561 15562 /* The FW expects the mac address write cmd to first be called with 15563 * one of these flags before calling it again with the multicast 15564 * enable flags. 15565 */ 15566 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 15567 15568 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 15569 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 15570 15571 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 15572 if (ret) { 15573 dev_err(&pf->pdev->dev, 15574 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 15575 return; 15576 } 15577 15578 flags = I40E_AQC_MC_MAG_EN 15579 | I40E_AQC_WOL_PRESERVE_ON_PFR 15580 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 15581 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 15582 if (ret) 15583 dev_err(&pf->pdev->dev, 15584 "Failed to enable Multicast Magic Packet wake up\n"); 15585 } 15586 15587 /** 15588 * i40e_shutdown - PCI callback for shutting down 15589 * @pdev: PCI device information struct 15590 **/ 15591 static void i40e_shutdown(struct pci_dev *pdev) 15592 { 15593 struct i40e_pf *pf = pci_get_drvdata(pdev); 15594 struct i40e_hw *hw = &pf->hw; 15595 15596 set_bit(__I40E_SUSPENDED, pf->state); 15597 set_bit(__I40E_DOWN, pf->state); 15598 15599 del_timer_sync(&pf->service_timer); 15600 cancel_work_sync(&pf->service_task); 15601 i40e_cloud_filter_exit(pf); 15602 i40e_fdir_teardown(pf); 15603 15604 /* Client close must be called explicitly here because the timer 15605 * has been stopped. 15606 */ 15607 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15608 15609 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 15610 i40e_enable_mc_magic_wake(pf); 15611 15612 i40e_prep_for_reset(pf, false); 15613 15614 wr32(hw, I40E_PFPM_APM, 15615 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 15616 wr32(hw, I40E_PFPM_WUFC, 15617 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 15618 15619 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 15620 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 15621 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 15622 free_irq(pf->pdev->irq, pf); 15623 15624 /* Since we're going to destroy queues during the 15625 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 15626 * whole section 15627 */ 15628 rtnl_lock(); 15629 i40e_clear_interrupt_scheme(pf); 15630 rtnl_unlock(); 15631 15632 if (system_state == SYSTEM_POWER_OFF) { 15633 pci_wake_from_d3(pdev, pf->wol_en); 15634 pci_set_power_state(pdev, PCI_D3hot); 15635 } 15636 } 15637 15638 /** 15639 * i40e_suspend - PM callback for moving to D3 15640 * @dev: generic device information structure 15641 **/ 15642 static int __maybe_unused i40e_suspend(struct device *dev) 15643 { 15644 struct i40e_pf *pf = dev_get_drvdata(dev); 15645 struct i40e_hw *hw = &pf->hw; 15646 15647 /* If we're already suspended, then there is nothing to do */ 15648 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 15649 return 0; 15650 15651 set_bit(__I40E_DOWN, pf->state); 15652 15653 /* Ensure service task will not be running */ 15654 del_timer_sync(&pf->service_timer); 15655 cancel_work_sync(&pf->service_task); 15656 15657 /* Client close must be called explicitly here because the timer 15658 * has been stopped. 15659 */ 15660 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15661 15662 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 15663 i40e_enable_mc_magic_wake(pf); 15664 15665 /* Since we're going to destroy queues during the 15666 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 15667 * whole section 15668 */ 15669 rtnl_lock(); 15670 15671 i40e_prep_for_reset(pf, true); 15672 15673 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 15674 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 15675 15676 /* Clear the interrupt scheme and release our IRQs so that the system 15677 * can safely hibernate even when there are a large number of CPUs. 15678 * Otherwise hibernation might fail when mapping all the vectors back 15679 * to CPU0. 15680 */ 15681 i40e_clear_interrupt_scheme(pf); 15682 15683 rtnl_unlock(); 15684 15685 return 0; 15686 } 15687 15688 /** 15689 * i40e_resume - PM callback for waking up from D3 15690 * @dev: generic device information structure 15691 **/ 15692 static int __maybe_unused i40e_resume(struct device *dev) 15693 { 15694 struct i40e_pf *pf = dev_get_drvdata(dev); 15695 int err; 15696 15697 /* If we're not suspended, then there is nothing to do */ 15698 if (!test_bit(__I40E_SUSPENDED, pf->state)) 15699 return 0; 15700 15701 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 15702 * since we're going to be restoring queues 15703 */ 15704 rtnl_lock(); 15705 15706 /* We cleared the interrupt scheme when we suspended, so we need to 15707 * restore it now to resume device functionality. 15708 */ 15709 err = i40e_restore_interrupt_scheme(pf); 15710 if (err) { 15711 dev_err(dev, "Cannot restore interrupt scheme: %d\n", 15712 err); 15713 } 15714 15715 clear_bit(__I40E_DOWN, pf->state); 15716 i40e_reset_and_rebuild(pf, false, true); 15717 15718 rtnl_unlock(); 15719 15720 /* Clear suspended state last after everything is recovered */ 15721 clear_bit(__I40E_SUSPENDED, pf->state); 15722 15723 /* Restart the service task */ 15724 mod_timer(&pf->service_timer, 15725 round_jiffies(jiffies + pf->service_timer_period)); 15726 15727 return 0; 15728 } 15729 15730 static const struct pci_error_handlers i40e_err_handler = { 15731 .error_detected = i40e_pci_error_detected, 15732 .slot_reset = i40e_pci_error_slot_reset, 15733 .reset_prepare = i40e_pci_error_reset_prepare, 15734 .reset_done = i40e_pci_error_reset_done, 15735 .resume = i40e_pci_error_resume, 15736 }; 15737 15738 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 15739 15740 static struct pci_driver i40e_driver = { 15741 .name = i40e_driver_name, 15742 .id_table = i40e_pci_tbl, 15743 .probe = i40e_probe, 15744 .remove = i40e_remove, 15745 .driver = { 15746 .pm = &i40e_pm_ops, 15747 }, 15748 .shutdown = i40e_shutdown, 15749 .err_handler = &i40e_err_handler, 15750 .sriov_configure = i40e_pci_sriov_configure, 15751 }; 15752 15753 /** 15754 * i40e_init_module - Driver registration routine 15755 * 15756 * i40e_init_module is the first routine called when the driver is 15757 * loaded. All it does is register with the PCI subsystem. 15758 **/ 15759 static int __init i40e_init_module(void) 15760 { 15761 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string); 15762 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 15763 15764 /* There is no need to throttle the number of active tasks because 15765 * each device limits its own task using a state bit for scheduling 15766 * the service task, and the device tasks do not interfere with each 15767 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 15768 * since we need to be able to guarantee forward progress even under 15769 * memory pressure. 15770 */ 15771 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 15772 if (!i40e_wq) { 15773 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 15774 return -ENOMEM; 15775 } 15776 15777 i40e_dbg_init(); 15778 return pci_register_driver(&i40e_driver); 15779 } 15780 module_init(i40e_init_module); 15781 15782 /** 15783 * i40e_exit_module - Driver exit cleanup routine 15784 * 15785 * i40e_exit_module is called just before the driver is removed 15786 * from memory. 15787 **/ 15788 static void __exit i40e_exit_module(void) 15789 { 15790 pci_unregister_driver(&i40e_driver); 15791 destroy_workqueue(i40e_wq); 15792 i40e_dbg_exit(); 15793 } 15794 module_exit(i40e_exit_module); 15795