1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/of_net.h> 6 #include <linux/pci.h> 7 #include <linux/bpf.h> 8 #include <generated/utsrelease.h> 9 10 /* Local includes */ 11 #include "i40e.h" 12 #include "i40e_diag.h" 13 #include "i40e_xsk.h" 14 #include <net/udp_tunnel.h> 15 #include <net/xdp_sock_drv.h> 16 /* All i40e tracepoints are defined by the include below, which 17 * must be included exactly once across the whole kernel with 18 * CREATE_TRACE_POINTS defined 19 */ 20 #define CREATE_TRACE_POINTS 21 #include "i40e_trace.h" 22 23 const char i40e_driver_name[] = "i40e"; 24 static const char i40e_driver_string[] = 25 "Intel(R) Ethernet Connection XL710 Network Driver"; 26 27 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation."; 28 29 /* a bit of forward declarations */ 30 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 31 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 32 static int i40e_add_vsi(struct i40e_vsi *vsi); 33 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 34 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 35 static int i40e_setup_misc_vector(struct i40e_pf *pf); 36 static void i40e_determine_queue_usage(struct i40e_pf *pf); 37 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 38 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 39 static int i40e_reset(struct i40e_pf *pf); 40 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 41 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf); 42 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf); 43 static bool i40e_check_recovery_mode(struct i40e_pf *pf); 44 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw); 45 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 46 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 47 static int i40e_get_capabilities(struct i40e_pf *pf, 48 enum i40e_admin_queue_opc list_type); 49 50 51 /* i40e_pci_tbl - PCI Device ID Table 52 * 53 * Last entry must be all 0s 54 * 55 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 56 * Class, Class Mask, private data (not used) } 57 */ 58 static const struct pci_device_id i40e_pci_tbl[] = { 59 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 60 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 61 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 62 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 63 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 64 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 83 /* required last entry */ 84 {0, } 85 }; 86 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 87 88 #define I40E_MAX_VF_COUNT 128 89 static int debug = -1; 90 module_param(debug, uint, 0); 91 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 92 93 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 94 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 95 MODULE_LICENSE("GPL v2"); 96 97 static struct workqueue_struct *i40e_wq; 98 99 /** 100 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 101 * @hw: pointer to the HW structure 102 * @mem: ptr to mem struct to fill out 103 * @size: size of memory requested 104 * @alignment: what to align the allocation to 105 **/ 106 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 107 u64 size, u32 alignment) 108 { 109 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 110 111 mem->size = ALIGN(size, alignment); 112 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa, 113 GFP_KERNEL); 114 if (!mem->va) 115 return -ENOMEM; 116 117 return 0; 118 } 119 120 /** 121 * i40e_free_dma_mem_d - OS specific memory free for shared code 122 * @hw: pointer to the HW structure 123 * @mem: ptr to mem struct to free 124 **/ 125 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 126 { 127 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 128 129 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 130 mem->va = NULL; 131 mem->pa = 0; 132 mem->size = 0; 133 134 return 0; 135 } 136 137 /** 138 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 139 * @hw: pointer to the HW structure 140 * @mem: ptr to mem struct to fill out 141 * @size: size of memory requested 142 **/ 143 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 144 u32 size) 145 { 146 mem->size = size; 147 mem->va = kzalloc(size, GFP_KERNEL); 148 149 if (!mem->va) 150 return -ENOMEM; 151 152 return 0; 153 } 154 155 /** 156 * i40e_free_virt_mem_d - OS specific memory free for shared code 157 * @hw: pointer to the HW structure 158 * @mem: ptr to mem struct to free 159 **/ 160 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 161 { 162 /* it's ok to kfree a NULL pointer */ 163 kfree(mem->va); 164 mem->va = NULL; 165 mem->size = 0; 166 167 return 0; 168 } 169 170 /** 171 * i40e_get_lump - find a lump of free generic resource 172 * @pf: board private structure 173 * @pile: the pile of resource to search 174 * @needed: the number of items needed 175 * @id: an owner id to stick on the items assigned 176 * 177 * Returns the base item index of the lump, or negative for error 178 * 179 * The search_hint trick and lack of advanced fit-finding only work 180 * because we're highly likely to have all the same size lump requests. 181 * Linear search time and any fragmentation should be minimal. 182 **/ 183 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 184 u16 needed, u16 id) 185 { 186 int ret = -ENOMEM; 187 int i, j; 188 189 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 190 dev_info(&pf->pdev->dev, 191 "param err: pile=%s needed=%d id=0x%04x\n", 192 pile ? "<valid>" : "<null>", needed, id); 193 return -EINVAL; 194 } 195 196 /* start the linear search with an imperfect hint */ 197 i = pile->search_hint; 198 while (i < pile->num_entries) { 199 /* skip already allocated entries */ 200 if (pile->list[i] & I40E_PILE_VALID_BIT) { 201 i++; 202 continue; 203 } 204 205 /* do we have enough in this lump? */ 206 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 207 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 208 break; 209 } 210 211 if (j == needed) { 212 /* there was enough, so assign it to the requestor */ 213 for (j = 0; j < needed; j++) 214 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 215 ret = i; 216 pile->search_hint = i + j; 217 break; 218 } 219 220 /* not enough, so skip over it and continue looking */ 221 i += j; 222 } 223 224 return ret; 225 } 226 227 /** 228 * i40e_put_lump - return a lump of generic resource 229 * @pile: the pile of resource to search 230 * @index: the base item index 231 * @id: the owner id of the items assigned 232 * 233 * Returns the count of items in the lump 234 **/ 235 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 236 { 237 int valid_id = (id | I40E_PILE_VALID_BIT); 238 int count = 0; 239 int i; 240 241 if (!pile || index >= pile->num_entries) 242 return -EINVAL; 243 244 for (i = index; 245 i < pile->num_entries && pile->list[i] == valid_id; 246 i++) { 247 pile->list[i] = 0; 248 count++; 249 } 250 251 if (count && index < pile->search_hint) 252 pile->search_hint = index; 253 254 return count; 255 } 256 257 /** 258 * i40e_find_vsi_from_id - searches for the vsi with the given id 259 * @pf: the pf structure to search for the vsi 260 * @id: id of the vsi it is searching for 261 **/ 262 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 263 { 264 int i; 265 266 for (i = 0; i < pf->num_alloc_vsi; i++) 267 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 268 return pf->vsi[i]; 269 270 return NULL; 271 } 272 273 /** 274 * i40e_service_event_schedule - Schedule the service task to wake up 275 * @pf: board private structure 276 * 277 * If not already scheduled, this puts the task into the work queue 278 **/ 279 void i40e_service_event_schedule(struct i40e_pf *pf) 280 { 281 if ((!test_bit(__I40E_DOWN, pf->state) && 282 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) || 283 test_bit(__I40E_RECOVERY_MODE, pf->state)) 284 queue_work(i40e_wq, &pf->service_task); 285 } 286 287 /** 288 * i40e_tx_timeout - Respond to a Tx Hang 289 * @netdev: network interface device structure 290 * 291 * If any port has noticed a Tx timeout, it is likely that the whole 292 * device is munged, not just the one netdev port, so go for the full 293 * reset. 294 **/ 295 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue) 296 { 297 struct i40e_netdev_priv *np = netdev_priv(netdev); 298 struct i40e_vsi *vsi = np->vsi; 299 struct i40e_pf *pf = vsi->back; 300 struct i40e_ring *tx_ring = NULL; 301 unsigned int i; 302 u32 head, val; 303 304 pf->tx_timeout_count++; 305 306 /* with txqueue index, find the tx_ring struct */ 307 for (i = 0; i < vsi->num_queue_pairs; i++) { 308 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 309 if (txqueue == 310 vsi->tx_rings[i]->queue_index) { 311 tx_ring = vsi->tx_rings[i]; 312 break; 313 } 314 } 315 } 316 317 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 318 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 319 else if (time_before(jiffies, 320 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 321 return; /* don't do any new action before the next timeout */ 322 323 /* don't kick off another recovery if one is already pending */ 324 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state)) 325 return; 326 327 if (tx_ring) { 328 head = i40e_get_head(tx_ring); 329 /* Read interrupt register */ 330 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 331 val = rd32(&pf->hw, 332 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 333 tx_ring->vsi->base_vector - 1)); 334 else 335 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 336 337 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 338 vsi->seid, txqueue, tx_ring->next_to_clean, 339 head, tx_ring->next_to_use, 340 readl(tx_ring->tail), val); 341 } 342 343 pf->tx_timeout_last_recovery = jiffies; 344 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n", 345 pf->tx_timeout_recovery_level, txqueue); 346 347 switch (pf->tx_timeout_recovery_level) { 348 case 1: 349 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 350 break; 351 case 2: 352 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 353 break; 354 case 3: 355 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 356 break; 357 default: 358 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 359 break; 360 } 361 362 i40e_service_event_schedule(pf); 363 pf->tx_timeout_recovery_level++; 364 } 365 366 /** 367 * i40e_get_vsi_stats_struct - Get System Network Statistics 368 * @vsi: the VSI we care about 369 * 370 * Returns the address of the device statistics structure. 371 * The statistics are actually updated from the service task. 372 **/ 373 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 374 { 375 return &vsi->net_stats; 376 } 377 378 /** 379 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 380 * @ring: Tx ring to get statistics from 381 * @stats: statistics entry to be updated 382 **/ 383 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 384 struct rtnl_link_stats64 *stats) 385 { 386 u64 bytes, packets; 387 unsigned int start; 388 389 do { 390 start = u64_stats_fetch_begin_irq(&ring->syncp); 391 packets = ring->stats.packets; 392 bytes = ring->stats.bytes; 393 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 394 395 stats->tx_packets += packets; 396 stats->tx_bytes += bytes; 397 } 398 399 /** 400 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 401 * @netdev: network interface device structure 402 * @stats: data structure to store statistics 403 * 404 * Returns the address of the device statistics structure. 405 * The statistics are actually updated from the service task. 406 **/ 407 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 408 struct rtnl_link_stats64 *stats) 409 { 410 struct i40e_netdev_priv *np = netdev_priv(netdev); 411 struct i40e_vsi *vsi = np->vsi; 412 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 413 struct i40e_ring *ring; 414 int i; 415 416 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 417 return; 418 419 if (!vsi->tx_rings) 420 return; 421 422 rcu_read_lock(); 423 for (i = 0; i < vsi->num_queue_pairs; i++) { 424 u64 bytes, packets; 425 unsigned int start; 426 427 ring = READ_ONCE(vsi->tx_rings[i]); 428 if (!ring) 429 continue; 430 i40e_get_netdev_stats_struct_tx(ring, stats); 431 432 if (i40e_enabled_xdp_vsi(vsi)) { 433 ring = READ_ONCE(vsi->xdp_rings[i]); 434 if (!ring) 435 continue; 436 i40e_get_netdev_stats_struct_tx(ring, stats); 437 } 438 439 ring = READ_ONCE(vsi->rx_rings[i]); 440 if (!ring) 441 continue; 442 do { 443 start = u64_stats_fetch_begin_irq(&ring->syncp); 444 packets = ring->stats.packets; 445 bytes = ring->stats.bytes; 446 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 447 448 stats->rx_packets += packets; 449 stats->rx_bytes += bytes; 450 451 } 452 rcu_read_unlock(); 453 454 /* following stats updated by i40e_watchdog_subtask() */ 455 stats->multicast = vsi_stats->multicast; 456 stats->tx_errors = vsi_stats->tx_errors; 457 stats->tx_dropped = vsi_stats->tx_dropped; 458 stats->rx_errors = vsi_stats->rx_errors; 459 stats->rx_dropped = vsi_stats->rx_dropped; 460 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 461 stats->rx_length_errors = vsi_stats->rx_length_errors; 462 } 463 464 /** 465 * i40e_vsi_reset_stats - Resets all stats of the given vsi 466 * @vsi: the VSI to have its stats reset 467 **/ 468 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 469 { 470 struct rtnl_link_stats64 *ns; 471 int i; 472 473 if (!vsi) 474 return; 475 476 ns = i40e_get_vsi_stats_struct(vsi); 477 memset(ns, 0, sizeof(*ns)); 478 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 479 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 480 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 481 if (vsi->rx_rings && vsi->rx_rings[0]) { 482 for (i = 0; i < vsi->num_queue_pairs; i++) { 483 memset(&vsi->rx_rings[i]->stats, 0, 484 sizeof(vsi->rx_rings[i]->stats)); 485 memset(&vsi->rx_rings[i]->rx_stats, 0, 486 sizeof(vsi->rx_rings[i]->rx_stats)); 487 memset(&vsi->tx_rings[i]->stats, 0, 488 sizeof(vsi->tx_rings[i]->stats)); 489 memset(&vsi->tx_rings[i]->tx_stats, 0, 490 sizeof(vsi->tx_rings[i]->tx_stats)); 491 } 492 } 493 vsi->stat_offsets_loaded = false; 494 } 495 496 /** 497 * i40e_pf_reset_stats - Reset all of the stats for the given PF 498 * @pf: the PF to be reset 499 **/ 500 void i40e_pf_reset_stats(struct i40e_pf *pf) 501 { 502 int i; 503 504 memset(&pf->stats, 0, sizeof(pf->stats)); 505 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 506 pf->stat_offsets_loaded = false; 507 508 for (i = 0; i < I40E_MAX_VEB; i++) { 509 if (pf->veb[i]) { 510 memset(&pf->veb[i]->stats, 0, 511 sizeof(pf->veb[i]->stats)); 512 memset(&pf->veb[i]->stats_offsets, 0, 513 sizeof(pf->veb[i]->stats_offsets)); 514 memset(&pf->veb[i]->tc_stats, 0, 515 sizeof(pf->veb[i]->tc_stats)); 516 memset(&pf->veb[i]->tc_stats_offsets, 0, 517 sizeof(pf->veb[i]->tc_stats_offsets)); 518 pf->veb[i]->stat_offsets_loaded = false; 519 } 520 } 521 pf->hw_csum_rx_error = 0; 522 } 523 524 /** 525 * i40e_stat_update48 - read and update a 48 bit stat from the chip 526 * @hw: ptr to the hardware info 527 * @hireg: the high 32 bit reg to read 528 * @loreg: the low 32 bit reg to read 529 * @offset_loaded: has the initial offset been loaded yet 530 * @offset: ptr to current offset value 531 * @stat: ptr to the stat 532 * 533 * Since the device stats are not reset at PFReset, they likely will not 534 * be zeroed when the driver starts. We'll save the first values read 535 * and use them as offsets to be subtracted from the raw values in order 536 * to report stats that count from zero. In the process, we also manage 537 * the potential roll-over. 538 **/ 539 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 540 bool offset_loaded, u64 *offset, u64 *stat) 541 { 542 u64 new_data; 543 544 if (hw->device_id == I40E_DEV_ID_QEMU) { 545 new_data = rd32(hw, loreg); 546 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 547 } else { 548 new_data = rd64(hw, loreg); 549 } 550 if (!offset_loaded) 551 *offset = new_data; 552 if (likely(new_data >= *offset)) 553 *stat = new_data - *offset; 554 else 555 *stat = (new_data + BIT_ULL(48)) - *offset; 556 *stat &= 0xFFFFFFFFFFFFULL; 557 } 558 559 /** 560 * i40e_stat_update32 - read and update a 32 bit stat from the chip 561 * @hw: ptr to the hardware info 562 * @reg: the hw reg to read 563 * @offset_loaded: has the initial offset been loaded yet 564 * @offset: ptr to current offset value 565 * @stat: ptr to the stat 566 **/ 567 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 568 bool offset_loaded, u64 *offset, u64 *stat) 569 { 570 u32 new_data; 571 572 new_data = rd32(hw, reg); 573 if (!offset_loaded) 574 *offset = new_data; 575 if (likely(new_data >= *offset)) 576 *stat = (u32)(new_data - *offset); 577 else 578 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 579 } 580 581 /** 582 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 583 * @hw: ptr to the hardware info 584 * @reg: the hw reg to read and clear 585 * @stat: ptr to the stat 586 **/ 587 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 588 { 589 u32 new_data = rd32(hw, reg); 590 591 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 592 *stat += new_data; 593 } 594 595 /** 596 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 597 * @vsi: the VSI to be updated 598 **/ 599 void i40e_update_eth_stats(struct i40e_vsi *vsi) 600 { 601 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 602 struct i40e_pf *pf = vsi->back; 603 struct i40e_hw *hw = &pf->hw; 604 struct i40e_eth_stats *oes; 605 struct i40e_eth_stats *es; /* device's eth stats */ 606 607 es = &vsi->eth_stats; 608 oes = &vsi->eth_stats_offsets; 609 610 /* Gather up the stats that the hw collects */ 611 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 612 vsi->stat_offsets_loaded, 613 &oes->tx_errors, &es->tx_errors); 614 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 615 vsi->stat_offsets_loaded, 616 &oes->rx_discards, &es->rx_discards); 617 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 618 vsi->stat_offsets_loaded, 619 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 620 621 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 622 I40E_GLV_GORCL(stat_idx), 623 vsi->stat_offsets_loaded, 624 &oes->rx_bytes, &es->rx_bytes); 625 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 626 I40E_GLV_UPRCL(stat_idx), 627 vsi->stat_offsets_loaded, 628 &oes->rx_unicast, &es->rx_unicast); 629 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 630 I40E_GLV_MPRCL(stat_idx), 631 vsi->stat_offsets_loaded, 632 &oes->rx_multicast, &es->rx_multicast); 633 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 634 I40E_GLV_BPRCL(stat_idx), 635 vsi->stat_offsets_loaded, 636 &oes->rx_broadcast, &es->rx_broadcast); 637 638 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 639 I40E_GLV_GOTCL(stat_idx), 640 vsi->stat_offsets_loaded, 641 &oes->tx_bytes, &es->tx_bytes); 642 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 643 I40E_GLV_UPTCL(stat_idx), 644 vsi->stat_offsets_loaded, 645 &oes->tx_unicast, &es->tx_unicast); 646 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 647 I40E_GLV_MPTCL(stat_idx), 648 vsi->stat_offsets_loaded, 649 &oes->tx_multicast, &es->tx_multicast); 650 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 651 I40E_GLV_BPTCL(stat_idx), 652 vsi->stat_offsets_loaded, 653 &oes->tx_broadcast, &es->tx_broadcast); 654 vsi->stat_offsets_loaded = true; 655 } 656 657 /** 658 * i40e_update_veb_stats - Update Switch component statistics 659 * @veb: the VEB being updated 660 **/ 661 void i40e_update_veb_stats(struct i40e_veb *veb) 662 { 663 struct i40e_pf *pf = veb->pf; 664 struct i40e_hw *hw = &pf->hw; 665 struct i40e_eth_stats *oes; 666 struct i40e_eth_stats *es; /* device's eth stats */ 667 struct i40e_veb_tc_stats *veb_oes; 668 struct i40e_veb_tc_stats *veb_es; 669 int i, idx = 0; 670 671 idx = veb->stats_idx; 672 es = &veb->stats; 673 oes = &veb->stats_offsets; 674 veb_es = &veb->tc_stats; 675 veb_oes = &veb->tc_stats_offsets; 676 677 /* Gather up the stats that the hw collects */ 678 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 679 veb->stat_offsets_loaded, 680 &oes->tx_discards, &es->tx_discards); 681 if (hw->revision_id > 0) 682 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 683 veb->stat_offsets_loaded, 684 &oes->rx_unknown_protocol, 685 &es->rx_unknown_protocol); 686 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 687 veb->stat_offsets_loaded, 688 &oes->rx_bytes, &es->rx_bytes); 689 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 690 veb->stat_offsets_loaded, 691 &oes->rx_unicast, &es->rx_unicast); 692 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 693 veb->stat_offsets_loaded, 694 &oes->rx_multicast, &es->rx_multicast); 695 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 696 veb->stat_offsets_loaded, 697 &oes->rx_broadcast, &es->rx_broadcast); 698 699 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 700 veb->stat_offsets_loaded, 701 &oes->tx_bytes, &es->tx_bytes); 702 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 703 veb->stat_offsets_loaded, 704 &oes->tx_unicast, &es->tx_unicast); 705 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 706 veb->stat_offsets_loaded, 707 &oes->tx_multicast, &es->tx_multicast); 708 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 709 veb->stat_offsets_loaded, 710 &oes->tx_broadcast, &es->tx_broadcast); 711 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 712 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 713 I40E_GLVEBTC_RPCL(i, idx), 714 veb->stat_offsets_loaded, 715 &veb_oes->tc_rx_packets[i], 716 &veb_es->tc_rx_packets[i]); 717 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 718 I40E_GLVEBTC_RBCL(i, idx), 719 veb->stat_offsets_loaded, 720 &veb_oes->tc_rx_bytes[i], 721 &veb_es->tc_rx_bytes[i]); 722 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 723 I40E_GLVEBTC_TPCL(i, idx), 724 veb->stat_offsets_loaded, 725 &veb_oes->tc_tx_packets[i], 726 &veb_es->tc_tx_packets[i]); 727 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 728 I40E_GLVEBTC_TBCL(i, idx), 729 veb->stat_offsets_loaded, 730 &veb_oes->tc_tx_bytes[i], 731 &veb_es->tc_tx_bytes[i]); 732 } 733 veb->stat_offsets_loaded = true; 734 } 735 736 /** 737 * i40e_update_vsi_stats - Update the vsi statistics counters. 738 * @vsi: the VSI to be updated 739 * 740 * There are a few instances where we store the same stat in a 741 * couple of different structs. This is partly because we have 742 * the netdev stats that need to be filled out, which is slightly 743 * different from the "eth_stats" defined by the chip and used in 744 * VF communications. We sort it out here. 745 **/ 746 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 747 { 748 struct i40e_pf *pf = vsi->back; 749 struct rtnl_link_stats64 *ons; 750 struct rtnl_link_stats64 *ns; /* netdev stats */ 751 struct i40e_eth_stats *oes; 752 struct i40e_eth_stats *es; /* device's eth stats */ 753 u32 tx_restart, tx_busy; 754 struct i40e_ring *p; 755 u32 rx_page, rx_buf; 756 u64 bytes, packets; 757 unsigned int start; 758 u64 tx_linearize; 759 u64 tx_force_wb; 760 u64 rx_p, rx_b; 761 u64 tx_p, tx_b; 762 u16 q; 763 764 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 765 test_bit(__I40E_CONFIG_BUSY, pf->state)) 766 return; 767 768 ns = i40e_get_vsi_stats_struct(vsi); 769 ons = &vsi->net_stats_offsets; 770 es = &vsi->eth_stats; 771 oes = &vsi->eth_stats_offsets; 772 773 /* Gather up the netdev and vsi stats that the driver collects 774 * on the fly during packet processing 775 */ 776 rx_b = rx_p = 0; 777 tx_b = tx_p = 0; 778 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 779 rx_page = 0; 780 rx_buf = 0; 781 rcu_read_lock(); 782 for (q = 0; q < vsi->num_queue_pairs; q++) { 783 /* locate Tx ring */ 784 p = READ_ONCE(vsi->tx_rings[q]); 785 if (!p) 786 continue; 787 788 do { 789 start = u64_stats_fetch_begin_irq(&p->syncp); 790 packets = p->stats.packets; 791 bytes = p->stats.bytes; 792 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 793 tx_b += bytes; 794 tx_p += packets; 795 tx_restart += p->tx_stats.restart_queue; 796 tx_busy += p->tx_stats.tx_busy; 797 tx_linearize += p->tx_stats.tx_linearize; 798 tx_force_wb += p->tx_stats.tx_force_wb; 799 800 /* locate Rx ring */ 801 p = READ_ONCE(vsi->rx_rings[q]); 802 if (!p) 803 continue; 804 805 do { 806 start = u64_stats_fetch_begin_irq(&p->syncp); 807 packets = p->stats.packets; 808 bytes = p->stats.bytes; 809 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 810 rx_b += bytes; 811 rx_p += packets; 812 rx_buf += p->rx_stats.alloc_buff_failed; 813 rx_page += p->rx_stats.alloc_page_failed; 814 } 815 rcu_read_unlock(); 816 vsi->tx_restart = tx_restart; 817 vsi->tx_busy = tx_busy; 818 vsi->tx_linearize = tx_linearize; 819 vsi->tx_force_wb = tx_force_wb; 820 vsi->rx_page_failed = rx_page; 821 vsi->rx_buf_failed = rx_buf; 822 823 ns->rx_packets = rx_p; 824 ns->rx_bytes = rx_b; 825 ns->tx_packets = tx_p; 826 ns->tx_bytes = tx_b; 827 828 /* update netdev stats from eth stats */ 829 i40e_update_eth_stats(vsi); 830 ons->tx_errors = oes->tx_errors; 831 ns->tx_errors = es->tx_errors; 832 ons->multicast = oes->rx_multicast; 833 ns->multicast = es->rx_multicast; 834 ons->rx_dropped = oes->rx_discards; 835 ns->rx_dropped = es->rx_discards; 836 ons->tx_dropped = oes->tx_discards; 837 ns->tx_dropped = es->tx_discards; 838 839 /* pull in a couple PF stats if this is the main vsi */ 840 if (vsi == pf->vsi[pf->lan_vsi]) { 841 ns->rx_crc_errors = pf->stats.crc_errors; 842 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 843 ns->rx_length_errors = pf->stats.rx_length_errors; 844 } 845 } 846 847 /** 848 * i40e_update_pf_stats - Update the PF statistics counters. 849 * @pf: the PF to be updated 850 **/ 851 static void i40e_update_pf_stats(struct i40e_pf *pf) 852 { 853 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 854 struct i40e_hw_port_stats *nsd = &pf->stats; 855 struct i40e_hw *hw = &pf->hw; 856 u32 val; 857 int i; 858 859 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 860 I40E_GLPRT_GORCL(hw->port), 861 pf->stat_offsets_loaded, 862 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 863 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 864 I40E_GLPRT_GOTCL(hw->port), 865 pf->stat_offsets_loaded, 866 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 867 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 868 pf->stat_offsets_loaded, 869 &osd->eth.rx_discards, 870 &nsd->eth.rx_discards); 871 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 872 I40E_GLPRT_UPRCL(hw->port), 873 pf->stat_offsets_loaded, 874 &osd->eth.rx_unicast, 875 &nsd->eth.rx_unicast); 876 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 877 I40E_GLPRT_MPRCL(hw->port), 878 pf->stat_offsets_loaded, 879 &osd->eth.rx_multicast, 880 &nsd->eth.rx_multicast); 881 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 882 I40E_GLPRT_BPRCL(hw->port), 883 pf->stat_offsets_loaded, 884 &osd->eth.rx_broadcast, 885 &nsd->eth.rx_broadcast); 886 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 887 I40E_GLPRT_UPTCL(hw->port), 888 pf->stat_offsets_loaded, 889 &osd->eth.tx_unicast, 890 &nsd->eth.tx_unicast); 891 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 892 I40E_GLPRT_MPTCL(hw->port), 893 pf->stat_offsets_loaded, 894 &osd->eth.tx_multicast, 895 &nsd->eth.tx_multicast); 896 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 897 I40E_GLPRT_BPTCL(hw->port), 898 pf->stat_offsets_loaded, 899 &osd->eth.tx_broadcast, 900 &nsd->eth.tx_broadcast); 901 902 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 903 pf->stat_offsets_loaded, 904 &osd->tx_dropped_link_down, 905 &nsd->tx_dropped_link_down); 906 907 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 908 pf->stat_offsets_loaded, 909 &osd->crc_errors, &nsd->crc_errors); 910 911 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->illegal_bytes, &nsd->illegal_bytes); 914 915 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 916 pf->stat_offsets_loaded, 917 &osd->mac_local_faults, 918 &nsd->mac_local_faults); 919 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 920 pf->stat_offsets_loaded, 921 &osd->mac_remote_faults, 922 &nsd->mac_remote_faults); 923 924 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 925 pf->stat_offsets_loaded, 926 &osd->rx_length_errors, 927 &nsd->rx_length_errors); 928 929 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 930 pf->stat_offsets_loaded, 931 &osd->link_xon_rx, &nsd->link_xon_rx); 932 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 933 pf->stat_offsets_loaded, 934 &osd->link_xon_tx, &nsd->link_xon_tx); 935 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 936 pf->stat_offsets_loaded, 937 &osd->link_xoff_rx, &nsd->link_xoff_rx); 938 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 939 pf->stat_offsets_loaded, 940 &osd->link_xoff_tx, &nsd->link_xoff_tx); 941 942 for (i = 0; i < 8; i++) { 943 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 944 pf->stat_offsets_loaded, 945 &osd->priority_xoff_rx[i], 946 &nsd->priority_xoff_rx[i]); 947 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 948 pf->stat_offsets_loaded, 949 &osd->priority_xon_rx[i], 950 &nsd->priority_xon_rx[i]); 951 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 952 pf->stat_offsets_loaded, 953 &osd->priority_xon_tx[i], 954 &nsd->priority_xon_tx[i]); 955 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 956 pf->stat_offsets_loaded, 957 &osd->priority_xoff_tx[i], 958 &nsd->priority_xoff_tx[i]); 959 i40e_stat_update32(hw, 960 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 961 pf->stat_offsets_loaded, 962 &osd->priority_xon_2_xoff[i], 963 &nsd->priority_xon_2_xoff[i]); 964 } 965 966 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 967 I40E_GLPRT_PRC64L(hw->port), 968 pf->stat_offsets_loaded, 969 &osd->rx_size_64, &nsd->rx_size_64); 970 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 971 I40E_GLPRT_PRC127L(hw->port), 972 pf->stat_offsets_loaded, 973 &osd->rx_size_127, &nsd->rx_size_127); 974 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 975 I40E_GLPRT_PRC255L(hw->port), 976 pf->stat_offsets_loaded, 977 &osd->rx_size_255, &nsd->rx_size_255); 978 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 979 I40E_GLPRT_PRC511L(hw->port), 980 pf->stat_offsets_loaded, 981 &osd->rx_size_511, &nsd->rx_size_511); 982 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 983 I40E_GLPRT_PRC1023L(hw->port), 984 pf->stat_offsets_loaded, 985 &osd->rx_size_1023, &nsd->rx_size_1023); 986 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 987 I40E_GLPRT_PRC1522L(hw->port), 988 pf->stat_offsets_loaded, 989 &osd->rx_size_1522, &nsd->rx_size_1522); 990 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 991 I40E_GLPRT_PRC9522L(hw->port), 992 pf->stat_offsets_loaded, 993 &osd->rx_size_big, &nsd->rx_size_big); 994 995 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 996 I40E_GLPRT_PTC64L(hw->port), 997 pf->stat_offsets_loaded, 998 &osd->tx_size_64, &nsd->tx_size_64); 999 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1000 I40E_GLPRT_PTC127L(hw->port), 1001 pf->stat_offsets_loaded, 1002 &osd->tx_size_127, &nsd->tx_size_127); 1003 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1004 I40E_GLPRT_PTC255L(hw->port), 1005 pf->stat_offsets_loaded, 1006 &osd->tx_size_255, &nsd->tx_size_255); 1007 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1008 I40E_GLPRT_PTC511L(hw->port), 1009 pf->stat_offsets_loaded, 1010 &osd->tx_size_511, &nsd->tx_size_511); 1011 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1012 I40E_GLPRT_PTC1023L(hw->port), 1013 pf->stat_offsets_loaded, 1014 &osd->tx_size_1023, &nsd->tx_size_1023); 1015 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1016 I40E_GLPRT_PTC1522L(hw->port), 1017 pf->stat_offsets_loaded, 1018 &osd->tx_size_1522, &nsd->tx_size_1522); 1019 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1020 I40E_GLPRT_PTC9522L(hw->port), 1021 pf->stat_offsets_loaded, 1022 &osd->tx_size_big, &nsd->tx_size_big); 1023 1024 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1025 pf->stat_offsets_loaded, 1026 &osd->rx_undersize, &nsd->rx_undersize); 1027 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->rx_fragments, &nsd->rx_fragments); 1030 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1031 pf->stat_offsets_loaded, 1032 &osd->rx_oversize, &nsd->rx_oversize); 1033 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1034 pf->stat_offsets_loaded, 1035 &osd->rx_jabber, &nsd->rx_jabber); 1036 1037 /* FDIR stats */ 1038 i40e_stat_update_and_clear32(hw, 1039 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1040 &nsd->fd_atr_match); 1041 i40e_stat_update_and_clear32(hw, 1042 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1043 &nsd->fd_sb_match); 1044 i40e_stat_update_and_clear32(hw, 1045 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1046 &nsd->fd_atr_tunnel_match); 1047 1048 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1049 nsd->tx_lpi_status = 1050 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1051 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1052 nsd->rx_lpi_status = 1053 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1054 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1055 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1056 pf->stat_offsets_loaded, 1057 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1058 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1059 pf->stat_offsets_loaded, 1060 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1061 1062 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1063 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1064 nsd->fd_sb_status = true; 1065 else 1066 nsd->fd_sb_status = false; 1067 1068 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1069 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1070 nsd->fd_atr_status = true; 1071 else 1072 nsd->fd_atr_status = false; 1073 1074 pf->stat_offsets_loaded = true; 1075 } 1076 1077 /** 1078 * i40e_update_stats - Update the various statistics counters. 1079 * @vsi: the VSI to be updated 1080 * 1081 * Update the various stats for this VSI and its related entities. 1082 **/ 1083 void i40e_update_stats(struct i40e_vsi *vsi) 1084 { 1085 struct i40e_pf *pf = vsi->back; 1086 1087 if (vsi == pf->vsi[pf->lan_vsi]) 1088 i40e_update_pf_stats(pf); 1089 1090 i40e_update_vsi_stats(vsi); 1091 } 1092 1093 /** 1094 * i40e_count_filters - counts VSI mac filters 1095 * @vsi: the VSI to be searched 1096 * 1097 * Returns count of mac filters 1098 **/ 1099 int i40e_count_filters(struct i40e_vsi *vsi) 1100 { 1101 struct i40e_mac_filter *f; 1102 struct hlist_node *h; 1103 int bkt; 1104 int cnt = 0; 1105 1106 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 1107 ++cnt; 1108 1109 return cnt; 1110 } 1111 1112 /** 1113 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1114 * @vsi: the VSI to be searched 1115 * @macaddr: the MAC address 1116 * @vlan: the vlan 1117 * 1118 * Returns ptr to the filter object or NULL 1119 **/ 1120 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1121 const u8 *macaddr, s16 vlan) 1122 { 1123 struct i40e_mac_filter *f; 1124 u64 key; 1125 1126 if (!vsi || !macaddr) 1127 return NULL; 1128 1129 key = i40e_addr_to_hkey(macaddr); 1130 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1131 if ((ether_addr_equal(macaddr, f->macaddr)) && 1132 (vlan == f->vlan)) 1133 return f; 1134 } 1135 return NULL; 1136 } 1137 1138 /** 1139 * i40e_find_mac - Find a mac addr in the macvlan filters list 1140 * @vsi: the VSI to be searched 1141 * @macaddr: the MAC address we are searching for 1142 * 1143 * Returns the first filter with the provided MAC address or NULL if 1144 * MAC address was not found 1145 **/ 1146 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1147 { 1148 struct i40e_mac_filter *f; 1149 u64 key; 1150 1151 if (!vsi || !macaddr) 1152 return NULL; 1153 1154 key = i40e_addr_to_hkey(macaddr); 1155 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1156 if ((ether_addr_equal(macaddr, f->macaddr))) 1157 return f; 1158 } 1159 return NULL; 1160 } 1161 1162 /** 1163 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1164 * @vsi: the VSI to be searched 1165 * 1166 * Returns true if VSI is in vlan mode or false otherwise 1167 **/ 1168 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1169 { 1170 /* If we have a PVID, always operate in VLAN mode */ 1171 if (vsi->info.pvid) 1172 return true; 1173 1174 /* We need to operate in VLAN mode whenever we have any filters with 1175 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1176 * time, incurring search cost repeatedly. However, we can notice two 1177 * things: 1178 * 1179 * 1) the only place where we can gain a VLAN filter is in 1180 * i40e_add_filter. 1181 * 1182 * 2) the only place where filters are actually removed is in 1183 * i40e_sync_filters_subtask. 1184 * 1185 * Thus, we can simply use a boolean value, has_vlan_filters which we 1186 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1187 * we have to perform the full search after deleting filters in 1188 * i40e_sync_filters_subtask, but we already have to search 1189 * filters here and can perform the check at the same time. This 1190 * results in avoiding embedding a loop for VLAN mode inside another 1191 * loop over all the filters, and should maintain correctness as noted 1192 * above. 1193 */ 1194 return vsi->has_vlan_filter; 1195 } 1196 1197 /** 1198 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1199 * @vsi: the VSI to configure 1200 * @tmp_add_list: list of filters ready to be added 1201 * @tmp_del_list: list of filters ready to be deleted 1202 * @vlan_filters: the number of active VLAN filters 1203 * 1204 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1205 * behave as expected. If we have any active VLAN filters remaining or about 1206 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1207 * so that they only match against untagged traffic. If we no longer have any 1208 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1209 * so that they match against both tagged and untagged traffic. In this way, 1210 * we ensure that we correctly receive the desired traffic. This ensures that 1211 * when we have an active VLAN we will receive only untagged traffic and 1212 * traffic matching active VLANs. If we have no active VLANs then we will 1213 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1214 * 1215 * Finally, in a similar fashion, this function also corrects filters when 1216 * there is an active PVID assigned to this VSI. 1217 * 1218 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1219 * 1220 * This function is only expected to be called from within 1221 * i40e_sync_vsi_filters. 1222 * 1223 * NOTE: This function expects to be called while under the 1224 * mac_filter_hash_lock 1225 */ 1226 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1227 struct hlist_head *tmp_add_list, 1228 struct hlist_head *tmp_del_list, 1229 int vlan_filters) 1230 { 1231 s16 pvid = le16_to_cpu(vsi->info.pvid); 1232 struct i40e_mac_filter *f, *add_head; 1233 struct i40e_new_mac_filter *new; 1234 struct hlist_node *h; 1235 int bkt, new_vlan; 1236 1237 /* To determine if a particular filter needs to be replaced we 1238 * have the three following conditions: 1239 * 1240 * a) if we have a PVID assigned, then all filters which are 1241 * not marked as VLAN=PVID must be replaced with filters that 1242 * are. 1243 * b) otherwise, if we have any active VLANS, all filters 1244 * which are marked as VLAN=-1 must be replaced with 1245 * filters marked as VLAN=0 1246 * c) finally, if we do not have any active VLANS, all filters 1247 * which are marked as VLAN=0 must be replaced with filters 1248 * marked as VLAN=-1 1249 */ 1250 1251 /* Update the filters about to be added in place */ 1252 hlist_for_each_entry(new, tmp_add_list, hlist) { 1253 if (pvid && new->f->vlan != pvid) 1254 new->f->vlan = pvid; 1255 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1256 new->f->vlan = 0; 1257 else if (!vlan_filters && new->f->vlan == 0) 1258 new->f->vlan = I40E_VLAN_ANY; 1259 } 1260 1261 /* Update the remaining active filters */ 1262 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1263 /* Combine the checks for whether a filter needs to be changed 1264 * and then determine the new VLAN inside the if block, in 1265 * order to avoid duplicating code for adding the new filter 1266 * then deleting the old filter. 1267 */ 1268 if ((pvid && f->vlan != pvid) || 1269 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1270 (!vlan_filters && f->vlan == 0)) { 1271 /* Determine the new vlan we will be adding */ 1272 if (pvid) 1273 new_vlan = pvid; 1274 else if (vlan_filters) 1275 new_vlan = 0; 1276 else 1277 new_vlan = I40E_VLAN_ANY; 1278 1279 /* Create the new filter */ 1280 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1281 if (!add_head) 1282 return -ENOMEM; 1283 1284 /* Create a temporary i40e_new_mac_filter */ 1285 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1286 if (!new) 1287 return -ENOMEM; 1288 1289 new->f = add_head; 1290 new->state = add_head->state; 1291 1292 /* Add the new filter to the tmp list */ 1293 hlist_add_head(&new->hlist, tmp_add_list); 1294 1295 /* Put the original filter into the delete list */ 1296 f->state = I40E_FILTER_REMOVE; 1297 hash_del(&f->hlist); 1298 hlist_add_head(&f->hlist, tmp_del_list); 1299 } 1300 } 1301 1302 vsi->has_vlan_filter = !!vlan_filters; 1303 1304 return 0; 1305 } 1306 1307 /** 1308 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1309 * @vsi: the PF Main VSI - inappropriate for any other VSI 1310 * @macaddr: the MAC address 1311 * 1312 * Remove whatever filter the firmware set up so the driver can manage 1313 * its own filtering intelligently. 1314 **/ 1315 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1316 { 1317 struct i40e_aqc_remove_macvlan_element_data element; 1318 struct i40e_pf *pf = vsi->back; 1319 1320 /* Only appropriate for the PF main VSI */ 1321 if (vsi->type != I40E_VSI_MAIN) 1322 return; 1323 1324 memset(&element, 0, sizeof(element)); 1325 ether_addr_copy(element.mac_addr, macaddr); 1326 element.vlan_tag = 0; 1327 /* Ignore error returns, some firmware does it this way... */ 1328 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1329 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1330 1331 memset(&element, 0, sizeof(element)); 1332 ether_addr_copy(element.mac_addr, macaddr); 1333 element.vlan_tag = 0; 1334 /* ...and some firmware does it this way. */ 1335 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1336 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1337 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1338 } 1339 1340 /** 1341 * i40e_add_filter - Add a mac/vlan filter to the VSI 1342 * @vsi: the VSI to be searched 1343 * @macaddr: the MAC address 1344 * @vlan: the vlan 1345 * 1346 * Returns ptr to the filter object or NULL when no memory available. 1347 * 1348 * NOTE: This function is expected to be called with mac_filter_hash_lock 1349 * being held. 1350 **/ 1351 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1352 const u8 *macaddr, s16 vlan) 1353 { 1354 struct i40e_mac_filter *f; 1355 u64 key; 1356 1357 if (!vsi || !macaddr) 1358 return NULL; 1359 1360 f = i40e_find_filter(vsi, macaddr, vlan); 1361 if (!f) { 1362 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1363 if (!f) 1364 return NULL; 1365 1366 /* Update the boolean indicating if we need to function in 1367 * VLAN mode. 1368 */ 1369 if (vlan >= 0) 1370 vsi->has_vlan_filter = true; 1371 1372 ether_addr_copy(f->macaddr, macaddr); 1373 f->vlan = vlan; 1374 f->state = I40E_FILTER_NEW; 1375 INIT_HLIST_NODE(&f->hlist); 1376 1377 key = i40e_addr_to_hkey(macaddr); 1378 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1379 1380 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1381 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1382 } 1383 1384 /* If we're asked to add a filter that has been marked for removal, it 1385 * is safe to simply restore it to active state. __i40e_del_filter 1386 * will have simply deleted any filters which were previously marked 1387 * NEW or FAILED, so if it is currently marked REMOVE it must have 1388 * previously been ACTIVE. Since we haven't yet run the sync filters 1389 * task, just restore this filter to the ACTIVE state so that the 1390 * sync task leaves it in place 1391 */ 1392 if (f->state == I40E_FILTER_REMOVE) 1393 f->state = I40E_FILTER_ACTIVE; 1394 1395 return f; 1396 } 1397 1398 /** 1399 * __i40e_del_filter - Remove a specific filter from the VSI 1400 * @vsi: VSI to remove from 1401 * @f: the filter to remove from the list 1402 * 1403 * This function should be called instead of i40e_del_filter only if you know 1404 * the exact filter you will remove already, such as via i40e_find_filter or 1405 * i40e_find_mac. 1406 * 1407 * NOTE: This function is expected to be called with mac_filter_hash_lock 1408 * being held. 1409 * ANOTHER NOTE: This function MUST be called from within the context of 1410 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1411 * instead of list_for_each_entry(). 1412 **/ 1413 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1414 { 1415 if (!f) 1416 return; 1417 1418 /* If the filter was never added to firmware then we can just delete it 1419 * directly and we don't want to set the status to remove or else an 1420 * admin queue command will unnecessarily fire. 1421 */ 1422 if ((f->state == I40E_FILTER_FAILED) || 1423 (f->state == I40E_FILTER_NEW)) { 1424 hash_del(&f->hlist); 1425 kfree(f); 1426 } else { 1427 f->state = I40E_FILTER_REMOVE; 1428 } 1429 1430 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1431 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1432 } 1433 1434 /** 1435 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1436 * @vsi: the VSI to be searched 1437 * @macaddr: the MAC address 1438 * @vlan: the VLAN 1439 * 1440 * NOTE: This function is expected to be called with mac_filter_hash_lock 1441 * being held. 1442 * ANOTHER NOTE: This function MUST be called from within the context of 1443 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1444 * instead of list_for_each_entry(). 1445 **/ 1446 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1447 { 1448 struct i40e_mac_filter *f; 1449 1450 if (!vsi || !macaddr) 1451 return; 1452 1453 f = i40e_find_filter(vsi, macaddr, vlan); 1454 __i40e_del_filter(vsi, f); 1455 } 1456 1457 /** 1458 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1459 * @vsi: the VSI to be searched 1460 * @macaddr: the mac address to be filtered 1461 * 1462 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1463 * go through all the macvlan filters and add a macvlan filter for each 1464 * unique vlan that already exists. If a PVID has been assigned, instead only 1465 * add the macaddr to that VLAN. 1466 * 1467 * Returns last filter added on success, else NULL 1468 **/ 1469 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1470 const u8 *macaddr) 1471 { 1472 struct i40e_mac_filter *f, *add = NULL; 1473 struct hlist_node *h; 1474 int bkt; 1475 1476 if (vsi->info.pvid) 1477 return i40e_add_filter(vsi, macaddr, 1478 le16_to_cpu(vsi->info.pvid)); 1479 1480 if (!i40e_is_vsi_in_vlan(vsi)) 1481 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1482 1483 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1484 if (f->state == I40E_FILTER_REMOVE) 1485 continue; 1486 add = i40e_add_filter(vsi, macaddr, f->vlan); 1487 if (!add) 1488 return NULL; 1489 } 1490 1491 return add; 1492 } 1493 1494 /** 1495 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1496 * @vsi: the VSI to be searched 1497 * @macaddr: the mac address to be removed 1498 * 1499 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1500 * associated with. 1501 * 1502 * Returns 0 for success, or error 1503 **/ 1504 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1505 { 1506 struct i40e_mac_filter *f; 1507 struct hlist_node *h; 1508 bool found = false; 1509 int bkt; 1510 1511 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1512 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1513 if (ether_addr_equal(macaddr, f->macaddr)) { 1514 __i40e_del_filter(vsi, f); 1515 found = true; 1516 } 1517 } 1518 1519 if (found) 1520 return 0; 1521 else 1522 return -ENOENT; 1523 } 1524 1525 /** 1526 * i40e_set_mac - NDO callback to set mac address 1527 * @netdev: network interface device structure 1528 * @p: pointer to an address structure 1529 * 1530 * Returns 0 on success, negative on failure 1531 **/ 1532 static int i40e_set_mac(struct net_device *netdev, void *p) 1533 { 1534 struct i40e_netdev_priv *np = netdev_priv(netdev); 1535 struct i40e_vsi *vsi = np->vsi; 1536 struct i40e_pf *pf = vsi->back; 1537 struct i40e_hw *hw = &pf->hw; 1538 struct sockaddr *addr = p; 1539 1540 if (!is_valid_ether_addr(addr->sa_data)) 1541 return -EADDRNOTAVAIL; 1542 1543 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1544 netdev_info(netdev, "already using mac address %pM\n", 1545 addr->sa_data); 1546 return 0; 1547 } 1548 1549 if (test_bit(__I40E_DOWN, pf->state) || 1550 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 1551 return -EADDRNOTAVAIL; 1552 1553 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1554 netdev_info(netdev, "returning to hw mac address %pM\n", 1555 hw->mac.addr); 1556 else 1557 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1558 1559 /* Copy the address first, so that we avoid a possible race with 1560 * .set_rx_mode(). 1561 * - Remove old address from MAC filter 1562 * - Copy new address 1563 * - Add new address to MAC filter 1564 */ 1565 spin_lock_bh(&vsi->mac_filter_hash_lock); 1566 i40e_del_mac_filter(vsi, netdev->dev_addr); 1567 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1568 i40e_add_mac_filter(vsi, netdev->dev_addr); 1569 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1570 1571 if (vsi->type == I40E_VSI_MAIN) { 1572 i40e_status ret; 1573 1574 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, 1575 addr->sa_data, NULL); 1576 if (ret) 1577 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1578 i40e_stat_str(hw, ret), 1579 i40e_aq_str(hw, hw->aq.asq_last_status)); 1580 } 1581 1582 /* schedule our worker thread which will take care of 1583 * applying the new filter changes 1584 */ 1585 i40e_service_event_schedule(pf); 1586 return 0; 1587 } 1588 1589 /** 1590 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1591 * @vsi: vsi structure 1592 * @seed: RSS hash seed 1593 **/ 1594 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1595 u8 *lut, u16 lut_size) 1596 { 1597 struct i40e_pf *pf = vsi->back; 1598 struct i40e_hw *hw = &pf->hw; 1599 int ret = 0; 1600 1601 if (seed) { 1602 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1603 (struct i40e_aqc_get_set_rss_key_data *)seed; 1604 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1605 if (ret) { 1606 dev_info(&pf->pdev->dev, 1607 "Cannot set RSS key, err %s aq_err %s\n", 1608 i40e_stat_str(hw, ret), 1609 i40e_aq_str(hw, hw->aq.asq_last_status)); 1610 return ret; 1611 } 1612 } 1613 if (lut) { 1614 bool pf_lut = vsi->type == I40E_VSI_MAIN; 1615 1616 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1617 if (ret) { 1618 dev_info(&pf->pdev->dev, 1619 "Cannot set RSS lut, err %s aq_err %s\n", 1620 i40e_stat_str(hw, ret), 1621 i40e_aq_str(hw, hw->aq.asq_last_status)); 1622 return ret; 1623 } 1624 } 1625 return ret; 1626 } 1627 1628 /** 1629 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1630 * @vsi: VSI structure 1631 **/ 1632 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1633 { 1634 struct i40e_pf *pf = vsi->back; 1635 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1636 u8 *lut; 1637 int ret; 1638 1639 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1640 return 0; 1641 if (!vsi->rss_size) 1642 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1643 vsi->num_queue_pairs); 1644 if (!vsi->rss_size) 1645 return -EINVAL; 1646 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1647 if (!lut) 1648 return -ENOMEM; 1649 1650 /* Use the user configured hash keys and lookup table if there is one, 1651 * otherwise use default 1652 */ 1653 if (vsi->rss_lut_user) 1654 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1655 else 1656 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1657 if (vsi->rss_hkey_user) 1658 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1659 else 1660 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1661 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1662 kfree(lut); 1663 return ret; 1664 } 1665 1666 /** 1667 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1668 * @vsi: the VSI being configured, 1669 * @ctxt: VSI context structure 1670 * @enabled_tc: number of traffic classes to enable 1671 * 1672 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1673 **/ 1674 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1675 struct i40e_vsi_context *ctxt, 1676 u8 enabled_tc) 1677 { 1678 u16 qcount = 0, max_qcount, qmap, sections = 0; 1679 int i, override_q, pow, num_qps, ret; 1680 u8 netdev_tc = 0, offset = 0; 1681 1682 if (vsi->type != I40E_VSI_MAIN) 1683 return -EINVAL; 1684 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1685 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1686 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1687 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1688 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1689 1690 /* find the next higher power-of-2 of num queue pairs */ 1691 pow = ilog2(num_qps); 1692 if (!is_power_of_2(num_qps)) 1693 pow++; 1694 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1695 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1696 1697 /* Setup queue offset/count for all TCs for given VSI */ 1698 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1699 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1700 /* See if the given TC is enabled for the given VSI */ 1701 if (vsi->tc_config.enabled_tc & BIT(i)) { 1702 offset = vsi->mqprio_qopt.qopt.offset[i]; 1703 qcount = vsi->mqprio_qopt.qopt.count[i]; 1704 if (qcount > max_qcount) 1705 max_qcount = qcount; 1706 vsi->tc_config.tc_info[i].qoffset = offset; 1707 vsi->tc_config.tc_info[i].qcount = qcount; 1708 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1709 } else { 1710 /* TC is not enabled so set the offset to 1711 * default queue and allocate one queue 1712 * for the given TC. 1713 */ 1714 vsi->tc_config.tc_info[i].qoffset = 0; 1715 vsi->tc_config.tc_info[i].qcount = 1; 1716 vsi->tc_config.tc_info[i].netdev_tc = 0; 1717 } 1718 } 1719 1720 /* Set actual Tx/Rx queue pairs */ 1721 vsi->num_queue_pairs = offset + qcount; 1722 1723 /* Setup queue TC[0].qmap for given VSI context */ 1724 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1725 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1726 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1727 ctxt->info.valid_sections |= cpu_to_le16(sections); 1728 1729 /* Reconfigure RSS for main VSI with max queue count */ 1730 vsi->rss_size = max_qcount; 1731 ret = i40e_vsi_config_rss(vsi); 1732 if (ret) { 1733 dev_info(&vsi->back->pdev->dev, 1734 "Failed to reconfig rss for num_queues (%u)\n", 1735 max_qcount); 1736 return ret; 1737 } 1738 vsi->reconfig_rss = true; 1739 dev_dbg(&vsi->back->pdev->dev, 1740 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1741 1742 /* Find queue count available for channel VSIs and starting offset 1743 * for channel VSIs 1744 */ 1745 override_q = vsi->mqprio_qopt.qopt.count[0]; 1746 if (override_q && override_q < vsi->num_queue_pairs) { 1747 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1748 vsi->next_base_queue = override_q; 1749 } 1750 return 0; 1751 } 1752 1753 /** 1754 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1755 * @vsi: the VSI being setup 1756 * @ctxt: VSI context structure 1757 * @enabled_tc: Enabled TCs bitmap 1758 * @is_add: True if called before Add VSI 1759 * 1760 * Setup VSI queue mapping for enabled traffic classes. 1761 **/ 1762 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1763 struct i40e_vsi_context *ctxt, 1764 u8 enabled_tc, 1765 bool is_add) 1766 { 1767 struct i40e_pf *pf = vsi->back; 1768 u16 sections = 0; 1769 u8 netdev_tc = 0; 1770 u16 numtc = 1; 1771 u16 qcount; 1772 u8 offset; 1773 u16 qmap; 1774 int i; 1775 u16 num_tc_qps = 0; 1776 1777 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1778 offset = 0; 1779 1780 /* Number of queues per enabled TC */ 1781 num_tc_qps = vsi->alloc_queue_pairs; 1782 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1783 /* Find numtc from enabled TC bitmap */ 1784 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1785 if (enabled_tc & BIT(i)) /* TC is enabled */ 1786 numtc++; 1787 } 1788 if (!numtc) { 1789 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1790 numtc = 1; 1791 } 1792 num_tc_qps = num_tc_qps / numtc; 1793 num_tc_qps = min_t(int, num_tc_qps, 1794 i40e_pf_get_max_q_per_tc(pf)); 1795 } 1796 1797 vsi->tc_config.numtc = numtc; 1798 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1799 1800 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 1801 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1802 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 1803 1804 /* Setup queue offset/count for all TCs for given VSI */ 1805 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1806 /* See if the given TC is enabled for the given VSI */ 1807 if (vsi->tc_config.enabled_tc & BIT(i)) { 1808 /* TC is enabled */ 1809 int pow, num_qps; 1810 1811 switch (vsi->type) { 1812 case I40E_VSI_MAIN: 1813 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | 1814 I40E_FLAG_FD_ATR_ENABLED)) || 1815 vsi->tc_config.enabled_tc != 1) { 1816 qcount = min_t(int, pf->alloc_rss_size, 1817 num_tc_qps); 1818 break; 1819 } 1820 /* fall through */ 1821 case I40E_VSI_FDIR: 1822 case I40E_VSI_SRIOV: 1823 case I40E_VSI_VMDQ2: 1824 default: 1825 qcount = num_tc_qps; 1826 WARN_ON(i != 0); 1827 break; 1828 } 1829 vsi->tc_config.tc_info[i].qoffset = offset; 1830 vsi->tc_config.tc_info[i].qcount = qcount; 1831 1832 /* find the next higher power-of-2 of num queue pairs */ 1833 num_qps = qcount; 1834 pow = 0; 1835 while (num_qps && (BIT_ULL(pow) < qcount)) { 1836 pow++; 1837 num_qps >>= 1; 1838 } 1839 1840 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1841 qmap = 1842 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1843 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1844 1845 offset += qcount; 1846 } else { 1847 /* TC is not enabled so set the offset to 1848 * default queue and allocate one queue 1849 * for the given TC. 1850 */ 1851 vsi->tc_config.tc_info[i].qoffset = 0; 1852 vsi->tc_config.tc_info[i].qcount = 1; 1853 vsi->tc_config.tc_info[i].netdev_tc = 0; 1854 1855 qmap = 0; 1856 } 1857 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1858 } 1859 1860 /* Set actual Tx/Rx queue pairs */ 1861 vsi->num_queue_pairs = offset; 1862 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1863 if (vsi->req_queue_pairs > 0) 1864 vsi->num_queue_pairs = vsi->req_queue_pairs; 1865 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1866 vsi->num_queue_pairs = pf->num_lan_msix; 1867 } 1868 1869 /* Scheduler section valid can only be set for ADD VSI */ 1870 if (is_add) { 1871 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1872 1873 ctxt->info.up_enable_bits = enabled_tc; 1874 } 1875 if (vsi->type == I40E_VSI_SRIOV) { 1876 ctxt->info.mapping_flags |= 1877 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1878 for (i = 0; i < vsi->num_queue_pairs; i++) 1879 ctxt->info.queue_mapping[i] = 1880 cpu_to_le16(vsi->base_queue + i); 1881 } else { 1882 ctxt->info.mapping_flags |= 1883 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1884 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1885 } 1886 ctxt->info.valid_sections |= cpu_to_le16(sections); 1887 } 1888 1889 /** 1890 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1891 * @netdev: the netdevice 1892 * @addr: address to add 1893 * 1894 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1895 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1896 */ 1897 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1898 { 1899 struct i40e_netdev_priv *np = netdev_priv(netdev); 1900 struct i40e_vsi *vsi = np->vsi; 1901 1902 if (i40e_add_mac_filter(vsi, addr)) 1903 return 0; 1904 else 1905 return -ENOMEM; 1906 } 1907 1908 /** 1909 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1910 * @netdev: the netdevice 1911 * @addr: address to add 1912 * 1913 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1914 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1915 */ 1916 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1917 { 1918 struct i40e_netdev_priv *np = netdev_priv(netdev); 1919 struct i40e_vsi *vsi = np->vsi; 1920 1921 /* Under some circumstances, we might receive a request to delete 1922 * our own device address from our uc list. Because we store the 1923 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1924 * such requests and not delete our device address from this list. 1925 */ 1926 if (ether_addr_equal(addr, netdev->dev_addr)) 1927 return 0; 1928 1929 i40e_del_mac_filter(vsi, addr); 1930 1931 return 0; 1932 } 1933 1934 /** 1935 * i40e_set_rx_mode - NDO callback to set the netdev filters 1936 * @netdev: network interface device structure 1937 **/ 1938 static void i40e_set_rx_mode(struct net_device *netdev) 1939 { 1940 struct i40e_netdev_priv *np = netdev_priv(netdev); 1941 struct i40e_vsi *vsi = np->vsi; 1942 1943 spin_lock_bh(&vsi->mac_filter_hash_lock); 1944 1945 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1946 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1947 1948 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1949 1950 /* check for other flag changes */ 1951 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1952 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1953 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1954 } 1955 } 1956 1957 /** 1958 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1959 * @vsi: Pointer to VSI struct 1960 * @from: Pointer to list which contains MAC filter entries - changes to 1961 * those entries needs to be undone. 1962 * 1963 * MAC filter entries from this list were slated for deletion. 1964 **/ 1965 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1966 struct hlist_head *from) 1967 { 1968 struct i40e_mac_filter *f; 1969 struct hlist_node *h; 1970 1971 hlist_for_each_entry_safe(f, h, from, hlist) { 1972 u64 key = i40e_addr_to_hkey(f->macaddr); 1973 1974 /* Move the element back into MAC filter list*/ 1975 hlist_del(&f->hlist); 1976 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1977 } 1978 } 1979 1980 /** 1981 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1982 * @vsi: Pointer to vsi struct 1983 * @from: Pointer to list which contains MAC filter entries - changes to 1984 * those entries needs to be undone. 1985 * 1986 * MAC filter entries from this list were slated for addition. 1987 **/ 1988 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 1989 struct hlist_head *from) 1990 { 1991 struct i40e_new_mac_filter *new; 1992 struct hlist_node *h; 1993 1994 hlist_for_each_entry_safe(new, h, from, hlist) { 1995 /* We can simply free the wrapper structure */ 1996 hlist_del(&new->hlist); 1997 kfree(new); 1998 } 1999 } 2000 2001 /** 2002 * i40e_next_entry - Get the next non-broadcast filter from a list 2003 * @next: pointer to filter in list 2004 * 2005 * Returns the next non-broadcast filter in the list. Required so that we 2006 * ignore broadcast filters within the list, since these are not handled via 2007 * the normal firmware update path. 2008 */ 2009 static 2010 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2011 { 2012 hlist_for_each_entry_continue(next, hlist) { 2013 if (!is_broadcast_ether_addr(next->f->macaddr)) 2014 return next; 2015 } 2016 2017 return NULL; 2018 } 2019 2020 /** 2021 * i40e_update_filter_state - Update filter state based on return data 2022 * from firmware 2023 * @count: Number of filters added 2024 * @add_list: return data from fw 2025 * @add_head: pointer to first filter in current batch 2026 * 2027 * MAC filter entries from list were slated to be added to device. Returns 2028 * number of successful filters. Note that 0 does NOT mean success! 2029 **/ 2030 static int 2031 i40e_update_filter_state(int count, 2032 struct i40e_aqc_add_macvlan_element_data *add_list, 2033 struct i40e_new_mac_filter *add_head) 2034 { 2035 int retval = 0; 2036 int i; 2037 2038 for (i = 0; i < count; i++) { 2039 /* Always check status of each filter. We don't need to check 2040 * the firmware return status because we pre-set the filter 2041 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2042 * request to the adminq. Thus, if it no longer matches then 2043 * we know the filter is active. 2044 */ 2045 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2046 add_head->state = I40E_FILTER_FAILED; 2047 } else { 2048 add_head->state = I40E_FILTER_ACTIVE; 2049 retval++; 2050 } 2051 2052 add_head = i40e_next_filter(add_head); 2053 if (!add_head) 2054 break; 2055 } 2056 2057 return retval; 2058 } 2059 2060 /** 2061 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2062 * @vsi: ptr to the VSI 2063 * @vsi_name: name to display in messages 2064 * @list: the list of filters to send to firmware 2065 * @num_del: the number of filters to delete 2066 * @retval: Set to -EIO on failure to delete 2067 * 2068 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2069 * *retval instead of a return value so that success does not force ret_val to 2070 * be set to 0. This ensures that a sequence of calls to this function 2071 * preserve the previous value of *retval on successful delete. 2072 */ 2073 static 2074 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2075 struct i40e_aqc_remove_macvlan_element_data *list, 2076 int num_del, int *retval) 2077 { 2078 struct i40e_hw *hw = &vsi->back->hw; 2079 i40e_status aq_ret; 2080 int aq_err; 2081 2082 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 2083 aq_err = hw->aq.asq_last_status; 2084 2085 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2086 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 2087 *retval = -EIO; 2088 dev_info(&vsi->back->pdev->dev, 2089 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 2090 vsi_name, i40e_stat_str(hw, aq_ret), 2091 i40e_aq_str(hw, aq_err)); 2092 } 2093 } 2094 2095 /** 2096 * i40e_aqc_add_filters - Request firmware to add a set of filters 2097 * @vsi: ptr to the VSI 2098 * @vsi_name: name to display in messages 2099 * @list: the list of filters to send to firmware 2100 * @add_head: Position in the add hlist 2101 * @num_add: the number of filters to add 2102 * 2103 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2104 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2105 * space for more filters. 2106 */ 2107 static 2108 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2109 struct i40e_aqc_add_macvlan_element_data *list, 2110 struct i40e_new_mac_filter *add_head, 2111 int num_add) 2112 { 2113 struct i40e_hw *hw = &vsi->back->hw; 2114 int aq_err, fcnt; 2115 2116 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 2117 aq_err = hw->aq.asq_last_status; 2118 fcnt = i40e_update_filter_state(num_add, list, add_head); 2119 2120 if (fcnt != num_add) { 2121 if (vsi->type == I40E_VSI_MAIN) { 2122 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2123 dev_warn(&vsi->back->pdev->dev, 2124 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2125 i40e_aq_str(hw, aq_err), vsi_name); 2126 } else if (vsi->type == I40E_VSI_SRIOV || 2127 vsi->type == I40E_VSI_VMDQ1 || 2128 vsi->type == I40E_VSI_VMDQ2) { 2129 dev_warn(&vsi->back->pdev->dev, 2130 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n", 2131 i40e_aq_str(hw, aq_err), vsi_name, vsi_name); 2132 } else { 2133 dev_warn(&vsi->back->pdev->dev, 2134 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n", 2135 i40e_aq_str(hw, aq_err), vsi_name, vsi->type); 2136 } 2137 } 2138 } 2139 2140 /** 2141 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2142 * @vsi: pointer to the VSI 2143 * @vsi_name: the VSI name 2144 * @f: filter data 2145 * 2146 * This function sets or clears the promiscuous broadcast flags for VLAN 2147 * filters in order to properly receive broadcast frames. Assumes that only 2148 * broadcast filters are passed. 2149 * 2150 * Returns status indicating success or failure; 2151 **/ 2152 static i40e_status 2153 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2154 struct i40e_mac_filter *f) 2155 { 2156 bool enable = f->state == I40E_FILTER_NEW; 2157 struct i40e_hw *hw = &vsi->back->hw; 2158 i40e_status aq_ret; 2159 2160 if (f->vlan == I40E_VLAN_ANY) { 2161 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2162 vsi->seid, 2163 enable, 2164 NULL); 2165 } else { 2166 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2167 vsi->seid, 2168 enable, 2169 f->vlan, 2170 NULL); 2171 } 2172 2173 if (aq_ret) { 2174 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2175 dev_warn(&vsi->back->pdev->dev, 2176 "Error %s, forcing overflow promiscuous on %s\n", 2177 i40e_aq_str(hw, hw->aq.asq_last_status), 2178 vsi_name); 2179 } 2180 2181 return aq_ret; 2182 } 2183 2184 /** 2185 * i40e_set_promiscuous - set promiscuous mode 2186 * @pf: board private structure 2187 * @promisc: promisc on or off 2188 * 2189 * There are different ways of setting promiscuous mode on a PF depending on 2190 * what state/environment we're in. This identifies and sets it appropriately. 2191 * Returns 0 on success. 2192 **/ 2193 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2194 { 2195 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2196 struct i40e_hw *hw = &pf->hw; 2197 i40e_status aq_ret; 2198 2199 if (vsi->type == I40E_VSI_MAIN && 2200 pf->lan_veb != I40E_NO_VEB && 2201 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2202 /* set defport ON for Main VSI instead of true promisc 2203 * this way we will get all unicast/multicast and VLAN 2204 * promisc behavior but will not get VF or VMDq traffic 2205 * replicated on the Main VSI. 2206 */ 2207 if (promisc) 2208 aq_ret = i40e_aq_set_default_vsi(hw, 2209 vsi->seid, 2210 NULL); 2211 else 2212 aq_ret = i40e_aq_clear_default_vsi(hw, 2213 vsi->seid, 2214 NULL); 2215 if (aq_ret) { 2216 dev_info(&pf->pdev->dev, 2217 "Set default VSI failed, err %s, aq_err %s\n", 2218 i40e_stat_str(hw, aq_ret), 2219 i40e_aq_str(hw, hw->aq.asq_last_status)); 2220 } 2221 } else { 2222 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2223 hw, 2224 vsi->seid, 2225 promisc, NULL, 2226 true); 2227 if (aq_ret) { 2228 dev_info(&pf->pdev->dev, 2229 "set unicast promisc failed, err %s, aq_err %s\n", 2230 i40e_stat_str(hw, aq_ret), 2231 i40e_aq_str(hw, hw->aq.asq_last_status)); 2232 } 2233 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2234 hw, 2235 vsi->seid, 2236 promisc, NULL); 2237 if (aq_ret) { 2238 dev_info(&pf->pdev->dev, 2239 "set multicast promisc failed, err %s, aq_err %s\n", 2240 i40e_stat_str(hw, aq_ret), 2241 i40e_aq_str(hw, hw->aq.asq_last_status)); 2242 } 2243 } 2244 2245 if (!aq_ret) 2246 pf->cur_promisc = promisc; 2247 2248 return aq_ret; 2249 } 2250 2251 /** 2252 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2253 * @vsi: ptr to the VSI 2254 * 2255 * Push any outstanding VSI filter changes through the AdminQ. 2256 * 2257 * Returns 0 or error value 2258 **/ 2259 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2260 { 2261 struct hlist_head tmp_add_list, tmp_del_list; 2262 struct i40e_mac_filter *f; 2263 struct i40e_new_mac_filter *new, *add_head = NULL; 2264 struct i40e_hw *hw = &vsi->back->hw; 2265 bool old_overflow, new_overflow; 2266 unsigned int failed_filters = 0; 2267 unsigned int vlan_filters = 0; 2268 char vsi_name[16] = "PF"; 2269 int filter_list_len = 0; 2270 i40e_status aq_ret = 0; 2271 u32 changed_flags = 0; 2272 struct hlist_node *h; 2273 struct i40e_pf *pf; 2274 int num_add = 0; 2275 int num_del = 0; 2276 int retval = 0; 2277 u16 cmd_flags; 2278 int list_size; 2279 int bkt; 2280 2281 /* empty array typed pointers, kcalloc later */ 2282 struct i40e_aqc_add_macvlan_element_data *add_list; 2283 struct i40e_aqc_remove_macvlan_element_data *del_list; 2284 2285 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2286 usleep_range(1000, 2000); 2287 pf = vsi->back; 2288 2289 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2290 2291 if (vsi->netdev) { 2292 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2293 vsi->current_netdev_flags = vsi->netdev->flags; 2294 } 2295 2296 INIT_HLIST_HEAD(&tmp_add_list); 2297 INIT_HLIST_HEAD(&tmp_del_list); 2298 2299 if (vsi->type == I40E_VSI_SRIOV) 2300 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2301 else if (vsi->type != I40E_VSI_MAIN) 2302 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2303 2304 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2305 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2306 2307 spin_lock_bh(&vsi->mac_filter_hash_lock); 2308 /* Create a list of filters to delete. */ 2309 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2310 if (f->state == I40E_FILTER_REMOVE) { 2311 /* Move the element into temporary del_list */ 2312 hash_del(&f->hlist); 2313 hlist_add_head(&f->hlist, &tmp_del_list); 2314 2315 /* Avoid counting removed filters */ 2316 continue; 2317 } 2318 if (f->state == I40E_FILTER_NEW) { 2319 /* Create a temporary i40e_new_mac_filter */ 2320 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2321 if (!new) 2322 goto err_no_memory_locked; 2323 2324 /* Store pointer to the real filter */ 2325 new->f = f; 2326 new->state = f->state; 2327 2328 /* Add it to the hash list */ 2329 hlist_add_head(&new->hlist, &tmp_add_list); 2330 } 2331 2332 /* Count the number of active (current and new) VLAN 2333 * filters we have now. Does not count filters which 2334 * are marked for deletion. 2335 */ 2336 if (f->vlan > 0) 2337 vlan_filters++; 2338 } 2339 2340 retval = i40e_correct_mac_vlan_filters(vsi, 2341 &tmp_add_list, 2342 &tmp_del_list, 2343 vlan_filters); 2344 if (retval) 2345 goto err_no_memory_locked; 2346 2347 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2348 } 2349 2350 /* Now process 'del_list' outside the lock */ 2351 if (!hlist_empty(&tmp_del_list)) { 2352 filter_list_len = hw->aq.asq_buf_size / 2353 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2354 list_size = filter_list_len * 2355 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2356 del_list = kzalloc(list_size, GFP_ATOMIC); 2357 if (!del_list) 2358 goto err_no_memory; 2359 2360 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2361 cmd_flags = 0; 2362 2363 /* handle broadcast filters by updating the broadcast 2364 * promiscuous flag and release filter list. 2365 */ 2366 if (is_broadcast_ether_addr(f->macaddr)) { 2367 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2368 2369 hlist_del(&f->hlist); 2370 kfree(f); 2371 continue; 2372 } 2373 2374 /* add to delete list */ 2375 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2376 if (f->vlan == I40E_VLAN_ANY) { 2377 del_list[num_del].vlan_tag = 0; 2378 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2379 } else { 2380 del_list[num_del].vlan_tag = 2381 cpu_to_le16((u16)(f->vlan)); 2382 } 2383 2384 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2385 del_list[num_del].flags = cmd_flags; 2386 num_del++; 2387 2388 /* flush a full buffer */ 2389 if (num_del == filter_list_len) { 2390 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2391 num_del, &retval); 2392 memset(del_list, 0, list_size); 2393 num_del = 0; 2394 } 2395 /* Release memory for MAC filter entries which were 2396 * synced up with HW. 2397 */ 2398 hlist_del(&f->hlist); 2399 kfree(f); 2400 } 2401 2402 if (num_del) { 2403 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2404 num_del, &retval); 2405 } 2406 2407 kfree(del_list); 2408 del_list = NULL; 2409 } 2410 2411 if (!hlist_empty(&tmp_add_list)) { 2412 /* Do all the adds now. */ 2413 filter_list_len = hw->aq.asq_buf_size / 2414 sizeof(struct i40e_aqc_add_macvlan_element_data); 2415 list_size = filter_list_len * 2416 sizeof(struct i40e_aqc_add_macvlan_element_data); 2417 add_list = kzalloc(list_size, GFP_ATOMIC); 2418 if (!add_list) 2419 goto err_no_memory; 2420 2421 num_add = 0; 2422 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2423 /* handle broadcast filters by updating the broadcast 2424 * promiscuous flag instead of adding a MAC filter. 2425 */ 2426 if (is_broadcast_ether_addr(new->f->macaddr)) { 2427 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2428 new->f)) 2429 new->state = I40E_FILTER_FAILED; 2430 else 2431 new->state = I40E_FILTER_ACTIVE; 2432 continue; 2433 } 2434 2435 /* add to add array */ 2436 if (num_add == 0) 2437 add_head = new; 2438 cmd_flags = 0; 2439 ether_addr_copy(add_list[num_add].mac_addr, 2440 new->f->macaddr); 2441 if (new->f->vlan == I40E_VLAN_ANY) { 2442 add_list[num_add].vlan_tag = 0; 2443 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2444 } else { 2445 add_list[num_add].vlan_tag = 2446 cpu_to_le16((u16)(new->f->vlan)); 2447 } 2448 add_list[num_add].queue_number = 0; 2449 /* set invalid match method for later detection */ 2450 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2451 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2452 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2453 num_add++; 2454 2455 /* flush a full buffer */ 2456 if (num_add == filter_list_len) { 2457 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2458 add_head, num_add); 2459 memset(add_list, 0, list_size); 2460 num_add = 0; 2461 } 2462 } 2463 if (num_add) { 2464 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2465 num_add); 2466 } 2467 /* Now move all of the filters from the temp add list back to 2468 * the VSI's list. 2469 */ 2470 spin_lock_bh(&vsi->mac_filter_hash_lock); 2471 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2472 /* Only update the state if we're still NEW */ 2473 if (new->f->state == I40E_FILTER_NEW) 2474 new->f->state = new->state; 2475 hlist_del(&new->hlist); 2476 kfree(new); 2477 } 2478 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2479 kfree(add_list); 2480 add_list = NULL; 2481 } 2482 2483 /* Determine the number of active and failed filters. */ 2484 spin_lock_bh(&vsi->mac_filter_hash_lock); 2485 vsi->active_filters = 0; 2486 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2487 if (f->state == I40E_FILTER_ACTIVE) 2488 vsi->active_filters++; 2489 else if (f->state == I40E_FILTER_FAILED) 2490 failed_filters++; 2491 } 2492 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2493 2494 /* Check if we are able to exit overflow promiscuous mode. We can 2495 * safely exit if we didn't just enter, we no longer have any failed 2496 * filters, and we have reduced filters below the threshold value. 2497 */ 2498 if (old_overflow && !failed_filters && 2499 vsi->active_filters < vsi->promisc_threshold) { 2500 dev_info(&pf->pdev->dev, 2501 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2502 vsi_name); 2503 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2504 vsi->promisc_threshold = 0; 2505 } 2506 2507 /* if the VF is not trusted do not do promisc */ 2508 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2509 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2510 goto out; 2511 } 2512 2513 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2514 2515 /* If we are entering overflow promiscuous, we need to calculate a new 2516 * threshold for when we are safe to exit 2517 */ 2518 if (!old_overflow && new_overflow) 2519 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2520 2521 /* check for changes in promiscuous modes */ 2522 if (changed_flags & IFF_ALLMULTI) { 2523 bool cur_multipromisc; 2524 2525 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2526 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2527 vsi->seid, 2528 cur_multipromisc, 2529 NULL); 2530 if (aq_ret) { 2531 retval = i40e_aq_rc_to_posix(aq_ret, 2532 hw->aq.asq_last_status); 2533 dev_info(&pf->pdev->dev, 2534 "set multi promisc failed on %s, err %s aq_err %s\n", 2535 vsi_name, 2536 i40e_stat_str(hw, aq_ret), 2537 i40e_aq_str(hw, hw->aq.asq_last_status)); 2538 } else { 2539 dev_info(&pf->pdev->dev, "%s is %s allmulti mode.\n", 2540 vsi->netdev->name, 2541 cur_multipromisc ? "entering" : "leaving"); 2542 } 2543 } 2544 2545 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2546 bool cur_promisc; 2547 2548 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2549 new_overflow); 2550 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2551 if (aq_ret) { 2552 retval = i40e_aq_rc_to_posix(aq_ret, 2553 hw->aq.asq_last_status); 2554 dev_info(&pf->pdev->dev, 2555 "Setting promiscuous %s failed on %s, err %s aq_err %s\n", 2556 cur_promisc ? "on" : "off", 2557 vsi_name, 2558 i40e_stat_str(hw, aq_ret), 2559 i40e_aq_str(hw, hw->aq.asq_last_status)); 2560 } 2561 } 2562 out: 2563 /* if something went wrong then set the changed flag so we try again */ 2564 if (retval) 2565 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2566 2567 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2568 return retval; 2569 2570 err_no_memory: 2571 /* Restore elements on the temporary add and delete lists */ 2572 spin_lock_bh(&vsi->mac_filter_hash_lock); 2573 err_no_memory_locked: 2574 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2575 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2576 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2577 2578 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2579 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2580 return -ENOMEM; 2581 } 2582 2583 /** 2584 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2585 * @pf: board private structure 2586 **/ 2587 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2588 { 2589 int v; 2590 2591 if (!pf) 2592 return; 2593 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2594 return; 2595 if (test_and_set_bit(__I40E_VF_DISABLE, pf->state)) { 2596 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 2597 return; 2598 } 2599 2600 for (v = 0; v < pf->num_alloc_vsi; v++) { 2601 if (pf->vsi[v] && 2602 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2603 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2604 2605 if (ret) { 2606 /* come back and try again later */ 2607 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2608 pf->state); 2609 break; 2610 } 2611 } 2612 } 2613 clear_bit(__I40E_VF_DISABLE, pf->state); 2614 } 2615 2616 /** 2617 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2618 * @vsi: the vsi 2619 **/ 2620 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2621 { 2622 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2623 return I40E_RXBUFFER_2048; 2624 else 2625 return I40E_RXBUFFER_3072; 2626 } 2627 2628 /** 2629 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2630 * @netdev: network interface device structure 2631 * @new_mtu: new value for maximum frame size 2632 * 2633 * Returns 0 on success, negative on failure 2634 **/ 2635 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2636 { 2637 struct i40e_netdev_priv *np = netdev_priv(netdev); 2638 struct i40e_vsi *vsi = np->vsi; 2639 struct i40e_pf *pf = vsi->back; 2640 2641 if (i40e_enabled_xdp_vsi(vsi)) { 2642 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2643 2644 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2645 return -EINVAL; 2646 } 2647 2648 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2649 netdev->mtu, new_mtu); 2650 netdev->mtu = new_mtu; 2651 if (netif_running(netdev)) 2652 i40e_vsi_reinit_locked(vsi); 2653 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2654 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2655 return 0; 2656 } 2657 2658 /** 2659 * i40e_ioctl - Access the hwtstamp interface 2660 * @netdev: network interface device structure 2661 * @ifr: interface request data 2662 * @cmd: ioctl command 2663 **/ 2664 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2665 { 2666 struct i40e_netdev_priv *np = netdev_priv(netdev); 2667 struct i40e_pf *pf = np->vsi->back; 2668 2669 switch (cmd) { 2670 case SIOCGHWTSTAMP: 2671 return i40e_ptp_get_ts_config(pf, ifr); 2672 case SIOCSHWTSTAMP: 2673 return i40e_ptp_set_ts_config(pf, ifr); 2674 default: 2675 return -EOPNOTSUPP; 2676 } 2677 } 2678 2679 /** 2680 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2681 * @vsi: the vsi being adjusted 2682 **/ 2683 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2684 { 2685 struct i40e_vsi_context ctxt; 2686 i40e_status ret; 2687 2688 /* Don't modify stripping options if a port VLAN is active */ 2689 if (vsi->info.pvid) 2690 return; 2691 2692 if ((vsi->info.valid_sections & 2693 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2694 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2695 return; /* already enabled */ 2696 2697 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2698 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2699 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2700 2701 ctxt.seid = vsi->seid; 2702 ctxt.info = vsi->info; 2703 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2704 if (ret) { 2705 dev_info(&vsi->back->pdev->dev, 2706 "update vlan stripping failed, err %s aq_err %s\n", 2707 i40e_stat_str(&vsi->back->hw, ret), 2708 i40e_aq_str(&vsi->back->hw, 2709 vsi->back->hw.aq.asq_last_status)); 2710 } 2711 } 2712 2713 /** 2714 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2715 * @vsi: the vsi being adjusted 2716 **/ 2717 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2718 { 2719 struct i40e_vsi_context ctxt; 2720 i40e_status ret; 2721 2722 /* Don't modify stripping options if a port VLAN is active */ 2723 if (vsi->info.pvid) 2724 return; 2725 2726 if ((vsi->info.valid_sections & 2727 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2728 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2729 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2730 return; /* already disabled */ 2731 2732 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2733 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2734 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2735 2736 ctxt.seid = vsi->seid; 2737 ctxt.info = vsi->info; 2738 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2739 if (ret) { 2740 dev_info(&vsi->back->pdev->dev, 2741 "update vlan stripping failed, err %s aq_err %s\n", 2742 i40e_stat_str(&vsi->back->hw, ret), 2743 i40e_aq_str(&vsi->back->hw, 2744 vsi->back->hw.aq.asq_last_status)); 2745 } 2746 } 2747 2748 /** 2749 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2750 * @vsi: the vsi being configured 2751 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2752 * 2753 * This is a helper function for adding a new MAC/VLAN filter with the 2754 * specified VLAN for each existing MAC address already in the hash table. 2755 * This function does *not* perform any accounting to update filters based on 2756 * VLAN mode. 2757 * 2758 * NOTE: this function expects to be called while under the 2759 * mac_filter_hash_lock 2760 **/ 2761 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2762 { 2763 struct i40e_mac_filter *f, *add_f; 2764 struct hlist_node *h; 2765 int bkt; 2766 2767 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2768 if (f->state == I40E_FILTER_REMOVE) 2769 continue; 2770 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2771 if (!add_f) { 2772 dev_info(&vsi->back->pdev->dev, 2773 "Could not add vlan filter %d for %pM\n", 2774 vid, f->macaddr); 2775 return -ENOMEM; 2776 } 2777 } 2778 2779 return 0; 2780 } 2781 2782 /** 2783 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2784 * @vsi: the VSI being configured 2785 * @vid: VLAN id to be added 2786 **/ 2787 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2788 { 2789 int err; 2790 2791 if (vsi->info.pvid) 2792 return -EINVAL; 2793 2794 /* The network stack will attempt to add VID=0, with the intention to 2795 * receive priority tagged packets with a VLAN of 0. Our HW receives 2796 * these packets by default when configured to receive untagged 2797 * packets, so we don't need to add a filter for this case. 2798 * Additionally, HW interprets adding a VID=0 filter as meaning to 2799 * receive *only* tagged traffic and stops receiving untagged traffic. 2800 * Thus, we do not want to actually add a filter for VID=0 2801 */ 2802 if (!vid) 2803 return 0; 2804 2805 /* Locked once because all functions invoked below iterates list*/ 2806 spin_lock_bh(&vsi->mac_filter_hash_lock); 2807 err = i40e_add_vlan_all_mac(vsi, vid); 2808 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2809 if (err) 2810 return err; 2811 2812 /* schedule our worker thread which will take care of 2813 * applying the new filter changes 2814 */ 2815 i40e_service_event_schedule(vsi->back); 2816 return 0; 2817 } 2818 2819 /** 2820 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2821 * @vsi: the vsi being configured 2822 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2823 * 2824 * This function should be used to remove all VLAN filters which match the 2825 * given VID. It does not schedule the service event and does not take the 2826 * mac_filter_hash_lock so it may be combined with other operations under 2827 * a single invocation of the mac_filter_hash_lock. 2828 * 2829 * NOTE: this function expects to be called while under the 2830 * mac_filter_hash_lock 2831 */ 2832 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2833 { 2834 struct i40e_mac_filter *f; 2835 struct hlist_node *h; 2836 int bkt; 2837 2838 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2839 if (f->vlan == vid) 2840 __i40e_del_filter(vsi, f); 2841 } 2842 } 2843 2844 /** 2845 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2846 * @vsi: the VSI being configured 2847 * @vid: VLAN id to be removed 2848 **/ 2849 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2850 { 2851 if (!vid || vsi->info.pvid) 2852 return; 2853 2854 spin_lock_bh(&vsi->mac_filter_hash_lock); 2855 i40e_rm_vlan_all_mac(vsi, vid); 2856 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2857 2858 /* schedule our worker thread which will take care of 2859 * applying the new filter changes 2860 */ 2861 i40e_service_event_schedule(vsi->back); 2862 } 2863 2864 /** 2865 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2866 * @netdev: network interface to be adjusted 2867 * @proto: unused protocol value 2868 * @vid: vlan id to be added 2869 * 2870 * net_device_ops implementation for adding vlan ids 2871 **/ 2872 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2873 __always_unused __be16 proto, u16 vid) 2874 { 2875 struct i40e_netdev_priv *np = netdev_priv(netdev); 2876 struct i40e_vsi *vsi = np->vsi; 2877 int ret = 0; 2878 2879 if (vid >= VLAN_N_VID) 2880 return -EINVAL; 2881 2882 ret = i40e_vsi_add_vlan(vsi, vid); 2883 if (!ret) 2884 set_bit(vid, vsi->active_vlans); 2885 2886 return ret; 2887 } 2888 2889 /** 2890 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 2891 * @netdev: network interface to be adjusted 2892 * @proto: unused protocol value 2893 * @vid: vlan id to be added 2894 **/ 2895 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 2896 __always_unused __be16 proto, u16 vid) 2897 { 2898 struct i40e_netdev_priv *np = netdev_priv(netdev); 2899 struct i40e_vsi *vsi = np->vsi; 2900 2901 if (vid >= VLAN_N_VID) 2902 return; 2903 set_bit(vid, vsi->active_vlans); 2904 } 2905 2906 /** 2907 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2908 * @netdev: network interface to be adjusted 2909 * @proto: unused protocol value 2910 * @vid: vlan id to be removed 2911 * 2912 * net_device_ops implementation for removing vlan ids 2913 **/ 2914 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2915 __always_unused __be16 proto, u16 vid) 2916 { 2917 struct i40e_netdev_priv *np = netdev_priv(netdev); 2918 struct i40e_vsi *vsi = np->vsi; 2919 2920 /* return code is ignored as there is nothing a user 2921 * can do about failure to remove and a log message was 2922 * already printed from the other function 2923 */ 2924 i40e_vsi_kill_vlan(vsi, vid); 2925 2926 clear_bit(vid, vsi->active_vlans); 2927 2928 return 0; 2929 } 2930 2931 /** 2932 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2933 * @vsi: the vsi being brought back up 2934 **/ 2935 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2936 { 2937 u16 vid; 2938 2939 if (!vsi->netdev) 2940 return; 2941 2942 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2943 i40e_vlan_stripping_enable(vsi); 2944 else 2945 i40e_vlan_stripping_disable(vsi); 2946 2947 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2948 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 2949 vid); 2950 } 2951 2952 /** 2953 * i40e_vsi_add_pvid - Add pvid for the VSI 2954 * @vsi: the vsi being adjusted 2955 * @vid: the vlan id to set as a PVID 2956 **/ 2957 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2958 { 2959 struct i40e_vsi_context ctxt; 2960 i40e_status ret; 2961 2962 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2963 vsi->info.pvid = cpu_to_le16(vid); 2964 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2965 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2966 I40E_AQ_VSI_PVLAN_EMOD_STR; 2967 2968 ctxt.seid = vsi->seid; 2969 ctxt.info = vsi->info; 2970 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2971 if (ret) { 2972 dev_info(&vsi->back->pdev->dev, 2973 "add pvid failed, err %s aq_err %s\n", 2974 i40e_stat_str(&vsi->back->hw, ret), 2975 i40e_aq_str(&vsi->back->hw, 2976 vsi->back->hw.aq.asq_last_status)); 2977 return -ENOENT; 2978 } 2979 2980 return 0; 2981 } 2982 2983 /** 2984 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2985 * @vsi: the vsi being adjusted 2986 * 2987 * Just use the vlan_rx_register() service to put it back to normal 2988 **/ 2989 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2990 { 2991 vsi->info.pvid = 0; 2992 2993 i40e_vlan_stripping_disable(vsi); 2994 } 2995 2996 /** 2997 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2998 * @vsi: ptr to the VSI 2999 * 3000 * If this function returns with an error, then it's possible one or 3001 * more of the rings is populated (while the rest are not). It is the 3002 * callers duty to clean those orphaned rings. 3003 * 3004 * Return 0 on success, negative on failure 3005 **/ 3006 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 3007 { 3008 int i, err = 0; 3009 3010 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3011 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 3012 3013 if (!i40e_enabled_xdp_vsi(vsi)) 3014 return err; 3015 3016 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3017 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3018 3019 return err; 3020 } 3021 3022 /** 3023 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3024 * @vsi: ptr to the VSI 3025 * 3026 * Free VSI's transmit software resources 3027 **/ 3028 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3029 { 3030 int i; 3031 3032 if (vsi->tx_rings) { 3033 for (i = 0; i < vsi->num_queue_pairs; i++) 3034 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3035 i40e_free_tx_resources(vsi->tx_rings[i]); 3036 } 3037 3038 if (vsi->xdp_rings) { 3039 for (i = 0; i < vsi->num_queue_pairs; i++) 3040 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3041 i40e_free_tx_resources(vsi->xdp_rings[i]); 3042 } 3043 } 3044 3045 /** 3046 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3047 * @vsi: ptr to the VSI 3048 * 3049 * If this function returns with an error, then it's possible one or 3050 * more of the rings is populated (while the rest are not). It is the 3051 * callers duty to clean those orphaned rings. 3052 * 3053 * Return 0 on success, negative on failure 3054 **/ 3055 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3056 { 3057 int i, err = 0; 3058 3059 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3060 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3061 return err; 3062 } 3063 3064 /** 3065 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3066 * @vsi: ptr to the VSI 3067 * 3068 * Free all receive software resources 3069 **/ 3070 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3071 { 3072 int i; 3073 3074 if (!vsi->rx_rings) 3075 return; 3076 3077 for (i = 0; i < vsi->num_queue_pairs; i++) 3078 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3079 i40e_free_rx_resources(vsi->rx_rings[i]); 3080 } 3081 3082 /** 3083 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3084 * @ring: The Tx ring to configure 3085 * 3086 * This enables/disables XPS for a given Tx descriptor ring 3087 * based on the TCs enabled for the VSI that ring belongs to. 3088 **/ 3089 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3090 { 3091 int cpu; 3092 3093 if (!ring->q_vector || !ring->netdev || ring->ch) 3094 return; 3095 3096 /* We only initialize XPS once, so as not to overwrite user settings */ 3097 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3098 return; 3099 3100 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3101 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3102 ring->queue_index); 3103 } 3104 3105 /** 3106 * i40e_xsk_umem - Retrieve the AF_XDP ZC if XDP and ZC is enabled 3107 * @ring: The Tx or Rx ring 3108 * 3109 * Returns the UMEM or NULL. 3110 **/ 3111 static struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring) 3112 { 3113 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi); 3114 int qid = ring->queue_index; 3115 3116 if (ring_is_xdp(ring)) 3117 qid -= ring->vsi->alloc_queue_pairs; 3118 3119 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps)) 3120 return NULL; 3121 3122 return xdp_get_umem_from_qid(ring->vsi->netdev, qid); 3123 } 3124 3125 /** 3126 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3127 * @ring: The Tx ring to configure 3128 * 3129 * Configure the Tx descriptor ring in the HMC context. 3130 **/ 3131 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3132 { 3133 struct i40e_vsi *vsi = ring->vsi; 3134 u16 pf_q = vsi->base_queue + ring->queue_index; 3135 struct i40e_hw *hw = &vsi->back->hw; 3136 struct i40e_hmc_obj_txq tx_ctx; 3137 i40e_status err = 0; 3138 u32 qtx_ctl = 0; 3139 3140 if (ring_is_xdp(ring)) 3141 ring->xsk_umem = i40e_xsk_umem(ring); 3142 3143 /* some ATR related tx ring init */ 3144 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3145 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3146 ring->atr_count = 0; 3147 } else { 3148 ring->atr_sample_rate = 0; 3149 } 3150 3151 /* configure XPS */ 3152 i40e_config_xps_tx_ring(ring); 3153 3154 /* clear the context structure first */ 3155 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3156 3157 tx_ctx.new_context = 1; 3158 tx_ctx.base = (ring->dma / 128); 3159 tx_ctx.qlen = ring->count; 3160 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3161 I40E_FLAG_FD_ATR_ENABLED)); 3162 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3163 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3164 if (vsi->type != I40E_VSI_FDIR) 3165 tx_ctx.head_wb_ena = 1; 3166 tx_ctx.head_wb_addr = ring->dma + 3167 (ring->count * sizeof(struct i40e_tx_desc)); 3168 3169 /* As part of VSI creation/update, FW allocates certain 3170 * Tx arbitration queue sets for each TC enabled for 3171 * the VSI. The FW returns the handles to these queue 3172 * sets as part of the response buffer to Add VSI, 3173 * Update VSI, etc. AQ commands. It is expected that 3174 * these queue set handles be associated with the Tx 3175 * queues by the driver as part of the TX queue context 3176 * initialization. This has to be done regardless of 3177 * DCB as by default everything is mapped to TC0. 3178 */ 3179 3180 if (ring->ch) 3181 tx_ctx.rdylist = 3182 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3183 3184 else 3185 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3186 3187 tx_ctx.rdylist_act = 0; 3188 3189 /* clear the context in the HMC */ 3190 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3191 if (err) { 3192 dev_info(&vsi->back->pdev->dev, 3193 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3194 ring->queue_index, pf_q, err); 3195 return -ENOMEM; 3196 } 3197 3198 /* set the context in the HMC */ 3199 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3200 if (err) { 3201 dev_info(&vsi->back->pdev->dev, 3202 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3203 ring->queue_index, pf_q, err); 3204 return -ENOMEM; 3205 } 3206 3207 /* Now associate this queue with this PCI function */ 3208 if (ring->ch) { 3209 if (ring->ch->type == I40E_VSI_VMDQ2) 3210 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3211 else 3212 return -EINVAL; 3213 3214 qtx_ctl |= (ring->ch->vsi_number << 3215 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3216 I40E_QTX_CTL_VFVM_INDX_MASK; 3217 } else { 3218 if (vsi->type == I40E_VSI_VMDQ2) { 3219 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3220 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3221 I40E_QTX_CTL_VFVM_INDX_MASK; 3222 } else { 3223 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3224 } 3225 } 3226 3227 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3228 I40E_QTX_CTL_PF_INDX_MASK); 3229 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3230 i40e_flush(hw); 3231 3232 /* cache tail off for easier writes later */ 3233 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3234 3235 return 0; 3236 } 3237 3238 /** 3239 * i40e_configure_rx_ring - Configure a receive ring context 3240 * @ring: The Rx ring to configure 3241 * 3242 * Configure the Rx descriptor ring in the HMC context. 3243 **/ 3244 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3245 { 3246 struct i40e_vsi *vsi = ring->vsi; 3247 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3248 u16 pf_q = vsi->base_queue + ring->queue_index; 3249 struct i40e_hw *hw = &vsi->back->hw; 3250 struct i40e_hmc_obj_rxq rx_ctx; 3251 i40e_status err = 0; 3252 bool ok; 3253 int ret; 3254 3255 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3256 3257 /* clear the context structure first */ 3258 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3259 3260 if (ring->vsi->type == I40E_VSI_MAIN) 3261 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 3262 3263 kfree(ring->rx_bi); 3264 ring->xsk_umem = i40e_xsk_umem(ring); 3265 if (ring->xsk_umem) { 3266 ret = i40e_alloc_rx_bi_zc(ring); 3267 if (ret) 3268 return ret; 3269 ring->rx_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem); 3270 /* For AF_XDP ZC, we disallow packets to span on 3271 * multiple buffers, thus letting us skip that 3272 * handling in the fast-path. 3273 */ 3274 chain_len = 1; 3275 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3276 MEM_TYPE_XSK_BUFF_POOL, 3277 NULL); 3278 if (ret) 3279 return ret; 3280 dev_info(&vsi->back->pdev->dev, 3281 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 3282 ring->queue_index); 3283 3284 } else { 3285 ret = i40e_alloc_rx_bi(ring); 3286 if (ret) 3287 return ret; 3288 ring->rx_buf_len = vsi->rx_buf_len; 3289 if (ring->vsi->type == I40E_VSI_MAIN) { 3290 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3291 MEM_TYPE_PAGE_SHARED, 3292 NULL); 3293 if (ret) 3294 return ret; 3295 } 3296 } 3297 3298 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3299 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3300 3301 rx_ctx.base = (ring->dma / 128); 3302 rx_ctx.qlen = ring->count; 3303 3304 /* use 32 byte descriptors */ 3305 rx_ctx.dsize = 1; 3306 3307 /* descriptor type is always zero 3308 * rx_ctx.dtype = 0; 3309 */ 3310 rx_ctx.hsplit_0 = 0; 3311 3312 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3313 if (hw->revision_id == 0) 3314 rx_ctx.lrxqthresh = 0; 3315 else 3316 rx_ctx.lrxqthresh = 1; 3317 rx_ctx.crcstrip = 1; 3318 rx_ctx.l2tsel = 1; 3319 /* this controls whether VLAN is stripped from inner headers */ 3320 rx_ctx.showiv = 0; 3321 /* set the prefena field to 1 because the manual says to */ 3322 rx_ctx.prefena = 1; 3323 3324 /* clear the context in the HMC */ 3325 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3326 if (err) { 3327 dev_info(&vsi->back->pdev->dev, 3328 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3329 ring->queue_index, pf_q, err); 3330 return -ENOMEM; 3331 } 3332 3333 /* set the context in the HMC */ 3334 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3335 if (err) { 3336 dev_info(&vsi->back->pdev->dev, 3337 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3338 ring->queue_index, pf_q, err); 3339 return -ENOMEM; 3340 } 3341 3342 /* configure Rx buffer alignment */ 3343 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3344 clear_ring_build_skb_enabled(ring); 3345 else 3346 set_ring_build_skb_enabled(ring); 3347 3348 /* cache tail for quicker writes, and clear the reg before use */ 3349 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3350 writel(0, ring->tail); 3351 3352 if (ring->xsk_umem) { 3353 xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq); 3354 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); 3355 } else { 3356 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3357 } 3358 if (!ok) { 3359 /* Log this in case the user has forgotten to give the kernel 3360 * any buffers, even later in the application. 3361 */ 3362 dev_info(&vsi->back->pdev->dev, 3363 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n", 3364 ring->xsk_umem ? "UMEM enabled " : "", 3365 ring->queue_index, pf_q); 3366 } 3367 3368 return 0; 3369 } 3370 3371 /** 3372 * i40e_vsi_configure_tx - Configure the VSI for Tx 3373 * @vsi: VSI structure describing this set of rings and resources 3374 * 3375 * Configure the Tx VSI for operation. 3376 **/ 3377 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3378 { 3379 int err = 0; 3380 u16 i; 3381 3382 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3383 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3384 3385 if (err || !i40e_enabled_xdp_vsi(vsi)) 3386 return err; 3387 3388 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3389 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3390 3391 return err; 3392 } 3393 3394 /** 3395 * i40e_vsi_configure_rx - Configure the VSI for Rx 3396 * @vsi: the VSI being configured 3397 * 3398 * Configure the Rx VSI for operation. 3399 **/ 3400 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3401 { 3402 int err = 0; 3403 u16 i; 3404 3405 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3406 vsi->max_frame = I40E_MAX_RXBUFFER; 3407 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3408 #if (PAGE_SIZE < 8192) 3409 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3410 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3411 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3412 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3413 #endif 3414 } else { 3415 vsi->max_frame = I40E_MAX_RXBUFFER; 3416 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3417 I40E_RXBUFFER_2048; 3418 } 3419 3420 /* set up individual rings */ 3421 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3422 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3423 3424 return err; 3425 } 3426 3427 /** 3428 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3429 * @vsi: ptr to the VSI 3430 **/ 3431 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3432 { 3433 struct i40e_ring *tx_ring, *rx_ring; 3434 u16 qoffset, qcount; 3435 int i, n; 3436 3437 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3438 /* Reset the TC information */ 3439 for (i = 0; i < vsi->num_queue_pairs; i++) { 3440 rx_ring = vsi->rx_rings[i]; 3441 tx_ring = vsi->tx_rings[i]; 3442 rx_ring->dcb_tc = 0; 3443 tx_ring->dcb_tc = 0; 3444 } 3445 return; 3446 } 3447 3448 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3449 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3450 continue; 3451 3452 qoffset = vsi->tc_config.tc_info[n].qoffset; 3453 qcount = vsi->tc_config.tc_info[n].qcount; 3454 for (i = qoffset; i < (qoffset + qcount); i++) { 3455 rx_ring = vsi->rx_rings[i]; 3456 tx_ring = vsi->tx_rings[i]; 3457 rx_ring->dcb_tc = n; 3458 tx_ring->dcb_tc = n; 3459 } 3460 } 3461 } 3462 3463 /** 3464 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3465 * @vsi: ptr to the VSI 3466 **/ 3467 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3468 { 3469 if (vsi->netdev) 3470 i40e_set_rx_mode(vsi->netdev); 3471 } 3472 3473 /** 3474 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3475 * @vsi: Pointer to the targeted VSI 3476 * 3477 * This function replays the hlist on the hw where all the SB Flow Director 3478 * filters were saved. 3479 **/ 3480 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3481 { 3482 struct i40e_fdir_filter *filter; 3483 struct i40e_pf *pf = vsi->back; 3484 struct hlist_node *node; 3485 3486 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3487 return; 3488 3489 /* Reset FDir counters as we're replaying all existing filters */ 3490 pf->fd_tcp4_filter_cnt = 0; 3491 pf->fd_udp4_filter_cnt = 0; 3492 pf->fd_sctp4_filter_cnt = 0; 3493 pf->fd_ip4_filter_cnt = 0; 3494 3495 hlist_for_each_entry_safe(filter, node, 3496 &pf->fdir_filter_list, fdir_node) { 3497 i40e_add_del_fdir(vsi, filter, true); 3498 } 3499 } 3500 3501 /** 3502 * i40e_vsi_configure - Set up the VSI for action 3503 * @vsi: the VSI being configured 3504 **/ 3505 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3506 { 3507 int err; 3508 3509 i40e_set_vsi_rx_mode(vsi); 3510 i40e_restore_vlan(vsi); 3511 i40e_vsi_config_dcb_rings(vsi); 3512 err = i40e_vsi_configure_tx(vsi); 3513 if (!err) 3514 err = i40e_vsi_configure_rx(vsi); 3515 3516 return err; 3517 } 3518 3519 /** 3520 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3521 * @vsi: the VSI being configured 3522 **/ 3523 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3524 { 3525 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3526 struct i40e_pf *pf = vsi->back; 3527 struct i40e_hw *hw = &pf->hw; 3528 u16 vector; 3529 int i, q; 3530 u32 qp; 3531 3532 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3533 * and PFINT_LNKLSTn registers, e.g.: 3534 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3535 */ 3536 qp = vsi->base_queue; 3537 vector = vsi->base_vector; 3538 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3539 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3540 3541 q_vector->rx.next_update = jiffies + 1; 3542 q_vector->rx.target_itr = 3543 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3544 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3545 q_vector->rx.target_itr >> 1); 3546 q_vector->rx.current_itr = q_vector->rx.target_itr; 3547 3548 q_vector->tx.next_update = jiffies + 1; 3549 q_vector->tx.target_itr = 3550 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3551 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3552 q_vector->tx.target_itr >> 1); 3553 q_vector->tx.current_itr = q_vector->tx.target_itr; 3554 3555 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3556 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3557 3558 /* Linked list for the queuepairs assigned to this vector */ 3559 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3560 for (q = 0; q < q_vector->num_ringpairs; q++) { 3561 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3562 u32 val; 3563 3564 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3565 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3566 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3567 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3568 (I40E_QUEUE_TYPE_TX << 3569 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3570 3571 wr32(hw, I40E_QINT_RQCTL(qp), val); 3572 3573 if (has_xdp) { 3574 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3575 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3576 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3577 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3578 (I40E_QUEUE_TYPE_TX << 3579 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3580 3581 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3582 } 3583 3584 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3585 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3586 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3587 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3588 (I40E_QUEUE_TYPE_RX << 3589 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3590 3591 /* Terminate the linked list */ 3592 if (q == (q_vector->num_ringpairs - 1)) 3593 val |= (I40E_QUEUE_END_OF_LIST << 3594 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3595 3596 wr32(hw, I40E_QINT_TQCTL(qp), val); 3597 qp++; 3598 } 3599 } 3600 3601 i40e_flush(hw); 3602 } 3603 3604 /** 3605 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3606 * @pf: pointer to private device data structure 3607 **/ 3608 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3609 { 3610 struct i40e_hw *hw = &pf->hw; 3611 u32 val; 3612 3613 /* clear things first */ 3614 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3615 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3616 3617 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3618 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3619 I40E_PFINT_ICR0_ENA_GRST_MASK | 3620 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3621 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3622 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3623 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3624 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3625 3626 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3627 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3628 3629 if (pf->flags & I40E_FLAG_PTP) 3630 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3631 3632 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3633 3634 /* SW_ITR_IDX = 0, but don't change INTENA */ 3635 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3636 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3637 3638 /* OTHER_ITR_IDX = 0 */ 3639 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3640 } 3641 3642 /** 3643 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3644 * @vsi: the VSI being configured 3645 **/ 3646 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3647 { 3648 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3649 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3650 struct i40e_pf *pf = vsi->back; 3651 struct i40e_hw *hw = &pf->hw; 3652 u32 val; 3653 3654 /* set the ITR configuration */ 3655 q_vector->rx.next_update = jiffies + 1; 3656 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 3657 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1); 3658 q_vector->rx.current_itr = q_vector->rx.target_itr; 3659 q_vector->tx.next_update = jiffies + 1; 3660 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 3661 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1); 3662 q_vector->tx.current_itr = q_vector->tx.target_itr; 3663 3664 i40e_enable_misc_int_causes(pf); 3665 3666 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3667 wr32(hw, I40E_PFINT_LNKLST0, 0); 3668 3669 /* Associate the queue pair to the vector and enable the queue int */ 3670 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3671 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3672 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3673 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3674 3675 wr32(hw, I40E_QINT_RQCTL(0), val); 3676 3677 if (i40e_enabled_xdp_vsi(vsi)) { 3678 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3679 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3680 (I40E_QUEUE_TYPE_TX 3681 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3682 3683 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3684 } 3685 3686 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3687 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3688 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3689 3690 wr32(hw, I40E_QINT_TQCTL(0), val); 3691 i40e_flush(hw); 3692 } 3693 3694 /** 3695 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3696 * @pf: board private structure 3697 **/ 3698 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3699 { 3700 struct i40e_hw *hw = &pf->hw; 3701 3702 wr32(hw, I40E_PFINT_DYN_CTL0, 3703 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3704 i40e_flush(hw); 3705 } 3706 3707 /** 3708 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3709 * @pf: board private structure 3710 **/ 3711 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 3712 { 3713 struct i40e_hw *hw = &pf->hw; 3714 u32 val; 3715 3716 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3717 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3718 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3719 3720 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3721 i40e_flush(hw); 3722 } 3723 3724 /** 3725 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3726 * @irq: interrupt number 3727 * @data: pointer to a q_vector 3728 **/ 3729 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3730 { 3731 struct i40e_q_vector *q_vector = data; 3732 3733 if (!q_vector->tx.ring && !q_vector->rx.ring) 3734 return IRQ_HANDLED; 3735 3736 napi_schedule_irqoff(&q_vector->napi); 3737 3738 return IRQ_HANDLED; 3739 } 3740 3741 /** 3742 * i40e_irq_affinity_notify - Callback for affinity changes 3743 * @notify: context as to what irq was changed 3744 * @mask: the new affinity mask 3745 * 3746 * This is a callback function used by the irq_set_affinity_notifier function 3747 * so that we may register to receive changes to the irq affinity masks. 3748 **/ 3749 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3750 const cpumask_t *mask) 3751 { 3752 struct i40e_q_vector *q_vector = 3753 container_of(notify, struct i40e_q_vector, affinity_notify); 3754 3755 cpumask_copy(&q_vector->affinity_mask, mask); 3756 } 3757 3758 /** 3759 * i40e_irq_affinity_release - Callback for affinity notifier release 3760 * @ref: internal core kernel usage 3761 * 3762 * This is a callback function used by the irq_set_affinity_notifier function 3763 * to inform the current notification subscriber that they will no longer 3764 * receive notifications. 3765 **/ 3766 static void i40e_irq_affinity_release(struct kref *ref) {} 3767 3768 /** 3769 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3770 * @vsi: the VSI being configured 3771 * @basename: name for the vector 3772 * 3773 * Allocates MSI-X vectors and requests interrupts from the kernel. 3774 **/ 3775 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3776 { 3777 int q_vectors = vsi->num_q_vectors; 3778 struct i40e_pf *pf = vsi->back; 3779 int base = vsi->base_vector; 3780 int rx_int_idx = 0; 3781 int tx_int_idx = 0; 3782 int vector, err; 3783 int irq_num; 3784 int cpu; 3785 3786 for (vector = 0; vector < q_vectors; vector++) { 3787 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3788 3789 irq_num = pf->msix_entries[base + vector].vector; 3790 3791 if (q_vector->tx.ring && q_vector->rx.ring) { 3792 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3793 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3794 tx_int_idx++; 3795 } else if (q_vector->rx.ring) { 3796 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3797 "%s-%s-%d", basename, "rx", rx_int_idx++); 3798 } else if (q_vector->tx.ring) { 3799 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3800 "%s-%s-%d", basename, "tx", tx_int_idx++); 3801 } else { 3802 /* skip this unused q_vector */ 3803 continue; 3804 } 3805 err = request_irq(irq_num, 3806 vsi->irq_handler, 3807 0, 3808 q_vector->name, 3809 q_vector); 3810 if (err) { 3811 dev_info(&pf->pdev->dev, 3812 "MSIX request_irq failed, error: %d\n", err); 3813 goto free_queue_irqs; 3814 } 3815 3816 /* register for affinity change notifications */ 3817 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3818 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3819 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3820 /* Spread affinity hints out across online CPUs. 3821 * 3822 * get_cpu_mask returns a static constant mask with 3823 * a permanent lifetime so it's ok to pass to 3824 * irq_set_affinity_hint without making a copy. 3825 */ 3826 cpu = cpumask_local_spread(q_vector->v_idx, -1); 3827 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 3828 } 3829 3830 vsi->irqs_ready = true; 3831 return 0; 3832 3833 free_queue_irqs: 3834 while (vector) { 3835 vector--; 3836 irq_num = pf->msix_entries[base + vector].vector; 3837 irq_set_affinity_notifier(irq_num, NULL); 3838 irq_set_affinity_hint(irq_num, NULL); 3839 free_irq(irq_num, &vsi->q_vectors[vector]); 3840 } 3841 return err; 3842 } 3843 3844 /** 3845 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3846 * @vsi: the VSI being un-configured 3847 **/ 3848 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3849 { 3850 struct i40e_pf *pf = vsi->back; 3851 struct i40e_hw *hw = &pf->hw; 3852 int base = vsi->base_vector; 3853 int i; 3854 3855 /* disable interrupt causation from each queue */ 3856 for (i = 0; i < vsi->num_queue_pairs; i++) { 3857 u32 val; 3858 3859 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3860 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3861 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3862 3863 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3864 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3865 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3866 3867 if (!i40e_enabled_xdp_vsi(vsi)) 3868 continue; 3869 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3870 } 3871 3872 /* disable each interrupt */ 3873 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3874 for (i = vsi->base_vector; 3875 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3876 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3877 3878 i40e_flush(hw); 3879 for (i = 0; i < vsi->num_q_vectors; i++) 3880 synchronize_irq(pf->msix_entries[i + base].vector); 3881 } else { 3882 /* Legacy and MSI mode - this stops all interrupt handling */ 3883 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3884 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3885 i40e_flush(hw); 3886 synchronize_irq(pf->pdev->irq); 3887 } 3888 } 3889 3890 /** 3891 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3892 * @vsi: the VSI being configured 3893 **/ 3894 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3895 { 3896 struct i40e_pf *pf = vsi->back; 3897 int i; 3898 3899 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3900 for (i = 0; i < vsi->num_q_vectors; i++) 3901 i40e_irq_dynamic_enable(vsi, i); 3902 } else { 3903 i40e_irq_dynamic_enable_icr0(pf); 3904 } 3905 3906 i40e_flush(&pf->hw); 3907 return 0; 3908 } 3909 3910 /** 3911 * i40e_free_misc_vector - Free the vector that handles non-queue events 3912 * @pf: board private structure 3913 **/ 3914 static void i40e_free_misc_vector(struct i40e_pf *pf) 3915 { 3916 /* Disable ICR 0 */ 3917 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3918 i40e_flush(&pf->hw); 3919 3920 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3921 synchronize_irq(pf->msix_entries[0].vector); 3922 free_irq(pf->msix_entries[0].vector, pf); 3923 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3924 } 3925 } 3926 3927 /** 3928 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3929 * @irq: interrupt number 3930 * @data: pointer to a q_vector 3931 * 3932 * This is the handler used for all MSI/Legacy interrupts, and deals 3933 * with both queue and non-queue interrupts. This is also used in 3934 * MSIX mode to handle the non-queue interrupts. 3935 **/ 3936 static irqreturn_t i40e_intr(int irq, void *data) 3937 { 3938 struct i40e_pf *pf = (struct i40e_pf *)data; 3939 struct i40e_hw *hw = &pf->hw; 3940 irqreturn_t ret = IRQ_NONE; 3941 u32 icr0, icr0_remaining; 3942 u32 val, ena_mask; 3943 3944 icr0 = rd32(hw, I40E_PFINT_ICR0); 3945 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3946 3947 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3948 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3949 goto enable_intr; 3950 3951 /* if interrupt but no bits showing, must be SWINT */ 3952 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3953 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3954 pf->sw_int_count++; 3955 3956 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3957 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3958 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3959 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3960 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3961 } 3962 3963 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3964 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3965 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3966 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3967 3968 /* We do not have a way to disarm Queue causes while leaving 3969 * interrupt enabled for all other causes, ideally 3970 * interrupt should be disabled while we are in NAPI but 3971 * this is not a performance path and napi_schedule() 3972 * can deal with rescheduling. 3973 */ 3974 if (!test_bit(__I40E_DOWN, pf->state)) 3975 napi_schedule_irqoff(&q_vector->napi); 3976 } 3977 3978 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3979 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3980 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 3981 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3982 } 3983 3984 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3985 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3986 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 3987 } 3988 3989 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3990 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3991 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 3992 } 3993 3994 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3995 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 3996 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 3997 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3998 val = rd32(hw, I40E_GLGEN_RSTAT); 3999 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 4000 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 4001 if (val == I40E_RESET_CORER) { 4002 pf->corer_count++; 4003 } else if (val == I40E_RESET_GLOBR) { 4004 pf->globr_count++; 4005 } else if (val == I40E_RESET_EMPR) { 4006 pf->empr_count++; 4007 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 4008 } 4009 } 4010 4011 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 4012 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 4013 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 4014 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 4015 rd32(hw, I40E_PFHMC_ERRORINFO), 4016 rd32(hw, I40E_PFHMC_ERRORDATA)); 4017 } 4018 4019 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 4020 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 4021 4022 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 4023 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4024 i40e_ptp_tx_hwtstamp(pf); 4025 } 4026 } 4027 4028 /* If a critical error is pending we have no choice but to reset the 4029 * device. 4030 * Report and mask out any remaining unexpected interrupts. 4031 */ 4032 icr0_remaining = icr0 & ena_mask; 4033 if (icr0_remaining) { 4034 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 4035 icr0_remaining); 4036 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 4037 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 4038 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 4039 dev_info(&pf->pdev->dev, "device will be reset\n"); 4040 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 4041 i40e_service_event_schedule(pf); 4042 } 4043 ena_mask &= ~icr0_remaining; 4044 } 4045 ret = IRQ_HANDLED; 4046 4047 enable_intr: 4048 /* re-enable interrupt causes */ 4049 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 4050 if (!test_bit(__I40E_DOWN, pf->state) || 4051 test_bit(__I40E_RECOVERY_MODE, pf->state)) { 4052 i40e_service_event_schedule(pf); 4053 i40e_irq_dynamic_enable_icr0(pf); 4054 } 4055 4056 return ret; 4057 } 4058 4059 /** 4060 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 4061 * @tx_ring: tx ring to clean 4062 * @budget: how many cleans we're allowed 4063 * 4064 * Returns true if there's any budget left (e.g. the clean is finished) 4065 **/ 4066 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 4067 { 4068 struct i40e_vsi *vsi = tx_ring->vsi; 4069 u16 i = tx_ring->next_to_clean; 4070 struct i40e_tx_buffer *tx_buf; 4071 struct i40e_tx_desc *tx_desc; 4072 4073 tx_buf = &tx_ring->tx_bi[i]; 4074 tx_desc = I40E_TX_DESC(tx_ring, i); 4075 i -= tx_ring->count; 4076 4077 do { 4078 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4079 4080 /* if next_to_watch is not set then there is no work pending */ 4081 if (!eop_desc) 4082 break; 4083 4084 /* prevent any other reads prior to eop_desc */ 4085 smp_rmb(); 4086 4087 /* if the descriptor isn't done, no work yet to do */ 4088 if (!(eop_desc->cmd_type_offset_bsz & 4089 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4090 break; 4091 4092 /* clear next_to_watch to prevent false hangs */ 4093 tx_buf->next_to_watch = NULL; 4094 4095 tx_desc->buffer_addr = 0; 4096 tx_desc->cmd_type_offset_bsz = 0; 4097 /* move past filter desc */ 4098 tx_buf++; 4099 tx_desc++; 4100 i++; 4101 if (unlikely(!i)) { 4102 i -= tx_ring->count; 4103 tx_buf = tx_ring->tx_bi; 4104 tx_desc = I40E_TX_DESC(tx_ring, 0); 4105 } 4106 /* unmap skb header data */ 4107 dma_unmap_single(tx_ring->dev, 4108 dma_unmap_addr(tx_buf, dma), 4109 dma_unmap_len(tx_buf, len), 4110 DMA_TO_DEVICE); 4111 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4112 kfree(tx_buf->raw_buf); 4113 4114 tx_buf->raw_buf = NULL; 4115 tx_buf->tx_flags = 0; 4116 tx_buf->next_to_watch = NULL; 4117 dma_unmap_len_set(tx_buf, len, 0); 4118 tx_desc->buffer_addr = 0; 4119 tx_desc->cmd_type_offset_bsz = 0; 4120 4121 /* move us past the eop_desc for start of next FD desc */ 4122 tx_buf++; 4123 tx_desc++; 4124 i++; 4125 if (unlikely(!i)) { 4126 i -= tx_ring->count; 4127 tx_buf = tx_ring->tx_bi; 4128 tx_desc = I40E_TX_DESC(tx_ring, 0); 4129 } 4130 4131 /* update budget accounting */ 4132 budget--; 4133 } while (likely(budget)); 4134 4135 i += tx_ring->count; 4136 tx_ring->next_to_clean = i; 4137 4138 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4139 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4140 4141 return budget > 0; 4142 } 4143 4144 /** 4145 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4146 * @irq: interrupt number 4147 * @data: pointer to a q_vector 4148 **/ 4149 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4150 { 4151 struct i40e_q_vector *q_vector = data; 4152 struct i40e_vsi *vsi; 4153 4154 if (!q_vector->tx.ring) 4155 return IRQ_HANDLED; 4156 4157 vsi = q_vector->tx.ring->vsi; 4158 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4159 4160 return IRQ_HANDLED; 4161 } 4162 4163 /** 4164 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4165 * @vsi: the VSI being configured 4166 * @v_idx: vector index 4167 * @qp_idx: queue pair index 4168 **/ 4169 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4170 { 4171 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4172 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4173 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4174 4175 tx_ring->q_vector = q_vector; 4176 tx_ring->next = q_vector->tx.ring; 4177 q_vector->tx.ring = tx_ring; 4178 q_vector->tx.count++; 4179 4180 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4181 if (i40e_enabled_xdp_vsi(vsi)) { 4182 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4183 4184 xdp_ring->q_vector = q_vector; 4185 xdp_ring->next = q_vector->tx.ring; 4186 q_vector->tx.ring = xdp_ring; 4187 q_vector->tx.count++; 4188 } 4189 4190 rx_ring->q_vector = q_vector; 4191 rx_ring->next = q_vector->rx.ring; 4192 q_vector->rx.ring = rx_ring; 4193 q_vector->rx.count++; 4194 } 4195 4196 /** 4197 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4198 * @vsi: the VSI being configured 4199 * 4200 * This function maps descriptor rings to the queue-specific vectors 4201 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4202 * one vector per queue pair, but on a constrained vector budget, we 4203 * group the queue pairs as "efficiently" as possible. 4204 **/ 4205 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4206 { 4207 int qp_remaining = vsi->num_queue_pairs; 4208 int q_vectors = vsi->num_q_vectors; 4209 int num_ringpairs; 4210 int v_start = 0; 4211 int qp_idx = 0; 4212 4213 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4214 * group them so there are multiple queues per vector. 4215 * It is also important to go through all the vectors available to be 4216 * sure that if we don't use all the vectors, that the remaining vectors 4217 * are cleared. This is especially important when decreasing the 4218 * number of queues in use. 4219 */ 4220 for (; v_start < q_vectors; v_start++) { 4221 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4222 4223 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4224 4225 q_vector->num_ringpairs = num_ringpairs; 4226 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4227 4228 q_vector->rx.count = 0; 4229 q_vector->tx.count = 0; 4230 q_vector->rx.ring = NULL; 4231 q_vector->tx.ring = NULL; 4232 4233 while (num_ringpairs--) { 4234 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4235 qp_idx++; 4236 qp_remaining--; 4237 } 4238 } 4239 } 4240 4241 /** 4242 * i40e_vsi_request_irq - Request IRQ from the OS 4243 * @vsi: the VSI being configured 4244 * @basename: name for the vector 4245 **/ 4246 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4247 { 4248 struct i40e_pf *pf = vsi->back; 4249 int err; 4250 4251 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4252 err = i40e_vsi_request_irq_msix(vsi, basename); 4253 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4254 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4255 pf->int_name, pf); 4256 else 4257 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4258 pf->int_name, pf); 4259 4260 if (err) 4261 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4262 4263 return err; 4264 } 4265 4266 #ifdef CONFIG_NET_POLL_CONTROLLER 4267 /** 4268 * i40e_netpoll - A Polling 'interrupt' handler 4269 * @netdev: network interface device structure 4270 * 4271 * This is used by netconsole to send skbs without having to re-enable 4272 * interrupts. It's not called while the normal interrupt routine is executing. 4273 **/ 4274 static void i40e_netpoll(struct net_device *netdev) 4275 { 4276 struct i40e_netdev_priv *np = netdev_priv(netdev); 4277 struct i40e_vsi *vsi = np->vsi; 4278 struct i40e_pf *pf = vsi->back; 4279 int i; 4280 4281 /* if interface is down do nothing */ 4282 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4283 return; 4284 4285 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4286 for (i = 0; i < vsi->num_q_vectors; i++) 4287 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4288 } else { 4289 i40e_intr(pf->pdev->irq, netdev); 4290 } 4291 } 4292 #endif 4293 4294 #define I40E_QTX_ENA_WAIT_COUNT 50 4295 4296 /** 4297 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4298 * @pf: the PF being configured 4299 * @pf_q: the PF queue 4300 * @enable: enable or disable state of the queue 4301 * 4302 * This routine will wait for the given Tx queue of the PF to reach the 4303 * enabled or disabled state. 4304 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4305 * multiple retries; else will return 0 in case of success. 4306 **/ 4307 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4308 { 4309 int i; 4310 u32 tx_reg; 4311 4312 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4313 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4314 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4315 break; 4316 4317 usleep_range(10, 20); 4318 } 4319 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4320 return -ETIMEDOUT; 4321 4322 return 0; 4323 } 4324 4325 /** 4326 * i40e_control_tx_q - Start or stop a particular Tx queue 4327 * @pf: the PF structure 4328 * @pf_q: the PF queue to configure 4329 * @enable: start or stop the queue 4330 * 4331 * This function enables or disables a single queue. Note that any delay 4332 * required after the operation is expected to be handled by the caller of 4333 * this function. 4334 **/ 4335 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4336 { 4337 struct i40e_hw *hw = &pf->hw; 4338 u32 tx_reg; 4339 int i; 4340 4341 /* warn the TX unit of coming changes */ 4342 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4343 if (!enable) 4344 usleep_range(10, 20); 4345 4346 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4347 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4348 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4349 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4350 break; 4351 usleep_range(1000, 2000); 4352 } 4353 4354 /* Skip if the queue is already in the requested state */ 4355 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4356 return; 4357 4358 /* turn on/off the queue */ 4359 if (enable) { 4360 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4361 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4362 } else { 4363 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4364 } 4365 4366 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4367 } 4368 4369 /** 4370 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4371 * @seid: VSI SEID 4372 * @pf: the PF structure 4373 * @pf_q: the PF queue to configure 4374 * @is_xdp: true if the queue is used for XDP 4375 * @enable: start or stop the queue 4376 **/ 4377 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4378 bool is_xdp, bool enable) 4379 { 4380 int ret; 4381 4382 i40e_control_tx_q(pf, pf_q, enable); 4383 4384 /* wait for the change to finish */ 4385 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4386 if (ret) { 4387 dev_info(&pf->pdev->dev, 4388 "VSI seid %d %sTx ring %d %sable timeout\n", 4389 seid, (is_xdp ? "XDP " : ""), pf_q, 4390 (enable ? "en" : "dis")); 4391 } 4392 4393 return ret; 4394 } 4395 4396 /** 4397 * i40e_vsi_control_tx - Start or stop a VSI's rings 4398 * @vsi: the VSI being configured 4399 * @enable: start or stop the rings 4400 **/ 4401 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4402 { 4403 struct i40e_pf *pf = vsi->back; 4404 int i, pf_q, ret = 0; 4405 4406 pf_q = vsi->base_queue; 4407 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4408 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4409 pf_q, 4410 false /*is xdp*/, enable); 4411 if (ret) 4412 break; 4413 4414 if (!i40e_enabled_xdp_vsi(vsi)) 4415 continue; 4416 4417 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4418 pf_q + vsi->alloc_queue_pairs, 4419 true /*is xdp*/, enable); 4420 if (ret) 4421 break; 4422 } 4423 return ret; 4424 } 4425 4426 /** 4427 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4428 * @pf: the PF being configured 4429 * @pf_q: the PF queue 4430 * @enable: enable or disable state of the queue 4431 * 4432 * This routine will wait for the given Rx queue of the PF to reach the 4433 * enabled or disabled state. 4434 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4435 * multiple retries; else will return 0 in case of success. 4436 **/ 4437 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4438 { 4439 int i; 4440 u32 rx_reg; 4441 4442 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4443 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4444 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4445 break; 4446 4447 usleep_range(10, 20); 4448 } 4449 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4450 return -ETIMEDOUT; 4451 4452 return 0; 4453 } 4454 4455 /** 4456 * i40e_control_rx_q - Start or stop a particular Rx queue 4457 * @pf: the PF structure 4458 * @pf_q: the PF queue to configure 4459 * @enable: start or stop the queue 4460 * 4461 * This function enables or disables a single queue. Note that 4462 * any delay required after the operation is expected to be 4463 * handled by the caller of this function. 4464 **/ 4465 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4466 { 4467 struct i40e_hw *hw = &pf->hw; 4468 u32 rx_reg; 4469 int i; 4470 4471 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4472 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4473 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4474 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4475 break; 4476 usleep_range(1000, 2000); 4477 } 4478 4479 /* Skip if the queue is already in the requested state */ 4480 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4481 return; 4482 4483 /* turn on/off the queue */ 4484 if (enable) 4485 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4486 else 4487 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4488 4489 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4490 } 4491 4492 /** 4493 * i40e_control_wait_rx_q 4494 * @pf: the PF structure 4495 * @pf_q: queue being configured 4496 * @enable: start or stop the rings 4497 * 4498 * This function enables or disables a single queue along with waiting 4499 * for the change to finish. The caller of this function should handle 4500 * the delays needed in the case of disabling queues. 4501 **/ 4502 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4503 { 4504 int ret = 0; 4505 4506 i40e_control_rx_q(pf, pf_q, enable); 4507 4508 /* wait for the change to finish */ 4509 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4510 if (ret) 4511 return ret; 4512 4513 return ret; 4514 } 4515 4516 /** 4517 * i40e_vsi_control_rx - Start or stop a VSI's rings 4518 * @vsi: the VSI being configured 4519 * @enable: start or stop the rings 4520 **/ 4521 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4522 { 4523 struct i40e_pf *pf = vsi->back; 4524 int i, pf_q, ret = 0; 4525 4526 pf_q = vsi->base_queue; 4527 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4528 ret = i40e_control_wait_rx_q(pf, pf_q, enable); 4529 if (ret) { 4530 dev_info(&pf->pdev->dev, 4531 "VSI seid %d Rx ring %d %sable timeout\n", 4532 vsi->seid, pf_q, (enable ? "en" : "dis")); 4533 break; 4534 } 4535 } 4536 4537 /* Due to HW errata, on Rx disable only, the register can indicate done 4538 * before it really is. Needs 50ms to be sure 4539 */ 4540 if (!enable) 4541 mdelay(50); 4542 4543 return ret; 4544 } 4545 4546 /** 4547 * i40e_vsi_start_rings - Start a VSI's rings 4548 * @vsi: the VSI being configured 4549 **/ 4550 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4551 { 4552 int ret = 0; 4553 4554 /* do rx first for enable and last for disable */ 4555 ret = i40e_vsi_control_rx(vsi, true); 4556 if (ret) 4557 return ret; 4558 ret = i40e_vsi_control_tx(vsi, true); 4559 4560 return ret; 4561 } 4562 4563 /** 4564 * i40e_vsi_stop_rings - Stop a VSI's rings 4565 * @vsi: the VSI being configured 4566 **/ 4567 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4568 { 4569 /* When port TX is suspended, don't wait */ 4570 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4571 return i40e_vsi_stop_rings_no_wait(vsi); 4572 4573 /* do rx first for enable and last for disable 4574 * Ignore return value, we need to shutdown whatever we can 4575 */ 4576 i40e_vsi_control_tx(vsi, false); 4577 i40e_vsi_control_rx(vsi, false); 4578 } 4579 4580 /** 4581 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4582 * @vsi: the VSI being shutdown 4583 * 4584 * This function stops all the rings for a VSI but does not delay to verify 4585 * that rings have been disabled. It is expected that the caller is shutting 4586 * down multiple VSIs at once and will delay together for all the VSIs after 4587 * initiating the shutdown. This is particularly useful for shutting down lots 4588 * of VFs together. Otherwise, a large delay can be incurred while configuring 4589 * each VSI in serial. 4590 **/ 4591 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4592 { 4593 struct i40e_pf *pf = vsi->back; 4594 int i, pf_q; 4595 4596 pf_q = vsi->base_queue; 4597 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4598 i40e_control_tx_q(pf, pf_q, false); 4599 i40e_control_rx_q(pf, pf_q, false); 4600 } 4601 } 4602 4603 /** 4604 * i40e_vsi_free_irq - Free the irq association with the OS 4605 * @vsi: the VSI being configured 4606 **/ 4607 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4608 { 4609 struct i40e_pf *pf = vsi->back; 4610 struct i40e_hw *hw = &pf->hw; 4611 int base = vsi->base_vector; 4612 u32 val, qp; 4613 int i; 4614 4615 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4616 if (!vsi->q_vectors) 4617 return; 4618 4619 if (!vsi->irqs_ready) 4620 return; 4621 4622 vsi->irqs_ready = false; 4623 for (i = 0; i < vsi->num_q_vectors; i++) { 4624 int irq_num; 4625 u16 vector; 4626 4627 vector = i + base; 4628 irq_num = pf->msix_entries[vector].vector; 4629 4630 /* free only the irqs that were actually requested */ 4631 if (!vsi->q_vectors[i] || 4632 !vsi->q_vectors[i]->num_ringpairs) 4633 continue; 4634 4635 /* clear the affinity notifier in the IRQ descriptor */ 4636 irq_set_affinity_notifier(irq_num, NULL); 4637 /* remove our suggested affinity mask for this IRQ */ 4638 irq_set_affinity_hint(irq_num, NULL); 4639 synchronize_irq(irq_num); 4640 free_irq(irq_num, vsi->q_vectors[i]); 4641 4642 /* Tear down the interrupt queue link list 4643 * 4644 * We know that they come in pairs and always 4645 * the Rx first, then the Tx. To clear the 4646 * link list, stick the EOL value into the 4647 * next_q field of the registers. 4648 */ 4649 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4650 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4651 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4652 val |= I40E_QUEUE_END_OF_LIST 4653 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4654 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4655 4656 while (qp != I40E_QUEUE_END_OF_LIST) { 4657 u32 next; 4658 4659 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4660 4661 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4662 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4663 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4664 I40E_QINT_RQCTL_INTEVENT_MASK); 4665 4666 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4667 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4668 4669 wr32(hw, I40E_QINT_RQCTL(qp), val); 4670 4671 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4672 4673 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4674 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4675 4676 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4677 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4678 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4679 I40E_QINT_TQCTL_INTEVENT_MASK); 4680 4681 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4682 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4683 4684 wr32(hw, I40E_QINT_TQCTL(qp), val); 4685 qp = next; 4686 } 4687 } 4688 } else { 4689 free_irq(pf->pdev->irq, pf); 4690 4691 val = rd32(hw, I40E_PFINT_LNKLST0); 4692 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4693 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4694 val |= I40E_QUEUE_END_OF_LIST 4695 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4696 wr32(hw, I40E_PFINT_LNKLST0, val); 4697 4698 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4699 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4700 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4701 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4702 I40E_QINT_RQCTL_INTEVENT_MASK); 4703 4704 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4705 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4706 4707 wr32(hw, I40E_QINT_RQCTL(qp), val); 4708 4709 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4710 4711 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4712 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4713 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4714 I40E_QINT_TQCTL_INTEVENT_MASK); 4715 4716 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4717 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4718 4719 wr32(hw, I40E_QINT_TQCTL(qp), val); 4720 } 4721 } 4722 4723 /** 4724 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4725 * @vsi: the VSI being configured 4726 * @v_idx: Index of vector to be freed 4727 * 4728 * This function frees the memory allocated to the q_vector. In addition if 4729 * NAPI is enabled it will delete any references to the NAPI struct prior 4730 * to freeing the q_vector. 4731 **/ 4732 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4733 { 4734 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4735 struct i40e_ring *ring; 4736 4737 if (!q_vector) 4738 return; 4739 4740 /* disassociate q_vector from rings */ 4741 i40e_for_each_ring(ring, q_vector->tx) 4742 ring->q_vector = NULL; 4743 4744 i40e_for_each_ring(ring, q_vector->rx) 4745 ring->q_vector = NULL; 4746 4747 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4748 if (vsi->netdev) 4749 netif_napi_del(&q_vector->napi); 4750 4751 vsi->q_vectors[v_idx] = NULL; 4752 4753 kfree_rcu(q_vector, rcu); 4754 } 4755 4756 /** 4757 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4758 * @vsi: the VSI being un-configured 4759 * 4760 * This frees the memory allocated to the q_vectors and 4761 * deletes references to the NAPI struct. 4762 **/ 4763 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4764 { 4765 int v_idx; 4766 4767 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4768 i40e_free_q_vector(vsi, v_idx); 4769 } 4770 4771 /** 4772 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4773 * @pf: board private structure 4774 **/ 4775 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4776 { 4777 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4778 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4779 pci_disable_msix(pf->pdev); 4780 kfree(pf->msix_entries); 4781 pf->msix_entries = NULL; 4782 kfree(pf->irq_pile); 4783 pf->irq_pile = NULL; 4784 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4785 pci_disable_msi(pf->pdev); 4786 } 4787 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4788 } 4789 4790 /** 4791 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4792 * @pf: board private structure 4793 * 4794 * We go through and clear interrupt specific resources and reset the structure 4795 * to pre-load conditions 4796 **/ 4797 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4798 { 4799 int i; 4800 4801 i40e_free_misc_vector(pf); 4802 4803 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4804 I40E_IWARP_IRQ_PILE_ID); 4805 4806 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4807 for (i = 0; i < pf->num_alloc_vsi; i++) 4808 if (pf->vsi[i]) 4809 i40e_vsi_free_q_vectors(pf->vsi[i]); 4810 i40e_reset_interrupt_capability(pf); 4811 } 4812 4813 /** 4814 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4815 * @vsi: the VSI being configured 4816 **/ 4817 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4818 { 4819 int q_idx; 4820 4821 if (!vsi->netdev) 4822 return; 4823 4824 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4825 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4826 4827 if (q_vector->rx.ring || q_vector->tx.ring) 4828 napi_enable(&q_vector->napi); 4829 } 4830 } 4831 4832 /** 4833 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4834 * @vsi: the VSI being configured 4835 **/ 4836 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4837 { 4838 int q_idx; 4839 4840 if (!vsi->netdev) 4841 return; 4842 4843 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4844 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4845 4846 if (q_vector->rx.ring || q_vector->tx.ring) 4847 napi_disable(&q_vector->napi); 4848 } 4849 } 4850 4851 /** 4852 * i40e_vsi_close - Shut down a VSI 4853 * @vsi: the vsi to be quelled 4854 **/ 4855 static void i40e_vsi_close(struct i40e_vsi *vsi) 4856 { 4857 struct i40e_pf *pf = vsi->back; 4858 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4859 i40e_down(vsi); 4860 i40e_vsi_free_irq(vsi); 4861 i40e_vsi_free_tx_resources(vsi); 4862 i40e_vsi_free_rx_resources(vsi); 4863 vsi->current_netdev_flags = 0; 4864 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 4865 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4866 set_bit(__I40E_CLIENT_RESET, pf->state); 4867 } 4868 4869 /** 4870 * i40e_quiesce_vsi - Pause a given VSI 4871 * @vsi: the VSI being paused 4872 **/ 4873 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4874 { 4875 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4876 return; 4877 4878 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4879 if (vsi->netdev && netif_running(vsi->netdev)) 4880 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4881 else 4882 i40e_vsi_close(vsi); 4883 } 4884 4885 /** 4886 * i40e_unquiesce_vsi - Resume a given VSI 4887 * @vsi: the VSI being resumed 4888 **/ 4889 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4890 { 4891 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4892 return; 4893 4894 if (vsi->netdev && netif_running(vsi->netdev)) 4895 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4896 else 4897 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4898 } 4899 4900 /** 4901 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4902 * @pf: the PF 4903 **/ 4904 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4905 { 4906 int v; 4907 4908 for (v = 0; v < pf->num_alloc_vsi; v++) { 4909 if (pf->vsi[v]) 4910 i40e_quiesce_vsi(pf->vsi[v]); 4911 } 4912 } 4913 4914 /** 4915 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4916 * @pf: the PF 4917 **/ 4918 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4919 { 4920 int v; 4921 4922 for (v = 0; v < pf->num_alloc_vsi; v++) { 4923 if (pf->vsi[v]) 4924 i40e_unquiesce_vsi(pf->vsi[v]); 4925 } 4926 } 4927 4928 /** 4929 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4930 * @vsi: the VSI being configured 4931 * 4932 * Wait until all queues on a given VSI have been disabled. 4933 **/ 4934 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4935 { 4936 struct i40e_pf *pf = vsi->back; 4937 int i, pf_q, ret; 4938 4939 pf_q = vsi->base_queue; 4940 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4941 /* Check and wait for the Tx queue */ 4942 ret = i40e_pf_txq_wait(pf, pf_q, false); 4943 if (ret) { 4944 dev_info(&pf->pdev->dev, 4945 "VSI seid %d Tx ring %d disable timeout\n", 4946 vsi->seid, pf_q); 4947 return ret; 4948 } 4949 4950 if (!i40e_enabled_xdp_vsi(vsi)) 4951 goto wait_rx; 4952 4953 /* Check and wait for the XDP Tx queue */ 4954 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4955 false); 4956 if (ret) { 4957 dev_info(&pf->pdev->dev, 4958 "VSI seid %d XDP Tx ring %d disable timeout\n", 4959 vsi->seid, pf_q); 4960 return ret; 4961 } 4962 wait_rx: 4963 /* Check and wait for the Rx queue */ 4964 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4965 if (ret) { 4966 dev_info(&pf->pdev->dev, 4967 "VSI seid %d Rx ring %d disable timeout\n", 4968 vsi->seid, pf_q); 4969 return ret; 4970 } 4971 } 4972 4973 return 0; 4974 } 4975 4976 #ifdef CONFIG_I40E_DCB 4977 /** 4978 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4979 * @pf: the PF 4980 * 4981 * This function waits for the queues to be in disabled state for all the 4982 * VSIs that are managed by this PF. 4983 **/ 4984 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4985 { 4986 int v, ret = 0; 4987 4988 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4989 if (pf->vsi[v]) { 4990 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4991 if (ret) 4992 break; 4993 } 4994 } 4995 4996 return ret; 4997 } 4998 4999 #endif 5000 5001 /** 5002 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 5003 * @pf: pointer to PF 5004 * 5005 * Get TC map for ISCSI PF type that will include iSCSI TC 5006 * and LAN TC. 5007 **/ 5008 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 5009 { 5010 struct i40e_dcb_app_priority_table app; 5011 struct i40e_hw *hw = &pf->hw; 5012 u8 enabled_tc = 1; /* TC0 is always enabled */ 5013 u8 tc, i; 5014 /* Get the iSCSI APP TLV */ 5015 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5016 5017 for (i = 0; i < dcbcfg->numapps; i++) { 5018 app = dcbcfg->app[i]; 5019 if (app.selector == I40E_APP_SEL_TCPIP && 5020 app.protocolid == I40E_APP_PROTOID_ISCSI) { 5021 tc = dcbcfg->etscfg.prioritytable[app.priority]; 5022 enabled_tc |= BIT(tc); 5023 break; 5024 } 5025 } 5026 5027 return enabled_tc; 5028 } 5029 5030 /** 5031 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5032 * @dcbcfg: the corresponding DCBx configuration structure 5033 * 5034 * Return the number of TCs from given DCBx configuration 5035 **/ 5036 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5037 { 5038 int i, tc_unused = 0; 5039 u8 num_tc = 0; 5040 u8 ret = 0; 5041 5042 /* Scan the ETS Config Priority Table to find 5043 * traffic class enabled for a given priority 5044 * and create a bitmask of enabled TCs 5045 */ 5046 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5047 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5048 5049 /* Now scan the bitmask to check for 5050 * contiguous TCs starting with TC0 5051 */ 5052 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5053 if (num_tc & BIT(i)) { 5054 if (!tc_unused) { 5055 ret++; 5056 } else { 5057 pr_err("Non-contiguous TC - Disabling DCB\n"); 5058 return 1; 5059 } 5060 } else { 5061 tc_unused = 1; 5062 } 5063 } 5064 5065 /* There is always at least TC0 */ 5066 if (!ret) 5067 ret = 1; 5068 5069 return ret; 5070 } 5071 5072 /** 5073 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5074 * @dcbcfg: the corresponding DCBx configuration structure 5075 * 5076 * Query the current DCB configuration and return the number of 5077 * traffic classes enabled from the given DCBX config 5078 **/ 5079 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5080 { 5081 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5082 u8 enabled_tc = 1; 5083 u8 i; 5084 5085 for (i = 0; i < num_tc; i++) 5086 enabled_tc |= BIT(i); 5087 5088 return enabled_tc; 5089 } 5090 5091 /** 5092 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5093 * @pf: PF being queried 5094 * 5095 * Query the current MQPRIO configuration and return the number of 5096 * traffic classes enabled. 5097 **/ 5098 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5099 { 5100 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 5101 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5102 u8 enabled_tc = 1, i; 5103 5104 for (i = 1; i < num_tc; i++) 5105 enabled_tc |= BIT(i); 5106 return enabled_tc; 5107 } 5108 5109 /** 5110 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5111 * @pf: PF being queried 5112 * 5113 * Return number of traffic classes enabled for the given PF 5114 **/ 5115 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5116 { 5117 struct i40e_hw *hw = &pf->hw; 5118 u8 i, enabled_tc = 1; 5119 u8 num_tc = 0; 5120 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5121 5122 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5123 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5124 5125 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5126 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5127 return 1; 5128 5129 /* SFP mode will be enabled for all TCs on port */ 5130 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5131 return i40e_dcb_get_num_tc(dcbcfg); 5132 5133 /* MFP mode return count of enabled TCs for this PF */ 5134 if (pf->hw.func_caps.iscsi) 5135 enabled_tc = i40e_get_iscsi_tc_map(pf); 5136 else 5137 return 1; /* Only TC0 */ 5138 5139 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5140 if (enabled_tc & BIT(i)) 5141 num_tc++; 5142 } 5143 return num_tc; 5144 } 5145 5146 /** 5147 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 5148 * @pf: PF being queried 5149 * 5150 * Return a bitmap for enabled traffic classes for this PF. 5151 **/ 5152 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5153 { 5154 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5155 return i40e_mqprio_get_enabled_tc(pf); 5156 5157 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5158 * default TC 5159 */ 5160 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5161 return I40E_DEFAULT_TRAFFIC_CLASS; 5162 5163 /* SFP mode we want PF to be enabled for all TCs */ 5164 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5165 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5166 5167 /* MFP enabled and iSCSI PF type */ 5168 if (pf->hw.func_caps.iscsi) 5169 return i40e_get_iscsi_tc_map(pf); 5170 else 5171 return I40E_DEFAULT_TRAFFIC_CLASS; 5172 } 5173 5174 /** 5175 * i40e_vsi_get_bw_info - Query VSI BW Information 5176 * @vsi: the VSI being queried 5177 * 5178 * Returns 0 on success, negative value on failure 5179 **/ 5180 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5181 { 5182 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5183 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5184 struct i40e_pf *pf = vsi->back; 5185 struct i40e_hw *hw = &pf->hw; 5186 i40e_status ret; 5187 u32 tc_bw_max; 5188 int i; 5189 5190 /* Get the VSI level BW configuration */ 5191 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5192 if (ret) { 5193 dev_info(&pf->pdev->dev, 5194 "couldn't get PF vsi bw config, err %s aq_err %s\n", 5195 i40e_stat_str(&pf->hw, ret), 5196 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5197 return -EINVAL; 5198 } 5199 5200 /* Get the VSI level BW configuration per TC */ 5201 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5202 NULL); 5203 if (ret) { 5204 dev_info(&pf->pdev->dev, 5205 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 5206 i40e_stat_str(&pf->hw, ret), 5207 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5208 return -EINVAL; 5209 } 5210 5211 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5212 dev_info(&pf->pdev->dev, 5213 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5214 bw_config.tc_valid_bits, 5215 bw_ets_config.tc_valid_bits); 5216 /* Still continuing */ 5217 } 5218 5219 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5220 vsi->bw_max_quanta = bw_config.max_bw; 5221 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5222 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5223 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5224 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5225 vsi->bw_ets_limit_credits[i] = 5226 le16_to_cpu(bw_ets_config.credits[i]); 5227 /* 3 bits out of 4 for each TC */ 5228 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5229 } 5230 5231 return 0; 5232 } 5233 5234 /** 5235 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5236 * @vsi: the VSI being configured 5237 * @enabled_tc: TC bitmap 5238 * @bw_share: BW shared credits per TC 5239 * 5240 * Returns 0 on success, negative value on failure 5241 **/ 5242 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5243 u8 *bw_share) 5244 { 5245 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5246 struct i40e_pf *pf = vsi->back; 5247 i40e_status ret; 5248 int i; 5249 5250 /* There is no need to reset BW when mqprio mode is on. */ 5251 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5252 return 0; 5253 if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5254 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5255 if (ret) 5256 dev_info(&pf->pdev->dev, 5257 "Failed to reset tx rate for vsi->seid %u\n", 5258 vsi->seid); 5259 return ret; 5260 } 5261 bw_data.tc_valid_bits = enabled_tc; 5262 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5263 bw_data.tc_bw_credits[i] = bw_share[i]; 5264 5265 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5266 if (ret) { 5267 dev_info(&pf->pdev->dev, 5268 "AQ command Config VSI BW allocation per TC failed = %d\n", 5269 pf->hw.aq.asq_last_status); 5270 return -EINVAL; 5271 } 5272 5273 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5274 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5275 5276 return 0; 5277 } 5278 5279 /** 5280 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5281 * @vsi: the VSI being configured 5282 * @enabled_tc: TC map to be enabled 5283 * 5284 **/ 5285 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5286 { 5287 struct net_device *netdev = vsi->netdev; 5288 struct i40e_pf *pf = vsi->back; 5289 struct i40e_hw *hw = &pf->hw; 5290 u8 netdev_tc = 0; 5291 int i; 5292 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5293 5294 if (!netdev) 5295 return; 5296 5297 if (!enabled_tc) { 5298 netdev_reset_tc(netdev); 5299 return; 5300 } 5301 5302 /* Set up actual enabled TCs on the VSI */ 5303 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5304 return; 5305 5306 /* set per TC queues for the VSI */ 5307 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5308 /* Only set TC queues for enabled tcs 5309 * 5310 * e.g. For a VSI that has TC0 and TC3 enabled the 5311 * enabled_tc bitmap would be 0x00001001; the driver 5312 * will set the numtc for netdev as 2 that will be 5313 * referenced by the netdev layer as TC 0 and 1. 5314 */ 5315 if (vsi->tc_config.enabled_tc & BIT(i)) 5316 netdev_set_tc_queue(netdev, 5317 vsi->tc_config.tc_info[i].netdev_tc, 5318 vsi->tc_config.tc_info[i].qcount, 5319 vsi->tc_config.tc_info[i].qoffset); 5320 } 5321 5322 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5323 return; 5324 5325 /* Assign UP2TC map for the VSI */ 5326 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5327 /* Get the actual TC# for the UP */ 5328 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5329 /* Get the mapped netdev TC# for the UP */ 5330 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5331 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5332 } 5333 } 5334 5335 /** 5336 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5337 * @vsi: the VSI being configured 5338 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5339 **/ 5340 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5341 struct i40e_vsi_context *ctxt) 5342 { 5343 /* copy just the sections touched not the entire info 5344 * since not all sections are valid as returned by 5345 * update vsi params 5346 */ 5347 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5348 memcpy(&vsi->info.queue_mapping, 5349 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5350 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5351 sizeof(vsi->info.tc_mapping)); 5352 } 5353 5354 /** 5355 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5356 * @vsi: VSI to be configured 5357 * @enabled_tc: TC bitmap 5358 * 5359 * This configures a particular VSI for TCs that are mapped to the 5360 * given TC bitmap. It uses default bandwidth share for TCs across 5361 * VSIs to configure TC for a particular VSI. 5362 * 5363 * NOTE: 5364 * It is expected that the VSI queues have been quisced before calling 5365 * this function. 5366 **/ 5367 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5368 { 5369 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5370 struct i40e_pf *pf = vsi->back; 5371 struct i40e_hw *hw = &pf->hw; 5372 struct i40e_vsi_context ctxt; 5373 int ret = 0; 5374 int i; 5375 5376 /* Check if enabled_tc is same as existing or new TCs */ 5377 if (vsi->tc_config.enabled_tc == enabled_tc && 5378 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5379 return ret; 5380 5381 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5382 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5383 if (enabled_tc & BIT(i)) 5384 bw_share[i] = 1; 5385 } 5386 5387 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5388 if (ret) { 5389 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5390 5391 dev_info(&pf->pdev->dev, 5392 "Failed configuring TC map %d for VSI %d\n", 5393 enabled_tc, vsi->seid); 5394 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5395 &bw_config, NULL); 5396 if (ret) { 5397 dev_info(&pf->pdev->dev, 5398 "Failed querying vsi bw info, err %s aq_err %s\n", 5399 i40e_stat_str(hw, ret), 5400 i40e_aq_str(hw, hw->aq.asq_last_status)); 5401 goto out; 5402 } 5403 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5404 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5405 5406 if (!valid_tc) 5407 valid_tc = bw_config.tc_valid_bits; 5408 /* Always enable TC0, no matter what */ 5409 valid_tc |= 1; 5410 dev_info(&pf->pdev->dev, 5411 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5412 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5413 enabled_tc = valid_tc; 5414 } 5415 5416 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5417 if (ret) { 5418 dev_err(&pf->pdev->dev, 5419 "Unable to configure TC map %d for VSI %d\n", 5420 enabled_tc, vsi->seid); 5421 goto out; 5422 } 5423 } 5424 5425 /* Update Queue Pairs Mapping for currently enabled UPs */ 5426 ctxt.seid = vsi->seid; 5427 ctxt.pf_num = vsi->back->hw.pf_id; 5428 ctxt.vf_num = 0; 5429 ctxt.uplink_seid = vsi->uplink_seid; 5430 ctxt.info = vsi->info; 5431 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { 5432 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5433 if (ret) 5434 goto out; 5435 } else { 5436 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5437 } 5438 5439 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5440 * queues changed. 5441 */ 5442 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5443 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5444 vsi->num_queue_pairs); 5445 ret = i40e_vsi_config_rss(vsi); 5446 if (ret) { 5447 dev_info(&vsi->back->pdev->dev, 5448 "Failed to reconfig rss for num_queues\n"); 5449 return ret; 5450 } 5451 vsi->reconfig_rss = false; 5452 } 5453 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5454 ctxt.info.valid_sections |= 5455 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5456 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5457 } 5458 5459 /* Update the VSI after updating the VSI queue-mapping 5460 * information 5461 */ 5462 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5463 if (ret) { 5464 dev_info(&pf->pdev->dev, 5465 "Update vsi tc config failed, err %s aq_err %s\n", 5466 i40e_stat_str(hw, ret), 5467 i40e_aq_str(hw, hw->aq.asq_last_status)); 5468 goto out; 5469 } 5470 /* update the local VSI info with updated queue map */ 5471 i40e_vsi_update_queue_map(vsi, &ctxt); 5472 vsi->info.valid_sections = 0; 5473 5474 /* Update current VSI BW information */ 5475 ret = i40e_vsi_get_bw_info(vsi); 5476 if (ret) { 5477 dev_info(&pf->pdev->dev, 5478 "Failed updating vsi bw info, err %s aq_err %s\n", 5479 i40e_stat_str(hw, ret), 5480 i40e_aq_str(hw, hw->aq.asq_last_status)); 5481 goto out; 5482 } 5483 5484 /* Update the netdev TC setup */ 5485 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5486 out: 5487 return ret; 5488 } 5489 5490 /** 5491 * i40e_get_link_speed - Returns link speed for the interface 5492 * @vsi: VSI to be configured 5493 * 5494 **/ 5495 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5496 { 5497 struct i40e_pf *pf = vsi->back; 5498 5499 switch (pf->hw.phy.link_info.link_speed) { 5500 case I40E_LINK_SPEED_40GB: 5501 return 40000; 5502 case I40E_LINK_SPEED_25GB: 5503 return 25000; 5504 case I40E_LINK_SPEED_20GB: 5505 return 20000; 5506 case I40E_LINK_SPEED_10GB: 5507 return 10000; 5508 case I40E_LINK_SPEED_1GB: 5509 return 1000; 5510 default: 5511 return -EINVAL; 5512 } 5513 } 5514 5515 /** 5516 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5517 * @vsi: VSI to be configured 5518 * @seid: seid of the channel/VSI 5519 * @max_tx_rate: max TX rate to be configured as BW limit 5520 * 5521 * Helper function to set BW limit for a given VSI 5522 **/ 5523 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5524 { 5525 struct i40e_pf *pf = vsi->back; 5526 u64 credits = 0; 5527 int speed = 0; 5528 int ret = 0; 5529 5530 speed = i40e_get_link_speed(vsi); 5531 if (max_tx_rate > speed) { 5532 dev_err(&pf->pdev->dev, 5533 "Invalid max tx rate %llu specified for VSI seid %d.", 5534 max_tx_rate, seid); 5535 return -EINVAL; 5536 } 5537 if (max_tx_rate && max_tx_rate < 50) { 5538 dev_warn(&pf->pdev->dev, 5539 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5540 max_tx_rate = 50; 5541 } 5542 5543 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5544 credits = max_tx_rate; 5545 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5546 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5547 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5548 if (ret) 5549 dev_err(&pf->pdev->dev, 5550 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n", 5551 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret), 5552 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5553 return ret; 5554 } 5555 5556 /** 5557 * i40e_remove_queue_channels - Remove queue channels for the TCs 5558 * @vsi: VSI to be configured 5559 * 5560 * Remove queue channels for the TCs 5561 **/ 5562 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5563 { 5564 enum i40e_admin_queue_err last_aq_status; 5565 struct i40e_cloud_filter *cfilter; 5566 struct i40e_channel *ch, *ch_tmp; 5567 struct i40e_pf *pf = vsi->back; 5568 struct hlist_node *node; 5569 int ret, i; 5570 5571 /* Reset rss size that was stored when reconfiguring rss for 5572 * channel VSIs with non-power-of-2 queue count. 5573 */ 5574 vsi->current_rss_size = 0; 5575 5576 /* perform cleanup for channels if they exist */ 5577 if (list_empty(&vsi->ch_list)) 5578 return; 5579 5580 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5581 struct i40e_vsi *p_vsi; 5582 5583 list_del(&ch->list); 5584 p_vsi = ch->parent_vsi; 5585 if (!p_vsi || !ch->initialized) { 5586 kfree(ch); 5587 continue; 5588 } 5589 /* Reset queue contexts */ 5590 for (i = 0; i < ch->num_queue_pairs; i++) { 5591 struct i40e_ring *tx_ring, *rx_ring; 5592 u16 pf_q; 5593 5594 pf_q = ch->base_queue + i; 5595 tx_ring = vsi->tx_rings[pf_q]; 5596 tx_ring->ch = NULL; 5597 5598 rx_ring = vsi->rx_rings[pf_q]; 5599 rx_ring->ch = NULL; 5600 } 5601 5602 /* Reset BW configured for this VSI via mqprio */ 5603 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 5604 if (ret) 5605 dev_info(&vsi->back->pdev->dev, 5606 "Failed to reset tx rate for ch->seid %u\n", 5607 ch->seid); 5608 5609 /* delete cloud filters associated with this channel */ 5610 hlist_for_each_entry_safe(cfilter, node, 5611 &pf->cloud_filter_list, cloud_node) { 5612 if (cfilter->seid != ch->seid) 5613 continue; 5614 5615 hash_del(&cfilter->cloud_node); 5616 if (cfilter->dst_port) 5617 ret = i40e_add_del_cloud_filter_big_buf(vsi, 5618 cfilter, 5619 false); 5620 else 5621 ret = i40e_add_del_cloud_filter(vsi, cfilter, 5622 false); 5623 last_aq_status = pf->hw.aq.asq_last_status; 5624 if (ret) 5625 dev_info(&pf->pdev->dev, 5626 "Failed to delete cloud filter, err %s aq_err %s\n", 5627 i40e_stat_str(&pf->hw, ret), 5628 i40e_aq_str(&pf->hw, last_aq_status)); 5629 kfree(cfilter); 5630 } 5631 5632 /* delete VSI from FW */ 5633 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 5634 NULL); 5635 if (ret) 5636 dev_err(&vsi->back->pdev->dev, 5637 "unable to remove channel (%d) for parent VSI(%d)\n", 5638 ch->seid, p_vsi->seid); 5639 kfree(ch); 5640 } 5641 INIT_LIST_HEAD(&vsi->ch_list); 5642 } 5643 5644 /** 5645 * i40e_is_any_channel - channel exist or not 5646 * @vsi: ptr to VSI to which channels are associated with 5647 * 5648 * Returns true or false if channel(s) exist for associated VSI or not 5649 **/ 5650 static bool i40e_is_any_channel(struct i40e_vsi *vsi) 5651 { 5652 struct i40e_channel *ch, *ch_tmp; 5653 5654 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5655 if (ch->initialized) 5656 return true; 5657 } 5658 5659 return false; 5660 } 5661 5662 /** 5663 * i40e_get_max_queues_for_channel 5664 * @vsi: ptr to VSI to which channels are associated with 5665 * 5666 * Helper function which returns max value among the queue counts set on the 5667 * channels/TCs created. 5668 **/ 5669 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 5670 { 5671 struct i40e_channel *ch, *ch_tmp; 5672 int max = 0; 5673 5674 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5675 if (!ch->initialized) 5676 continue; 5677 if (ch->num_queue_pairs > max) 5678 max = ch->num_queue_pairs; 5679 } 5680 5681 return max; 5682 } 5683 5684 /** 5685 * i40e_validate_num_queues - validate num_queues w.r.t channel 5686 * @pf: ptr to PF device 5687 * @num_queues: number of queues 5688 * @vsi: the parent VSI 5689 * @reconfig_rss: indicates should the RSS be reconfigured or not 5690 * 5691 * This function validates number of queues in the context of new channel 5692 * which is being established and determines if RSS should be reconfigured 5693 * or not for parent VSI. 5694 **/ 5695 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 5696 struct i40e_vsi *vsi, bool *reconfig_rss) 5697 { 5698 int max_ch_queues; 5699 5700 if (!reconfig_rss) 5701 return -EINVAL; 5702 5703 *reconfig_rss = false; 5704 if (vsi->current_rss_size) { 5705 if (num_queues > vsi->current_rss_size) { 5706 dev_dbg(&pf->pdev->dev, 5707 "Error: num_queues (%d) > vsi's current_size(%d)\n", 5708 num_queues, vsi->current_rss_size); 5709 return -EINVAL; 5710 } else if ((num_queues < vsi->current_rss_size) && 5711 (!is_power_of_2(num_queues))) { 5712 dev_dbg(&pf->pdev->dev, 5713 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 5714 num_queues, vsi->current_rss_size); 5715 return -EINVAL; 5716 } 5717 } 5718 5719 if (!is_power_of_2(num_queues)) { 5720 /* Find the max num_queues configured for channel if channel 5721 * exist. 5722 * if channel exist, then enforce 'num_queues' to be more than 5723 * max ever queues configured for channel. 5724 */ 5725 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 5726 if (num_queues < max_ch_queues) { 5727 dev_dbg(&pf->pdev->dev, 5728 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 5729 num_queues, max_ch_queues); 5730 return -EINVAL; 5731 } 5732 *reconfig_rss = true; 5733 } 5734 5735 return 0; 5736 } 5737 5738 /** 5739 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 5740 * @vsi: the VSI being setup 5741 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 5742 * 5743 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 5744 **/ 5745 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 5746 { 5747 struct i40e_pf *pf = vsi->back; 5748 u8 seed[I40E_HKEY_ARRAY_SIZE]; 5749 struct i40e_hw *hw = &pf->hw; 5750 int local_rss_size; 5751 u8 *lut; 5752 int ret; 5753 5754 if (!vsi->rss_size) 5755 return -EINVAL; 5756 5757 if (rss_size > vsi->rss_size) 5758 return -EINVAL; 5759 5760 local_rss_size = min_t(int, vsi->rss_size, rss_size); 5761 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 5762 if (!lut) 5763 return -ENOMEM; 5764 5765 /* Ignoring user configured lut if there is one */ 5766 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 5767 5768 /* Use user configured hash key if there is one, otherwise 5769 * use default. 5770 */ 5771 if (vsi->rss_hkey_user) 5772 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 5773 else 5774 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 5775 5776 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 5777 if (ret) { 5778 dev_info(&pf->pdev->dev, 5779 "Cannot set RSS lut, err %s aq_err %s\n", 5780 i40e_stat_str(hw, ret), 5781 i40e_aq_str(hw, hw->aq.asq_last_status)); 5782 kfree(lut); 5783 return ret; 5784 } 5785 kfree(lut); 5786 5787 /* Do the update w.r.t. storing rss_size */ 5788 if (!vsi->orig_rss_size) 5789 vsi->orig_rss_size = vsi->rss_size; 5790 vsi->current_rss_size = local_rss_size; 5791 5792 return ret; 5793 } 5794 5795 /** 5796 * i40e_channel_setup_queue_map - Setup a channel queue map 5797 * @pf: ptr to PF device 5798 * @vsi: the VSI being setup 5799 * @ctxt: VSI context structure 5800 * @ch: ptr to channel structure 5801 * 5802 * Setup queue map for a specific channel 5803 **/ 5804 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 5805 struct i40e_vsi_context *ctxt, 5806 struct i40e_channel *ch) 5807 { 5808 u16 qcount, qmap, sections = 0; 5809 u8 offset = 0; 5810 int pow; 5811 5812 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 5813 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 5814 5815 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 5816 ch->num_queue_pairs = qcount; 5817 5818 /* find the next higher power-of-2 of num queue pairs */ 5819 pow = ilog2(qcount); 5820 if (!is_power_of_2(qcount)) 5821 pow++; 5822 5823 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5824 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 5825 5826 /* Setup queue TC[0].qmap for given VSI context */ 5827 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 5828 5829 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 5830 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 5831 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 5832 ctxt->info.valid_sections |= cpu_to_le16(sections); 5833 } 5834 5835 /** 5836 * i40e_add_channel - add a channel by adding VSI 5837 * @pf: ptr to PF device 5838 * @uplink_seid: underlying HW switching element (VEB) ID 5839 * @ch: ptr to channel structure 5840 * 5841 * Add a channel (VSI) using add_vsi and queue_map 5842 **/ 5843 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 5844 struct i40e_channel *ch) 5845 { 5846 struct i40e_hw *hw = &pf->hw; 5847 struct i40e_vsi_context ctxt; 5848 u8 enabled_tc = 0x1; /* TC0 enabled */ 5849 int ret; 5850 5851 if (ch->type != I40E_VSI_VMDQ2) { 5852 dev_info(&pf->pdev->dev, 5853 "add new vsi failed, ch->type %d\n", ch->type); 5854 return -EINVAL; 5855 } 5856 5857 memset(&ctxt, 0, sizeof(ctxt)); 5858 ctxt.pf_num = hw->pf_id; 5859 ctxt.vf_num = 0; 5860 ctxt.uplink_seid = uplink_seid; 5861 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5862 if (ch->type == I40E_VSI_VMDQ2) 5863 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 5864 5865 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 5866 ctxt.info.valid_sections |= 5867 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5868 ctxt.info.switch_id = 5869 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 5870 } 5871 5872 /* Set queue map for a given VSI context */ 5873 i40e_channel_setup_queue_map(pf, &ctxt, ch); 5874 5875 /* Now time to create VSI */ 5876 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 5877 if (ret) { 5878 dev_info(&pf->pdev->dev, 5879 "add new vsi failed, err %s aq_err %s\n", 5880 i40e_stat_str(&pf->hw, ret), 5881 i40e_aq_str(&pf->hw, 5882 pf->hw.aq.asq_last_status)); 5883 return -ENOENT; 5884 } 5885 5886 /* Success, update channel, set enabled_tc only if the channel 5887 * is not a macvlan 5888 */ 5889 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; 5890 ch->seid = ctxt.seid; 5891 ch->vsi_number = ctxt.vsi_number; 5892 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); 5893 5894 /* copy just the sections touched not the entire info 5895 * since not all sections are valid as returned by 5896 * update vsi params 5897 */ 5898 ch->info.mapping_flags = ctxt.info.mapping_flags; 5899 memcpy(&ch->info.queue_mapping, 5900 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 5901 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 5902 sizeof(ctxt.info.tc_mapping)); 5903 5904 return 0; 5905 } 5906 5907 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 5908 u8 *bw_share) 5909 { 5910 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5911 i40e_status ret; 5912 int i; 5913 5914 bw_data.tc_valid_bits = ch->enabled_tc; 5915 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5916 bw_data.tc_bw_credits[i] = bw_share[i]; 5917 5918 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 5919 &bw_data, NULL); 5920 if (ret) { 5921 dev_info(&vsi->back->pdev->dev, 5922 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 5923 vsi->back->hw.aq.asq_last_status, ch->seid); 5924 return -EINVAL; 5925 } 5926 5927 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5928 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 5929 5930 return 0; 5931 } 5932 5933 /** 5934 * i40e_channel_config_tx_ring - config TX ring associated with new channel 5935 * @pf: ptr to PF device 5936 * @vsi: the VSI being setup 5937 * @ch: ptr to channel structure 5938 * 5939 * Configure TX rings associated with channel (VSI) since queues are being 5940 * from parent VSI. 5941 **/ 5942 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 5943 struct i40e_vsi *vsi, 5944 struct i40e_channel *ch) 5945 { 5946 i40e_status ret; 5947 int i; 5948 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5949 5950 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5951 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5952 if (ch->enabled_tc & BIT(i)) 5953 bw_share[i] = 1; 5954 } 5955 5956 /* configure BW for new VSI */ 5957 ret = i40e_channel_config_bw(vsi, ch, bw_share); 5958 if (ret) { 5959 dev_info(&vsi->back->pdev->dev, 5960 "Failed configuring TC map %d for channel (seid %u)\n", 5961 ch->enabled_tc, ch->seid); 5962 return ret; 5963 } 5964 5965 for (i = 0; i < ch->num_queue_pairs; i++) { 5966 struct i40e_ring *tx_ring, *rx_ring; 5967 u16 pf_q; 5968 5969 pf_q = ch->base_queue + i; 5970 5971 /* Get to TX ring ptr of main VSI, for re-setup TX queue 5972 * context 5973 */ 5974 tx_ring = vsi->tx_rings[pf_q]; 5975 tx_ring->ch = ch; 5976 5977 /* Get the RX ring ptr */ 5978 rx_ring = vsi->rx_rings[pf_q]; 5979 rx_ring->ch = ch; 5980 } 5981 5982 return 0; 5983 } 5984 5985 /** 5986 * i40e_setup_hw_channel - setup new channel 5987 * @pf: ptr to PF device 5988 * @vsi: the VSI being setup 5989 * @ch: ptr to channel structure 5990 * @uplink_seid: underlying HW switching element (VEB) ID 5991 * @type: type of channel to be created (VMDq2/VF) 5992 * 5993 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5994 * and configures TX rings accordingly 5995 **/ 5996 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 5997 struct i40e_vsi *vsi, 5998 struct i40e_channel *ch, 5999 u16 uplink_seid, u8 type) 6000 { 6001 int ret; 6002 6003 ch->initialized = false; 6004 ch->base_queue = vsi->next_base_queue; 6005 ch->type = type; 6006 6007 /* Proceed with creation of channel (VMDq2) VSI */ 6008 ret = i40e_add_channel(pf, uplink_seid, ch); 6009 if (ret) { 6010 dev_info(&pf->pdev->dev, 6011 "failed to add_channel using uplink_seid %u\n", 6012 uplink_seid); 6013 return ret; 6014 } 6015 6016 /* Mark the successful creation of channel */ 6017 ch->initialized = true; 6018 6019 /* Reconfigure TX queues using QTX_CTL register */ 6020 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 6021 if (ret) { 6022 dev_info(&pf->pdev->dev, 6023 "failed to configure TX rings for channel %u\n", 6024 ch->seid); 6025 return ret; 6026 } 6027 6028 /* update 'next_base_queue' */ 6029 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 6030 dev_dbg(&pf->pdev->dev, 6031 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 6032 ch->seid, ch->vsi_number, ch->stat_counter_idx, 6033 ch->num_queue_pairs, 6034 vsi->next_base_queue); 6035 return ret; 6036 } 6037 6038 /** 6039 * i40e_setup_channel - setup new channel using uplink element 6040 * @pf: ptr to PF device 6041 * @type: type of channel to be created (VMDq2/VF) 6042 * @uplink_seid: underlying HW switching element (VEB) ID 6043 * @ch: ptr to channel structure 6044 * 6045 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6046 * and uplink switching element (uplink_seid) 6047 **/ 6048 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 6049 struct i40e_channel *ch) 6050 { 6051 u8 vsi_type; 6052 u16 seid; 6053 int ret; 6054 6055 if (vsi->type == I40E_VSI_MAIN) { 6056 vsi_type = I40E_VSI_VMDQ2; 6057 } else { 6058 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6059 vsi->type); 6060 return false; 6061 } 6062 6063 /* underlying switching element */ 6064 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6065 6066 /* create channel (VSI), configure TX rings */ 6067 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6068 if (ret) { 6069 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6070 return false; 6071 } 6072 6073 return ch->initialized ? true : false; 6074 } 6075 6076 /** 6077 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6078 * @vsi: ptr to VSI which has PF backing 6079 * 6080 * Sets up switch mode correctly if it needs to be changed and perform 6081 * what are allowed modes. 6082 **/ 6083 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6084 { 6085 u8 mode; 6086 struct i40e_pf *pf = vsi->back; 6087 struct i40e_hw *hw = &pf->hw; 6088 int ret; 6089 6090 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6091 if (ret) 6092 return -EINVAL; 6093 6094 if (hw->dev_caps.switch_mode) { 6095 /* if switch mode is set, support mode2 (non-tunneled for 6096 * cloud filter) for now 6097 */ 6098 u32 switch_mode = hw->dev_caps.switch_mode & 6099 I40E_SWITCH_MODE_MASK; 6100 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6101 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6102 return 0; 6103 dev_err(&pf->pdev->dev, 6104 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6105 hw->dev_caps.switch_mode); 6106 return -EINVAL; 6107 } 6108 } 6109 6110 /* Set Bit 7 to be valid */ 6111 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6112 6113 /* Set L4type for TCP support */ 6114 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6115 6116 /* Set cloud filter mode */ 6117 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6118 6119 /* Prep mode field for set_switch_config */ 6120 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6121 pf->last_sw_conf_valid_flags, 6122 mode, NULL); 6123 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6124 dev_err(&pf->pdev->dev, 6125 "couldn't set switch config bits, err %s aq_err %s\n", 6126 i40e_stat_str(hw, ret), 6127 i40e_aq_str(hw, 6128 hw->aq.asq_last_status)); 6129 6130 return ret; 6131 } 6132 6133 /** 6134 * i40e_create_queue_channel - function to create channel 6135 * @vsi: VSI to be configured 6136 * @ch: ptr to channel (it contains channel specific params) 6137 * 6138 * This function creates channel (VSI) using num_queues specified by user, 6139 * reconfigs RSS if needed. 6140 **/ 6141 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6142 struct i40e_channel *ch) 6143 { 6144 struct i40e_pf *pf = vsi->back; 6145 bool reconfig_rss; 6146 int err; 6147 6148 if (!ch) 6149 return -EINVAL; 6150 6151 if (!ch->num_queue_pairs) { 6152 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6153 ch->num_queue_pairs); 6154 return -EINVAL; 6155 } 6156 6157 /* validate user requested num_queues for channel */ 6158 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6159 &reconfig_rss); 6160 if (err) { 6161 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6162 ch->num_queue_pairs); 6163 return -EINVAL; 6164 } 6165 6166 /* By default we are in VEPA mode, if this is the first VF/VMDq 6167 * VSI to be added switch to VEB mode. 6168 */ 6169 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) || 6170 (!i40e_is_any_channel(vsi))) { 6171 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) { 6172 dev_dbg(&pf->pdev->dev, 6173 "Failed to create channel. Override queues (%u) not power of 2\n", 6174 vsi->tc_config.tc_info[0].qcount); 6175 return -EINVAL; 6176 } 6177 6178 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6179 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6180 6181 if (vsi->type == I40E_VSI_MAIN) { 6182 if (pf->flags & I40E_FLAG_TC_MQPRIO) 6183 i40e_do_reset(pf, I40E_PF_RESET_FLAG, 6184 true); 6185 else 6186 i40e_do_reset_safe(pf, 6187 I40E_PF_RESET_FLAG); 6188 } 6189 } 6190 /* now onwards for main VSI, number of queues will be value 6191 * of TC0's queue count 6192 */ 6193 } 6194 6195 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6196 * it should be more than num_queues 6197 */ 6198 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6199 dev_dbg(&pf->pdev->dev, 6200 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6201 vsi->cnt_q_avail, ch->num_queue_pairs); 6202 return -EINVAL; 6203 } 6204 6205 /* reconfig_rss only if vsi type is MAIN_VSI */ 6206 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6207 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6208 if (err) { 6209 dev_info(&pf->pdev->dev, 6210 "Error: unable to reconfig rss for num_queues (%u)\n", 6211 ch->num_queue_pairs); 6212 return -EINVAL; 6213 } 6214 } 6215 6216 if (!i40e_setup_channel(pf, vsi, ch)) { 6217 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6218 return -EINVAL; 6219 } 6220 6221 dev_info(&pf->pdev->dev, 6222 "Setup channel (id:%u) utilizing num_queues %d\n", 6223 ch->seid, ch->num_queue_pairs); 6224 6225 /* configure VSI for BW limit */ 6226 if (ch->max_tx_rate) { 6227 u64 credits = ch->max_tx_rate; 6228 6229 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6230 return -EINVAL; 6231 6232 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6233 dev_dbg(&pf->pdev->dev, 6234 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6235 ch->max_tx_rate, 6236 credits, 6237 ch->seid); 6238 } 6239 6240 /* in case of VF, this will be main SRIOV VSI */ 6241 ch->parent_vsi = vsi; 6242 6243 /* and update main_vsi's count for queue_available to use */ 6244 vsi->cnt_q_avail -= ch->num_queue_pairs; 6245 6246 return 0; 6247 } 6248 6249 /** 6250 * i40e_configure_queue_channels - Add queue channel for the given TCs 6251 * @vsi: VSI to be configured 6252 * 6253 * Configures queue channel mapping to the given TCs 6254 **/ 6255 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6256 { 6257 struct i40e_channel *ch; 6258 u64 max_rate = 0; 6259 int ret = 0, i; 6260 6261 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6262 vsi->tc_seid_map[0] = vsi->seid; 6263 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6264 if (vsi->tc_config.enabled_tc & BIT(i)) { 6265 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6266 if (!ch) { 6267 ret = -ENOMEM; 6268 goto err_free; 6269 } 6270 6271 INIT_LIST_HEAD(&ch->list); 6272 ch->num_queue_pairs = 6273 vsi->tc_config.tc_info[i].qcount; 6274 ch->base_queue = 6275 vsi->tc_config.tc_info[i].qoffset; 6276 6277 /* Bandwidth limit through tc interface is in bytes/s, 6278 * change to Mbit/s 6279 */ 6280 max_rate = vsi->mqprio_qopt.max_rate[i]; 6281 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6282 ch->max_tx_rate = max_rate; 6283 6284 list_add_tail(&ch->list, &vsi->ch_list); 6285 6286 ret = i40e_create_queue_channel(vsi, ch); 6287 if (ret) { 6288 dev_err(&vsi->back->pdev->dev, 6289 "Failed creating queue channel with TC%d: queues %d\n", 6290 i, ch->num_queue_pairs); 6291 goto err_free; 6292 } 6293 vsi->tc_seid_map[i] = ch->seid; 6294 } 6295 } 6296 return ret; 6297 6298 err_free: 6299 i40e_remove_queue_channels(vsi); 6300 return ret; 6301 } 6302 6303 /** 6304 * i40e_veb_config_tc - Configure TCs for given VEB 6305 * @veb: given VEB 6306 * @enabled_tc: TC bitmap 6307 * 6308 * Configures given TC bitmap for VEB (switching) element 6309 **/ 6310 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6311 { 6312 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6313 struct i40e_pf *pf = veb->pf; 6314 int ret = 0; 6315 int i; 6316 6317 /* No TCs or already enabled TCs just return */ 6318 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6319 return ret; 6320 6321 bw_data.tc_valid_bits = enabled_tc; 6322 /* bw_data.absolute_credits is not set (relative) */ 6323 6324 /* Enable ETS TCs with equal BW Share for now */ 6325 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6326 if (enabled_tc & BIT(i)) 6327 bw_data.tc_bw_share_credits[i] = 1; 6328 } 6329 6330 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6331 &bw_data, NULL); 6332 if (ret) { 6333 dev_info(&pf->pdev->dev, 6334 "VEB bw config failed, err %s aq_err %s\n", 6335 i40e_stat_str(&pf->hw, ret), 6336 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6337 goto out; 6338 } 6339 6340 /* Update the BW information */ 6341 ret = i40e_veb_get_bw_info(veb); 6342 if (ret) { 6343 dev_info(&pf->pdev->dev, 6344 "Failed getting veb bw config, err %s aq_err %s\n", 6345 i40e_stat_str(&pf->hw, ret), 6346 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6347 } 6348 6349 out: 6350 return ret; 6351 } 6352 6353 #ifdef CONFIG_I40E_DCB 6354 /** 6355 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6356 * @pf: PF struct 6357 * 6358 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6359 * the caller would've quiesce all the VSIs before calling 6360 * this function 6361 **/ 6362 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6363 { 6364 u8 tc_map = 0; 6365 int ret; 6366 u8 v; 6367 6368 /* Enable the TCs available on PF to all VEBs */ 6369 tc_map = i40e_pf_get_tc_map(pf); 6370 for (v = 0; v < I40E_MAX_VEB; v++) { 6371 if (!pf->veb[v]) 6372 continue; 6373 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6374 if (ret) { 6375 dev_info(&pf->pdev->dev, 6376 "Failed configuring TC for VEB seid=%d\n", 6377 pf->veb[v]->seid); 6378 /* Will try to configure as many components */ 6379 } 6380 } 6381 6382 /* Update each VSI */ 6383 for (v = 0; v < pf->num_alloc_vsi; v++) { 6384 if (!pf->vsi[v]) 6385 continue; 6386 6387 /* - Enable all TCs for the LAN VSI 6388 * - For all others keep them at TC0 for now 6389 */ 6390 if (v == pf->lan_vsi) 6391 tc_map = i40e_pf_get_tc_map(pf); 6392 else 6393 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6394 6395 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6396 if (ret) { 6397 dev_info(&pf->pdev->dev, 6398 "Failed configuring TC for VSI seid=%d\n", 6399 pf->vsi[v]->seid); 6400 /* Will try to configure as many components */ 6401 } else { 6402 /* Re-configure VSI vectors based on updated TC map */ 6403 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6404 if (pf->vsi[v]->netdev) 6405 i40e_dcbnl_set_all(pf->vsi[v]); 6406 } 6407 } 6408 } 6409 6410 /** 6411 * i40e_resume_port_tx - Resume port Tx 6412 * @pf: PF struct 6413 * 6414 * Resume a port's Tx and issue a PF reset in case of failure to 6415 * resume. 6416 **/ 6417 static int i40e_resume_port_tx(struct i40e_pf *pf) 6418 { 6419 struct i40e_hw *hw = &pf->hw; 6420 int ret; 6421 6422 ret = i40e_aq_resume_port_tx(hw, NULL); 6423 if (ret) { 6424 dev_info(&pf->pdev->dev, 6425 "Resume Port Tx failed, err %s aq_err %s\n", 6426 i40e_stat_str(&pf->hw, ret), 6427 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6428 /* Schedule PF reset to recover */ 6429 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6430 i40e_service_event_schedule(pf); 6431 } 6432 6433 return ret; 6434 } 6435 6436 /** 6437 * i40e_init_pf_dcb - Initialize DCB configuration 6438 * @pf: PF being configured 6439 * 6440 * Query the current DCB configuration and cache it 6441 * in the hardware structure 6442 **/ 6443 static int i40e_init_pf_dcb(struct i40e_pf *pf) 6444 { 6445 struct i40e_hw *hw = &pf->hw; 6446 int err = 0; 6447 6448 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 6449 * Also do not enable DCBx if FW LLDP agent is disabled 6450 */ 6451 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) || 6452 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) { 6453 dev_info(&pf->pdev->dev, "DCB is not supported or FW LLDP is disabled\n"); 6454 err = I40E_NOT_SUPPORTED; 6455 goto out; 6456 } 6457 6458 err = i40e_init_dcb(hw, true); 6459 if (!err) { 6460 /* Device/Function is not DCBX capable */ 6461 if ((!hw->func_caps.dcb) || 6462 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 6463 dev_info(&pf->pdev->dev, 6464 "DCBX offload is not supported or is disabled for this PF.\n"); 6465 } else { 6466 /* When status is not DISABLED then DCBX in FW */ 6467 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 6468 DCB_CAP_DCBX_VER_IEEE; 6469 6470 pf->flags |= I40E_FLAG_DCB_CAPABLE; 6471 /* Enable DCB tagging only when more than one TC 6472 * or explicitly disable if only one TC 6473 */ 6474 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6475 pf->flags |= I40E_FLAG_DCB_ENABLED; 6476 else 6477 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6478 dev_dbg(&pf->pdev->dev, 6479 "DCBX offload is supported for this PF.\n"); 6480 } 6481 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 6482 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 6483 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP; 6484 } else { 6485 dev_info(&pf->pdev->dev, 6486 "Query for DCB configuration failed, err %s aq_err %s\n", 6487 i40e_stat_str(&pf->hw, err), 6488 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6489 } 6490 6491 out: 6492 return err; 6493 } 6494 #endif /* CONFIG_I40E_DCB */ 6495 6496 /** 6497 * i40e_print_link_message - print link up or down 6498 * @vsi: the VSI for which link needs a message 6499 * @isup: true of link is up, false otherwise 6500 */ 6501 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 6502 { 6503 enum i40e_aq_link_speed new_speed; 6504 struct i40e_pf *pf = vsi->back; 6505 char *speed = "Unknown"; 6506 char *fc = "Unknown"; 6507 char *fec = ""; 6508 char *req_fec = ""; 6509 char *an = ""; 6510 6511 if (isup) 6512 new_speed = pf->hw.phy.link_info.link_speed; 6513 else 6514 new_speed = I40E_LINK_SPEED_UNKNOWN; 6515 6516 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 6517 return; 6518 vsi->current_isup = isup; 6519 vsi->current_speed = new_speed; 6520 if (!isup) { 6521 netdev_info(vsi->netdev, "NIC Link is Down\n"); 6522 return; 6523 } 6524 6525 /* Warn user if link speed on NPAR enabled partition is not at 6526 * least 10GB 6527 */ 6528 if (pf->hw.func_caps.npar_enable && 6529 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 6530 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 6531 netdev_warn(vsi->netdev, 6532 "The partition detected link speed that is less than 10Gbps\n"); 6533 6534 switch (pf->hw.phy.link_info.link_speed) { 6535 case I40E_LINK_SPEED_40GB: 6536 speed = "40 G"; 6537 break; 6538 case I40E_LINK_SPEED_20GB: 6539 speed = "20 G"; 6540 break; 6541 case I40E_LINK_SPEED_25GB: 6542 speed = "25 G"; 6543 break; 6544 case I40E_LINK_SPEED_10GB: 6545 speed = "10 G"; 6546 break; 6547 case I40E_LINK_SPEED_5GB: 6548 speed = "5 G"; 6549 break; 6550 case I40E_LINK_SPEED_2_5GB: 6551 speed = "2.5 G"; 6552 break; 6553 case I40E_LINK_SPEED_1GB: 6554 speed = "1000 M"; 6555 break; 6556 case I40E_LINK_SPEED_100MB: 6557 speed = "100 M"; 6558 break; 6559 default: 6560 break; 6561 } 6562 6563 switch (pf->hw.fc.current_mode) { 6564 case I40E_FC_FULL: 6565 fc = "RX/TX"; 6566 break; 6567 case I40E_FC_TX_PAUSE: 6568 fc = "TX"; 6569 break; 6570 case I40E_FC_RX_PAUSE: 6571 fc = "RX"; 6572 break; 6573 default: 6574 fc = "None"; 6575 break; 6576 } 6577 6578 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 6579 req_fec = "None"; 6580 fec = "None"; 6581 an = "False"; 6582 6583 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6584 an = "True"; 6585 6586 if (pf->hw.phy.link_info.fec_info & 6587 I40E_AQ_CONFIG_FEC_KR_ENA) 6588 fec = "CL74 FC-FEC/BASE-R"; 6589 else if (pf->hw.phy.link_info.fec_info & 6590 I40E_AQ_CONFIG_FEC_RS_ENA) 6591 fec = "CL108 RS-FEC"; 6592 6593 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 6594 * both RS and FC are requested 6595 */ 6596 if (vsi->back->hw.phy.link_info.req_fec_info & 6597 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 6598 if (vsi->back->hw.phy.link_info.req_fec_info & 6599 I40E_AQ_REQUEST_FEC_RS) 6600 req_fec = "CL108 RS-FEC"; 6601 else 6602 req_fec = "CL74 FC-FEC/BASE-R"; 6603 } 6604 netdev_info(vsi->netdev, 6605 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 6606 speed, req_fec, fec, an, fc); 6607 } else { 6608 netdev_info(vsi->netdev, 6609 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", 6610 speed, fc); 6611 } 6612 6613 } 6614 6615 /** 6616 * i40e_up_complete - Finish the last steps of bringing up a connection 6617 * @vsi: the VSI being configured 6618 **/ 6619 static int i40e_up_complete(struct i40e_vsi *vsi) 6620 { 6621 struct i40e_pf *pf = vsi->back; 6622 int err; 6623 6624 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6625 i40e_vsi_configure_msix(vsi); 6626 else 6627 i40e_configure_msi_and_legacy(vsi); 6628 6629 /* start rings */ 6630 err = i40e_vsi_start_rings(vsi); 6631 if (err) 6632 return err; 6633 6634 clear_bit(__I40E_VSI_DOWN, vsi->state); 6635 i40e_napi_enable_all(vsi); 6636 i40e_vsi_enable_irq(vsi); 6637 6638 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 6639 (vsi->netdev)) { 6640 i40e_print_link_message(vsi, true); 6641 netif_tx_start_all_queues(vsi->netdev); 6642 netif_carrier_on(vsi->netdev); 6643 } 6644 6645 /* replay FDIR SB filters */ 6646 if (vsi->type == I40E_VSI_FDIR) { 6647 /* reset fd counters */ 6648 pf->fd_add_err = 0; 6649 pf->fd_atr_cnt = 0; 6650 i40e_fdir_filter_restore(vsi); 6651 } 6652 6653 /* On the next run of the service_task, notify any clients of the new 6654 * opened netdev 6655 */ 6656 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 6657 i40e_service_event_schedule(pf); 6658 6659 return 0; 6660 } 6661 6662 /** 6663 * i40e_vsi_reinit_locked - Reset the VSI 6664 * @vsi: the VSI being configured 6665 * 6666 * Rebuild the ring structs after some configuration 6667 * has changed, e.g. MTU size. 6668 **/ 6669 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 6670 { 6671 struct i40e_pf *pf = vsi->back; 6672 6673 WARN_ON(in_interrupt()); 6674 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 6675 usleep_range(1000, 2000); 6676 i40e_down(vsi); 6677 6678 i40e_up(vsi); 6679 clear_bit(__I40E_CONFIG_BUSY, pf->state); 6680 } 6681 6682 /** 6683 * i40e_up - Bring the connection back up after being down 6684 * @vsi: the VSI being configured 6685 **/ 6686 int i40e_up(struct i40e_vsi *vsi) 6687 { 6688 int err; 6689 6690 err = i40e_vsi_configure(vsi); 6691 if (!err) 6692 err = i40e_up_complete(vsi); 6693 6694 return err; 6695 } 6696 6697 /** 6698 * i40e_force_link_state - Force the link status 6699 * @pf: board private structure 6700 * @is_up: whether the link state should be forced up or down 6701 **/ 6702 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up) 6703 { 6704 struct i40e_aq_get_phy_abilities_resp abilities; 6705 struct i40e_aq_set_phy_config config = {0}; 6706 struct i40e_hw *hw = &pf->hw; 6707 i40e_status err; 6708 u64 mask; 6709 u8 speed; 6710 6711 /* Card might've been put in an unstable state by other drivers 6712 * and applications, which causes incorrect speed values being 6713 * set on startup. In order to clear speed registers, we call 6714 * get_phy_capabilities twice, once to get initial state of 6715 * available speeds, and once to get current PHY config. 6716 */ 6717 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 6718 NULL); 6719 if (err) { 6720 dev_err(&pf->pdev->dev, 6721 "failed to get phy cap., ret = %s last_status = %s\n", 6722 i40e_stat_str(hw, err), 6723 i40e_aq_str(hw, hw->aq.asq_last_status)); 6724 return err; 6725 } 6726 speed = abilities.link_speed; 6727 6728 /* Get the current phy config */ 6729 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 6730 NULL); 6731 if (err) { 6732 dev_err(&pf->pdev->dev, 6733 "failed to get phy cap., ret = %s last_status = %s\n", 6734 i40e_stat_str(hw, err), 6735 i40e_aq_str(hw, hw->aq.asq_last_status)); 6736 return err; 6737 } 6738 6739 /* If link needs to go up, but was not forced to go down, 6740 * and its speed values are OK, no need for a flap 6741 */ 6742 if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 6743 return I40E_SUCCESS; 6744 6745 /* To force link we need to set bits for all supported PHY types, 6746 * but there are now more than 32, so we need to split the bitmap 6747 * across two fields. 6748 */ 6749 mask = I40E_PHY_TYPES_BITMASK; 6750 config.phy_type = is_up ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 6751 config.phy_type_ext = is_up ? (u8)((mask >> 32) & 0xff) : 0; 6752 /* Copy the old settings, except of phy_type */ 6753 config.abilities = abilities.abilities; 6754 if (abilities.link_speed != 0) 6755 config.link_speed = abilities.link_speed; 6756 else 6757 config.link_speed = speed; 6758 config.eee_capability = abilities.eee_capability; 6759 config.eeer = abilities.eeer_val; 6760 config.low_power_ctrl = abilities.d3_lpan; 6761 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 6762 I40E_AQ_PHY_FEC_CONFIG_MASK; 6763 err = i40e_aq_set_phy_config(hw, &config, NULL); 6764 6765 if (err) { 6766 dev_err(&pf->pdev->dev, 6767 "set phy config ret = %s last_status = %s\n", 6768 i40e_stat_str(&pf->hw, err), 6769 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6770 return err; 6771 } 6772 6773 /* Update the link info */ 6774 err = i40e_update_link_info(hw); 6775 if (err) { 6776 /* Wait a little bit (on 40G cards it sometimes takes a really 6777 * long time for link to come back from the atomic reset) 6778 * and try once more 6779 */ 6780 msleep(1000); 6781 i40e_update_link_info(hw); 6782 } 6783 6784 i40e_aq_set_link_restart_an(hw, true, NULL); 6785 6786 return I40E_SUCCESS; 6787 } 6788 6789 /** 6790 * i40e_down - Shutdown the connection processing 6791 * @vsi: the VSI being stopped 6792 **/ 6793 void i40e_down(struct i40e_vsi *vsi) 6794 { 6795 int i; 6796 6797 /* It is assumed that the caller of this function 6798 * sets the vsi->state __I40E_VSI_DOWN bit. 6799 */ 6800 if (vsi->netdev) { 6801 netif_carrier_off(vsi->netdev); 6802 netif_tx_disable(vsi->netdev); 6803 } 6804 i40e_vsi_disable_irq(vsi); 6805 i40e_vsi_stop_rings(vsi); 6806 if (vsi->type == I40E_VSI_MAIN && 6807 vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) 6808 i40e_force_link_state(vsi->back, false); 6809 i40e_napi_disable_all(vsi); 6810 6811 for (i = 0; i < vsi->num_queue_pairs; i++) { 6812 i40e_clean_tx_ring(vsi->tx_rings[i]); 6813 if (i40e_enabled_xdp_vsi(vsi)) { 6814 /* Make sure that in-progress ndo_xdp_xmit and 6815 * ndo_xsk_wakeup calls are completed. 6816 */ 6817 synchronize_rcu(); 6818 i40e_clean_tx_ring(vsi->xdp_rings[i]); 6819 } 6820 i40e_clean_rx_ring(vsi->rx_rings[i]); 6821 } 6822 6823 } 6824 6825 /** 6826 * i40e_validate_mqprio_qopt- validate queue mapping info 6827 * @vsi: the VSI being configured 6828 * @mqprio_qopt: queue parametrs 6829 **/ 6830 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 6831 struct tc_mqprio_qopt_offload *mqprio_qopt) 6832 { 6833 u64 sum_max_rate = 0; 6834 u64 max_rate = 0; 6835 int i; 6836 6837 if (mqprio_qopt->qopt.offset[0] != 0 || 6838 mqprio_qopt->qopt.num_tc < 1 || 6839 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 6840 return -EINVAL; 6841 for (i = 0; ; i++) { 6842 if (!mqprio_qopt->qopt.count[i]) 6843 return -EINVAL; 6844 if (mqprio_qopt->min_rate[i]) { 6845 dev_err(&vsi->back->pdev->dev, 6846 "Invalid min tx rate (greater than 0) specified\n"); 6847 return -EINVAL; 6848 } 6849 max_rate = mqprio_qopt->max_rate[i]; 6850 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6851 sum_max_rate += max_rate; 6852 6853 if (i >= mqprio_qopt->qopt.num_tc - 1) 6854 break; 6855 if (mqprio_qopt->qopt.offset[i + 1] != 6856 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 6857 return -EINVAL; 6858 } 6859 if (vsi->num_queue_pairs < 6860 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 6861 return -EINVAL; 6862 } 6863 if (sum_max_rate > i40e_get_link_speed(vsi)) { 6864 dev_err(&vsi->back->pdev->dev, 6865 "Invalid max tx rate specified\n"); 6866 return -EINVAL; 6867 } 6868 return 0; 6869 } 6870 6871 /** 6872 * i40e_vsi_set_default_tc_config - set default values for tc configuration 6873 * @vsi: the VSI being configured 6874 **/ 6875 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 6876 { 6877 u16 qcount; 6878 int i; 6879 6880 /* Only TC0 is enabled */ 6881 vsi->tc_config.numtc = 1; 6882 vsi->tc_config.enabled_tc = 1; 6883 qcount = min_t(int, vsi->alloc_queue_pairs, 6884 i40e_pf_get_max_q_per_tc(vsi->back)); 6885 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6886 /* For the TC that is not enabled set the offset to to default 6887 * queue and allocate one queue for the given TC. 6888 */ 6889 vsi->tc_config.tc_info[i].qoffset = 0; 6890 if (i == 0) 6891 vsi->tc_config.tc_info[i].qcount = qcount; 6892 else 6893 vsi->tc_config.tc_info[i].qcount = 1; 6894 vsi->tc_config.tc_info[i].netdev_tc = 0; 6895 } 6896 } 6897 6898 /** 6899 * i40e_del_macvlan_filter 6900 * @hw: pointer to the HW structure 6901 * @seid: seid of the channel VSI 6902 * @macaddr: the mac address to apply as a filter 6903 * @aq_err: store the admin Q error 6904 * 6905 * This function deletes a mac filter on the channel VSI which serves as the 6906 * macvlan. Returns 0 on success. 6907 **/ 6908 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid, 6909 const u8 *macaddr, int *aq_err) 6910 { 6911 struct i40e_aqc_remove_macvlan_element_data element; 6912 i40e_status status; 6913 6914 memset(&element, 0, sizeof(element)); 6915 ether_addr_copy(element.mac_addr, macaddr); 6916 element.vlan_tag = 0; 6917 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 6918 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL); 6919 *aq_err = hw->aq.asq_last_status; 6920 6921 return status; 6922 } 6923 6924 /** 6925 * i40e_add_macvlan_filter 6926 * @hw: pointer to the HW structure 6927 * @seid: seid of the channel VSI 6928 * @macaddr: the mac address to apply as a filter 6929 * @aq_err: store the admin Q error 6930 * 6931 * This function adds a mac filter on the channel VSI which serves as the 6932 * macvlan. Returns 0 on success. 6933 **/ 6934 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid, 6935 const u8 *macaddr, int *aq_err) 6936 { 6937 struct i40e_aqc_add_macvlan_element_data element; 6938 i40e_status status; 6939 u16 cmd_flags = 0; 6940 6941 ether_addr_copy(element.mac_addr, macaddr); 6942 element.vlan_tag = 0; 6943 element.queue_number = 0; 6944 element.match_method = I40E_AQC_MM_ERR_NO_RES; 6945 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 6946 element.flags = cpu_to_le16(cmd_flags); 6947 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL); 6948 *aq_err = hw->aq.asq_last_status; 6949 6950 return status; 6951 } 6952 6953 /** 6954 * i40e_reset_ch_rings - Reset the queue contexts in a channel 6955 * @vsi: the VSI we want to access 6956 * @ch: the channel we want to access 6957 */ 6958 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch) 6959 { 6960 struct i40e_ring *tx_ring, *rx_ring; 6961 u16 pf_q; 6962 int i; 6963 6964 for (i = 0; i < ch->num_queue_pairs; i++) { 6965 pf_q = ch->base_queue + i; 6966 tx_ring = vsi->tx_rings[pf_q]; 6967 tx_ring->ch = NULL; 6968 rx_ring = vsi->rx_rings[pf_q]; 6969 rx_ring->ch = NULL; 6970 } 6971 } 6972 6973 /** 6974 * i40e_free_macvlan_channels 6975 * @vsi: the VSI we want to access 6976 * 6977 * This function frees the Qs of the channel VSI from 6978 * the stack and also deletes the channel VSIs which 6979 * serve as macvlans. 6980 */ 6981 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi) 6982 { 6983 struct i40e_channel *ch, *ch_tmp; 6984 int ret; 6985 6986 if (list_empty(&vsi->macvlan_list)) 6987 return; 6988 6989 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 6990 struct i40e_vsi *parent_vsi; 6991 6992 if (i40e_is_channel_macvlan(ch)) { 6993 i40e_reset_ch_rings(vsi, ch); 6994 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 6995 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev); 6996 netdev_set_sb_channel(ch->fwd->netdev, 0); 6997 kfree(ch->fwd); 6998 ch->fwd = NULL; 6999 } 7000 7001 list_del(&ch->list); 7002 parent_vsi = ch->parent_vsi; 7003 if (!parent_vsi || !ch->initialized) { 7004 kfree(ch); 7005 continue; 7006 } 7007 7008 /* remove the VSI */ 7009 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 7010 NULL); 7011 if (ret) 7012 dev_err(&vsi->back->pdev->dev, 7013 "unable to remove channel (%d) for parent VSI(%d)\n", 7014 ch->seid, parent_vsi->seid); 7015 kfree(ch); 7016 } 7017 vsi->macvlan_cnt = 0; 7018 } 7019 7020 /** 7021 * i40e_fwd_ring_up - bring the macvlan device up 7022 * @vsi: the VSI we want to access 7023 * @vdev: macvlan netdevice 7024 * @fwd: the private fwd structure 7025 */ 7026 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev, 7027 struct i40e_fwd_adapter *fwd) 7028 { 7029 int ret = 0, num_tc = 1, i, aq_err; 7030 struct i40e_channel *ch, *ch_tmp; 7031 struct i40e_pf *pf = vsi->back; 7032 struct i40e_hw *hw = &pf->hw; 7033 7034 if (list_empty(&vsi->macvlan_list)) 7035 return -EINVAL; 7036 7037 /* Go through the list and find an available channel */ 7038 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7039 if (!i40e_is_channel_macvlan(ch)) { 7040 ch->fwd = fwd; 7041 /* record configuration for macvlan interface in vdev */ 7042 for (i = 0; i < num_tc; i++) 7043 netdev_bind_sb_channel_queue(vsi->netdev, vdev, 7044 i, 7045 ch->num_queue_pairs, 7046 ch->base_queue); 7047 for (i = 0; i < ch->num_queue_pairs; i++) { 7048 struct i40e_ring *tx_ring, *rx_ring; 7049 u16 pf_q; 7050 7051 pf_q = ch->base_queue + i; 7052 7053 /* Get to TX ring ptr */ 7054 tx_ring = vsi->tx_rings[pf_q]; 7055 tx_ring->ch = ch; 7056 7057 /* Get the RX ring ptr */ 7058 rx_ring = vsi->rx_rings[pf_q]; 7059 rx_ring->ch = ch; 7060 } 7061 break; 7062 } 7063 } 7064 7065 /* Guarantee all rings are updated before we update the 7066 * MAC address filter. 7067 */ 7068 wmb(); 7069 7070 /* Add a mac filter */ 7071 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err); 7072 if (ret) { 7073 /* if we cannot add the MAC rule then disable the offload */ 7074 macvlan_release_l2fw_offload(vdev); 7075 for (i = 0; i < ch->num_queue_pairs; i++) { 7076 struct i40e_ring *rx_ring; 7077 u16 pf_q; 7078 7079 pf_q = ch->base_queue + i; 7080 rx_ring = vsi->rx_rings[pf_q]; 7081 rx_ring->netdev = NULL; 7082 } 7083 dev_info(&pf->pdev->dev, 7084 "Error adding mac filter on macvlan err %s, aq_err %s\n", 7085 i40e_stat_str(hw, ret), 7086 i40e_aq_str(hw, aq_err)); 7087 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n"); 7088 } 7089 7090 return ret; 7091 } 7092 7093 /** 7094 * i40e_setup_macvlans - create the channels which will be macvlans 7095 * @vsi: the VSI we want to access 7096 * @macvlan_cnt: no. of macvlans to be setup 7097 * @qcnt: no. of Qs per macvlan 7098 * @vdev: macvlan netdevice 7099 */ 7100 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt, 7101 struct net_device *vdev) 7102 { 7103 struct i40e_pf *pf = vsi->back; 7104 struct i40e_hw *hw = &pf->hw; 7105 struct i40e_vsi_context ctxt; 7106 u16 sections, qmap, num_qps; 7107 struct i40e_channel *ch; 7108 int i, pow, ret = 0; 7109 u8 offset = 0; 7110 7111 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt) 7112 return -EINVAL; 7113 7114 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt); 7115 7116 /* find the next higher power-of-2 of num queue pairs */ 7117 pow = fls(roundup_pow_of_two(num_qps) - 1); 7118 7119 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 7120 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 7121 7122 /* Setup context bits for the main VSI */ 7123 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 7124 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 7125 memset(&ctxt, 0, sizeof(ctxt)); 7126 ctxt.seid = vsi->seid; 7127 ctxt.pf_num = vsi->back->hw.pf_id; 7128 ctxt.vf_num = 0; 7129 ctxt.uplink_seid = vsi->uplink_seid; 7130 ctxt.info = vsi->info; 7131 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap); 7132 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 7133 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 7134 ctxt.info.valid_sections |= cpu_to_le16(sections); 7135 7136 /* Reconfigure RSS for main VSI with new max queue count */ 7137 vsi->rss_size = max_t(u16, num_qps, qcnt); 7138 ret = i40e_vsi_config_rss(vsi); 7139 if (ret) { 7140 dev_info(&pf->pdev->dev, 7141 "Failed to reconfig RSS for num_queues (%u)\n", 7142 vsi->rss_size); 7143 return ret; 7144 } 7145 vsi->reconfig_rss = true; 7146 dev_dbg(&vsi->back->pdev->dev, 7147 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size); 7148 vsi->next_base_queue = num_qps; 7149 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps; 7150 7151 /* Update the VSI after updating the VSI queue-mapping 7152 * information 7153 */ 7154 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7155 if (ret) { 7156 dev_info(&pf->pdev->dev, 7157 "Update vsi tc config failed, err %s aq_err %s\n", 7158 i40e_stat_str(hw, ret), 7159 i40e_aq_str(hw, hw->aq.asq_last_status)); 7160 return ret; 7161 } 7162 /* update the local VSI info with updated queue map */ 7163 i40e_vsi_update_queue_map(vsi, &ctxt); 7164 vsi->info.valid_sections = 0; 7165 7166 /* Create channels for macvlans */ 7167 INIT_LIST_HEAD(&vsi->macvlan_list); 7168 for (i = 0; i < macvlan_cnt; i++) { 7169 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 7170 if (!ch) { 7171 ret = -ENOMEM; 7172 goto err_free; 7173 } 7174 INIT_LIST_HEAD(&ch->list); 7175 ch->num_queue_pairs = qcnt; 7176 if (!i40e_setup_channel(pf, vsi, ch)) { 7177 ret = -EINVAL; 7178 kfree(ch); 7179 goto err_free; 7180 } 7181 ch->parent_vsi = vsi; 7182 vsi->cnt_q_avail -= ch->num_queue_pairs; 7183 vsi->macvlan_cnt++; 7184 list_add_tail(&ch->list, &vsi->macvlan_list); 7185 } 7186 7187 return ret; 7188 7189 err_free: 7190 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n"); 7191 i40e_free_macvlan_channels(vsi); 7192 7193 return ret; 7194 } 7195 7196 /** 7197 * i40e_fwd_add - configure macvlans 7198 * @netdev: net device to configure 7199 * @vdev: macvlan netdevice 7200 **/ 7201 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) 7202 { 7203 struct i40e_netdev_priv *np = netdev_priv(netdev); 7204 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors; 7205 struct i40e_vsi *vsi = np->vsi; 7206 struct i40e_pf *pf = vsi->back; 7207 struct i40e_fwd_adapter *fwd; 7208 int avail_macvlan, ret; 7209 7210 if ((pf->flags & I40E_FLAG_DCB_ENABLED)) { 7211 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); 7212 return ERR_PTR(-EINVAL); 7213 } 7214 if ((pf->flags & I40E_FLAG_TC_MQPRIO)) { 7215 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); 7216 return ERR_PTR(-EINVAL); 7217 } 7218 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) { 7219 netdev_info(netdev, "Not enough vectors available to support macvlans\n"); 7220 return ERR_PTR(-EINVAL); 7221 } 7222 7223 /* The macvlan device has to be a single Q device so that the 7224 * tc_to_txq field can be reused to pick the tx queue. 7225 */ 7226 if (netif_is_multiqueue(vdev)) 7227 return ERR_PTR(-ERANGE); 7228 7229 if (!vsi->macvlan_cnt) { 7230 /* reserve bit 0 for the pf device */ 7231 set_bit(0, vsi->fwd_bitmask); 7232 7233 /* Try to reserve as many queues as possible for macvlans. First 7234 * reserve 3/4th of max vectors, then half, then quarter and 7235 * calculate Qs per macvlan as you go 7236 */ 7237 vectors = pf->num_lan_msix; 7238 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) { 7239 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/ 7240 q_per_macvlan = 4; 7241 macvlan_cnt = (vectors - 32) / 4; 7242 } else if (vectors <= 64 && vectors > 32) { 7243 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/ 7244 q_per_macvlan = 2; 7245 macvlan_cnt = (vectors - 16) / 2; 7246 } else if (vectors <= 32 && vectors > 16) { 7247 /* allocate 1 Q per macvlan and 16 Qs to the PF*/ 7248 q_per_macvlan = 1; 7249 macvlan_cnt = vectors - 16; 7250 } else if (vectors <= 16 && vectors > 8) { 7251 /* allocate 1 Q per macvlan and 8 Qs to the PF */ 7252 q_per_macvlan = 1; 7253 macvlan_cnt = vectors - 8; 7254 } else { 7255 /* allocate 1 Q per macvlan and 1 Q to the PF */ 7256 q_per_macvlan = 1; 7257 macvlan_cnt = vectors - 1; 7258 } 7259 7260 if (macvlan_cnt == 0) 7261 return ERR_PTR(-EBUSY); 7262 7263 /* Quiesce VSI queues */ 7264 i40e_quiesce_vsi(vsi); 7265 7266 /* sets up the macvlans but does not "enable" them */ 7267 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan, 7268 vdev); 7269 if (ret) 7270 return ERR_PTR(ret); 7271 7272 /* Unquiesce VSI */ 7273 i40e_unquiesce_vsi(vsi); 7274 } 7275 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask, 7276 vsi->macvlan_cnt); 7277 if (avail_macvlan >= I40E_MAX_MACVLANS) 7278 return ERR_PTR(-EBUSY); 7279 7280 /* create the fwd struct */ 7281 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL); 7282 if (!fwd) 7283 return ERR_PTR(-ENOMEM); 7284 7285 set_bit(avail_macvlan, vsi->fwd_bitmask); 7286 fwd->bit_no = avail_macvlan; 7287 netdev_set_sb_channel(vdev, avail_macvlan); 7288 fwd->netdev = vdev; 7289 7290 if (!netif_running(netdev)) 7291 return fwd; 7292 7293 /* Set fwd ring up */ 7294 ret = i40e_fwd_ring_up(vsi, vdev, fwd); 7295 if (ret) { 7296 /* unbind the queues and drop the subordinate channel config */ 7297 netdev_unbind_sb_channel(netdev, vdev); 7298 netdev_set_sb_channel(vdev, 0); 7299 7300 kfree(fwd); 7301 return ERR_PTR(-EINVAL); 7302 } 7303 7304 return fwd; 7305 } 7306 7307 /** 7308 * i40e_del_all_macvlans - Delete all the mac filters on the channels 7309 * @vsi: the VSI we want to access 7310 */ 7311 static void i40e_del_all_macvlans(struct i40e_vsi *vsi) 7312 { 7313 struct i40e_channel *ch, *ch_tmp; 7314 struct i40e_pf *pf = vsi->back; 7315 struct i40e_hw *hw = &pf->hw; 7316 int aq_err, ret = 0; 7317 7318 if (list_empty(&vsi->macvlan_list)) 7319 return; 7320 7321 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7322 if (i40e_is_channel_macvlan(ch)) { 7323 ret = i40e_del_macvlan_filter(hw, ch->seid, 7324 i40e_channel_mac(ch), 7325 &aq_err); 7326 if (!ret) { 7327 /* Reset queue contexts */ 7328 i40e_reset_ch_rings(vsi, ch); 7329 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7330 netdev_unbind_sb_channel(vsi->netdev, 7331 ch->fwd->netdev); 7332 netdev_set_sb_channel(ch->fwd->netdev, 0); 7333 kfree(ch->fwd); 7334 ch->fwd = NULL; 7335 } 7336 } 7337 } 7338 } 7339 7340 /** 7341 * i40e_fwd_del - delete macvlan interfaces 7342 * @netdev: net device to configure 7343 * @vdev: macvlan netdevice 7344 */ 7345 static void i40e_fwd_del(struct net_device *netdev, void *vdev) 7346 { 7347 struct i40e_netdev_priv *np = netdev_priv(netdev); 7348 struct i40e_fwd_adapter *fwd = vdev; 7349 struct i40e_channel *ch, *ch_tmp; 7350 struct i40e_vsi *vsi = np->vsi; 7351 struct i40e_pf *pf = vsi->back; 7352 struct i40e_hw *hw = &pf->hw; 7353 int aq_err, ret = 0; 7354 7355 /* Find the channel associated with the macvlan and del mac filter */ 7356 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7357 if (i40e_is_channel_macvlan(ch) && 7358 ether_addr_equal(i40e_channel_mac(ch), 7359 fwd->netdev->dev_addr)) { 7360 ret = i40e_del_macvlan_filter(hw, ch->seid, 7361 i40e_channel_mac(ch), 7362 &aq_err); 7363 if (!ret) { 7364 /* Reset queue contexts */ 7365 i40e_reset_ch_rings(vsi, ch); 7366 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7367 netdev_unbind_sb_channel(netdev, fwd->netdev); 7368 netdev_set_sb_channel(fwd->netdev, 0); 7369 kfree(ch->fwd); 7370 ch->fwd = NULL; 7371 } else { 7372 dev_info(&pf->pdev->dev, 7373 "Error deleting mac filter on macvlan err %s, aq_err %s\n", 7374 i40e_stat_str(hw, ret), 7375 i40e_aq_str(hw, aq_err)); 7376 } 7377 break; 7378 } 7379 } 7380 } 7381 7382 /** 7383 * i40e_setup_tc - configure multiple traffic classes 7384 * @netdev: net device to configure 7385 * @type_data: tc offload data 7386 **/ 7387 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 7388 { 7389 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 7390 struct i40e_netdev_priv *np = netdev_priv(netdev); 7391 struct i40e_vsi *vsi = np->vsi; 7392 struct i40e_pf *pf = vsi->back; 7393 u8 enabled_tc = 0, num_tc, hw; 7394 bool need_reset = false; 7395 int old_queue_pairs; 7396 int ret = -EINVAL; 7397 u16 mode; 7398 int i; 7399 7400 old_queue_pairs = vsi->num_queue_pairs; 7401 num_tc = mqprio_qopt->qopt.num_tc; 7402 hw = mqprio_qopt->qopt.hw; 7403 mode = mqprio_qopt->mode; 7404 if (!hw) { 7405 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 7406 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 7407 goto config_tc; 7408 } 7409 7410 /* Check if MFP enabled */ 7411 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 7412 netdev_info(netdev, 7413 "Configuring TC not supported in MFP mode\n"); 7414 return ret; 7415 } 7416 switch (mode) { 7417 case TC_MQPRIO_MODE_DCB: 7418 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 7419 7420 /* Check if DCB enabled to continue */ 7421 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 7422 netdev_info(netdev, 7423 "DCB is not enabled for adapter\n"); 7424 return ret; 7425 } 7426 7427 /* Check whether tc count is within enabled limit */ 7428 if (num_tc > i40e_pf_get_num_tc(pf)) { 7429 netdev_info(netdev, 7430 "TC count greater than enabled on link for adapter\n"); 7431 return ret; 7432 } 7433 break; 7434 case TC_MQPRIO_MODE_CHANNEL: 7435 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 7436 netdev_info(netdev, 7437 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 7438 return ret; 7439 } 7440 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 7441 return ret; 7442 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 7443 if (ret) 7444 return ret; 7445 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 7446 sizeof(*mqprio_qopt)); 7447 pf->flags |= I40E_FLAG_TC_MQPRIO; 7448 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 7449 break; 7450 default: 7451 return -EINVAL; 7452 } 7453 7454 config_tc: 7455 /* Generate TC map for number of tc requested */ 7456 for (i = 0; i < num_tc; i++) 7457 enabled_tc |= BIT(i); 7458 7459 /* Requesting same TC configuration as already enabled */ 7460 if (enabled_tc == vsi->tc_config.enabled_tc && 7461 mode != TC_MQPRIO_MODE_CHANNEL) 7462 return 0; 7463 7464 /* Quiesce VSI queues */ 7465 i40e_quiesce_vsi(vsi); 7466 7467 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) 7468 i40e_remove_queue_channels(vsi); 7469 7470 /* Configure VSI for enabled TCs */ 7471 ret = i40e_vsi_config_tc(vsi, enabled_tc); 7472 if (ret) { 7473 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 7474 vsi->seid); 7475 need_reset = true; 7476 goto exit; 7477 } else { 7478 dev_info(&vsi->back->pdev->dev, 7479 "Setup channel (id:%u) utilizing num_queues %d\n", 7480 vsi->seid, vsi->tc_config.tc_info[0].qcount); 7481 } 7482 7483 if (pf->flags & I40E_FLAG_TC_MQPRIO) { 7484 if (vsi->mqprio_qopt.max_rate[0]) { 7485 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 7486 7487 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 7488 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 7489 if (!ret) { 7490 u64 credits = max_tx_rate; 7491 7492 do_div(credits, I40E_BW_CREDIT_DIVISOR); 7493 dev_dbg(&vsi->back->pdev->dev, 7494 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 7495 max_tx_rate, 7496 credits, 7497 vsi->seid); 7498 } else { 7499 need_reset = true; 7500 goto exit; 7501 } 7502 } 7503 ret = i40e_configure_queue_channels(vsi); 7504 if (ret) { 7505 vsi->num_queue_pairs = old_queue_pairs; 7506 netdev_info(netdev, 7507 "Failed configuring queue channels\n"); 7508 need_reset = true; 7509 goto exit; 7510 } 7511 } 7512 7513 exit: 7514 /* Reset the configuration data to defaults, only TC0 is enabled */ 7515 if (need_reset) { 7516 i40e_vsi_set_default_tc_config(vsi); 7517 need_reset = false; 7518 } 7519 7520 /* Unquiesce VSI */ 7521 i40e_unquiesce_vsi(vsi); 7522 return ret; 7523 } 7524 7525 /** 7526 * i40e_set_cld_element - sets cloud filter element data 7527 * @filter: cloud filter rule 7528 * @cld: ptr to cloud filter element data 7529 * 7530 * This is helper function to copy data into cloud filter element 7531 **/ 7532 static inline void 7533 i40e_set_cld_element(struct i40e_cloud_filter *filter, 7534 struct i40e_aqc_cloud_filters_element_data *cld) 7535 { 7536 int i, j; 7537 u32 ipa; 7538 7539 memset(cld, 0, sizeof(*cld)); 7540 ether_addr_copy(cld->outer_mac, filter->dst_mac); 7541 ether_addr_copy(cld->inner_mac, filter->src_mac); 7542 7543 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 7544 return; 7545 7546 if (filter->n_proto == ETH_P_IPV6) { 7547 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 7548 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); 7549 i++, j += 2) { 7550 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 7551 ipa = cpu_to_le32(ipa); 7552 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); 7553 } 7554 } else { 7555 ipa = be32_to_cpu(filter->dst_ipv4); 7556 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 7557 } 7558 7559 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 7560 7561 /* tenant_id is not supported by FW now, once the support is enabled 7562 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 7563 */ 7564 if (filter->tenant_id) 7565 return; 7566 } 7567 7568 /** 7569 * i40e_add_del_cloud_filter - Add/del cloud filter 7570 * @vsi: pointer to VSI 7571 * @filter: cloud filter rule 7572 * @add: if true, add, if false, delete 7573 * 7574 * Add or delete a cloud filter for a specific flow spec. 7575 * Returns 0 if the filter were successfully added. 7576 **/ 7577 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 7578 struct i40e_cloud_filter *filter, bool add) 7579 { 7580 struct i40e_aqc_cloud_filters_element_data cld_filter; 7581 struct i40e_pf *pf = vsi->back; 7582 int ret; 7583 static const u16 flag_table[128] = { 7584 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 7585 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 7586 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 7587 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 7588 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 7589 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 7590 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 7591 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 7592 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 7593 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 7594 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 7595 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 7596 [I40E_CLOUD_FILTER_FLAGS_IIP] = 7597 I40E_AQC_ADD_CLOUD_FILTER_IIP, 7598 }; 7599 7600 if (filter->flags >= ARRAY_SIZE(flag_table)) 7601 return I40E_ERR_CONFIG; 7602 7603 /* copy element needed to add cloud filter from filter */ 7604 i40e_set_cld_element(filter, &cld_filter); 7605 7606 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 7607 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 7608 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 7609 7610 if (filter->n_proto == ETH_P_IPV6) 7611 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7612 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7613 else 7614 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7615 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7616 7617 if (add) 7618 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 7619 &cld_filter, 1); 7620 else 7621 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 7622 &cld_filter, 1); 7623 if (ret) 7624 dev_dbg(&pf->pdev->dev, 7625 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 7626 add ? "add" : "delete", filter->dst_port, ret, 7627 pf->hw.aq.asq_last_status); 7628 else 7629 dev_info(&pf->pdev->dev, 7630 "%s cloud filter for VSI: %d\n", 7631 add ? "Added" : "Deleted", filter->seid); 7632 return ret; 7633 } 7634 7635 /** 7636 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 7637 * @vsi: pointer to VSI 7638 * @filter: cloud filter rule 7639 * @add: if true, add, if false, delete 7640 * 7641 * Add or delete a cloud filter for a specific flow spec using big buffer. 7642 * Returns 0 if the filter were successfully added. 7643 **/ 7644 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 7645 struct i40e_cloud_filter *filter, 7646 bool add) 7647 { 7648 struct i40e_aqc_cloud_filters_element_bb cld_filter; 7649 struct i40e_pf *pf = vsi->back; 7650 int ret; 7651 7652 /* Both (src/dst) valid mac_addr are not supported */ 7653 if ((is_valid_ether_addr(filter->dst_mac) && 7654 is_valid_ether_addr(filter->src_mac)) || 7655 (is_multicast_ether_addr(filter->dst_mac) && 7656 is_multicast_ether_addr(filter->src_mac))) 7657 return -EOPNOTSUPP; 7658 7659 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 7660 * ports are not supported via big buffer now. 7661 */ 7662 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 7663 return -EOPNOTSUPP; 7664 7665 /* adding filter using src_port/src_ip is not supported at this stage */ 7666 if (filter->src_port || filter->src_ipv4 || 7667 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 7668 return -EOPNOTSUPP; 7669 7670 /* copy element needed to add cloud filter from filter */ 7671 i40e_set_cld_element(filter, &cld_filter.element); 7672 7673 if (is_valid_ether_addr(filter->dst_mac) || 7674 is_valid_ether_addr(filter->src_mac) || 7675 is_multicast_ether_addr(filter->dst_mac) || 7676 is_multicast_ether_addr(filter->src_mac)) { 7677 /* MAC + IP : unsupported mode */ 7678 if (filter->dst_ipv4) 7679 return -EOPNOTSUPP; 7680 7681 /* since we validated that L4 port must be valid before 7682 * we get here, start with respective "flags" value 7683 * and update if vlan is present or not 7684 */ 7685 cld_filter.element.flags = 7686 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 7687 7688 if (filter->vlan_id) { 7689 cld_filter.element.flags = 7690 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 7691 } 7692 7693 } else if (filter->dst_ipv4 || 7694 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 7695 cld_filter.element.flags = 7696 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 7697 if (filter->n_proto == ETH_P_IPV6) 7698 cld_filter.element.flags |= 7699 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7700 else 7701 cld_filter.element.flags |= 7702 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7703 } else { 7704 dev_err(&pf->pdev->dev, 7705 "either mac or ip has to be valid for cloud filter\n"); 7706 return -EINVAL; 7707 } 7708 7709 /* Now copy L4 port in Byte 6..7 in general fields */ 7710 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 7711 be16_to_cpu(filter->dst_port); 7712 7713 if (add) { 7714 /* Validate current device switch mode, change if necessary */ 7715 ret = i40e_validate_and_set_switch_mode(vsi); 7716 if (ret) { 7717 dev_err(&pf->pdev->dev, 7718 "failed to set switch mode, ret %d\n", 7719 ret); 7720 return ret; 7721 } 7722 7723 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 7724 &cld_filter, 1); 7725 } else { 7726 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 7727 &cld_filter, 1); 7728 } 7729 7730 if (ret) 7731 dev_dbg(&pf->pdev->dev, 7732 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 7733 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 7734 else 7735 dev_info(&pf->pdev->dev, 7736 "%s cloud filter for VSI: %d, L4 port: %d\n", 7737 add ? "add" : "delete", filter->seid, 7738 ntohs(filter->dst_port)); 7739 return ret; 7740 } 7741 7742 /** 7743 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 7744 * @vsi: Pointer to VSI 7745 * @cls_flower: Pointer to struct flow_cls_offload 7746 * @filter: Pointer to cloud filter structure 7747 * 7748 **/ 7749 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 7750 struct flow_cls_offload *f, 7751 struct i40e_cloud_filter *filter) 7752 { 7753 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 7754 struct flow_dissector *dissector = rule->match.dissector; 7755 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 7756 struct i40e_pf *pf = vsi->back; 7757 u8 field_flags = 0; 7758 7759 if (dissector->used_keys & 7760 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 7761 BIT(FLOW_DISSECTOR_KEY_BASIC) | 7762 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 7763 BIT(FLOW_DISSECTOR_KEY_VLAN) | 7764 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 7765 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 7766 BIT(FLOW_DISSECTOR_KEY_PORTS) | 7767 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 7768 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 7769 dissector->used_keys); 7770 return -EOPNOTSUPP; 7771 } 7772 7773 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 7774 struct flow_match_enc_keyid match; 7775 7776 flow_rule_match_enc_keyid(rule, &match); 7777 if (match.mask->keyid != 0) 7778 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 7779 7780 filter->tenant_id = be32_to_cpu(match.key->keyid); 7781 } 7782 7783 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 7784 struct flow_match_basic match; 7785 7786 flow_rule_match_basic(rule, &match); 7787 n_proto_key = ntohs(match.key->n_proto); 7788 n_proto_mask = ntohs(match.mask->n_proto); 7789 7790 if (n_proto_key == ETH_P_ALL) { 7791 n_proto_key = 0; 7792 n_proto_mask = 0; 7793 } 7794 filter->n_proto = n_proto_key & n_proto_mask; 7795 filter->ip_proto = match.key->ip_proto; 7796 } 7797 7798 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 7799 struct flow_match_eth_addrs match; 7800 7801 flow_rule_match_eth_addrs(rule, &match); 7802 7803 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 7804 if (!is_zero_ether_addr(match.mask->dst)) { 7805 if (is_broadcast_ether_addr(match.mask->dst)) { 7806 field_flags |= I40E_CLOUD_FIELD_OMAC; 7807 } else { 7808 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 7809 match.mask->dst); 7810 return I40E_ERR_CONFIG; 7811 } 7812 } 7813 7814 if (!is_zero_ether_addr(match.mask->src)) { 7815 if (is_broadcast_ether_addr(match.mask->src)) { 7816 field_flags |= I40E_CLOUD_FIELD_IMAC; 7817 } else { 7818 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 7819 match.mask->src); 7820 return I40E_ERR_CONFIG; 7821 } 7822 } 7823 ether_addr_copy(filter->dst_mac, match.key->dst); 7824 ether_addr_copy(filter->src_mac, match.key->src); 7825 } 7826 7827 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 7828 struct flow_match_vlan match; 7829 7830 flow_rule_match_vlan(rule, &match); 7831 if (match.mask->vlan_id) { 7832 if (match.mask->vlan_id == VLAN_VID_MASK) { 7833 field_flags |= I40E_CLOUD_FIELD_IVLAN; 7834 7835 } else { 7836 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 7837 match.mask->vlan_id); 7838 return I40E_ERR_CONFIG; 7839 } 7840 } 7841 7842 filter->vlan_id = cpu_to_be16(match.key->vlan_id); 7843 } 7844 7845 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 7846 struct flow_match_control match; 7847 7848 flow_rule_match_control(rule, &match); 7849 addr_type = match.key->addr_type; 7850 } 7851 7852 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 7853 struct flow_match_ipv4_addrs match; 7854 7855 flow_rule_match_ipv4_addrs(rule, &match); 7856 if (match.mask->dst) { 7857 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 7858 field_flags |= I40E_CLOUD_FIELD_IIP; 7859 } else { 7860 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 7861 &match.mask->dst); 7862 return I40E_ERR_CONFIG; 7863 } 7864 } 7865 7866 if (match.mask->src) { 7867 if (match.mask->src == cpu_to_be32(0xffffffff)) { 7868 field_flags |= I40E_CLOUD_FIELD_IIP; 7869 } else { 7870 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 7871 &match.mask->src); 7872 return I40E_ERR_CONFIG; 7873 } 7874 } 7875 7876 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 7877 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 7878 return I40E_ERR_CONFIG; 7879 } 7880 filter->dst_ipv4 = match.key->dst; 7881 filter->src_ipv4 = match.key->src; 7882 } 7883 7884 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 7885 struct flow_match_ipv6_addrs match; 7886 7887 flow_rule_match_ipv6_addrs(rule, &match); 7888 7889 /* src and dest IPV6 address should not be LOOPBACK 7890 * (0:0:0:0:0:0:0:1), which can be represented as ::1 7891 */ 7892 if (ipv6_addr_loopback(&match.key->dst) || 7893 ipv6_addr_loopback(&match.key->src)) { 7894 dev_err(&pf->pdev->dev, 7895 "Bad ipv6, addr is LOOPBACK\n"); 7896 return I40E_ERR_CONFIG; 7897 } 7898 if (!ipv6_addr_any(&match.mask->dst) || 7899 !ipv6_addr_any(&match.mask->src)) 7900 field_flags |= I40E_CLOUD_FIELD_IIP; 7901 7902 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32, 7903 sizeof(filter->src_ipv6)); 7904 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32, 7905 sizeof(filter->dst_ipv6)); 7906 } 7907 7908 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 7909 struct flow_match_ports match; 7910 7911 flow_rule_match_ports(rule, &match); 7912 if (match.mask->src) { 7913 if (match.mask->src == cpu_to_be16(0xffff)) { 7914 field_flags |= I40E_CLOUD_FIELD_IIP; 7915 } else { 7916 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 7917 be16_to_cpu(match.mask->src)); 7918 return I40E_ERR_CONFIG; 7919 } 7920 } 7921 7922 if (match.mask->dst) { 7923 if (match.mask->dst == cpu_to_be16(0xffff)) { 7924 field_flags |= I40E_CLOUD_FIELD_IIP; 7925 } else { 7926 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 7927 be16_to_cpu(match.mask->dst)); 7928 return I40E_ERR_CONFIG; 7929 } 7930 } 7931 7932 filter->dst_port = match.key->dst; 7933 filter->src_port = match.key->src; 7934 7935 switch (filter->ip_proto) { 7936 case IPPROTO_TCP: 7937 case IPPROTO_UDP: 7938 break; 7939 default: 7940 dev_err(&pf->pdev->dev, 7941 "Only UDP and TCP transport are supported\n"); 7942 return -EINVAL; 7943 } 7944 } 7945 filter->flags = field_flags; 7946 return 0; 7947 } 7948 7949 /** 7950 * i40e_handle_tclass: Forward to a traffic class on the device 7951 * @vsi: Pointer to VSI 7952 * @tc: traffic class index on the device 7953 * @filter: Pointer to cloud filter structure 7954 * 7955 **/ 7956 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 7957 struct i40e_cloud_filter *filter) 7958 { 7959 struct i40e_channel *ch, *ch_tmp; 7960 7961 /* direct to a traffic class on the same device */ 7962 if (tc == 0) { 7963 filter->seid = vsi->seid; 7964 return 0; 7965 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 7966 if (!filter->dst_port) { 7967 dev_err(&vsi->back->pdev->dev, 7968 "Specify destination port to direct to traffic class that is not default\n"); 7969 return -EINVAL; 7970 } 7971 if (list_empty(&vsi->ch_list)) 7972 return -EINVAL; 7973 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 7974 list) { 7975 if (ch->seid == vsi->tc_seid_map[tc]) 7976 filter->seid = ch->seid; 7977 } 7978 return 0; 7979 } 7980 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 7981 return -EINVAL; 7982 } 7983 7984 /** 7985 * i40e_configure_clsflower - Configure tc flower filters 7986 * @vsi: Pointer to VSI 7987 * @cls_flower: Pointer to struct flow_cls_offload 7988 * 7989 **/ 7990 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 7991 struct flow_cls_offload *cls_flower) 7992 { 7993 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 7994 struct i40e_cloud_filter *filter = NULL; 7995 struct i40e_pf *pf = vsi->back; 7996 int err = 0; 7997 7998 if (tc < 0) { 7999 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 8000 return -EOPNOTSUPP; 8001 } 8002 8003 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 8004 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 8005 return -EBUSY; 8006 8007 if (pf->fdir_pf_active_filters || 8008 (!hlist_empty(&pf->fdir_filter_list))) { 8009 dev_err(&vsi->back->pdev->dev, 8010 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 8011 return -EINVAL; 8012 } 8013 8014 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 8015 dev_err(&vsi->back->pdev->dev, 8016 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 8017 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8018 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8019 } 8020 8021 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 8022 if (!filter) 8023 return -ENOMEM; 8024 8025 filter->cookie = cls_flower->cookie; 8026 8027 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 8028 if (err < 0) 8029 goto err; 8030 8031 err = i40e_handle_tclass(vsi, tc, filter); 8032 if (err < 0) 8033 goto err; 8034 8035 /* Add cloud filter */ 8036 if (filter->dst_port) 8037 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 8038 else 8039 err = i40e_add_del_cloud_filter(vsi, filter, true); 8040 8041 if (err) { 8042 dev_err(&pf->pdev->dev, 8043 "Failed to add cloud filter, err %s\n", 8044 i40e_stat_str(&pf->hw, err)); 8045 goto err; 8046 } 8047 8048 /* add filter to the ordered list */ 8049 INIT_HLIST_NODE(&filter->cloud_node); 8050 8051 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 8052 8053 pf->num_cloud_filters++; 8054 8055 return err; 8056 err: 8057 kfree(filter); 8058 return err; 8059 } 8060 8061 /** 8062 * i40e_find_cloud_filter - Find the could filter in the list 8063 * @vsi: Pointer to VSI 8064 * @cookie: filter specific cookie 8065 * 8066 **/ 8067 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 8068 unsigned long *cookie) 8069 { 8070 struct i40e_cloud_filter *filter = NULL; 8071 struct hlist_node *node2; 8072 8073 hlist_for_each_entry_safe(filter, node2, 8074 &vsi->back->cloud_filter_list, cloud_node) 8075 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 8076 return filter; 8077 return NULL; 8078 } 8079 8080 /** 8081 * i40e_delete_clsflower - Remove tc flower filters 8082 * @vsi: Pointer to VSI 8083 * @cls_flower: Pointer to struct flow_cls_offload 8084 * 8085 **/ 8086 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 8087 struct flow_cls_offload *cls_flower) 8088 { 8089 struct i40e_cloud_filter *filter = NULL; 8090 struct i40e_pf *pf = vsi->back; 8091 int err = 0; 8092 8093 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 8094 8095 if (!filter) 8096 return -EINVAL; 8097 8098 hash_del(&filter->cloud_node); 8099 8100 if (filter->dst_port) 8101 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 8102 else 8103 err = i40e_add_del_cloud_filter(vsi, filter, false); 8104 8105 kfree(filter); 8106 if (err) { 8107 dev_err(&pf->pdev->dev, 8108 "Failed to delete cloud filter, err %s\n", 8109 i40e_stat_str(&pf->hw, err)); 8110 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 8111 } 8112 8113 pf->num_cloud_filters--; 8114 if (!pf->num_cloud_filters) 8115 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8116 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8117 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8118 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8119 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8120 } 8121 return 0; 8122 } 8123 8124 /** 8125 * i40e_setup_tc_cls_flower - flower classifier offloads 8126 * @netdev: net device to configure 8127 * @type_data: offload data 8128 **/ 8129 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 8130 struct flow_cls_offload *cls_flower) 8131 { 8132 struct i40e_vsi *vsi = np->vsi; 8133 8134 switch (cls_flower->command) { 8135 case FLOW_CLS_REPLACE: 8136 return i40e_configure_clsflower(vsi, cls_flower); 8137 case FLOW_CLS_DESTROY: 8138 return i40e_delete_clsflower(vsi, cls_flower); 8139 case FLOW_CLS_STATS: 8140 return -EOPNOTSUPP; 8141 default: 8142 return -EOPNOTSUPP; 8143 } 8144 } 8145 8146 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8147 void *cb_priv) 8148 { 8149 struct i40e_netdev_priv *np = cb_priv; 8150 8151 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 8152 return -EOPNOTSUPP; 8153 8154 switch (type) { 8155 case TC_SETUP_CLSFLOWER: 8156 return i40e_setup_tc_cls_flower(np, type_data); 8157 8158 default: 8159 return -EOPNOTSUPP; 8160 } 8161 } 8162 8163 static LIST_HEAD(i40e_block_cb_list); 8164 8165 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8166 void *type_data) 8167 { 8168 struct i40e_netdev_priv *np = netdev_priv(netdev); 8169 8170 switch (type) { 8171 case TC_SETUP_QDISC_MQPRIO: 8172 return i40e_setup_tc(netdev, type_data); 8173 case TC_SETUP_BLOCK: 8174 return flow_block_cb_setup_simple(type_data, 8175 &i40e_block_cb_list, 8176 i40e_setup_tc_block_cb, 8177 np, np, true); 8178 default: 8179 return -EOPNOTSUPP; 8180 } 8181 } 8182 8183 /** 8184 * i40e_open - Called when a network interface is made active 8185 * @netdev: network interface device structure 8186 * 8187 * The open entry point is called when a network interface is made 8188 * active by the system (IFF_UP). At this point all resources needed 8189 * for transmit and receive operations are allocated, the interrupt 8190 * handler is registered with the OS, the netdev watchdog subtask is 8191 * enabled, and the stack is notified that the interface is ready. 8192 * 8193 * Returns 0 on success, negative value on failure 8194 **/ 8195 int i40e_open(struct net_device *netdev) 8196 { 8197 struct i40e_netdev_priv *np = netdev_priv(netdev); 8198 struct i40e_vsi *vsi = np->vsi; 8199 struct i40e_pf *pf = vsi->back; 8200 int err; 8201 8202 /* disallow open during test or if eeprom is broken */ 8203 if (test_bit(__I40E_TESTING, pf->state) || 8204 test_bit(__I40E_BAD_EEPROM, pf->state)) 8205 return -EBUSY; 8206 8207 netif_carrier_off(netdev); 8208 8209 if (i40e_force_link_state(pf, true)) 8210 return -EAGAIN; 8211 8212 err = i40e_vsi_open(vsi); 8213 if (err) 8214 return err; 8215 8216 /* configure global TSO hardware offload settings */ 8217 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 8218 TCP_FLAG_FIN) >> 16); 8219 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 8220 TCP_FLAG_FIN | 8221 TCP_FLAG_CWR) >> 16); 8222 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 8223 8224 udp_tunnel_get_rx_info(netdev); 8225 8226 return 0; 8227 } 8228 8229 /** 8230 * i40e_vsi_open - 8231 * @vsi: the VSI to open 8232 * 8233 * Finish initialization of the VSI. 8234 * 8235 * Returns 0 on success, negative value on failure 8236 * 8237 * Note: expects to be called while under rtnl_lock() 8238 **/ 8239 int i40e_vsi_open(struct i40e_vsi *vsi) 8240 { 8241 struct i40e_pf *pf = vsi->back; 8242 char int_name[I40E_INT_NAME_STR_LEN]; 8243 int err; 8244 8245 /* allocate descriptors */ 8246 err = i40e_vsi_setup_tx_resources(vsi); 8247 if (err) 8248 goto err_setup_tx; 8249 err = i40e_vsi_setup_rx_resources(vsi); 8250 if (err) 8251 goto err_setup_rx; 8252 8253 err = i40e_vsi_configure(vsi); 8254 if (err) 8255 goto err_setup_rx; 8256 8257 if (vsi->netdev) { 8258 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 8259 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 8260 err = i40e_vsi_request_irq(vsi, int_name); 8261 if (err) 8262 goto err_setup_rx; 8263 8264 /* Notify the stack of the actual queue counts. */ 8265 err = netif_set_real_num_tx_queues(vsi->netdev, 8266 vsi->num_queue_pairs); 8267 if (err) 8268 goto err_set_queues; 8269 8270 err = netif_set_real_num_rx_queues(vsi->netdev, 8271 vsi->num_queue_pairs); 8272 if (err) 8273 goto err_set_queues; 8274 8275 } else if (vsi->type == I40E_VSI_FDIR) { 8276 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 8277 dev_driver_string(&pf->pdev->dev), 8278 dev_name(&pf->pdev->dev)); 8279 err = i40e_vsi_request_irq(vsi, int_name); 8280 8281 } else { 8282 err = -EINVAL; 8283 goto err_setup_rx; 8284 } 8285 8286 err = i40e_up_complete(vsi); 8287 if (err) 8288 goto err_up_complete; 8289 8290 return 0; 8291 8292 err_up_complete: 8293 i40e_down(vsi); 8294 err_set_queues: 8295 i40e_vsi_free_irq(vsi); 8296 err_setup_rx: 8297 i40e_vsi_free_rx_resources(vsi); 8298 err_setup_tx: 8299 i40e_vsi_free_tx_resources(vsi); 8300 if (vsi == pf->vsi[pf->lan_vsi]) 8301 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 8302 8303 return err; 8304 } 8305 8306 /** 8307 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 8308 * @pf: Pointer to PF 8309 * 8310 * This function destroys the hlist where all the Flow Director 8311 * filters were saved. 8312 **/ 8313 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 8314 { 8315 struct i40e_fdir_filter *filter; 8316 struct i40e_flex_pit *pit_entry, *tmp; 8317 struct hlist_node *node2; 8318 8319 hlist_for_each_entry_safe(filter, node2, 8320 &pf->fdir_filter_list, fdir_node) { 8321 hlist_del(&filter->fdir_node); 8322 kfree(filter); 8323 } 8324 8325 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 8326 list_del(&pit_entry->list); 8327 kfree(pit_entry); 8328 } 8329 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 8330 8331 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 8332 list_del(&pit_entry->list); 8333 kfree(pit_entry); 8334 } 8335 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 8336 8337 pf->fdir_pf_active_filters = 0; 8338 pf->fd_tcp4_filter_cnt = 0; 8339 pf->fd_udp4_filter_cnt = 0; 8340 pf->fd_sctp4_filter_cnt = 0; 8341 pf->fd_ip4_filter_cnt = 0; 8342 8343 /* Reprogram the default input set for TCP/IPv4 */ 8344 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8345 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8346 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8347 8348 /* Reprogram the default input set for UDP/IPv4 */ 8349 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 8350 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8351 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8352 8353 /* Reprogram the default input set for SCTP/IPv4 */ 8354 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 8355 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8356 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8357 8358 /* Reprogram the default input set for Other/IPv4 */ 8359 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 8360 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 8361 8362 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 8363 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 8364 } 8365 8366 /** 8367 * i40e_cloud_filter_exit - Cleans up the cloud filters 8368 * @pf: Pointer to PF 8369 * 8370 * This function destroys the hlist where all the cloud filters 8371 * were saved. 8372 **/ 8373 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 8374 { 8375 struct i40e_cloud_filter *cfilter; 8376 struct hlist_node *node; 8377 8378 hlist_for_each_entry_safe(cfilter, node, 8379 &pf->cloud_filter_list, cloud_node) { 8380 hlist_del(&cfilter->cloud_node); 8381 kfree(cfilter); 8382 } 8383 pf->num_cloud_filters = 0; 8384 8385 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8386 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8387 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8388 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8389 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8390 } 8391 } 8392 8393 /** 8394 * i40e_close - Disables a network interface 8395 * @netdev: network interface device structure 8396 * 8397 * The close entry point is called when an interface is de-activated 8398 * by the OS. The hardware is still under the driver's control, but 8399 * this netdev interface is disabled. 8400 * 8401 * Returns 0, this is not allowed to fail 8402 **/ 8403 int i40e_close(struct net_device *netdev) 8404 { 8405 struct i40e_netdev_priv *np = netdev_priv(netdev); 8406 struct i40e_vsi *vsi = np->vsi; 8407 8408 i40e_vsi_close(vsi); 8409 8410 return 0; 8411 } 8412 8413 /** 8414 * i40e_do_reset - Start a PF or Core Reset sequence 8415 * @pf: board private structure 8416 * @reset_flags: which reset is requested 8417 * @lock_acquired: indicates whether or not the lock has been acquired 8418 * before this function was called. 8419 * 8420 * The essential difference in resets is that the PF Reset 8421 * doesn't clear the packet buffers, doesn't reset the PE 8422 * firmware, and doesn't bother the other PFs on the chip. 8423 **/ 8424 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 8425 { 8426 u32 val; 8427 8428 WARN_ON(in_interrupt()); 8429 8430 8431 /* do the biggest reset indicated */ 8432 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 8433 8434 /* Request a Global Reset 8435 * 8436 * This will start the chip's countdown to the actual full 8437 * chip reset event, and a warning interrupt to be sent 8438 * to all PFs, including the requestor. Our handler 8439 * for the warning interrupt will deal with the shutdown 8440 * and recovery of the switch setup. 8441 */ 8442 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 8443 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8444 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 8445 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 8446 8447 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 8448 8449 /* Request a Core Reset 8450 * 8451 * Same as Global Reset, except does *not* include the MAC/PHY 8452 */ 8453 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 8454 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8455 val |= I40E_GLGEN_RTRIG_CORER_MASK; 8456 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 8457 i40e_flush(&pf->hw); 8458 8459 } else if (reset_flags & I40E_PF_RESET_FLAG) { 8460 8461 /* Request a PF Reset 8462 * 8463 * Resets only the PF-specific registers 8464 * 8465 * This goes directly to the tear-down and rebuild of 8466 * the switch, since we need to do all the recovery as 8467 * for the Core Reset. 8468 */ 8469 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 8470 i40e_handle_reset_warning(pf, lock_acquired); 8471 8472 dev_info(&pf->pdev->dev, 8473 pf->flags & I40E_FLAG_DISABLE_FW_LLDP ? 8474 "FW LLDP is disabled\n" : 8475 "FW LLDP is enabled\n"); 8476 8477 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 8478 int v; 8479 8480 /* Find the VSI(s) that requested a re-init */ 8481 dev_info(&pf->pdev->dev, 8482 "VSI reinit requested\n"); 8483 for (v = 0; v < pf->num_alloc_vsi; v++) { 8484 struct i40e_vsi *vsi = pf->vsi[v]; 8485 8486 if (vsi != NULL && 8487 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 8488 vsi->state)) 8489 i40e_vsi_reinit_locked(pf->vsi[v]); 8490 } 8491 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 8492 int v; 8493 8494 /* Find the VSI(s) that needs to be brought down */ 8495 dev_info(&pf->pdev->dev, "VSI down requested\n"); 8496 for (v = 0; v < pf->num_alloc_vsi; v++) { 8497 struct i40e_vsi *vsi = pf->vsi[v]; 8498 8499 if (vsi != NULL && 8500 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 8501 vsi->state)) { 8502 set_bit(__I40E_VSI_DOWN, vsi->state); 8503 i40e_down(vsi); 8504 } 8505 } 8506 } else { 8507 dev_info(&pf->pdev->dev, 8508 "bad reset request 0x%08x\n", reset_flags); 8509 } 8510 } 8511 8512 #ifdef CONFIG_I40E_DCB 8513 /** 8514 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 8515 * @pf: board private structure 8516 * @old_cfg: current DCB config 8517 * @new_cfg: new DCB config 8518 **/ 8519 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 8520 struct i40e_dcbx_config *old_cfg, 8521 struct i40e_dcbx_config *new_cfg) 8522 { 8523 bool need_reconfig = false; 8524 8525 /* Check if ETS configuration has changed */ 8526 if (memcmp(&new_cfg->etscfg, 8527 &old_cfg->etscfg, 8528 sizeof(new_cfg->etscfg))) { 8529 /* If Priority Table has changed reconfig is needed */ 8530 if (memcmp(&new_cfg->etscfg.prioritytable, 8531 &old_cfg->etscfg.prioritytable, 8532 sizeof(new_cfg->etscfg.prioritytable))) { 8533 need_reconfig = true; 8534 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 8535 } 8536 8537 if (memcmp(&new_cfg->etscfg.tcbwtable, 8538 &old_cfg->etscfg.tcbwtable, 8539 sizeof(new_cfg->etscfg.tcbwtable))) 8540 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 8541 8542 if (memcmp(&new_cfg->etscfg.tsatable, 8543 &old_cfg->etscfg.tsatable, 8544 sizeof(new_cfg->etscfg.tsatable))) 8545 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 8546 } 8547 8548 /* Check if PFC configuration has changed */ 8549 if (memcmp(&new_cfg->pfc, 8550 &old_cfg->pfc, 8551 sizeof(new_cfg->pfc))) { 8552 need_reconfig = true; 8553 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 8554 } 8555 8556 /* Check if APP Table has changed */ 8557 if (memcmp(&new_cfg->app, 8558 &old_cfg->app, 8559 sizeof(new_cfg->app))) { 8560 need_reconfig = true; 8561 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 8562 } 8563 8564 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 8565 return need_reconfig; 8566 } 8567 8568 /** 8569 * i40e_handle_lldp_event - Handle LLDP Change MIB event 8570 * @pf: board private structure 8571 * @e: event info posted on ARQ 8572 **/ 8573 static int i40e_handle_lldp_event(struct i40e_pf *pf, 8574 struct i40e_arq_event_info *e) 8575 { 8576 struct i40e_aqc_lldp_get_mib *mib = 8577 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 8578 struct i40e_hw *hw = &pf->hw; 8579 struct i40e_dcbx_config tmp_dcbx_cfg; 8580 bool need_reconfig = false; 8581 int ret = 0; 8582 u8 type; 8583 8584 /* Not DCB capable or capability disabled */ 8585 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 8586 return ret; 8587 8588 /* Ignore if event is not for Nearest Bridge */ 8589 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 8590 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 8591 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 8592 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 8593 return ret; 8594 8595 /* Check MIB Type and return if event for Remote MIB update */ 8596 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 8597 dev_dbg(&pf->pdev->dev, 8598 "LLDP event mib type %s\n", type ? "remote" : "local"); 8599 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 8600 /* Update the remote cached instance and return */ 8601 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 8602 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 8603 &hw->remote_dcbx_config); 8604 goto exit; 8605 } 8606 8607 /* Store the old configuration */ 8608 tmp_dcbx_cfg = hw->local_dcbx_config; 8609 8610 /* Reset the old DCBx configuration data */ 8611 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 8612 /* Get updated DCBX data from firmware */ 8613 ret = i40e_get_dcb_config(&pf->hw); 8614 if (ret) { 8615 dev_info(&pf->pdev->dev, 8616 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 8617 i40e_stat_str(&pf->hw, ret), 8618 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8619 goto exit; 8620 } 8621 8622 /* No change detected in DCBX configs */ 8623 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 8624 sizeof(tmp_dcbx_cfg))) { 8625 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 8626 goto exit; 8627 } 8628 8629 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 8630 &hw->local_dcbx_config); 8631 8632 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 8633 8634 if (!need_reconfig) 8635 goto exit; 8636 8637 /* Enable DCB tagging only when more than one TC */ 8638 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 8639 pf->flags |= I40E_FLAG_DCB_ENABLED; 8640 else 8641 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 8642 8643 set_bit(__I40E_PORT_SUSPENDED, pf->state); 8644 /* Reconfiguration needed quiesce all VSIs */ 8645 i40e_pf_quiesce_all_vsi(pf); 8646 8647 /* Changes in configuration update VEB/VSI */ 8648 i40e_dcb_reconfigure(pf); 8649 8650 ret = i40e_resume_port_tx(pf); 8651 8652 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 8653 /* In case of error no point in resuming VSIs */ 8654 if (ret) 8655 goto exit; 8656 8657 /* Wait for the PF's queues to be disabled */ 8658 ret = i40e_pf_wait_queues_disabled(pf); 8659 if (ret) { 8660 /* Schedule PF reset to recover */ 8661 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8662 i40e_service_event_schedule(pf); 8663 } else { 8664 i40e_pf_unquiesce_all_vsi(pf); 8665 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 8666 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 8667 } 8668 8669 exit: 8670 return ret; 8671 } 8672 #endif /* CONFIG_I40E_DCB */ 8673 8674 /** 8675 * i40e_do_reset_safe - Protected reset path for userland calls. 8676 * @pf: board private structure 8677 * @reset_flags: which reset is requested 8678 * 8679 **/ 8680 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 8681 { 8682 rtnl_lock(); 8683 i40e_do_reset(pf, reset_flags, true); 8684 rtnl_unlock(); 8685 } 8686 8687 /** 8688 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 8689 * @pf: board private structure 8690 * @e: event info posted on ARQ 8691 * 8692 * Handler for LAN Queue Overflow Event generated by the firmware for PF 8693 * and VF queues 8694 **/ 8695 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 8696 struct i40e_arq_event_info *e) 8697 { 8698 struct i40e_aqc_lan_overflow *data = 8699 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 8700 u32 queue = le32_to_cpu(data->prtdcb_rupto); 8701 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 8702 struct i40e_hw *hw = &pf->hw; 8703 struct i40e_vf *vf; 8704 u16 vf_id; 8705 8706 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 8707 queue, qtx_ctl); 8708 8709 /* Queue belongs to VF, find the VF and issue VF reset */ 8710 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 8711 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 8712 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 8713 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 8714 vf_id -= hw->func_caps.vf_base_id; 8715 vf = &pf->vf[vf_id]; 8716 i40e_vc_notify_vf_reset(vf); 8717 /* Allow VF to process pending reset notification */ 8718 msleep(20); 8719 i40e_reset_vf(vf, false); 8720 } 8721 } 8722 8723 /** 8724 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 8725 * @pf: board private structure 8726 **/ 8727 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 8728 { 8729 u32 val, fcnt_prog; 8730 8731 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8732 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 8733 return fcnt_prog; 8734 } 8735 8736 /** 8737 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 8738 * @pf: board private structure 8739 **/ 8740 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 8741 { 8742 u32 val, fcnt_prog; 8743 8744 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8745 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 8746 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 8747 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 8748 return fcnt_prog; 8749 } 8750 8751 /** 8752 * i40e_get_global_fd_count - Get total FD filters programmed on device 8753 * @pf: board private structure 8754 **/ 8755 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 8756 { 8757 u32 val, fcnt_prog; 8758 8759 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 8760 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 8761 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 8762 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 8763 return fcnt_prog; 8764 } 8765 8766 /** 8767 * i40e_reenable_fdir_sb - Restore FDir SB capability 8768 * @pf: board private structure 8769 **/ 8770 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 8771 { 8772 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 8773 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8774 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8775 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 8776 } 8777 8778 /** 8779 * i40e_reenable_fdir_atr - Restore FDir ATR capability 8780 * @pf: board private structure 8781 **/ 8782 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 8783 { 8784 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 8785 /* ATR uses the same filtering logic as SB rules. It only 8786 * functions properly if the input set mask is at the default 8787 * settings. It is safe to restore the default input set 8788 * because there are no active TCPv4 filter rules. 8789 */ 8790 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8791 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8792 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8793 8794 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8795 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8796 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 8797 } 8798 } 8799 8800 /** 8801 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 8802 * @pf: board private structure 8803 * @filter: FDir filter to remove 8804 */ 8805 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 8806 struct i40e_fdir_filter *filter) 8807 { 8808 /* Update counters */ 8809 pf->fdir_pf_active_filters--; 8810 pf->fd_inv = 0; 8811 8812 switch (filter->flow_type) { 8813 case TCP_V4_FLOW: 8814 pf->fd_tcp4_filter_cnt--; 8815 break; 8816 case UDP_V4_FLOW: 8817 pf->fd_udp4_filter_cnt--; 8818 break; 8819 case SCTP_V4_FLOW: 8820 pf->fd_sctp4_filter_cnt--; 8821 break; 8822 case IP_USER_FLOW: 8823 switch (filter->ip4_proto) { 8824 case IPPROTO_TCP: 8825 pf->fd_tcp4_filter_cnt--; 8826 break; 8827 case IPPROTO_UDP: 8828 pf->fd_udp4_filter_cnt--; 8829 break; 8830 case IPPROTO_SCTP: 8831 pf->fd_sctp4_filter_cnt--; 8832 break; 8833 case IPPROTO_IP: 8834 pf->fd_ip4_filter_cnt--; 8835 break; 8836 } 8837 break; 8838 } 8839 8840 /* Remove the filter from the list and free memory */ 8841 hlist_del(&filter->fdir_node); 8842 kfree(filter); 8843 } 8844 8845 /** 8846 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 8847 * @pf: board private structure 8848 **/ 8849 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 8850 { 8851 struct i40e_fdir_filter *filter; 8852 u32 fcnt_prog, fcnt_avail; 8853 struct hlist_node *node; 8854 8855 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8856 return; 8857 8858 /* Check if we have enough room to re-enable FDir SB capability. */ 8859 fcnt_prog = i40e_get_global_fd_count(pf); 8860 fcnt_avail = pf->fdir_pf_filter_count; 8861 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 8862 (pf->fd_add_err == 0) || 8863 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 8864 i40e_reenable_fdir_sb(pf); 8865 8866 /* We should wait for even more space before re-enabling ATR. 8867 * Additionally, we cannot enable ATR as long as we still have TCP SB 8868 * rules active. 8869 */ 8870 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 8871 (pf->fd_tcp4_filter_cnt == 0)) 8872 i40e_reenable_fdir_atr(pf); 8873 8874 /* if hw had a problem adding a filter, delete it */ 8875 if (pf->fd_inv > 0) { 8876 hlist_for_each_entry_safe(filter, node, 8877 &pf->fdir_filter_list, fdir_node) 8878 if (filter->fd_id == pf->fd_inv) 8879 i40e_delete_invalid_filter(pf, filter); 8880 } 8881 } 8882 8883 #define I40E_MIN_FD_FLUSH_INTERVAL 10 8884 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 8885 /** 8886 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 8887 * @pf: board private structure 8888 **/ 8889 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 8890 { 8891 unsigned long min_flush_time; 8892 int flush_wait_retry = 50; 8893 bool disable_atr = false; 8894 int fd_room; 8895 int reg; 8896 8897 if (!time_after(jiffies, pf->fd_flush_timestamp + 8898 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 8899 return; 8900 8901 /* If the flush is happening too quick and we have mostly SB rules we 8902 * should not re-enable ATR for some time. 8903 */ 8904 min_flush_time = pf->fd_flush_timestamp + 8905 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 8906 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 8907 8908 if (!(time_after(jiffies, min_flush_time)) && 8909 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 8910 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8911 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 8912 disable_atr = true; 8913 } 8914 8915 pf->fd_flush_timestamp = jiffies; 8916 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8917 /* flush all filters */ 8918 wr32(&pf->hw, I40E_PFQF_CTL_1, 8919 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 8920 i40e_flush(&pf->hw); 8921 pf->fd_flush_cnt++; 8922 pf->fd_add_err = 0; 8923 do { 8924 /* Check FD flush status every 5-6msec */ 8925 usleep_range(5000, 6000); 8926 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 8927 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 8928 break; 8929 } while (flush_wait_retry--); 8930 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 8931 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 8932 } else { 8933 /* replay sideband filters */ 8934 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 8935 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 8936 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8937 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 8938 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8939 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 8940 } 8941 } 8942 8943 /** 8944 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 8945 * @pf: board private structure 8946 **/ 8947 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 8948 { 8949 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 8950 } 8951 8952 /** 8953 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 8954 * @pf: board private structure 8955 **/ 8956 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 8957 { 8958 8959 /* if interface is down do nothing */ 8960 if (test_bit(__I40E_DOWN, pf->state)) 8961 return; 8962 8963 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8964 i40e_fdir_flush_and_replay(pf); 8965 8966 i40e_fdir_check_and_reenable(pf); 8967 8968 } 8969 8970 /** 8971 * i40e_vsi_link_event - notify VSI of a link event 8972 * @vsi: vsi to be notified 8973 * @link_up: link up or down 8974 **/ 8975 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 8976 { 8977 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 8978 return; 8979 8980 switch (vsi->type) { 8981 case I40E_VSI_MAIN: 8982 if (!vsi->netdev || !vsi->netdev_registered) 8983 break; 8984 8985 if (link_up) { 8986 netif_carrier_on(vsi->netdev); 8987 netif_tx_wake_all_queues(vsi->netdev); 8988 } else { 8989 netif_carrier_off(vsi->netdev); 8990 netif_tx_stop_all_queues(vsi->netdev); 8991 } 8992 break; 8993 8994 case I40E_VSI_SRIOV: 8995 case I40E_VSI_VMDQ2: 8996 case I40E_VSI_CTRL: 8997 case I40E_VSI_IWARP: 8998 case I40E_VSI_MIRROR: 8999 default: 9000 /* there is no notification for other VSIs */ 9001 break; 9002 } 9003 } 9004 9005 /** 9006 * i40e_veb_link_event - notify elements on the veb of a link event 9007 * @veb: veb to be notified 9008 * @link_up: link up or down 9009 **/ 9010 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 9011 { 9012 struct i40e_pf *pf; 9013 int i; 9014 9015 if (!veb || !veb->pf) 9016 return; 9017 pf = veb->pf; 9018 9019 /* depth first... */ 9020 for (i = 0; i < I40E_MAX_VEB; i++) 9021 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 9022 i40e_veb_link_event(pf->veb[i], link_up); 9023 9024 /* ... now the local VSIs */ 9025 for (i = 0; i < pf->num_alloc_vsi; i++) 9026 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 9027 i40e_vsi_link_event(pf->vsi[i], link_up); 9028 } 9029 9030 /** 9031 * i40e_link_event - Update netif_carrier status 9032 * @pf: board private structure 9033 **/ 9034 static void i40e_link_event(struct i40e_pf *pf) 9035 { 9036 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9037 u8 new_link_speed, old_link_speed; 9038 i40e_status status; 9039 bool new_link, old_link; 9040 9041 /* set this to force the get_link_status call to refresh state */ 9042 pf->hw.phy.get_link_info = true; 9043 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 9044 status = i40e_get_link_status(&pf->hw, &new_link); 9045 9046 /* On success, disable temp link polling */ 9047 if (status == I40E_SUCCESS) { 9048 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9049 } else { 9050 /* Enable link polling temporarily until i40e_get_link_status 9051 * returns I40E_SUCCESS 9052 */ 9053 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9054 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 9055 status); 9056 return; 9057 } 9058 9059 old_link_speed = pf->hw.phy.link_info_old.link_speed; 9060 new_link_speed = pf->hw.phy.link_info.link_speed; 9061 9062 if (new_link == old_link && 9063 new_link_speed == old_link_speed && 9064 (test_bit(__I40E_VSI_DOWN, vsi->state) || 9065 new_link == netif_carrier_ok(vsi->netdev))) 9066 return; 9067 9068 i40e_print_link_message(vsi, new_link); 9069 9070 /* Notify the base of the switch tree connected to 9071 * the link. Floating VEBs are not notified. 9072 */ 9073 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 9074 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 9075 else 9076 i40e_vsi_link_event(vsi, new_link); 9077 9078 if (pf->vf) 9079 i40e_vc_notify_link_state(pf); 9080 9081 if (pf->flags & I40E_FLAG_PTP) 9082 i40e_ptp_set_increment(pf); 9083 } 9084 9085 /** 9086 * i40e_watchdog_subtask - periodic checks not using event driven response 9087 * @pf: board private structure 9088 **/ 9089 static void i40e_watchdog_subtask(struct i40e_pf *pf) 9090 { 9091 int i; 9092 9093 /* if interface is down do nothing */ 9094 if (test_bit(__I40E_DOWN, pf->state) || 9095 test_bit(__I40E_CONFIG_BUSY, pf->state)) 9096 return; 9097 9098 /* make sure we don't do these things too often */ 9099 if (time_before(jiffies, (pf->service_timer_previous + 9100 pf->service_timer_period))) 9101 return; 9102 pf->service_timer_previous = jiffies; 9103 9104 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 9105 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 9106 i40e_link_event(pf); 9107 9108 /* Update the stats for active netdevs so the network stack 9109 * can look at updated numbers whenever it cares to 9110 */ 9111 for (i = 0; i < pf->num_alloc_vsi; i++) 9112 if (pf->vsi[i] && pf->vsi[i]->netdev) 9113 i40e_update_stats(pf->vsi[i]); 9114 9115 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 9116 /* Update the stats for the active switching components */ 9117 for (i = 0; i < I40E_MAX_VEB; i++) 9118 if (pf->veb[i]) 9119 i40e_update_veb_stats(pf->veb[i]); 9120 } 9121 9122 i40e_ptp_rx_hang(pf); 9123 i40e_ptp_tx_hang(pf); 9124 } 9125 9126 /** 9127 * i40e_reset_subtask - Set up for resetting the device and driver 9128 * @pf: board private structure 9129 **/ 9130 static void i40e_reset_subtask(struct i40e_pf *pf) 9131 { 9132 u32 reset_flags = 0; 9133 9134 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 9135 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 9136 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 9137 } 9138 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 9139 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 9140 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9141 } 9142 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 9143 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 9144 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 9145 } 9146 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 9147 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 9148 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 9149 } 9150 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 9151 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 9152 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 9153 } 9154 9155 /* If there's a recovery already waiting, it takes 9156 * precedence before starting a new reset sequence. 9157 */ 9158 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 9159 i40e_prep_for_reset(pf, false); 9160 i40e_reset(pf); 9161 i40e_rebuild(pf, false, false); 9162 } 9163 9164 /* If we're already down or resetting, just bail */ 9165 if (reset_flags && 9166 !test_bit(__I40E_DOWN, pf->state) && 9167 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 9168 i40e_do_reset(pf, reset_flags, false); 9169 } 9170 } 9171 9172 /** 9173 * i40e_handle_link_event - Handle link event 9174 * @pf: board private structure 9175 * @e: event info posted on ARQ 9176 **/ 9177 static void i40e_handle_link_event(struct i40e_pf *pf, 9178 struct i40e_arq_event_info *e) 9179 { 9180 struct i40e_aqc_get_link_status *status = 9181 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 9182 9183 /* Do a new status request to re-enable LSE reporting 9184 * and load new status information into the hw struct 9185 * This completely ignores any state information 9186 * in the ARQ event info, instead choosing to always 9187 * issue the AQ update link status command. 9188 */ 9189 i40e_link_event(pf); 9190 9191 /* Check if module meets thermal requirements */ 9192 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 9193 dev_err(&pf->pdev->dev, 9194 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 9195 dev_err(&pf->pdev->dev, 9196 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 9197 } else { 9198 /* check for unqualified module, if link is down, suppress 9199 * the message if link was forced to be down. 9200 */ 9201 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 9202 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 9203 (!(status->link_info & I40E_AQ_LINK_UP)) && 9204 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 9205 dev_err(&pf->pdev->dev, 9206 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 9207 dev_err(&pf->pdev->dev, 9208 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 9209 } 9210 } 9211 } 9212 9213 /** 9214 * i40e_clean_adminq_subtask - Clean the AdminQ rings 9215 * @pf: board private structure 9216 **/ 9217 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 9218 { 9219 struct i40e_arq_event_info event; 9220 struct i40e_hw *hw = &pf->hw; 9221 u16 pending, i = 0; 9222 i40e_status ret; 9223 u16 opcode; 9224 u32 oldval; 9225 u32 val; 9226 9227 /* Do not run clean AQ when PF reset fails */ 9228 if (test_bit(__I40E_RESET_FAILED, pf->state)) 9229 return; 9230 9231 /* check for error indications */ 9232 val = rd32(&pf->hw, pf->hw.aq.arq.len); 9233 oldval = val; 9234 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 9235 if (hw->debug_mask & I40E_DEBUG_AQ) 9236 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 9237 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 9238 } 9239 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 9240 if (hw->debug_mask & I40E_DEBUG_AQ) 9241 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 9242 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 9243 pf->arq_overflows++; 9244 } 9245 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 9246 if (hw->debug_mask & I40E_DEBUG_AQ) 9247 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 9248 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 9249 } 9250 if (oldval != val) 9251 wr32(&pf->hw, pf->hw.aq.arq.len, val); 9252 9253 val = rd32(&pf->hw, pf->hw.aq.asq.len); 9254 oldval = val; 9255 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 9256 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9257 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 9258 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 9259 } 9260 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 9261 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9262 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 9263 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 9264 } 9265 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 9266 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9267 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 9268 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 9269 } 9270 if (oldval != val) 9271 wr32(&pf->hw, pf->hw.aq.asq.len, val); 9272 9273 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 9274 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 9275 if (!event.msg_buf) 9276 return; 9277 9278 do { 9279 ret = i40e_clean_arq_element(hw, &event, &pending); 9280 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 9281 break; 9282 else if (ret) { 9283 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 9284 break; 9285 } 9286 9287 opcode = le16_to_cpu(event.desc.opcode); 9288 switch (opcode) { 9289 9290 case i40e_aqc_opc_get_link_status: 9291 i40e_handle_link_event(pf, &event); 9292 break; 9293 case i40e_aqc_opc_send_msg_to_pf: 9294 ret = i40e_vc_process_vf_msg(pf, 9295 le16_to_cpu(event.desc.retval), 9296 le32_to_cpu(event.desc.cookie_high), 9297 le32_to_cpu(event.desc.cookie_low), 9298 event.msg_buf, 9299 event.msg_len); 9300 break; 9301 case i40e_aqc_opc_lldp_update_mib: 9302 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 9303 #ifdef CONFIG_I40E_DCB 9304 rtnl_lock(); 9305 ret = i40e_handle_lldp_event(pf, &event); 9306 rtnl_unlock(); 9307 #endif /* CONFIG_I40E_DCB */ 9308 break; 9309 case i40e_aqc_opc_event_lan_overflow: 9310 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 9311 i40e_handle_lan_overflow_event(pf, &event); 9312 break; 9313 case i40e_aqc_opc_send_msg_to_peer: 9314 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 9315 break; 9316 case i40e_aqc_opc_nvm_erase: 9317 case i40e_aqc_opc_nvm_update: 9318 case i40e_aqc_opc_oem_post_update: 9319 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 9320 "ARQ NVM operation 0x%04x completed\n", 9321 opcode); 9322 break; 9323 default: 9324 dev_info(&pf->pdev->dev, 9325 "ARQ: Unknown event 0x%04x ignored\n", 9326 opcode); 9327 break; 9328 } 9329 } while (i++ < pf->adminq_work_limit); 9330 9331 if (i < pf->adminq_work_limit) 9332 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 9333 9334 /* re-enable Admin queue interrupt cause */ 9335 val = rd32(hw, I40E_PFINT_ICR0_ENA); 9336 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 9337 wr32(hw, I40E_PFINT_ICR0_ENA, val); 9338 i40e_flush(hw); 9339 9340 kfree(event.msg_buf); 9341 } 9342 9343 /** 9344 * i40e_verify_eeprom - make sure eeprom is good to use 9345 * @pf: board private structure 9346 **/ 9347 static void i40e_verify_eeprom(struct i40e_pf *pf) 9348 { 9349 int err; 9350 9351 err = i40e_diag_eeprom_test(&pf->hw); 9352 if (err) { 9353 /* retry in case of garbage read */ 9354 err = i40e_diag_eeprom_test(&pf->hw); 9355 if (err) { 9356 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 9357 err); 9358 set_bit(__I40E_BAD_EEPROM, pf->state); 9359 } 9360 } 9361 9362 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 9363 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 9364 clear_bit(__I40E_BAD_EEPROM, pf->state); 9365 } 9366 } 9367 9368 /** 9369 * i40e_enable_pf_switch_lb 9370 * @pf: pointer to the PF structure 9371 * 9372 * enable switch loop back or die - no point in a return value 9373 **/ 9374 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 9375 { 9376 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9377 struct i40e_vsi_context ctxt; 9378 int ret; 9379 9380 ctxt.seid = pf->main_vsi_seid; 9381 ctxt.pf_num = pf->hw.pf_id; 9382 ctxt.vf_num = 0; 9383 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9384 if (ret) { 9385 dev_info(&pf->pdev->dev, 9386 "couldn't get PF vsi config, err %s aq_err %s\n", 9387 i40e_stat_str(&pf->hw, ret), 9388 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9389 return; 9390 } 9391 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9392 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9393 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9394 9395 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 9396 if (ret) { 9397 dev_info(&pf->pdev->dev, 9398 "update vsi switch failed, err %s aq_err %s\n", 9399 i40e_stat_str(&pf->hw, ret), 9400 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9401 } 9402 } 9403 9404 /** 9405 * i40e_disable_pf_switch_lb 9406 * @pf: pointer to the PF structure 9407 * 9408 * disable switch loop back or die - no point in a return value 9409 **/ 9410 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 9411 { 9412 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9413 struct i40e_vsi_context ctxt; 9414 int ret; 9415 9416 ctxt.seid = pf->main_vsi_seid; 9417 ctxt.pf_num = pf->hw.pf_id; 9418 ctxt.vf_num = 0; 9419 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9420 if (ret) { 9421 dev_info(&pf->pdev->dev, 9422 "couldn't get PF vsi config, err %s aq_err %s\n", 9423 i40e_stat_str(&pf->hw, ret), 9424 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9425 return; 9426 } 9427 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9428 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9429 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9430 9431 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 9432 if (ret) { 9433 dev_info(&pf->pdev->dev, 9434 "update vsi switch failed, err %s aq_err %s\n", 9435 i40e_stat_str(&pf->hw, ret), 9436 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9437 } 9438 } 9439 9440 /** 9441 * i40e_config_bridge_mode - Configure the HW bridge mode 9442 * @veb: pointer to the bridge instance 9443 * 9444 * Configure the loop back mode for the LAN VSI that is downlink to the 9445 * specified HW bridge instance. It is expected this function is called 9446 * when a new HW bridge is instantiated. 9447 **/ 9448 static void i40e_config_bridge_mode(struct i40e_veb *veb) 9449 { 9450 struct i40e_pf *pf = veb->pf; 9451 9452 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 9453 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 9454 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 9455 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 9456 i40e_disable_pf_switch_lb(pf); 9457 else 9458 i40e_enable_pf_switch_lb(pf); 9459 } 9460 9461 /** 9462 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 9463 * @veb: pointer to the VEB instance 9464 * 9465 * This is a recursive function that first builds the attached VSIs then 9466 * recurses in to build the next layer of VEB. We track the connections 9467 * through our own index numbers because the seid's from the HW could 9468 * change across the reset. 9469 **/ 9470 static int i40e_reconstitute_veb(struct i40e_veb *veb) 9471 { 9472 struct i40e_vsi *ctl_vsi = NULL; 9473 struct i40e_pf *pf = veb->pf; 9474 int v, veb_idx; 9475 int ret; 9476 9477 /* build VSI that owns this VEB, temporarily attached to base VEB */ 9478 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 9479 if (pf->vsi[v] && 9480 pf->vsi[v]->veb_idx == veb->idx && 9481 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 9482 ctl_vsi = pf->vsi[v]; 9483 break; 9484 } 9485 } 9486 if (!ctl_vsi) { 9487 dev_info(&pf->pdev->dev, 9488 "missing owner VSI for veb_idx %d\n", veb->idx); 9489 ret = -ENOENT; 9490 goto end_reconstitute; 9491 } 9492 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 9493 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 9494 ret = i40e_add_vsi(ctl_vsi); 9495 if (ret) { 9496 dev_info(&pf->pdev->dev, 9497 "rebuild of veb_idx %d owner VSI failed: %d\n", 9498 veb->idx, ret); 9499 goto end_reconstitute; 9500 } 9501 i40e_vsi_reset_stats(ctl_vsi); 9502 9503 /* create the VEB in the switch and move the VSI onto the VEB */ 9504 ret = i40e_add_veb(veb, ctl_vsi); 9505 if (ret) 9506 goto end_reconstitute; 9507 9508 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 9509 veb->bridge_mode = BRIDGE_MODE_VEB; 9510 else 9511 veb->bridge_mode = BRIDGE_MODE_VEPA; 9512 i40e_config_bridge_mode(veb); 9513 9514 /* create the remaining VSIs attached to this VEB */ 9515 for (v = 0; v < pf->num_alloc_vsi; v++) { 9516 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 9517 continue; 9518 9519 if (pf->vsi[v]->veb_idx == veb->idx) { 9520 struct i40e_vsi *vsi = pf->vsi[v]; 9521 9522 vsi->uplink_seid = veb->seid; 9523 ret = i40e_add_vsi(vsi); 9524 if (ret) { 9525 dev_info(&pf->pdev->dev, 9526 "rebuild of vsi_idx %d failed: %d\n", 9527 v, ret); 9528 goto end_reconstitute; 9529 } 9530 i40e_vsi_reset_stats(vsi); 9531 } 9532 } 9533 9534 /* create any VEBs attached to this VEB - RECURSION */ 9535 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 9536 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 9537 pf->veb[veb_idx]->uplink_seid = veb->seid; 9538 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 9539 if (ret) 9540 break; 9541 } 9542 } 9543 9544 end_reconstitute: 9545 return ret; 9546 } 9547 9548 /** 9549 * i40e_get_capabilities - get info about the HW 9550 * @pf: the PF struct 9551 **/ 9552 static int i40e_get_capabilities(struct i40e_pf *pf, 9553 enum i40e_admin_queue_opc list_type) 9554 { 9555 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 9556 u16 data_size; 9557 int buf_len; 9558 int err; 9559 9560 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 9561 do { 9562 cap_buf = kzalloc(buf_len, GFP_KERNEL); 9563 if (!cap_buf) 9564 return -ENOMEM; 9565 9566 /* this loads the data into the hw struct for us */ 9567 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 9568 &data_size, list_type, 9569 NULL); 9570 /* data loaded, buffer no longer needed */ 9571 kfree(cap_buf); 9572 9573 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 9574 /* retry with a larger buffer */ 9575 buf_len = data_size; 9576 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 9577 dev_info(&pf->pdev->dev, 9578 "capability discovery failed, err %s aq_err %s\n", 9579 i40e_stat_str(&pf->hw, err), 9580 i40e_aq_str(&pf->hw, 9581 pf->hw.aq.asq_last_status)); 9582 return -ENODEV; 9583 } 9584 } while (err); 9585 9586 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 9587 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9588 dev_info(&pf->pdev->dev, 9589 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 9590 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 9591 pf->hw.func_caps.num_msix_vectors, 9592 pf->hw.func_caps.num_msix_vectors_vf, 9593 pf->hw.func_caps.fd_filters_guaranteed, 9594 pf->hw.func_caps.fd_filters_best_effort, 9595 pf->hw.func_caps.num_tx_qp, 9596 pf->hw.func_caps.num_vsis); 9597 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 9598 dev_info(&pf->pdev->dev, 9599 "switch_mode=0x%04x, function_valid=0x%08x\n", 9600 pf->hw.dev_caps.switch_mode, 9601 pf->hw.dev_caps.valid_functions); 9602 dev_info(&pf->pdev->dev, 9603 "SR-IOV=%d, num_vfs for all function=%u\n", 9604 pf->hw.dev_caps.sr_iov_1_1, 9605 pf->hw.dev_caps.num_vfs); 9606 dev_info(&pf->pdev->dev, 9607 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 9608 pf->hw.dev_caps.num_vsis, 9609 pf->hw.dev_caps.num_rx_qp, 9610 pf->hw.dev_caps.num_tx_qp); 9611 } 9612 } 9613 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9614 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 9615 + pf->hw.func_caps.num_vfs) 9616 if (pf->hw.revision_id == 0 && 9617 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 9618 dev_info(&pf->pdev->dev, 9619 "got num_vsis %d, setting num_vsis to %d\n", 9620 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 9621 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 9622 } 9623 } 9624 return 0; 9625 } 9626 9627 static int i40e_vsi_clear(struct i40e_vsi *vsi); 9628 9629 /** 9630 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 9631 * @pf: board private structure 9632 **/ 9633 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 9634 { 9635 struct i40e_vsi *vsi; 9636 9637 /* quick workaround for an NVM issue that leaves a critical register 9638 * uninitialized 9639 */ 9640 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 9641 static const u32 hkey[] = { 9642 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 9643 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 9644 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 9645 0x95b3a76d}; 9646 int i; 9647 9648 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 9649 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 9650 } 9651 9652 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 9653 return; 9654 9655 /* find existing VSI and see if it needs configuring */ 9656 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9657 9658 /* create a new VSI if none exists */ 9659 if (!vsi) { 9660 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 9661 pf->vsi[pf->lan_vsi]->seid, 0); 9662 if (!vsi) { 9663 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 9664 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 9665 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 9666 return; 9667 } 9668 } 9669 9670 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 9671 } 9672 9673 /** 9674 * i40e_fdir_teardown - release the Flow Director resources 9675 * @pf: board private structure 9676 **/ 9677 static void i40e_fdir_teardown(struct i40e_pf *pf) 9678 { 9679 struct i40e_vsi *vsi; 9680 9681 i40e_fdir_filter_exit(pf); 9682 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9683 if (vsi) 9684 i40e_vsi_release(vsi); 9685 } 9686 9687 /** 9688 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 9689 * @vsi: PF main vsi 9690 * @seid: seid of main or channel VSIs 9691 * 9692 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 9693 * existed before reset 9694 **/ 9695 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 9696 { 9697 struct i40e_cloud_filter *cfilter; 9698 struct i40e_pf *pf = vsi->back; 9699 struct hlist_node *node; 9700 i40e_status ret; 9701 9702 /* Add cloud filters back if they exist */ 9703 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 9704 cloud_node) { 9705 if (cfilter->seid != seid) 9706 continue; 9707 9708 if (cfilter->dst_port) 9709 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 9710 true); 9711 else 9712 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 9713 9714 if (ret) { 9715 dev_dbg(&pf->pdev->dev, 9716 "Failed to rebuild cloud filter, err %s aq_err %s\n", 9717 i40e_stat_str(&pf->hw, ret), 9718 i40e_aq_str(&pf->hw, 9719 pf->hw.aq.asq_last_status)); 9720 return ret; 9721 } 9722 } 9723 return 0; 9724 } 9725 9726 /** 9727 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 9728 * @vsi: PF main vsi 9729 * 9730 * Rebuilds channel VSIs if they existed before reset 9731 **/ 9732 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 9733 { 9734 struct i40e_channel *ch, *ch_tmp; 9735 i40e_status ret; 9736 9737 if (list_empty(&vsi->ch_list)) 9738 return 0; 9739 9740 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 9741 if (!ch->initialized) 9742 break; 9743 /* Proceed with creation of channel (VMDq2) VSI */ 9744 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 9745 if (ret) { 9746 dev_info(&vsi->back->pdev->dev, 9747 "failed to rebuild channels using uplink_seid %u\n", 9748 vsi->uplink_seid); 9749 return ret; 9750 } 9751 /* Reconfigure TX queues using QTX_CTL register */ 9752 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 9753 if (ret) { 9754 dev_info(&vsi->back->pdev->dev, 9755 "failed to configure TX rings for channel %u\n", 9756 ch->seid); 9757 return ret; 9758 } 9759 /* update 'next_base_queue' */ 9760 vsi->next_base_queue = vsi->next_base_queue + 9761 ch->num_queue_pairs; 9762 if (ch->max_tx_rate) { 9763 u64 credits = ch->max_tx_rate; 9764 9765 if (i40e_set_bw_limit(vsi, ch->seid, 9766 ch->max_tx_rate)) 9767 return -EINVAL; 9768 9769 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9770 dev_dbg(&vsi->back->pdev->dev, 9771 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9772 ch->max_tx_rate, 9773 credits, 9774 ch->seid); 9775 } 9776 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 9777 if (ret) { 9778 dev_dbg(&vsi->back->pdev->dev, 9779 "Failed to rebuild cloud filters for channel VSI %u\n", 9780 ch->seid); 9781 return ret; 9782 } 9783 } 9784 return 0; 9785 } 9786 9787 /** 9788 * i40e_prep_for_reset - prep for the core to reset 9789 * @pf: board private structure 9790 * @lock_acquired: indicates whether or not the lock has been acquired 9791 * before this function was called. 9792 * 9793 * Close up the VFs and other things in prep for PF Reset. 9794 **/ 9795 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 9796 { 9797 struct i40e_hw *hw = &pf->hw; 9798 i40e_status ret = 0; 9799 u32 v; 9800 9801 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 9802 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9803 return; 9804 if (i40e_check_asq_alive(&pf->hw)) 9805 i40e_vc_notify_reset(pf); 9806 9807 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 9808 9809 /* quiesce the VSIs and their queues that are not already DOWN */ 9810 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 9811 if (!lock_acquired) 9812 rtnl_lock(); 9813 i40e_pf_quiesce_all_vsi(pf); 9814 if (!lock_acquired) 9815 rtnl_unlock(); 9816 9817 for (v = 0; v < pf->num_alloc_vsi; v++) { 9818 if (pf->vsi[v]) 9819 pf->vsi[v]->seid = 0; 9820 } 9821 9822 i40e_shutdown_adminq(&pf->hw); 9823 9824 /* call shutdown HMC */ 9825 if (hw->hmc.hmc_obj) { 9826 ret = i40e_shutdown_lan_hmc(hw); 9827 if (ret) 9828 dev_warn(&pf->pdev->dev, 9829 "shutdown_lan_hmc failed: %d\n", ret); 9830 } 9831 9832 /* Save the current PTP time so that we can restore the time after the 9833 * reset completes. 9834 */ 9835 i40e_ptp_save_hw_time(pf); 9836 } 9837 9838 /** 9839 * i40e_send_version - update firmware with driver version 9840 * @pf: PF struct 9841 */ 9842 static void i40e_send_version(struct i40e_pf *pf) 9843 { 9844 struct i40e_driver_version dv; 9845 9846 dv.major_version = 0xff; 9847 dv.minor_version = 0xff; 9848 dv.build_version = 0xff; 9849 dv.subbuild_version = 0; 9850 strlcpy(dv.driver_string, UTS_RELEASE, sizeof(dv.driver_string)); 9851 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 9852 } 9853 9854 /** 9855 * i40e_get_oem_version - get OEM specific version information 9856 * @hw: pointer to the hardware structure 9857 **/ 9858 static void i40e_get_oem_version(struct i40e_hw *hw) 9859 { 9860 u16 block_offset = 0xffff; 9861 u16 block_length = 0; 9862 u16 capabilities = 0; 9863 u16 gen_snap = 0; 9864 u16 release = 0; 9865 9866 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 9867 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 9868 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 9869 #define I40E_NVM_OEM_GEN_OFFSET 0x02 9870 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 9871 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 9872 #define I40E_NVM_OEM_LENGTH 3 9873 9874 /* Check if pointer to OEM version block is valid. */ 9875 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 9876 if (block_offset == 0xffff) 9877 return; 9878 9879 /* Check if OEM version block has correct length. */ 9880 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 9881 &block_length); 9882 if (block_length < I40E_NVM_OEM_LENGTH) 9883 return; 9884 9885 /* Check if OEM version format is as expected. */ 9886 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 9887 &capabilities); 9888 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 9889 return; 9890 9891 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 9892 &gen_snap); 9893 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 9894 &release); 9895 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 9896 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 9897 } 9898 9899 /** 9900 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 9901 * @pf: board private structure 9902 **/ 9903 static int i40e_reset(struct i40e_pf *pf) 9904 { 9905 struct i40e_hw *hw = &pf->hw; 9906 i40e_status ret; 9907 9908 ret = i40e_pf_reset(hw); 9909 if (ret) { 9910 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 9911 set_bit(__I40E_RESET_FAILED, pf->state); 9912 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9913 } else { 9914 pf->pfr_count++; 9915 } 9916 return ret; 9917 } 9918 9919 /** 9920 * i40e_rebuild - rebuild using a saved config 9921 * @pf: board private structure 9922 * @reinit: if the Main VSI needs to re-initialized. 9923 * @lock_acquired: indicates whether or not the lock has been acquired 9924 * before this function was called. 9925 **/ 9926 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 9927 { 9928 int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state); 9929 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9930 struct i40e_hw *hw = &pf->hw; 9931 u8 set_fc_aq_fail = 0; 9932 i40e_status ret; 9933 u32 val; 9934 int v; 9935 9936 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 9937 i40e_check_recovery_mode(pf)) { 9938 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev); 9939 } 9940 9941 if (test_bit(__I40E_DOWN, pf->state) && 9942 !test_bit(__I40E_RECOVERY_MODE, pf->state) && 9943 !old_recovery_mode_bit) 9944 goto clear_recovery; 9945 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 9946 9947 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 9948 ret = i40e_init_adminq(&pf->hw); 9949 if (ret) { 9950 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 9951 i40e_stat_str(&pf->hw, ret), 9952 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9953 goto clear_recovery; 9954 } 9955 i40e_get_oem_version(&pf->hw); 9956 9957 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 9958 ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) || 9959 hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) { 9960 /* The following delay is necessary for 4.33 firmware and older 9961 * to recover after EMP reset. 200 ms should suffice but we 9962 * put here 300 ms to be sure that FW is ready to operate 9963 * after reset. 9964 */ 9965 mdelay(300); 9966 } 9967 9968 /* re-verify the eeprom if we just had an EMP reset */ 9969 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 9970 i40e_verify_eeprom(pf); 9971 9972 /* if we are going out of or into recovery mode we have to act 9973 * accordingly with regard to resources initialization 9974 * and deinitialization 9975 */ 9976 if (test_bit(__I40E_RECOVERY_MODE, pf->state) || 9977 old_recovery_mode_bit) { 9978 if (i40e_get_capabilities(pf, 9979 i40e_aqc_opc_list_func_capabilities)) 9980 goto end_unlock; 9981 9982 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 9983 /* we're staying in recovery mode so we'll reinitialize 9984 * misc vector here 9985 */ 9986 if (i40e_setup_misc_vector_for_recovery_mode(pf)) 9987 goto end_unlock; 9988 } else { 9989 if (!lock_acquired) 9990 rtnl_lock(); 9991 /* we're going out of recovery mode so we'll free 9992 * the IRQ allocated specifically for recovery mode 9993 * and restore the interrupt scheme 9994 */ 9995 free_irq(pf->pdev->irq, pf); 9996 i40e_clear_interrupt_scheme(pf); 9997 if (i40e_restore_interrupt_scheme(pf)) 9998 goto end_unlock; 9999 } 10000 10001 /* tell the firmware that we're starting */ 10002 i40e_send_version(pf); 10003 10004 /* bail out in case recovery mode was detected, as there is 10005 * no need for further configuration. 10006 */ 10007 goto end_unlock; 10008 } 10009 10010 i40e_clear_pxe_mode(hw); 10011 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 10012 if (ret) 10013 goto end_core_reset; 10014 10015 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10016 hw->func_caps.num_rx_qp, 0, 0); 10017 if (ret) { 10018 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 10019 goto end_core_reset; 10020 } 10021 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10022 if (ret) { 10023 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 10024 goto end_core_reset; 10025 } 10026 10027 /* Enable FW to write a default DCB config on link-up */ 10028 i40e_aq_set_dcb_parameters(hw, true, NULL); 10029 10030 #ifdef CONFIG_I40E_DCB 10031 ret = i40e_init_pf_dcb(pf); 10032 if (ret) { 10033 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 10034 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 10035 /* Continue without DCB enabled */ 10036 } 10037 #endif /* CONFIG_I40E_DCB */ 10038 /* do basic switch setup */ 10039 if (!lock_acquired) 10040 rtnl_lock(); 10041 ret = i40e_setup_pf_switch(pf, reinit); 10042 if (ret) 10043 goto end_unlock; 10044 10045 /* The driver only wants link up/down and module qualification 10046 * reports from firmware. Note the negative logic. 10047 */ 10048 ret = i40e_aq_set_phy_int_mask(&pf->hw, 10049 ~(I40E_AQ_EVENT_LINK_UPDOWN | 10050 I40E_AQ_EVENT_MEDIA_NA | 10051 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 10052 if (ret) 10053 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 10054 i40e_stat_str(&pf->hw, ret), 10055 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10056 10057 /* make sure our flow control settings are restored */ 10058 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 10059 if (ret) 10060 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 10061 i40e_stat_str(&pf->hw, ret), 10062 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10063 10064 /* Rebuild the VSIs and VEBs that existed before reset. 10065 * They are still in our local switch element arrays, so only 10066 * need to rebuild the switch model in the HW. 10067 * 10068 * If there were VEBs but the reconstitution failed, we'll try 10069 * try to recover minimal use by getting the basic PF VSI working. 10070 */ 10071 if (vsi->uplink_seid != pf->mac_seid) { 10072 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 10073 /* find the one VEB connected to the MAC, and find orphans */ 10074 for (v = 0; v < I40E_MAX_VEB; v++) { 10075 if (!pf->veb[v]) 10076 continue; 10077 10078 if (pf->veb[v]->uplink_seid == pf->mac_seid || 10079 pf->veb[v]->uplink_seid == 0) { 10080 ret = i40e_reconstitute_veb(pf->veb[v]); 10081 10082 if (!ret) 10083 continue; 10084 10085 /* If Main VEB failed, we're in deep doodoo, 10086 * so give up rebuilding the switch and set up 10087 * for minimal rebuild of PF VSI. 10088 * If orphan failed, we'll report the error 10089 * but try to keep going. 10090 */ 10091 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 10092 dev_info(&pf->pdev->dev, 10093 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 10094 ret); 10095 vsi->uplink_seid = pf->mac_seid; 10096 break; 10097 } else if (pf->veb[v]->uplink_seid == 0) { 10098 dev_info(&pf->pdev->dev, 10099 "rebuild of orphan VEB failed: %d\n", 10100 ret); 10101 } 10102 } 10103 } 10104 } 10105 10106 if (vsi->uplink_seid == pf->mac_seid) { 10107 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 10108 /* no VEB, so rebuild only the Main VSI */ 10109 ret = i40e_add_vsi(vsi); 10110 if (ret) { 10111 dev_info(&pf->pdev->dev, 10112 "rebuild of Main VSI failed: %d\n", ret); 10113 goto end_unlock; 10114 } 10115 } 10116 10117 if (vsi->mqprio_qopt.max_rate[0]) { 10118 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 10119 u64 credits = 0; 10120 10121 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 10122 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 10123 if (ret) 10124 goto end_unlock; 10125 10126 credits = max_tx_rate; 10127 do_div(credits, I40E_BW_CREDIT_DIVISOR); 10128 dev_dbg(&vsi->back->pdev->dev, 10129 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 10130 max_tx_rate, 10131 credits, 10132 vsi->seid); 10133 } 10134 10135 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 10136 if (ret) 10137 goto end_unlock; 10138 10139 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 10140 * for this main VSI if they exist 10141 */ 10142 ret = i40e_rebuild_channels(vsi); 10143 if (ret) 10144 goto end_unlock; 10145 10146 /* Reconfigure hardware for allowing smaller MSS in the case 10147 * of TSO, so that we avoid the MDD being fired and causing 10148 * a reset in the case of small MSS+TSO. 10149 */ 10150 #define I40E_REG_MSS 0x000E64DC 10151 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 10152 #define I40E_64BYTE_MSS 0x400000 10153 val = rd32(hw, I40E_REG_MSS); 10154 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 10155 val &= ~I40E_REG_MSS_MIN_MASK; 10156 val |= I40E_64BYTE_MSS; 10157 wr32(hw, I40E_REG_MSS, val); 10158 } 10159 10160 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 10161 msleep(75); 10162 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 10163 if (ret) 10164 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 10165 i40e_stat_str(&pf->hw, ret), 10166 i40e_aq_str(&pf->hw, 10167 pf->hw.aq.asq_last_status)); 10168 } 10169 /* reinit the misc interrupt */ 10170 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10171 ret = i40e_setup_misc_vector(pf); 10172 10173 /* Add a filter to drop all Flow control frames from any VSI from being 10174 * transmitted. By doing so we stop a malicious VF from sending out 10175 * PAUSE or PFC frames and potentially controlling traffic for other 10176 * PF/VF VSIs. 10177 * The FW can still send Flow control frames if enabled. 10178 */ 10179 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 10180 pf->main_vsi_seid); 10181 10182 /* restart the VSIs that were rebuilt and running before the reset */ 10183 i40e_pf_unquiesce_all_vsi(pf); 10184 10185 /* Release the RTNL lock before we start resetting VFs */ 10186 if (!lock_acquired) 10187 rtnl_unlock(); 10188 10189 /* Restore promiscuous settings */ 10190 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 10191 if (ret) 10192 dev_warn(&pf->pdev->dev, 10193 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n", 10194 pf->cur_promisc ? "on" : "off", 10195 i40e_stat_str(&pf->hw, ret), 10196 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10197 10198 i40e_reset_all_vfs(pf, true); 10199 10200 /* tell the firmware that we're starting */ 10201 i40e_send_version(pf); 10202 10203 /* We've already released the lock, so don't do it again */ 10204 goto end_core_reset; 10205 10206 end_unlock: 10207 if (!lock_acquired) 10208 rtnl_unlock(); 10209 end_core_reset: 10210 clear_bit(__I40E_RESET_FAILED, pf->state); 10211 clear_recovery: 10212 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 10213 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state); 10214 } 10215 10216 /** 10217 * i40e_reset_and_rebuild - reset and rebuild using a saved config 10218 * @pf: board private structure 10219 * @reinit: if the Main VSI needs to re-initialized. 10220 * @lock_acquired: indicates whether or not the lock has been acquired 10221 * before this function was called. 10222 **/ 10223 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 10224 bool lock_acquired) 10225 { 10226 int ret; 10227 /* Now we wait for GRST to settle out. 10228 * We don't have to delete the VEBs or VSIs from the hw switch 10229 * because the reset will make them disappear. 10230 */ 10231 ret = i40e_reset(pf); 10232 if (!ret) 10233 i40e_rebuild(pf, reinit, lock_acquired); 10234 } 10235 10236 /** 10237 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 10238 * @pf: board private structure 10239 * 10240 * Close up the VFs and other things in prep for a Core Reset, 10241 * then get ready to rebuild the world. 10242 * @lock_acquired: indicates whether or not the lock has been acquired 10243 * before this function was called. 10244 **/ 10245 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 10246 { 10247 i40e_prep_for_reset(pf, lock_acquired); 10248 i40e_reset_and_rebuild(pf, false, lock_acquired); 10249 } 10250 10251 /** 10252 * i40e_handle_mdd_event 10253 * @pf: pointer to the PF structure 10254 * 10255 * Called from the MDD irq handler to identify possibly malicious vfs 10256 **/ 10257 static void i40e_handle_mdd_event(struct i40e_pf *pf) 10258 { 10259 struct i40e_hw *hw = &pf->hw; 10260 bool mdd_detected = false; 10261 struct i40e_vf *vf; 10262 u32 reg; 10263 int i; 10264 10265 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 10266 return; 10267 10268 /* find what triggered the MDD event */ 10269 reg = rd32(hw, I40E_GL_MDET_TX); 10270 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 10271 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 10272 I40E_GL_MDET_TX_PF_NUM_SHIFT; 10273 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 10274 I40E_GL_MDET_TX_VF_NUM_SHIFT; 10275 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 10276 I40E_GL_MDET_TX_EVENT_SHIFT; 10277 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 10278 I40E_GL_MDET_TX_QUEUE_SHIFT) - 10279 pf->hw.func_caps.base_queue; 10280 if (netif_msg_tx_err(pf)) 10281 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 10282 event, queue, pf_num, vf_num); 10283 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 10284 mdd_detected = true; 10285 } 10286 reg = rd32(hw, I40E_GL_MDET_RX); 10287 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 10288 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 10289 I40E_GL_MDET_RX_FUNCTION_SHIFT; 10290 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 10291 I40E_GL_MDET_RX_EVENT_SHIFT; 10292 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 10293 I40E_GL_MDET_RX_QUEUE_SHIFT) - 10294 pf->hw.func_caps.base_queue; 10295 if (netif_msg_rx_err(pf)) 10296 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 10297 event, queue, func); 10298 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 10299 mdd_detected = true; 10300 } 10301 10302 if (mdd_detected) { 10303 reg = rd32(hw, I40E_PF_MDET_TX); 10304 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 10305 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 10306 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n"); 10307 } 10308 reg = rd32(hw, I40E_PF_MDET_RX); 10309 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 10310 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 10311 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n"); 10312 } 10313 } 10314 10315 /* see if one of the VFs needs its hand slapped */ 10316 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 10317 vf = &(pf->vf[i]); 10318 reg = rd32(hw, I40E_VP_MDET_TX(i)); 10319 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 10320 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 10321 vf->num_mdd_events++; 10322 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 10323 i); 10324 dev_info(&pf->pdev->dev, 10325 "Use PF Control I/F to re-enable the VF\n"); 10326 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 10327 } 10328 10329 reg = rd32(hw, I40E_VP_MDET_RX(i)); 10330 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 10331 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 10332 vf->num_mdd_events++; 10333 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 10334 i); 10335 dev_info(&pf->pdev->dev, 10336 "Use PF Control I/F to re-enable the VF\n"); 10337 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 10338 } 10339 } 10340 10341 /* re-enable mdd interrupt cause */ 10342 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 10343 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 10344 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 10345 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 10346 i40e_flush(hw); 10347 } 10348 10349 static const char *i40e_tunnel_name(u8 type) 10350 { 10351 switch (type) { 10352 case UDP_TUNNEL_TYPE_VXLAN: 10353 return "vxlan"; 10354 case UDP_TUNNEL_TYPE_GENEVE: 10355 return "geneve"; 10356 default: 10357 return "unknown"; 10358 } 10359 } 10360 10361 /** 10362 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 10363 * @pf: board private structure 10364 **/ 10365 static void i40e_sync_udp_filters(struct i40e_pf *pf) 10366 { 10367 int i; 10368 10369 /* loop through and set pending bit for all active UDP filters */ 10370 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 10371 if (pf->udp_ports[i].port) 10372 pf->pending_udp_bitmap |= BIT_ULL(i); 10373 } 10374 10375 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 10376 } 10377 10378 /** 10379 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 10380 * @pf: board private structure 10381 **/ 10382 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 10383 { 10384 struct i40e_hw *hw = &pf->hw; 10385 u8 filter_index, type; 10386 u16 port; 10387 int i; 10388 10389 if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state)) 10390 return; 10391 10392 /* acquire RTNL to maintain state of flags and port requests */ 10393 rtnl_lock(); 10394 10395 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 10396 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 10397 struct i40e_udp_port_config *udp_port; 10398 i40e_status ret = 0; 10399 10400 udp_port = &pf->udp_ports[i]; 10401 pf->pending_udp_bitmap &= ~BIT_ULL(i); 10402 10403 port = READ_ONCE(udp_port->port); 10404 type = READ_ONCE(udp_port->type); 10405 filter_index = READ_ONCE(udp_port->filter_index); 10406 10407 /* release RTNL while we wait on AQ command */ 10408 rtnl_unlock(); 10409 10410 if (port) 10411 ret = i40e_aq_add_udp_tunnel(hw, port, 10412 type, 10413 &filter_index, 10414 NULL); 10415 else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED) 10416 ret = i40e_aq_del_udp_tunnel(hw, filter_index, 10417 NULL); 10418 10419 /* reacquire RTNL so we can update filter_index */ 10420 rtnl_lock(); 10421 10422 if (ret) { 10423 dev_info(&pf->pdev->dev, 10424 "%s %s port %d, index %d failed, err %s aq_err %s\n", 10425 i40e_tunnel_name(type), 10426 port ? "add" : "delete", 10427 port, 10428 filter_index, 10429 i40e_stat_str(&pf->hw, ret), 10430 i40e_aq_str(&pf->hw, 10431 pf->hw.aq.asq_last_status)); 10432 if (port) { 10433 /* failed to add, just reset port, 10434 * drop pending bit for any deletion 10435 */ 10436 udp_port->port = 0; 10437 pf->pending_udp_bitmap &= ~BIT_ULL(i); 10438 } 10439 } else if (port) { 10440 /* record filter index on success */ 10441 udp_port->filter_index = filter_index; 10442 } 10443 } 10444 } 10445 10446 rtnl_unlock(); 10447 } 10448 10449 /** 10450 * i40e_service_task - Run the driver's async subtasks 10451 * @work: pointer to work_struct containing our data 10452 **/ 10453 static void i40e_service_task(struct work_struct *work) 10454 { 10455 struct i40e_pf *pf = container_of(work, 10456 struct i40e_pf, 10457 service_task); 10458 unsigned long start_time = jiffies; 10459 10460 /* don't bother with service tasks if a reset is in progress */ 10461 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 10462 test_bit(__I40E_SUSPENDED, pf->state)) 10463 return; 10464 10465 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 10466 return; 10467 10468 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10469 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]); 10470 i40e_sync_filters_subtask(pf); 10471 i40e_reset_subtask(pf); 10472 i40e_handle_mdd_event(pf); 10473 i40e_vc_process_vflr_event(pf); 10474 i40e_watchdog_subtask(pf); 10475 i40e_fdir_reinit_subtask(pf); 10476 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 10477 /* Client subtask will reopen next time through. */ 10478 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], 10479 true); 10480 } else { 10481 i40e_client_subtask(pf); 10482 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 10483 pf->state)) 10484 i40e_notify_client_of_l2_param_changes( 10485 pf->vsi[pf->lan_vsi]); 10486 } 10487 i40e_sync_filters_subtask(pf); 10488 i40e_sync_udp_filters_subtask(pf); 10489 } else { 10490 i40e_reset_subtask(pf); 10491 } 10492 10493 i40e_clean_adminq_subtask(pf); 10494 10495 /* flush memory to make sure state is correct before next watchdog */ 10496 smp_mb__before_atomic(); 10497 clear_bit(__I40E_SERVICE_SCHED, pf->state); 10498 10499 /* If the tasks have taken longer than one timer cycle or there 10500 * is more work to be done, reschedule the service task now 10501 * rather than wait for the timer to tick again. 10502 */ 10503 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 10504 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 10505 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 10506 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 10507 i40e_service_event_schedule(pf); 10508 } 10509 10510 /** 10511 * i40e_service_timer - timer callback 10512 * @data: pointer to PF struct 10513 **/ 10514 static void i40e_service_timer(struct timer_list *t) 10515 { 10516 struct i40e_pf *pf = from_timer(pf, t, service_timer); 10517 10518 mod_timer(&pf->service_timer, 10519 round_jiffies(jiffies + pf->service_timer_period)); 10520 i40e_service_event_schedule(pf); 10521 } 10522 10523 /** 10524 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 10525 * @vsi: the VSI being configured 10526 **/ 10527 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 10528 { 10529 struct i40e_pf *pf = vsi->back; 10530 10531 switch (vsi->type) { 10532 case I40E_VSI_MAIN: 10533 vsi->alloc_queue_pairs = pf->num_lan_qps; 10534 if (!vsi->num_tx_desc) 10535 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10536 I40E_REQ_DESCRIPTOR_MULTIPLE); 10537 if (!vsi->num_rx_desc) 10538 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10539 I40E_REQ_DESCRIPTOR_MULTIPLE); 10540 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10541 vsi->num_q_vectors = pf->num_lan_msix; 10542 else 10543 vsi->num_q_vectors = 1; 10544 10545 break; 10546 10547 case I40E_VSI_FDIR: 10548 vsi->alloc_queue_pairs = 1; 10549 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT, 10550 I40E_REQ_DESCRIPTOR_MULTIPLE); 10551 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT, 10552 I40E_REQ_DESCRIPTOR_MULTIPLE); 10553 vsi->num_q_vectors = pf->num_fdsb_msix; 10554 break; 10555 10556 case I40E_VSI_VMDQ2: 10557 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 10558 if (!vsi->num_tx_desc) 10559 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10560 I40E_REQ_DESCRIPTOR_MULTIPLE); 10561 if (!vsi->num_rx_desc) 10562 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10563 I40E_REQ_DESCRIPTOR_MULTIPLE); 10564 vsi->num_q_vectors = pf->num_vmdq_msix; 10565 break; 10566 10567 case I40E_VSI_SRIOV: 10568 vsi->alloc_queue_pairs = pf->num_vf_qps; 10569 if (!vsi->num_tx_desc) 10570 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10571 I40E_REQ_DESCRIPTOR_MULTIPLE); 10572 if (!vsi->num_rx_desc) 10573 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10574 I40E_REQ_DESCRIPTOR_MULTIPLE); 10575 break; 10576 10577 default: 10578 WARN_ON(1); 10579 return -ENODATA; 10580 } 10581 10582 return 0; 10583 } 10584 10585 /** 10586 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 10587 * @vsi: VSI pointer 10588 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 10589 * 10590 * On error: returns error code (negative) 10591 * On success: returns 0 10592 **/ 10593 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 10594 { 10595 struct i40e_ring **next_rings; 10596 int size; 10597 int ret = 0; 10598 10599 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 10600 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 10601 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 10602 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 10603 if (!vsi->tx_rings) 10604 return -ENOMEM; 10605 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 10606 if (i40e_enabled_xdp_vsi(vsi)) { 10607 vsi->xdp_rings = next_rings; 10608 next_rings += vsi->alloc_queue_pairs; 10609 } 10610 vsi->rx_rings = next_rings; 10611 10612 if (alloc_qvectors) { 10613 /* allocate memory for q_vector pointers */ 10614 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 10615 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 10616 if (!vsi->q_vectors) { 10617 ret = -ENOMEM; 10618 goto err_vectors; 10619 } 10620 } 10621 return ret; 10622 10623 err_vectors: 10624 kfree(vsi->tx_rings); 10625 return ret; 10626 } 10627 10628 /** 10629 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 10630 * @pf: board private structure 10631 * @type: type of VSI 10632 * 10633 * On error: returns error code (negative) 10634 * On success: returns vsi index in PF (positive) 10635 **/ 10636 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 10637 { 10638 int ret = -ENODEV; 10639 struct i40e_vsi *vsi; 10640 int vsi_idx; 10641 int i; 10642 10643 /* Need to protect the allocation of the VSIs at the PF level */ 10644 mutex_lock(&pf->switch_mutex); 10645 10646 /* VSI list may be fragmented if VSI creation/destruction has 10647 * been happening. We can afford to do a quick scan to look 10648 * for any free VSIs in the list. 10649 * 10650 * find next empty vsi slot, looping back around if necessary 10651 */ 10652 i = pf->next_vsi; 10653 while (i < pf->num_alloc_vsi && pf->vsi[i]) 10654 i++; 10655 if (i >= pf->num_alloc_vsi) { 10656 i = 0; 10657 while (i < pf->next_vsi && pf->vsi[i]) 10658 i++; 10659 } 10660 10661 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 10662 vsi_idx = i; /* Found one! */ 10663 } else { 10664 ret = -ENODEV; 10665 goto unlock_pf; /* out of VSI slots! */ 10666 } 10667 pf->next_vsi = ++i; 10668 10669 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 10670 if (!vsi) { 10671 ret = -ENOMEM; 10672 goto unlock_pf; 10673 } 10674 vsi->type = type; 10675 vsi->back = pf; 10676 set_bit(__I40E_VSI_DOWN, vsi->state); 10677 vsi->flags = 0; 10678 vsi->idx = vsi_idx; 10679 vsi->int_rate_limit = 0; 10680 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 10681 pf->rss_table_size : 64; 10682 vsi->netdev_registered = false; 10683 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 10684 hash_init(vsi->mac_filter_hash); 10685 vsi->irqs_ready = false; 10686 10687 if (type == I40E_VSI_MAIN) { 10688 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL); 10689 if (!vsi->af_xdp_zc_qps) 10690 goto err_rings; 10691 } 10692 10693 ret = i40e_set_num_rings_in_vsi(vsi); 10694 if (ret) 10695 goto err_rings; 10696 10697 ret = i40e_vsi_alloc_arrays(vsi, true); 10698 if (ret) 10699 goto err_rings; 10700 10701 /* Setup default MSIX irq handler for VSI */ 10702 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 10703 10704 /* Initialize VSI lock */ 10705 spin_lock_init(&vsi->mac_filter_hash_lock); 10706 pf->vsi[vsi_idx] = vsi; 10707 ret = vsi_idx; 10708 goto unlock_pf; 10709 10710 err_rings: 10711 bitmap_free(vsi->af_xdp_zc_qps); 10712 pf->next_vsi = i - 1; 10713 kfree(vsi); 10714 unlock_pf: 10715 mutex_unlock(&pf->switch_mutex); 10716 return ret; 10717 } 10718 10719 /** 10720 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 10721 * @vsi: VSI pointer 10722 * @free_qvectors: a bool to specify if q_vectors need to be freed. 10723 * 10724 * On error: returns error code (negative) 10725 * On success: returns 0 10726 **/ 10727 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 10728 { 10729 /* free the ring and vector containers */ 10730 if (free_qvectors) { 10731 kfree(vsi->q_vectors); 10732 vsi->q_vectors = NULL; 10733 } 10734 kfree(vsi->tx_rings); 10735 vsi->tx_rings = NULL; 10736 vsi->rx_rings = NULL; 10737 vsi->xdp_rings = NULL; 10738 } 10739 10740 /** 10741 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 10742 * and lookup table 10743 * @vsi: Pointer to VSI structure 10744 */ 10745 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 10746 { 10747 if (!vsi) 10748 return; 10749 10750 kfree(vsi->rss_hkey_user); 10751 vsi->rss_hkey_user = NULL; 10752 10753 kfree(vsi->rss_lut_user); 10754 vsi->rss_lut_user = NULL; 10755 } 10756 10757 /** 10758 * i40e_vsi_clear - Deallocate the VSI provided 10759 * @vsi: the VSI being un-configured 10760 **/ 10761 static int i40e_vsi_clear(struct i40e_vsi *vsi) 10762 { 10763 struct i40e_pf *pf; 10764 10765 if (!vsi) 10766 return 0; 10767 10768 if (!vsi->back) 10769 goto free_vsi; 10770 pf = vsi->back; 10771 10772 mutex_lock(&pf->switch_mutex); 10773 if (!pf->vsi[vsi->idx]) { 10774 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 10775 vsi->idx, vsi->idx, vsi->type); 10776 goto unlock_vsi; 10777 } 10778 10779 if (pf->vsi[vsi->idx] != vsi) { 10780 dev_err(&pf->pdev->dev, 10781 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 10782 pf->vsi[vsi->idx]->idx, 10783 pf->vsi[vsi->idx]->type, 10784 vsi->idx, vsi->type); 10785 goto unlock_vsi; 10786 } 10787 10788 /* updates the PF for this cleared vsi */ 10789 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 10790 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 10791 10792 bitmap_free(vsi->af_xdp_zc_qps); 10793 i40e_vsi_free_arrays(vsi, true); 10794 i40e_clear_rss_config_user(vsi); 10795 10796 pf->vsi[vsi->idx] = NULL; 10797 if (vsi->idx < pf->next_vsi) 10798 pf->next_vsi = vsi->idx; 10799 10800 unlock_vsi: 10801 mutex_unlock(&pf->switch_mutex); 10802 free_vsi: 10803 kfree(vsi); 10804 10805 return 0; 10806 } 10807 10808 /** 10809 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 10810 * @vsi: the VSI being cleaned 10811 **/ 10812 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 10813 { 10814 int i; 10815 10816 if (vsi->tx_rings && vsi->tx_rings[0]) { 10817 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10818 kfree_rcu(vsi->tx_rings[i], rcu); 10819 WRITE_ONCE(vsi->tx_rings[i], NULL); 10820 WRITE_ONCE(vsi->rx_rings[i], NULL); 10821 if (vsi->xdp_rings) 10822 WRITE_ONCE(vsi->xdp_rings[i], NULL); 10823 } 10824 } 10825 } 10826 10827 /** 10828 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 10829 * @vsi: the VSI being configured 10830 **/ 10831 static int i40e_alloc_rings(struct i40e_vsi *vsi) 10832 { 10833 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 10834 struct i40e_pf *pf = vsi->back; 10835 struct i40e_ring *ring; 10836 10837 /* Set basic values in the rings to be used later during open() */ 10838 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10839 /* allocate space for both Tx and Rx in one shot */ 10840 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 10841 if (!ring) 10842 goto err_out; 10843 10844 ring->queue_index = i; 10845 ring->reg_idx = vsi->base_queue + i; 10846 ring->ring_active = false; 10847 ring->vsi = vsi; 10848 ring->netdev = vsi->netdev; 10849 ring->dev = &pf->pdev->dev; 10850 ring->count = vsi->num_tx_desc; 10851 ring->size = 0; 10852 ring->dcb_tc = 0; 10853 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10854 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10855 ring->itr_setting = pf->tx_itr_default; 10856 WRITE_ONCE(vsi->tx_rings[i], ring++); 10857 10858 if (!i40e_enabled_xdp_vsi(vsi)) 10859 goto setup_rx; 10860 10861 ring->queue_index = vsi->alloc_queue_pairs + i; 10862 ring->reg_idx = vsi->base_queue + ring->queue_index; 10863 ring->ring_active = false; 10864 ring->vsi = vsi; 10865 ring->netdev = NULL; 10866 ring->dev = &pf->pdev->dev; 10867 ring->count = vsi->num_tx_desc; 10868 ring->size = 0; 10869 ring->dcb_tc = 0; 10870 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10871 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10872 set_ring_xdp(ring); 10873 ring->itr_setting = pf->tx_itr_default; 10874 WRITE_ONCE(vsi->xdp_rings[i], ring++); 10875 10876 setup_rx: 10877 ring->queue_index = i; 10878 ring->reg_idx = vsi->base_queue + i; 10879 ring->ring_active = false; 10880 ring->vsi = vsi; 10881 ring->netdev = vsi->netdev; 10882 ring->dev = &pf->pdev->dev; 10883 ring->count = vsi->num_rx_desc; 10884 ring->size = 0; 10885 ring->dcb_tc = 0; 10886 ring->itr_setting = pf->rx_itr_default; 10887 WRITE_ONCE(vsi->rx_rings[i], ring); 10888 } 10889 10890 return 0; 10891 10892 err_out: 10893 i40e_vsi_clear_rings(vsi); 10894 return -ENOMEM; 10895 } 10896 10897 /** 10898 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 10899 * @pf: board private structure 10900 * @vectors: the number of MSI-X vectors to request 10901 * 10902 * Returns the number of vectors reserved, or error 10903 **/ 10904 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 10905 { 10906 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 10907 I40E_MIN_MSIX, vectors); 10908 if (vectors < 0) { 10909 dev_info(&pf->pdev->dev, 10910 "MSI-X vector reservation failed: %d\n", vectors); 10911 vectors = 0; 10912 } 10913 10914 return vectors; 10915 } 10916 10917 /** 10918 * i40e_init_msix - Setup the MSIX capability 10919 * @pf: board private structure 10920 * 10921 * Work with the OS to set up the MSIX vectors needed. 10922 * 10923 * Returns the number of vectors reserved or negative on failure 10924 **/ 10925 static int i40e_init_msix(struct i40e_pf *pf) 10926 { 10927 struct i40e_hw *hw = &pf->hw; 10928 int cpus, extra_vectors; 10929 int vectors_left; 10930 int v_budget, i; 10931 int v_actual; 10932 int iwarp_requested = 0; 10933 10934 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10935 return -ENODEV; 10936 10937 /* The number of vectors we'll request will be comprised of: 10938 * - Add 1 for "other" cause for Admin Queue events, etc. 10939 * - The number of LAN queue pairs 10940 * - Queues being used for RSS. 10941 * We don't need as many as max_rss_size vectors. 10942 * use rss_size instead in the calculation since that 10943 * is governed by number of cpus in the system. 10944 * - assumes symmetric Tx/Rx pairing 10945 * - The number of VMDq pairs 10946 * - The CPU count within the NUMA node if iWARP is enabled 10947 * Once we count this up, try the request. 10948 * 10949 * If we can't get what we want, we'll simplify to nearly nothing 10950 * and try again. If that still fails, we punt. 10951 */ 10952 vectors_left = hw->func_caps.num_msix_vectors; 10953 v_budget = 0; 10954 10955 /* reserve one vector for miscellaneous handler */ 10956 if (vectors_left) { 10957 v_budget++; 10958 vectors_left--; 10959 } 10960 10961 /* reserve some vectors for the main PF traffic queues. Initially we 10962 * only reserve at most 50% of the available vectors, in the case that 10963 * the number of online CPUs is large. This ensures that we can enable 10964 * extra features as well. Once we've enabled the other features, we 10965 * will use any remaining vectors to reach as close as we can to the 10966 * number of online CPUs. 10967 */ 10968 cpus = num_online_cpus(); 10969 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 10970 vectors_left -= pf->num_lan_msix; 10971 10972 /* reserve one vector for sideband flow director */ 10973 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10974 if (vectors_left) { 10975 pf->num_fdsb_msix = 1; 10976 v_budget++; 10977 vectors_left--; 10978 } else { 10979 pf->num_fdsb_msix = 0; 10980 } 10981 } 10982 10983 /* can we reserve enough for iWARP? */ 10984 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10985 iwarp_requested = pf->num_iwarp_msix; 10986 10987 if (!vectors_left) 10988 pf->num_iwarp_msix = 0; 10989 else if (vectors_left < pf->num_iwarp_msix) 10990 pf->num_iwarp_msix = 1; 10991 v_budget += pf->num_iwarp_msix; 10992 vectors_left -= pf->num_iwarp_msix; 10993 } 10994 10995 /* any vectors left over go for VMDq support */ 10996 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 10997 if (!vectors_left) { 10998 pf->num_vmdq_msix = 0; 10999 pf->num_vmdq_qps = 0; 11000 } else { 11001 int vmdq_vecs_wanted = 11002 pf->num_vmdq_vsis * pf->num_vmdq_qps; 11003 int vmdq_vecs = 11004 min_t(int, vectors_left, vmdq_vecs_wanted); 11005 11006 /* if we're short on vectors for what's desired, we limit 11007 * the queues per vmdq. If this is still more than are 11008 * available, the user will need to change the number of 11009 * queues/vectors used by the PF later with the ethtool 11010 * channels command 11011 */ 11012 if (vectors_left < vmdq_vecs_wanted) { 11013 pf->num_vmdq_qps = 1; 11014 vmdq_vecs_wanted = pf->num_vmdq_vsis; 11015 vmdq_vecs = min_t(int, 11016 vectors_left, 11017 vmdq_vecs_wanted); 11018 } 11019 pf->num_vmdq_msix = pf->num_vmdq_qps; 11020 11021 v_budget += vmdq_vecs; 11022 vectors_left -= vmdq_vecs; 11023 } 11024 } 11025 11026 /* On systems with a large number of SMP cores, we previously limited 11027 * the number of vectors for num_lan_msix to be at most 50% of the 11028 * available vectors, to allow for other features. Now, we add back 11029 * the remaining vectors. However, we ensure that the total 11030 * num_lan_msix will not exceed num_online_cpus(). To do this, we 11031 * calculate the number of vectors we can add without going over the 11032 * cap of CPUs. For systems with a small number of CPUs this will be 11033 * zero. 11034 */ 11035 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 11036 pf->num_lan_msix += extra_vectors; 11037 vectors_left -= extra_vectors; 11038 11039 WARN(vectors_left < 0, 11040 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 11041 11042 v_budget += pf->num_lan_msix; 11043 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 11044 GFP_KERNEL); 11045 if (!pf->msix_entries) 11046 return -ENOMEM; 11047 11048 for (i = 0; i < v_budget; i++) 11049 pf->msix_entries[i].entry = i; 11050 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 11051 11052 if (v_actual < I40E_MIN_MSIX) { 11053 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 11054 kfree(pf->msix_entries); 11055 pf->msix_entries = NULL; 11056 pci_disable_msix(pf->pdev); 11057 return -ENODEV; 11058 11059 } else if (v_actual == I40E_MIN_MSIX) { 11060 /* Adjust for minimal MSIX use */ 11061 pf->num_vmdq_vsis = 0; 11062 pf->num_vmdq_qps = 0; 11063 pf->num_lan_qps = 1; 11064 pf->num_lan_msix = 1; 11065 11066 } else if (v_actual != v_budget) { 11067 /* If we have limited resources, we will start with no vectors 11068 * for the special features and then allocate vectors to some 11069 * of these features based on the policy and at the end disable 11070 * the features that did not get any vectors. 11071 */ 11072 int vec; 11073 11074 dev_info(&pf->pdev->dev, 11075 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 11076 v_actual, v_budget); 11077 /* reserve the misc vector */ 11078 vec = v_actual - 1; 11079 11080 /* Scale vector usage down */ 11081 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 11082 pf->num_vmdq_vsis = 1; 11083 pf->num_vmdq_qps = 1; 11084 11085 /* partition out the remaining vectors */ 11086 switch (vec) { 11087 case 2: 11088 pf->num_lan_msix = 1; 11089 break; 11090 case 3: 11091 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11092 pf->num_lan_msix = 1; 11093 pf->num_iwarp_msix = 1; 11094 } else { 11095 pf->num_lan_msix = 2; 11096 } 11097 break; 11098 default: 11099 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11100 pf->num_iwarp_msix = min_t(int, (vec / 3), 11101 iwarp_requested); 11102 pf->num_vmdq_vsis = min_t(int, (vec / 3), 11103 I40E_DEFAULT_NUM_VMDQ_VSI); 11104 } else { 11105 pf->num_vmdq_vsis = min_t(int, (vec / 2), 11106 I40E_DEFAULT_NUM_VMDQ_VSI); 11107 } 11108 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11109 pf->num_fdsb_msix = 1; 11110 vec--; 11111 } 11112 pf->num_lan_msix = min_t(int, 11113 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 11114 pf->num_lan_msix); 11115 pf->num_lan_qps = pf->num_lan_msix; 11116 break; 11117 } 11118 } 11119 11120 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 11121 (pf->num_fdsb_msix == 0)) { 11122 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 11123 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 11124 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11125 } 11126 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 11127 (pf->num_vmdq_msix == 0)) { 11128 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 11129 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 11130 } 11131 11132 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 11133 (pf->num_iwarp_msix == 0)) { 11134 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 11135 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11136 } 11137 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 11138 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 11139 pf->num_lan_msix, 11140 pf->num_vmdq_msix * pf->num_vmdq_vsis, 11141 pf->num_fdsb_msix, 11142 pf->num_iwarp_msix); 11143 11144 return v_actual; 11145 } 11146 11147 /** 11148 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 11149 * @vsi: the VSI being configured 11150 * @v_idx: index of the vector in the vsi struct 11151 * @cpu: cpu to be used on affinity_mask 11152 * 11153 * We allocate one q_vector. If allocation fails we return -ENOMEM. 11154 **/ 11155 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 11156 { 11157 struct i40e_q_vector *q_vector; 11158 11159 /* allocate q_vector */ 11160 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 11161 if (!q_vector) 11162 return -ENOMEM; 11163 11164 q_vector->vsi = vsi; 11165 q_vector->v_idx = v_idx; 11166 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 11167 11168 if (vsi->netdev) 11169 netif_napi_add(vsi->netdev, &q_vector->napi, 11170 i40e_napi_poll, NAPI_POLL_WEIGHT); 11171 11172 /* tie q_vector and vsi together */ 11173 vsi->q_vectors[v_idx] = q_vector; 11174 11175 return 0; 11176 } 11177 11178 /** 11179 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 11180 * @vsi: the VSI being configured 11181 * 11182 * We allocate one q_vector per queue interrupt. If allocation fails we 11183 * return -ENOMEM. 11184 **/ 11185 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 11186 { 11187 struct i40e_pf *pf = vsi->back; 11188 int err, v_idx, num_q_vectors, current_cpu; 11189 11190 /* if not MSIX, give the one vector only to the LAN VSI */ 11191 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 11192 num_q_vectors = vsi->num_q_vectors; 11193 else if (vsi == pf->vsi[pf->lan_vsi]) 11194 num_q_vectors = 1; 11195 else 11196 return -EINVAL; 11197 11198 current_cpu = cpumask_first(cpu_online_mask); 11199 11200 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 11201 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 11202 if (err) 11203 goto err_out; 11204 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 11205 if (unlikely(current_cpu >= nr_cpu_ids)) 11206 current_cpu = cpumask_first(cpu_online_mask); 11207 } 11208 11209 return 0; 11210 11211 err_out: 11212 while (v_idx--) 11213 i40e_free_q_vector(vsi, v_idx); 11214 11215 return err; 11216 } 11217 11218 /** 11219 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 11220 * @pf: board private structure to initialize 11221 **/ 11222 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 11223 { 11224 int vectors = 0; 11225 ssize_t size; 11226 11227 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11228 vectors = i40e_init_msix(pf); 11229 if (vectors < 0) { 11230 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 11231 I40E_FLAG_IWARP_ENABLED | 11232 I40E_FLAG_RSS_ENABLED | 11233 I40E_FLAG_DCB_CAPABLE | 11234 I40E_FLAG_DCB_ENABLED | 11235 I40E_FLAG_SRIOV_ENABLED | 11236 I40E_FLAG_FD_SB_ENABLED | 11237 I40E_FLAG_FD_ATR_ENABLED | 11238 I40E_FLAG_VMDQ_ENABLED); 11239 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11240 11241 /* rework the queue expectations without MSIX */ 11242 i40e_determine_queue_usage(pf); 11243 } 11244 } 11245 11246 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 11247 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 11248 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 11249 vectors = pci_enable_msi(pf->pdev); 11250 if (vectors < 0) { 11251 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 11252 vectors); 11253 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 11254 } 11255 vectors = 1; /* one MSI or Legacy vector */ 11256 } 11257 11258 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 11259 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 11260 11261 /* set up vector assignment tracking */ 11262 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 11263 pf->irq_pile = kzalloc(size, GFP_KERNEL); 11264 if (!pf->irq_pile) 11265 return -ENOMEM; 11266 11267 pf->irq_pile->num_entries = vectors; 11268 pf->irq_pile->search_hint = 0; 11269 11270 /* track first vector for misc interrupts, ignore return */ 11271 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 11272 11273 return 0; 11274 } 11275 11276 /** 11277 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 11278 * @pf: private board data structure 11279 * 11280 * Restore the interrupt scheme that was cleared when we suspended the 11281 * device. This should be called during resume to re-allocate the q_vectors 11282 * and reacquire IRQs. 11283 */ 11284 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 11285 { 11286 int err, i; 11287 11288 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 11289 * scheme. We need to re-enabled them here in order to attempt to 11290 * re-acquire the MSI or MSI-X vectors 11291 */ 11292 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 11293 11294 err = i40e_init_interrupt_scheme(pf); 11295 if (err) 11296 return err; 11297 11298 /* Now that we've re-acquired IRQs, we need to remap the vectors and 11299 * rings together again. 11300 */ 11301 for (i = 0; i < pf->num_alloc_vsi; i++) { 11302 if (pf->vsi[i]) { 11303 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 11304 if (err) 11305 goto err_unwind; 11306 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 11307 } 11308 } 11309 11310 err = i40e_setup_misc_vector(pf); 11311 if (err) 11312 goto err_unwind; 11313 11314 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 11315 i40e_client_update_msix_info(pf); 11316 11317 return 0; 11318 11319 err_unwind: 11320 while (i--) { 11321 if (pf->vsi[i]) 11322 i40e_vsi_free_q_vectors(pf->vsi[i]); 11323 } 11324 11325 return err; 11326 } 11327 11328 /** 11329 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle 11330 * non queue events in recovery mode 11331 * @pf: board private structure 11332 * 11333 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage 11334 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode. 11335 * This is handled differently than in recovery mode since no Tx/Rx resources 11336 * are being allocated. 11337 **/ 11338 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf) 11339 { 11340 int err; 11341 11342 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11343 err = i40e_setup_misc_vector(pf); 11344 11345 if (err) { 11346 dev_info(&pf->pdev->dev, 11347 "MSI-X misc vector request failed, error %d\n", 11348 err); 11349 return err; 11350 } 11351 } else { 11352 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED; 11353 11354 err = request_irq(pf->pdev->irq, i40e_intr, flags, 11355 pf->int_name, pf); 11356 11357 if (err) { 11358 dev_info(&pf->pdev->dev, 11359 "MSI/legacy misc vector request failed, error %d\n", 11360 err); 11361 return err; 11362 } 11363 i40e_enable_misc_int_causes(pf); 11364 i40e_irq_dynamic_enable_icr0(pf); 11365 } 11366 11367 return 0; 11368 } 11369 11370 /** 11371 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 11372 * @pf: board private structure 11373 * 11374 * This sets up the handler for MSIX 0, which is used to manage the 11375 * non-queue interrupts, e.g. AdminQ and errors. This is not used 11376 * when in MSI or Legacy interrupt mode. 11377 **/ 11378 static int i40e_setup_misc_vector(struct i40e_pf *pf) 11379 { 11380 struct i40e_hw *hw = &pf->hw; 11381 int err = 0; 11382 11383 /* Only request the IRQ once, the first time through. */ 11384 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 11385 err = request_irq(pf->msix_entries[0].vector, 11386 i40e_intr, 0, pf->int_name, pf); 11387 if (err) { 11388 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 11389 dev_info(&pf->pdev->dev, 11390 "request_irq for %s failed: %d\n", 11391 pf->int_name, err); 11392 return -EFAULT; 11393 } 11394 } 11395 11396 i40e_enable_misc_int_causes(pf); 11397 11398 /* associate no queues to the misc vector */ 11399 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 11400 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1); 11401 11402 i40e_flush(hw); 11403 11404 i40e_irq_dynamic_enable_icr0(pf); 11405 11406 return err; 11407 } 11408 11409 /** 11410 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 11411 * @vsi: Pointer to vsi structure 11412 * @seed: Buffter to store the hash keys 11413 * @lut: Buffer to store the lookup table entries 11414 * @lut_size: Size of buffer to store the lookup table entries 11415 * 11416 * Return 0 on success, negative on failure 11417 */ 11418 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 11419 u8 *lut, u16 lut_size) 11420 { 11421 struct i40e_pf *pf = vsi->back; 11422 struct i40e_hw *hw = &pf->hw; 11423 int ret = 0; 11424 11425 if (seed) { 11426 ret = i40e_aq_get_rss_key(hw, vsi->id, 11427 (struct i40e_aqc_get_set_rss_key_data *)seed); 11428 if (ret) { 11429 dev_info(&pf->pdev->dev, 11430 "Cannot get RSS key, err %s aq_err %s\n", 11431 i40e_stat_str(&pf->hw, ret), 11432 i40e_aq_str(&pf->hw, 11433 pf->hw.aq.asq_last_status)); 11434 return ret; 11435 } 11436 } 11437 11438 if (lut) { 11439 bool pf_lut = vsi->type == I40E_VSI_MAIN; 11440 11441 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 11442 if (ret) { 11443 dev_info(&pf->pdev->dev, 11444 "Cannot get RSS lut, err %s aq_err %s\n", 11445 i40e_stat_str(&pf->hw, ret), 11446 i40e_aq_str(&pf->hw, 11447 pf->hw.aq.asq_last_status)); 11448 return ret; 11449 } 11450 } 11451 11452 return ret; 11453 } 11454 11455 /** 11456 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 11457 * @vsi: Pointer to vsi structure 11458 * @seed: RSS hash seed 11459 * @lut: Lookup table 11460 * @lut_size: Lookup table size 11461 * 11462 * Returns 0 on success, negative on failure 11463 **/ 11464 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 11465 const u8 *lut, u16 lut_size) 11466 { 11467 struct i40e_pf *pf = vsi->back; 11468 struct i40e_hw *hw = &pf->hw; 11469 u16 vf_id = vsi->vf_id; 11470 u8 i; 11471 11472 /* Fill out hash function seed */ 11473 if (seed) { 11474 u32 *seed_dw = (u32 *)seed; 11475 11476 if (vsi->type == I40E_VSI_MAIN) { 11477 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 11478 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 11479 } else if (vsi->type == I40E_VSI_SRIOV) { 11480 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 11481 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 11482 } else { 11483 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 11484 } 11485 } 11486 11487 if (lut) { 11488 u32 *lut_dw = (u32 *)lut; 11489 11490 if (vsi->type == I40E_VSI_MAIN) { 11491 if (lut_size != I40E_HLUT_ARRAY_SIZE) 11492 return -EINVAL; 11493 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11494 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 11495 } else if (vsi->type == I40E_VSI_SRIOV) { 11496 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 11497 return -EINVAL; 11498 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 11499 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 11500 } else { 11501 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 11502 } 11503 } 11504 i40e_flush(hw); 11505 11506 return 0; 11507 } 11508 11509 /** 11510 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 11511 * @vsi: Pointer to VSI structure 11512 * @seed: Buffer to store the keys 11513 * @lut: Buffer to store the lookup table entries 11514 * @lut_size: Size of buffer to store the lookup table entries 11515 * 11516 * Returns 0 on success, negative on failure 11517 */ 11518 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 11519 u8 *lut, u16 lut_size) 11520 { 11521 struct i40e_pf *pf = vsi->back; 11522 struct i40e_hw *hw = &pf->hw; 11523 u16 i; 11524 11525 if (seed) { 11526 u32 *seed_dw = (u32 *)seed; 11527 11528 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 11529 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 11530 } 11531 if (lut) { 11532 u32 *lut_dw = (u32 *)lut; 11533 11534 if (lut_size != I40E_HLUT_ARRAY_SIZE) 11535 return -EINVAL; 11536 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11537 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 11538 } 11539 11540 return 0; 11541 } 11542 11543 /** 11544 * i40e_config_rss - Configure RSS keys and lut 11545 * @vsi: Pointer to VSI structure 11546 * @seed: RSS hash seed 11547 * @lut: Lookup table 11548 * @lut_size: Lookup table size 11549 * 11550 * Returns 0 on success, negative on failure 11551 */ 11552 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 11553 { 11554 struct i40e_pf *pf = vsi->back; 11555 11556 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 11557 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 11558 else 11559 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 11560 } 11561 11562 /** 11563 * i40e_get_rss - Get RSS keys and lut 11564 * @vsi: Pointer to VSI structure 11565 * @seed: Buffer to store the keys 11566 * @lut: Buffer to store the lookup table entries 11567 * @lut_size: Size of buffer to store the lookup table entries 11568 * 11569 * Returns 0 on success, negative on failure 11570 */ 11571 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 11572 { 11573 struct i40e_pf *pf = vsi->back; 11574 11575 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 11576 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 11577 else 11578 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 11579 } 11580 11581 /** 11582 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 11583 * @pf: Pointer to board private structure 11584 * @lut: Lookup table 11585 * @rss_table_size: Lookup table size 11586 * @rss_size: Range of queue number for hashing 11587 */ 11588 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 11589 u16 rss_table_size, u16 rss_size) 11590 { 11591 u16 i; 11592 11593 for (i = 0; i < rss_table_size; i++) 11594 lut[i] = i % rss_size; 11595 } 11596 11597 /** 11598 * i40e_pf_config_rss - Prepare for RSS if used 11599 * @pf: board private structure 11600 **/ 11601 static int i40e_pf_config_rss(struct i40e_pf *pf) 11602 { 11603 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11604 u8 seed[I40E_HKEY_ARRAY_SIZE]; 11605 u8 *lut; 11606 struct i40e_hw *hw = &pf->hw; 11607 u32 reg_val; 11608 u64 hena; 11609 int ret; 11610 11611 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 11612 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 11613 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 11614 hena |= i40e_pf_get_default_rss_hena(pf); 11615 11616 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 11617 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 11618 11619 /* Determine the RSS table size based on the hardware capabilities */ 11620 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 11621 reg_val = (pf->rss_table_size == 512) ? 11622 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 11623 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 11624 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 11625 11626 /* Determine the RSS size of the VSI */ 11627 if (!vsi->rss_size) { 11628 u16 qcount; 11629 /* If the firmware does something weird during VSI init, we 11630 * could end up with zero TCs. Check for that to avoid 11631 * divide-by-zero. It probably won't pass traffic, but it also 11632 * won't panic. 11633 */ 11634 qcount = vsi->num_queue_pairs / 11635 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 11636 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11637 } 11638 if (!vsi->rss_size) 11639 return -EINVAL; 11640 11641 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 11642 if (!lut) 11643 return -ENOMEM; 11644 11645 /* Use user configured lut if there is one, otherwise use default */ 11646 if (vsi->rss_lut_user) 11647 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 11648 else 11649 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 11650 11651 /* Use user configured hash key if there is one, otherwise 11652 * use default. 11653 */ 11654 if (vsi->rss_hkey_user) 11655 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 11656 else 11657 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 11658 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 11659 kfree(lut); 11660 11661 return ret; 11662 } 11663 11664 /** 11665 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 11666 * @pf: board private structure 11667 * @queue_count: the requested queue count for rss. 11668 * 11669 * returns 0 if rss is not enabled, if enabled returns the final rss queue 11670 * count which may be different from the requested queue count. 11671 * Note: expects to be called while under rtnl_lock() 11672 **/ 11673 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 11674 { 11675 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11676 int new_rss_size; 11677 11678 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 11679 return 0; 11680 11681 queue_count = min_t(int, queue_count, num_online_cpus()); 11682 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 11683 11684 if (queue_count != vsi->num_queue_pairs) { 11685 u16 qcount; 11686 11687 vsi->req_queue_pairs = queue_count; 11688 i40e_prep_for_reset(pf, true); 11689 11690 pf->alloc_rss_size = new_rss_size; 11691 11692 i40e_reset_and_rebuild(pf, true, true); 11693 11694 /* Discard the user configured hash keys and lut, if less 11695 * queues are enabled. 11696 */ 11697 if (queue_count < vsi->rss_size) { 11698 i40e_clear_rss_config_user(vsi); 11699 dev_dbg(&pf->pdev->dev, 11700 "discard user configured hash keys and lut\n"); 11701 } 11702 11703 /* Reset vsi->rss_size, as number of enabled queues changed */ 11704 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 11705 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11706 11707 i40e_pf_config_rss(pf); 11708 } 11709 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 11710 vsi->req_queue_pairs, pf->rss_size_max); 11711 return pf->alloc_rss_size; 11712 } 11713 11714 /** 11715 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 11716 * @pf: board private structure 11717 **/ 11718 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 11719 { 11720 i40e_status status; 11721 bool min_valid, max_valid; 11722 u32 max_bw, min_bw; 11723 11724 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 11725 &min_valid, &max_valid); 11726 11727 if (!status) { 11728 if (min_valid) 11729 pf->min_bw = min_bw; 11730 if (max_valid) 11731 pf->max_bw = max_bw; 11732 } 11733 11734 return status; 11735 } 11736 11737 /** 11738 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 11739 * @pf: board private structure 11740 **/ 11741 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 11742 { 11743 struct i40e_aqc_configure_partition_bw_data bw_data; 11744 i40e_status status; 11745 11746 /* Set the valid bit for this PF */ 11747 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 11748 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 11749 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 11750 11751 /* Set the new bandwidths */ 11752 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 11753 11754 return status; 11755 } 11756 11757 /** 11758 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 11759 * @pf: board private structure 11760 **/ 11761 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 11762 { 11763 /* Commit temporary BW setting to permanent NVM image */ 11764 enum i40e_admin_queue_err last_aq_status; 11765 i40e_status ret; 11766 u16 nvm_word; 11767 11768 if (pf->hw.partition_id != 1) { 11769 dev_info(&pf->pdev->dev, 11770 "Commit BW only works on partition 1! This is partition %d", 11771 pf->hw.partition_id); 11772 ret = I40E_NOT_SUPPORTED; 11773 goto bw_commit_out; 11774 } 11775 11776 /* Acquire NVM for read access */ 11777 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 11778 last_aq_status = pf->hw.aq.asq_last_status; 11779 if (ret) { 11780 dev_info(&pf->pdev->dev, 11781 "Cannot acquire NVM for read access, err %s aq_err %s\n", 11782 i40e_stat_str(&pf->hw, ret), 11783 i40e_aq_str(&pf->hw, last_aq_status)); 11784 goto bw_commit_out; 11785 } 11786 11787 /* Read word 0x10 of NVM - SW compatibility word 1 */ 11788 ret = i40e_aq_read_nvm(&pf->hw, 11789 I40E_SR_NVM_CONTROL_WORD, 11790 0x10, sizeof(nvm_word), &nvm_word, 11791 false, NULL); 11792 /* Save off last admin queue command status before releasing 11793 * the NVM 11794 */ 11795 last_aq_status = pf->hw.aq.asq_last_status; 11796 i40e_release_nvm(&pf->hw); 11797 if (ret) { 11798 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 11799 i40e_stat_str(&pf->hw, ret), 11800 i40e_aq_str(&pf->hw, last_aq_status)); 11801 goto bw_commit_out; 11802 } 11803 11804 /* Wait a bit for NVM release to complete */ 11805 msleep(50); 11806 11807 /* Acquire NVM for write access */ 11808 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 11809 last_aq_status = pf->hw.aq.asq_last_status; 11810 if (ret) { 11811 dev_info(&pf->pdev->dev, 11812 "Cannot acquire NVM for write access, err %s aq_err %s\n", 11813 i40e_stat_str(&pf->hw, ret), 11814 i40e_aq_str(&pf->hw, last_aq_status)); 11815 goto bw_commit_out; 11816 } 11817 /* Write it back out unchanged to initiate update NVM, 11818 * which will force a write of the shadow (alt) RAM to 11819 * the NVM - thus storing the bandwidth values permanently. 11820 */ 11821 ret = i40e_aq_update_nvm(&pf->hw, 11822 I40E_SR_NVM_CONTROL_WORD, 11823 0x10, sizeof(nvm_word), 11824 &nvm_word, true, 0, NULL); 11825 /* Save off last admin queue command status before releasing 11826 * the NVM 11827 */ 11828 last_aq_status = pf->hw.aq.asq_last_status; 11829 i40e_release_nvm(&pf->hw); 11830 if (ret) 11831 dev_info(&pf->pdev->dev, 11832 "BW settings NOT SAVED, err %s aq_err %s\n", 11833 i40e_stat_str(&pf->hw, ret), 11834 i40e_aq_str(&pf->hw, last_aq_status)); 11835 bw_commit_out: 11836 11837 return ret; 11838 } 11839 11840 /** 11841 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 11842 * @pf: board private structure to initialize 11843 * 11844 * i40e_sw_init initializes the Adapter private data structure. 11845 * Fields are initialized based on PCI device information and 11846 * OS network device settings (MTU size). 11847 **/ 11848 static int i40e_sw_init(struct i40e_pf *pf) 11849 { 11850 int err = 0; 11851 int size; 11852 11853 /* Set default capability flags */ 11854 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 11855 I40E_FLAG_MSI_ENABLED | 11856 I40E_FLAG_MSIX_ENABLED; 11857 11858 /* Set default ITR */ 11859 pf->rx_itr_default = I40E_ITR_RX_DEF; 11860 pf->tx_itr_default = I40E_ITR_TX_DEF; 11861 11862 /* Depending on PF configurations, it is possible that the RSS 11863 * maximum might end up larger than the available queues 11864 */ 11865 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 11866 pf->alloc_rss_size = 1; 11867 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 11868 pf->rss_size_max = min_t(int, pf->rss_size_max, 11869 pf->hw.func_caps.num_tx_qp); 11870 if (pf->hw.func_caps.rss) { 11871 pf->flags |= I40E_FLAG_RSS_ENABLED; 11872 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 11873 num_online_cpus()); 11874 } 11875 11876 /* MFP mode enabled */ 11877 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 11878 pf->flags |= I40E_FLAG_MFP_ENABLED; 11879 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 11880 if (i40e_get_partition_bw_setting(pf)) { 11881 dev_warn(&pf->pdev->dev, 11882 "Could not get partition bw settings\n"); 11883 } else { 11884 dev_info(&pf->pdev->dev, 11885 "Partition BW Min = %8.8x, Max = %8.8x\n", 11886 pf->min_bw, pf->max_bw); 11887 11888 /* nudge the Tx scheduler */ 11889 i40e_set_partition_bw_setting(pf); 11890 } 11891 } 11892 11893 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 11894 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 11895 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 11896 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 11897 if (pf->flags & I40E_FLAG_MFP_ENABLED && 11898 pf->hw.num_partitions > 1) 11899 dev_info(&pf->pdev->dev, 11900 "Flow Director Sideband mode Disabled in MFP mode\n"); 11901 else 11902 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11903 pf->fdir_pf_filter_count = 11904 pf->hw.func_caps.fd_filters_guaranteed; 11905 pf->hw.fdir_shared_filter_count = 11906 pf->hw.func_caps.fd_filters_best_effort; 11907 } 11908 11909 if (pf->hw.mac.type == I40E_MAC_X722) { 11910 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 11911 I40E_HW_128_QP_RSS_CAPABLE | 11912 I40E_HW_ATR_EVICT_CAPABLE | 11913 I40E_HW_WB_ON_ITR_CAPABLE | 11914 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 11915 I40E_HW_NO_PCI_LINK_CHECK | 11916 I40E_HW_USE_SET_LLDP_MIB | 11917 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 11918 I40E_HW_PTP_L4_CAPABLE | 11919 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 11920 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 11921 11922 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 11923 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 11924 I40E_FDEVICT_PCTYPE_DEFAULT) { 11925 dev_warn(&pf->pdev->dev, 11926 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 11927 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 11928 } 11929 } else if ((pf->hw.aq.api_maj_ver > 1) || 11930 ((pf->hw.aq.api_maj_ver == 1) && 11931 (pf->hw.aq.api_min_ver > 4))) { 11932 /* Supported in FW API version higher than 1.4 */ 11933 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 11934 } 11935 11936 /* Enable HW ATR eviction if possible */ 11937 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 11938 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 11939 11940 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11941 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 11942 (pf->hw.aq.fw_maj_ver < 4))) { 11943 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 11944 /* No DCB support for FW < v4.33 */ 11945 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 11946 } 11947 11948 /* Disable FW LLDP if FW < v4.3 */ 11949 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11950 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 11951 (pf->hw.aq.fw_maj_ver < 4))) 11952 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 11953 11954 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 11955 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11956 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 11957 (pf->hw.aq.fw_maj_ver >= 5))) 11958 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 11959 11960 /* Enable PTP L4 if FW > v6.0 */ 11961 if (pf->hw.mac.type == I40E_MAC_XL710 && 11962 pf->hw.aq.fw_maj_ver >= 6) 11963 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 11964 11965 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 11966 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 11967 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 11968 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 11969 } 11970 11971 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 11972 pf->flags |= I40E_FLAG_IWARP_ENABLED; 11973 /* IWARP needs one extra vector for CQP just like MISC.*/ 11974 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 11975 } 11976 /* Stopping FW LLDP engine is supported on XL710 and X722 11977 * starting from FW versions determined in i40e_init_adminq. 11978 * Stopping the FW LLDP engine is not supported on XL710 11979 * if NPAR is functioning so unset this hw flag in this case. 11980 */ 11981 if (pf->hw.mac.type == I40E_MAC_XL710 && 11982 pf->hw.func_caps.npar_enable && 11983 (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE)) 11984 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE; 11985 11986 #ifdef CONFIG_PCI_IOV 11987 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 11988 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 11989 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 11990 pf->num_req_vfs = min_t(int, 11991 pf->hw.func_caps.num_vfs, 11992 I40E_MAX_VF_COUNT); 11993 } 11994 #endif /* CONFIG_PCI_IOV */ 11995 pf->eeprom_version = 0xDEAD; 11996 pf->lan_veb = I40E_NO_VEB; 11997 pf->lan_vsi = I40E_NO_VSI; 11998 11999 /* By default FW has this off for performance reasons */ 12000 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 12001 12002 /* set up queue assignment tracking */ 12003 size = sizeof(struct i40e_lump_tracking) 12004 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 12005 pf->qp_pile = kzalloc(size, GFP_KERNEL); 12006 if (!pf->qp_pile) { 12007 err = -ENOMEM; 12008 goto sw_init_done; 12009 } 12010 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 12011 pf->qp_pile->search_hint = 0; 12012 12013 pf->tx_timeout_recovery_level = 1; 12014 12015 mutex_init(&pf->switch_mutex); 12016 12017 sw_init_done: 12018 return err; 12019 } 12020 12021 /** 12022 * i40e_set_ntuple - set the ntuple feature flag and take action 12023 * @pf: board private structure to initialize 12024 * @features: the feature set that the stack is suggesting 12025 * 12026 * returns a bool to indicate if reset needs to happen 12027 **/ 12028 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 12029 { 12030 bool need_reset = false; 12031 12032 /* Check if Flow Director n-tuple support was enabled or disabled. If 12033 * the state changed, we need to reset. 12034 */ 12035 if (features & NETIF_F_NTUPLE) { 12036 /* Enable filters and mark for reset */ 12037 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 12038 need_reset = true; 12039 /* enable FD_SB only if there is MSI-X vector and no cloud 12040 * filters exist 12041 */ 12042 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 12043 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 12044 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 12045 } 12046 } else { 12047 /* turn off filters, mark for reset and clear SW filter list */ 12048 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 12049 need_reset = true; 12050 i40e_fdir_filter_exit(pf); 12051 } 12052 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 12053 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 12054 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 12055 12056 /* reset fd counters */ 12057 pf->fd_add_err = 0; 12058 pf->fd_atr_cnt = 0; 12059 /* if ATR was auto disabled it can be re-enabled. */ 12060 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 12061 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 12062 (I40E_DEBUG_FD & pf->hw.debug_mask)) 12063 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 12064 } 12065 return need_reset; 12066 } 12067 12068 /** 12069 * i40e_clear_rss_lut - clear the rx hash lookup table 12070 * @vsi: the VSI being configured 12071 **/ 12072 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 12073 { 12074 struct i40e_pf *pf = vsi->back; 12075 struct i40e_hw *hw = &pf->hw; 12076 u16 vf_id = vsi->vf_id; 12077 u8 i; 12078 12079 if (vsi->type == I40E_VSI_MAIN) { 12080 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12081 wr32(hw, I40E_PFQF_HLUT(i), 0); 12082 } else if (vsi->type == I40E_VSI_SRIOV) { 12083 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12084 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 12085 } else { 12086 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12087 } 12088 } 12089 12090 /** 12091 * i40e_set_features - set the netdev feature flags 12092 * @netdev: ptr to the netdev being adjusted 12093 * @features: the feature set that the stack is suggesting 12094 * Note: expects to be called while under rtnl_lock() 12095 **/ 12096 static int i40e_set_features(struct net_device *netdev, 12097 netdev_features_t features) 12098 { 12099 struct i40e_netdev_priv *np = netdev_priv(netdev); 12100 struct i40e_vsi *vsi = np->vsi; 12101 struct i40e_pf *pf = vsi->back; 12102 bool need_reset; 12103 12104 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 12105 i40e_pf_config_rss(pf); 12106 else if (!(features & NETIF_F_RXHASH) && 12107 netdev->features & NETIF_F_RXHASH) 12108 i40e_clear_rss_lut(vsi); 12109 12110 if (features & NETIF_F_HW_VLAN_CTAG_RX) 12111 i40e_vlan_stripping_enable(vsi); 12112 else 12113 i40e_vlan_stripping_disable(vsi); 12114 12115 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 12116 dev_err(&pf->pdev->dev, 12117 "Offloaded tc filters active, can't turn hw_tc_offload off"); 12118 return -EINVAL; 12119 } 12120 12121 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt) 12122 i40e_del_all_macvlans(vsi); 12123 12124 need_reset = i40e_set_ntuple(pf, features); 12125 12126 if (need_reset) 12127 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12128 12129 return 0; 12130 } 12131 12132 /** 12133 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 12134 * @pf: board private structure 12135 * @port: The UDP port to look up 12136 * 12137 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 12138 **/ 12139 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 12140 { 12141 u8 i; 12142 12143 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 12144 /* Do not report ports with pending deletions as 12145 * being available. 12146 */ 12147 if (!port && (pf->pending_udp_bitmap & BIT_ULL(i))) 12148 continue; 12149 if (pf->udp_ports[i].port == port) 12150 return i; 12151 } 12152 12153 return i; 12154 } 12155 12156 /** 12157 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 12158 * @netdev: This physical port's netdev 12159 * @ti: Tunnel endpoint information 12160 **/ 12161 static void i40e_udp_tunnel_add(struct net_device *netdev, 12162 struct udp_tunnel_info *ti) 12163 { 12164 struct i40e_netdev_priv *np = netdev_priv(netdev); 12165 struct i40e_vsi *vsi = np->vsi; 12166 struct i40e_pf *pf = vsi->back; 12167 u16 port = ntohs(ti->port); 12168 u8 next_idx; 12169 u8 idx; 12170 12171 idx = i40e_get_udp_port_idx(pf, port); 12172 12173 /* Check if port already exists */ 12174 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 12175 netdev_info(netdev, "port %d already offloaded\n", port); 12176 return; 12177 } 12178 12179 /* Now check if there is space to add the new port */ 12180 next_idx = i40e_get_udp_port_idx(pf, 0); 12181 12182 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 12183 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 12184 port); 12185 return; 12186 } 12187 12188 switch (ti->type) { 12189 case UDP_TUNNEL_TYPE_VXLAN: 12190 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 12191 break; 12192 case UDP_TUNNEL_TYPE_GENEVE: 12193 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) 12194 return; 12195 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 12196 break; 12197 default: 12198 return; 12199 } 12200 12201 /* New port: add it and mark its index in the bitmap */ 12202 pf->udp_ports[next_idx].port = port; 12203 pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED; 12204 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 12205 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 12206 } 12207 12208 /** 12209 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 12210 * @netdev: This physical port's netdev 12211 * @ti: Tunnel endpoint information 12212 **/ 12213 static void i40e_udp_tunnel_del(struct net_device *netdev, 12214 struct udp_tunnel_info *ti) 12215 { 12216 struct i40e_netdev_priv *np = netdev_priv(netdev); 12217 struct i40e_vsi *vsi = np->vsi; 12218 struct i40e_pf *pf = vsi->back; 12219 u16 port = ntohs(ti->port); 12220 u8 idx; 12221 12222 idx = i40e_get_udp_port_idx(pf, port); 12223 12224 /* Check if port already exists */ 12225 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 12226 goto not_found; 12227 12228 switch (ti->type) { 12229 case UDP_TUNNEL_TYPE_VXLAN: 12230 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 12231 goto not_found; 12232 break; 12233 case UDP_TUNNEL_TYPE_GENEVE: 12234 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 12235 goto not_found; 12236 break; 12237 default: 12238 goto not_found; 12239 } 12240 12241 /* if port exists, set it to 0 (mark for deletion) 12242 * and make it pending 12243 */ 12244 pf->udp_ports[idx].port = 0; 12245 12246 /* Toggle pending bit instead of setting it. This way if we are 12247 * deleting a port that has yet to be added we just clear the pending 12248 * bit and don't have to worry about it. 12249 */ 12250 pf->pending_udp_bitmap ^= BIT_ULL(idx); 12251 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 12252 12253 return; 12254 not_found: 12255 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 12256 port); 12257 } 12258 12259 static int i40e_get_phys_port_id(struct net_device *netdev, 12260 struct netdev_phys_item_id *ppid) 12261 { 12262 struct i40e_netdev_priv *np = netdev_priv(netdev); 12263 struct i40e_pf *pf = np->vsi->back; 12264 struct i40e_hw *hw = &pf->hw; 12265 12266 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 12267 return -EOPNOTSUPP; 12268 12269 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 12270 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 12271 12272 return 0; 12273 } 12274 12275 /** 12276 * i40e_ndo_fdb_add - add an entry to the hardware database 12277 * @ndm: the input from the stack 12278 * @tb: pointer to array of nladdr (unused) 12279 * @dev: the net device pointer 12280 * @addr: the MAC address entry being added 12281 * @vid: VLAN ID 12282 * @flags: instructions from stack about fdb operation 12283 */ 12284 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 12285 struct net_device *dev, 12286 const unsigned char *addr, u16 vid, 12287 u16 flags, 12288 struct netlink_ext_ack *extack) 12289 { 12290 struct i40e_netdev_priv *np = netdev_priv(dev); 12291 struct i40e_pf *pf = np->vsi->back; 12292 int err = 0; 12293 12294 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 12295 return -EOPNOTSUPP; 12296 12297 if (vid) { 12298 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 12299 return -EINVAL; 12300 } 12301 12302 /* Hardware does not support aging addresses so if a 12303 * ndm_state is given only allow permanent addresses 12304 */ 12305 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 12306 netdev_info(dev, "FDB only supports static addresses\n"); 12307 return -EINVAL; 12308 } 12309 12310 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 12311 err = dev_uc_add_excl(dev, addr); 12312 else if (is_multicast_ether_addr(addr)) 12313 err = dev_mc_add_excl(dev, addr); 12314 else 12315 err = -EINVAL; 12316 12317 /* Only return duplicate errors if NLM_F_EXCL is set */ 12318 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 12319 err = 0; 12320 12321 return err; 12322 } 12323 12324 /** 12325 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 12326 * @dev: the netdev being configured 12327 * @nlh: RTNL message 12328 * @flags: bridge flags 12329 * @extack: netlink extended ack 12330 * 12331 * Inserts a new hardware bridge if not already created and 12332 * enables the bridging mode requested (VEB or VEPA). If the 12333 * hardware bridge has already been inserted and the request 12334 * is to change the mode then that requires a PF reset to 12335 * allow rebuild of the components with required hardware 12336 * bridge mode enabled. 12337 * 12338 * Note: expects to be called while under rtnl_lock() 12339 **/ 12340 static int i40e_ndo_bridge_setlink(struct net_device *dev, 12341 struct nlmsghdr *nlh, 12342 u16 flags, 12343 struct netlink_ext_ack *extack) 12344 { 12345 struct i40e_netdev_priv *np = netdev_priv(dev); 12346 struct i40e_vsi *vsi = np->vsi; 12347 struct i40e_pf *pf = vsi->back; 12348 struct i40e_veb *veb = NULL; 12349 struct nlattr *attr, *br_spec; 12350 int i, rem; 12351 12352 /* Only for PF VSI for now */ 12353 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 12354 return -EOPNOTSUPP; 12355 12356 /* Find the HW bridge for PF VSI */ 12357 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12358 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12359 veb = pf->veb[i]; 12360 } 12361 12362 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12363 12364 nla_for_each_nested(attr, br_spec, rem) { 12365 __u16 mode; 12366 12367 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12368 continue; 12369 12370 mode = nla_get_u16(attr); 12371 if ((mode != BRIDGE_MODE_VEPA) && 12372 (mode != BRIDGE_MODE_VEB)) 12373 return -EINVAL; 12374 12375 /* Insert a new HW bridge */ 12376 if (!veb) { 12377 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 12378 vsi->tc_config.enabled_tc); 12379 if (veb) { 12380 veb->bridge_mode = mode; 12381 i40e_config_bridge_mode(veb); 12382 } else { 12383 /* No Bridge HW offload available */ 12384 return -ENOENT; 12385 } 12386 break; 12387 } else if (mode != veb->bridge_mode) { 12388 /* Existing HW bridge but different mode needs reset */ 12389 veb->bridge_mode = mode; 12390 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 12391 if (mode == BRIDGE_MODE_VEB) 12392 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 12393 else 12394 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 12395 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12396 break; 12397 } 12398 } 12399 12400 return 0; 12401 } 12402 12403 /** 12404 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 12405 * @skb: skb buff 12406 * @pid: process id 12407 * @seq: RTNL message seq # 12408 * @dev: the netdev being configured 12409 * @filter_mask: unused 12410 * @nlflags: netlink flags passed in 12411 * 12412 * Return the mode in which the hardware bridge is operating in 12413 * i.e VEB or VEPA. 12414 **/ 12415 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12416 struct net_device *dev, 12417 u32 __always_unused filter_mask, 12418 int nlflags) 12419 { 12420 struct i40e_netdev_priv *np = netdev_priv(dev); 12421 struct i40e_vsi *vsi = np->vsi; 12422 struct i40e_pf *pf = vsi->back; 12423 struct i40e_veb *veb = NULL; 12424 int i; 12425 12426 /* Only for PF VSI for now */ 12427 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 12428 return -EOPNOTSUPP; 12429 12430 /* Find the HW bridge for the PF VSI */ 12431 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12432 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12433 veb = pf->veb[i]; 12434 } 12435 12436 if (!veb) 12437 return 0; 12438 12439 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 12440 0, 0, nlflags, filter_mask, NULL); 12441 } 12442 12443 /** 12444 * i40e_features_check - Validate encapsulated packet conforms to limits 12445 * @skb: skb buff 12446 * @dev: This physical port's netdev 12447 * @features: Offload features that the stack believes apply 12448 **/ 12449 static netdev_features_t i40e_features_check(struct sk_buff *skb, 12450 struct net_device *dev, 12451 netdev_features_t features) 12452 { 12453 size_t len; 12454 12455 /* No point in doing any of this if neither checksum nor GSO are 12456 * being requested for this frame. We can rule out both by just 12457 * checking for CHECKSUM_PARTIAL 12458 */ 12459 if (skb->ip_summed != CHECKSUM_PARTIAL) 12460 return features; 12461 12462 /* We cannot support GSO if the MSS is going to be less than 12463 * 64 bytes. If it is then we need to drop support for GSO. 12464 */ 12465 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 12466 features &= ~NETIF_F_GSO_MASK; 12467 12468 /* MACLEN can support at most 63 words */ 12469 len = skb_network_header(skb) - skb->data; 12470 if (len & ~(63 * 2)) 12471 goto out_err; 12472 12473 /* IPLEN and EIPLEN can support at most 127 dwords */ 12474 len = skb_transport_header(skb) - skb_network_header(skb); 12475 if (len & ~(127 * 4)) 12476 goto out_err; 12477 12478 if (skb->encapsulation) { 12479 /* L4TUNLEN can support 127 words */ 12480 len = skb_inner_network_header(skb) - skb_transport_header(skb); 12481 if (len & ~(127 * 2)) 12482 goto out_err; 12483 12484 /* IPLEN can support at most 127 dwords */ 12485 len = skb_inner_transport_header(skb) - 12486 skb_inner_network_header(skb); 12487 if (len & ~(127 * 4)) 12488 goto out_err; 12489 } 12490 12491 /* No need to validate L4LEN as TCP is the only protocol with a 12492 * a flexible value and we support all possible values supported 12493 * by TCP, which is at most 15 dwords 12494 */ 12495 12496 return features; 12497 out_err: 12498 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 12499 } 12500 12501 /** 12502 * i40e_xdp_setup - add/remove an XDP program 12503 * @vsi: VSI to changed 12504 * @prog: XDP program 12505 **/ 12506 static int i40e_xdp_setup(struct i40e_vsi *vsi, 12507 struct bpf_prog *prog) 12508 { 12509 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 12510 struct i40e_pf *pf = vsi->back; 12511 struct bpf_prog *old_prog; 12512 bool need_reset; 12513 int i; 12514 12515 /* Don't allow frames that span over multiple buffers */ 12516 if (frame_size > vsi->rx_buf_len) 12517 return -EINVAL; 12518 12519 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 12520 return 0; 12521 12522 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 12523 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 12524 12525 if (need_reset) 12526 i40e_prep_for_reset(pf, true); 12527 12528 old_prog = xchg(&vsi->xdp_prog, prog); 12529 12530 if (need_reset) { 12531 if (!prog) 12532 /* Wait until ndo_xsk_wakeup completes. */ 12533 synchronize_rcu(); 12534 i40e_reset_and_rebuild(pf, true, true); 12535 } 12536 12537 for (i = 0; i < vsi->num_queue_pairs; i++) 12538 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 12539 12540 if (old_prog) 12541 bpf_prog_put(old_prog); 12542 12543 /* Kick start the NAPI context if there is an AF_XDP socket open 12544 * on that queue id. This so that receiving will start. 12545 */ 12546 if (need_reset && prog) 12547 for (i = 0; i < vsi->num_queue_pairs; i++) 12548 if (vsi->xdp_rings[i]->xsk_umem) 12549 (void)i40e_xsk_wakeup(vsi->netdev, i, 12550 XDP_WAKEUP_RX); 12551 12552 return 0; 12553 } 12554 12555 /** 12556 * i40e_enter_busy_conf - Enters busy config state 12557 * @vsi: vsi 12558 * 12559 * Returns 0 on success, <0 for failure. 12560 **/ 12561 static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 12562 { 12563 struct i40e_pf *pf = vsi->back; 12564 int timeout = 50; 12565 12566 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 12567 timeout--; 12568 if (!timeout) 12569 return -EBUSY; 12570 usleep_range(1000, 2000); 12571 } 12572 12573 return 0; 12574 } 12575 12576 /** 12577 * i40e_exit_busy_conf - Exits busy config state 12578 * @vsi: vsi 12579 **/ 12580 static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 12581 { 12582 struct i40e_pf *pf = vsi->back; 12583 12584 clear_bit(__I40E_CONFIG_BUSY, pf->state); 12585 } 12586 12587 /** 12588 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 12589 * @vsi: vsi 12590 * @queue_pair: queue pair 12591 **/ 12592 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 12593 { 12594 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 12595 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 12596 memset(&vsi->tx_rings[queue_pair]->stats, 0, 12597 sizeof(vsi->tx_rings[queue_pair]->stats)); 12598 if (i40e_enabled_xdp_vsi(vsi)) { 12599 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 12600 sizeof(vsi->xdp_rings[queue_pair]->stats)); 12601 } 12602 } 12603 12604 /** 12605 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 12606 * @vsi: vsi 12607 * @queue_pair: queue pair 12608 **/ 12609 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 12610 { 12611 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 12612 if (i40e_enabled_xdp_vsi(vsi)) { 12613 /* Make sure that in-progress ndo_xdp_xmit calls are 12614 * completed. 12615 */ 12616 synchronize_rcu(); 12617 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 12618 } 12619 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 12620 } 12621 12622 /** 12623 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 12624 * @vsi: vsi 12625 * @queue_pair: queue pair 12626 * @enable: true for enable, false for disable 12627 **/ 12628 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 12629 bool enable) 12630 { 12631 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12632 struct i40e_q_vector *q_vector = rxr->q_vector; 12633 12634 if (!vsi->netdev) 12635 return; 12636 12637 /* All rings in a qp belong to the same qvector. */ 12638 if (q_vector->rx.ring || q_vector->tx.ring) { 12639 if (enable) 12640 napi_enable(&q_vector->napi); 12641 else 12642 napi_disable(&q_vector->napi); 12643 } 12644 } 12645 12646 /** 12647 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 12648 * @vsi: vsi 12649 * @queue_pair: queue pair 12650 * @enable: true for enable, false for disable 12651 * 12652 * Returns 0 on success, <0 on failure. 12653 **/ 12654 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 12655 bool enable) 12656 { 12657 struct i40e_pf *pf = vsi->back; 12658 int pf_q, ret = 0; 12659 12660 pf_q = vsi->base_queue + queue_pair; 12661 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 12662 false /*is xdp*/, enable); 12663 if (ret) { 12664 dev_info(&pf->pdev->dev, 12665 "VSI seid %d Tx ring %d %sable timeout\n", 12666 vsi->seid, pf_q, (enable ? "en" : "dis")); 12667 return ret; 12668 } 12669 12670 i40e_control_rx_q(pf, pf_q, enable); 12671 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 12672 if (ret) { 12673 dev_info(&pf->pdev->dev, 12674 "VSI seid %d Rx ring %d %sable timeout\n", 12675 vsi->seid, pf_q, (enable ? "en" : "dis")); 12676 return ret; 12677 } 12678 12679 /* Due to HW errata, on Rx disable only, the register can 12680 * indicate done before it really is. Needs 50ms to be sure 12681 */ 12682 if (!enable) 12683 mdelay(50); 12684 12685 if (!i40e_enabled_xdp_vsi(vsi)) 12686 return ret; 12687 12688 ret = i40e_control_wait_tx_q(vsi->seid, pf, 12689 pf_q + vsi->alloc_queue_pairs, 12690 true /*is xdp*/, enable); 12691 if (ret) { 12692 dev_info(&pf->pdev->dev, 12693 "VSI seid %d XDP Tx ring %d %sable timeout\n", 12694 vsi->seid, pf_q, (enable ? "en" : "dis")); 12695 } 12696 12697 return ret; 12698 } 12699 12700 /** 12701 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 12702 * @vsi: vsi 12703 * @queue_pair: queue_pair 12704 **/ 12705 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 12706 { 12707 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12708 struct i40e_pf *pf = vsi->back; 12709 struct i40e_hw *hw = &pf->hw; 12710 12711 /* All rings in a qp belong to the same qvector. */ 12712 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 12713 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 12714 else 12715 i40e_irq_dynamic_enable_icr0(pf); 12716 12717 i40e_flush(hw); 12718 } 12719 12720 /** 12721 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 12722 * @vsi: vsi 12723 * @queue_pair: queue_pair 12724 **/ 12725 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 12726 { 12727 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12728 struct i40e_pf *pf = vsi->back; 12729 struct i40e_hw *hw = &pf->hw; 12730 12731 /* For simplicity, instead of removing the qp interrupt causes 12732 * from the interrupt linked list, we simply disable the interrupt, and 12733 * leave the list intact. 12734 * 12735 * All rings in a qp belong to the same qvector. 12736 */ 12737 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 12738 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 12739 12740 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 12741 i40e_flush(hw); 12742 synchronize_irq(pf->msix_entries[intpf].vector); 12743 } else { 12744 /* Legacy and MSI mode - this stops all interrupt handling */ 12745 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 12746 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 12747 i40e_flush(hw); 12748 synchronize_irq(pf->pdev->irq); 12749 } 12750 } 12751 12752 /** 12753 * i40e_queue_pair_disable - Disables a queue pair 12754 * @vsi: vsi 12755 * @queue_pair: queue pair 12756 * 12757 * Returns 0 on success, <0 on failure. 12758 **/ 12759 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 12760 { 12761 int err; 12762 12763 err = i40e_enter_busy_conf(vsi); 12764 if (err) 12765 return err; 12766 12767 i40e_queue_pair_disable_irq(vsi, queue_pair); 12768 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 12769 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 12770 i40e_queue_pair_clean_rings(vsi, queue_pair); 12771 i40e_queue_pair_reset_stats(vsi, queue_pair); 12772 12773 return err; 12774 } 12775 12776 /** 12777 * i40e_queue_pair_enable - Enables a queue pair 12778 * @vsi: vsi 12779 * @queue_pair: queue pair 12780 * 12781 * Returns 0 on success, <0 on failure. 12782 **/ 12783 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 12784 { 12785 int err; 12786 12787 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 12788 if (err) 12789 return err; 12790 12791 if (i40e_enabled_xdp_vsi(vsi)) { 12792 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 12793 if (err) 12794 return err; 12795 } 12796 12797 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 12798 if (err) 12799 return err; 12800 12801 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 12802 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 12803 i40e_queue_pair_enable_irq(vsi, queue_pair); 12804 12805 i40e_exit_busy_conf(vsi); 12806 12807 return err; 12808 } 12809 12810 /** 12811 * i40e_xdp - implements ndo_bpf for i40e 12812 * @dev: netdevice 12813 * @xdp: XDP command 12814 **/ 12815 static int i40e_xdp(struct net_device *dev, 12816 struct netdev_bpf *xdp) 12817 { 12818 struct i40e_netdev_priv *np = netdev_priv(dev); 12819 struct i40e_vsi *vsi = np->vsi; 12820 12821 if (vsi->type != I40E_VSI_MAIN) 12822 return -EINVAL; 12823 12824 switch (xdp->command) { 12825 case XDP_SETUP_PROG: 12826 return i40e_xdp_setup(vsi, xdp->prog); 12827 case XDP_QUERY_PROG: 12828 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0; 12829 return 0; 12830 case XDP_SETUP_XSK_UMEM: 12831 return i40e_xsk_umem_setup(vsi, xdp->xsk.umem, 12832 xdp->xsk.queue_id); 12833 default: 12834 return -EINVAL; 12835 } 12836 } 12837 12838 static const struct net_device_ops i40e_netdev_ops = { 12839 .ndo_open = i40e_open, 12840 .ndo_stop = i40e_close, 12841 .ndo_start_xmit = i40e_lan_xmit_frame, 12842 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 12843 .ndo_set_rx_mode = i40e_set_rx_mode, 12844 .ndo_validate_addr = eth_validate_addr, 12845 .ndo_set_mac_address = i40e_set_mac, 12846 .ndo_change_mtu = i40e_change_mtu, 12847 .ndo_do_ioctl = i40e_ioctl, 12848 .ndo_tx_timeout = i40e_tx_timeout, 12849 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 12850 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 12851 #ifdef CONFIG_NET_POLL_CONTROLLER 12852 .ndo_poll_controller = i40e_netpoll, 12853 #endif 12854 .ndo_setup_tc = __i40e_setup_tc, 12855 .ndo_set_features = i40e_set_features, 12856 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 12857 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 12858 .ndo_get_vf_stats = i40e_get_vf_stats, 12859 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 12860 .ndo_get_vf_config = i40e_ndo_get_vf_config, 12861 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 12862 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 12863 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 12864 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 12865 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 12866 .ndo_get_phys_port_id = i40e_get_phys_port_id, 12867 .ndo_fdb_add = i40e_ndo_fdb_add, 12868 .ndo_features_check = i40e_features_check, 12869 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 12870 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 12871 .ndo_bpf = i40e_xdp, 12872 .ndo_xdp_xmit = i40e_xdp_xmit, 12873 .ndo_xsk_wakeup = i40e_xsk_wakeup, 12874 .ndo_dfwd_add_station = i40e_fwd_add, 12875 .ndo_dfwd_del_station = i40e_fwd_del, 12876 }; 12877 12878 /** 12879 * i40e_config_netdev - Setup the netdev flags 12880 * @vsi: the VSI being configured 12881 * 12882 * Returns 0 on success, negative value on failure 12883 **/ 12884 static int i40e_config_netdev(struct i40e_vsi *vsi) 12885 { 12886 struct i40e_pf *pf = vsi->back; 12887 struct i40e_hw *hw = &pf->hw; 12888 struct i40e_netdev_priv *np; 12889 struct net_device *netdev; 12890 u8 broadcast[ETH_ALEN]; 12891 u8 mac_addr[ETH_ALEN]; 12892 int etherdev_size; 12893 netdev_features_t hw_enc_features; 12894 netdev_features_t hw_features; 12895 12896 etherdev_size = sizeof(struct i40e_netdev_priv); 12897 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 12898 if (!netdev) 12899 return -ENOMEM; 12900 12901 vsi->netdev = netdev; 12902 np = netdev_priv(netdev); 12903 np->vsi = vsi; 12904 12905 hw_enc_features = NETIF_F_SG | 12906 NETIF_F_IP_CSUM | 12907 NETIF_F_IPV6_CSUM | 12908 NETIF_F_HIGHDMA | 12909 NETIF_F_SOFT_FEATURES | 12910 NETIF_F_TSO | 12911 NETIF_F_TSO_ECN | 12912 NETIF_F_TSO6 | 12913 NETIF_F_GSO_GRE | 12914 NETIF_F_GSO_GRE_CSUM | 12915 NETIF_F_GSO_PARTIAL | 12916 NETIF_F_GSO_IPXIP4 | 12917 NETIF_F_GSO_IPXIP6 | 12918 NETIF_F_GSO_UDP_TUNNEL | 12919 NETIF_F_GSO_UDP_TUNNEL_CSUM | 12920 NETIF_F_GSO_UDP_L4 | 12921 NETIF_F_SCTP_CRC | 12922 NETIF_F_RXHASH | 12923 NETIF_F_RXCSUM | 12924 0; 12925 12926 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 12927 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 12928 12929 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 12930 12931 netdev->hw_enc_features |= hw_enc_features; 12932 12933 /* record features VLANs can make use of */ 12934 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 12935 12936 /* enable macvlan offloads */ 12937 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD; 12938 12939 hw_features = hw_enc_features | 12940 NETIF_F_HW_VLAN_CTAG_TX | 12941 NETIF_F_HW_VLAN_CTAG_RX; 12942 12943 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 12944 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 12945 12946 netdev->hw_features |= hw_features; 12947 12948 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 12949 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 12950 12951 if (vsi->type == I40E_VSI_MAIN) { 12952 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 12953 ether_addr_copy(mac_addr, hw->mac.perm_addr); 12954 /* The following steps are necessary for two reasons. First, 12955 * some older NVM configurations load a default MAC-VLAN 12956 * filter that will accept any tagged packet, and we want to 12957 * replace this with a normal filter. Additionally, it is 12958 * possible our MAC address was provided by the platform using 12959 * Open Firmware or similar. 12960 * 12961 * Thus, we need to remove the default filter and install one 12962 * specific to the MAC address. 12963 */ 12964 i40e_rm_default_mac_filter(vsi, mac_addr); 12965 spin_lock_bh(&vsi->mac_filter_hash_lock); 12966 i40e_add_mac_filter(vsi, mac_addr); 12967 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12968 } else { 12969 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 12970 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 12971 * the end, which is 4 bytes long, so force truncation of the 12972 * original name by IFNAMSIZ - 4 12973 */ 12974 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 12975 IFNAMSIZ - 4, 12976 pf->vsi[pf->lan_vsi]->netdev->name); 12977 eth_random_addr(mac_addr); 12978 12979 spin_lock_bh(&vsi->mac_filter_hash_lock); 12980 i40e_add_mac_filter(vsi, mac_addr); 12981 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12982 } 12983 12984 /* Add the broadcast filter so that we initially will receive 12985 * broadcast packets. Note that when a new VLAN is first added the 12986 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 12987 * specific filters as part of transitioning into "vlan" operation. 12988 * When more VLANs are added, the driver will copy each existing MAC 12989 * filter and add it for the new VLAN. 12990 * 12991 * Broadcast filters are handled specially by 12992 * i40e_sync_filters_subtask, as the driver must to set the broadcast 12993 * promiscuous bit instead of adding this directly as a MAC/VLAN 12994 * filter. The subtask will update the correct broadcast promiscuous 12995 * bits as VLANs become active or inactive. 12996 */ 12997 eth_broadcast_addr(broadcast); 12998 spin_lock_bh(&vsi->mac_filter_hash_lock); 12999 i40e_add_mac_filter(vsi, broadcast); 13000 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13001 13002 ether_addr_copy(netdev->dev_addr, mac_addr); 13003 ether_addr_copy(netdev->perm_addr, mac_addr); 13004 13005 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */ 13006 netdev->neigh_priv_len = sizeof(u32) * 4; 13007 13008 netdev->priv_flags |= IFF_UNICAST_FLT; 13009 netdev->priv_flags |= IFF_SUPP_NOFCS; 13010 /* Setup netdev TC information */ 13011 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 13012 13013 netdev->netdev_ops = &i40e_netdev_ops; 13014 netdev->watchdog_timeo = 5 * HZ; 13015 i40e_set_ethtool_ops(netdev); 13016 13017 /* MTU range: 68 - 9706 */ 13018 netdev->min_mtu = ETH_MIN_MTU; 13019 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 13020 13021 return 0; 13022 } 13023 13024 /** 13025 * i40e_vsi_delete - Delete a VSI from the switch 13026 * @vsi: the VSI being removed 13027 * 13028 * Returns 0 on success, negative value on failure 13029 **/ 13030 static void i40e_vsi_delete(struct i40e_vsi *vsi) 13031 { 13032 /* remove default VSI is not allowed */ 13033 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 13034 return; 13035 13036 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 13037 } 13038 13039 /** 13040 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 13041 * @vsi: the VSI being queried 13042 * 13043 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 13044 **/ 13045 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 13046 { 13047 struct i40e_veb *veb; 13048 struct i40e_pf *pf = vsi->back; 13049 13050 /* Uplink is not a bridge so default to VEB */ 13051 if (vsi->veb_idx >= I40E_MAX_VEB) 13052 return 1; 13053 13054 veb = pf->veb[vsi->veb_idx]; 13055 if (!veb) { 13056 dev_info(&pf->pdev->dev, 13057 "There is no veb associated with the bridge\n"); 13058 return -ENOENT; 13059 } 13060 13061 /* Uplink is a bridge in VEPA mode */ 13062 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 13063 return 0; 13064 } else { 13065 /* Uplink is a bridge in VEB mode */ 13066 return 1; 13067 } 13068 13069 /* VEPA is now default bridge, so return 0 */ 13070 return 0; 13071 } 13072 13073 /** 13074 * i40e_add_vsi - Add a VSI to the switch 13075 * @vsi: the VSI being configured 13076 * 13077 * This initializes a VSI context depending on the VSI type to be added and 13078 * passes it down to the add_vsi aq command. 13079 **/ 13080 static int i40e_add_vsi(struct i40e_vsi *vsi) 13081 { 13082 int ret = -ENODEV; 13083 struct i40e_pf *pf = vsi->back; 13084 struct i40e_hw *hw = &pf->hw; 13085 struct i40e_vsi_context ctxt; 13086 struct i40e_mac_filter *f; 13087 struct hlist_node *h; 13088 int bkt; 13089 13090 u8 enabled_tc = 0x1; /* TC0 enabled */ 13091 int f_count = 0; 13092 13093 memset(&ctxt, 0, sizeof(ctxt)); 13094 switch (vsi->type) { 13095 case I40E_VSI_MAIN: 13096 /* The PF's main VSI is already setup as part of the 13097 * device initialization, so we'll not bother with 13098 * the add_vsi call, but we will retrieve the current 13099 * VSI context. 13100 */ 13101 ctxt.seid = pf->main_vsi_seid; 13102 ctxt.pf_num = pf->hw.pf_id; 13103 ctxt.vf_num = 0; 13104 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 13105 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13106 if (ret) { 13107 dev_info(&pf->pdev->dev, 13108 "couldn't get PF vsi config, err %s aq_err %s\n", 13109 i40e_stat_str(&pf->hw, ret), 13110 i40e_aq_str(&pf->hw, 13111 pf->hw.aq.asq_last_status)); 13112 return -ENOENT; 13113 } 13114 vsi->info = ctxt.info; 13115 vsi->info.valid_sections = 0; 13116 13117 vsi->seid = ctxt.seid; 13118 vsi->id = ctxt.vsi_number; 13119 13120 enabled_tc = i40e_pf_get_tc_map(pf); 13121 13122 /* Source pruning is enabled by default, so the flag is 13123 * negative logic - if it's set, we need to fiddle with 13124 * the VSI to disable source pruning. 13125 */ 13126 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 13127 memset(&ctxt, 0, sizeof(ctxt)); 13128 ctxt.seid = pf->main_vsi_seid; 13129 ctxt.pf_num = pf->hw.pf_id; 13130 ctxt.vf_num = 0; 13131 ctxt.info.valid_sections |= 13132 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13133 ctxt.info.switch_id = 13134 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 13135 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13136 if (ret) { 13137 dev_info(&pf->pdev->dev, 13138 "update vsi failed, err %s aq_err %s\n", 13139 i40e_stat_str(&pf->hw, ret), 13140 i40e_aq_str(&pf->hw, 13141 pf->hw.aq.asq_last_status)); 13142 ret = -ENOENT; 13143 goto err; 13144 } 13145 } 13146 13147 /* MFP mode setup queue map and update VSI */ 13148 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 13149 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 13150 memset(&ctxt, 0, sizeof(ctxt)); 13151 ctxt.seid = pf->main_vsi_seid; 13152 ctxt.pf_num = pf->hw.pf_id; 13153 ctxt.vf_num = 0; 13154 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 13155 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13156 if (ret) { 13157 dev_info(&pf->pdev->dev, 13158 "update vsi failed, err %s aq_err %s\n", 13159 i40e_stat_str(&pf->hw, ret), 13160 i40e_aq_str(&pf->hw, 13161 pf->hw.aq.asq_last_status)); 13162 ret = -ENOENT; 13163 goto err; 13164 } 13165 /* update the local VSI info queue map */ 13166 i40e_vsi_update_queue_map(vsi, &ctxt); 13167 vsi->info.valid_sections = 0; 13168 } else { 13169 /* Default/Main VSI is only enabled for TC0 13170 * reconfigure it to enable all TCs that are 13171 * available on the port in SFP mode. 13172 * For MFP case the iSCSI PF would use this 13173 * flow to enable LAN+iSCSI TC. 13174 */ 13175 ret = i40e_vsi_config_tc(vsi, enabled_tc); 13176 if (ret) { 13177 /* Single TC condition is not fatal, 13178 * message and continue 13179 */ 13180 dev_info(&pf->pdev->dev, 13181 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 13182 enabled_tc, 13183 i40e_stat_str(&pf->hw, ret), 13184 i40e_aq_str(&pf->hw, 13185 pf->hw.aq.asq_last_status)); 13186 } 13187 } 13188 break; 13189 13190 case I40E_VSI_FDIR: 13191 ctxt.pf_num = hw->pf_id; 13192 ctxt.vf_num = 0; 13193 ctxt.uplink_seid = vsi->uplink_seid; 13194 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13195 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13196 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 13197 (i40e_is_vsi_uplink_mode_veb(vsi))) { 13198 ctxt.info.valid_sections |= 13199 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13200 ctxt.info.switch_id = 13201 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13202 } 13203 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13204 break; 13205 13206 case I40E_VSI_VMDQ2: 13207 ctxt.pf_num = hw->pf_id; 13208 ctxt.vf_num = 0; 13209 ctxt.uplink_seid = vsi->uplink_seid; 13210 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13211 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 13212 13213 /* This VSI is connected to VEB so the switch_id 13214 * should be set to zero by default. 13215 */ 13216 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13217 ctxt.info.valid_sections |= 13218 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13219 ctxt.info.switch_id = 13220 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13221 } 13222 13223 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13224 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13225 break; 13226 13227 case I40E_VSI_SRIOV: 13228 ctxt.pf_num = hw->pf_id; 13229 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 13230 ctxt.uplink_seid = vsi->uplink_seid; 13231 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13232 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 13233 13234 /* This VSI is connected to VEB so the switch_id 13235 * should be set to zero by default. 13236 */ 13237 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13238 ctxt.info.valid_sections |= 13239 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13240 ctxt.info.switch_id = 13241 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13242 } 13243 13244 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 13245 ctxt.info.valid_sections |= 13246 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 13247 ctxt.info.queueing_opt_flags |= 13248 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 13249 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 13250 } 13251 13252 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 13253 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 13254 if (pf->vf[vsi->vf_id].spoofchk) { 13255 ctxt.info.valid_sections |= 13256 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 13257 ctxt.info.sec_flags |= 13258 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 13259 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 13260 } 13261 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13262 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13263 break; 13264 13265 case I40E_VSI_IWARP: 13266 /* send down message to iWARP */ 13267 break; 13268 13269 default: 13270 return -ENODEV; 13271 } 13272 13273 if (vsi->type != I40E_VSI_MAIN) { 13274 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 13275 if (ret) { 13276 dev_info(&vsi->back->pdev->dev, 13277 "add vsi failed, err %s aq_err %s\n", 13278 i40e_stat_str(&pf->hw, ret), 13279 i40e_aq_str(&pf->hw, 13280 pf->hw.aq.asq_last_status)); 13281 ret = -ENOENT; 13282 goto err; 13283 } 13284 vsi->info = ctxt.info; 13285 vsi->info.valid_sections = 0; 13286 vsi->seid = ctxt.seid; 13287 vsi->id = ctxt.vsi_number; 13288 } 13289 13290 vsi->active_filters = 0; 13291 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 13292 spin_lock_bh(&vsi->mac_filter_hash_lock); 13293 /* If macvlan filters already exist, force them to get loaded */ 13294 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 13295 f->state = I40E_FILTER_NEW; 13296 f_count++; 13297 } 13298 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13299 13300 if (f_count) { 13301 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 13302 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 13303 } 13304 13305 /* Update VSI BW information */ 13306 ret = i40e_vsi_get_bw_info(vsi); 13307 if (ret) { 13308 dev_info(&pf->pdev->dev, 13309 "couldn't get vsi bw info, err %s aq_err %s\n", 13310 i40e_stat_str(&pf->hw, ret), 13311 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13312 /* VSI is already added so not tearing that up */ 13313 ret = 0; 13314 } 13315 13316 err: 13317 return ret; 13318 } 13319 13320 /** 13321 * i40e_vsi_release - Delete a VSI and free its resources 13322 * @vsi: the VSI being removed 13323 * 13324 * Returns 0 on success or < 0 on error 13325 **/ 13326 int i40e_vsi_release(struct i40e_vsi *vsi) 13327 { 13328 struct i40e_mac_filter *f; 13329 struct hlist_node *h; 13330 struct i40e_veb *veb = NULL; 13331 struct i40e_pf *pf; 13332 u16 uplink_seid; 13333 int i, n, bkt; 13334 13335 pf = vsi->back; 13336 13337 /* release of a VEB-owner or last VSI is not allowed */ 13338 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 13339 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 13340 vsi->seid, vsi->uplink_seid); 13341 return -ENODEV; 13342 } 13343 if (vsi == pf->vsi[pf->lan_vsi] && 13344 !test_bit(__I40E_DOWN, pf->state)) { 13345 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 13346 return -ENODEV; 13347 } 13348 13349 uplink_seid = vsi->uplink_seid; 13350 if (vsi->type != I40E_VSI_SRIOV) { 13351 if (vsi->netdev_registered) { 13352 vsi->netdev_registered = false; 13353 if (vsi->netdev) { 13354 /* results in a call to i40e_close() */ 13355 unregister_netdev(vsi->netdev); 13356 } 13357 } else { 13358 i40e_vsi_close(vsi); 13359 } 13360 i40e_vsi_disable_irq(vsi); 13361 } 13362 13363 spin_lock_bh(&vsi->mac_filter_hash_lock); 13364 13365 /* clear the sync flag on all filters */ 13366 if (vsi->netdev) { 13367 __dev_uc_unsync(vsi->netdev, NULL); 13368 __dev_mc_unsync(vsi->netdev, NULL); 13369 } 13370 13371 /* make sure any remaining filters are marked for deletion */ 13372 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 13373 __i40e_del_filter(vsi, f); 13374 13375 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13376 13377 i40e_sync_vsi_filters(vsi); 13378 13379 i40e_vsi_delete(vsi); 13380 i40e_vsi_free_q_vectors(vsi); 13381 if (vsi->netdev) { 13382 free_netdev(vsi->netdev); 13383 vsi->netdev = NULL; 13384 } 13385 i40e_vsi_clear_rings(vsi); 13386 i40e_vsi_clear(vsi); 13387 13388 /* If this was the last thing on the VEB, except for the 13389 * controlling VSI, remove the VEB, which puts the controlling 13390 * VSI onto the next level down in the switch. 13391 * 13392 * Well, okay, there's one more exception here: don't remove 13393 * the orphan VEBs yet. We'll wait for an explicit remove request 13394 * from up the network stack. 13395 */ 13396 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 13397 if (pf->vsi[i] && 13398 pf->vsi[i]->uplink_seid == uplink_seid && 13399 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13400 n++; /* count the VSIs */ 13401 } 13402 } 13403 for (i = 0; i < I40E_MAX_VEB; i++) { 13404 if (!pf->veb[i]) 13405 continue; 13406 if (pf->veb[i]->uplink_seid == uplink_seid) 13407 n++; /* count the VEBs */ 13408 if (pf->veb[i]->seid == uplink_seid) 13409 veb = pf->veb[i]; 13410 } 13411 if (n == 0 && veb && veb->uplink_seid != 0) 13412 i40e_veb_release(veb); 13413 13414 return 0; 13415 } 13416 13417 /** 13418 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 13419 * @vsi: ptr to the VSI 13420 * 13421 * This should only be called after i40e_vsi_mem_alloc() which allocates the 13422 * corresponding SW VSI structure and initializes num_queue_pairs for the 13423 * newly allocated VSI. 13424 * 13425 * Returns 0 on success or negative on failure 13426 **/ 13427 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 13428 { 13429 int ret = -ENOENT; 13430 struct i40e_pf *pf = vsi->back; 13431 13432 if (vsi->q_vectors[0]) { 13433 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 13434 vsi->seid); 13435 return -EEXIST; 13436 } 13437 13438 if (vsi->base_vector) { 13439 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 13440 vsi->seid, vsi->base_vector); 13441 return -EEXIST; 13442 } 13443 13444 ret = i40e_vsi_alloc_q_vectors(vsi); 13445 if (ret) { 13446 dev_info(&pf->pdev->dev, 13447 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 13448 vsi->num_q_vectors, vsi->seid, ret); 13449 vsi->num_q_vectors = 0; 13450 goto vector_setup_out; 13451 } 13452 13453 /* In Legacy mode, we do not have to get any other vector since we 13454 * piggyback on the misc/ICR0 for queue interrupts. 13455 */ 13456 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 13457 return ret; 13458 if (vsi->num_q_vectors) 13459 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 13460 vsi->num_q_vectors, vsi->idx); 13461 if (vsi->base_vector < 0) { 13462 dev_info(&pf->pdev->dev, 13463 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 13464 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 13465 i40e_vsi_free_q_vectors(vsi); 13466 ret = -ENOENT; 13467 goto vector_setup_out; 13468 } 13469 13470 vector_setup_out: 13471 return ret; 13472 } 13473 13474 /** 13475 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 13476 * @vsi: pointer to the vsi. 13477 * 13478 * This re-allocates a vsi's queue resources. 13479 * 13480 * Returns pointer to the successfully allocated and configured VSI sw struct 13481 * on success, otherwise returns NULL on failure. 13482 **/ 13483 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 13484 { 13485 u16 alloc_queue_pairs; 13486 struct i40e_pf *pf; 13487 u8 enabled_tc; 13488 int ret; 13489 13490 if (!vsi) 13491 return NULL; 13492 13493 pf = vsi->back; 13494 13495 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 13496 i40e_vsi_clear_rings(vsi); 13497 13498 i40e_vsi_free_arrays(vsi, false); 13499 i40e_set_num_rings_in_vsi(vsi); 13500 ret = i40e_vsi_alloc_arrays(vsi, false); 13501 if (ret) 13502 goto err_vsi; 13503 13504 alloc_queue_pairs = vsi->alloc_queue_pairs * 13505 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 13506 13507 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 13508 if (ret < 0) { 13509 dev_info(&pf->pdev->dev, 13510 "failed to get tracking for %d queues for VSI %d err %d\n", 13511 alloc_queue_pairs, vsi->seid, ret); 13512 goto err_vsi; 13513 } 13514 vsi->base_queue = ret; 13515 13516 /* Update the FW view of the VSI. Force a reset of TC and queue 13517 * layout configurations. 13518 */ 13519 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 13520 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 13521 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 13522 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 13523 if (vsi->type == I40E_VSI_MAIN) 13524 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 13525 13526 /* assign it some queues */ 13527 ret = i40e_alloc_rings(vsi); 13528 if (ret) 13529 goto err_rings; 13530 13531 /* map all of the rings to the q_vectors */ 13532 i40e_vsi_map_rings_to_vectors(vsi); 13533 return vsi; 13534 13535 err_rings: 13536 i40e_vsi_free_q_vectors(vsi); 13537 if (vsi->netdev_registered) { 13538 vsi->netdev_registered = false; 13539 unregister_netdev(vsi->netdev); 13540 free_netdev(vsi->netdev); 13541 vsi->netdev = NULL; 13542 } 13543 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13544 err_vsi: 13545 i40e_vsi_clear(vsi); 13546 return NULL; 13547 } 13548 13549 /** 13550 * i40e_vsi_setup - Set up a VSI by a given type 13551 * @pf: board private structure 13552 * @type: VSI type 13553 * @uplink_seid: the switch element to link to 13554 * @param1: usage depends upon VSI type. For VF types, indicates VF id 13555 * 13556 * This allocates the sw VSI structure and its queue resources, then add a VSI 13557 * to the identified VEB. 13558 * 13559 * Returns pointer to the successfully allocated and configure VSI sw struct on 13560 * success, otherwise returns NULL on failure. 13561 **/ 13562 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 13563 u16 uplink_seid, u32 param1) 13564 { 13565 struct i40e_vsi *vsi = NULL; 13566 struct i40e_veb *veb = NULL; 13567 u16 alloc_queue_pairs; 13568 int ret, i; 13569 int v_idx; 13570 13571 /* The requested uplink_seid must be either 13572 * - the PF's port seid 13573 * no VEB is needed because this is the PF 13574 * or this is a Flow Director special case VSI 13575 * - seid of an existing VEB 13576 * - seid of a VSI that owns an existing VEB 13577 * - seid of a VSI that doesn't own a VEB 13578 * a new VEB is created and the VSI becomes the owner 13579 * - seid of the PF VSI, which is what creates the first VEB 13580 * this is a special case of the previous 13581 * 13582 * Find which uplink_seid we were given and create a new VEB if needed 13583 */ 13584 for (i = 0; i < I40E_MAX_VEB; i++) { 13585 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 13586 veb = pf->veb[i]; 13587 break; 13588 } 13589 } 13590 13591 if (!veb && uplink_seid != pf->mac_seid) { 13592 13593 for (i = 0; i < pf->num_alloc_vsi; i++) { 13594 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 13595 vsi = pf->vsi[i]; 13596 break; 13597 } 13598 } 13599 if (!vsi) { 13600 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 13601 uplink_seid); 13602 return NULL; 13603 } 13604 13605 if (vsi->uplink_seid == pf->mac_seid) 13606 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 13607 vsi->tc_config.enabled_tc); 13608 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 13609 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 13610 vsi->tc_config.enabled_tc); 13611 if (veb) { 13612 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 13613 dev_info(&vsi->back->pdev->dev, 13614 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 13615 return NULL; 13616 } 13617 /* We come up by default in VEPA mode if SRIOV is not 13618 * already enabled, in which case we can't force VEPA 13619 * mode. 13620 */ 13621 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 13622 veb->bridge_mode = BRIDGE_MODE_VEPA; 13623 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 13624 } 13625 i40e_config_bridge_mode(veb); 13626 } 13627 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 13628 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 13629 veb = pf->veb[i]; 13630 } 13631 if (!veb) { 13632 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 13633 return NULL; 13634 } 13635 13636 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 13637 uplink_seid = veb->seid; 13638 } 13639 13640 /* get vsi sw struct */ 13641 v_idx = i40e_vsi_mem_alloc(pf, type); 13642 if (v_idx < 0) 13643 goto err_alloc; 13644 vsi = pf->vsi[v_idx]; 13645 if (!vsi) 13646 goto err_alloc; 13647 vsi->type = type; 13648 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 13649 13650 if (type == I40E_VSI_MAIN) 13651 pf->lan_vsi = v_idx; 13652 else if (type == I40E_VSI_SRIOV) 13653 vsi->vf_id = param1; 13654 /* assign it some queues */ 13655 alloc_queue_pairs = vsi->alloc_queue_pairs * 13656 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 13657 13658 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 13659 if (ret < 0) { 13660 dev_info(&pf->pdev->dev, 13661 "failed to get tracking for %d queues for VSI %d err=%d\n", 13662 alloc_queue_pairs, vsi->seid, ret); 13663 goto err_vsi; 13664 } 13665 vsi->base_queue = ret; 13666 13667 /* get a VSI from the hardware */ 13668 vsi->uplink_seid = uplink_seid; 13669 ret = i40e_add_vsi(vsi); 13670 if (ret) 13671 goto err_vsi; 13672 13673 switch (vsi->type) { 13674 /* setup the netdev if needed */ 13675 case I40E_VSI_MAIN: 13676 case I40E_VSI_VMDQ2: 13677 ret = i40e_config_netdev(vsi); 13678 if (ret) 13679 goto err_netdev; 13680 ret = register_netdev(vsi->netdev); 13681 if (ret) 13682 goto err_netdev; 13683 vsi->netdev_registered = true; 13684 netif_carrier_off(vsi->netdev); 13685 #ifdef CONFIG_I40E_DCB 13686 /* Setup DCB netlink interface */ 13687 i40e_dcbnl_setup(vsi); 13688 #endif /* CONFIG_I40E_DCB */ 13689 /* fall through */ 13690 13691 case I40E_VSI_FDIR: 13692 /* set up vectors and rings if needed */ 13693 ret = i40e_vsi_setup_vectors(vsi); 13694 if (ret) 13695 goto err_msix; 13696 13697 ret = i40e_alloc_rings(vsi); 13698 if (ret) 13699 goto err_rings; 13700 13701 /* map all of the rings to the q_vectors */ 13702 i40e_vsi_map_rings_to_vectors(vsi); 13703 13704 i40e_vsi_reset_stats(vsi); 13705 break; 13706 13707 default: 13708 /* no netdev or rings for the other VSI types */ 13709 break; 13710 } 13711 13712 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 13713 (vsi->type == I40E_VSI_VMDQ2)) { 13714 ret = i40e_vsi_config_rss(vsi); 13715 } 13716 return vsi; 13717 13718 err_rings: 13719 i40e_vsi_free_q_vectors(vsi); 13720 err_msix: 13721 if (vsi->netdev_registered) { 13722 vsi->netdev_registered = false; 13723 unregister_netdev(vsi->netdev); 13724 free_netdev(vsi->netdev); 13725 vsi->netdev = NULL; 13726 } 13727 err_netdev: 13728 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13729 err_vsi: 13730 i40e_vsi_clear(vsi); 13731 err_alloc: 13732 return NULL; 13733 } 13734 13735 /** 13736 * i40e_veb_get_bw_info - Query VEB BW information 13737 * @veb: the veb to query 13738 * 13739 * Query the Tx scheduler BW configuration data for given VEB 13740 **/ 13741 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 13742 { 13743 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 13744 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 13745 struct i40e_pf *pf = veb->pf; 13746 struct i40e_hw *hw = &pf->hw; 13747 u32 tc_bw_max; 13748 int ret = 0; 13749 int i; 13750 13751 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 13752 &bw_data, NULL); 13753 if (ret) { 13754 dev_info(&pf->pdev->dev, 13755 "query veb bw config failed, err %s aq_err %s\n", 13756 i40e_stat_str(&pf->hw, ret), 13757 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13758 goto out; 13759 } 13760 13761 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 13762 &ets_data, NULL); 13763 if (ret) { 13764 dev_info(&pf->pdev->dev, 13765 "query veb bw ets config failed, err %s aq_err %s\n", 13766 i40e_stat_str(&pf->hw, ret), 13767 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13768 goto out; 13769 } 13770 13771 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 13772 veb->bw_max_quanta = ets_data.tc_bw_max; 13773 veb->is_abs_credits = bw_data.absolute_credits_enable; 13774 veb->enabled_tc = ets_data.tc_valid_bits; 13775 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 13776 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 13777 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 13778 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 13779 veb->bw_tc_limit_credits[i] = 13780 le16_to_cpu(bw_data.tc_bw_limits[i]); 13781 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 13782 } 13783 13784 out: 13785 return ret; 13786 } 13787 13788 /** 13789 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 13790 * @pf: board private structure 13791 * 13792 * On error: returns error code (negative) 13793 * On success: returns vsi index in PF (positive) 13794 **/ 13795 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 13796 { 13797 int ret = -ENOENT; 13798 struct i40e_veb *veb; 13799 int i; 13800 13801 /* Need to protect the allocation of switch elements at the PF level */ 13802 mutex_lock(&pf->switch_mutex); 13803 13804 /* VEB list may be fragmented if VEB creation/destruction has 13805 * been happening. We can afford to do a quick scan to look 13806 * for any free slots in the list. 13807 * 13808 * find next empty veb slot, looping back around if necessary 13809 */ 13810 i = 0; 13811 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 13812 i++; 13813 if (i >= I40E_MAX_VEB) { 13814 ret = -ENOMEM; 13815 goto err_alloc_veb; /* out of VEB slots! */ 13816 } 13817 13818 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 13819 if (!veb) { 13820 ret = -ENOMEM; 13821 goto err_alloc_veb; 13822 } 13823 veb->pf = pf; 13824 veb->idx = i; 13825 veb->enabled_tc = 1; 13826 13827 pf->veb[i] = veb; 13828 ret = i; 13829 err_alloc_veb: 13830 mutex_unlock(&pf->switch_mutex); 13831 return ret; 13832 } 13833 13834 /** 13835 * i40e_switch_branch_release - Delete a branch of the switch tree 13836 * @branch: where to start deleting 13837 * 13838 * This uses recursion to find the tips of the branch to be 13839 * removed, deleting until we get back to and can delete this VEB. 13840 **/ 13841 static void i40e_switch_branch_release(struct i40e_veb *branch) 13842 { 13843 struct i40e_pf *pf = branch->pf; 13844 u16 branch_seid = branch->seid; 13845 u16 veb_idx = branch->idx; 13846 int i; 13847 13848 /* release any VEBs on this VEB - RECURSION */ 13849 for (i = 0; i < I40E_MAX_VEB; i++) { 13850 if (!pf->veb[i]) 13851 continue; 13852 if (pf->veb[i]->uplink_seid == branch->seid) 13853 i40e_switch_branch_release(pf->veb[i]); 13854 } 13855 13856 /* Release the VSIs on this VEB, but not the owner VSI. 13857 * 13858 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 13859 * the VEB itself, so don't use (*branch) after this loop. 13860 */ 13861 for (i = 0; i < pf->num_alloc_vsi; i++) { 13862 if (!pf->vsi[i]) 13863 continue; 13864 if (pf->vsi[i]->uplink_seid == branch_seid && 13865 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13866 i40e_vsi_release(pf->vsi[i]); 13867 } 13868 } 13869 13870 /* There's one corner case where the VEB might not have been 13871 * removed, so double check it here and remove it if needed. 13872 * This case happens if the veb was created from the debugfs 13873 * commands and no VSIs were added to it. 13874 */ 13875 if (pf->veb[veb_idx]) 13876 i40e_veb_release(pf->veb[veb_idx]); 13877 } 13878 13879 /** 13880 * i40e_veb_clear - remove veb struct 13881 * @veb: the veb to remove 13882 **/ 13883 static void i40e_veb_clear(struct i40e_veb *veb) 13884 { 13885 if (!veb) 13886 return; 13887 13888 if (veb->pf) { 13889 struct i40e_pf *pf = veb->pf; 13890 13891 mutex_lock(&pf->switch_mutex); 13892 if (pf->veb[veb->idx] == veb) 13893 pf->veb[veb->idx] = NULL; 13894 mutex_unlock(&pf->switch_mutex); 13895 } 13896 13897 kfree(veb); 13898 } 13899 13900 /** 13901 * i40e_veb_release - Delete a VEB and free its resources 13902 * @veb: the VEB being removed 13903 **/ 13904 void i40e_veb_release(struct i40e_veb *veb) 13905 { 13906 struct i40e_vsi *vsi = NULL; 13907 struct i40e_pf *pf; 13908 int i, n = 0; 13909 13910 pf = veb->pf; 13911 13912 /* find the remaining VSI and check for extras */ 13913 for (i = 0; i < pf->num_alloc_vsi; i++) { 13914 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 13915 n++; 13916 vsi = pf->vsi[i]; 13917 } 13918 } 13919 if (n != 1) { 13920 dev_info(&pf->pdev->dev, 13921 "can't remove VEB %d with %d VSIs left\n", 13922 veb->seid, n); 13923 return; 13924 } 13925 13926 /* move the remaining VSI to uplink veb */ 13927 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 13928 if (veb->uplink_seid) { 13929 vsi->uplink_seid = veb->uplink_seid; 13930 if (veb->uplink_seid == pf->mac_seid) 13931 vsi->veb_idx = I40E_NO_VEB; 13932 else 13933 vsi->veb_idx = veb->veb_idx; 13934 } else { 13935 /* floating VEB */ 13936 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 13937 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 13938 } 13939 13940 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 13941 i40e_veb_clear(veb); 13942 } 13943 13944 /** 13945 * i40e_add_veb - create the VEB in the switch 13946 * @veb: the VEB to be instantiated 13947 * @vsi: the controlling VSI 13948 **/ 13949 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 13950 { 13951 struct i40e_pf *pf = veb->pf; 13952 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 13953 int ret; 13954 13955 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 13956 veb->enabled_tc, false, 13957 &veb->seid, enable_stats, NULL); 13958 13959 /* get a VEB from the hardware */ 13960 if (ret) { 13961 dev_info(&pf->pdev->dev, 13962 "couldn't add VEB, err %s aq_err %s\n", 13963 i40e_stat_str(&pf->hw, ret), 13964 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13965 return -EPERM; 13966 } 13967 13968 /* get statistics counter */ 13969 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 13970 &veb->stats_idx, NULL, NULL, NULL); 13971 if (ret) { 13972 dev_info(&pf->pdev->dev, 13973 "couldn't get VEB statistics idx, err %s aq_err %s\n", 13974 i40e_stat_str(&pf->hw, ret), 13975 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13976 return -EPERM; 13977 } 13978 ret = i40e_veb_get_bw_info(veb); 13979 if (ret) { 13980 dev_info(&pf->pdev->dev, 13981 "couldn't get VEB bw info, err %s aq_err %s\n", 13982 i40e_stat_str(&pf->hw, ret), 13983 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13984 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 13985 return -ENOENT; 13986 } 13987 13988 vsi->uplink_seid = veb->seid; 13989 vsi->veb_idx = veb->idx; 13990 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 13991 13992 return 0; 13993 } 13994 13995 /** 13996 * i40e_veb_setup - Set up a VEB 13997 * @pf: board private structure 13998 * @flags: VEB setup flags 13999 * @uplink_seid: the switch element to link to 14000 * @vsi_seid: the initial VSI seid 14001 * @enabled_tc: Enabled TC bit-map 14002 * 14003 * This allocates the sw VEB structure and links it into the switch 14004 * It is possible and legal for this to be a duplicate of an already 14005 * existing VEB. It is also possible for both uplink and vsi seids 14006 * to be zero, in order to create a floating VEB. 14007 * 14008 * Returns pointer to the successfully allocated VEB sw struct on 14009 * success, otherwise returns NULL on failure. 14010 **/ 14011 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 14012 u16 uplink_seid, u16 vsi_seid, 14013 u8 enabled_tc) 14014 { 14015 struct i40e_veb *veb, *uplink_veb = NULL; 14016 int vsi_idx, veb_idx; 14017 int ret; 14018 14019 /* if one seid is 0, the other must be 0 to create a floating relay */ 14020 if ((uplink_seid == 0 || vsi_seid == 0) && 14021 (uplink_seid + vsi_seid != 0)) { 14022 dev_info(&pf->pdev->dev, 14023 "one, not both seid's are 0: uplink=%d vsi=%d\n", 14024 uplink_seid, vsi_seid); 14025 return NULL; 14026 } 14027 14028 /* make sure there is such a vsi and uplink */ 14029 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 14030 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 14031 break; 14032 if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) { 14033 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 14034 vsi_seid); 14035 return NULL; 14036 } 14037 14038 if (uplink_seid && uplink_seid != pf->mac_seid) { 14039 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 14040 if (pf->veb[veb_idx] && 14041 pf->veb[veb_idx]->seid == uplink_seid) { 14042 uplink_veb = pf->veb[veb_idx]; 14043 break; 14044 } 14045 } 14046 if (!uplink_veb) { 14047 dev_info(&pf->pdev->dev, 14048 "uplink seid %d not found\n", uplink_seid); 14049 return NULL; 14050 } 14051 } 14052 14053 /* get veb sw struct */ 14054 veb_idx = i40e_veb_mem_alloc(pf); 14055 if (veb_idx < 0) 14056 goto err_alloc; 14057 veb = pf->veb[veb_idx]; 14058 veb->flags = flags; 14059 veb->uplink_seid = uplink_seid; 14060 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 14061 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 14062 14063 /* create the VEB in the switch */ 14064 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 14065 if (ret) 14066 goto err_veb; 14067 if (vsi_idx == pf->lan_vsi) 14068 pf->lan_veb = veb->idx; 14069 14070 return veb; 14071 14072 err_veb: 14073 i40e_veb_clear(veb); 14074 err_alloc: 14075 return NULL; 14076 } 14077 14078 /** 14079 * i40e_setup_pf_switch_element - set PF vars based on switch type 14080 * @pf: board private structure 14081 * @ele: element we are building info from 14082 * @num_reported: total number of elements 14083 * @printconfig: should we print the contents 14084 * 14085 * helper function to assist in extracting a few useful SEID values. 14086 **/ 14087 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 14088 struct i40e_aqc_switch_config_element_resp *ele, 14089 u16 num_reported, bool printconfig) 14090 { 14091 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 14092 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 14093 u8 element_type = ele->element_type; 14094 u16 seid = le16_to_cpu(ele->seid); 14095 14096 if (printconfig) 14097 dev_info(&pf->pdev->dev, 14098 "type=%d seid=%d uplink=%d downlink=%d\n", 14099 element_type, seid, uplink_seid, downlink_seid); 14100 14101 switch (element_type) { 14102 case I40E_SWITCH_ELEMENT_TYPE_MAC: 14103 pf->mac_seid = seid; 14104 break; 14105 case I40E_SWITCH_ELEMENT_TYPE_VEB: 14106 /* Main VEB? */ 14107 if (uplink_seid != pf->mac_seid) 14108 break; 14109 if (pf->lan_veb >= I40E_MAX_VEB) { 14110 int v; 14111 14112 /* find existing or else empty VEB */ 14113 for (v = 0; v < I40E_MAX_VEB; v++) { 14114 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 14115 pf->lan_veb = v; 14116 break; 14117 } 14118 } 14119 if (pf->lan_veb >= I40E_MAX_VEB) { 14120 v = i40e_veb_mem_alloc(pf); 14121 if (v < 0) 14122 break; 14123 pf->lan_veb = v; 14124 } 14125 } 14126 if (pf->lan_veb >= I40E_MAX_VEB) 14127 break; 14128 14129 pf->veb[pf->lan_veb]->seid = seid; 14130 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 14131 pf->veb[pf->lan_veb]->pf = pf; 14132 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 14133 break; 14134 case I40E_SWITCH_ELEMENT_TYPE_VSI: 14135 if (num_reported != 1) 14136 break; 14137 /* This is immediately after a reset so we can assume this is 14138 * the PF's VSI 14139 */ 14140 pf->mac_seid = uplink_seid; 14141 pf->pf_seid = downlink_seid; 14142 pf->main_vsi_seid = seid; 14143 if (printconfig) 14144 dev_info(&pf->pdev->dev, 14145 "pf_seid=%d main_vsi_seid=%d\n", 14146 pf->pf_seid, pf->main_vsi_seid); 14147 break; 14148 case I40E_SWITCH_ELEMENT_TYPE_PF: 14149 case I40E_SWITCH_ELEMENT_TYPE_VF: 14150 case I40E_SWITCH_ELEMENT_TYPE_EMP: 14151 case I40E_SWITCH_ELEMENT_TYPE_BMC: 14152 case I40E_SWITCH_ELEMENT_TYPE_PE: 14153 case I40E_SWITCH_ELEMENT_TYPE_PA: 14154 /* ignore these for now */ 14155 break; 14156 default: 14157 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 14158 element_type, seid); 14159 break; 14160 } 14161 } 14162 14163 /** 14164 * i40e_fetch_switch_configuration - Get switch config from firmware 14165 * @pf: board private structure 14166 * @printconfig: should we print the contents 14167 * 14168 * Get the current switch configuration from the device and 14169 * extract a few useful SEID values. 14170 **/ 14171 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 14172 { 14173 struct i40e_aqc_get_switch_config_resp *sw_config; 14174 u16 next_seid = 0; 14175 int ret = 0; 14176 u8 *aq_buf; 14177 int i; 14178 14179 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 14180 if (!aq_buf) 14181 return -ENOMEM; 14182 14183 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 14184 do { 14185 u16 num_reported, num_total; 14186 14187 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 14188 I40E_AQ_LARGE_BUF, 14189 &next_seid, NULL); 14190 if (ret) { 14191 dev_info(&pf->pdev->dev, 14192 "get switch config failed err %s aq_err %s\n", 14193 i40e_stat_str(&pf->hw, ret), 14194 i40e_aq_str(&pf->hw, 14195 pf->hw.aq.asq_last_status)); 14196 kfree(aq_buf); 14197 return -ENOENT; 14198 } 14199 14200 num_reported = le16_to_cpu(sw_config->header.num_reported); 14201 num_total = le16_to_cpu(sw_config->header.num_total); 14202 14203 if (printconfig) 14204 dev_info(&pf->pdev->dev, 14205 "header: %d reported %d total\n", 14206 num_reported, num_total); 14207 14208 for (i = 0; i < num_reported; i++) { 14209 struct i40e_aqc_switch_config_element_resp *ele = 14210 &sw_config->element[i]; 14211 14212 i40e_setup_pf_switch_element(pf, ele, num_reported, 14213 printconfig); 14214 } 14215 } while (next_seid != 0); 14216 14217 kfree(aq_buf); 14218 return ret; 14219 } 14220 14221 /** 14222 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 14223 * @pf: board private structure 14224 * @reinit: if the Main VSI needs to re-initialized. 14225 * 14226 * Returns 0 on success, negative value on failure 14227 **/ 14228 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 14229 { 14230 u16 flags = 0; 14231 int ret; 14232 14233 /* find out what's out there already */ 14234 ret = i40e_fetch_switch_configuration(pf, false); 14235 if (ret) { 14236 dev_info(&pf->pdev->dev, 14237 "couldn't fetch switch config, err %s aq_err %s\n", 14238 i40e_stat_str(&pf->hw, ret), 14239 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14240 return ret; 14241 } 14242 i40e_pf_reset_stats(pf); 14243 14244 /* set the switch config bit for the whole device to 14245 * support limited promisc or true promisc 14246 * when user requests promisc. The default is limited 14247 * promisc. 14248 */ 14249 14250 if ((pf->hw.pf_id == 0) && 14251 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 14252 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14253 pf->last_sw_conf_flags = flags; 14254 } 14255 14256 if (pf->hw.pf_id == 0) { 14257 u16 valid_flags; 14258 14259 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14260 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 14261 NULL); 14262 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 14263 dev_info(&pf->pdev->dev, 14264 "couldn't set switch config bits, err %s aq_err %s\n", 14265 i40e_stat_str(&pf->hw, ret), 14266 i40e_aq_str(&pf->hw, 14267 pf->hw.aq.asq_last_status)); 14268 /* not a fatal problem, just keep going */ 14269 } 14270 pf->last_sw_conf_valid_flags = valid_flags; 14271 } 14272 14273 /* first time setup */ 14274 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 14275 struct i40e_vsi *vsi = NULL; 14276 u16 uplink_seid; 14277 14278 /* Set up the PF VSI associated with the PF's main VSI 14279 * that is already in the HW switch 14280 */ 14281 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 14282 uplink_seid = pf->veb[pf->lan_veb]->seid; 14283 else 14284 uplink_seid = pf->mac_seid; 14285 if (pf->lan_vsi == I40E_NO_VSI) 14286 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 14287 else if (reinit) 14288 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 14289 if (!vsi) { 14290 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 14291 i40e_cloud_filter_exit(pf); 14292 i40e_fdir_teardown(pf); 14293 return -EAGAIN; 14294 } 14295 } else { 14296 /* force a reset of TC and queue layout configurations */ 14297 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 14298 14299 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 14300 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 14301 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 14302 } 14303 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 14304 14305 i40e_fdir_sb_setup(pf); 14306 14307 /* Setup static PF queue filter control settings */ 14308 ret = i40e_setup_pf_filter_control(pf); 14309 if (ret) { 14310 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 14311 ret); 14312 /* Failure here should not stop continuing other steps */ 14313 } 14314 14315 /* enable RSS in the HW, even for only one queue, as the stack can use 14316 * the hash 14317 */ 14318 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 14319 i40e_pf_config_rss(pf); 14320 14321 /* fill in link information and enable LSE reporting */ 14322 i40e_link_event(pf); 14323 14324 /* Initialize user-specific link properties */ 14325 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 14326 I40E_AQ_AN_COMPLETED) ? true : false); 14327 14328 i40e_ptp_init(pf); 14329 14330 /* repopulate tunnel port filters */ 14331 i40e_sync_udp_filters(pf); 14332 14333 return ret; 14334 } 14335 14336 /** 14337 * i40e_determine_queue_usage - Work out queue distribution 14338 * @pf: board private structure 14339 **/ 14340 static void i40e_determine_queue_usage(struct i40e_pf *pf) 14341 { 14342 int queues_left; 14343 int q_max; 14344 14345 pf->num_lan_qps = 0; 14346 14347 /* Find the max queues to be put into basic use. We'll always be 14348 * using TC0, whether or not DCB is running, and TC0 will get the 14349 * big RSS set. 14350 */ 14351 queues_left = pf->hw.func_caps.num_tx_qp; 14352 14353 if ((queues_left == 1) || 14354 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 14355 /* one qp for PF, no queues for anything else */ 14356 queues_left = 0; 14357 pf->alloc_rss_size = pf->num_lan_qps = 1; 14358 14359 /* make sure all the fancies are disabled */ 14360 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 14361 I40E_FLAG_IWARP_ENABLED | 14362 I40E_FLAG_FD_SB_ENABLED | 14363 I40E_FLAG_FD_ATR_ENABLED | 14364 I40E_FLAG_DCB_CAPABLE | 14365 I40E_FLAG_DCB_ENABLED | 14366 I40E_FLAG_SRIOV_ENABLED | 14367 I40E_FLAG_VMDQ_ENABLED); 14368 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14369 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 14370 I40E_FLAG_FD_SB_ENABLED | 14371 I40E_FLAG_FD_ATR_ENABLED | 14372 I40E_FLAG_DCB_CAPABLE))) { 14373 /* one qp for PF */ 14374 pf->alloc_rss_size = pf->num_lan_qps = 1; 14375 queues_left -= pf->num_lan_qps; 14376 14377 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 14378 I40E_FLAG_IWARP_ENABLED | 14379 I40E_FLAG_FD_SB_ENABLED | 14380 I40E_FLAG_FD_ATR_ENABLED | 14381 I40E_FLAG_DCB_ENABLED | 14382 I40E_FLAG_VMDQ_ENABLED); 14383 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14384 } else { 14385 /* Not enough queues for all TCs */ 14386 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 14387 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 14388 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 14389 I40E_FLAG_DCB_ENABLED); 14390 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 14391 } 14392 14393 /* limit lan qps to the smaller of qps, cpus or msix */ 14394 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 14395 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 14396 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 14397 pf->num_lan_qps = q_max; 14398 14399 queues_left -= pf->num_lan_qps; 14400 } 14401 14402 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 14403 if (queues_left > 1) { 14404 queues_left -= 1; /* save 1 queue for FD */ 14405 } else { 14406 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 14407 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14408 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 14409 } 14410 } 14411 14412 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 14413 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 14414 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 14415 (queues_left / pf->num_vf_qps)); 14416 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 14417 } 14418 14419 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 14420 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 14421 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 14422 (queues_left / pf->num_vmdq_qps)); 14423 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 14424 } 14425 14426 pf->queues_left = queues_left; 14427 dev_dbg(&pf->pdev->dev, 14428 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 14429 pf->hw.func_caps.num_tx_qp, 14430 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 14431 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 14432 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 14433 queues_left); 14434 } 14435 14436 /** 14437 * i40e_setup_pf_filter_control - Setup PF static filter control 14438 * @pf: PF to be setup 14439 * 14440 * i40e_setup_pf_filter_control sets up a PF's initial filter control 14441 * settings. If PE/FCoE are enabled then it will also set the per PF 14442 * based filter sizes required for them. It also enables Flow director, 14443 * ethertype and macvlan type filter settings for the pf. 14444 * 14445 * Returns 0 on success, negative on failure 14446 **/ 14447 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 14448 { 14449 struct i40e_filter_control_settings *settings = &pf->filter_settings; 14450 14451 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 14452 14453 /* Flow Director is enabled */ 14454 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 14455 settings->enable_fdir = true; 14456 14457 /* Ethtype and MACVLAN filters enabled for PF */ 14458 settings->enable_ethtype = true; 14459 settings->enable_macvlan = true; 14460 14461 if (i40e_set_filter_control(&pf->hw, settings)) 14462 return -ENOENT; 14463 14464 return 0; 14465 } 14466 14467 #define INFO_STRING_LEN 255 14468 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 14469 static void i40e_print_features(struct i40e_pf *pf) 14470 { 14471 struct i40e_hw *hw = &pf->hw; 14472 char *buf; 14473 int i; 14474 14475 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 14476 if (!buf) 14477 return; 14478 14479 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 14480 #ifdef CONFIG_PCI_IOV 14481 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 14482 #endif 14483 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 14484 pf->hw.func_caps.num_vsis, 14485 pf->vsi[pf->lan_vsi]->num_queue_pairs); 14486 if (pf->flags & I40E_FLAG_RSS_ENABLED) 14487 i += scnprintf(&buf[i], REMAIN(i), " RSS"); 14488 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 14489 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); 14490 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 14491 i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); 14492 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); 14493 } 14494 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 14495 i += scnprintf(&buf[i], REMAIN(i), " DCB"); 14496 i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); 14497 i += scnprintf(&buf[i], REMAIN(i), " Geneve"); 14498 if (pf->flags & I40E_FLAG_PTP) 14499 i += scnprintf(&buf[i], REMAIN(i), " PTP"); 14500 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 14501 i += scnprintf(&buf[i], REMAIN(i), " VEB"); 14502 else 14503 i += scnprintf(&buf[i], REMAIN(i), " VEPA"); 14504 14505 dev_info(&pf->pdev->dev, "%s\n", buf); 14506 kfree(buf); 14507 WARN_ON(i > INFO_STRING_LEN); 14508 } 14509 14510 /** 14511 * i40e_get_platform_mac_addr - get platform-specific MAC address 14512 * @pdev: PCI device information struct 14513 * @pf: board private structure 14514 * 14515 * Look up the MAC address for the device. First we'll try 14516 * eth_platform_get_mac_address, which will check Open Firmware, or arch 14517 * specific fallback. Otherwise, we'll default to the stored value in 14518 * firmware. 14519 **/ 14520 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 14521 { 14522 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 14523 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 14524 } 14525 14526 /** 14527 * i40e_set_fec_in_flags - helper function for setting FEC options in flags 14528 * @fec_cfg: FEC option to set in flags 14529 * @flags: ptr to flags in which we set FEC option 14530 **/ 14531 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags) 14532 { 14533 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) 14534 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC; 14535 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) || 14536 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) { 14537 *flags |= I40E_FLAG_RS_FEC; 14538 *flags &= ~I40E_FLAG_BASE_R_FEC; 14539 } 14540 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) || 14541 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) { 14542 *flags |= I40E_FLAG_BASE_R_FEC; 14543 *flags &= ~I40E_FLAG_RS_FEC; 14544 } 14545 if (fec_cfg == 0) 14546 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC); 14547 } 14548 14549 /** 14550 * i40e_check_recovery_mode - check if we are running transition firmware 14551 * @pf: board private structure 14552 * 14553 * Check registers indicating the firmware runs in recovery mode. Sets the 14554 * appropriate driver state. 14555 * 14556 * Returns true if the recovery mode was detected, false otherwise 14557 **/ 14558 static bool i40e_check_recovery_mode(struct i40e_pf *pf) 14559 { 14560 u32 val = rd32(&pf->hw, I40E_GL_FWSTS); 14561 14562 if (val & I40E_GL_FWSTS_FWS1B_MASK) { 14563 dev_crit(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n"); 14564 dev_crit(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 14565 set_bit(__I40E_RECOVERY_MODE, pf->state); 14566 14567 return true; 14568 } 14569 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 14570 dev_info(&pf->pdev->dev, "Please do Power-On Reset to initialize adapter in normal mode with full functionality.\n"); 14571 14572 return false; 14573 } 14574 14575 /** 14576 * i40e_pf_loop_reset - perform reset in a loop. 14577 * @pf: board private structure 14578 * 14579 * This function is useful when a NIC is about to enter recovery mode. 14580 * When a NIC's internal data structures are corrupted the NIC's 14581 * firmware is going to enter recovery mode. 14582 * Right after a POR it takes about 7 minutes for firmware to enter 14583 * recovery mode. Until that time a NIC is in some kind of intermediate 14584 * state. After that time period the NIC almost surely enters 14585 * recovery mode. The only way for a driver to detect intermediate 14586 * state is to issue a series of pf-resets and check a return value. 14587 * If a PF reset returns success then the firmware could be in recovery 14588 * mode so the caller of this code needs to check for recovery mode 14589 * if this function returns success. There is a little chance that 14590 * firmware will hang in intermediate state forever. 14591 * Since waiting 7 minutes is quite a lot of time this function waits 14592 * 10 seconds and then gives up by returning an error. 14593 * 14594 * Return 0 on success, negative on failure. 14595 **/ 14596 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf) 14597 { 14598 /* wait max 10 seconds for PF reset to succeed */ 14599 const unsigned long time_end = jiffies + 10 * HZ; 14600 14601 struct i40e_hw *hw = &pf->hw; 14602 i40e_status ret; 14603 14604 ret = i40e_pf_reset(hw); 14605 while (ret != I40E_SUCCESS && time_before(jiffies, time_end)) { 14606 usleep_range(10000, 20000); 14607 ret = i40e_pf_reset(hw); 14608 } 14609 14610 if (ret == I40E_SUCCESS) 14611 pf->pfr_count++; 14612 else 14613 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); 14614 14615 return ret; 14616 } 14617 14618 /** 14619 * i40e_check_fw_empr - check if FW issued unexpected EMP Reset 14620 * @pf: board private structure 14621 * 14622 * Check FW registers to determine if FW issued unexpected EMP Reset. 14623 * Every time when unexpected EMP Reset occurs the FW increments 14624 * a counter of unexpected EMP Resets. When the counter reaches 10 14625 * the FW should enter the Recovery mode 14626 * 14627 * Returns true if FW issued unexpected EMP Reset 14628 **/ 14629 static bool i40e_check_fw_empr(struct i40e_pf *pf) 14630 { 14631 const u32 fw_sts = rd32(&pf->hw, I40E_GL_FWSTS) & 14632 I40E_GL_FWSTS_FWS1B_MASK; 14633 return (fw_sts > I40E_GL_FWSTS_FWS1B_EMPR_0) && 14634 (fw_sts <= I40E_GL_FWSTS_FWS1B_EMPR_10); 14635 } 14636 14637 /** 14638 * i40e_handle_resets - handle EMP resets and PF resets 14639 * @pf: board private structure 14640 * 14641 * Handle both EMP resets and PF resets and conclude whether there are 14642 * any issues regarding these resets. If there are any issues then 14643 * generate log entry. 14644 * 14645 * Return 0 if NIC is healthy or negative value when there are issues 14646 * with resets 14647 **/ 14648 static i40e_status i40e_handle_resets(struct i40e_pf *pf) 14649 { 14650 const i40e_status pfr = i40e_pf_loop_reset(pf); 14651 const bool is_empr = i40e_check_fw_empr(pf); 14652 14653 if (is_empr || pfr != I40E_SUCCESS) 14654 dev_crit(&pf->pdev->dev, "Entering recovery mode due to repeated FW resets. This may take several minutes. Refer to the Intel(R) Ethernet Adapters and Devices User Guide.\n"); 14655 14656 return is_empr ? I40E_ERR_RESET_FAILED : pfr; 14657 } 14658 14659 /** 14660 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode 14661 * @pf: board private structure 14662 * @hw: ptr to the hardware info 14663 * 14664 * This function does a minimal setup of all subsystems needed for running 14665 * recovery mode. 14666 * 14667 * Returns 0 on success, negative on failure 14668 **/ 14669 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw) 14670 { 14671 struct i40e_vsi *vsi; 14672 int err; 14673 int v_idx; 14674 14675 pci_save_state(pf->pdev); 14676 14677 /* set up periodic task facility */ 14678 timer_setup(&pf->service_timer, i40e_service_timer, 0); 14679 pf->service_timer_period = HZ; 14680 14681 INIT_WORK(&pf->service_task, i40e_service_task); 14682 clear_bit(__I40E_SERVICE_SCHED, pf->state); 14683 14684 err = i40e_init_interrupt_scheme(pf); 14685 if (err) 14686 goto err_switch_setup; 14687 14688 /* The number of VSIs reported by the FW is the minimum guaranteed 14689 * to us; HW supports far more and we share the remaining pool with 14690 * the other PFs. We allocate space for more than the guarantee with 14691 * the understanding that we might not get them all later. 14692 */ 14693 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 14694 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 14695 else 14696 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 14697 14698 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */ 14699 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 14700 GFP_KERNEL); 14701 if (!pf->vsi) { 14702 err = -ENOMEM; 14703 goto err_switch_setup; 14704 } 14705 14706 /* We allocate one VSI which is needed as absolute minimum 14707 * in order to register the netdev 14708 */ 14709 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN); 14710 if (v_idx < 0) 14711 goto err_switch_setup; 14712 pf->lan_vsi = v_idx; 14713 vsi = pf->vsi[v_idx]; 14714 if (!vsi) 14715 goto err_switch_setup; 14716 vsi->alloc_queue_pairs = 1; 14717 err = i40e_config_netdev(vsi); 14718 if (err) 14719 goto err_switch_setup; 14720 err = register_netdev(vsi->netdev); 14721 if (err) 14722 goto err_switch_setup; 14723 vsi->netdev_registered = true; 14724 i40e_dbg_pf_init(pf); 14725 14726 err = i40e_setup_misc_vector_for_recovery_mode(pf); 14727 if (err) 14728 goto err_switch_setup; 14729 14730 /* tell the firmware that we're starting */ 14731 i40e_send_version(pf); 14732 14733 /* since everything's happy, start the service_task timer */ 14734 mod_timer(&pf->service_timer, 14735 round_jiffies(jiffies + pf->service_timer_period)); 14736 14737 return 0; 14738 14739 err_switch_setup: 14740 i40e_reset_interrupt_capability(pf); 14741 del_timer_sync(&pf->service_timer); 14742 i40e_shutdown_adminq(hw); 14743 iounmap(hw->hw_addr); 14744 pci_disable_pcie_error_reporting(pf->pdev); 14745 pci_release_mem_regions(pf->pdev); 14746 pci_disable_device(pf->pdev); 14747 kfree(pf); 14748 14749 return err; 14750 } 14751 14752 /** 14753 * i40e_probe - Device initialization routine 14754 * @pdev: PCI device information struct 14755 * @ent: entry in i40e_pci_tbl 14756 * 14757 * i40e_probe initializes a PF identified by a pci_dev structure. 14758 * The OS initialization, configuring of the PF private structure, 14759 * and a hardware reset occur. 14760 * 14761 * Returns 0 on success, negative on failure 14762 **/ 14763 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 14764 { 14765 struct i40e_aq_get_phy_abilities_resp abilities; 14766 struct i40e_pf *pf; 14767 struct i40e_hw *hw; 14768 static u16 pfs_found; 14769 u16 wol_nvm_bits; 14770 u16 link_status; 14771 int err; 14772 u32 val; 14773 u32 i; 14774 u8 set_fc_aq_fail; 14775 14776 err = pci_enable_device_mem(pdev); 14777 if (err) 14778 return err; 14779 14780 /* set up for high or low dma */ 14781 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 14782 if (err) { 14783 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 14784 if (err) { 14785 dev_err(&pdev->dev, 14786 "DMA configuration failed: 0x%x\n", err); 14787 goto err_dma; 14788 } 14789 } 14790 14791 /* set up pci connections */ 14792 err = pci_request_mem_regions(pdev, i40e_driver_name); 14793 if (err) { 14794 dev_info(&pdev->dev, 14795 "pci_request_selected_regions failed %d\n", err); 14796 goto err_pci_reg; 14797 } 14798 14799 pci_enable_pcie_error_reporting(pdev); 14800 pci_set_master(pdev); 14801 14802 /* Now that we have a PCI connection, we need to do the 14803 * low level device setup. This is primarily setting up 14804 * the Admin Queue structures and then querying for the 14805 * device's current profile information. 14806 */ 14807 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 14808 if (!pf) { 14809 err = -ENOMEM; 14810 goto err_pf_alloc; 14811 } 14812 pf->next_vsi = 0; 14813 pf->pdev = pdev; 14814 set_bit(__I40E_DOWN, pf->state); 14815 14816 hw = &pf->hw; 14817 hw->back = pf; 14818 14819 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 14820 I40E_MAX_CSR_SPACE); 14821 /* We believe that the highest register to read is 14822 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size 14823 * is not less than that before mapping to prevent a 14824 * kernel panic. 14825 */ 14826 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) { 14827 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n", 14828 pf->ioremap_len); 14829 err = -ENOMEM; 14830 goto err_ioremap; 14831 } 14832 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 14833 if (!hw->hw_addr) { 14834 err = -EIO; 14835 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 14836 (unsigned int)pci_resource_start(pdev, 0), 14837 pf->ioremap_len, err); 14838 goto err_ioremap; 14839 } 14840 hw->vendor_id = pdev->vendor; 14841 hw->device_id = pdev->device; 14842 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 14843 hw->subsystem_vendor_id = pdev->subsystem_vendor; 14844 hw->subsystem_device_id = pdev->subsystem_device; 14845 hw->bus.device = PCI_SLOT(pdev->devfn); 14846 hw->bus.func = PCI_FUNC(pdev->devfn); 14847 hw->bus.bus_id = pdev->bus->number; 14848 pf->instance = pfs_found; 14849 14850 /* Select something other than the 802.1ad ethertype for the 14851 * switch to use internally and drop on ingress. 14852 */ 14853 hw->switch_tag = 0xffff; 14854 hw->first_tag = ETH_P_8021AD; 14855 hw->second_tag = ETH_P_8021Q; 14856 14857 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 14858 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 14859 INIT_LIST_HEAD(&pf->ddp_old_prof); 14860 14861 /* set up the locks for the AQ, do this only once in probe 14862 * and destroy them only once in remove 14863 */ 14864 mutex_init(&hw->aq.asq_mutex); 14865 mutex_init(&hw->aq.arq_mutex); 14866 14867 pf->msg_enable = netif_msg_init(debug, 14868 NETIF_MSG_DRV | 14869 NETIF_MSG_PROBE | 14870 NETIF_MSG_LINK); 14871 if (debug < -1) 14872 pf->hw.debug_mask = debug; 14873 14874 /* do a special CORER for clearing PXE mode once at init */ 14875 if (hw->revision_id == 0 && 14876 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 14877 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 14878 i40e_flush(hw); 14879 msleep(200); 14880 pf->corer_count++; 14881 14882 i40e_clear_pxe_mode(hw); 14883 } 14884 14885 /* Reset here to make sure all is clean and to define PF 'n' */ 14886 i40e_clear_hw(hw); 14887 14888 err = i40e_set_mac_type(hw); 14889 if (err) { 14890 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 14891 err); 14892 goto err_pf_reset; 14893 } 14894 14895 err = i40e_handle_resets(pf); 14896 if (err) 14897 goto err_pf_reset; 14898 14899 i40e_check_recovery_mode(pf); 14900 14901 hw->aq.num_arq_entries = I40E_AQ_LEN; 14902 hw->aq.num_asq_entries = I40E_AQ_LEN; 14903 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 14904 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 14905 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 14906 14907 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 14908 "%s-%s:misc", 14909 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 14910 14911 err = i40e_init_shared_code(hw); 14912 if (err) { 14913 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 14914 err); 14915 goto err_pf_reset; 14916 } 14917 14918 /* set up a default setting for link flow control */ 14919 pf->hw.fc.requested_mode = I40E_FC_NONE; 14920 14921 err = i40e_init_adminq(hw); 14922 if (err) { 14923 if (err == I40E_ERR_FIRMWARE_API_VERSION) 14924 dev_info(&pdev->dev, 14925 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", 14926 hw->aq.api_maj_ver, 14927 hw->aq.api_min_ver, 14928 I40E_FW_API_VERSION_MAJOR, 14929 I40E_FW_MINOR_VERSION(hw)); 14930 else 14931 dev_info(&pdev->dev, 14932 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 14933 14934 goto err_pf_reset; 14935 } 14936 i40e_get_oem_version(hw); 14937 14938 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */ 14939 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n", 14940 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 14941 hw->aq.api_maj_ver, hw->aq.api_min_ver, 14942 i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id, 14943 hw->subsystem_vendor_id, hw->subsystem_device_id); 14944 14945 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 14946 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 14947 dev_info(&pdev->dev, 14948 "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n", 14949 hw->aq.api_maj_ver, 14950 hw->aq.api_min_ver, 14951 I40E_FW_API_VERSION_MAJOR, 14952 I40E_FW_MINOR_VERSION(hw)); 14953 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 14954 dev_info(&pdev->dev, 14955 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n", 14956 hw->aq.api_maj_ver, 14957 hw->aq.api_min_ver, 14958 I40E_FW_API_VERSION_MAJOR, 14959 I40E_FW_MINOR_VERSION(hw)); 14960 14961 i40e_verify_eeprom(pf); 14962 14963 /* Rev 0 hardware was never productized */ 14964 if (hw->revision_id < 1) 14965 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 14966 14967 i40e_clear_pxe_mode(hw); 14968 14969 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 14970 if (err) 14971 goto err_adminq_setup; 14972 14973 err = i40e_sw_init(pf); 14974 if (err) { 14975 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 14976 goto err_sw_init; 14977 } 14978 14979 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 14980 return i40e_init_recovery_mode(pf, hw); 14981 14982 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 14983 hw->func_caps.num_rx_qp, 0, 0); 14984 if (err) { 14985 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 14986 goto err_init_lan_hmc; 14987 } 14988 14989 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 14990 if (err) { 14991 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 14992 err = -ENOENT; 14993 goto err_configure_lan_hmc; 14994 } 14995 14996 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 14997 * Ignore error return codes because if it was already disabled via 14998 * hardware settings this will fail 14999 */ 15000 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 15001 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 15002 i40e_aq_stop_lldp(hw, true, false, NULL); 15003 } 15004 15005 /* allow a platform config to override the HW addr */ 15006 i40e_get_platform_mac_addr(pdev, pf); 15007 15008 if (!is_valid_ether_addr(hw->mac.addr)) { 15009 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 15010 err = -EIO; 15011 goto err_mac_addr; 15012 } 15013 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 15014 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 15015 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 15016 if (is_valid_ether_addr(hw->mac.port_addr)) 15017 pf->hw_features |= I40E_HW_PORT_ID_VALID; 15018 15019 pci_set_drvdata(pdev, pf); 15020 pci_save_state(pdev); 15021 15022 dev_info(&pdev->dev, 15023 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ? 15024 "FW LLDP is disabled\n" : 15025 "FW LLDP is enabled\n"); 15026 15027 /* Enable FW to write default DCB config on link-up */ 15028 i40e_aq_set_dcb_parameters(hw, true, NULL); 15029 15030 #ifdef CONFIG_I40E_DCB 15031 err = i40e_init_pf_dcb(pf); 15032 if (err) { 15033 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 15034 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 15035 /* Continue without DCB enabled */ 15036 } 15037 #endif /* CONFIG_I40E_DCB */ 15038 15039 /* set up periodic task facility */ 15040 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15041 pf->service_timer_period = HZ; 15042 15043 INIT_WORK(&pf->service_task, i40e_service_task); 15044 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15045 15046 /* NVM bit on means WoL disabled for the port */ 15047 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 15048 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 15049 pf->wol_en = false; 15050 else 15051 pf->wol_en = true; 15052 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 15053 15054 /* set up the main switch operations */ 15055 i40e_determine_queue_usage(pf); 15056 err = i40e_init_interrupt_scheme(pf); 15057 if (err) 15058 goto err_switch_setup; 15059 15060 /* The number of VSIs reported by the FW is the minimum guaranteed 15061 * to us; HW supports far more and we share the remaining pool with 15062 * the other PFs. We allocate space for more than the guarantee with 15063 * the understanding that we might not get them all later. 15064 */ 15065 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15066 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15067 else 15068 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15069 15070 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 15071 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15072 GFP_KERNEL); 15073 if (!pf->vsi) { 15074 err = -ENOMEM; 15075 goto err_switch_setup; 15076 } 15077 15078 #ifdef CONFIG_PCI_IOV 15079 /* prep for VF support */ 15080 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15081 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15082 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15083 if (pci_num_vf(pdev)) 15084 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 15085 } 15086 #endif 15087 err = i40e_setup_pf_switch(pf, false); 15088 if (err) { 15089 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 15090 goto err_vsis; 15091 } 15092 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 15093 15094 /* Make sure flow control is set according to current settings */ 15095 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 15096 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 15097 dev_dbg(&pf->pdev->dev, 15098 "Set fc with err %s aq_err %s on get_phy_cap\n", 15099 i40e_stat_str(hw, err), 15100 i40e_aq_str(hw, hw->aq.asq_last_status)); 15101 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 15102 dev_dbg(&pf->pdev->dev, 15103 "Set fc with err %s aq_err %s on set_phy_config\n", 15104 i40e_stat_str(hw, err), 15105 i40e_aq_str(hw, hw->aq.asq_last_status)); 15106 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 15107 dev_dbg(&pf->pdev->dev, 15108 "Set fc with err %s aq_err %s on get_link_info\n", 15109 i40e_stat_str(hw, err), 15110 i40e_aq_str(hw, hw->aq.asq_last_status)); 15111 15112 /* if FDIR VSI was set up, start it now */ 15113 for (i = 0; i < pf->num_alloc_vsi; i++) { 15114 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 15115 i40e_vsi_open(pf->vsi[i]); 15116 break; 15117 } 15118 } 15119 15120 /* The driver only wants link up/down and module qualification 15121 * reports from firmware. Note the negative logic. 15122 */ 15123 err = i40e_aq_set_phy_int_mask(&pf->hw, 15124 ~(I40E_AQ_EVENT_LINK_UPDOWN | 15125 I40E_AQ_EVENT_MEDIA_NA | 15126 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 15127 if (err) 15128 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 15129 i40e_stat_str(&pf->hw, err), 15130 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15131 15132 /* Reconfigure hardware for allowing smaller MSS in the case 15133 * of TSO, so that we avoid the MDD being fired and causing 15134 * a reset in the case of small MSS+TSO. 15135 */ 15136 val = rd32(hw, I40E_REG_MSS); 15137 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 15138 val &= ~I40E_REG_MSS_MIN_MASK; 15139 val |= I40E_64BYTE_MSS; 15140 wr32(hw, I40E_REG_MSS, val); 15141 } 15142 15143 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 15144 msleep(75); 15145 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 15146 if (err) 15147 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 15148 i40e_stat_str(&pf->hw, err), 15149 i40e_aq_str(&pf->hw, 15150 pf->hw.aq.asq_last_status)); 15151 } 15152 /* The main driver is (mostly) up and happy. We need to set this state 15153 * before setting up the misc vector or we get a race and the vector 15154 * ends up disabled forever. 15155 */ 15156 clear_bit(__I40E_DOWN, pf->state); 15157 15158 /* In case of MSIX we are going to setup the misc vector right here 15159 * to handle admin queue events etc. In case of legacy and MSI 15160 * the misc functionality and queue processing is combined in 15161 * the same vector and that gets setup at open. 15162 */ 15163 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 15164 err = i40e_setup_misc_vector(pf); 15165 if (err) { 15166 dev_info(&pdev->dev, 15167 "setup of misc vector failed: %d\n", err); 15168 goto err_vsis; 15169 } 15170 } 15171 15172 #ifdef CONFIG_PCI_IOV 15173 /* prep for VF support */ 15174 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15175 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15176 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15177 /* disable link interrupts for VFs */ 15178 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 15179 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 15180 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 15181 i40e_flush(hw); 15182 15183 if (pci_num_vf(pdev)) { 15184 dev_info(&pdev->dev, 15185 "Active VFs found, allocating resources.\n"); 15186 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 15187 if (err) 15188 dev_info(&pdev->dev, 15189 "Error %d allocating resources for existing VFs\n", 15190 err); 15191 } 15192 } 15193 #endif /* CONFIG_PCI_IOV */ 15194 15195 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15196 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 15197 pf->num_iwarp_msix, 15198 I40E_IWARP_IRQ_PILE_ID); 15199 if (pf->iwarp_base_vector < 0) { 15200 dev_info(&pdev->dev, 15201 "failed to get tracking for %d vectors for IWARP err=%d\n", 15202 pf->num_iwarp_msix, pf->iwarp_base_vector); 15203 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 15204 } 15205 } 15206 15207 i40e_dbg_pf_init(pf); 15208 15209 /* tell the firmware that we're starting */ 15210 i40e_send_version(pf); 15211 15212 /* since everything's happy, start the service_task timer */ 15213 mod_timer(&pf->service_timer, 15214 round_jiffies(jiffies + pf->service_timer_period)); 15215 15216 /* add this PF to client device list and launch a client service task */ 15217 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15218 err = i40e_lan_add_device(pf); 15219 if (err) 15220 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 15221 err); 15222 } 15223 15224 #define PCI_SPEED_SIZE 8 15225 #define PCI_WIDTH_SIZE 8 15226 /* Devices on the IOSF bus do not have this information 15227 * and will report PCI Gen 1 x 1 by default so don't bother 15228 * checking them. 15229 */ 15230 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 15231 char speed[PCI_SPEED_SIZE] = "Unknown"; 15232 char width[PCI_WIDTH_SIZE] = "Unknown"; 15233 15234 /* Get the negotiated link width and speed from PCI config 15235 * space 15236 */ 15237 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 15238 &link_status); 15239 15240 i40e_set_pci_config_data(hw, link_status); 15241 15242 switch (hw->bus.speed) { 15243 case i40e_bus_speed_8000: 15244 strlcpy(speed, "8.0", PCI_SPEED_SIZE); break; 15245 case i40e_bus_speed_5000: 15246 strlcpy(speed, "5.0", PCI_SPEED_SIZE); break; 15247 case i40e_bus_speed_2500: 15248 strlcpy(speed, "2.5", PCI_SPEED_SIZE); break; 15249 default: 15250 break; 15251 } 15252 switch (hw->bus.width) { 15253 case i40e_bus_width_pcie_x8: 15254 strlcpy(width, "8", PCI_WIDTH_SIZE); break; 15255 case i40e_bus_width_pcie_x4: 15256 strlcpy(width, "4", PCI_WIDTH_SIZE); break; 15257 case i40e_bus_width_pcie_x2: 15258 strlcpy(width, "2", PCI_WIDTH_SIZE); break; 15259 case i40e_bus_width_pcie_x1: 15260 strlcpy(width, "1", PCI_WIDTH_SIZE); break; 15261 default: 15262 break; 15263 } 15264 15265 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 15266 speed, width); 15267 15268 if (hw->bus.width < i40e_bus_width_pcie_x8 || 15269 hw->bus.speed < i40e_bus_speed_8000) { 15270 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 15271 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 15272 } 15273 } 15274 15275 /* get the requested speeds from the fw */ 15276 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 15277 if (err) 15278 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 15279 i40e_stat_str(&pf->hw, err), 15280 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15281 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 15282 15283 /* set the FEC config due to the board capabilities */ 15284 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags); 15285 15286 /* get the supported phy types from the fw */ 15287 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 15288 if (err) 15289 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 15290 i40e_stat_str(&pf->hw, err), 15291 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15292 15293 /* make sure the MFS hasn't been set lower than the default */ 15294 #define MAX_FRAME_SIZE_DEFAULT 0x2600 15295 val = (rd32(&pf->hw, I40E_PRTGL_SAH) & 15296 I40E_PRTGL_SAH_MFS_MASK) >> I40E_PRTGL_SAH_MFS_SHIFT; 15297 if (val < MAX_FRAME_SIZE_DEFAULT) 15298 dev_warn(&pdev->dev, "MFS for port %x has been set below the default: %x\n", 15299 i, val); 15300 15301 /* Add a filter to drop all Flow control frames from any VSI from being 15302 * transmitted. By doing so we stop a malicious VF from sending out 15303 * PAUSE or PFC frames and potentially controlling traffic for other 15304 * PF/VF VSIs. 15305 * The FW can still send Flow control frames if enabled. 15306 */ 15307 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 15308 pf->main_vsi_seid); 15309 15310 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 15311 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 15312 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 15313 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 15314 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 15315 /* print a string summarizing features */ 15316 i40e_print_features(pf); 15317 15318 return 0; 15319 15320 /* Unwind what we've done if something failed in the setup */ 15321 err_vsis: 15322 set_bit(__I40E_DOWN, pf->state); 15323 i40e_clear_interrupt_scheme(pf); 15324 kfree(pf->vsi); 15325 err_switch_setup: 15326 i40e_reset_interrupt_capability(pf); 15327 del_timer_sync(&pf->service_timer); 15328 err_mac_addr: 15329 err_configure_lan_hmc: 15330 (void)i40e_shutdown_lan_hmc(hw); 15331 err_init_lan_hmc: 15332 kfree(pf->qp_pile); 15333 err_sw_init: 15334 err_adminq_setup: 15335 err_pf_reset: 15336 iounmap(hw->hw_addr); 15337 err_ioremap: 15338 kfree(pf); 15339 err_pf_alloc: 15340 pci_disable_pcie_error_reporting(pdev); 15341 pci_release_mem_regions(pdev); 15342 err_pci_reg: 15343 err_dma: 15344 pci_disable_device(pdev); 15345 return err; 15346 } 15347 15348 /** 15349 * i40e_remove - Device removal routine 15350 * @pdev: PCI device information struct 15351 * 15352 * i40e_remove is called by the PCI subsystem to alert the driver 15353 * that is should release a PCI device. This could be caused by a 15354 * Hot-Plug event, or because the driver is going to be removed from 15355 * memory. 15356 **/ 15357 static void i40e_remove(struct pci_dev *pdev) 15358 { 15359 struct i40e_pf *pf = pci_get_drvdata(pdev); 15360 struct i40e_hw *hw = &pf->hw; 15361 i40e_status ret_code; 15362 int i; 15363 15364 i40e_dbg_pf_exit(pf); 15365 15366 i40e_ptp_stop(pf); 15367 15368 /* Disable RSS in hw */ 15369 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 15370 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 15371 15372 /* no more scheduling of any task */ 15373 set_bit(__I40E_SUSPENDED, pf->state); 15374 set_bit(__I40E_DOWN, pf->state); 15375 if (pf->service_timer.function) 15376 del_timer_sync(&pf->service_timer); 15377 if (pf->service_task.func) 15378 cancel_work_sync(&pf->service_task); 15379 15380 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 15381 struct i40e_vsi *vsi = pf->vsi[0]; 15382 15383 /* We know that we have allocated only one vsi for this PF, 15384 * it was just for registering netdevice, so the interface 15385 * could be visible in the 'ifconfig' output 15386 */ 15387 unregister_netdev(vsi->netdev); 15388 free_netdev(vsi->netdev); 15389 15390 goto unmap; 15391 } 15392 15393 /* Client close must be called explicitly here because the timer 15394 * has been stopped. 15395 */ 15396 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15397 15398 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 15399 i40e_free_vfs(pf); 15400 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 15401 } 15402 15403 i40e_fdir_teardown(pf); 15404 15405 /* If there is a switch structure or any orphans, remove them. 15406 * This will leave only the PF's VSI remaining. 15407 */ 15408 for (i = 0; i < I40E_MAX_VEB; i++) { 15409 if (!pf->veb[i]) 15410 continue; 15411 15412 if (pf->veb[i]->uplink_seid == pf->mac_seid || 15413 pf->veb[i]->uplink_seid == 0) 15414 i40e_switch_branch_release(pf->veb[i]); 15415 } 15416 15417 /* Now we can shutdown the PF's VSI, just before we kill 15418 * adminq and hmc. 15419 */ 15420 if (pf->vsi[pf->lan_vsi]) 15421 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 15422 15423 i40e_cloud_filter_exit(pf); 15424 15425 /* remove attached clients */ 15426 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15427 ret_code = i40e_lan_del_device(pf); 15428 if (ret_code) 15429 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 15430 ret_code); 15431 } 15432 15433 /* shutdown and destroy the HMC */ 15434 if (hw->hmc.hmc_obj) { 15435 ret_code = i40e_shutdown_lan_hmc(hw); 15436 if (ret_code) 15437 dev_warn(&pdev->dev, 15438 "Failed to destroy the HMC resources: %d\n", 15439 ret_code); 15440 } 15441 15442 unmap: 15443 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 15444 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 15445 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 15446 free_irq(pf->pdev->irq, pf); 15447 15448 /* shutdown the adminq */ 15449 i40e_shutdown_adminq(hw); 15450 15451 /* destroy the locks only once, here */ 15452 mutex_destroy(&hw->aq.arq_mutex); 15453 mutex_destroy(&hw->aq.asq_mutex); 15454 15455 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 15456 rtnl_lock(); 15457 i40e_clear_interrupt_scheme(pf); 15458 for (i = 0; i < pf->num_alloc_vsi; i++) { 15459 if (pf->vsi[i]) { 15460 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) 15461 i40e_vsi_clear_rings(pf->vsi[i]); 15462 i40e_vsi_clear(pf->vsi[i]); 15463 pf->vsi[i] = NULL; 15464 } 15465 } 15466 rtnl_unlock(); 15467 15468 for (i = 0; i < I40E_MAX_VEB; i++) { 15469 kfree(pf->veb[i]); 15470 pf->veb[i] = NULL; 15471 } 15472 15473 kfree(pf->qp_pile); 15474 kfree(pf->vsi); 15475 15476 iounmap(hw->hw_addr); 15477 kfree(pf); 15478 pci_release_mem_regions(pdev); 15479 15480 pci_disable_pcie_error_reporting(pdev); 15481 pci_disable_device(pdev); 15482 } 15483 15484 /** 15485 * i40e_pci_error_detected - warning that something funky happened in PCI land 15486 * @pdev: PCI device information struct 15487 * @error: the type of PCI error 15488 * 15489 * Called to warn that something happened and the error handling steps 15490 * are in progress. Allows the driver to quiesce things, be ready for 15491 * remediation. 15492 **/ 15493 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 15494 enum pci_channel_state error) 15495 { 15496 struct i40e_pf *pf = pci_get_drvdata(pdev); 15497 15498 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 15499 15500 if (!pf) { 15501 dev_info(&pdev->dev, 15502 "Cannot recover - error happened during device probe\n"); 15503 return PCI_ERS_RESULT_DISCONNECT; 15504 } 15505 15506 /* shutdown all operations */ 15507 if (!test_bit(__I40E_SUSPENDED, pf->state)) 15508 i40e_prep_for_reset(pf, false); 15509 15510 /* Request a slot reset */ 15511 return PCI_ERS_RESULT_NEED_RESET; 15512 } 15513 15514 /** 15515 * i40e_pci_error_slot_reset - a PCI slot reset just happened 15516 * @pdev: PCI device information struct 15517 * 15518 * Called to find if the driver can work with the device now that 15519 * the pci slot has been reset. If a basic connection seems good 15520 * (registers are readable and have sane content) then return a 15521 * happy little PCI_ERS_RESULT_xxx. 15522 **/ 15523 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 15524 { 15525 struct i40e_pf *pf = pci_get_drvdata(pdev); 15526 pci_ers_result_t result; 15527 u32 reg; 15528 15529 dev_dbg(&pdev->dev, "%s\n", __func__); 15530 if (pci_enable_device_mem(pdev)) { 15531 dev_info(&pdev->dev, 15532 "Cannot re-enable PCI device after reset.\n"); 15533 result = PCI_ERS_RESULT_DISCONNECT; 15534 } else { 15535 pci_set_master(pdev); 15536 pci_restore_state(pdev); 15537 pci_save_state(pdev); 15538 pci_wake_from_d3(pdev, false); 15539 15540 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 15541 if (reg == 0) 15542 result = PCI_ERS_RESULT_RECOVERED; 15543 else 15544 result = PCI_ERS_RESULT_DISCONNECT; 15545 } 15546 15547 return result; 15548 } 15549 15550 /** 15551 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 15552 * @pdev: PCI device information struct 15553 */ 15554 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 15555 { 15556 struct i40e_pf *pf = pci_get_drvdata(pdev); 15557 15558 i40e_prep_for_reset(pf, false); 15559 } 15560 15561 /** 15562 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 15563 * @pdev: PCI device information struct 15564 */ 15565 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 15566 { 15567 struct i40e_pf *pf = pci_get_drvdata(pdev); 15568 15569 i40e_reset_and_rebuild(pf, false, false); 15570 } 15571 15572 /** 15573 * i40e_pci_error_resume - restart operations after PCI error recovery 15574 * @pdev: PCI device information struct 15575 * 15576 * Called to allow the driver to bring things back up after PCI error 15577 * and/or reset recovery has finished. 15578 **/ 15579 static void i40e_pci_error_resume(struct pci_dev *pdev) 15580 { 15581 struct i40e_pf *pf = pci_get_drvdata(pdev); 15582 15583 dev_dbg(&pdev->dev, "%s\n", __func__); 15584 if (test_bit(__I40E_SUSPENDED, pf->state)) 15585 return; 15586 15587 i40e_handle_reset_warning(pf, false); 15588 } 15589 15590 /** 15591 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 15592 * using the mac_address_write admin q function 15593 * @pf: pointer to i40e_pf struct 15594 **/ 15595 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 15596 { 15597 struct i40e_hw *hw = &pf->hw; 15598 i40e_status ret; 15599 u8 mac_addr[6]; 15600 u16 flags = 0; 15601 15602 /* Get current MAC address in case it's an LAA */ 15603 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 15604 ether_addr_copy(mac_addr, 15605 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 15606 } else { 15607 dev_err(&pf->pdev->dev, 15608 "Failed to retrieve MAC address; using default\n"); 15609 ether_addr_copy(mac_addr, hw->mac.addr); 15610 } 15611 15612 /* The FW expects the mac address write cmd to first be called with 15613 * one of these flags before calling it again with the multicast 15614 * enable flags. 15615 */ 15616 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 15617 15618 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 15619 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 15620 15621 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 15622 if (ret) { 15623 dev_err(&pf->pdev->dev, 15624 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 15625 return; 15626 } 15627 15628 flags = I40E_AQC_MC_MAG_EN 15629 | I40E_AQC_WOL_PRESERVE_ON_PFR 15630 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 15631 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 15632 if (ret) 15633 dev_err(&pf->pdev->dev, 15634 "Failed to enable Multicast Magic Packet wake up\n"); 15635 } 15636 15637 /** 15638 * i40e_shutdown - PCI callback for shutting down 15639 * @pdev: PCI device information struct 15640 **/ 15641 static void i40e_shutdown(struct pci_dev *pdev) 15642 { 15643 struct i40e_pf *pf = pci_get_drvdata(pdev); 15644 struct i40e_hw *hw = &pf->hw; 15645 15646 set_bit(__I40E_SUSPENDED, pf->state); 15647 set_bit(__I40E_DOWN, pf->state); 15648 15649 del_timer_sync(&pf->service_timer); 15650 cancel_work_sync(&pf->service_task); 15651 i40e_cloud_filter_exit(pf); 15652 i40e_fdir_teardown(pf); 15653 15654 /* Client close must be called explicitly here because the timer 15655 * has been stopped. 15656 */ 15657 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15658 15659 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 15660 i40e_enable_mc_magic_wake(pf); 15661 15662 i40e_prep_for_reset(pf, false); 15663 15664 wr32(hw, I40E_PFPM_APM, 15665 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 15666 wr32(hw, I40E_PFPM_WUFC, 15667 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 15668 15669 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 15670 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 15671 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 15672 free_irq(pf->pdev->irq, pf); 15673 15674 /* Since we're going to destroy queues during the 15675 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 15676 * whole section 15677 */ 15678 rtnl_lock(); 15679 i40e_clear_interrupt_scheme(pf); 15680 rtnl_unlock(); 15681 15682 if (system_state == SYSTEM_POWER_OFF) { 15683 pci_wake_from_d3(pdev, pf->wol_en); 15684 pci_set_power_state(pdev, PCI_D3hot); 15685 } 15686 } 15687 15688 /** 15689 * i40e_suspend - PM callback for moving to D3 15690 * @dev: generic device information structure 15691 **/ 15692 static int __maybe_unused i40e_suspend(struct device *dev) 15693 { 15694 struct i40e_pf *pf = dev_get_drvdata(dev); 15695 struct i40e_hw *hw = &pf->hw; 15696 15697 /* If we're already suspended, then there is nothing to do */ 15698 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 15699 return 0; 15700 15701 set_bit(__I40E_DOWN, pf->state); 15702 15703 /* Ensure service task will not be running */ 15704 del_timer_sync(&pf->service_timer); 15705 cancel_work_sync(&pf->service_task); 15706 15707 /* Client close must be called explicitly here because the timer 15708 * has been stopped. 15709 */ 15710 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15711 15712 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 15713 i40e_enable_mc_magic_wake(pf); 15714 15715 /* Since we're going to destroy queues during the 15716 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 15717 * whole section 15718 */ 15719 rtnl_lock(); 15720 15721 i40e_prep_for_reset(pf, true); 15722 15723 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 15724 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 15725 15726 /* Clear the interrupt scheme and release our IRQs so that the system 15727 * can safely hibernate even when there are a large number of CPUs. 15728 * Otherwise hibernation might fail when mapping all the vectors back 15729 * to CPU0. 15730 */ 15731 i40e_clear_interrupt_scheme(pf); 15732 15733 rtnl_unlock(); 15734 15735 return 0; 15736 } 15737 15738 /** 15739 * i40e_resume - PM callback for waking up from D3 15740 * @dev: generic device information structure 15741 **/ 15742 static int __maybe_unused i40e_resume(struct device *dev) 15743 { 15744 struct i40e_pf *pf = dev_get_drvdata(dev); 15745 int err; 15746 15747 /* If we're not suspended, then there is nothing to do */ 15748 if (!test_bit(__I40E_SUSPENDED, pf->state)) 15749 return 0; 15750 15751 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 15752 * since we're going to be restoring queues 15753 */ 15754 rtnl_lock(); 15755 15756 /* We cleared the interrupt scheme when we suspended, so we need to 15757 * restore it now to resume device functionality. 15758 */ 15759 err = i40e_restore_interrupt_scheme(pf); 15760 if (err) { 15761 dev_err(dev, "Cannot restore interrupt scheme: %d\n", 15762 err); 15763 } 15764 15765 clear_bit(__I40E_DOWN, pf->state); 15766 i40e_reset_and_rebuild(pf, false, true); 15767 15768 rtnl_unlock(); 15769 15770 /* Clear suspended state last after everything is recovered */ 15771 clear_bit(__I40E_SUSPENDED, pf->state); 15772 15773 /* Restart the service task */ 15774 mod_timer(&pf->service_timer, 15775 round_jiffies(jiffies + pf->service_timer_period)); 15776 15777 return 0; 15778 } 15779 15780 static const struct pci_error_handlers i40e_err_handler = { 15781 .error_detected = i40e_pci_error_detected, 15782 .slot_reset = i40e_pci_error_slot_reset, 15783 .reset_prepare = i40e_pci_error_reset_prepare, 15784 .reset_done = i40e_pci_error_reset_done, 15785 .resume = i40e_pci_error_resume, 15786 }; 15787 15788 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 15789 15790 static struct pci_driver i40e_driver = { 15791 .name = i40e_driver_name, 15792 .id_table = i40e_pci_tbl, 15793 .probe = i40e_probe, 15794 .remove = i40e_remove, 15795 .driver = { 15796 .pm = &i40e_pm_ops, 15797 }, 15798 .shutdown = i40e_shutdown, 15799 .err_handler = &i40e_err_handler, 15800 .sriov_configure = i40e_pci_sriov_configure, 15801 }; 15802 15803 /** 15804 * i40e_init_module - Driver registration routine 15805 * 15806 * i40e_init_module is the first routine called when the driver is 15807 * loaded. All it does is register with the PCI subsystem. 15808 **/ 15809 static int __init i40e_init_module(void) 15810 { 15811 pr_info("%s: %s\n", i40e_driver_name, i40e_driver_string); 15812 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 15813 15814 /* There is no need to throttle the number of active tasks because 15815 * each device limits its own task using a state bit for scheduling 15816 * the service task, and the device tasks do not interfere with each 15817 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 15818 * since we need to be able to guarantee forward progress even under 15819 * memory pressure. 15820 */ 15821 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 15822 if (!i40e_wq) { 15823 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 15824 return -ENOMEM; 15825 } 15826 15827 i40e_dbg_init(); 15828 return pci_register_driver(&i40e_driver); 15829 } 15830 module_init(i40e_init_module); 15831 15832 /** 15833 * i40e_exit_module - Driver exit cleanup routine 15834 * 15835 * i40e_exit_module is called just before the driver is removed 15836 * from memory. 15837 **/ 15838 static void __exit i40e_exit_module(void) 15839 { 15840 pci_unregister_driver(&i40e_driver); 15841 destroy_workqueue(i40e_wq); 15842 i40e_dbg_exit(); 15843 } 15844 module_exit(i40e_exit_module); 15845