1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/of_net.h> 6 #include <linux/pci.h> 7 #include <linux/bpf.h> 8 9 /* Local includes */ 10 #include "i40e.h" 11 #include "i40e_diag.h" 12 #include "i40e_xsk.h" 13 #include <net/udp_tunnel.h> 14 #include <net/xdp_sock_drv.h> 15 /* All i40e tracepoints are defined by the include below, which 16 * must be included exactly once across the whole kernel with 17 * CREATE_TRACE_POINTS defined 18 */ 19 #define CREATE_TRACE_POINTS 20 #include "i40e_trace.h" 21 22 const char i40e_driver_name[] = "i40e"; 23 static const char i40e_driver_string[] = 24 "Intel(R) Ethernet Connection XL710 Network Driver"; 25 26 #define DRV_KERN "-k" 27 28 #define DRV_VERSION_MAJOR 2 29 #define DRV_VERSION_MINOR 8 30 #define DRV_VERSION_BUILD 20 31 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 32 __stringify(DRV_VERSION_MINOR) "." \ 33 __stringify(DRV_VERSION_BUILD) DRV_KERN 34 const char i40e_driver_version_str[] = DRV_VERSION; 35 static const char i40e_copyright[] = "Copyright (c) 2013 - 2019 Intel Corporation."; 36 37 /* a bit of forward declarations */ 38 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 39 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 40 static int i40e_add_vsi(struct i40e_vsi *vsi); 41 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 42 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 43 static int i40e_setup_misc_vector(struct i40e_pf *pf); 44 static void i40e_determine_queue_usage(struct i40e_pf *pf); 45 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 46 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 47 static int i40e_reset(struct i40e_pf *pf); 48 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 49 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf); 50 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf); 51 static bool i40e_check_recovery_mode(struct i40e_pf *pf); 52 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw); 53 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 54 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 55 static int i40e_get_capabilities(struct i40e_pf *pf, 56 enum i40e_admin_queue_opc list_type); 57 58 59 /* i40e_pci_tbl - PCI Device ID Table 60 * 61 * Last entry must be all 0s 62 * 63 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 64 * Class, Class Mask, private data (not used) } 65 */ 66 static const struct pci_device_id i40e_pci_tbl[] = { 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_BC), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_SFP), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_B), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_X710_N3000), 0}, 88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_XXV710_N3000), 0}, 89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 91 /* required last entry */ 92 {0, } 93 }; 94 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 95 96 #define I40E_MAX_VF_COUNT 128 97 static int debug = -1; 98 module_param(debug, uint, 0); 99 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 100 101 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 102 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 103 MODULE_LICENSE("GPL v2"); 104 MODULE_VERSION(DRV_VERSION); 105 106 static struct workqueue_struct *i40e_wq; 107 108 /** 109 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 110 * @hw: pointer to the HW structure 111 * @mem: ptr to mem struct to fill out 112 * @size: size of memory requested 113 * @alignment: what to align the allocation to 114 **/ 115 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 116 u64 size, u32 alignment) 117 { 118 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 119 120 mem->size = ALIGN(size, alignment); 121 mem->va = dma_alloc_coherent(&pf->pdev->dev, mem->size, &mem->pa, 122 GFP_KERNEL); 123 if (!mem->va) 124 return -ENOMEM; 125 126 return 0; 127 } 128 129 /** 130 * i40e_free_dma_mem_d - OS specific memory free for shared code 131 * @hw: pointer to the HW structure 132 * @mem: ptr to mem struct to free 133 **/ 134 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 135 { 136 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 137 138 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 139 mem->va = NULL; 140 mem->pa = 0; 141 mem->size = 0; 142 143 return 0; 144 } 145 146 /** 147 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 148 * @hw: pointer to the HW structure 149 * @mem: ptr to mem struct to fill out 150 * @size: size of memory requested 151 **/ 152 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 153 u32 size) 154 { 155 mem->size = size; 156 mem->va = kzalloc(size, GFP_KERNEL); 157 158 if (!mem->va) 159 return -ENOMEM; 160 161 return 0; 162 } 163 164 /** 165 * i40e_free_virt_mem_d - OS specific memory free for shared code 166 * @hw: pointer to the HW structure 167 * @mem: ptr to mem struct to free 168 **/ 169 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 170 { 171 /* it's ok to kfree a NULL pointer */ 172 kfree(mem->va); 173 mem->va = NULL; 174 mem->size = 0; 175 176 return 0; 177 } 178 179 /** 180 * i40e_get_lump - find a lump of free generic resource 181 * @pf: board private structure 182 * @pile: the pile of resource to search 183 * @needed: the number of items needed 184 * @id: an owner id to stick on the items assigned 185 * 186 * Returns the base item index of the lump, or negative for error 187 * 188 * The search_hint trick and lack of advanced fit-finding only work 189 * because we're highly likely to have all the same size lump requests. 190 * Linear search time and any fragmentation should be minimal. 191 **/ 192 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 193 u16 needed, u16 id) 194 { 195 int ret = -ENOMEM; 196 int i, j; 197 198 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 199 dev_info(&pf->pdev->dev, 200 "param err: pile=%s needed=%d id=0x%04x\n", 201 pile ? "<valid>" : "<null>", needed, id); 202 return -EINVAL; 203 } 204 205 /* start the linear search with an imperfect hint */ 206 i = pile->search_hint; 207 while (i < pile->num_entries) { 208 /* skip already allocated entries */ 209 if (pile->list[i] & I40E_PILE_VALID_BIT) { 210 i++; 211 continue; 212 } 213 214 /* do we have enough in this lump? */ 215 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 216 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 217 break; 218 } 219 220 if (j == needed) { 221 /* there was enough, so assign it to the requestor */ 222 for (j = 0; j < needed; j++) 223 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 224 ret = i; 225 pile->search_hint = i + j; 226 break; 227 } 228 229 /* not enough, so skip over it and continue looking */ 230 i += j; 231 } 232 233 return ret; 234 } 235 236 /** 237 * i40e_put_lump - return a lump of generic resource 238 * @pile: the pile of resource to search 239 * @index: the base item index 240 * @id: the owner id of the items assigned 241 * 242 * Returns the count of items in the lump 243 **/ 244 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 245 { 246 int valid_id = (id | I40E_PILE_VALID_BIT); 247 int count = 0; 248 int i; 249 250 if (!pile || index >= pile->num_entries) 251 return -EINVAL; 252 253 for (i = index; 254 i < pile->num_entries && pile->list[i] == valid_id; 255 i++) { 256 pile->list[i] = 0; 257 count++; 258 } 259 260 if (count && index < pile->search_hint) 261 pile->search_hint = index; 262 263 return count; 264 } 265 266 /** 267 * i40e_find_vsi_from_id - searches for the vsi with the given id 268 * @pf: the pf structure to search for the vsi 269 * @id: id of the vsi it is searching for 270 **/ 271 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 272 { 273 int i; 274 275 for (i = 0; i < pf->num_alloc_vsi; i++) 276 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 277 return pf->vsi[i]; 278 279 return NULL; 280 } 281 282 /** 283 * i40e_service_event_schedule - Schedule the service task to wake up 284 * @pf: board private structure 285 * 286 * If not already scheduled, this puts the task into the work queue 287 **/ 288 void i40e_service_event_schedule(struct i40e_pf *pf) 289 { 290 if ((!test_bit(__I40E_DOWN, pf->state) && 291 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) || 292 test_bit(__I40E_RECOVERY_MODE, pf->state)) 293 queue_work(i40e_wq, &pf->service_task); 294 } 295 296 /** 297 * i40e_tx_timeout - Respond to a Tx Hang 298 * @netdev: network interface device structure 299 * 300 * If any port has noticed a Tx timeout, it is likely that the whole 301 * device is munged, not just the one netdev port, so go for the full 302 * reset. 303 **/ 304 static void i40e_tx_timeout(struct net_device *netdev, unsigned int txqueue) 305 { 306 struct i40e_netdev_priv *np = netdev_priv(netdev); 307 struct i40e_vsi *vsi = np->vsi; 308 struct i40e_pf *pf = vsi->back; 309 struct i40e_ring *tx_ring = NULL; 310 unsigned int i; 311 u32 head, val; 312 313 pf->tx_timeout_count++; 314 315 /* with txqueue index, find the tx_ring struct */ 316 for (i = 0; i < vsi->num_queue_pairs; i++) { 317 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 318 if (txqueue == 319 vsi->tx_rings[i]->queue_index) { 320 tx_ring = vsi->tx_rings[i]; 321 break; 322 } 323 } 324 } 325 326 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 327 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 328 else if (time_before(jiffies, 329 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 330 return; /* don't do any new action before the next timeout */ 331 332 /* don't kick off another recovery if one is already pending */ 333 if (test_and_set_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state)) 334 return; 335 336 if (tx_ring) { 337 head = i40e_get_head(tx_ring); 338 /* Read interrupt register */ 339 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 340 val = rd32(&pf->hw, 341 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 342 tx_ring->vsi->base_vector - 1)); 343 else 344 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 345 346 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 347 vsi->seid, txqueue, tx_ring->next_to_clean, 348 head, tx_ring->next_to_use, 349 readl(tx_ring->tail), val); 350 } 351 352 pf->tx_timeout_last_recovery = jiffies; 353 netdev_info(netdev, "tx_timeout recovery level %d, txqueue %d\n", 354 pf->tx_timeout_recovery_level, txqueue); 355 356 switch (pf->tx_timeout_recovery_level) { 357 case 1: 358 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 359 break; 360 case 2: 361 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 362 break; 363 case 3: 364 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 365 break; 366 default: 367 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 368 break; 369 } 370 371 i40e_service_event_schedule(pf); 372 pf->tx_timeout_recovery_level++; 373 } 374 375 /** 376 * i40e_get_vsi_stats_struct - Get System Network Statistics 377 * @vsi: the VSI we care about 378 * 379 * Returns the address of the device statistics structure. 380 * The statistics are actually updated from the service task. 381 **/ 382 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 383 { 384 return &vsi->net_stats; 385 } 386 387 /** 388 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 389 * @ring: Tx ring to get statistics from 390 * @stats: statistics entry to be updated 391 **/ 392 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 393 struct rtnl_link_stats64 *stats) 394 { 395 u64 bytes, packets; 396 unsigned int start; 397 398 do { 399 start = u64_stats_fetch_begin_irq(&ring->syncp); 400 packets = ring->stats.packets; 401 bytes = ring->stats.bytes; 402 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 403 404 stats->tx_packets += packets; 405 stats->tx_bytes += bytes; 406 } 407 408 /** 409 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 410 * @netdev: network interface device structure 411 * @stats: data structure to store statistics 412 * 413 * Returns the address of the device statistics structure. 414 * The statistics are actually updated from the service task. 415 **/ 416 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 417 struct rtnl_link_stats64 *stats) 418 { 419 struct i40e_netdev_priv *np = netdev_priv(netdev); 420 struct i40e_vsi *vsi = np->vsi; 421 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 422 struct i40e_ring *ring; 423 int i; 424 425 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 426 return; 427 428 if (!vsi->tx_rings) 429 return; 430 431 rcu_read_lock(); 432 for (i = 0; i < vsi->num_queue_pairs; i++) { 433 u64 bytes, packets; 434 unsigned int start; 435 436 ring = READ_ONCE(vsi->tx_rings[i]); 437 if (!ring) 438 continue; 439 i40e_get_netdev_stats_struct_tx(ring, stats); 440 441 if (i40e_enabled_xdp_vsi(vsi)) { 442 ring++; 443 i40e_get_netdev_stats_struct_tx(ring, stats); 444 } 445 446 ring++; 447 do { 448 start = u64_stats_fetch_begin_irq(&ring->syncp); 449 packets = ring->stats.packets; 450 bytes = ring->stats.bytes; 451 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 452 453 stats->rx_packets += packets; 454 stats->rx_bytes += bytes; 455 456 } 457 rcu_read_unlock(); 458 459 /* following stats updated by i40e_watchdog_subtask() */ 460 stats->multicast = vsi_stats->multicast; 461 stats->tx_errors = vsi_stats->tx_errors; 462 stats->tx_dropped = vsi_stats->tx_dropped; 463 stats->rx_errors = vsi_stats->rx_errors; 464 stats->rx_dropped = vsi_stats->rx_dropped; 465 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 466 stats->rx_length_errors = vsi_stats->rx_length_errors; 467 } 468 469 /** 470 * i40e_vsi_reset_stats - Resets all stats of the given vsi 471 * @vsi: the VSI to have its stats reset 472 **/ 473 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 474 { 475 struct rtnl_link_stats64 *ns; 476 int i; 477 478 if (!vsi) 479 return; 480 481 ns = i40e_get_vsi_stats_struct(vsi); 482 memset(ns, 0, sizeof(*ns)); 483 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 484 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 485 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 486 if (vsi->rx_rings && vsi->rx_rings[0]) { 487 for (i = 0; i < vsi->num_queue_pairs; i++) { 488 memset(&vsi->rx_rings[i]->stats, 0, 489 sizeof(vsi->rx_rings[i]->stats)); 490 memset(&vsi->rx_rings[i]->rx_stats, 0, 491 sizeof(vsi->rx_rings[i]->rx_stats)); 492 memset(&vsi->tx_rings[i]->stats, 0, 493 sizeof(vsi->tx_rings[i]->stats)); 494 memset(&vsi->tx_rings[i]->tx_stats, 0, 495 sizeof(vsi->tx_rings[i]->tx_stats)); 496 } 497 } 498 vsi->stat_offsets_loaded = false; 499 } 500 501 /** 502 * i40e_pf_reset_stats - Reset all of the stats for the given PF 503 * @pf: the PF to be reset 504 **/ 505 void i40e_pf_reset_stats(struct i40e_pf *pf) 506 { 507 int i; 508 509 memset(&pf->stats, 0, sizeof(pf->stats)); 510 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 511 pf->stat_offsets_loaded = false; 512 513 for (i = 0; i < I40E_MAX_VEB; i++) { 514 if (pf->veb[i]) { 515 memset(&pf->veb[i]->stats, 0, 516 sizeof(pf->veb[i]->stats)); 517 memset(&pf->veb[i]->stats_offsets, 0, 518 sizeof(pf->veb[i]->stats_offsets)); 519 memset(&pf->veb[i]->tc_stats, 0, 520 sizeof(pf->veb[i]->tc_stats)); 521 memset(&pf->veb[i]->tc_stats_offsets, 0, 522 sizeof(pf->veb[i]->tc_stats_offsets)); 523 pf->veb[i]->stat_offsets_loaded = false; 524 } 525 } 526 pf->hw_csum_rx_error = 0; 527 } 528 529 /** 530 * i40e_stat_update48 - read and update a 48 bit stat from the chip 531 * @hw: ptr to the hardware info 532 * @hireg: the high 32 bit reg to read 533 * @loreg: the low 32 bit reg to read 534 * @offset_loaded: has the initial offset been loaded yet 535 * @offset: ptr to current offset value 536 * @stat: ptr to the stat 537 * 538 * Since the device stats are not reset at PFReset, they likely will not 539 * be zeroed when the driver starts. We'll save the first values read 540 * and use them as offsets to be subtracted from the raw values in order 541 * to report stats that count from zero. In the process, we also manage 542 * the potential roll-over. 543 **/ 544 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 545 bool offset_loaded, u64 *offset, u64 *stat) 546 { 547 u64 new_data; 548 549 if (hw->device_id == I40E_DEV_ID_QEMU) { 550 new_data = rd32(hw, loreg); 551 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 552 } else { 553 new_data = rd64(hw, loreg); 554 } 555 if (!offset_loaded) 556 *offset = new_data; 557 if (likely(new_data >= *offset)) 558 *stat = new_data - *offset; 559 else 560 *stat = (new_data + BIT_ULL(48)) - *offset; 561 *stat &= 0xFFFFFFFFFFFFULL; 562 } 563 564 /** 565 * i40e_stat_update32 - read and update a 32 bit stat from the chip 566 * @hw: ptr to the hardware info 567 * @reg: the hw reg to read 568 * @offset_loaded: has the initial offset been loaded yet 569 * @offset: ptr to current offset value 570 * @stat: ptr to the stat 571 **/ 572 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 573 bool offset_loaded, u64 *offset, u64 *stat) 574 { 575 u32 new_data; 576 577 new_data = rd32(hw, reg); 578 if (!offset_loaded) 579 *offset = new_data; 580 if (likely(new_data >= *offset)) 581 *stat = (u32)(new_data - *offset); 582 else 583 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 584 } 585 586 /** 587 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 588 * @hw: ptr to the hardware info 589 * @reg: the hw reg to read and clear 590 * @stat: ptr to the stat 591 **/ 592 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 593 { 594 u32 new_data = rd32(hw, reg); 595 596 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 597 *stat += new_data; 598 } 599 600 /** 601 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 602 * @vsi: the VSI to be updated 603 **/ 604 void i40e_update_eth_stats(struct i40e_vsi *vsi) 605 { 606 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 607 struct i40e_pf *pf = vsi->back; 608 struct i40e_hw *hw = &pf->hw; 609 struct i40e_eth_stats *oes; 610 struct i40e_eth_stats *es; /* device's eth stats */ 611 612 es = &vsi->eth_stats; 613 oes = &vsi->eth_stats_offsets; 614 615 /* Gather up the stats that the hw collects */ 616 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 617 vsi->stat_offsets_loaded, 618 &oes->tx_errors, &es->tx_errors); 619 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 620 vsi->stat_offsets_loaded, 621 &oes->rx_discards, &es->rx_discards); 622 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 623 vsi->stat_offsets_loaded, 624 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 625 626 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 627 I40E_GLV_GORCL(stat_idx), 628 vsi->stat_offsets_loaded, 629 &oes->rx_bytes, &es->rx_bytes); 630 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 631 I40E_GLV_UPRCL(stat_idx), 632 vsi->stat_offsets_loaded, 633 &oes->rx_unicast, &es->rx_unicast); 634 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 635 I40E_GLV_MPRCL(stat_idx), 636 vsi->stat_offsets_loaded, 637 &oes->rx_multicast, &es->rx_multicast); 638 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 639 I40E_GLV_BPRCL(stat_idx), 640 vsi->stat_offsets_loaded, 641 &oes->rx_broadcast, &es->rx_broadcast); 642 643 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 644 I40E_GLV_GOTCL(stat_idx), 645 vsi->stat_offsets_loaded, 646 &oes->tx_bytes, &es->tx_bytes); 647 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 648 I40E_GLV_UPTCL(stat_idx), 649 vsi->stat_offsets_loaded, 650 &oes->tx_unicast, &es->tx_unicast); 651 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 652 I40E_GLV_MPTCL(stat_idx), 653 vsi->stat_offsets_loaded, 654 &oes->tx_multicast, &es->tx_multicast); 655 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 656 I40E_GLV_BPTCL(stat_idx), 657 vsi->stat_offsets_loaded, 658 &oes->tx_broadcast, &es->tx_broadcast); 659 vsi->stat_offsets_loaded = true; 660 } 661 662 /** 663 * i40e_update_veb_stats - Update Switch component statistics 664 * @veb: the VEB being updated 665 **/ 666 void i40e_update_veb_stats(struct i40e_veb *veb) 667 { 668 struct i40e_pf *pf = veb->pf; 669 struct i40e_hw *hw = &pf->hw; 670 struct i40e_eth_stats *oes; 671 struct i40e_eth_stats *es; /* device's eth stats */ 672 struct i40e_veb_tc_stats *veb_oes; 673 struct i40e_veb_tc_stats *veb_es; 674 int i, idx = 0; 675 676 idx = veb->stats_idx; 677 es = &veb->stats; 678 oes = &veb->stats_offsets; 679 veb_es = &veb->tc_stats; 680 veb_oes = &veb->tc_stats_offsets; 681 682 /* Gather up the stats that the hw collects */ 683 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 684 veb->stat_offsets_loaded, 685 &oes->tx_discards, &es->tx_discards); 686 if (hw->revision_id > 0) 687 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 688 veb->stat_offsets_loaded, 689 &oes->rx_unknown_protocol, 690 &es->rx_unknown_protocol); 691 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 692 veb->stat_offsets_loaded, 693 &oes->rx_bytes, &es->rx_bytes); 694 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 695 veb->stat_offsets_loaded, 696 &oes->rx_unicast, &es->rx_unicast); 697 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 698 veb->stat_offsets_loaded, 699 &oes->rx_multicast, &es->rx_multicast); 700 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 701 veb->stat_offsets_loaded, 702 &oes->rx_broadcast, &es->rx_broadcast); 703 704 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 705 veb->stat_offsets_loaded, 706 &oes->tx_bytes, &es->tx_bytes); 707 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 708 veb->stat_offsets_loaded, 709 &oes->tx_unicast, &es->tx_unicast); 710 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 711 veb->stat_offsets_loaded, 712 &oes->tx_multicast, &es->tx_multicast); 713 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 714 veb->stat_offsets_loaded, 715 &oes->tx_broadcast, &es->tx_broadcast); 716 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 717 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 718 I40E_GLVEBTC_RPCL(i, idx), 719 veb->stat_offsets_loaded, 720 &veb_oes->tc_rx_packets[i], 721 &veb_es->tc_rx_packets[i]); 722 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 723 I40E_GLVEBTC_RBCL(i, idx), 724 veb->stat_offsets_loaded, 725 &veb_oes->tc_rx_bytes[i], 726 &veb_es->tc_rx_bytes[i]); 727 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 728 I40E_GLVEBTC_TPCL(i, idx), 729 veb->stat_offsets_loaded, 730 &veb_oes->tc_tx_packets[i], 731 &veb_es->tc_tx_packets[i]); 732 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 733 I40E_GLVEBTC_TBCL(i, idx), 734 veb->stat_offsets_loaded, 735 &veb_oes->tc_tx_bytes[i], 736 &veb_es->tc_tx_bytes[i]); 737 } 738 veb->stat_offsets_loaded = true; 739 } 740 741 /** 742 * i40e_update_vsi_stats - Update the vsi statistics counters. 743 * @vsi: the VSI to be updated 744 * 745 * There are a few instances where we store the same stat in a 746 * couple of different structs. This is partly because we have 747 * the netdev stats that need to be filled out, which is slightly 748 * different from the "eth_stats" defined by the chip and used in 749 * VF communications. We sort it out here. 750 **/ 751 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 752 { 753 struct i40e_pf *pf = vsi->back; 754 struct rtnl_link_stats64 *ons; 755 struct rtnl_link_stats64 *ns; /* netdev stats */ 756 struct i40e_eth_stats *oes; 757 struct i40e_eth_stats *es; /* device's eth stats */ 758 u32 tx_restart, tx_busy; 759 struct i40e_ring *p; 760 u32 rx_page, rx_buf; 761 u64 bytes, packets; 762 unsigned int start; 763 u64 tx_linearize; 764 u64 tx_force_wb; 765 u64 rx_p, rx_b; 766 u64 tx_p, tx_b; 767 u16 q; 768 769 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 770 test_bit(__I40E_CONFIG_BUSY, pf->state)) 771 return; 772 773 ns = i40e_get_vsi_stats_struct(vsi); 774 ons = &vsi->net_stats_offsets; 775 es = &vsi->eth_stats; 776 oes = &vsi->eth_stats_offsets; 777 778 /* Gather up the netdev and vsi stats that the driver collects 779 * on the fly during packet processing 780 */ 781 rx_b = rx_p = 0; 782 tx_b = tx_p = 0; 783 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 784 rx_page = 0; 785 rx_buf = 0; 786 rcu_read_lock(); 787 for (q = 0; q < vsi->num_queue_pairs; q++) { 788 /* locate Tx ring */ 789 p = READ_ONCE(vsi->tx_rings[q]); 790 791 do { 792 start = u64_stats_fetch_begin_irq(&p->syncp); 793 packets = p->stats.packets; 794 bytes = p->stats.bytes; 795 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 796 tx_b += bytes; 797 tx_p += packets; 798 tx_restart += p->tx_stats.restart_queue; 799 tx_busy += p->tx_stats.tx_busy; 800 tx_linearize += p->tx_stats.tx_linearize; 801 tx_force_wb += p->tx_stats.tx_force_wb; 802 803 /* Rx queue is part of the same block as Tx queue */ 804 p = &p[1]; 805 do { 806 start = u64_stats_fetch_begin_irq(&p->syncp); 807 packets = p->stats.packets; 808 bytes = p->stats.bytes; 809 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 810 rx_b += bytes; 811 rx_p += packets; 812 rx_buf += p->rx_stats.alloc_buff_failed; 813 rx_page += p->rx_stats.alloc_page_failed; 814 } 815 rcu_read_unlock(); 816 vsi->tx_restart = tx_restart; 817 vsi->tx_busy = tx_busy; 818 vsi->tx_linearize = tx_linearize; 819 vsi->tx_force_wb = tx_force_wb; 820 vsi->rx_page_failed = rx_page; 821 vsi->rx_buf_failed = rx_buf; 822 823 ns->rx_packets = rx_p; 824 ns->rx_bytes = rx_b; 825 ns->tx_packets = tx_p; 826 ns->tx_bytes = tx_b; 827 828 /* update netdev stats from eth stats */ 829 i40e_update_eth_stats(vsi); 830 ons->tx_errors = oes->tx_errors; 831 ns->tx_errors = es->tx_errors; 832 ons->multicast = oes->rx_multicast; 833 ns->multicast = es->rx_multicast; 834 ons->rx_dropped = oes->rx_discards; 835 ns->rx_dropped = es->rx_discards; 836 ons->tx_dropped = oes->tx_discards; 837 ns->tx_dropped = es->tx_discards; 838 839 /* pull in a couple PF stats if this is the main vsi */ 840 if (vsi == pf->vsi[pf->lan_vsi]) { 841 ns->rx_crc_errors = pf->stats.crc_errors; 842 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 843 ns->rx_length_errors = pf->stats.rx_length_errors; 844 } 845 } 846 847 /** 848 * i40e_update_pf_stats - Update the PF statistics counters. 849 * @pf: the PF to be updated 850 **/ 851 static void i40e_update_pf_stats(struct i40e_pf *pf) 852 { 853 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 854 struct i40e_hw_port_stats *nsd = &pf->stats; 855 struct i40e_hw *hw = &pf->hw; 856 u32 val; 857 int i; 858 859 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 860 I40E_GLPRT_GORCL(hw->port), 861 pf->stat_offsets_loaded, 862 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 863 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 864 I40E_GLPRT_GOTCL(hw->port), 865 pf->stat_offsets_loaded, 866 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 867 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 868 pf->stat_offsets_loaded, 869 &osd->eth.rx_discards, 870 &nsd->eth.rx_discards); 871 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 872 I40E_GLPRT_UPRCL(hw->port), 873 pf->stat_offsets_loaded, 874 &osd->eth.rx_unicast, 875 &nsd->eth.rx_unicast); 876 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 877 I40E_GLPRT_MPRCL(hw->port), 878 pf->stat_offsets_loaded, 879 &osd->eth.rx_multicast, 880 &nsd->eth.rx_multicast); 881 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 882 I40E_GLPRT_BPRCL(hw->port), 883 pf->stat_offsets_loaded, 884 &osd->eth.rx_broadcast, 885 &nsd->eth.rx_broadcast); 886 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 887 I40E_GLPRT_UPTCL(hw->port), 888 pf->stat_offsets_loaded, 889 &osd->eth.tx_unicast, 890 &nsd->eth.tx_unicast); 891 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 892 I40E_GLPRT_MPTCL(hw->port), 893 pf->stat_offsets_loaded, 894 &osd->eth.tx_multicast, 895 &nsd->eth.tx_multicast); 896 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 897 I40E_GLPRT_BPTCL(hw->port), 898 pf->stat_offsets_loaded, 899 &osd->eth.tx_broadcast, 900 &nsd->eth.tx_broadcast); 901 902 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 903 pf->stat_offsets_loaded, 904 &osd->tx_dropped_link_down, 905 &nsd->tx_dropped_link_down); 906 907 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 908 pf->stat_offsets_loaded, 909 &osd->crc_errors, &nsd->crc_errors); 910 911 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->illegal_bytes, &nsd->illegal_bytes); 914 915 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 916 pf->stat_offsets_loaded, 917 &osd->mac_local_faults, 918 &nsd->mac_local_faults); 919 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 920 pf->stat_offsets_loaded, 921 &osd->mac_remote_faults, 922 &nsd->mac_remote_faults); 923 924 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 925 pf->stat_offsets_loaded, 926 &osd->rx_length_errors, 927 &nsd->rx_length_errors); 928 929 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 930 pf->stat_offsets_loaded, 931 &osd->link_xon_rx, &nsd->link_xon_rx); 932 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 933 pf->stat_offsets_loaded, 934 &osd->link_xon_tx, &nsd->link_xon_tx); 935 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 936 pf->stat_offsets_loaded, 937 &osd->link_xoff_rx, &nsd->link_xoff_rx); 938 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 939 pf->stat_offsets_loaded, 940 &osd->link_xoff_tx, &nsd->link_xoff_tx); 941 942 for (i = 0; i < 8; i++) { 943 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 944 pf->stat_offsets_loaded, 945 &osd->priority_xoff_rx[i], 946 &nsd->priority_xoff_rx[i]); 947 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 948 pf->stat_offsets_loaded, 949 &osd->priority_xon_rx[i], 950 &nsd->priority_xon_rx[i]); 951 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 952 pf->stat_offsets_loaded, 953 &osd->priority_xon_tx[i], 954 &nsd->priority_xon_tx[i]); 955 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 956 pf->stat_offsets_loaded, 957 &osd->priority_xoff_tx[i], 958 &nsd->priority_xoff_tx[i]); 959 i40e_stat_update32(hw, 960 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 961 pf->stat_offsets_loaded, 962 &osd->priority_xon_2_xoff[i], 963 &nsd->priority_xon_2_xoff[i]); 964 } 965 966 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 967 I40E_GLPRT_PRC64L(hw->port), 968 pf->stat_offsets_loaded, 969 &osd->rx_size_64, &nsd->rx_size_64); 970 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 971 I40E_GLPRT_PRC127L(hw->port), 972 pf->stat_offsets_loaded, 973 &osd->rx_size_127, &nsd->rx_size_127); 974 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 975 I40E_GLPRT_PRC255L(hw->port), 976 pf->stat_offsets_loaded, 977 &osd->rx_size_255, &nsd->rx_size_255); 978 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 979 I40E_GLPRT_PRC511L(hw->port), 980 pf->stat_offsets_loaded, 981 &osd->rx_size_511, &nsd->rx_size_511); 982 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 983 I40E_GLPRT_PRC1023L(hw->port), 984 pf->stat_offsets_loaded, 985 &osd->rx_size_1023, &nsd->rx_size_1023); 986 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 987 I40E_GLPRT_PRC1522L(hw->port), 988 pf->stat_offsets_loaded, 989 &osd->rx_size_1522, &nsd->rx_size_1522); 990 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 991 I40E_GLPRT_PRC9522L(hw->port), 992 pf->stat_offsets_loaded, 993 &osd->rx_size_big, &nsd->rx_size_big); 994 995 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 996 I40E_GLPRT_PTC64L(hw->port), 997 pf->stat_offsets_loaded, 998 &osd->tx_size_64, &nsd->tx_size_64); 999 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1000 I40E_GLPRT_PTC127L(hw->port), 1001 pf->stat_offsets_loaded, 1002 &osd->tx_size_127, &nsd->tx_size_127); 1003 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1004 I40E_GLPRT_PTC255L(hw->port), 1005 pf->stat_offsets_loaded, 1006 &osd->tx_size_255, &nsd->tx_size_255); 1007 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1008 I40E_GLPRT_PTC511L(hw->port), 1009 pf->stat_offsets_loaded, 1010 &osd->tx_size_511, &nsd->tx_size_511); 1011 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1012 I40E_GLPRT_PTC1023L(hw->port), 1013 pf->stat_offsets_loaded, 1014 &osd->tx_size_1023, &nsd->tx_size_1023); 1015 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1016 I40E_GLPRT_PTC1522L(hw->port), 1017 pf->stat_offsets_loaded, 1018 &osd->tx_size_1522, &nsd->tx_size_1522); 1019 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1020 I40E_GLPRT_PTC9522L(hw->port), 1021 pf->stat_offsets_loaded, 1022 &osd->tx_size_big, &nsd->tx_size_big); 1023 1024 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1025 pf->stat_offsets_loaded, 1026 &osd->rx_undersize, &nsd->rx_undersize); 1027 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->rx_fragments, &nsd->rx_fragments); 1030 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1031 pf->stat_offsets_loaded, 1032 &osd->rx_oversize, &nsd->rx_oversize); 1033 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1034 pf->stat_offsets_loaded, 1035 &osd->rx_jabber, &nsd->rx_jabber); 1036 1037 /* FDIR stats */ 1038 i40e_stat_update_and_clear32(hw, 1039 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1040 &nsd->fd_atr_match); 1041 i40e_stat_update_and_clear32(hw, 1042 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1043 &nsd->fd_sb_match); 1044 i40e_stat_update_and_clear32(hw, 1045 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1046 &nsd->fd_atr_tunnel_match); 1047 1048 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1049 nsd->tx_lpi_status = 1050 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1051 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1052 nsd->rx_lpi_status = 1053 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1054 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1055 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1056 pf->stat_offsets_loaded, 1057 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1058 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1059 pf->stat_offsets_loaded, 1060 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1061 1062 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1063 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1064 nsd->fd_sb_status = true; 1065 else 1066 nsd->fd_sb_status = false; 1067 1068 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1069 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1070 nsd->fd_atr_status = true; 1071 else 1072 nsd->fd_atr_status = false; 1073 1074 pf->stat_offsets_loaded = true; 1075 } 1076 1077 /** 1078 * i40e_update_stats - Update the various statistics counters. 1079 * @vsi: the VSI to be updated 1080 * 1081 * Update the various stats for this VSI and its related entities. 1082 **/ 1083 void i40e_update_stats(struct i40e_vsi *vsi) 1084 { 1085 struct i40e_pf *pf = vsi->back; 1086 1087 if (vsi == pf->vsi[pf->lan_vsi]) 1088 i40e_update_pf_stats(pf); 1089 1090 i40e_update_vsi_stats(vsi); 1091 } 1092 1093 /** 1094 * i40e_count_filters - counts VSI mac filters 1095 * @vsi: the VSI to be searched 1096 * 1097 * Returns count of mac filters 1098 **/ 1099 int i40e_count_filters(struct i40e_vsi *vsi) 1100 { 1101 struct i40e_mac_filter *f; 1102 struct hlist_node *h; 1103 int bkt; 1104 int cnt = 0; 1105 1106 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 1107 ++cnt; 1108 1109 return cnt; 1110 } 1111 1112 /** 1113 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1114 * @vsi: the VSI to be searched 1115 * @macaddr: the MAC address 1116 * @vlan: the vlan 1117 * 1118 * Returns ptr to the filter object or NULL 1119 **/ 1120 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1121 const u8 *macaddr, s16 vlan) 1122 { 1123 struct i40e_mac_filter *f; 1124 u64 key; 1125 1126 if (!vsi || !macaddr) 1127 return NULL; 1128 1129 key = i40e_addr_to_hkey(macaddr); 1130 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1131 if ((ether_addr_equal(macaddr, f->macaddr)) && 1132 (vlan == f->vlan)) 1133 return f; 1134 } 1135 return NULL; 1136 } 1137 1138 /** 1139 * i40e_find_mac - Find a mac addr in the macvlan filters list 1140 * @vsi: the VSI to be searched 1141 * @macaddr: the MAC address we are searching for 1142 * 1143 * Returns the first filter with the provided MAC address or NULL if 1144 * MAC address was not found 1145 **/ 1146 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1147 { 1148 struct i40e_mac_filter *f; 1149 u64 key; 1150 1151 if (!vsi || !macaddr) 1152 return NULL; 1153 1154 key = i40e_addr_to_hkey(macaddr); 1155 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1156 if ((ether_addr_equal(macaddr, f->macaddr))) 1157 return f; 1158 } 1159 return NULL; 1160 } 1161 1162 /** 1163 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1164 * @vsi: the VSI to be searched 1165 * 1166 * Returns true if VSI is in vlan mode or false otherwise 1167 **/ 1168 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1169 { 1170 /* If we have a PVID, always operate in VLAN mode */ 1171 if (vsi->info.pvid) 1172 return true; 1173 1174 /* We need to operate in VLAN mode whenever we have any filters with 1175 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1176 * time, incurring search cost repeatedly. However, we can notice two 1177 * things: 1178 * 1179 * 1) the only place where we can gain a VLAN filter is in 1180 * i40e_add_filter. 1181 * 1182 * 2) the only place where filters are actually removed is in 1183 * i40e_sync_filters_subtask. 1184 * 1185 * Thus, we can simply use a boolean value, has_vlan_filters which we 1186 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1187 * we have to perform the full search after deleting filters in 1188 * i40e_sync_filters_subtask, but we already have to search 1189 * filters here and can perform the check at the same time. This 1190 * results in avoiding embedding a loop for VLAN mode inside another 1191 * loop over all the filters, and should maintain correctness as noted 1192 * above. 1193 */ 1194 return vsi->has_vlan_filter; 1195 } 1196 1197 /** 1198 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1199 * @vsi: the VSI to configure 1200 * @tmp_add_list: list of filters ready to be added 1201 * @tmp_del_list: list of filters ready to be deleted 1202 * @vlan_filters: the number of active VLAN filters 1203 * 1204 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1205 * behave as expected. If we have any active VLAN filters remaining or about 1206 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1207 * so that they only match against untagged traffic. If we no longer have any 1208 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1209 * so that they match against both tagged and untagged traffic. In this way, 1210 * we ensure that we correctly receive the desired traffic. This ensures that 1211 * when we have an active VLAN we will receive only untagged traffic and 1212 * traffic matching active VLANs. If we have no active VLANs then we will 1213 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1214 * 1215 * Finally, in a similar fashion, this function also corrects filters when 1216 * there is an active PVID assigned to this VSI. 1217 * 1218 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1219 * 1220 * This function is only expected to be called from within 1221 * i40e_sync_vsi_filters. 1222 * 1223 * NOTE: This function expects to be called while under the 1224 * mac_filter_hash_lock 1225 */ 1226 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1227 struct hlist_head *tmp_add_list, 1228 struct hlist_head *tmp_del_list, 1229 int vlan_filters) 1230 { 1231 s16 pvid = le16_to_cpu(vsi->info.pvid); 1232 struct i40e_mac_filter *f, *add_head; 1233 struct i40e_new_mac_filter *new; 1234 struct hlist_node *h; 1235 int bkt, new_vlan; 1236 1237 /* To determine if a particular filter needs to be replaced we 1238 * have the three following conditions: 1239 * 1240 * a) if we have a PVID assigned, then all filters which are 1241 * not marked as VLAN=PVID must be replaced with filters that 1242 * are. 1243 * b) otherwise, if we have any active VLANS, all filters 1244 * which are marked as VLAN=-1 must be replaced with 1245 * filters marked as VLAN=0 1246 * c) finally, if we do not have any active VLANS, all filters 1247 * which are marked as VLAN=0 must be replaced with filters 1248 * marked as VLAN=-1 1249 */ 1250 1251 /* Update the filters about to be added in place */ 1252 hlist_for_each_entry(new, tmp_add_list, hlist) { 1253 if (pvid && new->f->vlan != pvid) 1254 new->f->vlan = pvid; 1255 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1256 new->f->vlan = 0; 1257 else if (!vlan_filters && new->f->vlan == 0) 1258 new->f->vlan = I40E_VLAN_ANY; 1259 } 1260 1261 /* Update the remaining active filters */ 1262 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1263 /* Combine the checks for whether a filter needs to be changed 1264 * and then determine the new VLAN inside the if block, in 1265 * order to avoid duplicating code for adding the new filter 1266 * then deleting the old filter. 1267 */ 1268 if ((pvid && f->vlan != pvid) || 1269 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1270 (!vlan_filters && f->vlan == 0)) { 1271 /* Determine the new vlan we will be adding */ 1272 if (pvid) 1273 new_vlan = pvid; 1274 else if (vlan_filters) 1275 new_vlan = 0; 1276 else 1277 new_vlan = I40E_VLAN_ANY; 1278 1279 /* Create the new filter */ 1280 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1281 if (!add_head) 1282 return -ENOMEM; 1283 1284 /* Create a temporary i40e_new_mac_filter */ 1285 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1286 if (!new) 1287 return -ENOMEM; 1288 1289 new->f = add_head; 1290 new->state = add_head->state; 1291 1292 /* Add the new filter to the tmp list */ 1293 hlist_add_head(&new->hlist, tmp_add_list); 1294 1295 /* Put the original filter into the delete list */ 1296 f->state = I40E_FILTER_REMOVE; 1297 hash_del(&f->hlist); 1298 hlist_add_head(&f->hlist, tmp_del_list); 1299 } 1300 } 1301 1302 vsi->has_vlan_filter = !!vlan_filters; 1303 1304 return 0; 1305 } 1306 1307 /** 1308 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1309 * @vsi: the PF Main VSI - inappropriate for any other VSI 1310 * @macaddr: the MAC address 1311 * 1312 * Remove whatever filter the firmware set up so the driver can manage 1313 * its own filtering intelligently. 1314 **/ 1315 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1316 { 1317 struct i40e_aqc_remove_macvlan_element_data element; 1318 struct i40e_pf *pf = vsi->back; 1319 1320 /* Only appropriate for the PF main VSI */ 1321 if (vsi->type != I40E_VSI_MAIN) 1322 return; 1323 1324 memset(&element, 0, sizeof(element)); 1325 ether_addr_copy(element.mac_addr, macaddr); 1326 element.vlan_tag = 0; 1327 /* Ignore error returns, some firmware does it this way... */ 1328 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1329 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1330 1331 memset(&element, 0, sizeof(element)); 1332 ether_addr_copy(element.mac_addr, macaddr); 1333 element.vlan_tag = 0; 1334 /* ...and some firmware does it this way. */ 1335 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1336 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1337 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1338 } 1339 1340 /** 1341 * i40e_add_filter - Add a mac/vlan filter to the VSI 1342 * @vsi: the VSI to be searched 1343 * @macaddr: the MAC address 1344 * @vlan: the vlan 1345 * 1346 * Returns ptr to the filter object or NULL when no memory available. 1347 * 1348 * NOTE: This function is expected to be called with mac_filter_hash_lock 1349 * being held. 1350 **/ 1351 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1352 const u8 *macaddr, s16 vlan) 1353 { 1354 struct i40e_mac_filter *f; 1355 u64 key; 1356 1357 if (!vsi || !macaddr) 1358 return NULL; 1359 1360 f = i40e_find_filter(vsi, macaddr, vlan); 1361 if (!f) { 1362 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1363 if (!f) 1364 return NULL; 1365 1366 /* Update the boolean indicating if we need to function in 1367 * VLAN mode. 1368 */ 1369 if (vlan >= 0) 1370 vsi->has_vlan_filter = true; 1371 1372 ether_addr_copy(f->macaddr, macaddr); 1373 f->vlan = vlan; 1374 f->state = I40E_FILTER_NEW; 1375 INIT_HLIST_NODE(&f->hlist); 1376 1377 key = i40e_addr_to_hkey(macaddr); 1378 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1379 1380 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1381 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1382 } 1383 1384 /* If we're asked to add a filter that has been marked for removal, it 1385 * is safe to simply restore it to active state. __i40e_del_filter 1386 * will have simply deleted any filters which were previously marked 1387 * NEW or FAILED, so if it is currently marked REMOVE it must have 1388 * previously been ACTIVE. Since we haven't yet run the sync filters 1389 * task, just restore this filter to the ACTIVE state so that the 1390 * sync task leaves it in place 1391 */ 1392 if (f->state == I40E_FILTER_REMOVE) 1393 f->state = I40E_FILTER_ACTIVE; 1394 1395 return f; 1396 } 1397 1398 /** 1399 * __i40e_del_filter - Remove a specific filter from the VSI 1400 * @vsi: VSI to remove from 1401 * @f: the filter to remove from the list 1402 * 1403 * This function should be called instead of i40e_del_filter only if you know 1404 * the exact filter you will remove already, such as via i40e_find_filter or 1405 * i40e_find_mac. 1406 * 1407 * NOTE: This function is expected to be called with mac_filter_hash_lock 1408 * being held. 1409 * ANOTHER NOTE: This function MUST be called from within the context of 1410 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1411 * instead of list_for_each_entry(). 1412 **/ 1413 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1414 { 1415 if (!f) 1416 return; 1417 1418 /* If the filter was never added to firmware then we can just delete it 1419 * directly and we don't want to set the status to remove or else an 1420 * admin queue command will unnecessarily fire. 1421 */ 1422 if ((f->state == I40E_FILTER_FAILED) || 1423 (f->state == I40E_FILTER_NEW)) { 1424 hash_del(&f->hlist); 1425 kfree(f); 1426 } else { 1427 f->state = I40E_FILTER_REMOVE; 1428 } 1429 1430 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1431 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1432 } 1433 1434 /** 1435 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1436 * @vsi: the VSI to be searched 1437 * @macaddr: the MAC address 1438 * @vlan: the VLAN 1439 * 1440 * NOTE: This function is expected to be called with mac_filter_hash_lock 1441 * being held. 1442 * ANOTHER NOTE: This function MUST be called from within the context of 1443 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1444 * instead of list_for_each_entry(). 1445 **/ 1446 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1447 { 1448 struct i40e_mac_filter *f; 1449 1450 if (!vsi || !macaddr) 1451 return; 1452 1453 f = i40e_find_filter(vsi, macaddr, vlan); 1454 __i40e_del_filter(vsi, f); 1455 } 1456 1457 /** 1458 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1459 * @vsi: the VSI to be searched 1460 * @macaddr: the mac address to be filtered 1461 * 1462 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1463 * go through all the macvlan filters and add a macvlan filter for each 1464 * unique vlan that already exists. If a PVID has been assigned, instead only 1465 * add the macaddr to that VLAN. 1466 * 1467 * Returns last filter added on success, else NULL 1468 **/ 1469 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1470 const u8 *macaddr) 1471 { 1472 struct i40e_mac_filter *f, *add = NULL; 1473 struct hlist_node *h; 1474 int bkt; 1475 1476 if (vsi->info.pvid) 1477 return i40e_add_filter(vsi, macaddr, 1478 le16_to_cpu(vsi->info.pvid)); 1479 1480 if (!i40e_is_vsi_in_vlan(vsi)) 1481 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1482 1483 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1484 if (f->state == I40E_FILTER_REMOVE) 1485 continue; 1486 add = i40e_add_filter(vsi, macaddr, f->vlan); 1487 if (!add) 1488 return NULL; 1489 } 1490 1491 return add; 1492 } 1493 1494 /** 1495 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1496 * @vsi: the VSI to be searched 1497 * @macaddr: the mac address to be removed 1498 * 1499 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1500 * associated with. 1501 * 1502 * Returns 0 for success, or error 1503 **/ 1504 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1505 { 1506 struct i40e_mac_filter *f; 1507 struct hlist_node *h; 1508 bool found = false; 1509 int bkt; 1510 1511 lockdep_assert_held(&vsi->mac_filter_hash_lock); 1512 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1513 if (ether_addr_equal(macaddr, f->macaddr)) { 1514 __i40e_del_filter(vsi, f); 1515 found = true; 1516 } 1517 } 1518 1519 if (found) 1520 return 0; 1521 else 1522 return -ENOENT; 1523 } 1524 1525 /** 1526 * i40e_set_mac - NDO callback to set mac address 1527 * @netdev: network interface device structure 1528 * @p: pointer to an address structure 1529 * 1530 * Returns 0 on success, negative on failure 1531 **/ 1532 static int i40e_set_mac(struct net_device *netdev, void *p) 1533 { 1534 struct i40e_netdev_priv *np = netdev_priv(netdev); 1535 struct i40e_vsi *vsi = np->vsi; 1536 struct i40e_pf *pf = vsi->back; 1537 struct i40e_hw *hw = &pf->hw; 1538 struct sockaddr *addr = p; 1539 1540 if (!is_valid_ether_addr(addr->sa_data)) 1541 return -EADDRNOTAVAIL; 1542 1543 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1544 netdev_info(netdev, "already using mac address %pM\n", 1545 addr->sa_data); 1546 return 0; 1547 } 1548 1549 if (test_bit(__I40E_DOWN, pf->state) || 1550 test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 1551 return -EADDRNOTAVAIL; 1552 1553 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1554 netdev_info(netdev, "returning to hw mac address %pM\n", 1555 hw->mac.addr); 1556 else 1557 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1558 1559 /* Copy the address first, so that we avoid a possible race with 1560 * .set_rx_mode(). 1561 * - Remove old address from MAC filter 1562 * - Copy new address 1563 * - Add new address to MAC filter 1564 */ 1565 spin_lock_bh(&vsi->mac_filter_hash_lock); 1566 i40e_del_mac_filter(vsi, netdev->dev_addr); 1567 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1568 i40e_add_mac_filter(vsi, netdev->dev_addr); 1569 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1570 1571 if (vsi->type == I40E_VSI_MAIN) { 1572 i40e_status ret; 1573 1574 ret = i40e_aq_mac_address_write(hw, I40E_AQC_WRITE_TYPE_LAA_WOL, 1575 addr->sa_data, NULL); 1576 if (ret) 1577 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1578 i40e_stat_str(hw, ret), 1579 i40e_aq_str(hw, hw->aq.asq_last_status)); 1580 } 1581 1582 /* schedule our worker thread which will take care of 1583 * applying the new filter changes 1584 */ 1585 i40e_service_event_schedule(pf); 1586 return 0; 1587 } 1588 1589 /** 1590 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1591 * @vsi: vsi structure 1592 * @seed: RSS hash seed 1593 **/ 1594 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1595 u8 *lut, u16 lut_size) 1596 { 1597 struct i40e_pf *pf = vsi->back; 1598 struct i40e_hw *hw = &pf->hw; 1599 int ret = 0; 1600 1601 if (seed) { 1602 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1603 (struct i40e_aqc_get_set_rss_key_data *)seed; 1604 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1605 if (ret) { 1606 dev_info(&pf->pdev->dev, 1607 "Cannot set RSS key, err %s aq_err %s\n", 1608 i40e_stat_str(hw, ret), 1609 i40e_aq_str(hw, hw->aq.asq_last_status)); 1610 return ret; 1611 } 1612 } 1613 if (lut) { 1614 bool pf_lut = vsi->type == I40E_VSI_MAIN; 1615 1616 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1617 if (ret) { 1618 dev_info(&pf->pdev->dev, 1619 "Cannot set RSS lut, err %s aq_err %s\n", 1620 i40e_stat_str(hw, ret), 1621 i40e_aq_str(hw, hw->aq.asq_last_status)); 1622 return ret; 1623 } 1624 } 1625 return ret; 1626 } 1627 1628 /** 1629 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1630 * @vsi: VSI structure 1631 **/ 1632 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1633 { 1634 struct i40e_pf *pf = vsi->back; 1635 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1636 u8 *lut; 1637 int ret; 1638 1639 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1640 return 0; 1641 if (!vsi->rss_size) 1642 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1643 vsi->num_queue_pairs); 1644 if (!vsi->rss_size) 1645 return -EINVAL; 1646 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1647 if (!lut) 1648 return -ENOMEM; 1649 1650 /* Use the user configured hash keys and lookup table if there is one, 1651 * otherwise use default 1652 */ 1653 if (vsi->rss_lut_user) 1654 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1655 else 1656 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1657 if (vsi->rss_hkey_user) 1658 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1659 else 1660 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1661 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1662 kfree(lut); 1663 return ret; 1664 } 1665 1666 /** 1667 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1668 * @vsi: the VSI being configured, 1669 * @ctxt: VSI context structure 1670 * @enabled_tc: number of traffic classes to enable 1671 * 1672 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1673 **/ 1674 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1675 struct i40e_vsi_context *ctxt, 1676 u8 enabled_tc) 1677 { 1678 u16 qcount = 0, max_qcount, qmap, sections = 0; 1679 int i, override_q, pow, num_qps, ret; 1680 u8 netdev_tc = 0, offset = 0; 1681 1682 if (vsi->type != I40E_VSI_MAIN) 1683 return -EINVAL; 1684 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1685 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1686 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1687 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1688 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1689 1690 /* find the next higher power-of-2 of num queue pairs */ 1691 pow = ilog2(num_qps); 1692 if (!is_power_of_2(num_qps)) 1693 pow++; 1694 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1695 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1696 1697 /* Setup queue offset/count for all TCs for given VSI */ 1698 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1699 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1700 /* See if the given TC is enabled for the given VSI */ 1701 if (vsi->tc_config.enabled_tc & BIT(i)) { 1702 offset = vsi->mqprio_qopt.qopt.offset[i]; 1703 qcount = vsi->mqprio_qopt.qopt.count[i]; 1704 if (qcount > max_qcount) 1705 max_qcount = qcount; 1706 vsi->tc_config.tc_info[i].qoffset = offset; 1707 vsi->tc_config.tc_info[i].qcount = qcount; 1708 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1709 } else { 1710 /* TC is not enabled so set the offset to 1711 * default queue and allocate one queue 1712 * for the given TC. 1713 */ 1714 vsi->tc_config.tc_info[i].qoffset = 0; 1715 vsi->tc_config.tc_info[i].qcount = 1; 1716 vsi->tc_config.tc_info[i].netdev_tc = 0; 1717 } 1718 } 1719 1720 /* Set actual Tx/Rx queue pairs */ 1721 vsi->num_queue_pairs = offset + qcount; 1722 1723 /* Setup queue TC[0].qmap for given VSI context */ 1724 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1725 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1726 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1727 ctxt->info.valid_sections |= cpu_to_le16(sections); 1728 1729 /* Reconfigure RSS for main VSI with max queue count */ 1730 vsi->rss_size = max_qcount; 1731 ret = i40e_vsi_config_rss(vsi); 1732 if (ret) { 1733 dev_info(&vsi->back->pdev->dev, 1734 "Failed to reconfig rss for num_queues (%u)\n", 1735 max_qcount); 1736 return ret; 1737 } 1738 vsi->reconfig_rss = true; 1739 dev_dbg(&vsi->back->pdev->dev, 1740 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1741 1742 /* Find queue count available for channel VSIs and starting offset 1743 * for channel VSIs 1744 */ 1745 override_q = vsi->mqprio_qopt.qopt.count[0]; 1746 if (override_q && override_q < vsi->num_queue_pairs) { 1747 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1748 vsi->next_base_queue = override_q; 1749 } 1750 return 0; 1751 } 1752 1753 /** 1754 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1755 * @vsi: the VSI being setup 1756 * @ctxt: VSI context structure 1757 * @enabled_tc: Enabled TCs bitmap 1758 * @is_add: True if called before Add VSI 1759 * 1760 * Setup VSI queue mapping for enabled traffic classes. 1761 **/ 1762 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1763 struct i40e_vsi_context *ctxt, 1764 u8 enabled_tc, 1765 bool is_add) 1766 { 1767 struct i40e_pf *pf = vsi->back; 1768 u16 sections = 0; 1769 u8 netdev_tc = 0; 1770 u16 numtc = 1; 1771 u16 qcount; 1772 u8 offset; 1773 u16 qmap; 1774 int i; 1775 u16 num_tc_qps = 0; 1776 1777 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1778 offset = 0; 1779 1780 /* Number of queues per enabled TC */ 1781 num_tc_qps = vsi->alloc_queue_pairs; 1782 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1783 /* Find numtc from enabled TC bitmap */ 1784 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1785 if (enabled_tc & BIT(i)) /* TC is enabled */ 1786 numtc++; 1787 } 1788 if (!numtc) { 1789 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1790 numtc = 1; 1791 } 1792 num_tc_qps = num_tc_qps / numtc; 1793 num_tc_qps = min_t(int, num_tc_qps, 1794 i40e_pf_get_max_q_per_tc(pf)); 1795 } 1796 1797 vsi->tc_config.numtc = numtc; 1798 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1799 1800 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 1801 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1802 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 1803 1804 /* Setup queue offset/count for all TCs for given VSI */ 1805 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1806 /* See if the given TC is enabled for the given VSI */ 1807 if (vsi->tc_config.enabled_tc & BIT(i)) { 1808 /* TC is enabled */ 1809 int pow, num_qps; 1810 1811 switch (vsi->type) { 1812 case I40E_VSI_MAIN: 1813 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | 1814 I40E_FLAG_FD_ATR_ENABLED)) || 1815 vsi->tc_config.enabled_tc != 1) { 1816 qcount = min_t(int, pf->alloc_rss_size, 1817 num_tc_qps); 1818 break; 1819 } 1820 /* fall through */ 1821 case I40E_VSI_FDIR: 1822 case I40E_VSI_SRIOV: 1823 case I40E_VSI_VMDQ2: 1824 default: 1825 qcount = num_tc_qps; 1826 WARN_ON(i != 0); 1827 break; 1828 } 1829 vsi->tc_config.tc_info[i].qoffset = offset; 1830 vsi->tc_config.tc_info[i].qcount = qcount; 1831 1832 /* find the next higher power-of-2 of num queue pairs */ 1833 num_qps = qcount; 1834 pow = 0; 1835 while (num_qps && (BIT_ULL(pow) < qcount)) { 1836 pow++; 1837 num_qps >>= 1; 1838 } 1839 1840 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1841 qmap = 1842 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1843 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1844 1845 offset += qcount; 1846 } else { 1847 /* TC is not enabled so set the offset to 1848 * default queue and allocate one queue 1849 * for the given TC. 1850 */ 1851 vsi->tc_config.tc_info[i].qoffset = 0; 1852 vsi->tc_config.tc_info[i].qcount = 1; 1853 vsi->tc_config.tc_info[i].netdev_tc = 0; 1854 1855 qmap = 0; 1856 } 1857 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1858 } 1859 1860 /* Set actual Tx/Rx queue pairs */ 1861 vsi->num_queue_pairs = offset; 1862 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1863 if (vsi->req_queue_pairs > 0) 1864 vsi->num_queue_pairs = vsi->req_queue_pairs; 1865 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1866 vsi->num_queue_pairs = pf->num_lan_msix; 1867 } 1868 1869 /* Scheduler section valid can only be set for ADD VSI */ 1870 if (is_add) { 1871 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1872 1873 ctxt->info.up_enable_bits = enabled_tc; 1874 } 1875 if (vsi->type == I40E_VSI_SRIOV) { 1876 ctxt->info.mapping_flags |= 1877 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1878 for (i = 0; i < vsi->num_queue_pairs; i++) 1879 ctxt->info.queue_mapping[i] = 1880 cpu_to_le16(vsi->base_queue + i); 1881 } else { 1882 ctxt->info.mapping_flags |= 1883 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1884 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1885 } 1886 ctxt->info.valid_sections |= cpu_to_le16(sections); 1887 } 1888 1889 /** 1890 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1891 * @netdev: the netdevice 1892 * @addr: address to add 1893 * 1894 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1895 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1896 */ 1897 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1898 { 1899 struct i40e_netdev_priv *np = netdev_priv(netdev); 1900 struct i40e_vsi *vsi = np->vsi; 1901 1902 if (i40e_add_mac_filter(vsi, addr)) 1903 return 0; 1904 else 1905 return -ENOMEM; 1906 } 1907 1908 /** 1909 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1910 * @netdev: the netdevice 1911 * @addr: address to add 1912 * 1913 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1914 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1915 */ 1916 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1917 { 1918 struct i40e_netdev_priv *np = netdev_priv(netdev); 1919 struct i40e_vsi *vsi = np->vsi; 1920 1921 /* Under some circumstances, we might receive a request to delete 1922 * our own device address from our uc list. Because we store the 1923 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1924 * such requests and not delete our device address from this list. 1925 */ 1926 if (ether_addr_equal(addr, netdev->dev_addr)) 1927 return 0; 1928 1929 i40e_del_mac_filter(vsi, addr); 1930 1931 return 0; 1932 } 1933 1934 /** 1935 * i40e_set_rx_mode - NDO callback to set the netdev filters 1936 * @netdev: network interface device structure 1937 **/ 1938 static void i40e_set_rx_mode(struct net_device *netdev) 1939 { 1940 struct i40e_netdev_priv *np = netdev_priv(netdev); 1941 struct i40e_vsi *vsi = np->vsi; 1942 1943 spin_lock_bh(&vsi->mac_filter_hash_lock); 1944 1945 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1946 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1947 1948 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1949 1950 /* check for other flag changes */ 1951 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1952 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1953 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1954 } 1955 } 1956 1957 /** 1958 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1959 * @vsi: Pointer to VSI struct 1960 * @from: Pointer to list which contains MAC filter entries - changes to 1961 * those entries needs to be undone. 1962 * 1963 * MAC filter entries from this list were slated for deletion. 1964 **/ 1965 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1966 struct hlist_head *from) 1967 { 1968 struct i40e_mac_filter *f; 1969 struct hlist_node *h; 1970 1971 hlist_for_each_entry_safe(f, h, from, hlist) { 1972 u64 key = i40e_addr_to_hkey(f->macaddr); 1973 1974 /* Move the element back into MAC filter list*/ 1975 hlist_del(&f->hlist); 1976 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1977 } 1978 } 1979 1980 /** 1981 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1982 * @vsi: Pointer to vsi struct 1983 * @from: Pointer to list which contains MAC filter entries - changes to 1984 * those entries needs to be undone. 1985 * 1986 * MAC filter entries from this list were slated for addition. 1987 **/ 1988 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 1989 struct hlist_head *from) 1990 { 1991 struct i40e_new_mac_filter *new; 1992 struct hlist_node *h; 1993 1994 hlist_for_each_entry_safe(new, h, from, hlist) { 1995 /* We can simply free the wrapper structure */ 1996 hlist_del(&new->hlist); 1997 kfree(new); 1998 } 1999 } 2000 2001 /** 2002 * i40e_next_entry - Get the next non-broadcast filter from a list 2003 * @next: pointer to filter in list 2004 * 2005 * Returns the next non-broadcast filter in the list. Required so that we 2006 * ignore broadcast filters within the list, since these are not handled via 2007 * the normal firmware update path. 2008 */ 2009 static 2010 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 2011 { 2012 hlist_for_each_entry_continue(next, hlist) { 2013 if (!is_broadcast_ether_addr(next->f->macaddr)) 2014 return next; 2015 } 2016 2017 return NULL; 2018 } 2019 2020 /** 2021 * i40e_update_filter_state - Update filter state based on return data 2022 * from firmware 2023 * @count: Number of filters added 2024 * @add_list: return data from fw 2025 * @add_head: pointer to first filter in current batch 2026 * 2027 * MAC filter entries from list were slated to be added to device. Returns 2028 * number of successful filters. Note that 0 does NOT mean success! 2029 **/ 2030 static int 2031 i40e_update_filter_state(int count, 2032 struct i40e_aqc_add_macvlan_element_data *add_list, 2033 struct i40e_new_mac_filter *add_head) 2034 { 2035 int retval = 0; 2036 int i; 2037 2038 for (i = 0; i < count; i++) { 2039 /* Always check status of each filter. We don't need to check 2040 * the firmware return status because we pre-set the filter 2041 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2042 * request to the adminq. Thus, if it no longer matches then 2043 * we know the filter is active. 2044 */ 2045 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2046 add_head->state = I40E_FILTER_FAILED; 2047 } else { 2048 add_head->state = I40E_FILTER_ACTIVE; 2049 retval++; 2050 } 2051 2052 add_head = i40e_next_filter(add_head); 2053 if (!add_head) 2054 break; 2055 } 2056 2057 return retval; 2058 } 2059 2060 /** 2061 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2062 * @vsi: ptr to the VSI 2063 * @vsi_name: name to display in messages 2064 * @list: the list of filters to send to firmware 2065 * @num_del: the number of filters to delete 2066 * @retval: Set to -EIO on failure to delete 2067 * 2068 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2069 * *retval instead of a return value so that success does not force ret_val to 2070 * be set to 0. This ensures that a sequence of calls to this function 2071 * preserve the previous value of *retval on successful delete. 2072 */ 2073 static 2074 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2075 struct i40e_aqc_remove_macvlan_element_data *list, 2076 int num_del, int *retval) 2077 { 2078 struct i40e_hw *hw = &vsi->back->hw; 2079 i40e_status aq_ret; 2080 int aq_err; 2081 2082 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 2083 aq_err = hw->aq.asq_last_status; 2084 2085 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2086 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 2087 *retval = -EIO; 2088 dev_info(&vsi->back->pdev->dev, 2089 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 2090 vsi_name, i40e_stat_str(hw, aq_ret), 2091 i40e_aq_str(hw, aq_err)); 2092 } 2093 } 2094 2095 /** 2096 * i40e_aqc_add_filters - Request firmware to add a set of filters 2097 * @vsi: ptr to the VSI 2098 * @vsi_name: name to display in messages 2099 * @list: the list of filters to send to firmware 2100 * @add_head: Position in the add hlist 2101 * @num_add: the number of filters to add 2102 * 2103 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2104 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2105 * space for more filters. 2106 */ 2107 static 2108 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2109 struct i40e_aqc_add_macvlan_element_data *list, 2110 struct i40e_new_mac_filter *add_head, 2111 int num_add) 2112 { 2113 struct i40e_hw *hw = &vsi->back->hw; 2114 int aq_err, fcnt; 2115 2116 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 2117 aq_err = hw->aq.asq_last_status; 2118 fcnt = i40e_update_filter_state(num_add, list, add_head); 2119 2120 if (fcnt != num_add) { 2121 if (vsi->type == I40E_VSI_MAIN) { 2122 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2123 dev_warn(&vsi->back->pdev->dev, 2124 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2125 i40e_aq_str(hw, aq_err), vsi_name); 2126 } else if (vsi->type == I40E_VSI_SRIOV || 2127 vsi->type == I40E_VSI_VMDQ1 || 2128 vsi->type == I40E_VSI_VMDQ2) { 2129 dev_warn(&vsi->back->pdev->dev, 2130 "Error %s adding RX filters on %s, please set promiscuous on manually for %s\n", 2131 i40e_aq_str(hw, aq_err), vsi_name, vsi_name); 2132 } else { 2133 dev_warn(&vsi->back->pdev->dev, 2134 "Error %s adding RX filters on %s, incorrect VSI type: %i.\n", 2135 i40e_aq_str(hw, aq_err), vsi_name, vsi->type); 2136 } 2137 } 2138 } 2139 2140 /** 2141 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2142 * @vsi: pointer to the VSI 2143 * @vsi_name: the VSI name 2144 * @f: filter data 2145 * 2146 * This function sets or clears the promiscuous broadcast flags for VLAN 2147 * filters in order to properly receive broadcast frames. Assumes that only 2148 * broadcast filters are passed. 2149 * 2150 * Returns status indicating success or failure; 2151 **/ 2152 static i40e_status 2153 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2154 struct i40e_mac_filter *f) 2155 { 2156 bool enable = f->state == I40E_FILTER_NEW; 2157 struct i40e_hw *hw = &vsi->back->hw; 2158 i40e_status aq_ret; 2159 2160 if (f->vlan == I40E_VLAN_ANY) { 2161 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2162 vsi->seid, 2163 enable, 2164 NULL); 2165 } else { 2166 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2167 vsi->seid, 2168 enable, 2169 f->vlan, 2170 NULL); 2171 } 2172 2173 if (aq_ret) { 2174 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2175 dev_warn(&vsi->back->pdev->dev, 2176 "Error %s, forcing overflow promiscuous on %s\n", 2177 i40e_aq_str(hw, hw->aq.asq_last_status), 2178 vsi_name); 2179 } 2180 2181 return aq_ret; 2182 } 2183 2184 /** 2185 * i40e_set_promiscuous - set promiscuous mode 2186 * @pf: board private structure 2187 * @promisc: promisc on or off 2188 * 2189 * There are different ways of setting promiscuous mode on a PF depending on 2190 * what state/environment we're in. This identifies and sets it appropriately. 2191 * Returns 0 on success. 2192 **/ 2193 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2194 { 2195 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2196 struct i40e_hw *hw = &pf->hw; 2197 i40e_status aq_ret; 2198 2199 if (vsi->type == I40E_VSI_MAIN && 2200 pf->lan_veb != I40E_NO_VEB && 2201 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2202 /* set defport ON for Main VSI instead of true promisc 2203 * this way we will get all unicast/multicast and VLAN 2204 * promisc behavior but will not get VF or VMDq traffic 2205 * replicated on the Main VSI. 2206 */ 2207 if (promisc) 2208 aq_ret = i40e_aq_set_default_vsi(hw, 2209 vsi->seid, 2210 NULL); 2211 else 2212 aq_ret = i40e_aq_clear_default_vsi(hw, 2213 vsi->seid, 2214 NULL); 2215 if (aq_ret) { 2216 dev_info(&pf->pdev->dev, 2217 "Set default VSI failed, err %s, aq_err %s\n", 2218 i40e_stat_str(hw, aq_ret), 2219 i40e_aq_str(hw, hw->aq.asq_last_status)); 2220 } 2221 } else { 2222 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2223 hw, 2224 vsi->seid, 2225 promisc, NULL, 2226 true); 2227 if (aq_ret) { 2228 dev_info(&pf->pdev->dev, 2229 "set unicast promisc failed, err %s, aq_err %s\n", 2230 i40e_stat_str(hw, aq_ret), 2231 i40e_aq_str(hw, hw->aq.asq_last_status)); 2232 } 2233 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2234 hw, 2235 vsi->seid, 2236 promisc, NULL); 2237 if (aq_ret) { 2238 dev_info(&pf->pdev->dev, 2239 "set multicast promisc failed, err %s, aq_err %s\n", 2240 i40e_stat_str(hw, aq_ret), 2241 i40e_aq_str(hw, hw->aq.asq_last_status)); 2242 } 2243 } 2244 2245 if (!aq_ret) 2246 pf->cur_promisc = promisc; 2247 2248 return aq_ret; 2249 } 2250 2251 /** 2252 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2253 * @vsi: ptr to the VSI 2254 * 2255 * Push any outstanding VSI filter changes through the AdminQ. 2256 * 2257 * Returns 0 or error value 2258 **/ 2259 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2260 { 2261 struct hlist_head tmp_add_list, tmp_del_list; 2262 struct i40e_mac_filter *f; 2263 struct i40e_new_mac_filter *new, *add_head = NULL; 2264 struct i40e_hw *hw = &vsi->back->hw; 2265 bool old_overflow, new_overflow; 2266 unsigned int failed_filters = 0; 2267 unsigned int vlan_filters = 0; 2268 char vsi_name[16] = "PF"; 2269 int filter_list_len = 0; 2270 i40e_status aq_ret = 0; 2271 u32 changed_flags = 0; 2272 struct hlist_node *h; 2273 struct i40e_pf *pf; 2274 int num_add = 0; 2275 int num_del = 0; 2276 int retval = 0; 2277 u16 cmd_flags; 2278 int list_size; 2279 int bkt; 2280 2281 /* empty array typed pointers, kcalloc later */ 2282 struct i40e_aqc_add_macvlan_element_data *add_list; 2283 struct i40e_aqc_remove_macvlan_element_data *del_list; 2284 2285 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2286 usleep_range(1000, 2000); 2287 pf = vsi->back; 2288 2289 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2290 2291 if (vsi->netdev) { 2292 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2293 vsi->current_netdev_flags = vsi->netdev->flags; 2294 } 2295 2296 INIT_HLIST_HEAD(&tmp_add_list); 2297 INIT_HLIST_HEAD(&tmp_del_list); 2298 2299 if (vsi->type == I40E_VSI_SRIOV) 2300 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2301 else if (vsi->type != I40E_VSI_MAIN) 2302 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2303 2304 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2305 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2306 2307 spin_lock_bh(&vsi->mac_filter_hash_lock); 2308 /* Create a list of filters to delete. */ 2309 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2310 if (f->state == I40E_FILTER_REMOVE) { 2311 /* Move the element into temporary del_list */ 2312 hash_del(&f->hlist); 2313 hlist_add_head(&f->hlist, &tmp_del_list); 2314 2315 /* Avoid counting removed filters */ 2316 continue; 2317 } 2318 if (f->state == I40E_FILTER_NEW) { 2319 /* Create a temporary i40e_new_mac_filter */ 2320 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2321 if (!new) 2322 goto err_no_memory_locked; 2323 2324 /* Store pointer to the real filter */ 2325 new->f = f; 2326 new->state = f->state; 2327 2328 /* Add it to the hash list */ 2329 hlist_add_head(&new->hlist, &tmp_add_list); 2330 } 2331 2332 /* Count the number of active (current and new) VLAN 2333 * filters we have now. Does not count filters which 2334 * are marked for deletion. 2335 */ 2336 if (f->vlan > 0) 2337 vlan_filters++; 2338 } 2339 2340 retval = i40e_correct_mac_vlan_filters(vsi, 2341 &tmp_add_list, 2342 &tmp_del_list, 2343 vlan_filters); 2344 if (retval) 2345 goto err_no_memory_locked; 2346 2347 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2348 } 2349 2350 /* Now process 'del_list' outside the lock */ 2351 if (!hlist_empty(&tmp_del_list)) { 2352 filter_list_len = hw->aq.asq_buf_size / 2353 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2354 list_size = filter_list_len * 2355 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2356 del_list = kzalloc(list_size, GFP_ATOMIC); 2357 if (!del_list) 2358 goto err_no_memory; 2359 2360 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2361 cmd_flags = 0; 2362 2363 /* handle broadcast filters by updating the broadcast 2364 * promiscuous flag and release filter list. 2365 */ 2366 if (is_broadcast_ether_addr(f->macaddr)) { 2367 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2368 2369 hlist_del(&f->hlist); 2370 kfree(f); 2371 continue; 2372 } 2373 2374 /* add to delete list */ 2375 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2376 if (f->vlan == I40E_VLAN_ANY) { 2377 del_list[num_del].vlan_tag = 0; 2378 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2379 } else { 2380 del_list[num_del].vlan_tag = 2381 cpu_to_le16((u16)(f->vlan)); 2382 } 2383 2384 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2385 del_list[num_del].flags = cmd_flags; 2386 num_del++; 2387 2388 /* flush a full buffer */ 2389 if (num_del == filter_list_len) { 2390 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2391 num_del, &retval); 2392 memset(del_list, 0, list_size); 2393 num_del = 0; 2394 } 2395 /* Release memory for MAC filter entries which were 2396 * synced up with HW. 2397 */ 2398 hlist_del(&f->hlist); 2399 kfree(f); 2400 } 2401 2402 if (num_del) { 2403 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2404 num_del, &retval); 2405 } 2406 2407 kfree(del_list); 2408 del_list = NULL; 2409 } 2410 2411 if (!hlist_empty(&tmp_add_list)) { 2412 /* Do all the adds now. */ 2413 filter_list_len = hw->aq.asq_buf_size / 2414 sizeof(struct i40e_aqc_add_macvlan_element_data); 2415 list_size = filter_list_len * 2416 sizeof(struct i40e_aqc_add_macvlan_element_data); 2417 add_list = kzalloc(list_size, GFP_ATOMIC); 2418 if (!add_list) 2419 goto err_no_memory; 2420 2421 num_add = 0; 2422 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2423 /* handle broadcast filters by updating the broadcast 2424 * promiscuous flag instead of adding a MAC filter. 2425 */ 2426 if (is_broadcast_ether_addr(new->f->macaddr)) { 2427 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2428 new->f)) 2429 new->state = I40E_FILTER_FAILED; 2430 else 2431 new->state = I40E_FILTER_ACTIVE; 2432 continue; 2433 } 2434 2435 /* add to add array */ 2436 if (num_add == 0) 2437 add_head = new; 2438 cmd_flags = 0; 2439 ether_addr_copy(add_list[num_add].mac_addr, 2440 new->f->macaddr); 2441 if (new->f->vlan == I40E_VLAN_ANY) { 2442 add_list[num_add].vlan_tag = 0; 2443 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2444 } else { 2445 add_list[num_add].vlan_tag = 2446 cpu_to_le16((u16)(new->f->vlan)); 2447 } 2448 add_list[num_add].queue_number = 0; 2449 /* set invalid match method for later detection */ 2450 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2451 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2452 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2453 num_add++; 2454 2455 /* flush a full buffer */ 2456 if (num_add == filter_list_len) { 2457 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2458 add_head, num_add); 2459 memset(add_list, 0, list_size); 2460 num_add = 0; 2461 } 2462 } 2463 if (num_add) { 2464 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2465 num_add); 2466 } 2467 /* Now move all of the filters from the temp add list back to 2468 * the VSI's list. 2469 */ 2470 spin_lock_bh(&vsi->mac_filter_hash_lock); 2471 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2472 /* Only update the state if we're still NEW */ 2473 if (new->f->state == I40E_FILTER_NEW) 2474 new->f->state = new->state; 2475 hlist_del(&new->hlist); 2476 kfree(new); 2477 } 2478 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2479 kfree(add_list); 2480 add_list = NULL; 2481 } 2482 2483 /* Determine the number of active and failed filters. */ 2484 spin_lock_bh(&vsi->mac_filter_hash_lock); 2485 vsi->active_filters = 0; 2486 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2487 if (f->state == I40E_FILTER_ACTIVE) 2488 vsi->active_filters++; 2489 else if (f->state == I40E_FILTER_FAILED) 2490 failed_filters++; 2491 } 2492 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2493 2494 /* Check if we are able to exit overflow promiscuous mode. We can 2495 * safely exit if we didn't just enter, we no longer have any failed 2496 * filters, and we have reduced filters below the threshold value. 2497 */ 2498 if (old_overflow && !failed_filters && 2499 vsi->active_filters < vsi->promisc_threshold) { 2500 dev_info(&pf->pdev->dev, 2501 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2502 vsi_name); 2503 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2504 vsi->promisc_threshold = 0; 2505 } 2506 2507 /* if the VF is not trusted do not do promisc */ 2508 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2509 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2510 goto out; 2511 } 2512 2513 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2514 2515 /* If we are entering overflow promiscuous, we need to calculate a new 2516 * threshold for when we are safe to exit 2517 */ 2518 if (!old_overflow && new_overflow) 2519 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2520 2521 /* check for changes in promiscuous modes */ 2522 if (changed_flags & IFF_ALLMULTI) { 2523 bool cur_multipromisc; 2524 2525 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2526 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2527 vsi->seid, 2528 cur_multipromisc, 2529 NULL); 2530 if (aq_ret) { 2531 retval = i40e_aq_rc_to_posix(aq_ret, 2532 hw->aq.asq_last_status); 2533 dev_info(&pf->pdev->dev, 2534 "set multi promisc failed on %s, err %s aq_err %s\n", 2535 vsi_name, 2536 i40e_stat_str(hw, aq_ret), 2537 i40e_aq_str(hw, hw->aq.asq_last_status)); 2538 } else { 2539 dev_info(&pf->pdev->dev, "%s is %s allmulti mode.\n", 2540 vsi->netdev->name, 2541 cur_multipromisc ? "entering" : "leaving"); 2542 } 2543 } 2544 2545 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2546 bool cur_promisc; 2547 2548 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2549 new_overflow); 2550 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2551 if (aq_ret) { 2552 retval = i40e_aq_rc_to_posix(aq_ret, 2553 hw->aq.asq_last_status); 2554 dev_info(&pf->pdev->dev, 2555 "Setting promiscuous %s failed on %s, err %s aq_err %s\n", 2556 cur_promisc ? "on" : "off", 2557 vsi_name, 2558 i40e_stat_str(hw, aq_ret), 2559 i40e_aq_str(hw, hw->aq.asq_last_status)); 2560 } 2561 } 2562 out: 2563 /* if something went wrong then set the changed flag so we try again */ 2564 if (retval) 2565 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2566 2567 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2568 return retval; 2569 2570 err_no_memory: 2571 /* Restore elements on the temporary add and delete lists */ 2572 spin_lock_bh(&vsi->mac_filter_hash_lock); 2573 err_no_memory_locked: 2574 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2575 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2576 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2577 2578 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2579 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2580 return -ENOMEM; 2581 } 2582 2583 /** 2584 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2585 * @pf: board private structure 2586 **/ 2587 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2588 { 2589 int v; 2590 2591 if (!pf) 2592 return; 2593 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2594 return; 2595 if (test_and_set_bit(__I40E_VF_DISABLE, pf->state)) { 2596 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 2597 return; 2598 } 2599 2600 for (v = 0; v < pf->num_alloc_vsi; v++) { 2601 if (pf->vsi[v] && 2602 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2603 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2604 2605 if (ret) { 2606 /* come back and try again later */ 2607 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2608 pf->state); 2609 break; 2610 } 2611 } 2612 } 2613 clear_bit(__I40E_VF_DISABLE, pf->state); 2614 } 2615 2616 /** 2617 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2618 * @vsi: the vsi 2619 **/ 2620 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2621 { 2622 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2623 return I40E_RXBUFFER_2048; 2624 else 2625 return I40E_RXBUFFER_3072; 2626 } 2627 2628 /** 2629 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2630 * @netdev: network interface device structure 2631 * @new_mtu: new value for maximum frame size 2632 * 2633 * Returns 0 on success, negative on failure 2634 **/ 2635 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2636 { 2637 struct i40e_netdev_priv *np = netdev_priv(netdev); 2638 struct i40e_vsi *vsi = np->vsi; 2639 struct i40e_pf *pf = vsi->back; 2640 2641 if (i40e_enabled_xdp_vsi(vsi)) { 2642 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2643 2644 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2645 return -EINVAL; 2646 } 2647 2648 netdev_dbg(netdev, "changing MTU from %d to %d\n", 2649 netdev->mtu, new_mtu); 2650 netdev->mtu = new_mtu; 2651 if (netif_running(netdev)) 2652 i40e_vsi_reinit_locked(vsi); 2653 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2654 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2655 return 0; 2656 } 2657 2658 /** 2659 * i40e_ioctl - Access the hwtstamp interface 2660 * @netdev: network interface device structure 2661 * @ifr: interface request data 2662 * @cmd: ioctl command 2663 **/ 2664 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2665 { 2666 struct i40e_netdev_priv *np = netdev_priv(netdev); 2667 struct i40e_pf *pf = np->vsi->back; 2668 2669 switch (cmd) { 2670 case SIOCGHWTSTAMP: 2671 return i40e_ptp_get_ts_config(pf, ifr); 2672 case SIOCSHWTSTAMP: 2673 return i40e_ptp_set_ts_config(pf, ifr); 2674 default: 2675 return -EOPNOTSUPP; 2676 } 2677 } 2678 2679 /** 2680 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2681 * @vsi: the vsi being adjusted 2682 **/ 2683 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2684 { 2685 struct i40e_vsi_context ctxt; 2686 i40e_status ret; 2687 2688 /* Don't modify stripping options if a port VLAN is active */ 2689 if (vsi->info.pvid) 2690 return; 2691 2692 if ((vsi->info.valid_sections & 2693 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2694 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2695 return; /* already enabled */ 2696 2697 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2698 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2699 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2700 2701 ctxt.seid = vsi->seid; 2702 ctxt.info = vsi->info; 2703 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2704 if (ret) { 2705 dev_info(&vsi->back->pdev->dev, 2706 "update vlan stripping failed, err %s aq_err %s\n", 2707 i40e_stat_str(&vsi->back->hw, ret), 2708 i40e_aq_str(&vsi->back->hw, 2709 vsi->back->hw.aq.asq_last_status)); 2710 } 2711 } 2712 2713 /** 2714 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2715 * @vsi: the vsi being adjusted 2716 **/ 2717 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2718 { 2719 struct i40e_vsi_context ctxt; 2720 i40e_status ret; 2721 2722 /* Don't modify stripping options if a port VLAN is active */ 2723 if (vsi->info.pvid) 2724 return; 2725 2726 if ((vsi->info.valid_sections & 2727 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2728 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2729 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2730 return; /* already disabled */ 2731 2732 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2733 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2734 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2735 2736 ctxt.seid = vsi->seid; 2737 ctxt.info = vsi->info; 2738 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2739 if (ret) { 2740 dev_info(&vsi->back->pdev->dev, 2741 "update vlan stripping failed, err %s aq_err %s\n", 2742 i40e_stat_str(&vsi->back->hw, ret), 2743 i40e_aq_str(&vsi->back->hw, 2744 vsi->back->hw.aq.asq_last_status)); 2745 } 2746 } 2747 2748 /** 2749 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2750 * @vsi: the vsi being configured 2751 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2752 * 2753 * This is a helper function for adding a new MAC/VLAN filter with the 2754 * specified VLAN for each existing MAC address already in the hash table. 2755 * This function does *not* perform any accounting to update filters based on 2756 * VLAN mode. 2757 * 2758 * NOTE: this function expects to be called while under the 2759 * mac_filter_hash_lock 2760 **/ 2761 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2762 { 2763 struct i40e_mac_filter *f, *add_f; 2764 struct hlist_node *h; 2765 int bkt; 2766 2767 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2768 if (f->state == I40E_FILTER_REMOVE) 2769 continue; 2770 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2771 if (!add_f) { 2772 dev_info(&vsi->back->pdev->dev, 2773 "Could not add vlan filter %d for %pM\n", 2774 vid, f->macaddr); 2775 return -ENOMEM; 2776 } 2777 } 2778 2779 return 0; 2780 } 2781 2782 /** 2783 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2784 * @vsi: the VSI being configured 2785 * @vid: VLAN id to be added 2786 **/ 2787 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2788 { 2789 int err; 2790 2791 if (vsi->info.pvid) 2792 return -EINVAL; 2793 2794 /* The network stack will attempt to add VID=0, with the intention to 2795 * receive priority tagged packets with a VLAN of 0. Our HW receives 2796 * these packets by default when configured to receive untagged 2797 * packets, so we don't need to add a filter for this case. 2798 * Additionally, HW interprets adding a VID=0 filter as meaning to 2799 * receive *only* tagged traffic and stops receiving untagged traffic. 2800 * Thus, we do not want to actually add a filter for VID=0 2801 */ 2802 if (!vid) 2803 return 0; 2804 2805 /* Locked once because all functions invoked below iterates list*/ 2806 spin_lock_bh(&vsi->mac_filter_hash_lock); 2807 err = i40e_add_vlan_all_mac(vsi, vid); 2808 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2809 if (err) 2810 return err; 2811 2812 /* schedule our worker thread which will take care of 2813 * applying the new filter changes 2814 */ 2815 i40e_service_event_schedule(vsi->back); 2816 return 0; 2817 } 2818 2819 /** 2820 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2821 * @vsi: the vsi being configured 2822 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2823 * 2824 * This function should be used to remove all VLAN filters which match the 2825 * given VID. It does not schedule the service event and does not take the 2826 * mac_filter_hash_lock so it may be combined with other operations under 2827 * a single invocation of the mac_filter_hash_lock. 2828 * 2829 * NOTE: this function expects to be called while under the 2830 * mac_filter_hash_lock 2831 */ 2832 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2833 { 2834 struct i40e_mac_filter *f; 2835 struct hlist_node *h; 2836 int bkt; 2837 2838 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2839 if (f->vlan == vid) 2840 __i40e_del_filter(vsi, f); 2841 } 2842 } 2843 2844 /** 2845 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2846 * @vsi: the VSI being configured 2847 * @vid: VLAN id to be removed 2848 **/ 2849 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2850 { 2851 if (!vid || vsi->info.pvid) 2852 return; 2853 2854 spin_lock_bh(&vsi->mac_filter_hash_lock); 2855 i40e_rm_vlan_all_mac(vsi, vid); 2856 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2857 2858 /* schedule our worker thread which will take care of 2859 * applying the new filter changes 2860 */ 2861 i40e_service_event_schedule(vsi->back); 2862 } 2863 2864 /** 2865 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2866 * @netdev: network interface to be adjusted 2867 * @proto: unused protocol value 2868 * @vid: vlan id to be added 2869 * 2870 * net_device_ops implementation for adding vlan ids 2871 **/ 2872 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2873 __always_unused __be16 proto, u16 vid) 2874 { 2875 struct i40e_netdev_priv *np = netdev_priv(netdev); 2876 struct i40e_vsi *vsi = np->vsi; 2877 int ret = 0; 2878 2879 if (vid >= VLAN_N_VID) 2880 return -EINVAL; 2881 2882 ret = i40e_vsi_add_vlan(vsi, vid); 2883 if (!ret) 2884 set_bit(vid, vsi->active_vlans); 2885 2886 return ret; 2887 } 2888 2889 /** 2890 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 2891 * @netdev: network interface to be adjusted 2892 * @proto: unused protocol value 2893 * @vid: vlan id to be added 2894 **/ 2895 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 2896 __always_unused __be16 proto, u16 vid) 2897 { 2898 struct i40e_netdev_priv *np = netdev_priv(netdev); 2899 struct i40e_vsi *vsi = np->vsi; 2900 2901 if (vid >= VLAN_N_VID) 2902 return; 2903 set_bit(vid, vsi->active_vlans); 2904 } 2905 2906 /** 2907 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2908 * @netdev: network interface to be adjusted 2909 * @proto: unused protocol value 2910 * @vid: vlan id to be removed 2911 * 2912 * net_device_ops implementation for removing vlan ids 2913 **/ 2914 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2915 __always_unused __be16 proto, u16 vid) 2916 { 2917 struct i40e_netdev_priv *np = netdev_priv(netdev); 2918 struct i40e_vsi *vsi = np->vsi; 2919 2920 /* return code is ignored as there is nothing a user 2921 * can do about failure to remove and a log message was 2922 * already printed from the other function 2923 */ 2924 i40e_vsi_kill_vlan(vsi, vid); 2925 2926 clear_bit(vid, vsi->active_vlans); 2927 2928 return 0; 2929 } 2930 2931 /** 2932 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2933 * @vsi: the vsi being brought back up 2934 **/ 2935 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2936 { 2937 u16 vid; 2938 2939 if (!vsi->netdev) 2940 return; 2941 2942 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2943 i40e_vlan_stripping_enable(vsi); 2944 else 2945 i40e_vlan_stripping_disable(vsi); 2946 2947 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2948 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 2949 vid); 2950 } 2951 2952 /** 2953 * i40e_vsi_add_pvid - Add pvid for the VSI 2954 * @vsi: the vsi being adjusted 2955 * @vid: the vlan id to set as a PVID 2956 **/ 2957 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2958 { 2959 struct i40e_vsi_context ctxt; 2960 i40e_status ret; 2961 2962 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2963 vsi->info.pvid = cpu_to_le16(vid); 2964 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2965 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2966 I40E_AQ_VSI_PVLAN_EMOD_STR; 2967 2968 ctxt.seid = vsi->seid; 2969 ctxt.info = vsi->info; 2970 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2971 if (ret) { 2972 dev_info(&vsi->back->pdev->dev, 2973 "add pvid failed, err %s aq_err %s\n", 2974 i40e_stat_str(&vsi->back->hw, ret), 2975 i40e_aq_str(&vsi->back->hw, 2976 vsi->back->hw.aq.asq_last_status)); 2977 return -ENOENT; 2978 } 2979 2980 return 0; 2981 } 2982 2983 /** 2984 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2985 * @vsi: the vsi being adjusted 2986 * 2987 * Just use the vlan_rx_register() service to put it back to normal 2988 **/ 2989 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2990 { 2991 vsi->info.pvid = 0; 2992 2993 i40e_vlan_stripping_disable(vsi); 2994 } 2995 2996 /** 2997 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2998 * @vsi: ptr to the VSI 2999 * 3000 * If this function returns with an error, then it's possible one or 3001 * more of the rings is populated (while the rest are not). It is the 3002 * callers duty to clean those orphaned rings. 3003 * 3004 * Return 0 on success, negative on failure 3005 **/ 3006 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 3007 { 3008 int i, err = 0; 3009 3010 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3011 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 3012 3013 if (!i40e_enabled_xdp_vsi(vsi)) 3014 return err; 3015 3016 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3017 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 3018 3019 return err; 3020 } 3021 3022 /** 3023 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 3024 * @vsi: ptr to the VSI 3025 * 3026 * Free VSI's transmit software resources 3027 **/ 3028 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 3029 { 3030 int i; 3031 3032 if (vsi->tx_rings) { 3033 for (i = 0; i < vsi->num_queue_pairs; i++) 3034 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 3035 i40e_free_tx_resources(vsi->tx_rings[i]); 3036 } 3037 3038 if (vsi->xdp_rings) { 3039 for (i = 0; i < vsi->num_queue_pairs; i++) 3040 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 3041 i40e_free_tx_resources(vsi->xdp_rings[i]); 3042 } 3043 } 3044 3045 /** 3046 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3047 * @vsi: ptr to the VSI 3048 * 3049 * If this function returns with an error, then it's possible one or 3050 * more of the rings is populated (while the rest are not). It is the 3051 * callers duty to clean those orphaned rings. 3052 * 3053 * Return 0 on success, negative on failure 3054 **/ 3055 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3056 { 3057 int i, err = 0; 3058 3059 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3060 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3061 return err; 3062 } 3063 3064 /** 3065 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3066 * @vsi: ptr to the VSI 3067 * 3068 * Free all receive software resources 3069 **/ 3070 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3071 { 3072 int i; 3073 3074 if (!vsi->rx_rings) 3075 return; 3076 3077 for (i = 0; i < vsi->num_queue_pairs; i++) 3078 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3079 i40e_free_rx_resources(vsi->rx_rings[i]); 3080 } 3081 3082 /** 3083 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3084 * @ring: The Tx ring to configure 3085 * 3086 * This enables/disables XPS for a given Tx descriptor ring 3087 * based on the TCs enabled for the VSI that ring belongs to. 3088 **/ 3089 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3090 { 3091 int cpu; 3092 3093 if (!ring->q_vector || !ring->netdev || ring->ch) 3094 return; 3095 3096 /* We only initialize XPS once, so as not to overwrite user settings */ 3097 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3098 return; 3099 3100 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3101 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3102 ring->queue_index); 3103 } 3104 3105 /** 3106 * i40e_xsk_umem - Retrieve the AF_XDP ZC if XDP and ZC is enabled 3107 * @ring: The Tx or Rx ring 3108 * 3109 * Returns the UMEM or NULL. 3110 **/ 3111 static struct xdp_umem *i40e_xsk_umem(struct i40e_ring *ring) 3112 { 3113 bool xdp_on = i40e_enabled_xdp_vsi(ring->vsi); 3114 int qid = ring->queue_index; 3115 3116 if (ring_is_xdp(ring)) 3117 qid -= ring->vsi->alloc_queue_pairs; 3118 3119 if (!xdp_on || !test_bit(qid, ring->vsi->af_xdp_zc_qps)) 3120 return NULL; 3121 3122 return xdp_get_umem_from_qid(ring->vsi->netdev, qid); 3123 } 3124 3125 /** 3126 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3127 * @ring: The Tx ring to configure 3128 * 3129 * Configure the Tx descriptor ring in the HMC context. 3130 **/ 3131 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3132 { 3133 struct i40e_vsi *vsi = ring->vsi; 3134 u16 pf_q = vsi->base_queue + ring->queue_index; 3135 struct i40e_hw *hw = &vsi->back->hw; 3136 struct i40e_hmc_obj_txq tx_ctx; 3137 i40e_status err = 0; 3138 u32 qtx_ctl = 0; 3139 3140 if (ring_is_xdp(ring)) 3141 ring->xsk_umem = i40e_xsk_umem(ring); 3142 3143 /* some ATR related tx ring init */ 3144 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3145 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3146 ring->atr_count = 0; 3147 } else { 3148 ring->atr_sample_rate = 0; 3149 } 3150 3151 /* configure XPS */ 3152 i40e_config_xps_tx_ring(ring); 3153 3154 /* clear the context structure first */ 3155 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3156 3157 tx_ctx.new_context = 1; 3158 tx_ctx.base = (ring->dma / 128); 3159 tx_ctx.qlen = ring->count; 3160 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3161 I40E_FLAG_FD_ATR_ENABLED)); 3162 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3163 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3164 if (vsi->type != I40E_VSI_FDIR) 3165 tx_ctx.head_wb_ena = 1; 3166 tx_ctx.head_wb_addr = ring->dma + 3167 (ring->count * sizeof(struct i40e_tx_desc)); 3168 3169 /* As part of VSI creation/update, FW allocates certain 3170 * Tx arbitration queue sets for each TC enabled for 3171 * the VSI. The FW returns the handles to these queue 3172 * sets as part of the response buffer to Add VSI, 3173 * Update VSI, etc. AQ commands. It is expected that 3174 * these queue set handles be associated with the Tx 3175 * queues by the driver as part of the TX queue context 3176 * initialization. This has to be done regardless of 3177 * DCB as by default everything is mapped to TC0. 3178 */ 3179 3180 if (ring->ch) 3181 tx_ctx.rdylist = 3182 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3183 3184 else 3185 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3186 3187 tx_ctx.rdylist_act = 0; 3188 3189 /* clear the context in the HMC */ 3190 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3191 if (err) { 3192 dev_info(&vsi->back->pdev->dev, 3193 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3194 ring->queue_index, pf_q, err); 3195 return -ENOMEM; 3196 } 3197 3198 /* set the context in the HMC */ 3199 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3200 if (err) { 3201 dev_info(&vsi->back->pdev->dev, 3202 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3203 ring->queue_index, pf_q, err); 3204 return -ENOMEM; 3205 } 3206 3207 /* Now associate this queue with this PCI function */ 3208 if (ring->ch) { 3209 if (ring->ch->type == I40E_VSI_VMDQ2) 3210 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3211 else 3212 return -EINVAL; 3213 3214 qtx_ctl |= (ring->ch->vsi_number << 3215 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3216 I40E_QTX_CTL_VFVM_INDX_MASK; 3217 } else { 3218 if (vsi->type == I40E_VSI_VMDQ2) { 3219 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3220 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3221 I40E_QTX_CTL_VFVM_INDX_MASK; 3222 } else { 3223 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3224 } 3225 } 3226 3227 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3228 I40E_QTX_CTL_PF_INDX_MASK); 3229 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3230 i40e_flush(hw); 3231 3232 /* cache tail off for easier writes later */ 3233 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3234 3235 return 0; 3236 } 3237 3238 /** 3239 * i40e_configure_rx_ring - Configure a receive ring context 3240 * @ring: The Rx ring to configure 3241 * 3242 * Configure the Rx descriptor ring in the HMC context. 3243 **/ 3244 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3245 { 3246 struct i40e_vsi *vsi = ring->vsi; 3247 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3248 u16 pf_q = vsi->base_queue + ring->queue_index; 3249 struct i40e_hw *hw = &vsi->back->hw; 3250 struct i40e_hmc_obj_rxq rx_ctx; 3251 i40e_status err = 0; 3252 bool ok; 3253 int ret; 3254 3255 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3256 3257 /* clear the context structure first */ 3258 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3259 3260 if (ring->vsi->type == I40E_VSI_MAIN) 3261 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); 3262 3263 kfree(ring->rx_bi); 3264 ring->xsk_umem = i40e_xsk_umem(ring); 3265 if (ring->xsk_umem) { 3266 ret = i40e_alloc_rx_bi_zc(ring); 3267 if (ret) 3268 return ret; 3269 ring->rx_buf_len = xsk_umem_get_rx_frame_size(ring->xsk_umem); 3270 /* For AF_XDP ZC, we disallow packets to span on 3271 * multiple buffers, thus letting us skip that 3272 * handling in the fast-path. 3273 */ 3274 chain_len = 1; 3275 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3276 MEM_TYPE_XSK_BUFF_POOL, 3277 NULL); 3278 if (ret) 3279 return ret; 3280 dev_info(&vsi->back->pdev->dev, 3281 "Registered XDP mem model MEM_TYPE_XSK_BUFF_POOL on Rx ring %d\n", 3282 ring->queue_index); 3283 3284 } else { 3285 ret = i40e_alloc_rx_bi(ring); 3286 if (ret) 3287 return ret; 3288 ring->rx_buf_len = vsi->rx_buf_len; 3289 if (ring->vsi->type == I40E_VSI_MAIN) { 3290 ret = xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, 3291 MEM_TYPE_PAGE_SHARED, 3292 NULL); 3293 if (ret) 3294 return ret; 3295 } 3296 } 3297 3298 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3299 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3300 3301 rx_ctx.base = (ring->dma / 128); 3302 rx_ctx.qlen = ring->count; 3303 3304 /* use 32 byte descriptors */ 3305 rx_ctx.dsize = 1; 3306 3307 /* descriptor type is always zero 3308 * rx_ctx.dtype = 0; 3309 */ 3310 rx_ctx.hsplit_0 = 0; 3311 3312 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3313 if (hw->revision_id == 0) 3314 rx_ctx.lrxqthresh = 0; 3315 else 3316 rx_ctx.lrxqthresh = 1; 3317 rx_ctx.crcstrip = 1; 3318 rx_ctx.l2tsel = 1; 3319 /* this controls whether VLAN is stripped from inner headers */ 3320 rx_ctx.showiv = 0; 3321 /* set the prefena field to 1 because the manual says to */ 3322 rx_ctx.prefena = 1; 3323 3324 /* clear the context in the HMC */ 3325 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3326 if (err) { 3327 dev_info(&vsi->back->pdev->dev, 3328 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3329 ring->queue_index, pf_q, err); 3330 return -ENOMEM; 3331 } 3332 3333 /* set the context in the HMC */ 3334 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3335 if (err) { 3336 dev_info(&vsi->back->pdev->dev, 3337 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3338 ring->queue_index, pf_q, err); 3339 return -ENOMEM; 3340 } 3341 3342 /* configure Rx buffer alignment */ 3343 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3344 clear_ring_build_skb_enabled(ring); 3345 else 3346 set_ring_build_skb_enabled(ring); 3347 3348 /* cache tail for quicker writes, and clear the reg before use */ 3349 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3350 writel(0, ring->tail); 3351 3352 if (ring->xsk_umem) { 3353 xsk_buff_set_rxq_info(ring->xsk_umem, &ring->xdp_rxq); 3354 ok = i40e_alloc_rx_buffers_zc(ring, I40E_DESC_UNUSED(ring)); 3355 } else { 3356 ok = !i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3357 } 3358 if (!ok) { 3359 /* Log this in case the user has forgotten to give the kernel 3360 * any buffers, even later in the application. 3361 */ 3362 dev_info(&vsi->back->pdev->dev, 3363 "Failed to allocate some buffers on %sRx ring %d (pf_q %d)\n", 3364 ring->xsk_umem ? "UMEM enabled " : "", 3365 ring->queue_index, pf_q); 3366 } 3367 3368 return 0; 3369 } 3370 3371 /** 3372 * i40e_vsi_configure_tx - Configure the VSI for Tx 3373 * @vsi: VSI structure describing this set of rings and resources 3374 * 3375 * Configure the Tx VSI for operation. 3376 **/ 3377 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3378 { 3379 int err = 0; 3380 u16 i; 3381 3382 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3383 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3384 3385 if (err || !i40e_enabled_xdp_vsi(vsi)) 3386 return err; 3387 3388 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3389 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3390 3391 return err; 3392 } 3393 3394 /** 3395 * i40e_vsi_configure_rx - Configure the VSI for Rx 3396 * @vsi: the VSI being configured 3397 * 3398 * Configure the Rx VSI for operation. 3399 **/ 3400 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3401 { 3402 int err = 0; 3403 u16 i; 3404 3405 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3406 vsi->max_frame = I40E_MAX_RXBUFFER; 3407 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3408 #if (PAGE_SIZE < 8192) 3409 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3410 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3411 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3412 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3413 #endif 3414 } else { 3415 vsi->max_frame = I40E_MAX_RXBUFFER; 3416 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3417 I40E_RXBUFFER_2048; 3418 } 3419 3420 /* set up individual rings */ 3421 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3422 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3423 3424 return err; 3425 } 3426 3427 /** 3428 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3429 * @vsi: ptr to the VSI 3430 **/ 3431 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3432 { 3433 struct i40e_ring *tx_ring, *rx_ring; 3434 u16 qoffset, qcount; 3435 int i, n; 3436 3437 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3438 /* Reset the TC information */ 3439 for (i = 0; i < vsi->num_queue_pairs; i++) { 3440 rx_ring = vsi->rx_rings[i]; 3441 tx_ring = vsi->tx_rings[i]; 3442 rx_ring->dcb_tc = 0; 3443 tx_ring->dcb_tc = 0; 3444 } 3445 return; 3446 } 3447 3448 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3449 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3450 continue; 3451 3452 qoffset = vsi->tc_config.tc_info[n].qoffset; 3453 qcount = vsi->tc_config.tc_info[n].qcount; 3454 for (i = qoffset; i < (qoffset + qcount); i++) { 3455 rx_ring = vsi->rx_rings[i]; 3456 tx_ring = vsi->tx_rings[i]; 3457 rx_ring->dcb_tc = n; 3458 tx_ring->dcb_tc = n; 3459 } 3460 } 3461 } 3462 3463 /** 3464 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3465 * @vsi: ptr to the VSI 3466 **/ 3467 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3468 { 3469 if (vsi->netdev) 3470 i40e_set_rx_mode(vsi->netdev); 3471 } 3472 3473 /** 3474 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3475 * @vsi: Pointer to the targeted VSI 3476 * 3477 * This function replays the hlist on the hw where all the SB Flow Director 3478 * filters were saved. 3479 **/ 3480 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3481 { 3482 struct i40e_fdir_filter *filter; 3483 struct i40e_pf *pf = vsi->back; 3484 struct hlist_node *node; 3485 3486 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3487 return; 3488 3489 /* Reset FDir counters as we're replaying all existing filters */ 3490 pf->fd_tcp4_filter_cnt = 0; 3491 pf->fd_udp4_filter_cnt = 0; 3492 pf->fd_sctp4_filter_cnt = 0; 3493 pf->fd_ip4_filter_cnt = 0; 3494 3495 hlist_for_each_entry_safe(filter, node, 3496 &pf->fdir_filter_list, fdir_node) { 3497 i40e_add_del_fdir(vsi, filter, true); 3498 } 3499 } 3500 3501 /** 3502 * i40e_vsi_configure - Set up the VSI for action 3503 * @vsi: the VSI being configured 3504 **/ 3505 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3506 { 3507 int err; 3508 3509 i40e_set_vsi_rx_mode(vsi); 3510 i40e_restore_vlan(vsi); 3511 i40e_vsi_config_dcb_rings(vsi); 3512 err = i40e_vsi_configure_tx(vsi); 3513 if (!err) 3514 err = i40e_vsi_configure_rx(vsi); 3515 3516 return err; 3517 } 3518 3519 /** 3520 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3521 * @vsi: the VSI being configured 3522 **/ 3523 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3524 { 3525 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3526 struct i40e_pf *pf = vsi->back; 3527 struct i40e_hw *hw = &pf->hw; 3528 u16 vector; 3529 int i, q; 3530 u32 qp; 3531 3532 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3533 * and PFINT_LNKLSTn registers, e.g.: 3534 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3535 */ 3536 qp = vsi->base_queue; 3537 vector = vsi->base_vector; 3538 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3539 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3540 3541 q_vector->rx.next_update = jiffies + 1; 3542 q_vector->rx.target_itr = 3543 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3544 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3545 q_vector->rx.target_itr >> 1); 3546 q_vector->rx.current_itr = q_vector->rx.target_itr; 3547 3548 q_vector->tx.next_update = jiffies + 1; 3549 q_vector->tx.target_itr = 3550 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3551 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3552 q_vector->tx.target_itr >> 1); 3553 q_vector->tx.current_itr = q_vector->tx.target_itr; 3554 3555 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3556 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3557 3558 /* Linked list for the queuepairs assigned to this vector */ 3559 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3560 for (q = 0; q < q_vector->num_ringpairs; q++) { 3561 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3562 u32 val; 3563 3564 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3565 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3566 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3567 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3568 (I40E_QUEUE_TYPE_TX << 3569 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3570 3571 wr32(hw, I40E_QINT_RQCTL(qp), val); 3572 3573 if (has_xdp) { 3574 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3575 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3576 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3577 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3578 (I40E_QUEUE_TYPE_TX << 3579 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3580 3581 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3582 } 3583 3584 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3585 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3586 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3587 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3588 (I40E_QUEUE_TYPE_RX << 3589 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3590 3591 /* Terminate the linked list */ 3592 if (q == (q_vector->num_ringpairs - 1)) 3593 val |= (I40E_QUEUE_END_OF_LIST << 3594 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3595 3596 wr32(hw, I40E_QINT_TQCTL(qp), val); 3597 qp++; 3598 } 3599 } 3600 3601 i40e_flush(hw); 3602 } 3603 3604 /** 3605 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3606 * @pf: pointer to private device data structure 3607 **/ 3608 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3609 { 3610 struct i40e_hw *hw = &pf->hw; 3611 u32 val; 3612 3613 /* clear things first */ 3614 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3615 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3616 3617 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3618 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3619 I40E_PFINT_ICR0_ENA_GRST_MASK | 3620 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3621 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3622 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3623 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3624 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3625 3626 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3627 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3628 3629 if (pf->flags & I40E_FLAG_PTP) 3630 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3631 3632 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3633 3634 /* SW_ITR_IDX = 0, but don't change INTENA */ 3635 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3636 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3637 3638 /* OTHER_ITR_IDX = 0 */ 3639 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3640 } 3641 3642 /** 3643 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3644 * @vsi: the VSI being configured 3645 **/ 3646 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3647 { 3648 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3649 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3650 struct i40e_pf *pf = vsi->back; 3651 struct i40e_hw *hw = &pf->hw; 3652 u32 val; 3653 3654 /* set the ITR configuration */ 3655 q_vector->rx.next_update = jiffies + 1; 3656 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 3657 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr >> 1); 3658 q_vector->rx.current_itr = q_vector->rx.target_itr; 3659 q_vector->tx.next_update = jiffies + 1; 3660 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 3661 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr >> 1); 3662 q_vector->tx.current_itr = q_vector->tx.target_itr; 3663 3664 i40e_enable_misc_int_causes(pf); 3665 3666 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3667 wr32(hw, I40E_PFINT_LNKLST0, 0); 3668 3669 /* Associate the queue pair to the vector and enable the queue int */ 3670 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3671 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3672 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3673 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3674 3675 wr32(hw, I40E_QINT_RQCTL(0), val); 3676 3677 if (i40e_enabled_xdp_vsi(vsi)) { 3678 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3679 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3680 (I40E_QUEUE_TYPE_TX 3681 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3682 3683 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3684 } 3685 3686 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3687 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3688 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3689 3690 wr32(hw, I40E_QINT_TQCTL(0), val); 3691 i40e_flush(hw); 3692 } 3693 3694 /** 3695 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3696 * @pf: board private structure 3697 **/ 3698 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3699 { 3700 struct i40e_hw *hw = &pf->hw; 3701 3702 wr32(hw, I40E_PFINT_DYN_CTL0, 3703 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3704 i40e_flush(hw); 3705 } 3706 3707 /** 3708 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3709 * @pf: board private structure 3710 **/ 3711 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 3712 { 3713 struct i40e_hw *hw = &pf->hw; 3714 u32 val; 3715 3716 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3717 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3718 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3719 3720 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3721 i40e_flush(hw); 3722 } 3723 3724 /** 3725 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3726 * @irq: interrupt number 3727 * @data: pointer to a q_vector 3728 **/ 3729 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3730 { 3731 struct i40e_q_vector *q_vector = data; 3732 3733 if (!q_vector->tx.ring && !q_vector->rx.ring) 3734 return IRQ_HANDLED; 3735 3736 napi_schedule_irqoff(&q_vector->napi); 3737 3738 return IRQ_HANDLED; 3739 } 3740 3741 /** 3742 * i40e_irq_affinity_notify - Callback for affinity changes 3743 * @notify: context as to what irq was changed 3744 * @mask: the new affinity mask 3745 * 3746 * This is a callback function used by the irq_set_affinity_notifier function 3747 * so that we may register to receive changes to the irq affinity masks. 3748 **/ 3749 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3750 const cpumask_t *mask) 3751 { 3752 struct i40e_q_vector *q_vector = 3753 container_of(notify, struct i40e_q_vector, affinity_notify); 3754 3755 cpumask_copy(&q_vector->affinity_mask, mask); 3756 } 3757 3758 /** 3759 * i40e_irq_affinity_release - Callback for affinity notifier release 3760 * @ref: internal core kernel usage 3761 * 3762 * This is a callback function used by the irq_set_affinity_notifier function 3763 * to inform the current notification subscriber that they will no longer 3764 * receive notifications. 3765 **/ 3766 static void i40e_irq_affinity_release(struct kref *ref) {} 3767 3768 /** 3769 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3770 * @vsi: the VSI being configured 3771 * @basename: name for the vector 3772 * 3773 * Allocates MSI-X vectors and requests interrupts from the kernel. 3774 **/ 3775 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3776 { 3777 int q_vectors = vsi->num_q_vectors; 3778 struct i40e_pf *pf = vsi->back; 3779 int base = vsi->base_vector; 3780 int rx_int_idx = 0; 3781 int tx_int_idx = 0; 3782 int vector, err; 3783 int irq_num; 3784 int cpu; 3785 3786 for (vector = 0; vector < q_vectors; vector++) { 3787 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3788 3789 irq_num = pf->msix_entries[base + vector].vector; 3790 3791 if (q_vector->tx.ring && q_vector->rx.ring) { 3792 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3793 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3794 tx_int_idx++; 3795 } else if (q_vector->rx.ring) { 3796 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3797 "%s-%s-%d", basename, "rx", rx_int_idx++); 3798 } else if (q_vector->tx.ring) { 3799 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3800 "%s-%s-%d", basename, "tx", tx_int_idx++); 3801 } else { 3802 /* skip this unused q_vector */ 3803 continue; 3804 } 3805 err = request_irq(irq_num, 3806 vsi->irq_handler, 3807 0, 3808 q_vector->name, 3809 q_vector); 3810 if (err) { 3811 dev_info(&pf->pdev->dev, 3812 "MSIX request_irq failed, error: %d\n", err); 3813 goto free_queue_irqs; 3814 } 3815 3816 /* register for affinity change notifications */ 3817 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3818 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3819 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3820 /* Spread affinity hints out across online CPUs. 3821 * 3822 * get_cpu_mask returns a static constant mask with 3823 * a permanent lifetime so it's ok to pass to 3824 * irq_set_affinity_hint without making a copy. 3825 */ 3826 cpu = cpumask_local_spread(q_vector->v_idx, -1); 3827 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 3828 } 3829 3830 vsi->irqs_ready = true; 3831 return 0; 3832 3833 free_queue_irqs: 3834 while (vector) { 3835 vector--; 3836 irq_num = pf->msix_entries[base + vector].vector; 3837 irq_set_affinity_notifier(irq_num, NULL); 3838 irq_set_affinity_hint(irq_num, NULL); 3839 free_irq(irq_num, &vsi->q_vectors[vector]); 3840 } 3841 return err; 3842 } 3843 3844 /** 3845 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3846 * @vsi: the VSI being un-configured 3847 **/ 3848 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3849 { 3850 struct i40e_pf *pf = vsi->back; 3851 struct i40e_hw *hw = &pf->hw; 3852 int base = vsi->base_vector; 3853 int i; 3854 3855 /* disable interrupt causation from each queue */ 3856 for (i = 0; i < vsi->num_queue_pairs; i++) { 3857 u32 val; 3858 3859 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3860 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3861 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3862 3863 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3864 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3865 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3866 3867 if (!i40e_enabled_xdp_vsi(vsi)) 3868 continue; 3869 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3870 } 3871 3872 /* disable each interrupt */ 3873 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3874 for (i = vsi->base_vector; 3875 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3876 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3877 3878 i40e_flush(hw); 3879 for (i = 0; i < vsi->num_q_vectors; i++) 3880 synchronize_irq(pf->msix_entries[i + base].vector); 3881 } else { 3882 /* Legacy and MSI mode - this stops all interrupt handling */ 3883 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3884 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3885 i40e_flush(hw); 3886 synchronize_irq(pf->pdev->irq); 3887 } 3888 } 3889 3890 /** 3891 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3892 * @vsi: the VSI being configured 3893 **/ 3894 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3895 { 3896 struct i40e_pf *pf = vsi->back; 3897 int i; 3898 3899 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3900 for (i = 0; i < vsi->num_q_vectors; i++) 3901 i40e_irq_dynamic_enable(vsi, i); 3902 } else { 3903 i40e_irq_dynamic_enable_icr0(pf); 3904 } 3905 3906 i40e_flush(&pf->hw); 3907 return 0; 3908 } 3909 3910 /** 3911 * i40e_free_misc_vector - Free the vector that handles non-queue events 3912 * @pf: board private structure 3913 **/ 3914 static void i40e_free_misc_vector(struct i40e_pf *pf) 3915 { 3916 /* Disable ICR 0 */ 3917 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3918 i40e_flush(&pf->hw); 3919 3920 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3921 synchronize_irq(pf->msix_entries[0].vector); 3922 free_irq(pf->msix_entries[0].vector, pf); 3923 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3924 } 3925 } 3926 3927 /** 3928 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3929 * @irq: interrupt number 3930 * @data: pointer to a q_vector 3931 * 3932 * This is the handler used for all MSI/Legacy interrupts, and deals 3933 * with both queue and non-queue interrupts. This is also used in 3934 * MSIX mode to handle the non-queue interrupts. 3935 **/ 3936 static irqreturn_t i40e_intr(int irq, void *data) 3937 { 3938 struct i40e_pf *pf = (struct i40e_pf *)data; 3939 struct i40e_hw *hw = &pf->hw; 3940 irqreturn_t ret = IRQ_NONE; 3941 u32 icr0, icr0_remaining; 3942 u32 val, ena_mask; 3943 3944 icr0 = rd32(hw, I40E_PFINT_ICR0); 3945 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3946 3947 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3948 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3949 goto enable_intr; 3950 3951 /* if interrupt but no bits showing, must be SWINT */ 3952 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3953 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3954 pf->sw_int_count++; 3955 3956 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3957 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3958 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3959 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3960 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3961 } 3962 3963 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3964 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3965 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3966 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3967 3968 /* We do not have a way to disarm Queue causes while leaving 3969 * interrupt enabled for all other causes, ideally 3970 * interrupt should be disabled while we are in NAPI but 3971 * this is not a performance path and napi_schedule() 3972 * can deal with rescheduling. 3973 */ 3974 if (!test_bit(__I40E_DOWN, pf->state)) 3975 napi_schedule_irqoff(&q_vector->napi); 3976 } 3977 3978 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3979 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3980 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 3981 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3982 } 3983 3984 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3985 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3986 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 3987 } 3988 3989 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3990 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3991 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 3992 } 3993 3994 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3995 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 3996 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 3997 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3998 val = rd32(hw, I40E_GLGEN_RSTAT); 3999 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 4000 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 4001 if (val == I40E_RESET_CORER) { 4002 pf->corer_count++; 4003 } else if (val == I40E_RESET_GLOBR) { 4004 pf->globr_count++; 4005 } else if (val == I40E_RESET_EMPR) { 4006 pf->empr_count++; 4007 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 4008 } 4009 } 4010 4011 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 4012 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 4013 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 4014 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 4015 rd32(hw, I40E_PFHMC_ERRORINFO), 4016 rd32(hw, I40E_PFHMC_ERRORDATA)); 4017 } 4018 4019 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 4020 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 4021 4022 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 4023 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 4024 i40e_ptp_tx_hwtstamp(pf); 4025 } 4026 } 4027 4028 /* If a critical error is pending we have no choice but to reset the 4029 * device. 4030 * Report and mask out any remaining unexpected interrupts. 4031 */ 4032 icr0_remaining = icr0 & ena_mask; 4033 if (icr0_remaining) { 4034 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 4035 icr0_remaining); 4036 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 4037 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 4038 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 4039 dev_info(&pf->pdev->dev, "device will be reset\n"); 4040 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 4041 i40e_service_event_schedule(pf); 4042 } 4043 ena_mask &= ~icr0_remaining; 4044 } 4045 ret = IRQ_HANDLED; 4046 4047 enable_intr: 4048 /* re-enable interrupt causes */ 4049 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 4050 if (!test_bit(__I40E_DOWN, pf->state) || 4051 test_bit(__I40E_RECOVERY_MODE, pf->state)) { 4052 i40e_service_event_schedule(pf); 4053 i40e_irq_dynamic_enable_icr0(pf); 4054 } 4055 4056 return ret; 4057 } 4058 4059 /** 4060 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 4061 * @tx_ring: tx ring to clean 4062 * @budget: how many cleans we're allowed 4063 * 4064 * Returns true if there's any budget left (e.g. the clean is finished) 4065 **/ 4066 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 4067 { 4068 struct i40e_vsi *vsi = tx_ring->vsi; 4069 u16 i = tx_ring->next_to_clean; 4070 struct i40e_tx_buffer *tx_buf; 4071 struct i40e_tx_desc *tx_desc; 4072 4073 tx_buf = &tx_ring->tx_bi[i]; 4074 tx_desc = I40E_TX_DESC(tx_ring, i); 4075 i -= tx_ring->count; 4076 4077 do { 4078 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 4079 4080 /* if next_to_watch is not set then there is no work pending */ 4081 if (!eop_desc) 4082 break; 4083 4084 /* prevent any other reads prior to eop_desc */ 4085 smp_rmb(); 4086 4087 /* if the descriptor isn't done, no work yet to do */ 4088 if (!(eop_desc->cmd_type_offset_bsz & 4089 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 4090 break; 4091 4092 /* clear next_to_watch to prevent false hangs */ 4093 tx_buf->next_to_watch = NULL; 4094 4095 tx_desc->buffer_addr = 0; 4096 tx_desc->cmd_type_offset_bsz = 0; 4097 /* move past filter desc */ 4098 tx_buf++; 4099 tx_desc++; 4100 i++; 4101 if (unlikely(!i)) { 4102 i -= tx_ring->count; 4103 tx_buf = tx_ring->tx_bi; 4104 tx_desc = I40E_TX_DESC(tx_ring, 0); 4105 } 4106 /* unmap skb header data */ 4107 dma_unmap_single(tx_ring->dev, 4108 dma_unmap_addr(tx_buf, dma), 4109 dma_unmap_len(tx_buf, len), 4110 DMA_TO_DEVICE); 4111 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 4112 kfree(tx_buf->raw_buf); 4113 4114 tx_buf->raw_buf = NULL; 4115 tx_buf->tx_flags = 0; 4116 tx_buf->next_to_watch = NULL; 4117 dma_unmap_len_set(tx_buf, len, 0); 4118 tx_desc->buffer_addr = 0; 4119 tx_desc->cmd_type_offset_bsz = 0; 4120 4121 /* move us past the eop_desc for start of next FD desc */ 4122 tx_buf++; 4123 tx_desc++; 4124 i++; 4125 if (unlikely(!i)) { 4126 i -= tx_ring->count; 4127 tx_buf = tx_ring->tx_bi; 4128 tx_desc = I40E_TX_DESC(tx_ring, 0); 4129 } 4130 4131 /* update budget accounting */ 4132 budget--; 4133 } while (likely(budget)); 4134 4135 i += tx_ring->count; 4136 tx_ring->next_to_clean = i; 4137 4138 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4139 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4140 4141 return budget > 0; 4142 } 4143 4144 /** 4145 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4146 * @irq: interrupt number 4147 * @data: pointer to a q_vector 4148 **/ 4149 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4150 { 4151 struct i40e_q_vector *q_vector = data; 4152 struct i40e_vsi *vsi; 4153 4154 if (!q_vector->tx.ring) 4155 return IRQ_HANDLED; 4156 4157 vsi = q_vector->tx.ring->vsi; 4158 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4159 4160 return IRQ_HANDLED; 4161 } 4162 4163 /** 4164 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4165 * @vsi: the VSI being configured 4166 * @v_idx: vector index 4167 * @qp_idx: queue pair index 4168 **/ 4169 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4170 { 4171 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4172 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4173 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4174 4175 tx_ring->q_vector = q_vector; 4176 tx_ring->next = q_vector->tx.ring; 4177 q_vector->tx.ring = tx_ring; 4178 q_vector->tx.count++; 4179 4180 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4181 if (i40e_enabled_xdp_vsi(vsi)) { 4182 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4183 4184 xdp_ring->q_vector = q_vector; 4185 xdp_ring->next = q_vector->tx.ring; 4186 q_vector->tx.ring = xdp_ring; 4187 q_vector->tx.count++; 4188 } 4189 4190 rx_ring->q_vector = q_vector; 4191 rx_ring->next = q_vector->rx.ring; 4192 q_vector->rx.ring = rx_ring; 4193 q_vector->rx.count++; 4194 } 4195 4196 /** 4197 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4198 * @vsi: the VSI being configured 4199 * 4200 * This function maps descriptor rings to the queue-specific vectors 4201 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4202 * one vector per queue pair, but on a constrained vector budget, we 4203 * group the queue pairs as "efficiently" as possible. 4204 **/ 4205 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4206 { 4207 int qp_remaining = vsi->num_queue_pairs; 4208 int q_vectors = vsi->num_q_vectors; 4209 int num_ringpairs; 4210 int v_start = 0; 4211 int qp_idx = 0; 4212 4213 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4214 * group them so there are multiple queues per vector. 4215 * It is also important to go through all the vectors available to be 4216 * sure that if we don't use all the vectors, that the remaining vectors 4217 * are cleared. This is especially important when decreasing the 4218 * number of queues in use. 4219 */ 4220 for (; v_start < q_vectors; v_start++) { 4221 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4222 4223 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4224 4225 q_vector->num_ringpairs = num_ringpairs; 4226 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4227 4228 q_vector->rx.count = 0; 4229 q_vector->tx.count = 0; 4230 q_vector->rx.ring = NULL; 4231 q_vector->tx.ring = NULL; 4232 4233 while (num_ringpairs--) { 4234 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4235 qp_idx++; 4236 qp_remaining--; 4237 } 4238 } 4239 } 4240 4241 /** 4242 * i40e_vsi_request_irq - Request IRQ from the OS 4243 * @vsi: the VSI being configured 4244 * @basename: name for the vector 4245 **/ 4246 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4247 { 4248 struct i40e_pf *pf = vsi->back; 4249 int err; 4250 4251 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4252 err = i40e_vsi_request_irq_msix(vsi, basename); 4253 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4254 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4255 pf->int_name, pf); 4256 else 4257 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4258 pf->int_name, pf); 4259 4260 if (err) 4261 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4262 4263 return err; 4264 } 4265 4266 #ifdef CONFIG_NET_POLL_CONTROLLER 4267 /** 4268 * i40e_netpoll - A Polling 'interrupt' handler 4269 * @netdev: network interface device structure 4270 * 4271 * This is used by netconsole to send skbs without having to re-enable 4272 * interrupts. It's not called while the normal interrupt routine is executing. 4273 **/ 4274 static void i40e_netpoll(struct net_device *netdev) 4275 { 4276 struct i40e_netdev_priv *np = netdev_priv(netdev); 4277 struct i40e_vsi *vsi = np->vsi; 4278 struct i40e_pf *pf = vsi->back; 4279 int i; 4280 4281 /* if interface is down do nothing */ 4282 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4283 return; 4284 4285 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4286 for (i = 0; i < vsi->num_q_vectors; i++) 4287 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4288 } else { 4289 i40e_intr(pf->pdev->irq, netdev); 4290 } 4291 } 4292 #endif 4293 4294 #define I40E_QTX_ENA_WAIT_COUNT 50 4295 4296 /** 4297 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4298 * @pf: the PF being configured 4299 * @pf_q: the PF queue 4300 * @enable: enable or disable state of the queue 4301 * 4302 * This routine will wait for the given Tx queue of the PF to reach the 4303 * enabled or disabled state. 4304 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4305 * multiple retries; else will return 0 in case of success. 4306 **/ 4307 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4308 { 4309 int i; 4310 u32 tx_reg; 4311 4312 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4313 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4314 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4315 break; 4316 4317 usleep_range(10, 20); 4318 } 4319 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4320 return -ETIMEDOUT; 4321 4322 return 0; 4323 } 4324 4325 /** 4326 * i40e_control_tx_q - Start or stop a particular Tx queue 4327 * @pf: the PF structure 4328 * @pf_q: the PF queue to configure 4329 * @enable: start or stop the queue 4330 * 4331 * This function enables or disables a single queue. Note that any delay 4332 * required after the operation is expected to be handled by the caller of 4333 * this function. 4334 **/ 4335 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4336 { 4337 struct i40e_hw *hw = &pf->hw; 4338 u32 tx_reg; 4339 int i; 4340 4341 /* warn the TX unit of coming changes */ 4342 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4343 if (!enable) 4344 usleep_range(10, 20); 4345 4346 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4347 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4348 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4349 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4350 break; 4351 usleep_range(1000, 2000); 4352 } 4353 4354 /* Skip if the queue is already in the requested state */ 4355 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4356 return; 4357 4358 /* turn on/off the queue */ 4359 if (enable) { 4360 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4361 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4362 } else { 4363 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4364 } 4365 4366 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4367 } 4368 4369 /** 4370 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4371 * @seid: VSI SEID 4372 * @pf: the PF structure 4373 * @pf_q: the PF queue to configure 4374 * @is_xdp: true if the queue is used for XDP 4375 * @enable: start or stop the queue 4376 **/ 4377 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4378 bool is_xdp, bool enable) 4379 { 4380 int ret; 4381 4382 i40e_control_tx_q(pf, pf_q, enable); 4383 4384 /* wait for the change to finish */ 4385 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4386 if (ret) { 4387 dev_info(&pf->pdev->dev, 4388 "VSI seid %d %sTx ring %d %sable timeout\n", 4389 seid, (is_xdp ? "XDP " : ""), pf_q, 4390 (enable ? "en" : "dis")); 4391 } 4392 4393 return ret; 4394 } 4395 4396 /** 4397 * i40e_vsi_control_tx - Start or stop a VSI's rings 4398 * @vsi: the VSI being configured 4399 * @enable: start or stop the rings 4400 **/ 4401 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4402 { 4403 struct i40e_pf *pf = vsi->back; 4404 int i, pf_q, ret = 0; 4405 4406 pf_q = vsi->base_queue; 4407 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4408 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4409 pf_q, 4410 false /*is xdp*/, enable); 4411 if (ret) 4412 break; 4413 4414 if (!i40e_enabled_xdp_vsi(vsi)) 4415 continue; 4416 4417 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4418 pf_q + vsi->alloc_queue_pairs, 4419 true /*is xdp*/, enable); 4420 if (ret) 4421 break; 4422 } 4423 return ret; 4424 } 4425 4426 /** 4427 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4428 * @pf: the PF being configured 4429 * @pf_q: the PF queue 4430 * @enable: enable or disable state of the queue 4431 * 4432 * This routine will wait for the given Rx queue of the PF to reach the 4433 * enabled or disabled state. 4434 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4435 * multiple retries; else will return 0 in case of success. 4436 **/ 4437 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4438 { 4439 int i; 4440 u32 rx_reg; 4441 4442 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4443 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4444 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4445 break; 4446 4447 usleep_range(10, 20); 4448 } 4449 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4450 return -ETIMEDOUT; 4451 4452 return 0; 4453 } 4454 4455 /** 4456 * i40e_control_rx_q - Start or stop a particular Rx queue 4457 * @pf: the PF structure 4458 * @pf_q: the PF queue to configure 4459 * @enable: start or stop the queue 4460 * 4461 * This function enables or disables a single queue. Note that 4462 * any delay required after the operation is expected to be 4463 * handled by the caller of this function. 4464 **/ 4465 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4466 { 4467 struct i40e_hw *hw = &pf->hw; 4468 u32 rx_reg; 4469 int i; 4470 4471 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4472 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4473 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4474 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4475 break; 4476 usleep_range(1000, 2000); 4477 } 4478 4479 /* Skip if the queue is already in the requested state */ 4480 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4481 return; 4482 4483 /* turn on/off the queue */ 4484 if (enable) 4485 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4486 else 4487 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4488 4489 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4490 } 4491 4492 /** 4493 * i40e_control_wait_rx_q 4494 * @pf: the PF structure 4495 * @pf_q: queue being configured 4496 * @enable: start or stop the rings 4497 * 4498 * This function enables or disables a single queue along with waiting 4499 * for the change to finish. The caller of this function should handle 4500 * the delays needed in the case of disabling queues. 4501 **/ 4502 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4503 { 4504 int ret = 0; 4505 4506 i40e_control_rx_q(pf, pf_q, enable); 4507 4508 /* wait for the change to finish */ 4509 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4510 if (ret) 4511 return ret; 4512 4513 return ret; 4514 } 4515 4516 /** 4517 * i40e_vsi_control_rx - Start or stop a VSI's rings 4518 * @vsi: the VSI being configured 4519 * @enable: start or stop the rings 4520 **/ 4521 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4522 { 4523 struct i40e_pf *pf = vsi->back; 4524 int i, pf_q, ret = 0; 4525 4526 pf_q = vsi->base_queue; 4527 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4528 ret = i40e_control_wait_rx_q(pf, pf_q, enable); 4529 if (ret) { 4530 dev_info(&pf->pdev->dev, 4531 "VSI seid %d Rx ring %d %sable timeout\n", 4532 vsi->seid, pf_q, (enable ? "en" : "dis")); 4533 break; 4534 } 4535 } 4536 4537 /* Due to HW errata, on Rx disable only, the register can indicate done 4538 * before it really is. Needs 50ms to be sure 4539 */ 4540 if (!enable) 4541 mdelay(50); 4542 4543 return ret; 4544 } 4545 4546 /** 4547 * i40e_vsi_start_rings - Start a VSI's rings 4548 * @vsi: the VSI being configured 4549 **/ 4550 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4551 { 4552 int ret = 0; 4553 4554 /* do rx first for enable and last for disable */ 4555 ret = i40e_vsi_control_rx(vsi, true); 4556 if (ret) 4557 return ret; 4558 ret = i40e_vsi_control_tx(vsi, true); 4559 4560 return ret; 4561 } 4562 4563 /** 4564 * i40e_vsi_stop_rings - Stop a VSI's rings 4565 * @vsi: the VSI being configured 4566 **/ 4567 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4568 { 4569 /* When port TX is suspended, don't wait */ 4570 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4571 return i40e_vsi_stop_rings_no_wait(vsi); 4572 4573 /* do rx first for enable and last for disable 4574 * Ignore return value, we need to shutdown whatever we can 4575 */ 4576 i40e_vsi_control_tx(vsi, false); 4577 i40e_vsi_control_rx(vsi, false); 4578 } 4579 4580 /** 4581 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4582 * @vsi: the VSI being shutdown 4583 * 4584 * This function stops all the rings for a VSI but does not delay to verify 4585 * that rings have been disabled. It is expected that the caller is shutting 4586 * down multiple VSIs at once and will delay together for all the VSIs after 4587 * initiating the shutdown. This is particularly useful for shutting down lots 4588 * of VFs together. Otherwise, a large delay can be incurred while configuring 4589 * each VSI in serial. 4590 **/ 4591 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4592 { 4593 struct i40e_pf *pf = vsi->back; 4594 int i, pf_q; 4595 4596 pf_q = vsi->base_queue; 4597 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4598 i40e_control_tx_q(pf, pf_q, false); 4599 i40e_control_rx_q(pf, pf_q, false); 4600 } 4601 } 4602 4603 /** 4604 * i40e_vsi_free_irq - Free the irq association with the OS 4605 * @vsi: the VSI being configured 4606 **/ 4607 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4608 { 4609 struct i40e_pf *pf = vsi->back; 4610 struct i40e_hw *hw = &pf->hw; 4611 int base = vsi->base_vector; 4612 u32 val, qp; 4613 int i; 4614 4615 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4616 if (!vsi->q_vectors) 4617 return; 4618 4619 if (!vsi->irqs_ready) 4620 return; 4621 4622 vsi->irqs_ready = false; 4623 for (i = 0; i < vsi->num_q_vectors; i++) { 4624 int irq_num; 4625 u16 vector; 4626 4627 vector = i + base; 4628 irq_num = pf->msix_entries[vector].vector; 4629 4630 /* free only the irqs that were actually requested */ 4631 if (!vsi->q_vectors[i] || 4632 !vsi->q_vectors[i]->num_ringpairs) 4633 continue; 4634 4635 /* clear the affinity notifier in the IRQ descriptor */ 4636 irq_set_affinity_notifier(irq_num, NULL); 4637 /* remove our suggested affinity mask for this IRQ */ 4638 irq_set_affinity_hint(irq_num, NULL); 4639 synchronize_irq(irq_num); 4640 free_irq(irq_num, vsi->q_vectors[i]); 4641 4642 /* Tear down the interrupt queue link list 4643 * 4644 * We know that they come in pairs and always 4645 * the Rx first, then the Tx. To clear the 4646 * link list, stick the EOL value into the 4647 * next_q field of the registers. 4648 */ 4649 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4650 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4651 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4652 val |= I40E_QUEUE_END_OF_LIST 4653 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4654 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4655 4656 while (qp != I40E_QUEUE_END_OF_LIST) { 4657 u32 next; 4658 4659 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4660 4661 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4662 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4663 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4664 I40E_QINT_RQCTL_INTEVENT_MASK); 4665 4666 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4667 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4668 4669 wr32(hw, I40E_QINT_RQCTL(qp), val); 4670 4671 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4672 4673 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4674 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4675 4676 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4677 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4678 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4679 I40E_QINT_TQCTL_INTEVENT_MASK); 4680 4681 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4682 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4683 4684 wr32(hw, I40E_QINT_TQCTL(qp), val); 4685 qp = next; 4686 } 4687 } 4688 } else { 4689 free_irq(pf->pdev->irq, pf); 4690 4691 val = rd32(hw, I40E_PFINT_LNKLST0); 4692 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4693 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4694 val |= I40E_QUEUE_END_OF_LIST 4695 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4696 wr32(hw, I40E_PFINT_LNKLST0, val); 4697 4698 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4699 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4700 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4701 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4702 I40E_QINT_RQCTL_INTEVENT_MASK); 4703 4704 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4705 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4706 4707 wr32(hw, I40E_QINT_RQCTL(qp), val); 4708 4709 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4710 4711 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4712 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4713 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4714 I40E_QINT_TQCTL_INTEVENT_MASK); 4715 4716 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4717 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4718 4719 wr32(hw, I40E_QINT_TQCTL(qp), val); 4720 } 4721 } 4722 4723 /** 4724 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4725 * @vsi: the VSI being configured 4726 * @v_idx: Index of vector to be freed 4727 * 4728 * This function frees the memory allocated to the q_vector. In addition if 4729 * NAPI is enabled it will delete any references to the NAPI struct prior 4730 * to freeing the q_vector. 4731 **/ 4732 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4733 { 4734 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4735 struct i40e_ring *ring; 4736 4737 if (!q_vector) 4738 return; 4739 4740 /* disassociate q_vector from rings */ 4741 i40e_for_each_ring(ring, q_vector->tx) 4742 ring->q_vector = NULL; 4743 4744 i40e_for_each_ring(ring, q_vector->rx) 4745 ring->q_vector = NULL; 4746 4747 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4748 if (vsi->netdev) 4749 netif_napi_del(&q_vector->napi); 4750 4751 vsi->q_vectors[v_idx] = NULL; 4752 4753 kfree_rcu(q_vector, rcu); 4754 } 4755 4756 /** 4757 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4758 * @vsi: the VSI being un-configured 4759 * 4760 * This frees the memory allocated to the q_vectors and 4761 * deletes references to the NAPI struct. 4762 **/ 4763 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4764 { 4765 int v_idx; 4766 4767 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4768 i40e_free_q_vector(vsi, v_idx); 4769 } 4770 4771 /** 4772 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4773 * @pf: board private structure 4774 **/ 4775 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4776 { 4777 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4778 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4779 pci_disable_msix(pf->pdev); 4780 kfree(pf->msix_entries); 4781 pf->msix_entries = NULL; 4782 kfree(pf->irq_pile); 4783 pf->irq_pile = NULL; 4784 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4785 pci_disable_msi(pf->pdev); 4786 } 4787 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4788 } 4789 4790 /** 4791 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4792 * @pf: board private structure 4793 * 4794 * We go through and clear interrupt specific resources and reset the structure 4795 * to pre-load conditions 4796 **/ 4797 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4798 { 4799 int i; 4800 4801 i40e_free_misc_vector(pf); 4802 4803 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4804 I40E_IWARP_IRQ_PILE_ID); 4805 4806 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4807 for (i = 0; i < pf->num_alloc_vsi; i++) 4808 if (pf->vsi[i]) 4809 i40e_vsi_free_q_vectors(pf->vsi[i]); 4810 i40e_reset_interrupt_capability(pf); 4811 } 4812 4813 /** 4814 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4815 * @vsi: the VSI being configured 4816 **/ 4817 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4818 { 4819 int q_idx; 4820 4821 if (!vsi->netdev) 4822 return; 4823 4824 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4825 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4826 4827 if (q_vector->rx.ring || q_vector->tx.ring) 4828 napi_enable(&q_vector->napi); 4829 } 4830 } 4831 4832 /** 4833 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4834 * @vsi: the VSI being configured 4835 **/ 4836 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4837 { 4838 int q_idx; 4839 4840 if (!vsi->netdev) 4841 return; 4842 4843 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4844 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4845 4846 if (q_vector->rx.ring || q_vector->tx.ring) 4847 napi_disable(&q_vector->napi); 4848 } 4849 } 4850 4851 /** 4852 * i40e_vsi_close - Shut down a VSI 4853 * @vsi: the vsi to be quelled 4854 **/ 4855 static void i40e_vsi_close(struct i40e_vsi *vsi) 4856 { 4857 struct i40e_pf *pf = vsi->back; 4858 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4859 i40e_down(vsi); 4860 i40e_vsi_free_irq(vsi); 4861 i40e_vsi_free_tx_resources(vsi); 4862 i40e_vsi_free_rx_resources(vsi); 4863 vsi->current_netdev_flags = 0; 4864 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 4865 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4866 set_bit(__I40E_CLIENT_RESET, pf->state); 4867 } 4868 4869 /** 4870 * i40e_quiesce_vsi - Pause a given VSI 4871 * @vsi: the VSI being paused 4872 **/ 4873 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4874 { 4875 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4876 return; 4877 4878 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4879 if (vsi->netdev && netif_running(vsi->netdev)) 4880 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4881 else 4882 i40e_vsi_close(vsi); 4883 } 4884 4885 /** 4886 * i40e_unquiesce_vsi - Resume a given VSI 4887 * @vsi: the VSI being resumed 4888 **/ 4889 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4890 { 4891 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4892 return; 4893 4894 if (vsi->netdev && netif_running(vsi->netdev)) 4895 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4896 else 4897 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4898 } 4899 4900 /** 4901 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4902 * @pf: the PF 4903 **/ 4904 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4905 { 4906 int v; 4907 4908 for (v = 0; v < pf->num_alloc_vsi; v++) { 4909 if (pf->vsi[v]) 4910 i40e_quiesce_vsi(pf->vsi[v]); 4911 } 4912 } 4913 4914 /** 4915 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4916 * @pf: the PF 4917 **/ 4918 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4919 { 4920 int v; 4921 4922 for (v = 0; v < pf->num_alloc_vsi; v++) { 4923 if (pf->vsi[v]) 4924 i40e_unquiesce_vsi(pf->vsi[v]); 4925 } 4926 } 4927 4928 /** 4929 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4930 * @vsi: the VSI being configured 4931 * 4932 * Wait until all queues on a given VSI have been disabled. 4933 **/ 4934 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4935 { 4936 struct i40e_pf *pf = vsi->back; 4937 int i, pf_q, ret; 4938 4939 pf_q = vsi->base_queue; 4940 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4941 /* Check and wait for the Tx queue */ 4942 ret = i40e_pf_txq_wait(pf, pf_q, false); 4943 if (ret) { 4944 dev_info(&pf->pdev->dev, 4945 "VSI seid %d Tx ring %d disable timeout\n", 4946 vsi->seid, pf_q); 4947 return ret; 4948 } 4949 4950 if (!i40e_enabled_xdp_vsi(vsi)) 4951 goto wait_rx; 4952 4953 /* Check and wait for the XDP Tx queue */ 4954 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4955 false); 4956 if (ret) { 4957 dev_info(&pf->pdev->dev, 4958 "VSI seid %d XDP Tx ring %d disable timeout\n", 4959 vsi->seid, pf_q); 4960 return ret; 4961 } 4962 wait_rx: 4963 /* Check and wait for the Rx queue */ 4964 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4965 if (ret) { 4966 dev_info(&pf->pdev->dev, 4967 "VSI seid %d Rx ring %d disable timeout\n", 4968 vsi->seid, pf_q); 4969 return ret; 4970 } 4971 } 4972 4973 return 0; 4974 } 4975 4976 #ifdef CONFIG_I40E_DCB 4977 /** 4978 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4979 * @pf: the PF 4980 * 4981 * This function waits for the queues to be in disabled state for all the 4982 * VSIs that are managed by this PF. 4983 **/ 4984 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4985 { 4986 int v, ret = 0; 4987 4988 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4989 if (pf->vsi[v]) { 4990 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4991 if (ret) 4992 break; 4993 } 4994 } 4995 4996 return ret; 4997 } 4998 4999 #endif 5000 5001 /** 5002 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 5003 * @pf: pointer to PF 5004 * 5005 * Get TC map for ISCSI PF type that will include iSCSI TC 5006 * and LAN TC. 5007 **/ 5008 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 5009 { 5010 struct i40e_dcb_app_priority_table app; 5011 struct i40e_hw *hw = &pf->hw; 5012 u8 enabled_tc = 1; /* TC0 is always enabled */ 5013 u8 tc, i; 5014 /* Get the iSCSI APP TLV */ 5015 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5016 5017 for (i = 0; i < dcbcfg->numapps; i++) { 5018 app = dcbcfg->app[i]; 5019 if (app.selector == I40E_APP_SEL_TCPIP && 5020 app.protocolid == I40E_APP_PROTOID_ISCSI) { 5021 tc = dcbcfg->etscfg.prioritytable[app.priority]; 5022 enabled_tc |= BIT(tc); 5023 break; 5024 } 5025 } 5026 5027 return enabled_tc; 5028 } 5029 5030 /** 5031 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 5032 * @dcbcfg: the corresponding DCBx configuration structure 5033 * 5034 * Return the number of TCs from given DCBx configuration 5035 **/ 5036 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 5037 { 5038 int i, tc_unused = 0; 5039 u8 num_tc = 0; 5040 u8 ret = 0; 5041 5042 /* Scan the ETS Config Priority Table to find 5043 * traffic class enabled for a given priority 5044 * and create a bitmask of enabled TCs 5045 */ 5046 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 5047 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 5048 5049 /* Now scan the bitmask to check for 5050 * contiguous TCs starting with TC0 5051 */ 5052 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5053 if (num_tc & BIT(i)) { 5054 if (!tc_unused) { 5055 ret++; 5056 } else { 5057 pr_err("Non-contiguous TC - Disabling DCB\n"); 5058 return 1; 5059 } 5060 } else { 5061 tc_unused = 1; 5062 } 5063 } 5064 5065 /* There is always at least TC0 */ 5066 if (!ret) 5067 ret = 1; 5068 5069 return ret; 5070 } 5071 5072 /** 5073 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 5074 * @dcbcfg: the corresponding DCBx configuration structure 5075 * 5076 * Query the current DCB configuration and return the number of 5077 * traffic classes enabled from the given DCBX config 5078 **/ 5079 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 5080 { 5081 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 5082 u8 enabled_tc = 1; 5083 u8 i; 5084 5085 for (i = 0; i < num_tc; i++) 5086 enabled_tc |= BIT(i); 5087 5088 return enabled_tc; 5089 } 5090 5091 /** 5092 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 5093 * @pf: PF being queried 5094 * 5095 * Query the current MQPRIO configuration and return the number of 5096 * traffic classes enabled. 5097 **/ 5098 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 5099 { 5100 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 5101 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 5102 u8 enabled_tc = 1, i; 5103 5104 for (i = 1; i < num_tc; i++) 5105 enabled_tc |= BIT(i); 5106 return enabled_tc; 5107 } 5108 5109 /** 5110 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 5111 * @pf: PF being queried 5112 * 5113 * Return number of traffic classes enabled for the given PF 5114 **/ 5115 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 5116 { 5117 struct i40e_hw *hw = &pf->hw; 5118 u8 i, enabled_tc = 1; 5119 u8 num_tc = 0; 5120 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5121 5122 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5123 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5124 5125 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5126 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5127 return 1; 5128 5129 /* SFP mode will be enabled for all TCs on port */ 5130 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5131 return i40e_dcb_get_num_tc(dcbcfg); 5132 5133 /* MFP mode return count of enabled TCs for this PF */ 5134 if (pf->hw.func_caps.iscsi) 5135 enabled_tc = i40e_get_iscsi_tc_map(pf); 5136 else 5137 return 1; /* Only TC0 */ 5138 5139 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5140 if (enabled_tc & BIT(i)) 5141 num_tc++; 5142 } 5143 return num_tc; 5144 } 5145 5146 /** 5147 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 5148 * @pf: PF being queried 5149 * 5150 * Return a bitmap for enabled traffic classes for this PF. 5151 **/ 5152 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5153 { 5154 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5155 return i40e_mqprio_get_enabled_tc(pf); 5156 5157 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5158 * default TC 5159 */ 5160 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5161 return I40E_DEFAULT_TRAFFIC_CLASS; 5162 5163 /* SFP mode we want PF to be enabled for all TCs */ 5164 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5165 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5166 5167 /* MFP enabled and iSCSI PF type */ 5168 if (pf->hw.func_caps.iscsi) 5169 return i40e_get_iscsi_tc_map(pf); 5170 else 5171 return I40E_DEFAULT_TRAFFIC_CLASS; 5172 } 5173 5174 /** 5175 * i40e_vsi_get_bw_info - Query VSI BW Information 5176 * @vsi: the VSI being queried 5177 * 5178 * Returns 0 on success, negative value on failure 5179 **/ 5180 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5181 { 5182 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5183 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5184 struct i40e_pf *pf = vsi->back; 5185 struct i40e_hw *hw = &pf->hw; 5186 i40e_status ret; 5187 u32 tc_bw_max; 5188 int i; 5189 5190 /* Get the VSI level BW configuration */ 5191 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5192 if (ret) { 5193 dev_info(&pf->pdev->dev, 5194 "couldn't get PF vsi bw config, err %s aq_err %s\n", 5195 i40e_stat_str(&pf->hw, ret), 5196 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5197 return -EINVAL; 5198 } 5199 5200 /* Get the VSI level BW configuration per TC */ 5201 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5202 NULL); 5203 if (ret) { 5204 dev_info(&pf->pdev->dev, 5205 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 5206 i40e_stat_str(&pf->hw, ret), 5207 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5208 return -EINVAL; 5209 } 5210 5211 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5212 dev_info(&pf->pdev->dev, 5213 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5214 bw_config.tc_valid_bits, 5215 bw_ets_config.tc_valid_bits); 5216 /* Still continuing */ 5217 } 5218 5219 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5220 vsi->bw_max_quanta = bw_config.max_bw; 5221 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5222 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5223 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5224 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5225 vsi->bw_ets_limit_credits[i] = 5226 le16_to_cpu(bw_ets_config.credits[i]); 5227 /* 3 bits out of 4 for each TC */ 5228 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5229 } 5230 5231 return 0; 5232 } 5233 5234 /** 5235 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5236 * @vsi: the VSI being configured 5237 * @enabled_tc: TC bitmap 5238 * @bw_share: BW shared credits per TC 5239 * 5240 * Returns 0 on success, negative value on failure 5241 **/ 5242 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5243 u8 *bw_share) 5244 { 5245 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5246 struct i40e_pf *pf = vsi->back; 5247 i40e_status ret; 5248 int i; 5249 5250 /* There is no need to reset BW when mqprio mode is on. */ 5251 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5252 return 0; 5253 if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5254 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5255 if (ret) 5256 dev_info(&pf->pdev->dev, 5257 "Failed to reset tx rate for vsi->seid %u\n", 5258 vsi->seid); 5259 return ret; 5260 } 5261 bw_data.tc_valid_bits = enabled_tc; 5262 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5263 bw_data.tc_bw_credits[i] = bw_share[i]; 5264 5265 ret = i40e_aq_config_vsi_tc_bw(&pf->hw, vsi->seid, &bw_data, NULL); 5266 if (ret) { 5267 dev_info(&pf->pdev->dev, 5268 "AQ command Config VSI BW allocation per TC failed = %d\n", 5269 pf->hw.aq.asq_last_status); 5270 return -EINVAL; 5271 } 5272 5273 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5274 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5275 5276 return 0; 5277 } 5278 5279 /** 5280 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5281 * @vsi: the VSI being configured 5282 * @enabled_tc: TC map to be enabled 5283 * 5284 **/ 5285 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5286 { 5287 struct net_device *netdev = vsi->netdev; 5288 struct i40e_pf *pf = vsi->back; 5289 struct i40e_hw *hw = &pf->hw; 5290 u8 netdev_tc = 0; 5291 int i; 5292 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5293 5294 if (!netdev) 5295 return; 5296 5297 if (!enabled_tc) { 5298 netdev_reset_tc(netdev); 5299 return; 5300 } 5301 5302 /* Set up actual enabled TCs on the VSI */ 5303 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5304 return; 5305 5306 /* set per TC queues for the VSI */ 5307 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5308 /* Only set TC queues for enabled tcs 5309 * 5310 * e.g. For a VSI that has TC0 and TC3 enabled the 5311 * enabled_tc bitmap would be 0x00001001; the driver 5312 * will set the numtc for netdev as 2 that will be 5313 * referenced by the netdev layer as TC 0 and 1. 5314 */ 5315 if (vsi->tc_config.enabled_tc & BIT(i)) 5316 netdev_set_tc_queue(netdev, 5317 vsi->tc_config.tc_info[i].netdev_tc, 5318 vsi->tc_config.tc_info[i].qcount, 5319 vsi->tc_config.tc_info[i].qoffset); 5320 } 5321 5322 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5323 return; 5324 5325 /* Assign UP2TC map for the VSI */ 5326 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5327 /* Get the actual TC# for the UP */ 5328 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5329 /* Get the mapped netdev TC# for the UP */ 5330 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5331 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5332 } 5333 } 5334 5335 /** 5336 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5337 * @vsi: the VSI being configured 5338 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5339 **/ 5340 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5341 struct i40e_vsi_context *ctxt) 5342 { 5343 /* copy just the sections touched not the entire info 5344 * since not all sections are valid as returned by 5345 * update vsi params 5346 */ 5347 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5348 memcpy(&vsi->info.queue_mapping, 5349 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5350 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5351 sizeof(vsi->info.tc_mapping)); 5352 } 5353 5354 /** 5355 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5356 * @vsi: VSI to be configured 5357 * @enabled_tc: TC bitmap 5358 * 5359 * This configures a particular VSI for TCs that are mapped to the 5360 * given TC bitmap. It uses default bandwidth share for TCs across 5361 * VSIs to configure TC for a particular VSI. 5362 * 5363 * NOTE: 5364 * It is expected that the VSI queues have been quisced before calling 5365 * this function. 5366 **/ 5367 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5368 { 5369 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5370 struct i40e_pf *pf = vsi->back; 5371 struct i40e_hw *hw = &pf->hw; 5372 struct i40e_vsi_context ctxt; 5373 int ret = 0; 5374 int i; 5375 5376 /* Check if enabled_tc is same as existing or new TCs */ 5377 if (vsi->tc_config.enabled_tc == enabled_tc && 5378 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5379 return ret; 5380 5381 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5382 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5383 if (enabled_tc & BIT(i)) 5384 bw_share[i] = 1; 5385 } 5386 5387 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5388 if (ret) { 5389 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5390 5391 dev_info(&pf->pdev->dev, 5392 "Failed configuring TC map %d for VSI %d\n", 5393 enabled_tc, vsi->seid); 5394 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5395 &bw_config, NULL); 5396 if (ret) { 5397 dev_info(&pf->pdev->dev, 5398 "Failed querying vsi bw info, err %s aq_err %s\n", 5399 i40e_stat_str(hw, ret), 5400 i40e_aq_str(hw, hw->aq.asq_last_status)); 5401 goto out; 5402 } 5403 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5404 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5405 5406 if (!valid_tc) 5407 valid_tc = bw_config.tc_valid_bits; 5408 /* Always enable TC0, no matter what */ 5409 valid_tc |= 1; 5410 dev_info(&pf->pdev->dev, 5411 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5412 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5413 enabled_tc = valid_tc; 5414 } 5415 5416 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5417 if (ret) { 5418 dev_err(&pf->pdev->dev, 5419 "Unable to configure TC map %d for VSI %d\n", 5420 enabled_tc, vsi->seid); 5421 goto out; 5422 } 5423 } 5424 5425 /* Update Queue Pairs Mapping for currently enabled UPs */ 5426 ctxt.seid = vsi->seid; 5427 ctxt.pf_num = vsi->back->hw.pf_id; 5428 ctxt.vf_num = 0; 5429 ctxt.uplink_seid = vsi->uplink_seid; 5430 ctxt.info = vsi->info; 5431 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { 5432 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5433 if (ret) 5434 goto out; 5435 } else { 5436 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5437 } 5438 5439 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5440 * queues changed. 5441 */ 5442 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5443 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5444 vsi->num_queue_pairs); 5445 ret = i40e_vsi_config_rss(vsi); 5446 if (ret) { 5447 dev_info(&vsi->back->pdev->dev, 5448 "Failed to reconfig rss for num_queues\n"); 5449 return ret; 5450 } 5451 vsi->reconfig_rss = false; 5452 } 5453 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5454 ctxt.info.valid_sections |= 5455 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5456 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5457 } 5458 5459 /* Update the VSI after updating the VSI queue-mapping 5460 * information 5461 */ 5462 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5463 if (ret) { 5464 dev_info(&pf->pdev->dev, 5465 "Update vsi tc config failed, err %s aq_err %s\n", 5466 i40e_stat_str(hw, ret), 5467 i40e_aq_str(hw, hw->aq.asq_last_status)); 5468 goto out; 5469 } 5470 /* update the local VSI info with updated queue map */ 5471 i40e_vsi_update_queue_map(vsi, &ctxt); 5472 vsi->info.valid_sections = 0; 5473 5474 /* Update current VSI BW information */ 5475 ret = i40e_vsi_get_bw_info(vsi); 5476 if (ret) { 5477 dev_info(&pf->pdev->dev, 5478 "Failed updating vsi bw info, err %s aq_err %s\n", 5479 i40e_stat_str(hw, ret), 5480 i40e_aq_str(hw, hw->aq.asq_last_status)); 5481 goto out; 5482 } 5483 5484 /* Update the netdev TC setup */ 5485 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5486 out: 5487 return ret; 5488 } 5489 5490 /** 5491 * i40e_get_link_speed - Returns link speed for the interface 5492 * @vsi: VSI to be configured 5493 * 5494 **/ 5495 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5496 { 5497 struct i40e_pf *pf = vsi->back; 5498 5499 switch (pf->hw.phy.link_info.link_speed) { 5500 case I40E_LINK_SPEED_40GB: 5501 return 40000; 5502 case I40E_LINK_SPEED_25GB: 5503 return 25000; 5504 case I40E_LINK_SPEED_20GB: 5505 return 20000; 5506 case I40E_LINK_SPEED_10GB: 5507 return 10000; 5508 case I40E_LINK_SPEED_1GB: 5509 return 1000; 5510 default: 5511 return -EINVAL; 5512 } 5513 } 5514 5515 /** 5516 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5517 * @vsi: VSI to be configured 5518 * @seid: seid of the channel/VSI 5519 * @max_tx_rate: max TX rate to be configured as BW limit 5520 * 5521 * Helper function to set BW limit for a given VSI 5522 **/ 5523 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5524 { 5525 struct i40e_pf *pf = vsi->back; 5526 u64 credits = 0; 5527 int speed = 0; 5528 int ret = 0; 5529 5530 speed = i40e_get_link_speed(vsi); 5531 if (max_tx_rate > speed) { 5532 dev_err(&pf->pdev->dev, 5533 "Invalid max tx rate %llu specified for VSI seid %d.", 5534 max_tx_rate, seid); 5535 return -EINVAL; 5536 } 5537 if (max_tx_rate && max_tx_rate < 50) { 5538 dev_warn(&pf->pdev->dev, 5539 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5540 max_tx_rate = 50; 5541 } 5542 5543 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5544 credits = max_tx_rate; 5545 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5546 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5547 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5548 if (ret) 5549 dev_err(&pf->pdev->dev, 5550 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n", 5551 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret), 5552 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5553 return ret; 5554 } 5555 5556 /** 5557 * i40e_remove_queue_channels - Remove queue channels for the TCs 5558 * @vsi: VSI to be configured 5559 * 5560 * Remove queue channels for the TCs 5561 **/ 5562 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5563 { 5564 enum i40e_admin_queue_err last_aq_status; 5565 struct i40e_cloud_filter *cfilter; 5566 struct i40e_channel *ch, *ch_tmp; 5567 struct i40e_pf *pf = vsi->back; 5568 struct hlist_node *node; 5569 int ret, i; 5570 5571 /* Reset rss size that was stored when reconfiguring rss for 5572 * channel VSIs with non-power-of-2 queue count. 5573 */ 5574 vsi->current_rss_size = 0; 5575 5576 /* perform cleanup for channels if they exist */ 5577 if (list_empty(&vsi->ch_list)) 5578 return; 5579 5580 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5581 struct i40e_vsi *p_vsi; 5582 5583 list_del(&ch->list); 5584 p_vsi = ch->parent_vsi; 5585 if (!p_vsi || !ch->initialized) { 5586 kfree(ch); 5587 continue; 5588 } 5589 /* Reset queue contexts */ 5590 for (i = 0; i < ch->num_queue_pairs; i++) { 5591 struct i40e_ring *tx_ring, *rx_ring; 5592 u16 pf_q; 5593 5594 pf_q = ch->base_queue + i; 5595 tx_ring = vsi->tx_rings[pf_q]; 5596 tx_ring->ch = NULL; 5597 5598 rx_ring = vsi->rx_rings[pf_q]; 5599 rx_ring->ch = NULL; 5600 } 5601 5602 /* Reset BW configured for this VSI via mqprio */ 5603 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 5604 if (ret) 5605 dev_info(&vsi->back->pdev->dev, 5606 "Failed to reset tx rate for ch->seid %u\n", 5607 ch->seid); 5608 5609 /* delete cloud filters associated with this channel */ 5610 hlist_for_each_entry_safe(cfilter, node, 5611 &pf->cloud_filter_list, cloud_node) { 5612 if (cfilter->seid != ch->seid) 5613 continue; 5614 5615 hash_del(&cfilter->cloud_node); 5616 if (cfilter->dst_port) 5617 ret = i40e_add_del_cloud_filter_big_buf(vsi, 5618 cfilter, 5619 false); 5620 else 5621 ret = i40e_add_del_cloud_filter(vsi, cfilter, 5622 false); 5623 last_aq_status = pf->hw.aq.asq_last_status; 5624 if (ret) 5625 dev_info(&pf->pdev->dev, 5626 "Failed to delete cloud filter, err %s aq_err %s\n", 5627 i40e_stat_str(&pf->hw, ret), 5628 i40e_aq_str(&pf->hw, last_aq_status)); 5629 kfree(cfilter); 5630 } 5631 5632 /* delete VSI from FW */ 5633 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 5634 NULL); 5635 if (ret) 5636 dev_err(&vsi->back->pdev->dev, 5637 "unable to remove channel (%d) for parent VSI(%d)\n", 5638 ch->seid, p_vsi->seid); 5639 kfree(ch); 5640 } 5641 INIT_LIST_HEAD(&vsi->ch_list); 5642 } 5643 5644 /** 5645 * i40e_is_any_channel - channel exist or not 5646 * @vsi: ptr to VSI to which channels are associated with 5647 * 5648 * Returns true or false if channel(s) exist for associated VSI or not 5649 **/ 5650 static bool i40e_is_any_channel(struct i40e_vsi *vsi) 5651 { 5652 struct i40e_channel *ch, *ch_tmp; 5653 5654 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5655 if (ch->initialized) 5656 return true; 5657 } 5658 5659 return false; 5660 } 5661 5662 /** 5663 * i40e_get_max_queues_for_channel 5664 * @vsi: ptr to VSI to which channels are associated with 5665 * 5666 * Helper function which returns max value among the queue counts set on the 5667 * channels/TCs created. 5668 **/ 5669 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 5670 { 5671 struct i40e_channel *ch, *ch_tmp; 5672 int max = 0; 5673 5674 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5675 if (!ch->initialized) 5676 continue; 5677 if (ch->num_queue_pairs > max) 5678 max = ch->num_queue_pairs; 5679 } 5680 5681 return max; 5682 } 5683 5684 /** 5685 * i40e_validate_num_queues - validate num_queues w.r.t channel 5686 * @pf: ptr to PF device 5687 * @num_queues: number of queues 5688 * @vsi: the parent VSI 5689 * @reconfig_rss: indicates should the RSS be reconfigured or not 5690 * 5691 * This function validates number of queues in the context of new channel 5692 * which is being established and determines if RSS should be reconfigured 5693 * or not for parent VSI. 5694 **/ 5695 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 5696 struct i40e_vsi *vsi, bool *reconfig_rss) 5697 { 5698 int max_ch_queues; 5699 5700 if (!reconfig_rss) 5701 return -EINVAL; 5702 5703 *reconfig_rss = false; 5704 if (vsi->current_rss_size) { 5705 if (num_queues > vsi->current_rss_size) { 5706 dev_dbg(&pf->pdev->dev, 5707 "Error: num_queues (%d) > vsi's current_size(%d)\n", 5708 num_queues, vsi->current_rss_size); 5709 return -EINVAL; 5710 } else if ((num_queues < vsi->current_rss_size) && 5711 (!is_power_of_2(num_queues))) { 5712 dev_dbg(&pf->pdev->dev, 5713 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 5714 num_queues, vsi->current_rss_size); 5715 return -EINVAL; 5716 } 5717 } 5718 5719 if (!is_power_of_2(num_queues)) { 5720 /* Find the max num_queues configured for channel if channel 5721 * exist. 5722 * if channel exist, then enforce 'num_queues' to be more than 5723 * max ever queues configured for channel. 5724 */ 5725 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 5726 if (num_queues < max_ch_queues) { 5727 dev_dbg(&pf->pdev->dev, 5728 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 5729 num_queues, max_ch_queues); 5730 return -EINVAL; 5731 } 5732 *reconfig_rss = true; 5733 } 5734 5735 return 0; 5736 } 5737 5738 /** 5739 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 5740 * @vsi: the VSI being setup 5741 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 5742 * 5743 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 5744 **/ 5745 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 5746 { 5747 struct i40e_pf *pf = vsi->back; 5748 u8 seed[I40E_HKEY_ARRAY_SIZE]; 5749 struct i40e_hw *hw = &pf->hw; 5750 int local_rss_size; 5751 u8 *lut; 5752 int ret; 5753 5754 if (!vsi->rss_size) 5755 return -EINVAL; 5756 5757 if (rss_size > vsi->rss_size) 5758 return -EINVAL; 5759 5760 local_rss_size = min_t(int, vsi->rss_size, rss_size); 5761 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 5762 if (!lut) 5763 return -ENOMEM; 5764 5765 /* Ignoring user configured lut if there is one */ 5766 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 5767 5768 /* Use user configured hash key if there is one, otherwise 5769 * use default. 5770 */ 5771 if (vsi->rss_hkey_user) 5772 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 5773 else 5774 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 5775 5776 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 5777 if (ret) { 5778 dev_info(&pf->pdev->dev, 5779 "Cannot set RSS lut, err %s aq_err %s\n", 5780 i40e_stat_str(hw, ret), 5781 i40e_aq_str(hw, hw->aq.asq_last_status)); 5782 kfree(lut); 5783 return ret; 5784 } 5785 kfree(lut); 5786 5787 /* Do the update w.r.t. storing rss_size */ 5788 if (!vsi->orig_rss_size) 5789 vsi->orig_rss_size = vsi->rss_size; 5790 vsi->current_rss_size = local_rss_size; 5791 5792 return ret; 5793 } 5794 5795 /** 5796 * i40e_channel_setup_queue_map - Setup a channel queue map 5797 * @pf: ptr to PF device 5798 * @vsi: the VSI being setup 5799 * @ctxt: VSI context structure 5800 * @ch: ptr to channel structure 5801 * 5802 * Setup queue map for a specific channel 5803 **/ 5804 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 5805 struct i40e_vsi_context *ctxt, 5806 struct i40e_channel *ch) 5807 { 5808 u16 qcount, qmap, sections = 0; 5809 u8 offset = 0; 5810 int pow; 5811 5812 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 5813 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 5814 5815 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 5816 ch->num_queue_pairs = qcount; 5817 5818 /* find the next higher power-of-2 of num queue pairs */ 5819 pow = ilog2(qcount); 5820 if (!is_power_of_2(qcount)) 5821 pow++; 5822 5823 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5824 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 5825 5826 /* Setup queue TC[0].qmap for given VSI context */ 5827 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 5828 5829 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 5830 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 5831 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 5832 ctxt->info.valid_sections |= cpu_to_le16(sections); 5833 } 5834 5835 /** 5836 * i40e_add_channel - add a channel by adding VSI 5837 * @pf: ptr to PF device 5838 * @uplink_seid: underlying HW switching element (VEB) ID 5839 * @ch: ptr to channel structure 5840 * 5841 * Add a channel (VSI) using add_vsi and queue_map 5842 **/ 5843 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 5844 struct i40e_channel *ch) 5845 { 5846 struct i40e_hw *hw = &pf->hw; 5847 struct i40e_vsi_context ctxt; 5848 u8 enabled_tc = 0x1; /* TC0 enabled */ 5849 int ret; 5850 5851 if (ch->type != I40E_VSI_VMDQ2) { 5852 dev_info(&pf->pdev->dev, 5853 "add new vsi failed, ch->type %d\n", ch->type); 5854 return -EINVAL; 5855 } 5856 5857 memset(&ctxt, 0, sizeof(ctxt)); 5858 ctxt.pf_num = hw->pf_id; 5859 ctxt.vf_num = 0; 5860 ctxt.uplink_seid = uplink_seid; 5861 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5862 if (ch->type == I40E_VSI_VMDQ2) 5863 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 5864 5865 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 5866 ctxt.info.valid_sections |= 5867 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5868 ctxt.info.switch_id = 5869 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 5870 } 5871 5872 /* Set queue map for a given VSI context */ 5873 i40e_channel_setup_queue_map(pf, &ctxt, ch); 5874 5875 /* Now time to create VSI */ 5876 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 5877 if (ret) { 5878 dev_info(&pf->pdev->dev, 5879 "add new vsi failed, err %s aq_err %s\n", 5880 i40e_stat_str(&pf->hw, ret), 5881 i40e_aq_str(&pf->hw, 5882 pf->hw.aq.asq_last_status)); 5883 return -ENOENT; 5884 } 5885 5886 /* Success, update channel, set enabled_tc only if the channel 5887 * is not a macvlan 5888 */ 5889 ch->enabled_tc = !i40e_is_channel_macvlan(ch) && enabled_tc; 5890 ch->seid = ctxt.seid; 5891 ch->vsi_number = ctxt.vsi_number; 5892 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); 5893 5894 /* copy just the sections touched not the entire info 5895 * since not all sections are valid as returned by 5896 * update vsi params 5897 */ 5898 ch->info.mapping_flags = ctxt.info.mapping_flags; 5899 memcpy(&ch->info.queue_mapping, 5900 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 5901 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 5902 sizeof(ctxt.info.tc_mapping)); 5903 5904 return 0; 5905 } 5906 5907 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 5908 u8 *bw_share) 5909 { 5910 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5911 i40e_status ret; 5912 int i; 5913 5914 bw_data.tc_valid_bits = ch->enabled_tc; 5915 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5916 bw_data.tc_bw_credits[i] = bw_share[i]; 5917 5918 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 5919 &bw_data, NULL); 5920 if (ret) { 5921 dev_info(&vsi->back->pdev->dev, 5922 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 5923 vsi->back->hw.aq.asq_last_status, ch->seid); 5924 return -EINVAL; 5925 } 5926 5927 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5928 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 5929 5930 return 0; 5931 } 5932 5933 /** 5934 * i40e_channel_config_tx_ring - config TX ring associated with new channel 5935 * @pf: ptr to PF device 5936 * @vsi: the VSI being setup 5937 * @ch: ptr to channel structure 5938 * 5939 * Configure TX rings associated with channel (VSI) since queues are being 5940 * from parent VSI. 5941 **/ 5942 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 5943 struct i40e_vsi *vsi, 5944 struct i40e_channel *ch) 5945 { 5946 i40e_status ret; 5947 int i; 5948 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5949 5950 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5951 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5952 if (ch->enabled_tc & BIT(i)) 5953 bw_share[i] = 1; 5954 } 5955 5956 /* configure BW for new VSI */ 5957 ret = i40e_channel_config_bw(vsi, ch, bw_share); 5958 if (ret) { 5959 dev_info(&vsi->back->pdev->dev, 5960 "Failed configuring TC map %d for channel (seid %u)\n", 5961 ch->enabled_tc, ch->seid); 5962 return ret; 5963 } 5964 5965 for (i = 0; i < ch->num_queue_pairs; i++) { 5966 struct i40e_ring *tx_ring, *rx_ring; 5967 u16 pf_q; 5968 5969 pf_q = ch->base_queue + i; 5970 5971 /* Get to TX ring ptr of main VSI, for re-setup TX queue 5972 * context 5973 */ 5974 tx_ring = vsi->tx_rings[pf_q]; 5975 tx_ring->ch = ch; 5976 5977 /* Get the RX ring ptr */ 5978 rx_ring = vsi->rx_rings[pf_q]; 5979 rx_ring->ch = ch; 5980 } 5981 5982 return 0; 5983 } 5984 5985 /** 5986 * i40e_setup_hw_channel - setup new channel 5987 * @pf: ptr to PF device 5988 * @vsi: the VSI being setup 5989 * @ch: ptr to channel structure 5990 * @uplink_seid: underlying HW switching element (VEB) ID 5991 * @type: type of channel to be created (VMDq2/VF) 5992 * 5993 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5994 * and configures TX rings accordingly 5995 **/ 5996 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 5997 struct i40e_vsi *vsi, 5998 struct i40e_channel *ch, 5999 u16 uplink_seid, u8 type) 6000 { 6001 int ret; 6002 6003 ch->initialized = false; 6004 ch->base_queue = vsi->next_base_queue; 6005 ch->type = type; 6006 6007 /* Proceed with creation of channel (VMDq2) VSI */ 6008 ret = i40e_add_channel(pf, uplink_seid, ch); 6009 if (ret) { 6010 dev_info(&pf->pdev->dev, 6011 "failed to add_channel using uplink_seid %u\n", 6012 uplink_seid); 6013 return ret; 6014 } 6015 6016 /* Mark the successful creation of channel */ 6017 ch->initialized = true; 6018 6019 /* Reconfigure TX queues using QTX_CTL register */ 6020 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 6021 if (ret) { 6022 dev_info(&pf->pdev->dev, 6023 "failed to configure TX rings for channel %u\n", 6024 ch->seid); 6025 return ret; 6026 } 6027 6028 /* update 'next_base_queue' */ 6029 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 6030 dev_dbg(&pf->pdev->dev, 6031 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 6032 ch->seid, ch->vsi_number, ch->stat_counter_idx, 6033 ch->num_queue_pairs, 6034 vsi->next_base_queue); 6035 return ret; 6036 } 6037 6038 /** 6039 * i40e_setup_channel - setup new channel using uplink element 6040 * @pf: ptr to PF device 6041 * @type: type of channel to be created (VMDq2/VF) 6042 * @uplink_seid: underlying HW switching element (VEB) ID 6043 * @ch: ptr to channel structure 6044 * 6045 * Setup new channel (VSI) based on specified type (VMDq2/VF) 6046 * and uplink switching element (uplink_seid) 6047 **/ 6048 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 6049 struct i40e_channel *ch) 6050 { 6051 u8 vsi_type; 6052 u16 seid; 6053 int ret; 6054 6055 if (vsi->type == I40E_VSI_MAIN) { 6056 vsi_type = I40E_VSI_VMDQ2; 6057 } else { 6058 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 6059 vsi->type); 6060 return false; 6061 } 6062 6063 /* underlying switching element */ 6064 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6065 6066 /* create channel (VSI), configure TX rings */ 6067 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 6068 if (ret) { 6069 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 6070 return false; 6071 } 6072 6073 return ch->initialized ? true : false; 6074 } 6075 6076 /** 6077 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 6078 * @vsi: ptr to VSI which has PF backing 6079 * 6080 * Sets up switch mode correctly if it needs to be changed and perform 6081 * what are allowed modes. 6082 **/ 6083 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 6084 { 6085 u8 mode; 6086 struct i40e_pf *pf = vsi->back; 6087 struct i40e_hw *hw = &pf->hw; 6088 int ret; 6089 6090 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 6091 if (ret) 6092 return -EINVAL; 6093 6094 if (hw->dev_caps.switch_mode) { 6095 /* if switch mode is set, support mode2 (non-tunneled for 6096 * cloud filter) for now 6097 */ 6098 u32 switch_mode = hw->dev_caps.switch_mode & 6099 I40E_SWITCH_MODE_MASK; 6100 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 6101 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 6102 return 0; 6103 dev_err(&pf->pdev->dev, 6104 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 6105 hw->dev_caps.switch_mode); 6106 return -EINVAL; 6107 } 6108 } 6109 6110 /* Set Bit 7 to be valid */ 6111 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 6112 6113 /* Set L4type for TCP support */ 6114 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 6115 6116 /* Set cloud filter mode */ 6117 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 6118 6119 /* Prep mode field for set_switch_config */ 6120 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 6121 pf->last_sw_conf_valid_flags, 6122 mode, NULL); 6123 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 6124 dev_err(&pf->pdev->dev, 6125 "couldn't set switch config bits, err %s aq_err %s\n", 6126 i40e_stat_str(hw, ret), 6127 i40e_aq_str(hw, 6128 hw->aq.asq_last_status)); 6129 6130 return ret; 6131 } 6132 6133 /** 6134 * i40e_create_queue_channel - function to create channel 6135 * @vsi: VSI to be configured 6136 * @ch: ptr to channel (it contains channel specific params) 6137 * 6138 * This function creates channel (VSI) using num_queues specified by user, 6139 * reconfigs RSS if needed. 6140 **/ 6141 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6142 struct i40e_channel *ch) 6143 { 6144 struct i40e_pf *pf = vsi->back; 6145 bool reconfig_rss; 6146 int err; 6147 6148 if (!ch) 6149 return -EINVAL; 6150 6151 if (!ch->num_queue_pairs) { 6152 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6153 ch->num_queue_pairs); 6154 return -EINVAL; 6155 } 6156 6157 /* validate user requested num_queues for channel */ 6158 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6159 &reconfig_rss); 6160 if (err) { 6161 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6162 ch->num_queue_pairs); 6163 return -EINVAL; 6164 } 6165 6166 /* By default we are in VEPA mode, if this is the first VF/VMDq 6167 * VSI to be added switch to VEB mode. 6168 */ 6169 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) || 6170 (!i40e_is_any_channel(vsi))) { 6171 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) { 6172 dev_dbg(&pf->pdev->dev, 6173 "Failed to create channel. Override queues (%u) not power of 2\n", 6174 vsi->tc_config.tc_info[0].qcount); 6175 return -EINVAL; 6176 } 6177 6178 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6179 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6180 6181 if (vsi->type == I40E_VSI_MAIN) { 6182 if (pf->flags & I40E_FLAG_TC_MQPRIO) 6183 i40e_do_reset(pf, I40E_PF_RESET_FLAG, 6184 true); 6185 else 6186 i40e_do_reset_safe(pf, 6187 I40E_PF_RESET_FLAG); 6188 } 6189 } 6190 /* now onwards for main VSI, number of queues will be value 6191 * of TC0's queue count 6192 */ 6193 } 6194 6195 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6196 * it should be more than num_queues 6197 */ 6198 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6199 dev_dbg(&pf->pdev->dev, 6200 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6201 vsi->cnt_q_avail, ch->num_queue_pairs); 6202 return -EINVAL; 6203 } 6204 6205 /* reconfig_rss only if vsi type is MAIN_VSI */ 6206 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6207 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6208 if (err) { 6209 dev_info(&pf->pdev->dev, 6210 "Error: unable to reconfig rss for num_queues (%u)\n", 6211 ch->num_queue_pairs); 6212 return -EINVAL; 6213 } 6214 } 6215 6216 if (!i40e_setup_channel(pf, vsi, ch)) { 6217 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6218 return -EINVAL; 6219 } 6220 6221 dev_info(&pf->pdev->dev, 6222 "Setup channel (id:%u) utilizing num_queues %d\n", 6223 ch->seid, ch->num_queue_pairs); 6224 6225 /* configure VSI for BW limit */ 6226 if (ch->max_tx_rate) { 6227 u64 credits = ch->max_tx_rate; 6228 6229 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6230 return -EINVAL; 6231 6232 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6233 dev_dbg(&pf->pdev->dev, 6234 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6235 ch->max_tx_rate, 6236 credits, 6237 ch->seid); 6238 } 6239 6240 /* in case of VF, this will be main SRIOV VSI */ 6241 ch->parent_vsi = vsi; 6242 6243 /* and update main_vsi's count for queue_available to use */ 6244 vsi->cnt_q_avail -= ch->num_queue_pairs; 6245 6246 return 0; 6247 } 6248 6249 /** 6250 * i40e_configure_queue_channels - Add queue channel for the given TCs 6251 * @vsi: VSI to be configured 6252 * 6253 * Configures queue channel mapping to the given TCs 6254 **/ 6255 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6256 { 6257 struct i40e_channel *ch; 6258 u64 max_rate = 0; 6259 int ret = 0, i; 6260 6261 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6262 vsi->tc_seid_map[0] = vsi->seid; 6263 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6264 if (vsi->tc_config.enabled_tc & BIT(i)) { 6265 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6266 if (!ch) { 6267 ret = -ENOMEM; 6268 goto err_free; 6269 } 6270 6271 INIT_LIST_HEAD(&ch->list); 6272 ch->num_queue_pairs = 6273 vsi->tc_config.tc_info[i].qcount; 6274 ch->base_queue = 6275 vsi->tc_config.tc_info[i].qoffset; 6276 6277 /* Bandwidth limit through tc interface is in bytes/s, 6278 * change to Mbit/s 6279 */ 6280 max_rate = vsi->mqprio_qopt.max_rate[i]; 6281 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6282 ch->max_tx_rate = max_rate; 6283 6284 list_add_tail(&ch->list, &vsi->ch_list); 6285 6286 ret = i40e_create_queue_channel(vsi, ch); 6287 if (ret) { 6288 dev_err(&vsi->back->pdev->dev, 6289 "Failed creating queue channel with TC%d: queues %d\n", 6290 i, ch->num_queue_pairs); 6291 goto err_free; 6292 } 6293 vsi->tc_seid_map[i] = ch->seid; 6294 } 6295 } 6296 return ret; 6297 6298 err_free: 6299 i40e_remove_queue_channels(vsi); 6300 return ret; 6301 } 6302 6303 /** 6304 * i40e_veb_config_tc - Configure TCs for given VEB 6305 * @veb: given VEB 6306 * @enabled_tc: TC bitmap 6307 * 6308 * Configures given TC bitmap for VEB (switching) element 6309 **/ 6310 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6311 { 6312 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6313 struct i40e_pf *pf = veb->pf; 6314 int ret = 0; 6315 int i; 6316 6317 /* No TCs or already enabled TCs just return */ 6318 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6319 return ret; 6320 6321 bw_data.tc_valid_bits = enabled_tc; 6322 /* bw_data.absolute_credits is not set (relative) */ 6323 6324 /* Enable ETS TCs with equal BW Share for now */ 6325 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6326 if (enabled_tc & BIT(i)) 6327 bw_data.tc_bw_share_credits[i] = 1; 6328 } 6329 6330 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6331 &bw_data, NULL); 6332 if (ret) { 6333 dev_info(&pf->pdev->dev, 6334 "VEB bw config failed, err %s aq_err %s\n", 6335 i40e_stat_str(&pf->hw, ret), 6336 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6337 goto out; 6338 } 6339 6340 /* Update the BW information */ 6341 ret = i40e_veb_get_bw_info(veb); 6342 if (ret) { 6343 dev_info(&pf->pdev->dev, 6344 "Failed getting veb bw config, err %s aq_err %s\n", 6345 i40e_stat_str(&pf->hw, ret), 6346 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6347 } 6348 6349 out: 6350 return ret; 6351 } 6352 6353 #ifdef CONFIG_I40E_DCB 6354 /** 6355 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6356 * @pf: PF struct 6357 * 6358 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6359 * the caller would've quiesce all the VSIs before calling 6360 * this function 6361 **/ 6362 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6363 { 6364 u8 tc_map = 0; 6365 int ret; 6366 u8 v; 6367 6368 /* Enable the TCs available on PF to all VEBs */ 6369 tc_map = i40e_pf_get_tc_map(pf); 6370 for (v = 0; v < I40E_MAX_VEB; v++) { 6371 if (!pf->veb[v]) 6372 continue; 6373 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6374 if (ret) { 6375 dev_info(&pf->pdev->dev, 6376 "Failed configuring TC for VEB seid=%d\n", 6377 pf->veb[v]->seid); 6378 /* Will try to configure as many components */ 6379 } 6380 } 6381 6382 /* Update each VSI */ 6383 for (v = 0; v < pf->num_alloc_vsi; v++) { 6384 if (!pf->vsi[v]) 6385 continue; 6386 6387 /* - Enable all TCs for the LAN VSI 6388 * - For all others keep them at TC0 for now 6389 */ 6390 if (v == pf->lan_vsi) 6391 tc_map = i40e_pf_get_tc_map(pf); 6392 else 6393 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6394 6395 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6396 if (ret) { 6397 dev_info(&pf->pdev->dev, 6398 "Failed configuring TC for VSI seid=%d\n", 6399 pf->vsi[v]->seid); 6400 /* Will try to configure as many components */ 6401 } else { 6402 /* Re-configure VSI vectors based on updated TC map */ 6403 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6404 if (pf->vsi[v]->netdev) 6405 i40e_dcbnl_set_all(pf->vsi[v]); 6406 } 6407 } 6408 } 6409 6410 /** 6411 * i40e_resume_port_tx - Resume port Tx 6412 * @pf: PF struct 6413 * 6414 * Resume a port's Tx and issue a PF reset in case of failure to 6415 * resume. 6416 **/ 6417 static int i40e_resume_port_tx(struct i40e_pf *pf) 6418 { 6419 struct i40e_hw *hw = &pf->hw; 6420 int ret; 6421 6422 ret = i40e_aq_resume_port_tx(hw, NULL); 6423 if (ret) { 6424 dev_info(&pf->pdev->dev, 6425 "Resume Port Tx failed, err %s aq_err %s\n", 6426 i40e_stat_str(&pf->hw, ret), 6427 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6428 /* Schedule PF reset to recover */ 6429 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6430 i40e_service_event_schedule(pf); 6431 } 6432 6433 return ret; 6434 } 6435 6436 /** 6437 * i40e_init_pf_dcb - Initialize DCB configuration 6438 * @pf: PF being configured 6439 * 6440 * Query the current DCB configuration and cache it 6441 * in the hardware structure 6442 **/ 6443 static int i40e_init_pf_dcb(struct i40e_pf *pf) 6444 { 6445 struct i40e_hw *hw = &pf->hw; 6446 int err = 0; 6447 6448 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 6449 * Also do not enable DCBx if FW LLDP agent is disabled 6450 */ 6451 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) || 6452 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) { 6453 dev_info(&pf->pdev->dev, "DCB is not supported or FW LLDP is disabled\n"); 6454 err = I40E_NOT_SUPPORTED; 6455 goto out; 6456 } 6457 6458 err = i40e_init_dcb(hw, true); 6459 if (!err) { 6460 /* Device/Function is not DCBX capable */ 6461 if ((!hw->func_caps.dcb) || 6462 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 6463 dev_info(&pf->pdev->dev, 6464 "DCBX offload is not supported or is disabled for this PF.\n"); 6465 } else { 6466 /* When status is not DISABLED then DCBX in FW */ 6467 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 6468 DCB_CAP_DCBX_VER_IEEE; 6469 6470 pf->flags |= I40E_FLAG_DCB_CAPABLE; 6471 /* Enable DCB tagging only when more than one TC 6472 * or explicitly disable if only one TC 6473 */ 6474 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6475 pf->flags |= I40E_FLAG_DCB_ENABLED; 6476 else 6477 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6478 dev_dbg(&pf->pdev->dev, 6479 "DCBX offload is supported for this PF.\n"); 6480 } 6481 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 6482 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 6483 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP; 6484 } else { 6485 dev_info(&pf->pdev->dev, 6486 "Query for DCB configuration failed, err %s aq_err %s\n", 6487 i40e_stat_str(&pf->hw, err), 6488 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6489 } 6490 6491 out: 6492 return err; 6493 } 6494 #endif /* CONFIG_I40E_DCB */ 6495 #define SPEED_SIZE 14 6496 #define FC_SIZE 8 6497 /** 6498 * i40e_print_link_message - print link up or down 6499 * @vsi: the VSI for which link needs a message 6500 * @isup: true of link is up, false otherwise 6501 */ 6502 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 6503 { 6504 enum i40e_aq_link_speed new_speed; 6505 struct i40e_pf *pf = vsi->back; 6506 char *speed = "Unknown"; 6507 char *fc = "Unknown"; 6508 char *fec = ""; 6509 char *req_fec = ""; 6510 char *an = ""; 6511 6512 if (isup) 6513 new_speed = pf->hw.phy.link_info.link_speed; 6514 else 6515 new_speed = I40E_LINK_SPEED_UNKNOWN; 6516 6517 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 6518 return; 6519 vsi->current_isup = isup; 6520 vsi->current_speed = new_speed; 6521 if (!isup) { 6522 netdev_info(vsi->netdev, "NIC Link is Down\n"); 6523 return; 6524 } 6525 6526 /* Warn user if link speed on NPAR enabled partition is not at 6527 * least 10GB 6528 */ 6529 if (pf->hw.func_caps.npar_enable && 6530 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 6531 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 6532 netdev_warn(vsi->netdev, 6533 "The partition detected link speed that is less than 10Gbps\n"); 6534 6535 switch (pf->hw.phy.link_info.link_speed) { 6536 case I40E_LINK_SPEED_40GB: 6537 speed = "40 G"; 6538 break; 6539 case I40E_LINK_SPEED_20GB: 6540 speed = "20 G"; 6541 break; 6542 case I40E_LINK_SPEED_25GB: 6543 speed = "25 G"; 6544 break; 6545 case I40E_LINK_SPEED_10GB: 6546 speed = "10 G"; 6547 break; 6548 case I40E_LINK_SPEED_5GB: 6549 speed = "5 G"; 6550 break; 6551 case I40E_LINK_SPEED_2_5GB: 6552 speed = "2.5 G"; 6553 break; 6554 case I40E_LINK_SPEED_1GB: 6555 speed = "1000 M"; 6556 break; 6557 case I40E_LINK_SPEED_100MB: 6558 speed = "100 M"; 6559 break; 6560 default: 6561 break; 6562 } 6563 6564 switch (pf->hw.fc.current_mode) { 6565 case I40E_FC_FULL: 6566 fc = "RX/TX"; 6567 break; 6568 case I40E_FC_TX_PAUSE: 6569 fc = "TX"; 6570 break; 6571 case I40E_FC_RX_PAUSE: 6572 fc = "RX"; 6573 break; 6574 default: 6575 fc = "None"; 6576 break; 6577 } 6578 6579 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 6580 req_fec = "None"; 6581 fec = "None"; 6582 an = "False"; 6583 6584 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6585 an = "True"; 6586 6587 if (pf->hw.phy.link_info.fec_info & 6588 I40E_AQ_CONFIG_FEC_KR_ENA) 6589 fec = "CL74 FC-FEC/BASE-R"; 6590 else if (pf->hw.phy.link_info.fec_info & 6591 I40E_AQ_CONFIG_FEC_RS_ENA) 6592 fec = "CL108 RS-FEC"; 6593 6594 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 6595 * both RS and FC are requested 6596 */ 6597 if (vsi->back->hw.phy.link_info.req_fec_info & 6598 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 6599 if (vsi->back->hw.phy.link_info.req_fec_info & 6600 I40E_AQ_REQUEST_FEC_RS) 6601 req_fec = "CL108 RS-FEC"; 6602 else 6603 req_fec = "CL74 FC-FEC/BASE-R"; 6604 } 6605 netdev_info(vsi->netdev, 6606 "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", 6607 speed, req_fec, fec, an, fc); 6608 } else { 6609 netdev_info(vsi->netdev, 6610 "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", 6611 speed, fc); 6612 } 6613 6614 } 6615 6616 /** 6617 * i40e_up_complete - Finish the last steps of bringing up a connection 6618 * @vsi: the VSI being configured 6619 **/ 6620 static int i40e_up_complete(struct i40e_vsi *vsi) 6621 { 6622 struct i40e_pf *pf = vsi->back; 6623 int err; 6624 6625 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6626 i40e_vsi_configure_msix(vsi); 6627 else 6628 i40e_configure_msi_and_legacy(vsi); 6629 6630 /* start rings */ 6631 err = i40e_vsi_start_rings(vsi); 6632 if (err) 6633 return err; 6634 6635 clear_bit(__I40E_VSI_DOWN, vsi->state); 6636 i40e_napi_enable_all(vsi); 6637 i40e_vsi_enable_irq(vsi); 6638 6639 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 6640 (vsi->netdev)) { 6641 i40e_print_link_message(vsi, true); 6642 netif_tx_start_all_queues(vsi->netdev); 6643 netif_carrier_on(vsi->netdev); 6644 } 6645 6646 /* replay FDIR SB filters */ 6647 if (vsi->type == I40E_VSI_FDIR) { 6648 /* reset fd counters */ 6649 pf->fd_add_err = 0; 6650 pf->fd_atr_cnt = 0; 6651 i40e_fdir_filter_restore(vsi); 6652 } 6653 6654 /* On the next run of the service_task, notify any clients of the new 6655 * opened netdev 6656 */ 6657 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 6658 i40e_service_event_schedule(pf); 6659 6660 return 0; 6661 } 6662 6663 /** 6664 * i40e_vsi_reinit_locked - Reset the VSI 6665 * @vsi: the VSI being configured 6666 * 6667 * Rebuild the ring structs after some configuration 6668 * has changed, e.g. MTU size. 6669 **/ 6670 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 6671 { 6672 struct i40e_pf *pf = vsi->back; 6673 6674 WARN_ON(in_interrupt()); 6675 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 6676 usleep_range(1000, 2000); 6677 i40e_down(vsi); 6678 6679 i40e_up(vsi); 6680 clear_bit(__I40E_CONFIG_BUSY, pf->state); 6681 } 6682 6683 /** 6684 * i40e_up - Bring the connection back up after being down 6685 * @vsi: the VSI being configured 6686 **/ 6687 int i40e_up(struct i40e_vsi *vsi) 6688 { 6689 int err; 6690 6691 err = i40e_vsi_configure(vsi); 6692 if (!err) 6693 err = i40e_up_complete(vsi); 6694 6695 return err; 6696 } 6697 6698 /** 6699 * i40e_force_link_state - Force the link status 6700 * @pf: board private structure 6701 * @is_up: whether the link state should be forced up or down 6702 **/ 6703 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up) 6704 { 6705 struct i40e_aq_get_phy_abilities_resp abilities; 6706 struct i40e_aq_set_phy_config config = {0}; 6707 struct i40e_hw *hw = &pf->hw; 6708 i40e_status err; 6709 u64 mask; 6710 u8 speed; 6711 6712 /* Card might've been put in an unstable state by other drivers 6713 * and applications, which causes incorrect speed values being 6714 * set on startup. In order to clear speed registers, we call 6715 * get_phy_capabilities twice, once to get initial state of 6716 * available speeds, and once to get current PHY config. 6717 */ 6718 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, 6719 NULL); 6720 if (err) { 6721 dev_err(&pf->pdev->dev, 6722 "failed to get phy cap., ret = %s last_status = %s\n", 6723 i40e_stat_str(hw, err), 6724 i40e_aq_str(hw, hw->aq.asq_last_status)); 6725 return err; 6726 } 6727 speed = abilities.link_speed; 6728 6729 /* Get the current phy config */ 6730 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 6731 NULL); 6732 if (err) { 6733 dev_err(&pf->pdev->dev, 6734 "failed to get phy cap., ret = %s last_status = %s\n", 6735 i40e_stat_str(hw, err), 6736 i40e_aq_str(hw, hw->aq.asq_last_status)); 6737 return err; 6738 } 6739 6740 /* If link needs to go up, but was not forced to go down, 6741 * and its speed values are OK, no need for a flap 6742 */ 6743 if (is_up && abilities.phy_type != 0 && abilities.link_speed != 0) 6744 return I40E_SUCCESS; 6745 6746 /* To force link we need to set bits for all supported PHY types, 6747 * but there are now more than 32, so we need to split the bitmap 6748 * across two fields. 6749 */ 6750 mask = I40E_PHY_TYPES_BITMASK; 6751 config.phy_type = is_up ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 6752 config.phy_type_ext = is_up ? (u8)((mask >> 32) & 0xff) : 0; 6753 /* Copy the old settings, except of phy_type */ 6754 config.abilities = abilities.abilities; 6755 if (abilities.link_speed != 0) 6756 config.link_speed = abilities.link_speed; 6757 else 6758 config.link_speed = speed; 6759 config.eee_capability = abilities.eee_capability; 6760 config.eeer = abilities.eeer_val; 6761 config.low_power_ctrl = abilities.d3_lpan; 6762 config.fec_config = abilities.fec_cfg_curr_mod_ext_info & 6763 I40E_AQ_PHY_FEC_CONFIG_MASK; 6764 err = i40e_aq_set_phy_config(hw, &config, NULL); 6765 6766 if (err) { 6767 dev_err(&pf->pdev->dev, 6768 "set phy config ret = %s last_status = %s\n", 6769 i40e_stat_str(&pf->hw, err), 6770 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6771 return err; 6772 } 6773 6774 /* Update the link info */ 6775 err = i40e_update_link_info(hw); 6776 if (err) { 6777 /* Wait a little bit (on 40G cards it sometimes takes a really 6778 * long time for link to come back from the atomic reset) 6779 * and try once more 6780 */ 6781 msleep(1000); 6782 i40e_update_link_info(hw); 6783 } 6784 6785 i40e_aq_set_link_restart_an(hw, true, NULL); 6786 6787 return I40E_SUCCESS; 6788 } 6789 6790 /** 6791 * i40e_down - Shutdown the connection processing 6792 * @vsi: the VSI being stopped 6793 **/ 6794 void i40e_down(struct i40e_vsi *vsi) 6795 { 6796 int i; 6797 6798 /* It is assumed that the caller of this function 6799 * sets the vsi->state __I40E_VSI_DOWN bit. 6800 */ 6801 if (vsi->netdev) { 6802 netif_carrier_off(vsi->netdev); 6803 netif_tx_disable(vsi->netdev); 6804 } 6805 i40e_vsi_disable_irq(vsi); 6806 i40e_vsi_stop_rings(vsi); 6807 if (vsi->type == I40E_VSI_MAIN && 6808 vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) 6809 i40e_force_link_state(vsi->back, false); 6810 i40e_napi_disable_all(vsi); 6811 6812 for (i = 0; i < vsi->num_queue_pairs; i++) { 6813 i40e_clean_tx_ring(vsi->tx_rings[i]); 6814 if (i40e_enabled_xdp_vsi(vsi)) { 6815 /* Make sure that in-progress ndo_xdp_xmit and 6816 * ndo_xsk_wakeup calls are completed. 6817 */ 6818 synchronize_rcu(); 6819 i40e_clean_tx_ring(vsi->xdp_rings[i]); 6820 } 6821 i40e_clean_rx_ring(vsi->rx_rings[i]); 6822 } 6823 6824 } 6825 6826 /** 6827 * i40e_validate_mqprio_qopt- validate queue mapping info 6828 * @vsi: the VSI being configured 6829 * @mqprio_qopt: queue parametrs 6830 **/ 6831 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 6832 struct tc_mqprio_qopt_offload *mqprio_qopt) 6833 { 6834 u64 sum_max_rate = 0; 6835 u64 max_rate = 0; 6836 int i; 6837 6838 if (mqprio_qopt->qopt.offset[0] != 0 || 6839 mqprio_qopt->qopt.num_tc < 1 || 6840 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 6841 return -EINVAL; 6842 for (i = 0; ; i++) { 6843 if (!mqprio_qopt->qopt.count[i]) 6844 return -EINVAL; 6845 if (mqprio_qopt->min_rate[i]) { 6846 dev_err(&vsi->back->pdev->dev, 6847 "Invalid min tx rate (greater than 0) specified\n"); 6848 return -EINVAL; 6849 } 6850 max_rate = mqprio_qopt->max_rate[i]; 6851 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6852 sum_max_rate += max_rate; 6853 6854 if (i >= mqprio_qopt->qopt.num_tc - 1) 6855 break; 6856 if (mqprio_qopt->qopt.offset[i + 1] != 6857 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 6858 return -EINVAL; 6859 } 6860 if (vsi->num_queue_pairs < 6861 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 6862 return -EINVAL; 6863 } 6864 if (sum_max_rate > i40e_get_link_speed(vsi)) { 6865 dev_err(&vsi->back->pdev->dev, 6866 "Invalid max tx rate specified\n"); 6867 return -EINVAL; 6868 } 6869 return 0; 6870 } 6871 6872 /** 6873 * i40e_vsi_set_default_tc_config - set default values for tc configuration 6874 * @vsi: the VSI being configured 6875 **/ 6876 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 6877 { 6878 u16 qcount; 6879 int i; 6880 6881 /* Only TC0 is enabled */ 6882 vsi->tc_config.numtc = 1; 6883 vsi->tc_config.enabled_tc = 1; 6884 qcount = min_t(int, vsi->alloc_queue_pairs, 6885 i40e_pf_get_max_q_per_tc(vsi->back)); 6886 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6887 /* For the TC that is not enabled set the offset to to default 6888 * queue and allocate one queue for the given TC. 6889 */ 6890 vsi->tc_config.tc_info[i].qoffset = 0; 6891 if (i == 0) 6892 vsi->tc_config.tc_info[i].qcount = qcount; 6893 else 6894 vsi->tc_config.tc_info[i].qcount = 1; 6895 vsi->tc_config.tc_info[i].netdev_tc = 0; 6896 } 6897 } 6898 6899 /** 6900 * i40e_del_macvlan_filter 6901 * @hw: pointer to the HW structure 6902 * @seid: seid of the channel VSI 6903 * @macaddr: the mac address to apply as a filter 6904 * @aq_err: store the admin Q error 6905 * 6906 * This function deletes a mac filter on the channel VSI which serves as the 6907 * macvlan. Returns 0 on success. 6908 **/ 6909 static i40e_status i40e_del_macvlan_filter(struct i40e_hw *hw, u16 seid, 6910 const u8 *macaddr, int *aq_err) 6911 { 6912 struct i40e_aqc_remove_macvlan_element_data element; 6913 i40e_status status; 6914 6915 memset(&element, 0, sizeof(element)); 6916 ether_addr_copy(element.mac_addr, macaddr); 6917 element.vlan_tag = 0; 6918 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 6919 status = i40e_aq_remove_macvlan(hw, seid, &element, 1, NULL); 6920 *aq_err = hw->aq.asq_last_status; 6921 6922 return status; 6923 } 6924 6925 /** 6926 * i40e_add_macvlan_filter 6927 * @hw: pointer to the HW structure 6928 * @seid: seid of the channel VSI 6929 * @macaddr: the mac address to apply as a filter 6930 * @aq_err: store the admin Q error 6931 * 6932 * This function adds a mac filter on the channel VSI which serves as the 6933 * macvlan. Returns 0 on success. 6934 **/ 6935 static i40e_status i40e_add_macvlan_filter(struct i40e_hw *hw, u16 seid, 6936 const u8 *macaddr, int *aq_err) 6937 { 6938 struct i40e_aqc_add_macvlan_element_data element; 6939 i40e_status status; 6940 u16 cmd_flags = 0; 6941 6942 ether_addr_copy(element.mac_addr, macaddr); 6943 element.vlan_tag = 0; 6944 element.queue_number = 0; 6945 element.match_method = I40E_AQC_MM_ERR_NO_RES; 6946 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 6947 element.flags = cpu_to_le16(cmd_flags); 6948 status = i40e_aq_add_macvlan(hw, seid, &element, 1, NULL); 6949 *aq_err = hw->aq.asq_last_status; 6950 6951 return status; 6952 } 6953 6954 /** 6955 * i40e_reset_ch_rings - Reset the queue contexts in a channel 6956 * @vsi: the VSI we want to access 6957 * @ch: the channel we want to access 6958 */ 6959 static void i40e_reset_ch_rings(struct i40e_vsi *vsi, struct i40e_channel *ch) 6960 { 6961 struct i40e_ring *tx_ring, *rx_ring; 6962 u16 pf_q; 6963 int i; 6964 6965 for (i = 0; i < ch->num_queue_pairs; i++) { 6966 pf_q = ch->base_queue + i; 6967 tx_ring = vsi->tx_rings[pf_q]; 6968 tx_ring->ch = NULL; 6969 rx_ring = vsi->rx_rings[pf_q]; 6970 rx_ring->ch = NULL; 6971 } 6972 } 6973 6974 /** 6975 * i40e_free_macvlan_channels 6976 * @vsi: the VSI we want to access 6977 * 6978 * This function frees the Qs of the channel VSI from 6979 * the stack and also deletes the channel VSIs which 6980 * serve as macvlans. 6981 */ 6982 static void i40e_free_macvlan_channels(struct i40e_vsi *vsi) 6983 { 6984 struct i40e_channel *ch, *ch_tmp; 6985 int ret; 6986 6987 if (list_empty(&vsi->macvlan_list)) 6988 return; 6989 6990 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 6991 struct i40e_vsi *parent_vsi; 6992 6993 if (i40e_is_channel_macvlan(ch)) { 6994 i40e_reset_ch_rings(vsi, ch); 6995 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 6996 netdev_unbind_sb_channel(vsi->netdev, ch->fwd->netdev); 6997 netdev_set_sb_channel(ch->fwd->netdev, 0); 6998 kfree(ch->fwd); 6999 ch->fwd = NULL; 7000 } 7001 7002 list_del(&ch->list); 7003 parent_vsi = ch->parent_vsi; 7004 if (!parent_vsi || !ch->initialized) { 7005 kfree(ch); 7006 continue; 7007 } 7008 7009 /* remove the VSI */ 7010 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 7011 NULL); 7012 if (ret) 7013 dev_err(&vsi->back->pdev->dev, 7014 "unable to remove channel (%d) for parent VSI(%d)\n", 7015 ch->seid, parent_vsi->seid); 7016 kfree(ch); 7017 } 7018 vsi->macvlan_cnt = 0; 7019 } 7020 7021 /** 7022 * i40e_fwd_ring_up - bring the macvlan device up 7023 * @vsi: the VSI we want to access 7024 * @vdev: macvlan netdevice 7025 * @fwd: the private fwd structure 7026 */ 7027 static int i40e_fwd_ring_up(struct i40e_vsi *vsi, struct net_device *vdev, 7028 struct i40e_fwd_adapter *fwd) 7029 { 7030 int ret = 0, num_tc = 1, i, aq_err; 7031 struct i40e_channel *ch, *ch_tmp; 7032 struct i40e_pf *pf = vsi->back; 7033 struct i40e_hw *hw = &pf->hw; 7034 7035 if (list_empty(&vsi->macvlan_list)) 7036 return -EINVAL; 7037 7038 /* Go through the list and find an available channel */ 7039 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7040 if (!i40e_is_channel_macvlan(ch)) { 7041 ch->fwd = fwd; 7042 /* record configuration for macvlan interface in vdev */ 7043 for (i = 0; i < num_tc; i++) 7044 netdev_bind_sb_channel_queue(vsi->netdev, vdev, 7045 i, 7046 ch->num_queue_pairs, 7047 ch->base_queue); 7048 for (i = 0; i < ch->num_queue_pairs; i++) { 7049 struct i40e_ring *tx_ring, *rx_ring; 7050 u16 pf_q; 7051 7052 pf_q = ch->base_queue + i; 7053 7054 /* Get to TX ring ptr */ 7055 tx_ring = vsi->tx_rings[pf_q]; 7056 tx_ring->ch = ch; 7057 7058 /* Get the RX ring ptr */ 7059 rx_ring = vsi->rx_rings[pf_q]; 7060 rx_ring->ch = ch; 7061 } 7062 break; 7063 } 7064 } 7065 7066 /* Guarantee all rings are updated before we update the 7067 * MAC address filter. 7068 */ 7069 wmb(); 7070 7071 /* Add a mac filter */ 7072 ret = i40e_add_macvlan_filter(hw, ch->seid, vdev->dev_addr, &aq_err); 7073 if (ret) { 7074 /* if we cannot add the MAC rule then disable the offload */ 7075 macvlan_release_l2fw_offload(vdev); 7076 for (i = 0; i < ch->num_queue_pairs; i++) { 7077 struct i40e_ring *rx_ring; 7078 u16 pf_q; 7079 7080 pf_q = ch->base_queue + i; 7081 rx_ring = vsi->rx_rings[pf_q]; 7082 rx_ring->netdev = NULL; 7083 } 7084 dev_info(&pf->pdev->dev, 7085 "Error adding mac filter on macvlan err %s, aq_err %s\n", 7086 i40e_stat_str(hw, ret), 7087 i40e_aq_str(hw, aq_err)); 7088 netdev_err(vdev, "L2fwd offload disabled to L2 filter error\n"); 7089 } 7090 7091 return ret; 7092 } 7093 7094 /** 7095 * i40e_setup_macvlans - create the channels which will be macvlans 7096 * @vsi: the VSI we want to access 7097 * @macvlan_cnt: no. of macvlans to be setup 7098 * @qcnt: no. of Qs per macvlan 7099 * @vdev: macvlan netdevice 7100 */ 7101 static int i40e_setup_macvlans(struct i40e_vsi *vsi, u16 macvlan_cnt, u16 qcnt, 7102 struct net_device *vdev) 7103 { 7104 struct i40e_pf *pf = vsi->back; 7105 struct i40e_hw *hw = &pf->hw; 7106 struct i40e_vsi_context ctxt; 7107 u16 sections, qmap, num_qps; 7108 struct i40e_channel *ch; 7109 int i, pow, ret = 0; 7110 u8 offset = 0; 7111 7112 if (vsi->type != I40E_VSI_MAIN || !macvlan_cnt) 7113 return -EINVAL; 7114 7115 num_qps = vsi->num_queue_pairs - (macvlan_cnt * qcnt); 7116 7117 /* find the next higher power-of-2 of num queue pairs */ 7118 pow = fls(roundup_pow_of_two(num_qps) - 1); 7119 7120 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 7121 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 7122 7123 /* Setup context bits for the main VSI */ 7124 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 7125 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 7126 memset(&ctxt, 0, sizeof(ctxt)); 7127 ctxt.seid = vsi->seid; 7128 ctxt.pf_num = vsi->back->hw.pf_id; 7129 ctxt.vf_num = 0; 7130 ctxt.uplink_seid = vsi->uplink_seid; 7131 ctxt.info = vsi->info; 7132 ctxt.info.tc_mapping[0] = cpu_to_le16(qmap); 7133 ctxt.info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 7134 ctxt.info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 7135 ctxt.info.valid_sections |= cpu_to_le16(sections); 7136 7137 /* Reconfigure RSS for main VSI with new max queue count */ 7138 vsi->rss_size = max_t(u16, num_qps, qcnt); 7139 ret = i40e_vsi_config_rss(vsi); 7140 if (ret) { 7141 dev_info(&pf->pdev->dev, 7142 "Failed to reconfig RSS for num_queues (%u)\n", 7143 vsi->rss_size); 7144 return ret; 7145 } 7146 vsi->reconfig_rss = true; 7147 dev_dbg(&vsi->back->pdev->dev, 7148 "Reconfigured RSS with num_queues (%u)\n", vsi->rss_size); 7149 vsi->next_base_queue = num_qps; 7150 vsi->cnt_q_avail = vsi->num_queue_pairs - num_qps; 7151 7152 /* Update the VSI after updating the VSI queue-mapping 7153 * information 7154 */ 7155 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7156 if (ret) { 7157 dev_info(&pf->pdev->dev, 7158 "Update vsi tc config failed, err %s aq_err %s\n", 7159 i40e_stat_str(hw, ret), 7160 i40e_aq_str(hw, hw->aq.asq_last_status)); 7161 return ret; 7162 } 7163 /* update the local VSI info with updated queue map */ 7164 i40e_vsi_update_queue_map(vsi, &ctxt); 7165 vsi->info.valid_sections = 0; 7166 7167 /* Create channels for macvlans */ 7168 INIT_LIST_HEAD(&vsi->macvlan_list); 7169 for (i = 0; i < macvlan_cnt; i++) { 7170 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 7171 if (!ch) { 7172 ret = -ENOMEM; 7173 goto err_free; 7174 } 7175 INIT_LIST_HEAD(&ch->list); 7176 ch->num_queue_pairs = qcnt; 7177 if (!i40e_setup_channel(pf, vsi, ch)) { 7178 ret = -EINVAL; 7179 kfree(ch); 7180 goto err_free; 7181 } 7182 ch->parent_vsi = vsi; 7183 vsi->cnt_q_avail -= ch->num_queue_pairs; 7184 vsi->macvlan_cnt++; 7185 list_add_tail(&ch->list, &vsi->macvlan_list); 7186 } 7187 7188 return ret; 7189 7190 err_free: 7191 dev_info(&pf->pdev->dev, "Failed to setup macvlans\n"); 7192 i40e_free_macvlan_channels(vsi); 7193 7194 return ret; 7195 } 7196 7197 /** 7198 * i40e_fwd_add - configure macvlans 7199 * @netdev: net device to configure 7200 * @vdev: macvlan netdevice 7201 **/ 7202 static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) 7203 { 7204 struct i40e_netdev_priv *np = netdev_priv(netdev); 7205 u16 q_per_macvlan = 0, macvlan_cnt = 0, vectors; 7206 struct i40e_vsi *vsi = np->vsi; 7207 struct i40e_pf *pf = vsi->back; 7208 struct i40e_fwd_adapter *fwd; 7209 int avail_macvlan, ret; 7210 7211 if ((pf->flags & I40E_FLAG_DCB_ENABLED)) { 7212 netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); 7213 return ERR_PTR(-EINVAL); 7214 } 7215 if ((pf->flags & I40E_FLAG_TC_MQPRIO)) { 7216 netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); 7217 return ERR_PTR(-EINVAL); 7218 } 7219 if (pf->num_lan_msix < I40E_MIN_MACVLAN_VECTORS) { 7220 netdev_info(netdev, "Not enough vectors available to support macvlans\n"); 7221 return ERR_PTR(-EINVAL); 7222 } 7223 7224 /* The macvlan device has to be a single Q device so that the 7225 * tc_to_txq field can be reused to pick the tx queue. 7226 */ 7227 if (netif_is_multiqueue(vdev)) 7228 return ERR_PTR(-ERANGE); 7229 7230 if (!vsi->macvlan_cnt) { 7231 /* reserve bit 0 for the pf device */ 7232 set_bit(0, vsi->fwd_bitmask); 7233 7234 /* Try to reserve as many queues as possible for macvlans. First 7235 * reserve 3/4th of max vectors, then half, then quarter and 7236 * calculate Qs per macvlan as you go 7237 */ 7238 vectors = pf->num_lan_msix; 7239 if (vectors <= I40E_MAX_MACVLANS && vectors > 64) { 7240 /* allocate 4 Qs per macvlan and 32 Qs to the PF*/ 7241 q_per_macvlan = 4; 7242 macvlan_cnt = (vectors - 32) / 4; 7243 } else if (vectors <= 64 && vectors > 32) { 7244 /* allocate 2 Qs per macvlan and 16 Qs to the PF*/ 7245 q_per_macvlan = 2; 7246 macvlan_cnt = (vectors - 16) / 2; 7247 } else if (vectors <= 32 && vectors > 16) { 7248 /* allocate 1 Q per macvlan and 16 Qs to the PF*/ 7249 q_per_macvlan = 1; 7250 macvlan_cnt = vectors - 16; 7251 } else if (vectors <= 16 && vectors > 8) { 7252 /* allocate 1 Q per macvlan and 8 Qs to the PF */ 7253 q_per_macvlan = 1; 7254 macvlan_cnt = vectors - 8; 7255 } else { 7256 /* allocate 1 Q per macvlan and 1 Q to the PF */ 7257 q_per_macvlan = 1; 7258 macvlan_cnt = vectors - 1; 7259 } 7260 7261 if (macvlan_cnt == 0) 7262 return ERR_PTR(-EBUSY); 7263 7264 /* Quiesce VSI queues */ 7265 i40e_quiesce_vsi(vsi); 7266 7267 /* sets up the macvlans but does not "enable" them */ 7268 ret = i40e_setup_macvlans(vsi, macvlan_cnt, q_per_macvlan, 7269 vdev); 7270 if (ret) 7271 return ERR_PTR(ret); 7272 7273 /* Unquiesce VSI */ 7274 i40e_unquiesce_vsi(vsi); 7275 } 7276 avail_macvlan = find_first_zero_bit(vsi->fwd_bitmask, 7277 vsi->macvlan_cnt); 7278 if (avail_macvlan >= I40E_MAX_MACVLANS) 7279 return ERR_PTR(-EBUSY); 7280 7281 /* create the fwd struct */ 7282 fwd = kzalloc(sizeof(*fwd), GFP_KERNEL); 7283 if (!fwd) 7284 return ERR_PTR(-ENOMEM); 7285 7286 set_bit(avail_macvlan, vsi->fwd_bitmask); 7287 fwd->bit_no = avail_macvlan; 7288 netdev_set_sb_channel(vdev, avail_macvlan); 7289 fwd->netdev = vdev; 7290 7291 if (!netif_running(netdev)) 7292 return fwd; 7293 7294 /* Set fwd ring up */ 7295 ret = i40e_fwd_ring_up(vsi, vdev, fwd); 7296 if (ret) { 7297 /* unbind the queues and drop the subordinate channel config */ 7298 netdev_unbind_sb_channel(netdev, vdev); 7299 netdev_set_sb_channel(vdev, 0); 7300 7301 kfree(fwd); 7302 return ERR_PTR(-EINVAL); 7303 } 7304 7305 return fwd; 7306 } 7307 7308 /** 7309 * i40e_del_all_macvlans - Delete all the mac filters on the channels 7310 * @vsi: the VSI we want to access 7311 */ 7312 static void i40e_del_all_macvlans(struct i40e_vsi *vsi) 7313 { 7314 struct i40e_channel *ch, *ch_tmp; 7315 struct i40e_pf *pf = vsi->back; 7316 struct i40e_hw *hw = &pf->hw; 7317 int aq_err, ret = 0; 7318 7319 if (list_empty(&vsi->macvlan_list)) 7320 return; 7321 7322 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7323 if (i40e_is_channel_macvlan(ch)) { 7324 ret = i40e_del_macvlan_filter(hw, ch->seid, 7325 i40e_channel_mac(ch), 7326 &aq_err); 7327 if (!ret) { 7328 /* Reset queue contexts */ 7329 i40e_reset_ch_rings(vsi, ch); 7330 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7331 netdev_unbind_sb_channel(vsi->netdev, 7332 ch->fwd->netdev); 7333 netdev_set_sb_channel(ch->fwd->netdev, 0); 7334 kfree(ch->fwd); 7335 ch->fwd = NULL; 7336 } 7337 } 7338 } 7339 } 7340 7341 /** 7342 * i40e_fwd_del - delete macvlan interfaces 7343 * @netdev: net device to configure 7344 * @vdev: macvlan netdevice 7345 */ 7346 static void i40e_fwd_del(struct net_device *netdev, void *vdev) 7347 { 7348 struct i40e_netdev_priv *np = netdev_priv(netdev); 7349 struct i40e_fwd_adapter *fwd = vdev; 7350 struct i40e_channel *ch, *ch_tmp; 7351 struct i40e_vsi *vsi = np->vsi; 7352 struct i40e_pf *pf = vsi->back; 7353 struct i40e_hw *hw = &pf->hw; 7354 int aq_err, ret = 0; 7355 7356 /* Find the channel associated with the macvlan and del mac filter */ 7357 list_for_each_entry_safe(ch, ch_tmp, &vsi->macvlan_list, list) { 7358 if (i40e_is_channel_macvlan(ch) && 7359 ether_addr_equal(i40e_channel_mac(ch), 7360 fwd->netdev->dev_addr)) { 7361 ret = i40e_del_macvlan_filter(hw, ch->seid, 7362 i40e_channel_mac(ch), 7363 &aq_err); 7364 if (!ret) { 7365 /* Reset queue contexts */ 7366 i40e_reset_ch_rings(vsi, ch); 7367 clear_bit(ch->fwd->bit_no, vsi->fwd_bitmask); 7368 netdev_unbind_sb_channel(netdev, fwd->netdev); 7369 netdev_set_sb_channel(fwd->netdev, 0); 7370 kfree(ch->fwd); 7371 ch->fwd = NULL; 7372 } else { 7373 dev_info(&pf->pdev->dev, 7374 "Error deleting mac filter on macvlan err %s, aq_err %s\n", 7375 i40e_stat_str(hw, ret), 7376 i40e_aq_str(hw, aq_err)); 7377 } 7378 break; 7379 } 7380 } 7381 } 7382 7383 /** 7384 * i40e_setup_tc - configure multiple traffic classes 7385 * @netdev: net device to configure 7386 * @type_data: tc offload data 7387 **/ 7388 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 7389 { 7390 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 7391 struct i40e_netdev_priv *np = netdev_priv(netdev); 7392 struct i40e_vsi *vsi = np->vsi; 7393 struct i40e_pf *pf = vsi->back; 7394 u8 enabled_tc = 0, num_tc, hw; 7395 bool need_reset = false; 7396 int old_queue_pairs; 7397 int ret = -EINVAL; 7398 u16 mode; 7399 int i; 7400 7401 old_queue_pairs = vsi->num_queue_pairs; 7402 num_tc = mqprio_qopt->qopt.num_tc; 7403 hw = mqprio_qopt->qopt.hw; 7404 mode = mqprio_qopt->mode; 7405 if (!hw) { 7406 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 7407 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 7408 goto config_tc; 7409 } 7410 7411 /* Check if MFP enabled */ 7412 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 7413 netdev_info(netdev, 7414 "Configuring TC not supported in MFP mode\n"); 7415 return ret; 7416 } 7417 switch (mode) { 7418 case TC_MQPRIO_MODE_DCB: 7419 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 7420 7421 /* Check if DCB enabled to continue */ 7422 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 7423 netdev_info(netdev, 7424 "DCB is not enabled for adapter\n"); 7425 return ret; 7426 } 7427 7428 /* Check whether tc count is within enabled limit */ 7429 if (num_tc > i40e_pf_get_num_tc(pf)) { 7430 netdev_info(netdev, 7431 "TC count greater than enabled on link for adapter\n"); 7432 return ret; 7433 } 7434 break; 7435 case TC_MQPRIO_MODE_CHANNEL: 7436 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 7437 netdev_info(netdev, 7438 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 7439 return ret; 7440 } 7441 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 7442 return ret; 7443 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 7444 if (ret) 7445 return ret; 7446 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 7447 sizeof(*mqprio_qopt)); 7448 pf->flags |= I40E_FLAG_TC_MQPRIO; 7449 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 7450 break; 7451 default: 7452 return -EINVAL; 7453 } 7454 7455 config_tc: 7456 /* Generate TC map for number of tc requested */ 7457 for (i = 0; i < num_tc; i++) 7458 enabled_tc |= BIT(i); 7459 7460 /* Requesting same TC configuration as already enabled */ 7461 if (enabled_tc == vsi->tc_config.enabled_tc && 7462 mode != TC_MQPRIO_MODE_CHANNEL) 7463 return 0; 7464 7465 /* Quiesce VSI queues */ 7466 i40e_quiesce_vsi(vsi); 7467 7468 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) 7469 i40e_remove_queue_channels(vsi); 7470 7471 /* Configure VSI for enabled TCs */ 7472 ret = i40e_vsi_config_tc(vsi, enabled_tc); 7473 if (ret) { 7474 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 7475 vsi->seid); 7476 need_reset = true; 7477 goto exit; 7478 } else { 7479 dev_info(&vsi->back->pdev->dev, 7480 "Setup channel (id:%u) utilizing num_queues %d\n", 7481 vsi->seid, vsi->tc_config.tc_info[0].qcount); 7482 } 7483 7484 if (pf->flags & I40E_FLAG_TC_MQPRIO) { 7485 if (vsi->mqprio_qopt.max_rate[0]) { 7486 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 7487 7488 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 7489 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 7490 if (!ret) { 7491 u64 credits = max_tx_rate; 7492 7493 do_div(credits, I40E_BW_CREDIT_DIVISOR); 7494 dev_dbg(&vsi->back->pdev->dev, 7495 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 7496 max_tx_rate, 7497 credits, 7498 vsi->seid); 7499 } else { 7500 need_reset = true; 7501 goto exit; 7502 } 7503 } 7504 ret = i40e_configure_queue_channels(vsi); 7505 if (ret) { 7506 vsi->num_queue_pairs = old_queue_pairs; 7507 netdev_info(netdev, 7508 "Failed configuring queue channels\n"); 7509 need_reset = true; 7510 goto exit; 7511 } 7512 } 7513 7514 exit: 7515 /* Reset the configuration data to defaults, only TC0 is enabled */ 7516 if (need_reset) { 7517 i40e_vsi_set_default_tc_config(vsi); 7518 need_reset = false; 7519 } 7520 7521 /* Unquiesce VSI */ 7522 i40e_unquiesce_vsi(vsi); 7523 return ret; 7524 } 7525 7526 /** 7527 * i40e_set_cld_element - sets cloud filter element data 7528 * @filter: cloud filter rule 7529 * @cld: ptr to cloud filter element data 7530 * 7531 * This is helper function to copy data into cloud filter element 7532 **/ 7533 static inline void 7534 i40e_set_cld_element(struct i40e_cloud_filter *filter, 7535 struct i40e_aqc_cloud_filters_element_data *cld) 7536 { 7537 int i, j; 7538 u32 ipa; 7539 7540 memset(cld, 0, sizeof(*cld)); 7541 ether_addr_copy(cld->outer_mac, filter->dst_mac); 7542 ether_addr_copy(cld->inner_mac, filter->src_mac); 7543 7544 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 7545 return; 7546 7547 if (filter->n_proto == ETH_P_IPV6) { 7548 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 7549 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); 7550 i++, j += 2) { 7551 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 7552 ipa = cpu_to_le32(ipa); 7553 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); 7554 } 7555 } else { 7556 ipa = be32_to_cpu(filter->dst_ipv4); 7557 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 7558 } 7559 7560 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 7561 7562 /* tenant_id is not supported by FW now, once the support is enabled 7563 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 7564 */ 7565 if (filter->tenant_id) 7566 return; 7567 } 7568 7569 /** 7570 * i40e_add_del_cloud_filter - Add/del cloud filter 7571 * @vsi: pointer to VSI 7572 * @filter: cloud filter rule 7573 * @add: if true, add, if false, delete 7574 * 7575 * Add or delete a cloud filter for a specific flow spec. 7576 * Returns 0 if the filter were successfully added. 7577 **/ 7578 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 7579 struct i40e_cloud_filter *filter, bool add) 7580 { 7581 struct i40e_aqc_cloud_filters_element_data cld_filter; 7582 struct i40e_pf *pf = vsi->back; 7583 int ret; 7584 static const u16 flag_table[128] = { 7585 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 7586 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 7587 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 7588 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 7589 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 7590 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 7591 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 7592 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 7593 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 7594 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 7595 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 7596 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 7597 [I40E_CLOUD_FILTER_FLAGS_IIP] = 7598 I40E_AQC_ADD_CLOUD_FILTER_IIP, 7599 }; 7600 7601 if (filter->flags >= ARRAY_SIZE(flag_table)) 7602 return I40E_ERR_CONFIG; 7603 7604 /* copy element needed to add cloud filter from filter */ 7605 i40e_set_cld_element(filter, &cld_filter); 7606 7607 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 7608 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 7609 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 7610 7611 if (filter->n_proto == ETH_P_IPV6) 7612 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7613 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7614 else 7615 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 7616 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7617 7618 if (add) 7619 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 7620 &cld_filter, 1); 7621 else 7622 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 7623 &cld_filter, 1); 7624 if (ret) 7625 dev_dbg(&pf->pdev->dev, 7626 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 7627 add ? "add" : "delete", filter->dst_port, ret, 7628 pf->hw.aq.asq_last_status); 7629 else 7630 dev_info(&pf->pdev->dev, 7631 "%s cloud filter for VSI: %d\n", 7632 add ? "Added" : "Deleted", filter->seid); 7633 return ret; 7634 } 7635 7636 /** 7637 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 7638 * @vsi: pointer to VSI 7639 * @filter: cloud filter rule 7640 * @add: if true, add, if false, delete 7641 * 7642 * Add or delete a cloud filter for a specific flow spec using big buffer. 7643 * Returns 0 if the filter were successfully added. 7644 **/ 7645 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 7646 struct i40e_cloud_filter *filter, 7647 bool add) 7648 { 7649 struct i40e_aqc_cloud_filters_element_bb cld_filter; 7650 struct i40e_pf *pf = vsi->back; 7651 int ret; 7652 7653 /* Both (src/dst) valid mac_addr are not supported */ 7654 if ((is_valid_ether_addr(filter->dst_mac) && 7655 is_valid_ether_addr(filter->src_mac)) || 7656 (is_multicast_ether_addr(filter->dst_mac) && 7657 is_multicast_ether_addr(filter->src_mac))) 7658 return -EOPNOTSUPP; 7659 7660 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 7661 * ports are not supported via big buffer now. 7662 */ 7663 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 7664 return -EOPNOTSUPP; 7665 7666 /* adding filter using src_port/src_ip is not supported at this stage */ 7667 if (filter->src_port || filter->src_ipv4 || 7668 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 7669 return -EOPNOTSUPP; 7670 7671 /* copy element needed to add cloud filter from filter */ 7672 i40e_set_cld_element(filter, &cld_filter.element); 7673 7674 if (is_valid_ether_addr(filter->dst_mac) || 7675 is_valid_ether_addr(filter->src_mac) || 7676 is_multicast_ether_addr(filter->dst_mac) || 7677 is_multicast_ether_addr(filter->src_mac)) { 7678 /* MAC + IP : unsupported mode */ 7679 if (filter->dst_ipv4) 7680 return -EOPNOTSUPP; 7681 7682 /* since we validated that L4 port must be valid before 7683 * we get here, start with respective "flags" value 7684 * and update if vlan is present or not 7685 */ 7686 cld_filter.element.flags = 7687 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 7688 7689 if (filter->vlan_id) { 7690 cld_filter.element.flags = 7691 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 7692 } 7693 7694 } else if (filter->dst_ipv4 || 7695 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 7696 cld_filter.element.flags = 7697 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 7698 if (filter->n_proto == ETH_P_IPV6) 7699 cld_filter.element.flags |= 7700 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7701 else 7702 cld_filter.element.flags |= 7703 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7704 } else { 7705 dev_err(&pf->pdev->dev, 7706 "either mac or ip has to be valid for cloud filter\n"); 7707 return -EINVAL; 7708 } 7709 7710 /* Now copy L4 port in Byte 6..7 in general fields */ 7711 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 7712 be16_to_cpu(filter->dst_port); 7713 7714 if (add) { 7715 /* Validate current device switch mode, change if necessary */ 7716 ret = i40e_validate_and_set_switch_mode(vsi); 7717 if (ret) { 7718 dev_err(&pf->pdev->dev, 7719 "failed to set switch mode, ret %d\n", 7720 ret); 7721 return ret; 7722 } 7723 7724 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 7725 &cld_filter, 1); 7726 } else { 7727 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 7728 &cld_filter, 1); 7729 } 7730 7731 if (ret) 7732 dev_dbg(&pf->pdev->dev, 7733 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 7734 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 7735 else 7736 dev_info(&pf->pdev->dev, 7737 "%s cloud filter for VSI: %d, L4 port: %d\n", 7738 add ? "add" : "delete", filter->seid, 7739 ntohs(filter->dst_port)); 7740 return ret; 7741 } 7742 7743 /** 7744 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 7745 * @vsi: Pointer to VSI 7746 * @cls_flower: Pointer to struct flow_cls_offload 7747 * @filter: Pointer to cloud filter structure 7748 * 7749 **/ 7750 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 7751 struct flow_cls_offload *f, 7752 struct i40e_cloud_filter *filter) 7753 { 7754 struct flow_rule *rule = flow_cls_offload_flow_rule(f); 7755 struct flow_dissector *dissector = rule->match.dissector; 7756 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 7757 struct i40e_pf *pf = vsi->back; 7758 u8 field_flags = 0; 7759 7760 if (dissector->used_keys & 7761 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 7762 BIT(FLOW_DISSECTOR_KEY_BASIC) | 7763 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 7764 BIT(FLOW_DISSECTOR_KEY_VLAN) | 7765 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 7766 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 7767 BIT(FLOW_DISSECTOR_KEY_PORTS) | 7768 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 7769 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 7770 dissector->used_keys); 7771 return -EOPNOTSUPP; 7772 } 7773 7774 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 7775 struct flow_match_enc_keyid match; 7776 7777 flow_rule_match_enc_keyid(rule, &match); 7778 if (match.mask->keyid != 0) 7779 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 7780 7781 filter->tenant_id = be32_to_cpu(match.key->keyid); 7782 } 7783 7784 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) { 7785 struct flow_match_basic match; 7786 7787 flow_rule_match_basic(rule, &match); 7788 n_proto_key = ntohs(match.key->n_proto); 7789 n_proto_mask = ntohs(match.mask->n_proto); 7790 7791 if (n_proto_key == ETH_P_ALL) { 7792 n_proto_key = 0; 7793 n_proto_mask = 0; 7794 } 7795 filter->n_proto = n_proto_key & n_proto_mask; 7796 filter->ip_proto = match.key->ip_proto; 7797 } 7798 7799 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 7800 struct flow_match_eth_addrs match; 7801 7802 flow_rule_match_eth_addrs(rule, &match); 7803 7804 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 7805 if (!is_zero_ether_addr(match.mask->dst)) { 7806 if (is_broadcast_ether_addr(match.mask->dst)) { 7807 field_flags |= I40E_CLOUD_FIELD_OMAC; 7808 } else { 7809 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 7810 match.mask->dst); 7811 return I40E_ERR_CONFIG; 7812 } 7813 } 7814 7815 if (!is_zero_ether_addr(match.mask->src)) { 7816 if (is_broadcast_ether_addr(match.mask->src)) { 7817 field_flags |= I40E_CLOUD_FIELD_IMAC; 7818 } else { 7819 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 7820 match.mask->src); 7821 return I40E_ERR_CONFIG; 7822 } 7823 } 7824 ether_addr_copy(filter->dst_mac, match.key->dst); 7825 ether_addr_copy(filter->src_mac, match.key->src); 7826 } 7827 7828 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) { 7829 struct flow_match_vlan match; 7830 7831 flow_rule_match_vlan(rule, &match); 7832 if (match.mask->vlan_id) { 7833 if (match.mask->vlan_id == VLAN_VID_MASK) { 7834 field_flags |= I40E_CLOUD_FIELD_IVLAN; 7835 7836 } else { 7837 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 7838 match.mask->vlan_id); 7839 return I40E_ERR_CONFIG; 7840 } 7841 } 7842 7843 filter->vlan_id = cpu_to_be16(match.key->vlan_id); 7844 } 7845 7846 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) { 7847 struct flow_match_control match; 7848 7849 flow_rule_match_control(rule, &match); 7850 addr_type = match.key->addr_type; 7851 } 7852 7853 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 7854 struct flow_match_ipv4_addrs match; 7855 7856 flow_rule_match_ipv4_addrs(rule, &match); 7857 if (match.mask->dst) { 7858 if (match.mask->dst == cpu_to_be32(0xffffffff)) { 7859 field_flags |= I40E_CLOUD_FIELD_IIP; 7860 } else { 7861 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 7862 &match.mask->dst); 7863 return I40E_ERR_CONFIG; 7864 } 7865 } 7866 7867 if (match.mask->src) { 7868 if (match.mask->src == cpu_to_be32(0xffffffff)) { 7869 field_flags |= I40E_CLOUD_FIELD_IIP; 7870 } else { 7871 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 7872 &match.mask->src); 7873 return I40E_ERR_CONFIG; 7874 } 7875 } 7876 7877 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 7878 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 7879 return I40E_ERR_CONFIG; 7880 } 7881 filter->dst_ipv4 = match.key->dst; 7882 filter->src_ipv4 = match.key->src; 7883 } 7884 7885 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 7886 struct flow_match_ipv6_addrs match; 7887 7888 flow_rule_match_ipv6_addrs(rule, &match); 7889 7890 /* src and dest IPV6 address should not be LOOPBACK 7891 * (0:0:0:0:0:0:0:1), which can be represented as ::1 7892 */ 7893 if (ipv6_addr_loopback(&match.key->dst) || 7894 ipv6_addr_loopback(&match.key->src)) { 7895 dev_err(&pf->pdev->dev, 7896 "Bad ipv6, addr is LOOPBACK\n"); 7897 return I40E_ERR_CONFIG; 7898 } 7899 if (!ipv6_addr_any(&match.mask->dst) || 7900 !ipv6_addr_any(&match.mask->src)) 7901 field_flags |= I40E_CLOUD_FIELD_IIP; 7902 7903 memcpy(&filter->src_ipv6, &match.key->src.s6_addr32, 7904 sizeof(filter->src_ipv6)); 7905 memcpy(&filter->dst_ipv6, &match.key->dst.s6_addr32, 7906 sizeof(filter->dst_ipv6)); 7907 } 7908 7909 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) { 7910 struct flow_match_ports match; 7911 7912 flow_rule_match_ports(rule, &match); 7913 if (match.mask->src) { 7914 if (match.mask->src == cpu_to_be16(0xffff)) { 7915 field_flags |= I40E_CLOUD_FIELD_IIP; 7916 } else { 7917 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 7918 be16_to_cpu(match.mask->src)); 7919 return I40E_ERR_CONFIG; 7920 } 7921 } 7922 7923 if (match.mask->dst) { 7924 if (match.mask->dst == cpu_to_be16(0xffff)) { 7925 field_flags |= I40E_CLOUD_FIELD_IIP; 7926 } else { 7927 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 7928 be16_to_cpu(match.mask->dst)); 7929 return I40E_ERR_CONFIG; 7930 } 7931 } 7932 7933 filter->dst_port = match.key->dst; 7934 filter->src_port = match.key->src; 7935 7936 switch (filter->ip_proto) { 7937 case IPPROTO_TCP: 7938 case IPPROTO_UDP: 7939 break; 7940 default: 7941 dev_err(&pf->pdev->dev, 7942 "Only UDP and TCP transport are supported\n"); 7943 return -EINVAL; 7944 } 7945 } 7946 filter->flags = field_flags; 7947 return 0; 7948 } 7949 7950 /** 7951 * i40e_handle_tclass: Forward to a traffic class on the device 7952 * @vsi: Pointer to VSI 7953 * @tc: traffic class index on the device 7954 * @filter: Pointer to cloud filter structure 7955 * 7956 **/ 7957 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 7958 struct i40e_cloud_filter *filter) 7959 { 7960 struct i40e_channel *ch, *ch_tmp; 7961 7962 /* direct to a traffic class on the same device */ 7963 if (tc == 0) { 7964 filter->seid = vsi->seid; 7965 return 0; 7966 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 7967 if (!filter->dst_port) { 7968 dev_err(&vsi->back->pdev->dev, 7969 "Specify destination port to direct to traffic class that is not default\n"); 7970 return -EINVAL; 7971 } 7972 if (list_empty(&vsi->ch_list)) 7973 return -EINVAL; 7974 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 7975 list) { 7976 if (ch->seid == vsi->tc_seid_map[tc]) 7977 filter->seid = ch->seid; 7978 } 7979 return 0; 7980 } 7981 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 7982 return -EINVAL; 7983 } 7984 7985 /** 7986 * i40e_configure_clsflower - Configure tc flower filters 7987 * @vsi: Pointer to VSI 7988 * @cls_flower: Pointer to struct flow_cls_offload 7989 * 7990 **/ 7991 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 7992 struct flow_cls_offload *cls_flower) 7993 { 7994 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 7995 struct i40e_cloud_filter *filter = NULL; 7996 struct i40e_pf *pf = vsi->back; 7997 int err = 0; 7998 7999 if (tc < 0) { 8000 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 8001 return -EOPNOTSUPP; 8002 } 8003 8004 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 8005 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 8006 return -EBUSY; 8007 8008 if (pf->fdir_pf_active_filters || 8009 (!hlist_empty(&pf->fdir_filter_list))) { 8010 dev_err(&vsi->back->pdev->dev, 8011 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 8012 return -EINVAL; 8013 } 8014 8015 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 8016 dev_err(&vsi->back->pdev->dev, 8017 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 8018 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8019 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8020 } 8021 8022 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 8023 if (!filter) 8024 return -ENOMEM; 8025 8026 filter->cookie = cls_flower->cookie; 8027 8028 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 8029 if (err < 0) 8030 goto err; 8031 8032 err = i40e_handle_tclass(vsi, tc, filter); 8033 if (err < 0) 8034 goto err; 8035 8036 /* Add cloud filter */ 8037 if (filter->dst_port) 8038 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 8039 else 8040 err = i40e_add_del_cloud_filter(vsi, filter, true); 8041 8042 if (err) { 8043 dev_err(&pf->pdev->dev, 8044 "Failed to add cloud filter, err %s\n", 8045 i40e_stat_str(&pf->hw, err)); 8046 goto err; 8047 } 8048 8049 /* add filter to the ordered list */ 8050 INIT_HLIST_NODE(&filter->cloud_node); 8051 8052 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 8053 8054 pf->num_cloud_filters++; 8055 8056 return err; 8057 err: 8058 kfree(filter); 8059 return err; 8060 } 8061 8062 /** 8063 * i40e_find_cloud_filter - Find the could filter in the list 8064 * @vsi: Pointer to VSI 8065 * @cookie: filter specific cookie 8066 * 8067 **/ 8068 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 8069 unsigned long *cookie) 8070 { 8071 struct i40e_cloud_filter *filter = NULL; 8072 struct hlist_node *node2; 8073 8074 hlist_for_each_entry_safe(filter, node2, 8075 &vsi->back->cloud_filter_list, cloud_node) 8076 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 8077 return filter; 8078 return NULL; 8079 } 8080 8081 /** 8082 * i40e_delete_clsflower - Remove tc flower filters 8083 * @vsi: Pointer to VSI 8084 * @cls_flower: Pointer to struct flow_cls_offload 8085 * 8086 **/ 8087 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 8088 struct flow_cls_offload *cls_flower) 8089 { 8090 struct i40e_cloud_filter *filter = NULL; 8091 struct i40e_pf *pf = vsi->back; 8092 int err = 0; 8093 8094 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 8095 8096 if (!filter) 8097 return -EINVAL; 8098 8099 hash_del(&filter->cloud_node); 8100 8101 if (filter->dst_port) 8102 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 8103 else 8104 err = i40e_add_del_cloud_filter(vsi, filter, false); 8105 8106 kfree(filter); 8107 if (err) { 8108 dev_err(&pf->pdev->dev, 8109 "Failed to delete cloud filter, err %s\n", 8110 i40e_stat_str(&pf->hw, err)); 8111 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 8112 } 8113 8114 pf->num_cloud_filters--; 8115 if (!pf->num_cloud_filters) 8116 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8117 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8118 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8119 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8120 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8121 } 8122 return 0; 8123 } 8124 8125 /** 8126 * i40e_setup_tc_cls_flower - flower classifier offloads 8127 * @netdev: net device to configure 8128 * @type_data: offload data 8129 **/ 8130 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 8131 struct flow_cls_offload *cls_flower) 8132 { 8133 struct i40e_vsi *vsi = np->vsi; 8134 8135 switch (cls_flower->command) { 8136 case FLOW_CLS_REPLACE: 8137 return i40e_configure_clsflower(vsi, cls_flower); 8138 case FLOW_CLS_DESTROY: 8139 return i40e_delete_clsflower(vsi, cls_flower); 8140 case FLOW_CLS_STATS: 8141 return -EOPNOTSUPP; 8142 default: 8143 return -EOPNOTSUPP; 8144 } 8145 } 8146 8147 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 8148 void *cb_priv) 8149 { 8150 struct i40e_netdev_priv *np = cb_priv; 8151 8152 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 8153 return -EOPNOTSUPP; 8154 8155 switch (type) { 8156 case TC_SETUP_CLSFLOWER: 8157 return i40e_setup_tc_cls_flower(np, type_data); 8158 8159 default: 8160 return -EOPNOTSUPP; 8161 } 8162 } 8163 8164 static LIST_HEAD(i40e_block_cb_list); 8165 8166 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 8167 void *type_data) 8168 { 8169 struct i40e_netdev_priv *np = netdev_priv(netdev); 8170 8171 switch (type) { 8172 case TC_SETUP_QDISC_MQPRIO: 8173 return i40e_setup_tc(netdev, type_data); 8174 case TC_SETUP_BLOCK: 8175 return flow_block_cb_setup_simple(type_data, 8176 &i40e_block_cb_list, 8177 i40e_setup_tc_block_cb, 8178 np, np, true); 8179 default: 8180 return -EOPNOTSUPP; 8181 } 8182 } 8183 8184 /** 8185 * i40e_open - Called when a network interface is made active 8186 * @netdev: network interface device structure 8187 * 8188 * The open entry point is called when a network interface is made 8189 * active by the system (IFF_UP). At this point all resources needed 8190 * for transmit and receive operations are allocated, the interrupt 8191 * handler is registered with the OS, the netdev watchdog subtask is 8192 * enabled, and the stack is notified that the interface is ready. 8193 * 8194 * Returns 0 on success, negative value on failure 8195 **/ 8196 int i40e_open(struct net_device *netdev) 8197 { 8198 struct i40e_netdev_priv *np = netdev_priv(netdev); 8199 struct i40e_vsi *vsi = np->vsi; 8200 struct i40e_pf *pf = vsi->back; 8201 int err; 8202 8203 /* disallow open during test or if eeprom is broken */ 8204 if (test_bit(__I40E_TESTING, pf->state) || 8205 test_bit(__I40E_BAD_EEPROM, pf->state)) 8206 return -EBUSY; 8207 8208 netif_carrier_off(netdev); 8209 8210 if (i40e_force_link_state(pf, true)) 8211 return -EAGAIN; 8212 8213 err = i40e_vsi_open(vsi); 8214 if (err) 8215 return err; 8216 8217 /* configure global TSO hardware offload settings */ 8218 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 8219 TCP_FLAG_FIN) >> 16); 8220 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 8221 TCP_FLAG_FIN | 8222 TCP_FLAG_CWR) >> 16); 8223 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 8224 8225 udp_tunnel_get_rx_info(netdev); 8226 8227 return 0; 8228 } 8229 8230 /** 8231 * i40e_vsi_open - 8232 * @vsi: the VSI to open 8233 * 8234 * Finish initialization of the VSI. 8235 * 8236 * Returns 0 on success, negative value on failure 8237 * 8238 * Note: expects to be called while under rtnl_lock() 8239 **/ 8240 int i40e_vsi_open(struct i40e_vsi *vsi) 8241 { 8242 struct i40e_pf *pf = vsi->back; 8243 char int_name[I40E_INT_NAME_STR_LEN]; 8244 int err; 8245 8246 /* allocate descriptors */ 8247 err = i40e_vsi_setup_tx_resources(vsi); 8248 if (err) 8249 goto err_setup_tx; 8250 err = i40e_vsi_setup_rx_resources(vsi); 8251 if (err) 8252 goto err_setup_rx; 8253 8254 err = i40e_vsi_configure(vsi); 8255 if (err) 8256 goto err_setup_rx; 8257 8258 if (vsi->netdev) { 8259 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 8260 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 8261 err = i40e_vsi_request_irq(vsi, int_name); 8262 if (err) 8263 goto err_setup_rx; 8264 8265 /* Notify the stack of the actual queue counts. */ 8266 err = netif_set_real_num_tx_queues(vsi->netdev, 8267 vsi->num_queue_pairs); 8268 if (err) 8269 goto err_set_queues; 8270 8271 err = netif_set_real_num_rx_queues(vsi->netdev, 8272 vsi->num_queue_pairs); 8273 if (err) 8274 goto err_set_queues; 8275 8276 } else if (vsi->type == I40E_VSI_FDIR) { 8277 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 8278 dev_driver_string(&pf->pdev->dev), 8279 dev_name(&pf->pdev->dev)); 8280 err = i40e_vsi_request_irq(vsi, int_name); 8281 8282 } else { 8283 err = -EINVAL; 8284 goto err_setup_rx; 8285 } 8286 8287 err = i40e_up_complete(vsi); 8288 if (err) 8289 goto err_up_complete; 8290 8291 return 0; 8292 8293 err_up_complete: 8294 i40e_down(vsi); 8295 err_set_queues: 8296 i40e_vsi_free_irq(vsi); 8297 err_setup_rx: 8298 i40e_vsi_free_rx_resources(vsi); 8299 err_setup_tx: 8300 i40e_vsi_free_tx_resources(vsi); 8301 if (vsi == pf->vsi[pf->lan_vsi]) 8302 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 8303 8304 return err; 8305 } 8306 8307 /** 8308 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 8309 * @pf: Pointer to PF 8310 * 8311 * This function destroys the hlist where all the Flow Director 8312 * filters were saved. 8313 **/ 8314 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 8315 { 8316 struct i40e_fdir_filter *filter; 8317 struct i40e_flex_pit *pit_entry, *tmp; 8318 struct hlist_node *node2; 8319 8320 hlist_for_each_entry_safe(filter, node2, 8321 &pf->fdir_filter_list, fdir_node) { 8322 hlist_del(&filter->fdir_node); 8323 kfree(filter); 8324 } 8325 8326 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 8327 list_del(&pit_entry->list); 8328 kfree(pit_entry); 8329 } 8330 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 8331 8332 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 8333 list_del(&pit_entry->list); 8334 kfree(pit_entry); 8335 } 8336 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 8337 8338 pf->fdir_pf_active_filters = 0; 8339 pf->fd_tcp4_filter_cnt = 0; 8340 pf->fd_udp4_filter_cnt = 0; 8341 pf->fd_sctp4_filter_cnt = 0; 8342 pf->fd_ip4_filter_cnt = 0; 8343 8344 /* Reprogram the default input set for TCP/IPv4 */ 8345 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8346 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8347 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8348 8349 /* Reprogram the default input set for UDP/IPv4 */ 8350 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 8351 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8352 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8353 8354 /* Reprogram the default input set for SCTP/IPv4 */ 8355 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 8356 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8357 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8358 8359 /* Reprogram the default input set for Other/IPv4 */ 8360 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 8361 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 8362 8363 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 8364 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 8365 } 8366 8367 /** 8368 * i40e_cloud_filter_exit - Cleans up the cloud filters 8369 * @pf: Pointer to PF 8370 * 8371 * This function destroys the hlist where all the cloud filters 8372 * were saved. 8373 **/ 8374 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 8375 { 8376 struct i40e_cloud_filter *cfilter; 8377 struct hlist_node *node; 8378 8379 hlist_for_each_entry_safe(cfilter, node, 8380 &pf->cloud_filter_list, cloud_node) { 8381 hlist_del(&cfilter->cloud_node); 8382 kfree(cfilter); 8383 } 8384 pf->num_cloud_filters = 0; 8385 8386 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 8387 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 8388 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8389 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 8390 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 8391 } 8392 } 8393 8394 /** 8395 * i40e_close - Disables a network interface 8396 * @netdev: network interface device structure 8397 * 8398 * The close entry point is called when an interface is de-activated 8399 * by the OS. The hardware is still under the driver's control, but 8400 * this netdev interface is disabled. 8401 * 8402 * Returns 0, this is not allowed to fail 8403 **/ 8404 int i40e_close(struct net_device *netdev) 8405 { 8406 struct i40e_netdev_priv *np = netdev_priv(netdev); 8407 struct i40e_vsi *vsi = np->vsi; 8408 8409 i40e_vsi_close(vsi); 8410 8411 return 0; 8412 } 8413 8414 /** 8415 * i40e_do_reset - Start a PF or Core Reset sequence 8416 * @pf: board private structure 8417 * @reset_flags: which reset is requested 8418 * @lock_acquired: indicates whether or not the lock has been acquired 8419 * before this function was called. 8420 * 8421 * The essential difference in resets is that the PF Reset 8422 * doesn't clear the packet buffers, doesn't reset the PE 8423 * firmware, and doesn't bother the other PFs on the chip. 8424 **/ 8425 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 8426 { 8427 u32 val; 8428 8429 WARN_ON(in_interrupt()); 8430 8431 8432 /* do the biggest reset indicated */ 8433 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 8434 8435 /* Request a Global Reset 8436 * 8437 * This will start the chip's countdown to the actual full 8438 * chip reset event, and a warning interrupt to be sent 8439 * to all PFs, including the requestor. Our handler 8440 * for the warning interrupt will deal with the shutdown 8441 * and recovery of the switch setup. 8442 */ 8443 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 8444 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8445 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 8446 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 8447 8448 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 8449 8450 /* Request a Core Reset 8451 * 8452 * Same as Global Reset, except does *not* include the MAC/PHY 8453 */ 8454 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 8455 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8456 val |= I40E_GLGEN_RTRIG_CORER_MASK; 8457 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 8458 i40e_flush(&pf->hw); 8459 8460 } else if (reset_flags & I40E_PF_RESET_FLAG) { 8461 8462 /* Request a PF Reset 8463 * 8464 * Resets only the PF-specific registers 8465 * 8466 * This goes directly to the tear-down and rebuild of 8467 * the switch, since we need to do all the recovery as 8468 * for the Core Reset. 8469 */ 8470 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 8471 i40e_handle_reset_warning(pf, lock_acquired); 8472 8473 dev_info(&pf->pdev->dev, 8474 pf->flags & I40E_FLAG_DISABLE_FW_LLDP ? 8475 "FW LLDP is disabled\n" : 8476 "FW LLDP is enabled\n"); 8477 8478 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 8479 int v; 8480 8481 /* Find the VSI(s) that requested a re-init */ 8482 dev_info(&pf->pdev->dev, 8483 "VSI reinit requested\n"); 8484 for (v = 0; v < pf->num_alloc_vsi; v++) { 8485 struct i40e_vsi *vsi = pf->vsi[v]; 8486 8487 if (vsi != NULL && 8488 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 8489 vsi->state)) 8490 i40e_vsi_reinit_locked(pf->vsi[v]); 8491 } 8492 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 8493 int v; 8494 8495 /* Find the VSI(s) that needs to be brought down */ 8496 dev_info(&pf->pdev->dev, "VSI down requested\n"); 8497 for (v = 0; v < pf->num_alloc_vsi; v++) { 8498 struct i40e_vsi *vsi = pf->vsi[v]; 8499 8500 if (vsi != NULL && 8501 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 8502 vsi->state)) { 8503 set_bit(__I40E_VSI_DOWN, vsi->state); 8504 i40e_down(vsi); 8505 } 8506 } 8507 } else { 8508 dev_info(&pf->pdev->dev, 8509 "bad reset request 0x%08x\n", reset_flags); 8510 } 8511 } 8512 8513 #ifdef CONFIG_I40E_DCB 8514 /** 8515 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 8516 * @pf: board private structure 8517 * @old_cfg: current DCB config 8518 * @new_cfg: new DCB config 8519 **/ 8520 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 8521 struct i40e_dcbx_config *old_cfg, 8522 struct i40e_dcbx_config *new_cfg) 8523 { 8524 bool need_reconfig = false; 8525 8526 /* Check if ETS configuration has changed */ 8527 if (memcmp(&new_cfg->etscfg, 8528 &old_cfg->etscfg, 8529 sizeof(new_cfg->etscfg))) { 8530 /* If Priority Table has changed reconfig is needed */ 8531 if (memcmp(&new_cfg->etscfg.prioritytable, 8532 &old_cfg->etscfg.prioritytable, 8533 sizeof(new_cfg->etscfg.prioritytable))) { 8534 need_reconfig = true; 8535 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 8536 } 8537 8538 if (memcmp(&new_cfg->etscfg.tcbwtable, 8539 &old_cfg->etscfg.tcbwtable, 8540 sizeof(new_cfg->etscfg.tcbwtable))) 8541 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 8542 8543 if (memcmp(&new_cfg->etscfg.tsatable, 8544 &old_cfg->etscfg.tsatable, 8545 sizeof(new_cfg->etscfg.tsatable))) 8546 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 8547 } 8548 8549 /* Check if PFC configuration has changed */ 8550 if (memcmp(&new_cfg->pfc, 8551 &old_cfg->pfc, 8552 sizeof(new_cfg->pfc))) { 8553 need_reconfig = true; 8554 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 8555 } 8556 8557 /* Check if APP Table has changed */ 8558 if (memcmp(&new_cfg->app, 8559 &old_cfg->app, 8560 sizeof(new_cfg->app))) { 8561 need_reconfig = true; 8562 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 8563 } 8564 8565 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 8566 return need_reconfig; 8567 } 8568 8569 /** 8570 * i40e_handle_lldp_event - Handle LLDP Change MIB event 8571 * @pf: board private structure 8572 * @e: event info posted on ARQ 8573 **/ 8574 static int i40e_handle_lldp_event(struct i40e_pf *pf, 8575 struct i40e_arq_event_info *e) 8576 { 8577 struct i40e_aqc_lldp_get_mib *mib = 8578 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 8579 struct i40e_hw *hw = &pf->hw; 8580 struct i40e_dcbx_config tmp_dcbx_cfg; 8581 bool need_reconfig = false; 8582 int ret = 0; 8583 u8 type; 8584 8585 /* Not DCB capable or capability disabled */ 8586 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 8587 return ret; 8588 8589 /* Ignore if event is not for Nearest Bridge */ 8590 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 8591 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 8592 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 8593 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 8594 return ret; 8595 8596 /* Check MIB Type and return if event for Remote MIB update */ 8597 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 8598 dev_dbg(&pf->pdev->dev, 8599 "LLDP event mib type %s\n", type ? "remote" : "local"); 8600 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 8601 /* Update the remote cached instance and return */ 8602 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 8603 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 8604 &hw->remote_dcbx_config); 8605 goto exit; 8606 } 8607 8608 /* Store the old configuration */ 8609 tmp_dcbx_cfg = hw->local_dcbx_config; 8610 8611 /* Reset the old DCBx configuration data */ 8612 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 8613 /* Get updated DCBX data from firmware */ 8614 ret = i40e_get_dcb_config(&pf->hw); 8615 if (ret) { 8616 dev_info(&pf->pdev->dev, 8617 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 8618 i40e_stat_str(&pf->hw, ret), 8619 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8620 goto exit; 8621 } 8622 8623 /* No change detected in DCBX configs */ 8624 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 8625 sizeof(tmp_dcbx_cfg))) { 8626 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 8627 goto exit; 8628 } 8629 8630 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 8631 &hw->local_dcbx_config); 8632 8633 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 8634 8635 if (!need_reconfig) 8636 goto exit; 8637 8638 /* Enable DCB tagging only when more than one TC */ 8639 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 8640 pf->flags |= I40E_FLAG_DCB_ENABLED; 8641 else 8642 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 8643 8644 set_bit(__I40E_PORT_SUSPENDED, pf->state); 8645 /* Reconfiguration needed quiesce all VSIs */ 8646 i40e_pf_quiesce_all_vsi(pf); 8647 8648 /* Changes in configuration update VEB/VSI */ 8649 i40e_dcb_reconfigure(pf); 8650 8651 ret = i40e_resume_port_tx(pf); 8652 8653 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 8654 /* In case of error no point in resuming VSIs */ 8655 if (ret) 8656 goto exit; 8657 8658 /* Wait for the PF's queues to be disabled */ 8659 ret = i40e_pf_wait_queues_disabled(pf); 8660 if (ret) { 8661 /* Schedule PF reset to recover */ 8662 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8663 i40e_service_event_schedule(pf); 8664 } else { 8665 i40e_pf_unquiesce_all_vsi(pf); 8666 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 8667 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 8668 } 8669 8670 exit: 8671 return ret; 8672 } 8673 #endif /* CONFIG_I40E_DCB */ 8674 8675 /** 8676 * i40e_do_reset_safe - Protected reset path for userland calls. 8677 * @pf: board private structure 8678 * @reset_flags: which reset is requested 8679 * 8680 **/ 8681 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 8682 { 8683 rtnl_lock(); 8684 i40e_do_reset(pf, reset_flags, true); 8685 rtnl_unlock(); 8686 } 8687 8688 /** 8689 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 8690 * @pf: board private structure 8691 * @e: event info posted on ARQ 8692 * 8693 * Handler for LAN Queue Overflow Event generated by the firmware for PF 8694 * and VF queues 8695 **/ 8696 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 8697 struct i40e_arq_event_info *e) 8698 { 8699 struct i40e_aqc_lan_overflow *data = 8700 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 8701 u32 queue = le32_to_cpu(data->prtdcb_rupto); 8702 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 8703 struct i40e_hw *hw = &pf->hw; 8704 struct i40e_vf *vf; 8705 u16 vf_id; 8706 8707 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 8708 queue, qtx_ctl); 8709 8710 /* Queue belongs to VF, find the VF and issue VF reset */ 8711 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 8712 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 8713 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 8714 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 8715 vf_id -= hw->func_caps.vf_base_id; 8716 vf = &pf->vf[vf_id]; 8717 i40e_vc_notify_vf_reset(vf); 8718 /* Allow VF to process pending reset notification */ 8719 msleep(20); 8720 i40e_reset_vf(vf, false); 8721 } 8722 } 8723 8724 /** 8725 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 8726 * @pf: board private structure 8727 **/ 8728 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 8729 { 8730 u32 val, fcnt_prog; 8731 8732 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8733 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 8734 return fcnt_prog; 8735 } 8736 8737 /** 8738 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 8739 * @pf: board private structure 8740 **/ 8741 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 8742 { 8743 u32 val, fcnt_prog; 8744 8745 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8746 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 8747 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 8748 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 8749 return fcnt_prog; 8750 } 8751 8752 /** 8753 * i40e_get_global_fd_count - Get total FD filters programmed on device 8754 * @pf: board private structure 8755 **/ 8756 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 8757 { 8758 u32 val, fcnt_prog; 8759 8760 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 8761 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 8762 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 8763 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 8764 return fcnt_prog; 8765 } 8766 8767 /** 8768 * i40e_reenable_fdir_sb - Restore FDir SB capability 8769 * @pf: board private structure 8770 **/ 8771 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 8772 { 8773 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 8774 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8775 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8776 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 8777 } 8778 8779 /** 8780 * i40e_reenable_fdir_atr - Restore FDir ATR capability 8781 * @pf: board private structure 8782 **/ 8783 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 8784 { 8785 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 8786 /* ATR uses the same filtering logic as SB rules. It only 8787 * functions properly if the input set mask is at the default 8788 * settings. It is safe to restore the default input set 8789 * because there are no active TCPv4 filter rules. 8790 */ 8791 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8792 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8793 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8794 8795 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8796 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8797 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 8798 } 8799 } 8800 8801 /** 8802 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 8803 * @pf: board private structure 8804 * @filter: FDir filter to remove 8805 */ 8806 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 8807 struct i40e_fdir_filter *filter) 8808 { 8809 /* Update counters */ 8810 pf->fdir_pf_active_filters--; 8811 pf->fd_inv = 0; 8812 8813 switch (filter->flow_type) { 8814 case TCP_V4_FLOW: 8815 pf->fd_tcp4_filter_cnt--; 8816 break; 8817 case UDP_V4_FLOW: 8818 pf->fd_udp4_filter_cnt--; 8819 break; 8820 case SCTP_V4_FLOW: 8821 pf->fd_sctp4_filter_cnt--; 8822 break; 8823 case IP_USER_FLOW: 8824 switch (filter->ip4_proto) { 8825 case IPPROTO_TCP: 8826 pf->fd_tcp4_filter_cnt--; 8827 break; 8828 case IPPROTO_UDP: 8829 pf->fd_udp4_filter_cnt--; 8830 break; 8831 case IPPROTO_SCTP: 8832 pf->fd_sctp4_filter_cnt--; 8833 break; 8834 case IPPROTO_IP: 8835 pf->fd_ip4_filter_cnt--; 8836 break; 8837 } 8838 break; 8839 } 8840 8841 /* Remove the filter from the list and free memory */ 8842 hlist_del(&filter->fdir_node); 8843 kfree(filter); 8844 } 8845 8846 /** 8847 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 8848 * @pf: board private structure 8849 **/ 8850 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 8851 { 8852 struct i40e_fdir_filter *filter; 8853 u32 fcnt_prog, fcnt_avail; 8854 struct hlist_node *node; 8855 8856 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8857 return; 8858 8859 /* Check if we have enough room to re-enable FDir SB capability. */ 8860 fcnt_prog = i40e_get_global_fd_count(pf); 8861 fcnt_avail = pf->fdir_pf_filter_count; 8862 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 8863 (pf->fd_add_err == 0) || 8864 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 8865 i40e_reenable_fdir_sb(pf); 8866 8867 /* We should wait for even more space before re-enabling ATR. 8868 * Additionally, we cannot enable ATR as long as we still have TCP SB 8869 * rules active. 8870 */ 8871 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 8872 (pf->fd_tcp4_filter_cnt == 0)) 8873 i40e_reenable_fdir_atr(pf); 8874 8875 /* if hw had a problem adding a filter, delete it */ 8876 if (pf->fd_inv > 0) { 8877 hlist_for_each_entry_safe(filter, node, 8878 &pf->fdir_filter_list, fdir_node) 8879 if (filter->fd_id == pf->fd_inv) 8880 i40e_delete_invalid_filter(pf, filter); 8881 } 8882 } 8883 8884 #define I40E_MIN_FD_FLUSH_INTERVAL 10 8885 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 8886 /** 8887 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 8888 * @pf: board private structure 8889 **/ 8890 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 8891 { 8892 unsigned long min_flush_time; 8893 int flush_wait_retry = 50; 8894 bool disable_atr = false; 8895 int fd_room; 8896 int reg; 8897 8898 if (!time_after(jiffies, pf->fd_flush_timestamp + 8899 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 8900 return; 8901 8902 /* If the flush is happening too quick and we have mostly SB rules we 8903 * should not re-enable ATR for some time. 8904 */ 8905 min_flush_time = pf->fd_flush_timestamp + 8906 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 8907 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 8908 8909 if (!(time_after(jiffies, min_flush_time)) && 8910 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 8911 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8912 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 8913 disable_atr = true; 8914 } 8915 8916 pf->fd_flush_timestamp = jiffies; 8917 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8918 /* flush all filters */ 8919 wr32(&pf->hw, I40E_PFQF_CTL_1, 8920 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 8921 i40e_flush(&pf->hw); 8922 pf->fd_flush_cnt++; 8923 pf->fd_add_err = 0; 8924 do { 8925 /* Check FD flush status every 5-6msec */ 8926 usleep_range(5000, 6000); 8927 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 8928 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 8929 break; 8930 } while (flush_wait_retry--); 8931 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 8932 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 8933 } else { 8934 /* replay sideband filters */ 8935 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 8936 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 8937 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8938 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 8939 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8940 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 8941 } 8942 } 8943 8944 /** 8945 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 8946 * @pf: board private structure 8947 **/ 8948 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 8949 { 8950 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 8951 } 8952 8953 /* We can see up to 256 filter programming desc in transit if the filters are 8954 * being applied really fast; before we see the first 8955 * filter miss error on Rx queue 0. Accumulating enough error messages before 8956 * reacting will make sure we don't cause flush too often. 8957 */ 8958 #define I40E_MAX_FD_PROGRAM_ERROR 256 8959 8960 /** 8961 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 8962 * @pf: board private structure 8963 **/ 8964 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 8965 { 8966 8967 /* if interface is down do nothing */ 8968 if (test_bit(__I40E_DOWN, pf->state)) 8969 return; 8970 8971 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8972 i40e_fdir_flush_and_replay(pf); 8973 8974 i40e_fdir_check_and_reenable(pf); 8975 8976 } 8977 8978 /** 8979 * i40e_vsi_link_event - notify VSI of a link event 8980 * @vsi: vsi to be notified 8981 * @link_up: link up or down 8982 **/ 8983 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 8984 { 8985 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 8986 return; 8987 8988 switch (vsi->type) { 8989 case I40E_VSI_MAIN: 8990 if (!vsi->netdev || !vsi->netdev_registered) 8991 break; 8992 8993 if (link_up) { 8994 netif_carrier_on(vsi->netdev); 8995 netif_tx_wake_all_queues(vsi->netdev); 8996 } else { 8997 netif_carrier_off(vsi->netdev); 8998 netif_tx_stop_all_queues(vsi->netdev); 8999 } 9000 break; 9001 9002 case I40E_VSI_SRIOV: 9003 case I40E_VSI_VMDQ2: 9004 case I40E_VSI_CTRL: 9005 case I40E_VSI_IWARP: 9006 case I40E_VSI_MIRROR: 9007 default: 9008 /* there is no notification for other VSIs */ 9009 break; 9010 } 9011 } 9012 9013 /** 9014 * i40e_veb_link_event - notify elements on the veb of a link event 9015 * @veb: veb to be notified 9016 * @link_up: link up or down 9017 **/ 9018 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 9019 { 9020 struct i40e_pf *pf; 9021 int i; 9022 9023 if (!veb || !veb->pf) 9024 return; 9025 pf = veb->pf; 9026 9027 /* depth first... */ 9028 for (i = 0; i < I40E_MAX_VEB; i++) 9029 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 9030 i40e_veb_link_event(pf->veb[i], link_up); 9031 9032 /* ... now the local VSIs */ 9033 for (i = 0; i < pf->num_alloc_vsi; i++) 9034 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 9035 i40e_vsi_link_event(pf->vsi[i], link_up); 9036 } 9037 9038 /** 9039 * i40e_link_event - Update netif_carrier status 9040 * @pf: board private structure 9041 **/ 9042 static void i40e_link_event(struct i40e_pf *pf) 9043 { 9044 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9045 u8 new_link_speed, old_link_speed; 9046 i40e_status status; 9047 bool new_link, old_link; 9048 9049 /* set this to force the get_link_status call to refresh state */ 9050 pf->hw.phy.get_link_info = true; 9051 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 9052 status = i40e_get_link_status(&pf->hw, &new_link); 9053 9054 /* On success, disable temp link polling */ 9055 if (status == I40E_SUCCESS) { 9056 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9057 } else { 9058 /* Enable link polling temporarily until i40e_get_link_status 9059 * returns I40E_SUCCESS 9060 */ 9061 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 9062 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 9063 status); 9064 return; 9065 } 9066 9067 old_link_speed = pf->hw.phy.link_info_old.link_speed; 9068 new_link_speed = pf->hw.phy.link_info.link_speed; 9069 9070 if (new_link == old_link && 9071 new_link_speed == old_link_speed && 9072 (test_bit(__I40E_VSI_DOWN, vsi->state) || 9073 new_link == netif_carrier_ok(vsi->netdev))) 9074 return; 9075 9076 i40e_print_link_message(vsi, new_link); 9077 9078 /* Notify the base of the switch tree connected to 9079 * the link. Floating VEBs are not notified. 9080 */ 9081 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 9082 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 9083 else 9084 i40e_vsi_link_event(vsi, new_link); 9085 9086 if (pf->vf) 9087 i40e_vc_notify_link_state(pf); 9088 9089 if (pf->flags & I40E_FLAG_PTP) 9090 i40e_ptp_set_increment(pf); 9091 } 9092 9093 /** 9094 * i40e_watchdog_subtask - periodic checks not using event driven response 9095 * @pf: board private structure 9096 **/ 9097 static void i40e_watchdog_subtask(struct i40e_pf *pf) 9098 { 9099 int i; 9100 9101 /* if interface is down do nothing */ 9102 if (test_bit(__I40E_DOWN, pf->state) || 9103 test_bit(__I40E_CONFIG_BUSY, pf->state)) 9104 return; 9105 9106 /* make sure we don't do these things too often */ 9107 if (time_before(jiffies, (pf->service_timer_previous + 9108 pf->service_timer_period))) 9109 return; 9110 pf->service_timer_previous = jiffies; 9111 9112 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 9113 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 9114 i40e_link_event(pf); 9115 9116 /* Update the stats for active netdevs so the network stack 9117 * can look at updated numbers whenever it cares to 9118 */ 9119 for (i = 0; i < pf->num_alloc_vsi; i++) 9120 if (pf->vsi[i] && pf->vsi[i]->netdev) 9121 i40e_update_stats(pf->vsi[i]); 9122 9123 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 9124 /* Update the stats for the active switching components */ 9125 for (i = 0; i < I40E_MAX_VEB; i++) 9126 if (pf->veb[i]) 9127 i40e_update_veb_stats(pf->veb[i]); 9128 } 9129 9130 i40e_ptp_rx_hang(pf); 9131 i40e_ptp_tx_hang(pf); 9132 } 9133 9134 /** 9135 * i40e_reset_subtask - Set up for resetting the device and driver 9136 * @pf: board private structure 9137 **/ 9138 static void i40e_reset_subtask(struct i40e_pf *pf) 9139 { 9140 u32 reset_flags = 0; 9141 9142 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 9143 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 9144 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 9145 } 9146 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 9147 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 9148 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9149 } 9150 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 9151 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 9152 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 9153 } 9154 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 9155 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 9156 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 9157 } 9158 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 9159 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 9160 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 9161 } 9162 9163 /* If there's a recovery already waiting, it takes 9164 * precedence before starting a new reset sequence. 9165 */ 9166 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 9167 i40e_prep_for_reset(pf, false); 9168 i40e_reset(pf); 9169 i40e_rebuild(pf, false, false); 9170 } 9171 9172 /* If we're already down or resetting, just bail */ 9173 if (reset_flags && 9174 !test_bit(__I40E_DOWN, pf->state) && 9175 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 9176 i40e_do_reset(pf, reset_flags, false); 9177 } 9178 } 9179 9180 /** 9181 * i40e_handle_link_event - Handle link event 9182 * @pf: board private structure 9183 * @e: event info posted on ARQ 9184 **/ 9185 static void i40e_handle_link_event(struct i40e_pf *pf, 9186 struct i40e_arq_event_info *e) 9187 { 9188 struct i40e_aqc_get_link_status *status = 9189 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 9190 9191 /* Do a new status request to re-enable LSE reporting 9192 * and load new status information into the hw struct 9193 * This completely ignores any state information 9194 * in the ARQ event info, instead choosing to always 9195 * issue the AQ update link status command. 9196 */ 9197 i40e_link_event(pf); 9198 9199 /* Check if module meets thermal requirements */ 9200 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 9201 dev_err(&pf->pdev->dev, 9202 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 9203 dev_err(&pf->pdev->dev, 9204 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 9205 } else { 9206 /* check for unqualified module, if link is down, suppress 9207 * the message if link was forced to be down. 9208 */ 9209 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 9210 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 9211 (!(status->link_info & I40E_AQ_LINK_UP)) && 9212 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 9213 dev_err(&pf->pdev->dev, 9214 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 9215 dev_err(&pf->pdev->dev, 9216 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 9217 } 9218 } 9219 } 9220 9221 /** 9222 * i40e_clean_adminq_subtask - Clean the AdminQ rings 9223 * @pf: board private structure 9224 **/ 9225 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 9226 { 9227 struct i40e_arq_event_info event; 9228 struct i40e_hw *hw = &pf->hw; 9229 u16 pending, i = 0; 9230 i40e_status ret; 9231 u16 opcode; 9232 u32 oldval; 9233 u32 val; 9234 9235 /* Do not run clean AQ when PF reset fails */ 9236 if (test_bit(__I40E_RESET_FAILED, pf->state)) 9237 return; 9238 9239 /* check for error indications */ 9240 val = rd32(&pf->hw, pf->hw.aq.arq.len); 9241 oldval = val; 9242 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 9243 if (hw->debug_mask & I40E_DEBUG_AQ) 9244 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 9245 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 9246 } 9247 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 9248 if (hw->debug_mask & I40E_DEBUG_AQ) 9249 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 9250 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 9251 pf->arq_overflows++; 9252 } 9253 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 9254 if (hw->debug_mask & I40E_DEBUG_AQ) 9255 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 9256 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 9257 } 9258 if (oldval != val) 9259 wr32(&pf->hw, pf->hw.aq.arq.len, val); 9260 9261 val = rd32(&pf->hw, pf->hw.aq.asq.len); 9262 oldval = val; 9263 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 9264 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9265 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 9266 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 9267 } 9268 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 9269 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9270 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 9271 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 9272 } 9273 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 9274 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 9275 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 9276 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 9277 } 9278 if (oldval != val) 9279 wr32(&pf->hw, pf->hw.aq.asq.len, val); 9280 9281 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 9282 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 9283 if (!event.msg_buf) 9284 return; 9285 9286 do { 9287 ret = i40e_clean_arq_element(hw, &event, &pending); 9288 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 9289 break; 9290 else if (ret) { 9291 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 9292 break; 9293 } 9294 9295 opcode = le16_to_cpu(event.desc.opcode); 9296 switch (opcode) { 9297 9298 case i40e_aqc_opc_get_link_status: 9299 i40e_handle_link_event(pf, &event); 9300 break; 9301 case i40e_aqc_opc_send_msg_to_pf: 9302 ret = i40e_vc_process_vf_msg(pf, 9303 le16_to_cpu(event.desc.retval), 9304 le32_to_cpu(event.desc.cookie_high), 9305 le32_to_cpu(event.desc.cookie_low), 9306 event.msg_buf, 9307 event.msg_len); 9308 break; 9309 case i40e_aqc_opc_lldp_update_mib: 9310 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 9311 #ifdef CONFIG_I40E_DCB 9312 rtnl_lock(); 9313 ret = i40e_handle_lldp_event(pf, &event); 9314 rtnl_unlock(); 9315 #endif /* CONFIG_I40E_DCB */ 9316 break; 9317 case i40e_aqc_opc_event_lan_overflow: 9318 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 9319 i40e_handle_lan_overflow_event(pf, &event); 9320 break; 9321 case i40e_aqc_opc_send_msg_to_peer: 9322 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 9323 break; 9324 case i40e_aqc_opc_nvm_erase: 9325 case i40e_aqc_opc_nvm_update: 9326 case i40e_aqc_opc_oem_post_update: 9327 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 9328 "ARQ NVM operation 0x%04x completed\n", 9329 opcode); 9330 break; 9331 default: 9332 dev_info(&pf->pdev->dev, 9333 "ARQ: Unknown event 0x%04x ignored\n", 9334 opcode); 9335 break; 9336 } 9337 } while (i++ < pf->adminq_work_limit); 9338 9339 if (i < pf->adminq_work_limit) 9340 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 9341 9342 /* re-enable Admin queue interrupt cause */ 9343 val = rd32(hw, I40E_PFINT_ICR0_ENA); 9344 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 9345 wr32(hw, I40E_PFINT_ICR0_ENA, val); 9346 i40e_flush(hw); 9347 9348 kfree(event.msg_buf); 9349 } 9350 9351 /** 9352 * i40e_verify_eeprom - make sure eeprom is good to use 9353 * @pf: board private structure 9354 **/ 9355 static void i40e_verify_eeprom(struct i40e_pf *pf) 9356 { 9357 int err; 9358 9359 err = i40e_diag_eeprom_test(&pf->hw); 9360 if (err) { 9361 /* retry in case of garbage read */ 9362 err = i40e_diag_eeprom_test(&pf->hw); 9363 if (err) { 9364 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 9365 err); 9366 set_bit(__I40E_BAD_EEPROM, pf->state); 9367 } 9368 } 9369 9370 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 9371 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 9372 clear_bit(__I40E_BAD_EEPROM, pf->state); 9373 } 9374 } 9375 9376 /** 9377 * i40e_enable_pf_switch_lb 9378 * @pf: pointer to the PF structure 9379 * 9380 * enable switch loop back or die - no point in a return value 9381 **/ 9382 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 9383 { 9384 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9385 struct i40e_vsi_context ctxt; 9386 int ret; 9387 9388 ctxt.seid = pf->main_vsi_seid; 9389 ctxt.pf_num = pf->hw.pf_id; 9390 ctxt.vf_num = 0; 9391 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9392 if (ret) { 9393 dev_info(&pf->pdev->dev, 9394 "couldn't get PF vsi config, err %s aq_err %s\n", 9395 i40e_stat_str(&pf->hw, ret), 9396 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9397 return; 9398 } 9399 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9400 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9401 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9402 9403 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 9404 if (ret) { 9405 dev_info(&pf->pdev->dev, 9406 "update vsi switch failed, err %s aq_err %s\n", 9407 i40e_stat_str(&pf->hw, ret), 9408 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9409 } 9410 } 9411 9412 /** 9413 * i40e_disable_pf_switch_lb 9414 * @pf: pointer to the PF structure 9415 * 9416 * disable switch loop back or die - no point in a return value 9417 **/ 9418 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 9419 { 9420 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9421 struct i40e_vsi_context ctxt; 9422 int ret; 9423 9424 ctxt.seid = pf->main_vsi_seid; 9425 ctxt.pf_num = pf->hw.pf_id; 9426 ctxt.vf_num = 0; 9427 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9428 if (ret) { 9429 dev_info(&pf->pdev->dev, 9430 "couldn't get PF vsi config, err %s aq_err %s\n", 9431 i40e_stat_str(&pf->hw, ret), 9432 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9433 return; 9434 } 9435 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9436 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9437 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9438 9439 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 9440 if (ret) { 9441 dev_info(&pf->pdev->dev, 9442 "update vsi switch failed, err %s aq_err %s\n", 9443 i40e_stat_str(&pf->hw, ret), 9444 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9445 } 9446 } 9447 9448 /** 9449 * i40e_config_bridge_mode - Configure the HW bridge mode 9450 * @veb: pointer to the bridge instance 9451 * 9452 * Configure the loop back mode for the LAN VSI that is downlink to the 9453 * specified HW bridge instance. It is expected this function is called 9454 * when a new HW bridge is instantiated. 9455 **/ 9456 static void i40e_config_bridge_mode(struct i40e_veb *veb) 9457 { 9458 struct i40e_pf *pf = veb->pf; 9459 9460 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 9461 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 9462 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 9463 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 9464 i40e_disable_pf_switch_lb(pf); 9465 else 9466 i40e_enable_pf_switch_lb(pf); 9467 } 9468 9469 /** 9470 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 9471 * @veb: pointer to the VEB instance 9472 * 9473 * This is a recursive function that first builds the attached VSIs then 9474 * recurses in to build the next layer of VEB. We track the connections 9475 * through our own index numbers because the seid's from the HW could 9476 * change across the reset. 9477 **/ 9478 static int i40e_reconstitute_veb(struct i40e_veb *veb) 9479 { 9480 struct i40e_vsi *ctl_vsi = NULL; 9481 struct i40e_pf *pf = veb->pf; 9482 int v, veb_idx; 9483 int ret; 9484 9485 /* build VSI that owns this VEB, temporarily attached to base VEB */ 9486 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 9487 if (pf->vsi[v] && 9488 pf->vsi[v]->veb_idx == veb->idx && 9489 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 9490 ctl_vsi = pf->vsi[v]; 9491 break; 9492 } 9493 } 9494 if (!ctl_vsi) { 9495 dev_info(&pf->pdev->dev, 9496 "missing owner VSI for veb_idx %d\n", veb->idx); 9497 ret = -ENOENT; 9498 goto end_reconstitute; 9499 } 9500 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 9501 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 9502 ret = i40e_add_vsi(ctl_vsi); 9503 if (ret) { 9504 dev_info(&pf->pdev->dev, 9505 "rebuild of veb_idx %d owner VSI failed: %d\n", 9506 veb->idx, ret); 9507 goto end_reconstitute; 9508 } 9509 i40e_vsi_reset_stats(ctl_vsi); 9510 9511 /* create the VEB in the switch and move the VSI onto the VEB */ 9512 ret = i40e_add_veb(veb, ctl_vsi); 9513 if (ret) 9514 goto end_reconstitute; 9515 9516 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 9517 veb->bridge_mode = BRIDGE_MODE_VEB; 9518 else 9519 veb->bridge_mode = BRIDGE_MODE_VEPA; 9520 i40e_config_bridge_mode(veb); 9521 9522 /* create the remaining VSIs attached to this VEB */ 9523 for (v = 0; v < pf->num_alloc_vsi; v++) { 9524 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 9525 continue; 9526 9527 if (pf->vsi[v]->veb_idx == veb->idx) { 9528 struct i40e_vsi *vsi = pf->vsi[v]; 9529 9530 vsi->uplink_seid = veb->seid; 9531 ret = i40e_add_vsi(vsi); 9532 if (ret) { 9533 dev_info(&pf->pdev->dev, 9534 "rebuild of vsi_idx %d failed: %d\n", 9535 v, ret); 9536 goto end_reconstitute; 9537 } 9538 i40e_vsi_reset_stats(vsi); 9539 } 9540 } 9541 9542 /* create any VEBs attached to this VEB - RECURSION */ 9543 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 9544 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 9545 pf->veb[veb_idx]->uplink_seid = veb->seid; 9546 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 9547 if (ret) 9548 break; 9549 } 9550 } 9551 9552 end_reconstitute: 9553 return ret; 9554 } 9555 9556 /** 9557 * i40e_get_capabilities - get info about the HW 9558 * @pf: the PF struct 9559 **/ 9560 static int i40e_get_capabilities(struct i40e_pf *pf, 9561 enum i40e_admin_queue_opc list_type) 9562 { 9563 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 9564 u16 data_size; 9565 int buf_len; 9566 int err; 9567 9568 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 9569 do { 9570 cap_buf = kzalloc(buf_len, GFP_KERNEL); 9571 if (!cap_buf) 9572 return -ENOMEM; 9573 9574 /* this loads the data into the hw struct for us */ 9575 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 9576 &data_size, list_type, 9577 NULL); 9578 /* data loaded, buffer no longer needed */ 9579 kfree(cap_buf); 9580 9581 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 9582 /* retry with a larger buffer */ 9583 buf_len = data_size; 9584 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 9585 dev_info(&pf->pdev->dev, 9586 "capability discovery failed, err %s aq_err %s\n", 9587 i40e_stat_str(&pf->hw, err), 9588 i40e_aq_str(&pf->hw, 9589 pf->hw.aq.asq_last_status)); 9590 return -ENODEV; 9591 } 9592 } while (err); 9593 9594 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 9595 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9596 dev_info(&pf->pdev->dev, 9597 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 9598 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 9599 pf->hw.func_caps.num_msix_vectors, 9600 pf->hw.func_caps.num_msix_vectors_vf, 9601 pf->hw.func_caps.fd_filters_guaranteed, 9602 pf->hw.func_caps.fd_filters_best_effort, 9603 pf->hw.func_caps.num_tx_qp, 9604 pf->hw.func_caps.num_vsis); 9605 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 9606 dev_info(&pf->pdev->dev, 9607 "switch_mode=0x%04x, function_valid=0x%08x\n", 9608 pf->hw.dev_caps.switch_mode, 9609 pf->hw.dev_caps.valid_functions); 9610 dev_info(&pf->pdev->dev, 9611 "SR-IOV=%d, num_vfs for all function=%u\n", 9612 pf->hw.dev_caps.sr_iov_1_1, 9613 pf->hw.dev_caps.num_vfs); 9614 dev_info(&pf->pdev->dev, 9615 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 9616 pf->hw.dev_caps.num_vsis, 9617 pf->hw.dev_caps.num_rx_qp, 9618 pf->hw.dev_caps.num_tx_qp); 9619 } 9620 } 9621 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9622 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 9623 + pf->hw.func_caps.num_vfs) 9624 if (pf->hw.revision_id == 0 && 9625 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 9626 dev_info(&pf->pdev->dev, 9627 "got num_vsis %d, setting num_vsis to %d\n", 9628 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 9629 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 9630 } 9631 } 9632 return 0; 9633 } 9634 9635 static int i40e_vsi_clear(struct i40e_vsi *vsi); 9636 9637 /** 9638 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 9639 * @pf: board private structure 9640 **/ 9641 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 9642 { 9643 struct i40e_vsi *vsi; 9644 9645 /* quick workaround for an NVM issue that leaves a critical register 9646 * uninitialized 9647 */ 9648 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 9649 static const u32 hkey[] = { 9650 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 9651 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 9652 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 9653 0x95b3a76d}; 9654 int i; 9655 9656 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 9657 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 9658 } 9659 9660 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 9661 return; 9662 9663 /* find existing VSI and see if it needs configuring */ 9664 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9665 9666 /* create a new VSI if none exists */ 9667 if (!vsi) { 9668 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 9669 pf->vsi[pf->lan_vsi]->seid, 0); 9670 if (!vsi) { 9671 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 9672 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 9673 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 9674 return; 9675 } 9676 } 9677 9678 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 9679 } 9680 9681 /** 9682 * i40e_fdir_teardown - release the Flow Director resources 9683 * @pf: board private structure 9684 **/ 9685 static void i40e_fdir_teardown(struct i40e_pf *pf) 9686 { 9687 struct i40e_vsi *vsi; 9688 9689 i40e_fdir_filter_exit(pf); 9690 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9691 if (vsi) 9692 i40e_vsi_release(vsi); 9693 } 9694 9695 /** 9696 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 9697 * @vsi: PF main vsi 9698 * @seid: seid of main or channel VSIs 9699 * 9700 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 9701 * existed before reset 9702 **/ 9703 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 9704 { 9705 struct i40e_cloud_filter *cfilter; 9706 struct i40e_pf *pf = vsi->back; 9707 struct hlist_node *node; 9708 i40e_status ret; 9709 9710 /* Add cloud filters back if they exist */ 9711 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 9712 cloud_node) { 9713 if (cfilter->seid != seid) 9714 continue; 9715 9716 if (cfilter->dst_port) 9717 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 9718 true); 9719 else 9720 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 9721 9722 if (ret) { 9723 dev_dbg(&pf->pdev->dev, 9724 "Failed to rebuild cloud filter, err %s aq_err %s\n", 9725 i40e_stat_str(&pf->hw, ret), 9726 i40e_aq_str(&pf->hw, 9727 pf->hw.aq.asq_last_status)); 9728 return ret; 9729 } 9730 } 9731 return 0; 9732 } 9733 9734 /** 9735 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 9736 * @vsi: PF main vsi 9737 * 9738 * Rebuilds channel VSIs if they existed before reset 9739 **/ 9740 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 9741 { 9742 struct i40e_channel *ch, *ch_tmp; 9743 i40e_status ret; 9744 9745 if (list_empty(&vsi->ch_list)) 9746 return 0; 9747 9748 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 9749 if (!ch->initialized) 9750 break; 9751 /* Proceed with creation of channel (VMDq2) VSI */ 9752 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 9753 if (ret) { 9754 dev_info(&vsi->back->pdev->dev, 9755 "failed to rebuild channels using uplink_seid %u\n", 9756 vsi->uplink_seid); 9757 return ret; 9758 } 9759 /* Reconfigure TX queues using QTX_CTL register */ 9760 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 9761 if (ret) { 9762 dev_info(&vsi->back->pdev->dev, 9763 "failed to configure TX rings for channel %u\n", 9764 ch->seid); 9765 return ret; 9766 } 9767 /* update 'next_base_queue' */ 9768 vsi->next_base_queue = vsi->next_base_queue + 9769 ch->num_queue_pairs; 9770 if (ch->max_tx_rate) { 9771 u64 credits = ch->max_tx_rate; 9772 9773 if (i40e_set_bw_limit(vsi, ch->seid, 9774 ch->max_tx_rate)) 9775 return -EINVAL; 9776 9777 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9778 dev_dbg(&vsi->back->pdev->dev, 9779 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9780 ch->max_tx_rate, 9781 credits, 9782 ch->seid); 9783 } 9784 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 9785 if (ret) { 9786 dev_dbg(&vsi->back->pdev->dev, 9787 "Failed to rebuild cloud filters for channel VSI %u\n", 9788 ch->seid); 9789 return ret; 9790 } 9791 } 9792 return 0; 9793 } 9794 9795 /** 9796 * i40e_prep_for_reset - prep for the core to reset 9797 * @pf: board private structure 9798 * @lock_acquired: indicates whether or not the lock has been acquired 9799 * before this function was called. 9800 * 9801 * Close up the VFs and other things in prep for PF Reset. 9802 **/ 9803 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 9804 { 9805 struct i40e_hw *hw = &pf->hw; 9806 i40e_status ret = 0; 9807 u32 v; 9808 9809 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 9810 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9811 return; 9812 if (i40e_check_asq_alive(&pf->hw)) 9813 i40e_vc_notify_reset(pf); 9814 9815 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 9816 9817 /* quiesce the VSIs and their queues that are not already DOWN */ 9818 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 9819 if (!lock_acquired) 9820 rtnl_lock(); 9821 i40e_pf_quiesce_all_vsi(pf); 9822 if (!lock_acquired) 9823 rtnl_unlock(); 9824 9825 for (v = 0; v < pf->num_alloc_vsi; v++) { 9826 if (pf->vsi[v]) 9827 pf->vsi[v]->seid = 0; 9828 } 9829 9830 i40e_shutdown_adminq(&pf->hw); 9831 9832 /* call shutdown HMC */ 9833 if (hw->hmc.hmc_obj) { 9834 ret = i40e_shutdown_lan_hmc(hw); 9835 if (ret) 9836 dev_warn(&pf->pdev->dev, 9837 "shutdown_lan_hmc failed: %d\n", ret); 9838 } 9839 9840 /* Save the current PTP time so that we can restore the time after the 9841 * reset completes. 9842 */ 9843 i40e_ptp_save_hw_time(pf); 9844 } 9845 9846 /** 9847 * i40e_send_version - update firmware with driver version 9848 * @pf: PF struct 9849 */ 9850 static void i40e_send_version(struct i40e_pf *pf) 9851 { 9852 struct i40e_driver_version dv; 9853 9854 dv.major_version = DRV_VERSION_MAJOR; 9855 dv.minor_version = DRV_VERSION_MINOR; 9856 dv.build_version = DRV_VERSION_BUILD; 9857 dv.subbuild_version = 0; 9858 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 9859 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 9860 } 9861 9862 /** 9863 * i40e_get_oem_version - get OEM specific version information 9864 * @hw: pointer to the hardware structure 9865 **/ 9866 static void i40e_get_oem_version(struct i40e_hw *hw) 9867 { 9868 u16 block_offset = 0xffff; 9869 u16 block_length = 0; 9870 u16 capabilities = 0; 9871 u16 gen_snap = 0; 9872 u16 release = 0; 9873 9874 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 9875 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 9876 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 9877 #define I40E_NVM_OEM_GEN_OFFSET 0x02 9878 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 9879 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 9880 #define I40E_NVM_OEM_LENGTH 3 9881 9882 /* Check if pointer to OEM version block is valid. */ 9883 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 9884 if (block_offset == 0xffff) 9885 return; 9886 9887 /* Check if OEM version block has correct length. */ 9888 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 9889 &block_length); 9890 if (block_length < I40E_NVM_OEM_LENGTH) 9891 return; 9892 9893 /* Check if OEM version format is as expected. */ 9894 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 9895 &capabilities); 9896 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 9897 return; 9898 9899 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 9900 &gen_snap); 9901 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 9902 &release); 9903 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 9904 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 9905 } 9906 9907 /** 9908 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 9909 * @pf: board private structure 9910 **/ 9911 static int i40e_reset(struct i40e_pf *pf) 9912 { 9913 struct i40e_hw *hw = &pf->hw; 9914 i40e_status ret; 9915 9916 ret = i40e_pf_reset(hw); 9917 if (ret) { 9918 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 9919 set_bit(__I40E_RESET_FAILED, pf->state); 9920 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9921 } else { 9922 pf->pfr_count++; 9923 } 9924 return ret; 9925 } 9926 9927 /** 9928 * i40e_rebuild - rebuild using a saved config 9929 * @pf: board private structure 9930 * @reinit: if the Main VSI needs to re-initialized. 9931 * @lock_acquired: indicates whether or not the lock has been acquired 9932 * before this function was called. 9933 **/ 9934 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 9935 { 9936 int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state); 9937 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9938 struct i40e_hw *hw = &pf->hw; 9939 u8 set_fc_aq_fail = 0; 9940 i40e_status ret; 9941 u32 val; 9942 int v; 9943 9944 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 9945 i40e_check_recovery_mode(pf)) { 9946 i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev); 9947 } 9948 9949 if (test_bit(__I40E_DOWN, pf->state) && 9950 !test_bit(__I40E_RECOVERY_MODE, pf->state) && 9951 !old_recovery_mode_bit) 9952 goto clear_recovery; 9953 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 9954 9955 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 9956 ret = i40e_init_adminq(&pf->hw); 9957 if (ret) { 9958 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 9959 i40e_stat_str(&pf->hw, ret), 9960 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9961 goto clear_recovery; 9962 } 9963 i40e_get_oem_version(&pf->hw); 9964 9965 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 9966 ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) || 9967 hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) { 9968 /* The following delay is necessary for 4.33 firmware and older 9969 * to recover after EMP reset. 200 ms should suffice but we 9970 * put here 300 ms to be sure that FW is ready to operate 9971 * after reset. 9972 */ 9973 mdelay(300); 9974 } 9975 9976 /* re-verify the eeprom if we just had an EMP reset */ 9977 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 9978 i40e_verify_eeprom(pf); 9979 9980 /* if we are going out of or into recovery mode we have to act 9981 * accordingly with regard to resources initialization 9982 * and deinitialization 9983 */ 9984 if (test_bit(__I40E_RECOVERY_MODE, pf->state) || 9985 old_recovery_mode_bit) { 9986 if (i40e_get_capabilities(pf, 9987 i40e_aqc_opc_list_func_capabilities)) 9988 goto end_unlock; 9989 9990 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 9991 /* we're staying in recovery mode so we'll reinitialize 9992 * misc vector here 9993 */ 9994 if (i40e_setup_misc_vector_for_recovery_mode(pf)) 9995 goto end_unlock; 9996 } else { 9997 if (!lock_acquired) 9998 rtnl_lock(); 9999 /* we're going out of recovery mode so we'll free 10000 * the IRQ allocated specifically for recovery mode 10001 * and restore the interrupt scheme 10002 */ 10003 free_irq(pf->pdev->irq, pf); 10004 i40e_clear_interrupt_scheme(pf); 10005 if (i40e_restore_interrupt_scheme(pf)) 10006 goto end_unlock; 10007 } 10008 10009 /* tell the firmware that we're starting */ 10010 i40e_send_version(pf); 10011 10012 /* bail out in case recovery mode was detected, as there is 10013 * no need for further configuration. 10014 */ 10015 goto end_unlock; 10016 } 10017 10018 i40e_clear_pxe_mode(hw); 10019 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 10020 if (ret) 10021 goto end_core_reset; 10022 10023 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 10024 hw->func_caps.num_rx_qp, 0, 0); 10025 if (ret) { 10026 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 10027 goto end_core_reset; 10028 } 10029 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 10030 if (ret) { 10031 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 10032 goto end_core_reset; 10033 } 10034 10035 /* Enable FW to write a default DCB config on link-up */ 10036 i40e_aq_set_dcb_parameters(hw, true, NULL); 10037 10038 #ifdef CONFIG_I40E_DCB 10039 ret = i40e_init_pf_dcb(pf); 10040 if (ret) { 10041 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 10042 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 10043 /* Continue without DCB enabled */ 10044 } 10045 #endif /* CONFIG_I40E_DCB */ 10046 /* do basic switch setup */ 10047 if (!lock_acquired) 10048 rtnl_lock(); 10049 ret = i40e_setup_pf_switch(pf, reinit); 10050 if (ret) 10051 goto end_unlock; 10052 10053 /* The driver only wants link up/down and module qualification 10054 * reports from firmware. Note the negative logic. 10055 */ 10056 ret = i40e_aq_set_phy_int_mask(&pf->hw, 10057 ~(I40E_AQ_EVENT_LINK_UPDOWN | 10058 I40E_AQ_EVENT_MEDIA_NA | 10059 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 10060 if (ret) 10061 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 10062 i40e_stat_str(&pf->hw, ret), 10063 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10064 10065 /* make sure our flow control settings are restored */ 10066 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 10067 if (ret) 10068 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 10069 i40e_stat_str(&pf->hw, ret), 10070 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10071 10072 /* Rebuild the VSIs and VEBs that existed before reset. 10073 * They are still in our local switch element arrays, so only 10074 * need to rebuild the switch model in the HW. 10075 * 10076 * If there were VEBs but the reconstitution failed, we'll try 10077 * try to recover minimal use by getting the basic PF VSI working. 10078 */ 10079 if (vsi->uplink_seid != pf->mac_seid) { 10080 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 10081 /* find the one VEB connected to the MAC, and find orphans */ 10082 for (v = 0; v < I40E_MAX_VEB; v++) { 10083 if (!pf->veb[v]) 10084 continue; 10085 10086 if (pf->veb[v]->uplink_seid == pf->mac_seid || 10087 pf->veb[v]->uplink_seid == 0) { 10088 ret = i40e_reconstitute_veb(pf->veb[v]); 10089 10090 if (!ret) 10091 continue; 10092 10093 /* If Main VEB failed, we're in deep doodoo, 10094 * so give up rebuilding the switch and set up 10095 * for minimal rebuild of PF VSI. 10096 * If orphan failed, we'll report the error 10097 * but try to keep going. 10098 */ 10099 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 10100 dev_info(&pf->pdev->dev, 10101 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 10102 ret); 10103 vsi->uplink_seid = pf->mac_seid; 10104 break; 10105 } else if (pf->veb[v]->uplink_seid == 0) { 10106 dev_info(&pf->pdev->dev, 10107 "rebuild of orphan VEB failed: %d\n", 10108 ret); 10109 } 10110 } 10111 } 10112 } 10113 10114 if (vsi->uplink_seid == pf->mac_seid) { 10115 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 10116 /* no VEB, so rebuild only the Main VSI */ 10117 ret = i40e_add_vsi(vsi); 10118 if (ret) { 10119 dev_info(&pf->pdev->dev, 10120 "rebuild of Main VSI failed: %d\n", ret); 10121 goto end_unlock; 10122 } 10123 } 10124 10125 if (vsi->mqprio_qopt.max_rate[0]) { 10126 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 10127 u64 credits = 0; 10128 10129 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 10130 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 10131 if (ret) 10132 goto end_unlock; 10133 10134 credits = max_tx_rate; 10135 do_div(credits, I40E_BW_CREDIT_DIVISOR); 10136 dev_dbg(&vsi->back->pdev->dev, 10137 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 10138 max_tx_rate, 10139 credits, 10140 vsi->seid); 10141 } 10142 10143 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 10144 if (ret) 10145 goto end_unlock; 10146 10147 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 10148 * for this main VSI if they exist 10149 */ 10150 ret = i40e_rebuild_channels(vsi); 10151 if (ret) 10152 goto end_unlock; 10153 10154 /* Reconfigure hardware for allowing smaller MSS in the case 10155 * of TSO, so that we avoid the MDD being fired and causing 10156 * a reset in the case of small MSS+TSO. 10157 */ 10158 #define I40E_REG_MSS 0x000E64DC 10159 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 10160 #define I40E_64BYTE_MSS 0x400000 10161 val = rd32(hw, I40E_REG_MSS); 10162 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 10163 val &= ~I40E_REG_MSS_MIN_MASK; 10164 val |= I40E_64BYTE_MSS; 10165 wr32(hw, I40E_REG_MSS, val); 10166 } 10167 10168 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 10169 msleep(75); 10170 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 10171 if (ret) 10172 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 10173 i40e_stat_str(&pf->hw, ret), 10174 i40e_aq_str(&pf->hw, 10175 pf->hw.aq.asq_last_status)); 10176 } 10177 /* reinit the misc interrupt */ 10178 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10179 ret = i40e_setup_misc_vector(pf); 10180 10181 /* Add a filter to drop all Flow control frames from any VSI from being 10182 * transmitted. By doing so we stop a malicious VF from sending out 10183 * PAUSE or PFC frames and potentially controlling traffic for other 10184 * PF/VF VSIs. 10185 * The FW can still send Flow control frames if enabled. 10186 */ 10187 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 10188 pf->main_vsi_seid); 10189 10190 /* restart the VSIs that were rebuilt and running before the reset */ 10191 i40e_pf_unquiesce_all_vsi(pf); 10192 10193 /* Release the RTNL lock before we start resetting VFs */ 10194 if (!lock_acquired) 10195 rtnl_unlock(); 10196 10197 /* Restore promiscuous settings */ 10198 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 10199 if (ret) 10200 dev_warn(&pf->pdev->dev, 10201 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n", 10202 pf->cur_promisc ? "on" : "off", 10203 i40e_stat_str(&pf->hw, ret), 10204 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10205 10206 i40e_reset_all_vfs(pf, true); 10207 10208 /* tell the firmware that we're starting */ 10209 i40e_send_version(pf); 10210 10211 /* We've already released the lock, so don't do it again */ 10212 goto end_core_reset; 10213 10214 end_unlock: 10215 if (!lock_acquired) 10216 rtnl_unlock(); 10217 end_core_reset: 10218 clear_bit(__I40E_RESET_FAILED, pf->state); 10219 clear_recovery: 10220 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 10221 clear_bit(__I40E_TIMEOUT_RECOVERY_PENDING, pf->state); 10222 } 10223 10224 /** 10225 * i40e_reset_and_rebuild - reset and rebuild using a saved config 10226 * @pf: board private structure 10227 * @reinit: if the Main VSI needs to re-initialized. 10228 * @lock_acquired: indicates whether or not the lock has been acquired 10229 * before this function was called. 10230 **/ 10231 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 10232 bool lock_acquired) 10233 { 10234 int ret; 10235 /* Now we wait for GRST to settle out. 10236 * We don't have to delete the VEBs or VSIs from the hw switch 10237 * because the reset will make them disappear. 10238 */ 10239 ret = i40e_reset(pf); 10240 if (!ret) 10241 i40e_rebuild(pf, reinit, lock_acquired); 10242 } 10243 10244 /** 10245 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 10246 * @pf: board private structure 10247 * 10248 * Close up the VFs and other things in prep for a Core Reset, 10249 * then get ready to rebuild the world. 10250 * @lock_acquired: indicates whether or not the lock has been acquired 10251 * before this function was called. 10252 **/ 10253 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 10254 { 10255 i40e_prep_for_reset(pf, lock_acquired); 10256 i40e_reset_and_rebuild(pf, false, lock_acquired); 10257 } 10258 10259 /** 10260 * i40e_handle_mdd_event 10261 * @pf: pointer to the PF structure 10262 * 10263 * Called from the MDD irq handler to identify possibly malicious vfs 10264 **/ 10265 static void i40e_handle_mdd_event(struct i40e_pf *pf) 10266 { 10267 struct i40e_hw *hw = &pf->hw; 10268 bool mdd_detected = false; 10269 struct i40e_vf *vf; 10270 u32 reg; 10271 int i; 10272 10273 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 10274 return; 10275 10276 /* find what triggered the MDD event */ 10277 reg = rd32(hw, I40E_GL_MDET_TX); 10278 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 10279 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 10280 I40E_GL_MDET_TX_PF_NUM_SHIFT; 10281 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 10282 I40E_GL_MDET_TX_VF_NUM_SHIFT; 10283 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 10284 I40E_GL_MDET_TX_EVENT_SHIFT; 10285 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 10286 I40E_GL_MDET_TX_QUEUE_SHIFT) - 10287 pf->hw.func_caps.base_queue; 10288 if (netif_msg_tx_err(pf)) 10289 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 10290 event, queue, pf_num, vf_num); 10291 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 10292 mdd_detected = true; 10293 } 10294 reg = rd32(hw, I40E_GL_MDET_RX); 10295 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 10296 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 10297 I40E_GL_MDET_RX_FUNCTION_SHIFT; 10298 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 10299 I40E_GL_MDET_RX_EVENT_SHIFT; 10300 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 10301 I40E_GL_MDET_RX_QUEUE_SHIFT) - 10302 pf->hw.func_caps.base_queue; 10303 if (netif_msg_rx_err(pf)) 10304 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 10305 event, queue, func); 10306 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 10307 mdd_detected = true; 10308 } 10309 10310 if (mdd_detected) { 10311 reg = rd32(hw, I40E_PF_MDET_TX); 10312 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 10313 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 10314 dev_dbg(&pf->pdev->dev, "TX driver issue detected on PF\n"); 10315 } 10316 reg = rd32(hw, I40E_PF_MDET_RX); 10317 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 10318 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 10319 dev_dbg(&pf->pdev->dev, "RX driver issue detected on PF\n"); 10320 } 10321 } 10322 10323 /* see if one of the VFs needs its hand slapped */ 10324 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 10325 vf = &(pf->vf[i]); 10326 reg = rd32(hw, I40E_VP_MDET_TX(i)); 10327 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 10328 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 10329 vf->num_mdd_events++; 10330 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 10331 i); 10332 dev_info(&pf->pdev->dev, 10333 "Use PF Control I/F to re-enable the VF\n"); 10334 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 10335 } 10336 10337 reg = rd32(hw, I40E_VP_MDET_RX(i)); 10338 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 10339 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 10340 vf->num_mdd_events++; 10341 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 10342 i); 10343 dev_info(&pf->pdev->dev, 10344 "Use PF Control I/F to re-enable the VF\n"); 10345 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 10346 } 10347 } 10348 10349 /* re-enable mdd interrupt cause */ 10350 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 10351 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 10352 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 10353 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 10354 i40e_flush(hw); 10355 } 10356 10357 static const char *i40e_tunnel_name(u8 type) 10358 { 10359 switch (type) { 10360 case UDP_TUNNEL_TYPE_VXLAN: 10361 return "vxlan"; 10362 case UDP_TUNNEL_TYPE_GENEVE: 10363 return "geneve"; 10364 default: 10365 return "unknown"; 10366 } 10367 } 10368 10369 /** 10370 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 10371 * @pf: board private structure 10372 **/ 10373 static void i40e_sync_udp_filters(struct i40e_pf *pf) 10374 { 10375 int i; 10376 10377 /* loop through and set pending bit for all active UDP filters */ 10378 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 10379 if (pf->udp_ports[i].port) 10380 pf->pending_udp_bitmap |= BIT_ULL(i); 10381 } 10382 10383 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 10384 } 10385 10386 /** 10387 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 10388 * @pf: board private structure 10389 **/ 10390 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 10391 { 10392 struct i40e_hw *hw = &pf->hw; 10393 u8 filter_index, type; 10394 u16 port; 10395 int i; 10396 10397 if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state)) 10398 return; 10399 10400 /* acquire RTNL to maintain state of flags and port requests */ 10401 rtnl_lock(); 10402 10403 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 10404 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 10405 struct i40e_udp_port_config *udp_port; 10406 i40e_status ret = 0; 10407 10408 udp_port = &pf->udp_ports[i]; 10409 pf->pending_udp_bitmap &= ~BIT_ULL(i); 10410 10411 port = READ_ONCE(udp_port->port); 10412 type = READ_ONCE(udp_port->type); 10413 filter_index = READ_ONCE(udp_port->filter_index); 10414 10415 /* release RTNL while we wait on AQ command */ 10416 rtnl_unlock(); 10417 10418 if (port) 10419 ret = i40e_aq_add_udp_tunnel(hw, port, 10420 type, 10421 &filter_index, 10422 NULL); 10423 else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED) 10424 ret = i40e_aq_del_udp_tunnel(hw, filter_index, 10425 NULL); 10426 10427 /* reacquire RTNL so we can update filter_index */ 10428 rtnl_lock(); 10429 10430 if (ret) { 10431 dev_info(&pf->pdev->dev, 10432 "%s %s port %d, index %d failed, err %s aq_err %s\n", 10433 i40e_tunnel_name(type), 10434 port ? "add" : "delete", 10435 port, 10436 filter_index, 10437 i40e_stat_str(&pf->hw, ret), 10438 i40e_aq_str(&pf->hw, 10439 pf->hw.aq.asq_last_status)); 10440 if (port) { 10441 /* failed to add, just reset port, 10442 * drop pending bit for any deletion 10443 */ 10444 udp_port->port = 0; 10445 pf->pending_udp_bitmap &= ~BIT_ULL(i); 10446 } 10447 } else if (port) { 10448 /* record filter index on success */ 10449 udp_port->filter_index = filter_index; 10450 } 10451 } 10452 } 10453 10454 rtnl_unlock(); 10455 } 10456 10457 /** 10458 * i40e_service_task - Run the driver's async subtasks 10459 * @work: pointer to work_struct containing our data 10460 **/ 10461 static void i40e_service_task(struct work_struct *work) 10462 { 10463 struct i40e_pf *pf = container_of(work, 10464 struct i40e_pf, 10465 service_task); 10466 unsigned long start_time = jiffies; 10467 10468 /* don't bother with service tasks if a reset is in progress */ 10469 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 10470 test_bit(__I40E_SUSPENDED, pf->state)) 10471 return; 10472 10473 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 10474 return; 10475 10476 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) { 10477 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]); 10478 i40e_sync_filters_subtask(pf); 10479 i40e_reset_subtask(pf); 10480 i40e_handle_mdd_event(pf); 10481 i40e_vc_process_vflr_event(pf); 10482 i40e_watchdog_subtask(pf); 10483 i40e_fdir_reinit_subtask(pf); 10484 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 10485 /* Client subtask will reopen next time through. */ 10486 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], 10487 true); 10488 } else { 10489 i40e_client_subtask(pf); 10490 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 10491 pf->state)) 10492 i40e_notify_client_of_l2_param_changes( 10493 pf->vsi[pf->lan_vsi]); 10494 } 10495 i40e_sync_filters_subtask(pf); 10496 i40e_sync_udp_filters_subtask(pf); 10497 } else { 10498 i40e_reset_subtask(pf); 10499 } 10500 10501 i40e_clean_adminq_subtask(pf); 10502 10503 /* flush memory to make sure state is correct before next watchdog */ 10504 smp_mb__before_atomic(); 10505 clear_bit(__I40E_SERVICE_SCHED, pf->state); 10506 10507 /* If the tasks have taken longer than one timer cycle or there 10508 * is more work to be done, reschedule the service task now 10509 * rather than wait for the timer to tick again. 10510 */ 10511 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 10512 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 10513 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 10514 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 10515 i40e_service_event_schedule(pf); 10516 } 10517 10518 /** 10519 * i40e_service_timer - timer callback 10520 * @data: pointer to PF struct 10521 **/ 10522 static void i40e_service_timer(struct timer_list *t) 10523 { 10524 struct i40e_pf *pf = from_timer(pf, t, service_timer); 10525 10526 mod_timer(&pf->service_timer, 10527 round_jiffies(jiffies + pf->service_timer_period)); 10528 i40e_service_event_schedule(pf); 10529 } 10530 10531 /** 10532 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 10533 * @vsi: the VSI being configured 10534 **/ 10535 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 10536 { 10537 struct i40e_pf *pf = vsi->back; 10538 10539 switch (vsi->type) { 10540 case I40E_VSI_MAIN: 10541 vsi->alloc_queue_pairs = pf->num_lan_qps; 10542 if (!vsi->num_tx_desc) 10543 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10544 I40E_REQ_DESCRIPTOR_MULTIPLE); 10545 if (!vsi->num_rx_desc) 10546 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10547 I40E_REQ_DESCRIPTOR_MULTIPLE); 10548 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10549 vsi->num_q_vectors = pf->num_lan_msix; 10550 else 10551 vsi->num_q_vectors = 1; 10552 10553 break; 10554 10555 case I40E_VSI_FDIR: 10556 vsi->alloc_queue_pairs = 1; 10557 vsi->num_tx_desc = ALIGN(I40E_FDIR_RING_COUNT, 10558 I40E_REQ_DESCRIPTOR_MULTIPLE); 10559 vsi->num_rx_desc = ALIGN(I40E_FDIR_RING_COUNT, 10560 I40E_REQ_DESCRIPTOR_MULTIPLE); 10561 vsi->num_q_vectors = pf->num_fdsb_msix; 10562 break; 10563 10564 case I40E_VSI_VMDQ2: 10565 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 10566 if (!vsi->num_tx_desc) 10567 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10568 I40E_REQ_DESCRIPTOR_MULTIPLE); 10569 if (!vsi->num_rx_desc) 10570 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10571 I40E_REQ_DESCRIPTOR_MULTIPLE); 10572 vsi->num_q_vectors = pf->num_vmdq_msix; 10573 break; 10574 10575 case I40E_VSI_SRIOV: 10576 vsi->alloc_queue_pairs = pf->num_vf_qps; 10577 if (!vsi->num_tx_desc) 10578 vsi->num_tx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10579 I40E_REQ_DESCRIPTOR_MULTIPLE); 10580 if (!vsi->num_rx_desc) 10581 vsi->num_rx_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 10582 I40E_REQ_DESCRIPTOR_MULTIPLE); 10583 break; 10584 10585 default: 10586 WARN_ON(1); 10587 return -ENODATA; 10588 } 10589 10590 return 0; 10591 } 10592 10593 /** 10594 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 10595 * @vsi: VSI pointer 10596 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 10597 * 10598 * On error: returns error code (negative) 10599 * On success: returns 0 10600 **/ 10601 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 10602 { 10603 struct i40e_ring **next_rings; 10604 int size; 10605 int ret = 0; 10606 10607 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 10608 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 10609 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 10610 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 10611 if (!vsi->tx_rings) 10612 return -ENOMEM; 10613 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 10614 if (i40e_enabled_xdp_vsi(vsi)) { 10615 vsi->xdp_rings = next_rings; 10616 next_rings += vsi->alloc_queue_pairs; 10617 } 10618 vsi->rx_rings = next_rings; 10619 10620 if (alloc_qvectors) { 10621 /* allocate memory for q_vector pointers */ 10622 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 10623 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 10624 if (!vsi->q_vectors) { 10625 ret = -ENOMEM; 10626 goto err_vectors; 10627 } 10628 } 10629 return ret; 10630 10631 err_vectors: 10632 kfree(vsi->tx_rings); 10633 return ret; 10634 } 10635 10636 /** 10637 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 10638 * @pf: board private structure 10639 * @type: type of VSI 10640 * 10641 * On error: returns error code (negative) 10642 * On success: returns vsi index in PF (positive) 10643 **/ 10644 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 10645 { 10646 int ret = -ENODEV; 10647 struct i40e_vsi *vsi; 10648 int vsi_idx; 10649 int i; 10650 10651 /* Need to protect the allocation of the VSIs at the PF level */ 10652 mutex_lock(&pf->switch_mutex); 10653 10654 /* VSI list may be fragmented if VSI creation/destruction has 10655 * been happening. We can afford to do a quick scan to look 10656 * for any free VSIs in the list. 10657 * 10658 * find next empty vsi slot, looping back around if necessary 10659 */ 10660 i = pf->next_vsi; 10661 while (i < pf->num_alloc_vsi && pf->vsi[i]) 10662 i++; 10663 if (i >= pf->num_alloc_vsi) { 10664 i = 0; 10665 while (i < pf->next_vsi && pf->vsi[i]) 10666 i++; 10667 } 10668 10669 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 10670 vsi_idx = i; /* Found one! */ 10671 } else { 10672 ret = -ENODEV; 10673 goto unlock_pf; /* out of VSI slots! */ 10674 } 10675 pf->next_vsi = ++i; 10676 10677 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 10678 if (!vsi) { 10679 ret = -ENOMEM; 10680 goto unlock_pf; 10681 } 10682 vsi->type = type; 10683 vsi->back = pf; 10684 set_bit(__I40E_VSI_DOWN, vsi->state); 10685 vsi->flags = 0; 10686 vsi->idx = vsi_idx; 10687 vsi->int_rate_limit = 0; 10688 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 10689 pf->rss_table_size : 64; 10690 vsi->netdev_registered = false; 10691 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 10692 hash_init(vsi->mac_filter_hash); 10693 vsi->irqs_ready = false; 10694 10695 if (type == I40E_VSI_MAIN) { 10696 vsi->af_xdp_zc_qps = bitmap_zalloc(pf->num_lan_qps, GFP_KERNEL); 10697 if (!vsi->af_xdp_zc_qps) 10698 goto err_rings; 10699 } 10700 10701 ret = i40e_set_num_rings_in_vsi(vsi); 10702 if (ret) 10703 goto err_rings; 10704 10705 ret = i40e_vsi_alloc_arrays(vsi, true); 10706 if (ret) 10707 goto err_rings; 10708 10709 /* Setup default MSIX irq handler for VSI */ 10710 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 10711 10712 /* Initialize VSI lock */ 10713 spin_lock_init(&vsi->mac_filter_hash_lock); 10714 pf->vsi[vsi_idx] = vsi; 10715 ret = vsi_idx; 10716 goto unlock_pf; 10717 10718 err_rings: 10719 bitmap_free(vsi->af_xdp_zc_qps); 10720 pf->next_vsi = i - 1; 10721 kfree(vsi); 10722 unlock_pf: 10723 mutex_unlock(&pf->switch_mutex); 10724 return ret; 10725 } 10726 10727 /** 10728 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 10729 * @vsi: VSI pointer 10730 * @free_qvectors: a bool to specify if q_vectors need to be freed. 10731 * 10732 * On error: returns error code (negative) 10733 * On success: returns 0 10734 **/ 10735 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 10736 { 10737 /* free the ring and vector containers */ 10738 if (free_qvectors) { 10739 kfree(vsi->q_vectors); 10740 vsi->q_vectors = NULL; 10741 } 10742 kfree(vsi->tx_rings); 10743 vsi->tx_rings = NULL; 10744 vsi->rx_rings = NULL; 10745 vsi->xdp_rings = NULL; 10746 } 10747 10748 /** 10749 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 10750 * and lookup table 10751 * @vsi: Pointer to VSI structure 10752 */ 10753 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 10754 { 10755 if (!vsi) 10756 return; 10757 10758 kfree(vsi->rss_hkey_user); 10759 vsi->rss_hkey_user = NULL; 10760 10761 kfree(vsi->rss_lut_user); 10762 vsi->rss_lut_user = NULL; 10763 } 10764 10765 /** 10766 * i40e_vsi_clear - Deallocate the VSI provided 10767 * @vsi: the VSI being un-configured 10768 **/ 10769 static int i40e_vsi_clear(struct i40e_vsi *vsi) 10770 { 10771 struct i40e_pf *pf; 10772 10773 if (!vsi) 10774 return 0; 10775 10776 if (!vsi->back) 10777 goto free_vsi; 10778 pf = vsi->back; 10779 10780 mutex_lock(&pf->switch_mutex); 10781 if (!pf->vsi[vsi->idx]) { 10782 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 10783 vsi->idx, vsi->idx, vsi->type); 10784 goto unlock_vsi; 10785 } 10786 10787 if (pf->vsi[vsi->idx] != vsi) { 10788 dev_err(&pf->pdev->dev, 10789 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 10790 pf->vsi[vsi->idx]->idx, 10791 pf->vsi[vsi->idx]->type, 10792 vsi->idx, vsi->type); 10793 goto unlock_vsi; 10794 } 10795 10796 /* updates the PF for this cleared vsi */ 10797 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 10798 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 10799 10800 bitmap_free(vsi->af_xdp_zc_qps); 10801 i40e_vsi_free_arrays(vsi, true); 10802 i40e_clear_rss_config_user(vsi); 10803 10804 pf->vsi[vsi->idx] = NULL; 10805 if (vsi->idx < pf->next_vsi) 10806 pf->next_vsi = vsi->idx; 10807 10808 unlock_vsi: 10809 mutex_unlock(&pf->switch_mutex); 10810 free_vsi: 10811 kfree(vsi); 10812 10813 return 0; 10814 } 10815 10816 /** 10817 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 10818 * @vsi: the VSI being cleaned 10819 **/ 10820 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 10821 { 10822 int i; 10823 10824 if (vsi->tx_rings && vsi->tx_rings[0]) { 10825 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10826 kfree_rcu(vsi->tx_rings[i], rcu); 10827 vsi->tx_rings[i] = NULL; 10828 vsi->rx_rings[i] = NULL; 10829 if (vsi->xdp_rings) 10830 vsi->xdp_rings[i] = NULL; 10831 } 10832 } 10833 } 10834 10835 /** 10836 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 10837 * @vsi: the VSI being configured 10838 **/ 10839 static int i40e_alloc_rings(struct i40e_vsi *vsi) 10840 { 10841 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 10842 struct i40e_pf *pf = vsi->back; 10843 struct i40e_ring *ring; 10844 10845 /* Set basic values in the rings to be used later during open() */ 10846 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10847 /* allocate space for both Tx and Rx in one shot */ 10848 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 10849 if (!ring) 10850 goto err_out; 10851 10852 ring->queue_index = i; 10853 ring->reg_idx = vsi->base_queue + i; 10854 ring->ring_active = false; 10855 ring->vsi = vsi; 10856 ring->netdev = vsi->netdev; 10857 ring->dev = &pf->pdev->dev; 10858 ring->count = vsi->num_tx_desc; 10859 ring->size = 0; 10860 ring->dcb_tc = 0; 10861 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10862 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10863 ring->itr_setting = pf->tx_itr_default; 10864 vsi->tx_rings[i] = ring++; 10865 10866 if (!i40e_enabled_xdp_vsi(vsi)) 10867 goto setup_rx; 10868 10869 ring->queue_index = vsi->alloc_queue_pairs + i; 10870 ring->reg_idx = vsi->base_queue + ring->queue_index; 10871 ring->ring_active = false; 10872 ring->vsi = vsi; 10873 ring->netdev = NULL; 10874 ring->dev = &pf->pdev->dev; 10875 ring->count = vsi->num_tx_desc; 10876 ring->size = 0; 10877 ring->dcb_tc = 0; 10878 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10879 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10880 set_ring_xdp(ring); 10881 ring->itr_setting = pf->tx_itr_default; 10882 vsi->xdp_rings[i] = ring++; 10883 10884 setup_rx: 10885 ring->queue_index = i; 10886 ring->reg_idx = vsi->base_queue + i; 10887 ring->ring_active = false; 10888 ring->vsi = vsi; 10889 ring->netdev = vsi->netdev; 10890 ring->dev = &pf->pdev->dev; 10891 ring->count = vsi->num_rx_desc; 10892 ring->size = 0; 10893 ring->dcb_tc = 0; 10894 ring->itr_setting = pf->rx_itr_default; 10895 vsi->rx_rings[i] = ring; 10896 } 10897 10898 return 0; 10899 10900 err_out: 10901 i40e_vsi_clear_rings(vsi); 10902 return -ENOMEM; 10903 } 10904 10905 /** 10906 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 10907 * @pf: board private structure 10908 * @vectors: the number of MSI-X vectors to request 10909 * 10910 * Returns the number of vectors reserved, or error 10911 **/ 10912 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 10913 { 10914 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 10915 I40E_MIN_MSIX, vectors); 10916 if (vectors < 0) { 10917 dev_info(&pf->pdev->dev, 10918 "MSI-X vector reservation failed: %d\n", vectors); 10919 vectors = 0; 10920 } 10921 10922 return vectors; 10923 } 10924 10925 /** 10926 * i40e_init_msix - Setup the MSIX capability 10927 * @pf: board private structure 10928 * 10929 * Work with the OS to set up the MSIX vectors needed. 10930 * 10931 * Returns the number of vectors reserved or negative on failure 10932 **/ 10933 static int i40e_init_msix(struct i40e_pf *pf) 10934 { 10935 struct i40e_hw *hw = &pf->hw; 10936 int cpus, extra_vectors; 10937 int vectors_left; 10938 int v_budget, i; 10939 int v_actual; 10940 int iwarp_requested = 0; 10941 10942 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10943 return -ENODEV; 10944 10945 /* The number of vectors we'll request will be comprised of: 10946 * - Add 1 for "other" cause for Admin Queue events, etc. 10947 * - The number of LAN queue pairs 10948 * - Queues being used for RSS. 10949 * We don't need as many as max_rss_size vectors. 10950 * use rss_size instead in the calculation since that 10951 * is governed by number of cpus in the system. 10952 * - assumes symmetric Tx/Rx pairing 10953 * - The number of VMDq pairs 10954 * - The CPU count within the NUMA node if iWARP is enabled 10955 * Once we count this up, try the request. 10956 * 10957 * If we can't get what we want, we'll simplify to nearly nothing 10958 * and try again. If that still fails, we punt. 10959 */ 10960 vectors_left = hw->func_caps.num_msix_vectors; 10961 v_budget = 0; 10962 10963 /* reserve one vector for miscellaneous handler */ 10964 if (vectors_left) { 10965 v_budget++; 10966 vectors_left--; 10967 } 10968 10969 /* reserve some vectors for the main PF traffic queues. Initially we 10970 * only reserve at most 50% of the available vectors, in the case that 10971 * the number of online CPUs is large. This ensures that we can enable 10972 * extra features as well. Once we've enabled the other features, we 10973 * will use any remaining vectors to reach as close as we can to the 10974 * number of online CPUs. 10975 */ 10976 cpus = num_online_cpus(); 10977 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 10978 vectors_left -= pf->num_lan_msix; 10979 10980 /* reserve one vector for sideband flow director */ 10981 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10982 if (vectors_left) { 10983 pf->num_fdsb_msix = 1; 10984 v_budget++; 10985 vectors_left--; 10986 } else { 10987 pf->num_fdsb_msix = 0; 10988 } 10989 } 10990 10991 /* can we reserve enough for iWARP? */ 10992 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10993 iwarp_requested = pf->num_iwarp_msix; 10994 10995 if (!vectors_left) 10996 pf->num_iwarp_msix = 0; 10997 else if (vectors_left < pf->num_iwarp_msix) 10998 pf->num_iwarp_msix = 1; 10999 v_budget += pf->num_iwarp_msix; 11000 vectors_left -= pf->num_iwarp_msix; 11001 } 11002 11003 /* any vectors left over go for VMDq support */ 11004 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 11005 if (!vectors_left) { 11006 pf->num_vmdq_msix = 0; 11007 pf->num_vmdq_qps = 0; 11008 } else { 11009 int vmdq_vecs_wanted = 11010 pf->num_vmdq_vsis * pf->num_vmdq_qps; 11011 int vmdq_vecs = 11012 min_t(int, vectors_left, vmdq_vecs_wanted); 11013 11014 /* if we're short on vectors for what's desired, we limit 11015 * the queues per vmdq. If this is still more than are 11016 * available, the user will need to change the number of 11017 * queues/vectors used by the PF later with the ethtool 11018 * channels command 11019 */ 11020 if (vectors_left < vmdq_vecs_wanted) { 11021 pf->num_vmdq_qps = 1; 11022 vmdq_vecs_wanted = pf->num_vmdq_vsis; 11023 vmdq_vecs = min_t(int, 11024 vectors_left, 11025 vmdq_vecs_wanted); 11026 } 11027 pf->num_vmdq_msix = pf->num_vmdq_qps; 11028 11029 v_budget += vmdq_vecs; 11030 vectors_left -= vmdq_vecs; 11031 } 11032 } 11033 11034 /* On systems with a large number of SMP cores, we previously limited 11035 * the number of vectors for num_lan_msix to be at most 50% of the 11036 * available vectors, to allow for other features. Now, we add back 11037 * the remaining vectors. However, we ensure that the total 11038 * num_lan_msix will not exceed num_online_cpus(). To do this, we 11039 * calculate the number of vectors we can add without going over the 11040 * cap of CPUs. For systems with a small number of CPUs this will be 11041 * zero. 11042 */ 11043 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 11044 pf->num_lan_msix += extra_vectors; 11045 vectors_left -= extra_vectors; 11046 11047 WARN(vectors_left < 0, 11048 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 11049 11050 v_budget += pf->num_lan_msix; 11051 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 11052 GFP_KERNEL); 11053 if (!pf->msix_entries) 11054 return -ENOMEM; 11055 11056 for (i = 0; i < v_budget; i++) 11057 pf->msix_entries[i].entry = i; 11058 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 11059 11060 if (v_actual < I40E_MIN_MSIX) { 11061 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 11062 kfree(pf->msix_entries); 11063 pf->msix_entries = NULL; 11064 pci_disable_msix(pf->pdev); 11065 return -ENODEV; 11066 11067 } else if (v_actual == I40E_MIN_MSIX) { 11068 /* Adjust for minimal MSIX use */ 11069 pf->num_vmdq_vsis = 0; 11070 pf->num_vmdq_qps = 0; 11071 pf->num_lan_qps = 1; 11072 pf->num_lan_msix = 1; 11073 11074 } else if (v_actual != v_budget) { 11075 /* If we have limited resources, we will start with no vectors 11076 * for the special features and then allocate vectors to some 11077 * of these features based on the policy and at the end disable 11078 * the features that did not get any vectors. 11079 */ 11080 int vec; 11081 11082 dev_info(&pf->pdev->dev, 11083 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 11084 v_actual, v_budget); 11085 /* reserve the misc vector */ 11086 vec = v_actual - 1; 11087 11088 /* Scale vector usage down */ 11089 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 11090 pf->num_vmdq_vsis = 1; 11091 pf->num_vmdq_qps = 1; 11092 11093 /* partition out the remaining vectors */ 11094 switch (vec) { 11095 case 2: 11096 pf->num_lan_msix = 1; 11097 break; 11098 case 3: 11099 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11100 pf->num_lan_msix = 1; 11101 pf->num_iwarp_msix = 1; 11102 } else { 11103 pf->num_lan_msix = 2; 11104 } 11105 break; 11106 default: 11107 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11108 pf->num_iwarp_msix = min_t(int, (vec / 3), 11109 iwarp_requested); 11110 pf->num_vmdq_vsis = min_t(int, (vec / 3), 11111 I40E_DEFAULT_NUM_VMDQ_VSI); 11112 } else { 11113 pf->num_vmdq_vsis = min_t(int, (vec / 2), 11114 I40E_DEFAULT_NUM_VMDQ_VSI); 11115 } 11116 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11117 pf->num_fdsb_msix = 1; 11118 vec--; 11119 } 11120 pf->num_lan_msix = min_t(int, 11121 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 11122 pf->num_lan_msix); 11123 pf->num_lan_qps = pf->num_lan_msix; 11124 break; 11125 } 11126 } 11127 11128 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 11129 (pf->num_fdsb_msix == 0)) { 11130 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 11131 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 11132 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11133 } 11134 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 11135 (pf->num_vmdq_msix == 0)) { 11136 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 11137 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 11138 } 11139 11140 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 11141 (pf->num_iwarp_msix == 0)) { 11142 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 11143 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11144 } 11145 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 11146 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 11147 pf->num_lan_msix, 11148 pf->num_vmdq_msix * pf->num_vmdq_vsis, 11149 pf->num_fdsb_msix, 11150 pf->num_iwarp_msix); 11151 11152 return v_actual; 11153 } 11154 11155 /** 11156 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 11157 * @vsi: the VSI being configured 11158 * @v_idx: index of the vector in the vsi struct 11159 * @cpu: cpu to be used on affinity_mask 11160 * 11161 * We allocate one q_vector. If allocation fails we return -ENOMEM. 11162 **/ 11163 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 11164 { 11165 struct i40e_q_vector *q_vector; 11166 11167 /* allocate q_vector */ 11168 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 11169 if (!q_vector) 11170 return -ENOMEM; 11171 11172 q_vector->vsi = vsi; 11173 q_vector->v_idx = v_idx; 11174 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 11175 11176 if (vsi->netdev) 11177 netif_napi_add(vsi->netdev, &q_vector->napi, 11178 i40e_napi_poll, NAPI_POLL_WEIGHT); 11179 11180 /* tie q_vector and vsi together */ 11181 vsi->q_vectors[v_idx] = q_vector; 11182 11183 return 0; 11184 } 11185 11186 /** 11187 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 11188 * @vsi: the VSI being configured 11189 * 11190 * We allocate one q_vector per queue interrupt. If allocation fails we 11191 * return -ENOMEM. 11192 **/ 11193 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 11194 { 11195 struct i40e_pf *pf = vsi->back; 11196 int err, v_idx, num_q_vectors, current_cpu; 11197 11198 /* if not MSIX, give the one vector only to the LAN VSI */ 11199 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 11200 num_q_vectors = vsi->num_q_vectors; 11201 else if (vsi == pf->vsi[pf->lan_vsi]) 11202 num_q_vectors = 1; 11203 else 11204 return -EINVAL; 11205 11206 current_cpu = cpumask_first(cpu_online_mask); 11207 11208 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 11209 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 11210 if (err) 11211 goto err_out; 11212 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 11213 if (unlikely(current_cpu >= nr_cpu_ids)) 11214 current_cpu = cpumask_first(cpu_online_mask); 11215 } 11216 11217 return 0; 11218 11219 err_out: 11220 while (v_idx--) 11221 i40e_free_q_vector(vsi, v_idx); 11222 11223 return err; 11224 } 11225 11226 /** 11227 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 11228 * @pf: board private structure to initialize 11229 **/ 11230 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 11231 { 11232 int vectors = 0; 11233 ssize_t size; 11234 11235 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11236 vectors = i40e_init_msix(pf); 11237 if (vectors < 0) { 11238 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 11239 I40E_FLAG_IWARP_ENABLED | 11240 I40E_FLAG_RSS_ENABLED | 11241 I40E_FLAG_DCB_CAPABLE | 11242 I40E_FLAG_DCB_ENABLED | 11243 I40E_FLAG_SRIOV_ENABLED | 11244 I40E_FLAG_FD_SB_ENABLED | 11245 I40E_FLAG_FD_ATR_ENABLED | 11246 I40E_FLAG_VMDQ_ENABLED); 11247 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11248 11249 /* rework the queue expectations without MSIX */ 11250 i40e_determine_queue_usage(pf); 11251 } 11252 } 11253 11254 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 11255 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 11256 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 11257 vectors = pci_enable_msi(pf->pdev); 11258 if (vectors < 0) { 11259 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 11260 vectors); 11261 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 11262 } 11263 vectors = 1; /* one MSI or Legacy vector */ 11264 } 11265 11266 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 11267 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 11268 11269 /* set up vector assignment tracking */ 11270 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 11271 pf->irq_pile = kzalloc(size, GFP_KERNEL); 11272 if (!pf->irq_pile) 11273 return -ENOMEM; 11274 11275 pf->irq_pile->num_entries = vectors; 11276 pf->irq_pile->search_hint = 0; 11277 11278 /* track first vector for misc interrupts, ignore return */ 11279 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 11280 11281 return 0; 11282 } 11283 11284 /** 11285 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 11286 * @pf: private board data structure 11287 * 11288 * Restore the interrupt scheme that was cleared when we suspended the 11289 * device. This should be called during resume to re-allocate the q_vectors 11290 * and reacquire IRQs. 11291 */ 11292 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 11293 { 11294 int err, i; 11295 11296 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 11297 * scheme. We need to re-enabled them here in order to attempt to 11298 * re-acquire the MSI or MSI-X vectors 11299 */ 11300 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 11301 11302 err = i40e_init_interrupt_scheme(pf); 11303 if (err) 11304 return err; 11305 11306 /* Now that we've re-acquired IRQs, we need to remap the vectors and 11307 * rings together again. 11308 */ 11309 for (i = 0; i < pf->num_alloc_vsi; i++) { 11310 if (pf->vsi[i]) { 11311 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 11312 if (err) 11313 goto err_unwind; 11314 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 11315 } 11316 } 11317 11318 err = i40e_setup_misc_vector(pf); 11319 if (err) 11320 goto err_unwind; 11321 11322 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 11323 i40e_client_update_msix_info(pf); 11324 11325 return 0; 11326 11327 err_unwind: 11328 while (i--) { 11329 if (pf->vsi[i]) 11330 i40e_vsi_free_q_vectors(pf->vsi[i]); 11331 } 11332 11333 return err; 11334 } 11335 11336 /** 11337 * i40e_setup_misc_vector_for_recovery_mode - Setup the misc vector to handle 11338 * non queue events in recovery mode 11339 * @pf: board private structure 11340 * 11341 * This sets up the handler for MSIX 0 or MSI/legacy, which is used to manage 11342 * the non-queue interrupts, e.g. AdminQ and errors in recovery mode. 11343 * This is handled differently than in recovery mode since no Tx/Rx resources 11344 * are being allocated. 11345 **/ 11346 static int i40e_setup_misc_vector_for_recovery_mode(struct i40e_pf *pf) 11347 { 11348 int err; 11349 11350 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11351 err = i40e_setup_misc_vector(pf); 11352 11353 if (err) { 11354 dev_info(&pf->pdev->dev, 11355 "MSI-X misc vector request failed, error %d\n", 11356 err); 11357 return err; 11358 } 11359 } else { 11360 u32 flags = pf->flags & I40E_FLAG_MSI_ENABLED ? 0 : IRQF_SHARED; 11361 11362 err = request_irq(pf->pdev->irq, i40e_intr, flags, 11363 pf->int_name, pf); 11364 11365 if (err) { 11366 dev_info(&pf->pdev->dev, 11367 "MSI/legacy misc vector request failed, error %d\n", 11368 err); 11369 return err; 11370 } 11371 i40e_enable_misc_int_causes(pf); 11372 i40e_irq_dynamic_enable_icr0(pf); 11373 } 11374 11375 return 0; 11376 } 11377 11378 /** 11379 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 11380 * @pf: board private structure 11381 * 11382 * This sets up the handler for MSIX 0, which is used to manage the 11383 * non-queue interrupts, e.g. AdminQ and errors. This is not used 11384 * when in MSI or Legacy interrupt mode. 11385 **/ 11386 static int i40e_setup_misc_vector(struct i40e_pf *pf) 11387 { 11388 struct i40e_hw *hw = &pf->hw; 11389 int err = 0; 11390 11391 /* Only request the IRQ once, the first time through. */ 11392 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 11393 err = request_irq(pf->msix_entries[0].vector, 11394 i40e_intr, 0, pf->int_name, pf); 11395 if (err) { 11396 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 11397 dev_info(&pf->pdev->dev, 11398 "request_irq for %s failed: %d\n", 11399 pf->int_name, err); 11400 return -EFAULT; 11401 } 11402 } 11403 11404 i40e_enable_misc_int_causes(pf); 11405 11406 /* associate no queues to the misc vector */ 11407 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 11408 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K >> 1); 11409 11410 i40e_flush(hw); 11411 11412 i40e_irq_dynamic_enable_icr0(pf); 11413 11414 return err; 11415 } 11416 11417 /** 11418 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 11419 * @vsi: Pointer to vsi structure 11420 * @seed: Buffter to store the hash keys 11421 * @lut: Buffer to store the lookup table entries 11422 * @lut_size: Size of buffer to store the lookup table entries 11423 * 11424 * Return 0 on success, negative on failure 11425 */ 11426 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 11427 u8 *lut, u16 lut_size) 11428 { 11429 struct i40e_pf *pf = vsi->back; 11430 struct i40e_hw *hw = &pf->hw; 11431 int ret = 0; 11432 11433 if (seed) { 11434 ret = i40e_aq_get_rss_key(hw, vsi->id, 11435 (struct i40e_aqc_get_set_rss_key_data *)seed); 11436 if (ret) { 11437 dev_info(&pf->pdev->dev, 11438 "Cannot get RSS key, err %s aq_err %s\n", 11439 i40e_stat_str(&pf->hw, ret), 11440 i40e_aq_str(&pf->hw, 11441 pf->hw.aq.asq_last_status)); 11442 return ret; 11443 } 11444 } 11445 11446 if (lut) { 11447 bool pf_lut = vsi->type == I40E_VSI_MAIN; 11448 11449 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 11450 if (ret) { 11451 dev_info(&pf->pdev->dev, 11452 "Cannot get RSS lut, err %s aq_err %s\n", 11453 i40e_stat_str(&pf->hw, ret), 11454 i40e_aq_str(&pf->hw, 11455 pf->hw.aq.asq_last_status)); 11456 return ret; 11457 } 11458 } 11459 11460 return ret; 11461 } 11462 11463 /** 11464 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 11465 * @vsi: Pointer to vsi structure 11466 * @seed: RSS hash seed 11467 * @lut: Lookup table 11468 * @lut_size: Lookup table size 11469 * 11470 * Returns 0 on success, negative on failure 11471 **/ 11472 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 11473 const u8 *lut, u16 lut_size) 11474 { 11475 struct i40e_pf *pf = vsi->back; 11476 struct i40e_hw *hw = &pf->hw; 11477 u16 vf_id = vsi->vf_id; 11478 u8 i; 11479 11480 /* Fill out hash function seed */ 11481 if (seed) { 11482 u32 *seed_dw = (u32 *)seed; 11483 11484 if (vsi->type == I40E_VSI_MAIN) { 11485 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 11486 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 11487 } else if (vsi->type == I40E_VSI_SRIOV) { 11488 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 11489 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 11490 } else { 11491 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 11492 } 11493 } 11494 11495 if (lut) { 11496 u32 *lut_dw = (u32 *)lut; 11497 11498 if (vsi->type == I40E_VSI_MAIN) { 11499 if (lut_size != I40E_HLUT_ARRAY_SIZE) 11500 return -EINVAL; 11501 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11502 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 11503 } else if (vsi->type == I40E_VSI_SRIOV) { 11504 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 11505 return -EINVAL; 11506 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 11507 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 11508 } else { 11509 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 11510 } 11511 } 11512 i40e_flush(hw); 11513 11514 return 0; 11515 } 11516 11517 /** 11518 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 11519 * @vsi: Pointer to VSI structure 11520 * @seed: Buffer to store the keys 11521 * @lut: Buffer to store the lookup table entries 11522 * @lut_size: Size of buffer to store the lookup table entries 11523 * 11524 * Returns 0 on success, negative on failure 11525 */ 11526 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 11527 u8 *lut, u16 lut_size) 11528 { 11529 struct i40e_pf *pf = vsi->back; 11530 struct i40e_hw *hw = &pf->hw; 11531 u16 i; 11532 11533 if (seed) { 11534 u32 *seed_dw = (u32 *)seed; 11535 11536 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 11537 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 11538 } 11539 if (lut) { 11540 u32 *lut_dw = (u32 *)lut; 11541 11542 if (lut_size != I40E_HLUT_ARRAY_SIZE) 11543 return -EINVAL; 11544 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11545 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 11546 } 11547 11548 return 0; 11549 } 11550 11551 /** 11552 * i40e_config_rss - Configure RSS keys and lut 11553 * @vsi: Pointer to VSI structure 11554 * @seed: RSS hash seed 11555 * @lut: Lookup table 11556 * @lut_size: Lookup table size 11557 * 11558 * Returns 0 on success, negative on failure 11559 */ 11560 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 11561 { 11562 struct i40e_pf *pf = vsi->back; 11563 11564 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 11565 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 11566 else 11567 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 11568 } 11569 11570 /** 11571 * i40e_get_rss - Get RSS keys and lut 11572 * @vsi: Pointer to VSI structure 11573 * @seed: Buffer to store the keys 11574 * @lut: Buffer to store the lookup table entries 11575 * @lut_size: Size of buffer to store the lookup table entries 11576 * 11577 * Returns 0 on success, negative on failure 11578 */ 11579 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 11580 { 11581 struct i40e_pf *pf = vsi->back; 11582 11583 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 11584 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 11585 else 11586 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 11587 } 11588 11589 /** 11590 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 11591 * @pf: Pointer to board private structure 11592 * @lut: Lookup table 11593 * @rss_table_size: Lookup table size 11594 * @rss_size: Range of queue number for hashing 11595 */ 11596 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 11597 u16 rss_table_size, u16 rss_size) 11598 { 11599 u16 i; 11600 11601 for (i = 0; i < rss_table_size; i++) 11602 lut[i] = i % rss_size; 11603 } 11604 11605 /** 11606 * i40e_pf_config_rss - Prepare for RSS if used 11607 * @pf: board private structure 11608 **/ 11609 static int i40e_pf_config_rss(struct i40e_pf *pf) 11610 { 11611 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11612 u8 seed[I40E_HKEY_ARRAY_SIZE]; 11613 u8 *lut; 11614 struct i40e_hw *hw = &pf->hw; 11615 u32 reg_val; 11616 u64 hena; 11617 int ret; 11618 11619 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 11620 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 11621 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 11622 hena |= i40e_pf_get_default_rss_hena(pf); 11623 11624 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 11625 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 11626 11627 /* Determine the RSS table size based on the hardware capabilities */ 11628 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 11629 reg_val = (pf->rss_table_size == 512) ? 11630 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 11631 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 11632 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 11633 11634 /* Determine the RSS size of the VSI */ 11635 if (!vsi->rss_size) { 11636 u16 qcount; 11637 /* If the firmware does something weird during VSI init, we 11638 * could end up with zero TCs. Check for that to avoid 11639 * divide-by-zero. It probably won't pass traffic, but it also 11640 * won't panic. 11641 */ 11642 qcount = vsi->num_queue_pairs / 11643 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 11644 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11645 } 11646 if (!vsi->rss_size) 11647 return -EINVAL; 11648 11649 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 11650 if (!lut) 11651 return -ENOMEM; 11652 11653 /* Use user configured lut if there is one, otherwise use default */ 11654 if (vsi->rss_lut_user) 11655 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 11656 else 11657 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 11658 11659 /* Use user configured hash key if there is one, otherwise 11660 * use default. 11661 */ 11662 if (vsi->rss_hkey_user) 11663 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 11664 else 11665 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 11666 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 11667 kfree(lut); 11668 11669 return ret; 11670 } 11671 11672 /** 11673 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 11674 * @pf: board private structure 11675 * @queue_count: the requested queue count for rss. 11676 * 11677 * returns 0 if rss is not enabled, if enabled returns the final rss queue 11678 * count which may be different from the requested queue count. 11679 * Note: expects to be called while under rtnl_lock() 11680 **/ 11681 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 11682 { 11683 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 11684 int new_rss_size; 11685 11686 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 11687 return 0; 11688 11689 queue_count = min_t(int, queue_count, num_online_cpus()); 11690 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 11691 11692 if (queue_count != vsi->num_queue_pairs) { 11693 u16 qcount; 11694 11695 vsi->req_queue_pairs = queue_count; 11696 i40e_prep_for_reset(pf, true); 11697 11698 pf->alloc_rss_size = new_rss_size; 11699 11700 i40e_reset_and_rebuild(pf, true, true); 11701 11702 /* Discard the user configured hash keys and lut, if less 11703 * queues are enabled. 11704 */ 11705 if (queue_count < vsi->rss_size) { 11706 i40e_clear_rss_config_user(vsi); 11707 dev_dbg(&pf->pdev->dev, 11708 "discard user configured hash keys and lut\n"); 11709 } 11710 11711 /* Reset vsi->rss_size, as number of enabled queues changed */ 11712 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 11713 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 11714 11715 i40e_pf_config_rss(pf); 11716 } 11717 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 11718 vsi->req_queue_pairs, pf->rss_size_max); 11719 return pf->alloc_rss_size; 11720 } 11721 11722 /** 11723 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 11724 * @pf: board private structure 11725 **/ 11726 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 11727 { 11728 i40e_status status; 11729 bool min_valid, max_valid; 11730 u32 max_bw, min_bw; 11731 11732 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 11733 &min_valid, &max_valid); 11734 11735 if (!status) { 11736 if (min_valid) 11737 pf->min_bw = min_bw; 11738 if (max_valid) 11739 pf->max_bw = max_bw; 11740 } 11741 11742 return status; 11743 } 11744 11745 /** 11746 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 11747 * @pf: board private structure 11748 **/ 11749 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 11750 { 11751 struct i40e_aqc_configure_partition_bw_data bw_data; 11752 i40e_status status; 11753 11754 /* Set the valid bit for this PF */ 11755 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 11756 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 11757 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 11758 11759 /* Set the new bandwidths */ 11760 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 11761 11762 return status; 11763 } 11764 11765 /** 11766 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 11767 * @pf: board private structure 11768 **/ 11769 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 11770 { 11771 /* Commit temporary BW setting to permanent NVM image */ 11772 enum i40e_admin_queue_err last_aq_status; 11773 i40e_status ret; 11774 u16 nvm_word; 11775 11776 if (pf->hw.partition_id != 1) { 11777 dev_info(&pf->pdev->dev, 11778 "Commit BW only works on partition 1! This is partition %d", 11779 pf->hw.partition_id); 11780 ret = I40E_NOT_SUPPORTED; 11781 goto bw_commit_out; 11782 } 11783 11784 /* Acquire NVM for read access */ 11785 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 11786 last_aq_status = pf->hw.aq.asq_last_status; 11787 if (ret) { 11788 dev_info(&pf->pdev->dev, 11789 "Cannot acquire NVM for read access, err %s aq_err %s\n", 11790 i40e_stat_str(&pf->hw, ret), 11791 i40e_aq_str(&pf->hw, last_aq_status)); 11792 goto bw_commit_out; 11793 } 11794 11795 /* Read word 0x10 of NVM - SW compatibility word 1 */ 11796 ret = i40e_aq_read_nvm(&pf->hw, 11797 I40E_SR_NVM_CONTROL_WORD, 11798 0x10, sizeof(nvm_word), &nvm_word, 11799 false, NULL); 11800 /* Save off last admin queue command status before releasing 11801 * the NVM 11802 */ 11803 last_aq_status = pf->hw.aq.asq_last_status; 11804 i40e_release_nvm(&pf->hw); 11805 if (ret) { 11806 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 11807 i40e_stat_str(&pf->hw, ret), 11808 i40e_aq_str(&pf->hw, last_aq_status)); 11809 goto bw_commit_out; 11810 } 11811 11812 /* Wait a bit for NVM release to complete */ 11813 msleep(50); 11814 11815 /* Acquire NVM for write access */ 11816 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 11817 last_aq_status = pf->hw.aq.asq_last_status; 11818 if (ret) { 11819 dev_info(&pf->pdev->dev, 11820 "Cannot acquire NVM for write access, err %s aq_err %s\n", 11821 i40e_stat_str(&pf->hw, ret), 11822 i40e_aq_str(&pf->hw, last_aq_status)); 11823 goto bw_commit_out; 11824 } 11825 /* Write it back out unchanged to initiate update NVM, 11826 * which will force a write of the shadow (alt) RAM to 11827 * the NVM - thus storing the bandwidth values permanently. 11828 */ 11829 ret = i40e_aq_update_nvm(&pf->hw, 11830 I40E_SR_NVM_CONTROL_WORD, 11831 0x10, sizeof(nvm_word), 11832 &nvm_word, true, 0, NULL); 11833 /* Save off last admin queue command status before releasing 11834 * the NVM 11835 */ 11836 last_aq_status = pf->hw.aq.asq_last_status; 11837 i40e_release_nvm(&pf->hw); 11838 if (ret) 11839 dev_info(&pf->pdev->dev, 11840 "BW settings NOT SAVED, err %s aq_err %s\n", 11841 i40e_stat_str(&pf->hw, ret), 11842 i40e_aq_str(&pf->hw, last_aq_status)); 11843 bw_commit_out: 11844 11845 return ret; 11846 } 11847 11848 /** 11849 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 11850 * @pf: board private structure to initialize 11851 * 11852 * i40e_sw_init initializes the Adapter private data structure. 11853 * Fields are initialized based on PCI device information and 11854 * OS network device settings (MTU size). 11855 **/ 11856 static int i40e_sw_init(struct i40e_pf *pf) 11857 { 11858 int err = 0; 11859 int size; 11860 11861 /* Set default capability flags */ 11862 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 11863 I40E_FLAG_MSI_ENABLED | 11864 I40E_FLAG_MSIX_ENABLED; 11865 11866 /* Set default ITR */ 11867 pf->rx_itr_default = I40E_ITR_RX_DEF; 11868 pf->tx_itr_default = I40E_ITR_TX_DEF; 11869 11870 /* Depending on PF configurations, it is possible that the RSS 11871 * maximum might end up larger than the available queues 11872 */ 11873 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 11874 pf->alloc_rss_size = 1; 11875 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 11876 pf->rss_size_max = min_t(int, pf->rss_size_max, 11877 pf->hw.func_caps.num_tx_qp); 11878 if (pf->hw.func_caps.rss) { 11879 pf->flags |= I40E_FLAG_RSS_ENABLED; 11880 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 11881 num_online_cpus()); 11882 } 11883 11884 /* MFP mode enabled */ 11885 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 11886 pf->flags |= I40E_FLAG_MFP_ENABLED; 11887 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 11888 if (i40e_get_partition_bw_setting(pf)) { 11889 dev_warn(&pf->pdev->dev, 11890 "Could not get partition bw settings\n"); 11891 } else { 11892 dev_info(&pf->pdev->dev, 11893 "Partition BW Min = %8.8x, Max = %8.8x\n", 11894 pf->min_bw, pf->max_bw); 11895 11896 /* nudge the Tx scheduler */ 11897 i40e_set_partition_bw_setting(pf); 11898 } 11899 } 11900 11901 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 11902 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 11903 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 11904 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 11905 if (pf->flags & I40E_FLAG_MFP_ENABLED && 11906 pf->hw.num_partitions > 1) 11907 dev_info(&pf->pdev->dev, 11908 "Flow Director Sideband mode Disabled in MFP mode\n"); 11909 else 11910 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11911 pf->fdir_pf_filter_count = 11912 pf->hw.func_caps.fd_filters_guaranteed; 11913 pf->hw.fdir_shared_filter_count = 11914 pf->hw.func_caps.fd_filters_best_effort; 11915 } 11916 11917 if (pf->hw.mac.type == I40E_MAC_X722) { 11918 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 11919 I40E_HW_128_QP_RSS_CAPABLE | 11920 I40E_HW_ATR_EVICT_CAPABLE | 11921 I40E_HW_WB_ON_ITR_CAPABLE | 11922 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 11923 I40E_HW_NO_PCI_LINK_CHECK | 11924 I40E_HW_USE_SET_LLDP_MIB | 11925 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 11926 I40E_HW_PTP_L4_CAPABLE | 11927 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 11928 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 11929 11930 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 11931 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 11932 I40E_FDEVICT_PCTYPE_DEFAULT) { 11933 dev_warn(&pf->pdev->dev, 11934 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 11935 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 11936 } 11937 } else if ((pf->hw.aq.api_maj_ver > 1) || 11938 ((pf->hw.aq.api_maj_ver == 1) && 11939 (pf->hw.aq.api_min_ver > 4))) { 11940 /* Supported in FW API version higher than 1.4 */ 11941 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 11942 } 11943 11944 /* Enable HW ATR eviction if possible */ 11945 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 11946 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 11947 11948 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11949 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 11950 (pf->hw.aq.fw_maj_ver < 4))) { 11951 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 11952 /* No DCB support for FW < v4.33 */ 11953 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 11954 } 11955 11956 /* Disable FW LLDP if FW < v4.3 */ 11957 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11958 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 11959 (pf->hw.aq.fw_maj_ver < 4))) 11960 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 11961 11962 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 11963 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11964 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 11965 (pf->hw.aq.fw_maj_ver >= 5))) 11966 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 11967 11968 /* Enable PTP L4 if FW > v6.0 */ 11969 if (pf->hw.mac.type == I40E_MAC_XL710 && 11970 pf->hw.aq.fw_maj_ver >= 6) 11971 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 11972 11973 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 11974 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 11975 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 11976 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 11977 } 11978 11979 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 11980 pf->flags |= I40E_FLAG_IWARP_ENABLED; 11981 /* IWARP needs one extra vector for CQP just like MISC.*/ 11982 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 11983 } 11984 /* Stopping FW LLDP engine is supported on XL710 and X722 11985 * starting from FW versions determined in i40e_init_adminq. 11986 * Stopping the FW LLDP engine is not supported on XL710 11987 * if NPAR is functioning so unset this hw flag in this case. 11988 */ 11989 if (pf->hw.mac.type == I40E_MAC_XL710 && 11990 pf->hw.func_caps.npar_enable && 11991 (pf->hw.flags & I40E_HW_FLAG_FW_LLDP_STOPPABLE)) 11992 pf->hw.flags &= ~I40E_HW_FLAG_FW_LLDP_STOPPABLE; 11993 11994 #ifdef CONFIG_PCI_IOV 11995 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 11996 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 11997 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 11998 pf->num_req_vfs = min_t(int, 11999 pf->hw.func_caps.num_vfs, 12000 I40E_MAX_VF_COUNT); 12001 } 12002 #endif /* CONFIG_PCI_IOV */ 12003 pf->eeprom_version = 0xDEAD; 12004 pf->lan_veb = I40E_NO_VEB; 12005 pf->lan_vsi = I40E_NO_VSI; 12006 12007 /* By default FW has this off for performance reasons */ 12008 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 12009 12010 /* set up queue assignment tracking */ 12011 size = sizeof(struct i40e_lump_tracking) 12012 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 12013 pf->qp_pile = kzalloc(size, GFP_KERNEL); 12014 if (!pf->qp_pile) { 12015 err = -ENOMEM; 12016 goto sw_init_done; 12017 } 12018 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 12019 pf->qp_pile->search_hint = 0; 12020 12021 pf->tx_timeout_recovery_level = 1; 12022 12023 mutex_init(&pf->switch_mutex); 12024 12025 sw_init_done: 12026 return err; 12027 } 12028 12029 /** 12030 * i40e_set_ntuple - set the ntuple feature flag and take action 12031 * @pf: board private structure to initialize 12032 * @features: the feature set that the stack is suggesting 12033 * 12034 * returns a bool to indicate if reset needs to happen 12035 **/ 12036 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 12037 { 12038 bool need_reset = false; 12039 12040 /* Check if Flow Director n-tuple support was enabled or disabled. If 12041 * the state changed, we need to reset. 12042 */ 12043 if (features & NETIF_F_NTUPLE) { 12044 /* Enable filters and mark for reset */ 12045 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 12046 need_reset = true; 12047 /* enable FD_SB only if there is MSI-X vector and no cloud 12048 * filters exist 12049 */ 12050 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 12051 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 12052 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 12053 } 12054 } else { 12055 /* turn off filters, mark for reset and clear SW filter list */ 12056 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 12057 need_reset = true; 12058 i40e_fdir_filter_exit(pf); 12059 } 12060 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 12061 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 12062 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 12063 12064 /* reset fd counters */ 12065 pf->fd_add_err = 0; 12066 pf->fd_atr_cnt = 0; 12067 /* if ATR was auto disabled it can be re-enabled. */ 12068 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 12069 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 12070 (I40E_DEBUG_FD & pf->hw.debug_mask)) 12071 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 12072 } 12073 return need_reset; 12074 } 12075 12076 /** 12077 * i40e_clear_rss_lut - clear the rx hash lookup table 12078 * @vsi: the VSI being configured 12079 **/ 12080 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 12081 { 12082 struct i40e_pf *pf = vsi->back; 12083 struct i40e_hw *hw = &pf->hw; 12084 u16 vf_id = vsi->vf_id; 12085 u8 i; 12086 12087 if (vsi->type == I40E_VSI_MAIN) { 12088 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 12089 wr32(hw, I40E_PFQF_HLUT(i), 0); 12090 } else if (vsi->type == I40E_VSI_SRIOV) { 12091 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 12092 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 12093 } else { 12094 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 12095 } 12096 } 12097 12098 /** 12099 * i40e_set_features - set the netdev feature flags 12100 * @netdev: ptr to the netdev being adjusted 12101 * @features: the feature set that the stack is suggesting 12102 * Note: expects to be called while under rtnl_lock() 12103 **/ 12104 static int i40e_set_features(struct net_device *netdev, 12105 netdev_features_t features) 12106 { 12107 struct i40e_netdev_priv *np = netdev_priv(netdev); 12108 struct i40e_vsi *vsi = np->vsi; 12109 struct i40e_pf *pf = vsi->back; 12110 bool need_reset; 12111 12112 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 12113 i40e_pf_config_rss(pf); 12114 else if (!(features & NETIF_F_RXHASH) && 12115 netdev->features & NETIF_F_RXHASH) 12116 i40e_clear_rss_lut(vsi); 12117 12118 if (features & NETIF_F_HW_VLAN_CTAG_RX) 12119 i40e_vlan_stripping_enable(vsi); 12120 else 12121 i40e_vlan_stripping_disable(vsi); 12122 12123 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 12124 dev_err(&pf->pdev->dev, 12125 "Offloaded tc filters active, can't turn hw_tc_offload off"); 12126 return -EINVAL; 12127 } 12128 12129 if (!(features & NETIF_F_HW_L2FW_DOFFLOAD) && vsi->macvlan_cnt) 12130 i40e_del_all_macvlans(vsi); 12131 12132 need_reset = i40e_set_ntuple(pf, features); 12133 12134 if (need_reset) 12135 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12136 12137 return 0; 12138 } 12139 12140 /** 12141 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 12142 * @pf: board private structure 12143 * @port: The UDP port to look up 12144 * 12145 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 12146 **/ 12147 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 12148 { 12149 u8 i; 12150 12151 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 12152 /* Do not report ports with pending deletions as 12153 * being available. 12154 */ 12155 if (!port && (pf->pending_udp_bitmap & BIT_ULL(i))) 12156 continue; 12157 if (pf->udp_ports[i].port == port) 12158 return i; 12159 } 12160 12161 return i; 12162 } 12163 12164 /** 12165 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 12166 * @netdev: This physical port's netdev 12167 * @ti: Tunnel endpoint information 12168 **/ 12169 static void i40e_udp_tunnel_add(struct net_device *netdev, 12170 struct udp_tunnel_info *ti) 12171 { 12172 struct i40e_netdev_priv *np = netdev_priv(netdev); 12173 struct i40e_vsi *vsi = np->vsi; 12174 struct i40e_pf *pf = vsi->back; 12175 u16 port = ntohs(ti->port); 12176 u8 next_idx; 12177 u8 idx; 12178 12179 idx = i40e_get_udp_port_idx(pf, port); 12180 12181 /* Check if port already exists */ 12182 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 12183 netdev_info(netdev, "port %d already offloaded\n", port); 12184 return; 12185 } 12186 12187 /* Now check if there is space to add the new port */ 12188 next_idx = i40e_get_udp_port_idx(pf, 0); 12189 12190 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 12191 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 12192 port); 12193 return; 12194 } 12195 12196 switch (ti->type) { 12197 case UDP_TUNNEL_TYPE_VXLAN: 12198 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 12199 break; 12200 case UDP_TUNNEL_TYPE_GENEVE: 12201 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) 12202 return; 12203 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 12204 break; 12205 default: 12206 return; 12207 } 12208 12209 /* New port: add it and mark its index in the bitmap */ 12210 pf->udp_ports[next_idx].port = port; 12211 pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED; 12212 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 12213 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 12214 } 12215 12216 /** 12217 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 12218 * @netdev: This physical port's netdev 12219 * @ti: Tunnel endpoint information 12220 **/ 12221 static void i40e_udp_tunnel_del(struct net_device *netdev, 12222 struct udp_tunnel_info *ti) 12223 { 12224 struct i40e_netdev_priv *np = netdev_priv(netdev); 12225 struct i40e_vsi *vsi = np->vsi; 12226 struct i40e_pf *pf = vsi->back; 12227 u16 port = ntohs(ti->port); 12228 u8 idx; 12229 12230 idx = i40e_get_udp_port_idx(pf, port); 12231 12232 /* Check if port already exists */ 12233 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 12234 goto not_found; 12235 12236 switch (ti->type) { 12237 case UDP_TUNNEL_TYPE_VXLAN: 12238 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 12239 goto not_found; 12240 break; 12241 case UDP_TUNNEL_TYPE_GENEVE: 12242 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 12243 goto not_found; 12244 break; 12245 default: 12246 goto not_found; 12247 } 12248 12249 /* if port exists, set it to 0 (mark for deletion) 12250 * and make it pending 12251 */ 12252 pf->udp_ports[idx].port = 0; 12253 12254 /* Toggle pending bit instead of setting it. This way if we are 12255 * deleting a port that has yet to be added we just clear the pending 12256 * bit and don't have to worry about it. 12257 */ 12258 pf->pending_udp_bitmap ^= BIT_ULL(idx); 12259 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 12260 12261 return; 12262 not_found: 12263 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 12264 port); 12265 } 12266 12267 static int i40e_get_phys_port_id(struct net_device *netdev, 12268 struct netdev_phys_item_id *ppid) 12269 { 12270 struct i40e_netdev_priv *np = netdev_priv(netdev); 12271 struct i40e_pf *pf = np->vsi->back; 12272 struct i40e_hw *hw = &pf->hw; 12273 12274 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 12275 return -EOPNOTSUPP; 12276 12277 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 12278 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 12279 12280 return 0; 12281 } 12282 12283 /** 12284 * i40e_ndo_fdb_add - add an entry to the hardware database 12285 * @ndm: the input from the stack 12286 * @tb: pointer to array of nladdr (unused) 12287 * @dev: the net device pointer 12288 * @addr: the MAC address entry being added 12289 * @vid: VLAN ID 12290 * @flags: instructions from stack about fdb operation 12291 */ 12292 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 12293 struct net_device *dev, 12294 const unsigned char *addr, u16 vid, 12295 u16 flags, 12296 struct netlink_ext_ack *extack) 12297 { 12298 struct i40e_netdev_priv *np = netdev_priv(dev); 12299 struct i40e_pf *pf = np->vsi->back; 12300 int err = 0; 12301 12302 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 12303 return -EOPNOTSUPP; 12304 12305 if (vid) { 12306 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 12307 return -EINVAL; 12308 } 12309 12310 /* Hardware does not support aging addresses so if a 12311 * ndm_state is given only allow permanent addresses 12312 */ 12313 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 12314 netdev_info(dev, "FDB only supports static addresses\n"); 12315 return -EINVAL; 12316 } 12317 12318 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 12319 err = dev_uc_add_excl(dev, addr); 12320 else if (is_multicast_ether_addr(addr)) 12321 err = dev_mc_add_excl(dev, addr); 12322 else 12323 err = -EINVAL; 12324 12325 /* Only return duplicate errors if NLM_F_EXCL is set */ 12326 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 12327 err = 0; 12328 12329 return err; 12330 } 12331 12332 /** 12333 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 12334 * @dev: the netdev being configured 12335 * @nlh: RTNL message 12336 * @flags: bridge flags 12337 * @extack: netlink extended ack 12338 * 12339 * Inserts a new hardware bridge if not already created and 12340 * enables the bridging mode requested (VEB or VEPA). If the 12341 * hardware bridge has already been inserted and the request 12342 * is to change the mode then that requires a PF reset to 12343 * allow rebuild of the components with required hardware 12344 * bridge mode enabled. 12345 * 12346 * Note: expects to be called while under rtnl_lock() 12347 **/ 12348 static int i40e_ndo_bridge_setlink(struct net_device *dev, 12349 struct nlmsghdr *nlh, 12350 u16 flags, 12351 struct netlink_ext_ack *extack) 12352 { 12353 struct i40e_netdev_priv *np = netdev_priv(dev); 12354 struct i40e_vsi *vsi = np->vsi; 12355 struct i40e_pf *pf = vsi->back; 12356 struct i40e_veb *veb = NULL; 12357 struct nlattr *attr, *br_spec; 12358 int i, rem; 12359 12360 /* Only for PF VSI for now */ 12361 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 12362 return -EOPNOTSUPP; 12363 12364 /* Find the HW bridge for PF VSI */ 12365 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12366 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12367 veb = pf->veb[i]; 12368 } 12369 12370 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 12371 12372 nla_for_each_nested(attr, br_spec, rem) { 12373 __u16 mode; 12374 12375 if (nla_type(attr) != IFLA_BRIDGE_MODE) 12376 continue; 12377 12378 mode = nla_get_u16(attr); 12379 if ((mode != BRIDGE_MODE_VEPA) && 12380 (mode != BRIDGE_MODE_VEB)) 12381 return -EINVAL; 12382 12383 /* Insert a new HW bridge */ 12384 if (!veb) { 12385 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 12386 vsi->tc_config.enabled_tc); 12387 if (veb) { 12388 veb->bridge_mode = mode; 12389 i40e_config_bridge_mode(veb); 12390 } else { 12391 /* No Bridge HW offload available */ 12392 return -ENOENT; 12393 } 12394 break; 12395 } else if (mode != veb->bridge_mode) { 12396 /* Existing HW bridge but different mode needs reset */ 12397 veb->bridge_mode = mode; 12398 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 12399 if (mode == BRIDGE_MODE_VEB) 12400 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 12401 else 12402 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 12403 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 12404 break; 12405 } 12406 } 12407 12408 return 0; 12409 } 12410 12411 /** 12412 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 12413 * @skb: skb buff 12414 * @pid: process id 12415 * @seq: RTNL message seq # 12416 * @dev: the netdev being configured 12417 * @filter_mask: unused 12418 * @nlflags: netlink flags passed in 12419 * 12420 * Return the mode in which the hardware bridge is operating in 12421 * i.e VEB or VEPA. 12422 **/ 12423 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 12424 struct net_device *dev, 12425 u32 __always_unused filter_mask, 12426 int nlflags) 12427 { 12428 struct i40e_netdev_priv *np = netdev_priv(dev); 12429 struct i40e_vsi *vsi = np->vsi; 12430 struct i40e_pf *pf = vsi->back; 12431 struct i40e_veb *veb = NULL; 12432 int i; 12433 12434 /* Only for PF VSI for now */ 12435 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 12436 return -EOPNOTSUPP; 12437 12438 /* Find the HW bridge for the PF VSI */ 12439 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12440 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12441 veb = pf->veb[i]; 12442 } 12443 12444 if (!veb) 12445 return 0; 12446 12447 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 12448 0, 0, nlflags, filter_mask, NULL); 12449 } 12450 12451 /** 12452 * i40e_features_check - Validate encapsulated packet conforms to limits 12453 * @skb: skb buff 12454 * @dev: This physical port's netdev 12455 * @features: Offload features that the stack believes apply 12456 **/ 12457 static netdev_features_t i40e_features_check(struct sk_buff *skb, 12458 struct net_device *dev, 12459 netdev_features_t features) 12460 { 12461 size_t len; 12462 12463 /* No point in doing any of this if neither checksum nor GSO are 12464 * being requested for this frame. We can rule out both by just 12465 * checking for CHECKSUM_PARTIAL 12466 */ 12467 if (skb->ip_summed != CHECKSUM_PARTIAL) 12468 return features; 12469 12470 /* We cannot support GSO if the MSS is going to be less than 12471 * 64 bytes. If it is then we need to drop support for GSO. 12472 */ 12473 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 12474 features &= ~NETIF_F_GSO_MASK; 12475 12476 /* MACLEN can support at most 63 words */ 12477 len = skb_network_header(skb) - skb->data; 12478 if (len & ~(63 * 2)) 12479 goto out_err; 12480 12481 /* IPLEN and EIPLEN can support at most 127 dwords */ 12482 len = skb_transport_header(skb) - skb_network_header(skb); 12483 if (len & ~(127 * 4)) 12484 goto out_err; 12485 12486 if (skb->encapsulation) { 12487 /* L4TUNLEN can support 127 words */ 12488 len = skb_inner_network_header(skb) - skb_transport_header(skb); 12489 if (len & ~(127 * 2)) 12490 goto out_err; 12491 12492 /* IPLEN can support at most 127 dwords */ 12493 len = skb_inner_transport_header(skb) - 12494 skb_inner_network_header(skb); 12495 if (len & ~(127 * 4)) 12496 goto out_err; 12497 } 12498 12499 /* No need to validate L4LEN as TCP is the only protocol with a 12500 * a flexible value and we support all possible values supported 12501 * by TCP, which is at most 15 dwords 12502 */ 12503 12504 return features; 12505 out_err: 12506 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 12507 } 12508 12509 /** 12510 * i40e_xdp_setup - add/remove an XDP program 12511 * @vsi: VSI to changed 12512 * @prog: XDP program 12513 **/ 12514 static int i40e_xdp_setup(struct i40e_vsi *vsi, 12515 struct bpf_prog *prog) 12516 { 12517 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 12518 struct i40e_pf *pf = vsi->back; 12519 struct bpf_prog *old_prog; 12520 bool need_reset; 12521 int i; 12522 12523 /* Don't allow frames that span over multiple buffers */ 12524 if (frame_size > vsi->rx_buf_len) 12525 return -EINVAL; 12526 12527 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 12528 return 0; 12529 12530 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 12531 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 12532 12533 if (need_reset) 12534 i40e_prep_for_reset(pf, true); 12535 12536 old_prog = xchg(&vsi->xdp_prog, prog); 12537 12538 if (need_reset) { 12539 if (!prog) 12540 /* Wait until ndo_xsk_wakeup completes. */ 12541 synchronize_rcu(); 12542 i40e_reset_and_rebuild(pf, true, true); 12543 } 12544 12545 for (i = 0; i < vsi->num_queue_pairs; i++) 12546 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 12547 12548 if (old_prog) 12549 bpf_prog_put(old_prog); 12550 12551 /* Kick start the NAPI context if there is an AF_XDP socket open 12552 * on that queue id. This so that receiving will start. 12553 */ 12554 if (need_reset && prog) 12555 for (i = 0; i < vsi->num_queue_pairs; i++) 12556 if (vsi->xdp_rings[i]->xsk_umem) 12557 (void)i40e_xsk_wakeup(vsi->netdev, i, 12558 XDP_WAKEUP_RX); 12559 12560 return 0; 12561 } 12562 12563 /** 12564 * i40e_enter_busy_conf - Enters busy config state 12565 * @vsi: vsi 12566 * 12567 * Returns 0 on success, <0 for failure. 12568 **/ 12569 static int i40e_enter_busy_conf(struct i40e_vsi *vsi) 12570 { 12571 struct i40e_pf *pf = vsi->back; 12572 int timeout = 50; 12573 12574 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) { 12575 timeout--; 12576 if (!timeout) 12577 return -EBUSY; 12578 usleep_range(1000, 2000); 12579 } 12580 12581 return 0; 12582 } 12583 12584 /** 12585 * i40e_exit_busy_conf - Exits busy config state 12586 * @vsi: vsi 12587 **/ 12588 static void i40e_exit_busy_conf(struct i40e_vsi *vsi) 12589 { 12590 struct i40e_pf *pf = vsi->back; 12591 12592 clear_bit(__I40E_CONFIG_BUSY, pf->state); 12593 } 12594 12595 /** 12596 * i40e_queue_pair_reset_stats - Resets all statistics for a queue pair 12597 * @vsi: vsi 12598 * @queue_pair: queue pair 12599 **/ 12600 static void i40e_queue_pair_reset_stats(struct i40e_vsi *vsi, int queue_pair) 12601 { 12602 memset(&vsi->rx_rings[queue_pair]->rx_stats, 0, 12603 sizeof(vsi->rx_rings[queue_pair]->rx_stats)); 12604 memset(&vsi->tx_rings[queue_pair]->stats, 0, 12605 sizeof(vsi->tx_rings[queue_pair]->stats)); 12606 if (i40e_enabled_xdp_vsi(vsi)) { 12607 memset(&vsi->xdp_rings[queue_pair]->stats, 0, 12608 sizeof(vsi->xdp_rings[queue_pair]->stats)); 12609 } 12610 } 12611 12612 /** 12613 * i40e_queue_pair_clean_rings - Cleans all the rings of a queue pair 12614 * @vsi: vsi 12615 * @queue_pair: queue pair 12616 **/ 12617 static void i40e_queue_pair_clean_rings(struct i40e_vsi *vsi, int queue_pair) 12618 { 12619 i40e_clean_tx_ring(vsi->tx_rings[queue_pair]); 12620 if (i40e_enabled_xdp_vsi(vsi)) { 12621 /* Make sure that in-progress ndo_xdp_xmit calls are 12622 * completed. 12623 */ 12624 synchronize_rcu(); 12625 i40e_clean_tx_ring(vsi->xdp_rings[queue_pair]); 12626 } 12627 i40e_clean_rx_ring(vsi->rx_rings[queue_pair]); 12628 } 12629 12630 /** 12631 * i40e_queue_pair_toggle_napi - Enables/disables NAPI for a queue pair 12632 * @vsi: vsi 12633 * @queue_pair: queue pair 12634 * @enable: true for enable, false for disable 12635 **/ 12636 static void i40e_queue_pair_toggle_napi(struct i40e_vsi *vsi, int queue_pair, 12637 bool enable) 12638 { 12639 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12640 struct i40e_q_vector *q_vector = rxr->q_vector; 12641 12642 if (!vsi->netdev) 12643 return; 12644 12645 /* All rings in a qp belong to the same qvector. */ 12646 if (q_vector->rx.ring || q_vector->tx.ring) { 12647 if (enable) 12648 napi_enable(&q_vector->napi); 12649 else 12650 napi_disable(&q_vector->napi); 12651 } 12652 } 12653 12654 /** 12655 * i40e_queue_pair_toggle_rings - Enables/disables all rings for a queue pair 12656 * @vsi: vsi 12657 * @queue_pair: queue pair 12658 * @enable: true for enable, false for disable 12659 * 12660 * Returns 0 on success, <0 on failure. 12661 **/ 12662 static int i40e_queue_pair_toggle_rings(struct i40e_vsi *vsi, int queue_pair, 12663 bool enable) 12664 { 12665 struct i40e_pf *pf = vsi->back; 12666 int pf_q, ret = 0; 12667 12668 pf_q = vsi->base_queue + queue_pair; 12669 ret = i40e_control_wait_tx_q(vsi->seid, pf, pf_q, 12670 false /*is xdp*/, enable); 12671 if (ret) { 12672 dev_info(&pf->pdev->dev, 12673 "VSI seid %d Tx ring %d %sable timeout\n", 12674 vsi->seid, pf_q, (enable ? "en" : "dis")); 12675 return ret; 12676 } 12677 12678 i40e_control_rx_q(pf, pf_q, enable); 12679 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 12680 if (ret) { 12681 dev_info(&pf->pdev->dev, 12682 "VSI seid %d Rx ring %d %sable timeout\n", 12683 vsi->seid, pf_q, (enable ? "en" : "dis")); 12684 return ret; 12685 } 12686 12687 /* Due to HW errata, on Rx disable only, the register can 12688 * indicate done before it really is. Needs 50ms to be sure 12689 */ 12690 if (!enable) 12691 mdelay(50); 12692 12693 if (!i40e_enabled_xdp_vsi(vsi)) 12694 return ret; 12695 12696 ret = i40e_control_wait_tx_q(vsi->seid, pf, 12697 pf_q + vsi->alloc_queue_pairs, 12698 true /*is xdp*/, enable); 12699 if (ret) { 12700 dev_info(&pf->pdev->dev, 12701 "VSI seid %d XDP Tx ring %d %sable timeout\n", 12702 vsi->seid, pf_q, (enable ? "en" : "dis")); 12703 } 12704 12705 return ret; 12706 } 12707 12708 /** 12709 * i40e_queue_pair_enable_irq - Enables interrupts for a queue pair 12710 * @vsi: vsi 12711 * @queue_pair: queue_pair 12712 **/ 12713 static void i40e_queue_pair_enable_irq(struct i40e_vsi *vsi, int queue_pair) 12714 { 12715 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12716 struct i40e_pf *pf = vsi->back; 12717 struct i40e_hw *hw = &pf->hw; 12718 12719 /* All rings in a qp belong to the same qvector. */ 12720 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 12721 i40e_irq_dynamic_enable(vsi, rxr->q_vector->v_idx); 12722 else 12723 i40e_irq_dynamic_enable_icr0(pf); 12724 12725 i40e_flush(hw); 12726 } 12727 12728 /** 12729 * i40e_queue_pair_disable_irq - Disables interrupts for a queue pair 12730 * @vsi: vsi 12731 * @queue_pair: queue_pair 12732 **/ 12733 static void i40e_queue_pair_disable_irq(struct i40e_vsi *vsi, int queue_pair) 12734 { 12735 struct i40e_ring *rxr = vsi->rx_rings[queue_pair]; 12736 struct i40e_pf *pf = vsi->back; 12737 struct i40e_hw *hw = &pf->hw; 12738 12739 /* For simplicity, instead of removing the qp interrupt causes 12740 * from the interrupt linked list, we simply disable the interrupt, and 12741 * leave the list intact. 12742 * 12743 * All rings in a qp belong to the same qvector. 12744 */ 12745 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 12746 u32 intpf = vsi->base_vector + rxr->q_vector->v_idx; 12747 12748 wr32(hw, I40E_PFINT_DYN_CTLN(intpf - 1), 0); 12749 i40e_flush(hw); 12750 synchronize_irq(pf->msix_entries[intpf].vector); 12751 } else { 12752 /* Legacy and MSI mode - this stops all interrupt handling */ 12753 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 12754 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 12755 i40e_flush(hw); 12756 synchronize_irq(pf->pdev->irq); 12757 } 12758 } 12759 12760 /** 12761 * i40e_queue_pair_disable - Disables a queue pair 12762 * @vsi: vsi 12763 * @queue_pair: queue pair 12764 * 12765 * Returns 0 on success, <0 on failure. 12766 **/ 12767 int i40e_queue_pair_disable(struct i40e_vsi *vsi, int queue_pair) 12768 { 12769 int err; 12770 12771 err = i40e_enter_busy_conf(vsi); 12772 if (err) 12773 return err; 12774 12775 i40e_queue_pair_disable_irq(vsi, queue_pair); 12776 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, false /* off */); 12777 i40e_queue_pair_toggle_napi(vsi, queue_pair, false /* off */); 12778 i40e_queue_pair_clean_rings(vsi, queue_pair); 12779 i40e_queue_pair_reset_stats(vsi, queue_pair); 12780 12781 return err; 12782 } 12783 12784 /** 12785 * i40e_queue_pair_enable - Enables a queue pair 12786 * @vsi: vsi 12787 * @queue_pair: queue pair 12788 * 12789 * Returns 0 on success, <0 on failure. 12790 **/ 12791 int i40e_queue_pair_enable(struct i40e_vsi *vsi, int queue_pair) 12792 { 12793 int err; 12794 12795 err = i40e_configure_tx_ring(vsi->tx_rings[queue_pair]); 12796 if (err) 12797 return err; 12798 12799 if (i40e_enabled_xdp_vsi(vsi)) { 12800 err = i40e_configure_tx_ring(vsi->xdp_rings[queue_pair]); 12801 if (err) 12802 return err; 12803 } 12804 12805 err = i40e_configure_rx_ring(vsi->rx_rings[queue_pair]); 12806 if (err) 12807 return err; 12808 12809 err = i40e_queue_pair_toggle_rings(vsi, queue_pair, true /* on */); 12810 i40e_queue_pair_toggle_napi(vsi, queue_pair, true /* on */); 12811 i40e_queue_pair_enable_irq(vsi, queue_pair); 12812 12813 i40e_exit_busy_conf(vsi); 12814 12815 return err; 12816 } 12817 12818 /** 12819 * i40e_xdp - implements ndo_bpf for i40e 12820 * @dev: netdevice 12821 * @xdp: XDP command 12822 **/ 12823 static int i40e_xdp(struct net_device *dev, 12824 struct netdev_bpf *xdp) 12825 { 12826 struct i40e_netdev_priv *np = netdev_priv(dev); 12827 struct i40e_vsi *vsi = np->vsi; 12828 12829 if (vsi->type != I40E_VSI_MAIN) 12830 return -EINVAL; 12831 12832 switch (xdp->command) { 12833 case XDP_SETUP_PROG: 12834 return i40e_xdp_setup(vsi, xdp->prog); 12835 case XDP_QUERY_PROG: 12836 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0; 12837 return 0; 12838 case XDP_SETUP_XSK_UMEM: 12839 return i40e_xsk_umem_setup(vsi, xdp->xsk.umem, 12840 xdp->xsk.queue_id); 12841 default: 12842 return -EINVAL; 12843 } 12844 } 12845 12846 static const struct net_device_ops i40e_netdev_ops = { 12847 .ndo_open = i40e_open, 12848 .ndo_stop = i40e_close, 12849 .ndo_start_xmit = i40e_lan_xmit_frame, 12850 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 12851 .ndo_set_rx_mode = i40e_set_rx_mode, 12852 .ndo_validate_addr = eth_validate_addr, 12853 .ndo_set_mac_address = i40e_set_mac, 12854 .ndo_change_mtu = i40e_change_mtu, 12855 .ndo_do_ioctl = i40e_ioctl, 12856 .ndo_tx_timeout = i40e_tx_timeout, 12857 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 12858 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 12859 #ifdef CONFIG_NET_POLL_CONTROLLER 12860 .ndo_poll_controller = i40e_netpoll, 12861 #endif 12862 .ndo_setup_tc = __i40e_setup_tc, 12863 .ndo_set_features = i40e_set_features, 12864 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 12865 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 12866 .ndo_get_vf_stats = i40e_get_vf_stats, 12867 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 12868 .ndo_get_vf_config = i40e_ndo_get_vf_config, 12869 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 12870 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 12871 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 12872 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 12873 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 12874 .ndo_get_phys_port_id = i40e_get_phys_port_id, 12875 .ndo_fdb_add = i40e_ndo_fdb_add, 12876 .ndo_features_check = i40e_features_check, 12877 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 12878 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 12879 .ndo_bpf = i40e_xdp, 12880 .ndo_xdp_xmit = i40e_xdp_xmit, 12881 .ndo_xsk_wakeup = i40e_xsk_wakeup, 12882 .ndo_dfwd_add_station = i40e_fwd_add, 12883 .ndo_dfwd_del_station = i40e_fwd_del, 12884 }; 12885 12886 /** 12887 * i40e_config_netdev - Setup the netdev flags 12888 * @vsi: the VSI being configured 12889 * 12890 * Returns 0 on success, negative value on failure 12891 **/ 12892 static int i40e_config_netdev(struct i40e_vsi *vsi) 12893 { 12894 struct i40e_pf *pf = vsi->back; 12895 struct i40e_hw *hw = &pf->hw; 12896 struct i40e_netdev_priv *np; 12897 struct net_device *netdev; 12898 u8 broadcast[ETH_ALEN]; 12899 u8 mac_addr[ETH_ALEN]; 12900 int etherdev_size; 12901 netdev_features_t hw_enc_features; 12902 netdev_features_t hw_features; 12903 12904 etherdev_size = sizeof(struct i40e_netdev_priv); 12905 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 12906 if (!netdev) 12907 return -ENOMEM; 12908 12909 vsi->netdev = netdev; 12910 np = netdev_priv(netdev); 12911 np->vsi = vsi; 12912 12913 hw_enc_features = NETIF_F_SG | 12914 NETIF_F_IP_CSUM | 12915 NETIF_F_IPV6_CSUM | 12916 NETIF_F_HIGHDMA | 12917 NETIF_F_SOFT_FEATURES | 12918 NETIF_F_TSO | 12919 NETIF_F_TSO_ECN | 12920 NETIF_F_TSO6 | 12921 NETIF_F_GSO_GRE | 12922 NETIF_F_GSO_GRE_CSUM | 12923 NETIF_F_GSO_PARTIAL | 12924 NETIF_F_GSO_IPXIP4 | 12925 NETIF_F_GSO_IPXIP6 | 12926 NETIF_F_GSO_UDP_TUNNEL | 12927 NETIF_F_GSO_UDP_TUNNEL_CSUM | 12928 NETIF_F_GSO_UDP_L4 | 12929 NETIF_F_SCTP_CRC | 12930 NETIF_F_RXHASH | 12931 NETIF_F_RXCSUM | 12932 0; 12933 12934 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 12935 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 12936 12937 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 12938 12939 netdev->hw_enc_features |= hw_enc_features; 12940 12941 /* record features VLANs can make use of */ 12942 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 12943 12944 /* enable macvlan offloads */ 12945 netdev->hw_features |= NETIF_F_HW_L2FW_DOFFLOAD; 12946 12947 hw_features = hw_enc_features | 12948 NETIF_F_HW_VLAN_CTAG_TX | 12949 NETIF_F_HW_VLAN_CTAG_RX; 12950 12951 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 12952 hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 12953 12954 netdev->hw_features |= hw_features; 12955 12956 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 12957 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 12958 12959 if (vsi->type == I40E_VSI_MAIN) { 12960 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 12961 ether_addr_copy(mac_addr, hw->mac.perm_addr); 12962 /* The following steps are necessary for two reasons. First, 12963 * some older NVM configurations load a default MAC-VLAN 12964 * filter that will accept any tagged packet, and we want to 12965 * replace this with a normal filter. Additionally, it is 12966 * possible our MAC address was provided by the platform using 12967 * Open Firmware or similar. 12968 * 12969 * Thus, we need to remove the default filter and install one 12970 * specific to the MAC address. 12971 */ 12972 i40e_rm_default_mac_filter(vsi, mac_addr); 12973 spin_lock_bh(&vsi->mac_filter_hash_lock); 12974 i40e_add_mac_filter(vsi, mac_addr); 12975 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12976 } else { 12977 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 12978 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 12979 * the end, which is 4 bytes long, so force truncation of the 12980 * original name by IFNAMSIZ - 4 12981 */ 12982 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 12983 IFNAMSIZ - 4, 12984 pf->vsi[pf->lan_vsi]->netdev->name); 12985 eth_random_addr(mac_addr); 12986 12987 spin_lock_bh(&vsi->mac_filter_hash_lock); 12988 i40e_add_mac_filter(vsi, mac_addr); 12989 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12990 } 12991 12992 /* Add the broadcast filter so that we initially will receive 12993 * broadcast packets. Note that when a new VLAN is first added the 12994 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 12995 * specific filters as part of transitioning into "vlan" operation. 12996 * When more VLANs are added, the driver will copy each existing MAC 12997 * filter and add it for the new VLAN. 12998 * 12999 * Broadcast filters are handled specially by 13000 * i40e_sync_filters_subtask, as the driver must to set the broadcast 13001 * promiscuous bit instead of adding this directly as a MAC/VLAN 13002 * filter. The subtask will update the correct broadcast promiscuous 13003 * bits as VLANs become active or inactive. 13004 */ 13005 eth_broadcast_addr(broadcast); 13006 spin_lock_bh(&vsi->mac_filter_hash_lock); 13007 i40e_add_mac_filter(vsi, broadcast); 13008 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13009 13010 ether_addr_copy(netdev->dev_addr, mac_addr); 13011 ether_addr_copy(netdev->perm_addr, mac_addr); 13012 13013 /* i40iw_net_event() reads 16 bytes from neigh->primary_key */ 13014 netdev->neigh_priv_len = sizeof(u32) * 4; 13015 13016 netdev->priv_flags |= IFF_UNICAST_FLT; 13017 netdev->priv_flags |= IFF_SUPP_NOFCS; 13018 /* Setup netdev TC information */ 13019 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 13020 13021 netdev->netdev_ops = &i40e_netdev_ops; 13022 netdev->watchdog_timeo = 5 * HZ; 13023 i40e_set_ethtool_ops(netdev); 13024 13025 /* MTU range: 68 - 9706 */ 13026 netdev->min_mtu = ETH_MIN_MTU; 13027 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 13028 13029 return 0; 13030 } 13031 13032 /** 13033 * i40e_vsi_delete - Delete a VSI from the switch 13034 * @vsi: the VSI being removed 13035 * 13036 * Returns 0 on success, negative value on failure 13037 **/ 13038 static void i40e_vsi_delete(struct i40e_vsi *vsi) 13039 { 13040 /* remove default VSI is not allowed */ 13041 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 13042 return; 13043 13044 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 13045 } 13046 13047 /** 13048 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 13049 * @vsi: the VSI being queried 13050 * 13051 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 13052 **/ 13053 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 13054 { 13055 struct i40e_veb *veb; 13056 struct i40e_pf *pf = vsi->back; 13057 13058 /* Uplink is not a bridge so default to VEB */ 13059 if (vsi->veb_idx >= I40E_MAX_VEB) 13060 return 1; 13061 13062 veb = pf->veb[vsi->veb_idx]; 13063 if (!veb) { 13064 dev_info(&pf->pdev->dev, 13065 "There is no veb associated with the bridge\n"); 13066 return -ENOENT; 13067 } 13068 13069 /* Uplink is a bridge in VEPA mode */ 13070 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 13071 return 0; 13072 } else { 13073 /* Uplink is a bridge in VEB mode */ 13074 return 1; 13075 } 13076 13077 /* VEPA is now default bridge, so return 0 */ 13078 return 0; 13079 } 13080 13081 /** 13082 * i40e_add_vsi - Add a VSI to the switch 13083 * @vsi: the VSI being configured 13084 * 13085 * This initializes a VSI context depending on the VSI type to be added and 13086 * passes it down to the add_vsi aq command. 13087 **/ 13088 static int i40e_add_vsi(struct i40e_vsi *vsi) 13089 { 13090 int ret = -ENODEV; 13091 struct i40e_pf *pf = vsi->back; 13092 struct i40e_hw *hw = &pf->hw; 13093 struct i40e_vsi_context ctxt; 13094 struct i40e_mac_filter *f; 13095 struct hlist_node *h; 13096 int bkt; 13097 13098 u8 enabled_tc = 0x1; /* TC0 enabled */ 13099 int f_count = 0; 13100 13101 memset(&ctxt, 0, sizeof(ctxt)); 13102 switch (vsi->type) { 13103 case I40E_VSI_MAIN: 13104 /* The PF's main VSI is already setup as part of the 13105 * device initialization, so we'll not bother with 13106 * the add_vsi call, but we will retrieve the current 13107 * VSI context. 13108 */ 13109 ctxt.seid = pf->main_vsi_seid; 13110 ctxt.pf_num = pf->hw.pf_id; 13111 ctxt.vf_num = 0; 13112 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 13113 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13114 if (ret) { 13115 dev_info(&pf->pdev->dev, 13116 "couldn't get PF vsi config, err %s aq_err %s\n", 13117 i40e_stat_str(&pf->hw, ret), 13118 i40e_aq_str(&pf->hw, 13119 pf->hw.aq.asq_last_status)); 13120 return -ENOENT; 13121 } 13122 vsi->info = ctxt.info; 13123 vsi->info.valid_sections = 0; 13124 13125 vsi->seid = ctxt.seid; 13126 vsi->id = ctxt.vsi_number; 13127 13128 enabled_tc = i40e_pf_get_tc_map(pf); 13129 13130 /* Source pruning is enabled by default, so the flag is 13131 * negative logic - if it's set, we need to fiddle with 13132 * the VSI to disable source pruning. 13133 */ 13134 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 13135 memset(&ctxt, 0, sizeof(ctxt)); 13136 ctxt.seid = pf->main_vsi_seid; 13137 ctxt.pf_num = pf->hw.pf_id; 13138 ctxt.vf_num = 0; 13139 ctxt.info.valid_sections |= 13140 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13141 ctxt.info.switch_id = 13142 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 13143 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13144 if (ret) { 13145 dev_info(&pf->pdev->dev, 13146 "update vsi failed, err %s aq_err %s\n", 13147 i40e_stat_str(&pf->hw, ret), 13148 i40e_aq_str(&pf->hw, 13149 pf->hw.aq.asq_last_status)); 13150 ret = -ENOENT; 13151 goto err; 13152 } 13153 } 13154 13155 /* MFP mode setup queue map and update VSI */ 13156 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 13157 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 13158 memset(&ctxt, 0, sizeof(ctxt)); 13159 ctxt.seid = pf->main_vsi_seid; 13160 ctxt.pf_num = pf->hw.pf_id; 13161 ctxt.vf_num = 0; 13162 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 13163 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 13164 if (ret) { 13165 dev_info(&pf->pdev->dev, 13166 "update vsi failed, err %s aq_err %s\n", 13167 i40e_stat_str(&pf->hw, ret), 13168 i40e_aq_str(&pf->hw, 13169 pf->hw.aq.asq_last_status)); 13170 ret = -ENOENT; 13171 goto err; 13172 } 13173 /* update the local VSI info queue map */ 13174 i40e_vsi_update_queue_map(vsi, &ctxt); 13175 vsi->info.valid_sections = 0; 13176 } else { 13177 /* Default/Main VSI is only enabled for TC0 13178 * reconfigure it to enable all TCs that are 13179 * available on the port in SFP mode. 13180 * For MFP case the iSCSI PF would use this 13181 * flow to enable LAN+iSCSI TC. 13182 */ 13183 ret = i40e_vsi_config_tc(vsi, enabled_tc); 13184 if (ret) { 13185 /* Single TC condition is not fatal, 13186 * message and continue 13187 */ 13188 dev_info(&pf->pdev->dev, 13189 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 13190 enabled_tc, 13191 i40e_stat_str(&pf->hw, ret), 13192 i40e_aq_str(&pf->hw, 13193 pf->hw.aq.asq_last_status)); 13194 } 13195 } 13196 break; 13197 13198 case I40E_VSI_FDIR: 13199 ctxt.pf_num = hw->pf_id; 13200 ctxt.vf_num = 0; 13201 ctxt.uplink_seid = vsi->uplink_seid; 13202 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13203 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 13204 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 13205 (i40e_is_vsi_uplink_mode_veb(vsi))) { 13206 ctxt.info.valid_sections |= 13207 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13208 ctxt.info.switch_id = 13209 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13210 } 13211 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13212 break; 13213 13214 case I40E_VSI_VMDQ2: 13215 ctxt.pf_num = hw->pf_id; 13216 ctxt.vf_num = 0; 13217 ctxt.uplink_seid = vsi->uplink_seid; 13218 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13219 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 13220 13221 /* This VSI is connected to VEB so the switch_id 13222 * should be set to zero by default. 13223 */ 13224 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13225 ctxt.info.valid_sections |= 13226 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13227 ctxt.info.switch_id = 13228 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13229 } 13230 13231 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13232 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13233 break; 13234 13235 case I40E_VSI_SRIOV: 13236 ctxt.pf_num = hw->pf_id; 13237 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 13238 ctxt.uplink_seid = vsi->uplink_seid; 13239 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 13240 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 13241 13242 /* This VSI is connected to VEB so the switch_id 13243 * should be set to zero by default. 13244 */ 13245 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 13246 ctxt.info.valid_sections |= 13247 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 13248 ctxt.info.switch_id = 13249 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 13250 } 13251 13252 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 13253 ctxt.info.valid_sections |= 13254 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 13255 ctxt.info.queueing_opt_flags |= 13256 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 13257 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 13258 } 13259 13260 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 13261 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 13262 if (pf->vf[vsi->vf_id].spoofchk) { 13263 ctxt.info.valid_sections |= 13264 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 13265 ctxt.info.sec_flags |= 13266 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 13267 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 13268 } 13269 /* Setup the VSI tx/rx queue map for TC0 only for now */ 13270 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 13271 break; 13272 13273 case I40E_VSI_IWARP: 13274 /* send down message to iWARP */ 13275 break; 13276 13277 default: 13278 return -ENODEV; 13279 } 13280 13281 if (vsi->type != I40E_VSI_MAIN) { 13282 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 13283 if (ret) { 13284 dev_info(&vsi->back->pdev->dev, 13285 "add vsi failed, err %s aq_err %s\n", 13286 i40e_stat_str(&pf->hw, ret), 13287 i40e_aq_str(&pf->hw, 13288 pf->hw.aq.asq_last_status)); 13289 ret = -ENOENT; 13290 goto err; 13291 } 13292 vsi->info = ctxt.info; 13293 vsi->info.valid_sections = 0; 13294 vsi->seid = ctxt.seid; 13295 vsi->id = ctxt.vsi_number; 13296 } 13297 13298 vsi->active_filters = 0; 13299 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 13300 spin_lock_bh(&vsi->mac_filter_hash_lock); 13301 /* If macvlan filters already exist, force them to get loaded */ 13302 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 13303 f->state = I40E_FILTER_NEW; 13304 f_count++; 13305 } 13306 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13307 13308 if (f_count) { 13309 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 13310 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 13311 } 13312 13313 /* Update VSI BW information */ 13314 ret = i40e_vsi_get_bw_info(vsi); 13315 if (ret) { 13316 dev_info(&pf->pdev->dev, 13317 "couldn't get vsi bw info, err %s aq_err %s\n", 13318 i40e_stat_str(&pf->hw, ret), 13319 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13320 /* VSI is already added so not tearing that up */ 13321 ret = 0; 13322 } 13323 13324 err: 13325 return ret; 13326 } 13327 13328 /** 13329 * i40e_vsi_release - Delete a VSI and free its resources 13330 * @vsi: the VSI being removed 13331 * 13332 * Returns 0 on success or < 0 on error 13333 **/ 13334 int i40e_vsi_release(struct i40e_vsi *vsi) 13335 { 13336 struct i40e_mac_filter *f; 13337 struct hlist_node *h; 13338 struct i40e_veb *veb = NULL; 13339 struct i40e_pf *pf; 13340 u16 uplink_seid; 13341 int i, n, bkt; 13342 13343 pf = vsi->back; 13344 13345 /* release of a VEB-owner or last VSI is not allowed */ 13346 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 13347 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 13348 vsi->seid, vsi->uplink_seid); 13349 return -ENODEV; 13350 } 13351 if (vsi == pf->vsi[pf->lan_vsi] && 13352 !test_bit(__I40E_DOWN, pf->state)) { 13353 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 13354 return -ENODEV; 13355 } 13356 13357 uplink_seid = vsi->uplink_seid; 13358 if (vsi->type != I40E_VSI_SRIOV) { 13359 if (vsi->netdev_registered) { 13360 vsi->netdev_registered = false; 13361 if (vsi->netdev) { 13362 /* results in a call to i40e_close() */ 13363 unregister_netdev(vsi->netdev); 13364 } 13365 } else { 13366 i40e_vsi_close(vsi); 13367 } 13368 i40e_vsi_disable_irq(vsi); 13369 } 13370 13371 spin_lock_bh(&vsi->mac_filter_hash_lock); 13372 13373 /* clear the sync flag on all filters */ 13374 if (vsi->netdev) { 13375 __dev_uc_unsync(vsi->netdev, NULL); 13376 __dev_mc_unsync(vsi->netdev, NULL); 13377 } 13378 13379 /* make sure any remaining filters are marked for deletion */ 13380 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 13381 __i40e_del_filter(vsi, f); 13382 13383 spin_unlock_bh(&vsi->mac_filter_hash_lock); 13384 13385 i40e_sync_vsi_filters(vsi); 13386 13387 i40e_vsi_delete(vsi); 13388 i40e_vsi_free_q_vectors(vsi); 13389 if (vsi->netdev) { 13390 free_netdev(vsi->netdev); 13391 vsi->netdev = NULL; 13392 } 13393 i40e_vsi_clear_rings(vsi); 13394 i40e_vsi_clear(vsi); 13395 13396 /* If this was the last thing on the VEB, except for the 13397 * controlling VSI, remove the VEB, which puts the controlling 13398 * VSI onto the next level down in the switch. 13399 * 13400 * Well, okay, there's one more exception here: don't remove 13401 * the orphan VEBs yet. We'll wait for an explicit remove request 13402 * from up the network stack. 13403 */ 13404 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 13405 if (pf->vsi[i] && 13406 pf->vsi[i]->uplink_seid == uplink_seid && 13407 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13408 n++; /* count the VSIs */ 13409 } 13410 } 13411 for (i = 0; i < I40E_MAX_VEB; i++) { 13412 if (!pf->veb[i]) 13413 continue; 13414 if (pf->veb[i]->uplink_seid == uplink_seid) 13415 n++; /* count the VEBs */ 13416 if (pf->veb[i]->seid == uplink_seid) 13417 veb = pf->veb[i]; 13418 } 13419 if (n == 0 && veb && veb->uplink_seid != 0) 13420 i40e_veb_release(veb); 13421 13422 return 0; 13423 } 13424 13425 /** 13426 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 13427 * @vsi: ptr to the VSI 13428 * 13429 * This should only be called after i40e_vsi_mem_alloc() which allocates the 13430 * corresponding SW VSI structure and initializes num_queue_pairs for the 13431 * newly allocated VSI. 13432 * 13433 * Returns 0 on success or negative on failure 13434 **/ 13435 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 13436 { 13437 int ret = -ENOENT; 13438 struct i40e_pf *pf = vsi->back; 13439 13440 if (vsi->q_vectors[0]) { 13441 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 13442 vsi->seid); 13443 return -EEXIST; 13444 } 13445 13446 if (vsi->base_vector) { 13447 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 13448 vsi->seid, vsi->base_vector); 13449 return -EEXIST; 13450 } 13451 13452 ret = i40e_vsi_alloc_q_vectors(vsi); 13453 if (ret) { 13454 dev_info(&pf->pdev->dev, 13455 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 13456 vsi->num_q_vectors, vsi->seid, ret); 13457 vsi->num_q_vectors = 0; 13458 goto vector_setup_out; 13459 } 13460 13461 /* In Legacy mode, we do not have to get any other vector since we 13462 * piggyback on the misc/ICR0 for queue interrupts. 13463 */ 13464 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 13465 return ret; 13466 if (vsi->num_q_vectors) 13467 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 13468 vsi->num_q_vectors, vsi->idx); 13469 if (vsi->base_vector < 0) { 13470 dev_info(&pf->pdev->dev, 13471 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 13472 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 13473 i40e_vsi_free_q_vectors(vsi); 13474 ret = -ENOENT; 13475 goto vector_setup_out; 13476 } 13477 13478 vector_setup_out: 13479 return ret; 13480 } 13481 13482 /** 13483 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 13484 * @vsi: pointer to the vsi. 13485 * 13486 * This re-allocates a vsi's queue resources. 13487 * 13488 * Returns pointer to the successfully allocated and configured VSI sw struct 13489 * on success, otherwise returns NULL on failure. 13490 **/ 13491 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 13492 { 13493 u16 alloc_queue_pairs; 13494 struct i40e_pf *pf; 13495 u8 enabled_tc; 13496 int ret; 13497 13498 if (!vsi) 13499 return NULL; 13500 13501 pf = vsi->back; 13502 13503 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 13504 i40e_vsi_clear_rings(vsi); 13505 13506 i40e_vsi_free_arrays(vsi, false); 13507 i40e_set_num_rings_in_vsi(vsi); 13508 ret = i40e_vsi_alloc_arrays(vsi, false); 13509 if (ret) 13510 goto err_vsi; 13511 13512 alloc_queue_pairs = vsi->alloc_queue_pairs * 13513 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 13514 13515 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 13516 if (ret < 0) { 13517 dev_info(&pf->pdev->dev, 13518 "failed to get tracking for %d queues for VSI %d err %d\n", 13519 alloc_queue_pairs, vsi->seid, ret); 13520 goto err_vsi; 13521 } 13522 vsi->base_queue = ret; 13523 13524 /* Update the FW view of the VSI. Force a reset of TC and queue 13525 * layout configurations. 13526 */ 13527 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 13528 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 13529 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 13530 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 13531 if (vsi->type == I40E_VSI_MAIN) 13532 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 13533 13534 /* assign it some queues */ 13535 ret = i40e_alloc_rings(vsi); 13536 if (ret) 13537 goto err_rings; 13538 13539 /* map all of the rings to the q_vectors */ 13540 i40e_vsi_map_rings_to_vectors(vsi); 13541 return vsi; 13542 13543 err_rings: 13544 i40e_vsi_free_q_vectors(vsi); 13545 if (vsi->netdev_registered) { 13546 vsi->netdev_registered = false; 13547 unregister_netdev(vsi->netdev); 13548 free_netdev(vsi->netdev); 13549 vsi->netdev = NULL; 13550 } 13551 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13552 err_vsi: 13553 i40e_vsi_clear(vsi); 13554 return NULL; 13555 } 13556 13557 /** 13558 * i40e_vsi_setup - Set up a VSI by a given type 13559 * @pf: board private structure 13560 * @type: VSI type 13561 * @uplink_seid: the switch element to link to 13562 * @param1: usage depends upon VSI type. For VF types, indicates VF id 13563 * 13564 * This allocates the sw VSI structure and its queue resources, then add a VSI 13565 * to the identified VEB. 13566 * 13567 * Returns pointer to the successfully allocated and configure VSI sw struct on 13568 * success, otherwise returns NULL on failure. 13569 **/ 13570 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 13571 u16 uplink_seid, u32 param1) 13572 { 13573 struct i40e_vsi *vsi = NULL; 13574 struct i40e_veb *veb = NULL; 13575 u16 alloc_queue_pairs; 13576 int ret, i; 13577 int v_idx; 13578 13579 /* The requested uplink_seid must be either 13580 * - the PF's port seid 13581 * no VEB is needed because this is the PF 13582 * or this is a Flow Director special case VSI 13583 * - seid of an existing VEB 13584 * - seid of a VSI that owns an existing VEB 13585 * - seid of a VSI that doesn't own a VEB 13586 * a new VEB is created and the VSI becomes the owner 13587 * - seid of the PF VSI, which is what creates the first VEB 13588 * this is a special case of the previous 13589 * 13590 * Find which uplink_seid we were given and create a new VEB if needed 13591 */ 13592 for (i = 0; i < I40E_MAX_VEB; i++) { 13593 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 13594 veb = pf->veb[i]; 13595 break; 13596 } 13597 } 13598 13599 if (!veb && uplink_seid != pf->mac_seid) { 13600 13601 for (i = 0; i < pf->num_alloc_vsi; i++) { 13602 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 13603 vsi = pf->vsi[i]; 13604 break; 13605 } 13606 } 13607 if (!vsi) { 13608 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 13609 uplink_seid); 13610 return NULL; 13611 } 13612 13613 if (vsi->uplink_seid == pf->mac_seid) 13614 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 13615 vsi->tc_config.enabled_tc); 13616 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 13617 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 13618 vsi->tc_config.enabled_tc); 13619 if (veb) { 13620 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 13621 dev_info(&vsi->back->pdev->dev, 13622 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 13623 return NULL; 13624 } 13625 /* We come up by default in VEPA mode if SRIOV is not 13626 * already enabled, in which case we can't force VEPA 13627 * mode. 13628 */ 13629 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 13630 veb->bridge_mode = BRIDGE_MODE_VEPA; 13631 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 13632 } 13633 i40e_config_bridge_mode(veb); 13634 } 13635 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 13636 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 13637 veb = pf->veb[i]; 13638 } 13639 if (!veb) { 13640 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 13641 return NULL; 13642 } 13643 13644 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 13645 uplink_seid = veb->seid; 13646 } 13647 13648 /* get vsi sw struct */ 13649 v_idx = i40e_vsi_mem_alloc(pf, type); 13650 if (v_idx < 0) 13651 goto err_alloc; 13652 vsi = pf->vsi[v_idx]; 13653 if (!vsi) 13654 goto err_alloc; 13655 vsi->type = type; 13656 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 13657 13658 if (type == I40E_VSI_MAIN) 13659 pf->lan_vsi = v_idx; 13660 else if (type == I40E_VSI_SRIOV) 13661 vsi->vf_id = param1; 13662 /* assign it some queues */ 13663 alloc_queue_pairs = vsi->alloc_queue_pairs * 13664 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 13665 13666 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 13667 if (ret < 0) { 13668 dev_info(&pf->pdev->dev, 13669 "failed to get tracking for %d queues for VSI %d err=%d\n", 13670 alloc_queue_pairs, vsi->seid, ret); 13671 goto err_vsi; 13672 } 13673 vsi->base_queue = ret; 13674 13675 /* get a VSI from the hardware */ 13676 vsi->uplink_seid = uplink_seid; 13677 ret = i40e_add_vsi(vsi); 13678 if (ret) 13679 goto err_vsi; 13680 13681 switch (vsi->type) { 13682 /* setup the netdev if needed */ 13683 case I40E_VSI_MAIN: 13684 case I40E_VSI_VMDQ2: 13685 ret = i40e_config_netdev(vsi); 13686 if (ret) 13687 goto err_netdev; 13688 ret = register_netdev(vsi->netdev); 13689 if (ret) 13690 goto err_netdev; 13691 vsi->netdev_registered = true; 13692 netif_carrier_off(vsi->netdev); 13693 #ifdef CONFIG_I40E_DCB 13694 /* Setup DCB netlink interface */ 13695 i40e_dcbnl_setup(vsi); 13696 #endif /* CONFIG_I40E_DCB */ 13697 /* fall through */ 13698 13699 case I40E_VSI_FDIR: 13700 /* set up vectors and rings if needed */ 13701 ret = i40e_vsi_setup_vectors(vsi); 13702 if (ret) 13703 goto err_msix; 13704 13705 ret = i40e_alloc_rings(vsi); 13706 if (ret) 13707 goto err_rings; 13708 13709 /* map all of the rings to the q_vectors */ 13710 i40e_vsi_map_rings_to_vectors(vsi); 13711 13712 i40e_vsi_reset_stats(vsi); 13713 break; 13714 13715 default: 13716 /* no netdev or rings for the other VSI types */ 13717 break; 13718 } 13719 13720 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 13721 (vsi->type == I40E_VSI_VMDQ2)) { 13722 ret = i40e_vsi_config_rss(vsi); 13723 } 13724 return vsi; 13725 13726 err_rings: 13727 i40e_vsi_free_q_vectors(vsi); 13728 err_msix: 13729 if (vsi->netdev_registered) { 13730 vsi->netdev_registered = false; 13731 unregister_netdev(vsi->netdev); 13732 free_netdev(vsi->netdev); 13733 vsi->netdev = NULL; 13734 } 13735 err_netdev: 13736 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 13737 err_vsi: 13738 i40e_vsi_clear(vsi); 13739 err_alloc: 13740 return NULL; 13741 } 13742 13743 /** 13744 * i40e_veb_get_bw_info - Query VEB BW information 13745 * @veb: the veb to query 13746 * 13747 * Query the Tx scheduler BW configuration data for given VEB 13748 **/ 13749 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 13750 { 13751 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 13752 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 13753 struct i40e_pf *pf = veb->pf; 13754 struct i40e_hw *hw = &pf->hw; 13755 u32 tc_bw_max; 13756 int ret = 0; 13757 int i; 13758 13759 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 13760 &bw_data, NULL); 13761 if (ret) { 13762 dev_info(&pf->pdev->dev, 13763 "query veb bw config failed, err %s aq_err %s\n", 13764 i40e_stat_str(&pf->hw, ret), 13765 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13766 goto out; 13767 } 13768 13769 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 13770 &ets_data, NULL); 13771 if (ret) { 13772 dev_info(&pf->pdev->dev, 13773 "query veb bw ets config failed, err %s aq_err %s\n", 13774 i40e_stat_str(&pf->hw, ret), 13775 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 13776 goto out; 13777 } 13778 13779 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 13780 veb->bw_max_quanta = ets_data.tc_bw_max; 13781 veb->is_abs_credits = bw_data.absolute_credits_enable; 13782 veb->enabled_tc = ets_data.tc_valid_bits; 13783 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 13784 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 13785 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 13786 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 13787 veb->bw_tc_limit_credits[i] = 13788 le16_to_cpu(bw_data.tc_bw_limits[i]); 13789 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 13790 } 13791 13792 out: 13793 return ret; 13794 } 13795 13796 /** 13797 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 13798 * @pf: board private structure 13799 * 13800 * On error: returns error code (negative) 13801 * On success: returns vsi index in PF (positive) 13802 **/ 13803 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 13804 { 13805 int ret = -ENOENT; 13806 struct i40e_veb *veb; 13807 int i; 13808 13809 /* Need to protect the allocation of switch elements at the PF level */ 13810 mutex_lock(&pf->switch_mutex); 13811 13812 /* VEB list may be fragmented if VEB creation/destruction has 13813 * been happening. We can afford to do a quick scan to look 13814 * for any free slots in the list. 13815 * 13816 * find next empty veb slot, looping back around if necessary 13817 */ 13818 i = 0; 13819 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 13820 i++; 13821 if (i >= I40E_MAX_VEB) { 13822 ret = -ENOMEM; 13823 goto err_alloc_veb; /* out of VEB slots! */ 13824 } 13825 13826 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 13827 if (!veb) { 13828 ret = -ENOMEM; 13829 goto err_alloc_veb; 13830 } 13831 veb->pf = pf; 13832 veb->idx = i; 13833 veb->enabled_tc = 1; 13834 13835 pf->veb[i] = veb; 13836 ret = i; 13837 err_alloc_veb: 13838 mutex_unlock(&pf->switch_mutex); 13839 return ret; 13840 } 13841 13842 /** 13843 * i40e_switch_branch_release - Delete a branch of the switch tree 13844 * @branch: where to start deleting 13845 * 13846 * This uses recursion to find the tips of the branch to be 13847 * removed, deleting until we get back to and can delete this VEB. 13848 **/ 13849 static void i40e_switch_branch_release(struct i40e_veb *branch) 13850 { 13851 struct i40e_pf *pf = branch->pf; 13852 u16 branch_seid = branch->seid; 13853 u16 veb_idx = branch->idx; 13854 int i; 13855 13856 /* release any VEBs on this VEB - RECURSION */ 13857 for (i = 0; i < I40E_MAX_VEB; i++) { 13858 if (!pf->veb[i]) 13859 continue; 13860 if (pf->veb[i]->uplink_seid == branch->seid) 13861 i40e_switch_branch_release(pf->veb[i]); 13862 } 13863 13864 /* Release the VSIs on this VEB, but not the owner VSI. 13865 * 13866 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 13867 * the VEB itself, so don't use (*branch) after this loop. 13868 */ 13869 for (i = 0; i < pf->num_alloc_vsi; i++) { 13870 if (!pf->vsi[i]) 13871 continue; 13872 if (pf->vsi[i]->uplink_seid == branch_seid && 13873 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 13874 i40e_vsi_release(pf->vsi[i]); 13875 } 13876 } 13877 13878 /* There's one corner case where the VEB might not have been 13879 * removed, so double check it here and remove it if needed. 13880 * This case happens if the veb was created from the debugfs 13881 * commands and no VSIs were added to it. 13882 */ 13883 if (pf->veb[veb_idx]) 13884 i40e_veb_release(pf->veb[veb_idx]); 13885 } 13886 13887 /** 13888 * i40e_veb_clear - remove veb struct 13889 * @veb: the veb to remove 13890 **/ 13891 static void i40e_veb_clear(struct i40e_veb *veb) 13892 { 13893 if (!veb) 13894 return; 13895 13896 if (veb->pf) { 13897 struct i40e_pf *pf = veb->pf; 13898 13899 mutex_lock(&pf->switch_mutex); 13900 if (pf->veb[veb->idx] == veb) 13901 pf->veb[veb->idx] = NULL; 13902 mutex_unlock(&pf->switch_mutex); 13903 } 13904 13905 kfree(veb); 13906 } 13907 13908 /** 13909 * i40e_veb_release - Delete a VEB and free its resources 13910 * @veb: the VEB being removed 13911 **/ 13912 void i40e_veb_release(struct i40e_veb *veb) 13913 { 13914 struct i40e_vsi *vsi = NULL; 13915 struct i40e_pf *pf; 13916 int i, n = 0; 13917 13918 pf = veb->pf; 13919 13920 /* find the remaining VSI and check for extras */ 13921 for (i = 0; i < pf->num_alloc_vsi; i++) { 13922 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 13923 n++; 13924 vsi = pf->vsi[i]; 13925 } 13926 } 13927 if (n != 1) { 13928 dev_info(&pf->pdev->dev, 13929 "can't remove VEB %d with %d VSIs left\n", 13930 veb->seid, n); 13931 return; 13932 } 13933 13934 /* move the remaining VSI to uplink veb */ 13935 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 13936 if (veb->uplink_seid) { 13937 vsi->uplink_seid = veb->uplink_seid; 13938 if (veb->uplink_seid == pf->mac_seid) 13939 vsi->veb_idx = I40E_NO_VEB; 13940 else 13941 vsi->veb_idx = veb->veb_idx; 13942 } else { 13943 /* floating VEB */ 13944 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 13945 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 13946 } 13947 13948 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 13949 i40e_veb_clear(veb); 13950 } 13951 13952 /** 13953 * i40e_add_veb - create the VEB in the switch 13954 * @veb: the VEB to be instantiated 13955 * @vsi: the controlling VSI 13956 **/ 13957 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 13958 { 13959 struct i40e_pf *pf = veb->pf; 13960 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 13961 int ret; 13962 13963 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 13964 veb->enabled_tc, false, 13965 &veb->seid, enable_stats, NULL); 13966 13967 /* get a VEB from the hardware */ 13968 if (ret) { 13969 dev_info(&pf->pdev->dev, 13970 "couldn't add VEB, err %s aq_err %s\n", 13971 i40e_stat_str(&pf->hw, ret), 13972 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13973 return -EPERM; 13974 } 13975 13976 /* get statistics counter */ 13977 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 13978 &veb->stats_idx, NULL, NULL, NULL); 13979 if (ret) { 13980 dev_info(&pf->pdev->dev, 13981 "couldn't get VEB statistics idx, err %s aq_err %s\n", 13982 i40e_stat_str(&pf->hw, ret), 13983 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13984 return -EPERM; 13985 } 13986 ret = i40e_veb_get_bw_info(veb); 13987 if (ret) { 13988 dev_info(&pf->pdev->dev, 13989 "couldn't get VEB bw info, err %s aq_err %s\n", 13990 i40e_stat_str(&pf->hw, ret), 13991 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13992 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 13993 return -ENOENT; 13994 } 13995 13996 vsi->uplink_seid = veb->seid; 13997 vsi->veb_idx = veb->idx; 13998 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 13999 14000 return 0; 14001 } 14002 14003 /** 14004 * i40e_veb_setup - Set up a VEB 14005 * @pf: board private structure 14006 * @flags: VEB setup flags 14007 * @uplink_seid: the switch element to link to 14008 * @vsi_seid: the initial VSI seid 14009 * @enabled_tc: Enabled TC bit-map 14010 * 14011 * This allocates the sw VEB structure and links it into the switch 14012 * It is possible and legal for this to be a duplicate of an already 14013 * existing VEB. It is also possible for both uplink and vsi seids 14014 * to be zero, in order to create a floating VEB. 14015 * 14016 * Returns pointer to the successfully allocated VEB sw struct on 14017 * success, otherwise returns NULL on failure. 14018 **/ 14019 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 14020 u16 uplink_seid, u16 vsi_seid, 14021 u8 enabled_tc) 14022 { 14023 struct i40e_veb *veb, *uplink_veb = NULL; 14024 int vsi_idx, veb_idx; 14025 int ret; 14026 14027 /* if one seid is 0, the other must be 0 to create a floating relay */ 14028 if ((uplink_seid == 0 || vsi_seid == 0) && 14029 (uplink_seid + vsi_seid != 0)) { 14030 dev_info(&pf->pdev->dev, 14031 "one, not both seid's are 0: uplink=%d vsi=%d\n", 14032 uplink_seid, vsi_seid); 14033 return NULL; 14034 } 14035 14036 /* make sure there is such a vsi and uplink */ 14037 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 14038 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 14039 break; 14040 if (vsi_idx == pf->num_alloc_vsi && vsi_seid != 0) { 14041 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 14042 vsi_seid); 14043 return NULL; 14044 } 14045 14046 if (uplink_seid && uplink_seid != pf->mac_seid) { 14047 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 14048 if (pf->veb[veb_idx] && 14049 pf->veb[veb_idx]->seid == uplink_seid) { 14050 uplink_veb = pf->veb[veb_idx]; 14051 break; 14052 } 14053 } 14054 if (!uplink_veb) { 14055 dev_info(&pf->pdev->dev, 14056 "uplink seid %d not found\n", uplink_seid); 14057 return NULL; 14058 } 14059 } 14060 14061 /* get veb sw struct */ 14062 veb_idx = i40e_veb_mem_alloc(pf); 14063 if (veb_idx < 0) 14064 goto err_alloc; 14065 veb = pf->veb[veb_idx]; 14066 veb->flags = flags; 14067 veb->uplink_seid = uplink_seid; 14068 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 14069 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 14070 14071 /* create the VEB in the switch */ 14072 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 14073 if (ret) 14074 goto err_veb; 14075 if (vsi_idx == pf->lan_vsi) 14076 pf->lan_veb = veb->idx; 14077 14078 return veb; 14079 14080 err_veb: 14081 i40e_veb_clear(veb); 14082 err_alloc: 14083 return NULL; 14084 } 14085 14086 /** 14087 * i40e_setup_pf_switch_element - set PF vars based on switch type 14088 * @pf: board private structure 14089 * @ele: element we are building info from 14090 * @num_reported: total number of elements 14091 * @printconfig: should we print the contents 14092 * 14093 * helper function to assist in extracting a few useful SEID values. 14094 **/ 14095 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 14096 struct i40e_aqc_switch_config_element_resp *ele, 14097 u16 num_reported, bool printconfig) 14098 { 14099 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 14100 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 14101 u8 element_type = ele->element_type; 14102 u16 seid = le16_to_cpu(ele->seid); 14103 14104 if (printconfig) 14105 dev_info(&pf->pdev->dev, 14106 "type=%d seid=%d uplink=%d downlink=%d\n", 14107 element_type, seid, uplink_seid, downlink_seid); 14108 14109 switch (element_type) { 14110 case I40E_SWITCH_ELEMENT_TYPE_MAC: 14111 pf->mac_seid = seid; 14112 break; 14113 case I40E_SWITCH_ELEMENT_TYPE_VEB: 14114 /* Main VEB? */ 14115 if (uplink_seid != pf->mac_seid) 14116 break; 14117 if (pf->lan_veb >= I40E_MAX_VEB) { 14118 int v; 14119 14120 /* find existing or else empty VEB */ 14121 for (v = 0; v < I40E_MAX_VEB; v++) { 14122 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 14123 pf->lan_veb = v; 14124 break; 14125 } 14126 } 14127 if (pf->lan_veb >= I40E_MAX_VEB) { 14128 v = i40e_veb_mem_alloc(pf); 14129 if (v < 0) 14130 break; 14131 pf->lan_veb = v; 14132 } 14133 } 14134 if (pf->lan_veb >= I40E_MAX_VEB) 14135 break; 14136 14137 pf->veb[pf->lan_veb]->seid = seid; 14138 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 14139 pf->veb[pf->lan_veb]->pf = pf; 14140 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 14141 break; 14142 case I40E_SWITCH_ELEMENT_TYPE_VSI: 14143 if (num_reported != 1) 14144 break; 14145 /* This is immediately after a reset so we can assume this is 14146 * the PF's VSI 14147 */ 14148 pf->mac_seid = uplink_seid; 14149 pf->pf_seid = downlink_seid; 14150 pf->main_vsi_seid = seid; 14151 if (printconfig) 14152 dev_info(&pf->pdev->dev, 14153 "pf_seid=%d main_vsi_seid=%d\n", 14154 pf->pf_seid, pf->main_vsi_seid); 14155 break; 14156 case I40E_SWITCH_ELEMENT_TYPE_PF: 14157 case I40E_SWITCH_ELEMENT_TYPE_VF: 14158 case I40E_SWITCH_ELEMENT_TYPE_EMP: 14159 case I40E_SWITCH_ELEMENT_TYPE_BMC: 14160 case I40E_SWITCH_ELEMENT_TYPE_PE: 14161 case I40E_SWITCH_ELEMENT_TYPE_PA: 14162 /* ignore these for now */ 14163 break; 14164 default: 14165 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 14166 element_type, seid); 14167 break; 14168 } 14169 } 14170 14171 /** 14172 * i40e_fetch_switch_configuration - Get switch config from firmware 14173 * @pf: board private structure 14174 * @printconfig: should we print the contents 14175 * 14176 * Get the current switch configuration from the device and 14177 * extract a few useful SEID values. 14178 **/ 14179 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 14180 { 14181 struct i40e_aqc_get_switch_config_resp *sw_config; 14182 u16 next_seid = 0; 14183 int ret = 0; 14184 u8 *aq_buf; 14185 int i; 14186 14187 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 14188 if (!aq_buf) 14189 return -ENOMEM; 14190 14191 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 14192 do { 14193 u16 num_reported, num_total; 14194 14195 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 14196 I40E_AQ_LARGE_BUF, 14197 &next_seid, NULL); 14198 if (ret) { 14199 dev_info(&pf->pdev->dev, 14200 "get switch config failed err %s aq_err %s\n", 14201 i40e_stat_str(&pf->hw, ret), 14202 i40e_aq_str(&pf->hw, 14203 pf->hw.aq.asq_last_status)); 14204 kfree(aq_buf); 14205 return -ENOENT; 14206 } 14207 14208 num_reported = le16_to_cpu(sw_config->header.num_reported); 14209 num_total = le16_to_cpu(sw_config->header.num_total); 14210 14211 if (printconfig) 14212 dev_info(&pf->pdev->dev, 14213 "header: %d reported %d total\n", 14214 num_reported, num_total); 14215 14216 for (i = 0; i < num_reported; i++) { 14217 struct i40e_aqc_switch_config_element_resp *ele = 14218 &sw_config->element[i]; 14219 14220 i40e_setup_pf_switch_element(pf, ele, num_reported, 14221 printconfig); 14222 } 14223 } while (next_seid != 0); 14224 14225 kfree(aq_buf); 14226 return ret; 14227 } 14228 14229 /** 14230 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 14231 * @pf: board private structure 14232 * @reinit: if the Main VSI needs to re-initialized. 14233 * 14234 * Returns 0 on success, negative value on failure 14235 **/ 14236 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 14237 { 14238 u16 flags = 0; 14239 int ret; 14240 14241 /* find out what's out there already */ 14242 ret = i40e_fetch_switch_configuration(pf, false); 14243 if (ret) { 14244 dev_info(&pf->pdev->dev, 14245 "couldn't fetch switch config, err %s aq_err %s\n", 14246 i40e_stat_str(&pf->hw, ret), 14247 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14248 return ret; 14249 } 14250 i40e_pf_reset_stats(pf); 14251 14252 /* set the switch config bit for the whole device to 14253 * support limited promisc or true promisc 14254 * when user requests promisc. The default is limited 14255 * promisc. 14256 */ 14257 14258 if ((pf->hw.pf_id == 0) && 14259 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 14260 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14261 pf->last_sw_conf_flags = flags; 14262 } 14263 14264 if (pf->hw.pf_id == 0) { 14265 u16 valid_flags; 14266 14267 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 14268 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 14269 NULL); 14270 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 14271 dev_info(&pf->pdev->dev, 14272 "couldn't set switch config bits, err %s aq_err %s\n", 14273 i40e_stat_str(&pf->hw, ret), 14274 i40e_aq_str(&pf->hw, 14275 pf->hw.aq.asq_last_status)); 14276 /* not a fatal problem, just keep going */ 14277 } 14278 pf->last_sw_conf_valid_flags = valid_flags; 14279 } 14280 14281 /* first time setup */ 14282 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 14283 struct i40e_vsi *vsi = NULL; 14284 u16 uplink_seid; 14285 14286 /* Set up the PF VSI associated with the PF's main VSI 14287 * that is already in the HW switch 14288 */ 14289 if (pf->lan_veb < I40E_MAX_VEB && pf->veb[pf->lan_veb]) 14290 uplink_seid = pf->veb[pf->lan_veb]->seid; 14291 else 14292 uplink_seid = pf->mac_seid; 14293 if (pf->lan_vsi == I40E_NO_VSI) 14294 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 14295 else if (reinit) 14296 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 14297 if (!vsi) { 14298 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 14299 i40e_cloud_filter_exit(pf); 14300 i40e_fdir_teardown(pf); 14301 return -EAGAIN; 14302 } 14303 } else { 14304 /* force a reset of TC and queue layout configurations */ 14305 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 14306 14307 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 14308 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 14309 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 14310 } 14311 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 14312 14313 i40e_fdir_sb_setup(pf); 14314 14315 /* Setup static PF queue filter control settings */ 14316 ret = i40e_setup_pf_filter_control(pf); 14317 if (ret) { 14318 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 14319 ret); 14320 /* Failure here should not stop continuing other steps */ 14321 } 14322 14323 /* enable RSS in the HW, even for only one queue, as the stack can use 14324 * the hash 14325 */ 14326 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 14327 i40e_pf_config_rss(pf); 14328 14329 /* fill in link information and enable LSE reporting */ 14330 i40e_link_event(pf); 14331 14332 /* Initialize user-specific link properties */ 14333 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 14334 I40E_AQ_AN_COMPLETED) ? true : false); 14335 14336 i40e_ptp_init(pf); 14337 14338 /* repopulate tunnel port filters */ 14339 i40e_sync_udp_filters(pf); 14340 14341 return ret; 14342 } 14343 14344 /** 14345 * i40e_determine_queue_usage - Work out queue distribution 14346 * @pf: board private structure 14347 **/ 14348 static void i40e_determine_queue_usage(struct i40e_pf *pf) 14349 { 14350 int queues_left; 14351 int q_max; 14352 14353 pf->num_lan_qps = 0; 14354 14355 /* Find the max queues to be put into basic use. We'll always be 14356 * using TC0, whether or not DCB is running, and TC0 will get the 14357 * big RSS set. 14358 */ 14359 queues_left = pf->hw.func_caps.num_tx_qp; 14360 14361 if ((queues_left == 1) || 14362 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 14363 /* one qp for PF, no queues for anything else */ 14364 queues_left = 0; 14365 pf->alloc_rss_size = pf->num_lan_qps = 1; 14366 14367 /* make sure all the fancies are disabled */ 14368 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 14369 I40E_FLAG_IWARP_ENABLED | 14370 I40E_FLAG_FD_SB_ENABLED | 14371 I40E_FLAG_FD_ATR_ENABLED | 14372 I40E_FLAG_DCB_CAPABLE | 14373 I40E_FLAG_DCB_ENABLED | 14374 I40E_FLAG_SRIOV_ENABLED | 14375 I40E_FLAG_VMDQ_ENABLED); 14376 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14377 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 14378 I40E_FLAG_FD_SB_ENABLED | 14379 I40E_FLAG_FD_ATR_ENABLED | 14380 I40E_FLAG_DCB_CAPABLE))) { 14381 /* one qp for PF */ 14382 pf->alloc_rss_size = pf->num_lan_qps = 1; 14383 queues_left -= pf->num_lan_qps; 14384 14385 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 14386 I40E_FLAG_IWARP_ENABLED | 14387 I40E_FLAG_FD_SB_ENABLED | 14388 I40E_FLAG_FD_ATR_ENABLED | 14389 I40E_FLAG_DCB_ENABLED | 14390 I40E_FLAG_VMDQ_ENABLED); 14391 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14392 } else { 14393 /* Not enough queues for all TCs */ 14394 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 14395 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 14396 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 14397 I40E_FLAG_DCB_ENABLED); 14398 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 14399 } 14400 14401 /* limit lan qps to the smaller of qps, cpus or msix */ 14402 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 14403 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 14404 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 14405 pf->num_lan_qps = q_max; 14406 14407 queues_left -= pf->num_lan_qps; 14408 } 14409 14410 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 14411 if (queues_left > 1) { 14412 queues_left -= 1; /* save 1 queue for FD */ 14413 } else { 14414 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 14415 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 14416 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 14417 } 14418 } 14419 14420 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 14421 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 14422 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 14423 (queues_left / pf->num_vf_qps)); 14424 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 14425 } 14426 14427 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 14428 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 14429 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 14430 (queues_left / pf->num_vmdq_qps)); 14431 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 14432 } 14433 14434 pf->queues_left = queues_left; 14435 dev_dbg(&pf->pdev->dev, 14436 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 14437 pf->hw.func_caps.num_tx_qp, 14438 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 14439 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 14440 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 14441 queues_left); 14442 } 14443 14444 /** 14445 * i40e_setup_pf_filter_control - Setup PF static filter control 14446 * @pf: PF to be setup 14447 * 14448 * i40e_setup_pf_filter_control sets up a PF's initial filter control 14449 * settings. If PE/FCoE are enabled then it will also set the per PF 14450 * based filter sizes required for them. It also enables Flow director, 14451 * ethertype and macvlan type filter settings for the pf. 14452 * 14453 * Returns 0 on success, negative on failure 14454 **/ 14455 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 14456 { 14457 struct i40e_filter_control_settings *settings = &pf->filter_settings; 14458 14459 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 14460 14461 /* Flow Director is enabled */ 14462 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 14463 settings->enable_fdir = true; 14464 14465 /* Ethtype and MACVLAN filters enabled for PF */ 14466 settings->enable_ethtype = true; 14467 settings->enable_macvlan = true; 14468 14469 if (i40e_set_filter_control(&pf->hw, settings)) 14470 return -ENOENT; 14471 14472 return 0; 14473 } 14474 14475 #define INFO_STRING_LEN 255 14476 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 14477 static void i40e_print_features(struct i40e_pf *pf) 14478 { 14479 struct i40e_hw *hw = &pf->hw; 14480 char *buf; 14481 int i; 14482 14483 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 14484 if (!buf) 14485 return; 14486 14487 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 14488 #ifdef CONFIG_PCI_IOV 14489 i += scnprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 14490 #endif 14491 i += scnprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 14492 pf->hw.func_caps.num_vsis, 14493 pf->vsi[pf->lan_vsi]->num_queue_pairs); 14494 if (pf->flags & I40E_FLAG_RSS_ENABLED) 14495 i += scnprintf(&buf[i], REMAIN(i), " RSS"); 14496 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 14497 i += scnprintf(&buf[i], REMAIN(i), " FD_ATR"); 14498 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 14499 i += scnprintf(&buf[i], REMAIN(i), " FD_SB"); 14500 i += scnprintf(&buf[i], REMAIN(i), " NTUPLE"); 14501 } 14502 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 14503 i += scnprintf(&buf[i], REMAIN(i), " DCB"); 14504 i += scnprintf(&buf[i], REMAIN(i), " VxLAN"); 14505 i += scnprintf(&buf[i], REMAIN(i), " Geneve"); 14506 if (pf->flags & I40E_FLAG_PTP) 14507 i += scnprintf(&buf[i], REMAIN(i), " PTP"); 14508 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 14509 i += scnprintf(&buf[i], REMAIN(i), " VEB"); 14510 else 14511 i += scnprintf(&buf[i], REMAIN(i), " VEPA"); 14512 14513 dev_info(&pf->pdev->dev, "%s\n", buf); 14514 kfree(buf); 14515 WARN_ON(i > INFO_STRING_LEN); 14516 } 14517 14518 /** 14519 * i40e_get_platform_mac_addr - get platform-specific MAC address 14520 * @pdev: PCI device information struct 14521 * @pf: board private structure 14522 * 14523 * Look up the MAC address for the device. First we'll try 14524 * eth_platform_get_mac_address, which will check Open Firmware, or arch 14525 * specific fallback. Otherwise, we'll default to the stored value in 14526 * firmware. 14527 **/ 14528 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 14529 { 14530 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 14531 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 14532 } 14533 14534 /** 14535 * i40e_set_fec_in_flags - helper function for setting FEC options in flags 14536 * @fec_cfg: FEC option to set in flags 14537 * @flags: ptr to flags in which we set FEC option 14538 **/ 14539 void i40e_set_fec_in_flags(u8 fec_cfg, u32 *flags) 14540 { 14541 if (fec_cfg & I40E_AQ_SET_FEC_AUTO) 14542 *flags |= I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC; 14543 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_RS) || 14544 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_RS)) { 14545 *flags |= I40E_FLAG_RS_FEC; 14546 *flags &= ~I40E_FLAG_BASE_R_FEC; 14547 } 14548 if ((fec_cfg & I40E_AQ_SET_FEC_REQUEST_KR) || 14549 (fec_cfg & I40E_AQ_SET_FEC_ABILITY_KR)) { 14550 *flags |= I40E_FLAG_BASE_R_FEC; 14551 *flags &= ~I40E_FLAG_RS_FEC; 14552 } 14553 if (fec_cfg == 0) 14554 *flags &= ~(I40E_FLAG_RS_FEC | I40E_FLAG_BASE_R_FEC); 14555 } 14556 14557 /** 14558 * i40e_check_recovery_mode - check if we are running transition firmware 14559 * @pf: board private structure 14560 * 14561 * Check registers indicating the firmware runs in recovery mode. Sets the 14562 * appropriate driver state. 14563 * 14564 * Returns true if the recovery mode was detected, false otherwise 14565 **/ 14566 static bool i40e_check_recovery_mode(struct i40e_pf *pf) 14567 { 14568 u32 val = rd32(&pf->hw, I40E_GL_FWSTS) & I40E_GL_FWSTS_FWS1B_MASK; 14569 bool is_recovery_mode = false; 14570 14571 if (pf->hw.mac.type == I40E_MAC_XL710) 14572 is_recovery_mode = 14573 val == I40E_XL710_GL_FWSTS_FWS1B_REC_MOD_CORER_MASK || 14574 val == I40E_XL710_GL_FWSTS_FWS1B_REC_MOD_GLOBR_MASK || 14575 val == I40E_XL710_GL_FWSTS_FWS1B_REC_MOD_TRANSITION_MASK || 14576 val == I40E_XL710_GL_FWSTS_FWS1B_REC_MOD_NVM_MASK; 14577 if (pf->hw.mac.type == I40E_MAC_X722) 14578 is_recovery_mode = 14579 val == I40E_X722_GL_FWSTS_FWS1B_REC_MOD_CORER_MASK || 14580 val == I40E_X722_GL_FWSTS_FWS1B_REC_MOD_GLOBR_MASK; 14581 if (is_recovery_mode) { 14582 dev_notice(&pf->pdev->dev, "Firmware recovery mode detected. Limiting functionality.\n"); 14583 dev_notice(&pf->pdev->dev, "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for details on firmware recovery mode.\n"); 14584 set_bit(__I40E_RECOVERY_MODE, pf->state); 14585 14586 return true; 14587 } 14588 if (test_and_clear_bit(__I40E_RECOVERY_MODE, pf->state)) 14589 dev_info(&pf->pdev->dev, "Reinitializing in normal mode with full functionality.\n"); 14590 14591 return false; 14592 } 14593 14594 /** 14595 * i40e_pf_loop_reset - perform reset in a loop. 14596 * @pf: board private structure 14597 * 14598 * This function is useful when a NIC is about to enter recovery mode. 14599 * When a NIC's internal data structures are corrupted the NIC's 14600 * firmware is going to enter recovery mode. 14601 * Right after a POR it takes about 7 minutes for firmware to enter 14602 * recovery mode. Until that time a NIC is in some kind of intermediate 14603 * state. After that time period the NIC almost surely enters 14604 * recovery mode. The only way for a driver to detect intermediate 14605 * state is to issue a series of pf-resets and check a return value. 14606 * If a PF reset returns success then the firmware could be in recovery 14607 * mode so the caller of this code needs to check for recovery mode 14608 * if this function returns success. There is a little chance that 14609 * firmware will hang in intermediate state forever. 14610 * Since waiting 7 minutes is quite a lot of time this function waits 14611 * 10 seconds and then gives up by returning an error. 14612 * 14613 * Return 0 on success, negative on failure. 14614 **/ 14615 static i40e_status i40e_pf_loop_reset(struct i40e_pf *pf) 14616 { 14617 const unsigned short MAX_CNT = 1000; 14618 const unsigned short MSECS = 10; 14619 struct i40e_hw *hw = &pf->hw; 14620 i40e_status ret; 14621 int cnt; 14622 14623 for (cnt = 0; cnt < MAX_CNT; ++cnt) { 14624 ret = i40e_pf_reset(hw); 14625 if (!ret) 14626 break; 14627 msleep(MSECS); 14628 } 14629 14630 if (cnt == MAX_CNT) { 14631 dev_info(&pf->pdev->dev, "PF reset failed: %d\n", ret); 14632 return ret; 14633 } 14634 14635 pf->pfr_count++; 14636 return ret; 14637 } 14638 14639 /** 14640 * i40e_init_recovery_mode - initialize subsystems needed in recovery mode 14641 * @pf: board private structure 14642 * @hw: ptr to the hardware info 14643 * 14644 * This function does a minimal setup of all subsystems needed for running 14645 * recovery mode. 14646 * 14647 * Returns 0 on success, negative on failure 14648 **/ 14649 static int i40e_init_recovery_mode(struct i40e_pf *pf, struct i40e_hw *hw) 14650 { 14651 struct i40e_vsi *vsi; 14652 int err; 14653 int v_idx; 14654 14655 pci_save_state(pf->pdev); 14656 14657 /* set up periodic task facility */ 14658 timer_setup(&pf->service_timer, i40e_service_timer, 0); 14659 pf->service_timer_period = HZ; 14660 14661 INIT_WORK(&pf->service_task, i40e_service_task); 14662 clear_bit(__I40E_SERVICE_SCHED, pf->state); 14663 14664 err = i40e_init_interrupt_scheme(pf); 14665 if (err) 14666 goto err_switch_setup; 14667 14668 /* The number of VSIs reported by the FW is the minimum guaranteed 14669 * to us; HW supports far more and we share the remaining pool with 14670 * the other PFs. We allocate space for more than the guarantee with 14671 * the understanding that we might not get them all later. 14672 */ 14673 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 14674 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 14675 else 14676 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 14677 14678 /* Set up the vsi struct and our local tracking of the MAIN PF vsi. */ 14679 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 14680 GFP_KERNEL); 14681 if (!pf->vsi) { 14682 err = -ENOMEM; 14683 goto err_switch_setup; 14684 } 14685 14686 /* We allocate one VSI which is needed as absolute minimum 14687 * in order to register the netdev 14688 */ 14689 v_idx = i40e_vsi_mem_alloc(pf, I40E_VSI_MAIN); 14690 if (v_idx < 0) 14691 goto err_switch_setup; 14692 pf->lan_vsi = v_idx; 14693 vsi = pf->vsi[v_idx]; 14694 if (!vsi) 14695 goto err_switch_setup; 14696 vsi->alloc_queue_pairs = 1; 14697 err = i40e_config_netdev(vsi); 14698 if (err) 14699 goto err_switch_setup; 14700 err = register_netdev(vsi->netdev); 14701 if (err) 14702 goto err_switch_setup; 14703 vsi->netdev_registered = true; 14704 i40e_dbg_pf_init(pf); 14705 14706 err = i40e_setup_misc_vector_for_recovery_mode(pf); 14707 if (err) 14708 goto err_switch_setup; 14709 14710 /* tell the firmware that we're starting */ 14711 i40e_send_version(pf); 14712 14713 /* since everything's happy, start the service_task timer */ 14714 mod_timer(&pf->service_timer, 14715 round_jiffies(jiffies + pf->service_timer_period)); 14716 14717 return 0; 14718 14719 err_switch_setup: 14720 i40e_reset_interrupt_capability(pf); 14721 del_timer_sync(&pf->service_timer); 14722 i40e_shutdown_adminq(hw); 14723 iounmap(hw->hw_addr); 14724 pci_disable_pcie_error_reporting(pf->pdev); 14725 pci_release_mem_regions(pf->pdev); 14726 pci_disable_device(pf->pdev); 14727 kfree(pf); 14728 14729 return err; 14730 } 14731 14732 /** 14733 * i40e_probe - Device initialization routine 14734 * @pdev: PCI device information struct 14735 * @ent: entry in i40e_pci_tbl 14736 * 14737 * i40e_probe initializes a PF identified by a pci_dev structure. 14738 * The OS initialization, configuring of the PF private structure, 14739 * and a hardware reset occur. 14740 * 14741 * Returns 0 on success, negative on failure 14742 **/ 14743 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 14744 { 14745 struct i40e_aq_get_phy_abilities_resp abilities; 14746 struct i40e_pf *pf; 14747 struct i40e_hw *hw; 14748 static u16 pfs_found; 14749 u16 wol_nvm_bits; 14750 u16 link_status; 14751 int err; 14752 u32 val; 14753 u32 i; 14754 u8 set_fc_aq_fail; 14755 14756 err = pci_enable_device_mem(pdev); 14757 if (err) 14758 return err; 14759 14760 /* set up for high or low dma */ 14761 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 14762 if (err) { 14763 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 14764 if (err) { 14765 dev_err(&pdev->dev, 14766 "DMA configuration failed: 0x%x\n", err); 14767 goto err_dma; 14768 } 14769 } 14770 14771 /* set up pci connections */ 14772 err = pci_request_mem_regions(pdev, i40e_driver_name); 14773 if (err) { 14774 dev_info(&pdev->dev, 14775 "pci_request_selected_regions failed %d\n", err); 14776 goto err_pci_reg; 14777 } 14778 14779 pci_enable_pcie_error_reporting(pdev); 14780 pci_set_master(pdev); 14781 14782 /* Now that we have a PCI connection, we need to do the 14783 * low level device setup. This is primarily setting up 14784 * the Admin Queue structures and then querying for the 14785 * device's current profile information. 14786 */ 14787 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 14788 if (!pf) { 14789 err = -ENOMEM; 14790 goto err_pf_alloc; 14791 } 14792 pf->next_vsi = 0; 14793 pf->pdev = pdev; 14794 set_bit(__I40E_DOWN, pf->state); 14795 14796 hw = &pf->hw; 14797 hw->back = pf; 14798 14799 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 14800 I40E_MAX_CSR_SPACE); 14801 /* We believe that the highest register to read is 14802 * I40E_GLGEN_STAT_CLEAR, so we check if the BAR size 14803 * is not less than that before mapping to prevent a 14804 * kernel panic. 14805 */ 14806 if (pf->ioremap_len < I40E_GLGEN_STAT_CLEAR) { 14807 dev_err(&pdev->dev, "Cannot map registers, bar size 0x%X too small, aborting\n", 14808 pf->ioremap_len); 14809 err = -ENOMEM; 14810 goto err_ioremap; 14811 } 14812 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 14813 if (!hw->hw_addr) { 14814 err = -EIO; 14815 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 14816 (unsigned int)pci_resource_start(pdev, 0), 14817 pf->ioremap_len, err); 14818 goto err_ioremap; 14819 } 14820 hw->vendor_id = pdev->vendor; 14821 hw->device_id = pdev->device; 14822 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 14823 hw->subsystem_vendor_id = pdev->subsystem_vendor; 14824 hw->subsystem_device_id = pdev->subsystem_device; 14825 hw->bus.device = PCI_SLOT(pdev->devfn); 14826 hw->bus.func = PCI_FUNC(pdev->devfn); 14827 hw->bus.bus_id = pdev->bus->number; 14828 pf->instance = pfs_found; 14829 14830 /* Select something other than the 802.1ad ethertype for the 14831 * switch to use internally and drop on ingress. 14832 */ 14833 hw->switch_tag = 0xffff; 14834 hw->first_tag = ETH_P_8021AD; 14835 hw->second_tag = ETH_P_8021Q; 14836 14837 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 14838 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 14839 INIT_LIST_HEAD(&pf->ddp_old_prof); 14840 14841 /* set up the locks for the AQ, do this only once in probe 14842 * and destroy them only once in remove 14843 */ 14844 mutex_init(&hw->aq.asq_mutex); 14845 mutex_init(&hw->aq.arq_mutex); 14846 14847 pf->msg_enable = netif_msg_init(debug, 14848 NETIF_MSG_DRV | 14849 NETIF_MSG_PROBE | 14850 NETIF_MSG_LINK); 14851 if (debug < -1) 14852 pf->hw.debug_mask = debug; 14853 14854 /* do a special CORER for clearing PXE mode once at init */ 14855 if (hw->revision_id == 0 && 14856 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 14857 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 14858 i40e_flush(hw); 14859 msleep(200); 14860 pf->corer_count++; 14861 14862 i40e_clear_pxe_mode(hw); 14863 } 14864 14865 /* Reset here to make sure all is clean and to define PF 'n' */ 14866 i40e_clear_hw(hw); 14867 14868 err = i40e_set_mac_type(hw); 14869 if (err) { 14870 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 14871 err); 14872 goto err_pf_reset; 14873 } 14874 14875 err = i40e_pf_loop_reset(pf); 14876 if (err) { 14877 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 14878 goto err_pf_reset; 14879 } 14880 14881 i40e_check_recovery_mode(pf); 14882 14883 hw->aq.num_arq_entries = I40E_AQ_LEN; 14884 hw->aq.num_asq_entries = I40E_AQ_LEN; 14885 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 14886 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 14887 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 14888 14889 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 14890 "%s-%s:misc", 14891 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 14892 14893 err = i40e_init_shared_code(hw); 14894 if (err) { 14895 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 14896 err); 14897 goto err_pf_reset; 14898 } 14899 14900 /* set up a default setting for link flow control */ 14901 pf->hw.fc.requested_mode = I40E_FC_NONE; 14902 14903 err = i40e_init_adminq(hw); 14904 if (err) { 14905 if (err == I40E_ERR_FIRMWARE_API_VERSION) 14906 dev_info(&pdev->dev, 14907 "The driver for the device stopped because the NVM image v%u.%u is newer than expected v%u.%u. You must install the most recent version of the network driver.\n", 14908 hw->aq.api_maj_ver, 14909 hw->aq.api_min_ver, 14910 I40E_FW_API_VERSION_MAJOR, 14911 I40E_FW_MINOR_VERSION(hw)); 14912 else 14913 dev_info(&pdev->dev, 14914 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 14915 14916 goto err_pf_reset; 14917 } 14918 i40e_get_oem_version(hw); 14919 14920 /* provide nvm, fw, api versions, vendor:device id, subsys vendor:device id */ 14921 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s [%04x:%04x] [%04x:%04x]\n", 14922 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 14923 hw->aq.api_maj_ver, hw->aq.api_min_ver, 14924 i40e_nvm_version_str(hw), hw->vendor_id, hw->device_id, 14925 hw->subsystem_vendor_id, hw->subsystem_device_id); 14926 14927 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 14928 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 14929 dev_info(&pdev->dev, 14930 "The driver for the device detected a newer version of the NVM image v%u.%u than expected v%u.%u. Please install the most recent version of the network driver.\n", 14931 hw->aq.api_maj_ver, 14932 hw->aq.api_min_ver, 14933 I40E_FW_API_VERSION_MAJOR, 14934 I40E_FW_MINOR_VERSION(hw)); 14935 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 14936 dev_info(&pdev->dev, 14937 "The driver for the device detected an older version of the NVM image v%u.%u than expected v%u.%u. Please update the NVM image.\n", 14938 hw->aq.api_maj_ver, 14939 hw->aq.api_min_ver, 14940 I40E_FW_API_VERSION_MAJOR, 14941 I40E_FW_MINOR_VERSION(hw)); 14942 14943 i40e_verify_eeprom(pf); 14944 14945 /* Rev 0 hardware was never productized */ 14946 if (hw->revision_id < 1) 14947 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 14948 14949 i40e_clear_pxe_mode(hw); 14950 14951 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 14952 if (err) 14953 goto err_adminq_setup; 14954 14955 err = i40e_sw_init(pf); 14956 if (err) { 14957 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 14958 goto err_sw_init; 14959 } 14960 14961 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) 14962 return i40e_init_recovery_mode(pf, hw); 14963 14964 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 14965 hw->func_caps.num_rx_qp, 0, 0); 14966 if (err) { 14967 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 14968 goto err_init_lan_hmc; 14969 } 14970 14971 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 14972 if (err) { 14973 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 14974 err = -ENOENT; 14975 goto err_configure_lan_hmc; 14976 } 14977 14978 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 14979 * Ignore error return codes because if it was already disabled via 14980 * hardware settings this will fail 14981 */ 14982 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 14983 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 14984 i40e_aq_stop_lldp(hw, true, false, NULL); 14985 } 14986 14987 /* allow a platform config to override the HW addr */ 14988 i40e_get_platform_mac_addr(pdev, pf); 14989 14990 if (!is_valid_ether_addr(hw->mac.addr)) { 14991 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 14992 err = -EIO; 14993 goto err_mac_addr; 14994 } 14995 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 14996 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 14997 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 14998 if (is_valid_ether_addr(hw->mac.port_addr)) 14999 pf->hw_features |= I40E_HW_PORT_ID_VALID; 15000 15001 pci_set_drvdata(pdev, pf); 15002 pci_save_state(pdev); 15003 15004 dev_info(&pdev->dev, 15005 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP) ? 15006 "FW LLDP is disabled\n" : 15007 "FW LLDP is enabled\n"); 15008 15009 /* Enable FW to write default DCB config on link-up */ 15010 i40e_aq_set_dcb_parameters(hw, true, NULL); 15011 15012 #ifdef CONFIG_I40E_DCB 15013 err = i40e_init_pf_dcb(pf); 15014 if (err) { 15015 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 15016 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 15017 /* Continue without DCB enabled */ 15018 } 15019 #endif /* CONFIG_I40E_DCB */ 15020 15021 /* set up periodic task facility */ 15022 timer_setup(&pf->service_timer, i40e_service_timer, 0); 15023 pf->service_timer_period = HZ; 15024 15025 INIT_WORK(&pf->service_task, i40e_service_task); 15026 clear_bit(__I40E_SERVICE_SCHED, pf->state); 15027 15028 /* NVM bit on means WoL disabled for the port */ 15029 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 15030 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 15031 pf->wol_en = false; 15032 else 15033 pf->wol_en = true; 15034 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 15035 15036 /* set up the main switch operations */ 15037 i40e_determine_queue_usage(pf); 15038 err = i40e_init_interrupt_scheme(pf); 15039 if (err) 15040 goto err_switch_setup; 15041 15042 /* The number of VSIs reported by the FW is the minimum guaranteed 15043 * to us; HW supports far more and we share the remaining pool with 15044 * the other PFs. We allocate space for more than the guarantee with 15045 * the understanding that we might not get them all later. 15046 */ 15047 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 15048 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 15049 else 15050 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 15051 15052 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 15053 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 15054 GFP_KERNEL); 15055 if (!pf->vsi) { 15056 err = -ENOMEM; 15057 goto err_switch_setup; 15058 } 15059 15060 #ifdef CONFIG_PCI_IOV 15061 /* prep for VF support */ 15062 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15063 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15064 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15065 if (pci_num_vf(pdev)) 15066 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 15067 } 15068 #endif 15069 err = i40e_setup_pf_switch(pf, false); 15070 if (err) { 15071 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 15072 goto err_vsis; 15073 } 15074 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 15075 15076 /* Make sure flow control is set according to current settings */ 15077 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 15078 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 15079 dev_dbg(&pf->pdev->dev, 15080 "Set fc with err %s aq_err %s on get_phy_cap\n", 15081 i40e_stat_str(hw, err), 15082 i40e_aq_str(hw, hw->aq.asq_last_status)); 15083 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 15084 dev_dbg(&pf->pdev->dev, 15085 "Set fc with err %s aq_err %s on set_phy_config\n", 15086 i40e_stat_str(hw, err), 15087 i40e_aq_str(hw, hw->aq.asq_last_status)); 15088 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 15089 dev_dbg(&pf->pdev->dev, 15090 "Set fc with err %s aq_err %s on get_link_info\n", 15091 i40e_stat_str(hw, err), 15092 i40e_aq_str(hw, hw->aq.asq_last_status)); 15093 15094 /* if FDIR VSI was set up, start it now */ 15095 for (i = 0; i < pf->num_alloc_vsi; i++) { 15096 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 15097 i40e_vsi_open(pf->vsi[i]); 15098 break; 15099 } 15100 } 15101 15102 /* The driver only wants link up/down and module qualification 15103 * reports from firmware. Note the negative logic. 15104 */ 15105 err = i40e_aq_set_phy_int_mask(&pf->hw, 15106 ~(I40E_AQ_EVENT_LINK_UPDOWN | 15107 I40E_AQ_EVENT_MEDIA_NA | 15108 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 15109 if (err) 15110 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 15111 i40e_stat_str(&pf->hw, err), 15112 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15113 15114 /* Reconfigure hardware for allowing smaller MSS in the case 15115 * of TSO, so that we avoid the MDD being fired and causing 15116 * a reset in the case of small MSS+TSO. 15117 */ 15118 val = rd32(hw, I40E_REG_MSS); 15119 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 15120 val &= ~I40E_REG_MSS_MIN_MASK; 15121 val |= I40E_64BYTE_MSS; 15122 wr32(hw, I40E_REG_MSS, val); 15123 } 15124 15125 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 15126 msleep(75); 15127 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 15128 if (err) 15129 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 15130 i40e_stat_str(&pf->hw, err), 15131 i40e_aq_str(&pf->hw, 15132 pf->hw.aq.asq_last_status)); 15133 } 15134 /* The main driver is (mostly) up and happy. We need to set this state 15135 * before setting up the misc vector or we get a race and the vector 15136 * ends up disabled forever. 15137 */ 15138 clear_bit(__I40E_DOWN, pf->state); 15139 15140 /* In case of MSIX we are going to setup the misc vector right here 15141 * to handle admin queue events etc. In case of legacy and MSI 15142 * the misc functionality and queue processing is combined in 15143 * the same vector and that gets setup at open. 15144 */ 15145 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 15146 err = i40e_setup_misc_vector(pf); 15147 if (err) { 15148 dev_info(&pdev->dev, 15149 "setup of misc vector failed: %d\n", err); 15150 goto err_vsis; 15151 } 15152 } 15153 15154 #ifdef CONFIG_PCI_IOV 15155 /* prep for VF support */ 15156 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 15157 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 15158 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 15159 /* disable link interrupts for VFs */ 15160 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 15161 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 15162 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 15163 i40e_flush(hw); 15164 15165 if (pci_num_vf(pdev)) { 15166 dev_info(&pdev->dev, 15167 "Active VFs found, allocating resources.\n"); 15168 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 15169 if (err) 15170 dev_info(&pdev->dev, 15171 "Error %d allocating resources for existing VFs\n", 15172 err); 15173 } 15174 } 15175 #endif /* CONFIG_PCI_IOV */ 15176 15177 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15178 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 15179 pf->num_iwarp_msix, 15180 I40E_IWARP_IRQ_PILE_ID); 15181 if (pf->iwarp_base_vector < 0) { 15182 dev_info(&pdev->dev, 15183 "failed to get tracking for %d vectors for IWARP err=%d\n", 15184 pf->num_iwarp_msix, pf->iwarp_base_vector); 15185 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 15186 } 15187 } 15188 15189 i40e_dbg_pf_init(pf); 15190 15191 /* tell the firmware that we're starting */ 15192 i40e_send_version(pf); 15193 15194 /* since everything's happy, start the service_task timer */ 15195 mod_timer(&pf->service_timer, 15196 round_jiffies(jiffies + pf->service_timer_period)); 15197 15198 /* add this PF to client device list and launch a client service task */ 15199 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15200 err = i40e_lan_add_device(pf); 15201 if (err) 15202 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 15203 err); 15204 } 15205 15206 #define PCI_SPEED_SIZE 8 15207 #define PCI_WIDTH_SIZE 8 15208 /* Devices on the IOSF bus do not have this information 15209 * and will report PCI Gen 1 x 1 by default so don't bother 15210 * checking them. 15211 */ 15212 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 15213 char speed[PCI_SPEED_SIZE] = "Unknown"; 15214 char width[PCI_WIDTH_SIZE] = "Unknown"; 15215 15216 /* Get the negotiated link width and speed from PCI config 15217 * space 15218 */ 15219 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 15220 &link_status); 15221 15222 i40e_set_pci_config_data(hw, link_status); 15223 15224 switch (hw->bus.speed) { 15225 case i40e_bus_speed_8000: 15226 strlcpy(speed, "8.0", PCI_SPEED_SIZE); break; 15227 case i40e_bus_speed_5000: 15228 strlcpy(speed, "5.0", PCI_SPEED_SIZE); break; 15229 case i40e_bus_speed_2500: 15230 strlcpy(speed, "2.5", PCI_SPEED_SIZE); break; 15231 default: 15232 break; 15233 } 15234 switch (hw->bus.width) { 15235 case i40e_bus_width_pcie_x8: 15236 strlcpy(width, "8", PCI_WIDTH_SIZE); break; 15237 case i40e_bus_width_pcie_x4: 15238 strlcpy(width, "4", PCI_WIDTH_SIZE); break; 15239 case i40e_bus_width_pcie_x2: 15240 strlcpy(width, "2", PCI_WIDTH_SIZE); break; 15241 case i40e_bus_width_pcie_x1: 15242 strlcpy(width, "1", PCI_WIDTH_SIZE); break; 15243 default: 15244 break; 15245 } 15246 15247 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 15248 speed, width); 15249 15250 if (hw->bus.width < i40e_bus_width_pcie_x8 || 15251 hw->bus.speed < i40e_bus_speed_8000) { 15252 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 15253 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 15254 } 15255 } 15256 15257 /* get the requested speeds from the fw */ 15258 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 15259 if (err) 15260 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 15261 i40e_stat_str(&pf->hw, err), 15262 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15263 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 15264 15265 /* set the FEC config due to the board capabilities */ 15266 i40e_set_fec_in_flags(abilities.fec_cfg_curr_mod_ext_info, &pf->flags); 15267 15268 /* get the supported phy types from the fw */ 15269 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 15270 if (err) 15271 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 15272 i40e_stat_str(&pf->hw, err), 15273 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 15274 15275 /* Add a filter to drop all Flow control frames from any VSI from being 15276 * transmitted. By doing so we stop a malicious VF from sending out 15277 * PAUSE or PFC frames and potentially controlling traffic for other 15278 * PF/VF VSIs. 15279 * The FW can still send Flow control frames if enabled. 15280 */ 15281 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 15282 pf->main_vsi_seid); 15283 15284 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 15285 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 15286 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 15287 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 15288 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 15289 /* print a string summarizing features */ 15290 i40e_print_features(pf); 15291 15292 return 0; 15293 15294 /* Unwind what we've done if something failed in the setup */ 15295 err_vsis: 15296 set_bit(__I40E_DOWN, pf->state); 15297 i40e_clear_interrupt_scheme(pf); 15298 kfree(pf->vsi); 15299 err_switch_setup: 15300 i40e_reset_interrupt_capability(pf); 15301 del_timer_sync(&pf->service_timer); 15302 err_mac_addr: 15303 err_configure_lan_hmc: 15304 (void)i40e_shutdown_lan_hmc(hw); 15305 err_init_lan_hmc: 15306 kfree(pf->qp_pile); 15307 err_sw_init: 15308 err_adminq_setup: 15309 err_pf_reset: 15310 iounmap(hw->hw_addr); 15311 err_ioremap: 15312 kfree(pf); 15313 err_pf_alloc: 15314 pci_disable_pcie_error_reporting(pdev); 15315 pci_release_mem_regions(pdev); 15316 err_pci_reg: 15317 err_dma: 15318 pci_disable_device(pdev); 15319 return err; 15320 } 15321 15322 /** 15323 * i40e_remove - Device removal routine 15324 * @pdev: PCI device information struct 15325 * 15326 * i40e_remove is called by the PCI subsystem to alert the driver 15327 * that is should release a PCI device. This could be caused by a 15328 * Hot-Plug event, or because the driver is going to be removed from 15329 * memory. 15330 **/ 15331 static void i40e_remove(struct pci_dev *pdev) 15332 { 15333 struct i40e_pf *pf = pci_get_drvdata(pdev); 15334 struct i40e_hw *hw = &pf->hw; 15335 i40e_status ret_code; 15336 int i; 15337 15338 i40e_dbg_pf_exit(pf); 15339 15340 i40e_ptp_stop(pf); 15341 15342 /* Disable RSS in hw */ 15343 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 15344 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 15345 15346 /* no more scheduling of any task */ 15347 set_bit(__I40E_SUSPENDED, pf->state); 15348 set_bit(__I40E_DOWN, pf->state); 15349 if (pf->service_timer.function) 15350 del_timer_sync(&pf->service_timer); 15351 if (pf->service_task.func) 15352 cancel_work_sync(&pf->service_task); 15353 15354 if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { 15355 struct i40e_vsi *vsi = pf->vsi[0]; 15356 15357 /* We know that we have allocated only one vsi for this PF, 15358 * it was just for registering netdevice, so the interface 15359 * could be visible in the 'ifconfig' output 15360 */ 15361 unregister_netdev(vsi->netdev); 15362 free_netdev(vsi->netdev); 15363 15364 goto unmap; 15365 } 15366 15367 /* Client close must be called explicitly here because the timer 15368 * has been stopped. 15369 */ 15370 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15371 15372 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 15373 i40e_free_vfs(pf); 15374 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 15375 } 15376 15377 i40e_fdir_teardown(pf); 15378 15379 /* If there is a switch structure or any orphans, remove them. 15380 * This will leave only the PF's VSI remaining. 15381 */ 15382 for (i = 0; i < I40E_MAX_VEB; i++) { 15383 if (!pf->veb[i]) 15384 continue; 15385 15386 if (pf->veb[i]->uplink_seid == pf->mac_seid || 15387 pf->veb[i]->uplink_seid == 0) 15388 i40e_switch_branch_release(pf->veb[i]); 15389 } 15390 15391 /* Now we can shutdown the PF's VSI, just before we kill 15392 * adminq and hmc. 15393 */ 15394 if (pf->vsi[pf->lan_vsi]) 15395 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 15396 15397 i40e_cloud_filter_exit(pf); 15398 15399 /* remove attached clients */ 15400 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 15401 ret_code = i40e_lan_del_device(pf); 15402 if (ret_code) 15403 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 15404 ret_code); 15405 } 15406 15407 /* shutdown and destroy the HMC */ 15408 if (hw->hmc.hmc_obj) { 15409 ret_code = i40e_shutdown_lan_hmc(hw); 15410 if (ret_code) 15411 dev_warn(&pdev->dev, 15412 "Failed to destroy the HMC resources: %d\n", 15413 ret_code); 15414 } 15415 15416 unmap: 15417 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 15418 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 15419 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 15420 free_irq(pf->pdev->irq, pf); 15421 15422 /* shutdown the adminq */ 15423 i40e_shutdown_adminq(hw); 15424 15425 /* destroy the locks only once, here */ 15426 mutex_destroy(&hw->aq.arq_mutex); 15427 mutex_destroy(&hw->aq.asq_mutex); 15428 15429 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 15430 rtnl_lock(); 15431 i40e_clear_interrupt_scheme(pf); 15432 for (i = 0; i < pf->num_alloc_vsi; i++) { 15433 if (pf->vsi[i]) { 15434 if (!test_bit(__I40E_RECOVERY_MODE, pf->state)) 15435 i40e_vsi_clear_rings(pf->vsi[i]); 15436 i40e_vsi_clear(pf->vsi[i]); 15437 pf->vsi[i] = NULL; 15438 } 15439 } 15440 rtnl_unlock(); 15441 15442 for (i = 0; i < I40E_MAX_VEB; i++) { 15443 kfree(pf->veb[i]); 15444 pf->veb[i] = NULL; 15445 } 15446 15447 kfree(pf->qp_pile); 15448 kfree(pf->vsi); 15449 15450 iounmap(hw->hw_addr); 15451 kfree(pf); 15452 pci_release_mem_regions(pdev); 15453 15454 pci_disable_pcie_error_reporting(pdev); 15455 pci_disable_device(pdev); 15456 } 15457 15458 /** 15459 * i40e_pci_error_detected - warning that something funky happened in PCI land 15460 * @pdev: PCI device information struct 15461 * @error: the type of PCI error 15462 * 15463 * Called to warn that something happened and the error handling steps 15464 * are in progress. Allows the driver to quiesce things, be ready for 15465 * remediation. 15466 **/ 15467 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 15468 enum pci_channel_state error) 15469 { 15470 struct i40e_pf *pf = pci_get_drvdata(pdev); 15471 15472 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 15473 15474 if (!pf) { 15475 dev_info(&pdev->dev, 15476 "Cannot recover - error happened during device probe\n"); 15477 return PCI_ERS_RESULT_DISCONNECT; 15478 } 15479 15480 /* shutdown all operations */ 15481 if (!test_bit(__I40E_SUSPENDED, pf->state)) 15482 i40e_prep_for_reset(pf, false); 15483 15484 /* Request a slot reset */ 15485 return PCI_ERS_RESULT_NEED_RESET; 15486 } 15487 15488 /** 15489 * i40e_pci_error_slot_reset - a PCI slot reset just happened 15490 * @pdev: PCI device information struct 15491 * 15492 * Called to find if the driver can work with the device now that 15493 * the pci slot has been reset. If a basic connection seems good 15494 * (registers are readable and have sane content) then return a 15495 * happy little PCI_ERS_RESULT_xxx. 15496 **/ 15497 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 15498 { 15499 struct i40e_pf *pf = pci_get_drvdata(pdev); 15500 pci_ers_result_t result; 15501 u32 reg; 15502 15503 dev_dbg(&pdev->dev, "%s\n", __func__); 15504 if (pci_enable_device_mem(pdev)) { 15505 dev_info(&pdev->dev, 15506 "Cannot re-enable PCI device after reset.\n"); 15507 result = PCI_ERS_RESULT_DISCONNECT; 15508 } else { 15509 pci_set_master(pdev); 15510 pci_restore_state(pdev); 15511 pci_save_state(pdev); 15512 pci_wake_from_d3(pdev, false); 15513 15514 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 15515 if (reg == 0) 15516 result = PCI_ERS_RESULT_RECOVERED; 15517 else 15518 result = PCI_ERS_RESULT_DISCONNECT; 15519 } 15520 15521 return result; 15522 } 15523 15524 /** 15525 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 15526 * @pdev: PCI device information struct 15527 */ 15528 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 15529 { 15530 struct i40e_pf *pf = pci_get_drvdata(pdev); 15531 15532 i40e_prep_for_reset(pf, false); 15533 } 15534 15535 /** 15536 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 15537 * @pdev: PCI device information struct 15538 */ 15539 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 15540 { 15541 struct i40e_pf *pf = pci_get_drvdata(pdev); 15542 15543 i40e_reset_and_rebuild(pf, false, false); 15544 } 15545 15546 /** 15547 * i40e_pci_error_resume - restart operations after PCI error recovery 15548 * @pdev: PCI device information struct 15549 * 15550 * Called to allow the driver to bring things back up after PCI error 15551 * and/or reset recovery has finished. 15552 **/ 15553 static void i40e_pci_error_resume(struct pci_dev *pdev) 15554 { 15555 struct i40e_pf *pf = pci_get_drvdata(pdev); 15556 15557 dev_dbg(&pdev->dev, "%s\n", __func__); 15558 if (test_bit(__I40E_SUSPENDED, pf->state)) 15559 return; 15560 15561 i40e_handle_reset_warning(pf, false); 15562 } 15563 15564 /** 15565 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 15566 * using the mac_address_write admin q function 15567 * @pf: pointer to i40e_pf struct 15568 **/ 15569 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 15570 { 15571 struct i40e_hw *hw = &pf->hw; 15572 i40e_status ret; 15573 u8 mac_addr[6]; 15574 u16 flags = 0; 15575 15576 /* Get current MAC address in case it's an LAA */ 15577 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 15578 ether_addr_copy(mac_addr, 15579 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 15580 } else { 15581 dev_err(&pf->pdev->dev, 15582 "Failed to retrieve MAC address; using default\n"); 15583 ether_addr_copy(mac_addr, hw->mac.addr); 15584 } 15585 15586 /* The FW expects the mac address write cmd to first be called with 15587 * one of these flags before calling it again with the multicast 15588 * enable flags. 15589 */ 15590 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 15591 15592 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 15593 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 15594 15595 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 15596 if (ret) { 15597 dev_err(&pf->pdev->dev, 15598 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 15599 return; 15600 } 15601 15602 flags = I40E_AQC_MC_MAG_EN 15603 | I40E_AQC_WOL_PRESERVE_ON_PFR 15604 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 15605 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 15606 if (ret) 15607 dev_err(&pf->pdev->dev, 15608 "Failed to enable Multicast Magic Packet wake up\n"); 15609 } 15610 15611 /** 15612 * i40e_shutdown - PCI callback for shutting down 15613 * @pdev: PCI device information struct 15614 **/ 15615 static void i40e_shutdown(struct pci_dev *pdev) 15616 { 15617 struct i40e_pf *pf = pci_get_drvdata(pdev); 15618 struct i40e_hw *hw = &pf->hw; 15619 15620 set_bit(__I40E_SUSPENDED, pf->state); 15621 set_bit(__I40E_DOWN, pf->state); 15622 15623 del_timer_sync(&pf->service_timer); 15624 cancel_work_sync(&pf->service_task); 15625 i40e_cloud_filter_exit(pf); 15626 i40e_fdir_teardown(pf); 15627 15628 /* Client close must be called explicitly here because the timer 15629 * has been stopped. 15630 */ 15631 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15632 15633 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 15634 i40e_enable_mc_magic_wake(pf); 15635 15636 i40e_prep_for_reset(pf, false); 15637 15638 wr32(hw, I40E_PFPM_APM, 15639 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 15640 wr32(hw, I40E_PFPM_WUFC, 15641 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 15642 15643 /* Free MSI/legacy interrupt 0 when in recovery mode. */ 15644 if (test_bit(__I40E_RECOVERY_MODE, pf->state) && 15645 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) 15646 free_irq(pf->pdev->irq, pf); 15647 15648 /* Since we're going to destroy queues during the 15649 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 15650 * whole section 15651 */ 15652 rtnl_lock(); 15653 i40e_clear_interrupt_scheme(pf); 15654 rtnl_unlock(); 15655 15656 if (system_state == SYSTEM_POWER_OFF) { 15657 pci_wake_from_d3(pdev, pf->wol_en); 15658 pci_set_power_state(pdev, PCI_D3hot); 15659 } 15660 } 15661 15662 /** 15663 * i40e_suspend - PM callback for moving to D3 15664 * @dev: generic device information structure 15665 **/ 15666 static int __maybe_unused i40e_suspend(struct device *dev) 15667 { 15668 struct i40e_pf *pf = dev_get_drvdata(dev); 15669 struct i40e_hw *hw = &pf->hw; 15670 15671 /* If we're already suspended, then there is nothing to do */ 15672 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 15673 return 0; 15674 15675 set_bit(__I40E_DOWN, pf->state); 15676 15677 /* Ensure service task will not be running */ 15678 del_timer_sync(&pf->service_timer); 15679 cancel_work_sync(&pf->service_task); 15680 15681 /* Client close must be called explicitly here because the timer 15682 * has been stopped. 15683 */ 15684 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 15685 15686 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 15687 i40e_enable_mc_magic_wake(pf); 15688 15689 /* Since we're going to destroy queues during the 15690 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 15691 * whole section 15692 */ 15693 rtnl_lock(); 15694 15695 i40e_prep_for_reset(pf, true); 15696 15697 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 15698 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 15699 15700 /* Clear the interrupt scheme and release our IRQs so that the system 15701 * can safely hibernate even when there are a large number of CPUs. 15702 * Otherwise hibernation might fail when mapping all the vectors back 15703 * to CPU0. 15704 */ 15705 i40e_clear_interrupt_scheme(pf); 15706 15707 rtnl_unlock(); 15708 15709 return 0; 15710 } 15711 15712 /** 15713 * i40e_resume - PM callback for waking up from D3 15714 * @dev: generic device information structure 15715 **/ 15716 static int __maybe_unused i40e_resume(struct device *dev) 15717 { 15718 struct i40e_pf *pf = dev_get_drvdata(dev); 15719 int err; 15720 15721 /* If we're not suspended, then there is nothing to do */ 15722 if (!test_bit(__I40E_SUSPENDED, pf->state)) 15723 return 0; 15724 15725 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 15726 * since we're going to be restoring queues 15727 */ 15728 rtnl_lock(); 15729 15730 /* We cleared the interrupt scheme when we suspended, so we need to 15731 * restore it now to resume device functionality. 15732 */ 15733 err = i40e_restore_interrupt_scheme(pf); 15734 if (err) { 15735 dev_err(dev, "Cannot restore interrupt scheme: %d\n", 15736 err); 15737 } 15738 15739 clear_bit(__I40E_DOWN, pf->state); 15740 i40e_reset_and_rebuild(pf, false, true); 15741 15742 rtnl_unlock(); 15743 15744 /* Clear suspended state last after everything is recovered */ 15745 clear_bit(__I40E_SUSPENDED, pf->state); 15746 15747 /* Restart the service task */ 15748 mod_timer(&pf->service_timer, 15749 round_jiffies(jiffies + pf->service_timer_period)); 15750 15751 return 0; 15752 } 15753 15754 static const struct pci_error_handlers i40e_err_handler = { 15755 .error_detected = i40e_pci_error_detected, 15756 .slot_reset = i40e_pci_error_slot_reset, 15757 .reset_prepare = i40e_pci_error_reset_prepare, 15758 .reset_done = i40e_pci_error_reset_done, 15759 .resume = i40e_pci_error_resume, 15760 }; 15761 15762 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 15763 15764 static struct pci_driver i40e_driver = { 15765 .name = i40e_driver_name, 15766 .id_table = i40e_pci_tbl, 15767 .probe = i40e_probe, 15768 .remove = i40e_remove, 15769 .driver = { 15770 .pm = &i40e_pm_ops, 15771 }, 15772 .shutdown = i40e_shutdown, 15773 .err_handler = &i40e_err_handler, 15774 .sriov_configure = i40e_pci_sriov_configure, 15775 }; 15776 15777 /** 15778 * i40e_init_module - Driver registration routine 15779 * 15780 * i40e_init_module is the first routine called when the driver is 15781 * loaded. All it does is register with the PCI subsystem. 15782 **/ 15783 static int __init i40e_init_module(void) 15784 { 15785 pr_info("%s: %s - version %s\n", i40e_driver_name, 15786 i40e_driver_string, i40e_driver_version_str); 15787 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 15788 15789 /* There is no need to throttle the number of active tasks because 15790 * each device limits its own task using a state bit for scheduling 15791 * the service task, and the device tasks do not interfere with each 15792 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 15793 * since we need to be able to guarantee forward progress even under 15794 * memory pressure. 15795 */ 15796 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 15797 if (!i40e_wq) { 15798 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 15799 return -ENOMEM; 15800 } 15801 15802 i40e_dbg_init(); 15803 return pci_register_driver(&i40e_driver); 15804 } 15805 module_init(i40e_init_module); 15806 15807 /** 15808 * i40e_exit_module - Driver exit cleanup routine 15809 * 15810 * i40e_exit_module is called just before the driver is removed 15811 * from memory. 15812 **/ 15813 static void __exit i40e_exit_module(void) 15814 { 15815 pci_unregister_driver(&i40e_driver); 15816 destroy_workqueue(i40e_wq); 15817 i40e_dbg_exit(); 15818 } 15819 module_exit(i40e_exit_module); 15820