1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include <linux/etherdevice.h> 5 #include <linux/of_net.h> 6 #include <linux/pci.h> 7 #include <linux/bpf.h> 8 9 /* Local includes */ 10 #include "i40e.h" 11 #include "i40e_diag.h" 12 #include <net/udp_tunnel.h> 13 /* All i40e tracepoints are defined by the include below, which 14 * must be included exactly once across the whole kernel with 15 * CREATE_TRACE_POINTS defined 16 */ 17 #define CREATE_TRACE_POINTS 18 #include "i40e_trace.h" 19 20 const char i40e_driver_name[] = "i40e"; 21 static const char i40e_driver_string[] = 22 "Intel(R) Ethernet Connection XL710 Network Driver"; 23 24 #define DRV_KERN "-k" 25 26 #define DRV_VERSION_MAJOR 2 27 #define DRV_VERSION_MINOR 3 28 #define DRV_VERSION_BUILD 2 29 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 30 __stringify(DRV_VERSION_MINOR) "." \ 31 __stringify(DRV_VERSION_BUILD) DRV_KERN 32 const char i40e_driver_version_str[] = DRV_VERSION; 33 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 34 35 /* a bit of forward declarations */ 36 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 37 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 38 static int i40e_add_vsi(struct i40e_vsi *vsi); 39 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 40 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 41 static int i40e_setup_misc_vector(struct i40e_pf *pf); 42 static void i40e_determine_queue_usage(struct i40e_pf *pf); 43 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 44 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 45 static int i40e_reset(struct i40e_pf *pf); 46 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 47 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 48 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 49 static int i40e_get_capabilities(struct i40e_pf *pf, 50 enum i40e_admin_queue_opc list_type); 51 52 53 /* i40e_pci_tbl - PCI Device ID Table 54 * 55 * Last entry must be all 0s 56 * 57 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 58 * Class, Class Mask, private data (not used) } 59 */ 60 static const struct pci_device_id i40e_pci_tbl[] = { 61 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 62 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 63 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 64 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 65 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 66 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 67 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 68 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 80 /* required last entry */ 81 {0, } 82 }; 83 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 84 85 #define I40E_MAX_VF_COUNT 128 86 static int debug = -1; 87 module_param(debug, uint, 0); 88 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 89 90 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 91 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 92 MODULE_LICENSE("GPL"); 93 MODULE_VERSION(DRV_VERSION); 94 95 static struct workqueue_struct *i40e_wq; 96 97 /** 98 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 99 * @hw: pointer to the HW structure 100 * @mem: ptr to mem struct to fill out 101 * @size: size of memory requested 102 * @alignment: what to align the allocation to 103 **/ 104 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 105 u64 size, u32 alignment) 106 { 107 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 108 109 mem->size = ALIGN(size, alignment); 110 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 111 &mem->pa, GFP_KERNEL); 112 if (!mem->va) 113 return -ENOMEM; 114 115 return 0; 116 } 117 118 /** 119 * i40e_free_dma_mem_d - OS specific memory free for shared code 120 * @hw: pointer to the HW structure 121 * @mem: ptr to mem struct to free 122 **/ 123 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 124 { 125 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 126 127 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 128 mem->va = NULL; 129 mem->pa = 0; 130 mem->size = 0; 131 132 return 0; 133 } 134 135 /** 136 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 137 * @hw: pointer to the HW structure 138 * @mem: ptr to mem struct to fill out 139 * @size: size of memory requested 140 **/ 141 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 142 u32 size) 143 { 144 mem->size = size; 145 mem->va = kzalloc(size, GFP_KERNEL); 146 147 if (!mem->va) 148 return -ENOMEM; 149 150 return 0; 151 } 152 153 /** 154 * i40e_free_virt_mem_d - OS specific memory free for shared code 155 * @hw: pointer to the HW structure 156 * @mem: ptr to mem struct to free 157 **/ 158 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 159 { 160 /* it's ok to kfree a NULL pointer */ 161 kfree(mem->va); 162 mem->va = NULL; 163 mem->size = 0; 164 165 return 0; 166 } 167 168 /** 169 * i40e_get_lump - find a lump of free generic resource 170 * @pf: board private structure 171 * @pile: the pile of resource to search 172 * @needed: the number of items needed 173 * @id: an owner id to stick on the items assigned 174 * 175 * Returns the base item index of the lump, or negative for error 176 * 177 * The search_hint trick and lack of advanced fit-finding only work 178 * because we're highly likely to have all the same size lump requests. 179 * Linear search time and any fragmentation should be minimal. 180 **/ 181 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 182 u16 needed, u16 id) 183 { 184 int ret = -ENOMEM; 185 int i, j; 186 187 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 188 dev_info(&pf->pdev->dev, 189 "param err: pile=%s needed=%d id=0x%04x\n", 190 pile ? "<valid>" : "<null>", needed, id); 191 return -EINVAL; 192 } 193 194 /* start the linear search with an imperfect hint */ 195 i = pile->search_hint; 196 while (i < pile->num_entries) { 197 /* skip already allocated entries */ 198 if (pile->list[i] & I40E_PILE_VALID_BIT) { 199 i++; 200 continue; 201 } 202 203 /* do we have enough in this lump? */ 204 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 205 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 206 break; 207 } 208 209 if (j == needed) { 210 /* there was enough, so assign it to the requestor */ 211 for (j = 0; j < needed; j++) 212 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 213 ret = i; 214 pile->search_hint = i + j; 215 break; 216 } 217 218 /* not enough, so skip over it and continue looking */ 219 i += j; 220 } 221 222 return ret; 223 } 224 225 /** 226 * i40e_put_lump - return a lump of generic resource 227 * @pile: the pile of resource to search 228 * @index: the base item index 229 * @id: the owner id of the items assigned 230 * 231 * Returns the count of items in the lump 232 **/ 233 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 234 { 235 int valid_id = (id | I40E_PILE_VALID_BIT); 236 int count = 0; 237 int i; 238 239 if (!pile || index >= pile->num_entries) 240 return -EINVAL; 241 242 for (i = index; 243 i < pile->num_entries && pile->list[i] == valid_id; 244 i++) { 245 pile->list[i] = 0; 246 count++; 247 } 248 249 if (count && index < pile->search_hint) 250 pile->search_hint = index; 251 252 return count; 253 } 254 255 /** 256 * i40e_find_vsi_from_id - searches for the vsi with the given id 257 * @pf: the pf structure to search for the vsi 258 * @id: id of the vsi it is searching for 259 **/ 260 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 261 { 262 int i; 263 264 for (i = 0; i < pf->num_alloc_vsi; i++) 265 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 266 return pf->vsi[i]; 267 268 return NULL; 269 } 270 271 /** 272 * i40e_service_event_schedule - Schedule the service task to wake up 273 * @pf: board private structure 274 * 275 * If not already scheduled, this puts the task into the work queue 276 **/ 277 void i40e_service_event_schedule(struct i40e_pf *pf) 278 { 279 if (!test_bit(__I40E_DOWN, pf->state) && 280 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 281 queue_work(i40e_wq, &pf->service_task); 282 } 283 284 /** 285 * i40e_tx_timeout - Respond to a Tx Hang 286 * @netdev: network interface device structure 287 * 288 * If any port has noticed a Tx timeout, it is likely that the whole 289 * device is munged, not just the one netdev port, so go for the full 290 * reset. 291 **/ 292 static void i40e_tx_timeout(struct net_device *netdev) 293 { 294 struct i40e_netdev_priv *np = netdev_priv(netdev); 295 struct i40e_vsi *vsi = np->vsi; 296 struct i40e_pf *pf = vsi->back; 297 struct i40e_ring *tx_ring = NULL; 298 unsigned int i, hung_queue = 0; 299 u32 head, val; 300 301 pf->tx_timeout_count++; 302 303 /* find the stopped queue the same way the stack does */ 304 for (i = 0; i < netdev->num_tx_queues; i++) { 305 struct netdev_queue *q; 306 unsigned long trans_start; 307 308 q = netdev_get_tx_queue(netdev, i); 309 trans_start = q->trans_start; 310 if (netif_xmit_stopped(q) && 311 time_after(jiffies, 312 (trans_start + netdev->watchdog_timeo))) { 313 hung_queue = i; 314 break; 315 } 316 } 317 318 if (i == netdev->num_tx_queues) { 319 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 320 } else { 321 /* now that we have an index, find the tx_ring struct */ 322 for (i = 0; i < vsi->num_queue_pairs; i++) { 323 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 324 if (hung_queue == 325 vsi->tx_rings[i]->queue_index) { 326 tx_ring = vsi->tx_rings[i]; 327 break; 328 } 329 } 330 } 331 } 332 333 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 334 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 335 else if (time_before(jiffies, 336 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 337 return; /* don't do any new action before the next timeout */ 338 339 if (tx_ring) { 340 head = i40e_get_head(tx_ring); 341 /* Read interrupt register */ 342 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 343 val = rd32(&pf->hw, 344 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 345 tx_ring->vsi->base_vector - 1)); 346 else 347 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 348 349 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 350 vsi->seid, hung_queue, tx_ring->next_to_clean, 351 head, tx_ring->next_to_use, 352 readl(tx_ring->tail), val); 353 } 354 355 pf->tx_timeout_last_recovery = jiffies; 356 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 357 pf->tx_timeout_recovery_level, hung_queue); 358 359 switch (pf->tx_timeout_recovery_level) { 360 case 1: 361 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 362 break; 363 case 2: 364 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 365 break; 366 case 3: 367 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 368 break; 369 default: 370 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 371 break; 372 } 373 374 i40e_service_event_schedule(pf); 375 pf->tx_timeout_recovery_level++; 376 } 377 378 /** 379 * i40e_get_vsi_stats_struct - Get System Network Statistics 380 * @vsi: the VSI we care about 381 * 382 * Returns the address of the device statistics structure. 383 * The statistics are actually updated from the service task. 384 **/ 385 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 386 { 387 return &vsi->net_stats; 388 } 389 390 /** 391 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 392 * @ring: Tx ring to get statistics from 393 * @stats: statistics entry to be updated 394 **/ 395 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 396 struct rtnl_link_stats64 *stats) 397 { 398 u64 bytes, packets; 399 unsigned int start; 400 401 do { 402 start = u64_stats_fetch_begin_irq(&ring->syncp); 403 packets = ring->stats.packets; 404 bytes = ring->stats.bytes; 405 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 406 407 stats->tx_packets += packets; 408 stats->tx_bytes += bytes; 409 } 410 411 /** 412 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 413 * @netdev: network interface device structure 414 * @stats: data structure to store statistics 415 * 416 * Returns the address of the device statistics structure. 417 * The statistics are actually updated from the service task. 418 **/ 419 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 420 struct rtnl_link_stats64 *stats) 421 { 422 struct i40e_netdev_priv *np = netdev_priv(netdev); 423 struct i40e_ring *tx_ring, *rx_ring; 424 struct i40e_vsi *vsi = np->vsi; 425 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 426 int i; 427 428 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 429 return; 430 431 if (!vsi->tx_rings) 432 return; 433 434 rcu_read_lock(); 435 for (i = 0; i < vsi->num_queue_pairs; i++) { 436 u64 bytes, packets; 437 unsigned int start; 438 439 tx_ring = READ_ONCE(vsi->tx_rings[i]); 440 if (!tx_ring) 441 continue; 442 i40e_get_netdev_stats_struct_tx(tx_ring, stats); 443 444 rx_ring = &tx_ring[1]; 445 446 do { 447 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 448 packets = rx_ring->stats.packets; 449 bytes = rx_ring->stats.bytes; 450 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 451 452 stats->rx_packets += packets; 453 stats->rx_bytes += bytes; 454 455 if (i40e_enabled_xdp_vsi(vsi)) 456 i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats); 457 } 458 rcu_read_unlock(); 459 460 /* following stats updated by i40e_watchdog_subtask() */ 461 stats->multicast = vsi_stats->multicast; 462 stats->tx_errors = vsi_stats->tx_errors; 463 stats->tx_dropped = vsi_stats->tx_dropped; 464 stats->rx_errors = vsi_stats->rx_errors; 465 stats->rx_dropped = vsi_stats->rx_dropped; 466 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 467 stats->rx_length_errors = vsi_stats->rx_length_errors; 468 } 469 470 /** 471 * i40e_vsi_reset_stats - Resets all stats of the given vsi 472 * @vsi: the VSI to have its stats reset 473 **/ 474 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 475 { 476 struct rtnl_link_stats64 *ns; 477 int i; 478 479 if (!vsi) 480 return; 481 482 ns = i40e_get_vsi_stats_struct(vsi); 483 memset(ns, 0, sizeof(*ns)); 484 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 485 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 486 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 487 if (vsi->rx_rings && vsi->rx_rings[0]) { 488 for (i = 0; i < vsi->num_queue_pairs; i++) { 489 memset(&vsi->rx_rings[i]->stats, 0, 490 sizeof(vsi->rx_rings[i]->stats)); 491 memset(&vsi->rx_rings[i]->rx_stats, 0, 492 sizeof(vsi->rx_rings[i]->rx_stats)); 493 memset(&vsi->tx_rings[i]->stats, 0, 494 sizeof(vsi->tx_rings[i]->stats)); 495 memset(&vsi->tx_rings[i]->tx_stats, 0, 496 sizeof(vsi->tx_rings[i]->tx_stats)); 497 } 498 } 499 vsi->stat_offsets_loaded = false; 500 } 501 502 /** 503 * i40e_pf_reset_stats - Reset all of the stats for the given PF 504 * @pf: the PF to be reset 505 **/ 506 void i40e_pf_reset_stats(struct i40e_pf *pf) 507 { 508 int i; 509 510 memset(&pf->stats, 0, sizeof(pf->stats)); 511 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 512 pf->stat_offsets_loaded = false; 513 514 for (i = 0; i < I40E_MAX_VEB; i++) { 515 if (pf->veb[i]) { 516 memset(&pf->veb[i]->stats, 0, 517 sizeof(pf->veb[i]->stats)); 518 memset(&pf->veb[i]->stats_offsets, 0, 519 sizeof(pf->veb[i]->stats_offsets)); 520 pf->veb[i]->stat_offsets_loaded = false; 521 } 522 } 523 pf->hw_csum_rx_error = 0; 524 } 525 526 /** 527 * i40e_stat_update48 - read and update a 48 bit stat from the chip 528 * @hw: ptr to the hardware info 529 * @hireg: the high 32 bit reg to read 530 * @loreg: the low 32 bit reg to read 531 * @offset_loaded: has the initial offset been loaded yet 532 * @offset: ptr to current offset value 533 * @stat: ptr to the stat 534 * 535 * Since the device stats are not reset at PFReset, they likely will not 536 * be zeroed when the driver starts. We'll save the first values read 537 * and use them as offsets to be subtracted from the raw values in order 538 * to report stats that count from zero. In the process, we also manage 539 * the potential roll-over. 540 **/ 541 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 542 bool offset_loaded, u64 *offset, u64 *stat) 543 { 544 u64 new_data; 545 546 if (hw->device_id == I40E_DEV_ID_QEMU) { 547 new_data = rd32(hw, loreg); 548 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 549 } else { 550 new_data = rd64(hw, loreg); 551 } 552 if (!offset_loaded) 553 *offset = new_data; 554 if (likely(new_data >= *offset)) 555 *stat = new_data - *offset; 556 else 557 *stat = (new_data + BIT_ULL(48)) - *offset; 558 *stat &= 0xFFFFFFFFFFFFULL; 559 } 560 561 /** 562 * i40e_stat_update32 - read and update a 32 bit stat from the chip 563 * @hw: ptr to the hardware info 564 * @reg: the hw reg to read 565 * @offset_loaded: has the initial offset been loaded yet 566 * @offset: ptr to current offset value 567 * @stat: ptr to the stat 568 **/ 569 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 570 bool offset_loaded, u64 *offset, u64 *stat) 571 { 572 u32 new_data; 573 574 new_data = rd32(hw, reg); 575 if (!offset_loaded) 576 *offset = new_data; 577 if (likely(new_data >= *offset)) 578 *stat = (u32)(new_data - *offset); 579 else 580 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 581 } 582 583 /** 584 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 585 * @hw: ptr to the hardware info 586 * @reg: the hw reg to read and clear 587 * @stat: ptr to the stat 588 **/ 589 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 590 { 591 u32 new_data = rd32(hw, reg); 592 593 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 594 *stat += new_data; 595 } 596 597 /** 598 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 599 * @vsi: the VSI to be updated 600 **/ 601 void i40e_update_eth_stats(struct i40e_vsi *vsi) 602 { 603 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 604 struct i40e_pf *pf = vsi->back; 605 struct i40e_hw *hw = &pf->hw; 606 struct i40e_eth_stats *oes; 607 struct i40e_eth_stats *es; /* device's eth stats */ 608 609 es = &vsi->eth_stats; 610 oes = &vsi->eth_stats_offsets; 611 612 /* Gather up the stats that the hw collects */ 613 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 614 vsi->stat_offsets_loaded, 615 &oes->tx_errors, &es->tx_errors); 616 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 617 vsi->stat_offsets_loaded, 618 &oes->rx_discards, &es->rx_discards); 619 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 620 vsi->stat_offsets_loaded, 621 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 622 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 623 vsi->stat_offsets_loaded, 624 &oes->tx_errors, &es->tx_errors); 625 626 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 627 I40E_GLV_GORCL(stat_idx), 628 vsi->stat_offsets_loaded, 629 &oes->rx_bytes, &es->rx_bytes); 630 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 631 I40E_GLV_UPRCL(stat_idx), 632 vsi->stat_offsets_loaded, 633 &oes->rx_unicast, &es->rx_unicast); 634 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 635 I40E_GLV_MPRCL(stat_idx), 636 vsi->stat_offsets_loaded, 637 &oes->rx_multicast, &es->rx_multicast); 638 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 639 I40E_GLV_BPRCL(stat_idx), 640 vsi->stat_offsets_loaded, 641 &oes->rx_broadcast, &es->rx_broadcast); 642 643 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 644 I40E_GLV_GOTCL(stat_idx), 645 vsi->stat_offsets_loaded, 646 &oes->tx_bytes, &es->tx_bytes); 647 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 648 I40E_GLV_UPTCL(stat_idx), 649 vsi->stat_offsets_loaded, 650 &oes->tx_unicast, &es->tx_unicast); 651 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 652 I40E_GLV_MPTCL(stat_idx), 653 vsi->stat_offsets_loaded, 654 &oes->tx_multicast, &es->tx_multicast); 655 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 656 I40E_GLV_BPTCL(stat_idx), 657 vsi->stat_offsets_loaded, 658 &oes->tx_broadcast, &es->tx_broadcast); 659 vsi->stat_offsets_loaded = true; 660 } 661 662 /** 663 * i40e_update_veb_stats - Update Switch component statistics 664 * @veb: the VEB being updated 665 **/ 666 static void i40e_update_veb_stats(struct i40e_veb *veb) 667 { 668 struct i40e_pf *pf = veb->pf; 669 struct i40e_hw *hw = &pf->hw; 670 struct i40e_eth_stats *oes; 671 struct i40e_eth_stats *es; /* device's eth stats */ 672 struct i40e_veb_tc_stats *veb_oes; 673 struct i40e_veb_tc_stats *veb_es; 674 int i, idx = 0; 675 676 idx = veb->stats_idx; 677 es = &veb->stats; 678 oes = &veb->stats_offsets; 679 veb_es = &veb->tc_stats; 680 veb_oes = &veb->tc_stats_offsets; 681 682 /* Gather up the stats that the hw collects */ 683 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 684 veb->stat_offsets_loaded, 685 &oes->tx_discards, &es->tx_discards); 686 if (hw->revision_id > 0) 687 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 688 veb->stat_offsets_loaded, 689 &oes->rx_unknown_protocol, 690 &es->rx_unknown_protocol); 691 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 692 veb->stat_offsets_loaded, 693 &oes->rx_bytes, &es->rx_bytes); 694 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 695 veb->stat_offsets_loaded, 696 &oes->rx_unicast, &es->rx_unicast); 697 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 698 veb->stat_offsets_loaded, 699 &oes->rx_multicast, &es->rx_multicast); 700 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 701 veb->stat_offsets_loaded, 702 &oes->rx_broadcast, &es->rx_broadcast); 703 704 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 705 veb->stat_offsets_loaded, 706 &oes->tx_bytes, &es->tx_bytes); 707 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 708 veb->stat_offsets_loaded, 709 &oes->tx_unicast, &es->tx_unicast); 710 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 711 veb->stat_offsets_loaded, 712 &oes->tx_multicast, &es->tx_multicast); 713 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 714 veb->stat_offsets_loaded, 715 &oes->tx_broadcast, &es->tx_broadcast); 716 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 717 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 718 I40E_GLVEBTC_RPCL(i, idx), 719 veb->stat_offsets_loaded, 720 &veb_oes->tc_rx_packets[i], 721 &veb_es->tc_rx_packets[i]); 722 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 723 I40E_GLVEBTC_RBCL(i, idx), 724 veb->stat_offsets_loaded, 725 &veb_oes->tc_rx_bytes[i], 726 &veb_es->tc_rx_bytes[i]); 727 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 728 I40E_GLVEBTC_TPCL(i, idx), 729 veb->stat_offsets_loaded, 730 &veb_oes->tc_tx_packets[i], 731 &veb_es->tc_tx_packets[i]); 732 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 733 I40E_GLVEBTC_TBCL(i, idx), 734 veb->stat_offsets_loaded, 735 &veb_oes->tc_tx_bytes[i], 736 &veb_es->tc_tx_bytes[i]); 737 } 738 veb->stat_offsets_loaded = true; 739 } 740 741 /** 742 * i40e_update_vsi_stats - Update the vsi statistics counters. 743 * @vsi: the VSI to be updated 744 * 745 * There are a few instances where we store the same stat in a 746 * couple of different structs. This is partly because we have 747 * the netdev stats that need to be filled out, which is slightly 748 * different from the "eth_stats" defined by the chip and used in 749 * VF communications. We sort it out here. 750 **/ 751 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 752 { 753 struct i40e_pf *pf = vsi->back; 754 struct rtnl_link_stats64 *ons; 755 struct rtnl_link_stats64 *ns; /* netdev stats */ 756 struct i40e_eth_stats *oes; 757 struct i40e_eth_stats *es; /* device's eth stats */ 758 u32 tx_restart, tx_busy; 759 struct i40e_ring *p; 760 u32 rx_page, rx_buf; 761 u64 bytes, packets; 762 unsigned int start; 763 u64 tx_linearize; 764 u64 tx_force_wb; 765 u64 rx_p, rx_b; 766 u64 tx_p, tx_b; 767 u16 q; 768 769 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 770 test_bit(__I40E_CONFIG_BUSY, pf->state)) 771 return; 772 773 ns = i40e_get_vsi_stats_struct(vsi); 774 ons = &vsi->net_stats_offsets; 775 es = &vsi->eth_stats; 776 oes = &vsi->eth_stats_offsets; 777 778 /* Gather up the netdev and vsi stats that the driver collects 779 * on the fly during packet processing 780 */ 781 rx_b = rx_p = 0; 782 tx_b = tx_p = 0; 783 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 784 rx_page = 0; 785 rx_buf = 0; 786 rcu_read_lock(); 787 for (q = 0; q < vsi->num_queue_pairs; q++) { 788 /* locate Tx ring */ 789 p = READ_ONCE(vsi->tx_rings[q]); 790 791 do { 792 start = u64_stats_fetch_begin_irq(&p->syncp); 793 packets = p->stats.packets; 794 bytes = p->stats.bytes; 795 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 796 tx_b += bytes; 797 tx_p += packets; 798 tx_restart += p->tx_stats.restart_queue; 799 tx_busy += p->tx_stats.tx_busy; 800 tx_linearize += p->tx_stats.tx_linearize; 801 tx_force_wb += p->tx_stats.tx_force_wb; 802 803 /* Rx queue is part of the same block as Tx queue */ 804 p = &p[1]; 805 do { 806 start = u64_stats_fetch_begin_irq(&p->syncp); 807 packets = p->stats.packets; 808 bytes = p->stats.bytes; 809 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 810 rx_b += bytes; 811 rx_p += packets; 812 rx_buf += p->rx_stats.alloc_buff_failed; 813 rx_page += p->rx_stats.alloc_page_failed; 814 } 815 rcu_read_unlock(); 816 vsi->tx_restart = tx_restart; 817 vsi->tx_busy = tx_busy; 818 vsi->tx_linearize = tx_linearize; 819 vsi->tx_force_wb = tx_force_wb; 820 vsi->rx_page_failed = rx_page; 821 vsi->rx_buf_failed = rx_buf; 822 823 ns->rx_packets = rx_p; 824 ns->rx_bytes = rx_b; 825 ns->tx_packets = tx_p; 826 ns->tx_bytes = tx_b; 827 828 /* update netdev stats from eth stats */ 829 i40e_update_eth_stats(vsi); 830 ons->tx_errors = oes->tx_errors; 831 ns->tx_errors = es->tx_errors; 832 ons->multicast = oes->rx_multicast; 833 ns->multicast = es->rx_multicast; 834 ons->rx_dropped = oes->rx_discards; 835 ns->rx_dropped = es->rx_discards; 836 ons->tx_dropped = oes->tx_discards; 837 ns->tx_dropped = es->tx_discards; 838 839 /* pull in a couple PF stats if this is the main vsi */ 840 if (vsi == pf->vsi[pf->lan_vsi]) { 841 ns->rx_crc_errors = pf->stats.crc_errors; 842 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 843 ns->rx_length_errors = pf->stats.rx_length_errors; 844 } 845 } 846 847 /** 848 * i40e_update_pf_stats - Update the PF statistics counters. 849 * @pf: the PF to be updated 850 **/ 851 static void i40e_update_pf_stats(struct i40e_pf *pf) 852 { 853 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 854 struct i40e_hw_port_stats *nsd = &pf->stats; 855 struct i40e_hw *hw = &pf->hw; 856 u32 val; 857 int i; 858 859 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 860 I40E_GLPRT_GORCL(hw->port), 861 pf->stat_offsets_loaded, 862 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 863 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 864 I40E_GLPRT_GOTCL(hw->port), 865 pf->stat_offsets_loaded, 866 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 867 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 868 pf->stat_offsets_loaded, 869 &osd->eth.rx_discards, 870 &nsd->eth.rx_discards); 871 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 872 I40E_GLPRT_UPRCL(hw->port), 873 pf->stat_offsets_loaded, 874 &osd->eth.rx_unicast, 875 &nsd->eth.rx_unicast); 876 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 877 I40E_GLPRT_MPRCL(hw->port), 878 pf->stat_offsets_loaded, 879 &osd->eth.rx_multicast, 880 &nsd->eth.rx_multicast); 881 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 882 I40E_GLPRT_BPRCL(hw->port), 883 pf->stat_offsets_loaded, 884 &osd->eth.rx_broadcast, 885 &nsd->eth.rx_broadcast); 886 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 887 I40E_GLPRT_UPTCL(hw->port), 888 pf->stat_offsets_loaded, 889 &osd->eth.tx_unicast, 890 &nsd->eth.tx_unicast); 891 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 892 I40E_GLPRT_MPTCL(hw->port), 893 pf->stat_offsets_loaded, 894 &osd->eth.tx_multicast, 895 &nsd->eth.tx_multicast); 896 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 897 I40E_GLPRT_BPTCL(hw->port), 898 pf->stat_offsets_loaded, 899 &osd->eth.tx_broadcast, 900 &nsd->eth.tx_broadcast); 901 902 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 903 pf->stat_offsets_loaded, 904 &osd->tx_dropped_link_down, 905 &nsd->tx_dropped_link_down); 906 907 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 908 pf->stat_offsets_loaded, 909 &osd->crc_errors, &nsd->crc_errors); 910 911 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->illegal_bytes, &nsd->illegal_bytes); 914 915 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 916 pf->stat_offsets_loaded, 917 &osd->mac_local_faults, 918 &nsd->mac_local_faults); 919 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 920 pf->stat_offsets_loaded, 921 &osd->mac_remote_faults, 922 &nsd->mac_remote_faults); 923 924 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 925 pf->stat_offsets_loaded, 926 &osd->rx_length_errors, 927 &nsd->rx_length_errors); 928 929 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 930 pf->stat_offsets_loaded, 931 &osd->link_xon_rx, &nsd->link_xon_rx); 932 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 933 pf->stat_offsets_loaded, 934 &osd->link_xon_tx, &nsd->link_xon_tx); 935 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 936 pf->stat_offsets_loaded, 937 &osd->link_xoff_rx, &nsd->link_xoff_rx); 938 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 939 pf->stat_offsets_loaded, 940 &osd->link_xoff_tx, &nsd->link_xoff_tx); 941 942 for (i = 0; i < 8; i++) { 943 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 944 pf->stat_offsets_loaded, 945 &osd->priority_xoff_rx[i], 946 &nsd->priority_xoff_rx[i]); 947 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 948 pf->stat_offsets_loaded, 949 &osd->priority_xon_rx[i], 950 &nsd->priority_xon_rx[i]); 951 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 952 pf->stat_offsets_loaded, 953 &osd->priority_xon_tx[i], 954 &nsd->priority_xon_tx[i]); 955 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 956 pf->stat_offsets_loaded, 957 &osd->priority_xoff_tx[i], 958 &nsd->priority_xoff_tx[i]); 959 i40e_stat_update32(hw, 960 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 961 pf->stat_offsets_loaded, 962 &osd->priority_xon_2_xoff[i], 963 &nsd->priority_xon_2_xoff[i]); 964 } 965 966 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 967 I40E_GLPRT_PRC64L(hw->port), 968 pf->stat_offsets_loaded, 969 &osd->rx_size_64, &nsd->rx_size_64); 970 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 971 I40E_GLPRT_PRC127L(hw->port), 972 pf->stat_offsets_loaded, 973 &osd->rx_size_127, &nsd->rx_size_127); 974 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 975 I40E_GLPRT_PRC255L(hw->port), 976 pf->stat_offsets_loaded, 977 &osd->rx_size_255, &nsd->rx_size_255); 978 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 979 I40E_GLPRT_PRC511L(hw->port), 980 pf->stat_offsets_loaded, 981 &osd->rx_size_511, &nsd->rx_size_511); 982 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 983 I40E_GLPRT_PRC1023L(hw->port), 984 pf->stat_offsets_loaded, 985 &osd->rx_size_1023, &nsd->rx_size_1023); 986 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 987 I40E_GLPRT_PRC1522L(hw->port), 988 pf->stat_offsets_loaded, 989 &osd->rx_size_1522, &nsd->rx_size_1522); 990 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 991 I40E_GLPRT_PRC9522L(hw->port), 992 pf->stat_offsets_loaded, 993 &osd->rx_size_big, &nsd->rx_size_big); 994 995 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 996 I40E_GLPRT_PTC64L(hw->port), 997 pf->stat_offsets_loaded, 998 &osd->tx_size_64, &nsd->tx_size_64); 999 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1000 I40E_GLPRT_PTC127L(hw->port), 1001 pf->stat_offsets_loaded, 1002 &osd->tx_size_127, &nsd->tx_size_127); 1003 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1004 I40E_GLPRT_PTC255L(hw->port), 1005 pf->stat_offsets_loaded, 1006 &osd->tx_size_255, &nsd->tx_size_255); 1007 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1008 I40E_GLPRT_PTC511L(hw->port), 1009 pf->stat_offsets_loaded, 1010 &osd->tx_size_511, &nsd->tx_size_511); 1011 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1012 I40E_GLPRT_PTC1023L(hw->port), 1013 pf->stat_offsets_loaded, 1014 &osd->tx_size_1023, &nsd->tx_size_1023); 1015 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1016 I40E_GLPRT_PTC1522L(hw->port), 1017 pf->stat_offsets_loaded, 1018 &osd->tx_size_1522, &nsd->tx_size_1522); 1019 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1020 I40E_GLPRT_PTC9522L(hw->port), 1021 pf->stat_offsets_loaded, 1022 &osd->tx_size_big, &nsd->tx_size_big); 1023 1024 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1025 pf->stat_offsets_loaded, 1026 &osd->rx_undersize, &nsd->rx_undersize); 1027 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->rx_fragments, &nsd->rx_fragments); 1030 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1031 pf->stat_offsets_loaded, 1032 &osd->rx_oversize, &nsd->rx_oversize); 1033 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1034 pf->stat_offsets_loaded, 1035 &osd->rx_jabber, &nsd->rx_jabber); 1036 1037 /* FDIR stats */ 1038 i40e_stat_update_and_clear32(hw, 1039 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1040 &nsd->fd_atr_match); 1041 i40e_stat_update_and_clear32(hw, 1042 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1043 &nsd->fd_sb_match); 1044 i40e_stat_update_and_clear32(hw, 1045 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1046 &nsd->fd_atr_tunnel_match); 1047 1048 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1049 nsd->tx_lpi_status = 1050 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1051 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1052 nsd->rx_lpi_status = 1053 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1054 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1055 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1056 pf->stat_offsets_loaded, 1057 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1058 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1059 pf->stat_offsets_loaded, 1060 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1061 1062 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1063 !test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 1064 nsd->fd_sb_status = true; 1065 else 1066 nsd->fd_sb_status = false; 1067 1068 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1069 !test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 1070 nsd->fd_atr_status = true; 1071 else 1072 nsd->fd_atr_status = false; 1073 1074 pf->stat_offsets_loaded = true; 1075 } 1076 1077 /** 1078 * i40e_update_stats - Update the various statistics counters. 1079 * @vsi: the VSI to be updated 1080 * 1081 * Update the various stats for this VSI and its related entities. 1082 **/ 1083 void i40e_update_stats(struct i40e_vsi *vsi) 1084 { 1085 struct i40e_pf *pf = vsi->back; 1086 1087 if (vsi == pf->vsi[pf->lan_vsi]) 1088 i40e_update_pf_stats(pf); 1089 1090 i40e_update_vsi_stats(vsi); 1091 } 1092 1093 /** 1094 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1095 * @vsi: the VSI to be searched 1096 * @macaddr: the MAC address 1097 * @vlan: the vlan 1098 * 1099 * Returns ptr to the filter object or NULL 1100 **/ 1101 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1102 const u8 *macaddr, s16 vlan) 1103 { 1104 struct i40e_mac_filter *f; 1105 u64 key; 1106 1107 if (!vsi || !macaddr) 1108 return NULL; 1109 1110 key = i40e_addr_to_hkey(macaddr); 1111 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1112 if ((ether_addr_equal(macaddr, f->macaddr)) && 1113 (vlan == f->vlan)) 1114 return f; 1115 } 1116 return NULL; 1117 } 1118 1119 /** 1120 * i40e_find_mac - Find a mac addr in the macvlan filters list 1121 * @vsi: the VSI to be searched 1122 * @macaddr: the MAC address we are searching for 1123 * 1124 * Returns the first filter with the provided MAC address or NULL if 1125 * MAC address was not found 1126 **/ 1127 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1128 { 1129 struct i40e_mac_filter *f; 1130 u64 key; 1131 1132 if (!vsi || !macaddr) 1133 return NULL; 1134 1135 key = i40e_addr_to_hkey(macaddr); 1136 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1137 if ((ether_addr_equal(macaddr, f->macaddr))) 1138 return f; 1139 } 1140 return NULL; 1141 } 1142 1143 /** 1144 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1145 * @vsi: the VSI to be searched 1146 * 1147 * Returns true if VSI is in vlan mode or false otherwise 1148 **/ 1149 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1150 { 1151 /* If we have a PVID, always operate in VLAN mode */ 1152 if (vsi->info.pvid) 1153 return true; 1154 1155 /* We need to operate in VLAN mode whenever we have any filters with 1156 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1157 * time, incurring search cost repeatedly. However, we can notice two 1158 * things: 1159 * 1160 * 1) the only place where we can gain a VLAN filter is in 1161 * i40e_add_filter. 1162 * 1163 * 2) the only place where filters are actually removed is in 1164 * i40e_sync_filters_subtask. 1165 * 1166 * Thus, we can simply use a boolean value, has_vlan_filters which we 1167 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1168 * we have to perform the full search after deleting filters in 1169 * i40e_sync_filters_subtask, but we already have to search 1170 * filters here and can perform the check at the same time. This 1171 * results in avoiding embedding a loop for VLAN mode inside another 1172 * loop over all the filters, and should maintain correctness as noted 1173 * above. 1174 */ 1175 return vsi->has_vlan_filter; 1176 } 1177 1178 /** 1179 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1180 * @vsi: the VSI to configure 1181 * @tmp_add_list: list of filters ready to be added 1182 * @tmp_del_list: list of filters ready to be deleted 1183 * @vlan_filters: the number of active VLAN filters 1184 * 1185 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1186 * behave as expected. If we have any active VLAN filters remaining or about 1187 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1188 * so that they only match against untagged traffic. If we no longer have any 1189 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1190 * so that they match against both tagged and untagged traffic. In this way, 1191 * we ensure that we correctly receive the desired traffic. This ensures that 1192 * when we have an active VLAN we will receive only untagged traffic and 1193 * traffic matching active VLANs. If we have no active VLANs then we will 1194 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1195 * 1196 * Finally, in a similar fashion, this function also corrects filters when 1197 * there is an active PVID assigned to this VSI. 1198 * 1199 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1200 * 1201 * This function is only expected to be called from within 1202 * i40e_sync_vsi_filters. 1203 * 1204 * NOTE: This function expects to be called while under the 1205 * mac_filter_hash_lock 1206 */ 1207 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1208 struct hlist_head *tmp_add_list, 1209 struct hlist_head *tmp_del_list, 1210 int vlan_filters) 1211 { 1212 s16 pvid = le16_to_cpu(vsi->info.pvid); 1213 struct i40e_mac_filter *f, *add_head; 1214 struct i40e_new_mac_filter *new; 1215 struct hlist_node *h; 1216 int bkt, new_vlan; 1217 1218 /* To determine if a particular filter needs to be replaced we 1219 * have the three following conditions: 1220 * 1221 * a) if we have a PVID assigned, then all filters which are 1222 * not marked as VLAN=PVID must be replaced with filters that 1223 * are. 1224 * b) otherwise, if we have any active VLANS, all filters 1225 * which are marked as VLAN=-1 must be replaced with 1226 * filters marked as VLAN=0 1227 * c) finally, if we do not have any active VLANS, all filters 1228 * which are marked as VLAN=0 must be replaced with filters 1229 * marked as VLAN=-1 1230 */ 1231 1232 /* Update the filters about to be added in place */ 1233 hlist_for_each_entry(new, tmp_add_list, hlist) { 1234 if (pvid && new->f->vlan != pvid) 1235 new->f->vlan = pvid; 1236 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1237 new->f->vlan = 0; 1238 else if (!vlan_filters && new->f->vlan == 0) 1239 new->f->vlan = I40E_VLAN_ANY; 1240 } 1241 1242 /* Update the remaining active filters */ 1243 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1244 /* Combine the checks for whether a filter needs to be changed 1245 * and then determine the new VLAN inside the if block, in 1246 * order to avoid duplicating code for adding the new filter 1247 * then deleting the old filter. 1248 */ 1249 if ((pvid && f->vlan != pvid) || 1250 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1251 (!vlan_filters && f->vlan == 0)) { 1252 /* Determine the new vlan we will be adding */ 1253 if (pvid) 1254 new_vlan = pvid; 1255 else if (vlan_filters) 1256 new_vlan = 0; 1257 else 1258 new_vlan = I40E_VLAN_ANY; 1259 1260 /* Create the new filter */ 1261 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1262 if (!add_head) 1263 return -ENOMEM; 1264 1265 /* Create a temporary i40e_new_mac_filter */ 1266 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1267 if (!new) 1268 return -ENOMEM; 1269 1270 new->f = add_head; 1271 new->state = add_head->state; 1272 1273 /* Add the new filter to the tmp list */ 1274 hlist_add_head(&new->hlist, tmp_add_list); 1275 1276 /* Put the original filter into the delete list */ 1277 f->state = I40E_FILTER_REMOVE; 1278 hash_del(&f->hlist); 1279 hlist_add_head(&f->hlist, tmp_del_list); 1280 } 1281 } 1282 1283 vsi->has_vlan_filter = !!vlan_filters; 1284 1285 return 0; 1286 } 1287 1288 /** 1289 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1290 * @vsi: the PF Main VSI - inappropriate for any other VSI 1291 * @macaddr: the MAC address 1292 * 1293 * Remove whatever filter the firmware set up so the driver can manage 1294 * its own filtering intelligently. 1295 **/ 1296 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1297 { 1298 struct i40e_aqc_remove_macvlan_element_data element; 1299 struct i40e_pf *pf = vsi->back; 1300 1301 /* Only appropriate for the PF main VSI */ 1302 if (vsi->type != I40E_VSI_MAIN) 1303 return; 1304 1305 memset(&element, 0, sizeof(element)); 1306 ether_addr_copy(element.mac_addr, macaddr); 1307 element.vlan_tag = 0; 1308 /* Ignore error returns, some firmware does it this way... */ 1309 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1310 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1311 1312 memset(&element, 0, sizeof(element)); 1313 ether_addr_copy(element.mac_addr, macaddr); 1314 element.vlan_tag = 0; 1315 /* ...and some firmware does it this way. */ 1316 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1317 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1318 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1319 } 1320 1321 /** 1322 * i40e_add_filter - Add a mac/vlan filter to the VSI 1323 * @vsi: the VSI to be searched 1324 * @macaddr: the MAC address 1325 * @vlan: the vlan 1326 * 1327 * Returns ptr to the filter object or NULL when no memory available. 1328 * 1329 * NOTE: This function is expected to be called with mac_filter_hash_lock 1330 * being held. 1331 **/ 1332 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1333 const u8 *macaddr, s16 vlan) 1334 { 1335 struct i40e_mac_filter *f; 1336 u64 key; 1337 1338 if (!vsi || !macaddr) 1339 return NULL; 1340 1341 f = i40e_find_filter(vsi, macaddr, vlan); 1342 if (!f) { 1343 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1344 if (!f) 1345 return NULL; 1346 1347 /* Update the boolean indicating if we need to function in 1348 * VLAN mode. 1349 */ 1350 if (vlan >= 0) 1351 vsi->has_vlan_filter = true; 1352 1353 ether_addr_copy(f->macaddr, macaddr); 1354 f->vlan = vlan; 1355 f->state = I40E_FILTER_NEW; 1356 INIT_HLIST_NODE(&f->hlist); 1357 1358 key = i40e_addr_to_hkey(macaddr); 1359 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1360 1361 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1362 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1363 } 1364 1365 /* If we're asked to add a filter that has been marked for removal, it 1366 * is safe to simply restore it to active state. __i40e_del_filter 1367 * will have simply deleted any filters which were previously marked 1368 * NEW or FAILED, so if it is currently marked REMOVE it must have 1369 * previously been ACTIVE. Since we haven't yet run the sync filters 1370 * task, just restore this filter to the ACTIVE state so that the 1371 * sync task leaves it in place 1372 */ 1373 if (f->state == I40E_FILTER_REMOVE) 1374 f->state = I40E_FILTER_ACTIVE; 1375 1376 return f; 1377 } 1378 1379 /** 1380 * __i40e_del_filter - Remove a specific filter from the VSI 1381 * @vsi: VSI to remove from 1382 * @f: the filter to remove from the list 1383 * 1384 * This function should be called instead of i40e_del_filter only if you know 1385 * the exact filter you will remove already, such as via i40e_find_filter or 1386 * i40e_find_mac. 1387 * 1388 * NOTE: This function is expected to be called with mac_filter_hash_lock 1389 * being held. 1390 * ANOTHER NOTE: This function MUST be called from within the context of 1391 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1392 * instead of list_for_each_entry(). 1393 **/ 1394 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1395 { 1396 if (!f) 1397 return; 1398 1399 /* If the filter was never added to firmware then we can just delete it 1400 * directly and we don't want to set the status to remove or else an 1401 * admin queue command will unnecessarily fire. 1402 */ 1403 if ((f->state == I40E_FILTER_FAILED) || 1404 (f->state == I40E_FILTER_NEW)) { 1405 hash_del(&f->hlist); 1406 kfree(f); 1407 } else { 1408 f->state = I40E_FILTER_REMOVE; 1409 } 1410 1411 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1412 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->state); 1413 } 1414 1415 /** 1416 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1417 * @vsi: the VSI to be searched 1418 * @macaddr: the MAC address 1419 * @vlan: the VLAN 1420 * 1421 * NOTE: This function is expected to be called with mac_filter_hash_lock 1422 * being held. 1423 * ANOTHER NOTE: This function MUST be called from within the context of 1424 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1425 * instead of list_for_each_entry(). 1426 **/ 1427 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1428 { 1429 struct i40e_mac_filter *f; 1430 1431 if (!vsi || !macaddr) 1432 return; 1433 1434 f = i40e_find_filter(vsi, macaddr, vlan); 1435 __i40e_del_filter(vsi, f); 1436 } 1437 1438 /** 1439 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1440 * @vsi: the VSI to be searched 1441 * @macaddr: the mac address to be filtered 1442 * 1443 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1444 * go through all the macvlan filters and add a macvlan filter for each 1445 * unique vlan that already exists. If a PVID has been assigned, instead only 1446 * add the macaddr to that VLAN. 1447 * 1448 * Returns last filter added on success, else NULL 1449 **/ 1450 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1451 const u8 *macaddr) 1452 { 1453 struct i40e_mac_filter *f, *add = NULL; 1454 struct hlist_node *h; 1455 int bkt; 1456 1457 if (vsi->info.pvid) 1458 return i40e_add_filter(vsi, macaddr, 1459 le16_to_cpu(vsi->info.pvid)); 1460 1461 if (!i40e_is_vsi_in_vlan(vsi)) 1462 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1463 1464 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1465 if (f->state == I40E_FILTER_REMOVE) 1466 continue; 1467 add = i40e_add_filter(vsi, macaddr, f->vlan); 1468 if (!add) 1469 return NULL; 1470 } 1471 1472 return add; 1473 } 1474 1475 /** 1476 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1477 * @vsi: the VSI to be searched 1478 * @macaddr: the mac address to be removed 1479 * 1480 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1481 * associated with. 1482 * 1483 * Returns 0 for success, or error 1484 **/ 1485 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1486 { 1487 struct i40e_mac_filter *f; 1488 struct hlist_node *h; 1489 bool found = false; 1490 int bkt; 1491 1492 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1493 "Missing mac_filter_hash_lock\n"); 1494 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1495 if (ether_addr_equal(macaddr, f->macaddr)) { 1496 __i40e_del_filter(vsi, f); 1497 found = true; 1498 } 1499 } 1500 1501 if (found) 1502 return 0; 1503 else 1504 return -ENOENT; 1505 } 1506 1507 /** 1508 * i40e_set_mac - NDO callback to set mac address 1509 * @netdev: network interface device structure 1510 * @p: pointer to an address structure 1511 * 1512 * Returns 0 on success, negative on failure 1513 **/ 1514 static int i40e_set_mac(struct net_device *netdev, void *p) 1515 { 1516 struct i40e_netdev_priv *np = netdev_priv(netdev); 1517 struct i40e_vsi *vsi = np->vsi; 1518 struct i40e_pf *pf = vsi->back; 1519 struct i40e_hw *hw = &pf->hw; 1520 struct sockaddr *addr = p; 1521 1522 if (!is_valid_ether_addr(addr->sa_data)) 1523 return -EADDRNOTAVAIL; 1524 1525 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1526 netdev_info(netdev, "already using mac address %pM\n", 1527 addr->sa_data); 1528 return 0; 1529 } 1530 1531 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 1532 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 1533 return -EADDRNOTAVAIL; 1534 1535 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1536 netdev_info(netdev, "returning to hw mac address %pM\n", 1537 hw->mac.addr); 1538 else 1539 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1540 1541 /* Copy the address first, so that we avoid a possible race with 1542 * .set_rx_mode(). If we copy after changing the address in the filter 1543 * list, we might open ourselves to a narrow race window where 1544 * .set_rx_mode could delete our dev_addr filter and prevent traffic 1545 * from passing. 1546 */ 1547 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1548 1549 spin_lock_bh(&vsi->mac_filter_hash_lock); 1550 i40e_del_mac_filter(vsi, netdev->dev_addr); 1551 i40e_add_mac_filter(vsi, addr->sa_data); 1552 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1553 if (vsi->type == I40E_VSI_MAIN) { 1554 i40e_status ret; 1555 1556 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1557 I40E_AQC_WRITE_TYPE_LAA_WOL, 1558 addr->sa_data, NULL); 1559 if (ret) 1560 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1561 i40e_stat_str(hw, ret), 1562 i40e_aq_str(hw, hw->aq.asq_last_status)); 1563 } 1564 1565 /* schedule our worker thread which will take care of 1566 * applying the new filter changes 1567 */ 1568 i40e_service_event_schedule(vsi->back); 1569 return 0; 1570 } 1571 1572 /** 1573 * i40e_config_rss_aq - Prepare for RSS using AQ commands 1574 * @vsi: vsi structure 1575 * @seed: RSS hash seed 1576 **/ 1577 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 1578 u8 *lut, u16 lut_size) 1579 { 1580 struct i40e_pf *pf = vsi->back; 1581 struct i40e_hw *hw = &pf->hw; 1582 int ret = 0; 1583 1584 if (seed) { 1585 struct i40e_aqc_get_set_rss_key_data *seed_dw = 1586 (struct i40e_aqc_get_set_rss_key_data *)seed; 1587 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 1588 if (ret) { 1589 dev_info(&pf->pdev->dev, 1590 "Cannot set RSS key, err %s aq_err %s\n", 1591 i40e_stat_str(hw, ret), 1592 i40e_aq_str(hw, hw->aq.asq_last_status)); 1593 return ret; 1594 } 1595 } 1596 if (lut) { 1597 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 1598 1599 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 1600 if (ret) { 1601 dev_info(&pf->pdev->dev, 1602 "Cannot set RSS lut, err %s aq_err %s\n", 1603 i40e_stat_str(hw, ret), 1604 i40e_aq_str(hw, hw->aq.asq_last_status)); 1605 return ret; 1606 } 1607 } 1608 return ret; 1609 } 1610 1611 /** 1612 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 1613 * @vsi: VSI structure 1614 **/ 1615 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 1616 { 1617 struct i40e_pf *pf = vsi->back; 1618 u8 seed[I40E_HKEY_ARRAY_SIZE]; 1619 u8 *lut; 1620 int ret; 1621 1622 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 1623 return 0; 1624 if (!vsi->rss_size) 1625 vsi->rss_size = min_t(int, pf->alloc_rss_size, 1626 vsi->num_queue_pairs); 1627 if (!vsi->rss_size) 1628 return -EINVAL; 1629 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 1630 if (!lut) 1631 return -ENOMEM; 1632 1633 /* Use the user configured hash keys and lookup table if there is one, 1634 * otherwise use default 1635 */ 1636 if (vsi->rss_lut_user) 1637 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 1638 else 1639 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 1640 if (vsi->rss_hkey_user) 1641 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 1642 else 1643 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 1644 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 1645 kfree(lut); 1646 return ret; 1647 } 1648 1649 /** 1650 * i40e_vsi_setup_queue_map_mqprio - Prepares mqprio based tc_config 1651 * @vsi: the VSI being configured, 1652 * @ctxt: VSI context structure 1653 * @enabled_tc: number of traffic classes to enable 1654 * 1655 * Prepares VSI tc_config to have queue configurations based on MQPRIO options. 1656 **/ 1657 static int i40e_vsi_setup_queue_map_mqprio(struct i40e_vsi *vsi, 1658 struct i40e_vsi_context *ctxt, 1659 u8 enabled_tc) 1660 { 1661 u16 qcount = 0, max_qcount, qmap, sections = 0; 1662 int i, override_q, pow, num_qps, ret; 1663 u8 netdev_tc = 0, offset = 0; 1664 1665 if (vsi->type != I40E_VSI_MAIN) 1666 return -EINVAL; 1667 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1668 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1669 vsi->tc_config.numtc = vsi->mqprio_qopt.qopt.num_tc; 1670 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1671 num_qps = vsi->mqprio_qopt.qopt.count[0]; 1672 1673 /* find the next higher power-of-2 of num queue pairs */ 1674 pow = ilog2(num_qps); 1675 if (!is_power_of_2(num_qps)) 1676 pow++; 1677 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1678 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1679 1680 /* Setup queue offset/count for all TCs for given VSI */ 1681 max_qcount = vsi->mqprio_qopt.qopt.count[0]; 1682 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1683 /* See if the given TC is enabled for the given VSI */ 1684 if (vsi->tc_config.enabled_tc & BIT(i)) { 1685 offset = vsi->mqprio_qopt.qopt.offset[i]; 1686 qcount = vsi->mqprio_qopt.qopt.count[i]; 1687 if (qcount > max_qcount) 1688 max_qcount = qcount; 1689 vsi->tc_config.tc_info[i].qoffset = offset; 1690 vsi->tc_config.tc_info[i].qcount = qcount; 1691 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1692 } else { 1693 /* TC is not enabled so set the offset to 1694 * default queue and allocate one queue 1695 * for the given TC. 1696 */ 1697 vsi->tc_config.tc_info[i].qoffset = 0; 1698 vsi->tc_config.tc_info[i].qcount = 1; 1699 vsi->tc_config.tc_info[i].netdev_tc = 0; 1700 } 1701 } 1702 1703 /* Set actual Tx/Rx queue pairs */ 1704 vsi->num_queue_pairs = offset + qcount; 1705 1706 /* Setup queue TC[0].qmap for given VSI context */ 1707 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 1708 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1709 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1710 ctxt->info.valid_sections |= cpu_to_le16(sections); 1711 1712 /* Reconfigure RSS for main VSI with max queue count */ 1713 vsi->rss_size = max_qcount; 1714 ret = i40e_vsi_config_rss(vsi); 1715 if (ret) { 1716 dev_info(&vsi->back->pdev->dev, 1717 "Failed to reconfig rss for num_queues (%u)\n", 1718 max_qcount); 1719 return ret; 1720 } 1721 vsi->reconfig_rss = true; 1722 dev_dbg(&vsi->back->pdev->dev, 1723 "Reconfigured rss with num_queues (%u)\n", max_qcount); 1724 1725 /* Find queue count available for channel VSIs and starting offset 1726 * for channel VSIs 1727 */ 1728 override_q = vsi->mqprio_qopt.qopt.count[0]; 1729 if (override_q && override_q < vsi->num_queue_pairs) { 1730 vsi->cnt_q_avail = vsi->num_queue_pairs - override_q; 1731 vsi->next_base_queue = override_q; 1732 } 1733 return 0; 1734 } 1735 1736 /** 1737 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1738 * @vsi: the VSI being setup 1739 * @ctxt: VSI context structure 1740 * @enabled_tc: Enabled TCs bitmap 1741 * @is_add: True if called before Add VSI 1742 * 1743 * Setup VSI queue mapping for enabled traffic classes. 1744 **/ 1745 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1746 struct i40e_vsi_context *ctxt, 1747 u8 enabled_tc, 1748 bool is_add) 1749 { 1750 struct i40e_pf *pf = vsi->back; 1751 u16 sections = 0; 1752 u8 netdev_tc = 0; 1753 u16 numtc = 1; 1754 u16 qcount; 1755 u8 offset; 1756 u16 qmap; 1757 int i; 1758 u16 num_tc_qps = 0; 1759 1760 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1761 offset = 0; 1762 1763 /* Number of queues per enabled TC */ 1764 num_tc_qps = vsi->alloc_queue_pairs; 1765 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1766 /* Find numtc from enabled TC bitmap */ 1767 for (i = 0, numtc = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1768 if (enabled_tc & BIT(i)) /* TC is enabled */ 1769 numtc++; 1770 } 1771 if (!numtc) { 1772 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1773 numtc = 1; 1774 } 1775 num_tc_qps = num_tc_qps / numtc; 1776 num_tc_qps = min_t(int, num_tc_qps, 1777 i40e_pf_get_max_q_per_tc(pf)); 1778 } 1779 1780 vsi->tc_config.numtc = numtc; 1781 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1782 1783 /* Do not allow use more TC queue pairs than MSI-X vectors exist */ 1784 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1785 num_tc_qps = min_t(int, num_tc_qps, pf->num_lan_msix); 1786 1787 /* Setup queue offset/count for all TCs for given VSI */ 1788 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1789 /* See if the given TC is enabled for the given VSI */ 1790 if (vsi->tc_config.enabled_tc & BIT(i)) { 1791 /* TC is enabled */ 1792 int pow, num_qps; 1793 1794 switch (vsi->type) { 1795 case I40E_VSI_MAIN: 1796 if (!(pf->flags & (I40E_FLAG_FD_SB_ENABLED | 1797 I40E_FLAG_FD_ATR_ENABLED)) || 1798 vsi->tc_config.enabled_tc != 1) { 1799 qcount = min_t(int, pf->alloc_rss_size, 1800 num_tc_qps); 1801 break; 1802 } 1803 case I40E_VSI_FDIR: 1804 case I40E_VSI_SRIOV: 1805 case I40E_VSI_VMDQ2: 1806 default: 1807 qcount = num_tc_qps; 1808 WARN_ON(i != 0); 1809 break; 1810 } 1811 vsi->tc_config.tc_info[i].qoffset = offset; 1812 vsi->tc_config.tc_info[i].qcount = qcount; 1813 1814 /* find the next higher power-of-2 of num queue pairs */ 1815 num_qps = qcount; 1816 pow = 0; 1817 while (num_qps && (BIT_ULL(pow) < qcount)) { 1818 pow++; 1819 num_qps >>= 1; 1820 } 1821 1822 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1823 qmap = 1824 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1825 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1826 1827 offset += qcount; 1828 } else { 1829 /* TC is not enabled so set the offset to 1830 * default queue and allocate one queue 1831 * for the given TC. 1832 */ 1833 vsi->tc_config.tc_info[i].qoffset = 0; 1834 vsi->tc_config.tc_info[i].qcount = 1; 1835 vsi->tc_config.tc_info[i].netdev_tc = 0; 1836 1837 qmap = 0; 1838 } 1839 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1840 } 1841 1842 /* Set actual Tx/Rx queue pairs */ 1843 vsi->num_queue_pairs = offset; 1844 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1845 if (vsi->req_queue_pairs > 0) 1846 vsi->num_queue_pairs = vsi->req_queue_pairs; 1847 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1848 vsi->num_queue_pairs = pf->num_lan_msix; 1849 } 1850 1851 /* Scheduler section valid can only be set for ADD VSI */ 1852 if (is_add) { 1853 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1854 1855 ctxt->info.up_enable_bits = enabled_tc; 1856 } 1857 if (vsi->type == I40E_VSI_SRIOV) { 1858 ctxt->info.mapping_flags |= 1859 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1860 for (i = 0; i < vsi->num_queue_pairs; i++) 1861 ctxt->info.queue_mapping[i] = 1862 cpu_to_le16(vsi->base_queue + i); 1863 } else { 1864 ctxt->info.mapping_flags |= 1865 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1866 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1867 } 1868 ctxt->info.valid_sections |= cpu_to_le16(sections); 1869 } 1870 1871 /** 1872 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1873 * @netdev: the netdevice 1874 * @addr: address to add 1875 * 1876 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1877 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1878 */ 1879 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1880 { 1881 struct i40e_netdev_priv *np = netdev_priv(netdev); 1882 struct i40e_vsi *vsi = np->vsi; 1883 1884 if (i40e_add_mac_filter(vsi, addr)) 1885 return 0; 1886 else 1887 return -ENOMEM; 1888 } 1889 1890 /** 1891 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1892 * @netdev: the netdevice 1893 * @addr: address to add 1894 * 1895 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1896 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1897 */ 1898 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1899 { 1900 struct i40e_netdev_priv *np = netdev_priv(netdev); 1901 struct i40e_vsi *vsi = np->vsi; 1902 1903 /* Under some circumstances, we might receive a request to delete 1904 * our own device address from our uc list. Because we store the 1905 * device address in the VSI's MAC/VLAN filter list, we need to ignore 1906 * such requests and not delete our device address from this list. 1907 */ 1908 if (ether_addr_equal(addr, netdev->dev_addr)) 1909 return 0; 1910 1911 i40e_del_mac_filter(vsi, addr); 1912 1913 return 0; 1914 } 1915 1916 /** 1917 * i40e_set_rx_mode - NDO callback to set the netdev filters 1918 * @netdev: network interface device structure 1919 **/ 1920 static void i40e_set_rx_mode(struct net_device *netdev) 1921 { 1922 struct i40e_netdev_priv *np = netdev_priv(netdev); 1923 struct i40e_vsi *vsi = np->vsi; 1924 1925 spin_lock_bh(&vsi->mac_filter_hash_lock); 1926 1927 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1928 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1929 1930 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1931 1932 /* check for other flag changes */ 1933 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1934 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1935 set_bit(__I40E_MACVLAN_SYNC_PENDING, vsi->back->state); 1936 } 1937 } 1938 1939 /** 1940 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1941 * @vsi: Pointer to VSI struct 1942 * @from: Pointer to list which contains MAC filter entries - changes to 1943 * those entries needs to be undone. 1944 * 1945 * MAC filter entries from this list were slated for deletion. 1946 **/ 1947 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1948 struct hlist_head *from) 1949 { 1950 struct i40e_mac_filter *f; 1951 struct hlist_node *h; 1952 1953 hlist_for_each_entry_safe(f, h, from, hlist) { 1954 u64 key = i40e_addr_to_hkey(f->macaddr); 1955 1956 /* Move the element back into MAC filter list*/ 1957 hlist_del(&f->hlist); 1958 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1959 } 1960 } 1961 1962 /** 1963 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1964 * @vsi: Pointer to vsi struct 1965 * @from: Pointer to list which contains MAC filter entries - changes to 1966 * those entries needs to be undone. 1967 * 1968 * MAC filter entries from this list were slated for addition. 1969 **/ 1970 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 1971 struct hlist_head *from) 1972 { 1973 struct i40e_new_mac_filter *new; 1974 struct hlist_node *h; 1975 1976 hlist_for_each_entry_safe(new, h, from, hlist) { 1977 /* We can simply free the wrapper structure */ 1978 hlist_del(&new->hlist); 1979 kfree(new); 1980 } 1981 } 1982 1983 /** 1984 * i40e_next_entry - Get the next non-broadcast filter from a list 1985 * @next: pointer to filter in list 1986 * 1987 * Returns the next non-broadcast filter in the list. Required so that we 1988 * ignore broadcast filters within the list, since these are not handled via 1989 * the normal firmware update path. 1990 */ 1991 static 1992 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 1993 { 1994 hlist_for_each_entry_continue(next, hlist) { 1995 if (!is_broadcast_ether_addr(next->f->macaddr)) 1996 return next; 1997 } 1998 1999 return NULL; 2000 } 2001 2002 /** 2003 * i40e_update_filter_state - Update filter state based on return data 2004 * from firmware 2005 * @count: Number of filters added 2006 * @add_list: return data from fw 2007 * @add_head: pointer to first filter in current batch 2008 * 2009 * MAC filter entries from list were slated to be added to device. Returns 2010 * number of successful filters. Note that 0 does NOT mean success! 2011 **/ 2012 static int 2013 i40e_update_filter_state(int count, 2014 struct i40e_aqc_add_macvlan_element_data *add_list, 2015 struct i40e_new_mac_filter *add_head) 2016 { 2017 int retval = 0; 2018 int i; 2019 2020 for (i = 0; i < count; i++) { 2021 /* Always check status of each filter. We don't need to check 2022 * the firmware return status because we pre-set the filter 2023 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 2024 * request to the adminq. Thus, if it no longer matches then 2025 * we know the filter is active. 2026 */ 2027 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 2028 add_head->state = I40E_FILTER_FAILED; 2029 } else { 2030 add_head->state = I40E_FILTER_ACTIVE; 2031 retval++; 2032 } 2033 2034 add_head = i40e_next_filter(add_head); 2035 if (!add_head) 2036 break; 2037 } 2038 2039 return retval; 2040 } 2041 2042 /** 2043 * i40e_aqc_del_filters - Request firmware to delete a set of filters 2044 * @vsi: ptr to the VSI 2045 * @vsi_name: name to display in messages 2046 * @list: the list of filters to send to firmware 2047 * @num_del: the number of filters to delete 2048 * @retval: Set to -EIO on failure to delete 2049 * 2050 * Send a request to firmware via AdminQ to delete a set of filters. Uses 2051 * *retval instead of a return value so that success does not force ret_val to 2052 * be set to 0. This ensures that a sequence of calls to this function 2053 * preserve the previous value of *retval on successful delete. 2054 */ 2055 static 2056 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 2057 struct i40e_aqc_remove_macvlan_element_data *list, 2058 int num_del, int *retval) 2059 { 2060 struct i40e_hw *hw = &vsi->back->hw; 2061 i40e_status aq_ret; 2062 int aq_err; 2063 2064 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 2065 aq_err = hw->aq.asq_last_status; 2066 2067 /* Explicitly ignore and do not report when firmware returns ENOENT */ 2068 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 2069 *retval = -EIO; 2070 dev_info(&vsi->back->pdev->dev, 2071 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 2072 vsi_name, i40e_stat_str(hw, aq_ret), 2073 i40e_aq_str(hw, aq_err)); 2074 } 2075 } 2076 2077 /** 2078 * i40e_aqc_add_filters - Request firmware to add a set of filters 2079 * @vsi: ptr to the VSI 2080 * @vsi_name: name to display in messages 2081 * @list: the list of filters to send to firmware 2082 * @add_head: Position in the add hlist 2083 * @num_add: the number of filters to add 2084 * 2085 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 2086 * __I40E_VSI_OVERFLOW_PROMISC bit in vsi->state if the firmware has run out of 2087 * space for more filters. 2088 */ 2089 static 2090 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 2091 struct i40e_aqc_add_macvlan_element_data *list, 2092 struct i40e_new_mac_filter *add_head, 2093 int num_add) 2094 { 2095 struct i40e_hw *hw = &vsi->back->hw; 2096 int aq_err, fcnt; 2097 2098 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 2099 aq_err = hw->aq.asq_last_status; 2100 fcnt = i40e_update_filter_state(num_add, list, add_head); 2101 2102 if (fcnt != num_add) { 2103 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2104 dev_warn(&vsi->back->pdev->dev, 2105 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 2106 i40e_aq_str(hw, aq_err), 2107 vsi_name); 2108 } 2109 } 2110 2111 /** 2112 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 2113 * @vsi: pointer to the VSI 2114 * @vsi_name: the VSI name 2115 * @f: filter data 2116 * 2117 * This function sets or clears the promiscuous broadcast flags for VLAN 2118 * filters in order to properly receive broadcast frames. Assumes that only 2119 * broadcast filters are passed. 2120 * 2121 * Returns status indicating success or failure; 2122 **/ 2123 static i40e_status 2124 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 2125 struct i40e_mac_filter *f) 2126 { 2127 bool enable = f->state == I40E_FILTER_NEW; 2128 struct i40e_hw *hw = &vsi->back->hw; 2129 i40e_status aq_ret; 2130 2131 if (f->vlan == I40E_VLAN_ANY) { 2132 aq_ret = i40e_aq_set_vsi_broadcast(hw, 2133 vsi->seid, 2134 enable, 2135 NULL); 2136 } else { 2137 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 2138 vsi->seid, 2139 enable, 2140 f->vlan, 2141 NULL); 2142 } 2143 2144 if (aq_ret) { 2145 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2146 dev_warn(&vsi->back->pdev->dev, 2147 "Error %s, forcing overflow promiscuous on %s\n", 2148 i40e_aq_str(hw, hw->aq.asq_last_status), 2149 vsi_name); 2150 } 2151 2152 return aq_ret; 2153 } 2154 2155 /** 2156 * i40e_set_promiscuous - set promiscuous mode 2157 * @pf: board private structure 2158 * @promisc: promisc on or off 2159 * 2160 * There are different ways of setting promiscuous mode on a PF depending on 2161 * what state/environment we're in. This identifies and sets it appropriately. 2162 * Returns 0 on success. 2163 **/ 2164 static int i40e_set_promiscuous(struct i40e_pf *pf, bool promisc) 2165 { 2166 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 2167 struct i40e_hw *hw = &pf->hw; 2168 i40e_status aq_ret; 2169 2170 if (vsi->type == I40E_VSI_MAIN && 2171 pf->lan_veb != I40E_NO_VEB && 2172 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2173 /* set defport ON for Main VSI instead of true promisc 2174 * this way we will get all unicast/multicast and VLAN 2175 * promisc behavior but will not get VF or VMDq traffic 2176 * replicated on the Main VSI. 2177 */ 2178 if (promisc) 2179 aq_ret = i40e_aq_set_default_vsi(hw, 2180 vsi->seid, 2181 NULL); 2182 else 2183 aq_ret = i40e_aq_clear_default_vsi(hw, 2184 vsi->seid, 2185 NULL); 2186 if (aq_ret) { 2187 dev_info(&pf->pdev->dev, 2188 "Set default VSI failed, err %s, aq_err %s\n", 2189 i40e_stat_str(hw, aq_ret), 2190 i40e_aq_str(hw, hw->aq.asq_last_status)); 2191 } 2192 } else { 2193 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2194 hw, 2195 vsi->seid, 2196 promisc, NULL, 2197 true); 2198 if (aq_ret) { 2199 dev_info(&pf->pdev->dev, 2200 "set unicast promisc failed, err %s, aq_err %s\n", 2201 i40e_stat_str(hw, aq_ret), 2202 i40e_aq_str(hw, hw->aq.asq_last_status)); 2203 } 2204 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2205 hw, 2206 vsi->seid, 2207 promisc, NULL); 2208 if (aq_ret) { 2209 dev_info(&pf->pdev->dev, 2210 "set multicast promisc failed, err %s, aq_err %s\n", 2211 i40e_stat_str(hw, aq_ret), 2212 i40e_aq_str(hw, hw->aq.asq_last_status)); 2213 } 2214 } 2215 2216 if (!aq_ret) 2217 pf->cur_promisc = promisc; 2218 2219 return aq_ret; 2220 } 2221 2222 /** 2223 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2224 * @vsi: ptr to the VSI 2225 * 2226 * Push any outstanding VSI filter changes through the AdminQ. 2227 * 2228 * Returns 0 or error value 2229 **/ 2230 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2231 { 2232 struct hlist_head tmp_add_list, tmp_del_list; 2233 struct i40e_mac_filter *f; 2234 struct i40e_new_mac_filter *new, *add_head = NULL; 2235 struct i40e_hw *hw = &vsi->back->hw; 2236 bool old_overflow, new_overflow; 2237 unsigned int failed_filters = 0; 2238 unsigned int vlan_filters = 0; 2239 char vsi_name[16] = "PF"; 2240 int filter_list_len = 0; 2241 i40e_status aq_ret = 0; 2242 u32 changed_flags = 0; 2243 struct hlist_node *h; 2244 struct i40e_pf *pf; 2245 int num_add = 0; 2246 int num_del = 0; 2247 int retval = 0; 2248 u16 cmd_flags; 2249 int list_size; 2250 int bkt; 2251 2252 /* empty array typed pointers, kcalloc later */ 2253 struct i40e_aqc_add_macvlan_element_data *add_list; 2254 struct i40e_aqc_remove_macvlan_element_data *del_list; 2255 2256 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2257 usleep_range(1000, 2000); 2258 pf = vsi->back; 2259 2260 old_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2261 2262 if (vsi->netdev) { 2263 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2264 vsi->current_netdev_flags = vsi->netdev->flags; 2265 } 2266 2267 INIT_HLIST_HEAD(&tmp_add_list); 2268 INIT_HLIST_HEAD(&tmp_del_list); 2269 2270 if (vsi->type == I40E_VSI_SRIOV) 2271 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2272 else if (vsi->type != I40E_VSI_MAIN) 2273 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2274 2275 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2276 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2277 2278 spin_lock_bh(&vsi->mac_filter_hash_lock); 2279 /* Create a list of filters to delete. */ 2280 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2281 if (f->state == I40E_FILTER_REMOVE) { 2282 /* Move the element into temporary del_list */ 2283 hash_del(&f->hlist); 2284 hlist_add_head(&f->hlist, &tmp_del_list); 2285 2286 /* Avoid counting removed filters */ 2287 continue; 2288 } 2289 if (f->state == I40E_FILTER_NEW) { 2290 /* Create a temporary i40e_new_mac_filter */ 2291 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2292 if (!new) 2293 goto err_no_memory_locked; 2294 2295 /* Store pointer to the real filter */ 2296 new->f = f; 2297 new->state = f->state; 2298 2299 /* Add it to the hash list */ 2300 hlist_add_head(&new->hlist, &tmp_add_list); 2301 } 2302 2303 /* Count the number of active (current and new) VLAN 2304 * filters we have now. Does not count filters which 2305 * are marked for deletion. 2306 */ 2307 if (f->vlan > 0) 2308 vlan_filters++; 2309 } 2310 2311 retval = i40e_correct_mac_vlan_filters(vsi, 2312 &tmp_add_list, 2313 &tmp_del_list, 2314 vlan_filters); 2315 if (retval) 2316 goto err_no_memory_locked; 2317 2318 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2319 } 2320 2321 /* Now process 'del_list' outside the lock */ 2322 if (!hlist_empty(&tmp_del_list)) { 2323 filter_list_len = hw->aq.asq_buf_size / 2324 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2325 list_size = filter_list_len * 2326 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2327 del_list = kzalloc(list_size, GFP_ATOMIC); 2328 if (!del_list) 2329 goto err_no_memory; 2330 2331 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2332 cmd_flags = 0; 2333 2334 /* handle broadcast filters by updating the broadcast 2335 * promiscuous flag and release filter list. 2336 */ 2337 if (is_broadcast_ether_addr(f->macaddr)) { 2338 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2339 2340 hlist_del(&f->hlist); 2341 kfree(f); 2342 continue; 2343 } 2344 2345 /* add to delete list */ 2346 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2347 if (f->vlan == I40E_VLAN_ANY) { 2348 del_list[num_del].vlan_tag = 0; 2349 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2350 } else { 2351 del_list[num_del].vlan_tag = 2352 cpu_to_le16((u16)(f->vlan)); 2353 } 2354 2355 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2356 del_list[num_del].flags = cmd_flags; 2357 num_del++; 2358 2359 /* flush a full buffer */ 2360 if (num_del == filter_list_len) { 2361 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2362 num_del, &retval); 2363 memset(del_list, 0, list_size); 2364 num_del = 0; 2365 } 2366 /* Release memory for MAC filter entries which were 2367 * synced up with HW. 2368 */ 2369 hlist_del(&f->hlist); 2370 kfree(f); 2371 } 2372 2373 if (num_del) { 2374 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2375 num_del, &retval); 2376 } 2377 2378 kfree(del_list); 2379 del_list = NULL; 2380 } 2381 2382 if (!hlist_empty(&tmp_add_list)) { 2383 /* Do all the adds now. */ 2384 filter_list_len = hw->aq.asq_buf_size / 2385 sizeof(struct i40e_aqc_add_macvlan_element_data); 2386 list_size = filter_list_len * 2387 sizeof(struct i40e_aqc_add_macvlan_element_data); 2388 add_list = kzalloc(list_size, GFP_ATOMIC); 2389 if (!add_list) 2390 goto err_no_memory; 2391 2392 num_add = 0; 2393 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2394 /* handle broadcast filters by updating the broadcast 2395 * promiscuous flag instead of adding a MAC filter. 2396 */ 2397 if (is_broadcast_ether_addr(new->f->macaddr)) { 2398 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2399 new->f)) 2400 new->state = I40E_FILTER_FAILED; 2401 else 2402 new->state = I40E_FILTER_ACTIVE; 2403 continue; 2404 } 2405 2406 /* add to add array */ 2407 if (num_add == 0) 2408 add_head = new; 2409 cmd_flags = 0; 2410 ether_addr_copy(add_list[num_add].mac_addr, 2411 new->f->macaddr); 2412 if (new->f->vlan == I40E_VLAN_ANY) { 2413 add_list[num_add].vlan_tag = 0; 2414 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2415 } else { 2416 add_list[num_add].vlan_tag = 2417 cpu_to_le16((u16)(new->f->vlan)); 2418 } 2419 add_list[num_add].queue_number = 0; 2420 /* set invalid match method for later detection */ 2421 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2422 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2423 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2424 num_add++; 2425 2426 /* flush a full buffer */ 2427 if (num_add == filter_list_len) { 2428 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2429 add_head, num_add); 2430 memset(add_list, 0, list_size); 2431 num_add = 0; 2432 } 2433 } 2434 if (num_add) { 2435 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2436 num_add); 2437 } 2438 /* Now move all of the filters from the temp add list back to 2439 * the VSI's list. 2440 */ 2441 spin_lock_bh(&vsi->mac_filter_hash_lock); 2442 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2443 /* Only update the state if we're still NEW */ 2444 if (new->f->state == I40E_FILTER_NEW) 2445 new->f->state = new->state; 2446 hlist_del(&new->hlist); 2447 kfree(new); 2448 } 2449 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2450 kfree(add_list); 2451 add_list = NULL; 2452 } 2453 2454 /* Determine the number of active and failed filters. */ 2455 spin_lock_bh(&vsi->mac_filter_hash_lock); 2456 vsi->active_filters = 0; 2457 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2458 if (f->state == I40E_FILTER_ACTIVE) 2459 vsi->active_filters++; 2460 else if (f->state == I40E_FILTER_FAILED) 2461 failed_filters++; 2462 } 2463 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2464 2465 /* Check if we are able to exit overflow promiscuous mode. We can 2466 * safely exit if we didn't just enter, we no longer have any failed 2467 * filters, and we have reduced filters below the threshold value. 2468 */ 2469 if (old_overflow && !failed_filters && 2470 vsi->active_filters < vsi->promisc_threshold) { 2471 dev_info(&pf->pdev->dev, 2472 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2473 vsi_name); 2474 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2475 vsi->promisc_threshold = 0; 2476 } 2477 2478 /* if the VF is not trusted do not do promisc */ 2479 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2480 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2481 goto out; 2482 } 2483 2484 new_overflow = test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2485 2486 /* If we are entering overflow promiscuous, we need to calculate a new 2487 * threshold for when we are safe to exit 2488 */ 2489 if (!old_overflow && new_overflow) 2490 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2491 2492 /* check for changes in promiscuous modes */ 2493 if (changed_flags & IFF_ALLMULTI) { 2494 bool cur_multipromisc; 2495 2496 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2497 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2498 vsi->seid, 2499 cur_multipromisc, 2500 NULL); 2501 if (aq_ret) { 2502 retval = i40e_aq_rc_to_posix(aq_ret, 2503 hw->aq.asq_last_status); 2504 dev_info(&pf->pdev->dev, 2505 "set multi promisc failed on %s, err %s aq_err %s\n", 2506 vsi_name, 2507 i40e_stat_str(hw, aq_ret), 2508 i40e_aq_str(hw, hw->aq.asq_last_status)); 2509 } 2510 } 2511 2512 if ((changed_flags & IFF_PROMISC) || old_overflow != new_overflow) { 2513 bool cur_promisc; 2514 2515 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2516 new_overflow); 2517 aq_ret = i40e_set_promiscuous(pf, cur_promisc); 2518 if (aq_ret) { 2519 retval = i40e_aq_rc_to_posix(aq_ret, 2520 hw->aq.asq_last_status); 2521 dev_info(&pf->pdev->dev, 2522 "Setting promiscuous %s failed on %s, err %s aq_err %s\n", 2523 cur_promisc ? "on" : "off", 2524 vsi_name, 2525 i40e_stat_str(hw, aq_ret), 2526 i40e_aq_str(hw, hw->aq.asq_last_status)); 2527 } 2528 } 2529 out: 2530 /* if something went wrong then set the changed flag so we try again */ 2531 if (retval) 2532 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2533 2534 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2535 return retval; 2536 2537 err_no_memory: 2538 /* Restore elements on the temporary add and delete lists */ 2539 spin_lock_bh(&vsi->mac_filter_hash_lock); 2540 err_no_memory_locked: 2541 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2542 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2543 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2544 2545 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2546 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2547 return -ENOMEM; 2548 } 2549 2550 /** 2551 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2552 * @pf: board private structure 2553 **/ 2554 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2555 { 2556 int v; 2557 2558 if (!pf) 2559 return; 2560 if (!test_and_clear_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state)) 2561 return; 2562 2563 for (v = 0; v < pf->num_alloc_vsi; v++) { 2564 if (pf->vsi[v] && 2565 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2566 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2567 2568 if (ret) { 2569 /* come back and try again later */ 2570 set_bit(__I40E_MACVLAN_SYNC_PENDING, 2571 pf->state); 2572 break; 2573 } 2574 } 2575 } 2576 } 2577 2578 /** 2579 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2580 * @vsi: the vsi 2581 **/ 2582 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2583 { 2584 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2585 return I40E_RXBUFFER_2048; 2586 else 2587 return I40E_RXBUFFER_3072; 2588 } 2589 2590 /** 2591 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2592 * @netdev: network interface device structure 2593 * @new_mtu: new value for maximum frame size 2594 * 2595 * Returns 0 on success, negative on failure 2596 **/ 2597 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2598 { 2599 struct i40e_netdev_priv *np = netdev_priv(netdev); 2600 struct i40e_vsi *vsi = np->vsi; 2601 struct i40e_pf *pf = vsi->back; 2602 2603 if (i40e_enabled_xdp_vsi(vsi)) { 2604 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2605 2606 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2607 return -EINVAL; 2608 } 2609 2610 netdev_info(netdev, "changing MTU from %d to %d\n", 2611 netdev->mtu, new_mtu); 2612 netdev->mtu = new_mtu; 2613 if (netif_running(netdev)) 2614 i40e_vsi_reinit_locked(vsi); 2615 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 2616 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 2617 return 0; 2618 } 2619 2620 /** 2621 * i40e_ioctl - Access the hwtstamp interface 2622 * @netdev: network interface device structure 2623 * @ifr: interface request data 2624 * @cmd: ioctl command 2625 **/ 2626 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2627 { 2628 struct i40e_netdev_priv *np = netdev_priv(netdev); 2629 struct i40e_pf *pf = np->vsi->back; 2630 2631 switch (cmd) { 2632 case SIOCGHWTSTAMP: 2633 return i40e_ptp_get_ts_config(pf, ifr); 2634 case SIOCSHWTSTAMP: 2635 return i40e_ptp_set_ts_config(pf, ifr); 2636 default: 2637 return -EOPNOTSUPP; 2638 } 2639 } 2640 2641 /** 2642 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2643 * @vsi: the vsi being adjusted 2644 **/ 2645 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2646 { 2647 struct i40e_vsi_context ctxt; 2648 i40e_status ret; 2649 2650 if ((vsi->info.valid_sections & 2651 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2652 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2653 return; /* already enabled */ 2654 2655 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2656 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2657 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2658 2659 ctxt.seid = vsi->seid; 2660 ctxt.info = vsi->info; 2661 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2662 if (ret) { 2663 dev_info(&vsi->back->pdev->dev, 2664 "update vlan stripping failed, err %s aq_err %s\n", 2665 i40e_stat_str(&vsi->back->hw, ret), 2666 i40e_aq_str(&vsi->back->hw, 2667 vsi->back->hw.aq.asq_last_status)); 2668 } 2669 } 2670 2671 /** 2672 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2673 * @vsi: the vsi being adjusted 2674 **/ 2675 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2676 { 2677 struct i40e_vsi_context ctxt; 2678 i40e_status ret; 2679 2680 if ((vsi->info.valid_sections & 2681 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2682 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2683 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2684 return; /* already disabled */ 2685 2686 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2687 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2688 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2689 2690 ctxt.seid = vsi->seid; 2691 ctxt.info = vsi->info; 2692 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2693 if (ret) { 2694 dev_info(&vsi->back->pdev->dev, 2695 "update vlan stripping failed, err %s aq_err %s\n", 2696 i40e_stat_str(&vsi->back->hw, ret), 2697 i40e_aq_str(&vsi->back->hw, 2698 vsi->back->hw.aq.asq_last_status)); 2699 } 2700 } 2701 2702 /** 2703 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2704 * @vsi: the vsi being configured 2705 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2706 * 2707 * This is a helper function for adding a new MAC/VLAN filter with the 2708 * specified VLAN for each existing MAC address already in the hash table. 2709 * This function does *not* perform any accounting to update filters based on 2710 * VLAN mode. 2711 * 2712 * NOTE: this function expects to be called while under the 2713 * mac_filter_hash_lock 2714 **/ 2715 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2716 { 2717 struct i40e_mac_filter *f, *add_f; 2718 struct hlist_node *h; 2719 int bkt; 2720 2721 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2722 if (f->state == I40E_FILTER_REMOVE) 2723 continue; 2724 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2725 if (!add_f) { 2726 dev_info(&vsi->back->pdev->dev, 2727 "Could not add vlan filter %d for %pM\n", 2728 vid, f->macaddr); 2729 return -ENOMEM; 2730 } 2731 } 2732 2733 return 0; 2734 } 2735 2736 /** 2737 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2738 * @vsi: the VSI being configured 2739 * @vid: VLAN id to be added 2740 **/ 2741 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2742 { 2743 int err; 2744 2745 if (vsi->info.pvid) 2746 return -EINVAL; 2747 2748 /* The network stack will attempt to add VID=0, with the intention to 2749 * receive priority tagged packets with a VLAN of 0. Our HW receives 2750 * these packets by default when configured to receive untagged 2751 * packets, so we don't need to add a filter for this case. 2752 * Additionally, HW interprets adding a VID=0 filter as meaning to 2753 * receive *only* tagged traffic and stops receiving untagged traffic. 2754 * Thus, we do not want to actually add a filter for VID=0 2755 */ 2756 if (!vid) 2757 return 0; 2758 2759 /* Locked once because all functions invoked below iterates list*/ 2760 spin_lock_bh(&vsi->mac_filter_hash_lock); 2761 err = i40e_add_vlan_all_mac(vsi, vid); 2762 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2763 if (err) 2764 return err; 2765 2766 /* schedule our worker thread which will take care of 2767 * applying the new filter changes 2768 */ 2769 i40e_service_event_schedule(vsi->back); 2770 return 0; 2771 } 2772 2773 /** 2774 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2775 * @vsi: the vsi being configured 2776 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2777 * 2778 * This function should be used to remove all VLAN filters which match the 2779 * given VID. It does not schedule the service event and does not take the 2780 * mac_filter_hash_lock so it may be combined with other operations under 2781 * a single invocation of the mac_filter_hash_lock. 2782 * 2783 * NOTE: this function expects to be called while under the 2784 * mac_filter_hash_lock 2785 */ 2786 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2787 { 2788 struct i40e_mac_filter *f; 2789 struct hlist_node *h; 2790 int bkt; 2791 2792 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2793 if (f->vlan == vid) 2794 __i40e_del_filter(vsi, f); 2795 } 2796 } 2797 2798 /** 2799 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2800 * @vsi: the VSI being configured 2801 * @vid: VLAN id to be removed 2802 **/ 2803 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2804 { 2805 if (!vid || vsi->info.pvid) 2806 return; 2807 2808 spin_lock_bh(&vsi->mac_filter_hash_lock); 2809 i40e_rm_vlan_all_mac(vsi, vid); 2810 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2811 2812 /* schedule our worker thread which will take care of 2813 * applying the new filter changes 2814 */ 2815 i40e_service_event_schedule(vsi->back); 2816 } 2817 2818 /** 2819 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2820 * @netdev: network interface to be adjusted 2821 * @proto: unused protocol value 2822 * @vid: vlan id to be added 2823 * 2824 * net_device_ops implementation for adding vlan ids 2825 **/ 2826 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2827 __always_unused __be16 proto, u16 vid) 2828 { 2829 struct i40e_netdev_priv *np = netdev_priv(netdev); 2830 struct i40e_vsi *vsi = np->vsi; 2831 int ret = 0; 2832 2833 if (vid >= VLAN_N_VID) 2834 return -EINVAL; 2835 2836 ret = i40e_vsi_add_vlan(vsi, vid); 2837 if (!ret) 2838 set_bit(vid, vsi->active_vlans); 2839 2840 return ret; 2841 } 2842 2843 /** 2844 * i40e_vlan_rx_add_vid_up - Add a vlan id filter to HW offload in UP path 2845 * @netdev: network interface to be adjusted 2846 * @proto: unused protocol value 2847 * @vid: vlan id to be added 2848 **/ 2849 static void i40e_vlan_rx_add_vid_up(struct net_device *netdev, 2850 __always_unused __be16 proto, u16 vid) 2851 { 2852 struct i40e_netdev_priv *np = netdev_priv(netdev); 2853 struct i40e_vsi *vsi = np->vsi; 2854 2855 if (vid >= VLAN_N_VID) 2856 return; 2857 set_bit(vid, vsi->active_vlans); 2858 } 2859 2860 /** 2861 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2862 * @netdev: network interface to be adjusted 2863 * @proto: unused protocol value 2864 * @vid: vlan id to be removed 2865 * 2866 * net_device_ops implementation for removing vlan ids 2867 **/ 2868 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2869 __always_unused __be16 proto, u16 vid) 2870 { 2871 struct i40e_netdev_priv *np = netdev_priv(netdev); 2872 struct i40e_vsi *vsi = np->vsi; 2873 2874 /* return code is ignored as there is nothing a user 2875 * can do about failure to remove and a log message was 2876 * already printed from the other function 2877 */ 2878 i40e_vsi_kill_vlan(vsi, vid); 2879 2880 clear_bit(vid, vsi->active_vlans); 2881 2882 return 0; 2883 } 2884 2885 /** 2886 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2887 * @vsi: the vsi being brought back up 2888 **/ 2889 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2890 { 2891 u16 vid; 2892 2893 if (!vsi->netdev) 2894 return; 2895 2896 if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 2897 i40e_vlan_stripping_enable(vsi); 2898 else 2899 i40e_vlan_stripping_disable(vsi); 2900 2901 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2902 i40e_vlan_rx_add_vid_up(vsi->netdev, htons(ETH_P_8021Q), 2903 vid); 2904 } 2905 2906 /** 2907 * i40e_vsi_add_pvid - Add pvid for the VSI 2908 * @vsi: the vsi being adjusted 2909 * @vid: the vlan id to set as a PVID 2910 **/ 2911 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2912 { 2913 struct i40e_vsi_context ctxt; 2914 i40e_status ret; 2915 2916 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2917 vsi->info.pvid = cpu_to_le16(vid); 2918 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2919 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2920 I40E_AQ_VSI_PVLAN_EMOD_STR; 2921 2922 ctxt.seid = vsi->seid; 2923 ctxt.info = vsi->info; 2924 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2925 if (ret) { 2926 dev_info(&vsi->back->pdev->dev, 2927 "add pvid failed, err %s aq_err %s\n", 2928 i40e_stat_str(&vsi->back->hw, ret), 2929 i40e_aq_str(&vsi->back->hw, 2930 vsi->back->hw.aq.asq_last_status)); 2931 return -ENOENT; 2932 } 2933 2934 return 0; 2935 } 2936 2937 /** 2938 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2939 * @vsi: the vsi being adjusted 2940 * 2941 * Just use the vlan_rx_register() service to put it back to normal 2942 **/ 2943 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2944 { 2945 i40e_vlan_stripping_disable(vsi); 2946 2947 vsi->info.pvid = 0; 2948 } 2949 2950 /** 2951 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2952 * @vsi: ptr to the VSI 2953 * 2954 * If this function returns with an error, then it's possible one or 2955 * more of the rings is populated (while the rest are not). It is the 2956 * callers duty to clean those orphaned rings. 2957 * 2958 * Return 0 on success, negative on failure 2959 **/ 2960 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2961 { 2962 int i, err = 0; 2963 2964 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2965 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2966 2967 if (!i40e_enabled_xdp_vsi(vsi)) 2968 return err; 2969 2970 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2971 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 2972 2973 return err; 2974 } 2975 2976 /** 2977 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 2978 * @vsi: ptr to the VSI 2979 * 2980 * Free VSI's transmit software resources 2981 **/ 2982 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 2983 { 2984 int i; 2985 2986 if (vsi->tx_rings) { 2987 for (i = 0; i < vsi->num_queue_pairs; i++) 2988 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 2989 i40e_free_tx_resources(vsi->tx_rings[i]); 2990 } 2991 2992 if (vsi->xdp_rings) { 2993 for (i = 0; i < vsi->num_queue_pairs; i++) 2994 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 2995 i40e_free_tx_resources(vsi->xdp_rings[i]); 2996 } 2997 } 2998 2999 /** 3000 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 3001 * @vsi: ptr to the VSI 3002 * 3003 * If this function returns with an error, then it's possible one or 3004 * more of the rings is populated (while the rest are not). It is the 3005 * callers duty to clean those orphaned rings. 3006 * 3007 * Return 0 on success, negative on failure 3008 **/ 3009 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 3010 { 3011 int i, err = 0; 3012 3013 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3014 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 3015 return err; 3016 } 3017 3018 /** 3019 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 3020 * @vsi: ptr to the VSI 3021 * 3022 * Free all receive software resources 3023 **/ 3024 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 3025 { 3026 int i; 3027 3028 if (!vsi->rx_rings) 3029 return; 3030 3031 for (i = 0; i < vsi->num_queue_pairs; i++) 3032 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 3033 i40e_free_rx_resources(vsi->rx_rings[i]); 3034 } 3035 3036 /** 3037 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 3038 * @ring: The Tx ring to configure 3039 * 3040 * This enables/disables XPS for a given Tx descriptor ring 3041 * based on the TCs enabled for the VSI that ring belongs to. 3042 **/ 3043 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 3044 { 3045 int cpu; 3046 3047 if (!ring->q_vector || !ring->netdev || ring->ch) 3048 return; 3049 3050 /* We only initialize XPS once, so as not to overwrite user settings */ 3051 if (test_and_set_bit(__I40E_TX_XPS_INIT_DONE, ring->state)) 3052 return; 3053 3054 cpu = cpumask_local_spread(ring->q_vector->v_idx, -1); 3055 netif_set_xps_queue(ring->netdev, get_cpu_mask(cpu), 3056 ring->queue_index); 3057 } 3058 3059 /** 3060 * i40e_configure_tx_ring - Configure a transmit ring context and rest 3061 * @ring: The Tx ring to configure 3062 * 3063 * Configure the Tx descriptor ring in the HMC context. 3064 **/ 3065 static int i40e_configure_tx_ring(struct i40e_ring *ring) 3066 { 3067 struct i40e_vsi *vsi = ring->vsi; 3068 u16 pf_q = vsi->base_queue + ring->queue_index; 3069 struct i40e_hw *hw = &vsi->back->hw; 3070 struct i40e_hmc_obj_txq tx_ctx; 3071 i40e_status err = 0; 3072 u32 qtx_ctl = 0; 3073 3074 /* some ATR related tx ring init */ 3075 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 3076 ring->atr_sample_rate = vsi->back->atr_sample_rate; 3077 ring->atr_count = 0; 3078 } else { 3079 ring->atr_sample_rate = 0; 3080 } 3081 3082 /* configure XPS */ 3083 i40e_config_xps_tx_ring(ring); 3084 3085 /* clear the context structure first */ 3086 memset(&tx_ctx, 0, sizeof(tx_ctx)); 3087 3088 tx_ctx.new_context = 1; 3089 tx_ctx.base = (ring->dma / 128); 3090 tx_ctx.qlen = ring->count; 3091 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 3092 I40E_FLAG_FD_ATR_ENABLED)); 3093 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 3094 /* FDIR VSI tx ring can still use RS bit and writebacks */ 3095 if (vsi->type != I40E_VSI_FDIR) 3096 tx_ctx.head_wb_ena = 1; 3097 tx_ctx.head_wb_addr = ring->dma + 3098 (ring->count * sizeof(struct i40e_tx_desc)); 3099 3100 /* As part of VSI creation/update, FW allocates certain 3101 * Tx arbitration queue sets for each TC enabled for 3102 * the VSI. The FW returns the handles to these queue 3103 * sets as part of the response buffer to Add VSI, 3104 * Update VSI, etc. AQ commands. It is expected that 3105 * these queue set handles be associated with the Tx 3106 * queues by the driver as part of the TX queue context 3107 * initialization. This has to be done regardless of 3108 * DCB as by default everything is mapped to TC0. 3109 */ 3110 3111 if (ring->ch) 3112 tx_ctx.rdylist = 3113 le16_to_cpu(ring->ch->info.qs_handle[ring->dcb_tc]); 3114 3115 else 3116 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 3117 3118 tx_ctx.rdylist_act = 0; 3119 3120 /* clear the context in the HMC */ 3121 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 3122 if (err) { 3123 dev_info(&vsi->back->pdev->dev, 3124 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 3125 ring->queue_index, pf_q, err); 3126 return -ENOMEM; 3127 } 3128 3129 /* set the context in the HMC */ 3130 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 3131 if (err) { 3132 dev_info(&vsi->back->pdev->dev, 3133 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 3134 ring->queue_index, pf_q, err); 3135 return -ENOMEM; 3136 } 3137 3138 /* Now associate this queue with this PCI function */ 3139 if (ring->ch) { 3140 if (ring->ch->type == I40E_VSI_VMDQ2) 3141 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3142 else 3143 return -EINVAL; 3144 3145 qtx_ctl |= (ring->ch->vsi_number << 3146 I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3147 I40E_QTX_CTL_VFVM_INDX_MASK; 3148 } else { 3149 if (vsi->type == I40E_VSI_VMDQ2) { 3150 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 3151 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 3152 I40E_QTX_CTL_VFVM_INDX_MASK; 3153 } else { 3154 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 3155 } 3156 } 3157 3158 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 3159 I40E_QTX_CTL_PF_INDX_MASK); 3160 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 3161 i40e_flush(hw); 3162 3163 /* cache tail off for easier writes later */ 3164 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 3165 3166 return 0; 3167 } 3168 3169 /** 3170 * i40e_configure_rx_ring - Configure a receive ring context 3171 * @ring: The Rx ring to configure 3172 * 3173 * Configure the Rx descriptor ring in the HMC context. 3174 **/ 3175 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3176 { 3177 struct i40e_vsi *vsi = ring->vsi; 3178 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3179 u16 pf_q = vsi->base_queue + ring->queue_index; 3180 struct i40e_hw *hw = &vsi->back->hw; 3181 struct i40e_hmc_obj_rxq rx_ctx; 3182 i40e_status err = 0; 3183 3184 bitmap_zero(ring->state, __I40E_RING_STATE_NBITS); 3185 3186 /* clear the context structure first */ 3187 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3188 3189 ring->rx_buf_len = vsi->rx_buf_len; 3190 3191 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3192 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3193 3194 rx_ctx.base = (ring->dma / 128); 3195 rx_ctx.qlen = ring->count; 3196 3197 /* use 32 byte descriptors */ 3198 rx_ctx.dsize = 1; 3199 3200 /* descriptor type is always zero 3201 * rx_ctx.dtype = 0; 3202 */ 3203 rx_ctx.hsplit_0 = 0; 3204 3205 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3206 if (hw->revision_id == 0) 3207 rx_ctx.lrxqthresh = 0; 3208 else 3209 rx_ctx.lrxqthresh = 1; 3210 rx_ctx.crcstrip = 1; 3211 rx_ctx.l2tsel = 1; 3212 /* this controls whether VLAN is stripped from inner headers */ 3213 rx_ctx.showiv = 0; 3214 /* set the prefena field to 1 because the manual says to */ 3215 rx_ctx.prefena = 1; 3216 3217 /* clear the context in the HMC */ 3218 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3219 if (err) { 3220 dev_info(&vsi->back->pdev->dev, 3221 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3222 ring->queue_index, pf_q, err); 3223 return -ENOMEM; 3224 } 3225 3226 /* set the context in the HMC */ 3227 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3228 if (err) { 3229 dev_info(&vsi->back->pdev->dev, 3230 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3231 ring->queue_index, pf_q, err); 3232 return -ENOMEM; 3233 } 3234 3235 /* configure Rx buffer alignment */ 3236 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3237 clear_ring_build_skb_enabled(ring); 3238 else 3239 set_ring_build_skb_enabled(ring); 3240 3241 /* cache tail for quicker writes, and clear the reg before use */ 3242 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3243 writel(0, ring->tail); 3244 3245 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3246 3247 return 0; 3248 } 3249 3250 /** 3251 * i40e_vsi_configure_tx - Configure the VSI for Tx 3252 * @vsi: VSI structure describing this set of rings and resources 3253 * 3254 * Configure the Tx VSI for operation. 3255 **/ 3256 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3257 { 3258 int err = 0; 3259 u16 i; 3260 3261 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3262 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3263 3264 if (!i40e_enabled_xdp_vsi(vsi)) 3265 return err; 3266 3267 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3268 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3269 3270 return err; 3271 } 3272 3273 /** 3274 * i40e_vsi_configure_rx - Configure the VSI for Rx 3275 * @vsi: the VSI being configured 3276 * 3277 * Configure the Rx VSI for operation. 3278 **/ 3279 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3280 { 3281 int err = 0; 3282 u16 i; 3283 3284 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3285 vsi->max_frame = I40E_MAX_RXBUFFER; 3286 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3287 #if (PAGE_SIZE < 8192) 3288 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3289 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3290 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3291 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3292 #endif 3293 } else { 3294 vsi->max_frame = I40E_MAX_RXBUFFER; 3295 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3296 I40E_RXBUFFER_2048; 3297 } 3298 3299 /* set up individual rings */ 3300 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3301 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3302 3303 return err; 3304 } 3305 3306 /** 3307 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3308 * @vsi: ptr to the VSI 3309 **/ 3310 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3311 { 3312 struct i40e_ring *tx_ring, *rx_ring; 3313 u16 qoffset, qcount; 3314 int i, n; 3315 3316 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3317 /* Reset the TC information */ 3318 for (i = 0; i < vsi->num_queue_pairs; i++) { 3319 rx_ring = vsi->rx_rings[i]; 3320 tx_ring = vsi->tx_rings[i]; 3321 rx_ring->dcb_tc = 0; 3322 tx_ring->dcb_tc = 0; 3323 } 3324 return; 3325 } 3326 3327 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3328 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3329 continue; 3330 3331 qoffset = vsi->tc_config.tc_info[n].qoffset; 3332 qcount = vsi->tc_config.tc_info[n].qcount; 3333 for (i = qoffset; i < (qoffset + qcount); i++) { 3334 rx_ring = vsi->rx_rings[i]; 3335 tx_ring = vsi->tx_rings[i]; 3336 rx_ring->dcb_tc = n; 3337 tx_ring->dcb_tc = n; 3338 } 3339 } 3340 } 3341 3342 /** 3343 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3344 * @vsi: ptr to the VSI 3345 **/ 3346 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3347 { 3348 if (vsi->netdev) 3349 i40e_set_rx_mode(vsi->netdev); 3350 } 3351 3352 /** 3353 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3354 * @vsi: Pointer to the targeted VSI 3355 * 3356 * This function replays the hlist on the hw where all the SB Flow Director 3357 * filters were saved. 3358 **/ 3359 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3360 { 3361 struct i40e_fdir_filter *filter; 3362 struct i40e_pf *pf = vsi->back; 3363 struct hlist_node *node; 3364 3365 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3366 return; 3367 3368 /* Reset FDir counters as we're replaying all existing filters */ 3369 pf->fd_tcp4_filter_cnt = 0; 3370 pf->fd_udp4_filter_cnt = 0; 3371 pf->fd_sctp4_filter_cnt = 0; 3372 pf->fd_ip4_filter_cnt = 0; 3373 3374 hlist_for_each_entry_safe(filter, node, 3375 &pf->fdir_filter_list, fdir_node) { 3376 i40e_add_del_fdir(vsi, filter, true); 3377 } 3378 } 3379 3380 /** 3381 * i40e_vsi_configure - Set up the VSI for action 3382 * @vsi: the VSI being configured 3383 **/ 3384 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3385 { 3386 int err; 3387 3388 i40e_set_vsi_rx_mode(vsi); 3389 i40e_restore_vlan(vsi); 3390 i40e_vsi_config_dcb_rings(vsi); 3391 err = i40e_vsi_configure_tx(vsi); 3392 if (!err) 3393 err = i40e_vsi_configure_rx(vsi); 3394 3395 return err; 3396 } 3397 3398 /** 3399 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3400 * @vsi: the VSI being configured 3401 **/ 3402 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3403 { 3404 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3405 struct i40e_pf *pf = vsi->back; 3406 struct i40e_hw *hw = &pf->hw; 3407 u16 vector; 3408 int i, q; 3409 u32 qp; 3410 3411 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3412 * and PFINT_LNKLSTn registers, e.g.: 3413 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3414 */ 3415 qp = vsi->base_queue; 3416 vector = vsi->base_vector; 3417 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3418 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3419 3420 q_vector->rx.next_update = jiffies + 1; 3421 q_vector->rx.target_itr = 3422 ITR_TO_REG(vsi->rx_rings[i]->itr_setting); 3423 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3424 q_vector->rx.target_itr); 3425 q_vector->rx.current_itr = q_vector->rx.target_itr; 3426 3427 q_vector->tx.next_update = jiffies + 1; 3428 q_vector->tx.target_itr = 3429 ITR_TO_REG(vsi->tx_rings[i]->itr_setting); 3430 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3431 q_vector->tx.target_itr); 3432 q_vector->tx.current_itr = q_vector->tx.target_itr; 3433 3434 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3435 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3436 3437 /* Linked list for the queuepairs assigned to this vector */ 3438 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3439 for (q = 0; q < q_vector->num_ringpairs; q++) { 3440 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3441 u32 val; 3442 3443 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3444 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3445 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3446 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3447 (I40E_QUEUE_TYPE_TX << 3448 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3449 3450 wr32(hw, I40E_QINT_RQCTL(qp), val); 3451 3452 if (has_xdp) { 3453 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3454 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3455 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3456 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3457 (I40E_QUEUE_TYPE_TX << 3458 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3459 3460 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3461 } 3462 3463 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3464 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3465 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3466 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3467 (I40E_QUEUE_TYPE_RX << 3468 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3469 3470 /* Terminate the linked list */ 3471 if (q == (q_vector->num_ringpairs - 1)) 3472 val |= (I40E_QUEUE_END_OF_LIST << 3473 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3474 3475 wr32(hw, I40E_QINT_TQCTL(qp), val); 3476 qp++; 3477 } 3478 } 3479 3480 i40e_flush(hw); 3481 } 3482 3483 /** 3484 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3485 * @pf: pointer to private device data structure 3486 **/ 3487 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3488 { 3489 struct i40e_hw *hw = &pf->hw; 3490 u32 val; 3491 3492 /* clear things first */ 3493 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3494 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3495 3496 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3497 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3498 I40E_PFINT_ICR0_ENA_GRST_MASK | 3499 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3500 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3501 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3502 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3503 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3504 3505 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3506 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3507 3508 if (pf->flags & I40E_FLAG_PTP) 3509 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3510 3511 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3512 3513 /* SW_ITR_IDX = 0, but don't change INTENA */ 3514 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3515 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3516 3517 /* OTHER_ITR_IDX = 0 */ 3518 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3519 } 3520 3521 /** 3522 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3523 * @vsi: the VSI being configured 3524 **/ 3525 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3526 { 3527 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3528 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3529 struct i40e_pf *pf = vsi->back; 3530 struct i40e_hw *hw = &pf->hw; 3531 u32 val; 3532 3533 /* set the ITR configuration */ 3534 q_vector->rx.next_update = jiffies + 1; 3535 q_vector->rx.target_itr = ITR_TO_REG(vsi->rx_rings[0]->itr_setting); 3536 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.target_itr); 3537 q_vector->rx.current_itr = q_vector->rx.target_itr; 3538 q_vector->tx.next_update = jiffies + 1; 3539 q_vector->tx.target_itr = ITR_TO_REG(vsi->tx_rings[0]->itr_setting); 3540 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.target_itr); 3541 q_vector->tx.current_itr = q_vector->tx.target_itr; 3542 3543 i40e_enable_misc_int_causes(pf); 3544 3545 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3546 wr32(hw, I40E_PFINT_LNKLST0, 0); 3547 3548 /* Associate the queue pair to the vector and enable the queue int */ 3549 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3550 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3551 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3552 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3553 3554 wr32(hw, I40E_QINT_RQCTL(0), val); 3555 3556 if (i40e_enabled_xdp_vsi(vsi)) { 3557 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3558 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3559 (I40E_QUEUE_TYPE_TX 3560 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3561 3562 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3563 } 3564 3565 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3566 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3567 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3568 3569 wr32(hw, I40E_QINT_TQCTL(0), val); 3570 i40e_flush(hw); 3571 } 3572 3573 /** 3574 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3575 * @pf: board private structure 3576 **/ 3577 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3578 { 3579 struct i40e_hw *hw = &pf->hw; 3580 3581 wr32(hw, I40E_PFINT_DYN_CTL0, 3582 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3583 i40e_flush(hw); 3584 } 3585 3586 /** 3587 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3588 * @pf: board private structure 3589 **/ 3590 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 3591 { 3592 struct i40e_hw *hw = &pf->hw; 3593 u32 val; 3594 3595 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3596 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 3597 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3598 3599 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3600 i40e_flush(hw); 3601 } 3602 3603 /** 3604 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3605 * @irq: interrupt number 3606 * @data: pointer to a q_vector 3607 **/ 3608 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3609 { 3610 struct i40e_q_vector *q_vector = data; 3611 3612 if (!q_vector->tx.ring && !q_vector->rx.ring) 3613 return IRQ_HANDLED; 3614 3615 napi_schedule_irqoff(&q_vector->napi); 3616 3617 return IRQ_HANDLED; 3618 } 3619 3620 /** 3621 * i40e_irq_affinity_notify - Callback for affinity changes 3622 * @notify: context as to what irq was changed 3623 * @mask: the new affinity mask 3624 * 3625 * This is a callback function used by the irq_set_affinity_notifier function 3626 * so that we may register to receive changes to the irq affinity masks. 3627 **/ 3628 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3629 const cpumask_t *mask) 3630 { 3631 struct i40e_q_vector *q_vector = 3632 container_of(notify, struct i40e_q_vector, affinity_notify); 3633 3634 cpumask_copy(&q_vector->affinity_mask, mask); 3635 } 3636 3637 /** 3638 * i40e_irq_affinity_release - Callback for affinity notifier release 3639 * @ref: internal core kernel usage 3640 * 3641 * This is a callback function used by the irq_set_affinity_notifier function 3642 * to inform the current notification subscriber that they will no longer 3643 * receive notifications. 3644 **/ 3645 static void i40e_irq_affinity_release(struct kref *ref) {} 3646 3647 /** 3648 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3649 * @vsi: the VSI being configured 3650 * @basename: name for the vector 3651 * 3652 * Allocates MSI-X vectors and requests interrupts from the kernel. 3653 **/ 3654 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3655 { 3656 int q_vectors = vsi->num_q_vectors; 3657 struct i40e_pf *pf = vsi->back; 3658 int base = vsi->base_vector; 3659 int rx_int_idx = 0; 3660 int tx_int_idx = 0; 3661 int vector, err; 3662 int irq_num; 3663 int cpu; 3664 3665 for (vector = 0; vector < q_vectors; vector++) { 3666 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3667 3668 irq_num = pf->msix_entries[base + vector].vector; 3669 3670 if (q_vector->tx.ring && q_vector->rx.ring) { 3671 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3672 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3673 tx_int_idx++; 3674 } else if (q_vector->rx.ring) { 3675 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3676 "%s-%s-%d", basename, "rx", rx_int_idx++); 3677 } else if (q_vector->tx.ring) { 3678 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3679 "%s-%s-%d", basename, "tx", tx_int_idx++); 3680 } else { 3681 /* skip this unused q_vector */ 3682 continue; 3683 } 3684 err = request_irq(irq_num, 3685 vsi->irq_handler, 3686 0, 3687 q_vector->name, 3688 q_vector); 3689 if (err) { 3690 dev_info(&pf->pdev->dev, 3691 "MSIX request_irq failed, error: %d\n", err); 3692 goto free_queue_irqs; 3693 } 3694 3695 /* register for affinity change notifications */ 3696 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3697 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3698 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3699 /* Spread affinity hints out across online CPUs. 3700 * 3701 * get_cpu_mask returns a static constant mask with 3702 * a permanent lifetime so it's ok to pass to 3703 * irq_set_affinity_hint without making a copy. 3704 */ 3705 cpu = cpumask_local_spread(q_vector->v_idx, -1); 3706 irq_set_affinity_hint(irq_num, get_cpu_mask(cpu)); 3707 } 3708 3709 vsi->irqs_ready = true; 3710 return 0; 3711 3712 free_queue_irqs: 3713 while (vector) { 3714 vector--; 3715 irq_num = pf->msix_entries[base + vector].vector; 3716 irq_set_affinity_notifier(irq_num, NULL); 3717 irq_set_affinity_hint(irq_num, NULL); 3718 free_irq(irq_num, &vsi->q_vectors[vector]); 3719 } 3720 return err; 3721 } 3722 3723 /** 3724 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3725 * @vsi: the VSI being un-configured 3726 **/ 3727 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3728 { 3729 struct i40e_pf *pf = vsi->back; 3730 struct i40e_hw *hw = &pf->hw; 3731 int base = vsi->base_vector; 3732 int i; 3733 3734 /* disable interrupt causation from each queue */ 3735 for (i = 0; i < vsi->num_queue_pairs; i++) { 3736 u32 val; 3737 3738 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3739 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3740 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3741 3742 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3743 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3744 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3745 3746 if (!i40e_enabled_xdp_vsi(vsi)) 3747 continue; 3748 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3749 } 3750 3751 /* disable each interrupt */ 3752 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3753 for (i = vsi->base_vector; 3754 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3755 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3756 3757 i40e_flush(hw); 3758 for (i = 0; i < vsi->num_q_vectors; i++) 3759 synchronize_irq(pf->msix_entries[i + base].vector); 3760 } else { 3761 /* Legacy and MSI mode - this stops all interrupt handling */ 3762 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3763 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3764 i40e_flush(hw); 3765 synchronize_irq(pf->pdev->irq); 3766 } 3767 } 3768 3769 /** 3770 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3771 * @vsi: the VSI being configured 3772 **/ 3773 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3774 { 3775 struct i40e_pf *pf = vsi->back; 3776 int i; 3777 3778 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3779 for (i = 0; i < vsi->num_q_vectors; i++) 3780 i40e_irq_dynamic_enable(vsi, i); 3781 } else { 3782 i40e_irq_dynamic_enable_icr0(pf); 3783 } 3784 3785 i40e_flush(&pf->hw); 3786 return 0; 3787 } 3788 3789 /** 3790 * i40e_free_misc_vector - Free the vector that handles non-queue events 3791 * @pf: board private structure 3792 **/ 3793 static void i40e_free_misc_vector(struct i40e_pf *pf) 3794 { 3795 /* Disable ICR 0 */ 3796 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3797 i40e_flush(&pf->hw); 3798 3799 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3800 synchronize_irq(pf->msix_entries[0].vector); 3801 free_irq(pf->msix_entries[0].vector, pf); 3802 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3803 } 3804 } 3805 3806 /** 3807 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3808 * @irq: interrupt number 3809 * @data: pointer to a q_vector 3810 * 3811 * This is the handler used for all MSI/Legacy interrupts, and deals 3812 * with both queue and non-queue interrupts. This is also used in 3813 * MSIX mode to handle the non-queue interrupts. 3814 **/ 3815 static irqreturn_t i40e_intr(int irq, void *data) 3816 { 3817 struct i40e_pf *pf = (struct i40e_pf *)data; 3818 struct i40e_hw *hw = &pf->hw; 3819 irqreturn_t ret = IRQ_NONE; 3820 u32 icr0, icr0_remaining; 3821 u32 val, ena_mask; 3822 3823 icr0 = rd32(hw, I40E_PFINT_ICR0); 3824 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3825 3826 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3827 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3828 goto enable_intr; 3829 3830 /* if interrupt but no bits showing, must be SWINT */ 3831 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3832 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3833 pf->sw_int_count++; 3834 3835 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3836 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3837 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3838 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3839 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3840 } 3841 3842 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3843 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3844 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3845 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3846 3847 /* We do not have a way to disarm Queue causes while leaving 3848 * interrupt enabled for all other causes, ideally 3849 * interrupt should be disabled while we are in NAPI but 3850 * this is not a performance path and napi_schedule() 3851 * can deal with rescheduling. 3852 */ 3853 if (!test_bit(__I40E_DOWN, pf->state)) 3854 napi_schedule_irqoff(&q_vector->napi); 3855 } 3856 3857 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3858 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3859 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 3860 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3861 } 3862 3863 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3864 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3865 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 3866 } 3867 3868 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3869 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3870 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 3871 } 3872 3873 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3874 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 3875 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 3876 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3877 val = rd32(hw, I40E_GLGEN_RSTAT); 3878 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3879 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3880 if (val == I40E_RESET_CORER) { 3881 pf->corer_count++; 3882 } else if (val == I40E_RESET_GLOBR) { 3883 pf->globr_count++; 3884 } else if (val == I40E_RESET_EMPR) { 3885 pf->empr_count++; 3886 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 3887 } 3888 } 3889 3890 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3891 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3892 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3893 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3894 rd32(hw, I40E_PFHMC_ERRORINFO), 3895 rd32(hw, I40E_PFHMC_ERRORDATA)); 3896 } 3897 3898 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3899 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3900 3901 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3902 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3903 i40e_ptp_tx_hwtstamp(pf); 3904 } 3905 } 3906 3907 /* If a critical error is pending we have no choice but to reset the 3908 * device. 3909 * Report and mask out any remaining unexpected interrupts. 3910 */ 3911 icr0_remaining = icr0 & ena_mask; 3912 if (icr0_remaining) { 3913 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3914 icr0_remaining); 3915 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3916 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3917 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3918 dev_info(&pf->pdev->dev, "device will be reset\n"); 3919 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 3920 i40e_service_event_schedule(pf); 3921 } 3922 ena_mask &= ~icr0_remaining; 3923 } 3924 ret = IRQ_HANDLED; 3925 3926 enable_intr: 3927 /* re-enable interrupt causes */ 3928 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3929 if (!test_bit(__I40E_DOWN, pf->state)) { 3930 i40e_service_event_schedule(pf); 3931 i40e_irq_dynamic_enable_icr0(pf); 3932 } 3933 3934 return ret; 3935 } 3936 3937 /** 3938 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3939 * @tx_ring: tx ring to clean 3940 * @budget: how many cleans we're allowed 3941 * 3942 * Returns true if there's any budget left (e.g. the clean is finished) 3943 **/ 3944 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3945 { 3946 struct i40e_vsi *vsi = tx_ring->vsi; 3947 u16 i = tx_ring->next_to_clean; 3948 struct i40e_tx_buffer *tx_buf; 3949 struct i40e_tx_desc *tx_desc; 3950 3951 tx_buf = &tx_ring->tx_bi[i]; 3952 tx_desc = I40E_TX_DESC(tx_ring, i); 3953 i -= tx_ring->count; 3954 3955 do { 3956 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3957 3958 /* if next_to_watch is not set then there is no work pending */ 3959 if (!eop_desc) 3960 break; 3961 3962 /* prevent any other reads prior to eop_desc */ 3963 smp_rmb(); 3964 3965 /* if the descriptor isn't done, no work yet to do */ 3966 if (!(eop_desc->cmd_type_offset_bsz & 3967 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3968 break; 3969 3970 /* clear next_to_watch to prevent false hangs */ 3971 tx_buf->next_to_watch = NULL; 3972 3973 tx_desc->buffer_addr = 0; 3974 tx_desc->cmd_type_offset_bsz = 0; 3975 /* move past filter desc */ 3976 tx_buf++; 3977 tx_desc++; 3978 i++; 3979 if (unlikely(!i)) { 3980 i -= tx_ring->count; 3981 tx_buf = tx_ring->tx_bi; 3982 tx_desc = I40E_TX_DESC(tx_ring, 0); 3983 } 3984 /* unmap skb header data */ 3985 dma_unmap_single(tx_ring->dev, 3986 dma_unmap_addr(tx_buf, dma), 3987 dma_unmap_len(tx_buf, len), 3988 DMA_TO_DEVICE); 3989 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 3990 kfree(tx_buf->raw_buf); 3991 3992 tx_buf->raw_buf = NULL; 3993 tx_buf->tx_flags = 0; 3994 tx_buf->next_to_watch = NULL; 3995 dma_unmap_len_set(tx_buf, len, 0); 3996 tx_desc->buffer_addr = 0; 3997 tx_desc->cmd_type_offset_bsz = 0; 3998 3999 /* move us past the eop_desc for start of next FD desc */ 4000 tx_buf++; 4001 tx_desc++; 4002 i++; 4003 if (unlikely(!i)) { 4004 i -= tx_ring->count; 4005 tx_buf = tx_ring->tx_bi; 4006 tx_desc = I40E_TX_DESC(tx_ring, 0); 4007 } 4008 4009 /* update budget accounting */ 4010 budget--; 4011 } while (likely(budget)); 4012 4013 i += tx_ring->count; 4014 tx_ring->next_to_clean = i; 4015 4016 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 4017 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 4018 4019 return budget > 0; 4020 } 4021 4022 /** 4023 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 4024 * @irq: interrupt number 4025 * @data: pointer to a q_vector 4026 **/ 4027 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 4028 { 4029 struct i40e_q_vector *q_vector = data; 4030 struct i40e_vsi *vsi; 4031 4032 if (!q_vector->tx.ring) 4033 return IRQ_HANDLED; 4034 4035 vsi = q_vector->tx.ring->vsi; 4036 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 4037 4038 return IRQ_HANDLED; 4039 } 4040 4041 /** 4042 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 4043 * @vsi: the VSI being configured 4044 * @v_idx: vector index 4045 * @qp_idx: queue pair index 4046 **/ 4047 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 4048 { 4049 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4050 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 4051 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 4052 4053 tx_ring->q_vector = q_vector; 4054 tx_ring->next = q_vector->tx.ring; 4055 q_vector->tx.ring = tx_ring; 4056 q_vector->tx.count++; 4057 4058 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 4059 if (i40e_enabled_xdp_vsi(vsi)) { 4060 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 4061 4062 xdp_ring->q_vector = q_vector; 4063 xdp_ring->next = q_vector->tx.ring; 4064 q_vector->tx.ring = xdp_ring; 4065 q_vector->tx.count++; 4066 } 4067 4068 rx_ring->q_vector = q_vector; 4069 rx_ring->next = q_vector->rx.ring; 4070 q_vector->rx.ring = rx_ring; 4071 q_vector->rx.count++; 4072 } 4073 4074 /** 4075 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 4076 * @vsi: the VSI being configured 4077 * 4078 * This function maps descriptor rings to the queue-specific vectors 4079 * we were allotted through the MSI-X enabling code. Ideally, we'd have 4080 * one vector per queue pair, but on a constrained vector budget, we 4081 * group the queue pairs as "efficiently" as possible. 4082 **/ 4083 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 4084 { 4085 int qp_remaining = vsi->num_queue_pairs; 4086 int q_vectors = vsi->num_q_vectors; 4087 int num_ringpairs; 4088 int v_start = 0; 4089 int qp_idx = 0; 4090 4091 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 4092 * group them so there are multiple queues per vector. 4093 * It is also important to go through all the vectors available to be 4094 * sure that if we don't use all the vectors, that the remaining vectors 4095 * are cleared. This is especially important when decreasing the 4096 * number of queues in use. 4097 */ 4098 for (; v_start < q_vectors; v_start++) { 4099 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 4100 4101 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 4102 4103 q_vector->num_ringpairs = num_ringpairs; 4104 q_vector->reg_idx = q_vector->v_idx + vsi->base_vector - 1; 4105 4106 q_vector->rx.count = 0; 4107 q_vector->tx.count = 0; 4108 q_vector->rx.ring = NULL; 4109 q_vector->tx.ring = NULL; 4110 4111 while (num_ringpairs--) { 4112 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 4113 qp_idx++; 4114 qp_remaining--; 4115 } 4116 } 4117 } 4118 4119 /** 4120 * i40e_vsi_request_irq - Request IRQ from the OS 4121 * @vsi: the VSI being configured 4122 * @basename: name for the vector 4123 **/ 4124 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 4125 { 4126 struct i40e_pf *pf = vsi->back; 4127 int err; 4128 4129 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4130 err = i40e_vsi_request_irq_msix(vsi, basename); 4131 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 4132 err = request_irq(pf->pdev->irq, i40e_intr, 0, 4133 pf->int_name, pf); 4134 else 4135 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 4136 pf->int_name, pf); 4137 4138 if (err) 4139 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 4140 4141 return err; 4142 } 4143 4144 #ifdef CONFIG_NET_POLL_CONTROLLER 4145 /** 4146 * i40e_netpoll - A Polling 'interrupt' handler 4147 * @netdev: network interface device structure 4148 * 4149 * This is used by netconsole to send skbs without having to re-enable 4150 * interrupts. It's not called while the normal interrupt routine is executing. 4151 **/ 4152 static void i40e_netpoll(struct net_device *netdev) 4153 { 4154 struct i40e_netdev_priv *np = netdev_priv(netdev); 4155 struct i40e_vsi *vsi = np->vsi; 4156 struct i40e_pf *pf = vsi->back; 4157 int i; 4158 4159 /* if interface is down do nothing */ 4160 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4161 return; 4162 4163 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4164 for (i = 0; i < vsi->num_q_vectors; i++) 4165 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 4166 } else { 4167 i40e_intr(pf->pdev->irq, netdev); 4168 } 4169 } 4170 #endif 4171 4172 #define I40E_QTX_ENA_WAIT_COUNT 50 4173 4174 /** 4175 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 4176 * @pf: the PF being configured 4177 * @pf_q: the PF queue 4178 * @enable: enable or disable state of the queue 4179 * 4180 * This routine will wait for the given Tx queue of the PF to reach the 4181 * enabled or disabled state. 4182 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4183 * multiple retries; else will return 0 in case of success. 4184 **/ 4185 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4186 { 4187 int i; 4188 u32 tx_reg; 4189 4190 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4191 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4192 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4193 break; 4194 4195 usleep_range(10, 20); 4196 } 4197 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4198 return -ETIMEDOUT; 4199 4200 return 0; 4201 } 4202 4203 /** 4204 * i40e_control_tx_q - Start or stop a particular Tx queue 4205 * @pf: the PF structure 4206 * @pf_q: the PF queue to configure 4207 * @enable: start or stop the queue 4208 * 4209 * This function enables or disables a single queue. Note that any delay 4210 * required after the operation is expected to be handled by the caller of 4211 * this function. 4212 **/ 4213 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4214 { 4215 struct i40e_hw *hw = &pf->hw; 4216 u32 tx_reg; 4217 int i; 4218 4219 /* warn the TX unit of coming changes */ 4220 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4221 if (!enable) 4222 usleep_range(10, 20); 4223 4224 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4225 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4226 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4227 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4228 break; 4229 usleep_range(1000, 2000); 4230 } 4231 4232 /* Skip if the queue is already in the requested state */ 4233 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4234 return; 4235 4236 /* turn on/off the queue */ 4237 if (enable) { 4238 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4239 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4240 } else { 4241 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4242 } 4243 4244 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4245 } 4246 4247 /** 4248 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4249 * @seid: VSI SEID 4250 * @pf: the PF structure 4251 * @pf_q: the PF queue to configure 4252 * @is_xdp: true if the queue is used for XDP 4253 * @enable: start or stop the queue 4254 **/ 4255 int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4256 bool is_xdp, bool enable) 4257 { 4258 int ret; 4259 4260 i40e_control_tx_q(pf, pf_q, enable); 4261 4262 /* wait for the change to finish */ 4263 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4264 if (ret) { 4265 dev_info(&pf->pdev->dev, 4266 "VSI seid %d %sTx ring %d %sable timeout\n", 4267 seid, (is_xdp ? "XDP " : ""), pf_q, 4268 (enable ? "en" : "dis")); 4269 } 4270 4271 return ret; 4272 } 4273 4274 /** 4275 * i40e_vsi_control_tx - Start or stop a VSI's rings 4276 * @vsi: the VSI being configured 4277 * @enable: start or stop the rings 4278 **/ 4279 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4280 { 4281 struct i40e_pf *pf = vsi->back; 4282 int i, pf_q, ret = 0; 4283 4284 pf_q = vsi->base_queue; 4285 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4286 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4287 pf_q, 4288 false /*is xdp*/, enable); 4289 if (ret) 4290 break; 4291 4292 if (!i40e_enabled_xdp_vsi(vsi)) 4293 continue; 4294 4295 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4296 pf_q + vsi->alloc_queue_pairs, 4297 true /*is xdp*/, enable); 4298 if (ret) 4299 break; 4300 } 4301 return ret; 4302 } 4303 4304 /** 4305 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4306 * @pf: the PF being configured 4307 * @pf_q: the PF queue 4308 * @enable: enable or disable state of the queue 4309 * 4310 * This routine will wait for the given Rx queue of the PF to reach the 4311 * enabled or disabled state. 4312 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4313 * multiple retries; else will return 0 in case of success. 4314 **/ 4315 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4316 { 4317 int i; 4318 u32 rx_reg; 4319 4320 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4321 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4322 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4323 break; 4324 4325 usleep_range(10, 20); 4326 } 4327 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4328 return -ETIMEDOUT; 4329 4330 return 0; 4331 } 4332 4333 /** 4334 * i40e_control_rx_q - Start or stop a particular Rx queue 4335 * @pf: the PF structure 4336 * @pf_q: the PF queue to configure 4337 * @enable: start or stop the queue 4338 * 4339 * This function enables or disables a single queue. Note that 4340 * any delay required after the operation is expected to be 4341 * handled by the caller of this function. 4342 **/ 4343 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4344 { 4345 struct i40e_hw *hw = &pf->hw; 4346 u32 rx_reg; 4347 int i; 4348 4349 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4350 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4351 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4352 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4353 break; 4354 usleep_range(1000, 2000); 4355 } 4356 4357 /* Skip if the queue is already in the requested state */ 4358 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4359 return; 4360 4361 /* turn on/off the queue */ 4362 if (enable) 4363 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4364 else 4365 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4366 4367 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4368 } 4369 4370 /** 4371 * i40e_control_wait_rx_q 4372 * @pf: the PF structure 4373 * @pf_q: queue being configured 4374 * @enable: start or stop the rings 4375 * 4376 * This function enables or disables a single queue along with waiting 4377 * for the change to finish. The caller of this function should handle 4378 * the delays needed in the case of disabling queues. 4379 **/ 4380 int i40e_control_wait_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4381 { 4382 int ret = 0; 4383 4384 i40e_control_rx_q(pf, pf_q, enable); 4385 4386 /* wait for the change to finish */ 4387 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4388 if (ret) 4389 return ret; 4390 4391 return ret; 4392 } 4393 4394 /** 4395 * i40e_vsi_control_rx - Start or stop a VSI's rings 4396 * @vsi: the VSI being configured 4397 * @enable: start or stop the rings 4398 **/ 4399 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4400 { 4401 struct i40e_pf *pf = vsi->back; 4402 int i, pf_q, ret = 0; 4403 4404 pf_q = vsi->base_queue; 4405 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4406 ret = i40e_control_wait_rx_q(pf, pf_q, enable); 4407 if (ret) { 4408 dev_info(&pf->pdev->dev, 4409 "VSI seid %d Rx ring %d %sable timeout\n", 4410 vsi->seid, pf_q, (enable ? "en" : "dis")); 4411 break; 4412 } 4413 } 4414 4415 /* Due to HW errata, on Rx disable only, the register can indicate done 4416 * before it really is. Needs 50ms to be sure 4417 */ 4418 if (!enable) 4419 mdelay(50); 4420 4421 return ret; 4422 } 4423 4424 /** 4425 * i40e_vsi_start_rings - Start a VSI's rings 4426 * @vsi: the VSI being configured 4427 **/ 4428 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4429 { 4430 int ret = 0; 4431 4432 /* do rx first for enable and last for disable */ 4433 ret = i40e_vsi_control_rx(vsi, true); 4434 if (ret) 4435 return ret; 4436 ret = i40e_vsi_control_tx(vsi, true); 4437 4438 return ret; 4439 } 4440 4441 /** 4442 * i40e_vsi_stop_rings - Stop a VSI's rings 4443 * @vsi: the VSI being configured 4444 **/ 4445 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4446 { 4447 /* When port TX is suspended, don't wait */ 4448 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4449 return i40e_vsi_stop_rings_no_wait(vsi); 4450 4451 /* do rx first for enable and last for disable 4452 * Ignore return value, we need to shutdown whatever we can 4453 */ 4454 i40e_vsi_control_tx(vsi, false); 4455 i40e_vsi_control_rx(vsi, false); 4456 } 4457 4458 /** 4459 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4460 * @vsi: the VSI being shutdown 4461 * 4462 * This function stops all the rings for a VSI but does not delay to verify 4463 * that rings have been disabled. It is expected that the caller is shutting 4464 * down multiple VSIs at once and will delay together for all the VSIs after 4465 * initiating the shutdown. This is particularly useful for shutting down lots 4466 * of VFs together. Otherwise, a large delay can be incurred while configuring 4467 * each VSI in serial. 4468 **/ 4469 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4470 { 4471 struct i40e_pf *pf = vsi->back; 4472 int i, pf_q; 4473 4474 pf_q = vsi->base_queue; 4475 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4476 i40e_control_tx_q(pf, pf_q, false); 4477 i40e_control_rx_q(pf, pf_q, false); 4478 } 4479 } 4480 4481 /** 4482 * i40e_vsi_free_irq - Free the irq association with the OS 4483 * @vsi: the VSI being configured 4484 **/ 4485 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4486 { 4487 struct i40e_pf *pf = vsi->back; 4488 struct i40e_hw *hw = &pf->hw; 4489 int base = vsi->base_vector; 4490 u32 val, qp; 4491 int i; 4492 4493 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4494 if (!vsi->q_vectors) 4495 return; 4496 4497 if (!vsi->irqs_ready) 4498 return; 4499 4500 vsi->irqs_ready = false; 4501 for (i = 0; i < vsi->num_q_vectors; i++) { 4502 int irq_num; 4503 u16 vector; 4504 4505 vector = i + base; 4506 irq_num = pf->msix_entries[vector].vector; 4507 4508 /* free only the irqs that were actually requested */ 4509 if (!vsi->q_vectors[i] || 4510 !vsi->q_vectors[i]->num_ringpairs) 4511 continue; 4512 4513 /* clear the affinity notifier in the IRQ descriptor */ 4514 irq_set_affinity_notifier(irq_num, NULL); 4515 /* remove our suggested affinity mask for this IRQ */ 4516 irq_set_affinity_hint(irq_num, NULL); 4517 synchronize_irq(irq_num); 4518 free_irq(irq_num, vsi->q_vectors[i]); 4519 4520 /* Tear down the interrupt queue link list 4521 * 4522 * We know that they come in pairs and always 4523 * the Rx first, then the Tx. To clear the 4524 * link list, stick the EOL value into the 4525 * next_q field of the registers. 4526 */ 4527 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4528 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4529 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4530 val |= I40E_QUEUE_END_OF_LIST 4531 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4532 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4533 4534 while (qp != I40E_QUEUE_END_OF_LIST) { 4535 u32 next; 4536 4537 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4538 4539 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4540 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4541 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4542 I40E_QINT_RQCTL_INTEVENT_MASK); 4543 4544 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4545 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4546 4547 wr32(hw, I40E_QINT_RQCTL(qp), val); 4548 4549 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4550 4551 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4552 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4553 4554 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4555 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4556 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4557 I40E_QINT_TQCTL_INTEVENT_MASK); 4558 4559 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4560 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4561 4562 wr32(hw, I40E_QINT_TQCTL(qp), val); 4563 qp = next; 4564 } 4565 } 4566 } else { 4567 free_irq(pf->pdev->irq, pf); 4568 4569 val = rd32(hw, I40E_PFINT_LNKLST0); 4570 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4571 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4572 val |= I40E_QUEUE_END_OF_LIST 4573 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4574 wr32(hw, I40E_PFINT_LNKLST0, val); 4575 4576 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4577 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4578 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4579 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4580 I40E_QINT_RQCTL_INTEVENT_MASK); 4581 4582 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4583 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4584 4585 wr32(hw, I40E_QINT_RQCTL(qp), val); 4586 4587 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4588 4589 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4590 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4591 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4592 I40E_QINT_TQCTL_INTEVENT_MASK); 4593 4594 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4595 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4596 4597 wr32(hw, I40E_QINT_TQCTL(qp), val); 4598 } 4599 } 4600 4601 /** 4602 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4603 * @vsi: the VSI being configured 4604 * @v_idx: Index of vector to be freed 4605 * 4606 * This function frees the memory allocated to the q_vector. In addition if 4607 * NAPI is enabled it will delete any references to the NAPI struct prior 4608 * to freeing the q_vector. 4609 **/ 4610 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4611 { 4612 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4613 struct i40e_ring *ring; 4614 4615 if (!q_vector) 4616 return; 4617 4618 /* disassociate q_vector from rings */ 4619 i40e_for_each_ring(ring, q_vector->tx) 4620 ring->q_vector = NULL; 4621 4622 i40e_for_each_ring(ring, q_vector->rx) 4623 ring->q_vector = NULL; 4624 4625 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4626 if (vsi->netdev) 4627 netif_napi_del(&q_vector->napi); 4628 4629 vsi->q_vectors[v_idx] = NULL; 4630 4631 kfree_rcu(q_vector, rcu); 4632 } 4633 4634 /** 4635 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4636 * @vsi: the VSI being un-configured 4637 * 4638 * This frees the memory allocated to the q_vectors and 4639 * deletes references to the NAPI struct. 4640 **/ 4641 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4642 { 4643 int v_idx; 4644 4645 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4646 i40e_free_q_vector(vsi, v_idx); 4647 } 4648 4649 /** 4650 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4651 * @pf: board private structure 4652 **/ 4653 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4654 { 4655 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4656 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4657 pci_disable_msix(pf->pdev); 4658 kfree(pf->msix_entries); 4659 pf->msix_entries = NULL; 4660 kfree(pf->irq_pile); 4661 pf->irq_pile = NULL; 4662 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4663 pci_disable_msi(pf->pdev); 4664 } 4665 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4666 } 4667 4668 /** 4669 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4670 * @pf: board private structure 4671 * 4672 * We go through and clear interrupt specific resources and reset the structure 4673 * to pre-load conditions 4674 **/ 4675 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4676 { 4677 int i; 4678 4679 i40e_free_misc_vector(pf); 4680 4681 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4682 I40E_IWARP_IRQ_PILE_ID); 4683 4684 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4685 for (i = 0; i < pf->num_alloc_vsi; i++) 4686 if (pf->vsi[i]) 4687 i40e_vsi_free_q_vectors(pf->vsi[i]); 4688 i40e_reset_interrupt_capability(pf); 4689 } 4690 4691 /** 4692 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4693 * @vsi: the VSI being configured 4694 **/ 4695 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4696 { 4697 int q_idx; 4698 4699 if (!vsi->netdev) 4700 return; 4701 4702 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4703 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4704 4705 if (q_vector->rx.ring || q_vector->tx.ring) 4706 napi_enable(&q_vector->napi); 4707 } 4708 } 4709 4710 /** 4711 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4712 * @vsi: the VSI being configured 4713 **/ 4714 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4715 { 4716 int q_idx; 4717 4718 if (!vsi->netdev) 4719 return; 4720 4721 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4722 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4723 4724 if (q_vector->rx.ring || q_vector->tx.ring) 4725 napi_disable(&q_vector->napi); 4726 } 4727 } 4728 4729 /** 4730 * i40e_vsi_close - Shut down a VSI 4731 * @vsi: the vsi to be quelled 4732 **/ 4733 static void i40e_vsi_close(struct i40e_vsi *vsi) 4734 { 4735 struct i40e_pf *pf = vsi->back; 4736 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4737 i40e_down(vsi); 4738 i40e_vsi_free_irq(vsi); 4739 i40e_vsi_free_tx_resources(vsi); 4740 i40e_vsi_free_rx_resources(vsi); 4741 vsi->current_netdev_flags = 0; 4742 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 4743 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4744 set_bit(__I40E_CLIENT_RESET, pf->state); 4745 } 4746 4747 /** 4748 * i40e_quiesce_vsi - Pause a given VSI 4749 * @vsi: the VSI being paused 4750 **/ 4751 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4752 { 4753 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4754 return; 4755 4756 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4757 if (vsi->netdev && netif_running(vsi->netdev)) 4758 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4759 else 4760 i40e_vsi_close(vsi); 4761 } 4762 4763 /** 4764 * i40e_unquiesce_vsi - Resume a given VSI 4765 * @vsi: the VSI being resumed 4766 **/ 4767 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4768 { 4769 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4770 return; 4771 4772 if (vsi->netdev && netif_running(vsi->netdev)) 4773 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4774 else 4775 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4776 } 4777 4778 /** 4779 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4780 * @pf: the PF 4781 **/ 4782 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4783 { 4784 int v; 4785 4786 for (v = 0; v < pf->num_alloc_vsi; v++) { 4787 if (pf->vsi[v]) 4788 i40e_quiesce_vsi(pf->vsi[v]); 4789 } 4790 } 4791 4792 /** 4793 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4794 * @pf: the PF 4795 **/ 4796 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4797 { 4798 int v; 4799 4800 for (v = 0; v < pf->num_alloc_vsi; v++) { 4801 if (pf->vsi[v]) 4802 i40e_unquiesce_vsi(pf->vsi[v]); 4803 } 4804 } 4805 4806 /** 4807 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4808 * @vsi: the VSI being configured 4809 * 4810 * Wait until all queues on a given VSI have been disabled. 4811 **/ 4812 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4813 { 4814 struct i40e_pf *pf = vsi->back; 4815 int i, pf_q, ret; 4816 4817 pf_q = vsi->base_queue; 4818 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4819 /* Check and wait for the Tx queue */ 4820 ret = i40e_pf_txq_wait(pf, pf_q, false); 4821 if (ret) { 4822 dev_info(&pf->pdev->dev, 4823 "VSI seid %d Tx ring %d disable timeout\n", 4824 vsi->seid, pf_q); 4825 return ret; 4826 } 4827 4828 if (!i40e_enabled_xdp_vsi(vsi)) 4829 goto wait_rx; 4830 4831 /* Check and wait for the XDP Tx queue */ 4832 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4833 false); 4834 if (ret) { 4835 dev_info(&pf->pdev->dev, 4836 "VSI seid %d XDP Tx ring %d disable timeout\n", 4837 vsi->seid, pf_q); 4838 return ret; 4839 } 4840 wait_rx: 4841 /* Check and wait for the Rx queue */ 4842 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4843 if (ret) { 4844 dev_info(&pf->pdev->dev, 4845 "VSI seid %d Rx ring %d disable timeout\n", 4846 vsi->seid, pf_q); 4847 return ret; 4848 } 4849 } 4850 4851 return 0; 4852 } 4853 4854 #ifdef CONFIG_I40E_DCB 4855 /** 4856 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4857 * @pf: the PF 4858 * 4859 * This function waits for the queues to be in disabled state for all the 4860 * VSIs that are managed by this PF. 4861 **/ 4862 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4863 { 4864 int v, ret = 0; 4865 4866 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4867 if (pf->vsi[v]) { 4868 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4869 if (ret) 4870 break; 4871 } 4872 } 4873 4874 return ret; 4875 } 4876 4877 #endif 4878 4879 /** 4880 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4881 * @pf: pointer to PF 4882 * 4883 * Get TC map for ISCSI PF type that will include iSCSI TC 4884 * and LAN TC. 4885 **/ 4886 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4887 { 4888 struct i40e_dcb_app_priority_table app; 4889 struct i40e_hw *hw = &pf->hw; 4890 u8 enabled_tc = 1; /* TC0 is always enabled */ 4891 u8 tc, i; 4892 /* Get the iSCSI APP TLV */ 4893 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4894 4895 for (i = 0; i < dcbcfg->numapps; i++) { 4896 app = dcbcfg->app[i]; 4897 if (app.selector == I40E_APP_SEL_TCPIP && 4898 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4899 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4900 enabled_tc |= BIT(tc); 4901 break; 4902 } 4903 } 4904 4905 return enabled_tc; 4906 } 4907 4908 /** 4909 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 4910 * @dcbcfg: the corresponding DCBx configuration structure 4911 * 4912 * Return the number of TCs from given DCBx configuration 4913 **/ 4914 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 4915 { 4916 int i, tc_unused = 0; 4917 u8 num_tc = 0; 4918 u8 ret = 0; 4919 4920 /* Scan the ETS Config Priority Table to find 4921 * traffic class enabled for a given priority 4922 * and create a bitmask of enabled TCs 4923 */ 4924 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 4925 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 4926 4927 /* Now scan the bitmask to check for 4928 * contiguous TCs starting with TC0 4929 */ 4930 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4931 if (num_tc & BIT(i)) { 4932 if (!tc_unused) { 4933 ret++; 4934 } else { 4935 pr_err("Non-contiguous TC - Disabling DCB\n"); 4936 return 1; 4937 } 4938 } else { 4939 tc_unused = 1; 4940 } 4941 } 4942 4943 /* There is always at least TC0 */ 4944 if (!ret) 4945 ret = 1; 4946 4947 return ret; 4948 } 4949 4950 /** 4951 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 4952 * @dcbcfg: the corresponding DCBx configuration structure 4953 * 4954 * Query the current DCB configuration and return the number of 4955 * traffic classes enabled from the given DCBX config 4956 **/ 4957 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 4958 { 4959 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 4960 u8 enabled_tc = 1; 4961 u8 i; 4962 4963 for (i = 0; i < num_tc; i++) 4964 enabled_tc |= BIT(i); 4965 4966 return enabled_tc; 4967 } 4968 4969 /** 4970 * i40e_mqprio_get_enabled_tc - Get enabled traffic classes 4971 * @pf: PF being queried 4972 * 4973 * Query the current MQPRIO configuration and return the number of 4974 * traffic classes enabled. 4975 **/ 4976 static u8 i40e_mqprio_get_enabled_tc(struct i40e_pf *pf) 4977 { 4978 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 4979 u8 num_tc = vsi->mqprio_qopt.qopt.num_tc; 4980 u8 enabled_tc = 1, i; 4981 4982 for (i = 1; i < num_tc; i++) 4983 enabled_tc |= BIT(i); 4984 return enabled_tc; 4985 } 4986 4987 /** 4988 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 4989 * @pf: PF being queried 4990 * 4991 * Return number of traffic classes enabled for the given PF 4992 **/ 4993 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 4994 { 4995 struct i40e_hw *hw = &pf->hw; 4996 u8 i, enabled_tc = 1; 4997 u8 num_tc = 0; 4998 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4999 5000 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5001 return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; 5002 5003 /* If neither MQPRIO nor DCB is enabled, then always use single TC */ 5004 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5005 return 1; 5006 5007 /* SFP mode will be enabled for all TCs on port */ 5008 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5009 return i40e_dcb_get_num_tc(dcbcfg); 5010 5011 /* MFP mode return count of enabled TCs for this PF */ 5012 if (pf->hw.func_caps.iscsi) 5013 enabled_tc = i40e_get_iscsi_tc_map(pf); 5014 else 5015 return 1; /* Only TC0 */ 5016 5017 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5018 if (enabled_tc & BIT(i)) 5019 num_tc++; 5020 } 5021 return num_tc; 5022 } 5023 5024 /** 5025 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 5026 * @pf: PF being queried 5027 * 5028 * Return a bitmap for enabled traffic classes for this PF. 5029 **/ 5030 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 5031 { 5032 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5033 return i40e_mqprio_get_enabled_tc(pf); 5034 5035 /* If neither MQPRIO nor DCB is enabled for this PF then just return 5036 * default TC 5037 */ 5038 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 5039 return I40E_DEFAULT_TRAFFIC_CLASS; 5040 5041 /* SFP mode we want PF to be enabled for all TCs */ 5042 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 5043 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 5044 5045 /* MFP enabled and iSCSI PF type */ 5046 if (pf->hw.func_caps.iscsi) 5047 return i40e_get_iscsi_tc_map(pf); 5048 else 5049 return I40E_DEFAULT_TRAFFIC_CLASS; 5050 } 5051 5052 /** 5053 * i40e_vsi_get_bw_info - Query VSI BW Information 5054 * @vsi: the VSI being queried 5055 * 5056 * Returns 0 on success, negative value on failure 5057 **/ 5058 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 5059 { 5060 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 5061 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5062 struct i40e_pf *pf = vsi->back; 5063 struct i40e_hw *hw = &pf->hw; 5064 i40e_status ret; 5065 u32 tc_bw_max; 5066 int i; 5067 5068 /* Get the VSI level BW configuration */ 5069 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 5070 if (ret) { 5071 dev_info(&pf->pdev->dev, 5072 "couldn't get PF vsi bw config, err %s aq_err %s\n", 5073 i40e_stat_str(&pf->hw, ret), 5074 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5075 return -EINVAL; 5076 } 5077 5078 /* Get the VSI level BW configuration per TC */ 5079 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 5080 NULL); 5081 if (ret) { 5082 dev_info(&pf->pdev->dev, 5083 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 5084 i40e_stat_str(&pf->hw, ret), 5085 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5086 return -EINVAL; 5087 } 5088 5089 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 5090 dev_info(&pf->pdev->dev, 5091 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 5092 bw_config.tc_valid_bits, 5093 bw_ets_config.tc_valid_bits); 5094 /* Still continuing */ 5095 } 5096 5097 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 5098 vsi->bw_max_quanta = bw_config.max_bw; 5099 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 5100 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 5101 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5102 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 5103 vsi->bw_ets_limit_credits[i] = 5104 le16_to_cpu(bw_ets_config.credits[i]); 5105 /* 3 bits out of 4 for each TC */ 5106 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 5107 } 5108 5109 return 0; 5110 } 5111 5112 /** 5113 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 5114 * @vsi: the VSI being configured 5115 * @enabled_tc: TC bitmap 5116 * @bw_share: BW shared credits per TC 5117 * 5118 * Returns 0 on success, negative value on failure 5119 **/ 5120 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 5121 u8 *bw_share) 5122 { 5123 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5124 i40e_status ret; 5125 int i; 5126 5127 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) 5128 return 0; 5129 if (!vsi->mqprio_qopt.qopt.hw) { 5130 ret = i40e_set_bw_limit(vsi, vsi->seid, 0); 5131 if (ret) 5132 dev_info(&vsi->back->pdev->dev, 5133 "Failed to reset tx rate for vsi->seid %u\n", 5134 vsi->seid); 5135 return ret; 5136 } 5137 bw_data.tc_valid_bits = enabled_tc; 5138 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5139 bw_data.tc_bw_credits[i] = bw_share[i]; 5140 5141 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 5142 NULL); 5143 if (ret) { 5144 dev_info(&vsi->back->pdev->dev, 5145 "AQ command Config VSI BW allocation per TC failed = %d\n", 5146 vsi->back->hw.aq.asq_last_status); 5147 return -EINVAL; 5148 } 5149 5150 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5151 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5152 5153 return 0; 5154 } 5155 5156 /** 5157 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5158 * @vsi: the VSI being configured 5159 * @enabled_tc: TC map to be enabled 5160 * 5161 **/ 5162 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5163 { 5164 struct net_device *netdev = vsi->netdev; 5165 struct i40e_pf *pf = vsi->back; 5166 struct i40e_hw *hw = &pf->hw; 5167 u8 netdev_tc = 0; 5168 int i; 5169 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5170 5171 if (!netdev) 5172 return; 5173 5174 if (!enabled_tc) { 5175 netdev_reset_tc(netdev); 5176 return; 5177 } 5178 5179 /* Set up actual enabled TCs on the VSI */ 5180 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5181 return; 5182 5183 /* set per TC queues for the VSI */ 5184 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5185 /* Only set TC queues for enabled tcs 5186 * 5187 * e.g. For a VSI that has TC0 and TC3 enabled the 5188 * enabled_tc bitmap would be 0x00001001; the driver 5189 * will set the numtc for netdev as 2 that will be 5190 * referenced by the netdev layer as TC 0 and 1. 5191 */ 5192 if (vsi->tc_config.enabled_tc & BIT(i)) 5193 netdev_set_tc_queue(netdev, 5194 vsi->tc_config.tc_info[i].netdev_tc, 5195 vsi->tc_config.tc_info[i].qcount, 5196 vsi->tc_config.tc_info[i].qoffset); 5197 } 5198 5199 if (pf->flags & I40E_FLAG_TC_MQPRIO) 5200 return; 5201 5202 /* Assign UP2TC map for the VSI */ 5203 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5204 /* Get the actual TC# for the UP */ 5205 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5206 /* Get the mapped netdev TC# for the UP */ 5207 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5208 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5209 } 5210 } 5211 5212 /** 5213 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5214 * @vsi: the VSI being configured 5215 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5216 **/ 5217 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5218 struct i40e_vsi_context *ctxt) 5219 { 5220 /* copy just the sections touched not the entire info 5221 * since not all sections are valid as returned by 5222 * update vsi params 5223 */ 5224 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5225 memcpy(&vsi->info.queue_mapping, 5226 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5227 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5228 sizeof(vsi->info.tc_mapping)); 5229 } 5230 5231 /** 5232 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5233 * @vsi: VSI to be configured 5234 * @enabled_tc: TC bitmap 5235 * 5236 * This configures a particular VSI for TCs that are mapped to the 5237 * given TC bitmap. It uses default bandwidth share for TCs across 5238 * VSIs to configure TC for a particular VSI. 5239 * 5240 * NOTE: 5241 * It is expected that the VSI queues have been quisced before calling 5242 * this function. 5243 **/ 5244 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5245 { 5246 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5247 struct i40e_pf *pf = vsi->back; 5248 struct i40e_hw *hw = &pf->hw; 5249 struct i40e_vsi_context ctxt; 5250 int ret = 0; 5251 int i; 5252 5253 /* Check if enabled_tc is same as existing or new TCs */ 5254 if (vsi->tc_config.enabled_tc == enabled_tc && 5255 vsi->mqprio_qopt.mode != TC_MQPRIO_MODE_CHANNEL) 5256 return ret; 5257 5258 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5259 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5260 if (enabled_tc & BIT(i)) 5261 bw_share[i] = 1; 5262 } 5263 5264 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5265 if (ret) { 5266 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 5267 5268 dev_info(&pf->pdev->dev, 5269 "Failed configuring TC map %d for VSI %d\n", 5270 enabled_tc, vsi->seid); 5271 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, 5272 &bw_config, NULL); 5273 if (ret) { 5274 dev_info(&pf->pdev->dev, 5275 "Failed querying vsi bw info, err %s aq_err %s\n", 5276 i40e_stat_str(hw, ret), 5277 i40e_aq_str(hw, hw->aq.asq_last_status)); 5278 goto out; 5279 } 5280 if ((bw_config.tc_valid_bits & enabled_tc) != enabled_tc) { 5281 u8 valid_tc = bw_config.tc_valid_bits & enabled_tc; 5282 5283 if (!valid_tc) 5284 valid_tc = bw_config.tc_valid_bits; 5285 /* Always enable TC0, no matter what */ 5286 valid_tc |= 1; 5287 dev_info(&pf->pdev->dev, 5288 "Requested tc 0x%x, but FW reports 0x%x as valid. Attempting to use 0x%x.\n", 5289 enabled_tc, bw_config.tc_valid_bits, valid_tc); 5290 enabled_tc = valid_tc; 5291 } 5292 5293 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5294 if (ret) { 5295 dev_err(&pf->pdev->dev, 5296 "Unable to configure TC map %d for VSI %d\n", 5297 enabled_tc, vsi->seid); 5298 goto out; 5299 } 5300 } 5301 5302 /* Update Queue Pairs Mapping for currently enabled UPs */ 5303 ctxt.seid = vsi->seid; 5304 ctxt.pf_num = vsi->back->hw.pf_id; 5305 ctxt.vf_num = 0; 5306 ctxt.uplink_seid = vsi->uplink_seid; 5307 ctxt.info = vsi->info; 5308 if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { 5309 ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); 5310 if (ret) 5311 goto out; 5312 } else { 5313 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5314 } 5315 5316 /* On destroying the qdisc, reset vsi->rss_size, as number of enabled 5317 * queues changed. 5318 */ 5319 if (!vsi->mqprio_qopt.qopt.hw && vsi->reconfig_rss) { 5320 vsi->rss_size = min_t(int, vsi->back->alloc_rss_size, 5321 vsi->num_queue_pairs); 5322 ret = i40e_vsi_config_rss(vsi); 5323 if (ret) { 5324 dev_info(&vsi->back->pdev->dev, 5325 "Failed to reconfig rss for num_queues\n"); 5326 return ret; 5327 } 5328 vsi->reconfig_rss = false; 5329 } 5330 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5331 ctxt.info.valid_sections |= 5332 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5333 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5334 } 5335 5336 /* Update the VSI after updating the VSI queue-mapping 5337 * information 5338 */ 5339 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 5340 if (ret) { 5341 dev_info(&pf->pdev->dev, 5342 "Update vsi tc config failed, err %s aq_err %s\n", 5343 i40e_stat_str(hw, ret), 5344 i40e_aq_str(hw, hw->aq.asq_last_status)); 5345 goto out; 5346 } 5347 /* update the local VSI info with updated queue map */ 5348 i40e_vsi_update_queue_map(vsi, &ctxt); 5349 vsi->info.valid_sections = 0; 5350 5351 /* Update current VSI BW information */ 5352 ret = i40e_vsi_get_bw_info(vsi); 5353 if (ret) { 5354 dev_info(&pf->pdev->dev, 5355 "Failed updating vsi bw info, err %s aq_err %s\n", 5356 i40e_stat_str(hw, ret), 5357 i40e_aq_str(hw, hw->aq.asq_last_status)); 5358 goto out; 5359 } 5360 5361 /* Update the netdev TC setup */ 5362 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5363 out: 5364 return ret; 5365 } 5366 5367 /** 5368 * i40e_get_link_speed - Returns link speed for the interface 5369 * @vsi: VSI to be configured 5370 * 5371 **/ 5372 static int i40e_get_link_speed(struct i40e_vsi *vsi) 5373 { 5374 struct i40e_pf *pf = vsi->back; 5375 5376 switch (pf->hw.phy.link_info.link_speed) { 5377 case I40E_LINK_SPEED_40GB: 5378 return 40000; 5379 case I40E_LINK_SPEED_25GB: 5380 return 25000; 5381 case I40E_LINK_SPEED_20GB: 5382 return 20000; 5383 case I40E_LINK_SPEED_10GB: 5384 return 10000; 5385 case I40E_LINK_SPEED_1GB: 5386 return 1000; 5387 default: 5388 return -EINVAL; 5389 } 5390 } 5391 5392 /** 5393 * i40e_set_bw_limit - setup BW limit for Tx traffic based on max_tx_rate 5394 * @vsi: VSI to be configured 5395 * @seid: seid of the channel/VSI 5396 * @max_tx_rate: max TX rate to be configured as BW limit 5397 * 5398 * Helper function to set BW limit for a given VSI 5399 **/ 5400 int i40e_set_bw_limit(struct i40e_vsi *vsi, u16 seid, u64 max_tx_rate) 5401 { 5402 struct i40e_pf *pf = vsi->back; 5403 u64 credits = 0; 5404 int speed = 0; 5405 int ret = 0; 5406 5407 speed = i40e_get_link_speed(vsi); 5408 if (max_tx_rate > speed) { 5409 dev_err(&pf->pdev->dev, 5410 "Invalid max tx rate %llu specified for VSI seid %d.", 5411 max_tx_rate, seid); 5412 return -EINVAL; 5413 } 5414 if (max_tx_rate && max_tx_rate < 50) { 5415 dev_warn(&pf->pdev->dev, 5416 "Setting max tx rate to minimum usable value of 50Mbps.\n"); 5417 max_tx_rate = 50; 5418 } 5419 5420 /* Tx rate credits are in values of 50Mbps, 0 is disabled */ 5421 credits = max_tx_rate; 5422 do_div(credits, I40E_BW_CREDIT_DIVISOR); 5423 ret = i40e_aq_config_vsi_bw_limit(&pf->hw, seid, credits, 5424 I40E_MAX_BW_INACTIVE_ACCUM, NULL); 5425 if (ret) 5426 dev_err(&pf->pdev->dev, 5427 "Failed set tx rate (%llu Mbps) for vsi->seid %u, err %s aq_err %s\n", 5428 max_tx_rate, seid, i40e_stat_str(&pf->hw, ret), 5429 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5430 return ret; 5431 } 5432 5433 /** 5434 * i40e_remove_queue_channels - Remove queue channels for the TCs 5435 * @vsi: VSI to be configured 5436 * 5437 * Remove queue channels for the TCs 5438 **/ 5439 static void i40e_remove_queue_channels(struct i40e_vsi *vsi) 5440 { 5441 enum i40e_admin_queue_err last_aq_status; 5442 struct i40e_cloud_filter *cfilter; 5443 struct i40e_channel *ch, *ch_tmp; 5444 struct i40e_pf *pf = vsi->back; 5445 struct hlist_node *node; 5446 int ret, i; 5447 5448 /* Reset rss size that was stored when reconfiguring rss for 5449 * channel VSIs with non-power-of-2 queue count. 5450 */ 5451 vsi->current_rss_size = 0; 5452 5453 /* perform cleanup for channels if they exist */ 5454 if (list_empty(&vsi->ch_list)) 5455 return; 5456 5457 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5458 struct i40e_vsi *p_vsi; 5459 5460 list_del(&ch->list); 5461 p_vsi = ch->parent_vsi; 5462 if (!p_vsi || !ch->initialized) { 5463 kfree(ch); 5464 continue; 5465 } 5466 /* Reset queue contexts */ 5467 for (i = 0; i < ch->num_queue_pairs; i++) { 5468 struct i40e_ring *tx_ring, *rx_ring; 5469 u16 pf_q; 5470 5471 pf_q = ch->base_queue + i; 5472 tx_ring = vsi->tx_rings[pf_q]; 5473 tx_ring->ch = NULL; 5474 5475 rx_ring = vsi->rx_rings[pf_q]; 5476 rx_ring->ch = NULL; 5477 } 5478 5479 /* Reset BW configured for this VSI via mqprio */ 5480 ret = i40e_set_bw_limit(vsi, ch->seid, 0); 5481 if (ret) 5482 dev_info(&vsi->back->pdev->dev, 5483 "Failed to reset tx rate for ch->seid %u\n", 5484 ch->seid); 5485 5486 /* delete cloud filters associated with this channel */ 5487 hlist_for_each_entry_safe(cfilter, node, 5488 &pf->cloud_filter_list, cloud_node) { 5489 if (cfilter->seid != ch->seid) 5490 continue; 5491 5492 hash_del(&cfilter->cloud_node); 5493 if (cfilter->dst_port) 5494 ret = i40e_add_del_cloud_filter_big_buf(vsi, 5495 cfilter, 5496 false); 5497 else 5498 ret = i40e_add_del_cloud_filter(vsi, cfilter, 5499 false); 5500 last_aq_status = pf->hw.aq.asq_last_status; 5501 if (ret) 5502 dev_info(&pf->pdev->dev, 5503 "Failed to delete cloud filter, err %s aq_err %s\n", 5504 i40e_stat_str(&pf->hw, ret), 5505 i40e_aq_str(&pf->hw, last_aq_status)); 5506 kfree(cfilter); 5507 } 5508 5509 /* delete VSI from FW */ 5510 ret = i40e_aq_delete_element(&vsi->back->hw, ch->seid, 5511 NULL); 5512 if (ret) 5513 dev_err(&vsi->back->pdev->dev, 5514 "unable to remove channel (%d) for parent VSI(%d)\n", 5515 ch->seid, p_vsi->seid); 5516 kfree(ch); 5517 } 5518 INIT_LIST_HEAD(&vsi->ch_list); 5519 } 5520 5521 /** 5522 * i40e_is_any_channel - channel exist or not 5523 * @vsi: ptr to VSI to which channels are associated with 5524 * 5525 * Returns true or false if channel(s) exist for associated VSI or not 5526 **/ 5527 static bool i40e_is_any_channel(struct i40e_vsi *vsi) 5528 { 5529 struct i40e_channel *ch, *ch_tmp; 5530 5531 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5532 if (ch->initialized) 5533 return true; 5534 } 5535 5536 return false; 5537 } 5538 5539 /** 5540 * i40e_get_max_queues_for_channel 5541 * @vsi: ptr to VSI to which channels are associated with 5542 * 5543 * Helper function which returns max value among the queue counts set on the 5544 * channels/TCs created. 5545 **/ 5546 static int i40e_get_max_queues_for_channel(struct i40e_vsi *vsi) 5547 { 5548 struct i40e_channel *ch, *ch_tmp; 5549 int max = 0; 5550 5551 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 5552 if (!ch->initialized) 5553 continue; 5554 if (ch->num_queue_pairs > max) 5555 max = ch->num_queue_pairs; 5556 } 5557 5558 return max; 5559 } 5560 5561 /** 5562 * i40e_validate_num_queues - validate num_queues w.r.t channel 5563 * @pf: ptr to PF device 5564 * @num_queues: number of queues 5565 * @vsi: the parent VSI 5566 * @reconfig_rss: indicates should the RSS be reconfigured or not 5567 * 5568 * This function validates number of queues in the context of new channel 5569 * which is being established and determines if RSS should be reconfigured 5570 * or not for parent VSI. 5571 **/ 5572 static int i40e_validate_num_queues(struct i40e_pf *pf, int num_queues, 5573 struct i40e_vsi *vsi, bool *reconfig_rss) 5574 { 5575 int max_ch_queues; 5576 5577 if (!reconfig_rss) 5578 return -EINVAL; 5579 5580 *reconfig_rss = false; 5581 if (vsi->current_rss_size) { 5582 if (num_queues > vsi->current_rss_size) { 5583 dev_dbg(&pf->pdev->dev, 5584 "Error: num_queues (%d) > vsi's current_size(%d)\n", 5585 num_queues, vsi->current_rss_size); 5586 return -EINVAL; 5587 } else if ((num_queues < vsi->current_rss_size) && 5588 (!is_power_of_2(num_queues))) { 5589 dev_dbg(&pf->pdev->dev, 5590 "Error: num_queues (%d) < vsi's current_size(%d), but not power of 2\n", 5591 num_queues, vsi->current_rss_size); 5592 return -EINVAL; 5593 } 5594 } 5595 5596 if (!is_power_of_2(num_queues)) { 5597 /* Find the max num_queues configured for channel if channel 5598 * exist. 5599 * if channel exist, then enforce 'num_queues' to be more than 5600 * max ever queues configured for channel. 5601 */ 5602 max_ch_queues = i40e_get_max_queues_for_channel(vsi); 5603 if (num_queues < max_ch_queues) { 5604 dev_dbg(&pf->pdev->dev, 5605 "Error: num_queues (%d) < max queues configured for channel(%d)\n", 5606 num_queues, max_ch_queues); 5607 return -EINVAL; 5608 } 5609 *reconfig_rss = true; 5610 } 5611 5612 return 0; 5613 } 5614 5615 /** 5616 * i40e_vsi_reconfig_rss - reconfig RSS based on specified rss_size 5617 * @vsi: the VSI being setup 5618 * @rss_size: size of RSS, accordingly LUT gets reprogrammed 5619 * 5620 * This function reconfigures RSS by reprogramming LUTs using 'rss_size' 5621 **/ 5622 static int i40e_vsi_reconfig_rss(struct i40e_vsi *vsi, u16 rss_size) 5623 { 5624 struct i40e_pf *pf = vsi->back; 5625 u8 seed[I40E_HKEY_ARRAY_SIZE]; 5626 struct i40e_hw *hw = &pf->hw; 5627 int local_rss_size; 5628 u8 *lut; 5629 int ret; 5630 5631 if (!vsi->rss_size) 5632 return -EINVAL; 5633 5634 if (rss_size > vsi->rss_size) 5635 return -EINVAL; 5636 5637 local_rss_size = min_t(int, vsi->rss_size, rss_size); 5638 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 5639 if (!lut) 5640 return -ENOMEM; 5641 5642 /* Ignoring user configured lut if there is one */ 5643 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, local_rss_size); 5644 5645 /* Use user configured hash key if there is one, otherwise 5646 * use default. 5647 */ 5648 if (vsi->rss_hkey_user) 5649 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 5650 else 5651 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 5652 5653 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 5654 if (ret) { 5655 dev_info(&pf->pdev->dev, 5656 "Cannot set RSS lut, err %s aq_err %s\n", 5657 i40e_stat_str(hw, ret), 5658 i40e_aq_str(hw, hw->aq.asq_last_status)); 5659 kfree(lut); 5660 return ret; 5661 } 5662 kfree(lut); 5663 5664 /* Do the update w.r.t. storing rss_size */ 5665 if (!vsi->orig_rss_size) 5666 vsi->orig_rss_size = vsi->rss_size; 5667 vsi->current_rss_size = local_rss_size; 5668 5669 return ret; 5670 } 5671 5672 /** 5673 * i40e_channel_setup_queue_map - Setup a channel queue map 5674 * @pf: ptr to PF device 5675 * @vsi: the VSI being setup 5676 * @ctxt: VSI context structure 5677 * @ch: ptr to channel structure 5678 * 5679 * Setup queue map for a specific channel 5680 **/ 5681 static void i40e_channel_setup_queue_map(struct i40e_pf *pf, 5682 struct i40e_vsi_context *ctxt, 5683 struct i40e_channel *ch) 5684 { 5685 u16 qcount, qmap, sections = 0; 5686 u8 offset = 0; 5687 int pow; 5688 5689 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 5690 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 5691 5692 qcount = min_t(int, ch->num_queue_pairs, pf->num_lan_msix); 5693 ch->num_queue_pairs = qcount; 5694 5695 /* find the next higher power-of-2 of num queue pairs */ 5696 pow = ilog2(qcount); 5697 if (!is_power_of_2(qcount)) 5698 pow++; 5699 5700 qmap = (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 5701 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 5702 5703 /* Setup queue TC[0].qmap for given VSI context */ 5704 ctxt->info.tc_mapping[0] = cpu_to_le16(qmap); 5705 5706 ctxt->info.up_enable_bits = 0x1; /* TC0 enabled */ 5707 ctxt->info.mapping_flags |= cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 5708 ctxt->info.queue_mapping[0] = cpu_to_le16(ch->base_queue); 5709 ctxt->info.valid_sections |= cpu_to_le16(sections); 5710 } 5711 5712 /** 5713 * i40e_add_channel - add a channel by adding VSI 5714 * @pf: ptr to PF device 5715 * @uplink_seid: underlying HW switching element (VEB) ID 5716 * @ch: ptr to channel structure 5717 * 5718 * Add a channel (VSI) using add_vsi and queue_map 5719 **/ 5720 static int i40e_add_channel(struct i40e_pf *pf, u16 uplink_seid, 5721 struct i40e_channel *ch) 5722 { 5723 struct i40e_hw *hw = &pf->hw; 5724 struct i40e_vsi_context ctxt; 5725 u8 enabled_tc = 0x1; /* TC0 enabled */ 5726 int ret; 5727 5728 if (ch->type != I40E_VSI_VMDQ2) { 5729 dev_info(&pf->pdev->dev, 5730 "add new vsi failed, ch->type %d\n", ch->type); 5731 return -EINVAL; 5732 } 5733 5734 memset(&ctxt, 0, sizeof(ctxt)); 5735 ctxt.pf_num = hw->pf_id; 5736 ctxt.vf_num = 0; 5737 ctxt.uplink_seid = uplink_seid; 5738 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 5739 if (ch->type == I40E_VSI_VMDQ2) 5740 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 5741 5742 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) { 5743 ctxt.info.valid_sections |= 5744 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 5745 ctxt.info.switch_id = 5746 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 5747 } 5748 5749 /* Set queue map for a given VSI context */ 5750 i40e_channel_setup_queue_map(pf, &ctxt, ch); 5751 5752 /* Now time to create VSI */ 5753 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 5754 if (ret) { 5755 dev_info(&pf->pdev->dev, 5756 "add new vsi failed, err %s aq_err %s\n", 5757 i40e_stat_str(&pf->hw, ret), 5758 i40e_aq_str(&pf->hw, 5759 pf->hw.aq.asq_last_status)); 5760 return -ENOENT; 5761 } 5762 5763 /* Success, update channel */ 5764 ch->enabled_tc = enabled_tc; 5765 ch->seid = ctxt.seid; 5766 ch->vsi_number = ctxt.vsi_number; 5767 ch->stat_counter_idx = cpu_to_le16(ctxt.info.stat_counter_idx); 5768 5769 /* copy just the sections touched not the entire info 5770 * since not all sections are valid as returned by 5771 * update vsi params 5772 */ 5773 ch->info.mapping_flags = ctxt.info.mapping_flags; 5774 memcpy(&ch->info.queue_mapping, 5775 &ctxt.info.queue_mapping, sizeof(ctxt.info.queue_mapping)); 5776 memcpy(&ch->info.tc_mapping, ctxt.info.tc_mapping, 5777 sizeof(ctxt.info.tc_mapping)); 5778 5779 return 0; 5780 } 5781 5782 static int i40e_channel_config_bw(struct i40e_vsi *vsi, struct i40e_channel *ch, 5783 u8 *bw_share) 5784 { 5785 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 5786 i40e_status ret; 5787 int i; 5788 5789 bw_data.tc_valid_bits = ch->enabled_tc; 5790 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5791 bw_data.tc_bw_credits[i] = bw_share[i]; 5792 5793 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, ch->seid, 5794 &bw_data, NULL); 5795 if (ret) { 5796 dev_info(&vsi->back->pdev->dev, 5797 "Config VSI BW allocation per TC failed, aq_err: %d for new_vsi->seid %u\n", 5798 vsi->back->hw.aq.asq_last_status, ch->seid); 5799 return -EINVAL; 5800 } 5801 5802 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5803 ch->info.qs_handle[i] = bw_data.qs_handles[i]; 5804 5805 return 0; 5806 } 5807 5808 /** 5809 * i40e_channel_config_tx_ring - config TX ring associated with new channel 5810 * @pf: ptr to PF device 5811 * @vsi: the VSI being setup 5812 * @ch: ptr to channel structure 5813 * 5814 * Configure TX rings associated with channel (VSI) since queues are being 5815 * from parent VSI. 5816 **/ 5817 static int i40e_channel_config_tx_ring(struct i40e_pf *pf, 5818 struct i40e_vsi *vsi, 5819 struct i40e_channel *ch) 5820 { 5821 i40e_status ret; 5822 int i; 5823 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5824 5825 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5826 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5827 if (ch->enabled_tc & BIT(i)) 5828 bw_share[i] = 1; 5829 } 5830 5831 /* configure BW for new VSI */ 5832 ret = i40e_channel_config_bw(vsi, ch, bw_share); 5833 if (ret) { 5834 dev_info(&vsi->back->pdev->dev, 5835 "Failed configuring TC map %d for channel (seid %u)\n", 5836 ch->enabled_tc, ch->seid); 5837 return ret; 5838 } 5839 5840 for (i = 0; i < ch->num_queue_pairs; i++) { 5841 struct i40e_ring *tx_ring, *rx_ring; 5842 u16 pf_q; 5843 5844 pf_q = ch->base_queue + i; 5845 5846 /* Get to TX ring ptr of main VSI, for re-setup TX queue 5847 * context 5848 */ 5849 tx_ring = vsi->tx_rings[pf_q]; 5850 tx_ring->ch = ch; 5851 5852 /* Get the RX ring ptr */ 5853 rx_ring = vsi->rx_rings[pf_q]; 5854 rx_ring->ch = ch; 5855 } 5856 5857 return 0; 5858 } 5859 5860 /** 5861 * i40e_setup_hw_channel - setup new channel 5862 * @pf: ptr to PF device 5863 * @vsi: the VSI being setup 5864 * @ch: ptr to channel structure 5865 * @uplink_seid: underlying HW switching element (VEB) ID 5866 * @type: type of channel to be created (VMDq2/VF) 5867 * 5868 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5869 * and configures TX rings accordingly 5870 **/ 5871 static inline int i40e_setup_hw_channel(struct i40e_pf *pf, 5872 struct i40e_vsi *vsi, 5873 struct i40e_channel *ch, 5874 u16 uplink_seid, u8 type) 5875 { 5876 int ret; 5877 5878 ch->initialized = false; 5879 ch->base_queue = vsi->next_base_queue; 5880 ch->type = type; 5881 5882 /* Proceed with creation of channel (VMDq2) VSI */ 5883 ret = i40e_add_channel(pf, uplink_seid, ch); 5884 if (ret) { 5885 dev_info(&pf->pdev->dev, 5886 "failed to add_channel using uplink_seid %u\n", 5887 uplink_seid); 5888 return ret; 5889 } 5890 5891 /* Mark the successful creation of channel */ 5892 ch->initialized = true; 5893 5894 /* Reconfigure TX queues using QTX_CTL register */ 5895 ret = i40e_channel_config_tx_ring(pf, vsi, ch); 5896 if (ret) { 5897 dev_info(&pf->pdev->dev, 5898 "failed to configure TX rings for channel %u\n", 5899 ch->seid); 5900 return ret; 5901 } 5902 5903 /* update 'next_base_queue' */ 5904 vsi->next_base_queue = vsi->next_base_queue + ch->num_queue_pairs; 5905 dev_dbg(&pf->pdev->dev, 5906 "Added channel: vsi_seid %u, vsi_number %u, stat_counter_idx %u, num_queue_pairs %u, pf->next_base_queue %d\n", 5907 ch->seid, ch->vsi_number, ch->stat_counter_idx, 5908 ch->num_queue_pairs, 5909 vsi->next_base_queue); 5910 return ret; 5911 } 5912 5913 /** 5914 * i40e_setup_channel - setup new channel using uplink element 5915 * @pf: ptr to PF device 5916 * @type: type of channel to be created (VMDq2/VF) 5917 * @uplink_seid: underlying HW switching element (VEB) ID 5918 * @ch: ptr to channel structure 5919 * 5920 * Setup new channel (VSI) based on specified type (VMDq2/VF) 5921 * and uplink switching element (uplink_seid) 5922 **/ 5923 static bool i40e_setup_channel(struct i40e_pf *pf, struct i40e_vsi *vsi, 5924 struct i40e_channel *ch) 5925 { 5926 u8 vsi_type; 5927 u16 seid; 5928 int ret; 5929 5930 if (vsi->type == I40E_VSI_MAIN) { 5931 vsi_type = I40E_VSI_VMDQ2; 5932 } else { 5933 dev_err(&pf->pdev->dev, "unsupported parent vsi type(%d)\n", 5934 vsi->type); 5935 return false; 5936 } 5937 5938 /* underlying switching element */ 5939 seid = pf->vsi[pf->lan_vsi]->uplink_seid; 5940 5941 /* create channel (VSI), configure TX rings */ 5942 ret = i40e_setup_hw_channel(pf, vsi, ch, seid, vsi_type); 5943 if (ret) { 5944 dev_err(&pf->pdev->dev, "failed to setup hw_channel\n"); 5945 return false; 5946 } 5947 5948 return ch->initialized ? true : false; 5949 } 5950 5951 /** 5952 * i40e_validate_and_set_switch_mode - sets up switch mode correctly 5953 * @vsi: ptr to VSI which has PF backing 5954 * 5955 * Sets up switch mode correctly if it needs to be changed and perform 5956 * what are allowed modes. 5957 **/ 5958 static int i40e_validate_and_set_switch_mode(struct i40e_vsi *vsi) 5959 { 5960 u8 mode; 5961 struct i40e_pf *pf = vsi->back; 5962 struct i40e_hw *hw = &pf->hw; 5963 int ret; 5964 5965 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_dev_capabilities); 5966 if (ret) 5967 return -EINVAL; 5968 5969 if (hw->dev_caps.switch_mode) { 5970 /* if switch mode is set, support mode2 (non-tunneled for 5971 * cloud filter) for now 5972 */ 5973 u32 switch_mode = hw->dev_caps.switch_mode & 5974 I40E_SWITCH_MODE_MASK; 5975 if (switch_mode >= I40E_CLOUD_FILTER_MODE1) { 5976 if (switch_mode == I40E_CLOUD_FILTER_MODE2) 5977 return 0; 5978 dev_err(&pf->pdev->dev, 5979 "Invalid switch_mode (%d), only non-tunneled mode for cloud filter is supported\n", 5980 hw->dev_caps.switch_mode); 5981 return -EINVAL; 5982 } 5983 } 5984 5985 /* Set Bit 7 to be valid */ 5986 mode = I40E_AQ_SET_SWITCH_BIT7_VALID; 5987 5988 /* Set L4type for TCP support */ 5989 mode |= I40E_AQ_SET_SWITCH_L4_TYPE_TCP; 5990 5991 /* Set cloud filter mode */ 5992 mode |= I40E_AQ_SET_SWITCH_MODE_NON_TUNNEL; 5993 5994 /* Prep mode field for set_switch_config */ 5995 ret = i40e_aq_set_switch_config(hw, pf->last_sw_conf_flags, 5996 pf->last_sw_conf_valid_flags, 5997 mode, NULL); 5998 if (ret && hw->aq.asq_last_status != I40E_AQ_RC_ESRCH) 5999 dev_err(&pf->pdev->dev, 6000 "couldn't set switch config bits, err %s aq_err %s\n", 6001 i40e_stat_str(hw, ret), 6002 i40e_aq_str(hw, 6003 hw->aq.asq_last_status)); 6004 6005 return ret; 6006 } 6007 6008 /** 6009 * i40e_create_queue_channel - function to create channel 6010 * @vsi: VSI to be configured 6011 * @ch: ptr to channel (it contains channel specific params) 6012 * 6013 * This function creates channel (VSI) using num_queues specified by user, 6014 * reconfigs RSS if needed. 6015 **/ 6016 int i40e_create_queue_channel(struct i40e_vsi *vsi, 6017 struct i40e_channel *ch) 6018 { 6019 struct i40e_pf *pf = vsi->back; 6020 bool reconfig_rss; 6021 int err; 6022 6023 if (!ch) 6024 return -EINVAL; 6025 6026 if (!ch->num_queue_pairs) { 6027 dev_err(&pf->pdev->dev, "Invalid num_queues requested: %d\n", 6028 ch->num_queue_pairs); 6029 return -EINVAL; 6030 } 6031 6032 /* validate user requested num_queues for channel */ 6033 err = i40e_validate_num_queues(pf, ch->num_queue_pairs, vsi, 6034 &reconfig_rss); 6035 if (err) { 6036 dev_info(&pf->pdev->dev, "Failed to validate num_queues (%d)\n", 6037 ch->num_queue_pairs); 6038 return -EINVAL; 6039 } 6040 6041 /* By default we are in VEPA mode, if this is the first VF/VMDq 6042 * VSI to be added switch to VEB mode. 6043 */ 6044 if ((!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) || 6045 (!i40e_is_any_channel(vsi))) { 6046 if (!is_power_of_2(vsi->tc_config.tc_info[0].qcount)) { 6047 dev_dbg(&pf->pdev->dev, 6048 "Failed to create channel. Override queues (%u) not power of 2\n", 6049 vsi->tc_config.tc_info[0].qcount); 6050 return -EINVAL; 6051 } 6052 6053 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 6054 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 6055 6056 if (vsi->type == I40E_VSI_MAIN) { 6057 if (pf->flags & I40E_FLAG_TC_MQPRIO) 6058 i40e_do_reset(pf, I40E_PF_RESET_FLAG, 6059 true); 6060 else 6061 i40e_do_reset_safe(pf, 6062 I40E_PF_RESET_FLAG); 6063 } 6064 } 6065 /* now onwards for main VSI, number of queues will be value 6066 * of TC0's queue count 6067 */ 6068 } 6069 6070 /* By this time, vsi->cnt_q_avail shall be set to non-zero and 6071 * it should be more than num_queues 6072 */ 6073 if (!vsi->cnt_q_avail || vsi->cnt_q_avail < ch->num_queue_pairs) { 6074 dev_dbg(&pf->pdev->dev, 6075 "Error: cnt_q_avail (%u) less than num_queues %d\n", 6076 vsi->cnt_q_avail, ch->num_queue_pairs); 6077 return -EINVAL; 6078 } 6079 6080 /* reconfig_rss only if vsi type is MAIN_VSI */ 6081 if (reconfig_rss && (vsi->type == I40E_VSI_MAIN)) { 6082 err = i40e_vsi_reconfig_rss(vsi, ch->num_queue_pairs); 6083 if (err) { 6084 dev_info(&pf->pdev->dev, 6085 "Error: unable to reconfig rss for num_queues (%u)\n", 6086 ch->num_queue_pairs); 6087 return -EINVAL; 6088 } 6089 } 6090 6091 if (!i40e_setup_channel(pf, vsi, ch)) { 6092 dev_info(&pf->pdev->dev, "Failed to setup channel\n"); 6093 return -EINVAL; 6094 } 6095 6096 dev_info(&pf->pdev->dev, 6097 "Setup channel (id:%u) utilizing num_queues %d\n", 6098 ch->seid, ch->num_queue_pairs); 6099 6100 /* configure VSI for BW limit */ 6101 if (ch->max_tx_rate) { 6102 u64 credits = ch->max_tx_rate; 6103 6104 if (i40e_set_bw_limit(vsi, ch->seid, ch->max_tx_rate)) 6105 return -EINVAL; 6106 6107 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6108 dev_dbg(&pf->pdev->dev, 6109 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6110 ch->max_tx_rate, 6111 credits, 6112 ch->seid); 6113 } 6114 6115 /* in case of VF, this will be main SRIOV VSI */ 6116 ch->parent_vsi = vsi; 6117 6118 /* and update main_vsi's count for queue_available to use */ 6119 vsi->cnt_q_avail -= ch->num_queue_pairs; 6120 6121 return 0; 6122 } 6123 6124 /** 6125 * i40e_configure_queue_channels - Add queue channel for the given TCs 6126 * @vsi: VSI to be configured 6127 * 6128 * Configures queue channel mapping to the given TCs 6129 **/ 6130 static int i40e_configure_queue_channels(struct i40e_vsi *vsi) 6131 { 6132 struct i40e_channel *ch; 6133 u64 max_rate = 0; 6134 int ret = 0, i; 6135 6136 /* Create app vsi with the TCs. Main VSI with TC0 is already set up */ 6137 vsi->tc_seid_map[0] = vsi->seid; 6138 for (i = 1; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6139 if (vsi->tc_config.enabled_tc & BIT(i)) { 6140 ch = kzalloc(sizeof(*ch), GFP_KERNEL); 6141 if (!ch) { 6142 ret = -ENOMEM; 6143 goto err_free; 6144 } 6145 6146 INIT_LIST_HEAD(&ch->list); 6147 ch->num_queue_pairs = 6148 vsi->tc_config.tc_info[i].qcount; 6149 ch->base_queue = 6150 vsi->tc_config.tc_info[i].qoffset; 6151 6152 /* Bandwidth limit through tc interface is in bytes/s, 6153 * change to Mbit/s 6154 */ 6155 max_rate = vsi->mqprio_qopt.max_rate[i]; 6156 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6157 ch->max_tx_rate = max_rate; 6158 6159 list_add_tail(&ch->list, &vsi->ch_list); 6160 6161 ret = i40e_create_queue_channel(vsi, ch); 6162 if (ret) { 6163 dev_err(&vsi->back->pdev->dev, 6164 "Failed creating queue channel with TC%d: queues %d\n", 6165 i, ch->num_queue_pairs); 6166 goto err_free; 6167 } 6168 vsi->tc_seid_map[i] = ch->seid; 6169 } 6170 } 6171 return ret; 6172 6173 err_free: 6174 i40e_remove_queue_channels(vsi); 6175 return ret; 6176 } 6177 6178 /** 6179 * i40e_veb_config_tc - Configure TCs for given VEB 6180 * @veb: given VEB 6181 * @enabled_tc: TC bitmap 6182 * 6183 * Configures given TC bitmap for VEB (switching) element 6184 **/ 6185 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 6186 { 6187 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 6188 struct i40e_pf *pf = veb->pf; 6189 int ret = 0; 6190 int i; 6191 6192 /* No TCs or already enabled TCs just return */ 6193 if (!enabled_tc || veb->enabled_tc == enabled_tc) 6194 return ret; 6195 6196 bw_data.tc_valid_bits = enabled_tc; 6197 /* bw_data.absolute_credits is not set (relative) */ 6198 6199 /* Enable ETS TCs with equal BW Share for now */ 6200 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6201 if (enabled_tc & BIT(i)) 6202 bw_data.tc_bw_share_credits[i] = 1; 6203 } 6204 6205 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 6206 &bw_data, NULL); 6207 if (ret) { 6208 dev_info(&pf->pdev->dev, 6209 "VEB bw config failed, err %s aq_err %s\n", 6210 i40e_stat_str(&pf->hw, ret), 6211 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6212 goto out; 6213 } 6214 6215 /* Update the BW information */ 6216 ret = i40e_veb_get_bw_info(veb); 6217 if (ret) { 6218 dev_info(&pf->pdev->dev, 6219 "Failed getting veb bw config, err %s aq_err %s\n", 6220 i40e_stat_str(&pf->hw, ret), 6221 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6222 } 6223 6224 out: 6225 return ret; 6226 } 6227 6228 #ifdef CONFIG_I40E_DCB 6229 /** 6230 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 6231 * @pf: PF struct 6232 * 6233 * Reconfigure VEB/VSIs on a given PF; it is assumed that 6234 * the caller would've quiesce all the VSIs before calling 6235 * this function 6236 **/ 6237 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 6238 { 6239 u8 tc_map = 0; 6240 int ret; 6241 u8 v; 6242 6243 /* Enable the TCs available on PF to all VEBs */ 6244 tc_map = i40e_pf_get_tc_map(pf); 6245 for (v = 0; v < I40E_MAX_VEB; v++) { 6246 if (!pf->veb[v]) 6247 continue; 6248 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 6249 if (ret) { 6250 dev_info(&pf->pdev->dev, 6251 "Failed configuring TC for VEB seid=%d\n", 6252 pf->veb[v]->seid); 6253 /* Will try to configure as many components */ 6254 } 6255 } 6256 6257 /* Update each VSI */ 6258 for (v = 0; v < pf->num_alloc_vsi; v++) { 6259 if (!pf->vsi[v]) 6260 continue; 6261 6262 /* - Enable all TCs for the LAN VSI 6263 * - For all others keep them at TC0 for now 6264 */ 6265 if (v == pf->lan_vsi) 6266 tc_map = i40e_pf_get_tc_map(pf); 6267 else 6268 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 6269 6270 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 6271 if (ret) { 6272 dev_info(&pf->pdev->dev, 6273 "Failed configuring TC for VSI seid=%d\n", 6274 pf->vsi[v]->seid); 6275 /* Will try to configure as many components */ 6276 } else { 6277 /* Re-configure VSI vectors based on updated TC map */ 6278 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 6279 if (pf->vsi[v]->netdev) 6280 i40e_dcbnl_set_all(pf->vsi[v]); 6281 } 6282 } 6283 } 6284 6285 /** 6286 * i40e_resume_port_tx - Resume port Tx 6287 * @pf: PF struct 6288 * 6289 * Resume a port's Tx and issue a PF reset in case of failure to 6290 * resume. 6291 **/ 6292 static int i40e_resume_port_tx(struct i40e_pf *pf) 6293 { 6294 struct i40e_hw *hw = &pf->hw; 6295 int ret; 6296 6297 ret = i40e_aq_resume_port_tx(hw, NULL); 6298 if (ret) { 6299 dev_info(&pf->pdev->dev, 6300 "Resume Port Tx failed, err %s aq_err %s\n", 6301 i40e_stat_str(&pf->hw, ret), 6302 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6303 /* Schedule PF reset to recover */ 6304 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6305 i40e_service_event_schedule(pf); 6306 } 6307 6308 return ret; 6309 } 6310 6311 /** 6312 * i40e_init_pf_dcb - Initialize DCB configuration 6313 * @pf: PF being configured 6314 * 6315 * Query the current DCB configuration and cache it 6316 * in the hardware structure 6317 **/ 6318 static int i40e_init_pf_dcb(struct i40e_pf *pf) 6319 { 6320 struct i40e_hw *hw = &pf->hw; 6321 int err = 0; 6322 6323 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable 6324 * Also do not enable DCBx if FW LLDP agent is disabled 6325 */ 6326 if ((pf->hw_features & I40E_HW_NO_DCB_SUPPORT) || 6327 (pf->flags & I40E_FLAG_DISABLE_FW_LLDP)) 6328 goto out; 6329 6330 /* Get the initial DCB configuration */ 6331 err = i40e_init_dcb(hw); 6332 if (!err) { 6333 /* Device/Function is not DCBX capable */ 6334 if ((!hw->func_caps.dcb) || 6335 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 6336 dev_info(&pf->pdev->dev, 6337 "DCBX offload is not supported or is disabled for this PF.\n"); 6338 } else { 6339 /* When status is not DISABLED then DCBX in FW */ 6340 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 6341 DCB_CAP_DCBX_VER_IEEE; 6342 6343 pf->flags |= I40E_FLAG_DCB_CAPABLE; 6344 /* Enable DCB tagging only when more than one TC 6345 * or explicitly disable if only one TC 6346 */ 6347 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6348 pf->flags |= I40E_FLAG_DCB_ENABLED; 6349 else 6350 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6351 dev_dbg(&pf->pdev->dev, 6352 "DCBX offload is supported for this PF.\n"); 6353 } 6354 } else if (pf->hw.aq.asq_last_status == I40E_AQ_RC_EPERM) { 6355 dev_info(&pf->pdev->dev, "FW LLDP disabled for this PF.\n"); 6356 pf->flags |= I40E_FLAG_DISABLE_FW_LLDP; 6357 } else { 6358 dev_info(&pf->pdev->dev, 6359 "Query for DCB configuration failed, err %s aq_err %s\n", 6360 i40e_stat_str(&pf->hw, err), 6361 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6362 } 6363 6364 out: 6365 return err; 6366 } 6367 #endif /* CONFIG_I40E_DCB */ 6368 #define SPEED_SIZE 14 6369 #define FC_SIZE 8 6370 /** 6371 * i40e_print_link_message - print link up or down 6372 * @vsi: the VSI for which link needs a message 6373 * @isup: true of link is up, false otherwise 6374 */ 6375 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 6376 { 6377 enum i40e_aq_link_speed new_speed; 6378 struct i40e_pf *pf = vsi->back; 6379 char *speed = "Unknown"; 6380 char *fc = "Unknown"; 6381 char *fec = ""; 6382 char *req_fec = ""; 6383 char *an = ""; 6384 6385 new_speed = pf->hw.phy.link_info.link_speed; 6386 6387 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 6388 return; 6389 vsi->current_isup = isup; 6390 vsi->current_speed = new_speed; 6391 if (!isup) { 6392 netdev_info(vsi->netdev, "NIC Link is Down\n"); 6393 return; 6394 } 6395 6396 /* Warn user if link speed on NPAR enabled partition is not at 6397 * least 10GB 6398 */ 6399 if (pf->hw.func_caps.npar_enable && 6400 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 6401 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 6402 netdev_warn(vsi->netdev, 6403 "The partition detected link speed that is less than 10Gbps\n"); 6404 6405 switch (pf->hw.phy.link_info.link_speed) { 6406 case I40E_LINK_SPEED_40GB: 6407 speed = "40 G"; 6408 break; 6409 case I40E_LINK_SPEED_20GB: 6410 speed = "20 G"; 6411 break; 6412 case I40E_LINK_SPEED_25GB: 6413 speed = "25 G"; 6414 break; 6415 case I40E_LINK_SPEED_10GB: 6416 speed = "10 G"; 6417 break; 6418 case I40E_LINK_SPEED_1GB: 6419 speed = "1000 M"; 6420 break; 6421 case I40E_LINK_SPEED_100MB: 6422 speed = "100 M"; 6423 break; 6424 default: 6425 break; 6426 } 6427 6428 switch (pf->hw.fc.current_mode) { 6429 case I40E_FC_FULL: 6430 fc = "RX/TX"; 6431 break; 6432 case I40E_FC_TX_PAUSE: 6433 fc = "TX"; 6434 break; 6435 case I40E_FC_RX_PAUSE: 6436 fc = "RX"; 6437 break; 6438 default: 6439 fc = "None"; 6440 break; 6441 } 6442 6443 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 6444 req_fec = ", Requested FEC: None"; 6445 fec = ", FEC: None"; 6446 an = ", Autoneg: False"; 6447 6448 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 6449 an = ", Autoneg: True"; 6450 6451 if (pf->hw.phy.link_info.fec_info & 6452 I40E_AQ_CONFIG_FEC_KR_ENA) 6453 fec = ", FEC: CL74 FC-FEC/BASE-R"; 6454 else if (pf->hw.phy.link_info.fec_info & 6455 I40E_AQ_CONFIG_FEC_RS_ENA) 6456 fec = ", FEC: CL108 RS-FEC"; 6457 6458 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 6459 * both RS and FC are requested 6460 */ 6461 if (vsi->back->hw.phy.link_info.req_fec_info & 6462 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 6463 if (vsi->back->hw.phy.link_info.req_fec_info & 6464 I40E_AQ_REQUEST_FEC_RS) 6465 req_fec = ", Requested FEC: CL108 RS-FEC"; 6466 else 6467 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R"; 6468 } 6469 } 6470 6471 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n", 6472 speed, req_fec, fec, an, fc); 6473 } 6474 6475 /** 6476 * i40e_up_complete - Finish the last steps of bringing up a connection 6477 * @vsi: the VSI being configured 6478 **/ 6479 static int i40e_up_complete(struct i40e_vsi *vsi) 6480 { 6481 struct i40e_pf *pf = vsi->back; 6482 int err; 6483 6484 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6485 i40e_vsi_configure_msix(vsi); 6486 else 6487 i40e_configure_msi_and_legacy(vsi); 6488 6489 /* start rings */ 6490 err = i40e_vsi_start_rings(vsi); 6491 if (err) 6492 return err; 6493 6494 clear_bit(__I40E_VSI_DOWN, vsi->state); 6495 i40e_napi_enable_all(vsi); 6496 i40e_vsi_enable_irq(vsi); 6497 6498 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 6499 (vsi->netdev)) { 6500 i40e_print_link_message(vsi, true); 6501 netif_tx_start_all_queues(vsi->netdev); 6502 netif_carrier_on(vsi->netdev); 6503 } 6504 6505 /* replay FDIR SB filters */ 6506 if (vsi->type == I40E_VSI_FDIR) { 6507 /* reset fd counters */ 6508 pf->fd_add_err = 0; 6509 pf->fd_atr_cnt = 0; 6510 i40e_fdir_filter_restore(vsi); 6511 } 6512 6513 /* On the next run of the service_task, notify any clients of the new 6514 * opened netdev 6515 */ 6516 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 6517 i40e_service_event_schedule(pf); 6518 6519 return 0; 6520 } 6521 6522 /** 6523 * i40e_vsi_reinit_locked - Reset the VSI 6524 * @vsi: the VSI being configured 6525 * 6526 * Rebuild the ring structs after some configuration 6527 * has changed, e.g. MTU size. 6528 **/ 6529 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 6530 { 6531 struct i40e_pf *pf = vsi->back; 6532 6533 WARN_ON(in_interrupt()); 6534 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 6535 usleep_range(1000, 2000); 6536 i40e_down(vsi); 6537 6538 i40e_up(vsi); 6539 clear_bit(__I40E_CONFIG_BUSY, pf->state); 6540 } 6541 6542 /** 6543 * i40e_up - Bring the connection back up after being down 6544 * @vsi: the VSI being configured 6545 **/ 6546 int i40e_up(struct i40e_vsi *vsi) 6547 { 6548 int err; 6549 6550 err = i40e_vsi_configure(vsi); 6551 if (!err) 6552 err = i40e_up_complete(vsi); 6553 6554 return err; 6555 } 6556 6557 /** 6558 * i40e_force_link_state - Force the link status 6559 * @pf: board private structure 6560 * @is_up: whether the link state should be forced up or down 6561 **/ 6562 static i40e_status i40e_force_link_state(struct i40e_pf *pf, bool is_up) 6563 { 6564 struct i40e_aq_get_phy_abilities_resp abilities; 6565 struct i40e_aq_set_phy_config config = {0}; 6566 struct i40e_hw *hw = &pf->hw; 6567 i40e_status err; 6568 u64 mask; 6569 6570 /* Get the current phy config */ 6571 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, 6572 NULL); 6573 if (err) { 6574 dev_err(&pf->pdev->dev, 6575 "failed to get phy cap., ret = %s last_status = %s\n", 6576 i40e_stat_str(hw, err), 6577 i40e_aq_str(hw, hw->aq.asq_last_status)); 6578 return err; 6579 } 6580 6581 /* If link needs to go up, but was not forced to go down, 6582 * no need for a flap 6583 */ 6584 if (is_up && abilities.phy_type != 0) 6585 return I40E_SUCCESS; 6586 6587 /* To force link we need to set bits for all supported PHY types, 6588 * but there are now more than 32, so we need to split the bitmap 6589 * across two fields. 6590 */ 6591 mask = I40E_PHY_TYPES_BITMASK; 6592 config.phy_type = is_up ? cpu_to_le32((u32)(mask & 0xffffffff)) : 0; 6593 config.phy_type_ext = is_up ? (u8)((mask >> 32) & 0xff) : 0; 6594 /* Copy the old settings, except of phy_type */ 6595 config.abilities = abilities.abilities; 6596 config.link_speed = abilities.link_speed; 6597 config.eee_capability = abilities.eee_capability; 6598 config.eeer = abilities.eeer_val; 6599 config.low_power_ctrl = abilities.d3_lpan; 6600 err = i40e_aq_set_phy_config(hw, &config, NULL); 6601 6602 if (err) { 6603 dev_err(&pf->pdev->dev, 6604 "set phy config ret = %s last_status = %s\n", 6605 i40e_stat_str(&pf->hw, err), 6606 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6607 return err; 6608 } 6609 6610 /* Update the link info */ 6611 err = i40e_update_link_info(hw); 6612 if (err) { 6613 /* Wait a little bit (on 40G cards it sometimes takes a really 6614 * long time for link to come back from the atomic reset) 6615 * and try once more 6616 */ 6617 msleep(1000); 6618 i40e_update_link_info(hw); 6619 } 6620 6621 i40e_aq_set_link_restart_an(hw, true, NULL); 6622 6623 return I40E_SUCCESS; 6624 } 6625 6626 /** 6627 * i40e_down - Shutdown the connection processing 6628 * @vsi: the VSI being stopped 6629 **/ 6630 void i40e_down(struct i40e_vsi *vsi) 6631 { 6632 int i; 6633 6634 /* It is assumed that the caller of this function 6635 * sets the vsi->state __I40E_VSI_DOWN bit. 6636 */ 6637 if (vsi->netdev) { 6638 netif_carrier_off(vsi->netdev); 6639 netif_tx_disable(vsi->netdev); 6640 } 6641 i40e_vsi_disable_irq(vsi); 6642 i40e_vsi_stop_rings(vsi); 6643 if (vsi->type == I40E_VSI_MAIN && 6644 vsi->back->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) 6645 i40e_force_link_state(vsi->back, false); 6646 i40e_napi_disable_all(vsi); 6647 6648 for (i = 0; i < vsi->num_queue_pairs; i++) { 6649 i40e_clean_tx_ring(vsi->tx_rings[i]); 6650 if (i40e_enabled_xdp_vsi(vsi)) 6651 i40e_clean_tx_ring(vsi->xdp_rings[i]); 6652 i40e_clean_rx_ring(vsi->rx_rings[i]); 6653 } 6654 6655 } 6656 6657 /** 6658 * i40e_validate_mqprio_qopt- validate queue mapping info 6659 * @vsi: the VSI being configured 6660 * @mqprio_qopt: queue parametrs 6661 **/ 6662 static int i40e_validate_mqprio_qopt(struct i40e_vsi *vsi, 6663 struct tc_mqprio_qopt_offload *mqprio_qopt) 6664 { 6665 u64 sum_max_rate = 0; 6666 u64 max_rate = 0; 6667 int i; 6668 6669 if (mqprio_qopt->qopt.offset[0] != 0 || 6670 mqprio_qopt->qopt.num_tc < 1 || 6671 mqprio_qopt->qopt.num_tc > I40E_MAX_TRAFFIC_CLASS) 6672 return -EINVAL; 6673 for (i = 0; ; i++) { 6674 if (!mqprio_qopt->qopt.count[i]) 6675 return -EINVAL; 6676 if (mqprio_qopt->min_rate[i]) { 6677 dev_err(&vsi->back->pdev->dev, 6678 "Invalid min tx rate (greater than 0) specified\n"); 6679 return -EINVAL; 6680 } 6681 max_rate = mqprio_qopt->max_rate[i]; 6682 do_div(max_rate, I40E_BW_MBPS_DIVISOR); 6683 sum_max_rate += max_rate; 6684 6685 if (i >= mqprio_qopt->qopt.num_tc - 1) 6686 break; 6687 if (mqprio_qopt->qopt.offset[i + 1] != 6688 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) 6689 return -EINVAL; 6690 } 6691 if (vsi->num_queue_pairs < 6692 (mqprio_qopt->qopt.offset[i] + mqprio_qopt->qopt.count[i])) { 6693 return -EINVAL; 6694 } 6695 if (sum_max_rate > i40e_get_link_speed(vsi)) { 6696 dev_err(&vsi->back->pdev->dev, 6697 "Invalid max tx rate specified\n"); 6698 return -EINVAL; 6699 } 6700 return 0; 6701 } 6702 6703 /** 6704 * i40e_vsi_set_default_tc_config - set default values for tc configuration 6705 * @vsi: the VSI being configured 6706 **/ 6707 static void i40e_vsi_set_default_tc_config(struct i40e_vsi *vsi) 6708 { 6709 u16 qcount; 6710 int i; 6711 6712 /* Only TC0 is enabled */ 6713 vsi->tc_config.numtc = 1; 6714 vsi->tc_config.enabled_tc = 1; 6715 qcount = min_t(int, vsi->alloc_queue_pairs, 6716 i40e_pf_get_max_q_per_tc(vsi->back)); 6717 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 6718 /* For the TC that is not enabled set the offset to to default 6719 * queue and allocate one queue for the given TC. 6720 */ 6721 vsi->tc_config.tc_info[i].qoffset = 0; 6722 if (i == 0) 6723 vsi->tc_config.tc_info[i].qcount = qcount; 6724 else 6725 vsi->tc_config.tc_info[i].qcount = 1; 6726 vsi->tc_config.tc_info[i].netdev_tc = 0; 6727 } 6728 } 6729 6730 /** 6731 * i40e_setup_tc - configure multiple traffic classes 6732 * @netdev: net device to configure 6733 * @type_data: tc offload data 6734 **/ 6735 static int i40e_setup_tc(struct net_device *netdev, void *type_data) 6736 { 6737 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data; 6738 struct i40e_netdev_priv *np = netdev_priv(netdev); 6739 struct i40e_vsi *vsi = np->vsi; 6740 struct i40e_pf *pf = vsi->back; 6741 u8 enabled_tc = 0, num_tc, hw; 6742 bool need_reset = false; 6743 int ret = -EINVAL; 6744 u16 mode; 6745 int i; 6746 6747 num_tc = mqprio_qopt->qopt.num_tc; 6748 hw = mqprio_qopt->qopt.hw; 6749 mode = mqprio_qopt->mode; 6750 if (!hw) { 6751 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 6752 memcpy(&vsi->mqprio_qopt, mqprio_qopt, sizeof(*mqprio_qopt)); 6753 goto config_tc; 6754 } 6755 6756 /* Check if MFP enabled */ 6757 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 6758 netdev_info(netdev, 6759 "Configuring TC not supported in MFP mode\n"); 6760 return ret; 6761 } 6762 switch (mode) { 6763 case TC_MQPRIO_MODE_DCB: 6764 pf->flags &= ~I40E_FLAG_TC_MQPRIO; 6765 6766 /* Check if DCB enabled to continue */ 6767 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 6768 netdev_info(netdev, 6769 "DCB is not enabled for adapter\n"); 6770 return ret; 6771 } 6772 6773 /* Check whether tc count is within enabled limit */ 6774 if (num_tc > i40e_pf_get_num_tc(pf)) { 6775 netdev_info(netdev, 6776 "TC count greater than enabled on link for adapter\n"); 6777 return ret; 6778 } 6779 break; 6780 case TC_MQPRIO_MODE_CHANNEL: 6781 if (pf->flags & I40E_FLAG_DCB_ENABLED) { 6782 netdev_info(netdev, 6783 "Full offload of TC Mqprio options is not supported when DCB is enabled\n"); 6784 return ret; 6785 } 6786 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 6787 return ret; 6788 ret = i40e_validate_mqprio_qopt(vsi, mqprio_qopt); 6789 if (ret) 6790 return ret; 6791 memcpy(&vsi->mqprio_qopt, mqprio_qopt, 6792 sizeof(*mqprio_qopt)); 6793 pf->flags |= I40E_FLAG_TC_MQPRIO; 6794 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6795 break; 6796 default: 6797 return -EINVAL; 6798 } 6799 6800 config_tc: 6801 /* Generate TC map for number of tc requested */ 6802 for (i = 0; i < num_tc; i++) 6803 enabled_tc |= BIT(i); 6804 6805 /* Requesting same TC configuration as already enabled */ 6806 if (enabled_tc == vsi->tc_config.enabled_tc && 6807 mode != TC_MQPRIO_MODE_CHANNEL) 6808 return 0; 6809 6810 /* Quiesce VSI queues */ 6811 i40e_quiesce_vsi(vsi); 6812 6813 if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) 6814 i40e_remove_queue_channels(vsi); 6815 6816 /* Configure VSI for enabled TCs */ 6817 ret = i40e_vsi_config_tc(vsi, enabled_tc); 6818 if (ret) { 6819 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 6820 vsi->seid); 6821 need_reset = true; 6822 goto exit; 6823 } 6824 6825 if (pf->flags & I40E_FLAG_TC_MQPRIO) { 6826 if (vsi->mqprio_qopt.max_rate[0]) { 6827 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 6828 6829 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 6830 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 6831 if (!ret) { 6832 u64 credits = max_tx_rate; 6833 6834 do_div(credits, I40E_BW_CREDIT_DIVISOR); 6835 dev_dbg(&vsi->back->pdev->dev, 6836 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 6837 max_tx_rate, 6838 credits, 6839 vsi->seid); 6840 } else { 6841 need_reset = true; 6842 goto exit; 6843 } 6844 } 6845 ret = i40e_configure_queue_channels(vsi); 6846 if (ret) { 6847 netdev_info(netdev, 6848 "Failed configuring queue channels\n"); 6849 need_reset = true; 6850 goto exit; 6851 } 6852 } 6853 6854 exit: 6855 /* Reset the configuration data to defaults, only TC0 is enabled */ 6856 if (need_reset) { 6857 i40e_vsi_set_default_tc_config(vsi); 6858 need_reset = false; 6859 } 6860 6861 /* Unquiesce VSI */ 6862 i40e_unquiesce_vsi(vsi); 6863 return ret; 6864 } 6865 6866 /** 6867 * i40e_set_cld_element - sets cloud filter element data 6868 * @filter: cloud filter rule 6869 * @cld: ptr to cloud filter element data 6870 * 6871 * This is helper function to copy data into cloud filter element 6872 **/ 6873 static inline void 6874 i40e_set_cld_element(struct i40e_cloud_filter *filter, 6875 struct i40e_aqc_cloud_filters_element_data *cld) 6876 { 6877 int i, j; 6878 u32 ipa; 6879 6880 memset(cld, 0, sizeof(*cld)); 6881 ether_addr_copy(cld->outer_mac, filter->dst_mac); 6882 ether_addr_copy(cld->inner_mac, filter->src_mac); 6883 6884 if (filter->n_proto != ETH_P_IP && filter->n_proto != ETH_P_IPV6) 6885 return; 6886 6887 if (filter->n_proto == ETH_P_IPV6) { 6888 #define IPV6_MAX_INDEX (ARRAY_SIZE(filter->dst_ipv6) - 1) 6889 for (i = 0, j = 0; i < ARRAY_SIZE(filter->dst_ipv6); 6890 i++, j += 2) { 6891 ipa = be32_to_cpu(filter->dst_ipv6[IPV6_MAX_INDEX - i]); 6892 ipa = cpu_to_le32(ipa); 6893 memcpy(&cld->ipaddr.raw_v6.data[j], &ipa, sizeof(ipa)); 6894 } 6895 } else { 6896 ipa = be32_to_cpu(filter->dst_ipv4); 6897 memcpy(&cld->ipaddr.v4.data, &ipa, sizeof(ipa)); 6898 } 6899 6900 cld->inner_vlan = cpu_to_le16(ntohs(filter->vlan_id)); 6901 6902 /* tenant_id is not supported by FW now, once the support is enabled 6903 * fill the cld->tenant_id with cpu_to_le32(filter->tenant_id) 6904 */ 6905 if (filter->tenant_id) 6906 return; 6907 } 6908 6909 /** 6910 * i40e_add_del_cloud_filter - Add/del cloud filter 6911 * @vsi: pointer to VSI 6912 * @filter: cloud filter rule 6913 * @add: if true, add, if false, delete 6914 * 6915 * Add or delete a cloud filter for a specific flow spec. 6916 * Returns 0 if the filter were successfully added. 6917 **/ 6918 int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, 6919 struct i40e_cloud_filter *filter, bool add) 6920 { 6921 struct i40e_aqc_cloud_filters_element_data cld_filter; 6922 struct i40e_pf *pf = vsi->back; 6923 int ret; 6924 static const u16 flag_table[128] = { 6925 [I40E_CLOUD_FILTER_FLAGS_OMAC] = 6926 I40E_AQC_ADD_CLOUD_FILTER_OMAC, 6927 [I40E_CLOUD_FILTER_FLAGS_IMAC] = 6928 I40E_AQC_ADD_CLOUD_FILTER_IMAC, 6929 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN] = 6930 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN, 6931 [I40E_CLOUD_FILTER_FLAGS_IMAC_TEN_ID] = 6932 I40E_AQC_ADD_CLOUD_FILTER_IMAC_TEN_ID, 6933 [I40E_CLOUD_FILTER_FLAGS_OMAC_TEN_ID_IMAC] = 6934 I40E_AQC_ADD_CLOUD_FILTER_OMAC_TEN_ID_IMAC, 6935 [I40E_CLOUD_FILTER_FLAGS_IMAC_IVLAN_TEN_ID] = 6936 I40E_AQC_ADD_CLOUD_FILTER_IMAC_IVLAN_TEN_ID, 6937 [I40E_CLOUD_FILTER_FLAGS_IIP] = 6938 I40E_AQC_ADD_CLOUD_FILTER_IIP, 6939 }; 6940 6941 if (filter->flags >= ARRAY_SIZE(flag_table)) 6942 return I40E_ERR_CONFIG; 6943 6944 /* copy element needed to add cloud filter from filter */ 6945 i40e_set_cld_element(filter, &cld_filter); 6946 6947 if (filter->tunnel_type != I40E_CLOUD_TNL_TYPE_NONE) 6948 cld_filter.flags = cpu_to_le16(filter->tunnel_type << 6949 I40E_AQC_ADD_CLOUD_TNL_TYPE_SHIFT); 6950 6951 if (filter->n_proto == ETH_P_IPV6) 6952 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 6953 I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 6954 else 6955 cld_filter.flags |= cpu_to_le16(flag_table[filter->flags] | 6956 I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 6957 6958 if (add) 6959 ret = i40e_aq_add_cloud_filters(&pf->hw, filter->seid, 6960 &cld_filter, 1); 6961 else 6962 ret = i40e_aq_rem_cloud_filters(&pf->hw, filter->seid, 6963 &cld_filter, 1); 6964 if (ret) 6965 dev_dbg(&pf->pdev->dev, 6966 "Failed to %s cloud filter using l4 port %u, err %d aq_err %d\n", 6967 add ? "add" : "delete", filter->dst_port, ret, 6968 pf->hw.aq.asq_last_status); 6969 else 6970 dev_info(&pf->pdev->dev, 6971 "%s cloud filter for VSI: %d\n", 6972 add ? "Added" : "Deleted", filter->seid); 6973 return ret; 6974 } 6975 6976 /** 6977 * i40e_add_del_cloud_filter_big_buf - Add/del cloud filter using big_buf 6978 * @vsi: pointer to VSI 6979 * @filter: cloud filter rule 6980 * @add: if true, add, if false, delete 6981 * 6982 * Add or delete a cloud filter for a specific flow spec using big buffer. 6983 * Returns 0 if the filter were successfully added. 6984 **/ 6985 int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, 6986 struct i40e_cloud_filter *filter, 6987 bool add) 6988 { 6989 struct i40e_aqc_cloud_filters_element_bb cld_filter; 6990 struct i40e_pf *pf = vsi->back; 6991 int ret; 6992 6993 /* Both (src/dst) valid mac_addr are not supported */ 6994 if ((is_valid_ether_addr(filter->dst_mac) && 6995 is_valid_ether_addr(filter->src_mac)) || 6996 (is_multicast_ether_addr(filter->dst_mac) && 6997 is_multicast_ether_addr(filter->src_mac))) 6998 return -EOPNOTSUPP; 6999 7000 /* Big buffer cloud filter needs 'L4 port' to be non-zero. Also, UDP 7001 * ports are not supported via big buffer now. 7002 */ 7003 if (!filter->dst_port || filter->ip_proto == IPPROTO_UDP) 7004 return -EOPNOTSUPP; 7005 7006 /* adding filter using src_port/src_ip is not supported at this stage */ 7007 if (filter->src_port || filter->src_ipv4 || 7008 !ipv6_addr_any(&filter->ip.v6.src_ip6)) 7009 return -EOPNOTSUPP; 7010 7011 /* copy element needed to add cloud filter from filter */ 7012 i40e_set_cld_element(filter, &cld_filter.element); 7013 7014 if (is_valid_ether_addr(filter->dst_mac) || 7015 is_valid_ether_addr(filter->src_mac) || 7016 is_multicast_ether_addr(filter->dst_mac) || 7017 is_multicast_ether_addr(filter->src_mac)) { 7018 /* MAC + IP : unsupported mode */ 7019 if (filter->dst_ipv4) 7020 return -EOPNOTSUPP; 7021 7022 /* since we validated that L4 port must be valid before 7023 * we get here, start with respective "flags" value 7024 * and update if vlan is present or not 7025 */ 7026 cld_filter.element.flags = 7027 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_PORT); 7028 7029 if (filter->vlan_id) { 7030 cld_filter.element.flags = 7031 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_MAC_VLAN_PORT); 7032 } 7033 7034 } else if (filter->dst_ipv4 || 7035 !ipv6_addr_any(&filter->ip.v6.dst_ip6)) { 7036 cld_filter.element.flags = 7037 cpu_to_le16(I40E_AQC_ADD_CLOUD_FILTER_IP_PORT); 7038 if (filter->n_proto == ETH_P_IPV6) 7039 cld_filter.element.flags |= 7040 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV6); 7041 else 7042 cld_filter.element.flags |= 7043 cpu_to_le16(I40E_AQC_ADD_CLOUD_FLAGS_IPV4); 7044 } else { 7045 dev_err(&pf->pdev->dev, 7046 "either mac or ip has to be valid for cloud filter\n"); 7047 return -EINVAL; 7048 } 7049 7050 /* Now copy L4 port in Byte 6..7 in general fields */ 7051 cld_filter.general_fields[I40E_AQC_ADD_CLOUD_FV_FLU_0X16_WORD0] = 7052 be16_to_cpu(filter->dst_port); 7053 7054 if (add) { 7055 /* Validate current device switch mode, change if necessary */ 7056 ret = i40e_validate_and_set_switch_mode(vsi); 7057 if (ret) { 7058 dev_err(&pf->pdev->dev, 7059 "failed to set switch mode, ret %d\n", 7060 ret); 7061 return ret; 7062 } 7063 7064 ret = i40e_aq_add_cloud_filters_bb(&pf->hw, filter->seid, 7065 &cld_filter, 1); 7066 } else { 7067 ret = i40e_aq_rem_cloud_filters_bb(&pf->hw, filter->seid, 7068 &cld_filter, 1); 7069 } 7070 7071 if (ret) 7072 dev_dbg(&pf->pdev->dev, 7073 "Failed to %s cloud filter(big buffer) err %d aq_err %d\n", 7074 add ? "add" : "delete", ret, pf->hw.aq.asq_last_status); 7075 else 7076 dev_info(&pf->pdev->dev, 7077 "%s cloud filter for VSI: %d, L4 port: %d\n", 7078 add ? "add" : "delete", filter->seid, 7079 ntohs(filter->dst_port)); 7080 return ret; 7081 } 7082 7083 /** 7084 * i40e_parse_cls_flower - Parse tc flower filters provided by kernel 7085 * @vsi: Pointer to VSI 7086 * @cls_flower: Pointer to struct tc_cls_flower_offload 7087 * @filter: Pointer to cloud filter structure 7088 * 7089 **/ 7090 static int i40e_parse_cls_flower(struct i40e_vsi *vsi, 7091 struct tc_cls_flower_offload *f, 7092 struct i40e_cloud_filter *filter) 7093 { 7094 u16 n_proto_mask = 0, n_proto_key = 0, addr_type = 0; 7095 struct i40e_pf *pf = vsi->back; 7096 u8 field_flags = 0; 7097 7098 if (f->dissector->used_keys & 7099 ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) | 7100 BIT(FLOW_DISSECTOR_KEY_BASIC) | 7101 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) | 7102 BIT(FLOW_DISSECTOR_KEY_VLAN) | 7103 BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) | 7104 BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) | 7105 BIT(FLOW_DISSECTOR_KEY_PORTS) | 7106 BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) { 7107 dev_err(&pf->pdev->dev, "Unsupported key used: 0x%x\n", 7108 f->dissector->used_keys); 7109 return -EOPNOTSUPP; 7110 } 7111 7112 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ENC_KEYID)) { 7113 struct flow_dissector_key_keyid *key = 7114 skb_flow_dissector_target(f->dissector, 7115 FLOW_DISSECTOR_KEY_ENC_KEYID, 7116 f->key); 7117 7118 struct flow_dissector_key_keyid *mask = 7119 skb_flow_dissector_target(f->dissector, 7120 FLOW_DISSECTOR_KEY_ENC_KEYID, 7121 f->mask); 7122 7123 if (mask->keyid != 0) 7124 field_flags |= I40E_CLOUD_FIELD_TEN_ID; 7125 7126 filter->tenant_id = be32_to_cpu(key->keyid); 7127 } 7128 7129 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_BASIC)) { 7130 struct flow_dissector_key_basic *key = 7131 skb_flow_dissector_target(f->dissector, 7132 FLOW_DISSECTOR_KEY_BASIC, 7133 f->key); 7134 7135 struct flow_dissector_key_basic *mask = 7136 skb_flow_dissector_target(f->dissector, 7137 FLOW_DISSECTOR_KEY_BASIC, 7138 f->mask); 7139 7140 n_proto_key = ntohs(key->n_proto); 7141 n_proto_mask = ntohs(mask->n_proto); 7142 7143 if (n_proto_key == ETH_P_ALL) { 7144 n_proto_key = 0; 7145 n_proto_mask = 0; 7146 } 7147 filter->n_proto = n_proto_key & n_proto_mask; 7148 filter->ip_proto = key->ip_proto; 7149 } 7150 7151 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_ETH_ADDRS)) { 7152 struct flow_dissector_key_eth_addrs *key = 7153 skb_flow_dissector_target(f->dissector, 7154 FLOW_DISSECTOR_KEY_ETH_ADDRS, 7155 f->key); 7156 7157 struct flow_dissector_key_eth_addrs *mask = 7158 skb_flow_dissector_target(f->dissector, 7159 FLOW_DISSECTOR_KEY_ETH_ADDRS, 7160 f->mask); 7161 7162 /* use is_broadcast and is_zero to check for all 0xf or 0 */ 7163 if (!is_zero_ether_addr(mask->dst)) { 7164 if (is_broadcast_ether_addr(mask->dst)) { 7165 field_flags |= I40E_CLOUD_FIELD_OMAC; 7166 } else { 7167 dev_err(&pf->pdev->dev, "Bad ether dest mask %pM\n", 7168 mask->dst); 7169 return I40E_ERR_CONFIG; 7170 } 7171 } 7172 7173 if (!is_zero_ether_addr(mask->src)) { 7174 if (is_broadcast_ether_addr(mask->src)) { 7175 field_flags |= I40E_CLOUD_FIELD_IMAC; 7176 } else { 7177 dev_err(&pf->pdev->dev, "Bad ether src mask %pM\n", 7178 mask->src); 7179 return I40E_ERR_CONFIG; 7180 } 7181 } 7182 ether_addr_copy(filter->dst_mac, key->dst); 7183 ether_addr_copy(filter->src_mac, key->src); 7184 } 7185 7186 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_VLAN)) { 7187 struct flow_dissector_key_vlan *key = 7188 skb_flow_dissector_target(f->dissector, 7189 FLOW_DISSECTOR_KEY_VLAN, 7190 f->key); 7191 struct flow_dissector_key_vlan *mask = 7192 skb_flow_dissector_target(f->dissector, 7193 FLOW_DISSECTOR_KEY_VLAN, 7194 f->mask); 7195 7196 if (mask->vlan_id) { 7197 if (mask->vlan_id == VLAN_VID_MASK) { 7198 field_flags |= I40E_CLOUD_FIELD_IVLAN; 7199 7200 } else { 7201 dev_err(&pf->pdev->dev, "Bad vlan mask 0x%04x\n", 7202 mask->vlan_id); 7203 return I40E_ERR_CONFIG; 7204 } 7205 } 7206 7207 filter->vlan_id = cpu_to_be16(key->vlan_id); 7208 } 7209 7210 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_CONTROL)) { 7211 struct flow_dissector_key_control *key = 7212 skb_flow_dissector_target(f->dissector, 7213 FLOW_DISSECTOR_KEY_CONTROL, 7214 f->key); 7215 7216 addr_type = key->addr_type; 7217 } 7218 7219 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) { 7220 struct flow_dissector_key_ipv4_addrs *key = 7221 skb_flow_dissector_target(f->dissector, 7222 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 7223 f->key); 7224 struct flow_dissector_key_ipv4_addrs *mask = 7225 skb_flow_dissector_target(f->dissector, 7226 FLOW_DISSECTOR_KEY_IPV4_ADDRS, 7227 f->mask); 7228 7229 if (mask->dst) { 7230 if (mask->dst == cpu_to_be32(0xffffffff)) { 7231 field_flags |= I40E_CLOUD_FIELD_IIP; 7232 } else { 7233 dev_err(&pf->pdev->dev, "Bad ip dst mask %pI4b\n", 7234 &mask->dst); 7235 return I40E_ERR_CONFIG; 7236 } 7237 } 7238 7239 if (mask->src) { 7240 if (mask->src == cpu_to_be32(0xffffffff)) { 7241 field_flags |= I40E_CLOUD_FIELD_IIP; 7242 } else { 7243 dev_err(&pf->pdev->dev, "Bad ip src mask %pI4b\n", 7244 &mask->src); 7245 return I40E_ERR_CONFIG; 7246 } 7247 } 7248 7249 if (field_flags & I40E_CLOUD_FIELD_TEN_ID) { 7250 dev_err(&pf->pdev->dev, "Tenant id not allowed for ip filter\n"); 7251 return I40E_ERR_CONFIG; 7252 } 7253 filter->dst_ipv4 = key->dst; 7254 filter->src_ipv4 = key->src; 7255 } 7256 7257 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) { 7258 struct flow_dissector_key_ipv6_addrs *key = 7259 skb_flow_dissector_target(f->dissector, 7260 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 7261 f->key); 7262 struct flow_dissector_key_ipv6_addrs *mask = 7263 skb_flow_dissector_target(f->dissector, 7264 FLOW_DISSECTOR_KEY_IPV6_ADDRS, 7265 f->mask); 7266 7267 /* src and dest IPV6 address should not be LOOPBACK 7268 * (0:0:0:0:0:0:0:1), which can be represented as ::1 7269 */ 7270 if (ipv6_addr_loopback(&key->dst) || 7271 ipv6_addr_loopback(&key->src)) { 7272 dev_err(&pf->pdev->dev, 7273 "Bad ipv6, addr is LOOPBACK\n"); 7274 return I40E_ERR_CONFIG; 7275 } 7276 if (!ipv6_addr_any(&mask->dst) || !ipv6_addr_any(&mask->src)) 7277 field_flags |= I40E_CLOUD_FIELD_IIP; 7278 7279 memcpy(&filter->src_ipv6, &key->src.s6_addr32, 7280 sizeof(filter->src_ipv6)); 7281 memcpy(&filter->dst_ipv6, &key->dst.s6_addr32, 7282 sizeof(filter->dst_ipv6)); 7283 } 7284 7285 if (dissector_uses_key(f->dissector, FLOW_DISSECTOR_KEY_PORTS)) { 7286 struct flow_dissector_key_ports *key = 7287 skb_flow_dissector_target(f->dissector, 7288 FLOW_DISSECTOR_KEY_PORTS, 7289 f->key); 7290 struct flow_dissector_key_ports *mask = 7291 skb_flow_dissector_target(f->dissector, 7292 FLOW_DISSECTOR_KEY_PORTS, 7293 f->mask); 7294 7295 if (mask->src) { 7296 if (mask->src == cpu_to_be16(0xffff)) { 7297 field_flags |= I40E_CLOUD_FIELD_IIP; 7298 } else { 7299 dev_err(&pf->pdev->dev, "Bad src port mask 0x%04x\n", 7300 be16_to_cpu(mask->src)); 7301 return I40E_ERR_CONFIG; 7302 } 7303 } 7304 7305 if (mask->dst) { 7306 if (mask->dst == cpu_to_be16(0xffff)) { 7307 field_flags |= I40E_CLOUD_FIELD_IIP; 7308 } else { 7309 dev_err(&pf->pdev->dev, "Bad dst port mask 0x%04x\n", 7310 be16_to_cpu(mask->dst)); 7311 return I40E_ERR_CONFIG; 7312 } 7313 } 7314 7315 filter->dst_port = key->dst; 7316 filter->src_port = key->src; 7317 7318 switch (filter->ip_proto) { 7319 case IPPROTO_TCP: 7320 case IPPROTO_UDP: 7321 break; 7322 default: 7323 dev_err(&pf->pdev->dev, 7324 "Only UDP and TCP transport are supported\n"); 7325 return -EINVAL; 7326 } 7327 } 7328 filter->flags = field_flags; 7329 return 0; 7330 } 7331 7332 /** 7333 * i40e_handle_tclass: Forward to a traffic class on the device 7334 * @vsi: Pointer to VSI 7335 * @tc: traffic class index on the device 7336 * @filter: Pointer to cloud filter structure 7337 * 7338 **/ 7339 static int i40e_handle_tclass(struct i40e_vsi *vsi, u32 tc, 7340 struct i40e_cloud_filter *filter) 7341 { 7342 struct i40e_channel *ch, *ch_tmp; 7343 7344 /* direct to a traffic class on the same device */ 7345 if (tc == 0) { 7346 filter->seid = vsi->seid; 7347 return 0; 7348 } else if (vsi->tc_config.enabled_tc & BIT(tc)) { 7349 if (!filter->dst_port) { 7350 dev_err(&vsi->back->pdev->dev, 7351 "Specify destination port to direct to traffic class that is not default\n"); 7352 return -EINVAL; 7353 } 7354 if (list_empty(&vsi->ch_list)) 7355 return -EINVAL; 7356 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, 7357 list) { 7358 if (ch->seid == vsi->tc_seid_map[tc]) 7359 filter->seid = ch->seid; 7360 } 7361 return 0; 7362 } 7363 dev_err(&vsi->back->pdev->dev, "TC is not enabled\n"); 7364 return -EINVAL; 7365 } 7366 7367 /** 7368 * i40e_configure_clsflower - Configure tc flower filters 7369 * @vsi: Pointer to VSI 7370 * @cls_flower: Pointer to struct tc_cls_flower_offload 7371 * 7372 **/ 7373 static int i40e_configure_clsflower(struct i40e_vsi *vsi, 7374 struct tc_cls_flower_offload *cls_flower) 7375 { 7376 int tc = tc_classid_to_hwtc(vsi->netdev, cls_flower->classid); 7377 struct i40e_cloud_filter *filter = NULL; 7378 struct i40e_pf *pf = vsi->back; 7379 int err = 0; 7380 7381 if (tc < 0) { 7382 dev_err(&vsi->back->pdev->dev, "Invalid traffic class\n"); 7383 return -EOPNOTSUPP; 7384 } 7385 7386 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state) || 7387 test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) 7388 return -EBUSY; 7389 7390 if (pf->fdir_pf_active_filters || 7391 (!hlist_empty(&pf->fdir_filter_list))) { 7392 dev_err(&vsi->back->pdev->dev, 7393 "Flow Director Sideband filters exists, turn ntuple off to configure cloud filters\n"); 7394 return -EINVAL; 7395 } 7396 7397 if (vsi->back->flags & I40E_FLAG_FD_SB_ENABLED) { 7398 dev_err(&vsi->back->pdev->dev, 7399 "Disable Flow Director Sideband, configuring Cloud filters via tc-flower\n"); 7400 vsi->back->flags &= ~I40E_FLAG_FD_SB_ENABLED; 7401 vsi->back->flags |= I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7402 } 7403 7404 filter = kzalloc(sizeof(*filter), GFP_KERNEL); 7405 if (!filter) 7406 return -ENOMEM; 7407 7408 filter->cookie = cls_flower->cookie; 7409 7410 err = i40e_parse_cls_flower(vsi, cls_flower, filter); 7411 if (err < 0) 7412 goto err; 7413 7414 err = i40e_handle_tclass(vsi, tc, filter); 7415 if (err < 0) 7416 goto err; 7417 7418 /* Add cloud filter */ 7419 if (filter->dst_port) 7420 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, true); 7421 else 7422 err = i40e_add_del_cloud_filter(vsi, filter, true); 7423 7424 if (err) { 7425 dev_err(&pf->pdev->dev, 7426 "Failed to add cloud filter, err %s\n", 7427 i40e_stat_str(&pf->hw, err)); 7428 goto err; 7429 } 7430 7431 /* add filter to the ordered list */ 7432 INIT_HLIST_NODE(&filter->cloud_node); 7433 7434 hlist_add_head(&filter->cloud_node, &pf->cloud_filter_list); 7435 7436 pf->num_cloud_filters++; 7437 7438 return err; 7439 err: 7440 kfree(filter); 7441 return err; 7442 } 7443 7444 /** 7445 * i40e_find_cloud_filter - Find the could filter in the list 7446 * @vsi: Pointer to VSI 7447 * @cookie: filter specific cookie 7448 * 7449 **/ 7450 static struct i40e_cloud_filter *i40e_find_cloud_filter(struct i40e_vsi *vsi, 7451 unsigned long *cookie) 7452 { 7453 struct i40e_cloud_filter *filter = NULL; 7454 struct hlist_node *node2; 7455 7456 hlist_for_each_entry_safe(filter, node2, 7457 &vsi->back->cloud_filter_list, cloud_node) 7458 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie))) 7459 return filter; 7460 return NULL; 7461 } 7462 7463 /** 7464 * i40e_delete_clsflower - Remove tc flower filters 7465 * @vsi: Pointer to VSI 7466 * @cls_flower: Pointer to struct tc_cls_flower_offload 7467 * 7468 **/ 7469 static int i40e_delete_clsflower(struct i40e_vsi *vsi, 7470 struct tc_cls_flower_offload *cls_flower) 7471 { 7472 struct i40e_cloud_filter *filter = NULL; 7473 struct i40e_pf *pf = vsi->back; 7474 int err = 0; 7475 7476 filter = i40e_find_cloud_filter(vsi, &cls_flower->cookie); 7477 7478 if (!filter) 7479 return -EINVAL; 7480 7481 hash_del(&filter->cloud_node); 7482 7483 if (filter->dst_port) 7484 err = i40e_add_del_cloud_filter_big_buf(vsi, filter, false); 7485 else 7486 err = i40e_add_del_cloud_filter(vsi, filter, false); 7487 7488 kfree(filter); 7489 if (err) { 7490 dev_err(&pf->pdev->dev, 7491 "Failed to delete cloud filter, err %s\n", 7492 i40e_stat_str(&pf->hw, err)); 7493 return i40e_aq_rc_to_posix(err, pf->hw.aq.asq_last_status); 7494 } 7495 7496 pf->num_cloud_filters--; 7497 if (!pf->num_cloud_filters) 7498 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 7499 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 7500 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 7501 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7502 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 7503 } 7504 return 0; 7505 } 7506 7507 /** 7508 * i40e_setup_tc_cls_flower - flower classifier offloads 7509 * @netdev: net device to configure 7510 * @type_data: offload data 7511 **/ 7512 static int i40e_setup_tc_cls_flower(struct i40e_netdev_priv *np, 7513 struct tc_cls_flower_offload *cls_flower) 7514 { 7515 struct i40e_vsi *vsi = np->vsi; 7516 7517 switch (cls_flower->command) { 7518 case TC_CLSFLOWER_REPLACE: 7519 return i40e_configure_clsflower(vsi, cls_flower); 7520 case TC_CLSFLOWER_DESTROY: 7521 return i40e_delete_clsflower(vsi, cls_flower); 7522 case TC_CLSFLOWER_STATS: 7523 return -EOPNOTSUPP; 7524 default: 7525 return -EINVAL; 7526 } 7527 } 7528 7529 static int i40e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 7530 void *cb_priv) 7531 { 7532 struct i40e_netdev_priv *np = cb_priv; 7533 7534 if (!tc_cls_can_offload_and_chain0(np->vsi->netdev, type_data)) 7535 return -EOPNOTSUPP; 7536 7537 switch (type) { 7538 case TC_SETUP_CLSFLOWER: 7539 return i40e_setup_tc_cls_flower(np, type_data); 7540 7541 default: 7542 return -EOPNOTSUPP; 7543 } 7544 } 7545 7546 static int i40e_setup_tc_block(struct net_device *dev, 7547 struct tc_block_offload *f) 7548 { 7549 struct i40e_netdev_priv *np = netdev_priv(dev); 7550 7551 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 7552 return -EOPNOTSUPP; 7553 7554 switch (f->command) { 7555 case TC_BLOCK_BIND: 7556 return tcf_block_cb_register(f->block, i40e_setup_tc_block_cb, 7557 np, np); 7558 case TC_BLOCK_UNBIND: 7559 tcf_block_cb_unregister(f->block, i40e_setup_tc_block_cb, np); 7560 return 0; 7561 default: 7562 return -EOPNOTSUPP; 7563 } 7564 } 7565 7566 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 7567 void *type_data) 7568 { 7569 switch (type) { 7570 case TC_SETUP_QDISC_MQPRIO: 7571 return i40e_setup_tc(netdev, type_data); 7572 case TC_SETUP_BLOCK: 7573 return i40e_setup_tc_block(netdev, type_data); 7574 default: 7575 return -EOPNOTSUPP; 7576 } 7577 } 7578 7579 /** 7580 * i40e_open - Called when a network interface is made active 7581 * @netdev: network interface device structure 7582 * 7583 * The open entry point is called when a network interface is made 7584 * active by the system (IFF_UP). At this point all resources needed 7585 * for transmit and receive operations are allocated, the interrupt 7586 * handler is registered with the OS, the netdev watchdog subtask is 7587 * enabled, and the stack is notified that the interface is ready. 7588 * 7589 * Returns 0 on success, negative value on failure 7590 **/ 7591 int i40e_open(struct net_device *netdev) 7592 { 7593 struct i40e_netdev_priv *np = netdev_priv(netdev); 7594 struct i40e_vsi *vsi = np->vsi; 7595 struct i40e_pf *pf = vsi->back; 7596 int err; 7597 7598 /* disallow open during test or if eeprom is broken */ 7599 if (test_bit(__I40E_TESTING, pf->state) || 7600 test_bit(__I40E_BAD_EEPROM, pf->state)) 7601 return -EBUSY; 7602 7603 netif_carrier_off(netdev); 7604 7605 if (i40e_force_link_state(pf, true)) 7606 return -EAGAIN; 7607 7608 err = i40e_vsi_open(vsi); 7609 if (err) 7610 return err; 7611 7612 /* configure global TSO hardware offload settings */ 7613 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 7614 TCP_FLAG_FIN) >> 16); 7615 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 7616 TCP_FLAG_FIN | 7617 TCP_FLAG_CWR) >> 16); 7618 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 7619 7620 udp_tunnel_get_rx_info(netdev); 7621 7622 return 0; 7623 } 7624 7625 /** 7626 * i40e_vsi_open - 7627 * @vsi: the VSI to open 7628 * 7629 * Finish initialization of the VSI. 7630 * 7631 * Returns 0 on success, negative value on failure 7632 * 7633 * Note: expects to be called while under rtnl_lock() 7634 **/ 7635 int i40e_vsi_open(struct i40e_vsi *vsi) 7636 { 7637 struct i40e_pf *pf = vsi->back; 7638 char int_name[I40E_INT_NAME_STR_LEN]; 7639 int err; 7640 7641 /* allocate descriptors */ 7642 err = i40e_vsi_setup_tx_resources(vsi); 7643 if (err) 7644 goto err_setup_tx; 7645 err = i40e_vsi_setup_rx_resources(vsi); 7646 if (err) 7647 goto err_setup_rx; 7648 7649 err = i40e_vsi_configure(vsi); 7650 if (err) 7651 goto err_setup_rx; 7652 7653 if (vsi->netdev) { 7654 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 7655 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 7656 err = i40e_vsi_request_irq(vsi, int_name); 7657 if (err) 7658 goto err_setup_rx; 7659 7660 /* Notify the stack of the actual queue counts. */ 7661 err = netif_set_real_num_tx_queues(vsi->netdev, 7662 vsi->num_queue_pairs); 7663 if (err) 7664 goto err_set_queues; 7665 7666 err = netif_set_real_num_rx_queues(vsi->netdev, 7667 vsi->num_queue_pairs); 7668 if (err) 7669 goto err_set_queues; 7670 7671 } else if (vsi->type == I40E_VSI_FDIR) { 7672 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 7673 dev_driver_string(&pf->pdev->dev), 7674 dev_name(&pf->pdev->dev)); 7675 err = i40e_vsi_request_irq(vsi, int_name); 7676 7677 } else { 7678 err = -EINVAL; 7679 goto err_setup_rx; 7680 } 7681 7682 err = i40e_up_complete(vsi); 7683 if (err) 7684 goto err_up_complete; 7685 7686 return 0; 7687 7688 err_up_complete: 7689 i40e_down(vsi); 7690 err_set_queues: 7691 i40e_vsi_free_irq(vsi); 7692 err_setup_rx: 7693 i40e_vsi_free_rx_resources(vsi); 7694 err_setup_tx: 7695 i40e_vsi_free_tx_resources(vsi); 7696 if (vsi == pf->vsi[pf->lan_vsi]) 7697 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 7698 7699 return err; 7700 } 7701 7702 /** 7703 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 7704 * @pf: Pointer to PF 7705 * 7706 * This function destroys the hlist where all the Flow Director 7707 * filters were saved. 7708 **/ 7709 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 7710 { 7711 struct i40e_fdir_filter *filter; 7712 struct i40e_flex_pit *pit_entry, *tmp; 7713 struct hlist_node *node2; 7714 7715 hlist_for_each_entry_safe(filter, node2, 7716 &pf->fdir_filter_list, fdir_node) { 7717 hlist_del(&filter->fdir_node); 7718 kfree(filter); 7719 } 7720 7721 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 7722 list_del(&pit_entry->list); 7723 kfree(pit_entry); 7724 } 7725 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 7726 7727 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 7728 list_del(&pit_entry->list); 7729 kfree(pit_entry); 7730 } 7731 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 7732 7733 pf->fdir_pf_active_filters = 0; 7734 pf->fd_tcp4_filter_cnt = 0; 7735 pf->fd_udp4_filter_cnt = 0; 7736 pf->fd_sctp4_filter_cnt = 0; 7737 pf->fd_ip4_filter_cnt = 0; 7738 7739 /* Reprogram the default input set for TCP/IPv4 */ 7740 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 7741 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7742 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7743 7744 /* Reprogram the default input set for UDP/IPv4 */ 7745 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 7746 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7747 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7748 7749 /* Reprogram the default input set for SCTP/IPv4 */ 7750 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 7751 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 7752 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 7753 7754 /* Reprogram the default input set for Other/IPv4 */ 7755 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 7756 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 7757 7758 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_FRAG_IPV4, 7759 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 7760 } 7761 7762 /** 7763 * i40e_cloud_filter_exit - Cleans up the cloud filters 7764 * @pf: Pointer to PF 7765 * 7766 * This function destroys the hlist where all the cloud filters 7767 * were saved. 7768 **/ 7769 static void i40e_cloud_filter_exit(struct i40e_pf *pf) 7770 { 7771 struct i40e_cloud_filter *cfilter; 7772 struct hlist_node *node; 7773 7774 hlist_for_each_entry_safe(cfilter, node, 7775 &pf->cloud_filter_list, cloud_node) { 7776 hlist_del(&cfilter->cloud_node); 7777 kfree(cfilter); 7778 } 7779 pf->num_cloud_filters = 0; 7780 7781 if ((pf->flags & I40E_FLAG_FD_SB_TO_CLOUD_FILTER) && 7782 !(pf->flags & I40E_FLAG_FD_SB_INACTIVE)) { 7783 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 7784 pf->flags &= ~I40E_FLAG_FD_SB_TO_CLOUD_FILTER; 7785 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 7786 } 7787 } 7788 7789 /** 7790 * i40e_close - Disables a network interface 7791 * @netdev: network interface device structure 7792 * 7793 * The close entry point is called when an interface is de-activated 7794 * by the OS. The hardware is still under the driver's control, but 7795 * this netdev interface is disabled. 7796 * 7797 * Returns 0, this is not allowed to fail 7798 **/ 7799 int i40e_close(struct net_device *netdev) 7800 { 7801 struct i40e_netdev_priv *np = netdev_priv(netdev); 7802 struct i40e_vsi *vsi = np->vsi; 7803 7804 i40e_vsi_close(vsi); 7805 7806 return 0; 7807 } 7808 7809 /** 7810 * i40e_do_reset - Start a PF or Core Reset sequence 7811 * @pf: board private structure 7812 * @reset_flags: which reset is requested 7813 * @lock_acquired: indicates whether or not the lock has been acquired 7814 * before this function was called. 7815 * 7816 * The essential difference in resets is that the PF Reset 7817 * doesn't clear the packet buffers, doesn't reset the PE 7818 * firmware, and doesn't bother the other PFs on the chip. 7819 **/ 7820 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 7821 { 7822 u32 val; 7823 7824 WARN_ON(in_interrupt()); 7825 7826 7827 /* do the biggest reset indicated */ 7828 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 7829 7830 /* Request a Global Reset 7831 * 7832 * This will start the chip's countdown to the actual full 7833 * chip reset event, and a warning interrupt to be sent 7834 * to all PFs, including the requestor. Our handler 7835 * for the warning interrupt will deal with the shutdown 7836 * and recovery of the switch setup. 7837 */ 7838 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 7839 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 7840 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 7841 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 7842 7843 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 7844 7845 /* Request a Core Reset 7846 * 7847 * Same as Global Reset, except does *not* include the MAC/PHY 7848 */ 7849 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 7850 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 7851 val |= I40E_GLGEN_RTRIG_CORER_MASK; 7852 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 7853 i40e_flush(&pf->hw); 7854 7855 } else if (reset_flags & I40E_PF_RESET_FLAG) { 7856 7857 /* Request a PF Reset 7858 * 7859 * Resets only the PF-specific registers 7860 * 7861 * This goes directly to the tear-down and rebuild of 7862 * the switch, since we need to do all the recovery as 7863 * for the Core Reset. 7864 */ 7865 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 7866 i40e_handle_reset_warning(pf, lock_acquired); 7867 7868 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 7869 int v; 7870 7871 /* Find the VSI(s) that requested a re-init */ 7872 dev_info(&pf->pdev->dev, 7873 "VSI reinit requested\n"); 7874 for (v = 0; v < pf->num_alloc_vsi; v++) { 7875 struct i40e_vsi *vsi = pf->vsi[v]; 7876 7877 if (vsi != NULL && 7878 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 7879 vsi->state)) 7880 i40e_vsi_reinit_locked(pf->vsi[v]); 7881 } 7882 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 7883 int v; 7884 7885 /* Find the VSI(s) that needs to be brought down */ 7886 dev_info(&pf->pdev->dev, "VSI down requested\n"); 7887 for (v = 0; v < pf->num_alloc_vsi; v++) { 7888 struct i40e_vsi *vsi = pf->vsi[v]; 7889 7890 if (vsi != NULL && 7891 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 7892 vsi->state)) { 7893 set_bit(__I40E_VSI_DOWN, vsi->state); 7894 i40e_down(vsi); 7895 } 7896 } 7897 } else { 7898 dev_info(&pf->pdev->dev, 7899 "bad reset request 0x%08x\n", reset_flags); 7900 } 7901 } 7902 7903 #ifdef CONFIG_I40E_DCB 7904 /** 7905 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 7906 * @pf: board private structure 7907 * @old_cfg: current DCB config 7908 * @new_cfg: new DCB config 7909 **/ 7910 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 7911 struct i40e_dcbx_config *old_cfg, 7912 struct i40e_dcbx_config *new_cfg) 7913 { 7914 bool need_reconfig = false; 7915 7916 /* Check if ETS configuration has changed */ 7917 if (memcmp(&new_cfg->etscfg, 7918 &old_cfg->etscfg, 7919 sizeof(new_cfg->etscfg))) { 7920 /* If Priority Table has changed reconfig is needed */ 7921 if (memcmp(&new_cfg->etscfg.prioritytable, 7922 &old_cfg->etscfg.prioritytable, 7923 sizeof(new_cfg->etscfg.prioritytable))) { 7924 need_reconfig = true; 7925 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 7926 } 7927 7928 if (memcmp(&new_cfg->etscfg.tcbwtable, 7929 &old_cfg->etscfg.tcbwtable, 7930 sizeof(new_cfg->etscfg.tcbwtable))) 7931 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 7932 7933 if (memcmp(&new_cfg->etscfg.tsatable, 7934 &old_cfg->etscfg.tsatable, 7935 sizeof(new_cfg->etscfg.tsatable))) 7936 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 7937 } 7938 7939 /* Check if PFC configuration has changed */ 7940 if (memcmp(&new_cfg->pfc, 7941 &old_cfg->pfc, 7942 sizeof(new_cfg->pfc))) { 7943 need_reconfig = true; 7944 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 7945 } 7946 7947 /* Check if APP Table has changed */ 7948 if (memcmp(&new_cfg->app, 7949 &old_cfg->app, 7950 sizeof(new_cfg->app))) { 7951 need_reconfig = true; 7952 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 7953 } 7954 7955 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 7956 return need_reconfig; 7957 } 7958 7959 /** 7960 * i40e_handle_lldp_event - Handle LLDP Change MIB event 7961 * @pf: board private structure 7962 * @e: event info posted on ARQ 7963 **/ 7964 static int i40e_handle_lldp_event(struct i40e_pf *pf, 7965 struct i40e_arq_event_info *e) 7966 { 7967 struct i40e_aqc_lldp_get_mib *mib = 7968 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 7969 struct i40e_hw *hw = &pf->hw; 7970 struct i40e_dcbx_config tmp_dcbx_cfg; 7971 bool need_reconfig = false; 7972 int ret = 0; 7973 u8 type; 7974 7975 /* Not DCB capable or capability disabled */ 7976 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 7977 return ret; 7978 7979 /* Ignore if event is not for Nearest Bridge */ 7980 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 7981 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 7982 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 7983 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 7984 return ret; 7985 7986 /* Check MIB Type and return if event for Remote MIB update */ 7987 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 7988 dev_dbg(&pf->pdev->dev, 7989 "LLDP event mib type %s\n", type ? "remote" : "local"); 7990 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 7991 /* Update the remote cached instance and return */ 7992 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 7993 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 7994 &hw->remote_dcbx_config); 7995 goto exit; 7996 } 7997 7998 /* Store the old configuration */ 7999 tmp_dcbx_cfg = hw->local_dcbx_config; 8000 8001 /* Reset the old DCBx configuration data */ 8002 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 8003 /* Get updated DCBX data from firmware */ 8004 ret = i40e_get_dcb_config(&pf->hw); 8005 if (ret) { 8006 dev_info(&pf->pdev->dev, 8007 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 8008 i40e_stat_str(&pf->hw, ret), 8009 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8010 goto exit; 8011 } 8012 8013 /* No change detected in DCBX configs */ 8014 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 8015 sizeof(tmp_dcbx_cfg))) { 8016 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 8017 goto exit; 8018 } 8019 8020 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 8021 &hw->local_dcbx_config); 8022 8023 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 8024 8025 if (!need_reconfig) 8026 goto exit; 8027 8028 /* Enable DCB tagging only when more than one TC */ 8029 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 8030 pf->flags |= I40E_FLAG_DCB_ENABLED; 8031 else 8032 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 8033 8034 set_bit(__I40E_PORT_SUSPENDED, pf->state); 8035 /* Reconfiguration needed quiesce all VSIs */ 8036 i40e_pf_quiesce_all_vsi(pf); 8037 8038 /* Changes in configuration update VEB/VSI */ 8039 i40e_dcb_reconfigure(pf); 8040 8041 ret = i40e_resume_port_tx(pf); 8042 8043 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 8044 /* In case of error no point in resuming VSIs */ 8045 if (ret) 8046 goto exit; 8047 8048 /* Wait for the PF's queues to be disabled */ 8049 ret = i40e_pf_wait_queues_disabled(pf); 8050 if (ret) { 8051 /* Schedule PF reset to recover */ 8052 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8053 i40e_service_event_schedule(pf); 8054 } else { 8055 i40e_pf_unquiesce_all_vsi(pf); 8056 set_bit(__I40E_CLIENT_SERVICE_REQUESTED, pf->state); 8057 set_bit(__I40E_CLIENT_L2_CHANGE, pf->state); 8058 } 8059 8060 exit: 8061 return ret; 8062 } 8063 #endif /* CONFIG_I40E_DCB */ 8064 8065 /** 8066 * i40e_do_reset_safe - Protected reset path for userland calls. 8067 * @pf: board private structure 8068 * @reset_flags: which reset is requested 8069 * 8070 **/ 8071 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 8072 { 8073 rtnl_lock(); 8074 i40e_do_reset(pf, reset_flags, true); 8075 rtnl_unlock(); 8076 } 8077 8078 /** 8079 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 8080 * @pf: board private structure 8081 * @e: event info posted on ARQ 8082 * 8083 * Handler for LAN Queue Overflow Event generated by the firmware for PF 8084 * and VF queues 8085 **/ 8086 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 8087 struct i40e_arq_event_info *e) 8088 { 8089 struct i40e_aqc_lan_overflow *data = 8090 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 8091 u32 queue = le32_to_cpu(data->prtdcb_rupto); 8092 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 8093 struct i40e_hw *hw = &pf->hw; 8094 struct i40e_vf *vf; 8095 u16 vf_id; 8096 8097 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 8098 queue, qtx_ctl); 8099 8100 /* Queue belongs to VF, find the VF and issue VF reset */ 8101 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 8102 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 8103 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 8104 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 8105 vf_id -= hw->func_caps.vf_base_id; 8106 vf = &pf->vf[vf_id]; 8107 i40e_vc_notify_vf_reset(vf); 8108 /* Allow VF to process pending reset notification */ 8109 msleep(20); 8110 i40e_reset_vf(vf, false); 8111 } 8112 } 8113 8114 /** 8115 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 8116 * @pf: board private structure 8117 **/ 8118 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 8119 { 8120 u32 val, fcnt_prog; 8121 8122 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8123 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 8124 return fcnt_prog; 8125 } 8126 8127 /** 8128 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 8129 * @pf: board private structure 8130 **/ 8131 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 8132 { 8133 u32 val, fcnt_prog; 8134 8135 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 8136 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 8137 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 8138 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 8139 return fcnt_prog; 8140 } 8141 8142 /** 8143 * i40e_get_global_fd_count - Get total FD filters programmed on device 8144 * @pf: board private structure 8145 **/ 8146 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 8147 { 8148 u32 val, fcnt_prog; 8149 8150 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 8151 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 8152 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 8153 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 8154 return fcnt_prog; 8155 } 8156 8157 /** 8158 * i40e_reenable_fdir_sb - Restore FDir SB capability 8159 * @pf: board private structure 8160 **/ 8161 static void i40e_reenable_fdir_sb(struct i40e_pf *pf) 8162 { 8163 if (test_and_clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) 8164 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8165 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8166 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 8167 } 8168 8169 /** 8170 * i40e_reenable_fdir_atr - Restore FDir ATR capability 8171 * @pf: board private structure 8172 **/ 8173 static void i40e_reenable_fdir_atr(struct i40e_pf *pf) 8174 { 8175 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) { 8176 /* ATR uses the same filtering logic as SB rules. It only 8177 * functions properly if the input set mask is at the default 8178 * settings. It is safe to restore the default input set 8179 * because there are no active TCPv4 filter rules. 8180 */ 8181 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 8182 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 8183 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 8184 8185 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8186 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8187 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 8188 } 8189 } 8190 8191 /** 8192 * i40e_delete_invalid_filter - Delete an invalid FDIR filter 8193 * @pf: board private structure 8194 * @filter: FDir filter to remove 8195 */ 8196 static void i40e_delete_invalid_filter(struct i40e_pf *pf, 8197 struct i40e_fdir_filter *filter) 8198 { 8199 /* Update counters */ 8200 pf->fdir_pf_active_filters--; 8201 pf->fd_inv = 0; 8202 8203 switch (filter->flow_type) { 8204 case TCP_V4_FLOW: 8205 pf->fd_tcp4_filter_cnt--; 8206 break; 8207 case UDP_V4_FLOW: 8208 pf->fd_udp4_filter_cnt--; 8209 break; 8210 case SCTP_V4_FLOW: 8211 pf->fd_sctp4_filter_cnt--; 8212 break; 8213 case IP_USER_FLOW: 8214 switch (filter->ip4_proto) { 8215 case IPPROTO_TCP: 8216 pf->fd_tcp4_filter_cnt--; 8217 break; 8218 case IPPROTO_UDP: 8219 pf->fd_udp4_filter_cnt--; 8220 break; 8221 case IPPROTO_SCTP: 8222 pf->fd_sctp4_filter_cnt--; 8223 break; 8224 case IPPROTO_IP: 8225 pf->fd_ip4_filter_cnt--; 8226 break; 8227 } 8228 break; 8229 } 8230 8231 /* Remove the filter from the list and free memory */ 8232 hlist_del(&filter->fdir_node); 8233 kfree(filter); 8234 } 8235 8236 /** 8237 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 8238 * @pf: board private structure 8239 **/ 8240 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 8241 { 8242 struct i40e_fdir_filter *filter; 8243 u32 fcnt_prog, fcnt_avail; 8244 struct hlist_node *node; 8245 8246 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8247 return; 8248 8249 /* Check if we have enough room to re-enable FDir SB capability. */ 8250 fcnt_prog = i40e_get_global_fd_count(pf); 8251 fcnt_avail = pf->fdir_pf_filter_count; 8252 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 8253 (pf->fd_add_err == 0) || 8254 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) 8255 i40e_reenable_fdir_sb(pf); 8256 8257 /* We should wait for even more space before re-enabling ATR. 8258 * Additionally, we cannot enable ATR as long as we still have TCP SB 8259 * rules active. 8260 */ 8261 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 8262 (pf->fd_tcp4_filter_cnt == 0)) 8263 i40e_reenable_fdir_atr(pf); 8264 8265 /* if hw had a problem adding a filter, delete it */ 8266 if (pf->fd_inv > 0) { 8267 hlist_for_each_entry_safe(filter, node, 8268 &pf->fdir_filter_list, fdir_node) 8269 if (filter->fd_id == pf->fd_inv) 8270 i40e_delete_invalid_filter(pf, filter); 8271 } 8272 } 8273 8274 #define I40E_MIN_FD_FLUSH_INTERVAL 10 8275 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 8276 /** 8277 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 8278 * @pf: board private structure 8279 **/ 8280 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 8281 { 8282 unsigned long min_flush_time; 8283 int flush_wait_retry = 50; 8284 bool disable_atr = false; 8285 int fd_room; 8286 int reg; 8287 8288 if (!time_after(jiffies, pf->fd_flush_timestamp + 8289 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 8290 return; 8291 8292 /* If the flush is happening too quick and we have mostly SB rules we 8293 * should not re-enable ATR for some time. 8294 */ 8295 min_flush_time = pf->fd_flush_timestamp + 8296 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 8297 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 8298 8299 if (!(time_after(jiffies, min_flush_time)) && 8300 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 8301 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8302 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 8303 disable_atr = true; 8304 } 8305 8306 pf->fd_flush_timestamp = jiffies; 8307 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8308 /* flush all filters */ 8309 wr32(&pf->hw, I40E_PFQF_CTL_1, 8310 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 8311 i40e_flush(&pf->hw); 8312 pf->fd_flush_cnt++; 8313 pf->fd_add_err = 0; 8314 do { 8315 /* Check FD flush status every 5-6msec */ 8316 usleep_range(5000, 6000); 8317 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 8318 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 8319 break; 8320 } while (flush_wait_retry--); 8321 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 8322 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 8323 } else { 8324 /* replay sideband filters */ 8325 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 8326 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 8327 clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 8328 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 8329 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8330 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 8331 } 8332 } 8333 8334 /** 8335 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 8336 * @pf: board private structure 8337 **/ 8338 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 8339 { 8340 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 8341 } 8342 8343 /* We can see up to 256 filter programming desc in transit if the filters are 8344 * being applied really fast; before we see the first 8345 * filter miss error on Rx queue 0. Accumulating enough error messages before 8346 * reacting will make sure we don't cause flush too often. 8347 */ 8348 #define I40E_MAX_FD_PROGRAM_ERROR 256 8349 8350 /** 8351 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 8352 * @pf: board private structure 8353 **/ 8354 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 8355 { 8356 8357 /* if interface is down do nothing */ 8358 if (test_bit(__I40E_DOWN, pf->state)) 8359 return; 8360 8361 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 8362 i40e_fdir_flush_and_replay(pf); 8363 8364 i40e_fdir_check_and_reenable(pf); 8365 8366 } 8367 8368 /** 8369 * i40e_vsi_link_event - notify VSI of a link event 8370 * @vsi: vsi to be notified 8371 * @link_up: link up or down 8372 **/ 8373 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 8374 { 8375 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 8376 return; 8377 8378 switch (vsi->type) { 8379 case I40E_VSI_MAIN: 8380 if (!vsi->netdev || !vsi->netdev_registered) 8381 break; 8382 8383 if (link_up) { 8384 netif_carrier_on(vsi->netdev); 8385 netif_tx_wake_all_queues(vsi->netdev); 8386 } else { 8387 netif_carrier_off(vsi->netdev); 8388 netif_tx_stop_all_queues(vsi->netdev); 8389 } 8390 break; 8391 8392 case I40E_VSI_SRIOV: 8393 case I40E_VSI_VMDQ2: 8394 case I40E_VSI_CTRL: 8395 case I40E_VSI_IWARP: 8396 case I40E_VSI_MIRROR: 8397 default: 8398 /* there is no notification for other VSIs */ 8399 break; 8400 } 8401 } 8402 8403 /** 8404 * i40e_veb_link_event - notify elements on the veb of a link event 8405 * @veb: veb to be notified 8406 * @link_up: link up or down 8407 **/ 8408 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 8409 { 8410 struct i40e_pf *pf; 8411 int i; 8412 8413 if (!veb || !veb->pf) 8414 return; 8415 pf = veb->pf; 8416 8417 /* depth first... */ 8418 for (i = 0; i < I40E_MAX_VEB; i++) 8419 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 8420 i40e_veb_link_event(pf->veb[i], link_up); 8421 8422 /* ... now the local VSIs */ 8423 for (i = 0; i < pf->num_alloc_vsi; i++) 8424 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 8425 i40e_vsi_link_event(pf->vsi[i], link_up); 8426 } 8427 8428 /** 8429 * i40e_link_event - Update netif_carrier status 8430 * @pf: board private structure 8431 **/ 8432 static void i40e_link_event(struct i40e_pf *pf) 8433 { 8434 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8435 u8 new_link_speed, old_link_speed; 8436 i40e_status status; 8437 bool new_link, old_link; 8438 8439 /* save off old link status information */ 8440 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 8441 8442 /* set this to force the get_link_status call to refresh state */ 8443 pf->hw.phy.get_link_info = true; 8444 8445 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 8446 8447 status = i40e_get_link_status(&pf->hw, &new_link); 8448 8449 /* On success, disable temp link polling */ 8450 if (status == I40E_SUCCESS) { 8451 clear_bit(__I40E_TEMP_LINK_POLLING, pf->state); 8452 } else { 8453 /* Enable link polling temporarily until i40e_get_link_status 8454 * returns I40E_SUCCESS 8455 */ 8456 set_bit(__I40E_TEMP_LINK_POLLING, pf->state); 8457 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 8458 status); 8459 return; 8460 } 8461 8462 old_link_speed = pf->hw.phy.link_info_old.link_speed; 8463 new_link_speed = pf->hw.phy.link_info.link_speed; 8464 8465 if (new_link == old_link && 8466 new_link_speed == old_link_speed && 8467 (test_bit(__I40E_VSI_DOWN, vsi->state) || 8468 new_link == netif_carrier_ok(vsi->netdev))) 8469 return; 8470 8471 i40e_print_link_message(vsi, new_link); 8472 8473 /* Notify the base of the switch tree connected to 8474 * the link. Floating VEBs are not notified. 8475 */ 8476 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 8477 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 8478 else 8479 i40e_vsi_link_event(vsi, new_link); 8480 8481 if (pf->vf) 8482 i40e_vc_notify_link_state(pf); 8483 8484 if (pf->flags & I40E_FLAG_PTP) 8485 i40e_ptp_set_increment(pf); 8486 } 8487 8488 /** 8489 * i40e_watchdog_subtask - periodic checks not using event driven response 8490 * @pf: board private structure 8491 **/ 8492 static void i40e_watchdog_subtask(struct i40e_pf *pf) 8493 { 8494 int i; 8495 8496 /* if interface is down do nothing */ 8497 if (test_bit(__I40E_DOWN, pf->state) || 8498 test_bit(__I40E_CONFIG_BUSY, pf->state)) 8499 return; 8500 8501 /* make sure we don't do these things too often */ 8502 if (time_before(jiffies, (pf->service_timer_previous + 8503 pf->service_timer_period))) 8504 return; 8505 pf->service_timer_previous = jiffies; 8506 8507 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 8508 test_bit(__I40E_TEMP_LINK_POLLING, pf->state)) 8509 i40e_link_event(pf); 8510 8511 /* Update the stats for active netdevs so the network stack 8512 * can look at updated numbers whenever it cares to 8513 */ 8514 for (i = 0; i < pf->num_alloc_vsi; i++) 8515 if (pf->vsi[i] && pf->vsi[i]->netdev) 8516 i40e_update_stats(pf->vsi[i]); 8517 8518 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 8519 /* Update the stats for the active switching components */ 8520 for (i = 0; i < I40E_MAX_VEB; i++) 8521 if (pf->veb[i]) 8522 i40e_update_veb_stats(pf->veb[i]); 8523 } 8524 8525 i40e_ptp_rx_hang(pf); 8526 i40e_ptp_tx_hang(pf); 8527 } 8528 8529 /** 8530 * i40e_reset_subtask - Set up for resetting the device and driver 8531 * @pf: board private structure 8532 **/ 8533 static void i40e_reset_subtask(struct i40e_pf *pf) 8534 { 8535 u32 reset_flags = 0; 8536 8537 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 8538 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 8539 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 8540 } 8541 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 8542 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 8543 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 8544 } 8545 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 8546 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 8547 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 8548 } 8549 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 8550 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 8551 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 8552 } 8553 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 8554 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 8555 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 8556 } 8557 8558 /* If there's a recovery already waiting, it takes 8559 * precedence before starting a new reset sequence. 8560 */ 8561 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 8562 i40e_prep_for_reset(pf, false); 8563 i40e_reset(pf); 8564 i40e_rebuild(pf, false, false); 8565 } 8566 8567 /* If we're already down or resetting, just bail */ 8568 if (reset_flags && 8569 !test_bit(__I40E_DOWN, pf->state) && 8570 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 8571 i40e_do_reset(pf, reset_flags, false); 8572 } 8573 } 8574 8575 /** 8576 * i40e_handle_link_event - Handle link event 8577 * @pf: board private structure 8578 * @e: event info posted on ARQ 8579 **/ 8580 static void i40e_handle_link_event(struct i40e_pf *pf, 8581 struct i40e_arq_event_info *e) 8582 { 8583 struct i40e_aqc_get_link_status *status = 8584 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 8585 8586 /* Do a new status request to re-enable LSE reporting 8587 * and load new status information into the hw struct 8588 * This completely ignores any state information 8589 * in the ARQ event info, instead choosing to always 8590 * issue the AQ update link status command. 8591 */ 8592 i40e_link_event(pf); 8593 8594 /* Check if module meets thermal requirements */ 8595 if (status->phy_type == I40E_PHY_TYPE_NOT_SUPPORTED_HIGH_TEMP) { 8596 dev_err(&pf->pdev->dev, 8597 "Rx/Tx is disabled on this device because the module does not meet thermal requirements.\n"); 8598 dev_err(&pf->pdev->dev, 8599 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 8600 } else { 8601 /* check for unqualified module, if link is down, suppress 8602 * the message if link was forced to be down. 8603 */ 8604 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 8605 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 8606 (!(status->link_info & I40E_AQ_LINK_UP)) && 8607 (!(pf->flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED))) { 8608 dev_err(&pf->pdev->dev, 8609 "Rx/Tx is disabled on this device because an unsupported SFP module type was detected.\n"); 8610 dev_err(&pf->pdev->dev, 8611 "Refer to the Intel(R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n"); 8612 } 8613 } 8614 } 8615 8616 /** 8617 * i40e_clean_adminq_subtask - Clean the AdminQ rings 8618 * @pf: board private structure 8619 **/ 8620 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 8621 { 8622 struct i40e_arq_event_info event; 8623 struct i40e_hw *hw = &pf->hw; 8624 u16 pending, i = 0; 8625 i40e_status ret; 8626 u16 opcode; 8627 u32 oldval; 8628 u32 val; 8629 8630 /* Do not run clean AQ when PF reset fails */ 8631 if (test_bit(__I40E_RESET_FAILED, pf->state)) 8632 return; 8633 8634 /* check for error indications */ 8635 val = rd32(&pf->hw, pf->hw.aq.arq.len); 8636 oldval = val; 8637 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 8638 if (hw->debug_mask & I40E_DEBUG_AQ) 8639 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 8640 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 8641 } 8642 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 8643 if (hw->debug_mask & I40E_DEBUG_AQ) 8644 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 8645 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 8646 pf->arq_overflows++; 8647 } 8648 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 8649 if (hw->debug_mask & I40E_DEBUG_AQ) 8650 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 8651 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 8652 } 8653 if (oldval != val) 8654 wr32(&pf->hw, pf->hw.aq.arq.len, val); 8655 8656 val = rd32(&pf->hw, pf->hw.aq.asq.len); 8657 oldval = val; 8658 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 8659 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8660 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 8661 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 8662 } 8663 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 8664 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8665 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 8666 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 8667 } 8668 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 8669 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 8670 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 8671 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 8672 } 8673 if (oldval != val) 8674 wr32(&pf->hw, pf->hw.aq.asq.len, val); 8675 8676 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 8677 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 8678 if (!event.msg_buf) 8679 return; 8680 8681 do { 8682 ret = i40e_clean_arq_element(hw, &event, &pending); 8683 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 8684 break; 8685 else if (ret) { 8686 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 8687 break; 8688 } 8689 8690 opcode = le16_to_cpu(event.desc.opcode); 8691 switch (opcode) { 8692 8693 case i40e_aqc_opc_get_link_status: 8694 i40e_handle_link_event(pf, &event); 8695 break; 8696 case i40e_aqc_opc_send_msg_to_pf: 8697 ret = i40e_vc_process_vf_msg(pf, 8698 le16_to_cpu(event.desc.retval), 8699 le32_to_cpu(event.desc.cookie_high), 8700 le32_to_cpu(event.desc.cookie_low), 8701 event.msg_buf, 8702 event.msg_len); 8703 break; 8704 case i40e_aqc_opc_lldp_update_mib: 8705 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 8706 #ifdef CONFIG_I40E_DCB 8707 rtnl_lock(); 8708 ret = i40e_handle_lldp_event(pf, &event); 8709 rtnl_unlock(); 8710 #endif /* CONFIG_I40E_DCB */ 8711 break; 8712 case i40e_aqc_opc_event_lan_overflow: 8713 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 8714 i40e_handle_lan_overflow_event(pf, &event); 8715 break; 8716 case i40e_aqc_opc_send_msg_to_peer: 8717 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 8718 break; 8719 case i40e_aqc_opc_nvm_erase: 8720 case i40e_aqc_opc_nvm_update: 8721 case i40e_aqc_opc_oem_post_update: 8722 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 8723 "ARQ NVM operation 0x%04x completed\n", 8724 opcode); 8725 break; 8726 default: 8727 dev_info(&pf->pdev->dev, 8728 "ARQ: Unknown event 0x%04x ignored\n", 8729 opcode); 8730 break; 8731 } 8732 } while (i++ < pf->adminq_work_limit); 8733 8734 if (i < pf->adminq_work_limit) 8735 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 8736 8737 /* re-enable Admin queue interrupt cause */ 8738 val = rd32(hw, I40E_PFINT_ICR0_ENA); 8739 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 8740 wr32(hw, I40E_PFINT_ICR0_ENA, val); 8741 i40e_flush(hw); 8742 8743 kfree(event.msg_buf); 8744 } 8745 8746 /** 8747 * i40e_verify_eeprom - make sure eeprom is good to use 8748 * @pf: board private structure 8749 **/ 8750 static void i40e_verify_eeprom(struct i40e_pf *pf) 8751 { 8752 int err; 8753 8754 err = i40e_diag_eeprom_test(&pf->hw); 8755 if (err) { 8756 /* retry in case of garbage read */ 8757 err = i40e_diag_eeprom_test(&pf->hw); 8758 if (err) { 8759 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 8760 err); 8761 set_bit(__I40E_BAD_EEPROM, pf->state); 8762 } 8763 } 8764 8765 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 8766 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 8767 clear_bit(__I40E_BAD_EEPROM, pf->state); 8768 } 8769 } 8770 8771 /** 8772 * i40e_enable_pf_switch_lb 8773 * @pf: pointer to the PF structure 8774 * 8775 * enable switch loop back or die - no point in a return value 8776 **/ 8777 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 8778 { 8779 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8780 struct i40e_vsi_context ctxt; 8781 int ret; 8782 8783 ctxt.seid = pf->main_vsi_seid; 8784 ctxt.pf_num = pf->hw.pf_id; 8785 ctxt.vf_num = 0; 8786 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 8787 if (ret) { 8788 dev_info(&pf->pdev->dev, 8789 "couldn't get PF vsi config, err %s aq_err %s\n", 8790 i40e_stat_str(&pf->hw, ret), 8791 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8792 return; 8793 } 8794 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 8795 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 8796 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 8797 8798 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 8799 if (ret) { 8800 dev_info(&pf->pdev->dev, 8801 "update vsi switch failed, err %s aq_err %s\n", 8802 i40e_stat_str(&pf->hw, ret), 8803 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8804 } 8805 } 8806 8807 /** 8808 * i40e_disable_pf_switch_lb 8809 * @pf: pointer to the PF structure 8810 * 8811 * disable switch loop back or die - no point in a return value 8812 **/ 8813 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 8814 { 8815 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8816 struct i40e_vsi_context ctxt; 8817 int ret; 8818 8819 ctxt.seid = pf->main_vsi_seid; 8820 ctxt.pf_num = pf->hw.pf_id; 8821 ctxt.vf_num = 0; 8822 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 8823 if (ret) { 8824 dev_info(&pf->pdev->dev, 8825 "couldn't get PF vsi config, err %s aq_err %s\n", 8826 i40e_stat_str(&pf->hw, ret), 8827 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8828 return; 8829 } 8830 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 8831 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 8832 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 8833 8834 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 8835 if (ret) { 8836 dev_info(&pf->pdev->dev, 8837 "update vsi switch failed, err %s aq_err %s\n", 8838 i40e_stat_str(&pf->hw, ret), 8839 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 8840 } 8841 } 8842 8843 /** 8844 * i40e_config_bridge_mode - Configure the HW bridge mode 8845 * @veb: pointer to the bridge instance 8846 * 8847 * Configure the loop back mode for the LAN VSI that is downlink to the 8848 * specified HW bridge instance. It is expected this function is called 8849 * when a new HW bridge is instantiated. 8850 **/ 8851 static void i40e_config_bridge_mode(struct i40e_veb *veb) 8852 { 8853 struct i40e_pf *pf = veb->pf; 8854 8855 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 8856 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 8857 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 8858 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 8859 i40e_disable_pf_switch_lb(pf); 8860 else 8861 i40e_enable_pf_switch_lb(pf); 8862 } 8863 8864 /** 8865 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 8866 * @veb: pointer to the VEB instance 8867 * 8868 * This is a recursive function that first builds the attached VSIs then 8869 * recurses in to build the next layer of VEB. We track the connections 8870 * through our own index numbers because the seid's from the HW could 8871 * change across the reset. 8872 **/ 8873 static int i40e_reconstitute_veb(struct i40e_veb *veb) 8874 { 8875 struct i40e_vsi *ctl_vsi = NULL; 8876 struct i40e_pf *pf = veb->pf; 8877 int v, veb_idx; 8878 int ret; 8879 8880 /* build VSI that owns this VEB, temporarily attached to base VEB */ 8881 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 8882 if (pf->vsi[v] && 8883 pf->vsi[v]->veb_idx == veb->idx && 8884 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 8885 ctl_vsi = pf->vsi[v]; 8886 break; 8887 } 8888 } 8889 if (!ctl_vsi) { 8890 dev_info(&pf->pdev->dev, 8891 "missing owner VSI for veb_idx %d\n", veb->idx); 8892 ret = -ENOENT; 8893 goto end_reconstitute; 8894 } 8895 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 8896 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 8897 ret = i40e_add_vsi(ctl_vsi); 8898 if (ret) { 8899 dev_info(&pf->pdev->dev, 8900 "rebuild of veb_idx %d owner VSI failed: %d\n", 8901 veb->idx, ret); 8902 goto end_reconstitute; 8903 } 8904 i40e_vsi_reset_stats(ctl_vsi); 8905 8906 /* create the VEB in the switch and move the VSI onto the VEB */ 8907 ret = i40e_add_veb(veb, ctl_vsi); 8908 if (ret) 8909 goto end_reconstitute; 8910 8911 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 8912 veb->bridge_mode = BRIDGE_MODE_VEB; 8913 else 8914 veb->bridge_mode = BRIDGE_MODE_VEPA; 8915 i40e_config_bridge_mode(veb); 8916 8917 /* create the remaining VSIs attached to this VEB */ 8918 for (v = 0; v < pf->num_alloc_vsi; v++) { 8919 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 8920 continue; 8921 8922 if (pf->vsi[v]->veb_idx == veb->idx) { 8923 struct i40e_vsi *vsi = pf->vsi[v]; 8924 8925 vsi->uplink_seid = veb->seid; 8926 ret = i40e_add_vsi(vsi); 8927 if (ret) { 8928 dev_info(&pf->pdev->dev, 8929 "rebuild of vsi_idx %d failed: %d\n", 8930 v, ret); 8931 goto end_reconstitute; 8932 } 8933 i40e_vsi_reset_stats(vsi); 8934 } 8935 } 8936 8937 /* create any VEBs attached to this VEB - RECURSION */ 8938 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 8939 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 8940 pf->veb[veb_idx]->uplink_seid = veb->seid; 8941 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 8942 if (ret) 8943 break; 8944 } 8945 } 8946 8947 end_reconstitute: 8948 return ret; 8949 } 8950 8951 /** 8952 * i40e_get_capabilities - get info about the HW 8953 * @pf: the PF struct 8954 **/ 8955 static int i40e_get_capabilities(struct i40e_pf *pf, 8956 enum i40e_admin_queue_opc list_type) 8957 { 8958 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 8959 u16 data_size; 8960 int buf_len; 8961 int err; 8962 8963 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 8964 do { 8965 cap_buf = kzalloc(buf_len, GFP_KERNEL); 8966 if (!cap_buf) 8967 return -ENOMEM; 8968 8969 /* this loads the data into the hw struct for us */ 8970 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 8971 &data_size, list_type, 8972 NULL); 8973 /* data loaded, buffer no longer needed */ 8974 kfree(cap_buf); 8975 8976 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 8977 /* retry with a larger buffer */ 8978 buf_len = data_size; 8979 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 8980 dev_info(&pf->pdev->dev, 8981 "capability discovery failed, err %s aq_err %s\n", 8982 i40e_stat_str(&pf->hw, err), 8983 i40e_aq_str(&pf->hw, 8984 pf->hw.aq.asq_last_status)); 8985 return -ENODEV; 8986 } 8987 } while (err); 8988 8989 if (pf->hw.debug_mask & I40E_DEBUG_USER) { 8990 if (list_type == i40e_aqc_opc_list_func_capabilities) { 8991 dev_info(&pf->pdev->dev, 8992 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 8993 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 8994 pf->hw.func_caps.num_msix_vectors, 8995 pf->hw.func_caps.num_msix_vectors_vf, 8996 pf->hw.func_caps.fd_filters_guaranteed, 8997 pf->hw.func_caps.fd_filters_best_effort, 8998 pf->hw.func_caps.num_tx_qp, 8999 pf->hw.func_caps.num_vsis); 9000 } else if (list_type == i40e_aqc_opc_list_dev_capabilities) { 9001 dev_info(&pf->pdev->dev, 9002 "switch_mode=0x%04x, function_valid=0x%08x\n", 9003 pf->hw.dev_caps.switch_mode, 9004 pf->hw.dev_caps.valid_functions); 9005 dev_info(&pf->pdev->dev, 9006 "SR-IOV=%d, num_vfs for all function=%u\n", 9007 pf->hw.dev_caps.sr_iov_1_1, 9008 pf->hw.dev_caps.num_vfs); 9009 dev_info(&pf->pdev->dev, 9010 "num_vsis=%u, num_rx:%u, num_tx=%u\n", 9011 pf->hw.dev_caps.num_vsis, 9012 pf->hw.dev_caps.num_rx_qp, 9013 pf->hw.dev_caps.num_tx_qp); 9014 } 9015 } 9016 if (list_type == i40e_aqc_opc_list_func_capabilities) { 9017 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 9018 + pf->hw.func_caps.num_vfs) 9019 if (pf->hw.revision_id == 0 && 9020 pf->hw.func_caps.num_vsis < DEF_NUM_VSI) { 9021 dev_info(&pf->pdev->dev, 9022 "got num_vsis %d, setting num_vsis to %d\n", 9023 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 9024 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 9025 } 9026 } 9027 return 0; 9028 } 9029 9030 static int i40e_vsi_clear(struct i40e_vsi *vsi); 9031 9032 /** 9033 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 9034 * @pf: board private structure 9035 **/ 9036 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 9037 { 9038 struct i40e_vsi *vsi; 9039 9040 /* quick workaround for an NVM issue that leaves a critical register 9041 * uninitialized 9042 */ 9043 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 9044 static const u32 hkey[] = { 9045 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 9046 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 9047 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 9048 0x95b3a76d}; 9049 int i; 9050 9051 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 9052 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 9053 } 9054 9055 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 9056 return; 9057 9058 /* find existing VSI and see if it needs configuring */ 9059 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9060 9061 /* create a new VSI if none exists */ 9062 if (!vsi) { 9063 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 9064 pf->vsi[pf->lan_vsi]->seid, 0); 9065 if (!vsi) { 9066 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 9067 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 9068 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 9069 return; 9070 } 9071 } 9072 9073 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 9074 } 9075 9076 /** 9077 * i40e_fdir_teardown - release the Flow Director resources 9078 * @pf: board private structure 9079 **/ 9080 static void i40e_fdir_teardown(struct i40e_pf *pf) 9081 { 9082 struct i40e_vsi *vsi; 9083 9084 i40e_fdir_filter_exit(pf); 9085 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 9086 if (vsi) 9087 i40e_vsi_release(vsi); 9088 } 9089 9090 /** 9091 * i40e_rebuild_cloud_filters - Rebuilds cloud filters for VSIs 9092 * @vsi: PF main vsi 9093 * @seid: seid of main or channel VSIs 9094 * 9095 * Rebuilds cloud filters associated with main VSI and channel VSIs if they 9096 * existed before reset 9097 **/ 9098 static int i40e_rebuild_cloud_filters(struct i40e_vsi *vsi, u16 seid) 9099 { 9100 struct i40e_cloud_filter *cfilter; 9101 struct i40e_pf *pf = vsi->back; 9102 struct hlist_node *node; 9103 i40e_status ret; 9104 9105 /* Add cloud filters back if they exist */ 9106 hlist_for_each_entry_safe(cfilter, node, &pf->cloud_filter_list, 9107 cloud_node) { 9108 if (cfilter->seid != seid) 9109 continue; 9110 9111 if (cfilter->dst_port) 9112 ret = i40e_add_del_cloud_filter_big_buf(vsi, cfilter, 9113 true); 9114 else 9115 ret = i40e_add_del_cloud_filter(vsi, cfilter, true); 9116 9117 if (ret) { 9118 dev_dbg(&pf->pdev->dev, 9119 "Failed to rebuild cloud filter, err %s aq_err %s\n", 9120 i40e_stat_str(&pf->hw, ret), 9121 i40e_aq_str(&pf->hw, 9122 pf->hw.aq.asq_last_status)); 9123 return ret; 9124 } 9125 } 9126 return 0; 9127 } 9128 9129 /** 9130 * i40e_rebuild_channels - Rebuilds channel VSIs if they existed before reset 9131 * @vsi: PF main vsi 9132 * 9133 * Rebuilds channel VSIs if they existed before reset 9134 **/ 9135 static int i40e_rebuild_channels(struct i40e_vsi *vsi) 9136 { 9137 struct i40e_channel *ch, *ch_tmp; 9138 i40e_status ret; 9139 9140 if (list_empty(&vsi->ch_list)) 9141 return 0; 9142 9143 list_for_each_entry_safe(ch, ch_tmp, &vsi->ch_list, list) { 9144 if (!ch->initialized) 9145 break; 9146 /* Proceed with creation of channel (VMDq2) VSI */ 9147 ret = i40e_add_channel(vsi->back, vsi->uplink_seid, ch); 9148 if (ret) { 9149 dev_info(&vsi->back->pdev->dev, 9150 "failed to rebuild channels using uplink_seid %u\n", 9151 vsi->uplink_seid); 9152 return ret; 9153 } 9154 /* Reconfigure TX queues using QTX_CTL register */ 9155 ret = i40e_channel_config_tx_ring(vsi->back, vsi, ch); 9156 if (ret) { 9157 dev_info(&vsi->back->pdev->dev, 9158 "failed to configure TX rings for channel %u\n", 9159 ch->seid); 9160 return ret; 9161 } 9162 /* update 'next_base_queue' */ 9163 vsi->next_base_queue = vsi->next_base_queue + 9164 ch->num_queue_pairs; 9165 if (ch->max_tx_rate) { 9166 u64 credits = ch->max_tx_rate; 9167 9168 if (i40e_set_bw_limit(vsi, ch->seid, 9169 ch->max_tx_rate)) 9170 return -EINVAL; 9171 9172 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9173 dev_dbg(&vsi->back->pdev->dev, 9174 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9175 ch->max_tx_rate, 9176 credits, 9177 ch->seid); 9178 } 9179 ret = i40e_rebuild_cloud_filters(vsi, ch->seid); 9180 if (ret) { 9181 dev_dbg(&vsi->back->pdev->dev, 9182 "Failed to rebuild cloud filters for channel VSI %u\n", 9183 ch->seid); 9184 return ret; 9185 } 9186 } 9187 return 0; 9188 } 9189 9190 /** 9191 * i40e_prep_for_reset - prep for the core to reset 9192 * @pf: board private structure 9193 * @lock_acquired: indicates whether or not the lock has been acquired 9194 * before this function was called. 9195 * 9196 * Close up the VFs and other things in prep for PF Reset. 9197 **/ 9198 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 9199 { 9200 struct i40e_hw *hw = &pf->hw; 9201 i40e_status ret = 0; 9202 u32 v; 9203 9204 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 9205 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9206 return; 9207 if (i40e_check_asq_alive(&pf->hw)) 9208 i40e_vc_notify_reset(pf); 9209 9210 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 9211 9212 /* quiesce the VSIs and their queues that are not already DOWN */ 9213 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 9214 if (!lock_acquired) 9215 rtnl_lock(); 9216 i40e_pf_quiesce_all_vsi(pf); 9217 if (!lock_acquired) 9218 rtnl_unlock(); 9219 9220 for (v = 0; v < pf->num_alloc_vsi; v++) { 9221 if (pf->vsi[v]) 9222 pf->vsi[v]->seid = 0; 9223 } 9224 9225 i40e_shutdown_adminq(&pf->hw); 9226 9227 /* call shutdown HMC */ 9228 if (hw->hmc.hmc_obj) { 9229 ret = i40e_shutdown_lan_hmc(hw); 9230 if (ret) 9231 dev_warn(&pf->pdev->dev, 9232 "shutdown_lan_hmc failed: %d\n", ret); 9233 } 9234 } 9235 9236 /** 9237 * i40e_send_version - update firmware with driver version 9238 * @pf: PF struct 9239 */ 9240 static void i40e_send_version(struct i40e_pf *pf) 9241 { 9242 struct i40e_driver_version dv; 9243 9244 dv.major_version = DRV_VERSION_MAJOR; 9245 dv.minor_version = DRV_VERSION_MINOR; 9246 dv.build_version = DRV_VERSION_BUILD; 9247 dv.subbuild_version = 0; 9248 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 9249 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 9250 } 9251 9252 /** 9253 * i40e_get_oem_version - get OEM specific version information 9254 * @hw: pointer to the hardware structure 9255 **/ 9256 static void i40e_get_oem_version(struct i40e_hw *hw) 9257 { 9258 u16 block_offset = 0xffff; 9259 u16 block_length = 0; 9260 u16 capabilities = 0; 9261 u16 gen_snap = 0; 9262 u16 release = 0; 9263 9264 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 9265 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 9266 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 9267 #define I40E_NVM_OEM_GEN_OFFSET 0x02 9268 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 9269 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 9270 #define I40E_NVM_OEM_LENGTH 3 9271 9272 /* Check if pointer to OEM version block is valid. */ 9273 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 9274 if (block_offset == 0xffff) 9275 return; 9276 9277 /* Check if OEM version block has correct length. */ 9278 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 9279 &block_length); 9280 if (block_length < I40E_NVM_OEM_LENGTH) 9281 return; 9282 9283 /* Check if OEM version format is as expected. */ 9284 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 9285 &capabilities); 9286 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 9287 return; 9288 9289 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 9290 &gen_snap); 9291 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 9292 &release); 9293 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 9294 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 9295 } 9296 9297 /** 9298 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 9299 * @pf: board private structure 9300 **/ 9301 static int i40e_reset(struct i40e_pf *pf) 9302 { 9303 struct i40e_hw *hw = &pf->hw; 9304 i40e_status ret; 9305 9306 ret = i40e_pf_reset(hw); 9307 if (ret) { 9308 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 9309 set_bit(__I40E_RESET_FAILED, pf->state); 9310 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9311 } else { 9312 pf->pfr_count++; 9313 } 9314 return ret; 9315 } 9316 9317 /** 9318 * i40e_rebuild - rebuild using a saved config 9319 * @pf: board private structure 9320 * @reinit: if the Main VSI needs to re-initialized. 9321 * @lock_acquired: indicates whether or not the lock has been acquired 9322 * before this function was called. 9323 **/ 9324 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 9325 { 9326 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 9327 struct i40e_hw *hw = &pf->hw; 9328 u8 set_fc_aq_fail = 0; 9329 i40e_status ret; 9330 u32 val; 9331 int v; 9332 9333 if (test_bit(__I40E_DOWN, pf->state)) 9334 goto clear_recovery; 9335 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 9336 9337 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 9338 ret = i40e_init_adminq(&pf->hw); 9339 if (ret) { 9340 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 9341 i40e_stat_str(&pf->hw, ret), 9342 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9343 goto clear_recovery; 9344 } 9345 i40e_get_oem_version(&pf->hw); 9346 9347 if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && 9348 ((hw->aq.fw_maj_ver == 4 && hw->aq.fw_min_ver <= 33) || 9349 hw->aq.fw_maj_ver < 4) && hw->mac.type == I40E_MAC_XL710) { 9350 /* The following delay is necessary for 4.33 firmware and older 9351 * to recover after EMP reset. 200 ms should suffice but we 9352 * put here 300 ms to be sure that FW is ready to operate 9353 * after reset. 9354 */ 9355 mdelay(300); 9356 } 9357 9358 /* re-verify the eeprom if we just had an EMP reset */ 9359 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 9360 i40e_verify_eeprom(pf); 9361 9362 i40e_clear_pxe_mode(hw); 9363 ret = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 9364 if (ret) 9365 goto end_core_reset; 9366 9367 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 9368 hw->func_caps.num_rx_qp, 0, 0); 9369 if (ret) { 9370 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 9371 goto end_core_reset; 9372 } 9373 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 9374 if (ret) { 9375 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 9376 goto end_core_reset; 9377 } 9378 9379 /* Enable FW to write a default DCB config on link-up */ 9380 i40e_aq_set_dcb_parameters(hw, true, NULL); 9381 9382 #ifdef CONFIG_I40E_DCB 9383 ret = i40e_init_pf_dcb(pf); 9384 if (ret) { 9385 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 9386 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 9387 /* Continue without DCB enabled */ 9388 } 9389 #endif /* CONFIG_I40E_DCB */ 9390 /* do basic switch setup */ 9391 if (!lock_acquired) 9392 rtnl_lock(); 9393 ret = i40e_setup_pf_switch(pf, reinit); 9394 if (ret) 9395 goto end_unlock; 9396 9397 /* The driver only wants link up/down and module qualification 9398 * reports from firmware. Note the negative logic. 9399 */ 9400 ret = i40e_aq_set_phy_int_mask(&pf->hw, 9401 ~(I40E_AQ_EVENT_LINK_UPDOWN | 9402 I40E_AQ_EVENT_MEDIA_NA | 9403 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 9404 if (ret) 9405 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 9406 i40e_stat_str(&pf->hw, ret), 9407 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9408 9409 /* make sure our flow control settings are restored */ 9410 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 9411 if (ret) 9412 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 9413 i40e_stat_str(&pf->hw, ret), 9414 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9415 9416 /* Rebuild the VSIs and VEBs that existed before reset. 9417 * They are still in our local switch element arrays, so only 9418 * need to rebuild the switch model in the HW. 9419 * 9420 * If there were VEBs but the reconstitution failed, we'll try 9421 * try to recover minimal use by getting the basic PF VSI working. 9422 */ 9423 if (vsi->uplink_seid != pf->mac_seid) { 9424 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 9425 /* find the one VEB connected to the MAC, and find orphans */ 9426 for (v = 0; v < I40E_MAX_VEB; v++) { 9427 if (!pf->veb[v]) 9428 continue; 9429 9430 if (pf->veb[v]->uplink_seid == pf->mac_seid || 9431 pf->veb[v]->uplink_seid == 0) { 9432 ret = i40e_reconstitute_veb(pf->veb[v]); 9433 9434 if (!ret) 9435 continue; 9436 9437 /* If Main VEB failed, we're in deep doodoo, 9438 * so give up rebuilding the switch and set up 9439 * for minimal rebuild of PF VSI. 9440 * If orphan failed, we'll report the error 9441 * but try to keep going. 9442 */ 9443 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 9444 dev_info(&pf->pdev->dev, 9445 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 9446 ret); 9447 vsi->uplink_seid = pf->mac_seid; 9448 break; 9449 } else if (pf->veb[v]->uplink_seid == 0) { 9450 dev_info(&pf->pdev->dev, 9451 "rebuild of orphan VEB failed: %d\n", 9452 ret); 9453 } 9454 } 9455 } 9456 } 9457 9458 if (vsi->uplink_seid == pf->mac_seid) { 9459 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 9460 /* no VEB, so rebuild only the Main VSI */ 9461 ret = i40e_add_vsi(vsi); 9462 if (ret) { 9463 dev_info(&pf->pdev->dev, 9464 "rebuild of Main VSI failed: %d\n", ret); 9465 goto end_unlock; 9466 } 9467 } 9468 9469 if (vsi->mqprio_qopt.max_rate[0]) { 9470 u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; 9471 u64 credits = 0; 9472 9473 do_div(max_tx_rate, I40E_BW_MBPS_DIVISOR); 9474 ret = i40e_set_bw_limit(vsi, vsi->seid, max_tx_rate); 9475 if (ret) 9476 goto end_unlock; 9477 9478 credits = max_tx_rate; 9479 do_div(credits, I40E_BW_CREDIT_DIVISOR); 9480 dev_dbg(&vsi->back->pdev->dev, 9481 "Set tx rate of %llu Mbps (count of 50Mbps %llu) for vsi->seid %u\n", 9482 max_tx_rate, 9483 credits, 9484 vsi->seid); 9485 } 9486 9487 ret = i40e_rebuild_cloud_filters(vsi, vsi->seid); 9488 if (ret) 9489 goto end_unlock; 9490 9491 /* PF Main VSI is rebuild by now, go ahead and rebuild channel VSIs 9492 * for this main VSI if they exist 9493 */ 9494 ret = i40e_rebuild_channels(vsi); 9495 if (ret) 9496 goto end_unlock; 9497 9498 /* Reconfigure hardware for allowing smaller MSS in the case 9499 * of TSO, so that we avoid the MDD being fired and causing 9500 * a reset in the case of small MSS+TSO. 9501 */ 9502 #define I40E_REG_MSS 0x000E64DC 9503 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 9504 #define I40E_64BYTE_MSS 0x400000 9505 val = rd32(hw, I40E_REG_MSS); 9506 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 9507 val &= ~I40E_REG_MSS_MIN_MASK; 9508 val |= I40E_64BYTE_MSS; 9509 wr32(hw, I40E_REG_MSS, val); 9510 } 9511 9512 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 9513 msleep(75); 9514 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 9515 if (ret) 9516 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 9517 i40e_stat_str(&pf->hw, ret), 9518 i40e_aq_str(&pf->hw, 9519 pf->hw.aq.asq_last_status)); 9520 } 9521 /* reinit the misc interrupt */ 9522 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 9523 ret = i40e_setup_misc_vector(pf); 9524 9525 /* Add a filter to drop all Flow control frames from any VSI from being 9526 * transmitted. By doing so we stop a malicious VF from sending out 9527 * PAUSE or PFC frames and potentially controlling traffic for other 9528 * PF/VF VSIs. 9529 * The FW can still send Flow control frames if enabled. 9530 */ 9531 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 9532 pf->main_vsi_seid); 9533 9534 /* restart the VSIs that were rebuilt and running before the reset */ 9535 i40e_pf_unquiesce_all_vsi(pf); 9536 9537 /* Release the RTNL lock before we start resetting VFs */ 9538 if (!lock_acquired) 9539 rtnl_unlock(); 9540 9541 /* Restore promiscuous settings */ 9542 ret = i40e_set_promiscuous(pf, pf->cur_promisc); 9543 if (ret) 9544 dev_warn(&pf->pdev->dev, 9545 "Failed to restore promiscuous setting: %s, err %s aq_err %s\n", 9546 pf->cur_promisc ? "on" : "off", 9547 i40e_stat_str(&pf->hw, ret), 9548 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9549 9550 i40e_reset_all_vfs(pf, true); 9551 9552 /* tell the firmware that we're starting */ 9553 i40e_send_version(pf); 9554 9555 /* We've already released the lock, so don't do it again */ 9556 goto end_core_reset; 9557 9558 end_unlock: 9559 if (!lock_acquired) 9560 rtnl_unlock(); 9561 end_core_reset: 9562 clear_bit(__I40E_RESET_FAILED, pf->state); 9563 clear_recovery: 9564 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 9565 } 9566 9567 /** 9568 * i40e_reset_and_rebuild - reset and rebuild using a saved config 9569 * @pf: board private structure 9570 * @reinit: if the Main VSI needs to re-initialized. 9571 * @lock_acquired: indicates whether or not the lock has been acquired 9572 * before this function was called. 9573 **/ 9574 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 9575 bool lock_acquired) 9576 { 9577 int ret; 9578 /* Now we wait for GRST to settle out. 9579 * We don't have to delete the VEBs or VSIs from the hw switch 9580 * because the reset will make them disappear. 9581 */ 9582 ret = i40e_reset(pf); 9583 if (!ret) 9584 i40e_rebuild(pf, reinit, lock_acquired); 9585 } 9586 9587 /** 9588 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 9589 * @pf: board private structure 9590 * 9591 * Close up the VFs and other things in prep for a Core Reset, 9592 * then get ready to rebuild the world. 9593 * @lock_acquired: indicates whether or not the lock has been acquired 9594 * before this function was called. 9595 **/ 9596 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 9597 { 9598 i40e_prep_for_reset(pf, lock_acquired); 9599 i40e_reset_and_rebuild(pf, false, lock_acquired); 9600 } 9601 9602 /** 9603 * i40e_handle_mdd_event 9604 * @pf: pointer to the PF structure 9605 * 9606 * Called from the MDD irq handler to identify possibly malicious vfs 9607 **/ 9608 static void i40e_handle_mdd_event(struct i40e_pf *pf) 9609 { 9610 struct i40e_hw *hw = &pf->hw; 9611 bool mdd_detected = false; 9612 bool pf_mdd_detected = false; 9613 struct i40e_vf *vf; 9614 u32 reg; 9615 int i; 9616 9617 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 9618 return; 9619 9620 /* find what triggered the MDD event */ 9621 reg = rd32(hw, I40E_GL_MDET_TX); 9622 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 9623 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 9624 I40E_GL_MDET_TX_PF_NUM_SHIFT; 9625 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 9626 I40E_GL_MDET_TX_VF_NUM_SHIFT; 9627 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 9628 I40E_GL_MDET_TX_EVENT_SHIFT; 9629 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 9630 I40E_GL_MDET_TX_QUEUE_SHIFT) - 9631 pf->hw.func_caps.base_queue; 9632 if (netif_msg_tx_err(pf)) 9633 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 9634 event, queue, pf_num, vf_num); 9635 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 9636 mdd_detected = true; 9637 } 9638 reg = rd32(hw, I40E_GL_MDET_RX); 9639 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 9640 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 9641 I40E_GL_MDET_RX_FUNCTION_SHIFT; 9642 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 9643 I40E_GL_MDET_RX_EVENT_SHIFT; 9644 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 9645 I40E_GL_MDET_RX_QUEUE_SHIFT) - 9646 pf->hw.func_caps.base_queue; 9647 if (netif_msg_rx_err(pf)) 9648 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 9649 event, queue, func); 9650 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 9651 mdd_detected = true; 9652 } 9653 9654 if (mdd_detected) { 9655 reg = rd32(hw, I40E_PF_MDET_TX); 9656 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 9657 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 9658 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 9659 pf_mdd_detected = true; 9660 } 9661 reg = rd32(hw, I40E_PF_MDET_RX); 9662 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 9663 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 9664 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 9665 pf_mdd_detected = true; 9666 } 9667 /* Queue belongs to the PF, initiate a reset */ 9668 if (pf_mdd_detected) { 9669 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 9670 i40e_service_event_schedule(pf); 9671 } 9672 } 9673 9674 /* see if one of the VFs needs its hand slapped */ 9675 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 9676 vf = &(pf->vf[i]); 9677 reg = rd32(hw, I40E_VP_MDET_TX(i)); 9678 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 9679 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 9680 vf->num_mdd_events++; 9681 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 9682 i); 9683 } 9684 9685 reg = rd32(hw, I40E_VP_MDET_RX(i)); 9686 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 9687 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 9688 vf->num_mdd_events++; 9689 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 9690 i); 9691 } 9692 9693 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 9694 dev_info(&pf->pdev->dev, 9695 "Too many MDD events on VF %d, disabled\n", i); 9696 dev_info(&pf->pdev->dev, 9697 "Use PF Control I/F to re-enable the VF\n"); 9698 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 9699 } 9700 } 9701 9702 /* re-enable mdd interrupt cause */ 9703 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 9704 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 9705 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 9706 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 9707 i40e_flush(hw); 9708 } 9709 9710 static const char *i40e_tunnel_name(u8 type) 9711 { 9712 switch (type) { 9713 case UDP_TUNNEL_TYPE_VXLAN: 9714 return "vxlan"; 9715 case UDP_TUNNEL_TYPE_GENEVE: 9716 return "geneve"; 9717 default: 9718 return "unknown"; 9719 } 9720 } 9721 9722 /** 9723 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 9724 * @pf: board private structure 9725 **/ 9726 static void i40e_sync_udp_filters(struct i40e_pf *pf) 9727 { 9728 int i; 9729 9730 /* loop through and set pending bit for all active UDP filters */ 9731 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9732 if (pf->udp_ports[i].port) 9733 pf->pending_udp_bitmap |= BIT_ULL(i); 9734 } 9735 9736 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 9737 } 9738 9739 /** 9740 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 9741 * @pf: board private structure 9742 **/ 9743 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 9744 { 9745 struct i40e_hw *hw = &pf->hw; 9746 u8 filter_index, type; 9747 u16 port; 9748 int i; 9749 9750 if (!test_and_clear_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state)) 9751 return; 9752 9753 /* acquire RTNL to maintain state of flags and port requests */ 9754 rtnl_lock(); 9755 9756 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9757 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 9758 struct i40e_udp_port_config *udp_port; 9759 i40e_status ret = 0; 9760 9761 udp_port = &pf->udp_ports[i]; 9762 pf->pending_udp_bitmap &= ~BIT_ULL(i); 9763 9764 port = READ_ONCE(udp_port->port); 9765 type = READ_ONCE(udp_port->type); 9766 filter_index = READ_ONCE(udp_port->filter_index); 9767 9768 /* release RTNL while we wait on AQ command */ 9769 rtnl_unlock(); 9770 9771 if (port) 9772 ret = i40e_aq_add_udp_tunnel(hw, port, 9773 type, 9774 &filter_index, 9775 NULL); 9776 else if (filter_index != I40E_UDP_PORT_INDEX_UNUSED) 9777 ret = i40e_aq_del_udp_tunnel(hw, filter_index, 9778 NULL); 9779 9780 /* reacquire RTNL so we can update filter_index */ 9781 rtnl_lock(); 9782 9783 if (ret) { 9784 dev_info(&pf->pdev->dev, 9785 "%s %s port %d, index %d failed, err %s aq_err %s\n", 9786 i40e_tunnel_name(type), 9787 port ? "add" : "delete", 9788 port, 9789 filter_index, 9790 i40e_stat_str(&pf->hw, ret), 9791 i40e_aq_str(&pf->hw, 9792 pf->hw.aq.asq_last_status)); 9793 if (port) { 9794 /* failed to add, just reset port, 9795 * drop pending bit for any deletion 9796 */ 9797 udp_port->port = 0; 9798 pf->pending_udp_bitmap &= ~BIT_ULL(i); 9799 } 9800 } else if (port) { 9801 /* record filter index on success */ 9802 udp_port->filter_index = filter_index; 9803 } 9804 } 9805 } 9806 9807 rtnl_unlock(); 9808 } 9809 9810 /** 9811 * i40e_service_task - Run the driver's async subtasks 9812 * @work: pointer to work_struct containing our data 9813 **/ 9814 static void i40e_service_task(struct work_struct *work) 9815 { 9816 struct i40e_pf *pf = container_of(work, 9817 struct i40e_pf, 9818 service_task); 9819 unsigned long start_time = jiffies; 9820 9821 /* don't bother with service tasks if a reset is in progress */ 9822 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 9823 return; 9824 9825 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 9826 return; 9827 9828 i40e_detect_recover_hung(pf->vsi[pf->lan_vsi]); 9829 i40e_sync_filters_subtask(pf); 9830 i40e_reset_subtask(pf); 9831 i40e_handle_mdd_event(pf); 9832 i40e_vc_process_vflr_event(pf); 9833 i40e_watchdog_subtask(pf); 9834 i40e_fdir_reinit_subtask(pf); 9835 if (test_and_clear_bit(__I40E_CLIENT_RESET, pf->state)) { 9836 /* Client subtask will reopen next time through. */ 9837 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true); 9838 } else { 9839 i40e_client_subtask(pf); 9840 if (test_and_clear_bit(__I40E_CLIENT_L2_CHANGE, 9841 pf->state)) 9842 i40e_notify_client_of_l2_param_changes( 9843 pf->vsi[pf->lan_vsi]); 9844 } 9845 i40e_sync_filters_subtask(pf); 9846 i40e_sync_udp_filters_subtask(pf); 9847 i40e_clean_adminq_subtask(pf); 9848 9849 /* flush memory to make sure state is correct before next watchdog */ 9850 smp_mb__before_atomic(); 9851 clear_bit(__I40E_SERVICE_SCHED, pf->state); 9852 9853 /* If the tasks have taken longer than one timer cycle or there 9854 * is more work to be done, reschedule the service task now 9855 * rather than wait for the timer to tick again. 9856 */ 9857 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 9858 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 9859 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 9860 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 9861 i40e_service_event_schedule(pf); 9862 } 9863 9864 /** 9865 * i40e_service_timer - timer callback 9866 * @data: pointer to PF struct 9867 **/ 9868 static void i40e_service_timer(struct timer_list *t) 9869 { 9870 struct i40e_pf *pf = from_timer(pf, t, service_timer); 9871 9872 mod_timer(&pf->service_timer, 9873 round_jiffies(jiffies + pf->service_timer_period)); 9874 i40e_service_event_schedule(pf); 9875 } 9876 9877 /** 9878 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 9879 * @vsi: the VSI being configured 9880 **/ 9881 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 9882 { 9883 struct i40e_pf *pf = vsi->back; 9884 9885 switch (vsi->type) { 9886 case I40E_VSI_MAIN: 9887 vsi->alloc_queue_pairs = pf->num_lan_qps; 9888 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9889 I40E_REQ_DESCRIPTOR_MULTIPLE); 9890 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 9891 vsi->num_q_vectors = pf->num_lan_msix; 9892 else 9893 vsi->num_q_vectors = 1; 9894 9895 break; 9896 9897 case I40E_VSI_FDIR: 9898 vsi->alloc_queue_pairs = 1; 9899 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 9900 I40E_REQ_DESCRIPTOR_MULTIPLE); 9901 vsi->num_q_vectors = pf->num_fdsb_msix; 9902 break; 9903 9904 case I40E_VSI_VMDQ2: 9905 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 9906 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9907 I40E_REQ_DESCRIPTOR_MULTIPLE); 9908 vsi->num_q_vectors = pf->num_vmdq_msix; 9909 break; 9910 9911 case I40E_VSI_SRIOV: 9912 vsi->alloc_queue_pairs = pf->num_vf_qps; 9913 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 9914 I40E_REQ_DESCRIPTOR_MULTIPLE); 9915 break; 9916 9917 default: 9918 WARN_ON(1); 9919 return -ENODATA; 9920 } 9921 9922 return 0; 9923 } 9924 9925 /** 9926 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 9927 * @vsi: VSI pointer 9928 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 9929 * 9930 * On error: returns error code (negative) 9931 * On success: returns 0 9932 **/ 9933 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 9934 { 9935 struct i40e_ring **next_rings; 9936 int size; 9937 int ret = 0; 9938 9939 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 9940 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 9941 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 9942 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 9943 if (!vsi->tx_rings) 9944 return -ENOMEM; 9945 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 9946 if (i40e_enabled_xdp_vsi(vsi)) { 9947 vsi->xdp_rings = next_rings; 9948 next_rings += vsi->alloc_queue_pairs; 9949 } 9950 vsi->rx_rings = next_rings; 9951 9952 if (alloc_qvectors) { 9953 /* allocate memory for q_vector pointers */ 9954 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 9955 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 9956 if (!vsi->q_vectors) { 9957 ret = -ENOMEM; 9958 goto err_vectors; 9959 } 9960 } 9961 return ret; 9962 9963 err_vectors: 9964 kfree(vsi->tx_rings); 9965 return ret; 9966 } 9967 9968 /** 9969 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 9970 * @pf: board private structure 9971 * @type: type of VSI 9972 * 9973 * On error: returns error code (negative) 9974 * On success: returns vsi index in PF (positive) 9975 **/ 9976 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 9977 { 9978 int ret = -ENODEV; 9979 struct i40e_vsi *vsi; 9980 int vsi_idx; 9981 int i; 9982 9983 /* Need to protect the allocation of the VSIs at the PF level */ 9984 mutex_lock(&pf->switch_mutex); 9985 9986 /* VSI list may be fragmented if VSI creation/destruction has 9987 * been happening. We can afford to do a quick scan to look 9988 * for any free VSIs in the list. 9989 * 9990 * find next empty vsi slot, looping back around if necessary 9991 */ 9992 i = pf->next_vsi; 9993 while (i < pf->num_alloc_vsi && pf->vsi[i]) 9994 i++; 9995 if (i >= pf->num_alloc_vsi) { 9996 i = 0; 9997 while (i < pf->next_vsi && pf->vsi[i]) 9998 i++; 9999 } 10000 10001 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 10002 vsi_idx = i; /* Found one! */ 10003 } else { 10004 ret = -ENODEV; 10005 goto unlock_pf; /* out of VSI slots! */ 10006 } 10007 pf->next_vsi = ++i; 10008 10009 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 10010 if (!vsi) { 10011 ret = -ENOMEM; 10012 goto unlock_pf; 10013 } 10014 vsi->type = type; 10015 vsi->back = pf; 10016 set_bit(__I40E_VSI_DOWN, vsi->state); 10017 vsi->flags = 0; 10018 vsi->idx = vsi_idx; 10019 vsi->int_rate_limit = 0; 10020 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 10021 pf->rss_table_size : 64; 10022 vsi->netdev_registered = false; 10023 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 10024 hash_init(vsi->mac_filter_hash); 10025 vsi->irqs_ready = false; 10026 10027 ret = i40e_set_num_rings_in_vsi(vsi); 10028 if (ret) 10029 goto err_rings; 10030 10031 ret = i40e_vsi_alloc_arrays(vsi, true); 10032 if (ret) 10033 goto err_rings; 10034 10035 /* Setup default MSIX irq handler for VSI */ 10036 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 10037 10038 /* Initialize VSI lock */ 10039 spin_lock_init(&vsi->mac_filter_hash_lock); 10040 pf->vsi[vsi_idx] = vsi; 10041 ret = vsi_idx; 10042 goto unlock_pf; 10043 10044 err_rings: 10045 pf->next_vsi = i - 1; 10046 kfree(vsi); 10047 unlock_pf: 10048 mutex_unlock(&pf->switch_mutex); 10049 return ret; 10050 } 10051 10052 /** 10053 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 10054 * @vsi: VSI pointer 10055 * @free_qvectors: a bool to specify if q_vectors need to be freed. 10056 * 10057 * On error: returns error code (negative) 10058 * On success: returns 0 10059 **/ 10060 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 10061 { 10062 /* free the ring and vector containers */ 10063 if (free_qvectors) { 10064 kfree(vsi->q_vectors); 10065 vsi->q_vectors = NULL; 10066 } 10067 kfree(vsi->tx_rings); 10068 vsi->tx_rings = NULL; 10069 vsi->rx_rings = NULL; 10070 vsi->xdp_rings = NULL; 10071 } 10072 10073 /** 10074 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 10075 * and lookup table 10076 * @vsi: Pointer to VSI structure 10077 */ 10078 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 10079 { 10080 if (!vsi) 10081 return; 10082 10083 kfree(vsi->rss_hkey_user); 10084 vsi->rss_hkey_user = NULL; 10085 10086 kfree(vsi->rss_lut_user); 10087 vsi->rss_lut_user = NULL; 10088 } 10089 10090 /** 10091 * i40e_vsi_clear - Deallocate the VSI provided 10092 * @vsi: the VSI being un-configured 10093 **/ 10094 static int i40e_vsi_clear(struct i40e_vsi *vsi) 10095 { 10096 struct i40e_pf *pf; 10097 10098 if (!vsi) 10099 return 0; 10100 10101 if (!vsi->back) 10102 goto free_vsi; 10103 pf = vsi->back; 10104 10105 mutex_lock(&pf->switch_mutex); 10106 if (!pf->vsi[vsi->idx]) { 10107 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](type %d)\n", 10108 vsi->idx, vsi->idx, vsi->type); 10109 goto unlock_vsi; 10110 } 10111 10112 if (pf->vsi[vsi->idx] != vsi) { 10113 dev_err(&pf->pdev->dev, 10114 "pf->vsi[%d](type %d) != vsi[%d](type %d): no free!\n", 10115 pf->vsi[vsi->idx]->idx, 10116 pf->vsi[vsi->idx]->type, 10117 vsi->idx, vsi->type); 10118 goto unlock_vsi; 10119 } 10120 10121 /* updates the PF for this cleared vsi */ 10122 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 10123 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 10124 10125 i40e_vsi_free_arrays(vsi, true); 10126 i40e_clear_rss_config_user(vsi); 10127 10128 pf->vsi[vsi->idx] = NULL; 10129 if (vsi->idx < pf->next_vsi) 10130 pf->next_vsi = vsi->idx; 10131 10132 unlock_vsi: 10133 mutex_unlock(&pf->switch_mutex); 10134 free_vsi: 10135 kfree(vsi); 10136 10137 return 0; 10138 } 10139 10140 /** 10141 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 10142 * @vsi: the VSI being cleaned 10143 **/ 10144 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 10145 { 10146 int i; 10147 10148 if (vsi->tx_rings && vsi->tx_rings[0]) { 10149 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10150 kfree_rcu(vsi->tx_rings[i], rcu); 10151 vsi->tx_rings[i] = NULL; 10152 vsi->rx_rings[i] = NULL; 10153 if (vsi->xdp_rings) 10154 vsi->xdp_rings[i] = NULL; 10155 } 10156 } 10157 } 10158 10159 /** 10160 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 10161 * @vsi: the VSI being configured 10162 **/ 10163 static int i40e_alloc_rings(struct i40e_vsi *vsi) 10164 { 10165 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 10166 struct i40e_pf *pf = vsi->back; 10167 struct i40e_ring *ring; 10168 10169 /* Set basic values in the rings to be used later during open() */ 10170 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 10171 /* allocate space for both Tx and Rx in one shot */ 10172 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 10173 if (!ring) 10174 goto err_out; 10175 10176 ring->queue_index = i; 10177 ring->reg_idx = vsi->base_queue + i; 10178 ring->ring_active = false; 10179 ring->vsi = vsi; 10180 ring->netdev = vsi->netdev; 10181 ring->dev = &pf->pdev->dev; 10182 ring->count = vsi->num_desc; 10183 ring->size = 0; 10184 ring->dcb_tc = 0; 10185 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10186 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10187 ring->itr_setting = pf->tx_itr_default; 10188 vsi->tx_rings[i] = ring++; 10189 10190 if (!i40e_enabled_xdp_vsi(vsi)) 10191 goto setup_rx; 10192 10193 ring->queue_index = vsi->alloc_queue_pairs + i; 10194 ring->reg_idx = vsi->base_queue + ring->queue_index; 10195 ring->ring_active = false; 10196 ring->vsi = vsi; 10197 ring->netdev = NULL; 10198 ring->dev = &pf->pdev->dev; 10199 ring->count = vsi->num_desc; 10200 ring->size = 0; 10201 ring->dcb_tc = 0; 10202 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 10203 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 10204 set_ring_xdp(ring); 10205 ring->itr_setting = pf->tx_itr_default; 10206 vsi->xdp_rings[i] = ring++; 10207 10208 setup_rx: 10209 ring->queue_index = i; 10210 ring->reg_idx = vsi->base_queue + i; 10211 ring->ring_active = false; 10212 ring->vsi = vsi; 10213 ring->netdev = vsi->netdev; 10214 ring->dev = &pf->pdev->dev; 10215 ring->count = vsi->num_desc; 10216 ring->size = 0; 10217 ring->dcb_tc = 0; 10218 ring->itr_setting = pf->rx_itr_default; 10219 vsi->rx_rings[i] = ring; 10220 } 10221 10222 return 0; 10223 10224 err_out: 10225 i40e_vsi_clear_rings(vsi); 10226 return -ENOMEM; 10227 } 10228 10229 /** 10230 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 10231 * @pf: board private structure 10232 * @vectors: the number of MSI-X vectors to request 10233 * 10234 * Returns the number of vectors reserved, or error 10235 **/ 10236 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 10237 { 10238 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 10239 I40E_MIN_MSIX, vectors); 10240 if (vectors < 0) { 10241 dev_info(&pf->pdev->dev, 10242 "MSI-X vector reservation failed: %d\n", vectors); 10243 vectors = 0; 10244 } 10245 10246 return vectors; 10247 } 10248 10249 /** 10250 * i40e_init_msix - Setup the MSIX capability 10251 * @pf: board private structure 10252 * 10253 * Work with the OS to set up the MSIX vectors needed. 10254 * 10255 * Returns the number of vectors reserved or negative on failure 10256 **/ 10257 static int i40e_init_msix(struct i40e_pf *pf) 10258 { 10259 struct i40e_hw *hw = &pf->hw; 10260 int cpus, extra_vectors; 10261 int vectors_left; 10262 int v_budget, i; 10263 int v_actual; 10264 int iwarp_requested = 0; 10265 10266 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10267 return -ENODEV; 10268 10269 /* The number of vectors we'll request will be comprised of: 10270 * - Add 1 for "other" cause for Admin Queue events, etc. 10271 * - The number of LAN queue pairs 10272 * - Queues being used for RSS. 10273 * We don't need as many as max_rss_size vectors. 10274 * use rss_size instead in the calculation since that 10275 * is governed by number of cpus in the system. 10276 * - assumes symmetric Tx/Rx pairing 10277 * - The number of VMDq pairs 10278 * - The CPU count within the NUMA node if iWARP is enabled 10279 * Once we count this up, try the request. 10280 * 10281 * If we can't get what we want, we'll simplify to nearly nothing 10282 * and try again. If that still fails, we punt. 10283 */ 10284 vectors_left = hw->func_caps.num_msix_vectors; 10285 v_budget = 0; 10286 10287 /* reserve one vector for miscellaneous handler */ 10288 if (vectors_left) { 10289 v_budget++; 10290 vectors_left--; 10291 } 10292 10293 /* reserve some vectors for the main PF traffic queues. Initially we 10294 * only reserve at most 50% of the available vectors, in the case that 10295 * the number of online CPUs is large. This ensures that we can enable 10296 * extra features as well. Once we've enabled the other features, we 10297 * will use any remaining vectors to reach as close as we can to the 10298 * number of online CPUs. 10299 */ 10300 cpus = num_online_cpus(); 10301 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 10302 vectors_left -= pf->num_lan_msix; 10303 10304 /* reserve one vector for sideband flow director */ 10305 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10306 if (vectors_left) { 10307 pf->num_fdsb_msix = 1; 10308 v_budget++; 10309 vectors_left--; 10310 } else { 10311 pf->num_fdsb_msix = 0; 10312 } 10313 } 10314 10315 /* can we reserve enough for iWARP? */ 10316 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10317 iwarp_requested = pf->num_iwarp_msix; 10318 10319 if (!vectors_left) 10320 pf->num_iwarp_msix = 0; 10321 else if (vectors_left < pf->num_iwarp_msix) 10322 pf->num_iwarp_msix = 1; 10323 v_budget += pf->num_iwarp_msix; 10324 vectors_left -= pf->num_iwarp_msix; 10325 } 10326 10327 /* any vectors left over go for VMDq support */ 10328 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 10329 if (!vectors_left) { 10330 pf->num_vmdq_msix = 0; 10331 pf->num_vmdq_qps = 0; 10332 } else { 10333 int vmdq_vecs_wanted = 10334 pf->num_vmdq_vsis * pf->num_vmdq_qps; 10335 int vmdq_vecs = 10336 min_t(int, vectors_left, vmdq_vecs_wanted); 10337 10338 /* if we're short on vectors for what's desired, we limit 10339 * the queues per vmdq. If this is still more than are 10340 * available, the user will need to change the number of 10341 * queues/vectors used by the PF later with the ethtool 10342 * channels command 10343 */ 10344 if (vectors_left < vmdq_vecs_wanted) { 10345 pf->num_vmdq_qps = 1; 10346 vmdq_vecs_wanted = pf->num_vmdq_vsis; 10347 vmdq_vecs = min_t(int, 10348 vectors_left, 10349 vmdq_vecs_wanted); 10350 } 10351 pf->num_vmdq_msix = pf->num_vmdq_qps; 10352 10353 v_budget += vmdq_vecs; 10354 vectors_left -= vmdq_vecs; 10355 } 10356 } 10357 10358 /* On systems with a large number of SMP cores, we previously limited 10359 * the number of vectors for num_lan_msix to be at most 50% of the 10360 * available vectors, to allow for other features. Now, we add back 10361 * the remaining vectors. However, we ensure that the total 10362 * num_lan_msix will not exceed num_online_cpus(). To do this, we 10363 * calculate the number of vectors we can add without going over the 10364 * cap of CPUs. For systems with a small number of CPUs this will be 10365 * zero. 10366 */ 10367 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 10368 pf->num_lan_msix += extra_vectors; 10369 vectors_left -= extra_vectors; 10370 10371 WARN(vectors_left < 0, 10372 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 10373 10374 v_budget += pf->num_lan_msix; 10375 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 10376 GFP_KERNEL); 10377 if (!pf->msix_entries) 10378 return -ENOMEM; 10379 10380 for (i = 0; i < v_budget; i++) 10381 pf->msix_entries[i].entry = i; 10382 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 10383 10384 if (v_actual < I40E_MIN_MSIX) { 10385 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 10386 kfree(pf->msix_entries); 10387 pf->msix_entries = NULL; 10388 pci_disable_msix(pf->pdev); 10389 return -ENODEV; 10390 10391 } else if (v_actual == I40E_MIN_MSIX) { 10392 /* Adjust for minimal MSIX use */ 10393 pf->num_vmdq_vsis = 0; 10394 pf->num_vmdq_qps = 0; 10395 pf->num_lan_qps = 1; 10396 pf->num_lan_msix = 1; 10397 10398 } else if (v_actual != v_budget) { 10399 /* If we have limited resources, we will start with no vectors 10400 * for the special features and then allocate vectors to some 10401 * of these features based on the policy and at the end disable 10402 * the features that did not get any vectors. 10403 */ 10404 int vec; 10405 10406 dev_info(&pf->pdev->dev, 10407 "MSI-X vector limit reached with %d, wanted %d, attempting to redistribute vectors\n", 10408 v_actual, v_budget); 10409 /* reserve the misc vector */ 10410 vec = v_actual - 1; 10411 10412 /* Scale vector usage down */ 10413 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 10414 pf->num_vmdq_vsis = 1; 10415 pf->num_vmdq_qps = 1; 10416 10417 /* partition out the remaining vectors */ 10418 switch (vec) { 10419 case 2: 10420 pf->num_lan_msix = 1; 10421 break; 10422 case 3: 10423 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10424 pf->num_lan_msix = 1; 10425 pf->num_iwarp_msix = 1; 10426 } else { 10427 pf->num_lan_msix = 2; 10428 } 10429 break; 10430 default: 10431 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 10432 pf->num_iwarp_msix = min_t(int, (vec / 3), 10433 iwarp_requested); 10434 pf->num_vmdq_vsis = min_t(int, (vec / 3), 10435 I40E_DEFAULT_NUM_VMDQ_VSI); 10436 } else { 10437 pf->num_vmdq_vsis = min_t(int, (vec / 2), 10438 I40E_DEFAULT_NUM_VMDQ_VSI); 10439 } 10440 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10441 pf->num_fdsb_msix = 1; 10442 vec--; 10443 } 10444 pf->num_lan_msix = min_t(int, 10445 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 10446 pf->num_lan_msix); 10447 pf->num_lan_qps = pf->num_lan_msix; 10448 break; 10449 } 10450 } 10451 10452 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 10453 (pf->num_fdsb_msix == 0)) { 10454 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 10455 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10456 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10457 } 10458 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 10459 (pf->num_vmdq_msix == 0)) { 10460 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 10461 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 10462 } 10463 10464 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 10465 (pf->num_iwarp_msix == 0)) { 10466 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 10467 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 10468 } 10469 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 10470 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 10471 pf->num_lan_msix, 10472 pf->num_vmdq_msix * pf->num_vmdq_vsis, 10473 pf->num_fdsb_msix, 10474 pf->num_iwarp_msix); 10475 10476 return v_actual; 10477 } 10478 10479 /** 10480 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 10481 * @vsi: the VSI being configured 10482 * @v_idx: index of the vector in the vsi struct 10483 * @cpu: cpu to be used on affinity_mask 10484 * 10485 * We allocate one q_vector. If allocation fails we return -ENOMEM. 10486 **/ 10487 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 10488 { 10489 struct i40e_q_vector *q_vector; 10490 10491 /* allocate q_vector */ 10492 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 10493 if (!q_vector) 10494 return -ENOMEM; 10495 10496 q_vector->vsi = vsi; 10497 q_vector->v_idx = v_idx; 10498 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 10499 10500 if (vsi->netdev) 10501 netif_napi_add(vsi->netdev, &q_vector->napi, 10502 i40e_napi_poll, NAPI_POLL_WEIGHT); 10503 10504 /* tie q_vector and vsi together */ 10505 vsi->q_vectors[v_idx] = q_vector; 10506 10507 return 0; 10508 } 10509 10510 /** 10511 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 10512 * @vsi: the VSI being configured 10513 * 10514 * We allocate one q_vector per queue interrupt. If allocation fails we 10515 * return -ENOMEM. 10516 **/ 10517 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 10518 { 10519 struct i40e_pf *pf = vsi->back; 10520 int err, v_idx, num_q_vectors, current_cpu; 10521 10522 /* if not MSIX, give the one vector only to the LAN VSI */ 10523 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 10524 num_q_vectors = vsi->num_q_vectors; 10525 else if (vsi == pf->vsi[pf->lan_vsi]) 10526 num_q_vectors = 1; 10527 else 10528 return -EINVAL; 10529 10530 current_cpu = cpumask_first(cpu_online_mask); 10531 10532 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 10533 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 10534 if (err) 10535 goto err_out; 10536 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 10537 if (unlikely(current_cpu >= nr_cpu_ids)) 10538 current_cpu = cpumask_first(cpu_online_mask); 10539 } 10540 10541 return 0; 10542 10543 err_out: 10544 while (v_idx--) 10545 i40e_free_q_vector(vsi, v_idx); 10546 10547 return err; 10548 } 10549 10550 /** 10551 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 10552 * @pf: board private structure to initialize 10553 **/ 10554 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 10555 { 10556 int vectors = 0; 10557 ssize_t size; 10558 10559 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 10560 vectors = i40e_init_msix(pf); 10561 if (vectors < 0) { 10562 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 10563 I40E_FLAG_IWARP_ENABLED | 10564 I40E_FLAG_RSS_ENABLED | 10565 I40E_FLAG_DCB_CAPABLE | 10566 I40E_FLAG_DCB_ENABLED | 10567 I40E_FLAG_SRIOV_ENABLED | 10568 I40E_FLAG_FD_SB_ENABLED | 10569 I40E_FLAG_FD_ATR_ENABLED | 10570 I40E_FLAG_VMDQ_ENABLED); 10571 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 10572 10573 /* rework the queue expectations without MSIX */ 10574 i40e_determine_queue_usage(pf); 10575 } 10576 } 10577 10578 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 10579 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 10580 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 10581 vectors = pci_enable_msi(pf->pdev); 10582 if (vectors < 0) { 10583 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 10584 vectors); 10585 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 10586 } 10587 vectors = 1; /* one MSI or Legacy vector */ 10588 } 10589 10590 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 10591 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 10592 10593 /* set up vector assignment tracking */ 10594 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 10595 pf->irq_pile = kzalloc(size, GFP_KERNEL); 10596 if (!pf->irq_pile) 10597 return -ENOMEM; 10598 10599 pf->irq_pile->num_entries = vectors; 10600 pf->irq_pile->search_hint = 0; 10601 10602 /* track first vector for misc interrupts, ignore return */ 10603 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 10604 10605 return 0; 10606 } 10607 10608 /** 10609 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 10610 * @pf: private board data structure 10611 * 10612 * Restore the interrupt scheme that was cleared when we suspended the 10613 * device. This should be called during resume to re-allocate the q_vectors 10614 * and reacquire IRQs. 10615 */ 10616 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 10617 { 10618 int err, i; 10619 10620 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 10621 * scheme. We need to re-enabled them here in order to attempt to 10622 * re-acquire the MSI or MSI-X vectors 10623 */ 10624 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 10625 10626 err = i40e_init_interrupt_scheme(pf); 10627 if (err) 10628 return err; 10629 10630 /* Now that we've re-acquired IRQs, we need to remap the vectors and 10631 * rings together again. 10632 */ 10633 for (i = 0; i < pf->num_alloc_vsi; i++) { 10634 if (pf->vsi[i]) { 10635 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 10636 if (err) 10637 goto err_unwind; 10638 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 10639 } 10640 } 10641 10642 err = i40e_setup_misc_vector(pf); 10643 if (err) 10644 goto err_unwind; 10645 10646 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 10647 i40e_client_update_msix_info(pf); 10648 10649 return 0; 10650 10651 err_unwind: 10652 while (i--) { 10653 if (pf->vsi[i]) 10654 i40e_vsi_free_q_vectors(pf->vsi[i]); 10655 } 10656 10657 return err; 10658 } 10659 10660 /** 10661 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 10662 * @pf: board private structure 10663 * 10664 * This sets up the handler for MSIX 0, which is used to manage the 10665 * non-queue interrupts, e.g. AdminQ and errors. This is not used 10666 * when in MSI or Legacy interrupt mode. 10667 **/ 10668 static int i40e_setup_misc_vector(struct i40e_pf *pf) 10669 { 10670 struct i40e_hw *hw = &pf->hw; 10671 int err = 0; 10672 10673 /* Only request the IRQ once, the first time through. */ 10674 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 10675 err = request_irq(pf->msix_entries[0].vector, 10676 i40e_intr, 0, pf->int_name, pf); 10677 if (err) { 10678 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 10679 dev_info(&pf->pdev->dev, 10680 "request_irq for %s failed: %d\n", 10681 pf->int_name, err); 10682 return -EFAULT; 10683 } 10684 } 10685 10686 i40e_enable_misc_int_causes(pf); 10687 10688 /* associate no queues to the misc vector */ 10689 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 10690 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 10691 10692 i40e_flush(hw); 10693 10694 i40e_irq_dynamic_enable_icr0(pf); 10695 10696 return err; 10697 } 10698 10699 /** 10700 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 10701 * @vsi: Pointer to vsi structure 10702 * @seed: Buffter to store the hash keys 10703 * @lut: Buffer to store the lookup table entries 10704 * @lut_size: Size of buffer to store the lookup table entries 10705 * 10706 * Return 0 on success, negative on failure 10707 */ 10708 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 10709 u8 *lut, u16 lut_size) 10710 { 10711 struct i40e_pf *pf = vsi->back; 10712 struct i40e_hw *hw = &pf->hw; 10713 int ret = 0; 10714 10715 if (seed) { 10716 ret = i40e_aq_get_rss_key(hw, vsi->id, 10717 (struct i40e_aqc_get_set_rss_key_data *)seed); 10718 if (ret) { 10719 dev_info(&pf->pdev->dev, 10720 "Cannot get RSS key, err %s aq_err %s\n", 10721 i40e_stat_str(&pf->hw, ret), 10722 i40e_aq_str(&pf->hw, 10723 pf->hw.aq.asq_last_status)); 10724 return ret; 10725 } 10726 } 10727 10728 if (lut) { 10729 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 10730 10731 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 10732 if (ret) { 10733 dev_info(&pf->pdev->dev, 10734 "Cannot get RSS lut, err %s aq_err %s\n", 10735 i40e_stat_str(&pf->hw, ret), 10736 i40e_aq_str(&pf->hw, 10737 pf->hw.aq.asq_last_status)); 10738 return ret; 10739 } 10740 } 10741 10742 return ret; 10743 } 10744 10745 /** 10746 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 10747 * @vsi: Pointer to vsi structure 10748 * @seed: RSS hash seed 10749 * @lut: Lookup table 10750 * @lut_size: Lookup table size 10751 * 10752 * Returns 0 on success, negative on failure 10753 **/ 10754 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 10755 const u8 *lut, u16 lut_size) 10756 { 10757 struct i40e_pf *pf = vsi->back; 10758 struct i40e_hw *hw = &pf->hw; 10759 u16 vf_id = vsi->vf_id; 10760 u8 i; 10761 10762 /* Fill out hash function seed */ 10763 if (seed) { 10764 u32 *seed_dw = (u32 *)seed; 10765 10766 if (vsi->type == I40E_VSI_MAIN) { 10767 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 10768 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 10769 } else if (vsi->type == I40E_VSI_SRIOV) { 10770 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 10771 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 10772 } else { 10773 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 10774 } 10775 } 10776 10777 if (lut) { 10778 u32 *lut_dw = (u32 *)lut; 10779 10780 if (vsi->type == I40E_VSI_MAIN) { 10781 if (lut_size != I40E_HLUT_ARRAY_SIZE) 10782 return -EINVAL; 10783 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 10784 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 10785 } else if (vsi->type == I40E_VSI_SRIOV) { 10786 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 10787 return -EINVAL; 10788 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 10789 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 10790 } else { 10791 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 10792 } 10793 } 10794 i40e_flush(hw); 10795 10796 return 0; 10797 } 10798 10799 /** 10800 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 10801 * @vsi: Pointer to VSI structure 10802 * @seed: Buffer to store the keys 10803 * @lut: Buffer to store the lookup table entries 10804 * @lut_size: Size of buffer to store the lookup table entries 10805 * 10806 * Returns 0 on success, negative on failure 10807 */ 10808 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 10809 u8 *lut, u16 lut_size) 10810 { 10811 struct i40e_pf *pf = vsi->back; 10812 struct i40e_hw *hw = &pf->hw; 10813 u16 i; 10814 10815 if (seed) { 10816 u32 *seed_dw = (u32 *)seed; 10817 10818 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 10819 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 10820 } 10821 if (lut) { 10822 u32 *lut_dw = (u32 *)lut; 10823 10824 if (lut_size != I40E_HLUT_ARRAY_SIZE) 10825 return -EINVAL; 10826 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 10827 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 10828 } 10829 10830 return 0; 10831 } 10832 10833 /** 10834 * i40e_config_rss - Configure RSS keys and lut 10835 * @vsi: Pointer to VSI structure 10836 * @seed: RSS hash seed 10837 * @lut: Lookup table 10838 * @lut_size: Lookup table size 10839 * 10840 * Returns 0 on success, negative on failure 10841 */ 10842 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 10843 { 10844 struct i40e_pf *pf = vsi->back; 10845 10846 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 10847 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 10848 else 10849 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 10850 } 10851 10852 /** 10853 * i40e_get_rss - Get RSS keys and lut 10854 * @vsi: Pointer to VSI structure 10855 * @seed: Buffer to store the keys 10856 * @lut: Buffer to store the lookup table entries 10857 * @lut_size: Size of buffer to store the lookup table entries 10858 * 10859 * Returns 0 on success, negative on failure 10860 */ 10861 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 10862 { 10863 struct i40e_pf *pf = vsi->back; 10864 10865 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 10866 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 10867 else 10868 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 10869 } 10870 10871 /** 10872 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 10873 * @pf: Pointer to board private structure 10874 * @lut: Lookup table 10875 * @rss_table_size: Lookup table size 10876 * @rss_size: Range of queue number for hashing 10877 */ 10878 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 10879 u16 rss_table_size, u16 rss_size) 10880 { 10881 u16 i; 10882 10883 for (i = 0; i < rss_table_size; i++) 10884 lut[i] = i % rss_size; 10885 } 10886 10887 /** 10888 * i40e_pf_config_rss - Prepare for RSS if used 10889 * @pf: board private structure 10890 **/ 10891 static int i40e_pf_config_rss(struct i40e_pf *pf) 10892 { 10893 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10894 u8 seed[I40E_HKEY_ARRAY_SIZE]; 10895 u8 *lut; 10896 struct i40e_hw *hw = &pf->hw; 10897 u32 reg_val; 10898 u64 hena; 10899 int ret; 10900 10901 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 10902 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 10903 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 10904 hena |= i40e_pf_get_default_rss_hena(pf); 10905 10906 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 10907 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 10908 10909 /* Determine the RSS table size based on the hardware capabilities */ 10910 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 10911 reg_val = (pf->rss_table_size == 512) ? 10912 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 10913 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 10914 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 10915 10916 /* Determine the RSS size of the VSI */ 10917 if (!vsi->rss_size) { 10918 u16 qcount; 10919 /* If the firmware does something weird during VSI init, we 10920 * could end up with zero TCs. Check for that to avoid 10921 * divide-by-zero. It probably won't pass traffic, but it also 10922 * won't panic. 10923 */ 10924 qcount = vsi->num_queue_pairs / 10925 (vsi->tc_config.numtc ? vsi->tc_config.numtc : 1); 10926 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 10927 } 10928 if (!vsi->rss_size) 10929 return -EINVAL; 10930 10931 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 10932 if (!lut) 10933 return -ENOMEM; 10934 10935 /* Use user configured lut if there is one, otherwise use default */ 10936 if (vsi->rss_lut_user) 10937 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 10938 else 10939 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 10940 10941 /* Use user configured hash key if there is one, otherwise 10942 * use default. 10943 */ 10944 if (vsi->rss_hkey_user) 10945 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 10946 else 10947 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 10948 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 10949 kfree(lut); 10950 10951 return ret; 10952 } 10953 10954 /** 10955 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 10956 * @pf: board private structure 10957 * @queue_count: the requested queue count for rss. 10958 * 10959 * returns 0 if rss is not enabled, if enabled returns the final rss queue 10960 * count which may be different from the requested queue count. 10961 * Note: expects to be called while under rtnl_lock() 10962 **/ 10963 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 10964 { 10965 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 10966 int new_rss_size; 10967 10968 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 10969 return 0; 10970 10971 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 10972 10973 if (queue_count != vsi->num_queue_pairs) { 10974 u16 qcount; 10975 10976 vsi->req_queue_pairs = queue_count; 10977 i40e_prep_for_reset(pf, true); 10978 10979 pf->alloc_rss_size = new_rss_size; 10980 10981 i40e_reset_and_rebuild(pf, true, true); 10982 10983 /* Discard the user configured hash keys and lut, if less 10984 * queues are enabled. 10985 */ 10986 if (queue_count < vsi->rss_size) { 10987 i40e_clear_rss_config_user(vsi); 10988 dev_dbg(&pf->pdev->dev, 10989 "discard user configured hash keys and lut\n"); 10990 } 10991 10992 /* Reset vsi->rss_size, as number of enabled queues changed */ 10993 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 10994 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 10995 10996 i40e_pf_config_rss(pf); 10997 } 10998 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 10999 vsi->req_queue_pairs, pf->rss_size_max); 11000 return pf->alloc_rss_size; 11001 } 11002 11003 /** 11004 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 11005 * @pf: board private structure 11006 **/ 11007 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 11008 { 11009 i40e_status status; 11010 bool min_valid, max_valid; 11011 u32 max_bw, min_bw; 11012 11013 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 11014 &min_valid, &max_valid); 11015 11016 if (!status) { 11017 if (min_valid) 11018 pf->min_bw = min_bw; 11019 if (max_valid) 11020 pf->max_bw = max_bw; 11021 } 11022 11023 return status; 11024 } 11025 11026 /** 11027 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 11028 * @pf: board private structure 11029 **/ 11030 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 11031 { 11032 struct i40e_aqc_configure_partition_bw_data bw_data; 11033 i40e_status status; 11034 11035 /* Set the valid bit for this PF */ 11036 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 11037 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 11038 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 11039 11040 /* Set the new bandwidths */ 11041 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 11042 11043 return status; 11044 } 11045 11046 /** 11047 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 11048 * @pf: board private structure 11049 **/ 11050 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 11051 { 11052 /* Commit temporary BW setting to permanent NVM image */ 11053 enum i40e_admin_queue_err last_aq_status; 11054 i40e_status ret; 11055 u16 nvm_word; 11056 11057 if (pf->hw.partition_id != 1) { 11058 dev_info(&pf->pdev->dev, 11059 "Commit BW only works on partition 1! This is partition %d", 11060 pf->hw.partition_id); 11061 ret = I40E_NOT_SUPPORTED; 11062 goto bw_commit_out; 11063 } 11064 11065 /* Acquire NVM for read access */ 11066 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 11067 last_aq_status = pf->hw.aq.asq_last_status; 11068 if (ret) { 11069 dev_info(&pf->pdev->dev, 11070 "Cannot acquire NVM for read access, err %s aq_err %s\n", 11071 i40e_stat_str(&pf->hw, ret), 11072 i40e_aq_str(&pf->hw, last_aq_status)); 11073 goto bw_commit_out; 11074 } 11075 11076 /* Read word 0x10 of NVM - SW compatibility word 1 */ 11077 ret = i40e_aq_read_nvm(&pf->hw, 11078 I40E_SR_NVM_CONTROL_WORD, 11079 0x10, sizeof(nvm_word), &nvm_word, 11080 false, NULL); 11081 /* Save off last admin queue command status before releasing 11082 * the NVM 11083 */ 11084 last_aq_status = pf->hw.aq.asq_last_status; 11085 i40e_release_nvm(&pf->hw); 11086 if (ret) { 11087 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 11088 i40e_stat_str(&pf->hw, ret), 11089 i40e_aq_str(&pf->hw, last_aq_status)); 11090 goto bw_commit_out; 11091 } 11092 11093 /* Wait a bit for NVM release to complete */ 11094 msleep(50); 11095 11096 /* Acquire NVM for write access */ 11097 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 11098 last_aq_status = pf->hw.aq.asq_last_status; 11099 if (ret) { 11100 dev_info(&pf->pdev->dev, 11101 "Cannot acquire NVM for write access, err %s aq_err %s\n", 11102 i40e_stat_str(&pf->hw, ret), 11103 i40e_aq_str(&pf->hw, last_aq_status)); 11104 goto bw_commit_out; 11105 } 11106 /* Write it back out unchanged to initiate update NVM, 11107 * which will force a write of the shadow (alt) RAM to 11108 * the NVM - thus storing the bandwidth values permanently. 11109 */ 11110 ret = i40e_aq_update_nvm(&pf->hw, 11111 I40E_SR_NVM_CONTROL_WORD, 11112 0x10, sizeof(nvm_word), 11113 &nvm_word, true, 0, NULL); 11114 /* Save off last admin queue command status before releasing 11115 * the NVM 11116 */ 11117 last_aq_status = pf->hw.aq.asq_last_status; 11118 i40e_release_nvm(&pf->hw); 11119 if (ret) 11120 dev_info(&pf->pdev->dev, 11121 "BW settings NOT SAVED, err %s aq_err %s\n", 11122 i40e_stat_str(&pf->hw, ret), 11123 i40e_aq_str(&pf->hw, last_aq_status)); 11124 bw_commit_out: 11125 11126 return ret; 11127 } 11128 11129 /** 11130 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 11131 * @pf: board private structure to initialize 11132 * 11133 * i40e_sw_init initializes the Adapter private data structure. 11134 * Fields are initialized based on PCI device information and 11135 * OS network device settings (MTU size). 11136 **/ 11137 static int i40e_sw_init(struct i40e_pf *pf) 11138 { 11139 int err = 0; 11140 int size; 11141 11142 /* Set default capability flags */ 11143 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 11144 I40E_FLAG_MSI_ENABLED | 11145 I40E_FLAG_MSIX_ENABLED; 11146 11147 /* Set default ITR */ 11148 pf->rx_itr_default = I40E_ITR_RX_DEF; 11149 pf->tx_itr_default = I40E_ITR_TX_DEF; 11150 11151 /* Depending on PF configurations, it is possible that the RSS 11152 * maximum might end up larger than the available queues 11153 */ 11154 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 11155 pf->alloc_rss_size = 1; 11156 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 11157 pf->rss_size_max = min_t(int, pf->rss_size_max, 11158 pf->hw.func_caps.num_tx_qp); 11159 if (pf->hw.func_caps.rss) { 11160 pf->flags |= I40E_FLAG_RSS_ENABLED; 11161 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 11162 num_online_cpus()); 11163 } 11164 11165 /* MFP mode enabled */ 11166 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 11167 pf->flags |= I40E_FLAG_MFP_ENABLED; 11168 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 11169 if (i40e_get_partition_bw_setting(pf)) { 11170 dev_warn(&pf->pdev->dev, 11171 "Could not get partition bw settings\n"); 11172 } else { 11173 dev_info(&pf->pdev->dev, 11174 "Partition BW Min = %8.8x, Max = %8.8x\n", 11175 pf->min_bw, pf->max_bw); 11176 11177 /* nudge the Tx scheduler */ 11178 i40e_set_partition_bw_setting(pf); 11179 } 11180 } 11181 11182 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 11183 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 11184 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 11185 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 11186 if (pf->flags & I40E_FLAG_MFP_ENABLED && 11187 pf->hw.num_partitions > 1) 11188 dev_info(&pf->pdev->dev, 11189 "Flow Director Sideband mode Disabled in MFP mode\n"); 11190 else 11191 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11192 pf->fdir_pf_filter_count = 11193 pf->hw.func_caps.fd_filters_guaranteed; 11194 pf->hw.fdir_shared_filter_count = 11195 pf->hw.func_caps.fd_filters_best_effort; 11196 } 11197 11198 if (pf->hw.mac.type == I40E_MAC_X722) { 11199 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 11200 I40E_HW_128_QP_RSS_CAPABLE | 11201 I40E_HW_ATR_EVICT_CAPABLE | 11202 I40E_HW_WB_ON_ITR_CAPABLE | 11203 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 11204 I40E_HW_NO_PCI_LINK_CHECK | 11205 I40E_HW_USE_SET_LLDP_MIB | 11206 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 11207 I40E_HW_PTP_L4_CAPABLE | 11208 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 11209 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 11210 11211 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 11212 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 11213 I40E_FDEVICT_PCTYPE_DEFAULT) { 11214 dev_warn(&pf->pdev->dev, 11215 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 11216 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 11217 } 11218 } else if ((pf->hw.aq.api_maj_ver > 1) || 11219 ((pf->hw.aq.api_maj_ver == 1) && 11220 (pf->hw.aq.api_min_ver > 4))) { 11221 /* Supported in FW API version higher than 1.4 */ 11222 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 11223 } 11224 11225 /* Enable HW ATR eviction if possible */ 11226 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 11227 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 11228 11229 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11230 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 11231 (pf->hw.aq.fw_maj_ver < 4))) { 11232 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 11233 /* No DCB support for FW < v4.33 */ 11234 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 11235 } 11236 11237 /* Disable FW LLDP if FW < v4.3 */ 11238 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11239 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 11240 (pf->hw.aq.fw_maj_ver < 4))) 11241 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 11242 11243 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 11244 if ((pf->hw.mac.type == I40E_MAC_XL710) && 11245 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 11246 (pf->hw.aq.fw_maj_ver >= 5))) 11247 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 11248 11249 /* Enable PTP L4 if FW > v6.0 */ 11250 if (pf->hw.mac.type == I40E_MAC_XL710 && 11251 pf->hw.aq.fw_maj_ver >= 6) 11252 pf->hw_features |= I40E_HW_PTP_L4_CAPABLE; 11253 11254 if (pf->hw.func_caps.vmdq && num_online_cpus() != 1) { 11255 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 11256 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 11257 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 11258 } 11259 11260 if (pf->hw.func_caps.iwarp && num_online_cpus() != 1) { 11261 pf->flags |= I40E_FLAG_IWARP_ENABLED; 11262 /* IWARP needs one extra vector for CQP just like MISC.*/ 11263 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 11264 } 11265 /* Stopping the FW LLDP engine is only supported on the 11266 * XL710 with a FW ver >= 1.7. Also, stopping FW LLDP 11267 * engine is not supported if NPAR is functioning on this 11268 * part 11269 */ 11270 if (pf->hw.mac.type == I40E_MAC_XL710 && 11271 !pf->hw.func_caps.npar_enable && 11272 (pf->hw.aq.api_maj_ver > 1 || 11273 (pf->hw.aq.api_maj_ver == 1 && pf->hw.aq.api_min_ver > 6))) 11274 pf->hw_features |= I40E_HW_STOPPABLE_FW_LLDP; 11275 11276 #ifdef CONFIG_PCI_IOV 11277 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 11278 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 11279 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 11280 pf->num_req_vfs = min_t(int, 11281 pf->hw.func_caps.num_vfs, 11282 I40E_MAX_VF_COUNT); 11283 } 11284 #endif /* CONFIG_PCI_IOV */ 11285 pf->eeprom_version = 0xDEAD; 11286 pf->lan_veb = I40E_NO_VEB; 11287 pf->lan_vsi = I40E_NO_VSI; 11288 11289 /* By default FW has this off for performance reasons */ 11290 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 11291 11292 /* set up queue assignment tracking */ 11293 size = sizeof(struct i40e_lump_tracking) 11294 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 11295 pf->qp_pile = kzalloc(size, GFP_KERNEL); 11296 if (!pf->qp_pile) { 11297 err = -ENOMEM; 11298 goto sw_init_done; 11299 } 11300 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 11301 pf->qp_pile->search_hint = 0; 11302 11303 pf->tx_timeout_recovery_level = 1; 11304 11305 mutex_init(&pf->switch_mutex); 11306 11307 sw_init_done: 11308 return err; 11309 } 11310 11311 /** 11312 * i40e_set_ntuple - set the ntuple feature flag and take action 11313 * @pf: board private structure to initialize 11314 * @features: the feature set that the stack is suggesting 11315 * 11316 * returns a bool to indicate if reset needs to happen 11317 **/ 11318 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 11319 { 11320 bool need_reset = false; 11321 11322 /* Check if Flow Director n-tuple support was enabled or disabled. If 11323 * the state changed, we need to reset. 11324 */ 11325 if (features & NETIF_F_NTUPLE) { 11326 /* Enable filters and mark for reset */ 11327 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 11328 need_reset = true; 11329 /* enable FD_SB only if there is MSI-X vector and no cloud 11330 * filters exist 11331 */ 11332 if (pf->num_fdsb_msix > 0 && !pf->num_cloud_filters) { 11333 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 11334 pf->flags &= ~I40E_FLAG_FD_SB_INACTIVE; 11335 } 11336 } else { 11337 /* turn off filters, mark for reset and clear SW filter list */ 11338 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11339 need_reset = true; 11340 i40e_fdir_filter_exit(pf); 11341 } 11342 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 11343 clear_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state); 11344 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 11345 11346 /* reset fd counters */ 11347 pf->fd_add_err = 0; 11348 pf->fd_atr_cnt = 0; 11349 /* if ATR was auto disabled it can be re-enabled. */ 11350 if (test_and_clear_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 11351 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 11352 (I40E_DEBUG_FD & pf->hw.debug_mask)) 11353 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 11354 } 11355 return need_reset; 11356 } 11357 11358 /** 11359 * i40e_clear_rss_lut - clear the rx hash lookup table 11360 * @vsi: the VSI being configured 11361 **/ 11362 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 11363 { 11364 struct i40e_pf *pf = vsi->back; 11365 struct i40e_hw *hw = &pf->hw; 11366 u16 vf_id = vsi->vf_id; 11367 u8 i; 11368 11369 if (vsi->type == I40E_VSI_MAIN) { 11370 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 11371 wr32(hw, I40E_PFQF_HLUT(i), 0); 11372 } else if (vsi->type == I40E_VSI_SRIOV) { 11373 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 11374 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 11375 } else { 11376 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 11377 } 11378 } 11379 11380 /** 11381 * i40e_set_features - set the netdev feature flags 11382 * @netdev: ptr to the netdev being adjusted 11383 * @features: the feature set that the stack is suggesting 11384 * Note: expects to be called while under rtnl_lock() 11385 **/ 11386 static int i40e_set_features(struct net_device *netdev, 11387 netdev_features_t features) 11388 { 11389 struct i40e_netdev_priv *np = netdev_priv(netdev); 11390 struct i40e_vsi *vsi = np->vsi; 11391 struct i40e_pf *pf = vsi->back; 11392 bool need_reset; 11393 11394 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 11395 i40e_pf_config_rss(pf); 11396 else if (!(features & NETIF_F_RXHASH) && 11397 netdev->features & NETIF_F_RXHASH) 11398 i40e_clear_rss_lut(vsi); 11399 11400 if (features & NETIF_F_HW_VLAN_CTAG_RX) 11401 i40e_vlan_stripping_enable(vsi); 11402 else 11403 i40e_vlan_stripping_disable(vsi); 11404 11405 if (!(features & NETIF_F_HW_TC) && pf->num_cloud_filters) { 11406 dev_err(&pf->pdev->dev, 11407 "Offloaded tc filters active, can't turn hw_tc_offload off"); 11408 return -EINVAL; 11409 } 11410 11411 need_reset = i40e_set_ntuple(pf, features); 11412 11413 if (need_reset) 11414 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 11415 11416 return 0; 11417 } 11418 11419 /** 11420 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 11421 * @pf: board private structure 11422 * @port: The UDP port to look up 11423 * 11424 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 11425 **/ 11426 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 11427 { 11428 u8 i; 11429 11430 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 11431 /* Do not report ports with pending deletions as 11432 * being available. 11433 */ 11434 if (!port && (pf->pending_udp_bitmap & BIT_ULL(i))) 11435 continue; 11436 if (pf->udp_ports[i].port == port) 11437 return i; 11438 } 11439 11440 return i; 11441 } 11442 11443 /** 11444 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 11445 * @netdev: This physical port's netdev 11446 * @ti: Tunnel endpoint information 11447 **/ 11448 static void i40e_udp_tunnel_add(struct net_device *netdev, 11449 struct udp_tunnel_info *ti) 11450 { 11451 struct i40e_netdev_priv *np = netdev_priv(netdev); 11452 struct i40e_vsi *vsi = np->vsi; 11453 struct i40e_pf *pf = vsi->back; 11454 u16 port = ntohs(ti->port); 11455 u8 next_idx; 11456 u8 idx; 11457 11458 idx = i40e_get_udp_port_idx(pf, port); 11459 11460 /* Check if port already exists */ 11461 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 11462 netdev_info(netdev, "port %d already offloaded\n", port); 11463 return; 11464 } 11465 11466 /* Now check if there is space to add the new port */ 11467 next_idx = i40e_get_udp_port_idx(pf, 0); 11468 11469 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 11470 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 11471 port); 11472 return; 11473 } 11474 11475 switch (ti->type) { 11476 case UDP_TUNNEL_TYPE_VXLAN: 11477 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 11478 break; 11479 case UDP_TUNNEL_TYPE_GENEVE: 11480 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) 11481 return; 11482 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 11483 break; 11484 default: 11485 return; 11486 } 11487 11488 /* New port: add it and mark its index in the bitmap */ 11489 pf->udp_ports[next_idx].port = port; 11490 pf->udp_ports[next_idx].filter_index = I40E_UDP_PORT_INDEX_UNUSED; 11491 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 11492 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 11493 } 11494 11495 /** 11496 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 11497 * @netdev: This physical port's netdev 11498 * @ti: Tunnel endpoint information 11499 **/ 11500 static void i40e_udp_tunnel_del(struct net_device *netdev, 11501 struct udp_tunnel_info *ti) 11502 { 11503 struct i40e_netdev_priv *np = netdev_priv(netdev); 11504 struct i40e_vsi *vsi = np->vsi; 11505 struct i40e_pf *pf = vsi->back; 11506 u16 port = ntohs(ti->port); 11507 u8 idx; 11508 11509 idx = i40e_get_udp_port_idx(pf, port); 11510 11511 /* Check if port already exists */ 11512 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 11513 goto not_found; 11514 11515 switch (ti->type) { 11516 case UDP_TUNNEL_TYPE_VXLAN: 11517 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 11518 goto not_found; 11519 break; 11520 case UDP_TUNNEL_TYPE_GENEVE: 11521 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 11522 goto not_found; 11523 break; 11524 default: 11525 goto not_found; 11526 } 11527 11528 /* if port exists, set it to 0 (mark for deletion) 11529 * and make it pending 11530 */ 11531 pf->udp_ports[idx].port = 0; 11532 11533 /* Toggle pending bit instead of setting it. This way if we are 11534 * deleting a port that has yet to be added we just clear the pending 11535 * bit and don't have to worry about it. 11536 */ 11537 pf->pending_udp_bitmap ^= BIT_ULL(idx); 11538 set_bit(__I40E_UDP_FILTER_SYNC_PENDING, pf->state); 11539 11540 return; 11541 not_found: 11542 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 11543 port); 11544 } 11545 11546 static int i40e_get_phys_port_id(struct net_device *netdev, 11547 struct netdev_phys_item_id *ppid) 11548 { 11549 struct i40e_netdev_priv *np = netdev_priv(netdev); 11550 struct i40e_pf *pf = np->vsi->back; 11551 struct i40e_hw *hw = &pf->hw; 11552 11553 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 11554 return -EOPNOTSUPP; 11555 11556 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 11557 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 11558 11559 return 0; 11560 } 11561 11562 /** 11563 * i40e_ndo_fdb_add - add an entry to the hardware database 11564 * @ndm: the input from the stack 11565 * @tb: pointer to array of nladdr (unused) 11566 * @dev: the net device pointer 11567 * @addr: the MAC address entry being added 11568 * @vid: VLAN ID 11569 * @flags: instructions from stack about fdb operation 11570 */ 11571 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 11572 struct net_device *dev, 11573 const unsigned char *addr, u16 vid, 11574 u16 flags) 11575 { 11576 struct i40e_netdev_priv *np = netdev_priv(dev); 11577 struct i40e_pf *pf = np->vsi->back; 11578 int err = 0; 11579 11580 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 11581 return -EOPNOTSUPP; 11582 11583 if (vid) { 11584 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 11585 return -EINVAL; 11586 } 11587 11588 /* Hardware does not support aging addresses so if a 11589 * ndm_state is given only allow permanent addresses 11590 */ 11591 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 11592 netdev_info(dev, "FDB only supports static addresses\n"); 11593 return -EINVAL; 11594 } 11595 11596 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 11597 err = dev_uc_add_excl(dev, addr); 11598 else if (is_multicast_ether_addr(addr)) 11599 err = dev_mc_add_excl(dev, addr); 11600 else 11601 err = -EINVAL; 11602 11603 /* Only return duplicate errors if NLM_F_EXCL is set */ 11604 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 11605 err = 0; 11606 11607 return err; 11608 } 11609 11610 /** 11611 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 11612 * @dev: the netdev being configured 11613 * @nlh: RTNL message 11614 * @flags: bridge flags 11615 * 11616 * Inserts a new hardware bridge if not already created and 11617 * enables the bridging mode requested (VEB or VEPA). If the 11618 * hardware bridge has already been inserted and the request 11619 * is to change the mode then that requires a PF reset to 11620 * allow rebuild of the components with required hardware 11621 * bridge mode enabled. 11622 * 11623 * Note: expects to be called while under rtnl_lock() 11624 **/ 11625 static int i40e_ndo_bridge_setlink(struct net_device *dev, 11626 struct nlmsghdr *nlh, 11627 u16 flags) 11628 { 11629 struct i40e_netdev_priv *np = netdev_priv(dev); 11630 struct i40e_vsi *vsi = np->vsi; 11631 struct i40e_pf *pf = vsi->back; 11632 struct i40e_veb *veb = NULL; 11633 struct nlattr *attr, *br_spec; 11634 int i, rem; 11635 11636 /* Only for PF VSI for now */ 11637 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 11638 return -EOPNOTSUPP; 11639 11640 /* Find the HW bridge for PF VSI */ 11641 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 11642 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 11643 veb = pf->veb[i]; 11644 } 11645 11646 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 11647 11648 nla_for_each_nested(attr, br_spec, rem) { 11649 __u16 mode; 11650 11651 if (nla_type(attr) != IFLA_BRIDGE_MODE) 11652 continue; 11653 11654 mode = nla_get_u16(attr); 11655 if ((mode != BRIDGE_MODE_VEPA) && 11656 (mode != BRIDGE_MODE_VEB)) 11657 return -EINVAL; 11658 11659 /* Insert a new HW bridge */ 11660 if (!veb) { 11661 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 11662 vsi->tc_config.enabled_tc); 11663 if (veb) { 11664 veb->bridge_mode = mode; 11665 i40e_config_bridge_mode(veb); 11666 } else { 11667 /* No Bridge HW offload available */ 11668 return -ENOENT; 11669 } 11670 break; 11671 } else if (mode != veb->bridge_mode) { 11672 /* Existing HW bridge but different mode needs reset */ 11673 veb->bridge_mode = mode; 11674 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 11675 if (mode == BRIDGE_MODE_VEB) 11676 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11677 else 11678 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 11679 i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); 11680 break; 11681 } 11682 } 11683 11684 return 0; 11685 } 11686 11687 /** 11688 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 11689 * @skb: skb buff 11690 * @pid: process id 11691 * @seq: RTNL message seq # 11692 * @dev: the netdev being configured 11693 * @filter_mask: unused 11694 * @nlflags: netlink flags passed in 11695 * 11696 * Return the mode in which the hardware bridge is operating in 11697 * i.e VEB or VEPA. 11698 **/ 11699 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 11700 struct net_device *dev, 11701 u32 __always_unused filter_mask, 11702 int nlflags) 11703 { 11704 struct i40e_netdev_priv *np = netdev_priv(dev); 11705 struct i40e_vsi *vsi = np->vsi; 11706 struct i40e_pf *pf = vsi->back; 11707 struct i40e_veb *veb = NULL; 11708 int i; 11709 11710 /* Only for PF VSI for now */ 11711 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 11712 return -EOPNOTSUPP; 11713 11714 /* Find the HW bridge for the PF VSI */ 11715 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 11716 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 11717 veb = pf->veb[i]; 11718 } 11719 11720 if (!veb) 11721 return 0; 11722 11723 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 11724 0, 0, nlflags, filter_mask, NULL); 11725 } 11726 11727 /** 11728 * i40e_features_check - Validate encapsulated packet conforms to limits 11729 * @skb: skb buff 11730 * @dev: This physical port's netdev 11731 * @features: Offload features that the stack believes apply 11732 **/ 11733 static netdev_features_t i40e_features_check(struct sk_buff *skb, 11734 struct net_device *dev, 11735 netdev_features_t features) 11736 { 11737 size_t len; 11738 11739 /* No point in doing any of this if neither checksum nor GSO are 11740 * being requested for this frame. We can rule out both by just 11741 * checking for CHECKSUM_PARTIAL 11742 */ 11743 if (skb->ip_summed != CHECKSUM_PARTIAL) 11744 return features; 11745 11746 /* We cannot support GSO if the MSS is going to be less than 11747 * 64 bytes. If it is then we need to drop support for GSO. 11748 */ 11749 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 11750 features &= ~NETIF_F_GSO_MASK; 11751 11752 /* MACLEN can support at most 63 words */ 11753 len = skb_network_header(skb) - skb->data; 11754 if (len & ~(63 * 2)) 11755 goto out_err; 11756 11757 /* IPLEN and EIPLEN can support at most 127 dwords */ 11758 len = skb_transport_header(skb) - skb_network_header(skb); 11759 if (len & ~(127 * 4)) 11760 goto out_err; 11761 11762 if (skb->encapsulation) { 11763 /* L4TUNLEN can support 127 words */ 11764 len = skb_inner_network_header(skb) - skb_transport_header(skb); 11765 if (len & ~(127 * 2)) 11766 goto out_err; 11767 11768 /* IPLEN can support at most 127 dwords */ 11769 len = skb_inner_transport_header(skb) - 11770 skb_inner_network_header(skb); 11771 if (len & ~(127 * 4)) 11772 goto out_err; 11773 } 11774 11775 /* No need to validate L4LEN as TCP is the only protocol with a 11776 * a flexible value and we support all possible values supported 11777 * by TCP, which is at most 15 dwords 11778 */ 11779 11780 return features; 11781 out_err: 11782 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 11783 } 11784 11785 /** 11786 * i40e_xdp_setup - add/remove an XDP program 11787 * @vsi: VSI to changed 11788 * @prog: XDP program 11789 **/ 11790 static int i40e_xdp_setup(struct i40e_vsi *vsi, 11791 struct bpf_prog *prog) 11792 { 11793 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 11794 struct i40e_pf *pf = vsi->back; 11795 struct bpf_prog *old_prog; 11796 bool need_reset; 11797 int i; 11798 11799 /* Don't allow frames that span over multiple buffers */ 11800 if (frame_size > vsi->rx_buf_len) 11801 return -EINVAL; 11802 11803 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 11804 return 0; 11805 11806 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 11807 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 11808 11809 if (need_reset) 11810 i40e_prep_for_reset(pf, true); 11811 11812 old_prog = xchg(&vsi->xdp_prog, prog); 11813 11814 if (need_reset) 11815 i40e_reset_and_rebuild(pf, true, true); 11816 11817 for (i = 0; i < vsi->num_queue_pairs; i++) 11818 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 11819 11820 if (old_prog) 11821 bpf_prog_put(old_prog); 11822 11823 return 0; 11824 } 11825 11826 /** 11827 * i40e_xdp - implements ndo_bpf for i40e 11828 * @dev: netdevice 11829 * @xdp: XDP command 11830 **/ 11831 static int i40e_xdp(struct net_device *dev, 11832 struct netdev_bpf *xdp) 11833 { 11834 struct i40e_netdev_priv *np = netdev_priv(dev); 11835 struct i40e_vsi *vsi = np->vsi; 11836 11837 if (vsi->type != I40E_VSI_MAIN) 11838 return -EINVAL; 11839 11840 switch (xdp->command) { 11841 case XDP_SETUP_PROG: 11842 return i40e_xdp_setup(vsi, xdp->prog); 11843 case XDP_QUERY_PROG: 11844 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi); 11845 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0; 11846 return 0; 11847 default: 11848 return -EINVAL; 11849 } 11850 } 11851 11852 static const struct net_device_ops i40e_netdev_ops = { 11853 .ndo_open = i40e_open, 11854 .ndo_stop = i40e_close, 11855 .ndo_start_xmit = i40e_lan_xmit_frame, 11856 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 11857 .ndo_set_rx_mode = i40e_set_rx_mode, 11858 .ndo_validate_addr = eth_validate_addr, 11859 .ndo_set_mac_address = i40e_set_mac, 11860 .ndo_change_mtu = i40e_change_mtu, 11861 .ndo_do_ioctl = i40e_ioctl, 11862 .ndo_tx_timeout = i40e_tx_timeout, 11863 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 11864 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 11865 #ifdef CONFIG_NET_POLL_CONTROLLER 11866 .ndo_poll_controller = i40e_netpoll, 11867 #endif 11868 .ndo_setup_tc = __i40e_setup_tc, 11869 .ndo_set_features = i40e_set_features, 11870 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 11871 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 11872 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 11873 .ndo_get_vf_config = i40e_ndo_get_vf_config, 11874 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 11875 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 11876 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 11877 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 11878 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 11879 .ndo_get_phys_port_id = i40e_get_phys_port_id, 11880 .ndo_fdb_add = i40e_ndo_fdb_add, 11881 .ndo_features_check = i40e_features_check, 11882 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 11883 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 11884 .ndo_bpf = i40e_xdp, 11885 .ndo_xdp_xmit = i40e_xdp_xmit, 11886 }; 11887 11888 /** 11889 * i40e_config_netdev - Setup the netdev flags 11890 * @vsi: the VSI being configured 11891 * 11892 * Returns 0 on success, negative value on failure 11893 **/ 11894 static int i40e_config_netdev(struct i40e_vsi *vsi) 11895 { 11896 struct i40e_pf *pf = vsi->back; 11897 struct i40e_hw *hw = &pf->hw; 11898 struct i40e_netdev_priv *np; 11899 struct net_device *netdev; 11900 u8 broadcast[ETH_ALEN]; 11901 u8 mac_addr[ETH_ALEN]; 11902 int etherdev_size; 11903 netdev_features_t hw_enc_features; 11904 netdev_features_t hw_features; 11905 11906 etherdev_size = sizeof(struct i40e_netdev_priv); 11907 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 11908 if (!netdev) 11909 return -ENOMEM; 11910 11911 vsi->netdev = netdev; 11912 np = netdev_priv(netdev); 11913 np->vsi = vsi; 11914 11915 hw_enc_features = NETIF_F_SG | 11916 NETIF_F_IP_CSUM | 11917 NETIF_F_IPV6_CSUM | 11918 NETIF_F_HIGHDMA | 11919 NETIF_F_SOFT_FEATURES | 11920 NETIF_F_TSO | 11921 NETIF_F_TSO_ECN | 11922 NETIF_F_TSO6 | 11923 NETIF_F_GSO_GRE | 11924 NETIF_F_GSO_GRE_CSUM | 11925 NETIF_F_GSO_PARTIAL | 11926 NETIF_F_GSO_UDP_TUNNEL | 11927 NETIF_F_GSO_UDP_TUNNEL_CSUM | 11928 NETIF_F_SCTP_CRC | 11929 NETIF_F_RXHASH | 11930 NETIF_F_RXCSUM | 11931 0; 11932 11933 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 11934 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 11935 11936 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 11937 11938 netdev->hw_enc_features |= hw_enc_features; 11939 11940 /* record features VLANs can make use of */ 11941 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 11942 11943 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 11944 netdev->hw_features |= NETIF_F_NTUPLE | NETIF_F_HW_TC; 11945 11946 hw_features = hw_enc_features | 11947 NETIF_F_HW_VLAN_CTAG_TX | 11948 NETIF_F_HW_VLAN_CTAG_RX; 11949 11950 netdev->hw_features |= hw_features; 11951 11952 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 11953 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 11954 11955 if (vsi->type == I40E_VSI_MAIN) { 11956 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 11957 ether_addr_copy(mac_addr, hw->mac.perm_addr); 11958 /* The following steps are necessary for two reasons. First, 11959 * some older NVM configurations load a default MAC-VLAN 11960 * filter that will accept any tagged packet, and we want to 11961 * replace this with a normal filter. Additionally, it is 11962 * possible our MAC address was provided by the platform using 11963 * Open Firmware or similar. 11964 * 11965 * Thus, we need to remove the default filter and install one 11966 * specific to the MAC address. 11967 */ 11968 i40e_rm_default_mac_filter(vsi, mac_addr); 11969 spin_lock_bh(&vsi->mac_filter_hash_lock); 11970 i40e_add_mac_filter(vsi, mac_addr); 11971 spin_unlock_bh(&vsi->mac_filter_hash_lock); 11972 } else { 11973 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 11974 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 11975 * the end, which is 4 bytes long, so force truncation of the 11976 * original name by IFNAMSIZ - 4 11977 */ 11978 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 11979 IFNAMSIZ - 4, 11980 pf->vsi[pf->lan_vsi]->netdev->name); 11981 random_ether_addr(mac_addr); 11982 11983 spin_lock_bh(&vsi->mac_filter_hash_lock); 11984 i40e_add_mac_filter(vsi, mac_addr); 11985 spin_unlock_bh(&vsi->mac_filter_hash_lock); 11986 } 11987 11988 /* Add the broadcast filter so that we initially will receive 11989 * broadcast packets. Note that when a new VLAN is first added the 11990 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 11991 * specific filters as part of transitioning into "vlan" operation. 11992 * When more VLANs are added, the driver will copy each existing MAC 11993 * filter and add it for the new VLAN. 11994 * 11995 * Broadcast filters are handled specially by 11996 * i40e_sync_filters_subtask, as the driver must to set the broadcast 11997 * promiscuous bit instead of adding this directly as a MAC/VLAN 11998 * filter. The subtask will update the correct broadcast promiscuous 11999 * bits as VLANs become active or inactive. 12000 */ 12001 eth_broadcast_addr(broadcast); 12002 spin_lock_bh(&vsi->mac_filter_hash_lock); 12003 i40e_add_mac_filter(vsi, broadcast); 12004 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12005 12006 ether_addr_copy(netdev->dev_addr, mac_addr); 12007 ether_addr_copy(netdev->perm_addr, mac_addr); 12008 12009 netdev->priv_flags |= IFF_UNICAST_FLT; 12010 netdev->priv_flags |= IFF_SUPP_NOFCS; 12011 /* Setup netdev TC information */ 12012 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 12013 12014 netdev->netdev_ops = &i40e_netdev_ops; 12015 netdev->watchdog_timeo = 5 * HZ; 12016 i40e_set_ethtool_ops(netdev); 12017 12018 /* MTU range: 68 - 9706 */ 12019 netdev->min_mtu = ETH_MIN_MTU; 12020 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 12021 12022 return 0; 12023 } 12024 12025 /** 12026 * i40e_vsi_delete - Delete a VSI from the switch 12027 * @vsi: the VSI being removed 12028 * 12029 * Returns 0 on success, negative value on failure 12030 **/ 12031 static void i40e_vsi_delete(struct i40e_vsi *vsi) 12032 { 12033 /* remove default VSI is not allowed */ 12034 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 12035 return; 12036 12037 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 12038 } 12039 12040 /** 12041 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 12042 * @vsi: the VSI being queried 12043 * 12044 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 12045 **/ 12046 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 12047 { 12048 struct i40e_veb *veb; 12049 struct i40e_pf *pf = vsi->back; 12050 12051 /* Uplink is not a bridge so default to VEB */ 12052 if (vsi->veb_idx == I40E_NO_VEB) 12053 return 1; 12054 12055 veb = pf->veb[vsi->veb_idx]; 12056 if (!veb) { 12057 dev_info(&pf->pdev->dev, 12058 "There is no veb associated with the bridge\n"); 12059 return -ENOENT; 12060 } 12061 12062 /* Uplink is a bridge in VEPA mode */ 12063 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 12064 return 0; 12065 } else { 12066 /* Uplink is a bridge in VEB mode */ 12067 return 1; 12068 } 12069 12070 /* VEPA is now default bridge, so return 0 */ 12071 return 0; 12072 } 12073 12074 /** 12075 * i40e_add_vsi - Add a VSI to the switch 12076 * @vsi: the VSI being configured 12077 * 12078 * This initializes a VSI context depending on the VSI type to be added and 12079 * passes it down to the add_vsi aq command. 12080 **/ 12081 static int i40e_add_vsi(struct i40e_vsi *vsi) 12082 { 12083 int ret = -ENODEV; 12084 struct i40e_pf *pf = vsi->back; 12085 struct i40e_hw *hw = &pf->hw; 12086 struct i40e_vsi_context ctxt; 12087 struct i40e_mac_filter *f; 12088 struct hlist_node *h; 12089 int bkt; 12090 12091 u8 enabled_tc = 0x1; /* TC0 enabled */ 12092 int f_count = 0; 12093 12094 memset(&ctxt, 0, sizeof(ctxt)); 12095 switch (vsi->type) { 12096 case I40E_VSI_MAIN: 12097 /* The PF's main VSI is already setup as part of the 12098 * device initialization, so we'll not bother with 12099 * the add_vsi call, but we will retrieve the current 12100 * VSI context. 12101 */ 12102 ctxt.seid = pf->main_vsi_seid; 12103 ctxt.pf_num = pf->hw.pf_id; 12104 ctxt.vf_num = 0; 12105 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 12106 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 12107 if (ret) { 12108 dev_info(&pf->pdev->dev, 12109 "couldn't get PF vsi config, err %s aq_err %s\n", 12110 i40e_stat_str(&pf->hw, ret), 12111 i40e_aq_str(&pf->hw, 12112 pf->hw.aq.asq_last_status)); 12113 return -ENOENT; 12114 } 12115 vsi->info = ctxt.info; 12116 vsi->info.valid_sections = 0; 12117 12118 vsi->seid = ctxt.seid; 12119 vsi->id = ctxt.vsi_number; 12120 12121 enabled_tc = i40e_pf_get_tc_map(pf); 12122 12123 /* Source pruning is enabled by default, so the flag is 12124 * negative logic - if it's set, we need to fiddle with 12125 * the VSI to disable source pruning. 12126 */ 12127 if (pf->flags & I40E_FLAG_SOURCE_PRUNING_DISABLED) { 12128 memset(&ctxt, 0, sizeof(ctxt)); 12129 ctxt.seid = pf->main_vsi_seid; 12130 ctxt.pf_num = pf->hw.pf_id; 12131 ctxt.vf_num = 0; 12132 ctxt.info.valid_sections |= 12133 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12134 ctxt.info.switch_id = 12135 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 12136 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 12137 if (ret) { 12138 dev_info(&pf->pdev->dev, 12139 "update vsi failed, err %s aq_err %s\n", 12140 i40e_stat_str(&pf->hw, ret), 12141 i40e_aq_str(&pf->hw, 12142 pf->hw.aq.asq_last_status)); 12143 ret = -ENOENT; 12144 goto err; 12145 } 12146 } 12147 12148 /* MFP mode setup queue map and update VSI */ 12149 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 12150 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 12151 memset(&ctxt, 0, sizeof(ctxt)); 12152 ctxt.seid = pf->main_vsi_seid; 12153 ctxt.pf_num = pf->hw.pf_id; 12154 ctxt.vf_num = 0; 12155 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 12156 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 12157 if (ret) { 12158 dev_info(&pf->pdev->dev, 12159 "update vsi failed, err %s aq_err %s\n", 12160 i40e_stat_str(&pf->hw, ret), 12161 i40e_aq_str(&pf->hw, 12162 pf->hw.aq.asq_last_status)); 12163 ret = -ENOENT; 12164 goto err; 12165 } 12166 /* update the local VSI info queue map */ 12167 i40e_vsi_update_queue_map(vsi, &ctxt); 12168 vsi->info.valid_sections = 0; 12169 } else { 12170 /* Default/Main VSI is only enabled for TC0 12171 * reconfigure it to enable all TCs that are 12172 * available on the port in SFP mode. 12173 * For MFP case the iSCSI PF would use this 12174 * flow to enable LAN+iSCSI TC. 12175 */ 12176 ret = i40e_vsi_config_tc(vsi, enabled_tc); 12177 if (ret) { 12178 /* Single TC condition is not fatal, 12179 * message and continue 12180 */ 12181 dev_info(&pf->pdev->dev, 12182 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 12183 enabled_tc, 12184 i40e_stat_str(&pf->hw, ret), 12185 i40e_aq_str(&pf->hw, 12186 pf->hw.aq.asq_last_status)); 12187 } 12188 } 12189 break; 12190 12191 case I40E_VSI_FDIR: 12192 ctxt.pf_num = hw->pf_id; 12193 ctxt.vf_num = 0; 12194 ctxt.uplink_seid = vsi->uplink_seid; 12195 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12196 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 12197 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 12198 (i40e_is_vsi_uplink_mode_veb(vsi))) { 12199 ctxt.info.valid_sections |= 12200 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12201 ctxt.info.switch_id = 12202 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12203 } 12204 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12205 break; 12206 12207 case I40E_VSI_VMDQ2: 12208 ctxt.pf_num = hw->pf_id; 12209 ctxt.vf_num = 0; 12210 ctxt.uplink_seid = vsi->uplink_seid; 12211 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12212 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 12213 12214 /* This VSI is connected to VEB so the switch_id 12215 * should be set to zero by default. 12216 */ 12217 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 12218 ctxt.info.valid_sections |= 12219 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12220 ctxt.info.switch_id = 12221 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12222 } 12223 12224 /* Setup the VSI tx/rx queue map for TC0 only for now */ 12225 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12226 break; 12227 12228 case I40E_VSI_SRIOV: 12229 ctxt.pf_num = hw->pf_id; 12230 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 12231 ctxt.uplink_seid = vsi->uplink_seid; 12232 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 12233 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 12234 12235 /* This VSI is connected to VEB so the switch_id 12236 * should be set to zero by default. 12237 */ 12238 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 12239 ctxt.info.valid_sections |= 12240 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 12241 ctxt.info.switch_id = 12242 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 12243 } 12244 12245 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 12246 ctxt.info.valid_sections |= 12247 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 12248 ctxt.info.queueing_opt_flags |= 12249 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 12250 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 12251 } 12252 12253 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 12254 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 12255 if (pf->vf[vsi->vf_id].spoofchk) { 12256 ctxt.info.valid_sections |= 12257 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 12258 ctxt.info.sec_flags |= 12259 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 12260 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 12261 } 12262 /* Setup the VSI tx/rx queue map for TC0 only for now */ 12263 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 12264 break; 12265 12266 case I40E_VSI_IWARP: 12267 /* send down message to iWARP */ 12268 break; 12269 12270 default: 12271 return -ENODEV; 12272 } 12273 12274 if (vsi->type != I40E_VSI_MAIN) { 12275 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 12276 if (ret) { 12277 dev_info(&vsi->back->pdev->dev, 12278 "add vsi failed, err %s aq_err %s\n", 12279 i40e_stat_str(&pf->hw, ret), 12280 i40e_aq_str(&pf->hw, 12281 pf->hw.aq.asq_last_status)); 12282 ret = -ENOENT; 12283 goto err; 12284 } 12285 vsi->info = ctxt.info; 12286 vsi->info.valid_sections = 0; 12287 vsi->seid = ctxt.seid; 12288 vsi->id = ctxt.vsi_number; 12289 } 12290 12291 vsi->active_filters = 0; 12292 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 12293 spin_lock_bh(&vsi->mac_filter_hash_lock); 12294 /* If macvlan filters already exist, force them to get loaded */ 12295 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 12296 f->state = I40E_FILTER_NEW; 12297 f_count++; 12298 } 12299 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12300 12301 if (f_count) { 12302 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 12303 set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); 12304 } 12305 12306 /* Update VSI BW information */ 12307 ret = i40e_vsi_get_bw_info(vsi); 12308 if (ret) { 12309 dev_info(&pf->pdev->dev, 12310 "couldn't get vsi bw info, err %s aq_err %s\n", 12311 i40e_stat_str(&pf->hw, ret), 12312 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12313 /* VSI is already added so not tearing that up */ 12314 ret = 0; 12315 } 12316 12317 err: 12318 return ret; 12319 } 12320 12321 /** 12322 * i40e_vsi_release - Delete a VSI and free its resources 12323 * @vsi: the VSI being removed 12324 * 12325 * Returns 0 on success or < 0 on error 12326 **/ 12327 int i40e_vsi_release(struct i40e_vsi *vsi) 12328 { 12329 struct i40e_mac_filter *f; 12330 struct hlist_node *h; 12331 struct i40e_veb *veb = NULL; 12332 struct i40e_pf *pf; 12333 u16 uplink_seid; 12334 int i, n, bkt; 12335 12336 pf = vsi->back; 12337 12338 /* release of a VEB-owner or last VSI is not allowed */ 12339 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 12340 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 12341 vsi->seid, vsi->uplink_seid); 12342 return -ENODEV; 12343 } 12344 if (vsi == pf->vsi[pf->lan_vsi] && 12345 !test_bit(__I40E_DOWN, pf->state)) { 12346 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 12347 return -ENODEV; 12348 } 12349 12350 uplink_seid = vsi->uplink_seid; 12351 if (vsi->type != I40E_VSI_SRIOV) { 12352 if (vsi->netdev_registered) { 12353 vsi->netdev_registered = false; 12354 if (vsi->netdev) { 12355 /* results in a call to i40e_close() */ 12356 unregister_netdev(vsi->netdev); 12357 } 12358 } else { 12359 i40e_vsi_close(vsi); 12360 } 12361 i40e_vsi_disable_irq(vsi); 12362 } 12363 12364 spin_lock_bh(&vsi->mac_filter_hash_lock); 12365 12366 /* clear the sync flag on all filters */ 12367 if (vsi->netdev) { 12368 __dev_uc_unsync(vsi->netdev, NULL); 12369 __dev_mc_unsync(vsi->netdev, NULL); 12370 } 12371 12372 /* make sure any remaining filters are marked for deletion */ 12373 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 12374 __i40e_del_filter(vsi, f); 12375 12376 spin_unlock_bh(&vsi->mac_filter_hash_lock); 12377 12378 i40e_sync_vsi_filters(vsi); 12379 12380 i40e_vsi_delete(vsi); 12381 i40e_vsi_free_q_vectors(vsi); 12382 if (vsi->netdev) { 12383 free_netdev(vsi->netdev); 12384 vsi->netdev = NULL; 12385 } 12386 i40e_vsi_clear_rings(vsi); 12387 i40e_vsi_clear(vsi); 12388 12389 /* If this was the last thing on the VEB, except for the 12390 * controlling VSI, remove the VEB, which puts the controlling 12391 * VSI onto the next level down in the switch. 12392 * 12393 * Well, okay, there's one more exception here: don't remove 12394 * the orphan VEBs yet. We'll wait for an explicit remove request 12395 * from up the network stack. 12396 */ 12397 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 12398 if (pf->vsi[i] && 12399 pf->vsi[i]->uplink_seid == uplink_seid && 12400 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 12401 n++; /* count the VSIs */ 12402 } 12403 } 12404 for (i = 0; i < I40E_MAX_VEB; i++) { 12405 if (!pf->veb[i]) 12406 continue; 12407 if (pf->veb[i]->uplink_seid == uplink_seid) 12408 n++; /* count the VEBs */ 12409 if (pf->veb[i]->seid == uplink_seid) 12410 veb = pf->veb[i]; 12411 } 12412 if (n == 0 && veb && veb->uplink_seid != 0) 12413 i40e_veb_release(veb); 12414 12415 return 0; 12416 } 12417 12418 /** 12419 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 12420 * @vsi: ptr to the VSI 12421 * 12422 * This should only be called after i40e_vsi_mem_alloc() which allocates the 12423 * corresponding SW VSI structure and initializes num_queue_pairs for the 12424 * newly allocated VSI. 12425 * 12426 * Returns 0 on success or negative on failure 12427 **/ 12428 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 12429 { 12430 int ret = -ENOENT; 12431 struct i40e_pf *pf = vsi->back; 12432 12433 if (vsi->q_vectors[0]) { 12434 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 12435 vsi->seid); 12436 return -EEXIST; 12437 } 12438 12439 if (vsi->base_vector) { 12440 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 12441 vsi->seid, vsi->base_vector); 12442 return -EEXIST; 12443 } 12444 12445 ret = i40e_vsi_alloc_q_vectors(vsi); 12446 if (ret) { 12447 dev_info(&pf->pdev->dev, 12448 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 12449 vsi->num_q_vectors, vsi->seid, ret); 12450 vsi->num_q_vectors = 0; 12451 goto vector_setup_out; 12452 } 12453 12454 /* In Legacy mode, we do not have to get any other vector since we 12455 * piggyback on the misc/ICR0 for queue interrupts. 12456 */ 12457 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 12458 return ret; 12459 if (vsi->num_q_vectors) 12460 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 12461 vsi->num_q_vectors, vsi->idx); 12462 if (vsi->base_vector < 0) { 12463 dev_info(&pf->pdev->dev, 12464 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 12465 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 12466 i40e_vsi_free_q_vectors(vsi); 12467 ret = -ENOENT; 12468 goto vector_setup_out; 12469 } 12470 12471 vector_setup_out: 12472 return ret; 12473 } 12474 12475 /** 12476 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 12477 * @vsi: pointer to the vsi. 12478 * 12479 * This re-allocates a vsi's queue resources. 12480 * 12481 * Returns pointer to the successfully allocated and configured VSI sw struct 12482 * on success, otherwise returns NULL on failure. 12483 **/ 12484 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 12485 { 12486 u16 alloc_queue_pairs; 12487 struct i40e_pf *pf; 12488 u8 enabled_tc; 12489 int ret; 12490 12491 if (!vsi) 12492 return NULL; 12493 12494 pf = vsi->back; 12495 12496 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 12497 i40e_vsi_clear_rings(vsi); 12498 12499 i40e_vsi_free_arrays(vsi, false); 12500 i40e_set_num_rings_in_vsi(vsi); 12501 ret = i40e_vsi_alloc_arrays(vsi, false); 12502 if (ret) 12503 goto err_vsi; 12504 12505 alloc_queue_pairs = vsi->alloc_queue_pairs * 12506 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 12507 12508 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 12509 if (ret < 0) { 12510 dev_info(&pf->pdev->dev, 12511 "failed to get tracking for %d queues for VSI %d err %d\n", 12512 alloc_queue_pairs, vsi->seid, ret); 12513 goto err_vsi; 12514 } 12515 vsi->base_queue = ret; 12516 12517 /* Update the FW view of the VSI. Force a reset of TC and queue 12518 * layout configurations. 12519 */ 12520 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 12521 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 12522 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 12523 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 12524 if (vsi->type == I40E_VSI_MAIN) 12525 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 12526 12527 /* assign it some queues */ 12528 ret = i40e_alloc_rings(vsi); 12529 if (ret) 12530 goto err_rings; 12531 12532 /* map all of the rings to the q_vectors */ 12533 i40e_vsi_map_rings_to_vectors(vsi); 12534 return vsi; 12535 12536 err_rings: 12537 i40e_vsi_free_q_vectors(vsi); 12538 if (vsi->netdev_registered) { 12539 vsi->netdev_registered = false; 12540 unregister_netdev(vsi->netdev); 12541 free_netdev(vsi->netdev); 12542 vsi->netdev = NULL; 12543 } 12544 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 12545 err_vsi: 12546 i40e_vsi_clear(vsi); 12547 return NULL; 12548 } 12549 12550 /** 12551 * i40e_vsi_setup - Set up a VSI by a given type 12552 * @pf: board private structure 12553 * @type: VSI type 12554 * @uplink_seid: the switch element to link to 12555 * @param1: usage depends upon VSI type. For VF types, indicates VF id 12556 * 12557 * This allocates the sw VSI structure and its queue resources, then add a VSI 12558 * to the identified VEB. 12559 * 12560 * Returns pointer to the successfully allocated and configure VSI sw struct on 12561 * success, otherwise returns NULL on failure. 12562 **/ 12563 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 12564 u16 uplink_seid, u32 param1) 12565 { 12566 struct i40e_vsi *vsi = NULL; 12567 struct i40e_veb *veb = NULL; 12568 u16 alloc_queue_pairs; 12569 int ret, i; 12570 int v_idx; 12571 12572 /* The requested uplink_seid must be either 12573 * - the PF's port seid 12574 * no VEB is needed because this is the PF 12575 * or this is a Flow Director special case VSI 12576 * - seid of an existing VEB 12577 * - seid of a VSI that owns an existing VEB 12578 * - seid of a VSI that doesn't own a VEB 12579 * a new VEB is created and the VSI becomes the owner 12580 * - seid of the PF VSI, which is what creates the first VEB 12581 * this is a special case of the previous 12582 * 12583 * Find which uplink_seid we were given and create a new VEB if needed 12584 */ 12585 for (i = 0; i < I40E_MAX_VEB; i++) { 12586 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 12587 veb = pf->veb[i]; 12588 break; 12589 } 12590 } 12591 12592 if (!veb && uplink_seid != pf->mac_seid) { 12593 12594 for (i = 0; i < pf->num_alloc_vsi; i++) { 12595 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 12596 vsi = pf->vsi[i]; 12597 break; 12598 } 12599 } 12600 if (!vsi) { 12601 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 12602 uplink_seid); 12603 return NULL; 12604 } 12605 12606 if (vsi->uplink_seid == pf->mac_seid) 12607 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 12608 vsi->tc_config.enabled_tc); 12609 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 12610 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 12611 vsi->tc_config.enabled_tc); 12612 if (veb) { 12613 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 12614 dev_info(&vsi->back->pdev->dev, 12615 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 12616 return NULL; 12617 } 12618 /* We come up by default in VEPA mode if SRIOV is not 12619 * already enabled, in which case we can't force VEPA 12620 * mode. 12621 */ 12622 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 12623 veb->bridge_mode = BRIDGE_MODE_VEPA; 12624 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 12625 } 12626 i40e_config_bridge_mode(veb); 12627 } 12628 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 12629 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 12630 veb = pf->veb[i]; 12631 } 12632 if (!veb) { 12633 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 12634 return NULL; 12635 } 12636 12637 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 12638 uplink_seid = veb->seid; 12639 } 12640 12641 /* get vsi sw struct */ 12642 v_idx = i40e_vsi_mem_alloc(pf, type); 12643 if (v_idx < 0) 12644 goto err_alloc; 12645 vsi = pf->vsi[v_idx]; 12646 if (!vsi) 12647 goto err_alloc; 12648 vsi->type = type; 12649 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 12650 12651 if (type == I40E_VSI_MAIN) 12652 pf->lan_vsi = v_idx; 12653 else if (type == I40E_VSI_SRIOV) 12654 vsi->vf_id = param1; 12655 /* assign it some queues */ 12656 alloc_queue_pairs = vsi->alloc_queue_pairs * 12657 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 12658 12659 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 12660 if (ret < 0) { 12661 dev_info(&pf->pdev->dev, 12662 "failed to get tracking for %d queues for VSI %d err=%d\n", 12663 alloc_queue_pairs, vsi->seid, ret); 12664 goto err_vsi; 12665 } 12666 vsi->base_queue = ret; 12667 12668 /* get a VSI from the hardware */ 12669 vsi->uplink_seid = uplink_seid; 12670 ret = i40e_add_vsi(vsi); 12671 if (ret) 12672 goto err_vsi; 12673 12674 switch (vsi->type) { 12675 /* setup the netdev if needed */ 12676 case I40E_VSI_MAIN: 12677 case I40E_VSI_VMDQ2: 12678 ret = i40e_config_netdev(vsi); 12679 if (ret) 12680 goto err_netdev; 12681 ret = register_netdev(vsi->netdev); 12682 if (ret) 12683 goto err_netdev; 12684 vsi->netdev_registered = true; 12685 netif_carrier_off(vsi->netdev); 12686 #ifdef CONFIG_I40E_DCB 12687 /* Setup DCB netlink interface */ 12688 i40e_dcbnl_setup(vsi); 12689 #endif /* CONFIG_I40E_DCB */ 12690 /* fall through */ 12691 12692 case I40E_VSI_FDIR: 12693 /* set up vectors and rings if needed */ 12694 ret = i40e_vsi_setup_vectors(vsi); 12695 if (ret) 12696 goto err_msix; 12697 12698 ret = i40e_alloc_rings(vsi); 12699 if (ret) 12700 goto err_rings; 12701 12702 /* map all of the rings to the q_vectors */ 12703 i40e_vsi_map_rings_to_vectors(vsi); 12704 12705 i40e_vsi_reset_stats(vsi); 12706 break; 12707 12708 default: 12709 /* no netdev or rings for the other VSI types */ 12710 break; 12711 } 12712 12713 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 12714 (vsi->type == I40E_VSI_VMDQ2)) { 12715 ret = i40e_vsi_config_rss(vsi); 12716 } 12717 return vsi; 12718 12719 err_rings: 12720 i40e_vsi_free_q_vectors(vsi); 12721 err_msix: 12722 if (vsi->netdev_registered) { 12723 vsi->netdev_registered = false; 12724 unregister_netdev(vsi->netdev); 12725 free_netdev(vsi->netdev); 12726 vsi->netdev = NULL; 12727 } 12728 err_netdev: 12729 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 12730 err_vsi: 12731 i40e_vsi_clear(vsi); 12732 err_alloc: 12733 return NULL; 12734 } 12735 12736 /** 12737 * i40e_veb_get_bw_info - Query VEB BW information 12738 * @veb: the veb to query 12739 * 12740 * Query the Tx scheduler BW configuration data for given VEB 12741 **/ 12742 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 12743 { 12744 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 12745 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 12746 struct i40e_pf *pf = veb->pf; 12747 struct i40e_hw *hw = &pf->hw; 12748 u32 tc_bw_max; 12749 int ret = 0; 12750 int i; 12751 12752 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 12753 &bw_data, NULL); 12754 if (ret) { 12755 dev_info(&pf->pdev->dev, 12756 "query veb bw config failed, err %s aq_err %s\n", 12757 i40e_stat_str(&pf->hw, ret), 12758 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 12759 goto out; 12760 } 12761 12762 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 12763 &ets_data, NULL); 12764 if (ret) { 12765 dev_info(&pf->pdev->dev, 12766 "query veb bw ets config failed, err %s aq_err %s\n", 12767 i40e_stat_str(&pf->hw, ret), 12768 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 12769 goto out; 12770 } 12771 12772 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 12773 veb->bw_max_quanta = ets_data.tc_bw_max; 12774 veb->is_abs_credits = bw_data.absolute_credits_enable; 12775 veb->enabled_tc = ets_data.tc_valid_bits; 12776 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 12777 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 12778 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 12779 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 12780 veb->bw_tc_limit_credits[i] = 12781 le16_to_cpu(bw_data.tc_bw_limits[i]); 12782 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 12783 } 12784 12785 out: 12786 return ret; 12787 } 12788 12789 /** 12790 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 12791 * @pf: board private structure 12792 * 12793 * On error: returns error code (negative) 12794 * On success: returns vsi index in PF (positive) 12795 **/ 12796 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 12797 { 12798 int ret = -ENOENT; 12799 struct i40e_veb *veb; 12800 int i; 12801 12802 /* Need to protect the allocation of switch elements at the PF level */ 12803 mutex_lock(&pf->switch_mutex); 12804 12805 /* VEB list may be fragmented if VEB creation/destruction has 12806 * been happening. We can afford to do a quick scan to look 12807 * for any free slots in the list. 12808 * 12809 * find next empty veb slot, looping back around if necessary 12810 */ 12811 i = 0; 12812 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 12813 i++; 12814 if (i >= I40E_MAX_VEB) { 12815 ret = -ENOMEM; 12816 goto err_alloc_veb; /* out of VEB slots! */ 12817 } 12818 12819 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 12820 if (!veb) { 12821 ret = -ENOMEM; 12822 goto err_alloc_veb; 12823 } 12824 veb->pf = pf; 12825 veb->idx = i; 12826 veb->enabled_tc = 1; 12827 12828 pf->veb[i] = veb; 12829 ret = i; 12830 err_alloc_veb: 12831 mutex_unlock(&pf->switch_mutex); 12832 return ret; 12833 } 12834 12835 /** 12836 * i40e_switch_branch_release - Delete a branch of the switch tree 12837 * @branch: where to start deleting 12838 * 12839 * This uses recursion to find the tips of the branch to be 12840 * removed, deleting until we get back to and can delete this VEB. 12841 **/ 12842 static void i40e_switch_branch_release(struct i40e_veb *branch) 12843 { 12844 struct i40e_pf *pf = branch->pf; 12845 u16 branch_seid = branch->seid; 12846 u16 veb_idx = branch->idx; 12847 int i; 12848 12849 /* release any VEBs on this VEB - RECURSION */ 12850 for (i = 0; i < I40E_MAX_VEB; i++) { 12851 if (!pf->veb[i]) 12852 continue; 12853 if (pf->veb[i]->uplink_seid == branch->seid) 12854 i40e_switch_branch_release(pf->veb[i]); 12855 } 12856 12857 /* Release the VSIs on this VEB, but not the owner VSI. 12858 * 12859 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 12860 * the VEB itself, so don't use (*branch) after this loop. 12861 */ 12862 for (i = 0; i < pf->num_alloc_vsi; i++) { 12863 if (!pf->vsi[i]) 12864 continue; 12865 if (pf->vsi[i]->uplink_seid == branch_seid && 12866 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 12867 i40e_vsi_release(pf->vsi[i]); 12868 } 12869 } 12870 12871 /* There's one corner case where the VEB might not have been 12872 * removed, so double check it here and remove it if needed. 12873 * This case happens if the veb was created from the debugfs 12874 * commands and no VSIs were added to it. 12875 */ 12876 if (pf->veb[veb_idx]) 12877 i40e_veb_release(pf->veb[veb_idx]); 12878 } 12879 12880 /** 12881 * i40e_veb_clear - remove veb struct 12882 * @veb: the veb to remove 12883 **/ 12884 static void i40e_veb_clear(struct i40e_veb *veb) 12885 { 12886 if (!veb) 12887 return; 12888 12889 if (veb->pf) { 12890 struct i40e_pf *pf = veb->pf; 12891 12892 mutex_lock(&pf->switch_mutex); 12893 if (pf->veb[veb->idx] == veb) 12894 pf->veb[veb->idx] = NULL; 12895 mutex_unlock(&pf->switch_mutex); 12896 } 12897 12898 kfree(veb); 12899 } 12900 12901 /** 12902 * i40e_veb_release - Delete a VEB and free its resources 12903 * @veb: the VEB being removed 12904 **/ 12905 void i40e_veb_release(struct i40e_veb *veb) 12906 { 12907 struct i40e_vsi *vsi = NULL; 12908 struct i40e_pf *pf; 12909 int i, n = 0; 12910 12911 pf = veb->pf; 12912 12913 /* find the remaining VSI and check for extras */ 12914 for (i = 0; i < pf->num_alloc_vsi; i++) { 12915 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 12916 n++; 12917 vsi = pf->vsi[i]; 12918 } 12919 } 12920 if (n != 1) { 12921 dev_info(&pf->pdev->dev, 12922 "can't remove VEB %d with %d VSIs left\n", 12923 veb->seid, n); 12924 return; 12925 } 12926 12927 /* move the remaining VSI to uplink veb */ 12928 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 12929 if (veb->uplink_seid) { 12930 vsi->uplink_seid = veb->uplink_seid; 12931 if (veb->uplink_seid == pf->mac_seid) 12932 vsi->veb_idx = I40E_NO_VEB; 12933 else 12934 vsi->veb_idx = veb->veb_idx; 12935 } else { 12936 /* floating VEB */ 12937 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 12938 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 12939 } 12940 12941 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 12942 i40e_veb_clear(veb); 12943 } 12944 12945 /** 12946 * i40e_add_veb - create the VEB in the switch 12947 * @veb: the VEB to be instantiated 12948 * @vsi: the controlling VSI 12949 **/ 12950 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 12951 { 12952 struct i40e_pf *pf = veb->pf; 12953 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 12954 int ret; 12955 12956 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 12957 veb->enabled_tc, false, 12958 &veb->seid, enable_stats, NULL); 12959 12960 /* get a VEB from the hardware */ 12961 if (ret) { 12962 dev_info(&pf->pdev->dev, 12963 "couldn't add VEB, err %s aq_err %s\n", 12964 i40e_stat_str(&pf->hw, ret), 12965 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12966 return -EPERM; 12967 } 12968 12969 /* get statistics counter */ 12970 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 12971 &veb->stats_idx, NULL, NULL, NULL); 12972 if (ret) { 12973 dev_info(&pf->pdev->dev, 12974 "couldn't get VEB statistics idx, err %s aq_err %s\n", 12975 i40e_stat_str(&pf->hw, ret), 12976 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12977 return -EPERM; 12978 } 12979 ret = i40e_veb_get_bw_info(veb); 12980 if (ret) { 12981 dev_info(&pf->pdev->dev, 12982 "couldn't get VEB bw info, err %s aq_err %s\n", 12983 i40e_stat_str(&pf->hw, ret), 12984 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 12985 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 12986 return -ENOENT; 12987 } 12988 12989 vsi->uplink_seid = veb->seid; 12990 vsi->veb_idx = veb->idx; 12991 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 12992 12993 return 0; 12994 } 12995 12996 /** 12997 * i40e_veb_setup - Set up a VEB 12998 * @pf: board private structure 12999 * @flags: VEB setup flags 13000 * @uplink_seid: the switch element to link to 13001 * @vsi_seid: the initial VSI seid 13002 * @enabled_tc: Enabled TC bit-map 13003 * 13004 * This allocates the sw VEB structure and links it into the switch 13005 * It is possible and legal for this to be a duplicate of an already 13006 * existing VEB. It is also possible for both uplink and vsi seids 13007 * to be zero, in order to create a floating VEB. 13008 * 13009 * Returns pointer to the successfully allocated VEB sw struct on 13010 * success, otherwise returns NULL on failure. 13011 **/ 13012 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 13013 u16 uplink_seid, u16 vsi_seid, 13014 u8 enabled_tc) 13015 { 13016 struct i40e_veb *veb, *uplink_veb = NULL; 13017 int vsi_idx, veb_idx; 13018 int ret; 13019 13020 /* if one seid is 0, the other must be 0 to create a floating relay */ 13021 if ((uplink_seid == 0 || vsi_seid == 0) && 13022 (uplink_seid + vsi_seid != 0)) { 13023 dev_info(&pf->pdev->dev, 13024 "one, not both seid's are 0: uplink=%d vsi=%d\n", 13025 uplink_seid, vsi_seid); 13026 return NULL; 13027 } 13028 13029 /* make sure there is such a vsi and uplink */ 13030 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 13031 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 13032 break; 13033 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 13034 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 13035 vsi_seid); 13036 return NULL; 13037 } 13038 13039 if (uplink_seid && uplink_seid != pf->mac_seid) { 13040 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 13041 if (pf->veb[veb_idx] && 13042 pf->veb[veb_idx]->seid == uplink_seid) { 13043 uplink_veb = pf->veb[veb_idx]; 13044 break; 13045 } 13046 } 13047 if (!uplink_veb) { 13048 dev_info(&pf->pdev->dev, 13049 "uplink seid %d not found\n", uplink_seid); 13050 return NULL; 13051 } 13052 } 13053 13054 /* get veb sw struct */ 13055 veb_idx = i40e_veb_mem_alloc(pf); 13056 if (veb_idx < 0) 13057 goto err_alloc; 13058 veb = pf->veb[veb_idx]; 13059 veb->flags = flags; 13060 veb->uplink_seid = uplink_seid; 13061 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 13062 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 13063 13064 /* create the VEB in the switch */ 13065 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 13066 if (ret) 13067 goto err_veb; 13068 if (vsi_idx == pf->lan_vsi) 13069 pf->lan_veb = veb->idx; 13070 13071 return veb; 13072 13073 err_veb: 13074 i40e_veb_clear(veb); 13075 err_alloc: 13076 return NULL; 13077 } 13078 13079 /** 13080 * i40e_setup_pf_switch_element - set PF vars based on switch type 13081 * @pf: board private structure 13082 * @ele: element we are building info from 13083 * @num_reported: total number of elements 13084 * @printconfig: should we print the contents 13085 * 13086 * helper function to assist in extracting a few useful SEID values. 13087 **/ 13088 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 13089 struct i40e_aqc_switch_config_element_resp *ele, 13090 u16 num_reported, bool printconfig) 13091 { 13092 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 13093 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 13094 u8 element_type = ele->element_type; 13095 u16 seid = le16_to_cpu(ele->seid); 13096 13097 if (printconfig) 13098 dev_info(&pf->pdev->dev, 13099 "type=%d seid=%d uplink=%d downlink=%d\n", 13100 element_type, seid, uplink_seid, downlink_seid); 13101 13102 switch (element_type) { 13103 case I40E_SWITCH_ELEMENT_TYPE_MAC: 13104 pf->mac_seid = seid; 13105 break; 13106 case I40E_SWITCH_ELEMENT_TYPE_VEB: 13107 /* Main VEB? */ 13108 if (uplink_seid != pf->mac_seid) 13109 break; 13110 if (pf->lan_veb == I40E_NO_VEB) { 13111 int v; 13112 13113 /* find existing or else empty VEB */ 13114 for (v = 0; v < I40E_MAX_VEB; v++) { 13115 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 13116 pf->lan_veb = v; 13117 break; 13118 } 13119 } 13120 if (pf->lan_veb == I40E_NO_VEB) { 13121 v = i40e_veb_mem_alloc(pf); 13122 if (v < 0) 13123 break; 13124 pf->lan_veb = v; 13125 } 13126 } 13127 13128 pf->veb[pf->lan_veb]->seid = seid; 13129 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 13130 pf->veb[pf->lan_veb]->pf = pf; 13131 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 13132 break; 13133 case I40E_SWITCH_ELEMENT_TYPE_VSI: 13134 if (num_reported != 1) 13135 break; 13136 /* This is immediately after a reset so we can assume this is 13137 * the PF's VSI 13138 */ 13139 pf->mac_seid = uplink_seid; 13140 pf->pf_seid = downlink_seid; 13141 pf->main_vsi_seid = seid; 13142 if (printconfig) 13143 dev_info(&pf->pdev->dev, 13144 "pf_seid=%d main_vsi_seid=%d\n", 13145 pf->pf_seid, pf->main_vsi_seid); 13146 break; 13147 case I40E_SWITCH_ELEMENT_TYPE_PF: 13148 case I40E_SWITCH_ELEMENT_TYPE_VF: 13149 case I40E_SWITCH_ELEMENT_TYPE_EMP: 13150 case I40E_SWITCH_ELEMENT_TYPE_BMC: 13151 case I40E_SWITCH_ELEMENT_TYPE_PE: 13152 case I40E_SWITCH_ELEMENT_TYPE_PA: 13153 /* ignore these for now */ 13154 break; 13155 default: 13156 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 13157 element_type, seid); 13158 break; 13159 } 13160 } 13161 13162 /** 13163 * i40e_fetch_switch_configuration - Get switch config from firmware 13164 * @pf: board private structure 13165 * @printconfig: should we print the contents 13166 * 13167 * Get the current switch configuration from the device and 13168 * extract a few useful SEID values. 13169 **/ 13170 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 13171 { 13172 struct i40e_aqc_get_switch_config_resp *sw_config; 13173 u16 next_seid = 0; 13174 int ret = 0; 13175 u8 *aq_buf; 13176 int i; 13177 13178 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 13179 if (!aq_buf) 13180 return -ENOMEM; 13181 13182 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 13183 do { 13184 u16 num_reported, num_total; 13185 13186 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 13187 I40E_AQ_LARGE_BUF, 13188 &next_seid, NULL); 13189 if (ret) { 13190 dev_info(&pf->pdev->dev, 13191 "get switch config failed err %s aq_err %s\n", 13192 i40e_stat_str(&pf->hw, ret), 13193 i40e_aq_str(&pf->hw, 13194 pf->hw.aq.asq_last_status)); 13195 kfree(aq_buf); 13196 return -ENOENT; 13197 } 13198 13199 num_reported = le16_to_cpu(sw_config->header.num_reported); 13200 num_total = le16_to_cpu(sw_config->header.num_total); 13201 13202 if (printconfig) 13203 dev_info(&pf->pdev->dev, 13204 "header: %d reported %d total\n", 13205 num_reported, num_total); 13206 13207 for (i = 0; i < num_reported; i++) { 13208 struct i40e_aqc_switch_config_element_resp *ele = 13209 &sw_config->element[i]; 13210 13211 i40e_setup_pf_switch_element(pf, ele, num_reported, 13212 printconfig); 13213 } 13214 } while (next_seid != 0); 13215 13216 kfree(aq_buf); 13217 return ret; 13218 } 13219 13220 /** 13221 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 13222 * @pf: board private structure 13223 * @reinit: if the Main VSI needs to re-initialized. 13224 * 13225 * Returns 0 on success, negative value on failure 13226 **/ 13227 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 13228 { 13229 u16 flags = 0; 13230 int ret; 13231 13232 /* find out what's out there already */ 13233 ret = i40e_fetch_switch_configuration(pf, false); 13234 if (ret) { 13235 dev_info(&pf->pdev->dev, 13236 "couldn't fetch switch config, err %s aq_err %s\n", 13237 i40e_stat_str(&pf->hw, ret), 13238 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13239 return ret; 13240 } 13241 i40e_pf_reset_stats(pf); 13242 13243 /* set the switch config bit for the whole device to 13244 * support limited promisc or true promisc 13245 * when user requests promisc. The default is limited 13246 * promisc. 13247 */ 13248 13249 if ((pf->hw.pf_id == 0) && 13250 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) { 13251 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 13252 pf->last_sw_conf_flags = flags; 13253 } 13254 13255 if (pf->hw.pf_id == 0) { 13256 u16 valid_flags; 13257 13258 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 13259 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 0, 13260 NULL); 13261 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 13262 dev_info(&pf->pdev->dev, 13263 "couldn't set switch config bits, err %s aq_err %s\n", 13264 i40e_stat_str(&pf->hw, ret), 13265 i40e_aq_str(&pf->hw, 13266 pf->hw.aq.asq_last_status)); 13267 /* not a fatal problem, just keep going */ 13268 } 13269 pf->last_sw_conf_valid_flags = valid_flags; 13270 } 13271 13272 /* first time setup */ 13273 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 13274 struct i40e_vsi *vsi = NULL; 13275 u16 uplink_seid; 13276 13277 /* Set up the PF VSI associated with the PF's main VSI 13278 * that is already in the HW switch 13279 */ 13280 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 13281 uplink_seid = pf->veb[pf->lan_veb]->seid; 13282 else 13283 uplink_seid = pf->mac_seid; 13284 if (pf->lan_vsi == I40E_NO_VSI) 13285 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 13286 else if (reinit) 13287 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 13288 if (!vsi) { 13289 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 13290 i40e_cloud_filter_exit(pf); 13291 i40e_fdir_teardown(pf); 13292 return -EAGAIN; 13293 } 13294 } else { 13295 /* force a reset of TC and queue layout configurations */ 13296 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 13297 13298 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 13299 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 13300 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 13301 } 13302 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 13303 13304 i40e_fdir_sb_setup(pf); 13305 13306 /* Setup static PF queue filter control settings */ 13307 ret = i40e_setup_pf_filter_control(pf); 13308 if (ret) { 13309 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 13310 ret); 13311 /* Failure here should not stop continuing other steps */ 13312 } 13313 13314 /* enable RSS in the HW, even for only one queue, as the stack can use 13315 * the hash 13316 */ 13317 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 13318 i40e_pf_config_rss(pf); 13319 13320 /* fill in link information and enable LSE reporting */ 13321 i40e_link_event(pf); 13322 13323 /* Initialize user-specific link properties */ 13324 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 13325 I40E_AQ_AN_COMPLETED) ? true : false); 13326 13327 i40e_ptp_init(pf); 13328 13329 /* repopulate tunnel port filters */ 13330 i40e_sync_udp_filters(pf); 13331 13332 return ret; 13333 } 13334 13335 /** 13336 * i40e_determine_queue_usage - Work out queue distribution 13337 * @pf: board private structure 13338 **/ 13339 static void i40e_determine_queue_usage(struct i40e_pf *pf) 13340 { 13341 int queues_left; 13342 int q_max; 13343 13344 pf->num_lan_qps = 0; 13345 13346 /* Find the max queues to be put into basic use. We'll always be 13347 * using TC0, whether or not DCB is running, and TC0 will get the 13348 * big RSS set. 13349 */ 13350 queues_left = pf->hw.func_caps.num_tx_qp; 13351 13352 if ((queues_left == 1) || 13353 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 13354 /* one qp for PF, no queues for anything else */ 13355 queues_left = 0; 13356 pf->alloc_rss_size = pf->num_lan_qps = 1; 13357 13358 /* make sure all the fancies are disabled */ 13359 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 13360 I40E_FLAG_IWARP_ENABLED | 13361 I40E_FLAG_FD_SB_ENABLED | 13362 I40E_FLAG_FD_ATR_ENABLED | 13363 I40E_FLAG_DCB_CAPABLE | 13364 I40E_FLAG_DCB_ENABLED | 13365 I40E_FLAG_SRIOV_ENABLED | 13366 I40E_FLAG_VMDQ_ENABLED); 13367 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13368 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 13369 I40E_FLAG_FD_SB_ENABLED | 13370 I40E_FLAG_FD_ATR_ENABLED | 13371 I40E_FLAG_DCB_CAPABLE))) { 13372 /* one qp for PF */ 13373 pf->alloc_rss_size = pf->num_lan_qps = 1; 13374 queues_left -= pf->num_lan_qps; 13375 13376 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 13377 I40E_FLAG_IWARP_ENABLED | 13378 I40E_FLAG_FD_SB_ENABLED | 13379 I40E_FLAG_FD_ATR_ENABLED | 13380 I40E_FLAG_DCB_ENABLED | 13381 I40E_FLAG_VMDQ_ENABLED); 13382 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13383 } else { 13384 /* Not enough queues for all TCs */ 13385 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 13386 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 13387 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 13388 I40E_FLAG_DCB_ENABLED); 13389 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 13390 } 13391 13392 /* limit lan qps to the smaller of qps, cpus or msix */ 13393 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 13394 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 13395 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 13396 pf->num_lan_qps = q_max; 13397 13398 queues_left -= pf->num_lan_qps; 13399 } 13400 13401 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 13402 if (queues_left > 1) { 13403 queues_left -= 1; /* save 1 queue for FD */ 13404 } else { 13405 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 13406 pf->flags |= I40E_FLAG_FD_SB_INACTIVE; 13407 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 13408 } 13409 } 13410 13411 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13412 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 13413 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 13414 (queues_left / pf->num_vf_qps)); 13415 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 13416 } 13417 13418 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 13419 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 13420 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 13421 (queues_left / pf->num_vmdq_qps)); 13422 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 13423 } 13424 13425 pf->queues_left = queues_left; 13426 dev_dbg(&pf->pdev->dev, 13427 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 13428 pf->hw.func_caps.num_tx_qp, 13429 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 13430 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 13431 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 13432 queues_left); 13433 } 13434 13435 /** 13436 * i40e_setup_pf_filter_control - Setup PF static filter control 13437 * @pf: PF to be setup 13438 * 13439 * i40e_setup_pf_filter_control sets up a PF's initial filter control 13440 * settings. If PE/FCoE are enabled then it will also set the per PF 13441 * based filter sizes required for them. It also enables Flow director, 13442 * ethertype and macvlan type filter settings for the pf. 13443 * 13444 * Returns 0 on success, negative on failure 13445 **/ 13446 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 13447 { 13448 struct i40e_filter_control_settings *settings = &pf->filter_settings; 13449 13450 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 13451 13452 /* Flow Director is enabled */ 13453 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 13454 settings->enable_fdir = true; 13455 13456 /* Ethtype and MACVLAN filters enabled for PF */ 13457 settings->enable_ethtype = true; 13458 settings->enable_macvlan = true; 13459 13460 if (i40e_set_filter_control(&pf->hw, settings)) 13461 return -ENOENT; 13462 13463 return 0; 13464 } 13465 13466 #define INFO_STRING_LEN 255 13467 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 13468 static void i40e_print_features(struct i40e_pf *pf) 13469 { 13470 struct i40e_hw *hw = &pf->hw; 13471 char *buf; 13472 int i; 13473 13474 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 13475 if (!buf) 13476 return; 13477 13478 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 13479 #ifdef CONFIG_PCI_IOV 13480 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 13481 #endif 13482 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 13483 pf->hw.func_caps.num_vsis, 13484 pf->vsi[pf->lan_vsi]->num_queue_pairs); 13485 if (pf->flags & I40E_FLAG_RSS_ENABLED) 13486 i += snprintf(&buf[i], REMAIN(i), " RSS"); 13487 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 13488 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 13489 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 13490 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 13491 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 13492 } 13493 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 13494 i += snprintf(&buf[i], REMAIN(i), " DCB"); 13495 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 13496 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 13497 if (pf->flags & I40E_FLAG_PTP) 13498 i += snprintf(&buf[i], REMAIN(i), " PTP"); 13499 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 13500 i += snprintf(&buf[i], REMAIN(i), " VEB"); 13501 else 13502 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 13503 13504 dev_info(&pf->pdev->dev, "%s\n", buf); 13505 kfree(buf); 13506 WARN_ON(i > INFO_STRING_LEN); 13507 } 13508 13509 /** 13510 * i40e_get_platform_mac_addr - get platform-specific MAC address 13511 * @pdev: PCI device information struct 13512 * @pf: board private structure 13513 * 13514 * Look up the MAC address for the device. First we'll try 13515 * eth_platform_get_mac_address, which will check Open Firmware, or arch 13516 * specific fallback. Otherwise, we'll default to the stored value in 13517 * firmware. 13518 **/ 13519 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 13520 { 13521 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 13522 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 13523 } 13524 13525 /** 13526 * i40e_probe - Device initialization routine 13527 * @pdev: PCI device information struct 13528 * @ent: entry in i40e_pci_tbl 13529 * 13530 * i40e_probe initializes a PF identified by a pci_dev structure. 13531 * The OS initialization, configuring of the PF private structure, 13532 * and a hardware reset occur. 13533 * 13534 * Returns 0 on success, negative on failure 13535 **/ 13536 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 13537 { 13538 struct i40e_aq_get_phy_abilities_resp abilities; 13539 struct i40e_pf *pf; 13540 struct i40e_hw *hw; 13541 static u16 pfs_found; 13542 u16 wol_nvm_bits; 13543 u16 link_status; 13544 int err; 13545 u32 val; 13546 u32 i; 13547 u8 set_fc_aq_fail; 13548 13549 err = pci_enable_device_mem(pdev); 13550 if (err) 13551 return err; 13552 13553 /* set up for high or low dma */ 13554 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 13555 if (err) { 13556 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 13557 if (err) { 13558 dev_err(&pdev->dev, 13559 "DMA configuration failed: 0x%x\n", err); 13560 goto err_dma; 13561 } 13562 } 13563 13564 /* set up pci connections */ 13565 err = pci_request_mem_regions(pdev, i40e_driver_name); 13566 if (err) { 13567 dev_info(&pdev->dev, 13568 "pci_request_selected_regions failed %d\n", err); 13569 goto err_pci_reg; 13570 } 13571 13572 pci_enable_pcie_error_reporting(pdev); 13573 pci_set_master(pdev); 13574 13575 /* Now that we have a PCI connection, we need to do the 13576 * low level device setup. This is primarily setting up 13577 * the Admin Queue structures and then querying for the 13578 * device's current profile information. 13579 */ 13580 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 13581 if (!pf) { 13582 err = -ENOMEM; 13583 goto err_pf_alloc; 13584 } 13585 pf->next_vsi = 0; 13586 pf->pdev = pdev; 13587 set_bit(__I40E_DOWN, pf->state); 13588 13589 hw = &pf->hw; 13590 hw->back = pf; 13591 13592 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 13593 I40E_MAX_CSR_SPACE); 13594 13595 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 13596 if (!hw->hw_addr) { 13597 err = -EIO; 13598 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 13599 (unsigned int)pci_resource_start(pdev, 0), 13600 pf->ioremap_len, err); 13601 goto err_ioremap; 13602 } 13603 hw->vendor_id = pdev->vendor; 13604 hw->device_id = pdev->device; 13605 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 13606 hw->subsystem_vendor_id = pdev->subsystem_vendor; 13607 hw->subsystem_device_id = pdev->subsystem_device; 13608 hw->bus.device = PCI_SLOT(pdev->devfn); 13609 hw->bus.func = PCI_FUNC(pdev->devfn); 13610 hw->bus.bus_id = pdev->bus->number; 13611 pf->instance = pfs_found; 13612 13613 /* Select something other than the 802.1ad ethertype for the 13614 * switch to use internally and drop on ingress. 13615 */ 13616 hw->switch_tag = 0xffff; 13617 hw->first_tag = ETH_P_8021AD; 13618 hw->second_tag = ETH_P_8021Q; 13619 13620 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 13621 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 13622 13623 /* set up the locks for the AQ, do this only once in probe 13624 * and destroy them only once in remove 13625 */ 13626 mutex_init(&hw->aq.asq_mutex); 13627 mutex_init(&hw->aq.arq_mutex); 13628 13629 pf->msg_enable = netif_msg_init(debug, 13630 NETIF_MSG_DRV | 13631 NETIF_MSG_PROBE | 13632 NETIF_MSG_LINK); 13633 if (debug < -1) 13634 pf->hw.debug_mask = debug; 13635 13636 /* do a special CORER for clearing PXE mode once at init */ 13637 if (hw->revision_id == 0 && 13638 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 13639 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 13640 i40e_flush(hw); 13641 msleep(200); 13642 pf->corer_count++; 13643 13644 i40e_clear_pxe_mode(hw); 13645 } 13646 13647 /* Reset here to make sure all is clean and to define PF 'n' */ 13648 i40e_clear_hw(hw); 13649 err = i40e_pf_reset(hw); 13650 if (err) { 13651 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 13652 goto err_pf_reset; 13653 } 13654 pf->pfr_count++; 13655 13656 hw->aq.num_arq_entries = I40E_AQ_LEN; 13657 hw->aq.num_asq_entries = I40E_AQ_LEN; 13658 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 13659 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 13660 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 13661 13662 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 13663 "%s-%s:misc", 13664 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 13665 13666 err = i40e_init_shared_code(hw); 13667 if (err) { 13668 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 13669 err); 13670 goto err_pf_reset; 13671 } 13672 13673 /* set up a default setting for link flow control */ 13674 pf->hw.fc.requested_mode = I40E_FC_NONE; 13675 13676 err = i40e_init_adminq(hw); 13677 if (err) { 13678 if (err == I40E_ERR_FIRMWARE_API_VERSION) 13679 dev_info(&pdev->dev, 13680 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 13681 else 13682 dev_info(&pdev->dev, 13683 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 13684 13685 goto err_pf_reset; 13686 } 13687 i40e_get_oem_version(hw); 13688 13689 /* provide nvm, fw, api versions */ 13690 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 13691 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 13692 hw->aq.api_maj_ver, hw->aq.api_min_ver, 13693 i40e_nvm_version_str(hw)); 13694 13695 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 13696 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 13697 dev_info(&pdev->dev, 13698 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 13699 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 13700 dev_info(&pdev->dev, 13701 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 13702 13703 i40e_verify_eeprom(pf); 13704 13705 /* Rev 0 hardware was never productized */ 13706 if (hw->revision_id < 1) 13707 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 13708 13709 i40e_clear_pxe_mode(hw); 13710 err = i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities); 13711 if (err) 13712 goto err_adminq_setup; 13713 13714 err = i40e_sw_init(pf); 13715 if (err) { 13716 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 13717 goto err_sw_init; 13718 } 13719 13720 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 13721 hw->func_caps.num_rx_qp, 0, 0); 13722 if (err) { 13723 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 13724 goto err_init_lan_hmc; 13725 } 13726 13727 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 13728 if (err) { 13729 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 13730 err = -ENOENT; 13731 goto err_configure_lan_hmc; 13732 } 13733 13734 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 13735 * Ignore error return codes because if it was already disabled via 13736 * hardware settings this will fail 13737 */ 13738 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 13739 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 13740 i40e_aq_stop_lldp(hw, true, NULL); 13741 } 13742 13743 /* allow a platform config to override the HW addr */ 13744 i40e_get_platform_mac_addr(pdev, pf); 13745 13746 if (!is_valid_ether_addr(hw->mac.addr)) { 13747 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 13748 err = -EIO; 13749 goto err_mac_addr; 13750 } 13751 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 13752 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 13753 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 13754 if (is_valid_ether_addr(hw->mac.port_addr)) 13755 pf->hw_features |= I40E_HW_PORT_ID_VALID; 13756 13757 pci_set_drvdata(pdev, pf); 13758 pci_save_state(pdev); 13759 13760 /* Enable FW to write default DCB config on link-up */ 13761 i40e_aq_set_dcb_parameters(hw, true, NULL); 13762 13763 #ifdef CONFIG_I40E_DCB 13764 err = i40e_init_pf_dcb(pf); 13765 if (err) { 13766 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 13767 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 13768 /* Continue without DCB enabled */ 13769 } 13770 #endif /* CONFIG_I40E_DCB */ 13771 13772 /* set up periodic task facility */ 13773 timer_setup(&pf->service_timer, i40e_service_timer, 0); 13774 pf->service_timer_period = HZ; 13775 13776 INIT_WORK(&pf->service_task, i40e_service_task); 13777 clear_bit(__I40E_SERVICE_SCHED, pf->state); 13778 13779 /* NVM bit on means WoL disabled for the port */ 13780 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 13781 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 13782 pf->wol_en = false; 13783 else 13784 pf->wol_en = true; 13785 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 13786 13787 /* set up the main switch operations */ 13788 i40e_determine_queue_usage(pf); 13789 err = i40e_init_interrupt_scheme(pf); 13790 if (err) 13791 goto err_switch_setup; 13792 13793 /* The number of VSIs reported by the FW is the minimum guaranteed 13794 * to us; HW supports far more and we share the remaining pool with 13795 * the other PFs. We allocate space for more than the guarantee with 13796 * the understanding that we might not get them all later. 13797 */ 13798 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 13799 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 13800 else 13801 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 13802 13803 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 13804 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 13805 GFP_KERNEL); 13806 if (!pf->vsi) { 13807 err = -ENOMEM; 13808 goto err_switch_setup; 13809 } 13810 13811 #ifdef CONFIG_PCI_IOV 13812 /* prep for VF support */ 13813 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13814 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 13815 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 13816 if (pci_num_vf(pdev)) 13817 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 13818 } 13819 #endif 13820 err = i40e_setup_pf_switch(pf, false); 13821 if (err) { 13822 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 13823 goto err_vsis; 13824 } 13825 INIT_LIST_HEAD(&pf->vsi[pf->lan_vsi]->ch_list); 13826 13827 /* Make sure flow control is set according to current settings */ 13828 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 13829 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 13830 dev_dbg(&pf->pdev->dev, 13831 "Set fc with err %s aq_err %s on get_phy_cap\n", 13832 i40e_stat_str(hw, err), 13833 i40e_aq_str(hw, hw->aq.asq_last_status)); 13834 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 13835 dev_dbg(&pf->pdev->dev, 13836 "Set fc with err %s aq_err %s on set_phy_config\n", 13837 i40e_stat_str(hw, err), 13838 i40e_aq_str(hw, hw->aq.asq_last_status)); 13839 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 13840 dev_dbg(&pf->pdev->dev, 13841 "Set fc with err %s aq_err %s on get_link_info\n", 13842 i40e_stat_str(hw, err), 13843 i40e_aq_str(hw, hw->aq.asq_last_status)); 13844 13845 /* if FDIR VSI was set up, start it now */ 13846 for (i = 0; i < pf->num_alloc_vsi; i++) { 13847 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 13848 i40e_vsi_open(pf->vsi[i]); 13849 break; 13850 } 13851 } 13852 13853 /* The driver only wants link up/down and module qualification 13854 * reports from firmware. Note the negative logic. 13855 */ 13856 err = i40e_aq_set_phy_int_mask(&pf->hw, 13857 ~(I40E_AQ_EVENT_LINK_UPDOWN | 13858 I40E_AQ_EVENT_MEDIA_NA | 13859 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 13860 if (err) 13861 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 13862 i40e_stat_str(&pf->hw, err), 13863 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 13864 13865 /* Reconfigure hardware for allowing smaller MSS in the case 13866 * of TSO, so that we avoid the MDD being fired and causing 13867 * a reset in the case of small MSS+TSO. 13868 */ 13869 val = rd32(hw, I40E_REG_MSS); 13870 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 13871 val &= ~I40E_REG_MSS_MIN_MASK; 13872 val |= I40E_64BYTE_MSS; 13873 wr32(hw, I40E_REG_MSS, val); 13874 } 13875 13876 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 13877 msleep(75); 13878 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 13879 if (err) 13880 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 13881 i40e_stat_str(&pf->hw, err), 13882 i40e_aq_str(&pf->hw, 13883 pf->hw.aq.asq_last_status)); 13884 } 13885 /* The main driver is (mostly) up and happy. We need to set this state 13886 * before setting up the misc vector or we get a race and the vector 13887 * ends up disabled forever. 13888 */ 13889 clear_bit(__I40E_DOWN, pf->state); 13890 13891 /* In case of MSIX we are going to setup the misc vector right here 13892 * to handle admin queue events etc. In case of legacy and MSI 13893 * the misc functionality and queue processing is combined in 13894 * the same vector and that gets setup at open. 13895 */ 13896 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 13897 err = i40e_setup_misc_vector(pf); 13898 if (err) { 13899 dev_info(&pdev->dev, 13900 "setup of misc vector failed: %d\n", err); 13901 goto err_vsis; 13902 } 13903 } 13904 13905 #ifdef CONFIG_PCI_IOV 13906 /* prep for VF support */ 13907 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 13908 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 13909 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 13910 /* disable link interrupts for VFs */ 13911 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 13912 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 13913 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 13914 i40e_flush(hw); 13915 13916 if (pci_num_vf(pdev)) { 13917 dev_info(&pdev->dev, 13918 "Active VFs found, allocating resources.\n"); 13919 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 13920 if (err) 13921 dev_info(&pdev->dev, 13922 "Error %d allocating resources for existing VFs\n", 13923 err); 13924 } 13925 } 13926 #endif /* CONFIG_PCI_IOV */ 13927 13928 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 13929 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 13930 pf->num_iwarp_msix, 13931 I40E_IWARP_IRQ_PILE_ID); 13932 if (pf->iwarp_base_vector < 0) { 13933 dev_info(&pdev->dev, 13934 "failed to get tracking for %d vectors for IWARP err=%d\n", 13935 pf->num_iwarp_msix, pf->iwarp_base_vector); 13936 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 13937 } 13938 } 13939 13940 i40e_dbg_pf_init(pf); 13941 13942 /* tell the firmware that we're starting */ 13943 i40e_send_version(pf); 13944 13945 /* since everything's happy, start the service_task timer */ 13946 mod_timer(&pf->service_timer, 13947 round_jiffies(jiffies + pf->service_timer_period)); 13948 13949 /* add this PF to client device list and launch a client service task */ 13950 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 13951 err = i40e_lan_add_device(pf); 13952 if (err) 13953 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 13954 err); 13955 } 13956 13957 #define PCI_SPEED_SIZE 8 13958 #define PCI_WIDTH_SIZE 8 13959 /* Devices on the IOSF bus do not have this information 13960 * and will report PCI Gen 1 x 1 by default so don't bother 13961 * checking them. 13962 */ 13963 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 13964 char speed[PCI_SPEED_SIZE] = "Unknown"; 13965 char width[PCI_WIDTH_SIZE] = "Unknown"; 13966 13967 /* Get the negotiated link width and speed from PCI config 13968 * space 13969 */ 13970 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 13971 &link_status); 13972 13973 i40e_set_pci_config_data(hw, link_status); 13974 13975 switch (hw->bus.speed) { 13976 case i40e_bus_speed_8000: 13977 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 13978 case i40e_bus_speed_5000: 13979 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 13980 case i40e_bus_speed_2500: 13981 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 13982 default: 13983 break; 13984 } 13985 switch (hw->bus.width) { 13986 case i40e_bus_width_pcie_x8: 13987 strncpy(width, "8", PCI_WIDTH_SIZE); break; 13988 case i40e_bus_width_pcie_x4: 13989 strncpy(width, "4", PCI_WIDTH_SIZE); break; 13990 case i40e_bus_width_pcie_x2: 13991 strncpy(width, "2", PCI_WIDTH_SIZE); break; 13992 case i40e_bus_width_pcie_x1: 13993 strncpy(width, "1", PCI_WIDTH_SIZE); break; 13994 default: 13995 break; 13996 } 13997 13998 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 13999 speed, width); 14000 14001 if (hw->bus.width < i40e_bus_width_pcie_x8 || 14002 hw->bus.speed < i40e_bus_speed_8000) { 14003 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 14004 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 14005 } 14006 } 14007 14008 /* get the requested speeds from the fw */ 14009 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 14010 if (err) 14011 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 14012 i40e_stat_str(&pf->hw, err), 14013 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14014 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 14015 14016 /* get the supported phy types from the fw */ 14017 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 14018 if (err) 14019 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 14020 i40e_stat_str(&pf->hw, err), 14021 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 14022 14023 /* Add a filter to drop all Flow control frames from any VSI from being 14024 * transmitted. By doing so we stop a malicious VF from sending out 14025 * PAUSE or PFC frames and potentially controlling traffic for other 14026 * PF/VF VSIs. 14027 * The FW can still send Flow control frames if enabled. 14028 */ 14029 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 14030 pf->main_vsi_seid); 14031 14032 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 14033 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 14034 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 14035 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 14036 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 14037 /* print a string summarizing features */ 14038 i40e_print_features(pf); 14039 14040 return 0; 14041 14042 /* Unwind what we've done if something failed in the setup */ 14043 err_vsis: 14044 set_bit(__I40E_DOWN, pf->state); 14045 i40e_clear_interrupt_scheme(pf); 14046 kfree(pf->vsi); 14047 err_switch_setup: 14048 i40e_reset_interrupt_capability(pf); 14049 del_timer_sync(&pf->service_timer); 14050 err_mac_addr: 14051 err_configure_lan_hmc: 14052 (void)i40e_shutdown_lan_hmc(hw); 14053 err_init_lan_hmc: 14054 kfree(pf->qp_pile); 14055 err_sw_init: 14056 err_adminq_setup: 14057 err_pf_reset: 14058 iounmap(hw->hw_addr); 14059 err_ioremap: 14060 kfree(pf); 14061 err_pf_alloc: 14062 pci_disable_pcie_error_reporting(pdev); 14063 pci_release_mem_regions(pdev); 14064 err_pci_reg: 14065 err_dma: 14066 pci_disable_device(pdev); 14067 return err; 14068 } 14069 14070 /** 14071 * i40e_remove - Device removal routine 14072 * @pdev: PCI device information struct 14073 * 14074 * i40e_remove is called by the PCI subsystem to alert the driver 14075 * that is should release a PCI device. This could be caused by a 14076 * Hot-Plug event, or because the driver is going to be removed from 14077 * memory. 14078 **/ 14079 static void i40e_remove(struct pci_dev *pdev) 14080 { 14081 struct i40e_pf *pf = pci_get_drvdata(pdev); 14082 struct i40e_hw *hw = &pf->hw; 14083 i40e_status ret_code; 14084 int i; 14085 14086 i40e_dbg_pf_exit(pf); 14087 14088 i40e_ptp_stop(pf); 14089 14090 /* Disable RSS in hw */ 14091 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 14092 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 14093 14094 /* no more scheduling of any task */ 14095 set_bit(__I40E_SUSPENDED, pf->state); 14096 set_bit(__I40E_DOWN, pf->state); 14097 if (pf->service_timer.function) 14098 del_timer_sync(&pf->service_timer); 14099 if (pf->service_task.func) 14100 cancel_work_sync(&pf->service_task); 14101 14102 /* Client close must be called explicitly here because the timer 14103 * has been stopped. 14104 */ 14105 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 14106 14107 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 14108 i40e_free_vfs(pf); 14109 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 14110 } 14111 14112 i40e_fdir_teardown(pf); 14113 14114 /* If there is a switch structure or any orphans, remove them. 14115 * This will leave only the PF's VSI remaining. 14116 */ 14117 for (i = 0; i < I40E_MAX_VEB; i++) { 14118 if (!pf->veb[i]) 14119 continue; 14120 14121 if (pf->veb[i]->uplink_seid == pf->mac_seid || 14122 pf->veb[i]->uplink_seid == 0) 14123 i40e_switch_branch_release(pf->veb[i]); 14124 } 14125 14126 /* Now we can shutdown the PF's VSI, just before we kill 14127 * adminq and hmc. 14128 */ 14129 if (pf->vsi[pf->lan_vsi]) 14130 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 14131 14132 i40e_cloud_filter_exit(pf); 14133 14134 /* remove attached clients */ 14135 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 14136 ret_code = i40e_lan_del_device(pf); 14137 if (ret_code) 14138 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 14139 ret_code); 14140 } 14141 14142 /* shutdown and destroy the HMC */ 14143 if (hw->hmc.hmc_obj) { 14144 ret_code = i40e_shutdown_lan_hmc(hw); 14145 if (ret_code) 14146 dev_warn(&pdev->dev, 14147 "Failed to destroy the HMC resources: %d\n", 14148 ret_code); 14149 } 14150 14151 /* shutdown the adminq */ 14152 i40e_shutdown_adminq(hw); 14153 14154 /* destroy the locks only once, here */ 14155 mutex_destroy(&hw->aq.arq_mutex); 14156 mutex_destroy(&hw->aq.asq_mutex); 14157 14158 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 14159 i40e_clear_interrupt_scheme(pf); 14160 for (i = 0; i < pf->num_alloc_vsi; i++) { 14161 if (pf->vsi[i]) { 14162 i40e_vsi_clear_rings(pf->vsi[i]); 14163 i40e_vsi_clear(pf->vsi[i]); 14164 pf->vsi[i] = NULL; 14165 } 14166 } 14167 14168 for (i = 0; i < I40E_MAX_VEB; i++) { 14169 kfree(pf->veb[i]); 14170 pf->veb[i] = NULL; 14171 } 14172 14173 kfree(pf->qp_pile); 14174 kfree(pf->vsi); 14175 14176 iounmap(hw->hw_addr); 14177 kfree(pf); 14178 pci_release_mem_regions(pdev); 14179 14180 pci_disable_pcie_error_reporting(pdev); 14181 pci_disable_device(pdev); 14182 } 14183 14184 /** 14185 * i40e_pci_error_detected - warning that something funky happened in PCI land 14186 * @pdev: PCI device information struct 14187 * @error: the type of PCI error 14188 * 14189 * Called to warn that something happened and the error handling steps 14190 * are in progress. Allows the driver to quiesce things, be ready for 14191 * remediation. 14192 **/ 14193 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 14194 enum pci_channel_state error) 14195 { 14196 struct i40e_pf *pf = pci_get_drvdata(pdev); 14197 14198 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 14199 14200 if (!pf) { 14201 dev_info(&pdev->dev, 14202 "Cannot recover - error happened during device probe\n"); 14203 return PCI_ERS_RESULT_DISCONNECT; 14204 } 14205 14206 /* shutdown all operations */ 14207 if (!test_bit(__I40E_SUSPENDED, pf->state)) 14208 i40e_prep_for_reset(pf, false); 14209 14210 /* Request a slot reset */ 14211 return PCI_ERS_RESULT_NEED_RESET; 14212 } 14213 14214 /** 14215 * i40e_pci_error_slot_reset - a PCI slot reset just happened 14216 * @pdev: PCI device information struct 14217 * 14218 * Called to find if the driver can work with the device now that 14219 * the pci slot has been reset. If a basic connection seems good 14220 * (registers are readable and have sane content) then return a 14221 * happy little PCI_ERS_RESULT_xxx. 14222 **/ 14223 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 14224 { 14225 struct i40e_pf *pf = pci_get_drvdata(pdev); 14226 pci_ers_result_t result; 14227 int err; 14228 u32 reg; 14229 14230 dev_dbg(&pdev->dev, "%s\n", __func__); 14231 if (pci_enable_device_mem(pdev)) { 14232 dev_info(&pdev->dev, 14233 "Cannot re-enable PCI device after reset.\n"); 14234 result = PCI_ERS_RESULT_DISCONNECT; 14235 } else { 14236 pci_set_master(pdev); 14237 pci_restore_state(pdev); 14238 pci_save_state(pdev); 14239 pci_wake_from_d3(pdev, false); 14240 14241 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 14242 if (reg == 0) 14243 result = PCI_ERS_RESULT_RECOVERED; 14244 else 14245 result = PCI_ERS_RESULT_DISCONNECT; 14246 } 14247 14248 err = pci_cleanup_aer_uncorrect_error_status(pdev); 14249 if (err) { 14250 dev_info(&pdev->dev, 14251 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 14252 err); 14253 /* non-fatal, continue */ 14254 } 14255 14256 return result; 14257 } 14258 14259 /** 14260 * i40e_pci_error_reset_prepare - prepare device driver for pci reset 14261 * @pdev: PCI device information struct 14262 */ 14263 static void i40e_pci_error_reset_prepare(struct pci_dev *pdev) 14264 { 14265 struct i40e_pf *pf = pci_get_drvdata(pdev); 14266 14267 i40e_prep_for_reset(pf, false); 14268 } 14269 14270 /** 14271 * i40e_pci_error_reset_done - pci reset done, device driver reset can begin 14272 * @pdev: PCI device information struct 14273 */ 14274 static void i40e_pci_error_reset_done(struct pci_dev *pdev) 14275 { 14276 struct i40e_pf *pf = pci_get_drvdata(pdev); 14277 14278 i40e_reset_and_rebuild(pf, false, false); 14279 } 14280 14281 /** 14282 * i40e_pci_error_resume - restart operations after PCI error recovery 14283 * @pdev: PCI device information struct 14284 * 14285 * Called to allow the driver to bring things back up after PCI error 14286 * and/or reset recovery has finished. 14287 **/ 14288 static void i40e_pci_error_resume(struct pci_dev *pdev) 14289 { 14290 struct i40e_pf *pf = pci_get_drvdata(pdev); 14291 14292 dev_dbg(&pdev->dev, "%s\n", __func__); 14293 if (test_bit(__I40E_SUSPENDED, pf->state)) 14294 return; 14295 14296 i40e_handle_reset_warning(pf, false); 14297 } 14298 14299 /** 14300 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 14301 * using the mac_address_write admin q function 14302 * @pf: pointer to i40e_pf struct 14303 **/ 14304 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 14305 { 14306 struct i40e_hw *hw = &pf->hw; 14307 i40e_status ret; 14308 u8 mac_addr[6]; 14309 u16 flags = 0; 14310 14311 /* Get current MAC address in case it's an LAA */ 14312 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 14313 ether_addr_copy(mac_addr, 14314 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 14315 } else { 14316 dev_err(&pf->pdev->dev, 14317 "Failed to retrieve MAC address; using default\n"); 14318 ether_addr_copy(mac_addr, hw->mac.addr); 14319 } 14320 14321 /* The FW expects the mac address write cmd to first be called with 14322 * one of these flags before calling it again with the multicast 14323 * enable flags. 14324 */ 14325 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 14326 14327 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 14328 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 14329 14330 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 14331 if (ret) { 14332 dev_err(&pf->pdev->dev, 14333 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 14334 return; 14335 } 14336 14337 flags = I40E_AQC_MC_MAG_EN 14338 | I40E_AQC_WOL_PRESERVE_ON_PFR 14339 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 14340 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 14341 if (ret) 14342 dev_err(&pf->pdev->dev, 14343 "Failed to enable Multicast Magic Packet wake up\n"); 14344 } 14345 14346 /** 14347 * i40e_shutdown - PCI callback for shutting down 14348 * @pdev: PCI device information struct 14349 **/ 14350 static void i40e_shutdown(struct pci_dev *pdev) 14351 { 14352 struct i40e_pf *pf = pci_get_drvdata(pdev); 14353 struct i40e_hw *hw = &pf->hw; 14354 14355 set_bit(__I40E_SUSPENDED, pf->state); 14356 set_bit(__I40E_DOWN, pf->state); 14357 rtnl_lock(); 14358 i40e_prep_for_reset(pf, true); 14359 rtnl_unlock(); 14360 14361 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14362 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14363 14364 del_timer_sync(&pf->service_timer); 14365 cancel_work_sync(&pf->service_task); 14366 i40e_cloud_filter_exit(pf); 14367 i40e_fdir_teardown(pf); 14368 14369 /* Client close must be called explicitly here because the timer 14370 * has been stopped. 14371 */ 14372 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 14373 14374 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 14375 i40e_enable_mc_magic_wake(pf); 14376 14377 i40e_prep_for_reset(pf, false); 14378 14379 wr32(hw, I40E_PFPM_APM, 14380 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14381 wr32(hw, I40E_PFPM_WUFC, 14382 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14383 14384 i40e_clear_interrupt_scheme(pf); 14385 14386 if (system_state == SYSTEM_POWER_OFF) { 14387 pci_wake_from_d3(pdev, pf->wol_en); 14388 pci_set_power_state(pdev, PCI_D3hot); 14389 } 14390 } 14391 14392 /** 14393 * i40e_suspend - PM callback for moving to D3 14394 * @dev: generic device information structure 14395 **/ 14396 static int __maybe_unused i40e_suspend(struct device *dev) 14397 { 14398 struct pci_dev *pdev = to_pci_dev(dev); 14399 struct i40e_pf *pf = pci_get_drvdata(pdev); 14400 struct i40e_hw *hw = &pf->hw; 14401 14402 /* If we're already suspended, then there is nothing to do */ 14403 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 14404 return 0; 14405 14406 set_bit(__I40E_DOWN, pf->state); 14407 14408 /* Ensure service task will not be running */ 14409 del_timer_sync(&pf->service_timer); 14410 cancel_work_sync(&pf->service_task); 14411 14412 /* Client close must be called explicitly here because the timer 14413 * has been stopped. 14414 */ 14415 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 14416 14417 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 14418 i40e_enable_mc_magic_wake(pf); 14419 14420 /* Since we're going to destroy queues during the 14421 * i40e_clear_interrupt_scheme() we should hold the RTNL lock for this 14422 * whole section 14423 */ 14424 rtnl_lock(); 14425 14426 i40e_prep_for_reset(pf, true); 14427 14428 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 14429 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 14430 14431 /* Clear the interrupt scheme and release our IRQs so that the system 14432 * can safely hibernate even when there are a large number of CPUs. 14433 * Otherwise hibernation might fail when mapping all the vectors back 14434 * to CPU0. 14435 */ 14436 i40e_clear_interrupt_scheme(pf); 14437 14438 rtnl_unlock(); 14439 14440 return 0; 14441 } 14442 14443 /** 14444 * i40e_resume - PM callback for waking up from D3 14445 * @dev: generic device information structure 14446 **/ 14447 static int __maybe_unused i40e_resume(struct device *dev) 14448 { 14449 struct pci_dev *pdev = to_pci_dev(dev); 14450 struct i40e_pf *pf = pci_get_drvdata(pdev); 14451 int err; 14452 14453 /* If we're not suspended, then there is nothing to do */ 14454 if (!test_bit(__I40E_SUSPENDED, pf->state)) 14455 return 0; 14456 14457 /* We need to hold the RTNL lock prior to restoring interrupt schemes, 14458 * since we're going to be restoring queues 14459 */ 14460 rtnl_lock(); 14461 14462 /* We cleared the interrupt scheme when we suspended, so we need to 14463 * restore it now to resume device functionality. 14464 */ 14465 err = i40e_restore_interrupt_scheme(pf); 14466 if (err) { 14467 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n", 14468 err); 14469 } 14470 14471 clear_bit(__I40E_DOWN, pf->state); 14472 i40e_reset_and_rebuild(pf, false, true); 14473 14474 rtnl_unlock(); 14475 14476 /* Clear suspended state last after everything is recovered */ 14477 clear_bit(__I40E_SUSPENDED, pf->state); 14478 14479 /* Restart the service task */ 14480 mod_timer(&pf->service_timer, 14481 round_jiffies(jiffies + pf->service_timer_period)); 14482 14483 return 0; 14484 } 14485 14486 static const struct pci_error_handlers i40e_err_handler = { 14487 .error_detected = i40e_pci_error_detected, 14488 .slot_reset = i40e_pci_error_slot_reset, 14489 .reset_prepare = i40e_pci_error_reset_prepare, 14490 .reset_done = i40e_pci_error_reset_done, 14491 .resume = i40e_pci_error_resume, 14492 }; 14493 14494 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 14495 14496 static struct pci_driver i40e_driver = { 14497 .name = i40e_driver_name, 14498 .id_table = i40e_pci_tbl, 14499 .probe = i40e_probe, 14500 .remove = i40e_remove, 14501 .driver = { 14502 .pm = &i40e_pm_ops, 14503 }, 14504 .shutdown = i40e_shutdown, 14505 .err_handler = &i40e_err_handler, 14506 .sriov_configure = i40e_pci_sriov_configure, 14507 }; 14508 14509 /** 14510 * i40e_init_module - Driver registration routine 14511 * 14512 * i40e_init_module is the first routine called when the driver is 14513 * loaded. All it does is register with the PCI subsystem. 14514 **/ 14515 static int __init i40e_init_module(void) 14516 { 14517 pr_info("%s: %s - version %s\n", i40e_driver_name, 14518 i40e_driver_string, i40e_driver_version_str); 14519 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 14520 14521 /* There is no need to throttle the number of active tasks because 14522 * each device limits its own task using a state bit for scheduling 14523 * the service task, and the device tasks do not interfere with each 14524 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 14525 * since we need to be able to guarantee forward progress even under 14526 * memory pressure. 14527 */ 14528 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 14529 if (!i40e_wq) { 14530 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 14531 return -ENOMEM; 14532 } 14533 14534 i40e_dbg_init(); 14535 return pci_register_driver(&i40e_driver); 14536 } 14537 module_init(i40e_init_module); 14538 14539 /** 14540 * i40e_exit_module - Driver exit cleanup routine 14541 * 14542 * i40e_exit_module is called just before the driver is removed 14543 * from memory. 14544 **/ 14545 static void __exit i40e_exit_module(void) 14546 { 14547 pci_unregister_driver(&i40e_driver); 14548 destroy_workqueue(i40e_wq); 14549 i40e_dbg_exit(); 14550 } 14551 module_exit(i40e_exit_module); 14552