1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2017 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/etherdevice.h> 28 #include <linux/of_net.h> 29 #include <linux/pci.h> 30 #include <linux/bpf.h> 31 32 /* Local includes */ 33 #include "i40e.h" 34 #include "i40e_diag.h" 35 #include <net/udp_tunnel.h> 36 /* All i40e tracepoints are defined by the include below, which 37 * must be included exactly once across the whole kernel with 38 * CREATE_TRACE_POINTS defined 39 */ 40 #define CREATE_TRACE_POINTS 41 #include "i40e_trace.h" 42 43 const char i40e_driver_name[] = "i40e"; 44 static const char i40e_driver_string[] = 45 "Intel(R) Ethernet Connection XL710 Network Driver"; 46 47 #define DRV_KERN "-k" 48 49 #define DRV_VERSION_MAJOR 2 50 #define DRV_VERSION_MINOR 1 51 #define DRV_VERSION_BUILD 14 52 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 53 __stringify(DRV_VERSION_MINOR) "." \ 54 __stringify(DRV_VERSION_BUILD) DRV_KERN 55 const char i40e_driver_version_str[] = DRV_VERSION; 56 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 57 58 /* a bit of forward declarations */ 59 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 60 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 61 static int i40e_add_vsi(struct i40e_vsi *vsi); 62 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 63 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 64 static int i40e_setup_misc_vector(struct i40e_pf *pf); 65 static void i40e_determine_queue_usage(struct i40e_pf *pf); 66 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 67 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 68 static int i40e_reset(struct i40e_pf *pf); 69 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 70 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 71 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 72 73 /* i40e_pci_tbl - PCI Device ID Table 74 * 75 * Last entry must be all 0s 76 * 77 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 78 * Class, Class Mask, private data (not used) } 79 */ 80 static const struct pci_device_id i40e_pci_tbl[] = { 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 91 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 92 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 93 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 94 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 95 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 96 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 97 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 98 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 99 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 100 /* required last entry */ 101 {0, } 102 }; 103 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 104 105 #define I40E_MAX_VF_COUNT 128 106 static int debug = -1; 107 module_param(debug, uint, 0); 108 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 109 110 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 111 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 112 MODULE_LICENSE("GPL"); 113 MODULE_VERSION(DRV_VERSION); 114 115 static struct workqueue_struct *i40e_wq; 116 117 /** 118 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 119 * @hw: pointer to the HW structure 120 * @mem: ptr to mem struct to fill out 121 * @size: size of memory requested 122 * @alignment: what to align the allocation to 123 **/ 124 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 125 u64 size, u32 alignment) 126 { 127 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 128 129 mem->size = ALIGN(size, alignment); 130 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 131 &mem->pa, GFP_KERNEL); 132 if (!mem->va) 133 return -ENOMEM; 134 135 return 0; 136 } 137 138 /** 139 * i40e_free_dma_mem_d - OS specific memory free for shared code 140 * @hw: pointer to the HW structure 141 * @mem: ptr to mem struct to free 142 **/ 143 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 144 { 145 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 146 147 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 148 mem->va = NULL; 149 mem->pa = 0; 150 mem->size = 0; 151 152 return 0; 153 } 154 155 /** 156 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 157 * @hw: pointer to the HW structure 158 * @mem: ptr to mem struct to fill out 159 * @size: size of memory requested 160 **/ 161 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 162 u32 size) 163 { 164 mem->size = size; 165 mem->va = kzalloc(size, GFP_KERNEL); 166 167 if (!mem->va) 168 return -ENOMEM; 169 170 return 0; 171 } 172 173 /** 174 * i40e_free_virt_mem_d - OS specific memory free for shared code 175 * @hw: pointer to the HW structure 176 * @mem: ptr to mem struct to free 177 **/ 178 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 179 { 180 /* it's ok to kfree a NULL pointer */ 181 kfree(mem->va); 182 mem->va = NULL; 183 mem->size = 0; 184 185 return 0; 186 } 187 188 /** 189 * i40e_get_lump - find a lump of free generic resource 190 * @pf: board private structure 191 * @pile: the pile of resource to search 192 * @needed: the number of items needed 193 * @id: an owner id to stick on the items assigned 194 * 195 * Returns the base item index of the lump, or negative for error 196 * 197 * The search_hint trick and lack of advanced fit-finding only work 198 * because we're highly likely to have all the same size lump requests. 199 * Linear search time and any fragmentation should be minimal. 200 **/ 201 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 202 u16 needed, u16 id) 203 { 204 int ret = -ENOMEM; 205 int i, j; 206 207 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 208 dev_info(&pf->pdev->dev, 209 "param err: pile=%p needed=%d id=0x%04x\n", 210 pile, needed, id); 211 return -EINVAL; 212 } 213 214 /* start the linear search with an imperfect hint */ 215 i = pile->search_hint; 216 while (i < pile->num_entries) { 217 /* skip already allocated entries */ 218 if (pile->list[i] & I40E_PILE_VALID_BIT) { 219 i++; 220 continue; 221 } 222 223 /* do we have enough in this lump? */ 224 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 225 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 226 break; 227 } 228 229 if (j == needed) { 230 /* there was enough, so assign it to the requestor */ 231 for (j = 0; j < needed; j++) 232 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 233 ret = i; 234 pile->search_hint = i + j; 235 break; 236 } 237 238 /* not enough, so skip over it and continue looking */ 239 i += j; 240 } 241 242 return ret; 243 } 244 245 /** 246 * i40e_put_lump - return a lump of generic resource 247 * @pile: the pile of resource to search 248 * @index: the base item index 249 * @id: the owner id of the items assigned 250 * 251 * Returns the count of items in the lump 252 **/ 253 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 254 { 255 int valid_id = (id | I40E_PILE_VALID_BIT); 256 int count = 0; 257 int i; 258 259 if (!pile || index >= pile->num_entries) 260 return -EINVAL; 261 262 for (i = index; 263 i < pile->num_entries && pile->list[i] == valid_id; 264 i++) { 265 pile->list[i] = 0; 266 count++; 267 } 268 269 if (count && index < pile->search_hint) 270 pile->search_hint = index; 271 272 return count; 273 } 274 275 /** 276 * i40e_find_vsi_from_id - searches for the vsi with the given id 277 * @pf - the pf structure to search for the vsi 278 * @id - id of the vsi it is searching for 279 **/ 280 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 281 { 282 int i; 283 284 for (i = 0; i < pf->num_alloc_vsi; i++) 285 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 286 return pf->vsi[i]; 287 288 return NULL; 289 } 290 291 /** 292 * i40e_service_event_schedule - Schedule the service task to wake up 293 * @pf: board private structure 294 * 295 * If not already scheduled, this puts the task into the work queue 296 **/ 297 void i40e_service_event_schedule(struct i40e_pf *pf) 298 { 299 if (!test_bit(__I40E_DOWN, pf->state) && 300 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 301 queue_work(i40e_wq, &pf->service_task); 302 } 303 304 /** 305 * i40e_tx_timeout - Respond to a Tx Hang 306 * @netdev: network interface device structure 307 * 308 * If any port has noticed a Tx timeout, it is likely that the whole 309 * device is munged, not just the one netdev port, so go for the full 310 * reset. 311 **/ 312 static void i40e_tx_timeout(struct net_device *netdev) 313 { 314 struct i40e_netdev_priv *np = netdev_priv(netdev); 315 struct i40e_vsi *vsi = np->vsi; 316 struct i40e_pf *pf = vsi->back; 317 struct i40e_ring *tx_ring = NULL; 318 unsigned int i, hung_queue = 0; 319 u32 head, val; 320 321 pf->tx_timeout_count++; 322 323 /* find the stopped queue the same way the stack does */ 324 for (i = 0; i < netdev->num_tx_queues; i++) { 325 struct netdev_queue *q; 326 unsigned long trans_start; 327 328 q = netdev_get_tx_queue(netdev, i); 329 trans_start = q->trans_start; 330 if (netif_xmit_stopped(q) && 331 time_after(jiffies, 332 (trans_start + netdev->watchdog_timeo))) { 333 hung_queue = i; 334 break; 335 } 336 } 337 338 if (i == netdev->num_tx_queues) { 339 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 340 } else { 341 /* now that we have an index, find the tx_ring struct */ 342 for (i = 0; i < vsi->num_queue_pairs; i++) { 343 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 344 if (hung_queue == 345 vsi->tx_rings[i]->queue_index) { 346 tx_ring = vsi->tx_rings[i]; 347 break; 348 } 349 } 350 } 351 } 352 353 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 354 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 355 else if (time_before(jiffies, 356 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 357 return; /* don't do any new action before the next timeout */ 358 359 if (tx_ring) { 360 head = i40e_get_head(tx_ring); 361 /* Read interrupt register */ 362 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 363 val = rd32(&pf->hw, 364 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 365 tx_ring->vsi->base_vector - 1)); 366 else 367 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 368 369 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 370 vsi->seid, hung_queue, tx_ring->next_to_clean, 371 head, tx_ring->next_to_use, 372 readl(tx_ring->tail), val); 373 } 374 375 pf->tx_timeout_last_recovery = jiffies; 376 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 377 pf->tx_timeout_recovery_level, hung_queue); 378 379 switch (pf->tx_timeout_recovery_level) { 380 case 1: 381 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 382 break; 383 case 2: 384 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 385 break; 386 case 3: 387 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 388 break; 389 default: 390 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 391 break; 392 } 393 394 i40e_service_event_schedule(pf); 395 pf->tx_timeout_recovery_level++; 396 } 397 398 /** 399 * i40e_get_vsi_stats_struct - Get System Network Statistics 400 * @vsi: the VSI we care about 401 * 402 * Returns the address of the device statistics structure. 403 * The statistics are actually updated from the service task. 404 **/ 405 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 406 { 407 return &vsi->net_stats; 408 } 409 410 /** 411 * i40e_get_netdev_stats_struct_tx - populate stats from a Tx ring 412 * @ring: Tx ring to get statistics from 413 * @stats: statistics entry to be updated 414 **/ 415 static void i40e_get_netdev_stats_struct_tx(struct i40e_ring *ring, 416 struct rtnl_link_stats64 *stats) 417 { 418 u64 bytes, packets; 419 unsigned int start; 420 421 do { 422 start = u64_stats_fetch_begin_irq(&ring->syncp); 423 packets = ring->stats.packets; 424 bytes = ring->stats.bytes; 425 } while (u64_stats_fetch_retry_irq(&ring->syncp, start)); 426 427 stats->tx_packets += packets; 428 stats->tx_bytes += bytes; 429 } 430 431 /** 432 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 433 * @netdev: network interface device structure 434 * 435 * Returns the address of the device statistics structure. 436 * The statistics are actually updated from the service task. 437 **/ 438 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 439 struct rtnl_link_stats64 *stats) 440 { 441 struct i40e_netdev_priv *np = netdev_priv(netdev); 442 struct i40e_ring *tx_ring, *rx_ring; 443 struct i40e_vsi *vsi = np->vsi; 444 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 445 int i; 446 447 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 448 return; 449 450 if (!vsi->tx_rings) 451 return; 452 453 rcu_read_lock(); 454 for (i = 0; i < vsi->num_queue_pairs; i++) { 455 u64 bytes, packets; 456 unsigned int start; 457 458 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]); 459 if (!tx_ring) 460 continue; 461 i40e_get_netdev_stats_struct_tx(tx_ring, stats); 462 463 rx_ring = &tx_ring[1]; 464 465 do { 466 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 467 packets = rx_ring->stats.packets; 468 bytes = rx_ring->stats.bytes; 469 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 470 471 stats->rx_packets += packets; 472 stats->rx_bytes += bytes; 473 474 if (i40e_enabled_xdp_vsi(vsi)) 475 i40e_get_netdev_stats_struct_tx(&rx_ring[1], stats); 476 } 477 rcu_read_unlock(); 478 479 /* following stats updated by i40e_watchdog_subtask() */ 480 stats->multicast = vsi_stats->multicast; 481 stats->tx_errors = vsi_stats->tx_errors; 482 stats->tx_dropped = vsi_stats->tx_dropped; 483 stats->rx_errors = vsi_stats->rx_errors; 484 stats->rx_dropped = vsi_stats->rx_dropped; 485 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 486 stats->rx_length_errors = vsi_stats->rx_length_errors; 487 } 488 489 /** 490 * i40e_vsi_reset_stats - Resets all stats of the given vsi 491 * @vsi: the VSI to have its stats reset 492 **/ 493 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 494 { 495 struct rtnl_link_stats64 *ns; 496 int i; 497 498 if (!vsi) 499 return; 500 501 ns = i40e_get_vsi_stats_struct(vsi); 502 memset(ns, 0, sizeof(*ns)); 503 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 504 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 505 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 506 if (vsi->rx_rings && vsi->rx_rings[0]) { 507 for (i = 0; i < vsi->num_queue_pairs; i++) { 508 memset(&vsi->rx_rings[i]->stats, 0, 509 sizeof(vsi->rx_rings[i]->stats)); 510 memset(&vsi->rx_rings[i]->rx_stats, 0, 511 sizeof(vsi->rx_rings[i]->rx_stats)); 512 memset(&vsi->tx_rings[i]->stats, 0, 513 sizeof(vsi->tx_rings[i]->stats)); 514 memset(&vsi->tx_rings[i]->tx_stats, 0, 515 sizeof(vsi->tx_rings[i]->tx_stats)); 516 } 517 } 518 vsi->stat_offsets_loaded = false; 519 } 520 521 /** 522 * i40e_pf_reset_stats - Reset all of the stats for the given PF 523 * @pf: the PF to be reset 524 **/ 525 void i40e_pf_reset_stats(struct i40e_pf *pf) 526 { 527 int i; 528 529 memset(&pf->stats, 0, sizeof(pf->stats)); 530 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 531 pf->stat_offsets_loaded = false; 532 533 for (i = 0; i < I40E_MAX_VEB; i++) { 534 if (pf->veb[i]) { 535 memset(&pf->veb[i]->stats, 0, 536 sizeof(pf->veb[i]->stats)); 537 memset(&pf->veb[i]->stats_offsets, 0, 538 sizeof(pf->veb[i]->stats_offsets)); 539 pf->veb[i]->stat_offsets_loaded = false; 540 } 541 } 542 pf->hw_csum_rx_error = 0; 543 } 544 545 /** 546 * i40e_stat_update48 - read and update a 48 bit stat from the chip 547 * @hw: ptr to the hardware info 548 * @hireg: the high 32 bit reg to read 549 * @loreg: the low 32 bit reg to read 550 * @offset_loaded: has the initial offset been loaded yet 551 * @offset: ptr to current offset value 552 * @stat: ptr to the stat 553 * 554 * Since the device stats are not reset at PFReset, they likely will not 555 * be zeroed when the driver starts. We'll save the first values read 556 * and use them as offsets to be subtracted from the raw values in order 557 * to report stats that count from zero. In the process, we also manage 558 * the potential roll-over. 559 **/ 560 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 561 bool offset_loaded, u64 *offset, u64 *stat) 562 { 563 u64 new_data; 564 565 if (hw->device_id == I40E_DEV_ID_QEMU) { 566 new_data = rd32(hw, loreg); 567 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 568 } else { 569 new_data = rd64(hw, loreg); 570 } 571 if (!offset_loaded) 572 *offset = new_data; 573 if (likely(new_data >= *offset)) 574 *stat = new_data - *offset; 575 else 576 *stat = (new_data + BIT_ULL(48)) - *offset; 577 *stat &= 0xFFFFFFFFFFFFULL; 578 } 579 580 /** 581 * i40e_stat_update32 - read and update a 32 bit stat from the chip 582 * @hw: ptr to the hardware info 583 * @reg: the hw reg to read 584 * @offset_loaded: has the initial offset been loaded yet 585 * @offset: ptr to current offset value 586 * @stat: ptr to the stat 587 **/ 588 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 589 bool offset_loaded, u64 *offset, u64 *stat) 590 { 591 u32 new_data; 592 593 new_data = rd32(hw, reg); 594 if (!offset_loaded) 595 *offset = new_data; 596 if (likely(new_data >= *offset)) 597 *stat = (u32)(new_data - *offset); 598 else 599 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 600 } 601 602 /** 603 * i40e_stat_update_and_clear32 - read and clear hw reg, update a 32 bit stat 604 * @hw: ptr to the hardware info 605 * @reg: the hw reg to read and clear 606 * @stat: ptr to the stat 607 **/ 608 static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) 609 { 610 u32 new_data = rd32(hw, reg); 611 612 wr32(hw, reg, 1); /* must write a nonzero value to clear register */ 613 *stat += new_data; 614 } 615 616 /** 617 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 618 * @vsi: the VSI to be updated 619 **/ 620 void i40e_update_eth_stats(struct i40e_vsi *vsi) 621 { 622 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 623 struct i40e_pf *pf = vsi->back; 624 struct i40e_hw *hw = &pf->hw; 625 struct i40e_eth_stats *oes; 626 struct i40e_eth_stats *es; /* device's eth stats */ 627 628 es = &vsi->eth_stats; 629 oes = &vsi->eth_stats_offsets; 630 631 /* Gather up the stats that the hw collects */ 632 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 633 vsi->stat_offsets_loaded, 634 &oes->tx_errors, &es->tx_errors); 635 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 636 vsi->stat_offsets_loaded, 637 &oes->rx_discards, &es->rx_discards); 638 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 639 vsi->stat_offsets_loaded, 640 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 641 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 642 vsi->stat_offsets_loaded, 643 &oes->tx_errors, &es->tx_errors); 644 645 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 646 I40E_GLV_GORCL(stat_idx), 647 vsi->stat_offsets_loaded, 648 &oes->rx_bytes, &es->rx_bytes); 649 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 650 I40E_GLV_UPRCL(stat_idx), 651 vsi->stat_offsets_loaded, 652 &oes->rx_unicast, &es->rx_unicast); 653 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 654 I40E_GLV_MPRCL(stat_idx), 655 vsi->stat_offsets_loaded, 656 &oes->rx_multicast, &es->rx_multicast); 657 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 658 I40E_GLV_BPRCL(stat_idx), 659 vsi->stat_offsets_loaded, 660 &oes->rx_broadcast, &es->rx_broadcast); 661 662 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 663 I40E_GLV_GOTCL(stat_idx), 664 vsi->stat_offsets_loaded, 665 &oes->tx_bytes, &es->tx_bytes); 666 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 667 I40E_GLV_UPTCL(stat_idx), 668 vsi->stat_offsets_loaded, 669 &oes->tx_unicast, &es->tx_unicast); 670 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 671 I40E_GLV_MPTCL(stat_idx), 672 vsi->stat_offsets_loaded, 673 &oes->tx_multicast, &es->tx_multicast); 674 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 675 I40E_GLV_BPTCL(stat_idx), 676 vsi->stat_offsets_loaded, 677 &oes->tx_broadcast, &es->tx_broadcast); 678 vsi->stat_offsets_loaded = true; 679 } 680 681 /** 682 * i40e_update_veb_stats - Update Switch component statistics 683 * @veb: the VEB being updated 684 **/ 685 static void i40e_update_veb_stats(struct i40e_veb *veb) 686 { 687 struct i40e_pf *pf = veb->pf; 688 struct i40e_hw *hw = &pf->hw; 689 struct i40e_eth_stats *oes; 690 struct i40e_eth_stats *es; /* device's eth stats */ 691 struct i40e_veb_tc_stats *veb_oes; 692 struct i40e_veb_tc_stats *veb_es; 693 int i, idx = 0; 694 695 idx = veb->stats_idx; 696 es = &veb->stats; 697 oes = &veb->stats_offsets; 698 veb_es = &veb->tc_stats; 699 veb_oes = &veb->tc_stats_offsets; 700 701 /* Gather up the stats that the hw collects */ 702 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 703 veb->stat_offsets_loaded, 704 &oes->tx_discards, &es->tx_discards); 705 if (hw->revision_id > 0) 706 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 707 veb->stat_offsets_loaded, 708 &oes->rx_unknown_protocol, 709 &es->rx_unknown_protocol); 710 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 711 veb->stat_offsets_loaded, 712 &oes->rx_bytes, &es->rx_bytes); 713 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 714 veb->stat_offsets_loaded, 715 &oes->rx_unicast, &es->rx_unicast); 716 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 717 veb->stat_offsets_loaded, 718 &oes->rx_multicast, &es->rx_multicast); 719 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 720 veb->stat_offsets_loaded, 721 &oes->rx_broadcast, &es->rx_broadcast); 722 723 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 724 veb->stat_offsets_loaded, 725 &oes->tx_bytes, &es->tx_bytes); 726 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 727 veb->stat_offsets_loaded, 728 &oes->tx_unicast, &es->tx_unicast); 729 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 730 veb->stat_offsets_loaded, 731 &oes->tx_multicast, &es->tx_multicast); 732 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 733 veb->stat_offsets_loaded, 734 &oes->tx_broadcast, &es->tx_broadcast); 735 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 736 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 737 I40E_GLVEBTC_RPCL(i, idx), 738 veb->stat_offsets_loaded, 739 &veb_oes->tc_rx_packets[i], 740 &veb_es->tc_rx_packets[i]); 741 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 742 I40E_GLVEBTC_RBCL(i, idx), 743 veb->stat_offsets_loaded, 744 &veb_oes->tc_rx_bytes[i], 745 &veb_es->tc_rx_bytes[i]); 746 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 747 I40E_GLVEBTC_TPCL(i, idx), 748 veb->stat_offsets_loaded, 749 &veb_oes->tc_tx_packets[i], 750 &veb_es->tc_tx_packets[i]); 751 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 752 I40E_GLVEBTC_TBCL(i, idx), 753 veb->stat_offsets_loaded, 754 &veb_oes->tc_tx_bytes[i], 755 &veb_es->tc_tx_bytes[i]); 756 } 757 veb->stat_offsets_loaded = true; 758 } 759 760 /** 761 * i40e_update_vsi_stats - Update the vsi statistics counters. 762 * @vsi: the VSI to be updated 763 * 764 * There are a few instances where we store the same stat in a 765 * couple of different structs. This is partly because we have 766 * the netdev stats that need to be filled out, which is slightly 767 * different from the "eth_stats" defined by the chip and used in 768 * VF communications. We sort it out here. 769 **/ 770 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 771 { 772 struct i40e_pf *pf = vsi->back; 773 struct rtnl_link_stats64 *ons; 774 struct rtnl_link_stats64 *ns; /* netdev stats */ 775 struct i40e_eth_stats *oes; 776 struct i40e_eth_stats *es; /* device's eth stats */ 777 u32 tx_restart, tx_busy; 778 struct i40e_ring *p; 779 u32 rx_page, rx_buf; 780 u64 bytes, packets; 781 unsigned int start; 782 u64 tx_linearize; 783 u64 tx_force_wb; 784 u64 rx_p, rx_b; 785 u64 tx_p, tx_b; 786 u16 q; 787 788 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 789 test_bit(__I40E_CONFIG_BUSY, pf->state)) 790 return; 791 792 ns = i40e_get_vsi_stats_struct(vsi); 793 ons = &vsi->net_stats_offsets; 794 es = &vsi->eth_stats; 795 oes = &vsi->eth_stats_offsets; 796 797 /* Gather up the netdev and vsi stats that the driver collects 798 * on the fly during packet processing 799 */ 800 rx_b = rx_p = 0; 801 tx_b = tx_p = 0; 802 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 803 rx_page = 0; 804 rx_buf = 0; 805 rcu_read_lock(); 806 for (q = 0; q < vsi->num_queue_pairs; q++) { 807 /* locate Tx ring */ 808 p = ACCESS_ONCE(vsi->tx_rings[q]); 809 810 do { 811 start = u64_stats_fetch_begin_irq(&p->syncp); 812 packets = p->stats.packets; 813 bytes = p->stats.bytes; 814 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 815 tx_b += bytes; 816 tx_p += packets; 817 tx_restart += p->tx_stats.restart_queue; 818 tx_busy += p->tx_stats.tx_busy; 819 tx_linearize += p->tx_stats.tx_linearize; 820 tx_force_wb += p->tx_stats.tx_force_wb; 821 822 /* Rx queue is part of the same block as Tx queue */ 823 p = &p[1]; 824 do { 825 start = u64_stats_fetch_begin_irq(&p->syncp); 826 packets = p->stats.packets; 827 bytes = p->stats.bytes; 828 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 829 rx_b += bytes; 830 rx_p += packets; 831 rx_buf += p->rx_stats.alloc_buff_failed; 832 rx_page += p->rx_stats.alloc_page_failed; 833 } 834 rcu_read_unlock(); 835 vsi->tx_restart = tx_restart; 836 vsi->tx_busy = tx_busy; 837 vsi->tx_linearize = tx_linearize; 838 vsi->tx_force_wb = tx_force_wb; 839 vsi->rx_page_failed = rx_page; 840 vsi->rx_buf_failed = rx_buf; 841 842 ns->rx_packets = rx_p; 843 ns->rx_bytes = rx_b; 844 ns->tx_packets = tx_p; 845 ns->tx_bytes = tx_b; 846 847 /* update netdev stats from eth stats */ 848 i40e_update_eth_stats(vsi); 849 ons->tx_errors = oes->tx_errors; 850 ns->tx_errors = es->tx_errors; 851 ons->multicast = oes->rx_multicast; 852 ns->multicast = es->rx_multicast; 853 ons->rx_dropped = oes->rx_discards; 854 ns->rx_dropped = es->rx_discards; 855 ons->tx_dropped = oes->tx_discards; 856 ns->tx_dropped = es->tx_discards; 857 858 /* pull in a couple PF stats if this is the main vsi */ 859 if (vsi == pf->vsi[pf->lan_vsi]) { 860 ns->rx_crc_errors = pf->stats.crc_errors; 861 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 862 ns->rx_length_errors = pf->stats.rx_length_errors; 863 } 864 } 865 866 /** 867 * i40e_update_pf_stats - Update the PF statistics counters. 868 * @pf: the PF to be updated 869 **/ 870 static void i40e_update_pf_stats(struct i40e_pf *pf) 871 { 872 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 873 struct i40e_hw_port_stats *nsd = &pf->stats; 874 struct i40e_hw *hw = &pf->hw; 875 u32 val; 876 int i; 877 878 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 879 I40E_GLPRT_GORCL(hw->port), 880 pf->stat_offsets_loaded, 881 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 882 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 883 I40E_GLPRT_GOTCL(hw->port), 884 pf->stat_offsets_loaded, 885 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 886 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 887 pf->stat_offsets_loaded, 888 &osd->eth.rx_discards, 889 &nsd->eth.rx_discards); 890 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 891 I40E_GLPRT_UPRCL(hw->port), 892 pf->stat_offsets_loaded, 893 &osd->eth.rx_unicast, 894 &nsd->eth.rx_unicast); 895 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 896 I40E_GLPRT_MPRCL(hw->port), 897 pf->stat_offsets_loaded, 898 &osd->eth.rx_multicast, 899 &nsd->eth.rx_multicast); 900 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 901 I40E_GLPRT_BPRCL(hw->port), 902 pf->stat_offsets_loaded, 903 &osd->eth.rx_broadcast, 904 &nsd->eth.rx_broadcast); 905 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 906 I40E_GLPRT_UPTCL(hw->port), 907 pf->stat_offsets_loaded, 908 &osd->eth.tx_unicast, 909 &nsd->eth.tx_unicast); 910 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 911 I40E_GLPRT_MPTCL(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->eth.tx_multicast, 914 &nsd->eth.tx_multicast); 915 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 916 I40E_GLPRT_BPTCL(hw->port), 917 pf->stat_offsets_loaded, 918 &osd->eth.tx_broadcast, 919 &nsd->eth.tx_broadcast); 920 921 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 922 pf->stat_offsets_loaded, 923 &osd->tx_dropped_link_down, 924 &nsd->tx_dropped_link_down); 925 926 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 927 pf->stat_offsets_loaded, 928 &osd->crc_errors, &nsd->crc_errors); 929 930 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 931 pf->stat_offsets_loaded, 932 &osd->illegal_bytes, &nsd->illegal_bytes); 933 934 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 935 pf->stat_offsets_loaded, 936 &osd->mac_local_faults, 937 &nsd->mac_local_faults); 938 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 939 pf->stat_offsets_loaded, 940 &osd->mac_remote_faults, 941 &nsd->mac_remote_faults); 942 943 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 944 pf->stat_offsets_loaded, 945 &osd->rx_length_errors, 946 &nsd->rx_length_errors); 947 948 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 949 pf->stat_offsets_loaded, 950 &osd->link_xon_rx, &nsd->link_xon_rx); 951 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 952 pf->stat_offsets_loaded, 953 &osd->link_xon_tx, &nsd->link_xon_tx); 954 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 955 pf->stat_offsets_loaded, 956 &osd->link_xoff_rx, &nsd->link_xoff_rx); 957 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 958 pf->stat_offsets_loaded, 959 &osd->link_xoff_tx, &nsd->link_xoff_tx); 960 961 for (i = 0; i < 8; i++) { 962 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 963 pf->stat_offsets_loaded, 964 &osd->priority_xoff_rx[i], 965 &nsd->priority_xoff_rx[i]); 966 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 967 pf->stat_offsets_loaded, 968 &osd->priority_xon_rx[i], 969 &nsd->priority_xon_rx[i]); 970 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 971 pf->stat_offsets_loaded, 972 &osd->priority_xon_tx[i], 973 &nsd->priority_xon_tx[i]); 974 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 975 pf->stat_offsets_loaded, 976 &osd->priority_xoff_tx[i], 977 &nsd->priority_xoff_tx[i]); 978 i40e_stat_update32(hw, 979 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 980 pf->stat_offsets_loaded, 981 &osd->priority_xon_2_xoff[i], 982 &nsd->priority_xon_2_xoff[i]); 983 } 984 985 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 986 I40E_GLPRT_PRC64L(hw->port), 987 pf->stat_offsets_loaded, 988 &osd->rx_size_64, &nsd->rx_size_64); 989 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 990 I40E_GLPRT_PRC127L(hw->port), 991 pf->stat_offsets_loaded, 992 &osd->rx_size_127, &nsd->rx_size_127); 993 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 994 I40E_GLPRT_PRC255L(hw->port), 995 pf->stat_offsets_loaded, 996 &osd->rx_size_255, &nsd->rx_size_255); 997 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 998 I40E_GLPRT_PRC511L(hw->port), 999 pf->stat_offsets_loaded, 1000 &osd->rx_size_511, &nsd->rx_size_511); 1001 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 1002 I40E_GLPRT_PRC1023L(hw->port), 1003 pf->stat_offsets_loaded, 1004 &osd->rx_size_1023, &nsd->rx_size_1023); 1005 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 1006 I40E_GLPRT_PRC1522L(hw->port), 1007 pf->stat_offsets_loaded, 1008 &osd->rx_size_1522, &nsd->rx_size_1522); 1009 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 1010 I40E_GLPRT_PRC9522L(hw->port), 1011 pf->stat_offsets_loaded, 1012 &osd->rx_size_big, &nsd->rx_size_big); 1013 1014 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 1015 I40E_GLPRT_PTC64L(hw->port), 1016 pf->stat_offsets_loaded, 1017 &osd->tx_size_64, &nsd->tx_size_64); 1018 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 1019 I40E_GLPRT_PTC127L(hw->port), 1020 pf->stat_offsets_loaded, 1021 &osd->tx_size_127, &nsd->tx_size_127); 1022 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 1023 I40E_GLPRT_PTC255L(hw->port), 1024 pf->stat_offsets_loaded, 1025 &osd->tx_size_255, &nsd->tx_size_255); 1026 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 1027 I40E_GLPRT_PTC511L(hw->port), 1028 pf->stat_offsets_loaded, 1029 &osd->tx_size_511, &nsd->tx_size_511); 1030 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 1031 I40E_GLPRT_PTC1023L(hw->port), 1032 pf->stat_offsets_loaded, 1033 &osd->tx_size_1023, &nsd->tx_size_1023); 1034 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1035 I40E_GLPRT_PTC1522L(hw->port), 1036 pf->stat_offsets_loaded, 1037 &osd->tx_size_1522, &nsd->tx_size_1522); 1038 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1039 I40E_GLPRT_PTC9522L(hw->port), 1040 pf->stat_offsets_loaded, 1041 &osd->tx_size_big, &nsd->tx_size_big); 1042 1043 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1044 pf->stat_offsets_loaded, 1045 &osd->rx_undersize, &nsd->rx_undersize); 1046 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1047 pf->stat_offsets_loaded, 1048 &osd->rx_fragments, &nsd->rx_fragments); 1049 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1050 pf->stat_offsets_loaded, 1051 &osd->rx_oversize, &nsd->rx_oversize); 1052 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1053 pf->stat_offsets_loaded, 1054 &osd->rx_jabber, &nsd->rx_jabber); 1055 1056 /* FDIR stats */ 1057 i40e_stat_update_and_clear32(hw, 1058 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(hw->pf_id)), 1059 &nsd->fd_atr_match); 1060 i40e_stat_update_and_clear32(hw, 1061 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(hw->pf_id)), 1062 &nsd->fd_sb_match); 1063 i40e_stat_update_and_clear32(hw, 1064 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(hw->pf_id)), 1065 &nsd->fd_atr_tunnel_match); 1066 1067 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1068 nsd->tx_lpi_status = 1069 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1070 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1071 nsd->rx_lpi_status = 1072 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1073 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1074 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1075 pf->stat_offsets_loaded, 1076 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1077 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1078 pf->stat_offsets_loaded, 1079 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1080 1081 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1082 !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED)) 1083 nsd->fd_sb_status = true; 1084 else 1085 nsd->fd_sb_status = false; 1086 1087 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1088 !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED)) 1089 nsd->fd_atr_status = true; 1090 else 1091 nsd->fd_atr_status = false; 1092 1093 pf->stat_offsets_loaded = true; 1094 } 1095 1096 /** 1097 * i40e_update_stats - Update the various statistics counters. 1098 * @vsi: the VSI to be updated 1099 * 1100 * Update the various stats for this VSI and its related entities. 1101 **/ 1102 void i40e_update_stats(struct i40e_vsi *vsi) 1103 { 1104 struct i40e_pf *pf = vsi->back; 1105 1106 if (vsi == pf->vsi[pf->lan_vsi]) 1107 i40e_update_pf_stats(pf); 1108 1109 i40e_update_vsi_stats(vsi); 1110 } 1111 1112 /** 1113 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1114 * @vsi: the VSI to be searched 1115 * @macaddr: the MAC address 1116 * @vlan: the vlan 1117 * 1118 * Returns ptr to the filter object or NULL 1119 **/ 1120 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1121 const u8 *macaddr, s16 vlan) 1122 { 1123 struct i40e_mac_filter *f; 1124 u64 key; 1125 1126 if (!vsi || !macaddr) 1127 return NULL; 1128 1129 key = i40e_addr_to_hkey(macaddr); 1130 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1131 if ((ether_addr_equal(macaddr, f->macaddr)) && 1132 (vlan == f->vlan)) 1133 return f; 1134 } 1135 return NULL; 1136 } 1137 1138 /** 1139 * i40e_find_mac - Find a mac addr in the macvlan filters list 1140 * @vsi: the VSI to be searched 1141 * @macaddr: the MAC address we are searching for 1142 * 1143 * Returns the first filter with the provided MAC address or NULL if 1144 * MAC address was not found 1145 **/ 1146 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1147 { 1148 struct i40e_mac_filter *f; 1149 u64 key; 1150 1151 if (!vsi || !macaddr) 1152 return NULL; 1153 1154 key = i40e_addr_to_hkey(macaddr); 1155 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1156 if ((ether_addr_equal(macaddr, f->macaddr))) 1157 return f; 1158 } 1159 return NULL; 1160 } 1161 1162 /** 1163 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1164 * @vsi: the VSI to be searched 1165 * 1166 * Returns true if VSI is in vlan mode or false otherwise 1167 **/ 1168 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1169 { 1170 /* If we have a PVID, always operate in VLAN mode */ 1171 if (vsi->info.pvid) 1172 return true; 1173 1174 /* We need to operate in VLAN mode whenever we have any filters with 1175 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1176 * time, incurring search cost repeatedly. However, we can notice two 1177 * things: 1178 * 1179 * 1) the only place where we can gain a VLAN filter is in 1180 * i40e_add_filter. 1181 * 1182 * 2) the only place where filters are actually removed is in 1183 * i40e_sync_filters_subtask. 1184 * 1185 * Thus, we can simply use a boolean value, has_vlan_filters which we 1186 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1187 * we have to perform the full search after deleting filters in 1188 * i40e_sync_filters_subtask, but we already have to search 1189 * filters here and can perform the check at the same time. This 1190 * results in avoiding embedding a loop for VLAN mode inside another 1191 * loop over all the filters, and should maintain correctness as noted 1192 * above. 1193 */ 1194 return vsi->has_vlan_filter; 1195 } 1196 1197 /** 1198 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1199 * @vsi: the VSI to configure 1200 * @tmp_add_list: list of filters ready to be added 1201 * @tmp_del_list: list of filters ready to be deleted 1202 * @vlan_filters: the number of active VLAN filters 1203 * 1204 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1205 * behave as expected. If we have any active VLAN filters remaining or about 1206 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1207 * so that they only match against untagged traffic. If we no longer have any 1208 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1209 * so that they match against both tagged and untagged traffic. In this way, 1210 * we ensure that we correctly receive the desired traffic. This ensures that 1211 * when we have an active VLAN we will receive only untagged traffic and 1212 * traffic matching active VLANs. If we have no active VLANs then we will 1213 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1214 * 1215 * Finally, in a similar fashion, this function also corrects filters when 1216 * there is an active PVID assigned to this VSI. 1217 * 1218 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1219 * 1220 * This function is only expected to be called from within 1221 * i40e_sync_vsi_filters. 1222 * 1223 * NOTE: This function expects to be called while under the 1224 * mac_filter_hash_lock 1225 */ 1226 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1227 struct hlist_head *tmp_add_list, 1228 struct hlist_head *tmp_del_list, 1229 int vlan_filters) 1230 { 1231 s16 pvid = le16_to_cpu(vsi->info.pvid); 1232 struct i40e_mac_filter *f, *add_head; 1233 struct i40e_new_mac_filter *new; 1234 struct hlist_node *h; 1235 int bkt, new_vlan; 1236 1237 /* To determine if a particular filter needs to be replaced we 1238 * have the three following conditions: 1239 * 1240 * a) if we have a PVID assigned, then all filters which are 1241 * not marked as VLAN=PVID must be replaced with filters that 1242 * are. 1243 * b) otherwise, if we have any active VLANS, all filters 1244 * which are marked as VLAN=-1 must be replaced with 1245 * filters marked as VLAN=0 1246 * c) finally, if we do not have any active VLANS, all filters 1247 * which are marked as VLAN=0 must be replaced with filters 1248 * marked as VLAN=-1 1249 */ 1250 1251 /* Update the filters about to be added in place */ 1252 hlist_for_each_entry(new, tmp_add_list, hlist) { 1253 if (pvid && new->f->vlan != pvid) 1254 new->f->vlan = pvid; 1255 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1256 new->f->vlan = 0; 1257 else if (!vlan_filters && new->f->vlan == 0) 1258 new->f->vlan = I40E_VLAN_ANY; 1259 } 1260 1261 /* Update the remaining active filters */ 1262 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1263 /* Combine the checks for whether a filter needs to be changed 1264 * and then determine the new VLAN inside the if block, in 1265 * order to avoid duplicating code for adding the new filter 1266 * then deleting the old filter. 1267 */ 1268 if ((pvid && f->vlan != pvid) || 1269 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1270 (!vlan_filters && f->vlan == 0)) { 1271 /* Determine the new vlan we will be adding */ 1272 if (pvid) 1273 new_vlan = pvid; 1274 else if (vlan_filters) 1275 new_vlan = 0; 1276 else 1277 new_vlan = I40E_VLAN_ANY; 1278 1279 /* Create the new filter */ 1280 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1281 if (!add_head) 1282 return -ENOMEM; 1283 1284 /* Create a temporary i40e_new_mac_filter */ 1285 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1286 if (!new) 1287 return -ENOMEM; 1288 1289 new->f = add_head; 1290 new->state = add_head->state; 1291 1292 /* Add the new filter to the tmp list */ 1293 hlist_add_head(&new->hlist, tmp_add_list); 1294 1295 /* Put the original filter into the delete list */ 1296 f->state = I40E_FILTER_REMOVE; 1297 hash_del(&f->hlist); 1298 hlist_add_head(&f->hlist, tmp_del_list); 1299 } 1300 } 1301 1302 vsi->has_vlan_filter = !!vlan_filters; 1303 1304 return 0; 1305 } 1306 1307 /** 1308 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1309 * @vsi: the PF Main VSI - inappropriate for any other VSI 1310 * @macaddr: the MAC address 1311 * 1312 * Remove whatever filter the firmware set up so the driver can manage 1313 * its own filtering intelligently. 1314 **/ 1315 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1316 { 1317 struct i40e_aqc_remove_macvlan_element_data element; 1318 struct i40e_pf *pf = vsi->back; 1319 1320 /* Only appropriate for the PF main VSI */ 1321 if (vsi->type != I40E_VSI_MAIN) 1322 return; 1323 1324 memset(&element, 0, sizeof(element)); 1325 ether_addr_copy(element.mac_addr, macaddr); 1326 element.vlan_tag = 0; 1327 /* Ignore error returns, some firmware does it this way... */ 1328 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1329 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1330 1331 memset(&element, 0, sizeof(element)); 1332 ether_addr_copy(element.mac_addr, macaddr); 1333 element.vlan_tag = 0; 1334 /* ...and some firmware does it this way. */ 1335 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1336 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1337 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1338 } 1339 1340 /** 1341 * i40e_add_filter - Add a mac/vlan filter to the VSI 1342 * @vsi: the VSI to be searched 1343 * @macaddr: the MAC address 1344 * @vlan: the vlan 1345 * 1346 * Returns ptr to the filter object or NULL when no memory available. 1347 * 1348 * NOTE: This function is expected to be called with mac_filter_hash_lock 1349 * being held. 1350 **/ 1351 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1352 const u8 *macaddr, s16 vlan) 1353 { 1354 struct i40e_mac_filter *f; 1355 u64 key; 1356 1357 if (!vsi || !macaddr) 1358 return NULL; 1359 1360 f = i40e_find_filter(vsi, macaddr, vlan); 1361 if (!f) { 1362 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1363 if (!f) 1364 return NULL; 1365 1366 /* Update the boolean indicating if we need to function in 1367 * VLAN mode. 1368 */ 1369 if (vlan >= 0) 1370 vsi->has_vlan_filter = true; 1371 1372 ether_addr_copy(f->macaddr, macaddr); 1373 f->vlan = vlan; 1374 /* If we're in overflow promisc mode, set the state directly 1375 * to failed, so we don't bother to try sending the filter 1376 * to the hardware. 1377 */ 1378 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state)) 1379 f->state = I40E_FILTER_FAILED; 1380 else 1381 f->state = I40E_FILTER_NEW; 1382 INIT_HLIST_NODE(&f->hlist); 1383 1384 key = i40e_addr_to_hkey(macaddr); 1385 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1386 1387 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1388 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1389 } 1390 1391 /* If we're asked to add a filter that has been marked for removal, it 1392 * is safe to simply restore it to active state. __i40e_del_filter 1393 * will have simply deleted any filters which were previously marked 1394 * NEW or FAILED, so if it is currently marked REMOVE it must have 1395 * previously been ACTIVE. Since we haven't yet run the sync filters 1396 * task, just restore this filter to the ACTIVE state so that the 1397 * sync task leaves it in place 1398 */ 1399 if (f->state == I40E_FILTER_REMOVE) 1400 f->state = I40E_FILTER_ACTIVE; 1401 1402 return f; 1403 } 1404 1405 /** 1406 * __i40e_del_filter - Remove a specific filter from the VSI 1407 * @vsi: VSI to remove from 1408 * @f: the filter to remove from the list 1409 * 1410 * This function should be called instead of i40e_del_filter only if you know 1411 * the exact filter you will remove already, such as via i40e_find_filter or 1412 * i40e_find_mac. 1413 * 1414 * NOTE: This function is expected to be called with mac_filter_hash_lock 1415 * being held. 1416 * ANOTHER NOTE: This function MUST be called from within the context of 1417 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1418 * instead of list_for_each_entry(). 1419 **/ 1420 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1421 { 1422 if (!f) 1423 return; 1424 1425 /* If the filter was never added to firmware then we can just delete it 1426 * directly and we don't want to set the status to remove or else an 1427 * admin queue command will unnecessarily fire. 1428 */ 1429 if ((f->state == I40E_FILTER_FAILED) || 1430 (f->state == I40E_FILTER_NEW)) { 1431 hash_del(&f->hlist); 1432 kfree(f); 1433 } else { 1434 f->state = I40E_FILTER_REMOVE; 1435 } 1436 1437 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1438 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1439 } 1440 1441 /** 1442 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1443 * @vsi: the VSI to be searched 1444 * @macaddr: the MAC address 1445 * @vlan: the VLAN 1446 * 1447 * NOTE: This function is expected to be called with mac_filter_hash_lock 1448 * being held. 1449 * ANOTHER NOTE: This function MUST be called from within the context of 1450 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1451 * instead of list_for_each_entry(). 1452 **/ 1453 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1454 { 1455 struct i40e_mac_filter *f; 1456 1457 if (!vsi || !macaddr) 1458 return; 1459 1460 f = i40e_find_filter(vsi, macaddr, vlan); 1461 __i40e_del_filter(vsi, f); 1462 } 1463 1464 /** 1465 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1466 * @vsi: the VSI to be searched 1467 * @macaddr: the mac address to be filtered 1468 * 1469 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1470 * go through all the macvlan filters and add a macvlan filter for each 1471 * unique vlan that already exists. If a PVID has been assigned, instead only 1472 * add the macaddr to that VLAN. 1473 * 1474 * Returns last filter added on success, else NULL 1475 **/ 1476 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1477 const u8 *macaddr) 1478 { 1479 struct i40e_mac_filter *f, *add = NULL; 1480 struct hlist_node *h; 1481 int bkt; 1482 1483 if (vsi->info.pvid) 1484 return i40e_add_filter(vsi, macaddr, 1485 le16_to_cpu(vsi->info.pvid)); 1486 1487 if (!i40e_is_vsi_in_vlan(vsi)) 1488 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1489 1490 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1491 if (f->state == I40E_FILTER_REMOVE) 1492 continue; 1493 add = i40e_add_filter(vsi, macaddr, f->vlan); 1494 if (!add) 1495 return NULL; 1496 } 1497 1498 return add; 1499 } 1500 1501 /** 1502 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1503 * @vsi: the VSI to be searched 1504 * @macaddr: the mac address to be removed 1505 * 1506 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1507 * associated with. 1508 * 1509 * Returns 0 for success, or error 1510 **/ 1511 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1512 { 1513 struct i40e_mac_filter *f; 1514 struct hlist_node *h; 1515 bool found = false; 1516 int bkt; 1517 1518 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1519 "Missing mac_filter_hash_lock\n"); 1520 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1521 if (ether_addr_equal(macaddr, f->macaddr)) { 1522 __i40e_del_filter(vsi, f); 1523 found = true; 1524 } 1525 } 1526 1527 if (found) 1528 return 0; 1529 else 1530 return -ENOENT; 1531 } 1532 1533 /** 1534 * i40e_set_mac - NDO callback to set mac address 1535 * @netdev: network interface device structure 1536 * @p: pointer to an address structure 1537 * 1538 * Returns 0 on success, negative on failure 1539 **/ 1540 static int i40e_set_mac(struct net_device *netdev, void *p) 1541 { 1542 struct i40e_netdev_priv *np = netdev_priv(netdev); 1543 struct i40e_vsi *vsi = np->vsi; 1544 struct i40e_pf *pf = vsi->back; 1545 struct i40e_hw *hw = &pf->hw; 1546 struct sockaddr *addr = p; 1547 1548 if (!is_valid_ether_addr(addr->sa_data)) 1549 return -EADDRNOTAVAIL; 1550 1551 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1552 netdev_info(netdev, "already using mac address %pM\n", 1553 addr->sa_data); 1554 return 0; 1555 } 1556 1557 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 1558 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 1559 return -EADDRNOTAVAIL; 1560 1561 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1562 netdev_info(netdev, "returning to hw mac address %pM\n", 1563 hw->mac.addr); 1564 else 1565 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1566 1567 spin_lock_bh(&vsi->mac_filter_hash_lock); 1568 i40e_del_mac_filter(vsi, netdev->dev_addr); 1569 i40e_add_mac_filter(vsi, addr->sa_data); 1570 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1571 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1572 if (vsi->type == I40E_VSI_MAIN) { 1573 i40e_status ret; 1574 1575 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1576 I40E_AQC_WRITE_TYPE_LAA_WOL, 1577 addr->sa_data, NULL); 1578 if (ret) 1579 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1580 i40e_stat_str(hw, ret), 1581 i40e_aq_str(hw, hw->aq.asq_last_status)); 1582 } 1583 1584 /* schedule our worker thread which will take care of 1585 * applying the new filter changes 1586 */ 1587 i40e_service_event_schedule(vsi->back); 1588 return 0; 1589 } 1590 1591 /** 1592 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1593 * @vsi: the VSI being setup 1594 * @ctxt: VSI context structure 1595 * @enabled_tc: Enabled TCs bitmap 1596 * @is_add: True if called before Add VSI 1597 * 1598 * Setup VSI queue mapping for enabled traffic classes. 1599 **/ 1600 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1601 struct i40e_vsi_context *ctxt, 1602 u8 enabled_tc, 1603 bool is_add) 1604 { 1605 struct i40e_pf *pf = vsi->back; 1606 u16 sections = 0; 1607 u8 netdev_tc = 0; 1608 u16 numtc = 0; 1609 u16 qcount; 1610 u8 offset; 1611 u16 qmap; 1612 int i; 1613 u16 num_tc_qps = 0; 1614 1615 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1616 offset = 0; 1617 1618 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1619 /* Find numtc from enabled TC bitmap */ 1620 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1621 if (enabled_tc & BIT(i)) /* TC is enabled */ 1622 numtc++; 1623 } 1624 if (!numtc) { 1625 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1626 numtc = 1; 1627 } 1628 } else { 1629 /* At least TC0 is enabled in case of non-DCB case */ 1630 numtc = 1; 1631 } 1632 1633 vsi->tc_config.numtc = numtc; 1634 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1635 /* Number of queues per enabled TC */ 1636 qcount = vsi->alloc_queue_pairs; 1637 1638 num_tc_qps = qcount / numtc; 1639 num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf)); 1640 1641 /* Setup queue offset/count for all TCs for given VSI */ 1642 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1643 /* See if the given TC is enabled for the given VSI */ 1644 if (vsi->tc_config.enabled_tc & BIT(i)) { 1645 /* TC is enabled */ 1646 int pow, num_qps; 1647 1648 switch (vsi->type) { 1649 case I40E_VSI_MAIN: 1650 qcount = min_t(int, pf->alloc_rss_size, 1651 num_tc_qps); 1652 break; 1653 case I40E_VSI_FDIR: 1654 case I40E_VSI_SRIOV: 1655 case I40E_VSI_VMDQ2: 1656 default: 1657 qcount = num_tc_qps; 1658 WARN_ON(i != 0); 1659 break; 1660 } 1661 vsi->tc_config.tc_info[i].qoffset = offset; 1662 vsi->tc_config.tc_info[i].qcount = qcount; 1663 1664 /* find the next higher power-of-2 of num queue pairs */ 1665 num_qps = qcount; 1666 pow = 0; 1667 while (num_qps && (BIT_ULL(pow) < qcount)) { 1668 pow++; 1669 num_qps >>= 1; 1670 } 1671 1672 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1673 qmap = 1674 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1675 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1676 1677 offset += qcount; 1678 } else { 1679 /* TC is not enabled so set the offset to 1680 * default queue and allocate one queue 1681 * for the given TC. 1682 */ 1683 vsi->tc_config.tc_info[i].qoffset = 0; 1684 vsi->tc_config.tc_info[i].qcount = 1; 1685 vsi->tc_config.tc_info[i].netdev_tc = 0; 1686 1687 qmap = 0; 1688 } 1689 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1690 } 1691 1692 /* Set actual Tx/Rx queue pairs */ 1693 vsi->num_queue_pairs = offset; 1694 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1695 if (vsi->req_queue_pairs > 0) 1696 vsi->num_queue_pairs = vsi->req_queue_pairs; 1697 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1698 vsi->num_queue_pairs = pf->num_lan_msix; 1699 } 1700 1701 /* Scheduler section valid can only be set for ADD VSI */ 1702 if (is_add) { 1703 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1704 1705 ctxt->info.up_enable_bits = enabled_tc; 1706 } 1707 if (vsi->type == I40E_VSI_SRIOV) { 1708 ctxt->info.mapping_flags |= 1709 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1710 for (i = 0; i < vsi->num_queue_pairs; i++) 1711 ctxt->info.queue_mapping[i] = 1712 cpu_to_le16(vsi->base_queue + i); 1713 } else { 1714 ctxt->info.mapping_flags |= 1715 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1716 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1717 } 1718 ctxt->info.valid_sections |= cpu_to_le16(sections); 1719 } 1720 1721 /** 1722 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1723 * @netdev: the netdevice 1724 * @addr: address to add 1725 * 1726 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1727 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1728 */ 1729 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1730 { 1731 struct i40e_netdev_priv *np = netdev_priv(netdev); 1732 struct i40e_vsi *vsi = np->vsi; 1733 1734 if (i40e_add_mac_filter(vsi, addr)) 1735 return 0; 1736 else 1737 return -ENOMEM; 1738 } 1739 1740 /** 1741 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1742 * @netdev: the netdevice 1743 * @addr: address to add 1744 * 1745 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1746 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1747 */ 1748 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1749 { 1750 struct i40e_netdev_priv *np = netdev_priv(netdev); 1751 struct i40e_vsi *vsi = np->vsi; 1752 1753 i40e_del_mac_filter(vsi, addr); 1754 1755 return 0; 1756 } 1757 1758 /** 1759 * i40e_set_rx_mode - NDO callback to set the netdev filters 1760 * @netdev: network interface device structure 1761 **/ 1762 static void i40e_set_rx_mode(struct net_device *netdev) 1763 { 1764 struct i40e_netdev_priv *np = netdev_priv(netdev); 1765 struct i40e_vsi *vsi = np->vsi; 1766 1767 spin_lock_bh(&vsi->mac_filter_hash_lock); 1768 1769 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1770 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1771 1772 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1773 1774 /* check for other flag changes */ 1775 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1776 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1777 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1778 } 1779 1780 /* schedule our worker thread which will take care of 1781 * applying the new filter changes 1782 */ 1783 i40e_service_event_schedule(vsi->back); 1784 } 1785 1786 /** 1787 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1788 * @vsi: Pointer to VSI struct 1789 * @from: Pointer to list which contains MAC filter entries - changes to 1790 * those entries needs to be undone. 1791 * 1792 * MAC filter entries from this list were slated for deletion. 1793 **/ 1794 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1795 struct hlist_head *from) 1796 { 1797 struct i40e_mac_filter *f; 1798 struct hlist_node *h; 1799 1800 hlist_for_each_entry_safe(f, h, from, hlist) { 1801 u64 key = i40e_addr_to_hkey(f->macaddr); 1802 1803 /* Move the element back into MAC filter list*/ 1804 hlist_del(&f->hlist); 1805 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1806 } 1807 } 1808 1809 /** 1810 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1811 * @vsi: Pointer to vsi struct 1812 * @from: Pointer to list which contains MAC filter entries - changes to 1813 * those entries needs to be undone. 1814 * 1815 * MAC filter entries from this list were slated for addition. 1816 **/ 1817 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 1818 struct hlist_head *from) 1819 { 1820 struct i40e_new_mac_filter *new; 1821 struct hlist_node *h; 1822 1823 hlist_for_each_entry_safe(new, h, from, hlist) { 1824 /* We can simply free the wrapper structure */ 1825 hlist_del(&new->hlist); 1826 kfree(new); 1827 } 1828 } 1829 1830 /** 1831 * i40e_next_entry - Get the next non-broadcast filter from a list 1832 * @next: pointer to filter in list 1833 * 1834 * Returns the next non-broadcast filter in the list. Required so that we 1835 * ignore broadcast filters within the list, since these are not handled via 1836 * the normal firmware update path. 1837 */ 1838 static 1839 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 1840 { 1841 hlist_for_each_entry_continue(next, hlist) { 1842 if (!is_broadcast_ether_addr(next->f->macaddr)) 1843 return next; 1844 } 1845 1846 return NULL; 1847 } 1848 1849 /** 1850 * i40e_update_filter_state - Update filter state based on return data 1851 * from firmware 1852 * @count: Number of filters added 1853 * @add_list: return data from fw 1854 * @head: pointer to first filter in current batch 1855 * 1856 * MAC filter entries from list were slated to be added to device. Returns 1857 * number of successful filters. Note that 0 does NOT mean success! 1858 **/ 1859 static int 1860 i40e_update_filter_state(int count, 1861 struct i40e_aqc_add_macvlan_element_data *add_list, 1862 struct i40e_new_mac_filter *add_head) 1863 { 1864 int retval = 0; 1865 int i; 1866 1867 for (i = 0; i < count; i++) { 1868 /* Always check status of each filter. We don't need to check 1869 * the firmware return status because we pre-set the filter 1870 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 1871 * request to the adminq. Thus, if it no longer matches then 1872 * we know the filter is active. 1873 */ 1874 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 1875 add_head->state = I40E_FILTER_FAILED; 1876 } else { 1877 add_head->state = I40E_FILTER_ACTIVE; 1878 retval++; 1879 } 1880 1881 add_head = i40e_next_filter(add_head); 1882 if (!add_head) 1883 break; 1884 } 1885 1886 return retval; 1887 } 1888 1889 /** 1890 * i40e_aqc_del_filters - Request firmware to delete a set of filters 1891 * @vsi: ptr to the VSI 1892 * @vsi_name: name to display in messages 1893 * @list: the list of filters to send to firmware 1894 * @num_del: the number of filters to delete 1895 * @retval: Set to -EIO on failure to delete 1896 * 1897 * Send a request to firmware via AdminQ to delete a set of filters. Uses 1898 * *retval instead of a return value so that success does not force ret_val to 1899 * be set to 0. This ensures that a sequence of calls to this function 1900 * preserve the previous value of *retval on successful delete. 1901 */ 1902 static 1903 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 1904 struct i40e_aqc_remove_macvlan_element_data *list, 1905 int num_del, int *retval) 1906 { 1907 struct i40e_hw *hw = &vsi->back->hw; 1908 i40e_status aq_ret; 1909 int aq_err; 1910 1911 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 1912 aq_err = hw->aq.asq_last_status; 1913 1914 /* Explicitly ignore and do not report when firmware returns ENOENT */ 1915 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 1916 *retval = -EIO; 1917 dev_info(&vsi->back->pdev->dev, 1918 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 1919 vsi_name, i40e_stat_str(hw, aq_ret), 1920 i40e_aq_str(hw, aq_err)); 1921 } 1922 } 1923 1924 /** 1925 * i40e_aqc_add_filters - Request firmware to add a set of filters 1926 * @vsi: ptr to the VSI 1927 * @vsi_name: name to display in messages 1928 * @list: the list of filters to send to firmware 1929 * @add_head: Position in the add hlist 1930 * @num_add: the number of filters to add 1931 * @promisc_change: set to true on exit if promiscuous mode was forced on 1932 * 1933 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 1934 * promisc_changed to true if the firmware has run out of space for more 1935 * filters. 1936 */ 1937 static 1938 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 1939 struct i40e_aqc_add_macvlan_element_data *list, 1940 struct i40e_new_mac_filter *add_head, 1941 int num_add, bool *promisc_changed) 1942 { 1943 struct i40e_hw *hw = &vsi->back->hw; 1944 int aq_err, fcnt; 1945 1946 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 1947 aq_err = hw->aq.asq_last_status; 1948 fcnt = i40e_update_filter_state(num_add, list, add_head); 1949 1950 if (fcnt != num_add) { 1951 *promisc_changed = true; 1952 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 1953 dev_warn(&vsi->back->pdev->dev, 1954 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 1955 i40e_aq_str(hw, aq_err), 1956 vsi_name); 1957 } 1958 } 1959 1960 /** 1961 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 1962 * @vsi: pointer to the VSI 1963 * @f: filter data 1964 * 1965 * This function sets or clears the promiscuous broadcast flags for VLAN 1966 * filters in order to properly receive broadcast frames. Assumes that only 1967 * broadcast filters are passed. 1968 * 1969 * Returns status indicating success or failure; 1970 **/ 1971 static i40e_status 1972 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 1973 struct i40e_mac_filter *f) 1974 { 1975 bool enable = f->state == I40E_FILTER_NEW; 1976 struct i40e_hw *hw = &vsi->back->hw; 1977 i40e_status aq_ret; 1978 1979 if (f->vlan == I40E_VLAN_ANY) { 1980 aq_ret = i40e_aq_set_vsi_broadcast(hw, 1981 vsi->seid, 1982 enable, 1983 NULL); 1984 } else { 1985 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 1986 vsi->seid, 1987 enable, 1988 f->vlan, 1989 NULL); 1990 } 1991 1992 if (aq_ret) 1993 dev_warn(&vsi->back->pdev->dev, 1994 "Error %s setting broadcast promiscuous mode on %s\n", 1995 i40e_aq_str(hw, hw->aq.asq_last_status), 1996 vsi_name); 1997 1998 return aq_ret; 1999 } 2000 2001 /** 2002 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 2003 * @vsi: ptr to the VSI 2004 * 2005 * Push any outstanding VSI filter changes through the AdminQ. 2006 * 2007 * Returns 0 or error value 2008 **/ 2009 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 2010 { 2011 struct hlist_head tmp_add_list, tmp_del_list; 2012 struct i40e_mac_filter *f; 2013 struct i40e_new_mac_filter *new, *add_head = NULL; 2014 struct i40e_hw *hw = &vsi->back->hw; 2015 unsigned int failed_filters = 0; 2016 unsigned int vlan_filters = 0; 2017 bool promisc_changed = false; 2018 char vsi_name[16] = "PF"; 2019 int filter_list_len = 0; 2020 i40e_status aq_ret = 0; 2021 u32 changed_flags = 0; 2022 struct hlist_node *h; 2023 struct i40e_pf *pf; 2024 int num_add = 0; 2025 int num_del = 0; 2026 int retval = 0; 2027 u16 cmd_flags; 2028 int list_size; 2029 int bkt; 2030 2031 /* empty array typed pointers, kcalloc later */ 2032 struct i40e_aqc_add_macvlan_element_data *add_list; 2033 struct i40e_aqc_remove_macvlan_element_data *del_list; 2034 2035 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2036 usleep_range(1000, 2000); 2037 pf = vsi->back; 2038 2039 if (vsi->netdev) { 2040 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2041 vsi->current_netdev_flags = vsi->netdev->flags; 2042 } 2043 2044 INIT_HLIST_HEAD(&tmp_add_list); 2045 INIT_HLIST_HEAD(&tmp_del_list); 2046 2047 if (vsi->type == I40E_VSI_SRIOV) 2048 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2049 else if (vsi->type != I40E_VSI_MAIN) 2050 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2051 2052 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2053 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2054 2055 spin_lock_bh(&vsi->mac_filter_hash_lock); 2056 /* Create a list of filters to delete. */ 2057 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2058 if (f->state == I40E_FILTER_REMOVE) { 2059 /* Move the element into temporary del_list */ 2060 hash_del(&f->hlist); 2061 hlist_add_head(&f->hlist, &tmp_del_list); 2062 2063 /* Avoid counting removed filters */ 2064 continue; 2065 } 2066 if (f->state == I40E_FILTER_NEW) { 2067 /* Create a temporary i40e_new_mac_filter */ 2068 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2069 if (!new) 2070 goto err_no_memory_locked; 2071 2072 /* Store pointer to the real filter */ 2073 new->f = f; 2074 new->state = f->state; 2075 2076 /* Add it to the hash list */ 2077 hlist_add_head(&new->hlist, &tmp_add_list); 2078 } 2079 2080 /* Count the number of active (current and new) VLAN 2081 * filters we have now. Does not count filters which 2082 * are marked for deletion. 2083 */ 2084 if (f->vlan > 0) 2085 vlan_filters++; 2086 } 2087 2088 retval = i40e_correct_mac_vlan_filters(vsi, 2089 &tmp_add_list, 2090 &tmp_del_list, 2091 vlan_filters); 2092 if (retval) 2093 goto err_no_memory_locked; 2094 2095 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2096 } 2097 2098 /* Now process 'del_list' outside the lock */ 2099 if (!hlist_empty(&tmp_del_list)) { 2100 filter_list_len = hw->aq.asq_buf_size / 2101 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2102 list_size = filter_list_len * 2103 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2104 del_list = kzalloc(list_size, GFP_ATOMIC); 2105 if (!del_list) 2106 goto err_no_memory; 2107 2108 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2109 cmd_flags = 0; 2110 2111 /* handle broadcast filters by updating the broadcast 2112 * promiscuous flag and release filter list. 2113 */ 2114 if (is_broadcast_ether_addr(f->macaddr)) { 2115 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2116 2117 hlist_del(&f->hlist); 2118 kfree(f); 2119 continue; 2120 } 2121 2122 /* add to delete list */ 2123 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2124 if (f->vlan == I40E_VLAN_ANY) { 2125 del_list[num_del].vlan_tag = 0; 2126 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2127 } else { 2128 del_list[num_del].vlan_tag = 2129 cpu_to_le16((u16)(f->vlan)); 2130 } 2131 2132 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2133 del_list[num_del].flags = cmd_flags; 2134 num_del++; 2135 2136 /* flush a full buffer */ 2137 if (num_del == filter_list_len) { 2138 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2139 num_del, &retval); 2140 memset(del_list, 0, list_size); 2141 num_del = 0; 2142 } 2143 /* Release memory for MAC filter entries which were 2144 * synced up with HW. 2145 */ 2146 hlist_del(&f->hlist); 2147 kfree(f); 2148 } 2149 2150 if (num_del) { 2151 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2152 num_del, &retval); 2153 } 2154 2155 kfree(del_list); 2156 del_list = NULL; 2157 } 2158 2159 if (!hlist_empty(&tmp_add_list)) { 2160 /* Do all the adds now. */ 2161 filter_list_len = hw->aq.asq_buf_size / 2162 sizeof(struct i40e_aqc_add_macvlan_element_data); 2163 list_size = filter_list_len * 2164 sizeof(struct i40e_aqc_add_macvlan_element_data); 2165 add_list = kzalloc(list_size, GFP_ATOMIC); 2166 if (!add_list) 2167 goto err_no_memory; 2168 2169 num_add = 0; 2170 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2171 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, 2172 vsi->state)) { 2173 new->state = I40E_FILTER_FAILED; 2174 continue; 2175 } 2176 2177 /* handle broadcast filters by updating the broadcast 2178 * promiscuous flag instead of adding a MAC filter. 2179 */ 2180 if (is_broadcast_ether_addr(new->f->macaddr)) { 2181 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2182 new->f)) 2183 new->state = I40E_FILTER_FAILED; 2184 else 2185 new->state = I40E_FILTER_ACTIVE; 2186 continue; 2187 } 2188 2189 /* add to add array */ 2190 if (num_add == 0) 2191 add_head = new; 2192 cmd_flags = 0; 2193 ether_addr_copy(add_list[num_add].mac_addr, 2194 new->f->macaddr); 2195 if (new->f->vlan == I40E_VLAN_ANY) { 2196 add_list[num_add].vlan_tag = 0; 2197 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2198 } else { 2199 add_list[num_add].vlan_tag = 2200 cpu_to_le16((u16)(new->f->vlan)); 2201 } 2202 add_list[num_add].queue_number = 0; 2203 /* set invalid match method for later detection */ 2204 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2205 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2206 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2207 num_add++; 2208 2209 /* flush a full buffer */ 2210 if (num_add == filter_list_len) { 2211 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2212 add_head, num_add, 2213 &promisc_changed); 2214 memset(add_list, 0, list_size); 2215 num_add = 0; 2216 } 2217 } 2218 if (num_add) { 2219 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2220 num_add, &promisc_changed); 2221 } 2222 /* Now move all of the filters from the temp add list back to 2223 * the VSI's list. 2224 */ 2225 spin_lock_bh(&vsi->mac_filter_hash_lock); 2226 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2227 /* Only update the state if we're still NEW */ 2228 if (new->f->state == I40E_FILTER_NEW) 2229 new->f->state = new->state; 2230 hlist_del(&new->hlist); 2231 kfree(new); 2232 } 2233 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2234 kfree(add_list); 2235 add_list = NULL; 2236 } 2237 2238 /* Determine the number of active and failed filters. */ 2239 spin_lock_bh(&vsi->mac_filter_hash_lock); 2240 vsi->active_filters = 0; 2241 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2242 if (f->state == I40E_FILTER_ACTIVE) 2243 vsi->active_filters++; 2244 else if (f->state == I40E_FILTER_FAILED) 2245 failed_filters++; 2246 } 2247 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2248 2249 /* If promiscuous mode has changed, we need to calculate a new 2250 * threshold for when we are safe to exit 2251 */ 2252 if (promisc_changed) 2253 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2254 2255 /* Check if we are able to exit overflow promiscuous mode. We can 2256 * safely exit if we didn't just enter, we no longer have any failed 2257 * filters, and we have reduced filters below the threshold value. 2258 */ 2259 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) && 2260 !promisc_changed && !failed_filters && 2261 (vsi->active_filters < vsi->promisc_threshold)) { 2262 dev_info(&pf->pdev->dev, 2263 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2264 vsi_name); 2265 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2266 promisc_changed = true; 2267 vsi->promisc_threshold = 0; 2268 } 2269 2270 /* if the VF is not trusted do not do promisc */ 2271 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2272 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2273 goto out; 2274 } 2275 2276 /* check for changes in promiscuous modes */ 2277 if (changed_flags & IFF_ALLMULTI) { 2278 bool cur_multipromisc; 2279 2280 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2281 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2282 vsi->seid, 2283 cur_multipromisc, 2284 NULL); 2285 if (aq_ret) { 2286 retval = i40e_aq_rc_to_posix(aq_ret, 2287 hw->aq.asq_last_status); 2288 dev_info(&pf->pdev->dev, 2289 "set multi promisc failed on %s, err %s aq_err %s\n", 2290 vsi_name, 2291 i40e_stat_str(hw, aq_ret), 2292 i40e_aq_str(hw, hw->aq.asq_last_status)); 2293 } 2294 } 2295 2296 if ((changed_flags & IFF_PROMISC) || promisc_changed) { 2297 bool cur_promisc; 2298 2299 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2300 test_bit(__I40E_VSI_OVERFLOW_PROMISC, 2301 vsi->state)); 2302 if ((vsi->type == I40E_VSI_MAIN) && 2303 (pf->lan_veb != I40E_NO_VEB) && 2304 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2305 /* set defport ON for Main VSI instead of true promisc 2306 * this way we will get all unicast/multicast and VLAN 2307 * promisc behavior but will not get VF or VMDq traffic 2308 * replicated on the Main VSI. 2309 */ 2310 if (pf->cur_promisc != cur_promisc) { 2311 pf->cur_promisc = cur_promisc; 2312 if (cur_promisc) 2313 aq_ret = 2314 i40e_aq_set_default_vsi(hw, 2315 vsi->seid, 2316 NULL); 2317 else 2318 aq_ret = 2319 i40e_aq_clear_default_vsi(hw, 2320 vsi->seid, 2321 NULL); 2322 if (aq_ret) { 2323 retval = i40e_aq_rc_to_posix(aq_ret, 2324 hw->aq.asq_last_status); 2325 dev_info(&pf->pdev->dev, 2326 "Set default VSI failed on %s, err %s, aq_err %s\n", 2327 vsi_name, 2328 i40e_stat_str(hw, aq_ret), 2329 i40e_aq_str(hw, 2330 hw->aq.asq_last_status)); 2331 } 2332 } 2333 } else { 2334 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2335 hw, 2336 vsi->seid, 2337 cur_promisc, NULL, 2338 true); 2339 if (aq_ret) { 2340 retval = 2341 i40e_aq_rc_to_posix(aq_ret, 2342 hw->aq.asq_last_status); 2343 dev_info(&pf->pdev->dev, 2344 "set unicast promisc failed on %s, err %s, aq_err %s\n", 2345 vsi_name, 2346 i40e_stat_str(hw, aq_ret), 2347 i40e_aq_str(hw, 2348 hw->aq.asq_last_status)); 2349 } 2350 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2351 hw, 2352 vsi->seid, 2353 cur_promisc, NULL); 2354 if (aq_ret) { 2355 retval = 2356 i40e_aq_rc_to_posix(aq_ret, 2357 hw->aq.asq_last_status); 2358 dev_info(&pf->pdev->dev, 2359 "set multicast promisc failed on %s, err %s, aq_err %s\n", 2360 vsi_name, 2361 i40e_stat_str(hw, aq_ret), 2362 i40e_aq_str(hw, 2363 hw->aq.asq_last_status)); 2364 } 2365 } 2366 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw, 2367 vsi->seid, 2368 cur_promisc, NULL); 2369 if (aq_ret) { 2370 retval = i40e_aq_rc_to_posix(aq_ret, 2371 pf->hw.aq.asq_last_status); 2372 dev_info(&pf->pdev->dev, 2373 "set brdcast promisc failed, err %s, aq_err %s\n", 2374 i40e_stat_str(hw, aq_ret), 2375 i40e_aq_str(hw, 2376 hw->aq.asq_last_status)); 2377 } 2378 } 2379 out: 2380 /* if something went wrong then set the changed flag so we try again */ 2381 if (retval) 2382 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2383 2384 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2385 return retval; 2386 2387 err_no_memory: 2388 /* Restore elements on the temporary add and delete lists */ 2389 spin_lock_bh(&vsi->mac_filter_hash_lock); 2390 err_no_memory_locked: 2391 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2392 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2393 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2394 2395 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2396 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2397 return -ENOMEM; 2398 } 2399 2400 /** 2401 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2402 * @pf: board private structure 2403 **/ 2404 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2405 { 2406 int v; 2407 2408 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC)) 2409 return; 2410 pf->flags &= ~I40E_FLAG_FILTER_SYNC; 2411 2412 for (v = 0; v < pf->num_alloc_vsi; v++) { 2413 if (pf->vsi[v] && 2414 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2415 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2416 2417 if (ret) { 2418 /* come back and try again later */ 2419 pf->flags |= I40E_FLAG_FILTER_SYNC; 2420 break; 2421 } 2422 } 2423 } 2424 } 2425 2426 /** 2427 * i40e_max_xdp_frame_size - returns the maximum allowed frame size for XDP 2428 * @vsi: the vsi 2429 **/ 2430 static int i40e_max_xdp_frame_size(struct i40e_vsi *vsi) 2431 { 2432 if (PAGE_SIZE >= 8192 || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 2433 return I40E_RXBUFFER_2048; 2434 else 2435 return I40E_RXBUFFER_3072; 2436 } 2437 2438 /** 2439 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2440 * @netdev: network interface device structure 2441 * @new_mtu: new value for maximum frame size 2442 * 2443 * Returns 0 on success, negative on failure 2444 **/ 2445 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2446 { 2447 struct i40e_netdev_priv *np = netdev_priv(netdev); 2448 struct i40e_vsi *vsi = np->vsi; 2449 struct i40e_pf *pf = vsi->back; 2450 2451 if (i40e_enabled_xdp_vsi(vsi)) { 2452 int frame_size = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 2453 2454 if (frame_size > i40e_max_xdp_frame_size(vsi)) 2455 return -EINVAL; 2456 } 2457 2458 netdev_info(netdev, "changing MTU from %d to %d\n", 2459 netdev->mtu, new_mtu); 2460 netdev->mtu = new_mtu; 2461 if (netif_running(netdev)) 2462 i40e_vsi_reinit_locked(vsi); 2463 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 2464 I40E_FLAG_CLIENT_L2_CHANGE); 2465 return 0; 2466 } 2467 2468 /** 2469 * i40e_ioctl - Access the hwtstamp interface 2470 * @netdev: network interface device structure 2471 * @ifr: interface request data 2472 * @cmd: ioctl command 2473 **/ 2474 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2475 { 2476 struct i40e_netdev_priv *np = netdev_priv(netdev); 2477 struct i40e_pf *pf = np->vsi->back; 2478 2479 switch (cmd) { 2480 case SIOCGHWTSTAMP: 2481 return i40e_ptp_get_ts_config(pf, ifr); 2482 case SIOCSHWTSTAMP: 2483 return i40e_ptp_set_ts_config(pf, ifr); 2484 default: 2485 return -EOPNOTSUPP; 2486 } 2487 } 2488 2489 /** 2490 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2491 * @vsi: the vsi being adjusted 2492 **/ 2493 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2494 { 2495 struct i40e_vsi_context ctxt; 2496 i40e_status ret; 2497 2498 if ((vsi->info.valid_sections & 2499 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2500 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2501 return; /* already enabled */ 2502 2503 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2504 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2505 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2506 2507 ctxt.seid = vsi->seid; 2508 ctxt.info = vsi->info; 2509 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2510 if (ret) { 2511 dev_info(&vsi->back->pdev->dev, 2512 "update vlan stripping failed, err %s aq_err %s\n", 2513 i40e_stat_str(&vsi->back->hw, ret), 2514 i40e_aq_str(&vsi->back->hw, 2515 vsi->back->hw.aq.asq_last_status)); 2516 } 2517 } 2518 2519 /** 2520 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2521 * @vsi: the vsi being adjusted 2522 **/ 2523 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2524 { 2525 struct i40e_vsi_context ctxt; 2526 i40e_status ret; 2527 2528 if ((vsi->info.valid_sections & 2529 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2530 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2531 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2532 return; /* already disabled */ 2533 2534 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2535 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2536 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2537 2538 ctxt.seid = vsi->seid; 2539 ctxt.info = vsi->info; 2540 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2541 if (ret) { 2542 dev_info(&vsi->back->pdev->dev, 2543 "update vlan stripping failed, err %s aq_err %s\n", 2544 i40e_stat_str(&vsi->back->hw, ret), 2545 i40e_aq_str(&vsi->back->hw, 2546 vsi->back->hw.aq.asq_last_status)); 2547 } 2548 } 2549 2550 /** 2551 * i40e_vlan_rx_register - Setup or shutdown vlan offload 2552 * @netdev: network interface to be adjusted 2553 * @features: netdev features to test if VLAN offload is enabled or not 2554 **/ 2555 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features) 2556 { 2557 struct i40e_netdev_priv *np = netdev_priv(netdev); 2558 struct i40e_vsi *vsi = np->vsi; 2559 2560 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2561 i40e_vlan_stripping_enable(vsi); 2562 else 2563 i40e_vlan_stripping_disable(vsi); 2564 } 2565 2566 /** 2567 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2568 * @vsi: the vsi being configured 2569 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2570 * 2571 * This is a helper function for adding a new MAC/VLAN filter with the 2572 * specified VLAN for each existing MAC address already in the hash table. 2573 * This function does *not* perform any accounting to update filters based on 2574 * VLAN mode. 2575 * 2576 * NOTE: this function expects to be called while under the 2577 * mac_filter_hash_lock 2578 **/ 2579 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2580 { 2581 struct i40e_mac_filter *f, *add_f; 2582 struct hlist_node *h; 2583 int bkt; 2584 2585 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2586 if (f->state == I40E_FILTER_REMOVE) 2587 continue; 2588 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2589 if (!add_f) { 2590 dev_info(&vsi->back->pdev->dev, 2591 "Could not add vlan filter %d for %pM\n", 2592 vid, f->macaddr); 2593 return -ENOMEM; 2594 } 2595 } 2596 2597 return 0; 2598 } 2599 2600 /** 2601 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2602 * @vsi: the VSI being configured 2603 * @vid: VLAN id to be added 2604 **/ 2605 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2606 { 2607 int err; 2608 2609 if (vsi->info.pvid) 2610 return -EINVAL; 2611 2612 /* The network stack will attempt to add VID=0, with the intention to 2613 * receive priority tagged packets with a VLAN of 0. Our HW receives 2614 * these packets by default when configured to receive untagged 2615 * packets, so we don't need to add a filter for this case. 2616 * Additionally, HW interprets adding a VID=0 filter as meaning to 2617 * receive *only* tagged traffic and stops receiving untagged traffic. 2618 * Thus, we do not want to actually add a filter for VID=0 2619 */ 2620 if (!vid) 2621 return 0; 2622 2623 /* Locked once because all functions invoked below iterates list*/ 2624 spin_lock_bh(&vsi->mac_filter_hash_lock); 2625 err = i40e_add_vlan_all_mac(vsi, vid); 2626 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2627 if (err) 2628 return err; 2629 2630 /* schedule our worker thread which will take care of 2631 * applying the new filter changes 2632 */ 2633 i40e_service_event_schedule(vsi->back); 2634 return 0; 2635 } 2636 2637 /** 2638 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2639 * @vsi: the vsi being configured 2640 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2641 * 2642 * This function should be used to remove all VLAN filters which match the 2643 * given VID. It does not schedule the service event and does not take the 2644 * mac_filter_hash_lock so it may be combined with other operations under 2645 * a single invocation of the mac_filter_hash_lock. 2646 * 2647 * NOTE: this function expects to be called while under the 2648 * mac_filter_hash_lock 2649 */ 2650 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2651 { 2652 struct i40e_mac_filter *f; 2653 struct hlist_node *h; 2654 int bkt; 2655 2656 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2657 if (f->vlan == vid) 2658 __i40e_del_filter(vsi, f); 2659 } 2660 } 2661 2662 /** 2663 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2664 * @vsi: the VSI being configured 2665 * @vid: VLAN id to be removed 2666 **/ 2667 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2668 { 2669 if (!vid || vsi->info.pvid) 2670 return; 2671 2672 spin_lock_bh(&vsi->mac_filter_hash_lock); 2673 i40e_rm_vlan_all_mac(vsi, vid); 2674 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2675 2676 /* schedule our worker thread which will take care of 2677 * applying the new filter changes 2678 */ 2679 i40e_service_event_schedule(vsi->back); 2680 } 2681 2682 /** 2683 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2684 * @netdev: network interface to be adjusted 2685 * @vid: vlan id to be added 2686 * 2687 * net_device_ops implementation for adding vlan ids 2688 **/ 2689 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2690 __always_unused __be16 proto, u16 vid) 2691 { 2692 struct i40e_netdev_priv *np = netdev_priv(netdev); 2693 struct i40e_vsi *vsi = np->vsi; 2694 int ret = 0; 2695 2696 if (vid >= VLAN_N_VID) 2697 return -EINVAL; 2698 2699 ret = i40e_vsi_add_vlan(vsi, vid); 2700 if (!ret) 2701 set_bit(vid, vsi->active_vlans); 2702 2703 return ret; 2704 } 2705 2706 /** 2707 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2708 * @netdev: network interface to be adjusted 2709 * @vid: vlan id to be removed 2710 * 2711 * net_device_ops implementation for removing vlan ids 2712 **/ 2713 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2714 __always_unused __be16 proto, u16 vid) 2715 { 2716 struct i40e_netdev_priv *np = netdev_priv(netdev); 2717 struct i40e_vsi *vsi = np->vsi; 2718 2719 /* return code is ignored as there is nothing a user 2720 * can do about failure to remove and a log message was 2721 * already printed from the other function 2722 */ 2723 i40e_vsi_kill_vlan(vsi, vid); 2724 2725 clear_bit(vid, vsi->active_vlans); 2726 2727 return 0; 2728 } 2729 2730 /** 2731 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2732 * @vsi: the vsi being brought back up 2733 **/ 2734 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2735 { 2736 u16 vid; 2737 2738 if (!vsi->netdev) 2739 return; 2740 2741 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features); 2742 2743 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2744 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), 2745 vid); 2746 } 2747 2748 /** 2749 * i40e_vsi_add_pvid - Add pvid for the VSI 2750 * @vsi: the vsi being adjusted 2751 * @vid: the vlan id to set as a PVID 2752 **/ 2753 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2754 { 2755 struct i40e_vsi_context ctxt; 2756 i40e_status ret; 2757 2758 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2759 vsi->info.pvid = cpu_to_le16(vid); 2760 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2761 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2762 I40E_AQ_VSI_PVLAN_EMOD_STR; 2763 2764 ctxt.seid = vsi->seid; 2765 ctxt.info = vsi->info; 2766 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2767 if (ret) { 2768 dev_info(&vsi->back->pdev->dev, 2769 "add pvid failed, err %s aq_err %s\n", 2770 i40e_stat_str(&vsi->back->hw, ret), 2771 i40e_aq_str(&vsi->back->hw, 2772 vsi->back->hw.aq.asq_last_status)); 2773 return -ENOENT; 2774 } 2775 2776 return 0; 2777 } 2778 2779 /** 2780 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2781 * @vsi: the vsi being adjusted 2782 * 2783 * Just use the vlan_rx_register() service to put it back to normal 2784 **/ 2785 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2786 { 2787 i40e_vlan_stripping_disable(vsi); 2788 2789 vsi->info.pvid = 0; 2790 } 2791 2792 /** 2793 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2794 * @vsi: ptr to the VSI 2795 * 2796 * If this function returns with an error, then it's possible one or 2797 * more of the rings is populated (while the rest are not). It is the 2798 * callers duty to clean those orphaned rings. 2799 * 2800 * Return 0 on success, negative on failure 2801 **/ 2802 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2803 { 2804 int i, err = 0; 2805 2806 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2807 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2808 2809 if (!i40e_enabled_xdp_vsi(vsi)) 2810 return err; 2811 2812 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2813 err = i40e_setup_tx_descriptors(vsi->xdp_rings[i]); 2814 2815 return err; 2816 } 2817 2818 /** 2819 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 2820 * @vsi: ptr to the VSI 2821 * 2822 * Free VSI's transmit software resources 2823 **/ 2824 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 2825 { 2826 int i; 2827 2828 if (vsi->tx_rings) { 2829 for (i = 0; i < vsi->num_queue_pairs; i++) 2830 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 2831 i40e_free_tx_resources(vsi->tx_rings[i]); 2832 } 2833 2834 if (vsi->xdp_rings) { 2835 for (i = 0; i < vsi->num_queue_pairs; i++) 2836 if (vsi->xdp_rings[i] && vsi->xdp_rings[i]->desc) 2837 i40e_free_tx_resources(vsi->xdp_rings[i]); 2838 } 2839 } 2840 2841 /** 2842 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 2843 * @vsi: ptr to the VSI 2844 * 2845 * If this function returns with an error, then it's possible one or 2846 * more of the rings is populated (while the rest are not). It is the 2847 * callers duty to clean those orphaned rings. 2848 * 2849 * Return 0 on success, negative on failure 2850 **/ 2851 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 2852 { 2853 int i, err = 0; 2854 2855 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2856 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 2857 return err; 2858 } 2859 2860 /** 2861 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 2862 * @vsi: ptr to the VSI 2863 * 2864 * Free all receive software resources 2865 **/ 2866 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 2867 { 2868 int i; 2869 2870 if (!vsi->rx_rings) 2871 return; 2872 2873 for (i = 0; i < vsi->num_queue_pairs; i++) 2874 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 2875 i40e_free_rx_resources(vsi->rx_rings[i]); 2876 } 2877 2878 /** 2879 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 2880 * @ring: The Tx ring to configure 2881 * 2882 * This enables/disables XPS for a given Tx descriptor ring 2883 * based on the TCs enabled for the VSI that ring belongs to. 2884 **/ 2885 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 2886 { 2887 struct i40e_vsi *vsi = ring->vsi; 2888 2889 if (!ring->q_vector || !ring->netdev) 2890 return; 2891 2892 if ((vsi->tc_config.numtc <= 1) && 2893 !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) { 2894 netif_set_xps_queue(ring->netdev, 2895 get_cpu_mask(ring->q_vector->v_idx), 2896 ring->queue_index); 2897 } 2898 2899 /* schedule our worker thread which will take care of 2900 * applying the new filter changes 2901 */ 2902 i40e_service_event_schedule(vsi->back); 2903 } 2904 2905 /** 2906 * i40e_configure_tx_ring - Configure a transmit ring context and rest 2907 * @ring: The Tx ring to configure 2908 * 2909 * Configure the Tx descriptor ring in the HMC context. 2910 **/ 2911 static int i40e_configure_tx_ring(struct i40e_ring *ring) 2912 { 2913 struct i40e_vsi *vsi = ring->vsi; 2914 u16 pf_q = vsi->base_queue + ring->queue_index; 2915 struct i40e_hw *hw = &vsi->back->hw; 2916 struct i40e_hmc_obj_txq tx_ctx; 2917 i40e_status err = 0; 2918 u32 qtx_ctl = 0; 2919 2920 /* some ATR related tx ring init */ 2921 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 2922 ring->atr_sample_rate = vsi->back->atr_sample_rate; 2923 ring->atr_count = 0; 2924 } else { 2925 ring->atr_sample_rate = 0; 2926 } 2927 2928 /* configure XPS */ 2929 i40e_config_xps_tx_ring(ring); 2930 2931 /* clear the context structure first */ 2932 memset(&tx_ctx, 0, sizeof(tx_ctx)); 2933 2934 tx_ctx.new_context = 1; 2935 tx_ctx.base = (ring->dma / 128); 2936 tx_ctx.qlen = ring->count; 2937 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 2938 I40E_FLAG_FD_ATR_ENABLED)); 2939 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 2940 /* FDIR VSI tx ring can still use RS bit and writebacks */ 2941 if (vsi->type != I40E_VSI_FDIR) 2942 tx_ctx.head_wb_ena = 1; 2943 tx_ctx.head_wb_addr = ring->dma + 2944 (ring->count * sizeof(struct i40e_tx_desc)); 2945 2946 /* As part of VSI creation/update, FW allocates certain 2947 * Tx arbitration queue sets for each TC enabled for 2948 * the VSI. The FW returns the handles to these queue 2949 * sets as part of the response buffer to Add VSI, 2950 * Update VSI, etc. AQ commands. It is expected that 2951 * these queue set handles be associated with the Tx 2952 * queues by the driver as part of the TX queue context 2953 * initialization. This has to be done regardless of 2954 * DCB as by default everything is mapped to TC0. 2955 */ 2956 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 2957 tx_ctx.rdylist_act = 0; 2958 2959 /* clear the context in the HMC */ 2960 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 2961 if (err) { 2962 dev_info(&vsi->back->pdev->dev, 2963 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 2964 ring->queue_index, pf_q, err); 2965 return -ENOMEM; 2966 } 2967 2968 /* set the context in the HMC */ 2969 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 2970 if (err) { 2971 dev_info(&vsi->back->pdev->dev, 2972 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 2973 ring->queue_index, pf_q, err); 2974 return -ENOMEM; 2975 } 2976 2977 /* Now associate this queue with this PCI function */ 2978 if (vsi->type == I40E_VSI_VMDQ2) { 2979 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 2980 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 2981 I40E_QTX_CTL_VFVM_INDX_MASK; 2982 } else { 2983 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 2984 } 2985 2986 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2987 I40E_QTX_CTL_PF_INDX_MASK); 2988 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 2989 i40e_flush(hw); 2990 2991 /* cache tail off for easier writes later */ 2992 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 2993 2994 return 0; 2995 } 2996 2997 /** 2998 * i40e_configure_rx_ring - Configure a receive ring context 2999 * @ring: The Rx ring to configure 3000 * 3001 * Configure the Rx descriptor ring in the HMC context. 3002 **/ 3003 static int i40e_configure_rx_ring(struct i40e_ring *ring) 3004 { 3005 struct i40e_vsi *vsi = ring->vsi; 3006 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 3007 u16 pf_q = vsi->base_queue + ring->queue_index; 3008 struct i40e_hw *hw = &vsi->back->hw; 3009 struct i40e_hmc_obj_rxq rx_ctx; 3010 i40e_status err = 0; 3011 3012 ring->state = 0; 3013 3014 /* clear the context structure first */ 3015 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3016 3017 ring->rx_buf_len = vsi->rx_buf_len; 3018 3019 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3020 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3021 3022 rx_ctx.base = (ring->dma / 128); 3023 rx_ctx.qlen = ring->count; 3024 3025 /* use 32 byte descriptors */ 3026 rx_ctx.dsize = 1; 3027 3028 /* descriptor type is always zero 3029 * rx_ctx.dtype = 0; 3030 */ 3031 rx_ctx.hsplit_0 = 0; 3032 3033 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3034 if (hw->revision_id == 0) 3035 rx_ctx.lrxqthresh = 0; 3036 else 3037 rx_ctx.lrxqthresh = 2; 3038 rx_ctx.crcstrip = 1; 3039 rx_ctx.l2tsel = 1; 3040 /* this controls whether VLAN is stripped from inner headers */ 3041 rx_ctx.showiv = 0; 3042 /* set the prefena field to 1 because the manual says to */ 3043 rx_ctx.prefena = 1; 3044 3045 /* clear the context in the HMC */ 3046 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3047 if (err) { 3048 dev_info(&vsi->back->pdev->dev, 3049 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3050 ring->queue_index, pf_q, err); 3051 return -ENOMEM; 3052 } 3053 3054 /* set the context in the HMC */ 3055 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3056 if (err) { 3057 dev_info(&vsi->back->pdev->dev, 3058 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3059 ring->queue_index, pf_q, err); 3060 return -ENOMEM; 3061 } 3062 3063 /* configure Rx buffer alignment */ 3064 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3065 clear_ring_build_skb_enabled(ring); 3066 else 3067 set_ring_build_skb_enabled(ring); 3068 3069 /* cache tail for quicker writes, and clear the reg before use */ 3070 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3071 writel(0, ring->tail); 3072 3073 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3074 3075 return 0; 3076 } 3077 3078 /** 3079 * i40e_vsi_configure_tx - Configure the VSI for Tx 3080 * @vsi: VSI structure describing this set of rings and resources 3081 * 3082 * Configure the Tx VSI for operation. 3083 **/ 3084 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3085 { 3086 int err = 0; 3087 u16 i; 3088 3089 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3090 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3091 3092 if (!i40e_enabled_xdp_vsi(vsi)) 3093 return err; 3094 3095 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3096 err = i40e_configure_tx_ring(vsi->xdp_rings[i]); 3097 3098 return err; 3099 } 3100 3101 /** 3102 * i40e_vsi_configure_rx - Configure the VSI for Rx 3103 * @vsi: the VSI being configured 3104 * 3105 * Configure the Rx VSI for operation. 3106 **/ 3107 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3108 { 3109 int err = 0; 3110 u16 i; 3111 3112 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3113 vsi->max_frame = I40E_MAX_RXBUFFER; 3114 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3115 #if (PAGE_SIZE < 8192) 3116 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3117 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3118 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3119 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3120 #endif 3121 } else { 3122 vsi->max_frame = I40E_MAX_RXBUFFER; 3123 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3124 I40E_RXBUFFER_2048; 3125 } 3126 3127 /* set up individual rings */ 3128 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3129 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3130 3131 return err; 3132 } 3133 3134 /** 3135 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3136 * @vsi: ptr to the VSI 3137 **/ 3138 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3139 { 3140 struct i40e_ring *tx_ring, *rx_ring; 3141 u16 qoffset, qcount; 3142 int i, n; 3143 3144 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3145 /* Reset the TC information */ 3146 for (i = 0; i < vsi->num_queue_pairs; i++) { 3147 rx_ring = vsi->rx_rings[i]; 3148 tx_ring = vsi->tx_rings[i]; 3149 rx_ring->dcb_tc = 0; 3150 tx_ring->dcb_tc = 0; 3151 } 3152 } 3153 3154 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3155 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3156 continue; 3157 3158 qoffset = vsi->tc_config.tc_info[n].qoffset; 3159 qcount = vsi->tc_config.tc_info[n].qcount; 3160 for (i = qoffset; i < (qoffset + qcount); i++) { 3161 rx_ring = vsi->rx_rings[i]; 3162 tx_ring = vsi->tx_rings[i]; 3163 rx_ring->dcb_tc = n; 3164 tx_ring->dcb_tc = n; 3165 } 3166 } 3167 } 3168 3169 /** 3170 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3171 * @vsi: ptr to the VSI 3172 **/ 3173 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3174 { 3175 if (vsi->netdev) 3176 i40e_set_rx_mode(vsi->netdev); 3177 } 3178 3179 /** 3180 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3181 * @vsi: Pointer to the targeted VSI 3182 * 3183 * This function replays the hlist on the hw where all the SB Flow Director 3184 * filters were saved. 3185 **/ 3186 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3187 { 3188 struct i40e_fdir_filter *filter; 3189 struct i40e_pf *pf = vsi->back; 3190 struct hlist_node *node; 3191 3192 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3193 return; 3194 3195 /* Reset FDir counters as we're replaying all existing filters */ 3196 pf->fd_tcp4_filter_cnt = 0; 3197 pf->fd_udp4_filter_cnt = 0; 3198 pf->fd_sctp4_filter_cnt = 0; 3199 pf->fd_ip4_filter_cnt = 0; 3200 3201 hlist_for_each_entry_safe(filter, node, 3202 &pf->fdir_filter_list, fdir_node) { 3203 i40e_add_del_fdir(vsi, filter, true); 3204 } 3205 } 3206 3207 /** 3208 * i40e_vsi_configure - Set up the VSI for action 3209 * @vsi: the VSI being configured 3210 **/ 3211 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3212 { 3213 int err; 3214 3215 i40e_set_vsi_rx_mode(vsi); 3216 i40e_restore_vlan(vsi); 3217 i40e_vsi_config_dcb_rings(vsi); 3218 err = i40e_vsi_configure_tx(vsi); 3219 if (!err) 3220 err = i40e_vsi_configure_rx(vsi); 3221 3222 return err; 3223 } 3224 3225 /** 3226 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3227 * @vsi: the VSI being configured 3228 **/ 3229 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3230 { 3231 bool has_xdp = i40e_enabled_xdp_vsi(vsi); 3232 struct i40e_pf *pf = vsi->back; 3233 struct i40e_hw *hw = &pf->hw; 3234 u16 vector; 3235 int i, q; 3236 u32 qp; 3237 3238 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3239 * and PFINT_LNKLSTn registers, e.g.: 3240 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3241 */ 3242 qp = vsi->base_queue; 3243 vector = vsi->base_vector; 3244 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3245 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3246 3247 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3248 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting); 3249 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3250 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3251 q_vector->rx.itr); 3252 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting); 3253 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3254 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3255 q_vector->tx.itr); 3256 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3257 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3258 3259 /* Linked list for the queuepairs assigned to this vector */ 3260 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3261 for (q = 0; q < q_vector->num_ringpairs; q++) { 3262 u32 nextqp = has_xdp ? qp + vsi->alloc_queue_pairs : qp; 3263 u32 val; 3264 3265 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3266 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3267 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3268 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT) | 3269 (I40E_QUEUE_TYPE_TX << 3270 I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3271 3272 wr32(hw, I40E_QINT_RQCTL(qp), val); 3273 3274 if (has_xdp) { 3275 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3276 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3277 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3278 (qp << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3279 (I40E_QUEUE_TYPE_TX << 3280 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3281 3282 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3283 } 3284 3285 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3286 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3287 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3288 ((qp + 1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT) | 3289 (I40E_QUEUE_TYPE_RX << 3290 I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3291 3292 /* Terminate the linked list */ 3293 if (q == (q_vector->num_ringpairs - 1)) 3294 val |= (I40E_QUEUE_END_OF_LIST << 3295 I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3296 3297 wr32(hw, I40E_QINT_TQCTL(qp), val); 3298 qp++; 3299 } 3300 } 3301 3302 i40e_flush(hw); 3303 } 3304 3305 /** 3306 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3307 * @hw: ptr to the hardware info 3308 **/ 3309 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3310 { 3311 struct i40e_hw *hw = &pf->hw; 3312 u32 val; 3313 3314 /* clear things first */ 3315 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3316 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3317 3318 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3319 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3320 I40E_PFINT_ICR0_ENA_GRST_MASK | 3321 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3322 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3323 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3324 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3325 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3326 3327 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3328 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3329 3330 if (pf->flags & I40E_FLAG_PTP) 3331 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3332 3333 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3334 3335 /* SW_ITR_IDX = 0, but don't change INTENA */ 3336 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3337 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3338 3339 /* OTHER_ITR_IDX = 0 */ 3340 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3341 } 3342 3343 /** 3344 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3345 * @vsi: the VSI being configured 3346 **/ 3347 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3348 { 3349 u32 nextqp = i40e_enabled_xdp_vsi(vsi) ? vsi->alloc_queue_pairs : 0; 3350 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3351 struct i40e_pf *pf = vsi->back; 3352 struct i40e_hw *hw = &pf->hw; 3353 u32 val; 3354 3355 /* set the ITR configuration */ 3356 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3357 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting); 3358 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3359 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 3360 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting); 3361 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3362 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 3363 3364 i40e_enable_misc_int_causes(pf); 3365 3366 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3367 wr32(hw, I40E_PFINT_LNKLST0, 0); 3368 3369 /* Associate the queue pair to the vector and enable the queue int */ 3370 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3371 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3372 (nextqp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3373 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3374 3375 wr32(hw, I40E_QINT_RQCTL(0), val); 3376 3377 if (i40e_enabled_xdp_vsi(vsi)) { 3378 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3379 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT)| 3380 (I40E_QUEUE_TYPE_TX 3381 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3382 3383 wr32(hw, I40E_QINT_TQCTL(nextqp), val); 3384 } 3385 3386 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3387 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3388 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3389 3390 wr32(hw, I40E_QINT_TQCTL(0), val); 3391 i40e_flush(hw); 3392 } 3393 3394 /** 3395 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3396 * @pf: board private structure 3397 **/ 3398 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3399 { 3400 struct i40e_hw *hw = &pf->hw; 3401 3402 wr32(hw, I40E_PFINT_DYN_CTL0, 3403 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3404 i40e_flush(hw); 3405 } 3406 3407 /** 3408 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3409 * @pf: board private structure 3410 * @clearpba: true when all pending interrupt events should be cleared 3411 **/ 3412 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba) 3413 { 3414 struct i40e_hw *hw = &pf->hw; 3415 u32 val; 3416 3417 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3418 (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) | 3419 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3420 3421 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3422 i40e_flush(hw); 3423 } 3424 3425 /** 3426 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3427 * @irq: interrupt number 3428 * @data: pointer to a q_vector 3429 **/ 3430 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3431 { 3432 struct i40e_q_vector *q_vector = data; 3433 3434 if (!q_vector->tx.ring && !q_vector->rx.ring) 3435 return IRQ_HANDLED; 3436 3437 napi_schedule_irqoff(&q_vector->napi); 3438 3439 return IRQ_HANDLED; 3440 } 3441 3442 /** 3443 * i40e_irq_affinity_notify - Callback for affinity changes 3444 * @notify: context as to what irq was changed 3445 * @mask: the new affinity mask 3446 * 3447 * This is a callback function used by the irq_set_affinity_notifier function 3448 * so that we may register to receive changes to the irq affinity masks. 3449 **/ 3450 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3451 const cpumask_t *mask) 3452 { 3453 struct i40e_q_vector *q_vector = 3454 container_of(notify, struct i40e_q_vector, affinity_notify); 3455 3456 cpumask_copy(&q_vector->affinity_mask, mask); 3457 } 3458 3459 /** 3460 * i40e_irq_affinity_release - Callback for affinity notifier release 3461 * @ref: internal core kernel usage 3462 * 3463 * This is a callback function used by the irq_set_affinity_notifier function 3464 * to inform the current notification subscriber that they will no longer 3465 * receive notifications. 3466 **/ 3467 static void i40e_irq_affinity_release(struct kref *ref) {} 3468 3469 /** 3470 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3471 * @vsi: the VSI being configured 3472 * @basename: name for the vector 3473 * 3474 * Allocates MSI-X vectors and requests interrupts from the kernel. 3475 **/ 3476 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3477 { 3478 int q_vectors = vsi->num_q_vectors; 3479 struct i40e_pf *pf = vsi->back; 3480 int base = vsi->base_vector; 3481 int rx_int_idx = 0; 3482 int tx_int_idx = 0; 3483 int vector, err; 3484 int irq_num; 3485 3486 for (vector = 0; vector < q_vectors; vector++) { 3487 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3488 3489 irq_num = pf->msix_entries[base + vector].vector; 3490 3491 if (q_vector->tx.ring && q_vector->rx.ring) { 3492 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3493 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3494 tx_int_idx++; 3495 } else if (q_vector->rx.ring) { 3496 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3497 "%s-%s-%d", basename, "rx", rx_int_idx++); 3498 } else if (q_vector->tx.ring) { 3499 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3500 "%s-%s-%d", basename, "tx", tx_int_idx++); 3501 } else { 3502 /* skip this unused q_vector */ 3503 continue; 3504 } 3505 err = request_irq(irq_num, 3506 vsi->irq_handler, 3507 0, 3508 q_vector->name, 3509 q_vector); 3510 if (err) { 3511 dev_info(&pf->pdev->dev, 3512 "MSIX request_irq failed, error: %d\n", err); 3513 goto free_queue_irqs; 3514 } 3515 3516 /* register for affinity change notifications */ 3517 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3518 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3519 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3520 /* get_cpu_mask returns a static constant mask with 3521 * a permanent lifetime so it's ok to use here. 3522 */ 3523 irq_set_affinity_hint(irq_num, get_cpu_mask(q_vector->v_idx)); 3524 } 3525 3526 vsi->irqs_ready = true; 3527 return 0; 3528 3529 free_queue_irqs: 3530 while (vector) { 3531 vector--; 3532 irq_num = pf->msix_entries[base + vector].vector; 3533 irq_set_affinity_notifier(irq_num, NULL); 3534 irq_set_affinity_hint(irq_num, NULL); 3535 free_irq(irq_num, &vsi->q_vectors[vector]); 3536 } 3537 return err; 3538 } 3539 3540 /** 3541 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3542 * @vsi: the VSI being un-configured 3543 **/ 3544 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3545 { 3546 struct i40e_pf *pf = vsi->back; 3547 struct i40e_hw *hw = &pf->hw; 3548 int base = vsi->base_vector; 3549 int i; 3550 3551 /* disable interrupt causation from each queue */ 3552 for (i = 0; i < vsi->num_queue_pairs; i++) { 3553 u32 val; 3554 3555 val = rd32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx)); 3556 val &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 3557 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val); 3558 3559 val = rd32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx)); 3560 val &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 3561 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), val); 3562 3563 if (!i40e_enabled_xdp_vsi(vsi)) 3564 continue; 3565 wr32(hw, I40E_QINT_TQCTL(vsi->xdp_rings[i]->reg_idx), 0); 3566 } 3567 3568 /* disable each interrupt */ 3569 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3570 for (i = vsi->base_vector; 3571 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3572 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3573 3574 i40e_flush(hw); 3575 for (i = 0; i < vsi->num_q_vectors; i++) 3576 synchronize_irq(pf->msix_entries[i + base].vector); 3577 } else { 3578 /* Legacy and MSI mode - this stops all interrupt handling */ 3579 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3580 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3581 i40e_flush(hw); 3582 synchronize_irq(pf->pdev->irq); 3583 } 3584 } 3585 3586 /** 3587 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3588 * @vsi: the VSI being configured 3589 **/ 3590 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3591 { 3592 struct i40e_pf *pf = vsi->back; 3593 int i; 3594 3595 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3596 for (i = 0; i < vsi->num_q_vectors; i++) 3597 i40e_irq_dynamic_enable(vsi, i); 3598 } else { 3599 i40e_irq_dynamic_enable_icr0(pf, true); 3600 } 3601 3602 i40e_flush(&pf->hw); 3603 return 0; 3604 } 3605 3606 /** 3607 * i40e_free_misc_vector - Free the vector that handles non-queue events 3608 * @pf: board private structure 3609 **/ 3610 static void i40e_free_misc_vector(struct i40e_pf *pf) 3611 { 3612 /* Disable ICR 0 */ 3613 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3614 i40e_flush(&pf->hw); 3615 3616 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 3617 synchronize_irq(pf->msix_entries[0].vector); 3618 free_irq(pf->msix_entries[0].vector, pf); 3619 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 3620 } 3621 } 3622 3623 /** 3624 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3625 * @irq: interrupt number 3626 * @data: pointer to a q_vector 3627 * 3628 * This is the handler used for all MSI/Legacy interrupts, and deals 3629 * with both queue and non-queue interrupts. This is also used in 3630 * MSIX mode to handle the non-queue interrupts. 3631 **/ 3632 static irqreturn_t i40e_intr(int irq, void *data) 3633 { 3634 struct i40e_pf *pf = (struct i40e_pf *)data; 3635 struct i40e_hw *hw = &pf->hw; 3636 irqreturn_t ret = IRQ_NONE; 3637 u32 icr0, icr0_remaining; 3638 u32 val, ena_mask; 3639 3640 icr0 = rd32(hw, I40E_PFINT_ICR0); 3641 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3642 3643 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3644 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3645 goto enable_intr; 3646 3647 /* if interrupt but no bits showing, must be SWINT */ 3648 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3649 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3650 pf->sw_int_count++; 3651 3652 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3653 (icr0 & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3654 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3655 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3656 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 3657 } 3658 3659 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3660 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3661 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3662 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3663 3664 /* We do not have a way to disarm Queue causes while leaving 3665 * interrupt enabled for all other causes, ideally 3666 * interrupt should be disabled while we are in NAPI but 3667 * this is not a performance path and napi_schedule() 3668 * can deal with rescheduling. 3669 */ 3670 if (!test_bit(__I40E_DOWN, pf->state)) 3671 napi_schedule_irqoff(&q_vector->napi); 3672 } 3673 3674 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3675 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3676 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 3677 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3678 } 3679 3680 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3681 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3682 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 3683 } 3684 3685 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3686 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3687 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 3688 } 3689 3690 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3691 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 3692 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 3693 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3694 val = rd32(hw, I40E_GLGEN_RSTAT); 3695 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3696 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3697 if (val == I40E_RESET_CORER) { 3698 pf->corer_count++; 3699 } else if (val == I40E_RESET_GLOBR) { 3700 pf->globr_count++; 3701 } else if (val == I40E_RESET_EMPR) { 3702 pf->empr_count++; 3703 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 3704 } 3705 } 3706 3707 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3708 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3709 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3710 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3711 rd32(hw, I40E_PFHMC_ERRORINFO), 3712 rd32(hw, I40E_PFHMC_ERRORDATA)); 3713 } 3714 3715 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3716 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3717 3718 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3719 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3720 i40e_ptp_tx_hwtstamp(pf); 3721 } 3722 } 3723 3724 /* If a critical error is pending we have no choice but to reset the 3725 * device. 3726 * Report and mask out any remaining unexpected interrupts. 3727 */ 3728 icr0_remaining = icr0 & ena_mask; 3729 if (icr0_remaining) { 3730 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3731 icr0_remaining); 3732 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3733 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3734 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3735 dev_info(&pf->pdev->dev, "device will be reset\n"); 3736 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 3737 i40e_service_event_schedule(pf); 3738 } 3739 ena_mask &= ~icr0_remaining; 3740 } 3741 ret = IRQ_HANDLED; 3742 3743 enable_intr: 3744 /* re-enable interrupt causes */ 3745 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3746 if (!test_bit(__I40E_DOWN, pf->state)) { 3747 i40e_service_event_schedule(pf); 3748 i40e_irq_dynamic_enable_icr0(pf, false); 3749 } 3750 3751 return ret; 3752 } 3753 3754 /** 3755 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3756 * @tx_ring: tx ring to clean 3757 * @budget: how many cleans we're allowed 3758 * 3759 * Returns true if there's any budget left (e.g. the clean is finished) 3760 **/ 3761 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3762 { 3763 struct i40e_vsi *vsi = tx_ring->vsi; 3764 u16 i = tx_ring->next_to_clean; 3765 struct i40e_tx_buffer *tx_buf; 3766 struct i40e_tx_desc *tx_desc; 3767 3768 tx_buf = &tx_ring->tx_bi[i]; 3769 tx_desc = I40E_TX_DESC(tx_ring, i); 3770 i -= tx_ring->count; 3771 3772 do { 3773 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3774 3775 /* if next_to_watch is not set then there is no work pending */ 3776 if (!eop_desc) 3777 break; 3778 3779 /* prevent any other reads prior to eop_desc */ 3780 read_barrier_depends(); 3781 3782 /* if the descriptor isn't done, no work yet to do */ 3783 if (!(eop_desc->cmd_type_offset_bsz & 3784 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3785 break; 3786 3787 /* clear next_to_watch to prevent false hangs */ 3788 tx_buf->next_to_watch = NULL; 3789 3790 tx_desc->buffer_addr = 0; 3791 tx_desc->cmd_type_offset_bsz = 0; 3792 /* move past filter desc */ 3793 tx_buf++; 3794 tx_desc++; 3795 i++; 3796 if (unlikely(!i)) { 3797 i -= tx_ring->count; 3798 tx_buf = tx_ring->tx_bi; 3799 tx_desc = I40E_TX_DESC(tx_ring, 0); 3800 } 3801 /* unmap skb header data */ 3802 dma_unmap_single(tx_ring->dev, 3803 dma_unmap_addr(tx_buf, dma), 3804 dma_unmap_len(tx_buf, len), 3805 DMA_TO_DEVICE); 3806 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 3807 kfree(tx_buf->raw_buf); 3808 3809 tx_buf->raw_buf = NULL; 3810 tx_buf->tx_flags = 0; 3811 tx_buf->next_to_watch = NULL; 3812 dma_unmap_len_set(tx_buf, len, 0); 3813 tx_desc->buffer_addr = 0; 3814 tx_desc->cmd_type_offset_bsz = 0; 3815 3816 /* move us past the eop_desc for start of next FD desc */ 3817 tx_buf++; 3818 tx_desc++; 3819 i++; 3820 if (unlikely(!i)) { 3821 i -= tx_ring->count; 3822 tx_buf = tx_ring->tx_bi; 3823 tx_desc = I40E_TX_DESC(tx_ring, 0); 3824 } 3825 3826 /* update budget accounting */ 3827 budget--; 3828 } while (likely(budget)); 3829 3830 i += tx_ring->count; 3831 tx_ring->next_to_clean = i; 3832 3833 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 3834 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 3835 3836 return budget > 0; 3837 } 3838 3839 /** 3840 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 3841 * @irq: interrupt number 3842 * @data: pointer to a q_vector 3843 **/ 3844 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 3845 { 3846 struct i40e_q_vector *q_vector = data; 3847 struct i40e_vsi *vsi; 3848 3849 if (!q_vector->tx.ring) 3850 return IRQ_HANDLED; 3851 3852 vsi = q_vector->tx.ring->vsi; 3853 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 3854 3855 return IRQ_HANDLED; 3856 } 3857 3858 /** 3859 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 3860 * @vsi: the VSI being configured 3861 * @v_idx: vector index 3862 * @qp_idx: queue pair index 3863 **/ 3864 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 3865 { 3866 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 3867 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 3868 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 3869 3870 tx_ring->q_vector = q_vector; 3871 tx_ring->next = q_vector->tx.ring; 3872 q_vector->tx.ring = tx_ring; 3873 q_vector->tx.count++; 3874 3875 /* Place XDP Tx ring in the same q_vector ring list as regular Tx */ 3876 if (i40e_enabled_xdp_vsi(vsi)) { 3877 struct i40e_ring *xdp_ring = vsi->xdp_rings[qp_idx]; 3878 3879 xdp_ring->q_vector = q_vector; 3880 xdp_ring->next = q_vector->tx.ring; 3881 q_vector->tx.ring = xdp_ring; 3882 q_vector->tx.count++; 3883 } 3884 3885 rx_ring->q_vector = q_vector; 3886 rx_ring->next = q_vector->rx.ring; 3887 q_vector->rx.ring = rx_ring; 3888 q_vector->rx.count++; 3889 } 3890 3891 /** 3892 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 3893 * @vsi: the VSI being configured 3894 * 3895 * This function maps descriptor rings to the queue-specific vectors 3896 * we were allotted through the MSI-X enabling code. Ideally, we'd have 3897 * one vector per queue pair, but on a constrained vector budget, we 3898 * group the queue pairs as "efficiently" as possible. 3899 **/ 3900 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 3901 { 3902 int qp_remaining = vsi->num_queue_pairs; 3903 int q_vectors = vsi->num_q_vectors; 3904 int num_ringpairs; 3905 int v_start = 0; 3906 int qp_idx = 0; 3907 3908 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 3909 * group them so there are multiple queues per vector. 3910 * It is also important to go through all the vectors available to be 3911 * sure that if we don't use all the vectors, that the remaining vectors 3912 * are cleared. This is especially important when decreasing the 3913 * number of queues in use. 3914 */ 3915 for (; v_start < q_vectors; v_start++) { 3916 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 3917 3918 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 3919 3920 q_vector->num_ringpairs = num_ringpairs; 3921 3922 q_vector->rx.count = 0; 3923 q_vector->tx.count = 0; 3924 q_vector->rx.ring = NULL; 3925 q_vector->tx.ring = NULL; 3926 3927 while (num_ringpairs--) { 3928 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 3929 qp_idx++; 3930 qp_remaining--; 3931 } 3932 } 3933 } 3934 3935 /** 3936 * i40e_vsi_request_irq - Request IRQ from the OS 3937 * @vsi: the VSI being configured 3938 * @basename: name for the vector 3939 **/ 3940 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 3941 { 3942 struct i40e_pf *pf = vsi->back; 3943 int err; 3944 3945 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 3946 err = i40e_vsi_request_irq_msix(vsi, basename); 3947 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 3948 err = request_irq(pf->pdev->irq, i40e_intr, 0, 3949 pf->int_name, pf); 3950 else 3951 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 3952 pf->int_name, pf); 3953 3954 if (err) 3955 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 3956 3957 return err; 3958 } 3959 3960 #ifdef CONFIG_NET_POLL_CONTROLLER 3961 /** 3962 * i40e_netpoll - A Polling 'interrupt' handler 3963 * @netdev: network interface device structure 3964 * 3965 * This is used by netconsole to send skbs without having to re-enable 3966 * interrupts. It's not called while the normal interrupt routine is executing. 3967 **/ 3968 static void i40e_netpoll(struct net_device *netdev) 3969 { 3970 struct i40e_netdev_priv *np = netdev_priv(netdev); 3971 struct i40e_vsi *vsi = np->vsi; 3972 struct i40e_pf *pf = vsi->back; 3973 int i; 3974 3975 /* if interface is down do nothing */ 3976 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 3977 return; 3978 3979 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3980 for (i = 0; i < vsi->num_q_vectors; i++) 3981 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 3982 } else { 3983 i40e_intr(pf->pdev->irq, netdev); 3984 } 3985 } 3986 #endif 3987 3988 #define I40E_QTX_ENA_WAIT_COUNT 50 3989 3990 /** 3991 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 3992 * @pf: the PF being configured 3993 * @pf_q: the PF queue 3994 * @enable: enable or disable state of the queue 3995 * 3996 * This routine will wait for the given Tx queue of the PF to reach the 3997 * enabled or disabled state. 3998 * Returns -ETIMEDOUT in case of failing to reach the requested state after 3999 * multiple retries; else will return 0 in case of success. 4000 **/ 4001 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4002 { 4003 int i; 4004 u32 tx_reg; 4005 4006 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4007 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 4008 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4009 break; 4010 4011 usleep_range(10, 20); 4012 } 4013 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4014 return -ETIMEDOUT; 4015 4016 return 0; 4017 } 4018 4019 /** 4020 * i40e_control_tx_q - Start or stop a particular Tx queue 4021 * @pf: the PF structure 4022 * @pf_q: the PF queue to configure 4023 * @enable: start or stop the queue 4024 * 4025 * This function enables or disables a single queue. Note that any delay 4026 * required after the operation is expected to be handled by the caller of 4027 * this function. 4028 **/ 4029 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 4030 { 4031 struct i40e_hw *hw = &pf->hw; 4032 u32 tx_reg; 4033 int i; 4034 4035 /* warn the TX unit of coming changes */ 4036 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 4037 if (!enable) 4038 usleep_range(10, 20); 4039 4040 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4041 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 4042 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 4043 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 4044 break; 4045 usleep_range(1000, 2000); 4046 } 4047 4048 /* Skip if the queue is already in the requested state */ 4049 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 4050 return; 4051 4052 /* turn on/off the queue */ 4053 if (enable) { 4054 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 4055 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 4056 } else { 4057 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 4058 } 4059 4060 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 4061 } 4062 4063 /** 4064 * i40e_control_wait_tx_q - Start/stop Tx queue and wait for completion 4065 * @seid: VSI SEID 4066 * @pf: the PF structure 4067 * @pf_q: the PF queue to configure 4068 * @is_xdp: true if the queue is used for XDP 4069 * @enable: start or stop the queue 4070 **/ 4071 static int i40e_control_wait_tx_q(int seid, struct i40e_pf *pf, int pf_q, 4072 bool is_xdp, bool enable) 4073 { 4074 int ret; 4075 4076 i40e_control_tx_q(pf, pf_q, enable); 4077 4078 /* wait for the change to finish */ 4079 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4080 if (ret) { 4081 dev_info(&pf->pdev->dev, 4082 "VSI seid %d %sTx ring %d %sable timeout\n", 4083 seid, (is_xdp ? "XDP " : ""), pf_q, 4084 (enable ? "en" : "dis")); 4085 } 4086 4087 return ret; 4088 } 4089 4090 /** 4091 * i40e_vsi_control_tx - Start or stop a VSI's rings 4092 * @vsi: the VSI being configured 4093 * @enable: start or stop the rings 4094 **/ 4095 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4096 { 4097 struct i40e_pf *pf = vsi->back; 4098 int i, pf_q, ret = 0; 4099 4100 pf_q = vsi->base_queue; 4101 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4102 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4103 pf_q, 4104 false /*is xdp*/, enable); 4105 if (ret) 4106 break; 4107 4108 if (!i40e_enabled_xdp_vsi(vsi)) 4109 continue; 4110 4111 ret = i40e_control_wait_tx_q(vsi->seid, pf, 4112 pf_q + vsi->alloc_queue_pairs, 4113 true /*is xdp*/, enable); 4114 if (ret) 4115 break; 4116 } 4117 4118 return ret; 4119 } 4120 4121 /** 4122 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4123 * @pf: the PF being configured 4124 * @pf_q: the PF queue 4125 * @enable: enable or disable state of the queue 4126 * 4127 * This routine will wait for the given Rx queue of the PF to reach the 4128 * enabled or disabled state. 4129 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4130 * multiple retries; else will return 0 in case of success. 4131 **/ 4132 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4133 { 4134 int i; 4135 u32 rx_reg; 4136 4137 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4138 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4139 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4140 break; 4141 4142 usleep_range(10, 20); 4143 } 4144 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4145 return -ETIMEDOUT; 4146 4147 return 0; 4148 } 4149 4150 /** 4151 * i40e_control_rx_q - Start or stop a particular Rx queue 4152 * @pf: the PF structure 4153 * @pf_q: the PF queue to configure 4154 * @enable: start or stop the queue 4155 * 4156 * This function enables or disables a single queue. Note that any delay 4157 * required after the operation is expected to be handled by the caller of 4158 * this function. 4159 **/ 4160 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4161 { 4162 struct i40e_hw *hw = &pf->hw; 4163 u32 rx_reg; 4164 int i; 4165 4166 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4167 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4168 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4169 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4170 break; 4171 usleep_range(1000, 2000); 4172 } 4173 4174 /* Skip if the queue is already in the requested state */ 4175 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4176 return; 4177 4178 /* turn on/off the queue */ 4179 if (enable) 4180 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4181 else 4182 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4183 4184 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4185 } 4186 4187 /** 4188 * i40e_vsi_control_rx - Start or stop a VSI's rings 4189 * @vsi: the VSI being configured 4190 * @enable: start or stop the rings 4191 **/ 4192 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4193 { 4194 struct i40e_pf *pf = vsi->back; 4195 int i, pf_q, ret = 0; 4196 4197 pf_q = vsi->base_queue; 4198 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4199 i40e_control_rx_q(pf, pf_q, enable); 4200 4201 /* wait for the change to finish */ 4202 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4203 if (ret) { 4204 dev_info(&pf->pdev->dev, 4205 "VSI seid %d Rx ring %d %sable timeout\n", 4206 vsi->seid, pf_q, (enable ? "en" : "dis")); 4207 break; 4208 } 4209 } 4210 4211 /* Due to HW errata, on Rx disable only, the register can indicate done 4212 * before it really is. Needs 50ms to be sure 4213 */ 4214 if (!enable) 4215 mdelay(50); 4216 4217 return ret; 4218 } 4219 4220 /** 4221 * i40e_vsi_start_rings - Start a VSI's rings 4222 * @vsi: the VSI being configured 4223 **/ 4224 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4225 { 4226 int ret = 0; 4227 4228 /* do rx first for enable and last for disable */ 4229 ret = i40e_vsi_control_rx(vsi, true); 4230 if (ret) 4231 return ret; 4232 ret = i40e_vsi_control_tx(vsi, true); 4233 4234 return ret; 4235 } 4236 4237 /** 4238 * i40e_vsi_stop_rings - Stop a VSI's rings 4239 * @vsi: the VSI being configured 4240 **/ 4241 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4242 { 4243 /* When port TX is suspended, don't wait */ 4244 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4245 return i40e_vsi_stop_rings_no_wait(vsi); 4246 4247 /* do rx first for enable and last for disable 4248 * Ignore return value, we need to shutdown whatever we can 4249 */ 4250 i40e_vsi_control_tx(vsi, false); 4251 i40e_vsi_control_rx(vsi, false); 4252 } 4253 4254 /** 4255 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4256 * @vsi: the VSI being shutdown 4257 * 4258 * This function stops all the rings for a VSI but does not delay to verify 4259 * that rings have been disabled. It is expected that the caller is shutting 4260 * down multiple VSIs at once and will delay together for all the VSIs after 4261 * initiating the shutdown. This is particularly useful for shutting down lots 4262 * of VFs together. Otherwise, a large delay can be incurred while configuring 4263 * each VSI in serial. 4264 **/ 4265 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4266 { 4267 struct i40e_pf *pf = vsi->back; 4268 int i, pf_q; 4269 4270 pf_q = vsi->base_queue; 4271 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4272 i40e_control_tx_q(pf, pf_q, false); 4273 i40e_control_rx_q(pf, pf_q, false); 4274 } 4275 } 4276 4277 /** 4278 * i40e_vsi_free_irq - Free the irq association with the OS 4279 * @vsi: the VSI being configured 4280 **/ 4281 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4282 { 4283 struct i40e_pf *pf = vsi->back; 4284 struct i40e_hw *hw = &pf->hw; 4285 int base = vsi->base_vector; 4286 u32 val, qp; 4287 int i; 4288 4289 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4290 if (!vsi->q_vectors) 4291 return; 4292 4293 if (!vsi->irqs_ready) 4294 return; 4295 4296 vsi->irqs_ready = false; 4297 for (i = 0; i < vsi->num_q_vectors; i++) { 4298 int irq_num; 4299 u16 vector; 4300 4301 vector = i + base; 4302 irq_num = pf->msix_entries[vector].vector; 4303 4304 /* free only the irqs that were actually requested */ 4305 if (!vsi->q_vectors[i] || 4306 !vsi->q_vectors[i]->num_ringpairs) 4307 continue; 4308 4309 /* clear the affinity notifier in the IRQ descriptor */ 4310 irq_set_affinity_notifier(irq_num, NULL); 4311 /* remove our suggested affinity mask for this IRQ */ 4312 irq_set_affinity_hint(irq_num, NULL); 4313 synchronize_irq(irq_num); 4314 free_irq(irq_num, vsi->q_vectors[i]); 4315 4316 /* Tear down the interrupt queue link list 4317 * 4318 * We know that they come in pairs and always 4319 * the Rx first, then the Tx. To clear the 4320 * link list, stick the EOL value into the 4321 * next_q field of the registers. 4322 */ 4323 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4324 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4325 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4326 val |= I40E_QUEUE_END_OF_LIST 4327 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4328 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4329 4330 while (qp != I40E_QUEUE_END_OF_LIST) { 4331 u32 next; 4332 4333 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4334 4335 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4336 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4337 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4338 I40E_QINT_RQCTL_INTEVENT_MASK); 4339 4340 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4341 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4342 4343 wr32(hw, I40E_QINT_RQCTL(qp), val); 4344 4345 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4346 4347 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4348 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4349 4350 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4351 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4352 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4353 I40E_QINT_TQCTL_INTEVENT_MASK); 4354 4355 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4356 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4357 4358 wr32(hw, I40E_QINT_TQCTL(qp), val); 4359 qp = next; 4360 } 4361 } 4362 } else { 4363 free_irq(pf->pdev->irq, pf); 4364 4365 val = rd32(hw, I40E_PFINT_LNKLST0); 4366 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4367 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4368 val |= I40E_QUEUE_END_OF_LIST 4369 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4370 wr32(hw, I40E_PFINT_LNKLST0, val); 4371 4372 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4373 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4374 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4375 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4376 I40E_QINT_RQCTL_INTEVENT_MASK); 4377 4378 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4379 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4380 4381 wr32(hw, I40E_QINT_RQCTL(qp), val); 4382 4383 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4384 4385 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4386 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4387 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4388 I40E_QINT_TQCTL_INTEVENT_MASK); 4389 4390 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4391 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4392 4393 wr32(hw, I40E_QINT_TQCTL(qp), val); 4394 } 4395 } 4396 4397 /** 4398 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4399 * @vsi: the VSI being configured 4400 * @v_idx: Index of vector to be freed 4401 * 4402 * This function frees the memory allocated to the q_vector. In addition if 4403 * NAPI is enabled it will delete any references to the NAPI struct prior 4404 * to freeing the q_vector. 4405 **/ 4406 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4407 { 4408 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4409 struct i40e_ring *ring; 4410 4411 if (!q_vector) 4412 return; 4413 4414 /* disassociate q_vector from rings */ 4415 i40e_for_each_ring(ring, q_vector->tx) 4416 ring->q_vector = NULL; 4417 4418 i40e_for_each_ring(ring, q_vector->rx) 4419 ring->q_vector = NULL; 4420 4421 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4422 if (vsi->netdev) 4423 netif_napi_del(&q_vector->napi); 4424 4425 vsi->q_vectors[v_idx] = NULL; 4426 4427 kfree_rcu(q_vector, rcu); 4428 } 4429 4430 /** 4431 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4432 * @vsi: the VSI being un-configured 4433 * 4434 * This frees the memory allocated to the q_vectors and 4435 * deletes references to the NAPI struct. 4436 **/ 4437 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4438 { 4439 int v_idx; 4440 4441 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4442 i40e_free_q_vector(vsi, v_idx); 4443 } 4444 4445 /** 4446 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4447 * @pf: board private structure 4448 **/ 4449 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4450 { 4451 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4452 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4453 pci_disable_msix(pf->pdev); 4454 kfree(pf->msix_entries); 4455 pf->msix_entries = NULL; 4456 kfree(pf->irq_pile); 4457 pf->irq_pile = NULL; 4458 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4459 pci_disable_msi(pf->pdev); 4460 } 4461 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4462 } 4463 4464 /** 4465 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4466 * @pf: board private structure 4467 * 4468 * We go through and clear interrupt specific resources and reset the structure 4469 * to pre-load conditions 4470 **/ 4471 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4472 { 4473 int i; 4474 4475 i40e_free_misc_vector(pf); 4476 4477 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4478 I40E_IWARP_IRQ_PILE_ID); 4479 4480 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4481 for (i = 0; i < pf->num_alloc_vsi; i++) 4482 if (pf->vsi[i]) 4483 i40e_vsi_free_q_vectors(pf->vsi[i]); 4484 i40e_reset_interrupt_capability(pf); 4485 } 4486 4487 /** 4488 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4489 * @vsi: the VSI being configured 4490 **/ 4491 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4492 { 4493 int q_idx; 4494 4495 if (!vsi->netdev) 4496 return; 4497 4498 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4499 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4500 4501 if (q_vector->rx.ring || q_vector->tx.ring) 4502 napi_enable(&q_vector->napi); 4503 } 4504 } 4505 4506 /** 4507 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4508 * @vsi: the VSI being configured 4509 **/ 4510 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4511 { 4512 int q_idx; 4513 4514 if (!vsi->netdev) 4515 return; 4516 4517 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4518 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4519 4520 if (q_vector->rx.ring || q_vector->tx.ring) 4521 napi_disable(&q_vector->napi); 4522 } 4523 } 4524 4525 /** 4526 * i40e_vsi_close - Shut down a VSI 4527 * @vsi: the vsi to be quelled 4528 **/ 4529 static void i40e_vsi_close(struct i40e_vsi *vsi) 4530 { 4531 struct i40e_pf *pf = vsi->back; 4532 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4533 i40e_down(vsi); 4534 i40e_vsi_free_irq(vsi); 4535 i40e_vsi_free_tx_resources(vsi); 4536 i40e_vsi_free_rx_resources(vsi); 4537 vsi->current_netdev_flags = 0; 4538 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 4539 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4540 pf->flags |= I40E_FLAG_CLIENT_RESET; 4541 } 4542 4543 /** 4544 * i40e_quiesce_vsi - Pause a given VSI 4545 * @vsi: the VSI being paused 4546 **/ 4547 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4548 { 4549 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4550 return; 4551 4552 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4553 if (vsi->netdev && netif_running(vsi->netdev)) 4554 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4555 else 4556 i40e_vsi_close(vsi); 4557 } 4558 4559 /** 4560 * i40e_unquiesce_vsi - Resume a given VSI 4561 * @vsi: the VSI being resumed 4562 **/ 4563 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4564 { 4565 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4566 return; 4567 4568 if (vsi->netdev && netif_running(vsi->netdev)) 4569 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4570 else 4571 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4572 } 4573 4574 /** 4575 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4576 * @pf: the PF 4577 **/ 4578 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4579 { 4580 int v; 4581 4582 for (v = 0; v < pf->num_alloc_vsi; v++) { 4583 if (pf->vsi[v]) 4584 i40e_quiesce_vsi(pf->vsi[v]); 4585 } 4586 } 4587 4588 /** 4589 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4590 * @pf: the PF 4591 **/ 4592 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4593 { 4594 int v; 4595 4596 for (v = 0; v < pf->num_alloc_vsi; v++) { 4597 if (pf->vsi[v]) 4598 i40e_unquiesce_vsi(pf->vsi[v]); 4599 } 4600 } 4601 4602 /** 4603 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4604 * @vsi: the VSI being configured 4605 * 4606 * Wait until all queues on a given VSI have been disabled. 4607 **/ 4608 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4609 { 4610 struct i40e_pf *pf = vsi->back; 4611 int i, pf_q, ret; 4612 4613 pf_q = vsi->base_queue; 4614 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4615 /* Check and wait for the Tx queue */ 4616 ret = i40e_pf_txq_wait(pf, pf_q, false); 4617 if (ret) { 4618 dev_info(&pf->pdev->dev, 4619 "VSI seid %d Tx ring %d disable timeout\n", 4620 vsi->seid, pf_q); 4621 return ret; 4622 } 4623 4624 if (!i40e_enabled_xdp_vsi(vsi)) 4625 goto wait_rx; 4626 4627 /* Check and wait for the XDP Tx queue */ 4628 ret = i40e_pf_txq_wait(pf, pf_q + vsi->alloc_queue_pairs, 4629 false); 4630 if (ret) { 4631 dev_info(&pf->pdev->dev, 4632 "VSI seid %d XDP Tx ring %d disable timeout\n", 4633 vsi->seid, pf_q); 4634 return ret; 4635 } 4636 wait_rx: 4637 /* Check and wait for the Rx queue */ 4638 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4639 if (ret) { 4640 dev_info(&pf->pdev->dev, 4641 "VSI seid %d Rx ring %d disable timeout\n", 4642 vsi->seid, pf_q); 4643 return ret; 4644 } 4645 } 4646 4647 return 0; 4648 } 4649 4650 #ifdef CONFIG_I40E_DCB 4651 /** 4652 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4653 * @pf: the PF 4654 * 4655 * This function waits for the queues to be in disabled state for all the 4656 * VSIs that are managed by this PF. 4657 **/ 4658 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4659 { 4660 int v, ret = 0; 4661 4662 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4663 if (pf->vsi[v]) { 4664 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4665 if (ret) 4666 break; 4667 } 4668 } 4669 4670 return ret; 4671 } 4672 4673 #endif 4674 4675 /** 4676 * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue 4677 * @q_idx: TX queue number 4678 * @vsi: Pointer to VSI struct 4679 * 4680 * This function checks specified queue for given VSI. Detects hung condition. 4681 * We proactively detect hung TX queues by checking if interrupts are disabled 4682 * but there are pending descriptors. If it appears hung, attempt to recover 4683 * by triggering a SW interrupt. 4684 **/ 4685 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi) 4686 { 4687 struct i40e_ring *tx_ring = NULL; 4688 struct i40e_pf *pf; 4689 u32 val, tx_pending; 4690 int i; 4691 4692 pf = vsi->back; 4693 4694 /* now that we have an index, find the tx_ring struct */ 4695 for (i = 0; i < vsi->num_queue_pairs; i++) { 4696 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 4697 if (q_idx == vsi->tx_rings[i]->queue_index) { 4698 tx_ring = vsi->tx_rings[i]; 4699 break; 4700 } 4701 } 4702 } 4703 4704 if (!tx_ring) 4705 return; 4706 4707 /* Read interrupt register */ 4708 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4709 val = rd32(&pf->hw, 4710 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 4711 tx_ring->vsi->base_vector - 1)); 4712 else 4713 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 4714 4715 tx_pending = i40e_get_tx_pending(tx_ring); 4716 4717 /* Interrupts are disabled and TX pending is non-zero, 4718 * trigger the SW interrupt (don't wait). Worst case 4719 * there will be one extra interrupt which may result 4720 * into not cleaning any queues because queues are cleaned. 4721 */ 4722 if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) 4723 i40e_force_wb(vsi, tx_ring->q_vector); 4724 } 4725 4726 /** 4727 * i40e_detect_recover_hung - Function to detect and recover hung_queues 4728 * @pf: pointer to PF struct 4729 * 4730 * LAN VSI has netdev and netdev has TX queues. This function is to check 4731 * each of those TX queues if they are hung, trigger recovery by issuing 4732 * SW interrupt. 4733 **/ 4734 static void i40e_detect_recover_hung(struct i40e_pf *pf) 4735 { 4736 struct net_device *netdev; 4737 struct i40e_vsi *vsi; 4738 unsigned int i; 4739 4740 /* Only for LAN VSI */ 4741 vsi = pf->vsi[pf->lan_vsi]; 4742 4743 if (!vsi) 4744 return; 4745 4746 /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */ 4747 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 4748 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 4749 return; 4750 4751 /* Make sure type is MAIN VSI */ 4752 if (vsi->type != I40E_VSI_MAIN) 4753 return; 4754 4755 netdev = vsi->netdev; 4756 if (!netdev) 4757 return; 4758 4759 /* Bail out if netif_carrier is not OK */ 4760 if (!netif_carrier_ok(netdev)) 4761 return; 4762 4763 /* Go thru' TX queues for netdev */ 4764 for (i = 0; i < netdev->num_tx_queues; i++) { 4765 struct netdev_queue *q; 4766 4767 q = netdev_get_tx_queue(netdev, i); 4768 if (q) 4769 i40e_detect_recover_hung_queue(i, vsi); 4770 } 4771 } 4772 4773 /** 4774 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4775 * @pf: pointer to PF 4776 * 4777 * Get TC map for ISCSI PF type that will include iSCSI TC 4778 * and LAN TC. 4779 **/ 4780 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4781 { 4782 struct i40e_dcb_app_priority_table app; 4783 struct i40e_hw *hw = &pf->hw; 4784 u8 enabled_tc = 1; /* TC0 is always enabled */ 4785 u8 tc, i; 4786 /* Get the iSCSI APP TLV */ 4787 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4788 4789 for (i = 0; i < dcbcfg->numapps; i++) { 4790 app = dcbcfg->app[i]; 4791 if (app.selector == I40E_APP_SEL_TCPIP && 4792 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4793 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4794 enabled_tc |= BIT(tc); 4795 break; 4796 } 4797 } 4798 4799 return enabled_tc; 4800 } 4801 4802 /** 4803 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 4804 * @dcbcfg: the corresponding DCBx configuration structure 4805 * 4806 * Return the number of TCs from given DCBx configuration 4807 **/ 4808 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 4809 { 4810 int i, tc_unused = 0; 4811 u8 num_tc = 0; 4812 u8 ret = 0; 4813 4814 /* Scan the ETS Config Priority Table to find 4815 * traffic class enabled for a given priority 4816 * and create a bitmask of enabled TCs 4817 */ 4818 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 4819 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 4820 4821 /* Now scan the bitmask to check for 4822 * contiguous TCs starting with TC0 4823 */ 4824 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4825 if (num_tc & BIT(i)) { 4826 if (!tc_unused) { 4827 ret++; 4828 } else { 4829 pr_err("Non-contiguous TC - Disabling DCB\n"); 4830 return 1; 4831 } 4832 } else { 4833 tc_unused = 1; 4834 } 4835 } 4836 4837 /* There is always at least TC0 */ 4838 if (!ret) 4839 ret = 1; 4840 4841 return ret; 4842 } 4843 4844 /** 4845 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 4846 * @dcbcfg: the corresponding DCBx configuration structure 4847 * 4848 * Query the current DCB configuration and return the number of 4849 * traffic classes enabled from the given DCBX config 4850 **/ 4851 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 4852 { 4853 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 4854 u8 enabled_tc = 1; 4855 u8 i; 4856 4857 for (i = 0; i < num_tc; i++) 4858 enabled_tc |= BIT(i); 4859 4860 return enabled_tc; 4861 } 4862 4863 /** 4864 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 4865 * @pf: PF being queried 4866 * 4867 * Return number of traffic classes enabled for the given PF 4868 **/ 4869 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 4870 { 4871 struct i40e_hw *hw = &pf->hw; 4872 u8 i, enabled_tc = 1; 4873 u8 num_tc = 0; 4874 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4875 4876 /* If DCB is not enabled then always in single TC */ 4877 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4878 return 1; 4879 4880 /* SFP mode will be enabled for all TCs on port */ 4881 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4882 return i40e_dcb_get_num_tc(dcbcfg); 4883 4884 /* MFP mode return count of enabled TCs for this PF */ 4885 if (pf->hw.func_caps.iscsi) 4886 enabled_tc = i40e_get_iscsi_tc_map(pf); 4887 else 4888 return 1; /* Only TC0 */ 4889 4890 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4891 if (enabled_tc & BIT(i)) 4892 num_tc++; 4893 } 4894 return num_tc; 4895 } 4896 4897 /** 4898 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 4899 * @pf: PF being queried 4900 * 4901 * Return a bitmap for enabled traffic classes for this PF. 4902 **/ 4903 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 4904 { 4905 /* If DCB is not enabled for this PF then just return default TC */ 4906 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4907 return I40E_DEFAULT_TRAFFIC_CLASS; 4908 4909 /* SFP mode we want PF to be enabled for all TCs */ 4910 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4911 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 4912 4913 /* MFP enabled and iSCSI PF type */ 4914 if (pf->hw.func_caps.iscsi) 4915 return i40e_get_iscsi_tc_map(pf); 4916 else 4917 return I40E_DEFAULT_TRAFFIC_CLASS; 4918 } 4919 4920 /** 4921 * i40e_vsi_get_bw_info - Query VSI BW Information 4922 * @vsi: the VSI being queried 4923 * 4924 * Returns 0 on success, negative value on failure 4925 **/ 4926 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 4927 { 4928 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 4929 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 4930 struct i40e_pf *pf = vsi->back; 4931 struct i40e_hw *hw = &pf->hw; 4932 i40e_status ret; 4933 u32 tc_bw_max; 4934 int i; 4935 4936 /* Get the VSI level BW configuration */ 4937 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 4938 if (ret) { 4939 dev_info(&pf->pdev->dev, 4940 "couldn't get PF vsi bw config, err %s aq_err %s\n", 4941 i40e_stat_str(&pf->hw, ret), 4942 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4943 return -EINVAL; 4944 } 4945 4946 /* Get the VSI level BW configuration per TC */ 4947 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 4948 NULL); 4949 if (ret) { 4950 dev_info(&pf->pdev->dev, 4951 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 4952 i40e_stat_str(&pf->hw, ret), 4953 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4954 return -EINVAL; 4955 } 4956 4957 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 4958 dev_info(&pf->pdev->dev, 4959 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 4960 bw_config.tc_valid_bits, 4961 bw_ets_config.tc_valid_bits); 4962 /* Still continuing */ 4963 } 4964 4965 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 4966 vsi->bw_max_quanta = bw_config.max_bw; 4967 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 4968 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 4969 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4970 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 4971 vsi->bw_ets_limit_credits[i] = 4972 le16_to_cpu(bw_ets_config.credits[i]); 4973 /* 3 bits out of 4 for each TC */ 4974 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 4975 } 4976 4977 return 0; 4978 } 4979 4980 /** 4981 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 4982 * @vsi: the VSI being configured 4983 * @enabled_tc: TC bitmap 4984 * @bw_credits: BW shared credits per TC 4985 * 4986 * Returns 0 on success, negative value on failure 4987 **/ 4988 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 4989 u8 *bw_share) 4990 { 4991 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 4992 i40e_status ret; 4993 int i; 4994 4995 bw_data.tc_valid_bits = enabled_tc; 4996 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4997 bw_data.tc_bw_credits[i] = bw_share[i]; 4998 4999 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 5000 NULL); 5001 if (ret) { 5002 dev_info(&vsi->back->pdev->dev, 5003 "AQ command Config VSI BW allocation per TC failed = %d\n", 5004 vsi->back->hw.aq.asq_last_status); 5005 return -EINVAL; 5006 } 5007 5008 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 5009 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 5010 5011 return 0; 5012 } 5013 5014 /** 5015 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 5016 * @vsi: the VSI being configured 5017 * @enabled_tc: TC map to be enabled 5018 * 5019 **/ 5020 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5021 { 5022 struct net_device *netdev = vsi->netdev; 5023 struct i40e_pf *pf = vsi->back; 5024 struct i40e_hw *hw = &pf->hw; 5025 u8 netdev_tc = 0; 5026 int i; 5027 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 5028 5029 if (!netdev) 5030 return; 5031 5032 if (!enabled_tc) { 5033 netdev_reset_tc(netdev); 5034 return; 5035 } 5036 5037 /* Set up actual enabled TCs on the VSI */ 5038 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 5039 return; 5040 5041 /* set per TC queues for the VSI */ 5042 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5043 /* Only set TC queues for enabled tcs 5044 * 5045 * e.g. For a VSI that has TC0 and TC3 enabled the 5046 * enabled_tc bitmap would be 0x00001001; the driver 5047 * will set the numtc for netdev as 2 that will be 5048 * referenced by the netdev layer as TC 0 and 1. 5049 */ 5050 if (vsi->tc_config.enabled_tc & BIT(i)) 5051 netdev_set_tc_queue(netdev, 5052 vsi->tc_config.tc_info[i].netdev_tc, 5053 vsi->tc_config.tc_info[i].qcount, 5054 vsi->tc_config.tc_info[i].qoffset); 5055 } 5056 5057 /* Assign UP2TC map for the VSI */ 5058 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 5059 /* Get the actual TC# for the UP */ 5060 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 5061 /* Get the mapped netdev TC# for the UP */ 5062 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 5063 netdev_set_prio_tc_map(netdev, i, netdev_tc); 5064 } 5065 } 5066 5067 /** 5068 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 5069 * @vsi: the VSI being configured 5070 * @ctxt: the ctxt buffer returned from AQ VSI update param command 5071 **/ 5072 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 5073 struct i40e_vsi_context *ctxt) 5074 { 5075 /* copy just the sections touched not the entire info 5076 * since not all sections are valid as returned by 5077 * update vsi params 5078 */ 5079 vsi->info.mapping_flags = ctxt->info.mapping_flags; 5080 memcpy(&vsi->info.queue_mapping, 5081 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 5082 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 5083 sizeof(vsi->info.tc_mapping)); 5084 } 5085 5086 /** 5087 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 5088 * @vsi: VSI to be configured 5089 * @enabled_tc: TC bitmap 5090 * 5091 * This configures a particular VSI for TCs that are mapped to the 5092 * given TC bitmap. It uses default bandwidth share for TCs across 5093 * VSIs to configure TC for a particular VSI. 5094 * 5095 * NOTE: 5096 * It is expected that the VSI queues have been quisced before calling 5097 * this function. 5098 **/ 5099 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 5100 { 5101 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 5102 struct i40e_vsi_context ctxt; 5103 int ret = 0; 5104 int i; 5105 5106 /* Check if enabled_tc is same as existing or new TCs */ 5107 if (vsi->tc_config.enabled_tc == enabled_tc) 5108 return ret; 5109 5110 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5111 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5112 if (enabled_tc & BIT(i)) 5113 bw_share[i] = 1; 5114 } 5115 5116 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5117 if (ret) { 5118 dev_info(&vsi->back->pdev->dev, 5119 "Failed configuring TC map %d for VSI %d\n", 5120 enabled_tc, vsi->seid); 5121 goto out; 5122 } 5123 5124 /* Update Queue Pairs Mapping for currently enabled UPs */ 5125 ctxt.seid = vsi->seid; 5126 ctxt.pf_num = vsi->back->hw.pf_id; 5127 ctxt.vf_num = 0; 5128 ctxt.uplink_seid = vsi->uplink_seid; 5129 ctxt.info = vsi->info; 5130 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5131 5132 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5133 ctxt.info.valid_sections |= 5134 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5135 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5136 } 5137 5138 /* Update the VSI after updating the VSI queue-mapping information */ 5139 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 5140 if (ret) { 5141 dev_info(&vsi->back->pdev->dev, 5142 "Update vsi tc config failed, err %s aq_err %s\n", 5143 i40e_stat_str(&vsi->back->hw, ret), 5144 i40e_aq_str(&vsi->back->hw, 5145 vsi->back->hw.aq.asq_last_status)); 5146 goto out; 5147 } 5148 /* update the local VSI info with updated queue map */ 5149 i40e_vsi_update_queue_map(vsi, &ctxt); 5150 vsi->info.valid_sections = 0; 5151 5152 /* Update current VSI BW information */ 5153 ret = i40e_vsi_get_bw_info(vsi); 5154 if (ret) { 5155 dev_info(&vsi->back->pdev->dev, 5156 "Failed updating vsi bw info, err %s aq_err %s\n", 5157 i40e_stat_str(&vsi->back->hw, ret), 5158 i40e_aq_str(&vsi->back->hw, 5159 vsi->back->hw.aq.asq_last_status)); 5160 goto out; 5161 } 5162 5163 /* Update the netdev TC setup */ 5164 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5165 out: 5166 return ret; 5167 } 5168 5169 /** 5170 * i40e_veb_config_tc - Configure TCs for given VEB 5171 * @veb: given VEB 5172 * @enabled_tc: TC bitmap 5173 * 5174 * Configures given TC bitmap for VEB (switching) element 5175 **/ 5176 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 5177 { 5178 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 5179 struct i40e_pf *pf = veb->pf; 5180 int ret = 0; 5181 int i; 5182 5183 /* No TCs or already enabled TCs just return */ 5184 if (!enabled_tc || veb->enabled_tc == enabled_tc) 5185 return ret; 5186 5187 bw_data.tc_valid_bits = enabled_tc; 5188 /* bw_data.absolute_credits is not set (relative) */ 5189 5190 /* Enable ETS TCs with equal BW Share for now */ 5191 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5192 if (enabled_tc & BIT(i)) 5193 bw_data.tc_bw_share_credits[i] = 1; 5194 } 5195 5196 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 5197 &bw_data, NULL); 5198 if (ret) { 5199 dev_info(&pf->pdev->dev, 5200 "VEB bw config failed, err %s aq_err %s\n", 5201 i40e_stat_str(&pf->hw, ret), 5202 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5203 goto out; 5204 } 5205 5206 /* Update the BW information */ 5207 ret = i40e_veb_get_bw_info(veb); 5208 if (ret) { 5209 dev_info(&pf->pdev->dev, 5210 "Failed getting veb bw config, err %s aq_err %s\n", 5211 i40e_stat_str(&pf->hw, ret), 5212 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5213 } 5214 5215 out: 5216 return ret; 5217 } 5218 5219 #ifdef CONFIG_I40E_DCB 5220 /** 5221 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 5222 * @pf: PF struct 5223 * 5224 * Reconfigure VEB/VSIs on a given PF; it is assumed that 5225 * the caller would've quiesce all the VSIs before calling 5226 * this function 5227 **/ 5228 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 5229 { 5230 u8 tc_map = 0; 5231 int ret; 5232 u8 v; 5233 5234 /* Enable the TCs available on PF to all VEBs */ 5235 tc_map = i40e_pf_get_tc_map(pf); 5236 for (v = 0; v < I40E_MAX_VEB; v++) { 5237 if (!pf->veb[v]) 5238 continue; 5239 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 5240 if (ret) { 5241 dev_info(&pf->pdev->dev, 5242 "Failed configuring TC for VEB seid=%d\n", 5243 pf->veb[v]->seid); 5244 /* Will try to configure as many components */ 5245 } 5246 } 5247 5248 /* Update each VSI */ 5249 for (v = 0; v < pf->num_alloc_vsi; v++) { 5250 if (!pf->vsi[v]) 5251 continue; 5252 5253 /* - Enable all TCs for the LAN VSI 5254 * - For all others keep them at TC0 for now 5255 */ 5256 if (v == pf->lan_vsi) 5257 tc_map = i40e_pf_get_tc_map(pf); 5258 else 5259 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 5260 5261 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 5262 if (ret) { 5263 dev_info(&pf->pdev->dev, 5264 "Failed configuring TC for VSI seid=%d\n", 5265 pf->vsi[v]->seid); 5266 /* Will try to configure as many components */ 5267 } else { 5268 /* Re-configure VSI vectors based on updated TC map */ 5269 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 5270 if (pf->vsi[v]->netdev) 5271 i40e_dcbnl_set_all(pf->vsi[v]); 5272 } 5273 } 5274 } 5275 5276 /** 5277 * i40e_resume_port_tx - Resume port Tx 5278 * @pf: PF struct 5279 * 5280 * Resume a port's Tx and issue a PF reset in case of failure to 5281 * resume. 5282 **/ 5283 static int i40e_resume_port_tx(struct i40e_pf *pf) 5284 { 5285 struct i40e_hw *hw = &pf->hw; 5286 int ret; 5287 5288 ret = i40e_aq_resume_port_tx(hw, NULL); 5289 if (ret) { 5290 dev_info(&pf->pdev->dev, 5291 "Resume Port Tx failed, err %s aq_err %s\n", 5292 i40e_stat_str(&pf->hw, ret), 5293 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5294 /* Schedule PF reset to recover */ 5295 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 5296 i40e_service_event_schedule(pf); 5297 } 5298 5299 return ret; 5300 } 5301 5302 /** 5303 * i40e_init_pf_dcb - Initialize DCB configuration 5304 * @pf: PF being configured 5305 * 5306 * Query the current DCB configuration and cache it 5307 * in the hardware structure 5308 **/ 5309 static int i40e_init_pf_dcb(struct i40e_pf *pf) 5310 { 5311 struct i40e_hw *hw = &pf->hw; 5312 int err = 0; 5313 5314 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */ 5315 if (pf->hw_features & I40E_HW_NO_DCB_SUPPORT) 5316 goto out; 5317 5318 /* Get the initial DCB configuration */ 5319 err = i40e_init_dcb(hw); 5320 if (!err) { 5321 /* Device/Function is not DCBX capable */ 5322 if ((!hw->func_caps.dcb) || 5323 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 5324 dev_info(&pf->pdev->dev, 5325 "DCBX offload is not supported or is disabled for this PF.\n"); 5326 } else { 5327 /* When status is not DISABLED then DCBX in FW */ 5328 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 5329 DCB_CAP_DCBX_VER_IEEE; 5330 5331 pf->flags |= I40E_FLAG_DCB_CAPABLE; 5332 /* Enable DCB tagging only when more than one TC 5333 * or explicitly disable if only one TC 5334 */ 5335 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5336 pf->flags |= I40E_FLAG_DCB_ENABLED; 5337 else 5338 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5339 dev_dbg(&pf->pdev->dev, 5340 "DCBX offload is supported for this PF.\n"); 5341 } 5342 } else { 5343 dev_info(&pf->pdev->dev, 5344 "Query for DCB configuration failed, err %s aq_err %s\n", 5345 i40e_stat_str(&pf->hw, err), 5346 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5347 } 5348 5349 out: 5350 return err; 5351 } 5352 #endif /* CONFIG_I40E_DCB */ 5353 #define SPEED_SIZE 14 5354 #define FC_SIZE 8 5355 /** 5356 * i40e_print_link_message - print link up or down 5357 * @vsi: the VSI for which link needs a message 5358 */ 5359 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 5360 { 5361 enum i40e_aq_link_speed new_speed; 5362 struct i40e_pf *pf = vsi->back; 5363 char *speed = "Unknown"; 5364 char *fc = "Unknown"; 5365 char *fec = ""; 5366 char *req_fec = ""; 5367 char *an = ""; 5368 5369 new_speed = pf->hw.phy.link_info.link_speed; 5370 5371 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 5372 return; 5373 vsi->current_isup = isup; 5374 vsi->current_speed = new_speed; 5375 if (!isup) { 5376 netdev_info(vsi->netdev, "NIC Link is Down\n"); 5377 return; 5378 } 5379 5380 /* Warn user if link speed on NPAR enabled partition is not at 5381 * least 10GB 5382 */ 5383 if (pf->hw.func_caps.npar_enable && 5384 (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 5385 pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 5386 netdev_warn(vsi->netdev, 5387 "The partition detected link speed that is less than 10Gbps\n"); 5388 5389 switch (pf->hw.phy.link_info.link_speed) { 5390 case I40E_LINK_SPEED_40GB: 5391 speed = "40 G"; 5392 break; 5393 case I40E_LINK_SPEED_20GB: 5394 speed = "20 G"; 5395 break; 5396 case I40E_LINK_SPEED_25GB: 5397 speed = "25 G"; 5398 break; 5399 case I40E_LINK_SPEED_10GB: 5400 speed = "10 G"; 5401 break; 5402 case I40E_LINK_SPEED_1GB: 5403 speed = "1000 M"; 5404 break; 5405 case I40E_LINK_SPEED_100MB: 5406 speed = "100 M"; 5407 break; 5408 default: 5409 break; 5410 } 5411 5412 switch (pf->hw.fc.current_mode) { 5413 case I40E_FC_FULL: 5414 fc = "RX/TX"; 5415 break; 5416 case I40E_FC_TX_PAUSE: 5417 fc = "TX"; 5418 break; 5419 case I40E_FC_RX_PAUSE: 5420 fc = "RX"; 5421 break; 5422 default: 5423 fc = "None"; 5424 break; 5425 } 5426 5427 if (pf->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 5428 req_fec = ", Requested FEC: None"; 5429 fec = ", FEC: None"; 5430 an = ", Autoneg: False"; 5431 5432 if (pf->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 5433 an = ", Autoneg: True"; 5434 5435 if (pf->hw.phy.link_info.fec_info & 5436 I40E_AQ_CONFIG_FEC_KR_ENA) 5437 fec = ", FEC: CL74 FC-FEC/BASE-R"; 5438 else if (pf->hw.phy.link_info.fec_info & 5439 I40E_AQ_CONFIG_FEC_RS_ENA) 5440 fec = ", FEC: CL108 RS-FEC"; 5441 5442 /* 'CL108 RS-FEC' should be displayed when RS is requested, or 5443 * both RS and FC are requested 5444 */ 5445 if (vsi->back->hw.phy.link_info.req_fec_info & 5446 (I40E_AQ_REQUEST_FEC_KR | I40E_AQ_REQUEST_FEC_RS)) { 5447 if (vsi->back->hw.phy.link_info.req_fec_info & 5448 I40E_AQ_REQUEST_FEC_RS) 5449 req_fec = ", Requested FEC: CL108 RS-FEC"; 5450 else 5451 req_fec = ", Requested FEC: CL74 FC-FEC/BASE-R"; 5452 } 5453 } 5454 5455 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s%s, Flow Control: %s\n", 5456 speed, req_fec, fec, an, fc); 5457 } 5458 5459 /** 5460 * i40e_up_complete - Finish the last steps of bringing up a connection 5461 * @vsi: the VSI being configured 5462 **/ 5463 static int i40e_up_complete(struct i40e_vsi *vsi) 5464 { 5465 struct i40e_pf *pf = vsi->back; 5466 int err; 5467 5468 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 5469 i40e_vsi_configure_msix(vsi); 5470 else 5471 i40e_configure_msi_and_legacy(vsi); 5472 5473 /* start rings */ 5474 err = i40e_vsi_start_rings(vsi); 5475 if (err) 5476 return err; 5477 5478 clear_bit(__I40E_VSI_DOWN, vsi->state); 5479 i40e_napi_enable_all(vsi); 5480 i40e_vsi_enable_irq(vsi); 5481 5482 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 5483 (vsi->netdev)) { 5484 i40e_print_link_message(vsi, true); 5485 netif_tx_start_all_queues(vsi->netdev); 5486 netif_carrier_on(vsi->netdev); 5487 } 5488 5489 /* replay FDIR SB filters */ 5490 if (vsi->type == I40E_VSI_FDIR) { 5491 /* reset fd counters */ 5492 pf->fd_add_err = 0; 5493 pf->fd_atr_cnt = 0; 5494 i40e_fdir_filter_restore(vsi); 5495 } 5496 5497 /* On the next run of the service_task, notify any clients of the new 5498 * opened netdev 5499 */ 5500 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 5501 i40e_service_event_schedule(pf); 5502 5503 return 0; 5504 } 5505 5506 /** 5507 * i40e_vsi_reinit_locked - Reset the VSI 5508 * @vsi: the VSI being configured 5509 * 5510 * Rebuild the ring structs after some configuration 5511 * has changed, e.g. MTU size. 5512 **/ 5513 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 5514 { 5515 struct i40e_pf *pf = vsi->back; 5516 5517 WARN_ON(in_interrupt()); 5518 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 5519 usleep_range(1000, 2000); 5520 i40e_down(vsi); 5521 5522 i40e_up(vsi); 5523 clear_bit(__I40E_CONFIG_BUSY, pf->state); 5524 } 5525 5526 /** 5527 * i40e_up - Bring the connection back up after being down 5528 * @vsi: the VSI being configured 5529 **/ 5530 int i40e_up(struct i40e_vsi *vsi) 5531 { 5532 int err; 5533 5534 err = i40e_vsi_configure(vsi); 5535 if (!err) 5536 err = i40e_up_complete(vsi); 5537 5538 return err; 5539 } 5540 5541 /** 5542 * i40e_down - Shutdown the connection processing 5543 * @vsi: the VSI being stopped 5544 **/ 5545 void i40e_down(struct i40e_vsi *vsi) 5546 { 5547 int i; 5548 5549 /* It is assumed that the caller of this function 5550 * sets the vsi->state __I40E_VSI_DOWN bit. 5551 */ 5552 if (vsi->netdev) { 5553 netif_carrier_off(vsi->netdev); 5554 netif_tx_disable(vsi->netdev); 5555 } 5556 i40e_vsi_disable_irq(vsi); 5557 i40e_vsi_stop_rings(vsi); 5558 i40e_napi_disable_all(vsi); 5559 5560 for (i = 0; i < vsi->num_queue_pairs; i++) { 5561 i40e_clean_tx_ring(vsi->tx_rings[i]); 5562 if (i40e_enabled_xdp_vsi(vsi)) 5563 i40e_clean_tx_ring(vsi->xdp_rings[i]); 5564 i40e_clean_rx_ring(vsi->rx_rings[i]); 5565 } 5566 5567 } 5568 5569 /** 5570 * i40e_setup_tc - configure multiple traffic classes 5571 * @netdev: net device to configure 5572 * @tc: number of traffic classes to enable 5573 **/ 5574 static int i40e_setup_tc(struct net_device *netdev, u8 tc) 5575 { 5576 struct i40e_netdev_priv *np = netdev_priv(netdev); 5577 struct i40e_vsi *vsi = np->vsi; 5578 struct i40e_pf *pf = vsi->back; 5579 u8 enabled_tc = 0; 5580 int ret = -EINVAL; 5581 int i; 5582 5583 /* Check if DCB enabled to continue */ 5584 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5585 netdev_info(netdev, "DCB is not enabled for adapter\n"); 5586 goto exit; 5587 } 5588 5589 /* Check if MFP enabled */ 5590 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 5591 netdev_info(netdev, "Configuring TC not supported in MFP mode\n"); 5592 goto exit; 5593 } 5594 5595 /* Check whether tc count is within enabled limit */ 5596 if (tc > i40e_pf_get_num_tc(pf)) { 5597 netdev_info(netdev, "TC count greater than enabled on link for adapter\n"); 5598 goto exit; 5599 } 5600 5601 /* Generate TC map for number of tc requested */ 5602 for (i = 0; i < tc; i++) 5603 enabled_tc |= BIT(i); 5604 5605 /* Requesting same TC configuration as already enabled */ 5606 if (enabled_tc == vsi->tc_config.enabled_tc) 5607 return 0; 5608 5609 /* Quiesce VSI queues */ 5610 i40e_quiesce_vsi(vsi); 5611 5612 /* Configure VSI for enabled TCs */ 5613 ret = i40e_vsi_config_tc(vsi, enabled_tc); 5614 if (ret) { 5615 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 5616 vsi->seid); 5617 goto exit; 5618 } 5619 5620 /* Unquiesce VSI */ 5621 i40e_unquiesce_vsi(vsi); 5622 5623 exit: 5624 return ret; 5625 } 5626 5627 static int __i40e_setup_tc(struct net_device *netdev, enum tc_setup_type type, 5628 void *type_data) 5629 { 5630 struct tc_mqprio_qopt *mqprio = type_data; 5631 5632 if (type != TC_SETUP_MQPRIO) 5633 return -EOPNOTSUPP; 5634 5635 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 5636 5637 return i40e_setup_tc(netdev, mqprio->num_tc); 5638 } 5639 5640 /** 5641 * i40e_open - Called when a network interface is made active 5642 * @netdev: network interface device structure 5643 * 5644 * The open entry point is called when a network interface is made 5645 * active by the system (IFF_UP). At this point all resources needed 5646 * for transmit and receive operations are allocated, the interrupt 5647 * handler is registered with the OS, the netdev watchdog subtask is 5648 * enabled, and the stack is notified that the interface is ready. 5649 * 5650 * Returns 0 on success, negative value on failure 5651 **/ 5652 int i40e_open(struct net_device *netdev) 5653 { 5654 struct i40e_netdev_priv *np = netdev_priv(netdev); 5655 struct i40e_vsi *vsi = np->vsi; 5656 struct i40e_pf *pf = vsi->back; 5657 int err; 5658 5659 /* disallow open during test or if eeprom is broken */ 5660 if (test_bit(__I40E_TESTING, pf->state) || 5661 test_bit(__I40E_BAD_EEPROM, pf->state)) 5662 return -EBUSY; 5663 5664 netif_carrier_off(netdev); 5665 5666 err = i40e_vsi_open(vsi); 5667 if (err) 5668 return err; 5669 5670 /* configure global TSO hardware offload settings */ 5671 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 5672 TCP_FLAG_FIN) >> 16); 5673 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 5674 TCP_FLAG_FIN | 5675 TCP_FLAG_CWR) >> 16); 5676 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 5677 5678 udp_tunnel_get_rx_info(netdev); 5679 5680 return 0; 5681 } 5682 5683 /** 5684 * i40e_vsi_open - 5685 * @vsi: the VSI to open 5686 * 5687 * Finish initialization of the VSI. 5688 * 5689 * Returns 0 on success, negative value on failure 5690 * 5691 * Note: expects to be called while under rtnl_lock() 5692 **/ 5693 int i40e_vsi_open(struct i40e_vsi *vsi) 5694 { 5695 struct i40e_pf *pf = vsi->back; 5696 char int_name[I40E_INT_NAME_STR_LEN]; 5697 int err; 5698 5699 /* allocate descriptors */ 5700 err = i40e_vsi_setup_tx_resources(vsi); 5701 if (err) 5702 goto err_setup_tx; 5703 err = i40e_vsi_setup_rx_resources(vsi); 5704 if (err) 5705 goto err_setup_rx; 5706 5707 err = i40e_vsi_configure(vsi); 5708 if (err) 5709 goto err_setup_rx; 5710 5711 if (vsi->netdev) { 5712 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 5713 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 5714 err = i40e_vsi_request_irq(vsi, int_name); 5715 if (err) 5716 goto err_setup_rx; 5717 5718 /* Notify the stack of the actual queue counts. */ 5719 err = netif_set_real_num_tx_queues(vsi->netdev, 5720 vsi->num_queue_pairs); 5721 if (err) 5722 goto err_set_queues; 5723 5724 err = netif_set_real_num_rx_queues(vsi->netdev, 5725 vsi->num_queue_pairs); 5726 if (err) 5727 goto err_set_queues; 5728 5729 } else if (vsi->type == I40E_VSI_FDIR) { 5730 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 5731 dev_driver_string(&pf->pdev->dev), 5732 dev_name(&pf->pdev->dev)); 5733 err = i40e_vsi_request_irq(vsi, int_name); 5734 5735 } else { 5736 err = -EINVAL; 5737 goto err_setup_rx; 5738 } 5739 5740 err = i40e_up_complete(vsi); 5741 if (err) 5742 goto err_up_complete; 5743 5744 return 0; 5745 5746 err_up_complete: 5747 i40e_down(vsi); 5748 err_set_queues: 5749 i40e_vsi_free_irq(vsi); 5750 err_setup_rx: 5751 i40e_vsi_free_rx_resources(vsi); 5752 err_setup_tx: 5753 i40e_vsi_free_tx_resources(vsi); 5754 if (vsi == pf->vsi[pf->lan_vsi]) 5755 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true); 5756 5757 return err; 5758 } 5759 5760 /** 5761 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 5762 * @pf: Pointer to PF 5763 * 5764 * This function destroys the hlist where all the Flow Director 5765 * filters were saved. 5766 **/ 5767 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 5768 { 5769 struct i40e_fdir_filter *filter; 5770 struct i40e_flex_pit *pit_entry, *tmp; 5771 struct hlist_node *node2; 5772 5773 hlist_for_each_entry_safe(filter, node2, 5774 &pf->fdir_filter_list, fdir_node) { 5775 hlist_del(&filter->fdir_node); 5776 kfree(filter); 5777 } 5778 5779 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 5780 list_del(&pit_entry->list); 5781 kfree(pit_entry); 5782 } 5783 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 5784 5785 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 5786 list_del(&pit_entry->list); 5787 kfree(pit_entry); 5788 } 5789 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 5790 5791 pf->fdir_pf_active_filters = 0; 5792 pf->fd_tcp4_filter_cnt = 0; 5793 pf->fd_udp4_filter_cnt = 0; 5794 pf->fd_sctp4_filter_cnt = 0; 5795 pf->fd_ip4_filter_cnt = 0; 5796 5797 /* Reprogram the default input set for TCP/IPv4 */ 5798 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 5799 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5800 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5801 5802 /* Reprogram the default input set for UDP/IPv4 */ 5803 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 5804 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5805 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5806 5807 /* Reprogram the default input set for SCTP/IPv4 */ 5808 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 5809 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5810 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5811 5812 /* Reprogram the default input set for Other/IPv4 */ 5813 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 5814 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 5815 } 5816 5817 /** 5818 * i40e_close - Disables a network interface 5819 * @netdev: network interface device structure 5820 * 5821 * The close entry point is called when an interface is de-activated 5822 * by the OS. The hardware is still under the driver's control, but 5823 * this netdev interface is disabled. 5824 * 5825 * Returns 0, this is not allowed to fail 5826 **/ 5827 int i40e_close(struct net_device *netdev) 5828 { 5829 struct i40e_netdev_priv *np = netdev_priv(netdev); 5830 struct i40e_vsi *vsi = np->vsi; 5831 5832 i40e_vsi_close(vsi); 5833 5834 return 0; 5835 } 5836 5837 /** 5838 * i40e_do_reset - Start a PF or Core Reset sequence 5839 * @pf: board private structure 5840 * @reset_flags: which reset is requested 5841 * @lock_acquired: indicates whether or not the lock has been acquired 5842 * before this function was called. 5843 * 5844 * The essential difference in resets is that the PF Reset 5845 * doesn't clear the packet buffers, doesn't reset the PE 5846 * firmware, and doesn't bother the other PFs on the chip. 5847 **/ 5848 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 5849 { 5850 u32 val; 5851 5852 WARN_ON(in_interrupt()); 5853 5854 5855 /* do the biggest reset indicated */ 5856 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 5857 5858 /* Request a Global Reset 5859 * 5860 * This will start the chip's countdown to the actual full 5861 * chip reset event, and a warning interrupt to be sent 5862 * to all PFs, including the requestor. Our handler 5863 * for the warning interrupt will deal with the shutdown 5864 * and recovery of the switch setup. 5865 */ 5866 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 5867 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5868 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 5869 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5870 5871 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 5872 5873 /* Request a Core Reset 5874 * 5875 * Same as Global Reset, except does *not* include the MAC/PHY 5876 */ 5877 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 5878 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5879 val |= I40E_GLGEN_RTRIG_CORER_MASK; 5880 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5881 i40e_flush(&pf->hw); 5882 5883 } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) { 5884 5885 /* Request a PF Reset 5886 * 5887 * Resets only the PF-specific registers 5888 * 5889 * This goes directly to the tear-down and rebuild of 5890 * the switch, since we need to do all the recovery as 5891 * for the Core Reset. 5892 */ 5893 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 5894 i40e_handle_reset_warning(pf, lock_acquired); 5895 5896 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 5897 int v; 5898 5899 /* Find the VSI(s) that requested a re-init */ 5900 dev_info(&pf->pdev->dev, 5901 "VSI reinit requested\n"); 5902 for (v = 0; v < pf->num_alloc_vsi; v++) { 5903 struct i40e_vsi *vsi = pf->vsi[v]; 5904 5905 if (vsi != NULL && 5906 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 5907 vsi->state)) 5908 i40e_vsi_reinit_locked(pf->vsi[v]); 5909 } 5910 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 5911 int v; 5912 5913 /* Find the VSI(s) that needs to be brought down */ 5914 dev_info(&pf->pdev->dev, "VSI down requested\n"); 5915 for (v = 0; v < pf->num_alloc_vsi; v++) { 5916 struct i40e_vsi *vsi = pf->vsi[v]; 5917 5918 if (vsi != NULL && 5919 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 5920 vsi->state)) { 5921 set_bit(__I40E_VSI_DOWN, vsi->state); 5922 i40e_down(vsi); 5923 } 5924 } 5925 } else { 5926 dev_info(&pf->pdev->dev, 5927 "bad reset request 0x%08x\n", reset_flags); 5928 } 5929 } 5930 5931 #ifdef CONFIG_I40E_DCB 5932 /** 5933 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 5934 * @pf: board private structure 5935 * @old_cfg: current DCB config 5936 * @new_cfg: new DCB config 5937 **/ 5938 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 5939 struct i40e_dcbx_config *old_cfg, 5940 struct i40e_dcbx_config *new_cfg) 5941 { 5942 bool need_reconfig = false; 5943 5944 /* Check if ETS configuration has changed */ 5945 if (memcmp(&new_cfg->etscfg, 5946 &old_cfg->etscfg, 5947 sizeof(new_cfg->etscfg))) { 5948 /* If Priority Table has changed reconfig is needed */ 5949 if (memcmp(&new_cfg->etscfg.prioritytable, 5950 &old_cfg->etscfg.prioritytable, 5951 sizeof(new_cfg->etscfg.prioritytable))) { 5952 need_reconfig = true; 5953 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 5954 } 5955 5956 if (memcmp(&new_cfg->etscfg.tcbwtable, 5957 &old_cfg->etscfg.tcbwtable, 5958 sizeof(new_cfg->etscfg.tcbwtable))) 5959 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 5960 5961 if (memcmp(&new_cfg->etscfg.tsatable, 5962 &old_cfg->etscfg.tsatable, 5963 sizeof(new_cfg->etscfg.tsatable))) 5964 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 5965 } 5966 5967 /* Check if PFC configuration has changed */ 5968 if (memcmp(&new_cfg->pfc, 5969 &old_cfg->pfc, 5970 sizeof(new_cfg->pfc))) { 5971 need_reconfig = true; 5972 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 5973 } 5974 5975 /* Check if APP Table has changed */ 5976 if (memcmp(&new_cfg->app, 5977 &old_cfg->app, 5978 sizeof(new_cfg->app))) { 5979 need_reconfig = true; 5980 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 5981 } 5982 5983 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 5984 return need_reconfig; 5985 } 5986 5987 /** 5988 * i40e_handle_lldp_event - Handle LLDP Change MIB event 5989 * @pf: board private structure 5990 * @e: event info posted on ARQ 5991 **/ 5992 static int i40e_handle_lldp_event(struct i40e_pf *pf, 5993 struct i40e_arq_event_info *e) 5994 { 5995 struct i40e_aqc_lldp_get_mib *mib = 5996 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 5997 struct i40e_hw *hw = &pf->hw; 5998 struct i40e_dcbx_config tmp_dcbx_cfg; 5999 bool need_reconfig = false; 6000 int ret = 0; 6001 u8 type; 6002 6003 /* Not DCB capable or capability disabled */ 6004 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 6005 return ret; 6006 6007 /* Ignore if event is not for Nearest Bridge */ 6008 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 6009 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 6010 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 6011 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 6012 return ret; 6013 6014 /* Check MIB Type and return if event for Remote MIB update */ 6015 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 6016 dev_dbg(&pf->pdev->dev, 6017 "LLDP event mib type %s\n", type ? "remote" : "local"); 6018 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 6019 /* Update the remote cached instance and return */ 6020 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 6021 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 6022 &hw->remote_dcbx_config); 6023 goto exit; 6024 } 6025 6026 /* Store the old configuration */ 6027 tmp_dcbx_cfg = hw->local_dcbx_config; 6028 6029 /* Reset the old DCBx configuration data */ 6030 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 6031 /* Get updated DCBX data from firmware */ 6032 ret = i40e_get_dcb_config(&pf->hw); 6033 if (ret) { 6034 dev_info(&pf->pdev->dev, 6035 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 6036 i40e_stat_str(&pf->hw, ret), 6037 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6038 goto exit; 6039 } 6040 6041 /* No change detected in DCBX configs */ 6042 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 6043 sizeof(tmp_dcbx_cfg))) { 6044 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 6045 goto exit; 6046 } 6047 6048 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 6049 &hw->local_dcbx_config); 6050 6051 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 6052 6053 if (!need_reconfig) 6054 goto exit; 6055 6056 /* Enable DCB tagging only when more than one TC */ 6057 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 6058 pf->flags |= I40E_FLAG_DCB_ENABLED; 6059 else 6060 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 6061 6062 set_bit(__I40E_PORT_SUSPENDED, pf->state); 6063 /* Reconfiguration needed quiesce all VSIs */ 6064 i40e_pf_quiesce_all_vsi(pf); 6065 6066 /* Changes in configuration update VEB/VSI */ 6067 i40e_dcb_reconfigure(pf); 6068 6069 ret = i40e_resume_port_tx(pf); 6070 6071 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 6072 /* In case of error no point in resuming VSIs */ 6073 if (ret) 6074 goto exit; 6075 6076 /* Wait for the PF's queues to be disabled */ 6077 ret = i40e_pf_wait_queues_disabled(pf); 6078 if (ret) { 6079 /* Schedule PF reset to recover */ 6080 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6081 i40e_service_event_schedule(pf); 6082 } else { 6083 i40e_pf_unquiesce_all_vsi(pf); 6084 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 6085 I40E_FLAG_CLIENT_L2_CHANGE); 6086 } 6087 6088 exit: 6089 return ret; 6090 } 6091 #endif /* CONFIG_I40E_DCB */ 6092 6093 /** 6094 * i40e_do_reset_safe - Protected reset path for userland calls. 6095 * @pf: board private structure 6096 * @reset_flags: which reset is requested 6097 * 6098 **/ 6099 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 6100 { 6101 rtnl_lock(); 6102 i40e_do_reset(pf, reset_flags, true); 6103 rtnl_unlock(); 6104 } 6105 6106 /** 6107 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 6108 * @pf: board private structure 6109 * @e: event info posted on ARQ 6110 * 6111 * Handler for LAN Queue Overflow Event generated by the firmware for PF 6112 * and VF queues 6113 **/ 6114 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 6115 struct i40e_arq_event_info *e) 6116 { 6117 struct i40e_aqc_lan_overflow *data = 6118 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 6119 u32 queue = le32_to_cpu(data->prtdcb_rupto); 6120 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 6121 struct i40e_hw *hw = &pf->hw; 6122 struct i40e_vf *vf; 6123 u16 vf_id; 6124 6125 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 6126 queue, qtx_ctl); 6127 6128 /* Queue belongs to VF, find the VF and issue VF reset */ 6129 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 6130 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 6131 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 6132 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 6133 vf_id -= hw->func_caps.vf_base_id; 6134 vf = &pf->vf[vf_id]; 6135 i40e_vc_notify_vf_reset(vf); 6136 /* Allow VF to process pending reset notification */ 6137 msleep(20); 6138 i40e_reset_vf(vf, false); 6139 } 6140 } 6141 6142 /** 6143 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 6144 * @pf: board private structure 6145 **/ 6146 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 6147 { 6148 u32 val, fcnt_prog; 6149 6150 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 6151 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 6152 return fcnt_prog; 6153 } 6154 6155 /** 6156 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 6157 * @pf: board private structure 6158 **/ 6159 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 6160 { 6161 u32 val, fcnt_prog; 6162 6163 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 6164 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 6165 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 6166 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 6167 return fcnt_prog; 6168 } 6169 6170 /** 6171 * i40e_get_global_fd_count - Get total FD filters programmed on device 6172 * @pf: board private structure 6173 **/ 6174 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 6175 { 6176 u32 val, fcnt_prog; 6177 6178 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 6179 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 6180 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 6181 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 6182 return fcnt_prog; 6183 } 6184 6185 /** 6186 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 6187 * @pf: board private structure 6188 **/ 6189 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 6190 { 6191 struct i40e_fdir_filter *filter; 6192 u32 fcnt_prog, fcnt_avail; 6193 struct hlist_node *node; 6194 6195 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 6196 return; 6197 6198 /* Check if we have enough room to re-enable FDir SB capability. */ 6199 fcnt_prog = i40e_get_global_fd_count(pf); 6200 fcnt_avail = pf->fdir_pf_filter_count; 6201 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 6202 (pf->fd_add_err == 0) || 6203 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) { 6204 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) { 6205 pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED; 6206 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 6207 (I40E_DEBUG_FD & pf->hw.debug_mask)) 6208 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 6209 } 6210 } 6211 6212 /* We should wait for even more space before re-enabling ATR. 6213 * Additionally, we cannot enable ATR as long as we still have TCP SB 6214 * rules active. 6215 */ 6216 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 6217 (pf->fd_tcp4_filter_cnt == 0)) { 6218 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) { 6219 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 6220 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 6221 (I40E_DEBUG_FD & pf->hw.debug_mask)) 6222 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 6223 } 6224 } 6225 6226 /* if hw had a problem adding a filter, delete it */ 6227 if (pf->fd_inv > 0) { 6228 hlist_for_each_entry_safe(filter, node, 6229 &pf->fdir_filter_list, fdir_node) { 6230 if (filter->fd_id == pf->fd_inv) { 6231 hlist_del(&filter->fdir_node); 6232 kfree(filter); 6233 pf->fdir_pf_active_filters--; 6234 } 6235 } 6236 } 6237 } 6238 6239 #define I40E_MIN_FD_FLUSH_INTERVAL 10 6240 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 6241 /** 6242 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 6243 * @pf: board private structure 6244 **/ 6245 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 6246 { 6247 unsigned long min_flush_time; 6248 int flush_wait_retry = 50; 6249 bool disable_atr = false; 6250 int fd_room; 6251 int reg; 6252 6253 if (!time_after(jiffies, pf->fd_flush_timestamp + 6254 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 6255 return; 6256 6257 /* If the flush is happening too quick and we have mostly SB rules we 6258 * should not re-enable ATR for some time. 6259 */ 6260 min_flush_time = pf->fd_flush_timestamp + 6261 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 6262 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 6263 6264 if (!(time_after(jiffies, min_flush_time)) && 6265 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 6266 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6267 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 6268 disable_atr = true; 6269 } 6270 6271 pf->fd_flush_timestamp = jiffies; 6272 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED; 6273 /* flush all filters */ 6274 wr32(&pf->hw, I40E_PFQF_CTL_1, 6275 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 6276 i40e_flush(&pf->hw); 6277 pf->fd_flush_cnt++; 6278 pf->fd_add_err = 0; 6279 do { 6280 /* Check FD flush status every 5-6msec */ 6281 usleep_range(5000, 6000); 6282 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 6283 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 6284 break; 6285 } while (flush_wait_retry--); 6286 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 6287 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 6288 } else { 6289 /* replay sideband filters */ 6290 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 6291 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 6292 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 6293 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 6294 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6295 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 6296 } 6297 } 6298 6299 /** 6300 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 6301 * @pf: board private structure 6302 **/ 6303 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 6304 { 6305 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 6306 } 6307 6308 /* We can see up to 256 filter programming desc in transit if the filters are 6309 * being applied really fast; before we see the first 6310 * filter miss error on Rx queue 0. Accumulating enough error messages before 6311 * reacting will make sure we don't cause flush too often. 6312 */ 6313 #define I40E_MAX_FD_PROGRAM_ERROR 256 6314 6315 /** 6316 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 6317 * @pf: board private structure 6318 **/ 6319 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 6320 { 6321 6322 /* if interface is down do nothing */ 6323 if (test_bit(__I40E_DOWN, pf->state)) 6324 return; 6325 6326 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 6327 i40e_fdir_flush_and_replay(pf); 6328 6329 i40e_fdir_check_and_reenable(pf); 6330 6331 } 6332 6333 /** 6334 * i40e_vsi_link_event - notify VSI of a link event 6335 * @vsi: vsi to be notified 6336 * @link_up: link up or down 6337 **/ 6338 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 6339 { 6340 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 6341 return; 6342 6343 switch (vsi->type) { 6344 case I40E_VSI_MAIN: 6345 if (!vsi->netdev || !vsi->netdev_registered) 6346 break; 6347 6348 if (link_up) { 6349 netif_carrier_on(vsi->netdev); 6350 netif_tx_wake_all_queues(vsi->netdev); 6351 } else { 6352 netif_carrier_off(vsi->netdev); 6353 netif_tx_stop_all_queues(vsi->netdev); 6354 } 6355 break; 6356 6357 case I40E_VSI_SRIOV: 6358 case I40E_VSI_VMDQ2: 6359 case I40E_VSI_CTRL: 6360 case I40E_VSI_IWARP: 6361 case I40E_VSI_MIRROR: 6362 default: 6363 /* there is no notification for other VSIs */ 6364 break; 6365 } 6366 } 6367 6368 /** 6369 * i40e_veb_link_event - notify elements on the veb of a link event 6370 * @veb: veb to be notified 6371 * @link_up: link up or down 6372 **/ 6373 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 6374 { 6375 struct i40e_pf *pf; 6376 int i; 6377 6378 if (!veb || !veb->pf) 6379 return; 6380 pf = veb->pf; 6381 6382 /* depth first... */ 6383 for (i = 0; i < I40E_MAX_VEB; i++) 6384 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 6385 i40e_veb_link_event(pf->veb[i], link_up); 6386 6387 /* ... now the local VSIs */ 6388 for (i = 0; i < pf->num_alloc_vsi; i++) 6389 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 6390 i40e_vsi_link_event(pf->vsi[i], link_up); 6391 } 6392 6393 /** 6394 * i40e_link_event - Update netif_carrier status 6395 * @pf: board private structure 6396 **/ 6397 static void i40e_link_event(struct i40e_pf *pf) 6398 { 6399 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6400 u8 new_link_speed, old_link_speed; 6401 i40e_status status; 6402 bool new_link, old_link; 6403 6404 /* save off old link status information */ 6405 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 6406 6407 /* set this to force the get_link_status call to refresh state */ 6408 pf->hw.phy.get_link_info = true; 6409 6410 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 6411 6412 status = i40e_get_link_status(&pf->hw, &new_link); 6413 6414 /* On success, disable temp link polling */ 6415 if (status == I40E_SUCCESS) { 6416 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING) 6417 pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING; 6418 } else { 6419 /* Enable link polling temporarily until i40e_get_link_status 6420 * returns I40E_SUCCESS 6421 */ 6422 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING; 6423 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 6424 status); 6425 return; 6426 } 6427 6428 old_link_speed = pf->hw.phy.link_info_old.link_speed; 6429 new_link_speed = pf->hw.phy.link_info.link_speed; 6430 6431 if (new_link == old_link && 6432 new_link_speed == old_link_speed && 6433 (test_bit(__I40E_VSI_DOWN, vsi->state) || 6434 new_link == netif_carrier_ok(vsi->netdev))) 6435 return; 6436 6437 i40e_print_link_message(vsi, new_link); 6438 6439 /* Notify the base of the switch tree connected to 6440 * the link. Floating VEBs are not notified. 6441 */ 6442 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 6443 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 6444 else 6445 i40e_vsi_link_event(vsi, new_link); 6446 6447 if (pf->vf) 6448 i40e_vc_notify_link_state(pf); 6449 6450 if (pf->flags & I40E_FLAG_PTP) 6451 i40e_ptp_set_increment(pf); 6452 } 6453 6454 /** 6455 * i40e_watchdog_subtask - periodic checks not using event driven response 6456 * @pf: board private structure 6457 **/ 6458 static void i40e_watchdog_subtask(struct i40e_pf *pf) 6459 { 6460 int i; 6461 6462 /* if interface is down do nothing */ 6463 if (test_bit(__I40E_DOWN, pf->state) || 6464 test_bit(__I40E_CONFIG_BUSY, pf->state)) 6465 return; 6466 6467 /* make sure we don't do these things too often */ 6468 if (time_before(jiffies, (pf->service_timer_previous + 6469 pf->service_timer_period))) 6470 return; 6471 pf->service_timer_previous = jiffies; 6472 6473 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 6474 (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)) 6475 i40e_link_event(pf); 6476 6477 /* Update the stats for active netdevs so the network stack 6478 * can look at updated numbers whenever it cares to 6479 */ 6480 for (i = 0; i < pf->num_alloc_vsi; i++) 6481 if (pf->vsi[i] && pf->vsi[i]->netdev) 6482 i40e_update_stats(pf->vsi[i]); 6483 6484 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 6485 /* Update the stats for the active switching components */ 6486 for (i = 0; i < I40E_MAX_VEB; i++) 6487 if (pf->veb[i]) 6488 i40e_update_veb_stats(pf->veb[i]); 6489 } 6490 6491 i40e_ptp_rx_hang(pf); 6492 i40e_ptp_tx_hang(pf); 6493 } 6494 6495 /** 6496 * i40e_reset_subtask - Set up for resetting the device and driver 6497 * @pf: board private structure 6498 **/ 6499 static void i40e_reset_subtask(struct i40e_pf *pf) 6500 { 6501 u32 reset_flags = 0; 6502 6503 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 6504 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 6505 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 6506 } 6507 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 6508 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 6509 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6510 } 6511 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 6512 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 6513 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 6514 } 6515 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 6516 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 6517 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 6518 } 6519 if (test_bit(__I40E_DOWN_REQUESTED, pf->state)) { 6520 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 6521 clear_bit(__I40E_DOWN_REQUESTED, pf->state); 6522 } 6523 6524 /* If there's a recovery already waiting, it takes 6525 * precedence before starting a new reset sequence. 6526 */ 6527 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 6528 i40e_prep_for_reset(pf, false); 6529 i40e_reset(pf); 6530 i40e_rebuild(pf, false, false); 6531 } 6532 6533 /* If we're already down or resetting, just bail */ 6534 if (reset_flags && 6535 !test_bit(__I40E_DOWN, pf->state) && 6536 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 6537 i40e_do_reset(pf, reset_flags, false); 6538 } 6539 } 6540 6541 /** 6542 * i40e_handle_link_event - Handle link event 6543 * @pf: board private structure 6544 * @e: event info posted on ARQ 6545 **/ 6546 static void i40e_handle_link_event(struct i40e_pf *pf, 6547 struct i40e_arq_event_info *e) 6548 { 6549 struct i40e_aqc_get_link_status *status = 6550 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 6551 6552 /* Do a new status request to re-enable LSE reporting 6553 * and load new status information into the hw struct 6554 * This completely ignores any state information 6555 * in the ARQ event info, instead choosing to always 6556 * issue the AQ update link status command. 6557 */ 6558 i40e_link_event(pf); 6559 6560 /* check for unqualified module, if link is down */ 6561 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 6562 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 6563 (!(status->link_info & I40E_AQ_LINK_UP))) 6564 dev_err(&pf->pdev->dev, 6565 "The driver failed to link because an unqualified module was detected.\n"); 6566 } 6567 6568 /** 6569 * i40e_clean_adminq_subtask - Clean the AdminQ rings 6570 * @pf: board private structure 6571 **/ 6572 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 6573 { 6574 struct i40e_arq_event_info event; 6575 struct i40e_hw *hw = &pf->hw; 6576 u16 pending, i = 0; 6577 i40e_status ret; 6578 u16 opcode; 6579 u32 oldval; 6580 u32 val; 6581 6582 /* Do not run clean AQ when PF reset fails */ 6583 if (test_bit(__I40E_RESET_FAILED, pf->state)) 6584 return; 6585 6586 /* check for error indications */ 6587 val = rd32(&pf->hw, pf->hw.aq.arq.len); 6588 oldval = val; 6589 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 6590 if (hw->debug_mask & I40E_DEBUG_AQ) 6591 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 6592 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 6593 } 6594 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 6595 if (hw->debug_mask & I40E_DEBUG_AQ) 6596 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 6597 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 6598 pf->arq_overflows++; 6599 } 6600 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 6601 if (hw->debug_mask & I40E_DEBUG_AQ) 6602 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 6603 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 6604 } 6605 if (oldval != val) 6606 wr32(&pf->hw, pf->hw.aq.arq.len, val); 6607 6608 val = rd32(&pf->hw, pf->hw.aq.asq.len); 6609 oldval = val; 6610 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 6611 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6612 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 6613 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 6614 } 6615 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 6616 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6617 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 6618 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 6619 } 6620 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 6621 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6622 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 6623 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 6624 } 6625 if (oldval != val) 6626 wr32(&pf->hw, pf->hw.aq.asq.len, val); 6627 6628 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 6629 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 6630 if (!event.msg_buf) 6631 return; 6632 6633 do { 6634 ret = i40e_clean_arq_element(hw, &event, &pending); 6635 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 6636 break; 6637 else if (ret) { 6638 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 6639 break; 6640 } 6641 6642 opcode = le16_to_cpu(event.desc.opcode); 6643 switch (opcode) { 6644 6645 case i40e_aqc_opc_get_link_status: 6646 i40e_handle_link_event(pf, &event); 6647 break; 6648 case i40e_aqc_opc_send_msg_to_pf: 6649 ret = i40e_vc_process_vf_msg(pf, 6650 le16_to_cpu(event.desc.retval), 6651 le32_to_cpu(event.desc.cookie_high), 6652 le32_to_cpu(event.desc.cookie_low), 6653 event.msg_buf, 6654 event.msg_len); 6655 break; 6656 case i40e_aqc_opc_lldp_update_mib: 6657 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 6658 #ifdef CONFIG_I40E_DCB 6659 rtnl_lock(); 6660 ret = i40e_handle_lldp_event(pf, &event); 6661 rtnl_unlock(); 6662 #endif /* CONFIG_I40E_DCB */ 6663 break; 6664 case i40e_aqc_opc_event_lan_overflow: 6665 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 6666 i40e_handle_lan_overflow_event(pf, &event); 6667 break; 6668 case i40e_aqc_opc_send_msg_to_peer: 6669 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 6670 break; 6671 case i40e_aqc_opc_nvm_erase: 6672 case i40e_aqc_opc_nvm_update: 6673 case i40e_aqc_opc_oem_post_update: 6674 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 6675 "ARQ NVM operation 0x%04x completed\n", 6676 opcode); 6677 break; 6678 default: 6679 dev_info(&pf->pdev->dev, 6680 "ARQ: Unknown event 0x%04x ignored\n", 6681 opcode); 6682 break; 6683 } 6684 } while (i++ < pf->adminq_work_limit); 6685 6686 if (i < pf->adminq_work_limit) 6687 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 6688 6689 /* re-enable Admin queue interrupt cause */ 6690 val = rd32(hw, I40E_PFINT_ICR0_ENA); 6691 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 6692 wr32(hw, I40E_PFINT_ICR0_ENA, val); 6693 i40e_flush(hw); 6694 6695 kfree(event.msg_buf); 6696 } 6697 6698 /** 6699 * i40e_verify_eeprom - make sure eeprom is good to use 6700 * @pf: board private structure 6701 **/ 6702 static void i40e_verify_eeprom(struct i40e_pf *pf) 6703 { 6704 int err; 6705 6706 err = i40e_diag_eeprom_test(&pf->hw); 6707 if (err) { 6708 /* retry in case of garbage read */ 6709 err = i40e_diag_eeprom_test(&pf->hw); 6710 if (err) { 6711 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 6712 err); 6713 set_bit(__I40E_BAD_EEPROM, pf->state); 6714 } 6715 } 6716 6717 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 6718 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 6719 clear_bit(__I40E_BAD_EEPROM, pf->state); 6720 } 6721 } 6722 6723 /** 6724 * i40e_enable_pf_switch_lb 6725 * @pf: pointer to the PF structure 6726 * 6727 * enable switch loop back or die - no point in a return value 6728 **/ 6729 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 6730 { 6731 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6732 struct i40e_vsi_context ctxt; 6733 int ret; 6734 6735 ctxt.seid = pf->main_vsi_seid; 6736 ctxt.pf_num = pf->hw.pf_id; 6737 ctxt.vf_num = 0; 6738 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6739 if (ret) { 6740 dev_info(&pf->pdev->dev, 6741 "couldn't get PF vsi config, err %s aq_err %s\n", 6742 i40e_stat_str(&pf->hw, ret), 6743 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6744 return; 6745 } 6746 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6747 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6748 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6749 6750 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6751 if (ret) { 6752 dev_info(&pf->pdev->dev, 6753 "update vsi switch failed, err %s aq_err %s\n", 6754 i40e_stat_str(&pf->hw, ret), 6755 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6756 } 6757 } 6758 6759 /** 6760 * i40e_disable_pf_switch_lb 6761 * @pf: pointer to the PF structure 6762 * 6763 * disable switch loop back or die - no point in a return value 6764 **/ 6765 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 6766 { 6767 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6768 struct i40e_vsi_context ctxt; 6769 int ret; 6770 6771 ctxt.seid = pf->main_vsi_seid; 6772 ctxt.pf_num = pf->hw.pf_id; 6773 ctxt.vf_num = 0; 6774 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6775 if (ret) { 6776 dev_info(&pf->pdev->dev, 6777 "couldn't get PF vsi config, err %s aq_err %s\n", 6778 i40e_stat_str(&pf->hw, ret), 6779 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6780 return; 6781 } 6782 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6783 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6784 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6785 6786 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6787 if (ret) { 6788 dev_info(&pf->pdev->dev, 6789 "update vsi switch failed, err %s aq_err %s\n", 6790 i40e_stat_str(&pf->hw, ret), 6791 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6792 } 6793 } 6794 6795 /** 6796 * i40e_config_bridge_mode - Configure the HW bridge mode 6797 * @veb: pointer to the bridge instance 6798 * 6799 * Configure the loop back mode for the LAN VSI that is downlink to the 6800 * specified HW bridge instance. It is expected this function is called 6801 * when a new HW bridge is instantiated. 6802 **/ 6803 static void i40e_config_bridge_mode(struct i40e_veb *veb) 6804 { 6805 struct i40e_pf *pf = veb->pf; 6806 6807 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 6808 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 6809 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 6810 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 6811 i40e_disable_pf_switch_lb(pf); 6812 else 6813 i40e_enable_pf_switch_lb(pf); 6814 } 6815 6816 /** 6817 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 6818 * @veb: pointer to the VEB instance 6819 * 6820 * This is a recursive function that first builds the attached VSIs then 6821 * recurses in to build the next layer of VEB. We track the connections 6822 * through our own index numbers because the seid's from the HW could 6823 * change across the reset. 6824 **/ 6825 static int i40e_reconstitute_veb(struct i40e_veb *veb) 6826 { 6827 struct i40e_vsi *ctl_vsi = NULL; 6828 struct i40e_pf *pf = veb->pf; 6829 int v, veb_idx; 6830 int ret; 6831 6832 /* build VSI that owns this VEB, temporarily attached to base VEB */ 6833 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 6834 if (pf->vsi[v] && 6835 pf->vsi[v]->veb_idx == veb->idx && 6836 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 6837 ctl_vsi = pf->vsi[v]; 6838 break; 6839 } 6840 } 6841 if (!ctl_vsi) { 6842 dev_info(&pf->pdev->dev, 6843 "missing owner VSI for veb_idx %d\n", veb->idx); 6844 ret = -ENOENT; 6845 goto end_reconstitute; 6846 } 6847 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 6848 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6849 ret = i40e_add_vsi(ctl_vsi); 6850 if (ret) { 6851 dev_info(&pf->pdev->dev, 6852 "rebuild of veb_idx %d owner VSI failed: %d\n", 6853 veb->idx, ret); 6854 goto end_reconstitute; 6855 } 6856 i40e_vsi_reset_stats(ctl_vsi); 6857 6858 /* create the VEB in the switch and move the VSI onto the VEB */ 6859 ret = i40e_add_veb(veb, ctl_vsi); 6860 if (ret) 6861 goto end_reconstitute; 6862 6863 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 6864 veb->bridge_mode = BRIDGE_MODE_VEB; 6865 else 6866 veb->bridge_mode = BRIDGE_MODE_VEPA; 6867 i40e_config_bridge_mode(veb); 6868 6869 /* create the remaining VSIs attached to this VEB */ 6870 for (v = 0; v < pf->num_alloc_vsi; v++) { 6871 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 6872 continue; 6873 6874 if (pf->vsi[v]->veb_idx == veb->idx) { 6875 struct i40e_vsi *vsi = pf->vsi[v]; 6876 6877 vsi->uplink_seid = veb->seid; 6878 ret = i40e_add_vsi(vsi); 6879 if (ret) { 6880 dev_info(&pf->pdev->dev, 6881 "rebuild of vsi_idx %d failed: %d\n", 6882 v, ret); 6883 goto end_reconstitute; 6884 } 6885 i40e_vsi_reset_stats(vsi); 6886 } 6887 } 6888 6889 /* create any VEBs attached to this VEB - RECURSION */ 6890 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 6891 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 6892 pf->veb[veb_idx]->uplink_seid = veb->seid; 6893 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 6894 if (ret) 6895 break; 6896 } 6897 } 6898 6899 end_reconstitute: 6900 return ret; 6901 } 6902 6903 /** 6904 * i40e_get_capabilities - get info about the HW 6905 * @pf: the PF struct 6906 **/ 6907 static int i40e_get_capabilities(struct i40e_pf *pf) 6908 { 6909 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 6910 u16 data_size; 6911 int buf_len; 6912 int err; 6913 6914 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 6915 do { 6916 cap_buf = kzalloc(buf_len, GFP_KERNEL); 6917 if (!cap_buf) 6918 return -ENOMEM; 6919 6920 /* this loads the data into the hw struct for us */ 6921 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 6922 &data_size, 6923 i40e_aqc_opc_list_func_capabilities, 6924 NULL); 6925 /* data loaded, buffer no longer needed */ 6926 kfree(cap_buf); 6927 6928 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 6929 /* retry with a larger buffer */ 6930 buf_len = data_size; 6931 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 6932 dev_info(&pf->pdev->dev, 6933 "capability discovery failed, err %s aq_err %s\n", 6934 i40e_stat_str(&pf->hw, err), 6935 i40e_aq_str(&pf->hw, 6936 pf->hw.aq.asq_last_status)); 6937 return -ENODEV; 6938 } 6939 } while (err); 6940 6941 if (pf->hw.debug_mask & I40E_DEBUG_USER) 6942 dev_info(&pf->pdev->dev, 6943 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 6944 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 6945 pf->hw.func_caps.num_msix_vectors, 6946 pf->hw.func_caps.num_msix_vectors_vf, 6947 pf->hw.func_caps.fd_filters_guaranteed, 6948 pf->hw.func_caps.fd_filters_best_effort, 6949 pf->hw.func_caps.num_tx_qp, 6950 pf->hw.func_caps.num_vsis); 6951 6952 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 6953 + pf->hw.func_caps.num_vfs) 6954 if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) { 6955 dev_info(&pf->pdev->dev, 6956 "got num_vsis %d, setting num_vsis to %d\n", 6957 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 6958 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 6959 } 6960 6961 return 0; 6962 } 6963 6964 static int i40e_vsi_clear(struct i40e_vsi *vsi); 6965 6966 /** 6967 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 6968 * @pf: board private structure 6969 **/ 6970 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 6971 { 6972 struct i40e_vsi *vsi; 6973 6974 /* quick workaround for an NVM issue that leaves a critical register 6975 * uninitialized 6976 */ 6977 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 6978 static const u32 hkey[] = { 6979 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 6980 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 6981 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 6982 0x95b3a76d}; 6983 int i; 6984 6985 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 6986 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 6987 } 6988 6989 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 6990 return; 6991 6992 /* find existing VSI and see if it needs configuring */ 6993 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6994 6995 /* create a new VSI if none exists */ 6996 if (!vsi) { 6997 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 6998 pf->vsi[pf->lan_vsi]->seid, 0); 6999 if (!vsi) { 7000 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 7001 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 7002 return; 7003 } 7004 } 7005 7006 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 7007 } 7008 7009 /** 7010 * i40e_fdir_teardown - release the Flow Director resources 7011 * @pf: board private structure 7012 **/ 7013 static void i40e_fdir_teardown(struct i40e_pf *pf) 7014 { 7015 struct i40e_vsi *vsi; 7016 7017 i40e_fdir_filter_exit(pf); 7018 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 7019 if (vsi) 7020 i40e_vsi_release(vsi); 7021 } 7022 7023 /** 7024 * i40e_prep_for_reset - prep for the core to reset 7025 * @pf: board private structure 7026 * @lock_acquired: indicates whether or not the lock has been acquired 7027 * before this function was called. 7028 * 7029 * Close up the VFs and other things in prep for PF Reset. 7030 **/ 7031 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 7032 { 7033 struct i40e_hw *hw = &pf->hw; 7034 i40e_status ret = 0; 7035 u32 v; 7036 7037 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 7038 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 7039 return; 7040 if (i40e_check_asq_alive(&pf->hw)) 7041 i40e_vc_notify_reset(pf); 7042 7043 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 7044 7045 /* quiesce the VSIs and their queues that are not already DOWN */ 7046 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 7047 if (!lock_acquired) 7048 rtnl_lock(); 7049 i40e_pf_quiesce_all_vsi(pf); 7050 if (!lock_acquired) 7051 rtnl_unlock(); 7052 7053 for (v = 0; v < pf->num_alloc_vsi; v++) { 7054 if (pf->vsi[v]) 7055 pf->vsi[v]->seid = 0; 7056 } 7057 7058 i40e_shutdown_adminq(&pf->hw); 7059 7060 /* call shutdown HMC */ 7061 if (hw->hmc.hmc_obj) { 7062 ret = i40e_shutdown_lan_hmc(hw); 7063 if (ret) 7064 dev_warn(&pf->pdev->dev, 7065 "shutdown_lan_hmc failed: %d\n", ret); 7066 } 7067 } 7068 7069 /** 7070 * i40e_send_version - update firmware with driver version 7071 * @pf: PF struct 7072 */ 7073 static void i40e_send_version(struct i40e_pf *pf) 7074 { 7075 struct i40e_driver_version dv; 7076 7077 dv.major_version = DRV_VERSION_MAJOR; 7078 dv.minor_version = DRV_VERSION_MINOR; 7079 dv.build_version = DRV_VERSION_BUILD; 7080 dv.subbuild_version = 0; 7081 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 7082 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 7083 } 7084 7085 /** 7086 * i40e_get_oem_version - get OEM specific version information 7087 * @hw: pointer to the hardware structure 7088 **/ 7089 static void i40e_get_oem_version(struct i40e_hw *hw) 7090 { 7091 u16 block_offset = 0xffff; 7092 u16 block_length = 0; 7093 u16 capabilities = 0; 7094 u16 gen_snap = 0; 7095 u16 release = 0; 7096 7097 #define I40E_SR_NVM_OEM_VERSION_PTR 0x1B 7098 #define I40E_NVM_OEM_LENGTH_OFFSET 0x00 7099 #define I40E_NVM_OEM_CAPABILITIES_OFFSET 0x01 7100 #define I40E_NVM_OEM_GEN_OFFSET 0x02 7101 #define I40E_NVM_OEM_RELEASE_OFFSET 0x03 7102 #define I40E_NVM_OEM_CAPABILITIES_MASK 0x000F 7103 #define I40E_NVM_OEM_LENGTH 3 7104 7105 /* Check if pointer to OEM version block is valid. */ 7106 i40e_read_nvm_word(hw, I40E_SR_NVM_OEM_VERSION_PTR, &block_offset); 7107 if (block_offset == 0xffff) 7108 return; 7109 7110 /* Check if OEM version block has correct length. */ 7111 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_LENGTH_OFFSET, 7112 &block_length); 7113 if (block_length < I40E_NVM_OEM_LENGTH) 7114 return; 7115 7116 /* Check if OEM version format is as expected. */ 7117 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_CAPABILITIES_OFFSET, 7118 &capabilities); 7119 if ((capabilities & I40E_NVM_OEM_CAPABILITIES_MASK) != 0) 7120 return; 7121 7122 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_GEN_OFFSET, 7123 &gen_snap); 7124 i40e_read_nvm_word(hw, block_offset + I40E_NVM_OEM_RELEASE_OFFSET, 7125 &release); 7126 hw->nvm.oem_ver = (gen_snap << I40E_OEM_SNAP_SHIFT) | release; 7127 hw->nvm.eetrack = I40E_OEM_EETRACK_ID; 7128 } 7129 7130 /** 7131 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 7132 * @pf: board private structure 7133 **/ 7134 static int i40e_reset(struct i40e_pf *pf) 7135 { 7136 struct i40e_hw *hw = &pf->hw; 7137 i40e_status ret; 7138 7139 ret = i40e_pf_reset(hw); 7140 if (ret) { 7141 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 7142 set_bit(__I40E_RESET_FAILED, pf->state); 7143 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 7144 } else { 7145 pf->pfr_count++; 7146 } 7147 return ret; 7148 } 7149 7150 /** 7151 * i40e_rebuild - rebuild using a saved config 7152 * @pf: board private structure 7153 * @reinit: if the Main VSI needs to re-initialized. 7154 * @lock_acquired: indicates whether or not the lock has been acquired 7155 * before this function was called. 7156 **/ 7157 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 7158 { 7159 struct i40e_hw *hw = &pf->hw; 7160 u8 set_fc_aq_fail = 0; 7161 i40e_status ret; 7162 u32 val; 7163 int v; 7164 7165 if (test_bit(__I40E_DOWN, pf->state)) 7166 goto clear_recovery; 7167 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 7168 7169 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 7170 ret = i40e_init_adminq(&pf->hw); 7171 if (ret) { 7172 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 7173 i40e_stat_str(&pf->hw, ret), 7174 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7175 goto clear_recovery; 7176 } 7177 i40e_get_oem_version(&pf->hw); 7178 7179 /* re-verify the eeprom if we just had an EMP reset */ 7180 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 7181 i40e_verify_eeprom(pf); 7182 7183 i40e_clear_pxe_mode(hw); 7184 ret = i40e_get_capabilities(pf); 7185 if (ret) 7186 goto end_core_reset; 7187 7188 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 7189 hw->func_caps.num_rx_qp, 0, 0); 7190 if (ret) { 7191 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 7192 goto end_core_reset; 7193 } 7194 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 7195 if (ret) { 7196 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 7197 goto end_core_reset; 7198 } 7199 7200 #ifdef CONFIG_I40E_DCB 7201 ret = i40e_init_pf_dcb(pf); 7202 if (ret) { 7203 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 7204 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 7205 /* Continue without DCB enabled */ 7206 } 7207 #endif /* CONFIG_I40E_DCB */ 7208 /* do basic switch setup */ 7209 if (!lock_acquired) 7210 rtnl_lock(); 7211 ret = i40e_setup_pf_switch(pf, reinit); 7212 if (ret) 7213 goto end_unlock; 7214 7215 /* The driver only wants link up/down and module qualification 7216 * reports from firmware. Note the negative logic. 7217 */ 7218 ret = i40e_aq_set_phy_int_mask(&pf->hw, 7219 ~(I40E_AQ_EVENT_LINK_UPDOWN | 7220 I40E_AQ_EVENT_MEDIA_NA | 7221 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 7222 if (ret) 7223 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 7224 i40e_stat_str(&pf->hw, ret), 7225 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7226 7227 /* make sure our flow control settings are restored */ 7228 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 7229 if (ret) 7230 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 7231 i40e_stat_str(&pf->hw, ret), 7232 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7233 7234 /* Rebuild the VSIs and VEBs that existed before reset. 7235 * They are still in our local switch element arrays, so only 7236 * need to rebuild the switch model in the HW. 7237 * 7238 * If there were VEBs but the reconstitution failed, we'll try 7239 * try to recover minimal use by getting the basic PF VSI working. 7240 */ 7241 if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) { 7242 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 7243 /* find the one VEB connected to the MAC, and find orphans */ 7244 for (v = 0; v < I40E_MAX_VEB; v++) { 7245 if (!pf->veb[v]) 7246 continue; 7247 7248 if (pf->veb[v]->uplink_seid == pf->mac_seid || 7249 pf->veb[v]->uplink_seid == 0) { 7250 ret = i40e_reconstitute_veb(pf->veb[v]); 7251 7252 if (!ret) 7253 continue; 7254 7255 /* If Main VEB failed, we're in deep doodoo, 7256 * so give up rebuilding the switch and set up 7257 * for minimal rebuild of PF VSI. 7258 * If orphan failed, we'll report the error 7259 * but try to keep going. 7260 */ 7261 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 7262 dev_info(&pf->pdev->dev, 7263 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 7264 ret); 7265 pf->vsi[pf->lan_vsi]->uplink_seid 7266 = pf->mac_seid; 7267 break; 7268 } else if (pf->veb[v]->uplink_seid == 0) { 7269 dev_info(&pf->pdev->dev, 7270 "rebuild of orphan VEB failed: %d\n", 7271 ret); 7272 } 7273 } 7274 } 7275 } 7276 7277 if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) { 7278 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 7279 /* no VEB, so rebuild only the Main VSI */ 7280 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]); 7281 if (ret) { 7282 dev_info(&pf->pdev->dev, 7283 "rebuild of Main VSI failed: %d\n", ret); 7284 goto end_unlock; 7285 } 7286 } 7287 7288 /* Reconfigure hardware for allowing smaller MSS in the case 7289 * of TSO, so that we avoid the MDD being fired and causing 7290 * a reset in the case of small MSS+TSO. 7291 */ 7292 #define I40E_REG_MSS 0x000E64DC 7293 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 7294 #define I40E_64BYTE_MSS 0x400000 7295 val = rd32(hw, I40E_REG_MSS); 7296 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 7297 val &= ~I40E_REG_MSS_MIN_MASK; 7298 val |= I40E_64BYTE_MSS; 7299 wr32(hw, I40E_REG_MSS, val); 7300 } 7301 7302 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 7303 msleep(75); 7304 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 7305 if (ret) 7306 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 7307 i40e_stat_str(&pf->hw, ret), 7308 i40e_aq_str(&pf->hw, 7309 pf->hw.aq.asq_last_status)); 7310 } 7311 /* reinit the misc interrupt */ 7312 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7313 ret = i40e_setup_misc_vector(pf); 7314 7315 /* Add a filter to drop all Flow control frames from any VSI from being 7316 * transmitted. By doing so we stop a malicious VF from sending out 7317 * PAUSE or PFC frames and potentially controlling traffic for other 7318 * PF/VF VSIs. 7319 * The FW can still send Flow control frames if enabled. 7320 */ 7321 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 7322 pf->main_vsi_seid); 7323 7324 /* restart the VSIs that were rebuilt and running before the reset */ 7325 i40e_pf_unquiesce_all_vsi(pf); 7326 7327 /* Release the RTNL lock before we start resetting VFs */ 7328 if (!lock_acquired) 7329 rtnl_unlock(); 7330 7331 i40e_reset_all_vfs(pf, true); 7332 7333 /* tell the firmware that we're starting */ 7334 i40e_send_version(pf); 7335 7336 /* We've already released the lock, so don't do it again */ 7337 goto end_core_reset; 7338 7339 end_unlock: 7340 if (!lock_acquired) 7341 rtnl_unlock(); 7342 end_core_reset: 7343 clear_bit(__I40E_RESET_FAILED, pf->state); 7344 clear_recovery: 7345 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 7346 } 7347 7348 /** 7349 * i40e_reset_and_rebuild - reset and rebuild using a saved config 7350 * @pf: board private structure 7351 * @reinit: if the Main VSI needs to re-initialized. 7352 * @lock_acquired: indicates whether or not the lock has been acquired 7353 * before this function was called. 7354 **/ 7355 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 7356 bool lock_acquired) 7357 { 7358 int ret; 7359 /* Now we wait for GRST to settle out. 7360 * We don't have to delete the VEBs or VSIs from the hw switch 7361 * because the reset will make them disappear. 7362 */ 7363 ret = i40e_reset(pf); 7364 if (!ret) 7365 i40e_rebuild(pf, reinit, lock_acquired); 7366 } 7367 7368 /** 7369 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 7370 * @pf: board private structure 7371 * 7372 * Close up the VFs and other things in prep for a Core Reset, 7373 * then get ready to rebuild the world. 7374 * @lock_acquired: indicates whether or not the lock has been acquired 7375 * before this function was called. 7376 **/ 7377 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 7378 { 7379 i40e_prep_for_reset(pf, lock_acquired); 7380 i40e_reset_and_rebuild(pf, false, lock_acquired); 7381 } 7382 7383 /** 7384 * i40e_handle_mdd_event 7385 * @pf: pointer to the PF structure 7386 * 7387 * Called from the MDD irq handler to identify possibly malicious vfs 7388 **/ 7389 static void i40e_handle_mdd_event(struct i40e_pf *pf) 7390 { 7391 struct i40e_hw *hw = &pf->hw; 7392 bool mdd_detected = false; 7393 bool pf_mdd_detected = false; 7394 struct i40e_vf *vf; 7395 u32 reg; 7396 int i; 7397 7398 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 7399 return; 7400 7401 /* find what triggered the MDD event */ 7402 reg = rd32(hw, I40E_GL_MDET_TX); 7403 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 7404 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 7405 I40E_GL_MDET_TX_PF_NUM_SHIFT; 7406 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 7407 I40E_GL_MDET_TX_VF_NUM_SHIFT; 7408 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 7409 I40E_GL_MDET_TX_EVENT_SHIFT; 7410 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 7411 I40E_GL_MDET_TX_QUEUE_SHIFT) - 7412 pf->hw.func_caps.base_queue; 7413 if (netif_msg_tx_err(pf)) 7414 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 7415 event, queue, pf_num, vf_num); 7416 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 7417 mdd_detected = true; 7418 } 7419 reg = rd32(hw, I40E_GL_MDET_RX); 7420 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 7421 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 7422 I40E_GL_MDET_RX_FUNCTION_SHIFT; 7423 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 7424 I40E_GL_MDET_RX_EVENT_SHIFT; 7425 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 7426 I40E_GL_MDET_RX_QUEUE_SHIFT) - 7427 pf->hw.func_caps.base_queue; 7428 if (netif_msg_rx_err(pf)) 7429 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 7430 event, queue, func); 7431 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 7432 mdd_detected = true; 7433 } 7434 7435 if (mdd_detected) { 7436 reg = rd32(hw, I40E_PF_MDET_TX); 7437 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 7438 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 7439 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 7440 pf_mdd_detected = true; 7441 } 7442 reg = rd32(hw, I40E_PF_MDET_RX); 7443 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 7444 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 7445 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 7446 pf_mdd_detected = true; 7447 } 7448 /* Queue belongs to the PF, initiate a reset */ 7449 if (pf_mdd_detected) { 7450 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 7451 i40e_service_event_schedule(pf); 7452 } 7453 } 7454 7455 /* see if one of the VFs needs its hand slapped */ 7456 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 7457 vf = &(pf->vf[i]); 7458 reg = rd32(hw, I40E_VP_MDET_TX(i)); 7459 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 7460 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 7461 vf->num_mdd_events++; 7462 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 7463 i); 7464 } 7465 7466 reg = rd32(hw, I40E_VP_MDET_RX(i)); 7467 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 7468 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 7469 vf->num_mdd_events++; 7470 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 7471 i); 7472 } 7473 7474 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 7475 dev_info(&pf->pdev->dev, 7476 "Too many MDD events on VF %d, disabled\n", i); 7477 dev_info(&pf->pdev->dev, 7478 "Use PF Control I/F to re-enable the VF\n"); 7479 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 7480 } 7481 } 7482 7483 /* re-enable mdd interrupt cause */ 7484 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 7485 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 7486 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 7487 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 7488 i40e_flush(hw); 7489 } 7490 7491 static const char *i40e_tunnel_name(struct i40e_udp_port_config *port) 7492 { 7493 switch (port->type) { 7494 case UDP_TUNNEL_TYPE_VXLAN: 7495 return "vxlan"; 7496 case UDP_TUNNEL_TYPE_GENEVE: 7497 return "geneve"; 7498 default: 7499 return "unknown"; 7500 } 7501 } 7502 7503 /** 7504 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 7505 * @pf: board private structure 7506 **/ 7507 static void i40e_sync_udp_filters(struct i40e_pf *pf) 7508 { 7509 int i; 7510 7511 /* loop through and set pending bit for all active UDP filters */ 7512 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 7513 if (pf->udp_ports[i].port) 7514 pf->pending_udp_bitmap |= BIT_ULL(i); 7515 } 7516 7517 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 7518 } 7519 7520 /** 7521 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 7522 * @pf: board private structure 7523 **/ 7524 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 7525 { 7526 struct i40e_hw *hw = &pf->hw; 7527 i40e_status ret; 7528 u16 port; 7529 int i; 7530 7531 if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC)) 7532 return; 7533 7534 pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC; 7535 7536 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 7537 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 7538 pf->pending_udp_bitmap &= ~BIT_ULL(i); 7539 port = pf->udp_ports[i].port; 7540 if (port) 7541 ret = i40e_aq_add_udp_tunnel(hw, port, 7542 pf->udp_ports[i].type, 7543 NULL, NULL); 7544 else 7545 ret = i40e_aq_del_udp_tunnel(hw, i, NULL); 7546 7547 if (ret) { 7548 dev_info(&pf->pdev->dev, 7549 "%s %s port %d, index %d failed, err %s aq_err %s\n", 7550 i40e_tunnel_name(&pf->udp_ports[i]), 7551 port ? "add" : "delete", 7552 port, i, 7553 i40e_stat_str(&pf->hw, ret), 7554 i40e_aq_str(&pf->hw, 7555 pf->hw.aq.asq_last_status)); 7556 pf->udp_ports[i].port = 0; 7557 } 7558 } 7559 } 7560 } 7561 7562 /** 7563 * i40e_service_task - Run the driver's async subtasks 7564 * @work: pointer to work_struct containing our data 7565 **/ 7566 static void i40e_service_task(struct work_struct *work) 7567 { 7568 struct i40e_pf *pf = container_of(work, 7569 struct i40e_pf, 7570 service_task); 7571 unsigned long start_time = jiffies; 7572 7573 /* don't bother with service tasks if a reset is in progress */ 7574 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 7575 return; 7576 7577 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 7578 return; 7579 7580 i40e_detect_recover_hung(pf); 7581 i40e_sync_filters_subtask(pf); 7582 i40e_reset_subtask(pf); 7583 i40e_handle_mdd_event(pf); 7584 i40e_vc_process_vflr_event(pf); 7585 i40e_watchdog_subtask(pf); 7586 i40e_fdir_reinit_subtask(pf); 7587 if (pf->flags & I40E_FLAG_CLIENT_RESET) { 7588 /* Client subtask will reopen next time through. */ 7589 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true); 7590 pf->flags &= ~I40E_FLAG_CLIENT_RESET; 7591 } else { 7592 i40e_client_subtask(pf); 7593 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) { 7594 i40e_notify_client_of_l2_param_changes( 7595 pf->vsi[pf->lan_vsi]); 7596 pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE; 7597 } 7598 } 7599 i40e_sync_filters_subtask(pf); 7600 i40e_sync_udp_filters_subtask(pf); 7601 i40e_clean_adminq_subtask(pf); 7602 7603 /* flush memory to make sure state is correct before next watchdog */ 7604 smp_mb__before_atomic(); 7605 clear_bit(__I40E_SERVICE_SCHED, pf->state); 7606 7607 /* If the tasks have taken longer than one timer cycle or there 7608 * is more work to be done, reschedule the service task now 7609 * rather than wait for the timer to tick again. 7610 */ 7611 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 7612 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 7613 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 7614 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 7615 i40e_service_event_schedule(pf); 7616 } 7617 7618 /** 7619 * i40e_service_timer - timer callback 7620 * @data: pointer to PF struct 7621 **/ 7622 static void i40e_service_timer(unsigned long data) 7623 { 7624 struct i40e_pf *pf = (struct i40e_pf *)data; 7625 7626 mod_timer(&pf->service_timer, 7627 round_jiffies(jiffies + pf->service_timer_period)); 7628 i40e_service_event_schedule(pf); 7629 } 7630 7631 /** 7632 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 7633 * @vsi: the VSI being configured 7634 **/ 7635 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 7636 { 7637 struct i40e_pf *pf = vsi->back; 7638 7639 switch (vsi->type) { 7640 case I40E_VSI_MAIN: 7641 vsi->alloc_queue_pairs = pf->num_lan_qps; 7642 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7643 I40E_REQ_DESCRIPTOR_MULTIPLE); 7644 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7645 vsi->num_q_vectors = pf->num_lan_msix; 7646 else 7647 vsi->num_q_vectors = 1; 7648 7649 break; 7650 7651 case I40E_VSI_FDIR: 7652 vsi->alloc_queue_pairs = 1; 7653 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 7654 I40E_REQ_DESCRIPTOR_MULTIPLE); 7655 vsi->num_q_vectors = pf->num_fdsb_msix; 7656 break; 7657 7658 case I40E_VSI_VMDQ2: 7659 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 7660 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7661 I40E_REQ_DESCRIPTOR_MULTIPLE); 7662 vsi->num_q_vectors = pf->num_vmdq_msix; 7663 break; 7664 7665 case I40E_VSI_SRIOV: 7666 vsi->alloc_queue_pairs = pf->num_vf_qps; 7667 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7668 I40E_REQ_DESCRIPTOR_MULTIPLE); 7669 break; 7670 7671 default: 7672 WARN_ON(1); 7673 return -ENODATA; 7674 } 7675 7676 return 0; 7677 } 7678 7679 /** 7680 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 7681 * @type: VSI pointer 7682 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 7683 * 7684 * On error: returns error code (negative) 7685 * On success: returns 0 7686 **/ 7687 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 7688 { 7689 struct i40e_ring **next_rings; 7690 int size; 7691 int ret = 0; 7692 7693 /* allocate memory for both Tx, XDP Tx and Rx ring pointers */ 7694 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 7695 (i40e_enabled_xdp_vsi(vsi) ? 3 : 2); 7696 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 7697 if (!vsi->tx_rings) 7698 return -ENOMEM; 7699 next_rings = vsi->tx_rings + vsi->alloc_queue_pairs; 7700 if (i40e_enabled_xdp_vsi(vsi)) { 7701 vsi->xdp_rings = next_rings; 7702 next_rings += vsi->alloc_queue_pairs; 7703 } 7704 vsi->rx_rings = next_rings; 7705 7706 if (alloc_qvectors) { 7707 /* allocate memory for q_vector pointers */ 7708 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 7709 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 7710 if (!vsi->q_vectors) { 7711 ret = -ENOMEM; 7712 goto err_vectors; 7713 } 7714 } 7715 return ret; 7716 7717 err_vectors: 7718 kfree(vsi->tx_rings); 7719 return ret; 7720 } 7721 7722 /** 7723 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 7724 * @pf: board private structure 7725 * @type: type of VSI 7726 * 7727 * On error: returns error code (negative) 7728 * On success: returns vsi index in PF (positive) 7729 **/ 7730 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 7731 { 7732 int ret = -ENODEV; 7733 struct i40e_vsi *vsi; 7734 int vsi_idx; 7735 int i; 7736 7737 /* Need to protect the allocation of the VSIs at the PF level */ 7738 mutex_lock(&pf->switch_mutex); 7739 7740 /* VSI list may be fragmented if VSI creation/destruction has 7741 * been happening. We can afford to do a quick scan to look 7742 * for any free VSIs in the list. 7743 * 7744 * find next empty vsi slot, looping back around if necessary 7745 */ 7746 i = pf->next_vsi; 7747 while (i < pf->num_alloc_vsi && pf->vsi[i]) 7748 i++; 7749 if (i >= pf->num_alloc_vsi) { 7750 i = 0; 7751 while (i < pf->next_vsi && pf->vsi[i]) 7752 i++; 7753 } 7754 7755 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 7756 vsi_idx = i; /* Found one! */ 7757 } else { 7758 ret = -ENODEV; 7759 goto unlock_pf; /* out of VSI slots! */ 7760 } 7761 pf->next_vsi = ++i; 7762 7763 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 7764 if (!vsi) { 7765 ret = -ENOMEM; 7766 goto unlock_pf; 7767 } 7768 vsi->type = type; 7769 vsi->back = pf; 7770 set_bit(__I40E_VSI_DOWN, vsi->state); 7771 vsi->flags = 0; 7772 vsi->idx = vsi_idx; 7773 vsi->int_rate_limit = 0; 7774 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 7775 pf->rss_table_size : 64; 7776 vsi->netdev_registered = false; 7777 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 7778 hash_init(vsi->mac_filter_hash); 7779 vsi->irqs_ready = false; 7780 7781 ret = i40e_set_num_rings_in_vsi(vsi); 7782 if (ret) 7783 goto err_rings; 7784 7785 ret = i40e_vsi_alloc_arrays(vsi, true); 7786 if (ret) 7787 goto err_rings; 7788 7789 /* Setup default MSIX irq handler for VSI */ 7790 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 7791 7792 /* Initialize VSI lock */ 7793 spin_lock_init(&vsi->mac_filter_hash_lock); 7794 pf->vsi[vsi_idx] = vsi; 7795 ret = vsi_idx; 7796 goto unlock_pf; 7797 7798 err_rings: 7799 pf->next_vsi = i - 1; 7800 kfree(vsi); 7801 unlock_pf: 7802 mutex_unlock(&pf->switch_mutex); 7803 return ret; 7804 } 7805 7806 /** 7807 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 7808 * @type: VSI pointer 7809 * @free_qvectors: a bool to specify if q_vectors need to be freed. 7810 * 7811 * On error: returns error code (negative) 7812 * On success: returns 0 7813 **/ 7814 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 7815 { 7816 /* free the ring and vector containers */ 7817 if (free_qvectors) { 7818 kfree(vsi->q_vectors); 7819 vsi->q_vectors = NULL; 7820 } 7821 kfree(vsi->tx_rings); 7822 vsi->tx_rings = NULL; 7823 vsi->rx_rings = NULL; 7824 vsi->xdp_rings = NULL; 7825 } 7826 7827 /** 7828 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 7829 * and lookup table 7830 * @vsi: Pointer to VSI structure 7831 */ 7832 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 7833 { 7834 if (!vsi) 7835 return; 7836 7837 kfree(vsi->rss_hkey_user); 7838 vsi->rss_hkey_user = NULL; 7839 7840 kfree(vsi->rss_lut_user); 7841 vsi->rss_lut_user = NULL; 7842 } 7843 7844 /** 7845 * i40e_vsi_clear - Deallocate the VSI provided 7846 * @vsi: the VSI being un-configured 7847 **/ 7848 static int i40e_vsi_clear(struct i40e_vsi *vsi) 7849 { 7850 struct i40e_pf *pf; 7851 7852 if (!vsi) 7853 return 0; 7854 7855 if (!vsi->back) 7856 goto free_vsi; 7857 pf = vsi->back; 7858 7859 mutex_lock(&pf->switch_mutex); 7860 if (!pf->vsi[vsi->idx]) { 7861 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n", 7862 vsi->idx, vsi->idx, vsi, vsi->type); 7863 goto unlock_vsi; 7864 } 7865 7866 if (pf->vsi[vsi->idx] != vsi) { 7867 dev_err(&pf->pdev->dev, 7868 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n", 7869 pf->vsi[vsi->idx]->idx, 7870 pf->vsi[vsi->idx], 7871 pf->vsi[vsi->idx]->type, 7872 vsi->idx, vsi, vsi->type); 7873 goto unlock_vsi; 7874 } 7875 7876 /* updates the PF for this cleared vsi */ 7877 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 7878 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 7879 7880 i40e_vsi_free_arrays(vsi, true); 7881 i40e_clear_rss_config_user(vsi); 7882 7883 pf->vsi[vsi->idx] = NULL; 7884 if (vsi->idx < pf->next_vsi) 7885 pf->next_vsi = vsi->idx; 7886 7887 unlock_vsi: 7888 mutex_unlock(&pf->switch_mutex); 7889 free_vsi: 7890 kfree(vsi); 7891 7892 return 0; 7893 } 7894 7895 /** 7896 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 7897 * @vsi: the VSI being cleaned 7898 **/ 7899 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 7900 { 7901 int i; 7902 7903 if (vsi->tx_rings && vsi->tx_rings[0]) { 7904 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7905 kfree_rcu(vsi->tx_rings[i], rcu); 7906 vsi->tx_rings[i] = NULL; 7907 vsi->rx_rings[i] = NULL; 7908 if (vsi->xdp_rings) 7909 vsi->xdp_rings[i] = NULL; 7910 } 7911 } 7912 } 7913 7914 /** 7915 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 7916 * @vsi: the VSI being configured 7917 **/ 7918 static int i40e_alloc_rings(struct i40e_vsi *vsi) 7919 { 7920 int i, qpv = i40e_enabled_xdp_vsi(vsi) ? 3 : 2; 7921 struct i40e_pf *pf = vsi->back; 7922 struct i40e_ring *ring; 7923 7924 /* Set basic values in the rings to be used later during open() */ 7925 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7926 /* allocate space for both Tx and Rx in one shot */ 7927 ring = kcalloc(qpv, sizeof(struct i40e_ring), GFP_KERNEL); 7928 if (!ring) 7929 goto err_out; 7930 7931 ring->queue_index = i; 7932 ring->reg_idx = vsi->base_queue + i; 7933 ring->ring_active = false; 7934 ring->vsi = vsi; 7935 ring->netdev = vsi->netdev; 7936 ring->dev = &pf->pdev->dev; 7937 ring->count = vsi->num_desc; 7938 ring->size = 0; 7939 ring->dcb_tc = 0; 7940 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 7941 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 7942 ring->tx_itr_setting = pf->tx_itr_default; 7943 vsi->tx_rings[i] = ring++; 7944 7945 if (!i40e_enabled_xdp_vsi(vsi)) 7946 goto setup_rx; 7947 7948 ring->queue_index = vsi->alloc_queue_pairs + i; 7949 ring->reg_idx = vsi->base_queue + ring->queue_index; 7950 ring->ring_active = false; 7951 ring->vsi = vsi; 7952 ring->netdev = NULL; 7953 ring->dev = &pf->pdev->dev; 7954 ring->count = vsi->num_desc; 7955 ring->size = 0; 7956 ring->dcb_tc = 0; 7957 if (vsi->back->hw_features & I40E_HW_WB_ON_ITR_CAPABLE) 7958 ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 7959 set_ring_xdp(ring); 7960 ring->tx_itr_setting = pf->tx_itr_default; 7961 vsi->xdp_rings[i] = ring++; 7962 7963 setup_rx: 7964 ring->queue_index = i; 7965 ring->reg_idx = vsi->base_queue + i; 7966 ring->ring_active = false; 7967 ring->vsi = vsi; 7968 ring->netdev = vsi->netdev; 7969 ring->dev = &pf->pdev->dev; 7970 ring->count = vsi->num_desc; 7971 ring->size = 0; 7972 ring->dcb_tc = 0; 7973 ring->rx_itr_setting = pf->rx_itr_default; 7974 vsi->rx_rings[i] = ring; 7975 } 7976 7977 return 0; 7978 7979 err_out: 7980 i40e_vsi_clear_rings(vsi); 7981 return -ENOMEM; 7982 } 7983 7984 /** 7985 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 7986 * @pf: board private structure 7987 * @vectors: the number of MSI-X vectors to request 7988 * 7989 * Returns the number of vectors reserved, or error 7990 **/ 7991 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 7992 { 7993 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 7994 I40E_MIN_MSIX, vectors); 7995 if (vectors < 0) { 7996 dev_info(&pf->pdev->dev, 7997 "MSI-X vector reservation failed: %d\n", vectors); 7998 vectors = 0; 7999 } 8000 8001 return vectors; 8002 } 8003 8004 /** 8005 * i40e_init_msix - Setup the MSIX capability 8006 * @pf: board private structure 8007 * 8008 * Work with the OS to set up the MSIX vectors needed. 8009 * 8010 * Returns the number of vectors reserved or negative on failure 8011 **/ 8012 static int i40e_init_msix(struct i40e_pf *pf) 8013 { 8014 struct i40e_hw *hw = &pf->hw; 8015 int cpus, extra_vectors; 8016 int vectors_left; 8017 int v_budget, i; 8018 int v_actual; 8019 int iwarp_requested = 0; 8020 8021 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 8022 return -ENODEV; 8023 8024 /* The number of vectors we'll request will be comprised of: 8025 * - Add 1 for "other" cause for Admin Queue events, etc. 8026 * - The number of LAN queue pairs 8027 * - Queues being used for RSS. 8028 * We don't need as many as max_rss_size vectors. 8029 * use rss_size instead in the calculation since that 8030 * is governed by number of cpus in the system. 8031 * - assumes symmetric Tx/Rx pairing 8032 * - The number of VMDq pairs 8033 * - The CPU count within the NUMA node if iWARP is enabled 8034 * Once we count this up, try the request. 8035 * 8036 * If we can't get what we want, we'll simplify to nearly nothing 8037 * and try again. If that still fails, we punt. 8038 */ 8039 vectors_left = hw->func_caps.num_msix_vectors; 8040 v_budget = 0; 8041 8042 /* reserve one vector for miscellaneous handler */ 8043 if (vectors_left) { 8044 v_budget++; 8045 vectors_left--; 8046 } 8047 8048 /* reserve some vectors for the main PF traffic queues. Initially we 8049 * only reserve at most 50% of the available vectors, in the case that 8050 * the number of online CPUs is large. This ensures that we can enable 8051 * extra features as well. Once we've enabled the other features, we 8052 * will use any remaining vectors to reach as close as we can to the 8053 * number of online CPUs. 8054 */ 8055 cpus = num_online_cpus(); 8056 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 8057 vectors_left -= pf->num_lan_msix; 8058 8059 /* reserve one vector for sideband flow director */ 8060 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 8061 if (vectors_left) { 8062 pf->num_fdsb_msix = 1; 8063 v_budget++; 8064 vectors_left--; 8065 } else { 8066 pf->num_fdsb_msix = 0; 8067 } 8068 } 8069 8070 /* can we reserve enough for iWARP? */ 8071 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 8072 iwarp_requested = pf->num_iwarp_msix; 8073 8074 if (!vectors_left) 8075 pf->num_iwarp_msix = 0; 8076 else if (vectors_left < pf->num_iwarp_msix) 8077 pf->num_iwarp_msix = 1; 8078 v_budget += pf->num_iwarp_msix; 8079 vectors_left -= pf->num_iwarp_msix; 8080 } 8081 8082 /* any vectors left over go for VMDq support */ 8083 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 8084 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps; 8085 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted); 8086 8087 if (!vectors_left) { 8088 pf->num_vmdq_msix = 0; 8089 pf->num_vmdq_qps = 0; 8090 } else { 8091 /* if we're short on vectors for what's desired, we limit 8092 * the queues per vmdq. If this is still more than are 8093 * available, the user will need to change the number of 8094 * queues/vectors used by the PF later with the ethtool 8095 * channels command 8096 */ 8097 if (vmdq_vecs < vmdq_vecs_wanted) 8098 pf->num_vmdq_qps = 1; 8099 pf->num_vmdq_msix = pf->num_vmdq_qps; 8100 8101 v_budget += vmdq_vecs; 8102 vectors_left -= vmdq_vecs; 8103 } 8104 } 8105 8106 /* On systems with a large number of SMP cores, we previously limited 8107 * the number of vectors for num_lan_msix to be at most 50% of the 8108 * available vectors, to allow for other features. Now, we add back 8109 * the remaining vectors. However, we ensure that the total 8110 * num_lan_msix will not exceed num_online_cpus(). To do this, we 8111 * calculate the number of vectors we can add without going over the 8112 * cap of CPUs. For systems with a small number of CPUs this will be 8113 * zero. 8114 */ 8115 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 8116 pf->num_lan_msix += extra_vectors; 8117 vectors_left -= extra_vectors; 8118 8119 WARN(vectors_left < 0, 8120 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 8121 8122 v_budget += pf->num_lan_msix; 8123 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 8124 GFP_KERNEL); 8125 if (!pf->msix_entries) 8126 return -ENOMEM; 8127 8128 for (i = 0; i < v_budget; i++) 8129 pf->msix_entries[i].entry = i; 8130 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 8131 8132 if (v_actual < I40E_MIN_MSIX) { 8133 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 8134 kfree(pf->msix_entries); 8135 pf->msix_entries = NULL; 8136 pci_disable_msix(pf->pdev); 8137 return -ENODEV; 8138 8139 } else if (v_actual == I40E_MIN_MSIX) { 8140 /* Adjust for minimal MSIX use */ 8141 pf->num_vmdq_vsis = 0; 8142 pf->num_vmdq_qps = 0; 8143 pf->num_lan_qps = 1; 8144 pf->num_lan_msix = 1; 8145 8146 } else if (!vectors_left) { 8147 /* If we have limited resources, we will start with no vectors 8148 * for the special features and then allocate vectors to some 8149 * of these features based on the policy and at the end disable 8150 * the features that did not get any vectors. 8151 */ 8152 int vec; 8153 8154 dev_info(&pf->pdev->dev, 8155 "MSI-X vector limit reached, attempting to redistribute vectors\n"); 8156 /* reserve the misc vector */ 8157 vec = v_actual - 1; 8158 8159 /* Scale vector usage down */ 8160 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 8161 pf->num_vmdq_vsis = 1; 8162 pf->num_vmdq_qps = 1; 8163 8164 /* partition out the remaining vectors */ 8165 switch (vec) { 8166 case 2: 8167 pf->num_lan_msix = 1; 8168 break; 8169 case 3: 8170 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 8171 pf->num_lan_msix = 1; 8172 pf->num_iwarp_msix = 1; 8173 } else { 8174 pf->num_lan_msix = 2; 8175 } 8176 break; 8177 default: 8178 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 8179 pf->num_iwarp_msix = min_t(int, (vec / 3), 8180 iwarp_requested); 8181 pf->num_vmdq_vsis = min_t(int, (vec / 3), 8182 I40E_DEFAULT_NUM_VMDQ_VSI); 8183 } else { 8184 pf->num_vmdq_vsis = min_t(int, (vec / 2), 8185 I40E_DEFAULT_NUM_VMDQ_VSI); 8186 } 8187 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 8188 pf->num_fdsb_msix = 1; 8189 vec--; 8190 } 8191 pf->num_lan_msix = min_t(int, 8192 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 8193 pf->num_lan_msix); 8194 pf->num_lan_qps = pf->num_lan_msix; 8195 break; 8196 } 8197 } 8198 8199 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 8200 (pf->num_fdsb_msix == 0)) { 8201 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 8202 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8203 } 8204 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 8205 (pf->num_vmdq_msix == 0)) { 8206 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 8207 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 8208 } 8209 8210 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 8211 (pf->num_iwarp_msix == 0)) { 8212 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 8213 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 8214 } 8215 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 8216 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 8217 pf->num_lan_msix, 8218 pf->num_vmdq_msix * pf->num_vmdq_vsis, 8219 pf->num_fdsb_msix, 8220 pf->num_iwarp_msix); 8221 8222 return v_actual; 8223 } 8224 8225 /** 8226 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 8227 * @vsi: the VSI being configured 8228 * @v_idx: index of the vector in the vsi struct 8229 * @cpu: cpu to be used on affinity_mask 8230 * 8231 * We allocate one q_vector. If allocation fails we return -ENOMEM. 8232 **/ 8233 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 8234 { 8235 struct i40e_q_vector *q_vector; 8236 8237 /* allocate q_vector */ 8238 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 8239 if (!q_vector) 8240 return -ENOMEM; 8241 8242 q_vector->vsi = vsi; 8243 q_vector->v_idx = v_idx; 8244 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask); 8245 8246 if (vsi->netdev) 8247 netif_napi_add(vsi->netdev, &q_vector->napi, 8248 i40e_napi_poll, NAPI_POLL_WEIGHT); 8249 8250 q_vector->rx.latency_range = I40E_LOW_LATENCY; 8251 q_vector->tx.latency_range = I40E_LOW_LATENCY; 8252 8253 /* tie q_vector and vsi together */ 8254 vsi->q_vectors[v_idx] = q_vector; 8255 8256 return 0; 8257 } 8258 8259 /** 8260 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 8261 * @vsi: the VSI being configured 8262 * 8263 * We allocate one q_vector per queue interrupt. If allocation fails we 8264 * return -ENOMEM. 8265 **/ 8266 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 8267 { 8268 struct i40e_pf *pf = vsi->back; 8269 int err, v_idx, num_q_vectors, current_cpu; 8270 8271 /* if not MSIX, give the one vector only to the LAN VSI */ 8272 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 8273 num_q_vectors = vsi->num_q_vectors; 8274 else if (vsi == pf->vsi[pf->lan_vsi]) 8275 num_q_vectors = 1; 8276 else 8277 return -EINVAL; 8278 8279 current_cpu = cpumask_first(cpu_online_mask); 8280 8281 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 8282 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 8283 if (err) 8284 goto err_out; 8285 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 8286 if (unlikely(current_cpu >= nr_cpu_ids)) 8287 current_cpu = cpumask_first(cpu_online_mask); 8288 } 8289 8290 return 0; 8291 8292 err_out: 8293 while (v_idx--) 8294 i40e_free_q_vector(vsi, v_idx); 8295 8296 return err; 8297 } 8298 8299 /** 8300 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 8301 * @pf: board private structure to initialize 8302 **/ 8303 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 8304 { 8305 int vectors = 0; 8306 ssize_t size; 8307 8308 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 8309 vectors = i40e_init_msix(pf); 8310 if (vectors < 0) { 8311 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 8312 I40E_FLAG_IWARP_ENABLED | 8313 I40E_FLAG_RSS_ENABLED | 8314 I40E_FLAG_DCB_CAPABLE | 8315 I40E_FLAG_DCB_ENABLED | 8316 I40E_FLAG_SRIOV_ENABLED | 8317 I40E_FLAG_FD_SB_ENABLED | 8318 I40E_FLAG_FD_ATR_ENABLED | 8319 I40E_FLAG_VMDQ_ENABLED); 8320 8321 /* rework the queue expectations without MSIX */ 8322 i40e_determine_queue_usage(pf); 8323 } 8324 } 8325 8326 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 8327 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 8328 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 8329 vectors = pci_enable_msi(pf->pdev); 8330 if (vectors < 0) { 8331 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 8332 vectors); 8333 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 8334 } 8335 vectors = 1; /* one MSI or Legacy vector */ 8336 } 8337 8338 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 8339 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 8340 8341 /* set up vector assignment tracking */ 8342 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 8343 pf->irq_pile = kzalloc(size, GFP_KERNEL); 8344 if (!pf->irq_pile) { 8345 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n"); 8346 return -ENOMEM; 8347 } 8348 pf->irq_pile->num_entries = vectors; 8349 pf->irq_pile->search_hint = 0; 8350 8351 /* track first vector for misc interrupts, ignore return */ 8352 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 8353 8354 return 0; 8355 } 8356 8357 #ifdef CONFIG_PM 8358 /** 8359 * i40e_restore_interrupt_scheme - Restore the interrupt scheme 8360 * @pf: private board data structure 8361 * 8362 * Restore the interrupt scheme that was cleared when we suspended the 8363 * device. This should be called during resume to re-allocate the q_vectors 8364 * and reacquire IRQs. 8365 */ 8366 static int i40e_restore_interrupt_scheme(struct i40e_pf *pf) 8367 { 8368 int err, i; 8369 8370 /* We cleared the MSI and MSI-X flags when disabling the old interrupt 8371 * scheme. We need to re-enabled them here in order to attempt to 8372 * re-acquire the MSI or MSI-X vectors 8373 */ 8374 pf->flags |= (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 8375 8376 err = i40e_init_interrupt_scheme(pf); 8377 if (err) 8378 return err; 8379 8380 /* Now that we've re-acquired IRQs, we need to remap the vectors and 8381 * rings together again. 8382 */ 8383 for (i = 0; i < pf->num_alloc_vsi; i++) { 8384 if (pf->vsi[i]) { 8385 err = i40e_vsi_alloc_q_vectors(pf->vsi[i]); 8386 if (err) 8387 goto err_unwind; 8388 i40e_vsi_map_rings_to_vectors(pf->vsi[i]); 8389 } 8390 } 8391 8392 err = i40e_setup_misc_vector(pf); 8393 if (err) 8394 goto err_unwind; 8395 8396 return 0; 8397 8398 err_unwind: 8399 while (i--) { 8400 if (pf->vsi[i]) 8401 i40e_vsi_free_q_vectors(pf->vsi[i]); 8402 } 8403 8404 return err; 8405 } 8406 #endif /* CONFIG_PM */ 8407 8408 /** 8409 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 8410 * @pf: board private structure 8411 * 8412 * This sets up the handler for MSIX 0, which is used to manage the 8413 * non-queue interrupts, e.g. AdminQ and errors. This is not used 8414 * when in MSI or Legacy interrupt mode. 8415 **/ 8416 static int i40e_setup_misc_vector(struct i40e_pf *pf) 8417 { 8418 struct i40e_hw *hw = &pf->hw; 8419 int err = 0; 8420 8421 /* Only request the IRQ once, the first time through. */ 8422 if (!test_and_set_bit(__I40E_MISC_IRQ_REQUESTED, pf->state)) { 8423 err = request_irq(pf->msix_entries[0].vector, 8424 i40e_intr, 0, pf->int_name, pf); 8425 if (err) { 8426 clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); 8427 dev_info(&pf->pdev->dev, 8428 "request_irq for %s failed: %d\n", 8429 pf->int_name, err); 8430 return -EFAULT; 8431 } 8432 } 8433 8434 i40e_enable_misc_int_causes(pf); 8435 8436 /* associate no queues to the misc vector */ 8437 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 8438 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 8439 8440 i40e_flush(hw); 8441 8442 i40e_irq_dynamic_enable_icr0(pf, true); 8443 8444 return err; 8445 } 8446 8447 /** 8448 * i40e_config_rss_aq - Prepare for RSS using AQ commands 8449 * @vsi: vsi structure 8450 * @seed: RSS hash seed 8451 **/ 8452 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8453 u8 *lut, u16 lut_size) 8454 { 8455 struct i40e_pf *pf = vsi->back; 8456 struct i40e_hw *hw = &pf->hw; 8457 int ret = 0; 8458 8459 if (seed) { 8460 struct i40e_aqc_get_set_rss_key_data *seed_dw = 8461 (struct i40e_aqc_get_set_rss_key_data *)seed; 8462 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 8463 if (ret) { 8464 dev_info(&pf->pdev->dev, 8465 "Cannot set RSS key, err %s aq_err %s\n", 8466 i40e_stat_str(hw, ret), 8467 i40e_aq_str(hw, hw->aq.asq_last_status)); 8468 return ret; 8469 } 8470 } 8471 if (lut) { 8472 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8473 8474 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8475 if (ret) { 8476 dev_info(&pf->pdev->dev, 8477 "Cannot set RSS lut, err %s aq_err %s\n", 8478 i40e_stat_str(hw, ret), 8479 i40e_aq_str(hw, hw->aq.asq_last_status)); 8480 return ret; 8481 } 8482 } 8483 return ret; 8484 } 8485 8486 /** 8487 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 8488 * @vsi: Pointer to vsi structure 8489 * @seed: Buffter to store the hash keys 8490 * @lut: Buffer to store the lookup table entries 8491 * @lut_size: Size of buffer to store the lookup table entries 8492 * 8493 * Return 0 on success, negative on failure 8494 */ 8495 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8496 u8 *lut, u16 lut_size) 8497 { 8498 struct i40e_pf *pf = vsi->back; 8499 struct i40e_hw *hw = &pf->hw; 8500 int ret = 0; 8501 8502 if (seed) { 8503 ret = i40e_aq_get_rss_key(hw, vsi->id, 8504 (struct i40e_aqc_get_set_rss_key_data *)seed); 8505 if (ret) { 8506 dev_info(&pf->pdev->dev, 8507 "Cannot get RSS key, err %s aq_err %s\n", 8508 i40e_stat_str(&pf->hw, ret), 8509 i40e_aq_str(&pf->hw, 8510 pf->hw.aq.asq_last_status)); 8511 return ret; 8512 } 8513 } 8514 8515 if (lut) { 8516 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8517 8518 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8519 if (ret) { 8520 dev_info(&pf->pdev->dev, 8521 "Cannot get RSS lut, err %s aq_err %s\n", 8522 i40e_stat_str(&pf->hw, ret), 8523 i40e_aq_str(&pf->hw, 8524 pf->hw.aq.asq_last_status)); 8525 return ret; 8526 } 8527 } 8528 8529 return ret; 8530 } 8531 8532 /** 8533 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 8534 * @vsi: VSI structure 8535 **/ 8536 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 8537 { 8538 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8539 struct i40e_pf *pf = vsi->back; 8540 u8 *lut; 8541 int ret; 8542 8543 if (!(pf->hw_features & I40E_HW_RSS_AQ_CAPABLE)) 8544 return 0; 8545 8546 if (!vsi->rss_size) 8547 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8548 vsi->num_queue_pairs); 8549 if (!vsi->rss_size) 8550 return -EINVAL; 8551 8552 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8553 if (!lut) 8554 return -ENOMEM; 8555 /* Use the user configured hash keys and lookup table if there is one, 8556 * otherwise use default 8557 */ 8558 if (vsi->rss_lut_user) 8559 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8560 else 8561 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8562 if (vsi->rss_hkey_user) 8563 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8564 else 8565 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8566 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 8567 kfree(lut); 8568 8569 return ret; 8570 } 8571 8572 /** 8573 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 8574 * @vsi: Pointer to vsi structure 8575 * @seed: RSS hash seed 8576 * @lut: Lookup table 8577 * @lut_size: Lookup table size 8578 * 8579 * Returns 0 on success, negative on failure 8580 **/ 8581 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 8582 const u8 *lut, u16 lut_size) 8583 { 8584 struct i40e_pf *pf = vsi->back; 8585 struct i40e_hw *hw = &pf->hw; 8586 u16 vf_id = vsi->vf_id; 8587 u8 i; 8588 8589 /* Fill out hash function seed */ 8590 if (seed) { 8591 u32 *seed_dw = (u32 *)seed; 8592 8593 if (vsi->type == I40E_VSI_MAIN) { 8594 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8595 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 8596 } else if (vsi->type == I40E_VSI_SRIOV) { 8597 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 8598 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 8599 } else { 8600 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 8601 } 8602 } 8603 8604 if (lut) { 8605 u32 *lut_dw = (u32 *)lut; 8606 8607 if (vsi->type == I40E_VSI_MAIN) { 8608 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8609 return -EINVAL; 8610 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8611 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 8612 } else if (vsi->type == I40E_VSI_SRIOV) { 8613 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 8614 return -EINVAL; 8615 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8616 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 8617 } else { 8618 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8619 } 8620 } 8621 i40e_flush(hw); 8622 8623 return 0; 8624 } 8625 8626 /** 8627 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 8628 * @vsi: Pointer to VSI structure 8629 * @seed: Buffer to store the keys 8630 * @lut: Buffer to store the lookup table entries 8631 * @lut_size: Size of buffer to store the lookup table entries 8632 * 8633 * Returns 0 on success, negative on failure 8634 */ 8635 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 8636 u8 *lut, u16 lut_size) 8637 { 8638 struct i40e_pf *pf = vsi->back; 8639 struct i40e_hw *hw = &pf->hw; 8640 u16 i; 8641 8642 if (seed) { 8643 u32 *seed_dw = (u32 *)seed; 8644 8645 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8646 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 8647 } 8648 if (lut) { 8649 u32 *lut_dw = (u32 *)lut; 8650 8651 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8652 return -EINVAL; 8653 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8654 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 8655 } 8656 8657 return 0; 8658 } 8659 8660 /** 8661 * i40e_config_rss - Configure RSS keys and lut 8662 * @vsi: Pointer to VSI structure 8663 * @seed: RSS hash seed 8664 * @lut: Lookup table 8665 * @lut_size: Lookup table size 8666 * 8667 * Returns 0 on success, negative on failure 8668 */ 8669 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8670 { 8671 struct i40e_pf *pf = vsi->back; 8672 8673 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 8674 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 8675 else 8676 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 8677 } 8678 8679 /** 8680 * i40e_get_rss - Get RSS keys and lut 8681 * @vsi: Pointer to VSI structure 8682 * @seed: Buffer to store the keys 8683 * @lut: Buffer to store the lookup table entries 8684 * lut_size: Size of buffer to store the lookup table entries 8685 * 8686 * Returns 0 on success, negative on failure 8687 */ 8688 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8689 { 8690 struct i40e_pf *pf = vsi->back; 8691 8692 if (pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) 8693 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 8694 else 8695 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 8696 } 8697 8698 /** 8699 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 8700 * @pf: Pointer to board private structure 8701 * @lut: Lookup table 8702 * @rss_table_size: Lookup table size 8703 * @rss_size: Range of queue number for hashing 8704 */ 8705 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 8706 u16 rss_table_size, u16 rss_size) 8707 { 8708 u16 i; 8709 8710 for (i = 0; i < rss_table_size; i++) 8711 lut[i] = i % rss_size; 8712 } 8713 8714 /** 8715 * i40e_pf_config_rss - Prepare for RSS if used 8716 * @pf: board private structure 8717 **/ 8718 static int i40e_pf_config_rss(struct i40e_pf *pf) 8719 { 8720 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8721 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8722 u8 *lut; 8723 struct i40e_hw *hw = &pf->hw; 8724 u32 reg_val; 8725 u64 hena; 8726 int ret; 8727 8728 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 8729 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 8730 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 8731 hena |= i40e_pf_get_default_rss_hena(pf); 8732 8733 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 8734 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 8735 8736 /* Determine the RSS table size based on the hardware capabilities */ 8737 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 8738 reg_val = (pf->rss_table_size == 512) ? 8739 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 8740 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 8741 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 8742 8743 /* Determine the RSS size of the VSI */ 8744 if (!vsi->rss_size) { 8745 u16 qcount; 8746 8747 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 8748 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 8749 } 8750 if (!vsi->rss_size) 8751 return -EINVAL; 8752 8753 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8754 if (!lut) 8755 return -ENOMEM; 8756 8757 /* Use user configured lut if there is one, otherwise use default */ 8758 if (vsi->rss_lut_user) 8759 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8760 else 8761 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8762 8763 /* Use user configured hash key if there is one, otherwise 8764 * use default. 8765 */ 8766 if (vsi->rss_hkey_user) 8767 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8768 else 8769 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8770 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 8771 kfree(lut); 8772 8773 return ret; 8774 } 8775 8776 /** 8777 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 8778 * @pf: board private structure 8779 * @queue_count: the requested queue count for rss. 8780 * 8781 * returns 0 if rss is not enabled, if enabled returns the final rss queue 8782 * count which may be different from the requested queue count. 8783 * Note: expects to be called while under rtnl_lock() 8784 **/ 8785 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 8786 { 8787 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8788 int new_rss_size; 8789 8790 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 8791 return 0; 8792 8793 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 8794 8795 if (queue_count != vsi->num_queue_pairs) { 8796 u16 qcount; 8797 8798 vsi->req_queue_pairs = queue_count; 8799 i40e_prep_for_reset(pf, true); 8800 8801 pf->alloc_rss_size = new_rss_size; 8802 8803 i40e_reset_and_rebuild(pf, true, true); 8804 8805 /* Discard the user configured hash keys and lut, if less 8806 * queues are enabled. 8807 */ 8808 if (queue_count < vsi->rss_size) { 8809 i40e_clear_rss_config_user(vsi); 8810 dev_dbg(&pf->pdev->dev, 8811 "discard user configured hash keys and lut\n"); 8812 } 8813 8814 /* Reset vsi->rss_size, as number of enabled queues changed */ 8815 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 8816 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 8817 8818 i40e_pf_config_rss(pf); 8819 } 8820 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 8821 vsi->req_queue_pairs, pf->rss_size_max); 8822 return pf->alloc_rss_size; 8823 } 8824 8825 /** 8826 * i40e_get_partition_bw_setting - Retrieve BW settings for this PF partition 8827 * @pf: board private structure 8828 **/ 8829 i40e_status i40e_get_partition_bw_setting(struct i40e_pf *pf) 8830 { 8831 i40e_status status; 8832 bool min_valid, max_valid; 8833 u32 max_bw, min_bw; 8834 8835 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 8836 &min_valid, &max_valid); 8837 8838 if (!status) { 8839 if (min_valid) 8840 pf->min_bw = min_bw; 8841 if (max_valid) 8842 pf->max_bw = max_bw; 8843 } 8844 8845 return status; 8846 } 8847 8848 /** 8849 * i40e_set_partition_bw_setting - Set BW settings for this PF partition 8850 * @pf: board private structure 8851 **/ 8852 i40e_status i40e_set_partition_bw_setting(struct i40e_pf *pf) 8853 { 8854 struct i40e_aqc_configure_partition_bw_data bw_data; 8855 i40e_status status; 8856 8857 /* Set the valid bit for this PF */ 8858 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 8859 bw_data.max_bw[pf->hw.pf_id] = pf->max_bw & I40E_ALT_BW_VALUE_MASK; 8860 bw_data.min_bw[pf->hw.pf_id] = pf->min_bw & I40E_ALT_BW_VALUE_MASK; 8861 8862 /* Set the new bandwidths */ 8863 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 8864 8865 return status; 8866 } 8867 8868 /** 8869 * i40e_commit_partition_bw_setting - Commit BW settings for this PF partition 8870 * @pf: board private structure 8871 **/ 8872 i40e_status i40e_commit_partition_bw_setting(struct i40e_pf *pf) 8873 { 8874 /* Commit temporary BW setting to permanent NVM image */ 8875 enum i40e_admin_queue_err last_aq_status; 8876 i40e_status ret; 8877 u16 nvm_word; 8878 8879 if (pf->hw.partition_id != 1) { 8880 dev_info(&pf->pdev->dev, 8881 "Commit BW only works on partition 1! This is partition %d", 8882 pf->hw.partition_id); 8883 ret = I40E_NOT_SUPPORTED; 8884 goto bw_commit_out; 8885 } 8886 8887 /* Acquire NVM for read access */ 8888 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 8889 last_aq_status = pf->hw.aq.asq_last_status; 8890 if (ret) { 8891 dev_info(&pf->pdev->dev, 8892 "Cannot acquire NVM for read access, err %s aq_err %s\n", 8893 i40e_stat_str(&pf->hw, ret), 8894 i40e_aq_str(&pf->hw, last_aq_status)); 8895 goto bw_commit_out; 8896 } 8897 8898 /* Read word 0x10 of NVM - SW compatibility word 1 */ 8899 ret = i40e_aq_read_nvm(&pf->hw, 8900 I40E_SR_NVM_CONTROL_WORD, 8901 0x10, sizeof(nvm_word), &nvm_word, 8902 false, NULL); 8903 /* Save off last admin queue command status before releasing 8904 * the NVM 8905 */ 8906 last_aq_status = pf->hw.aq.asq_last_status; 8907 i40e_release_nvm(&pf->hw); 8908 if (ret) { 8909 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 8910 i40e_stat_str(&pf->hw, ret), 8911 i40e_aq_str(&pf->hw, last_aq_status)); 8912 goto bw_commit_out; 8913 } 8914 8915 /* Wait a bit for NVM release to complete */ 8916 msleep(50); 8917 8918 /* Acquire NVM for write access */ 8919 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 8920 last_aq_status = pf->hw.aq.asq_last_status; 8921 if (ret) { 8922 dev_info(&pf->pdev->dev, 8923 "Cannot acquire NVM for write access, err %s aq_err %s\n", 8924 i40e_stat_str(&pf->hw, ret), 8925 i40e_aq_str(&pf->hw, last_aq_status)); 8926 goto bw_commit_out; 8927 } 8928 /* Write it back out unchanged to initiate update NVM, 8929 * which will force a write of the shadow (alt) RAM to 8930 * the NVM - thus storing the bandwidth values permanently. 8931 */ 8932 ret = i40e_aq_update_nvm(&pf->hw, 8933 I40E_SR_NVM_CONTROL_WORD, 8934 0x10, sizeof(nvm_word), 8935 &nvm_word, true, NULL); 8936 /* Save off last admin queue command status before releasing 8937 * the NVM 8938 */ 8939 last_aq_status = pf->hw.aq.asq_last_status; 8940 i40e_release_nvm(&pf->hw); 8941 if (ret) 8942 dev_info(&pf->pdev->dev, 8943 "BW settings NOT SAVED, err %s aq_err %s\n", 8944 i40e_stat_str(&pf->hw, ret), 8945 i40e_aq_str(&pf->hw, last_aq_status)); 8946 bw_commit_out: 8947 8948 return ret; 8949 } 8950 8951 /** 8952 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 8953 * @pf: board private structure to initialize 8954 * 8955 * i40e_sw_init initializes the Adapter private data structure. 8956 * Fields are initialized based on PCI device information and 8957 * OS network device settings (MTU size). 8958 **/ 8959 static int i40e_sw_init(struct i40e_pf *pf) 8960 { 8961 int err = 0; 8962 int size; 8963 8964 /* Set default capability flags */ 8965 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 8966 I40E_FLAG_MSI_ENABLED | 8967 I40E_FLAG_MSIX_ENABLED; 8968 8969 /* Set default ITR */ 8970 pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF; 8971 pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF; 8972 8973 /* Depending on PF configurations, it is possible that the RSS 8974 * maximum might end up larger than the available queues 8975 */ 8976 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 8977 pf->alloc_rss_size = 1; 8978 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 8979 pf->rss_size_max = min_t(int, pf->rss_size_max, 8980 pf->hw.func_caps.num_tx_qp); 8981 if (pf->hw.func_caps.rss) { 8982 pf->flags |= I40E_FLAG_RSS_ENABLED; 8983 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 8984 num_online_cpus()); 8985 } 8986 8987 /* MFP mode enabled */ 8988 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 8989 pf->flags |= I40E_FLAG_MFP_ENABLED; 8990 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 8991 if (i40e_get_partition_bw_setting(pf)) { 8992 dev_warn(&pf->pdev->dev, 8993 "Could not get partition bw settings\n"); 8994 } else { 8995 dev_info(&pf->pdev->dev, 8996 "Partition BW Min = %8.8x, Max = %8.8x\n", 8997 pf->min_bw, pf->max_bw); 8998 8999 /* nudge the Tx scheduler */ 9000 i40e_set_partition_bw_setting(pf); 9001 } 9002 } 9003 9004 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 9005 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 9006 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 9007 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 9008 if (pf->flags & I40E_FLAG_MFP_ENABLED && 9009 pf->hw.num_partitions > 1) 9010 dev_info(&pf->pdev->dev, 9011 "Flow Director Sideband mode Disabled in MFP mode\n"); 9012 else 9013 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 9014 pf->fdir_pf_filter_count = 9015 pf->hw.func_caps.fd_filters_guaranteed; 9016 pf->hw.fdir_shared_filter_count = 9017 pf->hw.func_caps.fd_filters_best_effort; 9018 } 9019 9020 if (pf->hw.mac.type == I40E_MAC_X722) { 9021 pf->hw_features |= (I40E_HW_RSS_AQ_CAPABLE | 9022 I40E_HW_128_QP_RSS_CAPABLE | 9023 I40E_HW_ATR_EVICT_CAPABLE | 9024 I40E_HW_WB_ON_ITR_CAPABLE | 9025 I40E_HW_MULTIPLE_TCP_UDP_RSS_PCTYPE | 9026 I40E_HW_NO_PCI_LINK_CHECK | 9027 I40E_HW_USE_SET_LLDP_MIB | 9028 I40E_HW_GENEVE_OFFLOAD_CAPABLE | 9029 I40E_HW_PTP_L4_CAPABLE | 9030 I40E_HW_WOL_MC_MAGIC_PKT_WAKE | 9031 I40E_HW_OUTER_UDP_CSUM_CAPABLE); 9032 9033 #define I40E_FDEVICT_PCTYPE_DEFAULT 0xc03 9034 if (rd32(&pf->hw, I40E_GLQF_FDEVICTENA(1)) != 9035 I40E_FDEVICT_PCTYPE_DEFAULT) { 9036 dev_warn(&pf->pdev->dev, 9037 "FD EVICT PCTYPES are not right, disable FD HW EVICT\n"); 9038 pf->hw_features &= ~I40E_HW_ATR_EVICT_CAPABLE; 9039 } 9040 } else if ((pf->hw.aq.api_maj_ver > 1) || 9041 ((pf->hw.aq.api_maj_ver == 1) && 9042 (pf->hw.aq.api_min_ver > 4))) { 9043 /* Supported in FW API version higher than 1.4 */ 9044 pf->hw_features |= I40E_HW_GENEVE_OFFLOAD_CAPABLE; 9045 } 9046 9047 /* Enable HW ATR eviction if possible */ 9048 if (pf->hw_features & I40E_HW_ATR_EVICT_CAPABLE) 9049 pf->flags |= I40E_FLAG_HW_ATR_EVICT_ENABLED; 9050 9051 if ((pf->hw.mac.type == I40E_MAC_XL710) && 9052 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 9053 (pf->hw.aq.fw_maj_ver < 4))) { 9054 pf->hw_features |= I40E_HW_RESTART_AUTONEG; 9055 /* No DCB support for FW < v4.33 */ 9056 pf->hw_features |= I40E_HW_NO_DCB_SUPPORT; 9057 } 9058 9059 /* Disable FW LLDP if FW < v4.3 */ 9060 if ((pf->hw.mac.type == I40E_MAC_XL710) && 9061 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 9062 (pf->hw.aq.fw_maj_ver < 4))) 9063 pf->hw_features |= I40E_HW_STOP_FW_LLDP; 9064 9065 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 9066 if ((pf->hw.mac.type == I40E_MAC_XL710) && 9067 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 9068 (pf->hw.aq.fw_maj_ver >= 5))) 9069 pf->hw_features |= I40E_HW_USE_SET_LLDP_MIB; 9070 9071 if (pf->hw.func_caps.vmdq) { 9072 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 9073 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 9074 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 9075 } 9076 9077 if (pf->hw.func_caps.iwarp) { 9078 pf->flags |= I40E_FLAG_IWARP_ENABLED; 9079 /* IWARP needs one extra vector for CQP just like MISC.*/ 9080 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 9081 } 9082 9083 #ifdef CONFIG_PCI_IOV 9084 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 9085 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 9086 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 9087 pf->num_req_vfs = min_t(int, 9088 pf->hw.func_caps.num_vfs, 9089 I40E_MAX_VF_COUNT); 9090 } 9091 #endif /* CONFIG_PCI_IOV */ 9092 pf->eeprom_version = 0xDEAD; 9093 pf->lan_veb = I40E_NO_VEB; 9094 pf->lan_vsi = I40E_NO_VSI; 9095 9096 /* By default FW has this off for performance reasons */ 9097 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 9098 9099 /* set up queue assignment tracking */ 9100 size = sizeof(struct i40e_lump_tracking) 9101 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 9102 pf->qp_pile = kzalloc(size, GFP_KERNEL); 9103 if (!pf->qp_pile) { 9104 err = -ENOMEM; 9105 goto sw_init_done; 9106 } 9107 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 9108 pf->qp_pile->search_hint = 0; 9109 9110 pf->tx_timeout_recovery_level = 1; 9111 9112 mutex_init(&pf->switch_mutex); 9113 9114 sw_init_done: 9115 return err; 9116 } 9117 9118 /** 9119 * i40e_set_ntuple - set the ntuple feature flag and take action 9120 * @pf: board private structure to initialize 9121 * @features: the feature set that the stack is suggesting 9122 * 9123 * returns a bool to indicate if reset needs to happen 9124 **/ 9125 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 9126 { 9127 bool need_reset = false; 9128 9129 /* Check if Flow Director n-tuple support was enabled or disabled. If 9130 * the state changed, we need to reset. 9131 */ 9132 if (features & NETIF_F_NTUPLE) { 9133 /* Enable filters and mark for reset */ 9134 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 9135 need_reset = true; 9136 /* enable FD_SB only if there is MSI-X vector */ 9137 if (pf->num_fdsb_msix > 0) 9138 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 9139 } else { 9140 /* turn off filters, mark for reset and clear SW filter list */ 9141 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 9142 need_reset = true; 9143 i40e_fdir_filter_exit(pf); 9144 } 9145 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED | 9146 I40E_FLAG_FD_SB_AUTO_DISABLED); 9147 /* reset fd counters */ 9148 pf->fd_add_err = 0; 9149 pf->fd_atr_cnt = 0; 9150 /* if ATR was auto disabled it can be re-enabled. */ 9151 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) { 9152 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 9153 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 9154 (I40E_DEBUG_FD & pf->hw.debug_mask)) 9155 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 9156 } 9157 } 9158 return need_reset; 9159 } 9160 9161 /** 9162 * i40e_clear_rss_lut - clear the rx hash lookup table 9163 * @vsi: the VSI being configured 9164 **/ 9165 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 9166 { 9167 struct i40e_pf *pf = vsi->back; 9168 struct i40e_hw *hw = &pf->hw; 9169 u16 vf_id = vsi->vf_id; 9170 u8 i; 9171 9172 if (vsi->type == I40E_VSI_MAIN) { 9173 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 9174 wr32(hw, I40E_PFQF_HLUT(i), 0); 9175 } else if (vsi->type == I40E_VSI_SRIOV) { 9176 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 9177 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 9178 } else { 9179 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 9180 } 9181 } 9182 9183 /** 9184 * i40e_set_features - set the netdev feature flags 9185 * @netdev: ptr to the netdev being adjusted 9186 * @features: the feature set that the stack is suggesting 9187 * Note: expects to be called while under rtnl_lock() 9188 **/ 9189 static int i40e_set_features(struct net_device *netdev, 9190 netdev_features_t features) 9191 { 9192 struct i40e_netdev_priv *np = netdev_priv(netdev); 9193 struct i40e_vsi *vsi = np->vsi; 9194 struct i40e_pf *pf = vsi->back; 9195 bool need_reset; 9196 9197 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 9198 i40e_pf_config_rss(pf); 9199 else if (!(features & NETIF_F_RXHASH) && 9200 netdev->features & NETIF_F_RXHASH) 9201 i40e_clear_rss_lut(vsi); 9202 9203 if (features & NETIF_F_HW_VLAN_CTAG_RX) 9204 i40e_vlan_stripping_enable(vsi); 9205 else 9206 i40e_vlan_stripping_disable(vsi); 9207 9208 need_reset = i40e_set_ntuple(pf, features); 9209 9210 if (need_reset) 9211 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true); 9212 9213 return 0; 9214 } 9215 9216 /** 9217 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 9218 * @pf: board private structure 9219 * @port: The UDP port to look up 9220 * 9221 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 9222 **/ 9223 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 9224 { 9225 u8 i; 9226 9227 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 9228 if (pf->udp_ports[i].port == port) 9229 return i; 9230 } 9231 9232 return i; 9233 } 9234 9235 /** 9236 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 9237 * @netdev: This physical port's netdev 9238 * @ti: Tunnel endpoint information 9239 **/ 9240 static void i40e_udp_tunnel_add(struct net_device *netdev, 9241 struct udp_tunnel_info *ti) 9242 { 9243 struct i40e_netdev_priv *np = netdev_priv(netdev); 9244 struct i40e_vsi *vsi = np->vsi; 9245 struct i40e_pf *pf = vsi->back; 9246 u16 port = ntohs(ti->port); 9247 u8 next_idx; 9248 u8 idx; 9249 9250 idx = i40e_get_udp_port_idx(pf, port); 9251 9252 /* Check if port already exists */ 9253 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 9254 netdev_info(netdev, "port %d already offloaded\n", port); 9255 return; 9256 } 9257 9258 /* Now check if there is space to add the new port */ 9259 next_idx = i40e_get_udp_port_idx(pf, 0); 9260 9261 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 9262 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 9263 port); 9264 return; 9265 } 9266 9267 switch (ti->type) { 9268 case UDP_TUNNEL_TYPE_VXLAN: 9269 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 9270 break; 9271 case UDP_TUNNEL_TYPE_GENEVE: 9272 if (!(pf->hw_features & I40E_HW_GENEVE_OFFLOAD_CAPABLE)) 9273 return; 9274 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 9275 break; 9276 default: 9277 return; 9278 } 9279 9280 /* New port: add it and mark its index in the bitmap */ 9281 pf->udp_ports[next_idx].port = port; 9282 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 9283 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 9284 } 9285 9286 /** 9287 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 9288 * @netdev: This physical port's netdev 9289 * @ti: Tunnel endpoint information 9290 **/ 9291 static void i40e_udp_tunnel_del(struct net_device *netdev, 9292 struct udp_tunnel_info *ti) 9293 { 9294 struct i40e_netdev_priv *np = netdev_priv(netdev); 9295 struct i40e_vsi *vsi = np->vsi; 9296 struct i40e_pf *pf = vsi->back; 9297 u16 port = ntohs(ti->port); 9298 u8 idx; 9299 9300 idx = i40e_get_udp_port_idx(pf, port); 9301 9302 /* Check if port already exists */ 9303 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 9304 goto not_found; 9305 9306 switch (ti->type) { 9307 case UDP_TUNNEL_TYPE_VXLAN: 9308 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 9309 goto not_found; 9310 break; 9311 case UDP_TUNNEL_TYPE_GENEVE: 9312 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 9313 goto not_found; 9314 break; 9315 default: 9316 goto not_found; 9317 } 9318 9319 /* if port exists, set it to 0 (mark for deletion) 9320 * and make it pending 9321 */ 9322 pf->udp_ports[idx].port = 0; 9323 pf->pending_udp_bitmap |= BIT_ULL(idx); 9324 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 9325 9326 return; 9327 not_found: 9328 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 9329 port); 9330 } 9331 9332 static int i40e_get_phys_port_id(struct net_device *netdev, 9333 struct netdev_phys_item_id *ppid) 9334 { 9335 struct i40e_netdev_priv *np = netdev_priv(netdev); 9336 struct i40e_pf *pf = np->vsi->back; 9337 struct i40e_hw *hw = &pf->hw; 9338 9339 if (!(pf->hw_features & I40E_HW_PORT_ID_VALID)) 9340 return -EOPNOTSUPP; 9341 9342 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 9343 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 9344 9345 return 0; 9346 } 9347 9348 /** 9349 * i40e_ndo_fdb_add - add an entry to the hardware database 9350 * @ndm: the input from the stack 9351 * @tb: pointer to array of nladdr (unused) 9352 * @dev: the net device pointer 9353 * @addr: the MAC address entry being added 9354 * @flags: instructions from stack about fdb operation 9355 */ 9356 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 9357 struct net_device *dev, 9358 const unsigned char *addr, u16 vid, 9359 u16 flags) 9360 { 9361 struct i40e_netdev_priv *np = netdev_priv(dev); 9362 struct i40e_pf *pf = np->vsi->back; 9363 int err = 0; 9364 9365 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 9366 return -EOPNOTSUPP; 9367 9368 if (vid) { 9369 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 9370 return -EINVAL; 9371 } 9372 9373 /* Hardware does not support aging addresses so if a 9374 * ndm_state is given only allow permanent addresses 9375 */ 9376 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 9377 netdev_info(dev, "FDB only supports static addresses\n"); 9378 return -EINVAL; 9379 } 9380 9381 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 9382 err = dev_uc_add_excl(dev, addr); 9383 else if (is_multicast_ether_addr(addr)) 9384 err = dev_mc_add_excl(dev, addr); 9385 else 9386 err = -EINVAL; 9387 9388 /* Only return duplicate errors if NLM_F_EXCL is set */ 9389 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 9390 err = 0; 9391 9392 return err; 9393 } 9394 9395 /** 9396 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 9397 * @dev: the netdev being configured 9398 * @nlh: RTNL message 9399 * 9400 * Inserts a new hardware bridge if not already created and 9401 * enables the bridging mode requested (VEB or VEPA). If the 9402 * hardware bridge has already been inserted and the request 9403 * is to change the mode then that requires a PF reset to 9404 * allow rebuild of the components with required hardware 9405 * bridge mode enabled. 9406 * 9407 * Note: expects to be called while under rtnl_lock() 9408 **/ 9409 static int i40e_ndo_bridge_setlink(struct net_device *dev, 9410 struct nlmsghdr *nlh, 9411 u16 flags) 9412 { 9413 struct i40e_netdev_priv *np = netdev_priv(dev); 9414 struct i40e_vsi *vsi = np->vsi; 9415 struct i40e_pf *pf = vsi->back; 9416 struct i40e_veb *veb = NULL; 9417 struct nlattr *attr, *br_spec; 9418 int i, rem; 9419 9420 /* Only for PF VSI for now */ 9421 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9422 return -EOPNOTSUPP; 9423 9424 /* Find the HW bridge for PF VSI */ 9425 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9426 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9427 veb = pf->veb[i]; 9428 } 9429 9430 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 9431 9432 nla_for_each_nested(attr, br_spec, rem) { 9433 __u16 mode; 9434 9435 if (nla_type(attr) != IFLA_BRIDGE_MODE) 9436 continue; 9437 9438 mode = nla_get_u16(attr); 9439 if ((mode != BRIDGE_MODE_VEPA) && 9440 (mode != BRIDGE_MODE_VEB)) 9441 return -EINVAL; 9442 9443 /* Insert a new HW bridge */ 9444 if (!veb) { 9445 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 9446 vsi->tc_config.enabled_tc); 9447 if (veb) { 9448 veb->bridge_mode = mode; 9449 i40e_config_bridge_mode(veb); 9450 } else { 9451 /* No Bridge HW offload available */ 9452 return -ENOENT; 9453 } 9454 break; 9455 } else if (mode != veb->bridge_mode) { 9456 /* Existing HW bridge but different mode needs reset */ 9457 veb->bridge_mode = mode; 9458 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 9459 if (mode == BRIDGE_MODE_VEB) 9460 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 9461 else 9462 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 9463 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), 9464 true); 9465 break; 9466 } 9467 } 9468 9469 return 0; 9470 } 9471 9472 /** 9473 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 9474 * @skb: skb buff 9475 * @pid: process id 9476 * @seq: RTNL message seq # 9477 * @dev: the netdev being configured 9478 * @filter_mask: unused 9479 * @nlflags: netlink flags passed in 9480 * 9481 * Return the mode in which the hardware bridge is operating in 9482 * i.e VEB or VEPA. 9483 **/ 9484 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 9485 struct net_device *dev, 9486 u32 __always_unused filter_mask, 9487 int nlflags) 9488 { 9489 struct i40e_netdev_priv *np = netdev_priv(dev); 9490 struct i40e_vsi *vsi = np->vsi; 9491 struct i40e_pf *pf = vsi->back; 9492 struct i40e_veb *veb = NULL; 9493 int i; 9494 9495 /* Only for PF VSI for now */ 9496 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9497 return -EOPNOTSUPP; 9498 9499 /* Find the HW bridge for the PF VSI */ 9500 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9501 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9502 veb = pf->veb[i]; 9503 } 9504 9505 if (!veb) 9506 return 0; 9507 9508 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 9509 0, 0, nlflags, filter_mask, NULL); 9510 } 9511 9512 /** 9513 * i40e_features_check - Validate encapsulated packet conforms to limits 9514 * @skb: skb buff 9515 * @dev: This physical port's netdev 9516 * @features: Offload features that the stack believes apply 9517 **/ 9518 static netdev_features_t i40e_features_check(struct sk_buff *skb, 9519 struct net_device *dev, 9520 netdev_features_t features) 9521 { 9522 size_t len; 9523 9524 /* No point in doing any of this if neither checksum nor GSO are 9525 * being requested for this frame. We can rule out both by just 9526 * checking for CHECKSUM_PARTIAL 9527 */ 9528 if (skb->ip_summed != CHECKSUM_PARTIAL) 9529 return features; 9530 9531 /* We cannot support GSO if the MSS is going to be less than 9532 * 64 bytes. If it is then we need to drop support for GSO. 9533 */ 9534 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 9535 features &= ~NETIF_F_GSO_MASK; 9536 9537 /* MACLEN can support at most 63 words */ 9538 len = skb_network_header(skb) - skb->data; 9539 if (len & ~(63 * 2)) 9540 goto out_err; 9541 9542 /* IPLEN and EIPLEN can support at most 127 dwords */ 9543 len = skb_transport_header(skb) - skb_network_header(skb); 9544 if (len & ~(127 * 4)) 9545 goto out_err; 9546 9547 if (skb->encapsulation) { 9548 /* L4TUNLEN can support 127 words */ 9549 len = skb_inner_network_header(skb) - skb_transport_header(skb); 9550 if (len & ~(127 * 2)) 9551 goto out_err; 9552 9553 /* IPLEN can support at most 127 dwords */ 9554 len = skb_inner_transport_header(skb) - 9555 skb_inner_network_header(skb); 9556 if (len & ~(127 * 4)) 9557 goto out_err; 9558 } 9559 9560 /* No need to validate L4LEN as TCP is the only protocol with a 9561 * a flexible value and we support all possible values supported 9562 * by TCP, which is at most 15 dwords 9563 */ 9564 9565 return features; 9566 out_err: 9567 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 9568 } 9569 9570 /** 9571 * i40e_xdp_setup - add/remove an XDP program 9572 * @vsi: VSI to changed 9573 * @prog: XDP program 9574 **/ 9575 static int i40e_xdp_setup(struct i40e_vsi *vsi, 9576 struct bpf_prog *prog) 9577 { 9578 int frame_size = vsi->netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 9579 struct i40e_pf *pf = vsi->back; 9580 struct bpf_prog *old_prog; 9581 bool need_reset; 9582 int i; 9583 9584 /* Don't allow frames that span over multiple buffers */ 9585 if (frame_size > vsi->rx_buf_len) 9586 return -EINVAL; 9587 9588 if (!i40e_enabled_xdp_vsi(vsi) && !prog) 9589 return 0; 9590 9591 /* When turning XDP on->off/off->on we reset and rebuild the rings. */ 9592 need_reset = (i40e_enabled_xdp_vsi(vsi) != !!prog); 9593 9594 if (need_reset) 9595 i40e_prep_for_reset(pf, true); 9596 9597 old_prog = xchg(&vsi->xdp_prog, prog); 9598 9599 if (need_reset) 9600 i40e_reset_and_rebuild(pf, true, true); 9601 9602 for (i = 0; i < vsi->num_queue_pairs; i++) 9603 WRITE_ONCE(vsi->rx_rings[i]->xdp_prog, vsi->xdp_prog); 9604 9605 if (old_prog) 9606 bpf_prog_put(old_prog); 9607 9608 return 0; 9609 } 9610 9611 /** 9612 * i40e_xdp - implements ndo_xdp for i40e 9613 * @dev: netdevice 9614 * @xdp: XDP command 9615 **/ 9616 static int i40e_xdp(struct net_device *dev, 9617 struct netdev_xdp *xdp) 9618 { 9619 struct i40e_netdev_priv *np = netdev_priv(dev); 9620 struct i40e_vsi *vsi = np->vsi; 9621 9622 if (vsi->type != I40E_VSI_MAIN) 9623 return -EINVAL; 9624 9625 switch (xdp->command) { 9626 case XDP_SETUP_PROG: 9627 return i40e_xdp_setup(vsi, xdp->prog); 9628 case XDP_QUERY_PROG: 9629 xdp->prog_attached = i40e_enabled_xdp_vsi(vsi); 9630 xdp->prog_id = vsi->xdp_prog ? vsi->xdp_prog->aux->id : 0; 9631 return 0; 9632 default: 9633 return -EINVAL; 9634 } 9635 } 9636 9637 static const struct net_device_ops i40e_netdev_ops = { 9638 .ndo_open = i40e_open, 9639 .ndo_stop = i40e_close, 9640 .ndo_start_xmit = i40e_lan_xmit_frame, 9641 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 9642 .ndo_set_rx_mode = i40e_set_rx_mode, 9643 .ndo_validate_addr = eth_validate_addr, 9644 .ndo_set_mac_address = i40e_set_mac, 9645 .ndo_change_mtu = i40e_change_mtu, 9646 .ndo_do_ioctl = i40e_ioctl, 9647 .ndo_tx_timeout = i40e_tx_timeout, 9648 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 9649 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 9650 #ifdef CONFIG_NET_POLL_CONTROLLER 9651 .ndo_poll_controller = i40e_netpoll, 9652 #endif 9653 .ndo_setup_tc = __i40e_setup_tc, 9654 .ndo_set_features = i40e_set_features, 9655 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 9656 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 9657 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 9658 .ndo_get_vf_config = i40e_ndo_get_vf_config, 9659 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 9660 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 9661 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 9662 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 9663 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 9664 .ndo_get_phys_port_id = i40e_get_phys_port_id, 9665 .ndo_fdb_add = i40e_ndo_fdb_add, 9666 .ndo_features_check = i40e_features_check, 9667 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 9668 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 9669 .ndo_xdp = i40e_xdp, 9670 }; 9671 9672 /** 9673 * i40e_config_netdev - Setup the netdev flags 9674 * @vsi: the VSI being configured 9675 * 9676 * Returns 0 on success, negative value on failure 9677 **/ 9678 static int i40e_config_netdev(struct i40e_vsi *vsi) 9679 { 9680 struct i40e_pf *pf = vsi->back; 9681 struct i40e_hw *hw = &pf->hw; 9682 struct i40e_netdev_priv *np; 9683 struct net_device *netdev; 9684 u8 broadcast[ETH_ALEN]; 9685 u8 mac_addr[ETH_ALEN]; 9686 int etherdev_size; 9687 netdev_features_t hw_enc_features; 9688 netdev_features_t hw_features; 9689 9690 etherdev_size = sizeof(struct i40e_netdev_priv); 9691 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 9692 if (!netdev) 9693 return -ENOMEM; 9694 9695 vsi->netdev = netdev; 9696 np = netdev_priv(netdev); 9697 np->vsi = vsi; 9698 9699 hw_enc_features = NETIF_F_SG | 9700 NETIF_F_IP_CSUM | 9701 NETIF_F_IPV6_CSUM | 9702 NETIF_F_HIGHDMA | 9703 NETIF_F_SOFT_FEATURES | 9704 NETIF_F_TSO | 9705 NETIF_F_TSO_ECN | 9706 NETIF_F_TSO6 | 9707 NETIF_F_GSO_GRE | 9708 NETIF_F_GSO_GRE_CSUM | 9709 NETIF_F_GSO_PARTIAL | 9710 NETIF_F_GSO_UDP_TUNNEL | 9711 NETIF_F_GSO_UDP_TUNNEL_CSUM | 9712 NETIF_F_SCTP_CRC | 9713 NETIF_F_RXHASH | 9714 NETIF_F_RXCSUM | 9715 0; 9716 9717 if (!(pf->hw_features & I40E_HW_OUTER_UDP_CSUM_CAPABLE)) 9718 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 9719 9720 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 9721 9722 netdev->hw_enc_features |= hw_enc_features; 9723 9724 /* record features VLANs can make use of */ 9725 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 9726 9727 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 9728 netdev->hw_features |= NETIF_F_NTUPLE; 9729 hw_features = hw_enc_features | 9730 NETIF_F_HW_VLAN_CTAG_TX | 9731 NETIF_F_HW_VLAN_CTAG_RX; 9732 9733 netdev->hw_features |= hw_features; 9734 9735 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 9736 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 9737 9738 if (vsi->type == I40E_VSI_MAIN) { 9739 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 9740 ether_addr_copy(mac_addr, hw->mac.perm_addr); 9741 /* The following steps are necessary for two reasons. First, 9742 * some older NVM configurations load a default MAC-VLAN 9743 * filter that will accept any tagged packet, and we want to 9744 * replace this with a normal filter. Additionally, it is 9745 * possible our MAC address was provided by the platform using 9746 * Open Firmware or similar. 9747 * 9748 * Thus, we need to remove the default filter and install one 9749 * specific to the MAC address. 9750 */ 9751 i40e_rm_default_mac_filter(vsi, mac_addr); 9752 spin_lock_bh(&vsi->mac_filter_hash_lock); 9753 i40e_add_mac_filter(vsi, mac_addr); 9754 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9755 } else { 9756 /* Relate the VSI_VMDQ name to the VSI_MAIN name. Note that we 9757 * are still limited by IFNAMSIZ, but we're adding 'v%d\0' to 9758 * the end, which is 4 bytes long, so force truncation of the 9759 * original name by IFNAMSIZ - 4 9760 */ 9761 snprintf(netdev->name, IFNAMSIZ, "%.*sv%%d", 9762 IFNAMSIZ - 4, 9763 pf->vsi[pf->lan_vsi]->netdev->name); 9764 random_ether_addr(mac_addr); 9765 9766 spin_lock_bh(&vsi->mac_filter_hash_lock); 9767 i40e_add_mac_filter(vsi, mac_addr); 9768 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9769 } 9770 9771 /* Add the broadcast filter so that we initially will receive 9772 * broadcast packets. Note that when a new VLAN is first added the 9773 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 9774 * specific filters as part of transitioning into "vlan" operation. 9775 * When more VLANs are added, the driver will copy each existing MAC 9776 * filter and add it for the new VLAN. 9777 * 9778 * Broadcast filters are handled specially by 9779 * i40e_sync_filters_subtask, as the driver must to set the broadcast 9780 * promiscuous bit instead of adding this directly as a MAC/VLAN 9781 * filter. The subtask will update the correct broadcast promiscuous 9782 * bits as VLANs become active or inactive. 9783 */ 9784 eth_broadcast_addr(broadcast); 9785 spin_lock_bh(&vsi->mac_filter_hash_lock); 9786 i40e_add_mac_filter(vsi, broadcast); 9787 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9788 9789 ether_addr_copy(netdev->dev_addr, mac_addr); 9790 ether_addr_copy(netdev->perm_addr, mac_addr); 9791 9792 netdev->priv_flags |= IFF_UNICAST_FLT; 9793 netdev->priv_flags |= IFF_SUPP_NOFCS; 9794 /* Setup netdev TC information */ 9795 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 9796 9797 netdev->netdev_ops = &i40e_netdev_ops; 9798 netdev->watchdog_timeo = 5 * HZ; 9799 i40e_set_ethtool_ops(netdev); 9800 9801 /* MTU range: 68 - 9706 */ 9802 netdev->min_mtu = ETH_MIN_MTU; 9803 netdev->max_mtu = I40E_MAX_RXBUFFER - I40E_PACKET_HDR_PAD; 9804 9805 return 0; 9806 } 9807 9808 /** 9809 * i40e_vsi_delete - Delete a VSI from the switch 9810 * @vsi: the VSI being removed 9811 * 9812 * Returns 0 on success, negative value on failure 9813 **/ 9814 static void i40e_vsi_delete(struct i40e_vsi *vsi) 9815 { 9816 /* remove default VSI is not allowed */ 9817 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 9818 return; 9819 9820 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 9821 } 9822 9823 /** 9824 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 9825 * @vsi: the VSI being queried 9826 * 9827 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 9828 **/ 9829 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 9830 { 9831 struct i40e_veb *veb; 9832 struct i40e_pf *pf = vsi->back; 9833 9834 /* Uplink is not a bridge so default to VEB */ 9835 if (vsi->veb_idx == I40E_NO_VEB) 9836 return 1; 9837 9838 veb = pf->veb[vsi->veb_idx]; 9839 if (!veb) { 9840 dev_info(&pf->pdev->dev, 9841 "There is no veb associated with the bridge\n"); 9842 return -ENOENT; 9843 } 9844 9845 /* Uplink is a bridge in VEPA mode */ 9846 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 9847 return 0; 9848 } else { 9849 /* Uplink is a bridge in VEB mode */ 9850 return 1; 9851 } 9852 9853 /* VEPA is now default bridge, so return 0 */ 9854 return 0; 9855 } 9856 9857 /** 9858 * i40e_add_vsi - Add a VSI to the switch 9859 * @vsi: the VSI being configured 9860 * 9861 * This initializes a VSI context depending on the VSI type to be added and 9862 * passes it down to the add_vsi aq command. 9863 **/ 9864 static int i40e_add_vsi(struct i40e_vsi *vsi) 9865 { 9866 int ret = -ENODEV; 9867 struct i40e_pf *pf = vsi->back; 9868 struct i40e_hw *hw = &pf->hw; 9869 struct i40e_vsi_context ctxt; 9870 struct i40e_mac_filter *f; 9871 struct hlist_node *h; 9872 int bkt; 9873 9874 u8 enabled_tc = 0x1; /* TC0 enabled */ 9875 int f_count = 0; 9876 9877 memset(&ctxt, 0, sizeof(ctxt)); 9878 switch (vsi->type) { 9879 case I40E_VSI_MAIN: 9880 /* The PF's main VSI is already setup as part of the 9881 * device initialization, so we'll not bother with 9882 * the add_vsi call, but we will retrieve the current 9883 * VSI context. 9884 */ 9885 ctxt.seid = pf->main_vsi_seid; 9886 ctxt.pf_num = pf->hw.pf_id; 9887 ctxt.vf_num = 0; 9888 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9889 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9890 if (ret) { 9891 dev_info(&pf->pdev->dev, 9892 "couldn't get PF vsi config, err %s aq_err %s\n", 9893 i40e_stat_str(&pf->hw, ret), 9894 i40e_aq_str(&pf->hw, 9895 pf->hw.aq.asq_last_status)); 9896 return -ENOENT; 9897 } 9898 vsi->info = ctxt.info; 9899 vsi->info.valid_sections = 0; 9900 9901 vsi->seid = ctxt.seid; 9902 vsi->id = ctxt.vsi_number; 9903 9904 enabled_tc = i40e_pf_get_tc_map(pf); 9905 9906 /* MFP mode setup queue map and update VSI */ 9907 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 9908 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 9909 memset(&ctxt, 0, sizeof(ctxt)); 9910 ctxt.seid = pf->main_vsi_seid; 9911 ctxt.pf_num = pf->hw.pf_id; 9912 ctxt.vf_num = 0; 9913 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 9914 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 9915 if (ret) { 9916 dev_info(&pf->pdev->dev, 9917 "update vsi failed, err %s aq_err %s\n", 9918 i40e_stat_str(&pf->hw, ret), 9919 i40e_aq_str(&pf->hw, 9920 pf->hw.aq.asq_last_status)); 9921 ret = -ENOENT; 9922 goto err; 9923 } 9924 /* update the local VSI info queue map */ 9925 i40e_vsi_update_queue_map(vsi, &ctxt); 9926 vsi->info.valid_sections = 0; 9927 } else { 9928 /* Default/Main VSI is only enabled for TC0 9929 * reconfigure it to enable all TCs that are 9930 * available on the port in SFP mode. 9931 * For MFP case the iSCSI PF would use this 9932 * flow to enable LAN+iSCSI TC. 9933 */ 9934 ret = i40e_vsi_config_tc(vsi, enabled_tc); 9935 if (ret) { 9936 /* Single TC condition is not fatal, 9937 * message and continue 9938 */ 9939 dev_info(&pf->pdev->dev, 9940 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 9941 enabled_tc, 9942 i40e_stat_str(&pf->hw, ret), 9943 i40e_aq_str(&pf->hw, 9944 pf->hw.aq.asq_last_status)); 9945 } 9946 } 9947 break; 9948 9949 case I40E_VSI_FDIR: 9950 ctxt.pf_num = hw->pf_id; 9951 ctxt.vf_num = 0; 9952 ctxt.uplink_seid = vsi->uplink_seid; 9953 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9954 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9955 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 9956 (i40e_is_vsi_uplink_mode_veb(vsi))) { 9957 ctxt.info.valid_sections |= 9958 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9959 ctxt.info.switch_id = 9960 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9961 } 9962 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9963 break; 9964 9965 case I40E_VSI_VMDQ2: 9966 ctxt.pf_num = hw->pf_id; 9967 ctxt.vf_num = 0; 9968 ctxt.uplink_seid = vsi->uplink_seid; 9969 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9970 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 9971 9972 /* This VSI is connected to VEB so the switch_id 9973 * should be set to zero by default. 9974 */ 9975 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9976 ctxt.info.valid_sections |= 9977 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9978 ctxt.info.switch_id = 9979 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9980 } 9981 9982 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9983 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9984 break; 9985 9986 case I40E_VSI_SRIOV: 9987 ctxt.pf_num = hw->pf_id; 9988 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 9989 ctxt.uplink_seid = vsi->uplink_seid; 9990 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9991 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 9992 9993 /* This VSI is connected to VEB so the switch_id 9994 * should be set to zero by default. 9995 */ 9996 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9997 ctxt.info.valid_sections |= 9998 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9999 ctxt.info.switch_id = 10000 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 10001 } 10002 10003 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 10004 ctxt.info.valid_sections |= 10005 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 10006 ctxt.info.queueing_opt_flags |= 10007 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 10008 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 10009 } 10010 10011 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 10012 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 10013 if (pf->vf[vsi->vf_id].spoofchk) { 10014 ctxt.info.valid_sections |= 10015 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 10016 ctxt.info.sec_flags |= 10017 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 10018 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 10019 } 10020 /* Setup the VSI tx/rx queue map for TC0 only for now */ 10021 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 10022 break; 10023 10024 case I40E_VSI_IWARP: 10025 /* send down message to iWARP */ 10026 break; 10027 10028 default: 10029 return -ENODEV; 10030 } 10031 10032 if (vsi->type != I40E_VSI_MAIN) { 10033 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 10034 if (ret) { 10035 dev_info(&vsi->back->pdev->dev, 10036 "add vsi failed, err %s aq_err %s\n", 10037 i40e_stat_str(&pf->hw, ret), 10038 i40e_aq_str(&pf->hw, 10039 pf->hw.aq.asq_last_status)); 10040 ret = -ENOENT; 10041 goto err; 10042 } 10043 vsi->info = ctxt.info; 10044 vsi->info.valid_sections = 0; 10045 vsi->seid = ctxt.seid; 10046 vsi->id = ctxt.vsi_number; 10047 } 10048 10049 vsi->active_filters = 0; 10050 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 10051 spin_lock_bh(&vsi->mac_filter_hash_lock); 10052 /* If macvlan filters already exist, force them to get loaded */ 10053 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 10054 f->state = I40E_FILTER_NEW; 10055 f_count++; 10056 } 10057 spin_unlock_bh(&vsi->mac_filter_hash_lock); 10058 10059 if (f_count) { 10060 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 10061 pf->flags |= I40E_FLAG_FILTER_SYNC; 10062 } 10063 10064 /* Update VSI BW information */ 10065 ret = i40e_vsi_get_bw_info(vsi); 10066 if (ret) { 10067 dev_info(&pf->pdev->dev, 10068 "couldn't get vsi bw info, err %s aq_err %s\n", 10069 i40e_stat_str(&pf->hw, ret), 10070 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10071 /* VSI is already added so not tearing that up */ 10072 ret = 0; 10073 } 10074 10075 err: 10076 return ret; 10077 } 10078 10079 /** 10080 * i40e_vsi_release - Delete a VSI and free its resources 10081 * @vsi: the VSI being removed 10082 * 10083 * Returns 0 on success or < 0 on error 10084 **/ 10085 int i40e_vsi_release(struct i40e_vsi *vsi) 10086 { 10087 struct i40e_mac_filter *f; 10088 struct hlist_node *h; 10089 struct i40e_veb *veb = NULL; 10090 struct i40e_pf *pf; 10091 u16 uplink_seid; 10092 int i, n, bkt; 10093 10094 pf = vsi->back; 10095 10096 /* release of a VEB-owner or last VSI is not allowed */ 10097 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 10098 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 10099 vsi->seid, vsi->uplink_seid); 10100 return -ENODEV; 10101 } 10102 if (vsi == pf->vsi[pf->lan_vsi] && 10103 !test_bit(__I40E_DOWN, pf->state)) { 10104 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 10105 return -ENODEV; 10106 } 10107 10108 uplink_seid = vsi->uplink_seid; 10109 if (vsi->type != I40E_VSI_SRIOV) { 10110 if (vsi->netdev_registered) { 10111 vsi->netdev_registered = false; 10112 if (vsi->netdev) { 10113 /* results in a call to i40e_close() */ 10114 unregister_netdev(vsi->netdev); 10115 } 10116 } else { 10117 i40e_vsi_close(vsi); 10118 } 10119 i40e_vsi_disable_irq(vsi); 10120 } 10121 10122 spin_lock_bh(&vsi->mac_filter_hash_lock); 10123 10124 /* clear the sync flag on all filters */ 10125 if (vsi->netdev) { 10126 __dev_uc_unsync(vsi->netdev, NULL); 10127 __dev_mc_unsync(vsi->netdev, NULL); 10128 } 10129 10130 /* make sure any remaining filters are marked for deletion */ 10131 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 10132 __i40e_del_filter(vsi, f); 10133 10134 spin_unlock_bh(&vsi->mac_filter_hash_lock); 10135 10136 i40e_sync_vsi_filters(vsi); 10137 10138 i40e_vsi_delete(vsi); 10139 i40e_vsi_free_q_vectors(vsi); 10140 if (vsi->netdev) { 10141 free_netdev(vsi->netdev); 10142 vsi->netdev = NULL; 10143 } 10144 i40e_vsi_clear_rings(vsi); 10145 i40e_vsi_clear(vsi); 10146 10147 /* If this was the last thing on the VEB, except for the 10148 * controlling VSI, remove the VEB, which puts the controlling 10149 * VSI onto the next level down in the switch. 10150 * 10151 * Well, okay, there's one more exception here: don't remove 10152 * the orphan VEBs yet. We'll wait for an explicit remove request 10153 * from up the network stack. 10154 */ 10155 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 10156 if (pf->vsi[i] && 10157 pf->vsi[i]->uplink_seid == uplink_seid && 10158 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 10159 n++; /* count the VSIs */ 10160 } 10161 } 10162 for (i = 0; i < I40E_MAX_VEB; i++) { 10163 if (!pf->veb[i]) 10164 continue; 10165 if (pf->veb[i]->uplink_seid == uplink_seid) 10166 n++; /* count the VEBs */ 10167 if (pf->veb[i]->seid == uplink_seid) 10168 veb = pf->veb[i]; 10169 } 10170 if (n == 0 && veb && veb->uplink_seid != 0) 10171 i40e_veb_release(veb); 10172 10173 return 0; 10174 } 10175 10176 /** 10177 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 10178 * @vsi: ptr to the VSI 10179 * 10180 * This should only be called after i40e_vsi_mem_alloc() which allocates the 10181 * corresponding SW VSI structure and initializes num_queue_pairs for the 10182 * newly allocated VSI. 10183 * 10184 * Returns 0 on success or negative on failure 10185 **/ 10186 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 10187 { 10188 int ret = -ENOENT; 10189 struct i40e_pf *pf = vsi->back; 10190 10191 if (vsi->q_vectors[0]) { 10192 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 10193 vsi->seid); 10194 return -EEXIST; 10195 } 10196 10197 if (vsi->base_vector) { 10198 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 10199 vsi->seid, vsi->base_vector); 10200 return -EEXIST; 10201 } 10202 10203 ret = i40e_vsi_alloc_q_vectors(vsi); 10204 if (ret) { 10205 dev_info(&pf->pdev->dev, 10206 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 10207 vsi->num_q_vectors, vsi->seid, ret); 10208 vsi->num_q_vectors = 0; 10209 goto vector_setup_out; 10210 } 10211 10212 /* In Legacy mode, we do not have to get any other vector since we 10213 * piggyback on the misc/ICR0 for queue interrupts. 10214 */ 10215 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 10216 return ret; 10217 if (vsi->num_q_vectors) 10218 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 10219 vsi->num_q_vectors, vsi->idx); 10220 if (vsi->base_vector < 0) { 10221 dev_info(&pf->pdev->dev, 10222 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 10223 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 10224 i40e_vsi_free_q_vectors(vsi); 10225 ret = -ENOENT; 10226 goto vector_setup_out; 10227 } 10228 10229 vector_setup_out: 10230 return ret; 10231 } 10232 10233 /** 10234 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 10235 * @vsi: pointer to the vsi. 10236 * 10237 * This re-allocates a vsi's queue resources. 10238 * 10239 * Returns pointer to the successfully allocated and configured VSI sw struct 10240 * on success, otherwise returns NULL on failure. 10241 **/ 10242 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 10243 { 10244 u16 alloc_queue_pairs; 10245 struct i40e_pf *pf; 10246 u8 enabled_tc; 10247 int ret; 10248 10249 if (!vsi) 10250 return NULL; 10251 10252 pf = vsi->back; 10253 10254 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 10255 i40e_vsi_clear_rings(vsi); 10256 10257 i40e_vsi_free_arrays(vsi, false); 10258 i40e_set_num_rings_in_vsi(vsi); 10259 ret = i40e_vsi_alloc_arrays(vsi, false); 10260 if (ret) 10261 goto err_vsi; 10262 10263 alloc_queue_pairs = vsi->alloc_queue_pairs * 10264 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 10265 10266 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 10267 if (ret < 0) { 10268 dev_info(&pf->pdev->dev, 10269 "failed to get tracking for %d queues for VSI %d err %d\n", 10270 alloc_queue_pairs, vsi->seid, ret); 10271 goto err_vsi; 10272 } 10273 vsi->base_queue = ret; 10274 10275 /* Update the FW view of the VSI. Force a reset of TC and queue 10276 * layout configurations. 10277 */ 10278 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 10279 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 10280 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 10281 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 10282 if (vsi->type == I40E_VSI_MAIN) 10283 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 10284 10285 /* assign it some queues */ 10286 ret = i40e_alloc_rings(vsi); 10287 if (ret) 10288 goto err_rings; 10289 10290 /* map all of the rings to the q_vectors */ 10291 i40e_vsi_map_rings_to_vectors(vsi); 10292 return vsi; 10293 10294 err_rings: 10295 i40e_vsi_free_q_vectors(vsi); 10296 if (vsi->netdev_registered) { 10297 vsi->netdev_registered = false; 10298 unregister_netdev(vsi->netdev); 10299 free_netdev(vsi->netdev); 10300 vsi->netdev = NULL; 10301 } 10302 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 10303 err_vsi: 10304 i40e_vsi_clear(vsi); 10305 return NULL; 10306 } 10307 10308 /** 10309 * i40e_vsi_setup - Set up a VSI by a given type 10310 * @pf: board private structure 10311 * @type: VSI type 10312 * @uplink_seid: the switch element to link to 10313 * @param1: usage depends upon VSI type. For VF types, indicates VF id 10314 * 10315 * This allocates the sw VSI structure and its queue resources, then add a VSI 10316 * to the identified VEB. 10317 * 10318 * Returns pointer to the successfully allocated and configure VSI sw struct on 10319 * success, otherwise returns NULL on failure. 10320 **/ 10321 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 10322 u16 uplink_seid, u32 param1) 10323 { 10324 struct i40e_vsi *vsi = NULL; 10325 struct i40e_veb *veb = NULL; 10326 u16 alloc_queue_pairs; 10327 int ret, i; 10328 int v_idx; 10329 10330 /* The requested uplink_seid must be either 10331 * - the PF's port seid 10332 * no VEB is needed because this is the PF 10333 * or this is a Flow Director special case VSI 10334 * - seid of an existing VEB 10335 * - seid of a VSI that owns an existing VEB 10336 * - seid of a VSI that doesn't own a VEB 10337 * a new VEB is created and the VSI becomes the owner 10338 * - seid of the PF VSI, which is what creates the first VEB 10339 * this is a special case of the previous 10340 * 10341 * Find which uplink_seid we were given and create a new VEB if needed 10342 */ 10343 for (i = 0; i < I40E_MAX_VEB; i++) { 10344 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 10345 veb = pf->veb[i]; 10346 break; 10347 } 10348 } 10349 10350 if (!veb && uplink_seid != pf->mac_seid) { 10351 10352 for (i = 0; i < pf->num_alloc_vsi; i++) { 10353 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 10354 vsi = pf->vsi[i]; 10355 break; 10356 } 10357 } 10358 if (!vsi) { 10359 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 10360 uplink_seid); 10361 return NULL; 10362 } 10363 10364 if (vsi->uplink_seid == pf->mac_seid) 10365 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 10366 vsi->tc_config.enabled_tc); 10367 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 10368 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 10369 vsi->tc_config.enabled_tc); 10370 if (veb) { 10371 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 10372 dev_info(&vsi->back->pdev->dev, 10373 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 10374 return NULL; 10375 } 10376 /* We come up by default in VEPA mode if SRIOV is not 10377 * already enabled, in which case we can't force VEPA 10378 * mode. 10379 */ 10380 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 10381 veb->bridge_mode = BRIDGE_MODE_VEPA; 10382 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 10383 } 10384 i40e_config_bridge_mode(veb); 10385 } 10386 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 10387 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 10388 veb = pf->veb[i]; 10389 } 10390 if (!veb) { 10391 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 10392 return NULL; 10393 } 10394 10395 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 10396 uplink_seid = veb->seid; 10397 } 10398 10399 /* get vsi sw struct */ 10400 v_idx = i40e_vsi_mem_alloc(pf, type); 10401 if (v_idx < 0) 10402 goto err_alloc; 10403 vsi = pf->vsi[v_idx]; 10404 if (!vsi) 10405 goto err_alloc; 10406 vsi->type = type; 10407 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 10408 10409 if (type == I40E_VSI_MAIN) 10410 pf->lan_vsi = v_idx; 10411 else if (type == I40E_VSI_SRIOV) 10412 vsi->vf_id = param1; 10413 /* assign it some queues */ 10414 alloc_queue_pairs = vsi->alloc_queue_pairs * 10415 (i40e_enabled_xdp_vsi(vsi) ? 2 : 1); 10416 10417 ret = i40e_get_lump(pf, pf->qp_pile, alloc_queue_pairs, vsi->idx); 10418 if (ret < 0) { 10419 dev_info(&pf->pdev->dev, 10420 "failed to get tracking for %d queues for VSI %d err=%d\n", 10421 alloc_queue_pairs, vsi->seid, ret); 10422 goto err_vsi; 10423 } 10424 vsi->base_queue = ret; 10425 10426 /* get a VSI from the hardware */ 10427 vsi->uplink_seid = uplink_seid; 10428 ret = i40e_add_vsi(vsi); 10429 if (ret) 10430 goto err_vsi; 10431 10432 switch (vsi->type) { 10433 /* setup the netdev if needed */ 10434 case I40E_VSI_MAIN: 10435 case I40E_VSI_VMDQ2: 10436 ret = i40e_config_netdev(vsi); 10437 if (ret) 10438 goto err_netdev; 10439 ret = register_netdev(vsi->netdev); 10440 if (ret) 10441 goto err_netdev; 10442 vsi->netdev_registered = true; 10443 netif_carrier_off(vsi->netdev); 10444 #ifdef CONFIG_I40E_DCB 10445 /* Setup DCB netlink interface */ 10446 i40e_dcbnl_setup(vsi); 10447 #endif /* CONFIG_I40E_DCB */ 10448 /* fall through */ 10449 10450 case I40E_VSI_FDIR: 10451 /* set up vectors and rings if needed */ 10452 ret = i40e_vsi_setup_vectors(vsi); 10453 if (ret) 10454 goto err_msix; 10455 10456 ret = i40e_alloc_rings(vsi); 10457 if (ret) 10458 goto err_rings; 10459 10460 /* map all of the rings to the q_vectors */ 10461 i40e_vsi_map_rings_to_vectors(vsi); 10462 10463 i40e_vsi_reset_stats(vsi); 10464 break; 10465 10466 default: 10467 /* no netdev or rings for the other VSI types */ 10468 break; 10469 } 10470 10471 if ((pf->hw_features & I40E_HW_RSS_AQ_CAPABLE) && 10472 (vsi->type == I40E_VSI_VMDQ2)) { 10473 ret = i40e_vsi_config_rss(vsi); 10474 } 10475 return vsi; 10476 10477 err_rings: 10478 i40e_vsi_free_q_vectors(vsi); 10479 err_msix: 10480 if (vsi->netdev_registered) { 10481 vsi->netdev_registered = false; 10482 unregister_netdev(vsi->netdev); 10483 free_netdev(vsi->netdev); 10484 vsi->netdev = NULL; 10485 } 10486 err_netdev: 10487 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 10488 err_vsi: 10489 i40e_vsi_clear(vsi); 10490 err_alloc: 10491 return NULL; 10492 } 10493 10494 /** 10495 * i40e_veb_get_bw_info - Query VEB BW information 10496 * @veb: the veb to query 10497 * 10498 * Query the Tx scheduler BW configuration data for given VEB 10499 **/ 10500 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 10501 { 10502 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 10503 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 10504 struct i40e_pf *pf = veb->pf; 10505 struct i40e_hw *hw = &pf->hw; 10506 u32 tc_bw_max; 10507 int ret = 0; 10508 int i; 10509 10510 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 10511 &bw_data, NULL); 10512 if (ret) { 10513 dev_info(&pf->pdev->dev, 10514 "query veb bw config failed, err %s aq_err %s\n", 10515 i40e_stat_str(&pf->hw, ret), 10516 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 10517 goto out; 10518 } 10519 10520 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 10521 &ets_data, NULL); 10522 if (ret) { 10523 dev_info(&pf->pdev->dev, 10524 "query veb bw ets config failed, err %s aq_err %s\n", 10525 i40e_stat_str(&pf->hw, ret), 10526 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 10527 goto out; 10528 } 10529 10530 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 10531 veb->bw_max_quanta = ets_data.tc_bw_max; 10532 veb->is_abs_credits = bw_data.absolute_credits_enable; 10533 veb->enabled_tc = ets_data.tc_valid_bits; 10534 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 10535 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 10536 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 10537 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 10538 veb->bw_tc_limit_credits[i] = 10539 le16_to_cpu(bw_data.tc_bw_limits[i]); 10540 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 10541 } 10542 10543 out: 10544 return ret; 10545 } 10546 10547 /** 10548 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 10549 * @pf: board private structure 10550 * 10551 * On error: returns error code (negative) 10552 * On success: returns vsi index in PF (positive) 10553 **/ 10554 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 10555 { 10556 int ret = -ENOENT; 10557 struct i40e_veb *veb; 10558 int i; 10559 10560 /* Need to protect the allocation of switch elements at the PF level */ 10561 mutex_lock(&pf->switch_mutex); 10562 10563 /* VEB list may be fragmented if VEB creation/destruction has 10564 * been happening. We can afford to do a quick scan to look 10565 * for any free slots in the list. 10566 * 10567 * find next empty veb slot, looping back around if necessary 10568 */ 10569 i = 0; 10570 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 10571 i++; 10572 if (i >= I40E_MAX_VEB) { 10573 ret = -ENOMEM; 10574 goto err_alloc_veb; /* out of VEB slots! */ 10575 } 10576 10577 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 10578 if (!veb) { 10579 ret = -ENOMEM; 10580 goto err_alloc_veb; 10581 } 10582 veb->pf = pf; 10583 veb->idx = i; 10584 veb->enabled_tc = 1; 10585 10586 pf->veb[i] = veb; 10587 ret = i; 10588 err_alloc_veb: 10589 mutex_unlock(&pf->switch_mutex); 10590 return ret; 10591 } 10592 10593 /** 10594 * i40e_switch_branch_release - Delete a branch of the switch tree 10595 * @branch: where to start deleting 10596 * 10597 * This uses recursion to find the tips of the branch to be 10598 * removed, deleting until we get back to and can delete this VEB. 10599 **/ 10600 static void i40e_switch_branch_release(struct i40e_veb *branch) 10601 { 10602 struct i40e_pf *pf = branch->pf; 10603 u16 branch_seid = branch->seid; 10604 u16 veb_idx = branch->idx; 10605 int i; 10606 10607 /* release any VEBs on this VEB - RECURSION */ 10608 for (i = 0; i < I40E_MAX_VEB; i++) { 10609 if (!pf->veb[i]) 10610 continue; 10611 if (pf->veb[i]->uplink_seid == branch->seid) 10612 i40e_switch_branch_release(pf->veb[i]); 10613 } 10614 10615 /* Release the VSIs on this VEB, but not the owner VSI. 10616 * 10617 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 10618 * the VEB itself, so don't use (*branch) after this loop. 10619 */ 10620 for (i = 0; i < pf->num_alloc_vsi; i++) { 10621 if (!pf->vsi[i]) 10622 continue; 10623 if (pf->vsi[i]->uplink_seid == branch_seid && 10624 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 10625 i40e_vsi_release(pf->vsi[i]); 10626 } 10627 } 10628 10629 /* There's one corner case where the VEB might not have been 10630 * removed, so double check it here and remove it if needed. 10631 * This case happens if the veb was created from the debugfs 10632 * commands and no VSIs were added to it. 10633 */ 10634 if (pf->veb[veb_idx]) 10635 i40e_veb_release(pf->veb[veb_idx]); 10636 } 10637 10638 /** 10639 * i40e_veb_clear - remove veb struct 10640 * @veb: the veb to remove 10641 **/ 10642 static void i40e_veb_clear(struct i40e_veb *veb) 10643 { 10644 if (!veb) 10645 return; 10646 10647 if (veb->pf) { 10648 struct i40e_pf *pf = veb->pf; 10649 10650 mutex_lock(&pf->switch_mutex); 10651 if (pf->veb[veb->idx] == veb) 10652 pf->veb[veb->idx] = NULL; 10653 mutex_unlock(&pf->switch_mutex); 10654 } 10655 10656 kfree(veb); 10657 } 10658 10659 /** 10660 * i40e_veb_release - Delete a VEB and free its resources 10661 * @veb: the VEB being removed 10662 **/ 10663 void i40e_veb_release(struct i40e_veb *veb) 10664 { 10665 struct i40e_vsi *vsi = NULL; 10666 struct i40e_pf *pf; 10667 int i, n = 0; 10668 10669 pf = veb->pf; 10670 10671 /* find the remaining VSI and check for extras */ 10672 for (i = 0; i < pf->num_alloc_vsi; i++) { 10673 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 10674 n++; 10675 vsi = pf->vsi[i]; 10676 } 10677 } 10678 if (n != 1) { 10679 dev_info(&pf->pdev->dev, 10680 "can't remove VEB %d with %d VSIs left\n", 10681 veb->seid, n); 10682 return; 10683 } 10684 10685 /* move the remaining VSI to uplink veb */ 10686 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 10687 if (veb->uplink_seid) { 10688 vsi->uplink_seid = veb->uplink_seid; 10689 if (veb->uplink_seid == pf->mac_seid) 10690 vsi->veb_idx = I40E_NO_VEB; 10691 else 10692 vsi->veb_idx = veb->veb_idx; 10693 } else { 10694 /* floating VEB */ 10695 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 10696 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 10697 } 10698 10699 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10700 i40e_veb_clear(veb); 10701 } 10702 10703 /** 10704 * i40e_add_veb - create the VEB in the switch 10705 * @veb: the VEB to be instantiated 10706 * @vsi: the controlling VSI 10707 **/ 10708 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 10709 { 10710 struct i40e_pf *pf = veb->pf; 10711 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 10712 int ret; 10713 10714 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 10715 veb->enabled_tc, false, 10716 &veb->seid, enable_stats, NULL); 10717 10718 /* get a VEB from the hardware */ 10719 if (ret) { 10720 dev_info(&pf->pdev->dev, 10721 "couldn't add VEB, err %s aq_err %s\n", 10722 i40e_stat_str(&pf->hw, ret), 10723 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10724 return -EPERM; 10725 } 10726 10727 /* get statistics counter */ 10728 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 10729 &veb->stats_idx, NULL, NULL, NULL); 10730 if (ret) { 10731 dev_info(&pf->pdev->dev, 10732 "couldn't get VEB statistics idx, err %s aq_err %s\n", 10733 i40e_stat_str(&pf->hw, ret), 10734 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10735 return -EPERM; 10736 } 10737 ret = i40e_veb_get_bw_info(veb); 10738 if (ret) { 10739 dev_info(&pf->pdev->dev, 10740 "couldn't get VEB bw info, err %s aq_err %s\n", 10741 i40e_stat_str(&pf->hw, ret), 10742 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10743 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10744 return -ENOENT; 10745 } 10746 10747 vsi->uplink_seid = veb->seid; 10748 vsi->veb_idx = veb->idx; 10749 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 10750 10751 return 0; 10752 } 10753 10754 /** 10755 * i40e_veb_setup - Set up a VEB 10756 * @pf: board private structure 10757 * @flags: VEB setup flags 10758 * @uplink_seid: the switch element to link to 10759 * @vsi_seid: the initial VSI seid 10760 * @enabled_tc: Enabled TC bit-map 10761 * 10762 * This allocates the sw VEB structure and links it into the switch 10763 * It is possible and legal for this to be a duplicate of an already 10764 * existing VEB. It is also possible for both uplink and vsi seids 10765 * to be zero, in order to create a floating VEB. 10766 * 10767 * Returns pointer to the successfully allocated VEB sw struct on 10768 * success, otherwise returns NULL on failure. 10769 **/ 10770 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 10771 u16 uplink_seid, u16 vsi_seid, 10772 u8 enabled_tc) 10773 { 10774 struct i40e_veb *veb, *uplink_veb = NULL; 10775 int vsi_idx, veb_idx; 10776 int ret; 10777 10778 /* if one seid is 0, the other must be 0 to create a floating relay */ 10779 if ((uplink_seid == 0 || vsi_seid == 0) && 10780 (uplink_seid + vsi_seid != 0)) { 10781 dev_info(&pf->pdev->dev, 10782 "one, not both seid's are 0: uplink=%d vsi=%d\n", 10783 uplink_seid, vsi_seid); 10784 return NULL; 10785 } 10786 10787 /* make sure there is such a vsi and uplink */ 10788 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 10789 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 10790 break; 10791 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 10792 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 10793 vsi_seid); 10794 return NULL; 10795 } 10796 10797 if (uplink_seid && uplink_seid != pf->mac_seid) { 10798 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 10799 if (pf->veb[veb_idx] && 10800 pf->veb[veb_idx]->seid == uplink_seid) { 10801 uplink_veb = pf->veb[veb_idx]; 10802 break; 10803 } 10804 } 10805 if (!uplink_veb) { 10806 dev_info(&pf->pdev->dev, 10807 "uplink seid %d not found\n", uplink_seid); 10808 return NULL; 10809 } 10810 } 10811 10812 /* get veb sw struct */ 10813 veb_idx = i40e_veb_mem_alloc(pf); 10814 if (veb_idx < 0) 10815 goto err_alloc; 10816 veb = pf->veb[veb_idx]; 10817 veb->flags = flags; 10818 veb->uplink_seid = uplink_seid; 10819 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 10820 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 10821 10822 /* create the VEB in the switch */ 10823 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 10824 if (ret) 10825 goto err_veb; 10826 if (vsi_idx == pf->lan_vsi) 10827 pf->lan_veb = veb->idx; 10828 10829 return veb; 10830 10831 err_veb: 10832 i40e_veb_clear(veb); 10833 err_alloc: 10834 return NULL; 10835 } 10836 10837 /** 10838 * i40e_setup_pf_switch_element - set PF vars based on switch type 10839 * @pf: board private structure 10840 * @ele: element we are building info from 10841 * @num_reported: total number of elements 10842 * @printconfig: should we print the contents 10843 * 10844 * helper function to assist in extracting a few useful SEID values. 10845 **/ 10846 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 10847 struct i40e_aqc_switch_config_element_resp *ele, 10848 u16 num_reported, bool printconfig) 10849 { 10850 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 10851 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 10852 u8 element_type = ele->element_type; 10853 u16 seid = le16_to_cpu(ele->seid); 10854 10855 if (printconfig) 10856 dev_info(&pf->pdev->dev, 10857 "type=%d seid=%d uplink=%d downlink=%d\n", 10858 element_type, seid, uplink_seid, downlink_seid); 10859 10860 switch (element_type) { 10861 case I40E_SWITCH_ELEMENT_TYPE_MAC: 10862 pf->mac_seid = seid; 10863 break; 10864 case I40E_SWITCH_ELEMENT_TYPE_VEB: 10865 /* Main VEB? */ 10866 if (uplink_seid != pf->mac_seid) 10867 break; 10868 if (pf->lan_veb == I40E_NO_VEB) { 10869 int v; 10870 10871 /* find existing or else empty VEB */ 10872 for (v = 0; v < I40E_MAX_VEB; v++) { 10873 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 10874 pf->lan_veb = v; 10875 break; 10876 } 10877 } 10878 if (pf->lan_veb == I40E_NO_VEB) { 10879 v = i40e_veb_mem_alloc(pf); 10880 if (v < 0) 10881 break; 10882 pf->lan_veb = v; 10883 } 10884 } 10885 10886 pf->veb[pf->lan_veb]->seid = seid; 10887 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 10888 pf->veb[pf->lan_veb]->pf = pf; 10889 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 10890 break; 10891 case I40E_SWITCH_ELEMENT_TYPE_VSI: 10892 if (num_reported != 1) 10893 break; 10894 /* This is immediately after a reset so we can assume this is 10895 * the PF's VSI 10896 */ 10897 pf->mac_seid = uplink_seid; 10898 pf->pf_seid = downlink_seid; 10899 pf->main_vsi_seid = seid; 10900 if (printconfig) 10901 dev_info(&pf->pdev->dev, 10902 "pf_seid=%d main_vsi_seid=%d\n", 10903 pf->pf_seid, pf->main_vsi_seid); 10904 break; 10905 case I40E_SWITCH_ELEMENT_TYPE_PF: 10906 case I40E_SWITCH_ELEMENT_TYPE_VF: 10907 case I40E_SWITCH_ELEMENT_TYPE_EMP: 10908 case I40E_SWITCH_ELEMENT_TYPE_BMC: 10909 case I40E_SWITCH_ELEMENT_TYPE_PE: 10910 case I40E_SWITCH_ELEMENT_TYPE_PA: 10911 /* ignore these for now */ 10912 break; 10913 default: 10914 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 10915 element_type, seid); 10916 break; 10917 } 10918 } 10919 10920 /** 10921 * i40e_fetch_switch_configuration - Get switch config from firmware 10922 * @pf: board private structure 10923 * @printconfig: should we print the contents 10924 * 10925 * Get the current switch configuration from the device and 10926 * extract a few useful SEID values. 10927 **/ 10928 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 10929 { 10930 struct i40e_aqc_get_switch_config_resp *sw_config; 10931 u16 next_seid = 0; 10932 int ret = 0; 10933 u8 *aq_buf; 10934 int i; 10935 10936 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 10937 if (!aq_buf) 10938 return -ENOMEM; 10939 10940 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 10941 do { 10942 u16 num_reported, num_total; 10943 10944 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 10945 I40E_AQ_LARGE_BUF, 10946 &next_seid, NULL); 10947 if (ret) { 10948 dev_info(&pf->pdev->dev, 10949 "get switch config failed err %s aq_err %s\n", 10950 i40e_stat_str(&pf->hw, ret), 10951 i40e_aq_str(&pf->hw, 10952 pf->hw.aq.asq_last_status)); 10953 kfree(aq_buf); 10954 return -ENOENT; 10955 } 10956 10957 num_reported = le16_to_cpu(sw_config->header.num_reported); 10958 num_total = le16_to_cpu(sw_config->header.num_total); 10959 10960 if (printconfig) 10961 dev_info(&pf->pdev->dev, 10962 "header: %d reported %d total\n", 10963 num_reported, num_total); 10964 10965 for (i = 0; i < num_reported; i++) { 10966 struct i40e_aqc_switch_config_element_resp *ele = 10967 &sw_config->element[i]; 10968 10969 i40e_setup_pf_switch_element(pf, ele, num_reported, 10970 printconfig); 10971 } 10972 } while (next_seid != 0); 10973 10974 kfree(aq_buf); 10975 return ret; 10976 } 10977 10978 /** 10979 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 10980 * @pf: board private structure 10981 * @reinit: if the Main VSI needs to re-initialized. 10982 * 10983 * Returns 0 on success, negative value on failure 10984 **/ 10985 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 10986 { 10987 u16 flags = 0; 10988 int ret; 10989 10990 /* find out what's out there already */ 10991 ret = i40e_fetch_switch_configuration(pf, false); 10992 if (ret) { 10993 dev_info(&pf->pdev->dev, 10994 "couldn't fetch switch config, err %s aq_err %s\n", 10995 i40e_stat_str(&pf->hw, ret), 10996 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10997 return ret; 10998 } 10999 i40e_pf_reset_stats(pf); 11000 11001 /* set the switch config bit for the whole device to 11002 * support limited promisc or true promisc 11003 * when user requests promisc. The default is limited 11004 * promisc. 11005 */ 11006 11007 if ((pf->hw.pf_id == 0) && 11008 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) 11009 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 11010 11011 if (pf->hw.pf_id == 0) { 11012 u16 valid_flags; 11013 11014 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 11015 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 11016 NULL); 11017 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 11018 dev_info(&pf->pdev->dev, 11019 "couldn't set switch config bits, err %s aq_err %s\n", 11020 i40e_stat_str(&pf->hw, ret), 11021 i40e_aq_str(&pf->hw, 11022 pf->hw.aq.asq_last_status)); 11023 /* not a fatal problem, just keep going */ 11024 } 11025 } 11026 11027 /* first time setup */ 11028 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 11029 struct i40e_vsi *vsi = NULL; 11030 u16 uplink_seid; 11031 11032 /* Set up the PF VSI associated with the PF's main VSI 11033 * that is already in the HW switch 11034 */ 11035 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 11036 uplink_seid = pf->veb[pf->lan_veb]->seid; 11037 else 11038 uplink_seid = pf->mac_seid; 11039 if (pf->lan_vsi == I40E_NO_VSI) 11040 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 11041 else if (reinit) 11042 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 11043 if (!vsi) { 11044 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 11045 i40e_fdir_teardown(pf); 11046 return -EAGAIN; 11047 } 11048 } else { 11049 /* force a reset of TC and queue layout configurations */ 11050 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 11051 11052 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 11053 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 11054 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 11055 } 11056 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 11057 11058 i40e_fdir_sb_setup(pf); 11059 11060 /* Setup static PF queue filter control settings */ 11061 ret = i40e_setup_pf_filter_control(pf); 11062 if (ret) { 11063 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 11064 ret); 11065 /* Failure here should not stop continuing other steps */ 11066 } 11067 11068 /* enable RSS in the HW, even for only one queue, as the stack can use 11069 * the hash 11070 */ 11071 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 11072 i40e_pf_config_rss(pf); 11073 11074 /* fill in link information and enable LSE reporting */ 11075 i40e_link_event(pf); 11076 11077 /* Initialize user-specific link properties */ 11078 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 11079 I40E_AQ_AN_COMPLETED) ? true : false); 11080 11081 i40e_ptp_init(pf); 11082 11083 /* repopulate tunnel port filters */ 11084 i40e_sync_udp_filters(pf); 11085 11086 return ret; 11087 } 11088 11089 /** 11090 * i40e_determine_queue_usage - Work out queue distribution 11091 * @pf: board private structure 11092 **/ 11093 static void i40e_determine_queue_usage(struct i40e_pf *pf) 11094 { 11095 int queues_left; 11096 int q_max; 11097 11098 pf->num_lan_qps = 0; 11099 11100 /* Find the max queues to be put into basic use. We'll always be 11101 * using TC0, whether or not DCB is running, and TC0 will get the 11102 * big RSS set. 11103 */ 11104 queues_left = pf->hw.func_caps.num_tx_qp; 11105 11106 if ((queues_left == 1) || 11107 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 11108 /* one qp for PF, no queues for anything else */ 11109 queues_left = 0; 11110 pf->alloc_rss_size = pf->num_lan_qps = 1; 11111 11112 /* make sure all the fancies are disabled */ 11113 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 11114 I40E_FLAG_IWARP_ENABLED | 11115 I40E_FLAG_FD_SB_ENABLED | 11116 I40E_FLAG_FD_ATR_ENABLED | 11117 I40E_FLAG_DCB_CAPABLE | 11118 I40E_FLAG_DCB_ENABLED | 11119 I40E_FLAG_SRIOV_ENABLED | 11120 I40E_FLAG_VMDQ_ENABLED); 11121 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 11122 I40E_FLAG_FD_SB_ENABLED | 11123 I40E_FLAG_FD_ATR_ENABLED | 11124 I40E_FLAG_DCB_CAPABLE))) { 11125 /* one qp for PF */ 11126 pf->alloc_rss_size = pf->num_lan_qps = 1; 11127 queues_left -= pf->num_lan_qps; 11128 11129 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 11130 I40E_FLAG_IWARP_ENABLED | 11131 I40E_FLAG_FD_SB_ENABLED | 11132 I40E_FLAG_FD_ATR_ENABLED | 11133 I40E_FLAG_DCB_ENABLED | 11134 I40E_FLAG_VMDQ_ENABLED); 11135 } else { 11136 /* Not enough queues for all TCs */ 11137 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 11138 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 11139 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 11140 I40E_FLAG_DCB_ENABLED); 11141 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 11142 } 11143 11144 /* limit lan qps to the smaller of qps, cpus or msix */ 11145 q_max = max_t(int, pf->rss_size_max, num_online_cpus()); 11146 q_max = min_t(int, q_max, pf->hw.func_caps.num_tx_qp); 11147 q_max = min_t(int, q_max, pf->hw.func_caps.num_msix_vectors); 11148 pf->num_lan_qps = q_max; 11149 11150 queues_left -= pf->num_lan_qps; 11151 } 11152 11153 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11154 if (queues_left > 1) { 11155 queues_left -= 1; /* save 1 queue for FD */ 11156 } else { 11157 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 11158 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 11159 } 11160 } 11161 11162 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11163 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 11164 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 11165 (queues_left / pf->num_vf_qps)); 11166 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 11167 } 11168 11169 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 11170 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 11171 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 11172 (queues_left / pf->num_vmdq_qps)); 11173 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 11174 } 11175 11176 pf->queues_left = queues_left; 11177 dev_dbg(&pf->pdev->dev, 11178 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 11179 pf->hw.func_caps.num_tx_qp, 11180 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 11181 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 11182 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 11183 queues_left); 11184 } 11185 11186 /** 11187 * i40e_setup_pf_filter_control - Setup PF static filter control 11188 * @pf: PF to be setup 11189 * 11190 * i40e_setup_pf_filter_control sets up a PF's initial filter control 11191 * settings. If PE/FCoE are enabled then it will also set the per PF 11192 * based filter sizes required for them. It also enables Flow director, 11193 * ethertype and macvlan type filter settings for the pf. 11194 * 11195 * Returns 0 on success, negative on failure 11196 **/ 11197 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 11198 { 11199 struct i40e_filter_control_settings *settings = &pf->filter_settings; 11200 11201 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 11202 11203 /* Flow Director is enabled */ 11204 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 11205 settings->enable_fdir = true; 11206 11207 /* Ethtype and MACVLAN filters enabled for PF */ 11208 settings->enable_ethtype = true; 11209 settings->enable_macvlan = true; 11210 11211 if (i40e_set_filter_control(&pf->hw, settings)) 11212 return -ENOENT; 11213 11214 return 0; 11215 } 11216 11217 #define INFO_STRING_LEN 255 11218 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 11219 static void i40e_print_features(struct i40e_pf *pf) 11220 { 11221 struct i40e_hw *hw = &pf->hw; 11222 char *buf; 11223 int i; 11224 11225 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 11226 if (!buf) 11227 return; 11228 11229 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 11230 #ifdef CONFIG_PCI_IOV 11231 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 11232 #endif 11233 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 11234 pf->hw.func_caps.num_vsis, 11235 pf->vsi[pf->lan_vsi]->num_queue_pairs); 11236 if (pf->flags & I40E_FLAG_RSS_ENABLED) 11237 i += snprintf(&buf[i], REMAIN(i), " RSS"); 11238 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 11239 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 11240 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 11241 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 11242 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 11243 } 11244 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 11245 i += snprintf(&buf[i], REMAIN(i), " DCB"); 11246 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 11247 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 11248 if (pf->flags & I40E_FLAG_PTP) 11249 i += snprintf(&buf[i], REMAIN(i), " PTP"); 11250 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 11251 i += snprintf(&buf[i], REMAIN(i), " VEB"); 11252 else 11253 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 11254 11255 dev_info(&pf->pdev->dev, "%s\n", buf); 11256 kfree(buf); 11257 WARN_ON(i > INFO_STRING_LEN); 11258 } 11259 11260 /** 11261 * i40e_get_platform_mac_addr - get platform-specific MAC address 11262 * @pdev: PCI device information struct 11263 * @pf: board private structure 11264 * 11265 * Look up the MAC address for the device. First we'll try 11266 * eth_platform_get_mac_address, which will check Open Firmware, or arch 11267 * specific fallback. Otherwise, we'll default to the stored value in 11268 * firmware. 11269 **/ 11270 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 11271 { 11272 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 11273 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 11274 } 11275 11276 /** 11277 * i40e_probe - Device initialization routine 11278 * @pdev: PCI device information struct 11279 * @ent: entry in i40e_pci_tbl 11280 * 11281 * i40e_probe initializes a PF identified by a pci_dev structure. 11282 * The OS initialization, configuring of the PF private structure, 11283 * and a hardware reset occur. 11284 * 11285 * Returns 0 on success, negative on failure 11286 **/ 11287 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 11288 { 11289 struct i40e_aq_get_phy_abilities_resp abilities; 11290 struct i40e_pf *pf; 11291 struct i40e_hw *hw; 11292 static u16 pfs_found; 11293 u16 wol_nvm_bits; 11294 u16 link_status; 11295 int err; 11296 u32 val; 11297 u32 i; 11298 u8 set_fc_aq_fail; 11299 11300 err = pci_enable_device_mem(pdev); 11301 if (err) 11302 return err; 11303 11304 /* set up for high or low dma */ 11305 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 11306 if (err) { 11307 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 11308 if (err) { 11309 dev_err(&pdev->dev, 11310 "DMA configuration failed: 0x%x\n", err); 11311 goto err_dma; 11312 } 11313 } 11314 11315 /* set up pci connections */ 11316 err = pci_request_mem_regions(pdev, i40e_driver_name); 11317 if (err) { 11318 dev_info(&pdev->dev, 11319 "pci_request_selected_regions failed %d\n", err); 11320 goto err_pci_reg; 11321 } 11322 11323 pci_enable_pcie_error_reporting(pdev); 11324 pci_set_master(pdev); 11325 11326 /* Now that we have a PCI connection, we need to do the 11327 * low level device setup. This is primarily setting up 11328 * the Admin Queue structures and then querying for the 11329 * device's current profile information. 11330 */ 11331 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 11332 if (!pf) { 11333 err = -ENOMEM; 11334 goto err_pf_alloc; 11335 } 11336 pf->next_vsi = 0; 11337 pf->pdev = pdev; 11338 set_bit(__I40E_DOWN, pf->state); 11339 11340 hw = &pf->hw; 11341 hw->back = pf; 11342 11343 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 11344 I40E_MAX_CSR_SPACE); 11345 11346 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 11347 if (!hw->hw_addr) { 11348 err = -EIO; 11349 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 11350 (unsigned int)pci_resource_start(pdev, 0), 11351 pf->ioremap_len, err); 11352 goto err_ioremap; 11353 } 11354 hw->vendor_id = pdev->vendor; 11355 hw->device_id = pdev->device; 11356 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 11357 hw->subsystem_vendor_id = pdev->subsystem_vendor; 11358 hw->subsystem_device_id = pdev->subsystem_device; 11359 hw->bus.device = PCI_SLOT(pdev->devfn); 11360 hw->bus.func = PCI_FUNC(pdev->devfn); 11361 hw->bus.bus_id = pdev->bus->number; 11362 pf->instance = pfs_found; 11363 11364 /* Select something other than the 802.1ad ethertype for the 11365 * switch to use internally and drop on ingress. 11366 */ 11367 hw->switch_tag = 0xffff; 11368 hw->first_tag = ETH_P_8021AD; 11369 hw->second_tag = ETH_P_8021Q; 11370 11371 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 11372 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 11373 11374 /* set up the locks for the AQ, do this only once in probe 11375 * and destroy them only once in remove 11376 */ 11377 mutex_init(&hw->aq.asq_mutex); 11378 mutex_init(&hw->aq.arq_mutex); 11379 11380 pf->msg_enable = netif_msg_init(debug, 11381 NETIF_MSG_DRV | 11382 NETIF_MSG_PROBE | 11383 NETIF_MSG_LINK); 11384 if (debug < -1) 11385 pf->hw.debug_mask = debug; 11386 11387 /* do a special CORER for clearing PXE mode once at init */ 11388 if (hw->revision_id == 0 && 11389 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 11390 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 11391 i40e_flush(hw); 11392 msleep(200); 11393 pf->corer_count++; 11394 11395 i40e_clear_pxe_mode(hw); 11396 } 11397 11398 /* Reset here to make sure all is clean and to define PF 'n' */ 11399 i40e_clear_hw(hw); 11400 err = i40e_pf_reset(hw); 11401 if (err) { 11402 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 11403 goto err_pf_reset; 11404 } 11405 pf->pfr_count++; 11406 11407 hw->aq.num_arq_entries = I40E_AQ_LEN; 11408 hw->aq.num_asq_entries = I40E_AQ_LEN; 11409 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 11410 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 11411 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 11412 11413 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 11414 "%s-%s:misc", 11415 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 11416 11417 err = i40e_init_shared_code(hw); 11418 if (err) { 11419 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 11420 err); 11421 goto err_pf_reset; 11422 } 11423 11424 /* set up a default setting for link flow control */ 11425 pf->hw.fc.requested_mode = I40E_FC_NONE; 11426 11427 err = i40e_init_adminq(hw); 11428 if (err) { 11429 if (err == I40E_ERR_FIRMWARE_API_VERSION) 11430 dev_info(&pdev->dev, 11431 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 11432 else 11433 dev_info(&pdev->dev, 11434 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 11435 11436 goto err_pf_reset; 11437 } 11438 i40e_get_oem_version(hw); 11439 11440 /* provide nvm, fw, api versions */ 11441 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 11442 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 11443 hw->aq.api_maj_ver, hw->aq.api_min_ver, 11444 i40e_nvm_version_str(hw)); 11445 11446 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 11447 hw->aq.api_min_ver > I40E_FW_MINOR_VERSION(hw)) 11448 dev_info(&pdev->dev, 11449 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 11450 else if (hw->aq.api_maj_ver == 1 && hw->aq.api_min_ver < 4) 11451 dev_info(&pdev->dev, 11452 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 11453 11454 i40e_verify_eeprom(pf); 11455 11456 /* Rev 0 hardware was never productized */ 11457 if (hw->revision_id < 1) 11458 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 11459 11460 i40e_clear_pxe_mode(hw); 11461 err = i40e_get_capabilities(pf); 11462 if (err) 11463 goto err_adminq_setup; 11464 11465 err = i40e_sw_init(pf); 11466 if (err) { 11467 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 11468 goto err_sw_init; 11469 } 11470 11471 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 11472 hw->func_caps.num_rx_qp, 0, 0); 11473 if (err) { 11474 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 11475 goto err_init_lan_hmc; 11476 } 11477 11478 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 11479 if (err) { 11480 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 11481 err = -ENOENT; 11482 goto err_configure_lan_hmc; 11483 } 11484 11485 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 11486 * Ignore error return codes because if it was already disabled via 11487 * hardware settings this will fail 11488 */ 11489 if (pf->hw_features & I40E_HW_STOP_FW_LLDP) { 11490 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 11491 i40e_aq_stop_lldp(hw, true, NULL); 11492 } 11493 11494 /* allow a platform config to override the HW addr */ 11495 i40e_get_platform_mac_addr(pdev, pf); 11496 11497 if (!is_valid_ether_addr(hw->mac.addr)) { 11498 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 11499 err = -EIO; 11500 goto err_mac_addr; 11501 } 11502 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 11503 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 11504 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 11505 if (is_valid_ether_addr(hw->mac.port_addr)) 11506 pf->hw_features |= I40E_HW_PORT_ID_VALID; 11507 11508 pci_set_drvdata(pdev, pf); 11509 pci_save_state(pdev); 11510 #ifdef CONFIG_I40E_DCB 11511 err = i40e_init_pf_dcb(pf); 11512 if (err) { 11513 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 11514 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 11515 /* Continue without DCB enabled */ 11516 } 11517 #endif /* CONFIG_I40E_DCB */ 11518 11519 /* set up periodic task facility */ 11520 setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf); 11521 pf->service_timer_period = HZ; 11522 11523 INIT_WORK(&pf->service_task, i40e_service_task); 11524 clear_bit(__I40E_SERVICE_SCHED, pf->state); 11525 11526 /* NVM bit on means WoL disabled for the port */ 11527 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 11528 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 11529 pf->wol_en = false; 11530 else 11531 pf->wol_en = true; 11532 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 11533 11534 /* set up the main switch operations */ 11535 i40e_determine_queue_usage(pf); 11536 err = i40e_init_interrupt_scheme(pf); 11537 if (err) 11538 goto err_switch_setup; 11539 11540 /* The number of VSIs reported by the FW is the minimum guaranteed 11541 * to us; HW supports far more and we share the remaining pool with 11542 * the other PFs. We allocate space for more than the guarantee with 11543 * the understanding that we might not get them all later. 11544 */ 11545 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 11546 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 11547 else 11548 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 11549 11550 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 11551 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 11552 GFP_KERNEL); 11553 if (!pf->vsi) { 11554 err = -ENOMEM; 11555 goto err_switch_setup; 11556 } 11557 11558 #ifdef CONFIG_PCI_IOV 11559 /* prep for VF support */ 11560 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11561 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11562 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 11563 if (pci_num_vf(pdev)) 11564 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11565 } 11566 #endif 11567 err = i40e_setup_pf_switch(pf, false); 11568 if (err) { 11569 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 11570 goto err_vsis; 11571 } 11572 11573 /* Make sure flow control is set according to current settings */ 11574 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 11575 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 11576 dev_dbg(&pf->pdev->dev, 11577 "Set fc with err %s aq_err %s on get_phy_cap\n", 11578 i40e_stat_str(hw, err), 11579 i40e_aq_str(hw, hw->aq.asq_last_status)); 11580 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 11581 dev_dbg(&pf->pdev->dev, 11582 "Set fc with err %s aq_err %s on set_phy_config\n", 11583 i40e_stat_str(hw, err), 11584 i40e_aq_str(hw, hw->aq.asq_last_status)); 11585 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 11586 dev_dbg(&pf->pdev->dev, 11587 "Set fc with err %s aq_err %s on get_link_info\n", 11588 i40e_stat_str(hw, err), 11589 i40e_aq_str(hw, hw->aq.asq_last_status)); 11590 11591 /* if FDIR VSI was set up, start it now */ 11592 for (i = 0; i < pf->num_alloc_vsi; i++) { 11593 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 11594 i40e_vsi_open(pf->vsi[i]); 11595 break; 11596 } 11597 } 11598 11599 /* The driver only wants link up/down and module qualification 11600 * reports from firmware. Note the negative logic. 11601 */ 11602 err = i40e_aq_set_phy_int_mask(&pf->hw, 11603 ~(I40E_AQ_EVENT_LINK_UPDOWN | 11604 I40E_AQ_EVENT_MEDIA_NA | 11605 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 11606 if (err) 11607 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 11608 i40e_stat_str(&pf->hw, err), 11609 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11610 11611 /* Reconfigure hardware for allowing smaller MSS in the case 11612 * of TSO, so that we avoid the MDD being fired and causing 11613 * a reset in the case of small MSS+TSO. 11614 */ 11615 val = rd32(hw, I40E_REG_MSS); 11616 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11617 val &= ~I40E_REG_MSS_MIN_MASK; 11618 val |= I40E_64BYTE_MSS; 11619 wr32(hw, I40E_REG_MSS, val); 11620 } 11621 11622 if (pf->hw_features & I40E_HW_RESTART_AUTONEG) { 11623 msleep(75); 11624 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11625 if (err) 11626 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 11627 i40e_stat_str(&pf->hw, err), 11628 i40e_aq_str(&pf->hw, 11629 pf->hw.aq.asq_last_status)); 11630 } 11631 /* The main driver is (mostly) up and happy. We need to set this state 11632 * before setting up the misc vector or we get a race and the vector 11633 * ends up disabled forever. 11634 */ 11635 clear_bit(__I40E_DOWN, pf->state); 11636 11637 /* In case of MSIX we are going to setup the misc vector right here 11638 * to handle admin queue events etc. In case of legacy and MSI 11639 * the misc functionality and queue processing is combined in 11640 * the same vector and that gets setup at open. 11641 */ 11642 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11643 err = i40e_setup_misc_vector(pf); 11644 if (err) { 11645 dev_info(&pdev->dev, 11646 "setup of misc vector failed: %d\n", err); 11647 goto err_vsis; 11648 } 11649 } 11650 11651 #ifdef CONFIG_PCI_IOV 11652 /* prep for VF support */ 11653 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11654 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11655 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 11656 /* disable link interrupts for VFs */ 11657 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 11658 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 11659 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 11660 i40e_flush(hw); 11661 11662 if (pci_num_vf(pdev)) { 11663 dev_info(&pdev->dev, 11664 "Active VFs found, allocating resources.\n"); 11665 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 11666 if (err) 11667 dev_info(&pdev->dev, 11668 "Error %d allocating resources for existing VFs\n", 11669 err); 11670 } 11671 } 11672 #endif /* CONFIG_PCI_IOV */ 11673 11674 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11675 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 11676 pf->num_iwarp_msix, 11677 I40E_IWARP_IRQ_PILE_ID); 11678 if (pf->iwarp_base_vector < 0) { 11679 dev_info(&pdev->dev, 11680 "failed to get tracking for %d vectors for IWARP err=%d\n", 11681 pf->num_iwarp_msix, pf->iwarp_base_vector); 11682 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11683 } 11684 } 11685 11686 i40e_dbg_pf_init(pf); 11687 11688 /* tell the firmware that we're starting */ 11689 i40e_send_version(pf); 11690 11691 /* since everything's happy, start the service_task timer */ 11692 mod_timer(&pf->service_timer, 11693 round_jiffies(jiffies + pf->service_timer_period)); 11694 11695 /* add this PF to client device list and launch a client service task */ 11696 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11697 err = i40e_lan_add_device(pf); 11698 if (err) 11699 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 11700 err); 11701 } 11702 11703 #define PCI_SPEED_SIZE 8 11704 #define PCI_WIDTH_SIZE 8 11705 /* Devices on the IOSF bus do not have this information 11706 * and will report PCI Gen 1 x 1 by default so don't bother 11707 * checking them. 11708 */ 11709 if (!(pf->hw_features & I40E_HW_NO_PCI_LINK_CHECK)) { 11710 char speed[PCI_SPEED_SIZE] = "Unknown"; 11711 char width[PCI_WIDTH_SIZE] = "Unknown"; 11712 11713 /* Get the negotiated link width and speed from PCI config 11714 * space 11715 */ 11716 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 11717 &link_status); 11718 11719 i40e_set_pci_config_data(hw, link_status); 11720 11721 switch (hw->bus.speed) { 11722 case i40e_bus_speed_8000: 11723 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 11724 case i40e_bus_speed_5000: 11725 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 11726 case i40e_bus_speed_2500: 11727 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 11728 default: 11729 break; 11730 } 11731 switch (hw->bus.width) { 11732 case i40e_bus_width_pcie_x8: 11733 strncpy(width, "8", PCI_WIDTH_SIZE); break; 11734 case i40e_bus_width_pcie_x4: 11735 strncpy(width, "4", PCI_WIDTH_SIZE); break; 11736 case i40e_bus_width_pcie_x2: 11737 strncpy(width, "2", PCI_WIDTH_SIZE); break; 11738 case i40e_bus_width_pcie_x1: 11739 strncpy(width, "1", PCI_WIDTH_SIZE); break; 11740 default: 11741 break; 11742 } 11743 11744 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 11745 speed, width); 11746 11747 if (hw->bus.width < i40e_bus_width_pcie_x8 || 11748 hw->bus.speed < i40e_bus_speed_8000) { 11749 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 11750 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 11751 } 11752 } 11753 11754 /* get the requested speeds from the fw */ 11755 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 11756 if (err) 11757 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 11758 i40e_stat_str(&pf->hw, err), 11759 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11760 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 11761 11762 /* get the supported phy types from the fw */ 11763 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 11764 if (err) 11765 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 11766 i40e_stat_str(&pf->hw, err), 11767 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11768 11769 /* Add a filter to drop all Flow control frames from any VSI from being 11770 * transmitted. By doing so we stop a malicious VF from sending out 11771 * PAUSE or PFC frames and potentially controlling traffic for other 11772 * PF/VF VSIs. 11773 * The FW can still send Flow control frames if enabled. 11774 */ 11775 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11776 pf->main_vsi_seid); 11777 11778 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 11779 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 11780 pf->hw_features |= I40E_HW_PHY_CONTROLS_LEDS; 11781 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 11782 pf->hw_features |= I40E_HW_HAVE_CRT_RETIMER; 11783 /* print a string summarizing features */ 11784 i40e_print_features(pf); 11785 11786 return 0; 11787 11788 /* Unwind what we've done if something failed in the setup */ 11789 err_vsis: 11790 set_bit(__I40E_DOWN, pf->state); 11791 i40e_clear_interrupt_scheme(pf); 11792 kfree(pf->vsi); 11793 err_switch_setup: 11794 i40e_reset_interrupt_capability(pf); 11795 del_timer_sync(&pf->service_timer); 11796 err_mac_addr: 11797 err_configure_lan_hmc: 11798 (void)i40e_shutdown_lan_hmc(hw); 11799 err_init_lan_hmc: 11800 kfree(pf->qp_pile); 11801 err_sw_init: 11802 err_adminq_setup: 11803 err_pf_reset: 11804 iounmap(hw->hw_addr); 11805 err_ioremap: 11806 kfree(pf); 11807 err_pf_alloc: 11808 pci_disable_pcie_error_reporting(pdev); 11809 pci_release_mem_regions(pdev); 11810 err_pci_reg: 11811 err_dma: 11812 pci_disable_device(pdev); 11813 return err; 11814 } 11815 11816 /** 11817 * i40e_remove - Device removal routine 11818 * @pdev: PCI device information struct 11819 * 11820 * i40e_remove is called by the PCI subsystem to alert the driver 11821 * that is should release a PCI device. This could be caused by a 11822 * Hot-Plug event, or because the driver is going to be removed from 11823 * memory. 11824 **/ 11825 static void i40e_remove(struct pci_dev *pdev) 11826 { 11827 struct i40e_pf *pf = pci_get_drvdata(pdev); 11828 struct i40e_hw *hw = &pf->hw; 11829 i40e_status ret_code; 11830 int i; 11831 11832 i40e_dbg_pf_exit(pf); 11833 11834 i40e_ptp_stop(pf); 11835 11836 /* Disable RSS in hw */ 11837 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 11838 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 11839 11840 /* no more scheduling of any task */ 11841 set_bit(__I40E_SUSPENDED, pf->state); 11842 set_bit(__I40E_DOWN, pf->state); 11843 if (pf->service_timer.data) 11844 del_timer_sync(&pf->service_timer); 11845 if (pf->service_task.func) 11846 cancel_work_sync(&pf->service_task); 11847 11848 /* Client close must be called explicitly here because the timer 11849 * has been stopped. 11850 */ 11851 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 11852 11853 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 11854 i40e_free_vfs(pf); 11855 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 11856 } 11857 11858 i40e_fdir_teardown(pf); 11859 11860 /* If there is a switch structure or any orphans, remove them. 11861 * This will leave only the PF's VSI remaining. 11862 */ 11863 for (i = 0; i < I40E_MAX_VEB; i++) { 11864 if (!pf->veb[i]) 11865 continue; 11866 11867 if (pf->veb[i]->uplink_seid == pf->mac_seid || 11868 pf->veb[i]->uplink_seid == 0) 11869 i40e_switch_branch_release(pf->veb[i]); 11870 } 11871 11872 /* Now we can shutdown the PF's VSI, just before we kill 11873 * adminq and hmc. 11874 */ 11875 if (pf->vsi[pf->lan_vsi]) 11876 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 11877 11878 /* remove attached clients */ 11879 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11880 ret_code = i40e_lan_del_device(pf); 11881 if (ret_code) 11882 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 11883 ret_code); 11884 } 11885 11886 /* shutdown and destroy the HMC */ 11887 if (hw->hmc.hmc_obj) { 11888 ret_code = i40e_shutdown_lan_hmc(hw); 11889 if (ret_code) 11890 dev_warn(&pdev->dev, 11891 "Failed to destroy the HMC resources: %d\n", 11892 ret_code); 11893 } 11894 11895 /* shutdown the adminq */ 11896 i40e_shutdown_adminq(hw); 11897 11898 /* destroy the locks only once, here */ 11899 mutex_destroy(&hw->aq.arq_mutex); 11900 mutex_destroy(&hw->aq.asq_mutex); 11901 11902 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 11903 i40e_clear_interrupt_scheme(pf); 11904 for (i = 0; i < pf->num_alloc_vsi; i++) { 11905 if (pf->vsi[i]) { 11906 i40e_vsi_clear_rings(pf->vsi[i]); 11907 i40e_vsi_clear(pf->vsi[i]); 11908 pf->vsi[i] = NULL; 11909 } 11910 } 11911 11912 for (i = 0; i < I40E_MAX_VEB; i++) { 11913 kfree(pf->veb[i]); 11914 pf->veb[i] = NULL; 11915 } 11916 11917 kfree(pf->qp_pile); 11918 kfree(pf->vsi); 11919 11920 iounmap(hw->hw_addr); 11921 kfree(pf); 11922 pci_release_mem_regions(pdev); 11923 11924 pci_disable_pcie_error_reporting(pdev); 11925 pci_disable_device(pdev); 11926 } 11927 11928 /** 11929 * i40e_pci_error_detected - warning that something funky happened in PCI land 11930 * @pdev: PCI device information struct 11931 * 11932 * Called to warn that something happened and the error handling steps 11933 * are in progress. Allows the driver to quiesce things, be ready for 11934 * remediation. 11935 **/ 11936 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 11937 enum pci_channel_state error) 11938 { 11939 struct i40e_pf *pf = pci_get_drvdata(pdev); 11940 11941 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 11942 11943 if (!pf) { 11944 dev_info(&pdev->dev, 11945 "Cannot recover - error happened during device probe\n"); 11946 return PCI_ERS_RESULT_DISCONNECT; 11947 } 11948 11949 /* shutdown all operations */ 11950 if (!test_bit(__I40E_SUSPENDED, pf->state)) 11951 i40e_prep_for_reset(pf, false); 11952 11953 /* Request a slot reset */ 11954 return PCI_ERS_RESULT_NEED_RESET; 11955 } 11956 11957 /** 11958 * i40e_pci_error_slot_reset - a PCI slot reset just happened 11959 * @pdev: PCI device information struct 11960 * 11961 * Called to find if the driver can work with the device now that 11962 * the pci slot has been reset. If a basic connection seems good 11963 * (registers are readable and have sane content) then return a 11964 * happy little PCI_ERS_RESULT_xxx. 11965 **/ 11966 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 11967 { 11968 struct i40e_pf *pf = pci_get_drvdata(pdev); 11969 pci_ers_result_t result; 11970 int err; 11971 u32 reg; 11972 11973 dev_dbg(&pdev->dev, "%s\n", __func__); 11974 if (pci_enable_device_mem(pdev)) { 11975 dev_info(&pdev->dev, 11976 "Cannot re-enable PCI device after reset.\n"); 11977 result = PCI_ERS_RESULT_DISCONNECT; 11978 } else { 11979 pci_set_master(pdev); 11980 pci_restore_state(pdev); 11981 pci_save_state(pdev); 11982 pci_wake_from_d3(pdev, false); 11983 11984 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 11985 if (reg == 0) 11986 result = PCI_ERS_RESULT_RECOVERED; 11987 else 11988 result = PCI_ERS_RESULT_DISCONNECT; 11989 } 11990 11991 err = pci_cleanup_aer_uncorrect_error_status(pdev); 11992 if (err) { 11993 dev_info(&pdev->dev, 11994 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 11995 err); 11996 /* non-fatal, continue */ 11997 } 11998 11999 return result; 12000 } 12001 12002 /** 12003 * i40e_pci_error_resume - restart operations after PCI error recovery 12004 * @pdev: PCI device information struct 12005 * 12006 * Called to allow the driver to bring things back up after PCI error 12007 * and/or reset recovery has finished. 12008 **/ 12009 static void i40e_pci_error_resume(struct pci_dev *pdev) 12010 { 12011 struct i40e_pf *pf = pci_get_drvdata(pdev); 12012 12013 dev_dbg(&pdev->dev, "%s\n", __func__); 12014 if (test_bit(__I40E_SUSPENDED, pf->state)) 12015 return; 12016 12017 i40e_handle_reset_warning(pf, false); 12018 } 12019 12020 /** 12021 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 12022 * using the mac_address_write admin q function 12023 * @pf: pointer to i40e_pf struct 12024 **/ 12025 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 12026 { 12027 struct i40e_hw *hw = &pf->hw; 12028 i40e_status ret; 12029 u8 mac_addr[6]; 12030 u16 flags = 0; 12031 12032 /* Get current MAC address in case it's an LAA */ 12033 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 12034 ether_addr_copy(mac_addr, 12035 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 12036 } else { 12037 dev_err(&pf->pdev->dev, 12038 "Failed to retrieve MAC address; using default\n"); 12039 ether_addr_copy(mac_addr, hw->mac.addr); 12040 } 12041 12042 /* The FW expects the mac address write cmd to first be called with 12043 * one of these flags before calling it again with the multicast 12044 * enable flags. 12045 */ 12046 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 12047 12048 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 12049 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 12050 12051 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 12052 if (ret) { 12053 dev_err(&pf->pdev->dev, 12054 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 12055 return; 12056 } 12057 12058 flags = I40E_AQC_MC_MAG_EN 12059 | I40E_AQC_WOL_PRESERVE_ON_PFR 12060 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 12061 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 12062 if (ret) 12063 dev_err(&pf->pdev->dev, 12064 "Failed to enable Multicast Magic Packet wake up\n"); 12065 } 12066 12067 /** 12068 * i40e_shutdown - PCI callback for shutting down 12069 * @pdev: PCI device information struct 12070 **/ 12071 static void i40e_shutdown(struct pci_dev *pdev) 12072 { 12073 struct i40e_pf *pf = pci_get_drvdata(pdev); 12074 struct i40e_hw *hw = &pf->hw; 12075 12076 set_bit(__I40E_SUSPENDED, pf->state); 12077 set_bit(__I40E_DOWN, pf->state); 12078 rtnl_lock(); 12079 i40e_prep_for_reset(pf, true); 12080 rtnl_unlock(); 12081 12082 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 12083 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 12084 12085 del_timer_sync(&pf->service_timer); 12086 cancel_work_sync(&pf->service_task); 12087 i40e_fdir_teardown(pf); 12088 12089 /* Client close must be called explicitly here because the timer 12090 * has been stopped. 12091 */ 12092 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 12093 12094 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 12095 i40e_enable_mc_magic_wake(pf); 12096 12097 i40e_prep_for_reset(pf, false); 12098 12099 wr32(hw, I40E_PFPM_APM, 12100 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 12101 wr32(hw, I40E_PFPM_WUFC, 12102 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 12103 12104 i40e_clear_interrupt_scheme(pf); 12105 12106 if (system_state == SYSTEM_POWER_OFF) { 12107 pci_wake_from_d3(pdev, pf->wol_en); 12108 pci_set_power_state(pdev, PCI_D3hot); 12109 } 12110 } 12111 12112 #ifdef CONFIG_PM 12113 /** 12114 * i40e_suspend - PM callback for moving to D3 12115 * @dev: generic device information structure 12116 **/ 12117 static int i40e_suspend(struct device *dev) 12118 { 12119 struct pci_dev *pdev = to_pci_dev(dev); 12120 struct i40e_pf *pf = pci_get_drvdata(pdev); 12121 struct i40e_hw *hw = &pf->hw; 12122 12123 /* If we're already suspended, then there is nothing to do */ 12124 if (test_and_set_bit(__I40E_SUSPENDED, pf->state)) 12125 return 0; 12126 12127 set_bit(__I40E_DOWN, pf->state); 12128 12129 /* Ensure service task will not be running */ 12130 del_timer_sync(&pf->service_timer); 12131 cancel_work_sync(&pf->service_task); 12132 12133 if (pf->wol_en && (pf->hw_features & I40E_HW_WOL_MC_MAGIC_PKT_WAKE)) 12134 i40e_enable_mc_magic_wake(pf); 12135 12136 i40e_prep_for_reset(pf, false); 12137 12138 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 12139 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 12140 12141 /* Clear the interrupt scheme and release our IRQs so that the system 12142 * can safely hibernate even when there are a large number of CPUs. 12143 * Otherwise hibernation might fail when mapping all the vectors back 12144 * to CPU0. 12145 */ 12146 i40e_clear_interrupt_scheme(pf); 12147 12148 return 0; 12149 } 12150 12151 /** 12152 * i40e_resume - PM callback for waking up from D3 12153 * @dev: generic device information structure 12154 **/ 12155 static int i40e_resume(struct device *dev) 12156 { 12157 struct pci_dev *pdev = to_pci_dev(dev); 12158 struct i40e_pf *pf = pci_get_drvdata(pdev); 12159 int err; 12160 12161 /* If we're not suspended, then there is nothing to do */ 12162 if (!test_bit(__I40E_SUSPENDED, pf->state)) 12163 return 0; 12164 12165 /* We cleared the interrupt scheme when we suspended, so we need to 12166 * restore it now to resume device functionality. 12167 */ 12168 err = i40e_restore_interrupt_scheme(pf); 12169 if (err) { 12170 dev_err(&pdev->dev, "Cannot restore interrupt scheme: %d\n", 12171 err); 12172 } 12173 12174 clear_bit(__I40E_DOWN, pf->state); 12175 i40e_reset_and_rebuild(pf, false, false); 12176 12177 /* Clear suspended state last after everything is recovered */ 12178 clear_bit(__I40E_SUSPENDED, pf->state); 12179 12180 /* Restart the service task */ 12181 mod_timer(&pf->service_timer, 12182 round_jiffies(jiffies + pf->service_timer_period)); 12183 12184 return 0; 12185 } 12186 12187 #endif /* CONFIG_PM */ 12188 12189 static const struct pci_error_handlers i40e_err_handler = { 12190 .error_detected = i40e_pci_error_detected, 12191 .slot_reset = i40e_pci_error_slot_reset, 12192 .resume = i40e_pci_error_resume, 12193 }; 12194 12195 static SIMPLE_DEV_PM_OPS(i40e_pm_ops, i40e_suspend, i40e_resume); 12196 12197 static struct pci_driver i40e_driver = { 12198 .name = i40e_driver_name, 12199 .id_table = i40e_pci_tbl, 12200 .probe = i40e_probe, 12201 .remove = i40e_remove, 12202 #ifdef CONFIG_PM 12203 .driver = { 12204 .pm = &i40e_pm_ops, 12205 }, 12206 #endif /* CONFIG_PM */ 12207 .shutdown = i40e_shutdown, 12208 .err_handler = &i40e_err_handler, 12209 .sriov_configure = i40e_pci_sriov_configure, 12210 }; 12211 12212 /** 12213 * i40e_init_module - Driver registration routine 12214 * 12215 * i40e_init_module is the first routine called when the driver is 12216 * loaded. All it does is register with the PCI subsystem. 12217 **/ 12218 static int __init i40e_init_module(void) 12219 { 12220 pr_info("%s: %s - version %s\n", i40e_driver_name, 12221 i40e_driver_string, i40e_driver_version_str); 12222 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 12223 12224 /* There is no need to throttle the number of active tasks because 12225 * each device limits its own task using a state bit for scheduling 12226 * the service task, and the device tasks do not interfere with each 12227 * other, so we don't set a max task limit. We must set WQ_MEM_RECLAIM 12228 * since we need to be able to guarantee forward progress even under 12229 * memory pressure. 12230 */ 12231 i40e_wq = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0, i40e_driver_name); 12232 if (!i40e_wq) { 12233 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 12234 return -ENOMEM; 12235 } 12236 12237 i40e_dbg_init(); 12238 return pci_register_driver(&i40e_driver); 12239 } 12240 module_init(i40e_init_module); 12241 12242 /** 12243 * i40e_exit_module - Driver exit cleanup routine 12244 * 12245 * i40e_exit_module is called just before the driver is removed 12246 * from memory. 12247 **/ 12248 static void __exit i40e_exit_module(void) 12249 { 12250 pci_unregister_driver(&i40e_driver); 12251 destroy_workqueue(i40e_wq); 12252 i40e_dbg_exit(); 12253 } 12254 module_exit(i40e_exit_module); 12255