1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2016 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/etherdevice.h> 28 #include <linux/of_net.h> 29 #include <linux/pci.h> 30 31 /* Local includes */ 32 #include "i40e.h" 33 #include "i40e_diag.h" 34 #include <net/udp_tunnel.h> 35 /* All i40e tracepoints are defined by the include below, which 36 * must be included exactly once across the whole kernel with 37 * CREATE_TRACE_POINTS defined 38 */ 39 #define CREATE_TRACE_POINTS 40 #include "i40e_trace.h" 41 42 const char i40e_driver_name[] = "i40e"; 43 static const char i40e_driver_string[] = 44 "Intel(R) Ethernet Connection XL710 Network Driver"; 45 46 #define DRV_KERN "-k" 47 48 #define DRV_VERSION_MAJOR 2 49 #define DRV_VERSION_MINOR 1 50 #define DRV_VERSION_BUILD 14 51 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 52 __stringify(DRV_VERSION_MINOR) "." \ 53 __stringify(DRV_VERSION_BUILD) DRV_KERN 54 const char i40e_driver_version_str[] = DRV_VERSION; 55 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 56 57 /* a bit of forward declarations */ 58 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 59 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired); 60 static int i40e_add_vsi(struct i40e_vsi *vsi); 61 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 62 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 63 static int i40e_setup_misc_vector(struct i40e_pf *pf); 64 static void i40e_determine_queue_usage(struct i40e_pf *pf); 65 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 66 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired); 67 static int i40e_reset(struct i40e_pf *pf); 68 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired); 69 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 70 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 71 72 /* i40e_pci_tbl - PCI Device ID Table 73 * 74 * Last entry must be all 0s 75 * 76 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 77 * Class, Class Mask, private data (not used) } 78 */ 79 static const struct pci_device_id i40e_pci_tbl[] = { 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 90 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 91 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 92 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 93 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 94 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 95 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 96 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 97 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 98 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 99 /* required last entry */ 100 {0, } 101 }; 102 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 103 104 #define I40E_MAX_VF_COUNT 128 105 static int debug = -1; 106 module_param(debug, uint, 0); 107 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 108 109 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 110 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 111 MODULE_LICENSE("GPL"); 112 MODULE_VERSION(DRV_VERSION); 113 114 static struct workqueue_struct *i40e_wq; 115 116 /** 117 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 118 * @hw: pointer to the HW structure 119 * @mem: ptr to mem struct to fill out 120 * @size: size of memory requested 121 * @alignment: what to align the allocation to 122 **/ 123 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 124 u64 size, u32 alignment) 125 { 126 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 127 128 mem->size = ALIGN(size, alignment); 129 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 130 &mem->pa, GFP_KERNEL); 131 if (!mem->va) 132 return -ENOMEM; 133 134 return 0; 135 } 136 137 /** 138 * i40e_free_dma_mem_d - OS specific memory free for shared code 139 * @hw: pointer to the HW structure 140 * @mem: ptr to mem struct to free 141 **/ 142 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 143 { 144 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 145 146 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 147 mem->va = NULL; 148 mem->pa = 0; 149 mem->size = 0; 150 151 return 0; 152 } 153 154 /** 155 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 156 * @hw: pointer to the HW structure 157 * @mem: ptr to mem struct to fill out 158 * @size: size of memory requested 159 **/ 160 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 161 u32 size) 162 { 163 mem->size = size; 164 mem->va = kzalloc(size, GFP_KERNEL); 165 166 if (!mem->va) 167 return -ENOMEM; 168 169 return 0; 170 } 171 172 /** 173 * i40e_free_virt_mem_d - OS specific memory free for shared code 174 * @hw: pointer to the HW structure 175 * @mem: ptr to mem struct to free 176 **/ 177 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 178 { 179 /* it's ok to kfree a NULL pointer */ 180 kfree(mem->va); 181 mem->va = NULL; 182 mem->size = 0; 183 184 return 0; 185 } 186 187 /** 188 * i40e_get_lump - find a lump of free generic resource 189 * @pf: board private structure 190 * @pile: the pile of resource to search 191 * @needed: the number of items needed 192 * @id: an owner id to stick on the items assigned 193 * 194 * Returns the base item index of the lump, or negative for error 195 * 196 * The search_hint trick and lack of advanced fit-finding only work 197 * because we're highly likely to have all the same size lump requests. 198 * Linear search time and any fragmentation should be minimal. 199 **/ 200 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 201 u16 needed, u16 id) 202 { 203 int ret = -ENOMEM; 204 int i, j; 205 206 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 207 dev_info(&pf->pdev->dev, 208 "param err: pile=%p needed=%d id=0x%04x\n", 209 pile, needed, id); 210 return -EINVAL; 211 } 212 213 /* start the linear search with an imperfect hint */ 214 i = pile->search_hint; 215 while (i < pile->num_entries) { 216 /* skip already allocated entries */ 217 if (pile->list[i] & I40E_PILE_VALID_BIT) { 218 i++; 219 continue; 220 } 221 222 /* do we have enough in this lump? */ 223 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 224 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 225 break; 226 } 227 228 if (j == needed) { 229 /* there was enough, so assign it to the requestor */ 230 for (j = 0; j < needed; j++) 231 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 232 ret = i; 233 pile->search_hint = i + j; 234 break; 235 } 236 237 /* not enough, so skip over it and continue looking */ 238 i += j; 239 } 240 241 return ret; 242 } 243 244 /** 245 * i40e_put_lump - return a lump of generic resource 246 * @pile: the pile of resource to search 247 * @index: the base item index 248 * @id: the owner id of the items assigned 249 * 250 * Returns the count of items in the lump 251 **/ 252 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 253 { 254 int valid_id = (id | I40E_PILE_VALID_BIT); 255 int count = 0; 256 int i; 257 258 if (!pile || index >= pile->num_entries) 259 return -EINVAL; 260 261 for (i = index; 262 i < pile->num_entries && pile->list[i] == valid_id; 263 i++) { 264 pile->list[i] = 0; 265 count++; 266 } 267 268 if (count && index < pile->search_hint) 269 pile->search_hint = index; 270 271 return count; 272 } 273 274 /** 275 * i40e_find_vsi_from_id - searches for the vsi with the given id 276 * @pf - the pf structure to search for the vsi 277 * @id - id of the vsi it is searching for 278 **/ 279 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 280 { 281 int i; 282 283 for (i = 0; i < pf->num_alloc_vsi; i++) 284 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 285 return pf->vsi[i]; 286 287 return NULL; 288 } 289 290 /** 291 * i40e_service_event_schedule - Schedule the service task to wake up 292 * @pf: board private structure 293 * 294 * If not already scheduled, this puts the task into the work queue 295 **/ 296 void i40e_service_event_schedule(struct i40e_pf *pf) 297 { 298 if (!test_bit(__I40E_VSI_DOWN, pf->state) && 299 !test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 300 queue_work(i40e_wq, &pf->service_task); 301 } 302 303 /** 304 * i40e_tx_timeout - Respond to a Tx Hang 305 * @netdev: network interface device structure 306 * 307 * If any port has noticed a Tx timeout, it is likely that the whole 308 * device is munged, not just the one netdev port, so go for the full 309 * reset. 310 **/ 311 static void i40e_tx_timeout(struct net_device *netdev) 312 { 313 struct i40e_netdev_priv *np = netdev_priv(netdev); 314 struct i40e_vsi *vsi = np->vsi; 315 struct i40e_pf *pf = vsi->back; 316 struct i40e_ring *tx_ring = NULL; 317 unsigned int i, hung_queue = 0; 318 u32 head, val; 319 320 pf->tx_timeout_count++; 321 322 /* find the stopped queue the same way the stack does */ 323 for (i = 0; i < netdev->num_tx_queues; i++) { 324 struct netdev_queue *q; 325 unsigned long trans_start; 326 327 q = netdev_get_tx_queue(netdev, i); 328 trans_start = q->trans_start; 329 if (netif_xmit_stopped(q) && 330 time_after(jiffies, 331 (trans_start + netdev->watchdog_timeo))) { 332 hung_queue = i; 333 break; 334 } 335 } 336 337 if (i == netdev->num_tx_queues) { 338 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 339 } else { 340 /* now that we have an index, find the tx_ring struct */ 341 for (i = 0; i < vsi->num_queue_pairs; i++) { 342 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 343 if (hung_queue == 344 vsi->tx_rings[i]->queue_index) { 345 tx_ring = vsi->tx_rings[i]; 346 break; 347 } 348 } 349 } 350 } 351 352 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 353 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 354 else if (time_before(jiffies, 355 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 356 return; /* don't do any new action before the next timeout */ 357 358 if (tx_ring) { 359 head = i40e_get_head(tx_ring); 360 /* Read interrupt register */ 361 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 362 val = rd32(&pf->hw, 363 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 364 tx_ring->vsi->base_vector - 1)); 365 else 366 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 367 368 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 369 vsi->seid, hung_queue, tx_ring->next_to_clean, 370 head, tx_ring->next_to_use, 371 readl(tx_ring->tail), val); 372 } 373 374 pf->tx_timeout_last_recovery = jiffies; 375 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 376 pf->tx_timeout_recovery_level, hung_queue); 377 378 switch (pf->tx_timeout_recovery_level) { 379 case 1: 380 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 381 break; 382 case 2: 383 set_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 384 break; 385 case 3: 386 set_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 387 break; 388 default: 389 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 390 break; 391 } 392 393 i40e_service_event_schedule(pf); 394 pf->tx_timeout_recovery_level++; 395 } 396 397 /** 398 * i40e_get_vsi_stats_struct - Get System Network Statistics 399 * @vsi: the VSI we care about 400 * 401 * Returns the address of the device statistics structure. 402 * The statistics are actually updated from the service task. 403 **/ 404 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 405 { 406 return &vsi->net_stats; 407 } 408 409 /** 410 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 411 * @netdev: network interface device structure 412 * 413 * Returns the address of the device statistics structure. 414 * The statistics are actually updated from the service task. 415 **/ 416 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 417 struct rtnl_link_stats64 *stats) 418 { 419 struct i40e_netdev_priv *np = netdev_priv(netdev); 420 struct i40e_ring *tx_ring, *rx_ring; 421 struct i40e_vsi *vsi = np->vsi; 422 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 423 int i; 424 425 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 426 return; 427 428 if (!vsi->tx_rings) 429 return; 430 431 rcu_read_lock(); 432 for (i = 0; i < vsi->num_queue_pairs; i++) { 433 u64 bytes, packets; 434 unsigned int start; 435 436 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]); 437 if (!tx_ring) 438 continue; 439 440 do { 441 start = u64_stats_fetch_begin_irq(&tx_ring->syncp); 442 packets = tx_ring->stats.packets; 443 bytes = tx_ring->stats.bytes; 444 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); 445 446 stats->tx_packets += packets; 447 stats->tx_bytes += bytes; 448 rx_ring = &tx_ring[1]; 449 450 do { 451 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 452 packets = rx_ring->stats.packets; 453 bytes = rx_ring->stats.bytes; 454 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 455 456 stats->rx_packets += packets; 457 stats->rx_bytes += bytes; 458 } 459 rcu_read_unlock(); 460 461 /* following stats updated by i40e_watchdog_subtask() */ 462 stats->multicast = vsi_stats->multicast; 463 stats->tx_errors = vsi_stats->tx_errors; 464 stats->tx_dropped = vsi_stats->tx_dropped; 465 stats->rx_errors = vsi_stats->rx_errors; 466 stats->rx_dropped = vsi_stats->rx_dropped; 467 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 468 stats->rx_length_errors = vsi_stats->rx_length_errors; 469 } 470 471 /** 472 * i40e_vsi_reset_stats - Resets all stats of the given vsi 473 * @vsi: the VSI to have its stats reset 474 **/ 475 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 476 { 477 struct rtnl_link_stats64 *ns; 478 int i; 479 480 if (!vsi) 481 return; 482 483 ns = i40e_get_vsi_stats_struct(vsi); 484 memset(ns, 0, sizeof(*ns)); 485 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 486 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 487 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 488 if (vsi->rx_rings && vsi->rx_rings[0]) { 489 for (i = 0; i < vsi->num_queue_pairs; i++) { 490 memset(&vsi->rx_rings[i]->stats, 0, 491 sizeof(vsi->rx_rings[i]->stats)); 492 memset(&vsi->rx_rings[i]->rx_stats, 0, 493 sizeof(vsi->rx_rings[i]->rx_stats)); 494 memset(&vsi->tx_rings[i]->stats, 0, 495 sizeof(vsi->tx_rings[i]->stats)); 496 memset(&vsi->tx_rings[i]->tx_stats, 0, 497 sizeof(vsi->tx_rings[i]->tx_stats)); 498 } 499 } 500 vsi->stat_offsets_loaded = false; 501 } 502 503 /** 504 * i40e_pf_reset_stats - Reset all of the stats for the given PF 505 * @pf: the PF to be reset 506 **/ 507 void i40e_pf_reset_stats(struct i40e_pf *pf) 508 { 509 int i; 510 511 memset(&pf->stats, 0, sizeof(pf->stats)); 512 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 513 pf->stat_offsets_loaded = false; 514 515 for (i = 0; i < I40E_MAX_VEB; i++) { 516 if (pf->veb[i]) { 517 memset(&pf->veb[i]->stats, 0, 518 sizeof(pf->veb[i]->stats)); 519 memset(&pf->veb[i]->stats_offsets, 0, 520 sizeof(pf->veb[i]->stats_offsets)); 521 pf->veb[i]->stat_offsets_loaded = false; 522 } 523 } 524 pf->hw_csum_rx_error = 0; 525 } 526 527 /** 528 * i40e_stat_update48 - read and update a 48 bit stat from the chip 529 * @hw: ptr to the hardware info 530 * @hireg: the high 32 bit reg to read 531 * @loreg: the low 32 bit reg to read 532 * @offset_loaded: has the initial offset been loaded yet 533 * @offset: ptr to current offset value 534 * @stat: ptr to the stat 535 * 536 * Since the device stats are not reset at PFReset, they likely will not 537 * be zeroed when the driver starts. We'll save the first values read 538 * and use them as offsets to be subtracted from the raw values in order 539 * to report stats that count from zero. In the process, we also manage 540 * the potential roll-over. 541 **/ 542 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 543 bool offset_loaded, u64 *offset, u64 *stat) 544 { 545 u64 new_data; 546 547 if (hw->device_id == I40E_DEV_ID_QEMU) { 548 new_data = rd32(hw, loreg); 549 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 550 } else { 551 new_data = rd64(hw, loreg); 552 } 553 if (!offset_loaded) 554 *offset = new_data; 555 if (likely(new_data >= *offset)) 556 *stat = new_data - *offset; 557 else 558 *stat = (new_data + BIT_ULL(48)) - *offset; 559 *stat &= 0xFFFFFFFFFFFFULL; 560 } 561 562 /** 563 * i40e_stat_update32 - read and update a 32 bit stat from the chip 564 * @hw: ptr to the hardware info 565 * @reg: the hw reg to read 566 * @offset_loaded: has the initial offset been loaded yet 567 * @offset: ptr to current offset value 568 * @stat: ptr to the stat 569 **/ 570 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 571 bool offset_loaded, u64 *offset, u64 *stat) 572 { 573 u32 new_data; 574 575 new_data = rd32(hw, reg); 576 if (!offset_loaded) 577 *offset = new_data; 578 if (likely(new_data >= *offset)) 579 *stat = (u32)(new_data - *offset); 580 else 581 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 582 } 583 584 /** 585 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 586 * @vsi: the VSI to be updated 587 **/ 588 void i40e_update_eth_stats(struct i40e_vsi *vsi) 589 { 590 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 591 struct i40e_pf *pf = vsi->back; 592 struct i40e_hw *hw = &pf->hw; 593 struct i40e_eth_stats *oes; 594 struct i40e_eth_stats *es; /* device's eth stats */ 595 596 es = &vsi->eth_stats; 597 oes = &vsi->eth_stats_offsets; 598 599 /* Gather up the stats that the hw collects */ 600 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 601 vsi->stat_offsets_loaded, 602 &oes->tx_errors, &es->tx_errors); 603 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 604 vsi->stat_offsets_loaded, 605 &oes->rx_discards, &es->rx_discards); 606 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 607 vsi->stat_offsets_loaded, 608 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 609 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 610 vsi->stat_offsets_loaded, 611 &oes->tx_errors, &es->tx_errors); 612 613 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 614 I40E_GLV_GORCL(stat_idx), 615 vsi->stat_offsets_loaded, 616 &oes->rx_bytes, &es->rx_bytes); 617 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 618 I40E_GLV_UPRCL(stat_idx), 619 vsi->stat_offsets_loaded, 620 &oes->rx_unicast, &es->rx_unicast); 621 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 622 I40E_GLV_MPRCL(stat_idx), 623 vsi->stat_offsets_loaded, 624 &oes->rx_multicast, &es->rx_multicast); 625 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 626 I40E_GLV_BPRCL(stat_idx), 627 vsi->stat_offsets_loaded, 628 &oes->rx_broadcast, &es->rx_broadcast); 629 630 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 631 I40E_GLV_GOTCL(stat_idx), 632 vsi->stat_offsets_loaded, 633 &oes->tx_bytes, &es->tx_bytes); 634 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 635 I40E_GLV_UPTCL(stat_idx), 636 vsi->stat_offsets_loaded, 637 &oes->tx_unicast, &es->tx_unicast); 638 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 639 I40E_GLV_MPTCL(stat_idx), 640 vsi->stat_offsets_loaded, 641 &oes->tx_multicast, &es->tx_multicast); 642 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 643 I40E_GLV_BPTCL(stat_idx), 644 vsi->stat_offsets_loaded, 645 &oes->tx_broadcast, &es->tx_broadcast); 646 vsi->stat_offsets_loaded = true; 647 } 648 649 /** 650 * i40e_update_veb_stats - Update Switch component statistics 651 * @veb: the VEB being updated 652 **/ 653 static void i40e_update_veb_stats(struct i40e_veb *veb) 654 { 655 struct i40e_pf *pf = veb->pf; 656 struct i40e_hw *hw = &pf->hw; 657 struct i40e_eth_stats *oes; 658 struct i40e_eth_stats *es; /* device's eth stats */ 659 struct i40e_veb_tc_stats *veb_oes; 660 struct i40e_veb_tc_stats *veb_es; 661 int i, idx = 0; 662 663 idx = veb->stats_idx; 664 es = &veb->stats; 665 oes = &veb->stats_offsets; 666 veb_es = &veb->tc_stats; 667 veb_oes = &veb->tc_stats_offsets; 668 669 /* Gather up the stats that the hw collects */ 670 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 671 veb->stat_offsets_loaded, 672 &oes->tx_discards, &es->tx_discards); 673 if (hw->revision_id > 0) 674 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 675 veb->stat_offsets_loaded, 676 &oes->rx_unknown_protocol, 677 &es->rx_unknown_protocol); 678 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 679 veb->stat_offsets_loaded, 680 &oes->rx_bytes, &es->rx_bytes); 681 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 682 veb->stat_offsets_loaded, 683 &oes->rx_unicast, &es->rx_unicast); 684 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 685 veb->stat_offsets_loaded, 686 &oes->rx_multicast, &es->rx_multicast); 687 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 688 veb->stat_offsets_loaded, 689 &oes->rx_broadcast, &es->rx_broadcast); 690 691 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 692 veb->stat_offsets_loaded, 693 &oes->tx_bytes, &es->tx_bytes); 694 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 695 veb->stat_offsets_loaded, 696 &oes->tx_unicast, &es->tx_unicast); 697 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 698 veb->stat_offsets_loaded, 699 &oes->tx_multicast, &es->tx_multicast); 700 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 701 veb->stat_offsets_loaded, 702 &oes->tx_broadcast, &es->tx_broadcast); 703 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 704 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 705 I40E_GLVEBTC_RPCL(i, idx), 706 veb->stat_offsets_loaded, 707 &veb_oes->tc_rx_packets[i], 708 &veb_es->tc_rx_packets[i]); 709 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 710 I40E_GLVEBTC_RBCL(i, idx), 711 veb->stat_offsets_loaded, 712 &veb_oes->tc_rx_bytes[i], 713 &veb_es->tc_rx_bytes[i]); 714 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 715 I40E_GLVEBTC_TPCL(i, idx), 716 veb->stat_offsets_loaded, 717 &veb_oes->tc_tx_packets[i], 718 &veb_es->tc_tx_packets[i]); 719 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 720 I40E_GLVEBTC_TBCL(i, idx), 721 veb->stat_offsets_loaded, 722 &veb_oes->tc_tx_bytes[i], 723 &veb_es->tc_tx_bytes[i]); 724 } 725 veb->stat_offsets_loaded = true; 726 } 727 728 /** 729 * i40e_update_vsi_stats - Update the vsi statistics counters. 730 * @vsi: the VSI to be updated 731 * 732 * There are a few instances where we store the same stat in a 733 * couple of different structs. This is partly because we have 734 * the netdev stats that need to be filled out, which is slightly 735 * different from the "eth_stats" defined by the chip and used in 736 * VF communications. We sort it out here. 737 **/ 738 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 739 { 740 struct i40e_pf *pf = vsi->back; 741 struct rtnl_link_stats64 *ons; 742 struct rtnl_link_stats64 *ns; /* netdev stats */ 743 struct i40e_eth_stats *oes; 744 struct i40e_eth_stats *es; /* device's eth stats */ 745 u32 tx_restart, tx_busy; 746 struct i40e_ring *p; 747 u32 rx_page, rx_buf; 748 u64 bytes, packets; 749 unsigned int start; 750 u64 tx_linearize; 751 u64 tx_force_wb; 752 u64 rx_p, rx_b; 753 u64 tx_p, tx_b; 754 u16 q; 755 756 if (test_bit(__I40E_VSI_DOWN, vsi->state) || 757 test_bit(__I40E_CONFIG_BUSY, pf->state)) 758 return; 759 760 ns = i40e_get_vsi_stats_struct(vsi); 761 ons = &vsi->net_stats_offsets; 762 es = &vsi->eth_stats; 763 oes = &vsi->eth_stats_offsets; 764 765 /* Gather up the netdev and vsi stats that the driver collects 766 * on the fly during packet processing 767 */ 768 rx_b = rx_p = 0; 769 tx_b = tx_p = 0; 770 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 771 rx_page = 0; 772 rx_buf = 0; 773 rcu_read_lock(); 774 for (q = 0; q < vsi->num_queue_pairs; q++) { 775 /* locate Tx ring */ 776 p = ACCESS_ONCE(vsi->tx_rings[q]); 777 778 do { 779 start = u64_stats_fetch_begin_irq(&p->syncp); 780 packets = p->stats.packets; 781 bytes = p->stats.bytes; 782 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 783 tx_b += bytes; 784 tx_p += packets; 785 tx_restart += p->tx_stats.restart_queue; 786 tx_busy += p->tx_stats.tx_busy; 787 tx_linearize += p->tx_stats.tx_linearize; 788 tx_force_wb += p->tx_stats.tx_force_wb; 789 790 /* Rx queue is part of the same block as Tx queue */ 791 p = &p[1]; 792 do { 793 start = u64_stats_fetch_begin_irq(&p->syncp); 794 packets = p->stats.packets; 795 bytes = p->stats.bytes; 796 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 797 rx_b += bytes; 798 rx_p += packets; 799 rx_buf += p->rx_stats.alloc_buff_failed; 800 rx_page += p->rx_stats.alloc_page_failed; 801 } 802 rcu_read_unlock(); 803 vsi->tx_restart = tx_restart; 804 vsi->tx_busy = tx_busy; 805 vsi->tx_linearize = tx_linearize; 806 vsi->tx_force_wb = tx_force_wb; 807 vsi->rx_page_failed = rx_page; 808 vsi->rx_buf_failed = rx_buf; 809 810 ns->rx_packets = rx_p; 811 ns->rx_bytes = rx_b; 812 ns->tx_packets = tx_p; 813 ns->tx_bytes = tx_b; 814 815 /* update netdev stats from eth stats */ 816 i40e_update_eth_stats(vsi); 817 ons->tx_errors = oes->tx_errors; 818 ns->tx_errors = es->tx_errors; 819 ons->multicast = oes->rx_multicast; 820 ns->multicast = es->rx_multicast; 821 ons->rx_dropped = oes->rx_discards; 822 ns->rx_dropped = es->rx_discards; 823 ons->tx_dropped = oes->tx_discards; 824 ns->tx_dropped = es->tx_discards; 825 826 /* pull in a couple PF stats if this is the main vsi */ 827 if (vsi == pf->vsi[pf->lan_vsi]) { 828 ns->rx_crc_errors = pf->stats.crc_errors; 829 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 830 ns->rx_length_errors = pf->stats.rx_length_errors; 831 } 832 } 833 834 /** 835 * i40e_update_pf_stats - Update the PF statistics counters. 836 * @pf: the PF to be updated 837 **/ 838 static void i40e_update_pf_stats(struct i40e_pf *pf) 839 { 840 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 841 struct i40e_hw_port_stats *nsd = &pf->stats; 842 struct i40e_hw *hw = &pf->hw; 843 u32 val; 844 int i; 845 846 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 847 I40E_GLPRT_GORCL(hw->port), 848 pf->stat_offsets_loaded, 849 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 850 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 851 I40E_GLPRT_GOTCL(hw->port), 852 pf->stat_offsets_loaded, 853 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 854 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 855 pf->stat_offsets_loaded, 856 &osd->eth.rx_discards, 857 &nsd->eth.rx_discards); 858 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 859 I40E_GLPRT_UPRCL(hw->port), 860 pf->stat_offsets_loaded, 861 &osd->eth.rx_unicast, 862 &nsd->eth.rx_unicast); 863 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 864 I40E_GLPRT_MPRCL(hw->port), 865 pf->stat_offsets_loaded, 866 &osd->eth.rx_multicast, 867 &nsd->eth.rx_multicast); 868 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 869 I40E_GLPRT_BPRCL(hw->port), 870 pf->stat_offsets_loaded, 871 &osd->eth.rx_broadcast, 872 &nsd->eth.rx_broadcast); 873 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 874 I40E_GLPRT_UPTCL(hw->port), 875 pf->stat_offsets_loaded, 876 &osd->eth.tx_unicast, 877 &nsd->eth.tx_unicast); 878 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 879 I40E_GLPRT_MPTCL(hw->port), 880 pf->stat_offsets_loaded, 881 &osd->eth.tx_multicast, 882 &nsd->eth.tx_multicast); 883 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 884 I40E_GLPRT_BPTCL(hw->port), 885 pf->stat_offsets_loaded, 886 &osd->eth.tx_broadcast, 887 &nsd->eth.tx_broadcast); 888 889 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 890 pf->stat_offsets_loaded, 891 &osd->tx_dropped_link_down, 892 &nsd->tx_dropped_link_down); 893 894 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 895 pf->stat_offsets_loaded, 896 &osd->crc_errors, &nsd->crc_errors); 897 898 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 899 pf->stat_offsets_loaded, 900 &osd->illegal_bytes, &nsd->illegal_bytes); 901 902 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 903 pf->stat_offsets_loaded, 904 &osd->mac_local_faults, 905 &nsd->mac_local_faults); 906 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 907 pf->stat_offsets_loaded, 908 &osd->mac_remote_faults, 909 &nsd->mac_remote_faults); 910 911 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->rx_length_errors, 914 &nsd->rx_length_errors); 915 916 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 917 pf->stat_offsets_loaded, 918 &osd->link_xon_rx, &nsd->link_xon_rx); 919 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 920 pf->stat_offsets_loaded, 921 &osd->link_xon_tx, &nsd->link_xon_tx); 922 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 923 pf->stat_offsets_loaded, 924 &osd->link_xoff_rx, &nsd->link_xoff_rx); 925 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 926 pf->stat_offsets_loaded, 927 &osd->link_xoff_tx, &nsd->link_xoff_tx); 928 929 for (i = 0; i < 8; i++) { 930 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 931 pf->stat_offsets_loaded, 932 &osd->priority_xoff_rx[i], 933 &nsd->priority_xoff_rx[i]); 934 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 935 pf->stat_offsets_loaded, 936 &osd->priority_xon_rx[i], 937 &nsd->priority_xon_rx[i]); 938 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 939 pf->stat_offsets_loaded, 940 &osd->priority_xon_tx[i], 941 &nsd->priority_xon_tx[i]); 942 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 943 pf->stat_offsets_loaded, 944 &osd->priority_xoff_tx[i], 945 &nsd->priority_xoff_tx[i]); 946 i40e_stat_update32(hw, 947 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 948 pf->stat_offsets_loaded, 949 &osd->priority_xon_2_xoff[i], 950 &nsd->priority_xon_2_xoff[i]); 951 } 952 953 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 954 I40E_GLPRT_PRC64L(hw->port), 955 pf->stat_offsets_loaded, 956 &osd->rx_size_64, &nsd->rx_size_64); 957 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 958 I40E_GLPRT_PRC127L(hw->port), 959 pf->stat_offsets_loaded, 960 &osd->rx_size_127, &nsd->rx_size_127); 961 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 962 I40E_GLPRT_PRC255L(hw->port), 963 pf->stat_offsets_loaded, 964 &osd->rx_size_255, &nsd->rx_size_255); 965 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 966 I40E_GLPRT_PRC511L(hw->port), 967 pf->stat_offsets_loaded, 968 &osd->rx_size_511, &nsd->rx_size_511); 969 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 970 I40E_GLPRT_PRC1023L(hw->port), 971 pf->stat_offsets_loaded, 972 &osd->rx_size_1023, &nsd->rx_size_1023); 973 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 974 I40E_GLPRT_PRC1522L(hw->port), 975 pf->stat_offsets_loaded, 976 &osd->rx_size_1522, &nsd->rx_size_1522); 977 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 978 I40E_GLPRT_PRC9522L(hw->port), 979 pf->stat_offsets_loaded, 980 &osd->rx_size_big, &nsd->rx_size_big); 981 982 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 983 I40E_GLPRT_PTC64L(hw->port), 984 pf->stat_offsets_loaded, 985 &osd->tx_size_64, &nsd->tx_size_64); 986 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 987 I40E_GLPRT_PTC127L(hw->port), 988 pf->stat_offsets_loaded, 989 &osd->tx_size_127, &nsd->tx_size_127); 990 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 991 I40E_GLPRT_PTC255L(hw->port), 992 pf->stat_offsets_loaded, 993 &osd->tx_size_255, &nsd->tx_size_255); 994 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 995 I40E_GLPRT_PTC511L(hw->port), 996 pf->stat_offsets_loaded, 997 &osd->tx_size_511, &nsd->tx_size_511); 998 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 999 I40E_GLPRT_PTC1023L(hw->port), 1000 pf->stat_offsets_loaded, 1001 &osd->tx_size_1023, &nsd->tx_size_1023); 1002 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 1003 I40E_GLPRT_PTC1522L(hw->port), 1004 pf->stat_offsets_loaded, 1005 &osd->tx_size_1522, &nsd->tx_size_1522); 1006 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1007 I40E_GLPRT_PTC9522L(hw->port), 1008 pf->stat_offsets_loaded, 1009 &osd->tx_size_big, &nsd->tx_size_big); 1010 1011 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1012 pf->stat_offsets_loaded, 1013 &osd->rx_undersize, &nsd->rx_undersize); 1014 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1015 pf->stat_offsets_loaded, 1016 &osd->rx_fragments, &nsd->rx_fragments); 1017 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1018 pf->stat_offsets_loaded, 1019 &osd->rx_oversize, &nsd->rx_oversize); 1020 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1021 pf->stat_offsets_loaded, 1022 &osd->rx_jabber, &nsd->rx_jabber); 1023 1024 /* FDIR stats */ 1025 i40e_stat_update32(hw, 1026 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)), 1027 pf->stat_offsets_loaded, 1028 &osd->fd_atr_match, &nsd->fd_atr_match); 1029 i40e_stat_update32(hw, 1030 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)), 1031 pf->stat_offsets_loaded, 1032 &osd->fd_sb_match, &nsd->fd_sb_match); 1033 i40e_stat_update32(hw, 1034 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)), 1035 pf->stat_offsets_loaded, 1036 &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match); 1037 1038 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1039 nsd->tx_lpi_status = 1040 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1041 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1042 nsd->rx_lpi_status = 1043 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1044 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1045 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1046 pf->stat_offsets_loaded, 1047 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1048 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1049 pf->stat_offsets_loaded, 1050 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1051 1052 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1053 !(pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED)) 1054 nsd->fd_sb_status = true; 1055 else 1056 nsd->fd_sb_status = false; 1057 1058 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1059 !(pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED)) 1060 nsd->fd_atr_status = true; 1061 else 1062 nsd->fd_atr_status = false; 1063 1064 pf->stat_offsets_loaded = true; 1065 } 1066 1067 /** 1068 * i40e_update_stats - Update the various statistics counters. 1069 * @vsi: the VSI to be updated 1070 * 1071 * Update the various stats for this VSI and its related entities. 1072 **/ 1073 void i40e_update_stats(struct i40e_vsi *vsi) 1074 { 1075 struct i40e_pf *pf = vsi->back; 1076 1077 if (vsi == pf->vsi[pf->lan_vsi]) 1078 i40e_update_pf_stats(pf); 1079 1080 i40e_update_vsi_stats(vsi); 1081 } 1082 1083 /** 1084 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1085 * @vsi: the VSI to be searched 1086 * @macaddr: the MAC address 1087 * @vlan: the vlan 1088 * 1089 * Returns ptr to the filter object or NULL 1090 **/ 1091 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1092 const u8 *macaddr, s16 vlan) 1093 { 1094 struct i40e_mac_filter *f; 1095 u64 key; 1096 1097 if (!vsi || !macaddr) 1098 return NULL; 1099 1100 key = i40e_addr_to_hkey(macaddr); 1101 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1102 if ((ether_addr_equal(macaddr, f->macaddr)) && 1103 (vlan == f->vlan)) 1104 return f; 1105 } 1106 return NULL; 1107 } 1108 1109 /** 1110 * i40e_find_mac - Find a mac addr in the macvlan filters list 1111 * @vsi: the VSI to be searched 1112 * @macaddr: the MAC address we are searching for 1113 * 1114 * Returns the first filter with the provided MAC address or NULL if 1115 * MAC address was not found 1116 **/ 1117 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1118 { 1119 struct i40e_mac_filter *f; 1120 u64 key; 1121 1122 if (!vsi || !macaddr) 1123 return NULL; 1124 1125 key = i40e_addr_to_hkey(macaddr); 1126 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1127 if ((ether_addr_equal(macaddr, f->macaddr))) 1128 return f; 1129 } 1130 return NULL; 1131 } 1132 1133 /** 1134 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1135 * @vsi: the VSI to be searched 1136 * 1137 * Returns true if VSI is in vlan mode or false otherwise 1138 **/ 1139 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1140 { 1141 /* If we have a PVID, always operate in VLAN mode */ 1142 if (vsi->info.pvid) 1143 return true; 1144 1145 /* We need to operate in VLAN mode whenever we have any filters with 1146 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1147 * time, incurring search cost repeatedly. However, we can notice two 1148 * things: 1149 * 1150 * 1) the only place where we can gain a VLAN filter is in 1151 * i40e_add_filter. 1152 * 1153 * 2) the only place where filters are actually removed is in 1154 * i40e_sync_filters_subtask. 1155 * 1156 * Thus, we can simply use a boolean value, has_vlan_filters which we 1157 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1158 * we have to perform the full search after deleting filters in 1159 * i40e_sync_filters_subtask, but we already have to search 1160 * filters here and can perform the check at the same time. This 1161 * results in avoiding embedding a loop for VLAN mode inside another 1162 * loop over all the filters, and should maintain correctness as noted 1163 * above. 1164 */ 1165 return vsi->has_vlan_filter; 1166 } 1167 1168 /** 1169 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1170 * @vsi: the VSI to configure 1171 * @tmp_add_list: list of filters ready to be added 1172 * @tmp_del_list: list of filters ready to be deleted 1173 * @vlan_filters: the number of active VLAN filters 1174 * 1175 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1176 * behave as expected. If we have any active VLAN filters remaining or about 1177 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1178 * so that they only match against untagged traffic. If we no longer have any 1179 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1180 * so that they match against both tagged and untagged traffic. In this way, 1181 * we ensure that we correctly receive the desired traffic. This ensures that 1182 * when we have an active VLAN we will receive only untagged traffic and 1183 * traffic matching active VLANs. If we have no active VLANs then we will 1184 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1185 * 1186 * Finally, in a similar fashion, this function also corrects filters when 1187 * there is an active PVID assigned to this VSI. 1188 * 1189 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1190 * 1191 * This function is only expected to be called from within 1192 * i40e_sync_vsi_filters. 1193 * 1194 * NOTE: This function expects to be called while under the 1195 * mac_filter_hash_lock 1196 */ 1197 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1198 struct hlist_head *tmp_add_list, 1199 struct hlist_head *tmp_del_list, 1200 int vlan_filters) 1201 { 1202 s16 pvid = le16_to_cpu(vsi->info.pvid); 1203 struct i40e_mac_filter *f, *add_head; 1204 struct i40e_new_mac_filter *new; 1205 struct hlist_node *h; 1206 int bkt, new_vlan; 1207 1208 /* To determine if a particular filter needs to be replaced we 1209 * have the three following conditions: 1210 * 1211 * a) if we have a PVID assigned, then all filters which are 1212 * not marked as VLAN=PVID must be replaced with filters that 1213 * are. 1214 * b) otherwise, if we have any active VLANS, all filters 1215 * which are marked as VLAN=-1 must be replaced with 1216 * filters marked as VLAN=0 1217 * c) finally, if we do not have any active VLANS, all filters 1218 * which are marked as VLAN=0 must be replaced with filters 1219 * marked as VLAN=-1 1220 */ 1221 1222 /* Update the filters about to be added in place */ 1223 hlist_for_each_entry(new, tmp_add_list, hlist) { 1224 if (pvid && new->f->vlan != pvid) 1225 new->f->vlan = pvid; 1226 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1227 new->f->vlan = 0; 1228 else if (!vlan_filters && new->f->vlan == 0) 1229 new->f->vlan = I40E_VLAN_ANY; 1230 } 1231 1232 /* Update the remaining active filters */ 1233 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1234 /* Combine the checks for whether a filter needs to be changed 1235 * and then determine the new VLAN inside the if block, in 1236 * order to avoid duplicating code for adding the new filter 1237 * then deleting the old filter. 1238 */ 1239 if ((pvid && f->vlan != pvid) || 1240 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1241 (!vlan_filters && f->vlan == 0)) { 1242 /* Determine the new vlan we will be adding */ 1243 if (pvid) 1244 new_vlan = pvid; 1245 else if (vlan_filters) 1246 new_vlan = 0; 1247 else 1248 new_vlan = I40E_VLAN_ANY; 1249 1250 /* Create the new filter */ 1251 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1252 if (!add_head) 1253 return -ENOMEM; 1254 1255 /* Create a temporary i40e_new_mac_filter */ 1256 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1257 if (!new) 1258 return -ENOMEM; 1259 1260 new->f = add_head; 1261 new->state = add_head->state; 1262 1263 /* Add the new filter to the tmp list */ 1264 hlist_add_head(&new->hlist, tmp_add_list); 1265 1266 /* Put the original filter into the delete list */ 1267 f->state = I40E_FILTER_REMOVE; 1268 hash_del(&f->hlist); 1269 hlist_add_head(&f->hlist, tmp_del_list); 1270 } 1271 } 1272 1273 vsi->has_vlan_filter = !!vlan_filters; 1274 1275 return 0; 1276 } 1277 1278 /** 1279 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1280 * @vsi: the PF Main VSI - inappropriate for any other VSI 1281 * @macaddr: the MAC address 1282 * 1283 * Remove whatever filter the firmware set up so the driver can manage 1284 * its own filtering intelligently. 1285 **/ 1286 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1287 { 1288 struct i40e_aqc_remove_macvlan_element_data element; 1289 struct i40e_pf *pf = vsi->back; 1290 1291 /* Only appropriate for the PF main VSI */ 1292 if (vsi->type != I40E_VSI_MAIN) 1293 return; 1294 1295 memset(&element, 0, sizeof(element)); 1296 ether_addr_copy(element.mac_addr, macaddr); 1297 element.vlan_tag = 0; 1298 /* Ignore error returns, some firmware does it this way... */ 1299 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1300 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1301 1302 memset(&element, 0, sizeof(element)); 1303 ether_addr_copy(element.mac_addr, macaddr); 1304 element.vlan_tag = 0; 1305 /* ...and some firmware does it this way. */ 1306 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1307 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1308 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1309 } 1310 1311 /** 1312 * i40e_add_filter - Add a mac/vlan filter to the VSI 1313 * @vsi: the VSI to be searched 1314 * @macaddr: the MAC address 1315 * @vlan: the vlan 1316 * 1317 * Returns ptr to the filter object or NULL when no memory available. 1318 * 1319 * NOTE: This function is expected to be called with mac_filter_hash_lock 1320 * being held. 1321 **/ 1322 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1323 const u8 *macaddr, s16 vlan) 1324 { 1325 struct i40e_mac_filter *f; 1326 u64 key; 1327 1328 if (!vsi || !macaddr) 1329 return NULL; 1330 1331 f = i40e_find_filter(vsi, macaddr, vlan); 1332 if (!f) { 1333 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1334 if (!f) 1335 return NULL; 1336 1337 /* Update the boolean indicating if we need to function in 1338 * VLAN mode. 1339 */ 1340 if (vlan >= 0) 1341 vsi->has_vlan_filter = true; 1342 1343 ether_addr_copy(f->macaddr, macaddr); 1344 f->vlan = vlan; 1345 /* If we're in overflow promisc mode, set the state directly 1346 * to failed, so we don't bother to try sending the filter 1347 * to the hardware. 1348 */ 1349 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state)) 1350 f->state = I40E_FILTER_FAILED; 1351 else 1352 f->state = I40E_FILTER_NEW; 1353 INIT_HLIST_NODE(&f->hlist); 1354 1355 key = i40e_addr_to_hkey(macaddr); 1356 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1357 1358 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1359 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1360 } 1361 1362 /* If we're asked to add a filter that has been marked for removal, it 1363 * is safe to simply restore it to active state. __i40e_del_filter 1364 * will have simply deleted any filters which were previously marked 1365 * NEW or FAILED, so if it is currently marked REMOVE it must have 1366 * previously been ACTIVE. Since we haven't yet run the sync filters 1367 * task, just restore this filter to the ACTIVE state so that the 1368 * sync task leaves it in place 1369 */ 1370 if (f->state == I40E_FILTER_REMOVE) 1371 f->state = I40E_FILTER_ACTIVE; 1372 1373 return f; 1374 } 1375 1376 /** 1377 * __i40e_del_filter - Remove a specific filter from the VSI 1378 * @vsi: VSI to remove from 1379 * @f: the filter to remove from the list 1380 * 1381 * This function should be called instead of i40e_del_filter only if you know 1382 * the exact filter you will remove already, such as via i40e_find_filter or 1383 * i40e_find_mac. 1384 * 1385 * NOTE: This function is expected to be called with mac_filter_hash_lock 1386 * being held. 1387 * ANOTHER NOTE: This function MUST be called from within the context of 1388 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1389 * instead of list_for_each_entry(). 1390 **/ 1391 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1392 { 1393 if (!f) 1394 return; 1395 1396 /* If the filter was never added to firmware then we can just delete it 1397 * directly and we don't want to set the status to remove or else an 1398 * admin queue command will unnecessarily fire. 1399 */ 1400 if ((f->state == I40E_FILTER_FAILED) || 1401 (f->state == I40E_FILTER_NEW)) { 1402 hash_del(&f->hlist); 1403 kfree(f); 1404 } else { 1405 f->state = I40E_FILTER_REMOVE; 1406 } 1407 1408 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1409 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1410 } 1411 1412 /** 1413 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1414 * @vsi: the VSI to be searched 1415 * @macaddr: the MAC address 1416 * @vlan: the VLAN 1417 * 1418 * NOTE: This function is expected to be called with mac_filter_hash_lock 1419 * being held. 1420 * ANOTHER NOTE: This function MUST be called from within the context of 1421 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1422 * instead of list_for_each_entry(). 1423 **/ 1424 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1425 { 1426 struct i40e_mac_filter *f; 1427 1428 if (!vsi || !macaddr) 1429 return; 1430 1431 f = i40e_find_filter(vsi, macaddr, vlan); 1432 __i40e_del_filter(vsi, f); 1433 } 1434 1435 /** 1436 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1437 * @vsi: the VSI to be searched 1438 * @macaddr: the mac address to be filtered 1439 * 1440 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1441 * go through all the macvlan filters and add a macvlan filter for each 1442 * unique vlan that already exists. If a PVID has been assigned, instead only 1443 * add the macaddr to that VLAN. 1444 * 1445 * Returns last filter added on success, else NULL 1446 **/ 1447 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1448 const u8 *macaddr) 1449 { 1450 struct i40e_mac_filter *f, *add = NULL; 1451 struct hlist_node *h; 1452 int bkt; 1453 1454 if (vsi->info.pvid) 1455 return i40e_add_filter(vsi, macaddr, 1456 le16_to_cpu(vsi->info.pvid)); 1457 1458 if (!i40e_is_vsi_in_vlan(vsi)) 1459 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1460 1461 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1462 if (f->state == I40E_FILTER_REMOVE) 1463 continue; 1464 add = i40e_add_filter(vsi, macaddr, f->vlan); 1465 if (!add) 1466 return NULL; 1467 } 1468 1469 return add; 1470 } 1471 1472 /** 1473 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1474 * @vsi: the VSI to be searched 1475 * @macaddr: the mac address to be removed 1476 * 1477 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1478 * associated with. 1479 * 1480 * Returns 0 for success, or error 1481 **/ 1482 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1483 { 1484 struct i40e_mac_filter *f; 1485 struct hlist_node *h; 1486 bool found = false; 1487 int bkt; 1488 1489 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1490 "Missing mac_filter_hash_lock\n"); 1491 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1492 if (ether_addr_equal(macaddr, f->macaddr)) { 1493 __i40e_del_filter(vsi, f); 1494 found = true; 1495 } 1496 } 1497 1498 if (found) 1499 return 0; 1500 else 1501 return -ENOENT; 1502 } 1503 1504 /** 1505 * i40e_set_mac - NDO callback to set mac address 1506 * @netdev: network interface device structure 1507 * @p: pointer to an address structure 1508 * 1509 * Returns 0 on success, negative on failure 1510 **/ 1511 static int i40e_set_mac(struct net_device *netdev, void *p) 1512 { 1513 struct i40e_netdev_priv *np = netdev_priv(netdev); 1514 struct i40e_vsi *vsi = np->vsi; 1515 struct i40e_pf *pf = vsi->back; 1516 struct i40e_hw *hw = &pf->hw; 1517 struct sockaddr *addr = p; 1518 1519 if (!is_valid_ether_addr(addr->sa_data)) 1520 return -EADDRNOTAVAIL; 1521 1522 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1523 netdev_info(netdev, "already using mac address %pM\n", 1524 addr->sa_data); 1525 return 0; 1526 } 1527 1528 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 1529 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 1530 return -EADDRNOTAVAIL; 1531 1532 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1533 netdev_info(netdev, "returning to hw mac address %pM\n", 1534 hw->mac.addr); 1535 else 1536 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1537 1538 spin_lock_bh(&vsi->mac_filter_hash_lock); 1539 i40e_del_mac_filter(vsi, netdev->dev_addr); 1540 i40e_add_mac_filter(vsi, addr->sa_data); 1541 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1542 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1543 if (vsi->type == I40E_VSI_MAIN) { 1544 i40e_status ret; 1545 1546 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1547 I40E_AQC_WRITE_TYPE_LAA_WOL, 1548 addr->sa_data, NULL); 1549 if (ret) 1550 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1551 i40e_stat_str(hw, ret), 1552 i40e_aq_str(hw, hw->aq.asq_last_status)); 1553 } 1554 1555 /* schedule our worker thread which will take care of 1556 * applying the new filter changes 1557 */ 1558 i40e_service_event_schedule(vsi->back); 1559 return 0; 1560 } 1561 1562 /** 1563 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1564 * @vsi: the VSI being setup 1565 * @ctxt: VSI context structure 1566 * @enabled_tc: Enabled TCs bitmap 1567 * @is_add: True if called before Add VSI 1568 * 1569 * Setup VSI queue mapping for enabled traffic classes. 1570 **/ 1571 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1572 struct i40e_vsi_context *ctxt, 1573 u8 enabled_tc, 1574 bool is_add) 1575 { 1576 struct i40e_pf *pf = vsi->back; 1577 u16 sections = 0; 1578 u8 netdev_tc = 0; 1579 u16 numtc = 0; 1580 u16 qcount; 1581 u8 offset; 1582 u16 qmap; 1583 int i; 1584 u16 num_tc_qps = 0; 1585 1586 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1587 offset = 0; 1588 1589 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1590 /* Find numtc from enabled TC bitmap */ 1591 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1592 if (enabled_tc & BIT(i)) /* TC is enabled */ 1593 numtc++; 1594 } 1595 if (!numtc) { 1596 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1597 numtc = 1; 1598 } 1599 } else { 1600 /* At least TC0 is enabled in case of non-DCB case */ 1601 numtc = 1; 1602 } 1603 1604 vsi->tc_config.numtc = numtc; 1605 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1606 /* Number of queues per enabled TC */ 1607 qcount = vsi->alloc_queue_pairs; 1608 1609 num_tc_qps = qcount / numtc; 1610 num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf)); 1611 1612 /* Setup queue offset/count for all TCs for given VSI */ 1613 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1614 /* See if the given TC is enabled for the given VSI */ 1615 if (vsi->tc_config.enabled_tc & BIT(i)) { 1616 /* TC is enabled */ 1617 int pow, num_qps; 1618 1619 switch (vsi->type) { 1620 case I40E_VSI_MAIN: 1621 qcount = min_t(int, pf->alloc_rss_size, 1622 num_tc_qps); 1623 break; 1624 case I40E_VSI_FDIR: 1625 case I40E_VSI_SRIOV: 1626 case I40E_VSI_VMDQ2: 1627 default: 1628 qcount = num_tc_qps; 1629 WARN_ON(i != 0); 1630 break; 1631 } 1632 vsi->tc_config.tc_info[i].qoffset = offset; 1633 vsi->tc_config.tc_info[i].qcount = qcount; 1634 1635 /* find the next higher power-of-2 of num queue pairs */ 1636 num_qps = qcount; 1637 pow = 0; 1638 while (num_qps && (BIT_ULL(pow) < qcount)) { 1639 pow++; 1640 num_qps >>= 1; 1641 } 1642 1643 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1644 qmap = 1645 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1646 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1647 1648 offset += qcount; 1649 } else { 1650 /* TC is not enabled so set the offset to 1651 * default queue and allocate one queue 1652 * for the given TC. 1653 */ 1654 vsi->tc_config.tc_info[i].qoffset = 0; 1655 vsi->tc_config.tc_info[i].qcount = 1; 1656 vsi->tc_config.tc_info[i].netdev_tc = 0; 1657 1658 qmap = 0; 1659 } 1660 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1661 } 1662 1663 /* Set actual Tx/Rx queue pairs */ 1664 vsi->num_queue_pairs = offset; 1665 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1666 if (vsi->req_queue_pairs > 0) 1667 vsi->num_queue_pairs = vsi->req_queue_pairs; 1668 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1669 vsi->num_queue_pairs = pf->num_lan_msix; 1670 } 1671 1672 /* Scheduler section valid can only be set for ADD VSI */ 1673 if (is_add) { 1674 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1675 1676 ctxt->info.up_enable_bits = enabled_tc; 1677 } 1678 if (vsi->type == I40E_VSI_SRIOV) { 1679 ctxt->info.mapping_flags |= 1680 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1681 for (i = 0; i < vsi->num_queue_pairs; i++) 1682 ctxt->info.queue_mapping[i] = 1683 cpu_to_le16(vsi->base_queue + i); 1684 } else { 1685 ctxt->info.mapping_flags |= 1686 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1687 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1688 } 1689 ctxt->info.valid_sections |= cpu_to_le16(sections); 1690 } 1691 1692 /** 1693 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1694 * @netdev: the netdevice 1695 * @addr: address to add 1696 * 1697 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1698 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1699 */ 1700 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1701 { 1702 struct i40e_netdev_priv *np = netdev_priv(netdev); 1703 struct i40e_vsi *vsi = np->vsi; 1704 1705 if (i40e_add_mac_filter(vsi, addr)) 1706 return 0; 1707 else 1708 return -ENOMEM; 1709 } 1710 1711 /** 1712 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1713 * @netdev: the netdevice 1714 * @addr: address to add 1715 * 1716 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1717 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1718 */ 1719 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1720 { 1721 struct i40e_netdev_priv *np = netdev_priv(netdev); 1722 struct i40e_vsi *vsi = np->vsi; 1723 1724 i40e_del_mac_filter(vsi, addr); 1725 1726 return 0; 1727 } 1728 1729 /** 1730 * i40e_set_rx_mode - NDO callback to set the netdev filters 1731 * @netdev: network interface device structure 1732 **/ 1733 static void i40e_set_rx_mode(struct net_device *netdev) 1734 { 1735 struct i40e_netdev_priv *np = netdev_priv(netdev); 1736 struct i40e_vsi *vsi = np->vsi; 1737 1738 spin_lock_bh(&vsi->mac_filter_hash_lock); 1739 1740 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1741 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1742 1743 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1744 1745 /* check for other flag changes */ 1746 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1747 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1748 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1749 } 1750 1751 /* schedule our worker thread which will take care of 1752 * applying the new filter changes 1753 */ 1754 i40e_service_event_schedule(vsi->back); 1755 } 1756 1757 /** 1758 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1759 * @vsi: Pointer to VSI struct 1760 * @from: Pointer to list which contains MAC filter entries - changes to 1761 * those entries needs to be undone. 1762 * 1763 * MAC filter entries from this list were slated for deletion. 1764 **/ 1765 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1766 struct hlist_head *from) 1767 { 1768 struct i40e_mac_filter *f; 1769 struct hlist_node *h; 1770 1771 hlist_for_each_entry_safe(f, h, from, hlist) { 1772 u64 key = i40e_addr_to_hkey(f->macaddr); 1773 1774 /* Move the element back into MAC filter list*/ 1775 hlist_del(&f->hlist); 1776 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1777 } 1778 } 1779 1780 /** 1781 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1782 * @vsi: Pointer to vsi struct 1783 * @from: Pointer to list which contains MAC filter entries - changes to 1784 * those entries needs to be undone. 1785 * 1786 * MAC filter entries from this list were slated for addition. 1787 **/ 1788 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 1789 struct hlist_head *from) 1790 { 1791 struct i40e_new_mac_filter *new; 1792 struct hlist_node *h; 1793 1794 hlist_for_each_entry_safe(new, h, from, hlist) { 1795 /* We can simply free the wrapper structure */ 1796 hlist_del(&new->hlist); 1797 kfree(new); 1798 } 1799 } 1800 1801 /** 1802 * i40e_next_entry - Get the next non-broadcast filter from a list 1803 * @next: pointer to filter in list 1804 * 1805 * Returns the next non-broadcast filter in the list. Required so that we 1806 * ignore broadcast filters within the list, since these are not handled via 1807 * the normal firmware update path. 1808 */ 1809 static 1810 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 1811 { 1812 hlist_for_each_entry_continue(next, hlist) { 1813 if (!is_broadcast_ether_addr(next->f->macaddr)) 1814 return next; 1815 } 1816 1817 return NULL; 1818 } 1819 1820 /** 1821 * i40e_update_filter_state - Update filter state based on return data 1822 * from firmware 1823 * @count: Number of filters added 1824 * @add_list: return data from fw 1825 * @head: pointer to first filter in current batch 1826 * 1827 * MAC filter entries from list were slated to be added to device. Returns 1828 * number of successful filters. Note that 0 does NOT mean success! 1829 **/ 1830 static int 1831 i40e_update_filter_state(int count, 1832 struct i40e_aqc_add_macvlan_element_data *add_list, 1833 struct i40e_new_mac_filter *add_head) 1834 { 1835 int retval = 0; 1836 int i; 1837 1838 for (i = 0; i < count; i++) { 1839 /* Always check status of each filter. We don't need to check 1840 * the firmware return status because we pre-set the filter 1841 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 1842 * request to the adminq. Thus, if it no longer matches then 1843 * we know the filter is active. 1844 */ 1845 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 1846 add_head->state = I40E_FILTER_FAILED; 1847 } else { 1848 add_head->state = I40E_FILTER_ACTIVE; 1849 retval++; 1850 } 1851 1852 add_head = i40e_next_filter(add_head); 1853 if (!add_head) 1854 break; 1855 } 1856 1857 return retval; 1858 } 1859 1860 /** 1861 * i40e_aqc_del_filters - Request firmware to delete a set of filters 1862 * @vsi: ptr to the VSI 1863 * @vsi_name: name to display in messages 1864 * @list: the list of filters to send to firmware 1865 * @num_del: the number of filters to delete 1866 * @retval: Set to -EIO on failure to delete 1867 * 1868 * Send a request to firmware via AdminQ to delete a set of filters. Uses 1869 * *retval instead of a return value so that success does not force ret_val to 1870 * be set to 0. This ensures that a sequence of calls to this function 1871 * preserve the previous value of *retval on successful delete. 1872 */ 1873 static 1874 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 1875 struct i40e_aqc_remove_macvlan_element_data *list, 1876 int num_del, int *retval) 1877 { 1878 struct i40e_hw *hw = &vsi->back->hw; 1879 i40e_status aq_ret; 1880 int aq_err; 1881 1882 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 1883 aq_err = hw->aq.asq_last_status; 1884 1885 /* Explicitly ignore and do not report when firmware returns ENOENT */ 1886 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 1887 *retval = -EIO; 1888 dev_info(&vsi->back->pdev->dev, 1889 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 1890 vsi_name, i40e_stat_str(hw, aq_ret), 1891 i40e_aq_str(hw, aq_err)); 1892 } 1893 } 1894 1895 /** 1896 * i40e_aqc_add_filters - Request firmware to add a set of filters 1897 * @vsi: ptr to the VSI 1898 * @vsi_name: name to display in messages 1899 * @list: the list of filters to send to firmware 1900 * @add_head: Position in the add hlist 1901 * @num_add: the number of filters to add 1902 * @promisc_change: set to true on exit if promiscuous mode was forced on 1903 * 1904 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 1905 * promisc_changed to true if the firmware has run out of space for more 1906 * filters. 1907 */ 1908 static 1909 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 1910 struct i40e_aqc_add_macvlan_element_data *list, 1911 struct i40e_new_mac_filter *add_head, 1912 int num_add, bool *promisc_changed) 1913 { 1914 struct i40e_hw *hw = &vsi->back->hw; 1915 int aq_err, fcnt; 1916 1917 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 1918 aq_err = hw->aq.asq_last_status; 1919 fcnt = i40e_update_filter_state(num_add, list, add_head); 1920 1921 if (fcnt != num_add) { 1922 *promisc_changed = true; 1923 set_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 1924 dev_warn(&vsi->back->pdev->dev, 1925 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 1926 i40e_aq_str(hw, aq_err), 1927 vsi_name); 1928 } 1929 } 1930 1931 /** 1932 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 1933 * @vsi: pointer to the VSI 1934 * @f: filter data 1935 * 1936 * This function sets or clears the promiscuous broadcast flags for VLAN 1937 * filters in order to properly receive broadcast frames. Assumes that only 1938 * broadcast filters are passed. 1939 * 1940 * Returns status indicating success or failure; 1941 **/ 1942 static i40e_status 1943 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 1944 struct i40e_mac_filter *f) 1945 { 1946 bool enable = f->state == I40E_FILTER_NEW; 1947 struct i40e_hw *hw = &vsi->back->hw; 1948 i40e_status aq_ret; 1949 1950 if (f->vlan == I40E_VLAN_ANY) { 1951 aq_ret = i40e_aq_set_vsi_broadcast(hw, 1952 vsi->seid, 1953 enable, 1954 NULL); 1955 } else { 1956 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 1957 vsi->seid, 1958 enable, 1959 f->vlan, 1960 NULL); 1961 } 1962 1963 if (aq_ret) 1964 dev_warn(&vsi->back->pdev->dev, 1965 "Error %s setting broadcast promiscuous mode on %s\n", 1966 i40e_aq_str(hw, hw->aq.asq_last_status), 1967 vsi_name); 1968 1969 return aq_ret; 1970 } 1971 1972 /** 1973 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 1974 * @vsi: ptr to the VSI 1975 * 1976 * Push any outstanding VSI filter changes through the AdminQ. 1977 * 1978 * Returns 0 or error value 1979 **/ 1980 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 1981 { 1982 struct hlist_head tmp_add_list, tmp_del_list; 1983 struct i40e_mac_filter *f; 1984 struct i40e_new_mac_filter *new, *add_head = NULL; 1985 struct i40e_hw *hw = &vsi->back->hw; 1986 unsigned int failed_filters = 0; 1987 unsigned int vlan_filters = 0; 1988 bool promisc_changed = false; 1989 char vsi_name[16] = "PF"; 1990 int filter_list_len = 0; 1991 i40e_status aq_ret = 0; 1992 u32 changed_flags = 0; 1993 struct hlist_node *h; 1994 struct i40e_pf *pf; 1995 int num_add = 0; 1996 int num_del = 0; 1997 int retval = 0; 1998 u16 cmd_flags; 1999 int list_size; 2000 int bkt; 2001 2002 /* empty array typed pointers, kcalloc later */ 2003 struct i40e_aqc_add_macvlan_element_data *add_list; 2004 struct i40e_aqc_remove_macvlan_element_data *del_list; 2005 2006 while (test_and_set_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state)) 2007 usleep_range(1000, 2000); 2008 pf = vsi->back; 2009 2010 if (vsi->netdev) { 2011 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2012 vsi->current_netdev_flags = vsi->netdev->flags; 2013 } 2014 2015 INIT_HLIST_HEAD(&tmp_add_list); 2016 INIT_HLIST_HEAD(&tmp_del_list); 2017 2018 if (vsi->type == I40E_VSI_SRIOV) 2019 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2020 else if (vsi->type != I40E_VSI_MAIN) 2021 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2022 2023 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2024 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2025 2026 spin_lock_bh(&vsi->mac_filter_hash_lock); 2027 /* Create a list of filters to delete. */ 2028 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2029 if (f->state == I40E_FILTER_REMOVE) { 2030 /* Move the element into temporary del_list */ 2031 hash_del(&f->hlist); 2032 hlist_add_head(&f->hlist, &tmp_del_list); 2033 2034 /* Avoid counting removed filters */ 2035 continue; 2036 } 2037 if (f->state == I40E_FILTER_NEW) { 2038 /* Create a temporary i40e_new_mac_filter */ 2039 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2040 if (!new) 2041 goto err_no_memory_locked; 2042 2043 /* Store pointer to the real filter */ 2044 new->f = f; 2045 new->state = f->state; 2046 2047 /* Add it to the hash list */ 2048 hlist_add_head(&new->hlist, &tmp_add_list); 2049 } 2050 2051 /* Count the number of active (current and new) VLAN 2052 * filters we have now. Does not count filters which 2053 * are marked for deletion. 2054 */ 2055 if (f->vlan > 0) 2056 vlan_filters++; 2057 } 2058 2059 retval = i40e_correct_mac_vlan_filters(vsi, 2060 &tmp_add_list, 2061 &tmp_del_list, 2062 vlan_filters); 2063 if (retval) 2064 goto err_no_memory_locked; 2065 2066 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2067 } 2068 2069 /* Now process 'del_list' outside the lock */ 2070 if (!hlist_empty(&tmp_del_list)) { 2071 filter_list_len = hw->aq.asq_buf_size / 2072 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2073 list_size = filter_list_len * 2074 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2075 del_list = kzalloc(list_size, GFP_ATOMIC); 2076 if (!del_list) 2077 goto err_no_memory; 2078 2079 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2080 cmd_flags = 0; 2081 2082 /* handle broadcast filters by updating the broadcast 2083 * promiscuous flag and release filter list. 2084 */ 2085 if (is_broadcast_ether_addr(f->macaddr)) { 2086 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2087 2088 hlist_del(&f->hlist); 2089 kfree(f); 2090 continue; 2091 } 2092 2093 /* add to delete list */ 2094 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2095 if (f->vlan == I40E_VLAN_ANY) { 2096 del_list[num_del].vlan_tag = 0; 2097 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2098 } else { 2099 del_list[num_del].vlan_tag = 2100 cpu_to_le16((u16)(f->vlan)); 2101 } 2102 2103 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2104 del_list[num_del].flags = cmd_flags; 2105 num_del++; 2106 2107 /* flush a full buffer */ 2108 if (num_del == filter_list_len) { 2109 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2110 num_del, &retval); 2111 memset(del_list, 0, list_size); 2112 num_del = 0; 2113 } 2114 /* Release memory for MAC filter entries which were 2115 * synced up with HW. 2116 */ 2117 hlist_del(&f->hlist); 2118 kfree(f); 2119 } 2120 2121 if (num_del) { 2122 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2123 num_del, &retval); 2124 } 2125 2126 kfree(del_list); 2127 del_list = NULL; 2128 } 2129 2130 if (!hlist_empty(&tmp_add_list)) { 2131 /* Do all the adds now. */ 2132 filter_list_len = hw->aq.asq_buf_size / 2133 sizeof(struct i40e_aqc_add_macvlan_element_data); 2134 list_size = filter_list_len * 2135 sizeof(struct i40e_aqc_add_macvlan_element_data); 2136 add_list = kzalloc(list_size, GFP_ATOMIC); 2137 if (!add_list) 2138 goto err_no_memory; 2139 2140 num_add = 0; 2141 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2142 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, 2143 vsi->state)) { 2144 new->state = I40E_FILTER_FAILED; 2145 continue; 2146 } 2147 2148 /* handle broadcast filters by updating the broadcast 2149 * promiscuous flag instead of adding a MAC filter. 2150 */ 2151 if (is_broadcast_ether_addr(new->f->macaddr)) { 2152 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2153 new->f)) 2154 new->state = I40E_FILTER_FAILED; 2155 else 2156 new->state = I40E_FILTER_ACTIVE; 2157 continue; 2158 } 2159 2160 /* add to add array */ 2161 if (num_add == 0) 2162 add_head = new; 2163 cmd_flags = 0; 2164 ether_addr_copy(add_list[num_add].mac_addr, 2165 new->f->macaddr); 2166 if (new->f->vlan == I40E_VLAN_ANY) { 2167 add_list[num_add].vlan_tag = 0; 2168 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2169 } else { 2170 add_list[num_add].vlan_tag = 2171 cpu_to_le16((u16)(new->f->vlan)); 2172 } 2173 add_list[num_add].queue_number = 0; 2174 /* set invalid match method for later detection */ 2175 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2176 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2177 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2178 num_add++; 2179 2180 /* flush a full buffer */ 2181 if (num_add == filter_list_len) { 2182 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2183 add_head, num_add, 2184 &promisc_changed); 2185 memset(add_list, 0, list_size); 2186 num_add = 0; 2187 } 2188 } 2189 if (num_add) { 2190 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2191 num_add, &promisc_changed); 2192 } 2193 /* Now move all of the filters from the temp add list back to 2194 * the VSI's list. 2195 */ 2196 spin_lock_bh(&vsi->mac_filter_hash_lock); 2197 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2198 /* Only update the state if we're still NEW */ 2199 if (new->f->state == I40E_FILTER_NEW) 2200 new->f->state = new->state; 2201 hlist_del(&new->hlist); 2202 kfree(new); 2203 } 2204 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2205 kfree(add_list); 2206 add_list = NULL; 2207 } 2208 2209 /* Determine the number of active and failed filters. */ 2210 spin_lock_bh(&vsi->mac_filter_hash_lock); 2211 vsi->active_filters = 0; 2212 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2213 if (f->state == I40E_FILTER_ACTIVE) 2214 vsi->active_filters++; 2215 else if (f->state == I40E_FILTER_FAILED) 2216 failed_filters++; 2217 } 2218 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2219 2220 /* If promiscuous mode has changed, we need to calculate a new 2221 * threshold for when we are safe to exit 2222 */ 2223 if (promisc_changed) 2224 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2225 2226 /* Check if we are able to exit overflow promiscuous mode. We can 2227 * safely exit if we didn't just enter, we no longer have any failed 2228 * filters, and we have reduced filters below the threshold value. 2229 */ 2230 if (test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state) && 2231 !promisc_changed && !failed_filters && 2232 (vsi->active_filters < vsi->promisc_threshold)) { 2233 dev_info(&pf->pdev->dev, 2234 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2235 vsi_name); 2236 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2237 promisc_changed = true; 2238 vsi->promisc_threshold = 0; 2239 } 2240 2241 /* if the VF is not trusted do not do promisc */ 2242 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2243 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 2244 goto out; 2245 } 2246 2247 /* check for changes in promiscuous modes */ 2248 if (changed_flags & IFF_ALLMULTI) { 2249 bool cur_multipromisc; 2250 2251 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2252 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2253 vsi->seid, 2254 cur_multipromisc, 2255 NULL); 2256 if (aq_ret) { 2257 retval = i40e_aq_rc_to_posix(aq_ret, 2258 hw->aq.asq_last_status); 2259 dev_info(&pf->pdev->dev, 2260 "set multi promisc failed on %s, err %s aq_err %s\n", 2261 vsi_name, 2262 i40e_stat_str(hw, aq_ret), 2263 i40e_aq_str(hw, hw->aq.asq_last_status)); 2264 } 2265 } 2266 if ((changed_flags & IFF_PROMISC) || 2267 (promisc_changed && 2268 test_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state))) { 2269 bool cur_promisc; 2270 2271 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2272 test_bit(__I40E_VSI_OVERFLOW_PROMISC, 2273 vsi->state)); 2274 if ((vsi->type == I40E_VSI_MAIN) && 2275 (pf->lan_veb != I40E_NO_VEB) && 2276 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2277 /* set defport ON for Main VSI instead of true promisc 2278 * this way we will get all unicast/multicast and VLAN 2279 * promisc behavior but will not get VF or VMDq traffic 2280 * replicated on the Main VSI. 2281 */ 2282 if (pf->cur_promisc != cur_promisc) { 2283 pf->cur_promisc = cur_promisc; 2284 if (cur_promisc) 2285 aq_ret = 2286 i40e_aq_set_default_vsi(hw, 2287 vsi->seid, 2288 NULL); 2289 else 2290 aq_ret = 2291 i40e_aq_clear_default_vsi(hw, 2292 vsi->seid, 2293 NULL); 2294 if (aq_ret) { 2295 retval = i40e_aq_rc_to_posix(aq_ret, 2296 hw->aq.asq_last_status); 2297 dev_info(&pf->pdev->dev, 2298 "Set default VSI failed on %s, err %s, aq_err %s\n", 2299 vsi_name, 2300 i40e_stat_str(hw, aq_ret), 2301 i40e_aq_str(hw, 2302 hw->aq.asq_last_status)); 2303 } 2304 } 2305 } else { 2306 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2307 hw, 2308 vsi->seid, 2309 cur_promisc, NULL, 2310 true); 2311 if (aq_ret) { 2312 retval = 2313 i40e_aq_rc_to_posix(aq_ret, 2314 hw->aq.asq_last_status); 2315 dev_info(&pf->pdev->dev, 2316 "set unicast promisc failed on %s, err %s, aq_err %s\n", 2317 vsi_name, 2318 i40e_stat_str(hw, aq_ret), 2319 i40e_aq_str(hw, 2320 hw->aq.asq_last_status)); 2321 } 2322 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2323 hw, 2324 vsi->seid, 2325 cur_promisc, NULL); 2326 if (aq_ret) { 2327 retval = 2328 i40e_aq_rc_to_posix(aq_ret, 2329 hw->aq.asq_last_status); 2330 dev_info(&pf->pdev->dev, 2331 "set multicast promisc failed on %s, err %s, aq_err %s\n", 2332 vsi_name, 2333 i40e_stat_str(hw, aq_ret), 2334 i40e_aq_str(hw, 2335 hw->aq.asq_last_status)); 2336 } 2337 } 2338 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw, 2339 vsi->seid, 2340 cur_promisc, NULL); 2341 if (aq_ret) { 2342 retval = i40e_aq_rc_to_posix(aq_ret, 2343 pf->hw.aq.asq_last_status); 2344 dev_info(&pf->pdev->dev, 2345 "set brdcast promisc failed, err %s, aq_err %s\n", 2346 i40e_stat_str(hw, aq_ret), 2347 i40e_aq_str(hw, 2348 hw->aq.asq_last_status)); 2349 } 2350 } 2351 out: 2352 /* if something went wrong then set the changed flag so we try again */ 2353 if (retval) 2354 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2355 2356 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2357 return retval; 2358 2359 err_no_memory: 2360 /* Restore elements on the temporary add and delete lists */ 2361 spin_lock_bh(&vsi->mac_filter_hash_lock); 2362 err_no_memory_locked: 2363 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2364 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2365 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2366 2367 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2368 clear_bit(__I40E_VSI_SYNCING_FILTERS, vsi->state); 2369 return -ENOMEM; 2370 } 2371 2372 /** 2373 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2374 * @pf: board private structure 2375 **/ 2376 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2377 { 2378 int v; 2379 2380 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC)) 2381 return; 2382 pf->flags &= ~I40E_FLAG_FILTER_SYNC; 2383 2384 for (v = 0; v < pf->num_alloc_vsi; v++) { 2385 if (pf->vsi[v] && 2386 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2387 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2388 2389 if (ret) { 2390 /* come back and try again later */ 2391 pf->flags |= I40E_FLAG_FILTER_SYNC; 2392 break; 2393 } 2394 } 2395 } 2396 } 2397 2398 /** 2399 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2400 * @netdev: network interface device structure 2401 * @new_mtu: new value for maximum frame size 2402 * 2403 * Returns 0 on success, negative on failure 2404 **/ 2405 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2406 { 2407 struct i40e_netdev_priv *np = netdev_priv(netdev); 2408 struct i40e_vsi *vsi = np->vsi; 2409 struct i40e_pf *pf = vsi->back; 2410 2411 netdev_info(netdev, "changing MTU from %d to %d\n", 2412 netdev->mtu, new_mtu); 2413 netdev->mtu = new_mtu; 2414 if (netif_running(netdev)) 2415 i40e_vsi_reinit_locked(vsi); 2416 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 2417 I40E_FLAG_CLIENT_L2_CHANGE); 2418 return 0; 2419 } 2420 2421 /** 2422 * i40e_ioctl - Access the hwtstamp interface 2423 * @netdev: network interface device structure 2424 * @ifr: interface request data 2425 * @cmd: ioctl command 2426 **/ 2427 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2428 { 2429 struct i40e_netdev_priv *np = netdev_priv(netdev); 2430 struct i40e_pf *pf = np->vsi->back; 2431 2432 switch (cmd) { 2433 case SIOCGHWTSTAMP: 2434 return i40e_ptp_get_ts_config(pf, ifr); 2435 case SIOCSHWTSTAMP: 2436 return i40e_ptp_set_ts_config(pf, ifr); 2437 default: 2438 return -EOPNOTSUPP; 2439 } 2440 } 2441 2442 /** 2443 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2444 * @vsi: the vsi being adjusted 2445 **/ 2446 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2447 { 2448 struct i40e_vsi_context ctxt; 2449 i40e_status ret; 2450 2451 if ((vsi->info.valid_sections & 2452 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2453 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2454 return; /* already enabled */ 2455 2456 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2457 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2458 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2459 2460 ctxt.seid = vsi->seid; 2461 ctxt.info = vsi->info; 2462 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2463 if (ret) { 2464 dev_info(&vsi->back->pdev->dev, 2465 "update vlan stripping failed, err %s aq_err %s\n", 2466 i40e_stat_str(&vsi->back->hw, ret), 2467 i40e_aq_str(&vsi->back->hw, 2468 vsi->back->hw.aq.asq_last_status)); 2469 } 2470 } 2471 2472 /** 2473 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2474 * @vsi: the vsi being adjusted 2475 **/ 2476 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2477 { 2478 struct i40e_vsi_context ctxt; 2479 i40e_status ret; 2480 2481 if ((vsi->info.valid_sections & 2482 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2483 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2484 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2485 return; /* already disabled */ 2486 2487 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2488 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2489 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2490 2491 ctxt.seid = vsi->seid; 2492 ctxt.info = vsi->info; 2493 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2494 if (ret) { 2495 dev_info(&vsi->back->pdev->dev, 2496 "update vlan stripping failed, err %s aq_err %s\n", 2497 i40e_stat_str(&vsi->back->hw, ret), 2498 i40e_aq_str(&vsi->back->hw, 2499 vsi->back->hw.aq.asq_last_status)); 2500 } 2501 } 2502 2503 /** 2504 * i40e_vlan_rx_register - Setup or shutdown vlan offload 2505 * @netdev: network interface to be adjusted 2506 * @features: netdev features to test if VLAN offload is enabled or not 2507 **/ 2508 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features) 2509 { 2510 struct i40e_netdev_priv *np = netdev_priv(netdev); 2511 struct i40e_vsi *vsi = np->vsi; 2512 2513 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2514 i40e_vlan_stripping_enable(vsi); 2515 else 2516 i40e_vlan_stripping_disable(vsi); 2517 } 2518 2519 /** 2520 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2521 * @vsi: the vsi being configured 2522 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2523 * 2524 * This is a helper function for adding a new MAC/VLAN filter with the 2525 * specified VLAN for each existing MAC address already in the hash table. 2526 * This function does *not* perform any accounting to update filters based on 2527 * VLAN mode. 2528 * 2529 * NOTE: this function expects to be called while under the 2530 * mac_filter_hash_lock 2531 **/ 2532 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2533 { 2534 struct i40e_mac_filter *f, *add_f; 2535 struct hlist_node *h; 2536 int bkt; 2537 2538 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2539 if (f->state == I40E_FILTER_REMOVE) 2540 continue; 2541 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2542 if (!add_f) { 2543 dev_info(&vsi->back->pdev->dev, 2544 "Could not add vlan filter %d for %pM\n", 2545 vid, f->macaddr); 2546 return -ENOMEM; 2547 } 2548 } 2549 2550 return 0; 2551 } 2552 2553 /** 2554 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2555 * @vsi: the VSI being configured 2556 * @vid: VLAN id to be added 2557 **/ 2558 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2559 { 2560 int err; 2561 2562 if (!vid || vsi->info.pvid) 2563 return -EINVAL; 2564 2565 /* Locked once because all functions invoked below iterates list*/ 2566 spin_lock_bh(&vsi->mac_filter_hash_lock); 2567 err = i40e_add_vlan_all_mac(vsi, vid); 2568 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2569 if (err) 2570 return err; 2571 2572 /* schedule our worker thread which will take care of 2573 * applying the new filter changes 2574 */ 2575 i40e_service_event_schedule(vsi->back); 2576 return 0; 2577 } 2578 2579 /** 2580 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2581 * @vsi: the vsi being configured 2582 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2583 * 2584 * This function should be used to remove all VLAN filters which match the 2585 * given VID. It does not schedule the service event and does not take the 2586 * mac_filter_hash_lock so it may be combined with other operations under 2587 * a single invocation of the mac_filter_hash_lock. 2588 * 2589 * NOTE: this function expects to be called while under the 2590 * mac_filter_hash_lock 2591 */ 2592 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2593 { 2594 struct i40e_mac_filter *f; 2595 struct hlist_node *h; 2596 int bkt; 2597 2598 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2599 if (f->vlan == vid) 2600 __i40e_del_filter(vsi, f); 2601 } 2602 } 2603 2604 /** 2605 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2606 * @vsi: the VSI being configured 2607 * @vid: VLAN id to be removed 2608 **/ 2609 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2610 { 2611 if (!vid || vsi->info.pvid) 2612 return; 2613 2614 spin_lock_bh(&vsi->mac_filter_hash_lock); 2615 i40e_rm_vlan_all_mac(vsi, vid); 2616 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2617 2618 /* schedule our worker thread which will take care of 2619 * applying the new filter changes 2620 */ 2621 i40e_service_event_schedule(vsi->back); 2622 } 2623 2624 /** 2625 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2626 * @netdev: network interface to be adjusted 2627 * @vid: vlan id to be added 2628 * 2629 * net_device_ops implementation for adding vlan ids 2630 **/ 2631 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2632 __always_unused __be16 proto, u16 vid) 2633 { 2634 struct i40e_netdev_priv *np = netdev_priv(netdev); 2635 struct i40e_vsi *vsi = np->vsi; 2636 int ret = 0; 2637 2638 if (vid >= VLAN_N_VID) 2639 return -EINVAL; 2640 2641 /* If the network stack called us with vid = 0 then 2642 * it is asking to receive priority tagged packets with 2643 * vlan id 0. Our HW receives them by default when configured 2644 * to receive untagged packets so there is no need to add an 2645 * extra filter for vlan 0 tagged packets. 2646 */ 2647 if (vid) 2648 ret = i40e_vsi_add_vlan(vsi, vid); 2649 2650 if (!ret) 2651 set_bit(vid, vsi->active_vlans); 2652 2653 return ret; 2654 } 2655 2656 /** 2657 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2658 * @netdev: network interface to be adjusted 2659 * @vid: vlan id to be removed 2660 * 2661 * net_device_ops implementation for removing vlan ids 2662 **/ 2663 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2664 __always_unused __be16 proto, u16 vid) 2665 { 2666 struct i40e_netdev_priv *np = netdev_priv(netdev); 2667 struct i40e_vsi *vsi = np->vsi; 2668 2669 /* return code is ignored as there is nothing a user 2670 * can do about failure to remove and a log message was 2671 * already printed from the other function 2672 */ 2673 i40e_vsi_kill_vlan(vsi, vid); 2674 2675 clear_bit(vid, vsi->active_vlans); 2676 2677 return 0; 2678 } 2679 2680 /** 2681 * i40e_macaddr_init - explicitly write the mac address filters 2682 * 2683 * @vsi: pointer to the vsi 2684 * @macaddr: the MAC address 2685 * 2686 * This is needed when the macaddr has been obtained by other 2687 * means than the default, e.g., from Open Firmware or IDPROM. 2688 * Returns 0 on success, negative on failure 2689 **/ 2690 static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr) 2691 { 2692 int ret; 2693 struct i40e_aqc_add_macvlan_element_data element; 2694 2695 ret = i40e_aq_mac_address_write(&vsi->back->hw, 2696 I40E_AQC_WRITE_TYPE_LAA_WOL, 2697 macaddr, NULL); 2698 if (ret) { 2699 dev_info(&vsi->back->pdev->dev, 2700 "Addr change for VSI failed: %d\n", ret); 2701 return -EADDRNOTAVAIL; 2702 } 2703 2704 memset(&element, 0, sizeof(element)); 2705 ether_addr_copy(element.mac_addr, macaddr); 2706 element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH); 2707 ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL); 2708 if (ret) { 2709 dev_info(&vsi->back->pdev->dev, 2710 "add filter failed err %s aq_err %s\n", 2711 i40e_stat_str(&vsi->back->hw, ret), 2712 i40e_aq_str(&vsi->back->hw, 2713 vsi->back->hw.aq.asq_last_status)); 2714 } 2715 return ret; 2716 } 2717 2718 /** 2719 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2720 * @vsi: the vsi being brought back up 2721 **/ 2722 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2723 { 2724 u16 vid; 2725 2726 if (!vsi->netdev) 2727 return; 2728 2729 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features); 2730 2731 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2732 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), 2733 vid); 2734 } 2735 2736 /** 2737 * i40e_vsi_add_pvid - Add pvid for the VSI 2738 * @vsi: the vsi being adjusted 2739 * @vid: the vlan id to set as a PVID 2740 **/ 2741 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2742 { 2743 struct i40e_vsi_context ctxt; 2744 i40e_status ret; 2745 2746 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2747 vsi->info.pvid = cpu_to_le16(vid); 2748 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2749 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2750 I40E_AQ_VSI_PVLAN_EMOD_STR; 2751 2752 ctxt.seid = vsi->seid; 2753 ctxt.info = vsi->info; 2754 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2755 if (ret) { 2756 dev_info(&vsi->back->pdev->dev, 2757 "add pvid failed, err %s aq_err %s\n", 2758 i40e_stat_str(&vsi->back->hw, ret), 2759 i40e_aq_str(&vsi->back->hw, 2760 vsi->back->hw.aq.asq_last_status)); 2761 return -ENOENT; 2762 } 2763 2764 return 0; 2765 } 2766 2767 /** 2768 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2769 * @vsi: the vsi being adjusted 2770 * 2771 * Just use the vlan_rx_register() service to put it back to normal 2772 **/ 2773 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2774 { 2775 i40e_vlan_stripping_disable(vsi); 2776 2777 vsi->info.pvid = 0; 2778 } 2779 2780 /** 2781 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2782 * @vsi: ptr to the VSI 2783 * 2784 * If this function returns with an error, then it's possible one or 2785 * more of the rings is populated (while the rest are not). It is the 2786 * callers duty to clean those orphaned rings. 2787 * 2788 * Return 0 on success, negative on failure 2789 **/ 2790 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2791 { 2792 int i, err = 0; 2793 2794 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2795 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2796 2797 return err; 2798 } 2799 2800 /** 2801 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 2802 * @vsi: ptr to the VSI 2803 * 2804 * Free VSI's transmit software resources 2805 **/ 2806 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 2807 { 2808 int i; 2809 2810 if (!vsi->tx_rings) 2811 return; 2812 2813 for (i = 0; i < vsi->num_queue_pairs; i++) 2814 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 2815 i40e_free_tx_resources(vsi->tx_rings[i]); 2816 } 2817 2818 /** 2819 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 2820 * @vsi: ptr to the VSI 2821 * 2822 * If this function returns with an error, then it's possible one or 2823 * more of the rings is populated (while the rest are not). It is the 2824 * callers duty to clean those orphaned rings. 2825 * 2826 * Return 0 on success, negative on failure 2827 **/ 2828 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 2829 { 2830 int i, err = 0; 2831 2832 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2833 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 2834 return err; 2835 } 2836 2837 /** 2838 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 2839 * @vsi: ptr to the VSI 2840 * 2841 * Free all receive software resources 2842 **/ 2843 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 2844 { 2845 int i; 2846 2847 if (!vsi->rx_rings) 2848 return; 2849 2850 for (i = 0; i < vsi->num_queue_pairs; i++) 2851 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 2852 i40e_free_rx_resources(vsi->rx_rings[i]); 2853 } 2854 2855 /** 2856 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 2857 * @ring: The Tx ring to configure 2858 * 2859 * This enables/disables XPS for a given Tx descriptor ring 2860 * based on the TCs enabled for the VSI that ring belongs to. 2861 **/ 2862 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 2863 { 2864 struct i40e_vsi *vsi = ring->vsi; 2865 cpumask_var_t mask; 2866 2867 if (!ring->q_vector || !ring->netdev) 2868 return; 2869 2870 /* Single TC mode enable XPS */ 2871 if (vsi->tc_config.numtc <= 1) { 2872 if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) 2873 netif_set_xps_queue(ring->netdev, 2874 &ring->q_vector->affinity_mask, 2875 ring->queue_index); 2876 } else if (alloc_cpumask_var(&mask, GFP_KERNEL)) { 2877 /* Disable XPS to allow selection based on TC */ 2878 bitmap_zero(cpumask_bits(mask), nr_cpumask_bits); 2879 netif_set_xps_queue(ring->netdev, mask, ring->queue_index); 2880 free_cpumask_var(mask); 2881 } 2882 2883 /* schedule our worker thread which will take care of 2884 * applying the new filter changes 2885 */ 2886 i40e_service_event_schedule(vsi->back); 2887 } 2888 2889 /** 2890 * i40e_configure_tx_ring - Configure a transmit ring context and rest 2891 * @ring: The Tx ring to configure 2892 * 2893 * Configure the Tx descriptor ring in the HMC context. 2894 **/ 2895 static int i40e_configure_tx_ring(struct i40e_ring *ring) 2896 { 2897 struct i40e_vsi *vsi = ring->vsi; 2898 u16 pf_q = vsi->base_queue + ring->queue_index; 2899 struct i40e_hw *hw = &vsi->back->hw; 2900 struct i40e_hmc_obj_txq tx_ctx; 2901 i40e_status err = 0; 2902 u32 qtx_ctl = 0; 2903 2904 /* some ATR related tx ring init */ 2905 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 2906 ring->atr_sample_rate = vsi->back->atr_sample_rate; 2907 ring->atr_count = 0; 2908 } else { 2909 ring->atr_sample_rate = 0; 2910 } 2911 2912 /* configure XPS */ 2913 i40e_config_xps_tx_ring(ring); 2914 2915 /* clear the context structure first */ 2916 memset(&tx_ctx, 0, sizeof(tx_ctx)); 2917 2918 tx_ctx.new_context = 1; 2919 tx_ctx.base = (ring->dma / 128); 2920 tx_ctx.qlen = ring->count; 2921 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 2922 I40E_FLAG_FD_ATR_ENABLED)); 2923 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 2924 /* FDIR VSI tx ring can still use RS bit and writebacks */ 2925 if (vsi->type != I40E_VSI_FDIR) 2926 tx_ctx.head_wb_ena = 1; 2927 tx_ctx.head_wb_addr = ring->dma + 2928 (ring->count * sizeof(struct i40e_tx_desc)); 2929 2930 /* As part of VSI creation/update, FW allocates certain 2931 * Tx arbitration queue sets for each TC enabled for 2932 * the VSI. The FW returns the handles to these queue 2933 * sets as part of the response buffer to Add VSI, 2934 * Update VSI, etc. AQ commands. It is expected that 2935 * these queue set handles be associated with the Tx 2936 * queues by the driver as part of the TX queue context 2937 * initialization. This has to be done regardless of 2938 * DCB as by default everything is mapped to TC0. 2939 */ 2940 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 2941 tx_ctx.rdylist_act = 0; 2942 2943 /* clear the context in the HMC */ 2944 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 2945 if (err) { 2946 dev_info(&vsi->back->pdev->dev, 2947 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 2948 ring->queue_index, pf_q, err); 2949 return -ENOMEM; 2950 } 2951 2952 /* set the context in the HMC */ 2953 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 2954 if (err) { 2955 dev_info(&vsi->back->pdev->dev, 2956 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 2957 ring->queue_index, pf_q, err); 2958 return -ENOMEM; 2959 } 2960 2961 /* Now associate this queue with this PCI function */ 2962 if (vsi->type == I40E_VSI_VMDQ2) { 2963 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 2964 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 2965 I40E_QTX_CTL_VFVM_INDX_MASK; 2966 } else { 2967 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 2968 } 2969 2970 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2971 I40E_QTX_CTL_PF_INDX_MASK); 2972 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 2973 i40e_flush(hw); 2974 2975 /* cache tail off for easier writes later */ 2976 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 2977 2978 return 0; 2979 } 2980 2981 /** 2982 * i40e_configure_rx_ring - Configure a receive ring context 2983 * @ring: The Rx ring to configure 2984 * 2985 * Configure the Rx descriptor ring in the HMC context. 2986 **/ 2987 static int i40e_configure_rx_ring(struct i40e_ring *ring) 2988 { 2989 struct i40e_vsi *vsi = ring->vsi; 2990 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 2991 u16 pf_q = vsi->base_queue + ring->queue_index; 2992 struct i40e_hw *hw = &vsi->back->hw; 2993 struct i40e_hmc_obj_rxq rx_ctx; 2994 i40e_status err = 0; 2995 2996 ring->state = 0; 2997 2998 /* clear the context structure first */ 2999 memset(&rx_ctx, 0, sizeof(rx_ctx)); 3000 3001 ring->rx_buf_len = vsi->rx_buf_len; 3002 3003 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 3004 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3005 3006 rx_ctx.base = (ring->dma / 128); 3007 rx_ctx.qlen = ring->count; 3008 3009 /* use 32 byte descriptors */ 3010 rx_ctx.dsize = 1; 3011 3012 /* descriptor type is always zero 3013 * rx_ctx.dtype = 0; 3014 */ 3015 rx_ctx.hsplit_0 = 0; 3016 3017 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3018 if (hw->revision_id == 0) 3019 rx_ctx.lrxqthresh = 0; 3020 else 3021 rx_ctx.lrxqthresh = 2; 3022 rx_ctx.crcstrip = 1; 3023 rx_ctx.l2tsel = 1; 3024 /* this controls whether VLAN is stripped from inner headers */ 3025 rx_ctx.showiv = 0; 3026 /* set the prefena field to 1 because the manual says to */ 3027 rx_ctx.prefena = 1; 3028 3029 /* clear the context in the HMC */ 3030 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3031 if (err) { 3032 dev_info(&vsi->back->pdev->dev, 3033 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3034 ring->queue_index, pf_q, err); 3035 return -ENOMEM; 3036 } 3037 3038 /* set the context in the HMC */ 3039 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3040 if (err) { 3041 dev_info(&vsi->back->pdev->dev, 3042 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3043 ring->queue_index, pf_q, err); 3044 return -ENOMEM; 3045 } 3046 3047 /* configure Rx buffer alignment */ 3048 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) 3049 clear_ring_build_skb_enabled(ring); 3050 else 3051 set_ring_build_skb_enabled(ring); 3052 3053 /* cache tail for quicker writes, and clear the reg before use */ 3054 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3055 writel(0, ring->tail); 3056 3057 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3058 3059 return 0; 3060 } 3061 3062 /** 3063 * i40e_vsi_configure_tx - Configure the VSI for Tx 3064 * @vsi: VSI structure describing this set of rings and resources 3065 * 3066 * Configure the Tx VSI for operation. 3067 **/ 3068 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3069 { 3070 int err = 0; 3071 u16 i; 3072 3073 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3074 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3075 3076 return err; 3077 } 3078 3079 /** 3080 * i40e_vsi_configure_rx - Configure the VSI for Rx 3081 * @vsi: the VSI being configured 3082 * 3083 * Configure the Rx VSI for operation. 3084 **/ 3085 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3086 { 3087 int err = 0; 3088 u16 i; 3089 3090 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3091 vsi->max_frame = I40E_MAX_RXBUFFER; 3092 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3093 #if (PAGE_SIZE < 8192) 3094 } else if (!I40E_2K_TOO_SMALL_WITH_PADDING && 3095 (vsi->netdev->mtu <= ETH_DATA_LEN)) { 3096 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3097 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3098 #endif 3099 } else { 3100 vsi->max_frame = I40E_MAX_RXBUFFER; 3101 vsi->rx_buf_len = (PAGE_SIZE < 8192) ? I40E_RXBUFFER_3072 : 3102 I40E_RXBUFFER_2048; 3103 } 3104 3105 /* set up individual rings */ 3106 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3107 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3108 3109 return err; 3110 } 3111 3112 /** 3113 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3114 * @vsi: ptr to the VSI 3115 **/ 3116 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3117 { 3118 struct i40e_ring *tx_ring, *rx_ring; 3119 u16 qoffset, qcount; 3120 int i, n; 3121 3122 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3123 /* Reset the TC information */ 3124 for (i = 0; i < vsi->num_queue_pairs; i++) { 3125 rx_ring = vsi->rx_rings[i]; 3126 tx_ring = vsi->tx_rings[i]; 3127 rx_ring->dcb_tc = 0; 3128 tx_ring->dcb_tc = 0; 3129 } 3130 } 3131 3132 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3133 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3134 continue; 3135 3136 qoffset = vsi->tc_config.tc_info[n].qoffset; 3137 qcount = vsi->tc_config.tc_info[n].qcount; 3138 for (i = qoffset; i < (qoffset + qcount); i++) { 3139 rx_ring = vsi->rx_rings[i]; 3140 tx_ring = vsi->tx_rings[i]; 3141 rx_ring->dcb_tc = n; 3142 tx_ring->dcb_tc = n; 3143 } 3144 } 3145 } 3146 3147 /** 3148 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3149 * @vsi: ptr to the VSI 3150 **/ 3151 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3152 { 3153 struct i40e_pf *pf = vsi->back; 3154 int err; 3155 3156 if (vsi->netdev) 3157 i40e_set_rx_mode(vsi->netdev); 3158 3159 if (!!(pf->flags & I40E_FLAG_PF_MAC)) { 3160 err = i40e_macaddr_init(vsi, pf->hw.mac.addr); 3161 if (err) { 3162 dev_warn(&pf->pdev->dev, 3163 "could not set up macaddr; err %d\n", err); 3164 } 3165 } 3166 } 3167 3168 /** 3169 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3170 * @vsi: Pointer to the targeted VSI 3171 * 3172 * This function replays the hlist on the hw where all the SB Flow Director 3173 * filters were saved. 3174 **/ 3175 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3176 { 3177 struct i40e_fdir_filter *filter; 3178 struct i40e_pf *pf = vsi->back; 3179 struct hlist_node *node; 3180 3181 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3182 return; 3183 3184 /* Reset FDir counters as we're replaying all existing filters */ 3185 pf->fd_tcp4_filter_cnt = 0; 3186 pf->fd_udp4_filter_cnt = 0; 3187 pf->fd_sctp4_filter_cnt = 0; 3188 pf->fd_ip4_filter_cnt = 0; 3189 3190 hlist_for_each_entry_safe(filter, node, 3191 &pf->fdir_filter_list, fdir_node) { 3192 i40e_add_del_fdir(vsi, filter, true); 3193 } 3194 } 3195 3196 /** 3197 * i40e_vsi_configure - Set up the VSI for action 3198 * @vsi: the VSI being configured 3199 **/ 3200 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3201 { 3202 int err; 3203 3204 i40e_set_vsi_rx_mode(vsi); 3205 i40e_restore_vlan(vsi); 3206 i40e_vsi_config_dcb_rings(vsi); 3207 err = i40e_vsi_configure_tx(vsi); 3208 if (!err) 3209 err = i40e_vsi_configure_rx(vsi); 3210 3211 return err; 3212 } 3213 3214 /** 3215 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3216 * @vsi: the VSI being configured 3217 **/ 3218 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3219 { 3220 struct i40e_pf *pf = vsi->back; 3221 struct i40e_hw *hw = &pf->hw; 3222 u16 vector; 3223 int i, q; 3224 u32 qp; 3225 3226 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3227 * and PFINT_LNKLSTn registers, e.g.: 3228 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3229 */ 3230 qp = vsi->base_queue; 3231 vector = vsi->base_vector; 3232 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3233 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3234 3235 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3236 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting); 3237 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3238 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3239 q_vector->rx.itr); 3240 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting); 3241 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3242 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3243 q_vector->tx.itr); 3244 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3245 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3246 3247 /* Linked list for the queuepairs assigned to this vector */ 3248 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3249 for (q = 0; q < q_vector->num_ringpairs; q++) { 3250 u32 val; 3251 3252 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3253 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3254 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3255 (qp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3256 (I40E_QUEUE_TYPE_TX 3257 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3258 3259 wr32(hw, I40E_QINT_RQCTL(qp), val); 3260 3261 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3262 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3263 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3264 ((qp+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)| 3265 (I40E_QUEUE_TYPE_RX 3266 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3267 3268 /* Terminate the linked list */ 3269 if (q == (q_vector->num_ringpairs - 1)) 3270 val |= (I40E_QUEUE_END_OF_LIST 3271 << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3272 3273 wr32(hw, I40E_QINT_TQCTL(qp), val); 3274 qp++; 3275 } 3276 } 3277 3278 i40e_flush(hw); 3279 } 3280 3281 /** 3282 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3283 * @hw: ptr to the hardware info 3284 **/ 3285 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3286 { 3287 struct i40e_hw *hw = &pf->hw; 3288 u32 val; 3289 3290 /* clear things first */ 3291 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3292 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3293 3294 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3295 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3296 I40E_PFINT_ICR0_ENA_GRST_MASK | 3297 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3298 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3299 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3300 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3301 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3302 3303 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3304 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3305 3306 if (pf->flags & I40E_FLAG_PTP) 3307 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3308 3309 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3310 3311 /* SW_ITR_IDX = 0, but don't change INTENA */ 3312 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3313 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3314 3315 /* OTHER_ITR_IDX = 0 */ 3316 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3317 } 3318 3319 /** 3320 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3321 * @vsi: the VSI being configured 3322 **/ 3323 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3324 { 3325 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3326 struct i40e_pf *pf = vsi->back; 3327 struct i40e_hw *hw = &pf->hw; 3328 u32 val; 3329 3330 /* set the ITR configuration */ 3331 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3332 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting); 3333 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3334 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 3335 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting); 3336 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3337 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 3338 3339 i40e_enable_misc_int_causes(pf); 3340 3341 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3342 wr32(hw, I40E_PFINT_LNKLST0, 0); 3343 3344 /* Associate the queue pair to the vector and enable the queue int */ 3345 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3346 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3347 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3348 3349 wr32(hw, I40E_QINT_RQCTL(0), val); 3350 3351 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3352 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3353 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3354 3355 wr32(hw, I40E_QINT_TQCTL(0), val); 3356 i40e_flush(hw); 3357 } 3358 3359 /** 3360 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3361 * @pf: board private structure 3362 **/ 3363 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3364 { 3365 struct i40e_hw *hw = &pf->hw; 3366 3367 wr32(hw, I40E_PFINT_DYN_CTL0, 3368 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3369 i40e_flush(hw); 3370 } 3371 3372 /** 3373 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3374 * @pf: board private structure 3375 * @clearpba: true when all pending interrupt events should be cleared 3376 **/ 3377 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba) 3378 { 3379 struct i40e_hw *hw = &pf->hw; 3380 u32 val; 3381 3382 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3383 (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) | 3384 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3385 3386 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3387 i40e_flush(hw); 3388 } 3389 3390 /** 3391 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3392 * @irq: interrupt number 3393 * @data: pointer to a q_vector 3394 **/ 3395 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3396 { 3397 struct i40e_q_vector *q_vector = data; 3398 3399 if (!q_vector->tx.ring && !q_vector->rx.ring) 3400 return IRQ_HANDLED; 3401 3402 napi_schedule_irqoff(&q_vector->napi); 3403 3404 return IRQ_HANDLED; 3405 } 3406 3407 /** 3408 * i40e_irq_affinity_notify - Callback for affinity changes 3409 * @notify: context as to what irq was changed 3410 * @mask: the new affinity mask 3411 * 3412 * This is a callback function used by the irq_set_affinity_notifier function 3413 * so that we may register to receive changes to the irq affinity masks. 3414 **/ 3415 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3416 const cpumask_t *mask) 3417 { 3418 struct i40e_q_vector *q_vector = 3419 container_of(notify, struct i40e_q_vector, affinity_notify); 3420 3421 q_vector->affinity_mask = *mask; 3422 } 3423 3424 /** 3425 * i40e_irq_affinity_release - Callback for affinity notifier release 3426 * @ref: internal core kernel usage 3427 * 3428 * This is a callback function used by the irq_set_affinity_notifier function 3429 * to inform the current notification subscriber that they will no longer 3430 * receive notifications. 3431 **/ 3432 static void i40e_irq_affinity_release(struct kref *ref) {} 3433 3434 /** 3435 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3436 * @vsi: the VSI being configured 3437 * @basename: name for the vector 3438 * 3439 * Allocates MSI-X vectors and requests interrupts from the kernel. 3440 **/ 3441 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3442 { 3443 int q_vectors = vsi->num_q_vectors; 3444 struct i40e_pf *pf = vsi->back; 3445 int base = vsi->base_vector; 3446 int rx_int_idx = 0; 3447 int tx_int_idx = 0; 3448 int vector, err; 3449 int irq_num; 3450 3451 for (vector = 0; vector < q_vectors; vector++) { 3452 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3453 3454 irq_num = pf->msix_entries[base + vector].vector; 3455 3456 if (q_vector->tx.ring && q_vector->rx.ring) { 3457 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3458 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3459 tx_int_idx++; 3460 } else if (q_vector->rx.ring) { 3461 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3462 "%s-%s-%d", basename, "rx", rx_int_idx++); 3463 } else if (q_vector->tx.ring) { 3464 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3465 "%s-%s-%d", basename, "tx", tx_int_idx++); 3466 } else { 3467 /* skip this unused q_vector */ 3468 continue; 3469 } 3470 err = request_irq(irq_num, 3471 vsi->irq_handler, 3472 0, 3473 q_vector->name, 3474 q_vector); 3475 if (err) { 3476 dev_info(&pf->pdev->dev, 3477 "MSIX request_irq failed, error: %d\n", err); 3478 goto free_queue_irqs; 3479 } 3480 3481 /* register for affinity change notifications */ 3482 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3483 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3484 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3485 /* assign the mask for this irq */ 3486 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 3487 } 3488 3489 vsi->irqs_ready = true; 3490 return 0; 3491 3492 free_queue_irqs: 3493 while (vector) { 3494 vector--; 3495 irq_num = pf->msix_entries[base + vector].vector; 3496 irq_set_affinity_notifier(irq_num, NULL); 3497 irq_set_affinity_hint(irq_num, NULL); 3498 free_irq(irq_num, &vsi->q_vectors[vector]); 3499 } 3500 return err; 3501 } 3502 3503 /** 3504 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3505 * @vsi: the VSI being un-configured 3506 **/ 3507 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3508 { 3509 struct i40e_pf *pf = vsi->back; 3510 struct i40e_hw *hw = &pf->hw; 3511 int base = vsi->base_vector; 3512 int i; 3513 3514 for (i = 0; i < vsi->num_queue_pairs; i++) { 3515 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0); 3516 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0); 3517 } 3518 3519 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3520 for (i = vsi->base_vector; 3521 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3522 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3523 3524 i40e_flush(hw); 3525 for (i = 0; i < vsi->num_q_vectors; i++) 3526 synchronize_irq(pf->msix_entries[i + base].vector); 3527 } else { 3528 /* Legacy and MSI mode - this stops all interrupt handling */ 3529 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3530 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3531 i40e_flush(hw); 3532 synchronize_irq(pf->pdev->irq); 3533 } 3534 } 3535 3536 /** 3537 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3538 * @vsi: the VSI being configured 3539 **/ 3540 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3541 { 3542 struct i40e_pf *pf = vsi->back; 3543 int i; 3544 3545 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3546 for (i = 0; i < vsi->num_q_vectors; i++) 3547 i40e_irq_dynamic_enable(vsi, i); 3548 } else { 3549 i40e_irq_dynamic_enable_icr0(pf, true); 3550 } 3551 3552 i40e_flush(&pf->hw); 3553 return 0; 3554 } 3555 3556 /** 3557 * i40e_stop_misc_vector - Stop the vector that handles non-queue events 3558 * @pf: board private structure 3559 **/ 3560 static void i40e_stop_misc_vector(struct i40e_pf *pf) 3561 { 3562 /* Disable ICR 0 */ 3563 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3564 i40e_flush(&pf->hw); 3565 } 3566 3567 /** 3568 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3569 * @irq: interrupt number 3570 * @data: pointer to a q_vector 3571 * 3572 * This is the handler used for all MSI/Legacy interrupts, and deals 3573 * with both queue and non-queue interrupts. This is also used in 3574 * MSIX mode to handle the non-queue interrupts. 3575 **/ 3576 static irqreturn_t i40e_intr(int irq, void *data) 3577 { 3578 struct i40e_pf *pf = (struct i40e_pf *)data; 3579 struct i40e_hw *hw = &pf->hw; 3580 irqreturn_t ret = IRQ_NONE; 3581 u32 icr0, icr0_remaining; 3582 u32 val, ena_mask; 3583 3584 icr0 = rd32(hw, I40E_PFINT_ICR0); 3585 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3586 3587 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3588 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3589 goto enable_intr; 3590 3591 /* if interrupt but no bits showing, must be SWINT */ 3592 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3593 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3594 pf->sw_int_count++; 3595 3596 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3597 (ena_mask & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3598 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3599 icr0 &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3600 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3601 } 3602 3603 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3604 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3605 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3606 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3607 3608 /* We do not have a way to disarm Queue causes while leaving 3609 * interrupt enabled for all other causes, ideally 3610 * interrupt should be disabled while we are in NAPI but 3611 * this is not a performance path and napi_schedule() 3612 * can deal with rescheduling. 3613 */ 3614 if (!test_bit(__I40E_VSI_DOWN, pf->state)) 3615 napi_schedule_irqoff(&q_vector->napi); 3616 } 3617 3618 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3619 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3620 set_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 3621 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3622 } 3623 3624 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3625 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3626 set_bit(__I40E_MDD_EVENT_PENDING, pf->state); 3627 } 3628 3629 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3630 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3631 set_bit(__I40E_VFLR_EVENT_PENDING, pf->state); 3632 } 3633 3634 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3635 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 3636 set_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 3637 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3638 val = rd32(hw, I40E_GLGEN_RSTAT); 3639 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3640 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3641 if (val == I40E_RESET_CORER) { 3642 pf->corer_count++; 3643 } else if (val == I40E_RESET_GLOBR) { 3644 pf->globr_count++; 3645 } else if (val == I40E_RESET_EMPR) { 3646 pf->empr_count++; 3647 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state); 3648 } 3649 } 3650 3651 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3652 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3653 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3654 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3655 rd32(hw, I40E_PFHMC_ERRORINFO), 3656 rd32(hw, I40E_PFHMC_ERRORDATA)); 3657 } 3658 3659 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3660 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3661 3662 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3663 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3664 i40e_ptp_tx_hwtstamp(pf); 3665 } 3666 } 3667 3668 /* If a critical error is pending we have no choice but to reset the 3669 * device. 3670 * Report and mask out any remaining unexpected interrupts. 3671 */ 3672 icr0_remaining = icr0 & ena_mask; 3673 if (icr0_remaining) { 3674 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3675 icr0_remaining); 3676 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3677 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3678 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3679 dev_info(&pf->pdev->dev, "device will be reset\n"); 3680 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 3681 i40e_service_event_schedule(pf); 3682 } 3683 ena_mask &= ~icr0_remaining; 3684 } 3685 ret = IRQ_HANDLED; 3686 3687 enable_intr: 3688 /* re-enable interrupt causes */ 3689 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3690 if (!test_bit(__I40E_VSI_DOWN, pf->state)) { 3691 i40e_service_event_schedule(pf); 3692 i40e_irq_dynamic_enable_icr0(pf, false); 3693 } 3694 3695 return ret; 3696 } 3697 3698 /** 3699 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3700 * @tx_ring: tx ring to clean 3701 * @budget: how many cleans we're allowed 3702 * 3703 * Returns true if there's any budget left (e.g. the clean is finished) 3704 **/ 3705 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3706 { 3707 struct i40e_vsi *vsi = tx_ring->vsi; 3708 u16 i = tx_ring->next_to_clean; 3709 struct i40e_tx_buffer *tx_buf; 3710 struct i40e_tx_desc *tx_desc; 3711 3712 tx_buf = &tx_ring->tx_bi[i]; 3713 tx_desc = I40E_TX_DESC(tx_ring, i); 3714 i -= tx_ring->count; 3715 3716 do { 3717 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3718 3719 /* if next_to_watch is not set then there is no work pending */ 3720 if (!eop_desc) 3721 break; 3722 3723 /* prevent any other reads prior to eop_desc */ 3724 read_barrier_depends(); 3725 3726 /* if the descriptor isn't done, no work yet to do */ 3727 if (!(eop_desc->cmd_type_offset_bsz & 3728 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3729 break; 3730 3731 /* clear next_to_watch to prevent false hangs */ 3732 tx_buf->next_to_watch = NULL; 3733 3734 tx_desc->buffer_addr = 0; 3735 tx_desc->cmd_type_offset_bsz = 0; 3736 /* move past filter desc */ 3737 tx_buf++; 3738 tx_desc++; 3739 i++; 3740 if (unlikely(!i)) { 3741 i -= tx_ring->count; 3742 tx_buf = tx_ring->tx_bi; 3743 tx_desc = I40E_TX_DESC(tx_ring, 0); 3744 } 3745 /* unmap skb header data */ 3746 dma_unmap_single(tx_ring->dev, 3747 dma_unmap_addr(tx_buf, dma), 3748 dma_unmap_len(tx_buf, len), 3749 DMA_TO_DEVICE); 3750 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 3751 kfree(tx_buf->raw_buf); 3752 3753 tx_buf->raw_buf = NULL; 3754 tx_buf->tx_flags = 0; 3755 tx_buf->next_to_watch = NULL; 3756 dma_unmap_len_set(tx_buf, len, 0); 3757 tx_desc->buffer_addr = 0; 3758 tx_desc->cmd_type_offset_bsz = 0; 3759 3760 /* move us past the eop_desc for start of next FD desc */ 3761 tx_buf++; 3762 tx_desc++; 3763 i++; 3764 if (unlikely(!i)) { 3765 i -= tx_ring->count; 3766 tx_buf = tx_ring->tx_bi; 3767 tx_desc = I40E_TX_DESC(tx_ring, 0); 3768 } 3769 3770 /* update budget accounting */ 3771 budget--; 3772 } while (likely(budget)); 3773 3774 i += tx_ring->count; 3775 tx_ring->next_to_clean = i; 3776 3777 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 3778 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 3779 3780 return budget > 0; 3781 } 3782 3783 /** 3784 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 3785 * @irq: interrupt number 3786 * @data: pointer to a q_vector 3787 **/ 3788 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 3789 { 3790 struct i40e_q_vector *q_vector = data; 3791 struct i40e_vsi *vsi; 3792 3793 if (!q_vector->tx.ring) 3794 return IRQ_HANDLED; 3795 3796 vsi = q_vector->tx.ring->vsi; 3797 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 3798 3799 return IRQ_HANDLED; 3800 } 3801 3802 /** 3803 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 3804 * @vsi: the VSI being configured 3805 * @v_idx: vector index 3806 * @qp_idx: queue pair index 3807 **/ 3808 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 3809 { 3810 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 3811 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 3812 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 3813 3814 tx_ring->q_vector = q_vector; 3815 tx_ring->next = q_vector->tx.ring; 3816 q_vector->tx.ring = tx_ring; 3817 q_vector->tx.count++; 3818 3819 rx_ring->q_vector = q_vector; 3820 rx_ring->next = q_vector->rx.ring; 3821 q_vector->rx.ring = rx_ring; 3822 q_vector->rx.count++; 3823 } 3824 3825 /** 3826 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 3827 * @vsi: the VSI being configured 3828 * 3829 * This function maps descriptor rings to the queue-specific vectors 3830 * we were allotted through the MSI-X enabling code. Ideally, we'd have 3831 * one vector per queue pair, but on a constrained vector budget, we 3832 * group the queue pairs as "efficiently" as possible. 3833 **/ 3834 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 3835 { 3836 int qp_remaining = vsi->num_queue_pairs; 3837 int q_vectors = vsi->num_q_vectors; 3838 int num_ringpairs; 3839 int v_start = 0; 3840 int qp_idx = 0; 3841 3842 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 3843 * group them so there are multiple queues per vector. 3844 * It is also important to go through all the vectors available to be 3845 * sure that if we don't use all the vectors, that the remaining vectors 3846 * are cleared. This is especially important when decreasing the 3847 * number of queues in use. 3848 */ 3849 for (; v_start < q_vectors; v_start++) { 3850 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 3851 3852 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 3853 3854 q_vector->num_ringpairs = num_ringpairs; 3855 3856 q_vector->rx.count = 0; 3857 q_vector->tx.count = 0; 3858 q_vector->rx.ring = NULL; 3859 q_vector->tx.ring = NULL; 3860 3861 while (num_ringpairs--) { 3862 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 3863 qp_idx++; 3864 qp_remaining--; 3865 } 3866 } 3867 } 3868 3869 /** 3870 * i40e_vsi_request_irq - Request IRQ from the OS 3871 * @vsi: the VSI being configured 3872 * @basename: name for the vector 3873 **/ 3874 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 3875 { 3876 struct i40e_pf *pf = vsi->back; 3877 int err; 3878 3879 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 3880 err = i40e_vsi_request_irq_msix(vsi, basename); 3881 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 3882 err = request_irq(pf->pdev->irq, i40e_intr, 0, 3883 pf->int_name, pf); 3884 else 3885 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 3886 pf->int_name, pf); 3887 3888 if (err) 3889 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 3890 3891 return err; 3892 } 3893 3894 #ifdef CONFIG_NET_POLL_CONTROLLER 3895 /** 3896 * i40e_netpoll - A Polling 'interrupt' handler 3897 * @netdev: network interface device structure 3898 * 3899 * This is used by netconsole to send skbs without having to re-enable 3900 * interrupts. It's not called while the normal interrupt routine is executing. 3901 **/ 3902 static void i40e_netpoll(struct net_device *netdev) 3903 { 3904 struct i40e_netdev_priv *np = netdev_priv(netdev); 3905 struct i40e_vsi *vsi = np->vsi; 3906 struct i40e_pf *pf = vsi->back; 3907 int i; 3908 3909 /* if interface is down do nothing */ 3910 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 3911 return; 3912 3913 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3914 for (i = 0; i < vsi->num_q_vectors; i++) 3915 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 3916 } else { 3917 i40e_intr(pf->pdev->irq, netdev); 3918 } 3919 } 3920 #endif 3921 3922 #define I40E_QTX_ENA_WAIT_COUNT 50 3923 3924 /** 3925 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 3926 * @pf: the PF being configured 3927 * @pf_q: the PF queue 3928 * @enable: enable or disable state of the queue 3929 * 3930 * This routine will wait for the given Tx queue of the PF to reach the 3931 * enabled or disabled state. 3932 * Returns -ETIMEDOUT in case of failing to reach the requested state after 3933 * multiple retries; else will return 0 in case of success. 3934 **/ 3935 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 3936 { 3937 int i; 3938 u32 tx_reg; 3939 3940 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 3941 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 3942 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3943 break; 3944 3945 usleep_range(10, 20); 3946 } 3947 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 3948 return -ETIMEDOUT; 3949 3950 return 0; 3951 } 3952 3953 /** 3954 * i40e_control_tx_q - Start or stop a particular Tx queue 3955 * @pf: the PF structure 3956 * @pf_q: the PF queue to configure 3957 * @enable: start or stop the queue 3958 * 3959 * This function enables or disables a single queue. Note that any delay 3960 * required after the operation is expected to be handled by the caller of 3961 * this function. 3962 **/ 3963 static void i40e_control_tx_q(struct i40e_pf *pf, int pf_q, bool enable) 3964 { 3965 struct i40e_hw *hw = &pf->hw; 3966 u32 tx_reg; 3967 int i; 3968 3969 /* warn the TX unit of coming changes */ 3970 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 3971 if (!enable) 3972 usleep_range(10, 20); 3973 3974 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 3975 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 3976 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 3977 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 3978 break; 3979 usleep_range(1000, 2000); 3980 } 3981 3982 /* Skip if the queue is already in the requested state */ 3983 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3984 return; 3985 3986 /* turn on/off the queue */ 3987 if (enable) { 3988 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 3989 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 3990 } else { 3991 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 3992 } 3993 3994 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 3995 } 3996 3997 /** 3998 * i40e_vsi_control_tx - Start or stop a VSI's rings 3999 * @vsi: the VSI being configured 4000 * @enable: start or stop the rings 4001 **/ 4002 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 4003 { 4004 struct i40e_pf *pf = vsi->back; 4005 int i, pf_q, ret = 0; 4006 4007 pf_q = vsi->base_queue; 4008 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4009 i40e_control_tx_q(pf, pf_q, enable); 4010 4011 /* wait for the change to finish */ 4012 ret = i40e_pf_txq_wait(pf, pf_q, enable); 4013 if (ret) { 4014 dev_info(&pf->pdev->dev, 4015 "VSI seid %d Tx ring %d %sable timeout\n", 4016 vsi->seid, pf_q, (enable ? "en" : "dis")); 4017 break; 4018 } 4019 } 4020 4021 return ret; 4022 } 4023 4024 /** 4025 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 4026 * @pf: the PF being configured 4027 * @pf_q: the PF queue 4028 * @enable: enable or disable state of the queue 4029 * 4030 * This routine will wait for the given Rx queue of the PF to reach the 4031 * enabled or disabled state. 4032 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4033 * multiple retries; else will return 0 in case of success. 4034 **/ 4035 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4036 { 4037 int i; 4038 u32 rx_reg; 4039 4040 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4041 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4042 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4043 break; 4044 4045 usleep_range(10, 20); 4046 } 4047 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4048 return -ETIMEDOUT; 4049 4050 return 0; 4051 } 4052 4053 /** 4054 * i40e_control_rx_q - Start or stop a particular Rx queue 4055 * @pf: the PF structure 4056 * @pf_q: the PF queue to configure 4057 * @enable: start or stop the queue 4058 * 4059 * This function enables or disables a single queue. Note that any delay 4060 * required after the operation is expected to be handled by the caller of 4061 * this function. 4062 **/ 4063 static void i40e_control_rx_q(struct i40e_pf *pf, int pf_q, bool enable) 4064 { 4065 struct i40e_hw *hw = &pf->hw; 4066 u32 rx_reg; 4067 int i; 4068 4069 for (i = 0; i < I40E_QTX_ENA_WAIT_COUNT; i++) { 4070 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4071 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4072 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4073 break; 4074 usleep_range(1000, 2000); 4075 } 4076 4077 /* Skip if the queue is already in the requested state */ 4078 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4079 return; 4080 4081 /* turn on/off the queue */ 4082 if (enable) 4083 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4084 else 4085 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4086 4087 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4088 } 4089 4090 /** 4091 * i40e_vsi_control_rx - Start or stop a VSI's rings 4092 * @vsi: the VSI being configured 4093 * @enable: start or stop the rings 4094 **/ 4095 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4096 { 4097 struct i40e_pf *pf = vsi->back; 4098 int i, pf_q, ret = 0; 4099 4100 pf_q = vsi->base_queue; 4101 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4102 i40e_control_rx_q(pf, pf_q, enable); 4103 4104 /* wait for the change to finish */ 4105 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4106 if (ret) { 4107 dev_info(&pf->pdev->dev, 4108 "VSI seid %d Rx ring %d %sable timeout\n", 4109 vsi->seid, pf_q, (enable ? "en" : "dis")); 4110 break; 4111 } 4112 } 4113 4114 /* Due to HW errata, on Rx disable only, the register can indicate done 4115 * before it really is. Needs 50ms to be sure 4116 */ 4117 if (!enable) 4118 mdelay(50); 4119 4120 return ret; 4121 } 4122 4123 /** 4124 * i40e_vsi_start_rings - Start a VSI's rings 4125 * @vsi: the VSI being configured 4126 **/ 4127 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4128 { 4129 int ret = 0; 4130 4131 /* do rx first for enable and last for disable */ 4132 ret = i40e_vsi_control_rx(vsi, true); 4133 if (ret) 4134 return ret; 4135 ret = i40e_vsi_control_tx(vsi, true); 4136 4137 return ret; 4138 } 4139 4140 /** 4141 * i40e_vsi_stop_rings - Stop a VSI's rings 4142 * @vsi: the VSI being configured 4143 **/ 4144 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4145 { 4146 /* When port TX is suspended, don't wait */ 4147 if (test_bit(__I40E_PORT_SUSPENDED, vsi->back->state)) 4148 return i40e_vsi_stop_rings_no_wait(vsi); 4149 4150 /* do rx first for enable and last for disable 4151 * Ignore return value, we need to shutdown whatever we can 4152 */ 4153 i40e_vsi_control_tx(vsi, false); 4154 i40e_vsi_control_rx(vsi, false); 4155 } 4156 4157 /** 4158 * i40e_vsi_stop_rings_no_wait - Stop a VSI's rings and do not delay 4159 * @vsi: the VSI being shutdown 4160 * 4161 * This function stops all the rings for a VSI but does not delay to verify 4162 * that rings have been disabled. It is expected that the caller is shutting 4163 * down multiple VSIs at once and will delay together for all the VSIs after 4164 * initiating the shutdown. This is particularly useful for shutting down lots 4165 * of VFs together. Otherwise, a large delay can be incurred while configuring 4166 * each VSI in serial. 4167 **/ 4168 void i40e_vsi_stop_rings_no_wait(struct i40e_vsi *vsi) 4169 { 4170 struct i40e_pf *pf = vsi->back; 4171 int i, pf_q; 4172 4173 pf_q = vsi->base_queue; 4174 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4175 i40e_control_tx_q(pf, pf_q, false); 4176 i40e_control_rx_q(pf, pf_q, false); 4177 } 4178 } 4179 4180 /** 4181 * i40e_vsi_free_irq - Free the irq association with the OS 4182 * @vsi: the VSI being configured 4183 **/ 4184 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4185 { 4186 struct i40e_pf *pf = vsi->back; 4187 struct i40e_hw *hw = &pf->hw; 4188 int base = vsi->base_vector; 4189 u32 val, qp; 4190 int i; 4191 4192 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4193 if (!vsi->q_vectors) 4194 return; 4195 4196 if (!vsi->irqs_ready) 4197 return; 4198 4199 vsi->irqs_ready = false; 4200 for (i = 0; i < vsi->num_q_vectors; i++) { 4201 int irq_num; 4202 u16 vector; 4203 4204 vector = i + base; 4205 irq_num = pf->msix_entries[vector].vector; 4206 4207 /* free only the irqs that were actually requested */ 4208 if (!vsi->q_vectors[i] || 4209 !vsi->q_vectors[i]->num_ringpairs) 4210 continue; 4211 4212 /* clear the affinity notifier in the IRQ descriptor */ 4213 irq_set_affinity_notifier(irq_num, NULL); 4214 /* clear the affinity_mask in the IRQ descriptor */ 4215 irq_set_affinity_hint(irq_num, NULL); 4216 synchronize_irq(irq_num); 4217 free_irq(irq_num, vsi->q_vectors[i]); 4218 4219 /* Tear down the interrupt queue link list 4220 * 4221 * We know that they come in pairs and always 4222 * the Rx first, then the Tx. To clear the 4223 * link list, stick the EOL value into the 4224 * next_q field of the registers. 4225 */ 4226 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4227 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4228 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4229 val |= I40E_QUEUE_END_OF_LIST 4230 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4231 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4232 4233 while (qp != I40E_QUEUE_END_OF_LIST) { 4234 u32 next; 4235 4236 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4237 4238 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4239 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4240 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4241 I40E_QINT_RQCTL_INTEVENT_MASK); 4242 4243 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4244 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4245 4246 wr32(hw, I40E_QINT_RQCTL(qp), val); 4247 4248 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4249 4250 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4251 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4252 4253 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4254 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4255 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4256 I40E_QINT_TQCTL_INTEVENT_MASK); 4257 4258 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4259 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4260 4261 wr32(hw, I40E_QINT_TQCTL(qp), val); 4262 qp = next; 4263 } 4264 } 4265 } else { 4266 free_irq(pf->pdev->irq, pf); 4267 4268 val = rd32(hw, I40E_PFINT_LNKLST0); 4269 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4270 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4271 val |= I40E_QUEUE_END_OF_LIST 4272 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4273 wr32(hw, I40E_PFINT_LNKLST0, val); 4274 4275 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4276 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4277 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4278 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4279 I40E_QINT_RQCTL_INTEVENT_MASK); 4280 4281 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4282 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4283 4284 wr32(hw, I40E_QINT_RQCTL(qp), val); 4285 4286 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4287 4288 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4289 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4290 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4291 I40E_QINT_TQCTL_INTEVENT_MASK); 4292 4293 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4294 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4295 4296 wr32(hw, I40E_QINT_TQCTL(qp), val); 4297 } 4298 } 4299 4300 /** 4301 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4302 * @vsi: the VSI being configured 4303 * @v_idx: Index of vector to be freed 4304 * 4305 * This function frees the memory allocated to the q_vector. In addition if 4306 * NAPI is enabled it will delete any references to the NAPI struct prior 4307 * to freeing the q_vector. 4308 **/ 4309 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4310 { 4311 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4312 struct i40e_ring *ring; 4313 4314 if (!q_vector) 4315 return; 4316 4317 /* disassociate q_vector from rings */ 4318 i40e_for_each_ring(ring, q_vector->tx) 4319 ring->q_vector = NULL; 4320 4321 i40e_for_each_ring(ring, q_vector->rx) 4322 ring->q_vector = NULL; 4323 4324 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4325 if (vsi->netdev) 4326 netif_napi_del(&q_vector->napi); 4327 4328 vsi->q_vectors[v_idx] = NULL; 4329 4330 kfree_rcu(q_vector, rcu); 4331 } 4332 4333 /** 4334 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4335 * @vsi: the VSI being un-configured 4336 * 4337 * This frees the memory allocated to the q_vectors and 4338 * deletes references to the NAPI struct. 4339 **/ 4340 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4341 { 4342 int v_idx; 4343 4344 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4345 i40e_free_q_vector(vsi, v_idx); 4346 } 4347 4348 /** 4349 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4350 * @pf: board private structure 4351 **/ 4352 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4353 { 4354 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4355 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4356 pci_disable_msix(pf->pdev); 4357 kfree(pf->msix_entries); 4358 pf->msix_entries = NULL; 4359 kfree(pf->irq_pile); 4360 pf->irq_pile = NULL; 4361 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4362 pci_disable_msi(pf->pdev); 4363 } 4364 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4365 } 4366 4367 /** 4368 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4369 * @pf: board private structure 4370 * 4371 * We go through and clear interrupt specific resources and reset the structure 4372 * to pre-load conditions 4373 **/ 4374 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4375 { 4376 int i; 4377 4378 i40e_stop_misc_vector(pf); 4379 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 4380 synchronize_irq(pf->msix_entries[0].vector); 4381 free_irq(pf->msix_entries[0].vector, pf); 4382 } 4383 4384 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4385 I40E_IWARP_IRQ_PILE_ID); 4386 4387 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4388 for (i = 0; i < pf->num_alloc_vsi; i++) 4389 if (pf->vsi[i]) 4390 i40e_vsi_free_q_vectors(pf->vsi[i]); 4391 i40e_reset_interrupt_capability(pf); 4392 } 4393 4394 /** 4395 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4396 * @vsi: the VSI being configured 4397 **/ 4398 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4399 { 4400 int q_idx; 4401 4402 if (!vsi->netdev) 4403 return; 4404 4405 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4406 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4407 4408 if (q_vector->rx.ring || q_vector->tx.ring) 4409 napi_enable(&q_vector->napi); 4410 } 4411 } 4412 4413 /** 4414 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4415 * @vsi: the VSI being configured 4416 **/ 4417 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4418 { 4419 int q_idx; 4420 4421 if (!vsi->netdev) 4422 return; 4423 4424 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4425 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4426 4427 if (q_vector->rx.ring || q_vector->tx.ring) 4428 napi_disable(&q_vector->napi); 4429 } 4430 } 4431 4432 /** 4433 * i40e_vsi_close - Shut down a VSI 4434 * @vsi: the vsi to be quelled 4435 **/ 4436 static void i40e_vsi_close(struct i40e_vsi *vsi) 4437 { 4438 struct i40e_pf *pf = vsi->back; 4439 if (!test_and_set_bit(__I40E_VSI_DOWN, vsi->state)) 4440 i40e_down(vsi); 4441 i40e_vsi_free_irq(vsi); 4442 i40e_vsi_free_tx_resources(vsi); 4443 i40e_vsi_free_rx_resources(vsi); 4444 vsi->current_netdev_flags = 0; 4445 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 4446 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 4447 pf->flags |= I40E_FLAG_CLIENT_RESET; 4448 } 4449 4450 /** 4451 * i40e_quiesce_vsi - Pause a given VSI 4452 * @vsi: the VSI being paused 4453 **/ 4454 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4455 { 4456 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 4457 return; 4458 4459 set_bit(__I40E_VSI_NEEDS_RESTART, vsi->state); 4460 if (vsi->netdev && netif_running(vsi->netdev)) 4461 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4462 else 4463 i40e_vsi_close(vsi); 4464 } 4465 4466 /** 4467 * i40e_unquiesce_vsi - Resume a given VSI 4468 * @vsi: the VSI being resumed 4469 **/ 4470 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4471 { 4472 if (!test_and_clear_bit(__I40E_VSI_NEEDS_RESTART, vsi->state)) 4473 return; 4474 4475 if (vsi->netdev && netif_running(vsi->netdev)) 4476 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4477 else 4478 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4479 } 4480 4481 /** 4482 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4483 * @pf: the PF 4484 **/ 4485 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4486 { 4487 int v; 4488 4489 for (v = 0; v < pf->num_alloc_vsi; v++) { 4490 if (pf->vsi[v]) 4491 i40e_quiesce_vsi(pf->vsi[v]); 4492 } 4493 } 4494 4495 /** 4496 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4497 * @pf: the PF 4498 **/ 4499 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4500 { 4501 int v; 4502 4503 for (v = 0; v < pf->num_alloc_vsi; v++) { 4504 if (pf->vsi[v]) 4505 i40e_unquiesce_vsi(pf->vsi[v]); 4506 } 4507 } 4508 4509 /** 4510 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4511 * @vsi: the VSI being configured 4512 * 4513 * Wait until all queues on a given VSI have been disabled. 4514 **/ 4515 int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4516 { 4517 struct i40e_pf *pf = vsi->back; 4518 int i, pf_q, ret; 4519 4520 pf_q = vsi->base_queue; 4521 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4522 /* Check and wait for the Tx queue */ 4523 ret = i40e_pf_txq_wait(pf, pf_q, false); 4524 if (ret) { 4525 dev_info(&pf->pdev->dev, 4526 "VSI seid %d Tx ring %d disable timeout\n", 4527 vsi->seid, pf_q); 4528 return ret; 4529 } 4530 /* Check and wait for the Tx queue */ 4531 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4532 if (ret) { 4533 dev_info(&pf->pdev->dev, 4534 "VSI seid %d Rx ring %d disable timeout\n", 4535 vsi->seid, pf_q); 4536 return ret; 4537 } 4538 } 4539 4540 return 0; 4541 } 4542 4543 #ifdef CONFIG_I40E_DCB 4544 /** 4545 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4546 * @pf: the PF 4547 * 4548 * This function waits for the queues to be in disabled state for all the 4549 * VSIs that are managed by this PF. 4550 **/ 4551 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4552 { 4553 int v, ret = 0; 4554 4555 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4556 if (pf->vsi[v]) { 4557 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4558 if (ret) 4559 break; 4560 } 4561 } 4562 4563 return ret; 4564 } 4565 4566 #endif 4567 4568 /** 4569 * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue 4570 * @q_idx: TX queue number 4571 * @vsi: Pointer to VSI struct 4572 * 4573 * This function checks specified queue for given VSI. Detects hung condition. 4574 * We proactively detect hung TX queues by checking if interrupts are disabled 4575 * but there are pending descriptors. If it appears hung, attempt to recover 4576 * by triggering a SW interrupt. 4577 **/ 4578 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi) 4579 { 4580 struct i40e_ring *tx_ring = NULL; 4581 struct i40e_pf *pf; 4582 u32 val, tx_pending; 4583 int i; 4584 4585 pf = vsi->back; 4586 4587 /* now that we have an index, find the tx_ring struct */ 4588 for (i = 0; i < vsi->num_queue_pairs; i++) { 4589 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 4590 if (q_idx == vsi->tx_rings[i]->queue_index) { 4591 tx_ring = vsi->tx_rings[i]; 4592 break; 4593 } 4594 } 4595 } 4596 4597 if (!tx_ring) 4598 return; 4599 4600 /* Read interrupt register */ 4601 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4602 val = rd32(&pf->hw, 4603 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 4604 tx_ring->vsi->base_vector - 1)); 4605 else 4606 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 4607 4608 tx_pending = i40e_get_tx_pending(tx_ring); 4609 4610 /* Interrupts are disabled and TX pending is non-zero, 4611 * trigger the SW interrupt (don't wait). Worst case 4612 * there will be one extra interrupt which may result 4613 * into not cleaning any queues because queues are cleaned. 4614 */ 4615 if (tx_pending && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) 4616 i40e_force_wb(vsi, tx_ring->q_vector); 4617 } 4618 4619 /** 4620 * i40e_detect_recover_hung - Function to detect and recover hung_queues 4621 * @pf: pointer to PF struct 4622 * 4623 * LAN VSI has netdev and netdev has TX queues. This function is to check 4624 * each of those TX queues if they are hung, trigger recovery by issuing 4625 * SW interrupt. 4626 **/ 4627 static void i40e_detect_recover_hung(struct i40e_pf *pf) 4628 { 4629 struct net_device *netdev; 4630 struct i40e_vsi *vsi; 4631 int i; 4632 4633 /* Only for LAN VSI */ 4634 vsi = pf->vsi[pf->lan_vsi]; 4635 4636 if (!vsi) 4637 return; 4638 4639 /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */ 4640 if (test_bit(__I40E_VSI_DOWN, vsi->back->state) || 4641 test_bit(__I40E_RESET_RECOVERY_PENDING, vsi->back->state)) 4642 return; 4643 4644 /* Make sure type is MAIN VSI */ 4645 if (vsi->type != I40E_VSI_MAIN) 4646 return; 4647 4648 netdev = vsi->netdev; 4649 if (!netdev) 4650 return; 4651 4652 /* Bail out if netif_carrier is not OK */ 4653 if (!netif_carrier_ok(netdev)) 4654 return; 4655 4656 /* Go thru' TX queues for netdev */ 4657 for (i = 0; i < netdev->num_tx_queues; i++) { 4658 struct netdev_queue *q; 4659 4660 q = netdev_get_tx_queue(netdev, i); 4661 if (q) 4662 i40e_detect_recover_hung_queue(i, vsi); 4663 } 4664 } 4665 4666 /** 4667 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4668 * @pf: pointer to PF 4669 * 4670 * Get TC map for ISCSI PF type that will include iSCSI TC 4671 * and LAN TC. 4672 **/ 4673 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4674 { 4675 struct i40e_dcb_app_priority_table app; 4676 struct i40e_hw *hw = &pf->hw; 4677 u8 enabled_tc = 1; /* TC0 is always enabled */ 4678 u8 tc, i; 4679 /* Get the iSCSI APP TLV */ 4680 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4681 4682 for (i = 0; i < dcbcfg->numapps; i++) { 4683 app = dcbcfg->app[i]; 4684 if (app.selector == I40E_APP_SEL_TCPIP && 4685 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4686 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4687 enabled_tc |= BIT(tc); 4688 break; 4689 } 4690 } 4691 4692 return enabled_tc; 4693 } 4694 4695 /** 4696 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 4697 * @dcbcfg: the corresponding DCBx configuration structure 4698 * 4699 * Return the number of TCs from given DCBx configuration 4700 **/ 4701 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 4702 { 4703 int i, tc_unused = 0; 4704 u8 num_tc = 0; 4705 u8 ret = 0; 4706 4707 /* Scan the ETS Config Priority Table to find 4708 * traffic class enabled for a given priority 4709 * and create a bitmask of enabled TCs 4710 */ 4711 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 4712 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 4713 4714 /* Now scan the bitmask to check for 4715 * contiguous TCs starting with TC0 4716 */ 4717 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4718 if (num_tc & BIT(i)) { 4719 if (!tc_unused) { 4720 ret++; 4721 } else { 4722 pr_err("Non-contiguous TC - Disabling DCB\n"); 4723 return 1; 4724 } 4725 } else { 4726 tc_unused = 1; 4727 } 4728 } 4729 4730 /* There is always at least TC0 */ 4731 if (!ret) 4732 ret = 1; 4733 4734 return ret; 4735 } 4736 4737 /** 4738 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 4739 * @dcbcfg: the corresponding DCBx configuration structure 4740 * 4741 * Query the current DCB configuration and return the number of 4742 * traffic classes enabled from the given DCBX config 4743 **/ 4744 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 4745 { 4746 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 4747 u8 enabled_tc = 1; 4748 u8 i; 4749 4750 for (i = 0; i < num_tc; i++) 4751 enabled_tc |= BIT(i); 4752 4753 return enabled_tc; 4754 } 4755 4756 /** 4757 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 4758 * @pf: PF being queried 4759 * 4760 * Return number of traffic classes enabled for the given PF 4761 **/ 4762 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 4763 { 4764 struct i40e_hw *hw = &pf->hw; 4765 u8 i, enabled_tc = 1; 4766 u8 num_tc = 0; 4767 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4768 4769 /* If DCB is not enabled then always in single TC */ 4770 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4771 return 1; 4772 4773 /* SFP mode will be enabled for all TCs on port */ 4774 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4775 return i40e_dcb_get_num_tc(dcbcfg); 4776 4777 /* MFP mode return count of enabled TCs for this PF */ 4778 if (pf->hw.func_caps.iscsi) 4779 enabled_tc = i40e_get_iscsi_tc_map(pf); 4780 else 4781 return 1; /* Only TC0 */ 4782 4783 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4784 if (enabled_tc & BIT(i)) 4785 num_tc++; 4786 } 4787 return num_tc; 4788 } 4789 4790 /** 4791 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 4792 * @pf: PF being queried 4793 * 4794 * Return a bitmap for enabled traffic classes for this PF. 4795 **/ 4796 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 4797 { 4798 /* If DCB is not enabled for this PF then just return default TC */ 4799 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4800 return I40E_DEFAULT_TRAFFIC_CLASS; 4801 4802 /* SFP mode we want PF to be enabled for all TCs */ 4803 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4804 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 4805 4806 /* MFP enabled and iSCSI PF type */ 4807 if (pf->hw.func_caps.iscsi) 4808 return i40e_get_iscsi_tc_map(pf); 4809 else 4810 return I40E_DEFAULT_TRAFFIC_CLASS; 4811 } 4812 4813 /** 4814 * i40e_vsi_get_bw_info - Query VSI BW Information 4815 * @vsi: the VSI being queried 4816 * 4817 * Returns 0 on success, negative value on failure 4818 **/ 4819 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 4820 { 4821 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 4822 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 4823 struct i40e_pf *pf = vsi->back; 4824 struct i40e_hw *hw = &pf->hw; 4825 i40e_status ret; 4826 u32 tc_bw_max; 4827 int i; 4828 4829 /* Get the VSI level BW configuration */ 4830 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 4831 if (ret) { 4832 dev_info(&pf->pdev->dev, 4833 "couldn't get PF vsi bw config, err %s aq_err %s\n", 4834 i40e_stat_str(&pf->hw, ret), 4835 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4836 return -EINVAL; 4837 } 4838 4839 /* Get the VSI level BW configuration per TC */ 4840 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 4841 NULL); 4842 if (ret) { 4843 dev_info(&pf->pdev->dev, 4844 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 4845 i40e_stat_str(&pf->hw, ret), 4846 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4847 return -EINVAL; 4848 } 4849 4850 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 4851 dev_info(&pf->pdev->dev, 4852 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 4853 bw_config.tc_valid_bits, 4854 bw_ets_config.tc_valid_bits); 4855 /* Still continuing */ 4856 } 4857 4858 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 4859 vsi->bw_max_quanta = bw_config.max_bw; 4860 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 4861 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 4862 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4863 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 4864 vsi->bw_ets_limit_credits[i] = 4865 le16_to_cpu(bw_ets_config.credits[i]); 4866 /* 3 bits out of 4 for each TC */ 4867 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 4868 } 4869 4870 return 0; 4871 } 4872 4873 /** 4874 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 4875 * @vsi: the VSI being configured 4876 * @enabled_tc: TC bitmap 4877 * @bw_credits: BW shared credits per TC 4878 * 4879 * Returns 0 on success, negative value on failure 4880 **/ 4881 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 4882 u8 *bw_share) 4883 { 4884 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 4885 i40e_status ret; 4886 int i; 4887 4888 bw_data.tc_valid_bits = enabled_tc; 4889 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4890 bw_data.tc_bw_credits[i] = bw_share[i]; 4891 4892 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 4893 NULL); 4894 if (ret) { 4895 dev_info(&vsi->back->pdev->dev, 4896 "AQ command Config VSI BW allocation per TC failed = %d\n", 4897 vsi->back->hw.aq.asq_last_status); 4898 return -EINVAL; 4899 } 4900 4901 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4902 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 4903 4904 return 0; 4905 } 4906 4907 /** 4908 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 4909 * @vsi: the VSI being configured 4910 * @enabled_tc: TC map to be enabled 4911 * 4912 **/ 4913 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 4914 { 4915 struct net_device *netdev = vsi->netdev; 4916 struct i40e_pf *pf = vsi->back; 4917 struct i40e_hw *hw = &pf->hw; 4918 u8 netdev_tc = 0; 4919 int i; 4920 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4921 4922 if (!netdev) 4923 return; 4924 4925 if (!enabled_tc) { 4926 netdev_reset_tc(netdev); 4927 return; 4928 } 4929 4930 /* Set up actual enabled TCs on the VSI */ 4931 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 4932 return; 4933 4934 /* set per TC queues for the VSI */ 4935 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4936 /* Only set TC queues for enabled tcs 4937 * 4938 * e.g. For a VSI that has TC0 and TC3 enabled the 4939 * enabled_tc bitmap would be 0x00001001; the driver 4940 * will set the numtc for netdev as 2 that will be 4941 * referenced by the netdev layer as TC 0 and 1. 4942 */ 4943 if (vsi->tc_config.enabled_tc & BIT(i)) 4944 netdev_set_tc_queue(netdev, 4945 vsi->tc_config.tc_info[i].netdev_tc, 4946 vsi->tc_config.tc_info[i].qcount, 4947 vsi->tc_config.tc_info[i].qoffset); 4948 } 4949 4950 /* Assign UP2TC map for the VSI */ 4951 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 4952 /* Get the actual TC# for the UP */ 4953 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 4954 /* Get the mapped netdev TC# for the UP */ 4955 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 4956 netdev_set_prio_tc_map(netdev, i, netdev_tc); 4957 } 4958 } 4959 4960 /** 4961 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 4962 * @vsi: the VSI being configured 4963 * @ctxt: the ctxt buffer returned from AQ VSI update param command 4964 **/ 4965 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 4966 struct i40e_vsi_context *ctxt) 4967 { 4968 /* copy just the sections touched not the entire info 4969 * since not all sections are valid as returned by 4970 * update vsi params 4971 */ 4972 vsi->info.mapping_flags = ctxt->info.mapping_flags; 4973 memcpy(&vsi->info.queue_mapping, 4974 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 4975 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 4976 sizeof(vsi->info.tc_mapping)); 4977 } 4978 4979 /** 4980 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 4981 * @vsi: VSI to be configured 4982 * @enabled_tc: TC bitmap 4983 * 4984 * This configures a particular VSI for TCs that are mapped to the 4985 * given TC bitmap. It uses default bandwidth share for TCs across 4986 * VSIs to configure TC for a particular VSI. 4987 * 4988 * NOTE: 4989 * It is expected that the VSI queues have been quisced before calling 4990 * this function. 4991 **/ 4992 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 4993 { 4994 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 4995 struct i40e_vsi_context ctxt; 4996 int ret = 0; 4997 int i; 4998 4999 /* Check if enabled_tc is same as existing or new TCs */ 5000 if (vsi->tc_config.enabled_tc == enabled_tc) 5001 return ret; 5002 5003 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 5004 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5005 if (enabled_tc & BIT(i)) 5006 bw_share[i] = 1; 5007 } 5008 5009 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 5010 if (ret) { 5011 dev_info(&vsi->back->pdev->dev, 5012 "Failed configuring TC map %d for VSI %d\n", 5013 enabled_tc, vsi->seid); 5014 goto out; 5015 } 5016 5017 /* Update Queue Pairs Mapping for currently enabled UPs */ 5018 ctxt.seid = vsi->seid; 5019 ctxt.pf_num = vsi->back->hw.pf_id; 5020 ctxt.vf_num = 0; 5021 ctxt.uplink_seid = vsi->uplink_seid; 5022 ctxt.info = vsi->info; 5023 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 5024 5025 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 5026 ctxt.info.valid_sections |= 5027 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 5028 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 5029 } 5030 5031 /* Update the VSI after updating the VSI queue-mapping information */ 5032 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 5033 if (ret) { 5034 dev_info(&vsi->back->pdev->dev, 5035 "Update vsi tc config failed, err %s aq_err %s\n", 5036 i40e_stat_str(&vsi->back->hw, ret), 5037 i40e_aq_str(&vsi->back->hw, 5038 vsi->back->hw.aq.asq_last_status)); 5039 goto out; 5040 } 5041 /* update the local VSI info with updated queue map */ 5042 i40e_vsi_update_queue_map(vsi, &ctxt); 5043 vsi->info.valid_sections = 0; 5044 5045 /* Update current VSI BW information */ 5046 ret = i40e_vsi_get_bw_info(vsi); 5047 if (ret) { 5048 dev_info(&vsi->back->pdev->dev, 5049 "Failed updating vsi bw info, err %s aq_err %s\n", 5050 i40e_stat_str(&vsi->back->hw, ret), 5051 i40e_aq_str(&vsi->back->hw, 5052 vsi->back->hw.aq.asq_last_status)); 5053 goto out; 5054 } 5055 5056 /* Update the netdev TC setup */ 5057 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5058 out: 5059 return ret; 5060 } 5061 5062 /** 5063 * i40e_veb_config_tc - Configure TCs for given VEB 5064 * @veb: given VEB 5065 * @enabled_tc: TC bitmap 5066 * 5067 * Configures given TC bitmap for VEB (switching) element 5068 **/ 5069 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 5070 { 5071 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 5072 struct i40e_pf *pf = veb->pf; 5073 int ret = 0; 5074 int i; 5075 5076 /* No TCs or already enabled TCs just return */ 5077 if (!enabled_tc || veb->enabled_tc == enabled_tc) 5078 return ret; 5079 5080 bw_data.tc_valid_bits = enabled_tc; 5081 /* bw_data.absolute_credits is not set (relative) */ 5082 5083 /* Enable ETS TCs with equal BW Share for now */ 5084 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5085 if (enabled_tc & BIT(i)) 5086 bw_data.tc_bw_share_credits[i] = 1; 5087 } 5088 5089 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 5090 &bw_data, NULL); 5091 if (ret) { 5092 dev_info(&pf->pdev->dev, 5093 "VEB bw config failed, err %s aq_err %s\n", 5094 i40e_stat_str(&pf->hw, ret), 5095 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5096 goto out; 5097 } 5098 5099 /* Update the BW information */ 5100 ret = i40e_veb_get_bw_info(veb); 5101 if (ret) { 5102 dev_info(&pf->pdev->dev, 5103 "Failed getting veb bw config, err %s aq_err %s\n", 5104 i40e_stat_str(&pf->hw, ret), 5105 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5106 } 5107 5108 out: 5109 return ret; 5110 } 5111 5112 #ifdef CONFIG_I40E_DCB 5113 /** 5114 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 5115 * @pf: PF struct 5116 * 5117 * Reconfigure VEB/VSIs on a given PF; it is assumed that 5118 * the caller would've quiesce all the VSIs before calling 5119 * this function 5120 **/ 5121 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 5122 { 5123 u8 tc_map = 0; 5124 int ret; 5125 u8 v; 5126 5127 /* Enable the TCs available on PF to all VEBs */ 5128 tc_map = i40e_pf_get_tc_map(pf); 5129 for (v = 0; v < I40E_MAX_VEB; v++) { 5130 if (!pf->veb[v]) 5131 continue; 5132 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 5133 if (ret) { 5134 dev_info(&pf->pdev->dev, 5135 "Failed configuring TC for VEB seid=%d\n", 5136 pf->veb[v]->seid); 5137 /* Will try to configure as many components */ 5138 } 5139 } 5140 5141 /* Update each VSI */ 5142 for (v = 0; v < pf->num_alloc_vsi; v++) { 5143 if (!pf->vsi[v]) 5144 continue; 5145 5146 /* - Enable all TCs for the LAN VSI 5147 * - For all others keep them at TC0 for now 5148 */ 5149 if (v == pf->lan_vsi) 5150 tc_map = i40e_pf_get_tc_map(pf); 5151 else 5152 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 5153 5154 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 5155 if (ret) { 5156 dev_info(&pf->pdev->dev, 5157 "Failed configuring TC for VSI seid=%d\n", 5158 pf->vsi[v]->seid); 5159 /* Will try to configure as many components */ 5160 } else { 5161 /* Re-configure VSI vectors based on updated TC map */ 5162 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 5163 if (pf->vsi[v]->netdev) 5164 i40e_dcbnl_set_all(pf->vsi[v]); 5165 } 5166 } 5167 } 5168 5169 /** 5170 * i40e_resume_port_tx - Resume port Tx 5171 * @pf: PF struct 5172 * 5173 * Resume a port's Tx and issue a PF reset in case of failure to 5174 * resume. 5175 **/ 5176 static int i40e_resume_port_tx(struct i40e_pf *pf) 5177 { 5178 struct i40e_hw *hw = &pf->hw; 5179 int ret; 5180 5181 ret = i40e_aq_resume_port_tx(hw, NULL); 5182 if (ret) { 5183 dev_info(&pf->pdev->dev, 5184 "Resume Port Tx failed, err %s aq_err %s\n", 5185 i40e_stat_str(&pf->hw, ret), 5186 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5187 /* Schedule PF reset to recover */ 5188 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 5189 i40e_service_event_schedule(pf); 5190 } 5191 5192 return ret; 5193 } 5194 5195 /** 5196 * i40e_init_pf_dcb - Initialize DCB configuration 5197 * @pf: PF being configured 5198 * 5199 * Query the current DCB configuration and cache it 5200 * in the hardware structure 5201 **/ 5202 static int i40e_init_pf_dcb(struct i40e_pf *pf) 5203 { 5204 struct i40e_hw *hw = &pf->hw; 5205 int err = 0; 5206 5207 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */ 5208 if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT) 5209 goto out; 5210 5211 /* Get the initial DCB configuration */ 5212 err = i40e_init_dcb(hw); 5213 if (!err) { 5214 /* Device/Function is not DCBX capable */ 5215 if ((!hw->func_caps.dcb) || 5216 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 5217 dev_info(&pf->pdev->dev, 5218 "DCBX offload is not supported or is disabled for this PF.\n"); 5219 } else { 5220 /* When status is not DISABLED then DCBX in FW */ 5221 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 5222 DCB_CAP_DCBX_VER_IEEE; 5223 5224 pf->flags |= I40E_FLAG_DCB_CAPABLE; 5225 /* Enable DCB tagging only when more than one TC 5226 * or explicitly disable if only one TC 5227 */ 5228 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5229 pf->flags |= I40E_FLAG_DCB_ENABLED; 5230 else 5231 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5232 dev_dbg(&pf->pdev->dev, 5233 "DCBX offload is supported for this PF.\n"); 5234 } 5235 } else { 5236 dev_info(&pf->pdev->dev, 5237 "Query for DCB configuration failed, err %s aq_err %s\n", 5238 i40e_stat_str(&pf->hw, err), 5239 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5240 } 5241 5242 out: 5243 return err; 5244 } 5245 #endif /* CONFIG_I40E_DCB */ 5246 #define SPEED_SIZE 14 5247 #define FC_SIZE 8 5248 /** 5249 * i40e_print_link_message - print link up or down 5250 * @vsi: the VSI for which link needs a message 5251 */ 5252 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 5253 { 5254 enum i40e_aq_link_speed new_speed; 5255 char *speed = "Unknown"; 5256 char *fc = "Unknown"; 5257 char *fec = ""; 5258 char *an = ""; 5259 5260 new_speed = vsi->back->hw.phy.link_info.link_speed; 5261 5262 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 5263 return; 5264 vsi->current_isup = isup; 5265 vsi->current_speed = new_speed; 5266 if (!isup) { 5267 netdev_info(vsi->netdev, "NIC Link is Down\n"); 5268 return; 5269 } 5270 5271 /* Warn user if link speed on NPAR enabled partition is not at 5272 * least 10GB 5273 */ 5274 if (vsi->back->hw.func_caps.npar_enable && 5275 (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 5276 vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 5277 netdev_warn(vsi->netdev, 5278 "The partition detected link speed that is less than 10Gbps\n"); 5279 5280 switch (vsi->back->hw.phy.link_info.link_speed) { 5281 case I40E_LINK_SPEED_40GB: 5282 speed = "40 G"; 5283 break; 5284 case I40E_LINK_SPEED_20GB: 5285 speed = "20 G"; 5286 break; 5287 case I40E_LINK_SPEED_25GB: 5288 speed = "25 G"; 5289 break; 5290 case I40E_LINK_SPEED_10GB: 5291 speed = "10 G"; 5292 break; 5293 case I40E_LINK_SPEED_1GB: 5294 speed = "1000 M"; 5295 break; 5296 case I40E_LINK_SPEED_100MB: 5297 speed = "100 M"; 5298 break; 5299 default: 5300 break; 5301 } 5302 5303 switch (vsi->back->hw.fc.current_mode) { 5304 case I40E_FC_FULL: 5305 fc = "RX/TX"; 5306 break; 5307 case I40E_FC_TX_PAUSE: 5308 fc = "TX"; 5309 break; 5310 case I40E_FC_RX_PAUSE: 5311 fc = "RX"; 5312 break; 5313 default: 5314 fc = "None"; 5315 break; 5316 } 5317 5318 if (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 5319 fec = ", FEC: None"; 5320 an = ", Autoneg: False"; 5321 5322 if (vsi->back->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 5323 an = ", Autoneg: True"; 5324 5325 if (vsi->back->hw.phy.link_info.fec_info & 5326 I40E_AQ_CONFIG_FEC_KR_ENA) 5327 fec = ", FEC: CL74 FC-FEC/BASE-R"; 5328 else if (vsi->back->hw.phy.link_info.fec_info & 5329 I40E_AQ_CONFIG_FEC_RS_ENA) 5330 fec = ", FEC: CL108 RS-FEC"; 5331 } 5332 5333 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s, Flow Control: %s\n", 5334 speed, fec, an, fc); 5335 } 5336 5337 /** 5338 * i40e_up_complete - Finish the last steps of bringing up a connection 5339 * @vsi: the VSI being configured 5340 **/ 5341 static int i40e_up_complete(struct i40e_vsi *vsi) 5342 { 5343 struct i40e_pf *pf = vsi->back; 5344 int err; 5345 5346 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 5347 i40e_vsi_configure_msix(vsi); 5348 else 5349 i40e_configure_msi_and_legacy(vsi); 5350 5351 /* start rings */ 5352 err = i40e_vsi_start_rings(vsi); 5353 if (err) 5354 return err; 5355 5356 clear_bit(__I40E_VSI_DOWN, vsi->state); 5357 i40e_napi_enable_all(vsi); 5358 i40e_vsi_enable_irq(vsi); 5359 5360 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 5361 (vsi->netdev)) { 5362 i40e_print_link_message(vsi, true); 5363 netif_tx_start_all_queues(vsi->netdev); 5364 netif_carrier_on(vsi->netdev); 5365 } else if (vsi->netdev) { 5366 i40e_print_link_message(vsi, false); 5367 /* need to check for qualified module here*/ 5368 if ((pf->hw.phy.link_info.link_info & 5369 I40E_AQ_MEDIA_AVAILABLE) && 5370 (!(pf->hw.phy.link_info.an_info & 5371 I40E_AQ_QUALIFIED_MODULE))) 5372 netdev_err(vsi->netdev, 5373 "the driver failed to link because an unqualified module was detected."); 5374 } 5375 5376 /* replay FDIR SB filters */ 5377 if (vsi->type == I40E_VSI_FDIR) { 5378 /* reset fd counters */ 5379 pf->fd_add_err = 0; 5380 pf->fd_atr_cnt = 0; 5381 i40e_fdir_filter_restore(vsi); 5382 } 5383 5384 /* On the next run of the service_task, notify any clients of the new 5385 * opened netdev 5386 */ 5387 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 5388 i40e_service_event_schedule(pf); 5389 5390 return 0; 5391 } 5392 5393 /** 5394 * i40e_vsi_reinit_locked - Reset the VSI 5395 * @vsi: the VSI being configured 5396 * 5397 * Rebuild the ring structs after some configuration 5398 * has changed, e.g. MTU size. 5399 **/ 5400 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 5401 { 5402 struct i40e_pf *pf = vsi->back; 5403 5404 WARN_ON(in_interrupt()); 5405 while (test_and_set_bit(__I40E_CONFIG_BUSY, pf->state)) 5406 usleep_range(1000, 2000); 5407 i40e_down(vsi); 5408 5409 i40e_up(vsi); 5410 clear_bit(__I40E_CONFIG_BUSY, pf->state); 5411 } 5412 5413 /** 5414 * i40e_up - Bring the connection back up after being down 5415 * @vsi: the VSI being configured 5416 **/ 5417 int i40e_up(struct i40e_vsi *vsi) 5418 { 5419 int err; 5420 5421 err = i40e_vsi_configure(vsi); 5422 if (!err) 5423 err = i40e_up_complete(vsi); 5424 5425 return err; 5426 } 5427 5428 /** 5429 * i40e_down - Shutdown the connection processing 5430 * @vsi: the VSI being stopped 5431 **/ 5432 void i40e_down(struct i40e_vsi *vsi) 5433 { 5434 int i; 5435 5436 /* It is assumed that the caller of this function 5437 * sets the vsi->state __I40E_VSI_DOWN bit. 5438 */ 5439 if (vsi->netdev) { 5440 netif_carrier_off(vsi->netdev); 5441 netif_tx_disable(vsi->netdev); 5442 } 5443 i40e_vsi_disable_irq(vsi); 5444 i40e_vsi_stop_rings(vsi); 5445 i40e_napi_disable_all(vsi); 5446 5447 for (i = 0; i < vsi->num_queue_pairs; i++) { 5448 i40e_clean_tx_ring(vsi->tx_rings[i]); 5449 i40e_clean_rx_ring(vsi->rx_rings[i]); 5450 } 5451 5452 } 5453 5454 /** 5455 * i40e_setup_tc - configure multiple traffic classes 5456 * @netdev: net device to configure 5457 * @tc: number of traffic classes to enable 5458 **/ 5459 static int i40e_setup_tc(struct net_device *netdev, u8 tc) 5460 { 5461 struct i40e_netdev_priv *np = netdev_priv(netdev); 5462 struct i40e_vsi *vsi = np->vsi; 5463 struct i40e_pf *pf = vsi->back; 5464 u8 enabled_tc = 0; 5465 int ret = -EINVAL; 5466 int i; 5467 5468 /* Check if DCB enabled to continue */ 5469 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5470 netdev_info(netdev, "DCB is not enabled for adapter\n"); 5471 goto exit; 5472 } 5473 5474 /* Check if MFP enabled */ 5475 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 5476 netdev_info(netdev, "Configuring TC not supported in MFP mode\n"); 5477 goto exit; 5478 } 5479 5480 /* Check whether tc count is within enabled limit */ 5481 if (tc > i40e_pf_get_num_tc(pf)) { 5482 netdev_info(netdev, "TC count greater than enabled on link for adapter\n"); 5483 goto exit; 5484 } 5485 5486 /* Generate TC map for number of tc requested */ 5487 for (i = 0; i < tc; i++) 5488 enabled_tc |= BIT(i); 5489 5490 /* Requesting same TC configuration as already enabled */ 5491 if (enabled_tc == vsi->tc_config.enabled_tc) 5492 return 0; 5493 5494 /* Quiesce VSI queues */ 5495 i40e_quiesce_vsi(vsi); 5496 5497 /* Configure VSI for enabled TCs */ 5498 ret = i40e_vsi_config_tc(vsi, enabled_tc); 5499 if (ret) { 5500 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 5501 vsi->seid); 5502 goto exit; 5503 } 5504 5505 /* Unquiesce VSI */ 5506 i40e_unquiesce_vsi(vsi); 5507 5508 exit: 5509 return ret; 5510 } 5511 5512 static int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto, 5513 struct tc_to_netdev *tc) 5514 { 5515 if (tc->type != TC_SETUP_MQPRIO) 5516 return -EINVAL; 5517 5518 tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 5519 5520 return i40e_setup_tc(netdev, tc->mqprio->num_tc); 5521 } 5522 5523 /** 5524 * i40e_open - Called when a network interface is made active 5525 * @netdev: network interface device structure 5526 * 5527 * The open entry point is called when a network interface is made 5528 * active by the system (IFF_UP). At this point all resources needed 5529 * for transmit and receive operations are allocated, the interrupt 5530 * handler is registered with the OS, the netdev watchdog subtask is 5531 * enabled, and the stack is notified that the interface is ready. 5532 * 5533 * Returns 0 on success, negative value on failure 5534 **/ 5535 int i40e_open(struct net_device *netdev) 5536 { 5537 struct i40e_netdev_priv *np = netdev_priv(netdev); 5538 struct i40e_vsi *vsi = np->vsi; 5539 struct i40e_pf *pf = vsi->back; 5540 int err; 5541 5542 /* disallow open during test or if eeprom is broken */ 5543 if (test_bit(__I40E_TESTING, pf->state) || 5544 test_bit(__I40E_BAD_EEPROM, pf->state)) 5545 return -EBUSY; 5546 5547 netif_carrier_off(netdev); 5548 5549 err = i40e_vsi_open(vsi); 5550 if (err) 5551 return err; 5552 5553 /* configure global TSO hardware offload settings */ 5554 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 5555 TCP_FLAG_FIN) >> 16); 5556 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 5557 TCP_FLAG_FIN | 5558 TCP_FLAG_CWR) >> 16); 5559 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 5560 5561 udp_tunnel_get_rx_info(netdev); 5562 5563 return 0; 5564 } 5565 5566 /** 5567 * i40e_vsi_open - 5568 * @vsi: the VSI to open 5569 * 5570 * Finish initialization of the VSI. 5571 * 5572 * Returns 0 on success, negative value on failure 5573 * 5574 * Note: expects to be called while under rtnl_lock() 5575 **/ 5576 int i40e_vsi_open(struct i40e_vsi *vsi) 5577 { 5578 struct i40e_pf *pf = vsi->back; 5579 char int_name[I40E_INT_NAME_STR_LEN]; 5580 int err; 5581 5582 /* allocate descriptors */ 5583 err = i40e_vsi_setup_tx_resources(vsi); 5584 if (err) 5585 goto err_setup_tx; 5586 err = i40e_vsi_setup_rx_resources(vsi); 5587 if (err) 5588 goto err_setup_rx; 5589 5590 err = i40e_vsi_configure(vsi); 5591 if (err) 5592 goto err_setup_rx; 5593 5594 if (vsi->netdev) { 5595 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 5596 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 5597 err = i40e_vsi_request_irq(vsi, int_name); 5598 if (err) 5599 goto err_setup_rx; 5600 5601 /* Notify the stack of the actual queue counts. */ 5602 err = netif_set_real_num_tx_queues(vsi->netdev, 5603 vsi->num_queue_pairs); 5604 if (err) 5605 goto err_set_queues; 5606 5607 err = netif_set_real_num_rx_queues(vsi->netdev, 5608 vsi->num_queue_pairs); 5609 if (err) 5610 goto err_set_queues; 5611 5612 } else if (vsi->type == I40E_VSI_FDIR) { 5613 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 5614 dev_driver_string(&pf->pdev->dev), 5615 dev_name(&pf->pdev->dev)); 5616 err = i40e_vsi_request_irq(vsi, int_name); 5617 5618 } else { 5619 err = -EINVAL; 5620 goto err_setup_rx; 5621 } 5622 5623 err = i40e_up_complete(vsi); 5624 if (err) 5625 goto err_up_complete; 5626 5627 return 0; 5628 5629 err_up_complete: 5630 i40e_down(vsi); 5631 err_set_queues: 5632 i40e_vsi_free_irq(vsi); 5633 err_setup_rx: 5634 i40e_vsi_free_rx_resources(vsi); 5635 err_setup_tx: 5636 i40e_vsi_free_tx_resources(vsi); 5637 if (vsi == pf->vsi[pf->lan_vsi]) 5638 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true); 5639 5640 return err; 5641 } 5642 5643 /** 5644 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 5645 * @pf: Pointer to PF 5646 * 5647 * This function destroys the hlist where all the Flow Director 5648 * filters were saved. 5649 **/ 5650 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 5651 { 5652 struct i40e_fdir_filter *filter; 5653 struct i40e_flex_pit *pit_entry, *tmp; 5654 struct hlist_node *node2; 5655 5656 hlist_for_each_entry_safe(filter, node2, 5657 &pf->fdir_filter_list, fdir_node) { 5658 hlist_del(&filter->fdir_node); 5659 kfree(filter); 5660 } 5661 5662 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 5663 list_del(&pit_entry->list); 5664 kfree(pit_entry); 5665 } 5666 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 5667 5668 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 5669 list_del(&pit_entry->list); 5670 kfree(pit_entry); 5671 } 5672 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 5673 5674 pf->fdir_pf_active_filters = 0; 5675 pf->fd_tcp4_filter_cnt = 0; 5676 pf->fd_udp4_filter_cnt = 0; 5677 pf->fd_sctp4_filter_cnt = 0; 5678 pf->fd_ip4_filter_cnt = 0; 5679 5680 /* Reprogram the default input set for TCP/IPv4 */ 5681 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 5682 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5683 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5684 5685 /* Reprogram the default input set for UDP/IPv4 */ 5686 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 5687 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5688 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5689 5690 /* Reprogram the default input set for SCTP/IPv4 */ 5691 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 5692 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5693 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5694 5695 /* Reprogram the default input set for Other/IPv4 */ 5696 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 5697 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 5698 } 5699 5700 /** 5701 * i40e_close - Disables a network interface 5702 * @netdev: network interface device structure 5703 * 5704 * The close entry point is called when an interface is de-activated 5705 * by the OS. The hardware is still under the driver's control, but 5706 * this netdev interface is disabled. 5707 * 5708 * Returns 0, this is not allowed to fail 5709 **/ 5710 int i40e_close(struct net_device *netdev) 5711 { 5712 struct i40e_netdev_priv *np = netdev_priv(netdev); 5713 struct i40e_vsi *vsi = np->vsi; 5714 5715 i40e_vsi_close(vsi); 5716 5717 return 0; 5718 } 5719 5720 /** 5721 * i40e_do_reset - Start a PF or Core Reset sequence 5722 * @pf: board private structure 5723 * @reset_flags: which reset is requested 5724 * @lock_acquired: indicates whether or not the lock has been acquired 5725 * before this function was called. 5726 * 5727 * The essential difference in resets is that the PF Reset 5728 * doesn't clear the packet buffers, doesn't reset the PE 5729 * firmware, and doesn't bother the other PFs on the chip. 5730 **/ 5731 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags, bool lock_acquired) 5732 { 5733 u32 val; 5734 5735 WARN_ON(in_interrupt()); 5736 5737 5738 /* do the biggest reset indicated */ 5739 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 5740 5741 /* Request a Global Reset 5742 * 5743 * This will start the chip's countdown to the actual full 5744 * chip reset event, and a warning interrupt to be sent 5745 * to all PFs, including the requestor. Our handler 5746 * for the warning interrupt will deal with the shutdown 5747 * and recovery of the switch setup. 5748 */ 5749 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 5750 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5751 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 5752 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5753 5754 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 5755 5756 /* Request a Core Reset 5757 * 5758 * Same as Global Reset, except does *not* include the MAC/PHY 5759 */ 5760 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 5761 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5762 val |= I40E_GLGEN_RTRIG_CORER_MASK; 5763 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5764 i40e_flush(&pf->hw); 5765 5766 } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) { 5767 5768 /* Request a PF Reset 5769 * 5770 * Resets only the PF-specific registers 5771 * 5772 * This goes directly to the tear-down and rebuild of 5773 * the switch, since we need to do all the recovery as 5774 * for the Core Reset. 5775 */ 5776 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 5777 i40e_handle_reset_warning(pf, lock_acquired); 5778 5779 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 5780 int v; 5781 5782 /* Find the VSI(s) that requested a re-init */ 5783 dev_info(&pf->pdev->dev, 5784 "VSI reinit requested\n"); 5785 for (v = 0; v < pf->num_alloc_vsi; v++) { 5786 struct i40e_vsi *vsi = pf->vsi[v]; 5787 5788 if (vsi != NULL && 5789 test_and_clear_bit(__I40E_VSI_REINIT_REQUESTED, 5790 vsi->state)) 5791 i40e_vsi_reinit_locked(pf->vsi[v]); 5792 } 5793 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 5794 int v; 5795 5796 /* Find the VSI(s) that needs to be brought down */ 5797 dev_info(&pf->pdev->dev, "VSI down requested\n"); 5798 for (v = 0; v < pf->num_alloc_vsi; v++) { 5799 struct i40e_vsi *vsi = pf->vsi[v]; 5800 5801 if (vsi != NULL && 5802 test_and_clear_bit(__I40E_VSI_DOWN_REQUESTED, 5803 vsi->state)) { 5804 set_bit(__I40E_VSI_DOWN, vsi->state); 5805 i40e_down(vsi); 5806 } 5807 } 5808 } else { 5809 dev_info(&pf->pdev->dev, 5810 "bad reset request 0x%08x\n", reset_flags); 5811 } 5812 } 5813 5814 #ifdef CONFIG_I40E_DCB 5815 /** 5816 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 5817 * @pf: board private structure 5818 * @old_cfg: current DCB config 5819 * @new_cfg: new DCB config 5820 **/ 5821 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 5822 struct i40e_dcbx_config *old_cfg, 5823 struct i40e_dcbx_config *new_cfg) 5824 { 5825 bool need_reconfig = false; 5826 5827 /* Check if ETS configuration has changed */ 5828 if (memcmp(&new_cfg->etscfg, 5829 &old_cfg->etscfg, 5830 sizeof(new_cfg->etscfg))) { 5831 /* If Priority Table has changed reconfig is needed */ 5832 if (memcmp(&new_cfg->etscfg.prioritytable, 5833 &old_cfg->etscfg.prioritytable, 5834 sizeof(new_cfg->etscfg.prioritytable))) { 5835 need_reconfig = true; 5836 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 5837 } 5838 5839 if (memcmp(&new_cfg->etscfg.tcbwtable, 5840 &old_cfg->etscfg.tcbwtable, 5841 sizeof(new_cfg->etscfg.tcbwtable))) 5842 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 5843 5844 if (memcmp(&new_cfg->etscfg.tsatable, 5845 &old_cfg->etscfg.tsatable, 5846 sizeof(new_cfg->etscfg.tsatable))) 5847 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 5848 } 5849 5850 /* Check if PFC configuration has changed */ 5851 if (memcmp(&new_cfg->pfc, 5852 &old_cfg->pfc, 5853 sizeof(new_cfg->pfc))) { 5854 need_reconfig = true; 5855 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 5856 } 5857 5858 /* Check if APP Table has changed */ 5859 if (memcmp(&new_cfg->app, 5860 &old_cfg->app, 5861 sizeof(new_cfg->app))) { 5862 need_reconfig = true; 5863 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 5864 } 5865 5866 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 5867 return need_reconfig; 5868 } 5869 5870 /** 5871 * i40e_handle_lldp_event - Handle LLDP Change MIB event 5872 * @pf: board private structure 5873 * @e: event info posted on ARQ 5874 **/ 5875 static int i40e_handle_lldp_event(struct i40e_pf *pf, 5876 struct i40e_arq_event_info *e) 5877 { 5878 struct i40e_aqc_lldp_get_mib *mib = 5879 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 5880 struct i40e_hw *hw = &pf->hw; 5881 struct i40e_dcbx_config tmp_dcbx_cfg; 5882 bool need_reconfig = false; 5883 int ret = 0; 5884 u8 type; 5885 5886 /* Not DCB capable or capability disabled */ 5887 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 5888 return ret; 5889 5890 /* Ignore if event is not for Nearest Bridge */ 5891 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 5892 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 5893 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 5894 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 5895 return ret; 5896 5897 /* Check MIB Type and return if event for Remote MIB update */ 5898 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 5899 dev_dbg(&pf->pdev->dev, 5900 "LLDP event mib type %s\n", type ? "remote" : "local"); 5901 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 5902 /* Update the remote cached instance and return */ 5903 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 5904 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 5905 &hw->remote_dcbx_config); 5906 goto exit; 5907 } 5908 5909 /* Store the old configuration */ 5910 tmp_dcbx_cfg = hw->local_dcbx_config; 5911 5912 /* Reset the old DCBx configuration data */ 5913 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 5914 /* Get updated DCBX data from firmware */ 5915 ret = i40e_get_dcb_config(&pf->hw); 5916 if (ret) { 5917 dev_info(&pf->pdev->dev, 5918 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 5919 i40e_stat_str(&pf->hw, ret), 5920 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5921 goto exit; 5922 } 5923 5924 /* No change detected in DCBX configs */ 5925 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 5926 sizeof(tmp_dcbx_cfg))) { 5927 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 5928 goto exit; 5929 } 5930 5931 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 5932 &hw->local_dcbx_config); 5933 5934 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 5935 5936 if (!need_reconfig) 5937 goto exit; 5938 5939 /* Enable DCB tagging only when more than one TC */ 5940 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5941 pf->flags |= I40E_FLAG_DCB_ENABLED; 5942 else 5943 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5944 5945 set_bit(__I40E_PORT_SUSPENDED, pf->state); 5946 /* Reconfiguration needed quiesce all VSIs */ 5947 i40e_pf_quiesce_all_vsi(pf); 5948 5949 /* Changes in configuration update VEB/VSI */ 5950 i40e_dcb_reconfigure(pf); 5951 5952 ret = i40e_resume_port_tx(pf); 5953 5954 clear_bit(__I40E_PORT_SUSPENDED, pf->state); 5955 /* In case of error no point in resuming VSIs */ 5956 if (ret) 5957 goto exit; 5958 5959 /* Wait for the PF's queues to be disabled */ 5960 ret = i40e_pf_wait_queues_disabled(pf); 5961 if (ret) { 5962 /* Schedule PF reset to recover */ 5963 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 5964 i40e_service_event_schedule(pf); 5965 } else { 5966 i40e_pf_unquiesce_all_vsi(pf); 5967 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 5968 I40E_FLAG_CLIENT_L2_CHANGE); 5969 } 5970 5971 exit: 5972 return ret; 5973 } 5974 #endif /* CONFIG_I40E_DCB */ 5975 5976 /** 5977 * i40e_do_reset_safe - Protected reset path for userland calls. 5978 * @pf: board private structure 5979 * @reset_flags: which reset is requested 5980 * 5981 **/ 5982 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 5983 { 5984 rtnl_lock(); 5985 i40e_do_reset(pf, reset_flags, true); 5986 rtnl_unlock(); 5987 } 5988 5989 /** 5990 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 5991 * @pf: board private structure 5992 * @e: event info posted on ARQ 5993 * 5994 * Handler for LAN Queue Overflow Event generated by the firmware for PF 5995 * and VF queues 5996 **/ 5997 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 5998 struct i40e_arq_event_info *e) 5999 { 6000 struct i40e_aqc_lan_overflow *data = 6001 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 6002 u32 queue = le32_to_cpu(data->prtdcb_rupto); 6003 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 6004 struct i40e_hw *hw = &pf->hw; 6005 struct i40e_vf *vf; 6006 u16 vf_id; 6007 6008 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 6009 queue, qtx_ctl); 6010 6011 /* Queue belongs to VF, find the VF and issue VF reset */ 6012 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 6013 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 6014 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 6015 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 6016 vf_id -= hw->func_caps.vf_base_id; 6017 vf = &pf->vf[vf_id]; 6018 i40e_vc_notify_vf_reset(vf); 6019 /* Allow VF to process pending reset notification */ 6020 msleep(20); 6021 i40e_reset_vf(vf, false); 6022 } 6023 } 6024 6025 /** 6026 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 6027 * @pf: board private structure 6028 **/ 6029 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 6030 { 6031 u32 val, fcnt_prog; 6032 6033 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 6034 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 6035 return fcnt_prog; 6036 } 6037 6038 /** 6039 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 6040 * @pf: board private structure 6041 **/ 6042 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 6043 { 6044 u32 val, fcnt_prog; 6045 6046 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 6047 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 6048 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 6049 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 6050 return fcnt_prog; 6051 } 6052 6053 /** 6054 * i40e_get_global_fd_count - Get total FD filters programmed on device 6055 * @pf: board private structure 6056 **/ 6057 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 6058 { 6059 u32 val, fcnt_prog; 6060 6061 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 6062 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 6063 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 6064 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 6065 return fcnt_prog; 6066 } 6067 6068 /** 6069 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 6070 * @pf: board private structure 6071 **/ 6072 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 6073 { 6074 struct i40e_fdir_filter *filter; 6075 u32 fcnt_prog, fcnt_avail; 6076 struct hlist_node *node; 6077 6078 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 6079 return; 6080 6081 /* Check if we have enough room to re-enable FDir SB capability. */ 6082 fcnt_prog = i40e_get_global_fd_count(pf); 6083 fcnt_avail = pf->fdir_pf_filter_count; 6084 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 6085 (pf->fd_add_err == 0) || 6086 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) { 6087 if (pf->flags & I40E_FLAG_FD_SB_AUTO_DISABLED) { 6088 pf->flags &= ~I40E_FLAG_FD_SB_AUTO_DISABLED; 6089 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 6090 (I40E_DEBUG_FD & pf->hw.debug_mask)) 6091 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 6092 } 6093 } 6094 6095 /* We should wait for even more space before re-enabling ATR. 6096 * Additionally, we cannot enable ATR as long as we still have TCP SB 6097 * rules active. 6098 */ 6099 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) && 6100 (pf->fd_tcp4_filter_cnt == 0)) { 6101 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) { 6102 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 6103 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 6104 (I40E_DEBUG_FD & pf->hw.debug_mask)) 6105 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 6106 } 6107 } 6108 6109 /* if hw had a problem adding a filter, delete it */ 6110 if (pf->fd_inv > 0) { 6111 hlist_for_each_entry_safe(filter, node, 6112 &pf->fdir_filter_list, fdir_node) { 6113 if (filter->fd_id == pf->fd_inv) { 6114 hlist_del(&filter->fdir_node); 6115 kfree(filter); 6116 pf->fdir_pf_active_filters--; 6117 } 6118 } 6119 } 6120 } 6121 6122 #define I40E_MIN_FD_FLUSH_INTERVAL 10 6123 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 6124 /** 6125 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 6126 * @pf: board private structure 6127 **/ 6128 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 6129 { 6130 unsigned long min_flush_time; 6131 int flush_wait_retry = 50; 6132 bool disable_atr = false; 6133 int fd_room; 6134 int reg; 6135 6136 if (!time_after(jiffies, pf->fd_flush_timestamp + 6137 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 6138 return; 6139 6140 /* If the flush is happening too quick and we have mostly SB rules we 6141 * should not re-enable ATR for some time. 6142 */ 6143 min_flush_time = pf->fd_flush_timestamp + 6144 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 6145 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 6146 6147 if (!(time_after(jiffies, min_flush_time)) && 6148 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 6149 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6150 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 6151 disable_atr = true; 6152 } 6153 6154 pf->fd_flush_timestamp = jiffies; 6155 pf->flags |= I40E_FLAG_FD_ATR_AUTO_DISABLED; 6156 /* flush all filters */ 6157 wr32(&pf->hw, I40E_PFQF_CTL_1, 6158 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 6159 i40e_flush(&pf->hw); 6160 pf->fd_flush_cnt++; 6161 pf->fd_add_err = 0; 6162 do { 6163 /* Check FD flush status every 5-6msec */ 6164 usleep_range(5000, 6000); 6165 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 6166 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 6167 break; 6168 } while (flush_wait_retry--); 6169 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 6170 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 6171 } else { 6172 /* replay sideband filters */ 6173 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 6174 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 6175 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 6176 clear_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 6177 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6178 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 6179 } 6180 } 6181 6182 /** 6183 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 6184 * @pf: board private structure 6185 **/ 6186 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 6187 { 6188 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 6189 } 6190 6191 /* We can see up to 256 filter programming desc in transit if the filters are 6192 * being applied really fast; before we see the first 6193 * filter miss error on Rx queue 0. Accumulating enough error messages before 6194 * reacting will make sure we don't cause flush too often. 6195 */ 6196 #define I40E_MAX_FD_PROGRAM_ERROR 256 6197 6198 /** 6199 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 6200 * @pf: board private structure 6201 **/ 6202 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 6203 { 6204 6205 /* if interface is down do nothing */ 6206 if (test_bit(__I40E_VSI_DOWN, pf->state)) 6207 return; 6208 6209 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 6210 i40e_fdir_flush_and_replay(pf); 6211 6212 i40e_fdir_check_and_reenable(pf); 6213 6214 } 6215 6216 /** 6217 * i40e_vsi_link_event - notify VSI of a link event 6218 * @vsi: vsi to be notified 6219 * @link_up: link up or down 6220 **/ 6221 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 6222 { 6223 if (!vsi || test_bit(__I40E_VSI_DOWN, vsi->state)) 6224 return; 6225 6226 switch (vsi->type) { 6227 case I40E_VSI_MAIN: 6228 if (!vsi->netdev || !vsi->netdev_registered) 6229 break; 6230 6231 if (link_up) { 6232 netif_carrier_on(vsi->netdev); 6233 netif_tx_wake_all_queues(vsi->netdev); 6234 } else { 6235 netif_carrier_off(vsi->netdev); 6236 netif_tx_stop_all_queues(vsi->netdev); 6237 } 6238 break; 6239 6240 case I40E_VSI_SRIOV: 6241 case I40E_VSI_VMDQ2: 6242 case I40E_VSI_CTRL: 6243 case I40E_VSI_IWARP: 6244 case I40E_VSI_MIRROR: 6245 default: 6246 /* there is no notification for other VSIs */ 6247 break; 6248 } 6249 } 6250 6251 /** 6252 * i40e_veb_link_event - notify elements on the veb of a link event 6253 * @veb: veb to be notified 6254 * @link_up: link up or down 6255 **/ 6256 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 6257 { 6258 struct i40e_pf *pf; 6259 int i; 6260 6261 if (!veb || !veb->pf) 6262 return; 6263 pf = veb->pf; 6264 6265 /* depth first... */ 6266 for (i = 0; i < I40E_MAX_VEB; i++) 6267 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 6268 i40e_veb_link_event(pf->veb[i], link_up); 6269 6270 /* ... now the local VSIs */ 6271 for (i = 0; i < pf->num_alloc_vsi; i++) 6272 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 6273 i40e_vsi_link_event(pf->vsi[i], link_up); 6274 } 6275 6276 /** 6277 * i40e_link_event - Update netif_carrier status 6278 * @pf: board private structure 6279 **/ 6280 static void i40e_link_event(struct i40e_pf *pf) 6281 { 6282 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6283 u8 new_link_speed, old_link_speed; 6284 i40e_status status; 6285 bool new_link, old_link; 6286 6287 /* save off old link status information */ 6288 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 6289 6290 /* set this to force the get_link_status call to refresh state */ 6291 pf->hw.phy.get_link_info = true; 6292 6293 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 6294 6295 status = i40e_get_link_status(&pf->hw, &new_link); 6296 6297 /* On success, disable temp link polling */ 6298 if (status == I40E_SUCCESS) { 6299 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING) 6300 pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING; 6301 } else { 6302 /* Enable link polling temporarily until i40e_get_link_status 6303 * returns I40E_SUCCESS 6304 */ 6305 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING; 6306 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 6307 status); 6308 return; 6309 } 6310 6311 old_link_speed = pf->hw.phy.link_info_old.link_speed; 6312 new_link_speed = pf->hw.phy.link_info.link_speed; 6313 6314 if (new_link == old_link && 6315 new_link_speed == old_link_speed && 6316 (test_bit(__I40E_VSI_DOWN, vsi->state) || 6317 new_link == netif_carrier_ok(vsi->netdev))) 6318 return; 6319 6320 if (!test_bit(__I40E_VSI_DOWN, vsi->state)) 6321 i40e_print_link_message(vsi, new_link); 6322 6323 /* Notify the base of the switch tree connected to 6324 * the link. Floating VEBs are not notified. 6325 */ 6326 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 6327 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 6328 else 6329 i40e_vsi_link_event(vsi, new_link); 6330 6331 if (pf->vf) 6332 i40e_vc_notify_link_state(pf); 6333 6334 if (pf->flags & I40E_FLAG_PTP) 6335 i40e_ptp_set_increment(pf); 6336 } 6337 6338 /** 6339 * i40e_watchdog_subtask - periodic checks not using event driven response 6340 * @pf: board private structure 6341 **/ 6342 static void i40e_watchdog_subtask(struct i40e_pf *pf) 6343 { 6344 int i; 6345 6346 /* if interface is down do nothing */ 6347 if (test_bit(__I40E_VSI_DOWN, pf->state) || 6348 test_bit(__I40E_CONFIG_BUSY, pf->state)) 6349 return; 6350 6351 /* make sure we don't do these things too often */ 6352 if (time_before(jiffies, (pf->service_timer_previous + 6353 pf->service_timer_period))) 6354 return; 6355 pf->service_timer_previous = jiffies; 6356 6357 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 6358 (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)) 6359 i40e_link_event(pf); 6360 6361 /* Update the stats for active netdevs so the network stack 6362 * can look at updated numbers whenever it cares to 6363 */ 6364 for (i = 0; i < pf->num_alloc_vsi; i++) 6365 if (pf->vsi[i] && pf->vsi[i]->netdev) 6366 i40e_update_stats(pf->vsi[i]); 6367 6368 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 6369 /* Update the stats for the active switching components */ 6370 for (i = 0; i < I40E_MAX_VEB; i++) 6371 if (pf->veb[i]) 6372 i40e_update_veb_stats(pf->veb[i]); 6373 } 6374 6375 i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]); 6376 } 6377 6378 /** 6379 * i40e_reset_subtask - Set up for resetting the device and driver 6380 * @pf: board private structure 6381 **/ 6382 static void i40e_reset_subtask(struct i40e_pf *pf) 6383 { 6384 u32 reset_flags = 0; 6385 6386 if (test_bit(__I40E_REINIT_REQUESTED, pf->state)) { 6387 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 6388 clear_bit(__I40E_REINIT_REQUESTED, pf->state); 6389 } 6390 if (test_bit(__I40E_PF_RESET_REQUESTED, pf->state)) { 6391 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 6392 clear_bit(__I40E_PF_RESET_REQUESTED, pf->state); 6393 } 6394 if (test_bit(__I40E_CORE_RESET_REQUESTED, pf->state)) { 6395 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 6396 clear_bit(__I40E_CORE_RESET_REQUESTED, pf->state); 6397 } 6398 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state)) { 6399 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 6400 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, pf->state); 6401 } 6402 if (test_bit(__I40E_VSI_DOWN_REQUESTED, pf->state)) { 6403 reset_flags |= BIT(__I40E_VSI_DOWN_REQUESTED); 6404 clear_bit(__I40E_VSI_DOWN_REQUESTED, pf->state); 6405 } 6406 6407 /* If there's a recovery already waiting, it takes 6408 * precedence before starting a new reset sequence. 6409 */ 6410 if (test_bit(__I40E_RESET_INTR_RECEIVED, pf->state)) { 6411 i40e_prep_for_reset(pf, false); 6412 i40e_reset(pf); 6413 i40e_rebuild(pf, false, false); 6414 } 6415 6416 /* If we're already down or resetting, just bail */ 6417 if (reset_flags && 6418 !test_bit(__I40E_VSI_DOWN, pf->state) && 6419 !test_bit(__I40E_CONFIG_BUSY, pf->state)) { 6420 rtnl_lock(); 6421 i40e_do_reset(pf, reset_flags, true); 6422 rtnl_unlock(); 6423 } 6424 } 6425 6426 /** 6427 * i40e_handle_link_event - Handle link event 6428 * @pf: board private structure 6429 * @e: event info posted on ARQ 6430 **/ 6431 static void i40e_handle_link_event(struct i40e_pf *pf, 6432 struct i40e_arq_event_info *e) 6433 { 6434 struct i40e_aqc_get_link_status *status = 6435 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 6436 6437 /* Do a new status request to re-enable LSE reporting 6438 * and load new status information into the hw struct 6439 * This completely ignores any state information 6440 * in the ARQ event info, instead choosing to always 6441 * issue the AQ update link status command. 6442 */ 6443 i40e_link_event(pf); 6444 6445 /* check for unqualified module, if link is down */ 6446 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 6447 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 6448 (!(status->link_info & I40E_AQ_LINK_UP))) 6449 dev_err(&pf->pdev->dev, 6450 "The driver failed to link because an unqualified module was detected.\n"); 6451 } 6452 6453 /** 6454 * i40e_clean_adminq_subtask - Clean the AdminQ rings 6455 * @pf: board private structure 6456 **/ 6457 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 6458 { 6459 struct i40e_arq_event_info event; 6460 struct i40e_hw *hw = &pf->hw; 6461 u16 pending, i = 0; 6462 i40e_status ret; 6463 u16 opcode; 6464 u32 oldval; 6465 u32 val; 6466 6467 /* Do not run clean AQ when PF reset fails */ 6468 if (test_bit(__I40E_RESET_FAILED, pf->state)) 6469 return; 6470 6471 /* check for error indications */ 6472 val = rd32(&pf->hw, pf->hw.aq.arq.len); 6473 oldval = val; 6474 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 6475 if (hw->debug_mask & I40E_DEBUG_AQ) 6476 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 6477 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 6478 } 6479 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 6480 if (hw->debug_mask & I40E_DEBUG_AQ) 6481 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 6482 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 6483 pf->arq_overflows++; 6484 } 6485 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 6486 if (hw->debug_mask & I40E_DEBUG_AQ) 6487 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 6488 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 6489 } 6490 if (oldval != val) 6491 wr32(&pf->hw, pf->hw.aq.arq.len, val); 6492 6493 val = rd32(&pf->hw, pf->hw.aq.asq.len); 6494 oldval = val; 6495 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 6496 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6497 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 6498 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 6499 } 6500 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 6501 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6502 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 6503 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 6504 } 6505 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 6506 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6507 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 6508 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 6509 } 6510 if (oldval != val) 6511 wr32(&pf->hw, pf->hw.aq.asq.len, val); 6512 6513 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 6514 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 6515 if (!event.msg_buf) 6516 return; 6517 6518 do { 6519 ret = i40e_clean_arq_element(hw, &event, &pending); 6520 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 6521 break; 6522 else if (ret) { 6523 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 6524 break; 6525 } 6526 6527 opcode = le16_to_cpu(event.desc.opcode); 6528 switch (opcode) { 6529 6530 case i40e_aqc_opc_get_link_status: 6531 i40e_handle_link_event(pf, &event); 6532 break; 6533 case i40e_aqc_opc_send_msg_to_pf: 6534 ret = i40e_vc_process_vf_msg(pf, 6535 le16_to_cpu(event.desc.retval), 6536 le32_to_cpu(event.desc.cookie_high), 6537 le32_to_cpu(event.desc.cookie_low), 6538 event.msg_buf, 6539 event.msg_len); 6540 break; 6541 case i40e_aqc_opc_lldp_update_mib: 6542 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 6543 #ifdef CONFIG_I40E_DCB 6544 rtnl_lock(); 6545 ret = i40e_handle_lldp_event(pf, &event); 6546 rtnl_unlock(); 6547 #endif /* CONFIG_I40E_DCB */ 6548 break; 6549 case i40e_aqc_opc_event_lan_overflow: 6550 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 6551 i40e_handle_lan_overflow_event(pf, &event); 6552 break; 6553 case i40e_aqc_opc_send_msg_to_peer: 6554 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 6555 break; 6556 case i40e_aqc_opc_nvm_erase: 6557 case i40e_aqc_opc_nvm_update: 6558 case i40e_aqc_opc_oem_post_update: 6559 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 6560 "ARQ NVM operation 0x%04x completed\n", 6561 opcode); 6562 break; 6563 default: 6564 dev_info(&pf->pdev->dev, 6565 "ARQ: Unknown event 0x%04x ignored\n", 6566 opcode); 6567 break; 6568 } 6569 } while (i++ < pf->adminq_work_limit); 6570 6571 if (i < pf->adminq_work_limit) 6572 clear_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state); 6573 6574 /* re-enable Admin queue interrupt cause */ 6575 val = rd32(hw, I40E_PFINT_ICR0_ENA); 6576 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 6577 wr32(hw, I40E_PFINT_ICR0_ENA, val); 6578 i40e_flush(hw); 6579 6580 kfree(event.msg_buf); 6581 } 6582 6583 /** 6584 * i40e_verify_eeprom - make sure eeprom is good to use 6585 * @pf: board private structure 6586 **/ 6587 static void i40e_verify_eeprom(struct i40e_pf *pf) 6588 { 6589 int err; 6590 6591 err = i40e_diag_eeprom_test(&pf->hw); 6592 if (err) { 6593 /* retry in case of garbage read */ 6594 err = i40e_diag_eeprom_test(&pf->hw); 6595 if (err) { 6596 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 6597 err); 6598 set_bit(__I40E_BAD_EEPROM, pf->state); 6599 } 6600 } 6601 6602 if (!err && test_bit(__I40E_BAD_EEPROM, pf->state)) { 6603 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 6604 clear_bit(__I40E_BAD_EEPROM, pf->state); 6605 } 6606 } 6607 6608 /** 6609 * i40e_enable_pf_switch_lb 6610 * @pf: pointer to the PF structure 6611 * 6612 * enable switch loop back or die - no point in a return value 6613 **/ 6614 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 6615 { 6616 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6617 struct i40e_vsi_context ctxt; 6618 int ret; 6619 6620 ctxt.seid = pf->main_vsi_seid; 6621 ctxt.pf_num = pf->hw.pf_id; 6622 ctxt.vf_num = 0; 6623 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6624 if (ret) { 6625 dev_info(&pf->pdev->dev, 6626 "couldn't get PF vsi config, err %s aq_err %s\n", 6627 i40e_stat_str(&pf->hw, ret), 6628 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6629 return; 6630 } 6631 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6632 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6633 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6634 6635 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6636 if (ret) { 6637 dev_info(&pf->pdev->dev, 6638 "update vsi switch failed, err %s aq_err %s\n", 6639 i40e_stat_str(&pf->hw, ret), 6640 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6641 } 6642 } 6643 6644 /** 6645 * i40e_disable_pf_switch_lb 6646 * @pf: pointer to the PF structure 6647 * 6648 * disable switch loop back or die - no point in a return value 6649 **/ 6650 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 6651 { 6652 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6653 struct i40e_vsi_context ctxt; 6654 int ret; 6655 6656 ctxt.seid = pf->main_vsi_seid; 6657 ctxt.pf_num = pf->hw.pf_id; 6658 ctxt.vf_num = 0; 6659 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6660 if (ret) { 6661 dev_info(&pf->pdev->dev, 6662 "couldn't get PF vsi config, err %s aq_err %s\n", 6663 i40e_stat_str(&pf->hw, ret), 6664 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6665 return; 6666 } 6667 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6668 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6669 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6670 6671 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6672 if (ret) { 6673 dev_info(&pf->pdev->dev, 6674 "update vsi switch failed, err %s aq_err %s\n", 6675 i40e_stat_str(&pf->hw, ret), 6676 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6677 } 6678 } 6679 6680 /** 6681 * i40e_config_bridge_mode - Configure the HW bridge mode 6682 * @veb: pointer to the bridge instance 6683 * 6684 * Configure the loop back mode for the LAN VSI that is downlink to the 6685 * specified HW bridge instance. It is expected this function is called 6686 * when a new HW bridge is instantiated. 6687 **/ 6688 static void i40e_config_bridge_mode(struct i40e_veb *veb) 6689 { 6690 struct i40e_pf *pf = veb->pf; 6691 6692 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 6693 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 6694 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 6695 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 6696 i40e_disable_pf_switch_lb(pf); 6697 else 6698 i40e_enable_pf_switch_lb(pf); 6699 } 6700 6701 /** 6702 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 6703 * @veb: pointer to the VEB instance 6704 * 6705 * This is a recursive function that first builds the attached VSIs then 6706 * recurses in to build the next layer of VEB. We track the connections 6707 * through our own index numbers because the seid's from the HW could 6708 * change across the reset. 6709 **/ 6710 static int i40e_reconstitute_veb(struct i40e_veb *veb) 6711 { 6712 struct i40e_vsi *ctl_vsi = NULL; 6713 struct i40e_pf *pf = veb->pf; 6714 int v, veb_idx; 6715 int ret; 6716 6717 /* build VSI that owns this VEB, temporarily attached to base VEB */ 6718 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 6719 if (pf->vsi[v] && 6720 pf->vsi[v]->veb_idx == veb->idx && 6721 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 6722 ctl_vsi = pf->vsi[v]; 6723 break; 6724 } 6725 } 6726 if (!ctl_vsi) { 6727 dev_info(&pf->pdev->dev, 6728 "missing owner VSI for veb_idx %d\n", veb->idx); 6729 ret = -ENOENT; 6730 goto end_reconstitute; 6731 } 6732 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 6733 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6734 ret = i40e_add_vsi(ctl_vsi); 6735 if (ret) { 6736 dev_info(&pf->pdev->dev, 6737 "rebuild of veb_idx %d owner VSI failed: %d\n", 6738 veb->idx, ret); 6739 goto end_reconstitute; 6740 } 6741 i40e_vsi_reset_stats(ctl_vsi); 6742 6743 /* create the VEB in the switch and move the VSI onto the VEB */ 6744 ret = i40e_add_veb(veb, ctl_vsi); 6745 if (ret) 6746 goto end_reconstitute; 6747 6748 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 6749 veb->bridge_mode = BRIDGE_MODE_VEB; 6750 else 6751 veb->bridge_mode = BRIDGE_MODE_VEPA; 6752 i40e_config_bridge_mode(veb); 6753 6754 /* create the remaining VSIs attached to this VEB */ 6755 for (v = 0; v < pf->num_alloc_vsi; v++) { 6756 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 6757 continue; 6758 6759 if (pf->vsi[v]->veb_idx == veb->idx) { 6760 struct i40e_vsi *vsi = pf->vsi[v]; 6761 6762 vsi->uplink_seid = veb->seid; 6763 ret = i40e_add_vsi(vsi); 6764 if (ret) { 6765 dev_info(&pf->pdev->dev, 6766 "rebuild of vsi_idx %d failed: %d\n", 6767 v, ret); 6768 goto end_reconstitute; 6769 } 6770 i40e_vsi_reset_stats(vsi); 6771 } 6772 } 6773 6774 /* create any VEBs attached to this VEB - RECURSION */ 6775 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 6776 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 6777 pf->veb[veb_idx]->uplink_seid = veb->seid; 6778 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 6779 if (ret) 6780 break; 6781 } 6782 } 6783 6784 end_reconstitute: 6785 return ret; 6786 } 6787 6788 /** 6789 * i40e_get_capabilities - get info about the HW 6790 * @pf: the PF struct 6791 **/ 6792 static int i40e_get_capabilities(struct i40e_pf *pf) 6793 { 6794 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 6795 u16 data_size; 6796 int buf_len; 6797 int err; 6798 6799 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 6800 do { 6801 cap_buf = kzalloc(buf_len, GFP_KERNEL); 6802 if (!cap_buf) 6803 return -ENOMEM; 6804 6805 /* this loads the data into the hw struct for us */ 6806 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 6807 &data_size, 6808 i40e_aqc_opc_list_func_capabilities, 6809 NULL); 6810 /* data loaded, buffer no longer needed */ 6811 kfree(cap_buf); 6812 6813 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 6814 /* retry with a larger buffer */ 6815 buf_len = data_size; 6816 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 6817 dev_info(&pf->pdev->dev, 6818 "capability discovery failed, err %s aq_err %s\n", 6819 i40e_stat_str(&pf->hw, err), 6820 i40e_aq_str(&pf->hw, 6821 pf->hw.aq.asq_last_status)); 6822 return -ENODEV; 6823 } 6824 } while (err); 6825 6826 if (pf->hw.debug_mask & I40E_DEBUG_USER) 6827 dev_info(&pf->pdev->dev, 6828 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 6829 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 6830 pf->hw.func_caps.num_msix_vectors, 6831 pf->hw.func_caps.num_msix_vectors_vf, 6832 pf->hw.func_caps.fd_filters_guaranteed, 6833 pf->hw.func_caps.fd_filters_best_effort, 6834 pf->hw.func_caps.num_tx_qp, 6835 pf->hw.func_caps.num_vsis); 6836 6837 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 6838 + pf->hw.func_caps.num_vfs) 6839 if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) { 6840 dev_info(&pf->pdev->dev, 6841 "got num_vsis %d, setting num_vsis to %d\n", 6842 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 6843 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 6844 } 6845 6846 return 0; 6847 } 6848 6849 static int i40e_vsi_clear(struct i40e_vsi *vsi); 6850 6851 /** 6852 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 6853 * @pf: board private structure 6854 **/ 6855 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 6856 { 6857 struct i40e_vsi *vsi; 6858 6859 /* quick workaround for an NVM issue that leaves a critical register 6860 * uninitialized 6861 */ 6862 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 6863 static const u32 hkey[] = { 6864 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 6865 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 6866 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 6867 0x95b3a76d}; 6868 int i; 6869 6870 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 6871 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 6872 } 6873 6874 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 6875 return; 6876 6877 /* find existing VSI and see if it needs configuring */ 6878 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6879 6880 /* create a new VSI if none exists */ 6881 if (!vsi) { 6882 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 6883 pf->vsi[pf->lan_vsi]->seid, 0); 6884 if (!vsi) { 6885 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 6886 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 6887 return; 6888 } 6889 } 6890 6891 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 6892 } 6893 6894 /** 6895 * i40e_fdir_teardown - release the Flow Director resources 6896 * @pf: board private structure 6897 **/ 6898 static void i40e_fdir_teardown(struct i40e_pf *pf) 6899 { 6900 struct i40e_vsi *vsi; 6901 6902 i40e_fdir_filter_exit(pf); 6903 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6904 if (vsi) 6905 i40e_vsi_release(vsi); 6906 } 6907 6908 /** 6909 * i40e_prep_for_reset - prep for the core to reset 6910 * @pf: board private structure 6911 * @lock_acquired: indicates whether or not the lock has been acquired 6912 * before this function was called. 6913 * 6914 * Close up the VFs and other things in prep for PF Reset. 6915 **/ 6916 static void i40e_prep_for_reset(struct i40e_pf *pf, bool lock_acquired) 6917 { 6918 struct i40e_hw *hw = &pf->hw; 6919 i40e_status ret = 0; 6920 u32 v; 6921 6922 clear_bit(__I40E_RESET_INTR_RECEIVED, pf->state); 6923 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 6924 return; 6925 if (i40e_check_asq_alive(&pf->hw)) 6926 i40e_vc_notify_reset(pf); 6927 6928 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 6929 6930 /* quiesce the VSIs and their queues that are not already DOWN */ 6931 /* pf_quiesce_all_vsi modifies netdev structures -rtnl_lock needed */ 6932 if (!lock_acquired) 6933 rtnl_lock(); 6934 i40e_pf_quiesce_all_vsi(pf); 6935 if (!lock_acquired) 6936 rtnl_unlock(); 6937 6938 for (v = 0; v < pf->num_alloc_vsi; v++) { 6939 if (pf->vsi[v]) 6940 pf->vsi[v]->seid = 0; 6941 } 6942 6943 i40e_shutdown_adminq(&pf->hw); 6944 6945 /* call shutdown HMC */ 6946 if (hw->hmc.hmc_obj) { 6947 ret = i40e_shutdown_lan_hmc(hw); 6948 if (ret) 6949 dev_warn(&pf->pdev->dev, 6950 "shutdown_lan_hmc failed: %d\n", ret); 6951 } 6952 } 6953 6954 /** 6955 * i40e_send_version - update firmware with driver version 6956 * @pf: PF struct 6957 */ 6958 static void i40e_send_version(struct i40e_pf *pf) 6959 { 6960 struct i40e_driver_version dv; 6961 6962 dv.major_version = DRV_VERSION_MAJOR; 6963 dv.minor_version = DRV_VERSION_MINOR; 6964 dv.build_version = DRV_VERSION_BUILD; 6965 dv.subbuild_version = 0; 6966 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 6967 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 6968 } 6969 6970 /** 6971 * i40e_reset - wait for core reset to finish reset, reset pf if corer not seen 6972 * @pf: board private structure 6973 **/ 6974 static int i40e_reset(struct i40e_pf *pf) 6975 { 6976 struct i40e_hw *hw = &pf->hw; 6977 i40e_status ret; 6978 6979 ret = i40e_pf_reset(hw); 6980 if (ret) { 6981 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 6982 set_bit(__I40E_RESET_FAILED, pf->state); 6983 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 6984 } else { 6985 pf->pfr_count++; 6986 } 6987 return ret; 6988 } 6989 6990 /** 6991 * i40e_rebuild - rebuild using a saved config 6992 * @pf: board private structure 6993 * @reinit: if the Main VSI needs to re-initialized. 6994 * @lock_acquired: indicates whether or not the lock has been acquired 6995 * before this function was called. 6996 **/ 6997 static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) 6998 { 6999 struct i40e_hw *hw = &pf->hw; 7000 u8 set_fc_aq_fail = 0; 7001 i40e_status ret; 7002 u32 val; 7003 int v; 7004 7005 if (test_bit(__I40E_VSI_DOWN, pf->state)) 7006 goto clear_recovery; 7007 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 7008 7009 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 7010 ret = i40e_init_adminq(&pf->hw); 7011 if (ret) { 7012 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 7013 i40e_stat_str(&pf->hw, ret), 7014 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7015 goto clear_recovery; 7016 } 7017 7018 /* re-verify the eeprom if we just had an EMP reset */ 7019 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state)) 7020 i40e_verify_eeprom(pf); 7021 7022 i40e_clear_pxe_mode(hw); 7023 ret = i40e_get_capabilities(pf); 7024 if (ret) 7025 goto end_core_reset; 7026 7027 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 7028 hw->func_caps.num_rx_qp, 0, 0); 7029 if (ret) { 7030 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 7031 goto end_core_reset; 7032 } 7033 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 7034 if (ret) { 7035 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 7036 goto end_core_reset; 7037 } 7038 7039 #ifdef CONFIG_I40E_DCB 7040 ret = i40e_init_pf_dcb(pf); 7041 if (ret) { 7042 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 7043 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 7044 /* Continue without DCB enabled */ 7045 } 7046 #endif /* CONFIG_I40E_DCB */ 7047 /* do basic switch setup */ 7048 if (!lock_acquired) 7049 rtnl_lock(); 7050 ret = i40e_setup_pf_switch(pf, reinit); 7051 if (ret) 7052 goto end_unlock; 7053 7054 /* The driver only wants link up/down and module qualification 7055 * reports from firmware. Note the negative logic. 7056 */ 7057 ret = i40e_aq_set_phy_int_mask(&pf->hw, 7058 ~(I40E_AQ_EVENT_LINK_UPDOWN | 7059 I40E_AQ_EVENT_MEDIA_NA | 7060 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 7061 if (ret) 7062 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 7063 i40e_stat_str(&pf->hw, ret), 7064 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7065 7066 /* make sure our flow control settings are restored */ 7067 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 7068 if (ret) 7069 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 7070 i40e_stat_str(&pf->hw, ret), 7071 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7072 7073 /* Rebuild the VSIs and VEBs that existed before reset. 7074 * They are still in our local switch element arrays, so only 7075 * need to rebuild the switch model in the HW. 7076 * 7077 * If there were VEBs but the reconstitution failed, we'll try 7078 * try to recover minimal use by getting the basic PF VSI working. 7079 */ 7080 if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) { 7081 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 7082 /* find the one VEB connected to the MAC, and find orphans */ 7083 for (v = 0; v < I40E_MAX_VEB; v++) { 7084 if (!pf->veb[v]) 7085 continue; 7086 7087 if (pf->veb[v]->uplink_seid == pf->mac_seid || 7088 pf->veb[v]->uplink_seid == 0) { 7089 ret = i40e_reconstitute_veb(pf->veb[v]); 7090 7091 if (!ret) 7092 continue; 7093 7094 /* If Main VEB failed, we're in deep doodoo, 7095 * so give up rebuilding the switch and set up 7096 * for minimal rebuild of PF VSI. 7097 * If orphan failed, we'll report the error 7098 * but try to keep going. 7099 */ 7100 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 7101 dev_info(&pf->pdev->dev, 7102 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 7103 ret); 7104 pf->vsi[pf->lan_vsi]->uplink_seid 7105 = pf->mac_seid; 7106 break; 7107 } else if (pf->veb[v]->uplink_seid == 0) { 7108 dev_info(&pf->pdev->dev, 7109 "rebuild of orphan VEB failed: %d\n", 7110 ret); 7111 } 7112 } 7113 } 7114 } 7115 7116 if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) { 7117 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 7118 /* no VEB, so rebuild only the Main VSI */ 7119 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]); 7120 if (ret) { 7121 dev_info(&pf->pdev->dev, 7122 "rebuild of Main VSI failed: %d\n", ret); 7123 goto end_unlock; 7124 } 7125 } 7126 7127 /* Reconfigure hardware for allowing smaller MSS in the case 7128 * of TSO, so that we avoid the MDD being fired and causing 7129 * a reset in the case of small MSS+TSO. 7130 */ 7131 #define I40E_REG_MSS 0x000E64DC 7132 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 7133 #define I40E_64BYTE_MSS 0x400000 7134 val = rd32(hw, I40E_REG_MSS); 7135 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 7136 val &= ~I40E_REG_MSS_MIN_MASK; 7137 val |= I40E_64BYTE_MSS; 7138 wr32(hw, I40E_REG_MSS, val); 7139 } 7140 7141 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) { 7142 msleep(75); 7143 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 7144 if (ret) 7145 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 7146 i40e_stat_str(&pf->hw, ret), 7147 i40e_aq_str(&pf->hw, 7148 pf->hw.aq.asq_last_status)); 7149 } 7150 /* reinit the misc interrupt */ 7151 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7152 ret = i40e_setup_misc_vector(pf); 7153 7154 /* Add a filter to drop all Flow control frames from any VSI from being 7155 * transmitted. By doing so we stop a malicious VF from sending out 7156 * PAUSE or PFC frames and potentially controlling traffic for other 7157 * PF/VF VSIs. 7158 * The FW can still send Flow control frames if enabled. 7159 */ 7160 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 7161 pf->main_vsi_seid); 7162 7163 /* restart the VSIs that were rebuilt and running before the reset */ 7164 i40e_pf_unquiesce_all_vsi(pf); 7165 7166 /* Release the RTNL lock before we start resetting VFs */ 7167 if (!lock_acquired) 7168 rtnl_unlock(); 7169 7170 i40e_reset_all_vfs(pf, true); 7171 7172 /* tell the firmware that we're starting */ 7173 i40e_send_version(pf); 7174 7175 /* We've already released the lock, so don't do it again */ 7176 goto end_core_reset; 7177 7178 end_unlock: 7179 if (!lock_acquired) 7180 rtnl_unlock(); 7181 end_core_reset: 7182 clear_bit(__I40E_RESET_FAILED, pf->state); 7183 clear_recovery: 7184 clear_bit(__I40E_RESET_RECOVERY_PENDING, pf->state); 7185 } 7186 7187 /** 7188 * i40e_reset_and_rebuild - reset and rebuild using a saved config 7189 * @pf: board private structure 7190 * @reinit: if the Main VSI needs to re-initialized. 7191 * @lock_acquired: indicates whether or not the lock has been acquired 7192 * before this function was called. 7193 **/ 7194 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit, 7195 bool lock_acquired) 7196 { 7197 int ret; 7198 /* Now we wait for GRST to settle out. 7199 * We don't have to delete the VEBs or VSIs from the hw switch 7200 * because the reset will make them disappear. 7201 */ 7202 ret = i40e_reset(pf); 7203 if (!ret) 7204 i40e_rebuild(pf, reinit, lock_acquired); 7205 } 7206 7207 /** 7208 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 7209 * @pf: board private structure 7210 * 7211 * Close up the VFs and other things in prep for a Core Reset, 7212 * then get ready to rebuild the world. 7213 * @lock_acquired: indicates whether or not the lock has been acquired 7214 * before this function was called. 7215 **/ 7216 static void i40e_handle_reset_warning(struct i40e_pf *pf, bool lock_acquired) 7217 { 7218 i40e_prep_for_reset(pf, lock_acquired); 7219 i40e_reset_and_rebuild(pf, false, lock_acquired); 7220 } 7221 7222 /** 7223 * i40e_handle_mdd_event 7224 * @pf: pointer to the PF structure 7225 * 7226 * Called from the MDD irq handler to identify possibly malicious vfs 7227 **/ 7228 static void i40e_handle_mdd_event(struct i40e_pf *pf) 7229 { 7230 struct i40e_hw *hw = &pf->hw; 7231 bool mdd_detected = false; 7232 bool pf_mdd_detected = false; 7233 struct i40e_vf *vf; 7234 u32 reg; 7235 int i; 7236 7237 if (!test_bit(__I40E_MDD_EVENT_PENDING, pf->state)) 7238 return; 7239 7240 /* find what triggered the MDD event */ 7241 reg = rd32(hw, I40E_GL_MDET_TX); 7242 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 7243 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 7244 I40E_GL_MDET_TX_PF_NUM_SHIFT; 7245 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 7246 I40E_GL_MDET_TX_VF_NUM_SHIFT; 7247 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 7248 I40E_GL_MDET_TX_EVENT_SHIFT; 7249 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 7250 I40E_GL_MDET_TX_QUEUE_SHIFT) - 7251 pf->hw.func_caps.base_queue; 7252 if (netif_msg_tx_err(pf)) 7253 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 7254 event, queue, pf_num, vf_num); 7255 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 7256 mdd_detected = true; 7257 } 7258 reg = rd32(hw, I40E_GL_MDET_RX); 7259 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 7260 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 7261 I40E_GL_MDET_RX_FUNCTION_SHIFT; 7262 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 7263 I40E_GL_MDET_RX_EVENT_SHIFT; 7264 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 7265 I40E_GL_MDET_RX_QUEUE_SHIFT) - 7266 pf->hw.func_caps.base_queue; 7267 if (netif_msg_rx_err(pf)) 7268 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 7269 event, queue, func); 7270 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 7271 mdd_detected = true; 7272 } 7273 7274 if (mdd_detected) { 7275 reg = rd32(hw, I40E_PF_MDET_TX); 7276 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 7277 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 7278 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 7279 pf_mdd_detected = true; 7280 } 7281 reg = rd32(hw, I40E_PF_MDET_RX); 7282 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 7283 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 7284 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 7285 pf_mdd_detected = true; 7286 } 7287 /* Queue belongs to the PF, initiate a reset */ 7288 if (pf_mdd_detected) { 7289 set_bit(__I40E_PF_RESET_REQUESTED, pf->state); 7290 i40e_service_event_schedule(pf); 7291 } 7292 } 7293 7294 /* see if one of the VFs needs its hand slapped */ 7295 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 7296 vf = &(pf->vf[i]); 7297 reg = rd32(hw, I40E_VP_MDET_TX(i)); 7298 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 7299 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 7300 vf->num_mdd_events++; 7301 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 7302 i); 7303 } 7304 7305 reg = rd32(hw, I40E_VP_MDET_RX(i)); 7306 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 7307 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 7308 vf->num_mdd_events++; 7309 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 7310 i); 7311 } 7312 7313 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 7314 dev_info(&pf->pdev->dev, 7315 "Too many MDD events on VF %d, disabled\n", i); 7316 dev_info(&pf->pdev->dev, 7317 "Use PF Control I/F to re-enable the VF\n"); 7318 set_bit(I40E_VF_STATE_DISABLED, &vf->vf_states); 7319 } 7320 } 7321 7322 /* re-enable mdd interrupt cause */ 7323 clear_bit(__I40E_MDD_EVENT_PENDING, pf->state); 7324 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 7325 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 7326 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 7327 i40e_flush(hw); 7328 } 7329 7330 /** 7331 * i40e_sync_udp_filters - Trigger a sync event for existing UDP filters 7332 * @pf: board private structure 7333 **/ 7334 static void i40e_sync_udp_filters(struct i40e_pf *pf) 7335 { 7336 int i; 7337 7338 /* loop through and set pending bit for all active UDP filters */ 7339 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 7340 if (pf->udp_ports[i].port) 7341 pf->pending_udp_bitmap |= BIT_ULL(i); 7342 } 7343 7344 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 7345 } 7346 7347 /** 7348 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 7349 * @pf: board private structure 7350 **/ 7351 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 7352 { 7353 struct i40e_hw *hw = &pf->hw; 7354 i40e_status ret; 7355 u16 port; 7356 int i; 7357 7358 if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC)) 7359 return; 7360 7361 pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC; 7362 7363 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 7364 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 7365 pf->pending_udp_bitmap &= ~BIT_ULL(i); 7366 port = pf->udp_ports[i].port; 7367 if (port) 7368 ret = i40e_aq_add_udp_tunnel(hw, port, 7369 pf->udp_ports[i].type, 7370 NULL, NULL); 7371 else 7372 ret = i40e_aq_del_udp_tunnel(hw, i, NULL); 7373 7374 if (ret) { 7375 dev_dbg(&pf->pdev->dev, 7376 "%s %s port %d, index %d failed, err %s aq_err %s\n", 7377 pf->udp_ports[i].type ? "vxlan" : "geneve", 7378 port ? "add" : "delete", 7379 port, i, 7380 i40e_stat_str(&pf->hw, ret), 7381 i40e_aq_str(&pf->hw, 7382 pf->hw.aq.asq_last_status)); 7383 pf->udp_ports[i].port = 0; 7384 } 7385 } 7386 } 7387 } 7388 7389 /** 7390 * i40e_service_task - Run the driver's async subtasks 7391 * @work: pointer to work_struct containing our data 7392 **/ 7393 static void i40e_service_task(struct work_struct *work) 7394 { 7395 struct i40e_pf *pf = container_of(work, 7396 struct i40e_pf, 7397 service_task); 7398 unsigned long start_time = jiffies; 7399 7400 /* don't bother with service tasks if a reset is in progress */ 7401 if (test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) 7402 return; 7403 7404 if (test_and_set_bit(__I40E_SERVICE_SCHED, pf->state)) 7405 return; 7406 7407 i40e_detect_recover_hung(pf); 7408 i40e_sync_filters_subtask(pf); 7409 i40e_reset_subtask(pf); 7410 i40e_handle_mdd_event(pf); 7411 i40e_vc_process_vflr_event(pf); 7412 i40e_watchdog_subtask(pf); 7413 i40e_fdir_reinit_subtask(pf); 7414 if (pf->flags & I40E_FLAG_CLIENT_RESET) { 7415 /* Client subtask will reopen next time through. */ 7416 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true); 7417 pf->flags &= ~I40E_FLAG_CLIENT_RESET; 7418 } else { 7419 i40e_client_subtask(pf); 7420 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) { 7421 i40e_notify_client_of_l2_param_changes( 7422 pf->vsi[pf->lan_vsi]); 7423 pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE; 7424 } 7425 } 7426 i40e_sync_filters_subtask(pf); 7427 i40e_sync_udp_filters_subtask(pf); 7428 i40e_clean_adminq_subtask(pf); 7429 7430 /* flush memory to make sure state is correct before next watchdog */ 7431 smp_mb__before_atomic(); 7432 clear_bit(__I40E_SERVICE_SCHED, pf->state); 7433 7434 /* If the tasks have taken longer than one timer cycle or there 7435 * is more work to be done, reschedule the service task now 7436 * rather than wait for the timer to tick again. 7437 */ 7438 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 7439 test_bit(__I40E_ADMINQ_EVENT_PENDING, pf->state) || 7440 test_bit(__I40E_MDD_EVENT_PENDING, pf->state) || 7441 test_bit(__I40E_VFLR_EVENT_PENDING, pf->state)) 7442 i40e_service_event_schedule(pf); 7443 } 7444 7445 /** 7446 * i40e_service_timer - timer callback 7447 * @data: pointer to PF struct 7448 **/ 7449 static void i40e_service_timer(unsigned long data) 7450 { 7451 struct i40e_pf *pf = (struct i40e_pf *)data; 7452 7453 mod_timer(&pf->service_timer, 7454 round_jiffies(jiffies + pf->service_timer_period)); 7455 i40e_service_event_schedule(pf); 7456 } 7457 7458 /** 7459 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 7460 * @vsi: the VSI being configured 7461 **/ 7462 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 7463 { 7464 struct i40e_pf *pf = vsi->back; 7465 7466 switch (vsi->type) { 7467 case I40E_VSI_MAIN: 7468 vsi->alloc_queue_pairs = pf->num_lan_qps; 7469 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7470 I40E_REQ_DESCRIPTOR_MULTIPLE); 7471 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7472 vsi->num_q_vectors = pf->num_lan_msix; 7473 else 7474 vsi->num_q_vectors = 1; 7475 7476 break; 7477 7478 case I40E_VSI_FDIR: 7479 vsi->alloc_queue_pairs = 1; 7480 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 7481 I40E_REQ_DESCRIPTOR_MULTIPLE); 7482 vsi->num_q_vectors = pf->num_fdsb_msix; 7483 break; 7484 7485 case I40E_VSI_VMDQ2: 7486 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 7487 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7488 I40E_REQ_DESCRIPTOR_MULTIPLE); 7489 vsi->num_q_vectors = pf->num_vmdq_msix; 7490 break; 7491 7492 case I40E_VSI_SRIOV: 7493 vsi->alloc_queue_pairs = pf->num_vf_qps; 7494 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7495 I40E_REQ_DESCRIPTOR_MULTIPLE); 7496 break; 7497 7498 default: 7499 WARN_ON(1); 7500 return -ENODATA; 7501 } 7502 7503 return 0; 7504 } 7505 7506 /** 7507 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 7508 * @type: VSI pointer 7509 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 7510 * 7511 * On error: returns error code (negative) 7512 * On success: returns 0 7513 **/ 7514 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 7515 { 7516 int size; 7517 int ret = 0; 7518 7519 /* allocate memory for both Tx and Rx ring pointers */ 7520 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2; 7521 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 7522 if (!vsi->tx_rings) 7523 return -ENOMEM; 7524 vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs]; 7525 7526 if (alloc_qvectors) { 7527 /* allocate memory for q_vector pointers */ 7528 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 7529 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 7530 if (!vsi->q_vectors) { 7531 ret = -ENOMEM; 7532 goto err_vectors; 7533 } 7534 } 7535 return ret; 7536 7537 err_vectors: 7538 kfree(vsi->tx_rings); 7539 return ret; 7540 } 7541 7542 /** 7543 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 7544 * @pf: board private structure 7545 * @type: type of VSI 7546 * 7547 * On error: returns error code (negative) 7548 * On success: returns vsi index in PF (positive) 7549 **/ 7550 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 7551 { 7552 int ret = -ENODEV; 7553 struct i40e_vsi *vsi; 7554 int vsi_idx; 7555 int i; 7556 7557 /* Need to protect the allocation of the VSIs at the PF level */ 7558 mutex_lock(&pf->switch_mutex); 7559 7560 /* VSI list may be fragmented if VSI creation/destruction has 7561 * been happening. We can afford to do a quick scan to look 7562 * for any free VSIs in the list. 7563 * 7564 * find next empty vsi slot, looping back around if necessary 7565 */ 7566 i = pf->next_vsi; 7567 while (i < pf->num_alloc_vsi && pf->vsi[i]) 7568 i++; 7569 if (i >= pf->num_alloc_vsi) { 7570 i = 0; 7571 while (i < pf->next_vsi && pf->vsi[i]) 7572 i++; 7573 } 7574 7575 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 7576 vsi_idx = i; /* Found one! */ 7577 } else { 7578 ret = -ENODEV; 7579 goto unlock_pf; /* out of VSI slots! */ 7580 } 7581 pf->next_vsi = ++i; 7582 7583 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 7584 if (!vsi) { 7585 ret = -ENOMEM; 7586 goto unlock_pf; 7587 } 7588 vsi->type = type; 7589 vsi->back = pf; 7590 set_bit(__I40E_VSI_DOWN, vsi->state); 7591 vsi->flags = 0; 7592 vsi->idx = vsi_idx; 7593 vsi->int_rate_limit = 0; 7594 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 7595 pf->rss_table_size : 64; 7596 vsi->netdev_registered = false; 7597 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 7598 hash_init(vsi->mac_filter_hash); 7599 vsi->irqs_ready = false; 7600 7601 ret = i40e_set_num_rings_in_vsi(vsi); 7602 if (ret) 7603 goto err_rings; 7604 7605 ret = i40e_vsi_alloc_arrays(vsi, true); 7606 if (ret) 7607 goto err_rings; 7608 7609 /* Setup default MSIX irq handler for VSI */ 7610 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 7611 7612 /* Initialize VSI lock */ 7613 spin_lock_init(&vsi->mac_filter_hash_lock); 7614 pf->vsi[vsi_idx] = vsi; 7615 ret = vsi_idx; 7616 goto unlock_pf; 7617 7618 err_rings: 7619 pf->next_vsi = i - 1; 7620 kfree(vsi); 7621 unlock_pf: 7622 mutex_unlock(&pf->switch_mutex); 7623 return ret; 7624 } 7625 7626 /** 7627 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 7628 * @type: VSI pointer 7629 * @free_qvectors: a bool to specify if q_vectors need to be freed. 7630 * 7631 * On error: returns error code (negative) 7632 * On success: returns 0 7633 **/ 7634 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 7635 { 7636 /* free the ring and vector containers */ 7637 if (free_qvectors) { 7638 kfree(vsi->q_vectors); 7639 vsi->q_vectors = NULL; 7640 } 7641 kfree(vsi->tx_rings); 7642 vsi->tx_rings = NULL; 7643 vsi->rx_rings = NULL; 7644 } 7645 7646 /** 7647 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 7648 * and lookup table 7649 * @vsi: Pointer to VSI structure 7650 */ 7651 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 7652 { 7653 if (!vsi) 7654 return; 7655 7656 kfree(vsi->rss_hkey_user); 7657 vsi->rss_hkey_user = NULL; 7658 7659 kfree(vsi->rss_lut_user); 7660 vsi->rss_lut_user = NULL; 7661 } 7662 7663 /** 7664 * i40e_vsi_clear - Deallocate the VSI provided 7665 * @vsi: the VSI being un-configured 7666 **/ 7667 static int i40e_vsi_clear(struct i40e_vsi *vsi) 7668 { 7669 struct i40e_pf *pf; 7670 7671 if (!vsi) 7672 return 0; 7673 7674 if (!vsi->back) 7675 goto free_vsi; 7676 pf = vsi->back; 7677 7678 mutex_lock(&pf->switch_mutex); 7679 if (!pf->vsi[vsi->idx]) { 7680 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n", 7681 vsi->idx, vsi->idx, vsi, vsi->type); 7682 goto unlock_vsi; 7683 } 7684 7685 if (pf->vsi[vsi->idx] != vsi) { 7686 dev_err(&pf->pdev->dev, 7687 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n", 7688 pf->vsi[vsi->idx]->idx, 7689 pf->vsi[vsi->idx], 7690 pf->vsi[vsi->idx]->type, 7691 vsi->idx, vsi, vsi->type); 7692 goto unlock_vsi; 7693 } 7694 7695 /* updates the PF for this cleared vsi */ 7696 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 7697 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 7698 7699 i40e_vsi_free_arrays(vsi, true); 7700 i40e_clear_rss_config_user(vsi); 7701 7702 pf->vsi[vsi->idx] = NULL; 7703 if (vsi->idx < pf->next_vsi) 7704 pf->next_vsi = vsi->idx; 7705 7706 unlock_vsi: 7707 mutex_unlock(&pf->switch_mutex); 7708 free_vsi: 7709 kfree(vsi); 7710 7711 return 0; 7712 } 7713 7714 /** 7715 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 7716 * @vsi: the VSI being cleaned 7717 **/ 7718 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 7719 { 7720 int i; 7721 7722 if (vsi->tx_rings && vsi->tx_rings[0]) { 7723 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7724 kfree_rcu(vsi->tx_rings[i], rcu); 7725 vsi->tx_rings[i] = NULL; 7726 vsi->rx_rings[i] = NULL; 7727 } 7728 } 7729 } 7730 7731 /** 7732 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 7733 * @vsi: the VSI being configured 7734 **/ 7735 static int i40e_alloc_rings(struct i40e_vsi *vsi) 7736 { 7737 struct i40e_ring *tx_ring, *rx_ring; 7738 struct i40e_pf *pf = vsi->back; 7739 int i; 7740 7741 /* Set basic values in the rings to be used later during open() */ 7742 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7743 /* allocate space for both Tx and Rx in one shot */ 7744 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL); 7745 if (!tx_ring) 7746 goto err_out; 7747 7748 tx_ring->queue_index = i; 7749 tx_ring->reg_idx = vsi->base_queue + i; 7750 tx_ring->ring_active = false; 7751 tx_ring->vsi = vsi; 7752 tx_ring->netdev = vsi->netdev; 7753 tx_ring->dev = &pf->pdev->dev; 7754 tx_ring->count = vsi->num_desc; 7755 tx_ring->size = 0; 7756 tx_ring->dcb_tc = 0; 7757 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE) 7758 tx_ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 7759 tx_ring->tx_itr_setting = pf->tx_itr_default; 7760 vsi->tx_rings[i] = tx_ring; 7761 7762 rx_ring = &tx_ring[1]; 7763 rx_ring->queue_index = i; 7764 rx_ring->reg_idx = vsi->base_queue + i; 7765 rx_ring->ring_active = false; 7766 rx_ring->vsi = vsi; 7767 rx_ring->netdev = vsi->netdev; 7768 rx_ring->dev = &pf->pdev->dev; 7769 rx_ring->count = vsi->num_desc; 7770 rx_ring->size = 0; 7771 rx_ring->dcb_tc = 0; 7772 rx_ring->rx_itr_setting = pf->rx_itr_default; 7773 vsi->rx_rings[i] = rx_ring; 7774 } 7775 7776 return 0; 7777 7778 err_out: 7779 i40e_vsi_clear_rings(vsi); 7780 return -ENOMEM; 7781 } 7782 7783 /** 7784 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 7785 * @pf: board private structure 7786 * @vectors: the number of MSI-X vectors to request 7787 * 7788 * Returns the number of vectors reserved, or error 7789 **/ 7790 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 7791 { 7792 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 7793 I40E_MIN_MSIX, vectors); 7794 if (vectors < 0) { 7795 dev_info(&pf->pdev->dev, 7796 "MSI-X vector reservation failed: %d\n", vectors); 7797 vectors = 0; 7798 } 7799 7800 return vectors; 7801 } 7802 7803 /** 7804 * i40e_init_msix - Setup the MSIX capability 7805 * @pf: board private structure 7806 * 7807 * Work with the OS to set up the MSIX vectors needed. 7808 * 7809 * Returns the number of vectors reserved or negative on failure 7810 **/ 7811 static int i40e_init_msix(struct i40e_pf *pf) 7812 { 7813 struct i40e_hw *hw = &pf->hw; 7814 int cpus, extra_vectors; 7815 int vectors_left; 7816 int v_budget, i; 7817 int v_actual; 7818 int iwarp_requested = 0; 7819 7820 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 7821 return -ENODEV; 7822 7823 /* The number of vectors we'll request will be comprised of: 7824 * - Add 1 for "other" cause for Admin Queue events, etc. 7825 * - The number of LAN queue pairs 7826 * - Queues being used for RSS. 7827 * We don't need as many as max_rss_size vectors. 7828 * use rss_size instead in the calculation since that 7829 * is governed by number of cpus in the system. 7830 * - assumes symmetric Tx/Rx pairing 7831 * - The number of VMDq pairs 7832 * - The CPU count within the NUMA node if iWARP is enabled 7833 * Once we count this up, try the request. 7834 * 7835 * If we can't get what we want, we'll simplify to nearly nothing 7836 * and try again. If that still fails, we punt. 7837 */ 7838 vectors_left = hw->func_caps.num_msix_vectors; 7839 v_budget = 0; 7840 7841 /* reserve one vector for miscellaneous handler */ 7842 if (vectors_left) { 7843 v_budget++; 7844 vectors_left--; 7845 } 7846 7847 /* reserve some vectors for the main PF traffic queues. Initially we 7848 * only reserve at most 50% of the available vectors, in the case that 7849 * the number of online CPUs is large. This ensures that we can enable 7850 * extra features as well. Once we've enabled the other features, we 7851 * will use any remaining vectors to reach as close as we can to the 7852 * number of online CPUs. 7853 */ 7854 cpus = num_online_cpus(); 7855 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 7856 vectors_left -= pf->num_lan_msix; 7857 7858 /* reserve one vector for sideband flow director */ 7859 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 7860 if (vectors_left) { 7861 pf->num_fdsb_msix = 1; 7862 v_budget++; 7863 vectors_left--; 7864 } else { 7865 pf->num_fdsb_msix = 0; 7866 } 7867 } 7868 7869 /* can we reserve enough for iWARP? */ 7870 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7871 iwarp_requested = pf->num_iwarp_msix; 7872 7873 if (!vectors_left) 7874 pf->num_iwarp_msix = 0; 7875 else if (vectors_left < pf->num_iwarp_msix) 7876 pf->num_iwarp_msix = 1; 7877 v_budget += pf->num_iwarp_msix; 7878 vectors_left -= pf->num_iwarp_msix; 7879 } 7880 7881 /* any vectors left over go for VMDq support */ 7882 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 7883 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps; 7884 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted); 7885 7886 if (!vectors_left) { 7887 pf->num_vmdq_msix = 0; 7888 pf->num_vmdq_qps = 0; 7889 } else { 7890 /* if we're short on vectors for what's desired, we limit 7891 * the queues per vmdq. If this is still more than are 7892 * available, the user will need to change the number of 7893 * queues/vectors used by the PF later with the ethtool 7894 * channels command 7895 */ 7896 if (vmdq_vecs < vmdq_vecs_wanted) 7897 pf->num_vmdq_qps = 1; 7898 pf->num_vmdq_msix = pf->num_vmdq_qps; 7899 7900 v_budget += vmdq_vecs; 7901 vectors_left -= vmdq_vecs; 7902 } 7903 } 7904 7905 /* On systems with a large number of SMP cores, we previously limited 7906 * the number of vectors for num_lan_msix to be at most 50% of the 7907 * available vectors, to allow for other features. Now, we add back 7908 * the remaining vectors. However, we ensure that the total 7909 * num_lan_msix will not exceed num_online_cpus(). To do this, we 7910 * calculate the number of vectors we can add without going over the 7911 * cap of CPUs. For systems with a small number of CPUs this will be 7912 * zero. 7913 */ 7914 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 7915 pf->num_lan_msix += extra_vectors; 7916 vectors_left -= extra_vectors; 7917 7918 WARN(vectors_left < 0, 7919 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 7920 7921 v_budget += pf->num_lan_msix; 7922 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 7923 GFP_KERNEL); 7924 if (!pf->msix_entries) 7925 return -ENOMEM; 7926 7927 for (i = 0; i < v_budget; i++) 7928 pf->msix_entries[i].entry = i; 7929 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 7930 7931 if (v_actual < I40E_MIN_MSIX) { 7932 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 7933 kfree(pf->msix_entries); 7934 pf->msix_entries = NULL; 7935 pci_disable_msix(pf->pdev); 7936 return -ENODEV; 7937 7938 } else if (v_actual == I40E_MIN_MSIX) { 7939 /* Adjust for minimal MSIX use */ 7940 pf->num_vmdq_vsis = 0; 7941 pf->num_vmdq_qps = 0; 7942 pf->num_lan_qps = 1; 7943 pf->num_lan_msix = 1; 7944 7945 } else if (!vectors_left) { 7946 /* If we have limited resources, we will start with no vectors 7947 * for the special features and then allocate vectors to some 7948 * of these features based on the policy and at the end disable 7949 * the features that did not get any vectors. 7950 */ 7951 int vec; 7952 7953 dev_info(&pf->pdev->dev, 7954 "MSI-X vector limit reached, attempting to redistribute vectors\n"); 7955 /* reserve the misc vector */ 7956 vec = v_actual - 1; 7957 7958 /* Scale vector usage down */ 7959 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 7960 pf->num_vmdq_vsis = 1; 7961 pf->num_vmdq_qps = 1; 7962 7963 /* partition out the remaining vectors */ 7964 switch (vec) { 7965 case 2: 7966 pf->num_lan_msix = 1; 7967 break; 7968 case 3: 7969 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7970 pf->num_lan_msix = 1; 7971 pf->num_iwarp_msix = 1; 7972 } else { 7973 pf->num_lan_msix = 2; 7974 } 7975 break; 7976 default: 7977 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7978 pf->num_iwarp_msix = min_t(int, (vec / 3), 7979 iwarp_requested); 7980 pf->num_vmdq_vsis = min_t(int, (vec / 3), 7981 I40E_DEFAULT_NUM_VMDQ_VSI); 7982 } else { 7983 pf->num_vmdq_vsis = min_t(int, (vec / 2), 7984 I40E_DEFAULT_NUM_VMDQ_VSI); 7985 } 7986 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 7987 pf->num_fdsb_msix = 1; 7988 vec--; 7989 } 7990 pf->num_lan_msix = min_t(int, 7991 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 7992 pf->num_lan_msix); 7993 pf->num_lan_qps = pf->num_lan_msix; 7994 break; 7995 } 7996 } 7997 7998 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 7999 (pf->num_fdsb_msix == 0)) { 8000 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 8001 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8002 } 8003 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 8004 (pf->num_vmdq_msix == 0)) { 8005 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 8006 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 8007 } 8008 8009 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 8010 (pf->num_iwarp_msix == 0)) { 8011 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 8012 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 8013 } 8014 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 8015 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 8016 pf->num_lan_msix, 8017 pf->num_vmdq_msix * pf->num_vmdq_vsis, 8018 pf->num_fdsb_msix, 8019 pf->num_iwarp_msix); 8020 8021 return v_actual; 8022 } 8023 8024 /** 8025 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 8026 * @vsi: the VSI being configured 8027 * @v_idx: index of the vector in the vsi struct 8028 * @cpu: cpu to be used on affinity_mask 8029 * 8030 * We allocate one q_vector. If allocation fails we return -ENOMEM. 8031 **/ 8032 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 8033 { 8034 struct i40e_q_vector *q_vector; 8035 8036 /* allocate q_vector */ 8037 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 8038 if (!q_vector) 8039 return -ENOMEM; 8040 8041 q_vector->vsi = vsi; 8042 q_vector->v_idx = v_idx; 8043 cpumask_set_cpu(cpu, &q_vector->affinity_mask); 8044 8045 if (vsi->netdev) 8046 netif_napi_add(vsi->netdev, &q_vector->napi, 8047 i40e_napi_poll, NAPI_POLL_WEIGHT); 8048 8049 q_vector->rx.latency_range = I40E_LOW_LATENCY; 8050 q_vector->tx.latency_range = I40E_LOW_LATENCY; 8051 8052 /* tie q_vector and vsi together */ 8053 vsi->q_vectors[v_idx] = q_vector; 8054 8055 return 0; 8056 } 8057 8058 /** 8059 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 8060 * @vsi: the VSI being configured 8061 * 8062 * We allocate one q_vector per queue interrupt. If allocation fails we 8063 * return -ENOMEM. 8064 **/ 8065 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 8066 { 8067 struct i40e_pf *pf = vsi->back; 8068 int err, v_idx, num_q_vectors, current_cpu; 8069 8070 /* if not MSIX, give the one vector only to the LAN VSI */ 8071 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 8072 num_q_vectors = vsi->num_q_vectors; 8073 else if (vsi == pf->vsi[pf->lan_vsi]) 8074 num_q_vectors = 1; 8075 else 8076 return -EINVAL; 8077 8078 current_cpu = cpumask_first(cpu_online_mask); 8079 8080 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 8081 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 8082 if (err) 8083 goto err_out; 8084 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 8085 if (unlikely(current_cpu >= nr_cpu_ids)) 8086 current_cpu = cpumask_first(cpu_online_mask); 8087 } 8088 8089 return 0; 8090 8091 err_out: 8092 while (v_idx--) 8093 i40e_free_q_vector(vsi, v_idx); 8094 8095 return err; 8096 } 8097 8098 /** 8099 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 8100 * @pf: board private structure to initialize 8101 **/ 8102 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 8103 { 8104 int vectors = 0; 8105 ssize_t size; 8106 8107 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 8108 vectors = i40e_init_msix(pf); 8109 if (vectors < 0) { 8110 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 8111 I40E_FLAG_IWARP_ENABLED | 8112 I40E_FLAG_RSS_ENABLED | 8113 I40E_FLAG_DCB_CAPABLE | 8114 I40E_FLAG_DCB_ENABLED | 8115 I40E_FLAG_SRIOV_ENABLED | 8116 I40E_FLAG_FD_SB_ENABLED | 8117 I40E_FLAG_FD_ATR_ENABLED | 8118 I40E_FLAG_VMDQ_ENABLED); 8119 8120 /* rework the queue expectations without MSIX */ 8121 i40e_determine_queue_usage(pf); 8122 } 8123 } 8124 8125 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 8126 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 8127 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 8128 vectors = pci_enable_msi(pf->pdev); 8129 if (vectors < 0) { 8130 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 8131 vectors); 8132 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 8133 } 8134 vectors = 1; /* one MSI or Legacy vector */ 8135 } 8136 8137 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 8138 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 8139 8140 /* set up vector assignment tracking */ 8141 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 8142 pf->irq_pile = kzalloc(size, GFP_KERNEL); 8143 if (!pf->irq_pile) { 8144 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n"); 8145 return -ENOMEM; 8146 } 8147 pf->irq_pile->num_entries = vectors; 8148 pf->irq_pile->search_hint = 0; 8149 8150 /* track first vector for misc interrupts, ignore return */ 8151 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 8152 8153 return 0; 8154 } 8155 8156 /** 8157 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 8158 * @pf: board private structure 8159 * 8160 * This sets up the handler for MSIX 0, which is used to manage the 8161 * non-queue interrupts, e.g. AdminQ and errors. This is not used 8162 * when in MSI or Legacy interrupt mode. 8163 **/ 8164 static int i40e_setup_misc_vector(struct i40e_pf *pf) 8165 { 8166 struct i40e_hw *hw = &pf->hw; 8167 int err = 0; 8168 8169 /* Only request the irq if this is the first time through, and 8170 * not when we're rebuilding after a Reset 8171 */ 8172 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, pf->state)) { 8173 err = request_irq(pf->msix_entries[0].vector, 8174 i40e_intr, 0, pf->int_name, pf); 8175 if (err) { 8176 dev_info(&pf->pdev->dev, 8177 "request_irq for %s failed: %d\n", 8178 pf->int_name, err); 8179 return -EFAULT; 8180 } 8181 } 8182 8183 i40e_enable_misc_int_causes(pf); 8184 8185 /* associate no queues to the misc vector */ 8186 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 8187 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 8188 8189 i40e_flush(hw); 8190 8191 i40e_irq_dynamic_enable_icr0(pf, true); 8192 8193 return err; 8194 } 8195 8196 /** 8197 * i40e_config_rss_aq - Prepare for RSS using AQ commands 8198 * @vsi: vsi structure 8199 * @seed: RSS hash seed 8200 **/ 8201 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8202 u8 *lut, u16 lut_size) 8203 { 8204 struct i40e_pf *pf = vsi->back; 8205 struct i40e_hw *hw = &pf->hw; 8206 int ret = 0; 8207 8208 if (seed) { 8209 struct i40e_aqc_get_set_rss_key_data *seed_dw = 8210 (struct i40e_aqc_get_set_rss_key_data *)seed; 8211 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 8212 if (ret) { 8213 dev_info(&pf->pdev->dev, 8214 "Cannot set RSS key, err %s aq_err %s\n", 8215 i40e_stat_str(hw, ret), 8216 i40e_aq_str(hw, hw->aq.asq_last_status)); 8217 return ret; 8218 } 8219 } 8220 if (lut) { 8221 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8222 8223 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8224 if (ret) { 8225 dev_info(&pf->pdev->dev, 8226 "Cannot set RSS lut, err %s aq_err %s\n", 8227 i40e_stat_str(hw, ret), 8228 i40e_aq_str(hw, hw->aq.asq_last_status)); 8229 return ret; 8230 } 8231 } 8232 return ret; 8233 } 8234 8235 /** 8236 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 8237 * @vsi: Pointer to vsi structure 8238 * @seed: Buffter to store the hash keys 8239 * @lut: Buffer to store the lookup table entries 8240 * @lut_size: Size of buffer to store the lookup table entries 8241 * 8242 * Return 0 on success, negative on failure 8243 */ 8244 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8245 u8 *lut, u16 lut_size) 8246 { 8247 struct i40e_pf *pf = vsi->back; 8248 struct i40e_hw *hw = &pf->hw; 8249 int ret = 0; 8250 8251 if (seed) { 8252 ret = i40e_aq_get_rss_key(hw, vsi->id, 8253 (struct i40e_aqc_get_set_rss_key_data *)seed); 8254 if (ret) { 8255 dev_info(&pf->pdev->dev, 8256 "Cannot get RSS key, err %s aq_err %s\n", 8257 i40e_stat_str(&pf->hw, ret), 8258 i40e_aq_str(&pf->hw, 8259 pf->hw.aq.asq_last_status)); 8260 return ret; 8261 } 8262 } 8263 8264 if (lut) { 8265 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8266 8267 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8268 if (ret) { 8269 dev_info(&pf->pdev->dev, 8270 "Cannot get RSS lut, err %s aq_err %s\n", 8271 i40e_stat_str(&pf->hw, ret), 8272 i40e_aq_str(&pf->hw, 8273 pf->hw.aq.asq_last_status)); 8274 return ret; 8275 } 8276 } 8277 8278 return ret; 8279 } 8280 8281 /** 8282 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 8283 * @vsi: VSI structure 8284 **/ 8285 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 8286 { 8287 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8288 struct i40e_pf *pf = vsi->back; 8289 u8 *lut; 8290 int ret; 8291 8292 if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)) 8293 return 0; 8294 8295 if (!vsi->rss_size) 8296 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8297 vsi->num_queue_pairs); 8298 if (!vsi->rss_size) 8299 return -EINVAL; 8300 8301 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8302 if (!lut) 8303 return -ENOMEM; 8304 /* Use the user configured hash keys and lookup table if there is one, 8305 * otherwise use default 8306 */ 8307 if (vsi->rss_lut_user) 8308 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8309 else 8310 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8311 if (vsi->rss_hkey_user) 8312 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8313 else 8314 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8315 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 8316 kfree(lut); 8317 8318 return ret; 8319 } 8320 8321 /** 8322 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 8323 * @vsi: Pointer to vsi structure 8324 * @seed: RSS hash seed 8325 * @lut: Lookup table 8326 * @lut_size: Lookup table size 8327 * 8328 * Returns 0 on success, negative on failure 8329 **/ 8330 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 8331 const u8 *lut, u16 lut_size) 8332 { 8333 struct i40e_pf *pf = vsi->back; 8334 struct i40e_hw *hw = &pf->hw; 8335 u16 vf_id = vsi->vf_id; 8336 u8 i; 8337 8338 /* Fill out hash function seed */ 8339 if (seed) { 8340 u32 *seed_dw = (u32 *)seed; 8341 8342 if (vsi->type == I40E_VSI_MAIN) { 8343 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8344 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 8345 } else if (vsi->type == I40E_VSI_SRIOV) { 8346 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 8347 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 8348 } else { 8349 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 8350 } 8351 } 8352 8353 if (lut) { 8354 u32 *lut_dw = (u32 *)lut; 8355 8356 if (vsi->type == I40E_VSI_MAIN) { 8357 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8358 return -EINVAL; 8359 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8360 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 8361 } else if (vsi->type == I40E_VSI_SRIOV) { 8362 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 8363 return -EINVAL; 8364 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8365 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 8366 } else { 8367 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8368 } 8369 } 8370 i40e_flush(hw); 8371 8372 return 0; 8373 } 8374 8375 /** 8376 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 8377 * @vsi: Pointer to VSI structure 8378 * @seed: Buffer to store the keys 8379 * @lut: Buffer to store the lookup table entries 8380 * @lut_size: Size of buffer to store the lookup table entries 8381 * 8382 * Returns 0 on success, negative on failure 8383 */ 8384 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 8385 u8 *lut, u16 lut_size) 8386 { 8387 struct i40e_pf *pf = vsi->back; 8388 struct i40e_hw *hw = &pf->hw; 8389 u16 i; 8390 8391 if (seed) { 8392 u32 *seed_dw = (u32 *)seed; 8393 8394 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8395 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 8396 } 8397 if (lut) { 8398 u32 *lut_dw = (u32 *)lut; 8399 8400 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8401 return -EINVAL; 8402 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8403 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 8404 } 8405 8406 return 0; 8407 } 8408 8409 /** 8410 * i40e_config_rss - Configure RSS keys and lut 8411 * @vsi: Pointer to VSI structure 8412 * @seed: RSS hash seed 8413 * @lut: Lookup table 8414 * @lut_size: Lookup table size 8415 * 8416 * Returns 0 on success, negative on failure 8417 */ 8418 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8419 { 8420 struct i40e_pf *pf = vsi->back; 8421 8422 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) 8423 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 8424 else 8425 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 8426 } 8427 8428 /** 8429 * i40e_get_rss - Get RSS keys and lut 8430 * @vsi: Pointer to VSI structure 8431 * @seed: Buffer to store the keys 8432 * @lut: Buffer to store the lookup table entries 8433 * lut_size: Size of buffer to store the lookup table entries 8434 * 8435 * Returns 0 on success, negative on failure 8436 */ 8437 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8438 { 8439 struct i40e_pf *pf = vsi->back; 8440 8441 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) 8442 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 8443 else 8444 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 8445 } 8446 8447 /** 8448 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 8449 * @pf: Pointer to board private structure 8450 * @lut: Lookup table 8451 * @rss_table_size: Lookup table size 8452 * @rss_size: Range of queue number for hashing 8453 */ 8454 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 8455 u16 rss_table_size, u16 rss_size) 8456 { 8457 u16 i; 8458 8459 for (i = 0; i < rss_table_size; i++) 8460 lut[i] = i % rss_size; 8461 } 8462 8463 /** 8464 * i40e_pf_config_rss - Prepare for RSS if used 8465 * @pf: board private structure 8466 **/ 8467 static int i40e_pf_config_rss(struct i40e_pf *pf) 8468 { 8469 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8470 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8471 u8 *lut; 8472 struct i40e_hw *hw = &pf->hw; 8473 u32 reg_val; 8474 u64 hena; 8475 int ret; 8476 8477 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 8478 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 8479 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 8480 hena |= i40e_pf_get_default_rss_hena(pf); 8481 8482 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 8483 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 8484 8485 /* Determine the RSS table size based on the hardware capabilities */ 8486 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 8487 reg_val = (pf->rss_table_size == 512) ? 8488 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 8489 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 8490 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 8491 8492 /* Determine the RSS size of the VSI */ 8493 if (!vsi->rss_size) { 8494 u16 qcount; 8495 8496 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 8497 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 8498 } 8499 if (!vsi->rss_size) 8500 return -EINVAL; 8501 8502 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8503 if (!lut) 8504 return -ENOMEM; 8505 8506 /* Use user configured lut if there is one, otherwise use default */ 8507 if (vsi->rss_lut_user) 8508 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8509 else 8510 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8511 8512 /* Use user configured hash key if there is one, otherwise 8513 * use default. 8514 */ 8515 if (vsi->rss_hkey_user) 8516 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8517 else 8518 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8519 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 8520 kfree(lut); 8521 8522 return ret; 8523 } 8524 8525 /** 8526 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 8527 * @pf: board private structure 8528 * @queue_count: the requested queue count for rss. 8529 * 8530 * returns 0 if rss is not enabled, if enabled returns the final rss queue 8531 * count which may be different from the requested queue count. 8532 * Note: expects to be called while under rtnl_lock() 8533 **/ 8534 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 8535 { 8536 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8537 int new_rss_size; 8538 8539 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 8540 return 0; 8541 8542 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 8543 8544 if (queue_count != vsi->num_queue_pairs) { 8545 u16 qcount; 8546 8547 vsi->req_queue_pairs = queue_count; 8548 i40e_prep_for_reset(pf, true); 8549 8550 pf->alloc_rss_size = new_rss_size; 8551 8552 i40e_reset_and_rebuild(pf, true, true); 8553 8554 /* Discard the user configured hash keys and lut, if less 8555 * queues are enabled. 8556 */ 8557 if (queue_count < vsi->rss_size) { 8558 i40e_clear_rss_config_user(vsi); 8559 dev_dbg(&pf->pdev->dev, 8560 "discard user configured hash keys and lut\n"); 8561 } 8562 8563 /* Reset vsi->rss_size, as number of enabled queues changed */ 8564 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 8565 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 8566 8567 i40e_pf_config_rss(pf); 8568 } 8569 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 8570 vsi->req_queue_pairs, pf->rss_size_max); 8571 return pf->alloc_rss_size; 8572 } 8573 8574 /** 8575 * i40e_get_npar_bw_setting - Retrieve BW settings for this PF partition 8576 * @pf: board private structure 8577 **/ 8578 i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf) 8579 { 8580 i40e_status status; 8581 bool min_valid, max_valid; 8582 u32 max_bw, min_bw; 8583 8584 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 8585 &min_valid, &max_valid); 8586 8587 if (!status) { 8588 if (min_valid) 8589 pf->npar_min_bw = min_bw; 8590 if (max_valid) 8591 pf->npar_max_bw = max_bw; 8592 } 8593 8594 return status; 8595 } 8596 8597 /** 8598 * i40e_set_npar_bw_setting - Set BW settings for this PF partition 8599 * @pf: board private structure 8600 **/ 8601 i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf) 8602 { 8603 struct i40e_aqc_configure_partition_bw_data bw_data; 8604 i40e_status status; 8605 8606 /* Set the valid bit for this PF */ 8607 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 8608 bw_data.max_bw[pf->hw.pf_id] = pf->npar_max_bw & I40E_ALT_BW_VALUE_MASK; 8609 bw_data.min_bw[pf->hw.pf_id] = pf->npar_min_bw & I40E_ALT_BW_VALUE_MASK; 8610 8611 /* Set the new bandwidths */ 8612 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 8613 8614 return status; 8615 } 8616 8617 /** 8618 * i40e_commit_npar_bw_setting - Commit BW settings for this PF partition 8619 * @pf: board private structure 8620 **/ 8621 i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf) 8622 { 8623 /* Commit temporary BW setting to permanent NVM image */ 8624 enum i40e_admin_queue_err last_aq_status; 8625 i40e_status ret; 8626 u16 nvm_word; 8627 8628 if (pf->hw.partition_id != 1) { 8629 dev_info(&pf->pdev->dev, 8630 "Commit BW only works on partition 1! This is partition %d", 8631 pf->hw.partition_id); 8632 ret = I40E_NOT_SUPPORTED; 8633 goto bw_commit_out; 8634 } 8635 8636 /* Acquire NVM for read access */ 8637 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 8638 last_aq_status = pf->hw.aq.asq_last_status; 8639 if (ret) { 8640 dev_info(&pf->pdev->dev, 8641 "Cannot acquire NVM for read access, err %s aq_err %s\n", 8642 i40e_stat_str(&pf->hw, ret), 8643 i40e_aq_str(&pf->hw, last_aq_status)); 8644 goto bw_commit_out; 8645 } 8646 8647 /* Read word 0x10 of NVM - SW compatibility word 1 */ 8648 ret = i40e_aq_read_nvm(&pf->hw, 8649 I40E_SR_NVM_CONTROL_WORD, 8650 0x10, sizeof(nvm_word), &nvm_word, 8651 false, NULL); 8652 /* Save off last admin queue command status before releasing 8653 * the NVM 8654 */ 8655 last_aq_status = pf->hw.aq.asq_last_status; 8656 i40e_release_nvm(&pf->hw); 8657 if (ret) { 8658 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 8659 i40e_stat_str(&pf->hw, ret), 8660 i40e_aq_str(&pf->hw, last_aq_status)); 8661 goto bw_commit_out; 8662 } 8663 8664 /* Wait a bit for NVM release to complete */ 8665 msleep(50); 8666 8667 /* Acquire NVM for write access */ 8668 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 8669 last_aq_status = pf->hw.aq.asq_last_status; 8670 if (ret) { 8671 dev_info(&pf->pdev->dev, 8672 "Cannot acquire NVM for write access, err %s aq_err %s\n", 8673 i40e_stat_str(&pf->hw, ret), 8674 i40e_aq_str(&pf->hw, last_aq_status)); 8675 goto bw_commit_out; 8676 } 8677 /* Write it back out unchanged to initiate update NVM, 8678 * which will force a write of the shadow (alt) RAM to 8679 * the NVM - thus storing the bandwidth values permanently. 8680 */ 8681 ret = i40e_aq_update_nvm(&pf->hw, 8682 I40E_SR_NVM_CONTROL_WORD, 8683 0x10, sizeof(nvm_word), 8684 &nvm_word, true, NULL); 8685 /* Save off last admin queue command status before releasing 8686 * the NVM 8687 */ 8688 last_aq_status = pf->hw.aq.asq_last_status; 8689 i40e_release_nvm(&pf->hw); 8690 if (ret) 8691 dev_info(&pf->pdev->dev, 8692 "BW settings NOT SAVED, err %s aq_err %s\n", 8693 i40e_stat_str(&pf->hw, ret), 8694 i40e_aq_str(&pf->hw, last_aq_status)); 8695 bw_commit_out: 8696 8697 return ret; 8698 } 8699 8700 /** 8701 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 8702 * @pf: board private structure to initialize 8703 * 8704 * i40e_sw_init initializes the Adapter private data structure. 8705 * Fields are initialized based on PCI device information and 8706 * OS network device settings (MTU size). 8707 **/ 8708 static int i40e_sw_init(struct i40e_pf *pf) 8709 { 8710 int err = 0; 8711 int size; 8712 8713 /* Set default capability flags */ 8714 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 8715 I40E_FLAG_MSI_ENABLED | 8716 I40E_FLAG_MSIX_ENABLED; 8717 8718 /* Set default ITR */ 8719 pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF; 8720 pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF; 8721 8722 /* Depending on PF configurations, it is possible that the RSS 8723 * maximum might end up larger than the available queues 8724 */ 8725 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 8726 pf->alloc_rss_size = 1; 8727 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 8728 pf->rss_size_max = min_t(int, pf->rss_size_max, 8729 pf->hw.func_caps.num_tx_qp); 8730 if (pf->hw.func_caps.rss) { 8731 pf->flags |= I40E_FLAG_RSS_ENABLED; 8732 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 8733 num_online_cpus()); 8734 } 8735 8736 /* MFP mode enabled */ 8737 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 8738 pf->flags |= I40E_FLAG_MFP_ENABLED; 8739 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 8740 if (i40e_get_npar_bw_setting(pf)) 8741 dev_warn(&pf->pdev->dev, 8742 "Could not get NPAR bw settings\n"); 8743 else 8744 dev_info(&pf->pdev->dev, 8745 "Min BW = %8.8x, Max BW = %8.8x\n", 8746 pf->npar_min_bw, pf->npar_max_bw); 8747 } 8748 8749 /* FW/NVM is not yet fixed in this regard */ 8750 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 8751 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 8752 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 8753 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 8754 if (pf->flags & I40E_FLAG_MFP_ENABLED && 8755 pf->hw.num_partitions > 1) 8756 dev_info(&pf->pdev->dev, 8757 "Flow Director Sideband mode Disabled in MFP mode\n"); 8758 else 8759 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8760 pf->fdir_pf_filter_count = 8761 pf->hw.func_caps.fd_filters_guaranteed; 8762 pf->hw.fdir_shared_filter_count = 8763 pf->hw.func_caps.fd_filters_best_effort; 8764 } 8765 8766 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8767 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 8768 (pf->hw.aq.fw_maj_ver < 4))) { 8769 pf->flags |= I40E_FLAG_RESTART_AUTONEG; 8770 /* No DCB support for FW < v4.33 */ 8771 pf->flags |= I40E_FLAG_NO_DCB_SUPPORT; 8772 } 8773 8774 /* Disable FW LLDP if FW < v4.3 */ 8775 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8776 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 8777 (pf->hw.aq.fw_maj_ver < 4))) 8778 pf->flags |= I40E_FLAG_STOP_FW_LLDP; 8779 8780 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 8781 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8782 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 8783 (pf->hw.aq.fw_maj_ver >= 5))) 8784 pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB; 8785 8786 if (pf->hw.func_caps.vmdq) { 8787 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 8788 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 8789 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 8790 } 8791 8792 if (pf->hw.func_caps.iwarp) { 8793 pf->flags |= I40E_FLAG_IWARP_ENABLED; 8794 /* IWARP needs one extra vector for CQP just like MISC.*/ 8795 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 8796 } 8797 8798 #ifdef CONFIG_PCI_IOV 8799 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 8800 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 8801 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 8802 pf->num_req_vfs = min_t(int, 8803 pf->hw.func_caps.num_vfs, 8804 I40E_MAX_VF_COUNT); 8805 } 8806 #endif /* CONFIG_PCI_IOV */ 8807 if (pf->hw.mac.type == I40E_MAC_X722) { 8808 pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE 8809 | I40E_FLAG_128_QP_RSS_CAPABLE 8810 | I40E_FLAG_HW_ATR_EVICT_CAPABLE 8811 | I40E_FLAG_OUTER_UDP_CSUM_CAPABLE 8812 | I40E_FLAG_WB_ON_ITR_CAPABLE 8813 | I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE 8814 | I40E_FLAG_NO_PCI_LINK_CHECK 8815 | I40E_FLAG_USE_SET_LLDP_MIB 8816 | I40E_FLAG_GENEVE_OFFLOAD_CAPABLE 8817 | I40E_FLAG_PTP_L4_CAPABLE 8818 | I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE; 8819 } else if ((pf->hw.aq.api_maj_ver > 1) || 8820 ((pf->hw.aq.api_maj_ver == 1) && 8821 (pf->hw.aq.api_min_ver > 4))) { 8822 /* Supported in FW API version higher than 1.4 */ 8823 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE; 8824 pf->flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE; 8825 } else { 8826 pf->flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE; 8827 } 8828 8829 pf->eeprom_version = 0xDEAD; 8830 pf->lan_veb = I40E_NO_VEB; 8831 pf->lan_vsi = I40E_NO_VSI; 8832 8833 /* By default FW has this off for performance reasons */ 8834 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 8835 8836 /* set up queue assignment tracking */ 8837 size = sizeof(struct i40e_lump_tracking) 8838 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 8839 pf->qp_pile = kzalloc(size, GFP_KERNEL); 8840 if (!pf->qp_pile) { 8841 err = -ENOMEM; 8842 goto sw_init_done; 8843 } 8844 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 8845 pf->qp_pile->search_hint = 0; 8846 8847 pf->tx_timeout_recovery_level = 1; 8848 8849 mutex_init(&pf->switch_mutex); 8850 8851 /* If NPAR is enabled nudge the Tx scheduler */ 8852 if (pf->hw.func_caps.npar_enable && (!i40e_get_npar_bw_setting(pf))) 8853 i40e_set_npar_bw_setting(pf); 8854 8855 sw_init_done: 8856 return err; 8857 } 8858 8859 /** 8860 * i40e_set_ntuple - set the ntuple feature flag and take action 8861 * @pf: board private structure to initialize 8862 * @features: the feature set that the stack is suggesting 8863 * 8864 * returns a bool to indicate if reset needs to happen 8865 **/ 8866 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 8867 { 8868 bool need_reset = false; 8869 8870 /* Check if Flow Director n-tuple support was enabled or disabled. If 8871 * the state changed, we need to reset. 8872 */ 8873 if (features & NETIF_F_NTUPLE) { 8874 /* Enable filters and mark for reset */ 8875 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 8876 need_reset = true; 8877 /* enable FD_SB only if there is MSI-X vector */ 8878 if (pf->num_fdsb_msix > 0) 8879 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8880 } else { 8881 /* turn off filters, mark for reset and clear SW filter list */ 8882 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 8883 need_reset = true; 8884 i40e_fdir_filter_exit(pf); 8885 } 8886 pf->flags &= ~(I40E_FLAG_FD_SB_ENABLED | 8887 I40E_FLAG_FD_SB_AUTO_DISABLED); 8888 /* reset fd counters */ 8889 pf->fd_add_err = 0; 8890 pf->fd_atr_cnt = 0; 8891 /* if ATR was auto disabled it can be re-enabled. */ 8892 if (pf->flags & I40E_FLAG_FD_ATR_AUTO_DISABLED) { 8893 pf->flags &= ~I40E_FLAG_FD_ATR_AUTO_DISABLED; 8894 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8895 (I40E_DEBUG_FD & pf->hw.debug_mask)) 8896 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 8897 } 8898 } 8899 return need_reset; 8900 } 8901 8902 /** 8903 * i40e_clear_rss_lut - clear the rx hash lookup table 8904 * @vsi: the VSI being configured 8905 **/ 8906 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 8907 { 8908 struct i40e_pf *pf = vsi->back; 8909 struct i40e_hw *hw = &pf->hw; 8910 u16 vf_id = vsi->vf_id; 8911 u8 i; 8912 8913 if (vsi->type == I40E_VSI_MAIN) { 8914 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8915 wr32(hw, I40E_PFQF_HLUT(i), 0); 8916 } else if (vsi->type == I40E_VSI_SRIOV) { 8917 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8918 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 8919 } else { 8920 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8921 } 8922 } 8923 8924 /** 8925 * i40e_set_features - set the netdev feature flags 8926 * @netdev: ptr to the netdev being adjusted 8927 * @features: the feature set that the stack is suggesting 8928 * Note: expects to be called while under rtnl_lock() 8929 **/ 8930 static int i40e_set_features(struct net_device *netdev, 8931 netdev_features_t features) 8932 { 8933 struct i40e_netdev_priv *np = netdev_priv(netdev); 8934 struct i40e_vsi *vsi = np->vsi; 8935 struct i40e_pf *pf = vsi->back; 8936 bool need_reset; 8937 8938 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 8939 i40e_pf_config_rss(pf); 8940 else if (!(features & NETIF_F_RXHASH) && 8941 netdev->features & NETIF_F_RXHASH) 8942 i40e_clear_rss_lut(vsi); 8943 8944 if (features & NETIF_F_HW_VLAN_CTAG_RX) 8945 i40e_vlan_stripping_enable(vsi); 8946 else 8947 i40e_vlan_stripping_disable(vsi); 8948 8949 need_reset = i40e_set_ntuple(pf, features); 8950 8951 if (need_reset) 8952 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), true); 8953 8954 return 0; 8955 } 8956 8957 /** 8958 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 8959 * @pf: board private structure 8960 * @port: The UDP port to look up 8961 * 8962 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 8963 **/ 8964 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 8965 { 8966 u8 i; 8967 8968 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 8969 if (pf->udp_ports[i].port == port) 8970 return i; 8971 } 8972 8973 return i; 8974 } 8975 8976 /** 8977 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 8978 * @netdev: This physical port's netdev 8979 * @ti: Tunnel endpoint information 8980 **/ 8981 static void i40e_udp_tunnel_add(struct net_device *netdev, 8982 struct udp_tunnel_info *ti) 8983 { 8984 struct i40e_netdev_priv *np = netdev_priv(netdev); 8985 struct i40e_vsi *vsi = np->vsi; 8986 struct i40e_pf *pf = vsi->back; 8987 u16 port = ntohs(ti->port); 8988 u8 next_idx; 8989 u8 idx; 8990 8991 idx = i40e_get_udp_port_idx(pf, port); 8992 8993 /* Check if port already exists */ 8994 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 8995 netdev_info(netdev, "port %d already offloaded\n", port); 8996 return; 8997 } 8998 8999 /* Now check if there is space to add the new port */ 9000 next_idx = i40e_get_udp_port_idx(pf, 0); 9001 9002 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 9003 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 9004 port); 9005 return; 9006 } 9007 9008 switch (ti->type) { 9009 case UDP_TUNNEL_TYPE_VXLAN: 9010 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 9011 break; 9012 case UDP_TUNNEL_TYPE_GENEVE: 9013 if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE)) 9014 return; 9015 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 9016 break; 9017 default: 9018 return; 9019 } 9020 9021 /* New port: add it and mark its index in the bitmap */ 9022 pf->udp_ports[next_idx].port = port; 9023 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 9024 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 9025 } 9026 9027 /** 9028 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 9029 * @netdev: This physical port's netdev 9030 * @ti: Tunnel endpoint information 9031 **/ 9032 static void i40e_udp_tunnel_del(struct net_device *netdev, 9033 struct udp_tunnel_info *ti) 9034 { 9035 struct i40e_netdev_priv *np = netdev_priv(netdev); 9036 struct i40e_vsi *vsi = np->vsi; 9037 struct i40e_pf *pf = vsi->back; 9038 u16 port = ntohs(ti->port); 9039 u8 idx; 9040 9041 idx = i40e_get_udp_port_idx(pf, port); 9042 9043 /* Check if port already exists */ 9044 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 9045 goto not_found; 9046 9047 switch (ti->type) { 9048 case UDP_TUNNEL_TYPE_VXLAN: 9049 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 9050 goto not_found; 9051 break; 9052 case UDP_TUNNEL_TYPE_GENEVE: 9053 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 9054 goto not_found; 9055 break; 9056 default: 9057 goto not_found; 9058 } 9059 9060 /* if port exists, set it to 0 (mark for deletion) 9061 * and make it pending 9062 */ 9063 pf->udp_ports[idx].port = 0; 9064 pf->pending_udp_bitmap |= BIT_ULL(idx); 9065 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 9066 9067 return; 9068 not_found: 9069 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 9070 port); 9071 } 9072 9073 static int i40e_get_phys_port_id(struct net_device *netdev, 9074 struct netdev_phys_item_id *ppid) 9075 { 9076 struct i40e_netdev_priv *np = netdev_priv(netdev); 9077 struct i40e_pf *pf = np->vsi->back; 9078 struct i40e_hw *hw = &pf->hw; 9079 9080 if (!(pf->flags & I40E_FLAG_PORT_ID_VALID)) 9081 return -EOPNOTSUPP; 9082 9083 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 9084 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 9085 9086 return 0; 9087 } 9088 9089 /** 9090 * i40e_ndo_fdb_add - add an entry to the hardware database 9091 * @ndm: the input from the stack 9092 * @tb: pointer to array of nladdr (unused) 9093 * @dev: the net device pointer 9094 * @addr: the MAC address entry being added 9095 * @flags: instructions from stack about fdb operation 9096 */ 9097 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 9098 struct net_device *dev, 9099 const unsigned char *addr, u16 vid, 9100 u16 flags) 9101 { 9102 struct i40e_netdev_priv *np = netdev_priv(dev); 9103 struct i40e_pf *pf = np->vsi->back; 9104 int err = 0; 9105 9106 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 9107 return -EOPNOTSUPP; 9108 9109 if (vid) { 9110 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 9111 return -EINVAL; 9112 } 9113 9114 /* Hardware does not support aging addresses so if a 9115 * ndm_state is given only allow permanent addresses 9116 */ 9117 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 9118 netdev_info(dev, "FDB only supports static addresses\n"); 9119 return -EINVAL; 9120 } 9121 9122 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 9123 err = dev_uc_add_excl(dev, addr); 9124 else if (is_multicast_ether_addr(addr)) 9125 err = dev_mc_add_excl(dev, addr); 9126 else 9127 err = -EINVAL; 9128 9129 /* Only return duplicate errors if NLM_F_EXCL is set */ 9130 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 9131 err = 0; 9132 9133 return err; 9134 } 9135 9136 /** 9137 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 9138 * @dev: the netdev being configured 9139 * @nlh: RTNL message 9140 * 9141 * Inserts a new hardware bridge if not already created and 9142 * enables the bridging mode requested (VEB or VEPA). If the 9143 * hardware bridge has already been inserted and the request 9144 * is to change the mode then that requires a PF reset to 9145 * allow rebuild of the components with required hardware 9146 * bridge mode enabled. 9147 * 9148 * Note: expects to be called while under rtnl_lock() 9149 **/ 9150 static int i40e_ndo_bridge_setlink(struct net_device *dev, 9151 struct nlmsghdr *nlh, 9152 u16 flags) 9153 { 9154 struct i40e_netdev_priv *np = netdev_priv(dev); 9155 struct i40e_vsi *vsi = np->vsi; 9156 struct i40e_pf *pf = vsi->back; 9157 struct i40e_veb *veb = NULL; 9158 struct nlattr *attr, *br_spec; 9159 int i, rem; 9160 9161 /* Only for PF VSI for now */ 9162 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9163 return -EOPNOTSUPP; 9164 9165 /* Find the HW bridge for PF VSI */ 9166 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9167 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9168 veb = pf->veb[i]; 9169 } 9170 9171 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 9172 9173 nla_for_each_nested(attr, br_spec, rem) { 9174 __u16 mode; 9175 9176 if (nla_type(attr) != IFLA_BRIDGE_MODE) 9177 continue; 9178 9179 mode = nla_get_u16(attr); 9180 if ((mode != BRIDGE_MODE_VEPA) && 9181 (mode != BRIDGE_MODE_VEB)) 9182 return -EINVAL; 9183 9184 /* Insert a new HW bridge */ 9185 if (!veb) { 9186 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 9187 vsi->tc_config.enabled_tc); 9188 if (veb) { 9189 veb->bridge_mode = mode; 9190 i40e_config_bridge_mode(veb); 9191 } else { 9192 /* No Bridge HW offload available */ 9193 return -ENOENT; 9194 } 9195 break; 9196 } else if (mode != veb->bridge_mode) { 9197 /* Existing HW bridge but different mode needs reset */ 9198 veb->bridge_mode = mode; 9199 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 9200 if (mode == BRIDGE_MODE_VEB) 9201 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 9202 else 9203 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 9204 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED), 9205 true); 9206 break; 9207 } 9208 } 9209 9210 return 0; 9211 } 9212 9213 /** 9214 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 9215 * @skb: skb buff 9216 * @pid: process id 9217 * @seq: RTNL message seq # 9218 * @dev: the netdev being configured 9219 * @filter_mask: unused 9220 * @nlflags: netlink flags passed in 9221 * 9222 * Return the mode in which the hardware bridge is operating in 9223 * i.e VEB or VEPA. 9224 **/ 9225 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 9226 struct net_device *dev, 9227 u32 __always_unused filter_mask, 9228 int nlflags) 9229 { 9230 struct i40e_netdev_priv *np = netdev_priv(dev); 9231 struct i40e_vsi *vsi = np->vsi; 9232 struct i40e_pf *pf = vsi->back; 9233 struct i40e_veb *veb = NULL; 9234 int i; 9235 9236 /* Only for PF VSI for now */ 9237 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9238 return -EOPNOTSUPP; 9239 9240 /* Find the HW bridge for the PF VSI */ 9241 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9242 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9243 veb = pf->veb[i]; 9244 } 9245 9246 if (!veb) 9247 return 0; 9248 9249 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 9250 0, 0, nlflags, filter_mask, NULL); 9251 } 9252 9253 /** 9254 * i40e_features_check - Validate encapsulated packet conforms to limits 9255 * @skb: skb buff 9256 * @dev: This physical port's netdev 9257 * @features: Offload features that the stack believes apply 9258 **/ 9259 static netdev_features_t i40e_features_check(struct sk_buff *skb, 9260 struct net_device *dev, 9261 netdev_features_t features) 9262 { 9263 size_t len; 9264 9265 /* No point in doing any of this if neither checksum nor GSO are 9266 * being requested for this frame. We can rule out both by just 9267 * checking for CHECKSUM_PARTIAL 9268 */ 9269 if (skb->ip_summed != CHECKSUM_PARTIAL) 9270 return features; 9271 9272 /* We cannot support GSO if the MSS is going to be less than 9273 * 64 bytes. If it is then we need to drop support for GSO. 9274 */ 9275 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 9276 features &= ~NETIF_F_GSO_MASK; 9277 9278 /* MACLEN can support at most 63 words */ 9279 len = skb_network_header(skb) - skb->data; 9280 if (len & ~(63 * 2)) 9281 goto out_err; 9282 9283 /* IPLEN and EIPLEN can support at most 127 dwords */ 9284 len = skb_transport_header(skb) - skb_network_header(skb); 9285 if (len & ~(127 * 4)) 9286 goto out_err; 9287 9288 if (skb->encapsulation) { 9289 /* L4TUNLEN can support 127 words */ 9290 len = skb_inner_network_header(skb) - skb_transport_header(skb); 9291 if (len & ~(127 * 2)) 9292 goto out_err; 9293 9294 /* IPLEN can support at most 127 dwords */ 9295 len = skb_inner_transport_header(skb) - 9296 skb_inner_network_header(skb); 9297 if (len & ~(127 * 4)) 9298 goto out_err; 9299 } 9300 9301 /* No need to validate L4LEN as TCP is the only protocol with a 9302 * a flexible value and we support all possible values supported 9303 * by TCP, which is at most 15 dwords 9304 */ 9305 9306 return features; 9307 out_err: 9308 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 9309 } 9310 9311 static const struct net_device_ops i40e_netdev_ops = { 9312 .ndo_open = i40e_open, 9313 .ndo_stop = i40e_close, 9314 .ndo_start_xmit = i40e_lan_xmit_frame, 9315 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 9316 .ndo_set_rx_mode = i40e_set_rx_mode, 9317 .ndo_validate_addr = eth_validate_addr, 9318 .ndo_set_mac_address = i40e_set_mac, 9319 .ndo_change_mtu = i40e_change_mtu, 9320 .ndo_do_ioctl = i40e_ioctl, 9321 .ndo_tx_timeout = i40e_tx_timeout, 9322 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 9323 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 9324 #ifdef CONFIG_NET_POLL_CONTROLLER 9325 .ndo_poll_controller = i40e_netpoll, 9326 #endif 9327 .ndo_setup_tc = __i40e_setup_tc, 9328 .ndo_set_features = i40e_set_features, 9329 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 9330 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 9331 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 9332 .ndo_get_vf_config = i40e_ndo_get_vf_config, 9333 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 9334 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 9335 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 9336 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 9337 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 9338 .ndo_get_phys_port_id = i40e_get_phys_port_id, 9339 .ndo_fdb_add = i40e_ndo_fdb_add, 9340 .ndo_features_check = i40e_features_check, 9341 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 9342 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 9343 }; 9344 9345 /** 9346 * i40e_config_netdev - Setup the netdev flags 9347 * @vsi: the VSI being configured 9348 * 9349 * Returns 0 on success, negative value on failure 9350 **/ 9351 static int i40e_config_netdev(struct i40e_vsi *vsi) 9352 { 9353 struct i40e_pf *pf = vsi->back; 9354 struct i40e_hw *hw = &pf->hw; 9355 struct i40e_netdev_priv *np; 9356 struct net_device *netdev; 9357 u8 broadcast[ETH_ALEN]; 9358 u8 mac_addr[ETH_ALEN]; 9359 int etherdev_size; 9360 netdev_features_t hw_enc_features; 9361 netdev_features_t hw_features; 9362 9363 etherdev_size = sizeof(struct i40e_netdev_priv); 9364 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 9365 if (!netdev) 9366 return -ENOMEM; 9367 9368 vsi->netdev = netdev; 9369 np = netdev_priv(netdev); 9370 np->vsi = vsi; 9371 9372 hw_enc_features = NETIF_F_SG | 9373 NETIF_F_IP_CSUM | 9374 NETIF_F_IPV6_CSUM | 9375 NETIF_F_HIGHDMA | 9376 NETIF_F_SOFT_FEATURES | 9377 NETIF_F_TSO | 9378 NETIF_F_TSO_ECN | 9379 NETIF_F_TSO6 | 9380 NETIF_F_GSO_GRE | 9381 NETIF_F_GSO_GRE_CSUM | 9382 NETIF_F_GSO_PARTIAL | 9383 NETIF_F_GSO_UDP_TUNNEL | 9384 NETIF_F_GSO_UDP_TUNNEL_CSUM | 9385 NETIF_F_SCTP_CRC | 9386 NETIF_F_RXHASH | 9387 NETIF_F_RXCSUM | 9388 0; 9389 9390 if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE)) 9391 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 9392 9393 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 9394 9395 netdev->hw_enc_features |= hw_enc_features; 9396 9397 /* record features VLANs can make use of */ 9398 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID; 9399 9400 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 9401 netdev->hw_features |= NETIF_F_NTUPLE; 9402 hw_features = hw_enc_features | 9403 NETIF_F_HW_VLAN_CTAG_TX | 9404 NETIF_F_HW_VLAN_CTAG_RX; 9405 9406 netdev->hw_features |= hw_features; 9407 9408 netdev->features |= hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 9409 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 9410 9411 if (vsi->type == I40E_VSI_MAIN) { 9412 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 9413 ether_addr_copy(mac_addr, hw->mac.perm_addr); 9414 /* The following steps are necessary for two reasons. First, 9415 * some older NVM configurations load a default MAC-VLAN 9416 * filter that will accept any tagged packet, and we want to 9417 * replace this with a normal filter. Additionally, it is 9418 * possible our MAC address was provided by the platform using 9419 * Open Firmware or similar. 9420 * 9421 * Thus, we need to remove the default filter and install one 9422 * specific to the MAC address. 9423 */ 9424 i40e_rm_default_mac_filter(vsi, mac_addr); 9425 spin_lock_bh(&vsi->mac_filter_hash_lock); 9426 i40e_add_mac_filter(vsi, mac_addr); 9427 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9428 } else { 9429 /* relate the VSI_VMDQ name to the VSI_MAIN name */ 9430 snprintf(netdev->name, IFNAMSIZ, "%sv%%d", 9431 pf->vsi[pf->lan_vsi]->netdev->name); 9432 random_ether_addr(mac_addr); 9433 9434 spin_lock_bh(&vsi->mac_filter_hash_lock); 9435 i40e_add_mac_filter(vsi, mac_addr); 9436 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9437 } 9438 9439 /* Add the broadcast filter so that we initially will receive 9440 * broadcast packets. Note that when a new VLAN is first added the 9441 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 9442 * specific filters as part of transitioning into "vlan" operation. 9443 * When more VLANs are added, the driver will copy each existing MAC 9444 * filter and add it for the new VLAN. 9445 * 9446 * Broadcast filters are handled specially by 9447 * i40e_sync_filters_subtask, as the driver must to set the broadcast 9448 * promiscuous bit instead of adding this directly as a MAC/VLAN 9449 * filter. The subtask will update the correct broadcast promiscuous 9450 * bits as VLANs become active or inactive. 9451 */ 9452 eth_broadcast_addr(broadcast); 9453 spin_lock_bh(&vsi->mac_filter_hash_lock); 9454 i40e_add_mac_filter(vsi, broadcast); 9455 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9456 9457 ether_addr_copy(netdev->dev_addr, mac_addr); 9458 ether_addr_copy(netdev->perm_addr, mac_addr); 9459 9460 netdev->priv_flags |= IFF_UNICAST_FLT; 9461 netdev->priv_flags |= IFF_SUPP_NOFCS; 9462 /* Setup netdev TC information */ 9463 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 9464 9465 netdev->netdev_ops = &i40e_netdev_ops; 9466 netdev->watchdog_timeo = 5 * HZ; 9467 i40e_set_ethtool_ops(netdev); 9468 9469 /* MTU range: 68 - 9706 */ 9470 netdev->min_mtu = ETH_MIN_MTU; 9471 netdev->max_mtu = I40E_MAX_RXBUFFER - 9472 (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN); 9473 9474 return 0; 9475 } 9476 9477 /** 9478 * i40e_vsi_delete - Delete a VSI from the switch 9479 * @vsi: the VSI being removed 9480 * 9481 * Returns 0 on success, negative value on failure 9482 **/ 9483 static void i40e_vsi_delete(struct i40e_vsi *vsi) 9484 { 9485 /* remove default VSI is not allowed */ 9486 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 9487 return; 9488 9489 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 9490 } 9491 9492 /** 9493 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 9494 * @vsi: the VSI being queried 9495 * 9496 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 9497 **/ 9498 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 9499 { 9500 struct i40e_veb *veb; 9501 struct i40e_pf *pf = vsi->back; 9502 9503 /* Uplink is not a bridge so default to VEB */ 9504 if (vsi->veb_idx == I40E_NO_VEB) 9505 return 1; 9506 9507 veb = pf->veb[vsi->veb_idx]; 9508 if (!veb) { 9509 dev_info(&pf->pdev->dev, 9510 "There is no veb associated with the bridge\n"); 9511 return -ENOENT; 9512 } 9513 9514 /* Uplink is a bridge in VEPA mode */ 9515 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 9516 return 0; 9517 } else { 9518 /* Uplink is a bridge in VEB mode */ 9519 return 1; 9520 } 9521 9522 /* VEPA is now default bridge, so return 0 */ 9523 return 0; 9524 } 9525 9526 /** 9527 * i40e_add_vsi - Add a VSI to the switch 9528 * @vsi: the VSI being configured 9529 * 9530 * This initializes a VSI context depending on the VSI type to be added and 9531 * passes it down to the add_vsi aq command. 9532 **/ 9533 static int i40e_add_vsi(struct i40e_vsi *vsi) 9534 { 9535 int ret = -ENODEV; 9536 struct i40e_pf *pf = vsi->back; 9537 struct i40e_hw *hw = &pf->hw; 9538 struct i40e_vsi_context ctxt; 9539 struct i40e_mac_filter *f; 9540 struct hlist_node *h; 9541 int bkt; 9542 9543 u8 enabled_tc = 0x1; /* TC0 enabled */ 9544 int f_count = 0; 9545 9546 memset(&ctxt, 0, sizeof(ctxt)); 9547 switch (vsi->type) { 9548 case I40E_VSI_MAIN: 9549 /* The PF's main VSI is already setup as part of the 9550 * device initialization, so we'll not bother with 9551 * the add_vsi call, but we will retrieve the current 9552 * VSI context. 9553 */ 9554 ctxt.seid = pf->main_vsi_seid; 9555 ctxt.pf_num = pf->hw.pf_id; 9556 ctxt.vf_num = 0; 9557 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9558 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9559 if (ret) { 9560 dev_info(&pf->pdev->dev, 9561 "couldn't get PF vsi config, err %s aq_err %s\n", 9562 i40e_stat_str(&pf->hw, ret), 9563 i40e_aq_str(&pf->hw, 9564 pf->hw.aq.asq_last_status)); 9565 return -ENOENT; 9566 } 9567 vsi->info = ctxt.info; 9568 vsi->info.valid_sections = 0; 9569 9570 vsi->seid = ctxt.seid; 9571 vsi->id = ctxt.vsi_number; 9572 9573 enabled_tc = i40e_pf_get_tc_map(pf); 9574 9575 /* MFP mode setup queue map and update VSI */ 9576 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 9577 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 9578 memset(&ctxt, 0, sizeof(ctxt)); 9579 ctxt.seid = pf->main_vsi_seid; 9580 ctxt.pf_num = pf->hw.pf_id; 9581 ctxt.vf_num = 0; 9582 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 9583 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 9584 if (ret) { 9585 dev_info(&pf->pdev->dev, 9586 "update vsi failed, err %s aq_err %s\n", 9587 i40e_stat_str(&pf->hw, ret), 9588 i40e_aq_str(&pf->hw, 9589 pf->hw.aq.asq_last_status)); 9590 ret = -ENOENT; 9591 goto err; 9592 } 9593 /* update the local VSI info queue map */ 9594 i40e_vsi_update_queue_map(vsi, &ctxt); 9595 vsi->info.valid_sections = 0; 9596 } else { 9597 /* Default/Main VSI is only enabled for TC0 9598 * reconfigure it to enable all TCs that are 9599 * available on the port in SFP mode. 9600 * For MFP case the iSCSI PF would use this 9601 * flow to enable LAN+iSCSI TC. 9602 */ 9603 ret = i40e_vsi_config_tc(vsi, enabled_tc); 9604 if (ret) { 9605 dev_info(&pf->pdev->dev, 9606 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 9607 enabled_tc, 9608 i40e_stat_str(&pf->hw, ret), 9609 i40e_aq_str(&pf->hw, 9610 pf->hw.aq.asq_last_status)); 9611 ret = -ENOENT; 9612 } 9613 } 9614 break; 9615 9616 case I40E_VSI_FDIR: 9617 ctxt.pf_num = hw->pf_id; 9618 ctxt.vf_num = 0; 9619 ctxt.uplink_seid = vsi->uplink_seid; 9620 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9621 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9622 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 9623 (i40e_is_vsi_uplink_mode_veb(vsi))) { 9624 ctxt.info.valid_sections |= 9625 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9626 ctxt.info.switch_id = 9627 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9628 } 9629 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9630 break; 9631 9632 case I40E_VSI_VMDQ2: 9633 ctxt.pf_num = hw->pf_id; 9634 ctxt.vf_num = 0; 9635 ctxt.uplink_seid = vsi->uplink_seid; 9636 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9637 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 9638 9639 /* This VSI is connected to VEB so the switch_id 9640 * should be set to zero by default. 9641 */ 9642 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9643 ctxt.info.valid_sections |= 9644 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9645 ctxt.info.switch_id = 9646 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9647 } 9648 9649 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9650 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9651 break; 9652 9653 case I40E_VSI_SRIOV: 9654 ctxt.pf_num = hw->pf_id; 9655 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 9656 ctxt.uplink_seid = vsi->uplink_seid; 9657 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9658 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 9659 9660 /* This VSI is connected to VEB so the switch_id 9661 * should be set to zero by default. 9662 */ 9663 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9664 ctxt.info.valid_sections |= 9665 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9666 ctxt.info.switch_id = 9667 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9668 } 9669 9670 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 9671 ctxt.info.valid_sections |= 9672 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 9673 ctxt.info.queueing_opt_flags |= 9674 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 9675 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 9676 } 9677 9678 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 9679 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 9680 if (pf->vf[vsi->vf_id].spoofchk) { 9681 ctxt.info.valid_sections |= 9682 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 9683 ctxt.info.sec_flags |= 9684 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 9685 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 9686 } 9687 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9688 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9689 break; 9690 9691 case I40E_VSI_IWARP: 9692 /* send down message to iWARP */ 9693 break; 9694 9695 default: 9696 return -ENODEV; 9697 } 9698 9699 if (vsi->type != I40E_VSI_MAIN) { 9700 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 9701 if (ret) { 9702 dev_info(&vsi->back->pdev->dev, 9703 "add vsi failed, err %s aq_err %s\n", 9704 i40e_stat_str(&pf->hw, ret), 9705 i40e_aq_str(&pf->hw, 9706 pf->hw.aq.asq_last_status)); 9707 ret = -ENOENT; 9708 goto err; 9709 } 9710 vsi->info = ctxt.info; 9711 vsi->info.valid_sections = 0; 9712 vsi->seid = ctxt.seid; 9713 vsi->id = ctxt.vsi_number; 9714 } 9715 9716 vsi->active_filters = 0; 9717 clear_bit(__I40E_VSI_OVERFLOW_PROMISC, vsi->state); 9718 spin_lock_bh(&vsi->mac_filter_hash_lock); 9719 /* If macvlan filters already exist, force them to get loaded */ 9720 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 9721 f->state = I40E_FILTER_NEW; 9722 f_count++; 9723 } 9724 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9725 9726 if (f_count) { 9727 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 9728 pf->flags |= I40E_FLAG_FILTER_SYNC; 9729 } 9730 9731 /* Update VSI BW information */ 9732 ret = i40e_vsi_get_bw_info(vsi); 9733 if (ret) { 9734 dev_info(&pf->pdev->dev, 9735 "couldn't get vsi bw info, err %s aq_err %s\n", 9736 i40e_stat_str(&pf->hw, ret), 9737 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9738 /* VSI is already added so not tearing that up */ 9739 ret = 0; 9740 } 9741 9742 err: 9743 return ret; 9744 } 9745 9746 /** 9747 * i40e_vsi_release - Delete a VSI and free its resources 9748 * @vsi: the VSI being removed 9749 * 9750 * Returns 0 on success or < 0 on error 9751 **/ 9752 int i40e_vsi_release(struct i40e_vsi *vsi) 9753 { 9754 struct i40e_mac_filter *f; 9755 struct hlist_node *h; 9756 struct i40e_veb *veb = NULL; 9757 struct i40e_pf *pf; 9758 u16 uplink_seid; 9759 int i, n, bkt; 9760 9761 pf = vsi->back; 9762 9763 /* release of a VEB-owner or last VSI is not allowed */ 9764 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 9765 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 9766 vsi->seid, vsi->uplink_seid); 9767 return -ENODEV; 9768 } 9769 if (vsi == pf->vsi[pf->lan_vsi] && 9770 !test_bit(__I40E_VSI_DOWN, pf->state)) { 9771 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 9772 return -ENODEV; 9773 } 9774 9775 uplink_seid = vsi->uplink_seid; 9776 if (vsi->type != I40E_VSI_SRIOV) { 9777 if (vsi->netdev_registered) { 9778 vsi->netdev_registered = false; 9779 if (vsi->netdev) { 9780 /* results in a call to i40e_close() */ 9781 unregister_netdev(vsi->netdev); 9782 } 9783 } else { 9784 i40e_vsi_close(vsi); 9785 } 9786 i40e_vsi_disable_irq(vsi); 9787 } 9788 9789 spin_lock_bh(&vsi->mac_filter_hash_lock); 9790 9791 /* clear the sync flag on all filters */ 9792 if (vsi->netdev) { 9793 __dev_uc_unsync(vsi->netdev, NULL); 9794 __dev_mc_unsync(vsi->netdev, NULL); 9795 } 9796 9797 /* make sure any remaining filters are marked for deletion */ 9798 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 9799 __i40e_del_filter(vsi, f); 9800 9801 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9802 9803 i40e_sync_vsi_filters(vsi); 9804 9805 i40e_vsi_delete(vsi); 9806 i40e_vsi_free_q_vectors(vsi); 9807 if (vsi->netdev) { 9808 free_netdev(vsi->netdev); 9809 vsi->netdev = NULL; 9810 } 9811 i40e_vsi_clear_rings(vsi); 9812 i40e_vsi_clear(vsi); 9813 9814 /* If this was the last thing on the VEB, except for the 9815 * controlling VSI, remove the VEB, which puts the controlling 9816 * VSI onto the next level down in the switch. 9817 * 9818 * Well, okay, there's one more exception here: don't remove 9819 * the orphan VEBs yet. We'll wait for an explicit remove request 9820 * from up the network stack. 9821 */ 9822 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 9823 if (pf->vsi[i] && 9824 pf->vsi[i]->uplink_seid == uplink_seid && 9825 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 9826 n++; /* count the VSIs */ 9827 } 9828 } 9829 for (i = 0; i < I40E_MAX_VEB; i++) { 9830 if (!pf->veb[i]) 9831 continue; 9832 if (pf->veb[i]->uplink_seid == uplink_seid) 9833 n++; /* count the VEBs */ 9834 if (pf->veb[i]->seid == uplink_seid) 9835 veb = pf->veb[i]; 9836 } 9837 if (n == 0 && veb && veb->uplink_seid != 0) 9838 i40e_veb_release(veb); 9839 9840 return 0; 9841 } 9842 9843 /** 9844 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 9845 * @vsi: ptr to the VSI 9846 * 9847 * This should only be called after i40e_vsi_mem_alloc() which allocates the 9848 * corresponding SW VSI structure and initializes num_queue_pairs for the 9849 * newly allocated VSI. 9850 * 9851 * Returns 0 on success or negative on failure 9852 **/ 9853 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 9854 { 9855 int ret = -ENOENT; 9856 struct i40e_pf *pf = vsi->back; 9857 9858 if (vsi->q_vectors[0]) { 9859 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 9860 vsi->seid); 9861 return -EEXIST; 9862 } 9863 9864 if (vsi->base_vector) { 9865 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 9866 vsi->seid, vsi->base_vector); 9867 return -EEXIST; 9868 } 9869 9870 ret = i40e_vsi_alloc_q_vectors(vsi); 9871 if (ret) { 9872 dev_info(&pf->pdev->dev, 9873 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 9874 vsi->num_q_vectors, vsi->seid, ret); 9875 vsi->num_q_vectors = 0; 9876 goto vector_setup_out; 9877 } 9878 9879 /* In Legacy mode, we do not have to get any other vector since we 9880 * piggyback on the misc/ICR0 for queue interrupts. 9881 */ 9882 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 9883 return ret; 9884 if (vsi->num_q_vectors) 9885 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 9886 vsi->num_q_vectors, vsi->idx); 9887 if (vsi->base_vector < 0) { 9888 dev_info(&pf->pdev->dev, 9889 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 9890 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 9891 i40e_vsi_free_q_vectors(vsi); 9892 ret = -ENOENT; 9893 goto vector_setup_out; 9894 } 9895 9896 vector_setup_out: 9897 return ret; 9898 } 9899 9900 /** 9901 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 9902 * @vsi: pointer to the vsi. 9903 * 9904 * This re-allocates a vsi's queue resources. 9905 * 9906 * Returns pointer to the successfully allocated and configured VSI sw struct 9907 * on success, otherwise returns NULL on failure. 9908 **/ 9909 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 9910 { 9911 struct i40e_pf *pf; 9912 u8 enabled_tc; 9913 int ret; 9914 9915 if (!vsi) 9916 return NULL; 9917 9918 pf = vsi->back; 9919 9920 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 9921 i40e_vsi_clear_rings(vsi); 9922 9923 i40e_vsi_free_arrays(vsi, false); 9924 i40e_set_num_rings_in_vsi(vsi); 9925 ret = i40e_vsi_alloc_arrays(vsi, false); 9926 if (ret) 9927 goto err_vsi; 9928 9929 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx); 9930 if (ret < 0) { 9931 dev_info(&pf->pdev->dev, 9932 "failed to get tracking for %d queues for VSI %d err %d\n", 9933 vsi->alloc_queue_pairs, vsi->seid, ret); 9934 goto err_vsi; 9935 } 9936 vsi->base_queue = ret; 9937 9938 /* Update the FW view of the VSI. Force a reset of TC and queue 9939 * layout configurations. 9940 */ 9941 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 9942 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 9943 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 9944 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 9945 if (vsi->type == I40E_VSI_MAIN) 9946 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 9947 9948 /* assign it some queues */ 9949 ret = i40e_alloc_rings(vsi); 9950 if (ret) 9951 goto err_rings; 9952 9953 /* map all of the rings to the q_vectors */ 9954 i40e_vsi_map_rings_to_vectors(vsi); 9955 return vsi; 9956 9957 err_rings: 9958 i40e_vsi_free_q_vectors(vsi); 9959 if (vsi->netdev_registered) { 9960 vsi->netdev_registered = false; 9961 unregister_netdev(vsi->netdev); 9962 free_netdev(vsi->netdev); 9963 vsi->netdev = NULL; 9964 } 9965 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 9966 err_vsi: 9967 i40e_vsi_clear(vsi); 9968 return NULL; 9969 } 9970 9971 /** 9972 * i40e_vsi_setup - Set up a VSI by a given type 9973 * @pf: board private structure 9974 * @type: VSI type 9975 * @uplink_seid: the switch element to link to 9976 * @param1: usage depends upon VSI type. For VF types, indicates VF id 9977 * 9978 * This allocates the sw VSI structure and its queue resources, then add a VSI 9979 * to the identified VEB. 9980 * 9981 * Returns pointer to the successfully allocated and configure VSI sw struct on 9982 * success, otherwise returns NULL on failure. 9983 **/ 9984 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 9985 u16 uplink_seid, u32 param1) 9986 { 9987 struct i40e_vsi *vsi = NULL; 9988 struct i40e_veb *veb = NULL; 9989 int ret, i; 9990 int v_idx; 9991 9992 /* The requested uplink_seid must be either 9993 * - the PF's port seid 9994 * no VEB is needed because this is the PF 9995 * or this is a Flow Director special case VSI 9996 * - seid of an existing VEB 9997 * - seid of a VSI that owns an existing VEB 9998 * - seid of a VSI that doesn't own a VEB 9999 * a new VEB is created and the VSI becomes the owner 10000 * - seid of the PF VSI, which is what creates the first VEB 10001 * this is a special case of the previous 10002 * 10003 * Find which uplink_seid we were given and create a new VEB if needed 10004 */ 10005 for (i = 0; i < I40E_MAX_VEB; i++) { 10006 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 10007 veb = pf->veb[i]; 10008 break; 10009 } 10010 } 10011 10012 if (!veb && uplink_seid != pf->mac_seid) { 10013 10014 for (i = 0; i < pf->num_alloc_vsi; i++) { 10015 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 10016 vsi = pf->vsi[i]; 10017 break; 10018 } 10019 } 10020 if (!vsi) { 10021 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 10022 uplink_seid); 10023 return NULL; 10024 } 10025 10026 if (vsi->uplink_seid == pf->mac_seid) 10027 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 10028 vsi->tc_config.enabled_tc); 10029 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 10030 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 10031 vsi->tc_config.enabled_tc); 10032 if (veb) { 10033 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 10034 dev_info(&vsi->back->pdev->dev, 10035 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 10036 return NULL; 10037 } 10038 /* We come up by default in VEPA mode if SRIOV is not 10039 * already enabled, in which case we can't force VEPA 10040 * mode. 10041 */ 10042 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 10043 veb->bridge_mode = BRIDGE_MODE_VEPA; 10044 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 10045 } 10046 i40e_config_bridge_mode(veb); 10047 } 10048 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 10049 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 10050 veb = pf->veb[i]; 10051 } 10052 if (!veb) { 10053 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 10054 return NULL; 10055 } 10056 10057 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 10058 uplink_seid = veb->seid; 10059 } 10060 10061 /* get vsi sw struct */ 10062 v_idx = i40e_vsi_mem_alloc(pf, type); 10063 if (v_idx < 0) 10064 goto err_alloc; 10065 vsi = pf->vsi[v_idx]; 10066 if (!vsi) 10067 goto err_alloc; 10068 vsi->type = type; 10069 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 10070 10071 if (type == I40E_VSI_MAIN) 10072 pf->lan_vsi = v_idx; 10073 else if (type == I40E_VSI_SRIOV) 10074 vsi->vf_id = param1; 10075 /* assign it some queues */ 10076 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, 10077 vsi->idx); 10078 if (ret < 0) { 10079 dev_info(&pf->pdev->dev, 10080 "failed to get tracking for %d queues for VSI %d err=%d\n", 10081 vsi->alloc_queue_pairs, vsi->seid, ret); 10082 goto err_vsi; 10083 } 10084 vsi->base_queue = ret; 10085 10086 /* get a VSI from the hardware */ 10087 vsi->uplink_seid = uplink_seid; 10088 ret = i40e_add_vsi(vsi); 10089 if (ret) 10090 goto err_vsi; 10091 10092 switch (vsi->type) { 10093 /* setup the netdev if needed */ 10094 case I40E_VSI_MAIN: 10095 /* Apply relevant filters if a platform-specific mac 10096 * address was selected. 10097 */ 10098 if (!!(pf->flags & I40E_FLAG_PF_MAC)) { 10099 ret = i40e_macaddr_init(vsi, pf->hw.mac.addr); 10100 if (ret) { 10101 dev_warn(&pf->pdev->dev, 10102 "could not set up macaddr; err %d\n", 10103 ret); 10104 } 10105 } 10106 case I40E_VSI_VMDQ2: 10107 ret = i40e_config_netdev(vsi); 10108 if (ret) 10109 goto err_netdev; 10110 ret = register_netdev(vsi->netdev); 10111 if (ret) 10112 goto err_netdev; 10113 vsi->netdev_registered = true; 10114 netif_carrier_off(vsi->netdev); 10115 #ifdef CONFIG_I40E_DCB 10116 /* Setup DCB netlink interface */ 10117 i40e_dcbnl_setup(vsi); 10118 #endif /* CONFIG_I40E_DCB */ 10119 /* fall through */ 10120 10121 case I40E_VSI_FDIR: 10122 /* set up vectors and rings if needed */ 10123 ret = i40e_vsi_setup_vectors(vsi); 10124 if (ret) 10125 goto err_msix; 10126 10127 ret = i40e_alloc_rings(vsi); 10128 if (ret) 10129 goto err_rings; 10130 10131 /* map all of the rings to the q_vectors */ 10132 i40e_vsi_map_rings_to_vectors(vsi); 10133 10134 i40e_vsi_reset_stats(vsi); 10135 break; 10136 10137 default: 10138 /* no netdev or rings for the other VSI types */ 10139 break; 10140 } 10141 10142 if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) && 10143 (vsi->type == I40E_VSI_VMDQ2)) { 10144 ret = i40e_vsi_config_rss(vsi); 10145 } 10146 return vsi; 10147 10148 err_rings: 10149 i40e_vsi_free_q_vectors(vsi); 10150 err_msix: 10151 if (vsi->netdev_registered) { 10152 vsi->netdev_registered = false; 10153 unregister_netdev(vsi->netdev); 10154 free_netdev(vsi->netdev); 10155 vsi->netdev = NULL; 10156 } 10157 err_netdev: 10158 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 10159 err_vsi: 10160 i40e_vsi_clear(vsi); 10161 err_alloc: 10162 return NULL; 10163 } 10164 10165 /** 10166 * i40e_veb_get_bw_info - Query VEB BW information 10167 * @veb: the veb to query 10168 * 10169 * Query the Tx scheduler BW configuration data for given VEB 10170 **/ 10171 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 10172 { 10173 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 10174 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 10175 struct i40e_pf *pf = veb->pf; 10176 struct i40e_hw *hw = &pf->hw; 10177 u32 tc_bw_max; 10178 int ret = 0; 10179 int i; 10180 10181 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 10182 &bw_data, NULL); 10183 if (ret) { 10184 dev_info(&pf->pdev->dev, 10185 "query veb bw config failed, err %s aq_err %s\n", 10186 i40e_stat_str(&pf->hw, ret), 10187 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 10188 goto out; 10189 } 10190 10191 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 10192 &ets_data, NULL); 10193 if (ret) { 10194 dev_info(&pf->pdev->dev, 10195 "query veb bw ets config failed, err %s aq_err %s\n", 10196 i40e_stat_str(&pf->hw, ret), 10197 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 10198 goto out; 10199 } 10200 10201 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 10202 veb->bw_max_quanta = ets_data.tc_bw_max; 10203 veb->is_abs_credits = bw_data.absolute_credits_enable; 10204 veb->enabled_tc = ets_data.tc_valid_bits; 10205 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 10206 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 10207 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 10208 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 10209 veb->bw_tc_limit_credits[i] = 10210 le16_to_cpu(bw_data.tc_bw_limits[i]); 10211 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 10212 } 10213 10214 out: 10215 return ret; 10216 } 10217 10218 /** 10219 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 10220 * @pf: board private structure 10221 * 10222 * On error: returns error code (negative) 10223 * On success: returns vsi index in PF (positive) 10224 **/ 10225 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 10226 { 10227 int ret = -ENOENT; 10228 struct i40e_veb *veb; 10229 int i; 10230 10231 /* Need to protect the allocation of switch elements at the PF level */ 10232 mutex_lock(&pf->switch_mutex); 10233 10234 /* VEB list may be fragmented if VEB creation/destruction has 10235 * been happening. We can afford to do a quick scan to look 10236 * for any free slots in the list. 10237 * 10238 * find next empty veb slot, looping back around if necessary 10239 */ 10240 i = 0; 10241 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 10242 i++; 10243 if (i >= I40E_MAX_VEB) { 10244 ret = -ENOMEM; 10245 goto err_alloc_veb; /* out of VEB slots! */ 10246 } 10247 10248 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 10249 if (!veb) { 10250 ret = -ENOMEM; 10251 goto err_alloc_veb; 10252 } 10253 veb->pf = pf; 10254 veb->idx = i; 10255 veb->enabled_tc = 1; 10256 10257 pf->veb[i] = veb; 10258 ret = i; 10259 err_alloc_veb: 10260 mutex_unlock(&pf->switch_mutex); 10261 return ret; 10262 } 10263 10264 /** 10265 * i40e_switch_branch_release - Delete a branch of the switch tree 10266 * @branch: where to start deleting 10267 * 10268 * This uses recursion to find the tips of the branch to be 10269 * removed, deleting until we get back to and can delete this VEB. 10270 **/ 10271 static void i40e_switch_branch_release(struct i40e_veb *branch) 10272 { 10273 struct i40e_pf *pf = branch->pf; 10274 u16 branch_seid = branch->seid; 10275 u16 veb_idx = branch->idx; 10276 int i; 10277 10278 /* release any VEBs on this VEB - RECURSION */ 10279 for (i = 0; i < I40E_MAX_VEB; i++) { 10280 if (!pf->veb[i]) 10281 continue; 10282 if (pf->veb[i]->uplink_seid == branch->seid) 10283 i40e_switch_branch_release(pf->veb[i]); 10284 } 10285 10286 /* Release the VSIs on this VEB, but not the owner VSI. 10287 * 10288 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 10289 * the VEB itself, so don't use (*branch) after this loop. 10290 */ 10291 for (i = 0; i < pf->num_alloc_vsi; i++) { 10292 if (!pf->vsi[i]) 10293 continue; 10294 if (pf->vsi[i]->uplink_seid == branch_seid && 10295 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 10296 i40e_vsi_release(pf->vsi[i]); 10297 } 10298 } 10299 10300 /* There's one corner case where the VEB might not have been 10301 * removed, so double check it here and remove it if needed. 10302 * This case happens if the veb was created from the debugfs 10303 * commands and no VSIs were added to it. 10304 */ 10305 if (pf->veb[veb_idx]) 10306 i40e_veb_release(pf->veb[veb_idx]); 10307 } 10308 10309 /** 10310 * i40e_veb_clear - remove veb struct 10311 * @veb: the veb to remove 10312 **/ 10313 static void i40e_veb_clear(struct i40e_veb *veb) 10314 { 10315 if (!veb) 10316 return; 10317 10318 if (veb->pf) { 10319 struct i40e_pf *pf = veb->pf; 10320 10321 mutex_lock(&pf->switch_mutex); 10322 if (pf->veb[veb->idx] == veb) 10323 pf->veb[veb->idx] = NULL; 10324 mutex_unlock(&pf->switch_mutex); 10325 } 10326 10327 kfree(veb); 10328 } 10329 10330 /** 10331 * i40e_veb_release - Delete a VEB and free its resources 10332 * @veb: the VEB being removed 10333 **/ 10334 void i40e_veb_release(struct i40e_veb *veb) 10335 { 10336 struct i40e_vsi *vsi = NULL; 10337 struct i40e_pf *pf; 10338 int i, n = 0; 10339 10340 pf = veb->pf; 10341 10342 /* find the remaining VSI and check for extras */ 10343 for (i = 0; i < pf->num_alloc_vsi; i++) { 10344 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 10345 n++; 10346 vsi = pf->vsi[i]; 10347 } 10348 } 10349 if (n != 1) { 10350 dev_info(&pf->pdev->dev, 10351 "can't remove VEB %d with %d VSIs left\n", 10352 veb->seid, n); 10353 return; 10354 } 10355 10356 /* move the remaining VSI to uplink veb */ 10357 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 10358 if (veb->uplink_seid) { 10359 vsi->uplink_seid = veb->uplink_seid; 10360 if (veb->uplink_seid == pf->mac_seid) 10361 vsi->veb_idx = I40E_NO_VEB; 10362 else 10363 vsi->veb_idx = veb->veb_idx; 10364 } else { 10365 /* floating VEB */ 10366 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 10367 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 10368 } 10369 10370 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10371 i40e_veb_clear(veb); 10372 } 10373 10374 /** 10375 * i40e_add_veb - create the VEB in the switch 10376 * @veb: the VEB to be instantiated 10377 * @vsi: the controlling VSI 10378 **/ 10379 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 10380 { 10381 struct i40e_pf *pf = veb->pf; 10382 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 10383 int ret; 10384 10385 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 10386 veb->enabled_tc, false, 10387 &veb->seid, enable_stats, NULL); 10388 10389 /* get a VEB from the hardware */ 10390 if (ret) { 10391 dev_info(&pf->pdev->dev, 10392 "couldn't add VEB, err %s aq_err %s\n", 10393 i40e_stat_str(&pf->hw, ret), 10394 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10395 return -EPERM; 10396 } 10397 10398 /* get statistics counter */ 10399 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 10400 &veb->stats_idx, NULL, NULL, NULL); 10401 if (ret) { 10402 dev_info(&pf->pdev->dev, 10403 "couldn't get VEB statistics idx, err %s aq_err %s\n", 10404 i40e_stat_str(&pf->hw, ret), 10405 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10406 return -EPERM; 10407 } 10408 ret = i40e_veb_get_bw_info(veb); 10409 if (ret) { 10410 dev_info(&pf->pdev->dev, 10411 "couldn't get VEB bw info, err %s aq_err %s\n", 10412 i40e_stat_str(&pf->hw, ret), 10413 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10414 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10415 return -ENOENT; 10416 } 10417 10418 vsi->uplink_seid = veb->seid; 10419 vsi->veb_idx = veb->idx; 10420 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 10421 10422 return 0; 10423 } 10424 10425 /** 10426 * i40e_veb_setup - Set up a VEB 10427 * @pf: board private structure 10428 * @flags: VEB setup flags 10429 * @uplink_seid: the switch element to link to 10430 * @vsi_seid: the initial VSI seid 10431 * @enabled_tc: Enabled TC bit-map 10432 * 10433 * This allocates the sw VEB structure and links it into the switch 10434 * It is possible and legal for this to be a duplicate of an already 10435 * existing VEB. It is also possible for both uplink and vsi seids 10436 * to be zero, in order to create a floating VEB. 10437 * 10438 * Returns pointer to the successfully allocated VEB sw struct on 10439 * success, otherwise returns NULL on failure. 10440 **/ 10441 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 10442 u16 uplink_seid, u16 vsi_seid, 10443 u8 enabled_tc) 10444 { 10445 struct i40e_veb *veb, *uplink_veb = NULL; 10446 int vsi_idx, veb_idx; 10447 int ret; 10448 10449 /* if one seid is 0, the other must be 0 to create a floating relay */ 10450 if ((uplink_seid == 0 || vsi_seid == 0) && 10451 (uplink_seid + vsi_seid != 0)) { 10452 dev_info(&pf->pdev->dev, 10453 "one, not both seid's are 0: uplink=%d vsi=%d\n", 10454 uplink_seid, vsi_seid); 10455 return NULL; 10456 } 10457 10458 /* make sure there is such a vsi and uplink */ 10459 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 10460 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 10461 break; 10462 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 10463 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 10464 vsi_seid); 10465 return NULL; 10466 } 10467 10468 if (uplink_seid && uplink_seid != pf->mac_seid) { 10469 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 10470 if (pf->veb[veb_idx] && 10471 pf->veb[veb_idx]->seid == uplink_seid) { 10472 uplink_veb = pf->veb[veb_idx]; 10473 break; 10474 } 10475 } 10476 if (!uplink_veb) { 10477 dev_info(&pf->pdev->dev, 10478 "uplink seid %d not found\n", uplink_seid); 10479 return NULL; 10480 } 10481 } 10482 10483 /* get veb sw struct */ 10484 veb_idx = i40e_veb_mem_alloc(pf); 10485 if (veb_idx < 0) 10486 goto err_alloc; 10487 veb = pf->veb[veb_idx]; 10488 veb->flags = flags; 10489 veb->uplink_seid = uplink_seid; 10490 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 10491 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 10492 10493 /* create the VEB in the switch */ 10494 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 10495 if (ret) 10496 goto err_veb; 10497 if (vsi_idx == pf->lan_vsi) 10498 pf->lan_veb = veb->idx; 10499 10500 return veb; 10501 10502 err_veb: 10503 i40e_veb_clear(veb); 10504 err_alloc: 10505 return NULL; 10506 } 10507 10508 /** 10509 * i40e_setup_pf_switch_element - set PF vars based on switch type 10510 * @pf: board private structure 10511 * @ele: element we are building info from 10512 * @num_reported: total number of elements 10513 * @printconfig: should we print the contents 10514 * 10515 * helper function to assist in extracting a few useful SEID values. 10516 **/ 10517 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 10518 struct i40e_aqc_switch_config_element_resp *ele, 10519 u16 num_reported, bool printconfig) 10520 { 10521 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 10522 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 10523 u8 element_type = ele->element_type; 10524 u16 seid = le16_to_cpu(ele->seid); 10525 10526 if (printconfig) 10527 dev_info(&pf->pdev->dev, 10528 "type=%d seid=%d uplink=%d downlink=%d\n", 10529 element_type, seid, uplink_seid, downlink_seid); 10530 10531 switch (element_type) { 10532 case I40E_SWITCH_ELEMENT_TYPE_MAC: 10533 pf->mac_seid = seid; 10534 break; 10535 case I40E_SWITCH_ELEMENT_TYPE_VEB: 10536 /* Main VEB? */ 10537 if (uplink_seid != pf->mac_seid) 10538 break; 10539 if (pf->lan_veb == I40E_NO_VEB) { 10540 int v; 10541 10542 /* find existing or else empty VEB */ 10543 for (v = 0; v < I40E_MAX_VEB; v++) { 10544 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 10545 pf->lan_veb = v; 10546 break; 10547 } 10548 } 10549 if (pf->lan_veb == I40E_NO_VEB) { 10550 v = i40e_veb_mem_alloc(pf); 10551 if (v < 0) 10552 break; 10553 pf->lan_veb = v; 10554 } 10555 } 10556 10557 pf->veb[pf->lan_veb]->seid = seid; 10558 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 10559 pf->veb[pf->lan_veb]->pf = pf; 10560 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 10561 break; 10562 case I40E_SWITCH_ELEMENT_TYPE_VSI: 10563 if (num_reported != 1) 10564 break; 10565 /* This is immediately after a reset so we can assume this is 10566 * the PF's VSI 10567 */ 10568 pf->mac_seid = uplink_seid; 10569 pf->pf_seid = downlink_seid; 10570 pf->main_vsi_seid = seid; 10571 if (printconfig) 10572 dev_info(&pf->pdev->dev, 10573 "pf_seid=%d main_vsi_seid=%d\n", 10574 pf->pf_seid, pf->main_vsi_seid); 10575 break; 10576 case I40E_SWITCH_ELEMENT_TYPE_PF: 10577 case I40E_SWITCH_ELEMENT_TYPE_VF: 10578 case I40E_SWITCH_ELEMENT_TYPE_EMP: 10579 case I40E_SWITCH_ELEMENT_TYPE_BMC: 10580 case I40E_SWITCH_ELEMENT_TYPE_PE: 10581 case I40E_SWITCH_ELEMENT_TYPE_PA: 10582 /* ignore these for now */ 10583 break; 10584 default: 10585 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 10586 element_type, seid); 10587 break; 10588 } 10589 } 10590 10591 /** 10592 * i40e_fetch_switch_configuration - Get switch config from firmware 10593 * @pf: board private structure 10594 * @printconfig: should we print the contents 10595 * 10596 * Get the current switch configuration from the device and 10597 * extract a few useful SEID values. 10598 **/ 10599 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 10600 { 10601 struct i40e_aqc_get_switch_config_resp *sw_config; 10602 u16 next_seid = 0; 10603 int ret = 0; 10604 u8 *aq_buf; 10605 int i; 10606 10607 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 10608 if (!aq_buf) 10609 return -ENOMEM; 10610 10611 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 10612 do { 10613 u16 num_reported, num_total; 10614 10615 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 10616 I40E_AQ_LARGE_BUF, 10617 &next_seid, NULL); 10618 if (ret) { 10619 dev_info(&pf->pdev->dev, 10620 "get switch config failed err %s aq_err %s\n", 10621 i40e_stat_str(&pf->hw, ret), 10622 i40e_aq_str(&pf->hw, 10623 pf->hw.aq.asq_last_status)); 10624 kfree(aq_buf); 10625 return -ENOENT; 10626 } 10627 10628 num_reported = le16_to_cpu(sw_config->header.num_reported); 10629 num_total = le16_to_cpu(sw_config->header.num_total); 10630 10631 if (printconfig) 10632 dev_info(&pf->pdev->dev, 10633 "header: %d reported %d total\n", 10634 num_reported, num_total); 10635 10636 for (i = 0; i < num_reported; i++) { 10637 struct i40e_aqc_switch_config_element_resp *ele = 10638 &sw_config->element[i]; 10639 10640 i40e_setup_pf_switch_element(pf, ele, num_reported, 10641 printconfig); 10642 } 10643 } while (next_seid != 0); 10644 10645 kfree(aq_buf); 10646 return ret; 10647 } 10648 10649 /** 10650 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 10651 * @pf: board private structure 10652 * @reinit: if the Main VSI needs to re-initialized. 10653 * 10654 * Returns 0 on success, negative value on failure 10655 **/ 10656 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 10657 { 10658 u16 flags = 0; 10659 int ret; 10660 10661 /* find out what's out there already */ 10662 ret = i40e_fetch_switch_configuration(pf, false); 10663 if (ret) { 10664 dev_info(&pf->pdev->dev, 10665 "couldn't fetch switch config, err %s aq_err %s\n", 10666 i40e_stat_str(&pf->hw, ret), 10667 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10668 return ret; 10669 } 10670 i40e_pf_reset_stats(pf); 10671 10672 /* set the switch config bit for the whole device to 10673 * support limited promisc or true promisc 10674 * when user requests promisc. The default is limited 10675 * promisc. 10676 */ 10677 10678 if ((pf->hw.pf_id == 0) && 10679 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) 10680 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 10681 10682 if (pf->hw.pf_id == 0) { 10683 u16 valid_flags; 10684 10685 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 10686 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 10687 NULL); 10688 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 10689 dev_info(&pf->pdev->dev, 10690 "couldn't set switch config bits, err %s aq_err %s\n", 10691 i40e_stat_str(&pf->hw, ret), 10692 i40e_aq_str(&pf->hw, 10693 pf->hw.aq.asq_last_status)); 10694 /* not a fatal problem, just keep going */ 10695 } 10696 } 10697 10698 /* first time setup */ 10699 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 10700 struct i40e_vsi *vsi = NULL; 10701 u16 uplink_seid; 10702 10703 /* Set up the PF VSI associated with the PF's main VSI 10704 * that is already in the HW switch 10705 */ 10706 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 10707 uplink_seid = pf->veb[pf->lan_veb]->seid; 10708 else 10709 uplink_seid = pf->mac_seid; 10710 if (pf->lan_vsi == I40E_NO_VSI) 10711 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 10712 else if (reinit) 10713 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 10714 if (!vsi) { 10715 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 10716 i40e_fdir_teardown(pf); 10717 return -EAGAIN; 10718 } 10719 } else { 10720 /* force a reset of TC and queue layout configurations */ 10721 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 10722 10723 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 10724 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 10725 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 10726 } 10727 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 10728 10729 i40e_fdir_sb_setup(pf); 10730 10731 /* Setup static PF queue filter control settings */ 10732 ret = i40e_setup_pf_filter_control(pf); 10733 if (ret) { 10734 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 10735 ret); 10736 /* Failure here should not stop continuing other steps */ 10737 } 10738 10739 /* enable RSS in the HW, even for only one queue, as the stack can use 10740 * the hash 10741 */ 10742 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 10743 i40e_pf_config_rss(pf); 10744 10745 /* fill in link information and enable LSE reporting */ 10746 i40e_link_event(pf); 10747 10748 /* Initialize user-specific link properties */ 10749 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 10750 I40E_AQ_AN_COMPLETED) ? true : false); 10751 10752 i40e_ptp_init(pf); 10753 10754 /* repopulate tunnel port filters */ 10755 i40e_sync_udp_filters(pf); 10756 10757 return ret; 10758 } 10759 10760 /** 10761 * i40e_determine_queue_usage - Work out queue distribution 10762 * @pf: board private structure 10763 **/ 10764 static void i40e_determine_queue_usage(struct i40e_pf *pf) 10765 { 10766 int queues_left; 10767 10768 pf->num_lan_qps = 0; 10769 10770 /* Find the max queues to be put into basic use. We'll always be 10771 * using TC0, whether or not DCB is running, and TC0 will get the 10772 * big RSS set. 10773 */ 10774 queues_left = pf->hw.func_caps.num_tx_qp; 10775 10776 if ((queues_left == 1) || 10777 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 10778 /* one qp for PF, no queues for anything else */ 10779 queues_left = 0; 10780 pf->alloc_rss_size = pf->num_lan_qps = 1; 10781 10782 /* make sure all the fancies are disabled */ 10783 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 10784 I40E_FLAG_IWARP_ENABLED | 10785 I40E_FLAG_FD_SB_ENABLED | 10786 I40E_FLAG_FD_ATR_ENABLED | 10787 I40E_FLAG_DCB_CAPABLE | 10788 I40E_FLAG_DCB_ENABLED | 10789 I40E_FLAG_SRIOV_ENABLED | 10790 I40E_FLAG_VMDQ_ENABLED); 10791 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 10792 I40E_FLAG_FD_SB_ENABLED | 10793 I40E_FLAG_FD_ATR_ENABLED | 10794 I40E_FLAG_DCB_CAPABLE))) { 10795 /* one qp for PF */ 10796 pf->alloc_rss_size = pf->num_lan_qps = 1; 10797 queues_left -= pf->num_lan_qps; 10798 10799 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 10800 I40E_FLAG_IWARP_ENABLED | 10801 I40E_FLAG_FD_SB_ENABLED | 10802 I40E_FLAG_FD_ATR_ENABLED | 10803 I40E_FLAG_DCB_ENABLED | 10804 I40E_FLAG_VMDQ_ENABLED); 10805 } else { 10806 /* Not enough queues for all TCs */ 10807 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 10808 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 10809 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 10810 I40E_FLAG_DCB_ENABLED); 10811 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 10812 } 10813 pf->num_lan_qps = max_t(int, pf->rss_size_max, 10814 num_online_cpus()); 10815 pf->num_lan_qps = min_t(int, pf->num_lan_qps, 10816 pf->hw.func_caps.num_tx_qp); 10817 10818 queues_left -= pf->num_lan_qps; 10819 } 10820 10821 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10822 if (queues_left > 1) { 10823 queues_left -= 1; /* save 1 queue for FD */ 10824 } else { 10825 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10826 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 10827 } 10828 } 10829 10830 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 10831 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 10832 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 10833 (queues_left / pf->num_vf_qps)); 10834 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 10835 } 10836 10837 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 10838 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 10839 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 10840 (queues_left / pf->num_vmdq_qps)); 10841 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 10842 } 10843 10844 pf->queues_left = queues_left; 10845 dev_dbg(&pf->pdev->dev, 10846 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 10847 pf->hw.func_caps.num_tx_qp, 10848 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 10849 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 10850 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 10851 queues_left); 10852 } 10853 10854 /** 10855 * i40e_setup_pf_filter_control - Setup PF static filter control 10856 * @pf: PF to be setup 10857 * 10858 * i40e_setup_pf_filter_control sets up a PF's initial filter control 10859 * settings. If PE/FCoE are enabled then it will also set the per PF 10860 * based filter sizes required for them. It also enables Flow director, 10861 * ethertype and macvlan type filter settings for the pf. 10862 * 10863 * Returns 0 on success, negative on failure 10864 **/ 10865 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 10866 { 10867 struct i40e_filter_control_settings *settings = &pf->filter_settings; 10868 10869 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 10870 10871 /* Flow Director is enabled */ 10872 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 10873 settings->enable_fdir = true; 10874 10875 /* Ethtype and MACVLAN filters enabled for PF */ 10876 settings->enable_ethtype = true; 10877 settings->enable_macvlan = true; 10878 10879 if (i40e_set_filter_control(&pf->hw, settings)) 10880 return -ENOENT; 10881 10882 return 0; 10883 } 10884 10885 #define INFO_STRING_LEN 255 10886 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 10887 static void i40e_print_features(struct i40e_pf *pf) 10888 { 10889 struct i40e_hw *hw = &pf->hw; 10890 char *buf; 10891 int i; 10892 10893 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 10894 if (!buf) 10895 return; 10896 10897 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 10898 #ifdef CONFIG_PCI_IOV 10899 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 10900 #endif 10901 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 10902 pf->hw.func_caps.num_vsis, 10903 pf->vsi[pf->lan_vsi]->num_queue_pairs); 10904 if (pf->flags & I40E_FLAG_RSS_ENABLED) 10905 i += snprintf(&buf[i], REMAIN(i), " RSS"); 10906 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 10907 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 10908 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10909 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 10910 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 10911 } 10912 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 10913 i += snprintf(&buf[i], REMAIN(i), " DCB"); 10914 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 10915 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 10916 if (pf->flags & I40E_FLAG_PTP) 10917 i += snprintf(&buf[i], REMAIN(i), " PTP"); 10918 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 10919 i += snprintf(&buf[i], REMAIN(i), " VEB"); 10920 else 10921 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 10922 10923 dev_info(&pf->pdev->dev, "%s\n", buf); 10924 kfree(buf); 10925 WARN_ON(i > INFO_STRING_LEN); 10926 } 10927 10928 /** 10929 * i40e_get_platform_mac_addr - get platform-specific MAC address 10930 * @pdev: PCI device information struct 10931 * @pf: board private structure 10932 * 10933 * Look up the MAC address for the device. First we'll try 10934 * eth_platform_get_mac_address, which will check Open Firmware, or arch 10935 * specific fallback. Otherwise, we'll default to the stored value in 10936 * firmware. 10937 **/ 10938 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 10939 { 10940 if (eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 10941 i40e_get_mac_addr(&pf->hw, pf->hw.mac.addr); 10942 } 10943 10944 /** 10945 * i40e_probe - Device initialization routine 10946 * @pdev: PCI device information struct 10947 * @ent: entry in i40e_pci_tbl 10948 * 10949 * i40e_probe initializes a PF identified by a pci_dev structure. 10950 * The OS initialization, configuring of the PF private structure, 10951 * and a hardware reset occur. 10952 * 10953 * Returns 0 on success, negative on failure 10954 **/ 10955 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 10956 { 10957 struct i40e_aq_get_phy_abilities_resp abilities; 10958 struct i40e_pf *pf; 10959 struct i40e_hw *hw; 10960 static u16 pfs_found; 10961 u16 wol_nvm_bits; 10962 u16 link_status; 10963 int err; 10964 u32 val; 10965 u32 i; 10966 u8 set_fc_aq_fail; 10967 10968 err = pci_enable_device_mem(pdev); 10969 if (err) 10970 return err; 10971 10972 /* set up for high or low dma */ 10973 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 10974 if (err) { 10975 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 10976 if (err) { 10977 dev_err(&pdev->dev, 10978 "DMA configuration failed: 0x%x\n", err); 10979 goto err_dma; 10980 } 10981 } 10982 10983 /* set up pci connections */ 10984 err = pci_request_mem_regions(pdev, i40e_driver_name); 10985 if (err) { 10986 dev_info(&pdev->dev, 10987 "pci_request_selected_regions failed %d\n", err); 10988 goto err_pci_reg; 10989 } 10990 10991 pci_enable_pcie_error_reporting(pdev); 10992 pci_set_master(pdev); 10993 10994 /* Now that we have a PCI connection, we need to do the 10995 * low level device setup. This is primarily setting up 10996 * the Admin Queue structures and then querying for the 10997 * device's current profile information. 10998 */ 10999 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 11000 if (!pf) { 11001 err = -ENOMEM; 11002 goto err_pf_alloc; 11003 } 11004 pf->next_vsi = 0; 11005 pf->pdev = pdev; 11006 set_bit(__I40E_VSI_DOWN, pf->state); 11007 11008 hw = &pf->hw; 11009 hw->back = pf; 11010 11011 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 11012 I40E_MAX_CSR_SPACE); 11013 11014 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 11015 if (!hw->hw_addr) { 11016 err = -EIO; 11017 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 11018 (unsigned int)pci_resource_start(pdev, 0), 11019 pf->ioremap_len, err); 11020 goto err_ioremap; 11021 } 11022 hw->vendor_id = pdev->vendor; 11023 hw->device_id = pdev->device; 11024 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 11025 hw->subsystem_vendor_id = pdev->subsystem_vendor; 11026 hw->subsystem_device_id = pdev->subsystem_device; 11027 hw->bus.device = PCI_SLOT(pdev->devfn); 11028 hw->bus.func = PCI_FUNC(pdev->devfn); 11029 hw->bus.bus_id = pdev->bus->number; 11030 pf->instance = pfs_found; 11031 11032 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 11033 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 11034 11035 /* set up the locks for the AQ, do this only once in probe 11036 * and destroy them only once in remove 11037 */ 11038 mutex_init(&hw->aq.asq_mutex); 11039 mutex_init(&hw->aq.arq_mutex); 11040 11041 pf->msg_enable = netif_msg_init(debug, 11042 NETIF_MSG_DRV | 11043 NETIF_MSG_PROBE | 11044 NETIF_MSG_LINK); 11045 if (debug < -1) 11046 pf->hw.debug_mask = debug; 11047 11048 /* do a special CORER for clearing PXE mode once at init */ 11049 if (hw->revision_id == 0 && 11050 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 11051 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 11052 i40e_flush(hw); 11053 msleep(200); 11054 pf->corer_count++; 11055 11056 i40e_clear_pxe_mode(hw); 11057 } 11058 11059 /* Reset here to make sure all is clean and to define PF 'n' */ 11060 i40e_clear_hw(hw); 11061 err = i40e_pf_reset(hw); 11062 if (err) { 11063 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 11064 goto err_pf_reset; 11065 } 11066 pf->pfr_count++; 11067 11068 hw->aq.num_arq_entries = I40E_AQ_LEN; 11069 hw->aq.num_asq_entries = I40E_AQ_LEN; 11070 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 11071 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 11072 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 11073 11074 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 11075 "%s-%s:misc", 11076 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 11077 11078 err = i40e_init_shared_code(hw); 11079 if (err) { 11080 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 11081 err); 11082 goto err_pf_reset; 11083 } 11084 11085 /* set up a default setting for link flow control */ 11086 pf->hw.fc.requested_mode = I40E_FC_NONE; 11087 11088 err = i40e_init_adminq(hw); 11089 if (err) { 11090 if (err == I40E_ERR_FIRMWARE_API_VERSION) 11091 dev_info(&pdev->dev, 11092 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 11093 else 11094 dev_info(&pdev->dev, 11095 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 11096 11097 goto err_pf_reset; 11098 } 11099 11100 /* provide nvm, fw, api versions */ 11101 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 11102 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 11103 hw->aq.api_maj_ver, hw->aq.api_min_ver, 11104 i40e_nvm_version_str(hw)); 11105 11106 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 11107 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) 11108 dev_info(&pdev->dev, 11109 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 11110 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 11111 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) 11112 dev_info(&pdev->dev, 11113 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 11114 11115 i40e_verify_eeprom(pf); 11116 11117 /* Rev 0 hardware was never productized */ 11118 if (hw->revision_id < 1) 11119 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 11120 11121 i40e_clear_pxe_mode(hw); 11122 err = i40e_get_capabilities(pf); 11123 if (err) 11124 goto err_adminq_setup; 11125 11126 err = i40e_sw_init(pf); 11127 if (err) { 11128 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 11129 goto err_sw_init; 11130 } 11131 11132 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 11133 hw->func_caps.num_rx_qp, 0, 0); 11134 if (err) { 11135 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 11136 goto err_init_lan_hmc; 11137 } 11138 11139 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 11140 if (err) { 11141 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 11142 err = -ENOENT; 11143 goto err_configure_lan_hmc; 11144 } 11145 11146 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 11147 * Ignore error return codes because if it was already disabled via 11148 * hardware settings this will fail 11149 */ 11150 if (pf->flags & I40E_FLAG_STOP_FW_LLDP) { 11151 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 11152 i40e_aq_stop_lldp(hw, true, NULL); 11153 } 11154 11155 /* allow a platform config to override the HW addr */ 11156 i40e_get_platform_mac_addr(pdev, pf); 11157 11158 if (!is_valid_ether_addr(hw->mac.addr)) { 11159 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 11160 err = -EIO; 11161 goto err_mac_addr; 11162 } 11163 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 11164 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 11165 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 11166 if (is_valid_ether_addr(hw->mac.port_addr)) 11167 pf->flags |= I40E_FLAG_PORT_ID_VALID; 11168 11169 pci_set_drvdata(pdev, pf); 11170 pci_save_state(pdev); 11171 #ifdef CONFIG_I40E_DCB 11172 err = i40e_init_pf_dcb(pf); 11173 if (err) { 11174 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 11175 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 11176 /* Continue without DCB enabled */ 11177 } 11178 #endif /* CONFIG_I40E_DCB */ 11179 11180 /* set up periodic task facility */ 11181 setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf); 11182 pf->service_timer_period = HZ; 11183 11184 INIT_WORK(&pf->service_task, i40e_service_task); 11185 clear_bit(__I40E_SERVICE_SCHED, pf->state); 11186 11187 /* NVM bit on means WoL disabled for the port */ 11188 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 11189 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 11190 pf->wol_en = false; 11191 else 11192 pf->wol_en = true; 11193 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 11194 11195 /* set up the main switch operations */ 11196 i40e_determine_queue_usage(pf); 11197 err = i40e_init_interrupt_scheme(pf); 11198 if (err) 11199 goto err_switch_setup; 11200 11201 /* The number of VSIs reported by the FW is the minimum guaranteed 11202 * to us; HW supports far more and we share the remaining pool with 11203 * the other PFs. We allocate space for more than the guarantee with 11204 * the understanding that we might not get them all later. 11205 */ 11206 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 11207 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 11208 else 11209 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 11210 11211 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 11212 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 11213 GFP_KERNEL); 11214 if (!pf->vsi) { 11215 err = -ENOMEM; 11216 goto err_switch_setup; 11217 } 11218 11219 #ifdef CONFIG_PCI_IOV 11220 /* prep for VF support */ 11221 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11222 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11223 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 11224 if (pci_num_vf(pdev)) 11225 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11226 } 11227 #endif 11228 err = i40e_setup_pf_switch(pf, false); 11229 if (err) { 11230 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 11231 goto err_vsis; 11232 } 11233 11234 /* Make sure flow control is set according to current settings */ 11235 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 11236 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 11237 dev_dbg(&pf->pdev->dev, 11238 "Set fc with err %s aq_err %s on get_phy_cap\n", 11239 i40e_stat_str(hw, err), 11240 i40e_aq_str(hw, hw->aq.asq_last_status)); 11241 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 11242 dev_dbg(&pf->pdev->dev, 11243 "Set fc with err %s aq_err %s on set_phy_config\n", 11244 i40e_stat_str(hw, err), 11245 i40e_aq_str(hw, hw->aq.asq_last_status)); 11246 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 11247 dev_dbg(&pf->pdev->dev, 11248 "Set fc with err %s aq_err %s on get_link_info\n", 11249 i40e_stat_str(hw, err), 11250 i40e_aq_str(hw, hw->aq.asq_last_status)); 11251 11252 /* if FDIR VSI was set up, start it now */ 11253 for (i = 0; i < pf->num_alloc_vsi; i++) { 11254 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 11255 i40e_vsi_open(pf->vsi[i]); 11256 break; 11257 } 11258 } 11259 11260 /* The driver only wants link up/down and module qualification 11261 * reports from firmware. Note the negative logic. 11262 */ 11263 err = i40e_aq_set_phy_int_mask(&pf->hw, 11264 ~(I40E_AQ_EVENT_LINK_UPDOWN | 11265 I40E_AQ_EVENT_MEDIA_NA | 11266 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 11267 if (err) 11268 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 11269 i40e_stat_str(&pf->hw, err), 11270 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11271 11272 /* Reconfigure hardware for allowing smaller MSS in the case 11273 * of TSO, so that we avoid the MDD being fired and causing 11274 * a reset in the case of small MSS+TSO. 11275 */ 11276 val = rd32(hw, I40E_REG_MSS); 11277 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11278 val &= ~I40E_REG_MSS_MIN_MASK; 11279 val |= I40E_64BYTE_MSS; 11280 wr32(hw, I40E_REG_MSS, val); 11281 } 11282 11283 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) { 11284 msleep(75); 11285 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11286 if (err) 11287 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 11288 i40e_stat_str(&pf->hw, err), 11289 i40e_aq_str(&pf->hw, 11290 pf->hw.aq.asq_last_status)); 11291 } 11292 /* The main driver is (mostly) up and happy. We need to set this state 11293 * before setting up the misc vector or we get a race and the vector 11294 * ends up disabled forever. 11295 */ 11296 clear_bit(__I40E_VSI_DOWN, pf->state); 11297 11298 /* In case of MSIX we are going to setup the misc vector right here 11299 * to handle admin queue events etc. In case of legacy and MSI 11300 * the misc functionality and queue processing is combined in 11301 * the same vector and that gets setup at open. 11302 */ 11303 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11304 err = i40e_setup_misc_vector(pf); 11305 if (err) { 11306 dev_info(&pdev->dev, 11307 "setup of misc vector failed: %d\n", err); 11308 goto err_vsis; 11309 } 11310 } 11311 11312 #ifdef CONFIG_PCI_IOV 11313 /* prep for VF support */ 11314 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11315 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11316 !test_bit(__I40E_BAD_EEPROM, pf->state)) { 11317 /* disable link interrupts for VFs */ 11318 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 11319 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 11320 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 11321 i40e_flush(hw); 11322 11323 if (pci_num_vf(pdev)) { 11324 dev_info(&pdev->dev, 11325 "Active VFs found, allocating resources.\n"); 11326 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 11327 if (err) 11328 dev_info(&pdev->dev, 11329 "Error %d allocating resources for existing VFs\n", 11330 err); 11331 } 11332 } 11333 #endif /* CONFIG_PCI_IOV */ 11334 11335 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11336 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 11337 pf->num_iwarp_msix, 11338 I40E_IWARP_IRQ_PILE_ID); 11339 if (pf->iwarp_base_vector < 0) { 11340 dev_info(&pdev->dev, 11341 "failed to get tracking for %d vectors for IWARP err=%d\n", 11342 pf->num_iwarp_msix, pf->iwarp_base_vector); 11343 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11344 } 11345 } 11346 11347 i40e_dbg_pf_init(pf); 11348 11349 /* tell the firmware that we're starting */ 11350 i40e_send_version(pf); 11351 11352 /* since everything's happy, start the service_task timer */ 11353 mod_timer(&pf->service_timer, 11354 round_jiffies(jiffies + pf->service_timer_period)); 11355 11356 /* add this PF to client device list and launch a client service task */ 11357 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11358 err = i40e_lan_add_device(pf); 11359 if (err) 11360 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 11361 err); 11362 } 11363 11364 #define PCI_SPEED_SIZE 8 11365 #define PCI_WIDTH_SIZE 8 11366 /* Devices on the IOSF bus do not have this information 11367 * and will report PCI Gen 1 x 1 by default so don't bother 11368 * checking them. 11369 */ 11370 if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) { 11371 char speed[PCI_SPEED_SIZE] = "Unknown"; 11372 char width[PCI_WIDTH_SIZE] = "Unknown"; 11373 11374 /* Get the negotiated link width and speed from PCI config 11375 * space 11376 */ 11377 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 11378 &link_status); 11379 11380 i40e_set_pci_config_data(hw, link_status); 11381 11382 switch (hw->bus.speed) { 11383 case i40e_bus_speed_8000: 11384 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 11385 case i40e_bus_speed_5000: 11386 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 11387 case i40e_bus_speed_2500: 11388 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 11389 default: 11390 break; 11391 } 11392 switch (hw->bus.width) { 11393 case i40e_bus_width_pcie_x8: 11394 strncpy(width, "8", PCI_WIDTH_SIZE); break; 11395 case i40e_bus_width_pcie_x4: 11396 strncpy(width, "4", PCI_WIDTH_SIZE); break; 11397 case i40e_bus_width_pcie_x2: 11398 strncpy(width, "2", PCI_WIDTH_SIZE); break; 11399 case i40e_bus_width_pcie_x1: 11400 strncpy(width, "1", PCI_WIDTH_SIZE); break; 11401 default: 11402 break; 11403 } 11404 11405 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 11406 speed, width); 11407 11408 if (hw->bus.width < i40e_bus_width_pcie_x8 || 11409 hw->bus.speed < i40e_bus_speed_8000) { 11410 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 11411 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 11412 } 11413 } 11414 11415 /* get the requested speeds from the fw */ 11416 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 11417 if (err) 11418 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 11419 i40e_stat_str(&pf->hw, err), 11420 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11421 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 11422 11423 /* get the supported phy types from the fw */ 11424 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 11425 if (err) 11426 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 11427 i40e_stat_str(&pf->hw, err), 11428 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11429 11430 /* Add a filter to drop all Flow control frames from any VSI from being 11431 * transmitted. By doing so we stop a malicious VF from sending out 11432 * PAUSE or PFC frames and potentially controlling traffic for other 11433 * PF/VF VSIs. 11434 * The FW can still send Flow control frames if enabled. 11435 */ 11436 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11437 pf->main_vsi_seid); 11438 11439 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 11440 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 11441 pf->flags |= I40E_FLAG_PHY_CONTROLS_LEDS; 11442 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 11443 pf->flags |= I40E_FLAG_HAVE_CRT_RETIMER; 11444 /* print a string summarizing features */ 11445 i40e_print_features(pf); 11446 11447 return 0; 11448 11449 /* Unwind what we've done if something failed in the setup */ 11450 err_vsis: 11451 set_bit(__I40E_VSI_DOWN, pf->state); 11452 i40e_clear_interrupt_scheme(pf); 11453 kfree(pf->vsi); 11454 err_switch_setup: 11455 i40e_reset_interrupt_capability(pf); 11456 del_timer_sync(&pf->service_timer); 11457 err_mac_addr: 11458 err_configure_lan_hmc: 11459 (void)i40e_shutdown_lan_hmc(hw); 11460 err_init_lan_hmc: 11461 kfree(pf->qp_pile); 11462 err_sw_init: 11463 err_adminq_setup: 11464 err_pf_reset: 11465 iounmap(hw->hw_addr); 11466 err_ioremap: 11467 kfree(pf); 11468 err_pf_alloc: 11469 pci_disable_pcie_error_reporting(pdev); 11470 pci_release_mem_regions(pdev); 11471 err_pci_reg: 11472 err_dma: 11473 pci_disable_device(pdev); 11474 return err; 11475 } 11476 11477 /** 11478 * i40e_remove - Device removal routine 11479 * @pdev: PCI device information struct 11480 * 11481 * i40e_remove is called by the PCI subsystem to alert the driver 11482 * that is should release a PCI device. This could be caused by a 11483 * Hot-Plug event, or because the driver is going to be removed from 11484 * memory. 11485 **/ 11486 static void i40e_remove(struct pci_dev *pdev) 11487 { 11488 struct i40e_pf *pf = pci_get_drvdata(pdev); 11489 struct i40e_hw *hw = &pf->hw; 11490 i40e_status ret_code; 11491 int i; 11492 11493 i40e_dbg_pf_exit(pf); 11494 11495 i40e_ptp_stop(pf); 11496 11497 /* Disable RSS in hw */ 11498 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 11499 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 11500 11501 /* no more scheduling of any task */ 11502 set_bit(__I40E_SUSPENDED, pf->state); 11503 set_bit(__I40E_VSI_DOWN, pf->state); 11504 if (pf->service_timer.data) 11505 del_timer_sync(&pf->service_timer); 11506 if (pf->service_task.func) 11507 cancel_work_sync(&pf->service_task); 11508 11509 /* Client close must be called explicitly here because the timer 11510 * has been stopped. 11511 */ 11512 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 11513 11514 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 11515 i40e_free_vfs(pf); 11516 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 11517 } 11518 11519 i40e_fdir_teardown(pf); 11520 11521 /* If there is a switch structure or any orphans, remove them. 11522 * This will leave only the PF's VSI remaining. 11523 */ 11524 for (i = 0; i < I40E_MAX_VEB; i++) { 11525 if (!pf->veb[i]) 11526 continue; 11527 11528 if (pf->veb[i]->uplink_seid == pf->mac_seid || 11529 pf->veb[i]->uplink_seid == 0) 11530 i40e_switch_branch_release(pf->veb[i]); 11531 } 11532 11533 /* Now we can shutdown the PF's VSI, just before we kill 11534 * adminq and hmc. 11535 */ 11536 if (pf->vsi[pf->lan_vsi]) 11537 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 11538 11539 /* remove attached clients */ 11540 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11541 ret_code = i40e_lan_del_device(pf); 11542 if (ret_code) 11543 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 11544 ret_code); 11545 } 11546 11547 /* shutdown and destroy the HMC */ 11548 if (hw->hmc.hmc_obj) { 11549 ret_code = i40e_shutdown_lan_hmc(hw); 11550 if (ret_code) 11551 dev_warn(&pdev->dev, 11552 "Failed to destroy the HMC resources: %d\n", 11553 ret_code); 11554 } 11555 11556 /* shutdown the adminq */ 11557 i40e_shutdown_adminq(hw); 11558 11559 /* destroy the locks only once, here */ 11560 mutex_destroy(&hw->aq.arq_mutex); 11561 mutex_destroy(&hw->aq.asq_mutex); 11562 11563 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 11564 i40e_clear_interrupt_scheme(pf); 11565 for (i = 0; i < pf->num_alloc_vsi; i++) { 11566 if (pf->vsi[i]) { 11567 i40e_vsi_clear_rings(pf->vsi[i]); 11568 i40e_vsi_clear(pf->vsi[i]); 11569 pf->vsi[i] = NULL; 11570 } 11571 } 11572 11573 for (i = 0; i < I40E_MAX_VEB; i++) { 11574 kfree(pf->veb[i]); 11575 pf->veb[i] = NULL; 11576 } 11577 11578 kfree(pf->qp_pile); 11579 kfree(pf->vsi); 11580 11581 iounmap(hw->hw_addr); 11582 kfree(pf); 11583 pci_release_mem_regions(pdev); 11584 11585 pci_disable_pcie_error_reporting(pdev); 11586 pci_disable_device(pdev); 11587 } 11588 11589 /** 11590 * i40e_pci_error_detected - warning that something funky happened in PCI land 11591 * @pdev: PCI device information struct 11592 * 11593 * Called to warn that something happened and the error handling steps 11594 * are in progress. Allows the driver to quiesce things, be ready for 11595 * remediation. 11596 **/ 11597 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 11598 enum pci_channel_state error) 11599 { 11600 struct i40e_pf *pf = pci_get_drvdata(pdev); 11601 11602 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 11603 11604 if (!pf) { 11605 dev_info(&pdev->dev, 11606 "Cannot recover - error happened during device probe\n"); 11607 return PCI_ERS_RESULT_DISCONNECT; 11608 } 11609 11610 /* shutdown all operations */ 11611 if (!test_bit(__I40E_SUSPENDED, pf->state)) { 11612 rtnl_lock(); 11613 i40e_prep_for_reset(pf, true); 11614 rtnl_unlock(); 11615 } 11616 11617 /* Request a slot reset */ 11618 return PCI_ERS_RESULT_NEED_RESET; 11619 } 11620 11621 /** 11622 * i40e_pci_error_slot_reset - a PCI slot reset just happened 11623 * @pdev: PCI device information struct 11624 * 11625 * Called to find if the driver can work with the device now that 11626 * the pci slot has been reset. If a basic connection seems good 11627 * (registers are readable and have sane content) then return a 11628 * happy little PCI_ERS_RESULT_xxx. 11629 **/ 11630 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 11631 { 11632 struct i40e_pf *pf = pci_get_drvdata(pdev); 11633 pci_ers_result_t result; 11634 int err; 11635 u32 reg; 11636 11637 dev_dbg(&pdev->dev, "%s\n", __func__); 11638 if (pci_enable_device_mem(pdev)) { 11639 dev_info(&pdev->dev, 11640 "Cannot re-enable PCI device after reset.\n"); 11641 result = PCI_ERS_RESULT_DISCONNECT; 11642 } else { 11643 pci_set_master(pdev); 11644 pci_restore_state(pdev); 11645 pci_save_state(pdev); 11646 pci_wake_from_d3(pdev, false); 11647 11648 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 11649 if (reg == 0) 11650 result = PCI_ERS_RESULT_RECOVERED; 11651 else 11652 result = PCI_ERS_RESULT_DISCONNECT; 11653 } 11654 11655 err = pci_cleanup_aer_uncorrect_error_status(pdev); 11656 if (err) { 11657 dev_info(&pdev->dev, 11658 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 11659 err); 11660 /* non-fatal, continue */ 11661 } 11662 11663 return result; 11664 } 11665 11666 /** 11667 * i40e_pci_error_resume - restart operations after PCI error recovery 11668 * @pdev: PCI device information struct 11669 * 11670 * Called to allow the driver to bring things back up after PCI error 11671 * and/or reset recovery has finished. 11672 **/ 11673 static void i40e_pci_error_resume(struct pci_dev *pdev) 11674 { 11675 struct i40e_pf *pf = pci_get_drvdata(pdev); 11676 11677 dev_dbg(&pdev->dev, "%s\n", __func__); 11678 if (test_bit(__I40E_SUSPENDED, pf->state)) 11679 return; 11680 11681 rtnl_lock(); 11682 i40e_handle_reset_warning(pf, true); 11683 rtnl_unlock(); 11684 } 11685 11686 /** 11687 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 11688 * using the mac_address_write admin q function 11689 * @pf: pointer to i40e_pf struct 11690 **/ 11691 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 11692 { 11693 struct i40e_hw *hw = &pf->hw; 11694 i40e_status ret; 11695 u8 mac_addr[6]; 11696 u16 flags = 0; 11697 11698 /* Get current MAC address in case it's an LAA */ 11699 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 11700 ether_addr_copy(mac_addr, 11701 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 11702 } else { 11703 dev_err(&pf->pdev->dev, 11704 "Failed to retrieve MAC address; using default\n"); 11705 ether_addr_copy(mac_addr, hw->mac.addr); 11706 } 11707 11708 /* The FW expects the mac address write cmd to first be called with 11709 * one of these flags before calling it again with the multicast 11710 * enable flags. 11711 */ 11712 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 11713 11714 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 11715 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 11716 11717 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 11718 if (ret) { 11719 dev_err(&pf->pdev->dev, 11720 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 11721 return; 11722 } 11723 11724 flags = I40E_AQC_MC_MAG_EN 11725 | I40E_AQC_WOL_PRESERVE_ON_PFR 11726 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 11727 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 11728 if (ret) 11729 dev_err(&pf->pdev->dev, 11730 "Failed to enable Multicast Magic Packet wake up\n"); 11731 } 11732 11733 /** 11734 * i40e_shutdown - PCI callback for shutting down 11735 * @pdev: PCI device information struct 11736 **/ 11737 static void i40e_shutdown(struct pci_dev *pdev) 11738 { 11739 struct i40e_pf *pf = pci_get_drvdata(pdev); 11740 struct i40e_hw *hw = &pf->hw; 11741 11742 set_bit(__I40E_SUSPENDED, pf->state); 11743 set_bit(__I40E_VSI_DOWN, pf->state); 11744 rtnl_lock(); 11745 i40e_prep_for_reset(pf, true); 11746 rtnl_unlock(); 11747 11748 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11749 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11750 11751 del_timer_sync(&pf->service_timer); 11752 cancel_work_sync(&pf->service_task); 11753 i40e_fdir_teardown(pf); 11754 11755 /* Client close must be called explicitly here because the timer 11756 * has been stopped. 11757 */ 11758 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], false); 11759 11760 if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE)) 11761 i40e_enable_mc_magic_wake(pf); 11762 11763 rtnl_lock(); 11764 i40e_prep_for_reset(pf, true); 11765 rtnl_unlock(); 11766 11767 wr32(hw, I40E_PFPM_APM, 11768 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11769 wr32(hw, I40E_PFPM_WUFC, 11770 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11771 11772 i40e_clear_interrupt_scheme(pf); 11773 11774 if (system_state == SYSTEM_POWER_OFF) { 11775 pci_wake_from_d3(pdev, pf->wol_en); 11776 pci_set_power_state(pdev, PCI_D3hot); 11777 } 11778 } 11779 11780 #ifdef CONFIG_PM 11781 /** 11782 * i40e_suspend - PCI callback for moving to D3 11783 * @pdev: PCI device information struct 11784 **/ 11785 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state) 11786 { 11787 struct i40e_pf *pf = pci_get_drvdata(pdev); 11788 struct i40e_hw *hw = &pf->hw; 11789 int retval = 0; 11790 11791 set_bit(__I40E_SUSPENDED, pf->state); 11792 set_bit(__I40E_VSI_DOWN, pf->state); 11793 11794 if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE)) 11795 i40e_enable_mc_magic_wake(pf); 11796 11797 rtnl_lock(); 11798 i40e_prep_for_reset(pf, true); 11799 rtnl_unlock(); 11800 11801 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11802 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11803 11804 i40e_stop_misc_vector(pf); 11805 11806 retval = pci_save_state(pdev); 11807 if (retval) 11808 return retval; 11809 11810 pci_wake_from_d3(pdev, pf->wol_en); 11811 pci_set_power_state(pdev, PCI_D3hot); 11812 11813 return retval; 11814 } 11815 11816 /** 11817 * i40e_resume - PCI callback for waking up from D3 11818 * @pdev: PCI device information struct 11819 **/ 11820 static int i40e_resume(struct pci_dev *pdev) 11821 { 11822 struct i40e_pf *pf = pci_get_drvdata(pdev); 11823 u32 err; 11824 11825 pci_set_power_state(pdev, PCI_D0); 11826 pci_restore_state(pdev); 11827 /* pci_restore_state() clears dev->state_saves, so 11828 * call pci_save_state() again to restore it. 11829 */ 11830 pci_save_state(pdev); 11831 11832 err = pci_enable_device_mem(pdev); 11833 if (err) { 11834 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 11835 return err; 11836 } 11837 pci_set_master(pdev); 11838 11839 /* no wakeup events while running */ 11840 pci_wake_from_d3(pdev, false); 11841 11842 /* handling the reset will rebuild the device state */ 11843 if (test_and_clear_bit(__I40E_SUSPENDED, pf->state)) { 11844 clear_bit(__I40E_VSI_DOWN, pf->state); 11845 rtnl_lock(); 11846 i40e_reset_and_rebuild(pf, false, true); 11847 rtnl_unlock(); 11848 } 11849 11850 return 0; 11851 } 11852 11853 #endif 11854 static const struct pci_error_handlers i40e_err_handler = { 11855 .error_detected = i40e_pci_error_detected, 11856 .slot_reset = i40e_pci_error_slot_reset, 11857 .resume = i40e_pci_error_resume, 11858 }; 11859 11860 static struct pci_driver i40e_driver = { 11861 .name = i40e_driver_name, 11862 .id_table = i40e_pci_tbl, 11863 .probe = i40e_probe, 11864 .remove = i40e_remove, 11865 #ifdef CONFIG_PM 11866 .suspend = i40e_suspend, 11867 .resume = i40e_resume, 11868 #endif 11869 .shutdown = i40e_shutdown, 11870 .err_handler = &i40e_err_handler, 11871 .sriov_configure = i40e_pci_sriov_configure, 11872 }; 11873 11874 /** 11875 * i40e_init_module - Driver registration routine 11876 * 11877 * i40e_init_module is the first routine called when the driver is 11878 * loaded. All it does is register with the PCI subsystem. 11879 **/ 11880 static int __init i40e_init_module(void) 11881 { 11882 pr_info("%s: %s - version %s\n", i40e_driver_name, 11883 i40e_driver_string, i40e_driver_version_str); 11884 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 11885 11886 /* we will see if single thread per module is enough for now, 11887 * it can't be any worse than using the system workqueue which 11888 * was already single threaded 11889 */ 11890 i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1, 11891 i40e_driver_name); 11892 if (!i40e_wq) { 11893 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 11894 return -ENOMEM; 11895 } 11896 11897 i40e_dbg_init(); 11898 return pci_register_driver(&i40e_driver); 11899 } 11900 module_init(i40e_init_module); 11901 11902 /** 11903 * i40e_exit_module - Driver exit cleanup routine 11904 * 11905 * i40e_exit_module is called just before the driver is removed 11906 * from memory. 11907 **/ 11908 static void __exit i40e_exit_module(void) 11909 { 11910 pci_unregister_driver(&i40e_driver); 11911 destroy_workqueue(i40e_wq); 11912 i40e_dbg_exit(); 11913 } 11914 module_exit(i40e_exit_module); 11915