1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2016 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 #include <linux/etherdevice.h> 28 #include <linux/of_net.h> 29 #include <linux/pci.h> 30 31 /* Local includes */ 32 #include "i40e.h" 33 #include "i40e_diag.h" 34 #include <net/udp_tunnel.h> 35 36 const char i40e_driver_name[] = "i40e"; 37 static const char i40e_driver_string[] = 38 "Intel(R) Ethernet Connection XL710 Network Driver"; 39 40 #define DRV_KERN "-k" 41 42 #define DRV_VERSION_MAJOR 2 43 #define DRV_VERSION_MINOR 1 44 #define DRV_VERSION_BUILD 7 45 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 46 __stringify(DRV_VERSION_MINOR) "." \ 47 __stringify(DRV_VERSION_BUILD) DRV_KERN 48 const char i40e_driver_version_str[] = DRV_VERSION; 49 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 50 51 /* a bit of forward declarations */ 52 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 53 static void i40e_handle_reset_warning(struct i40e_pf *pf); 54 static int i40e_add_vsi(struct i40e_vsi *vsi); 55 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 56 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 57 static int i40e_setup_misc_vector(struct i40e_pf *pf); 58 static void i40e_determine_queue_usage(struct i40e_pf *pf); 59 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 60 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 61 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 62 63 /* i40e_pci_tbl - PCI Device ID Table 64 * 65 * Last entry must be all 0s 66 * 67 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 68 * Class, Class Mask, private data (not used) } 69 */ 70 static const struct pci_device_id i40e_pci_tbl[] = { 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 77 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 78 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T), 0}, 79 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T4), 0}, 80 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_X722), 0}, 81 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_X722), 0}, 82 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_X722), 0}, 83 {PCI_VDEVICE(INTEL, I40E_DEV_ID_1G_BASE_T_X722), 0}, 84 {PCI_VDEVICE(INTEL, I40E_DEV_ID_10G_BASE_T_X722), 0}, 85 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_I_X722), 0}, 86 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2), 0}, 87 {PCI_VDEVICE(INTEL, I40E_DEV_ID_20G_KR2_A), 0}, 88 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_B), 0}, 89 {PCI_VDEVICE(INTEL, I40E_DEV_ID_25G_SFP28), 0}, 90 /* required last entry */ 91 {0, } 92 }; 93 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 94 95 #define I40E_MAX_VF_COUNT 128 96 static int debug = -1; 97 module_param(debug, uint, 0); 98 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all), Debug mask (0x8XXXXXXX)"); 99 100 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 101 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 102 MODULE_LICENSE("GPL"); 103 MODULE_VERSION(DRV_VERSION); 104 105 static struct workqueue_struct *i40e_wq; 106 107 /** 108 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 109 * @hw: pointer to the HW structure 110 * @mem: ptr to mem struct to fill out 111 * @size: size of memory requested 112 * @alignment: what to align the allocation to 113 **/ 114 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 115 u64 size, u32 alignment) 116 { 117 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 118 119 mem->size = ALIGN(size, alignment); 120 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 121 &mem->pa, GFP_KERNEL); 122 if (!mem->va) 123 return -ENOMEM; 124 125 return 0; 126 } 127 128 /** 129 * i40e_free_dma_mem_d - OS specific memory free for shared code 130 * @hw: pointer to the HW structure 131 * @mem: ptr to mem struct to free 132 **/ 133 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 134 { 135 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 136 137 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 138 mem->va = NULL; 139 mem->pa = 0; 140 mem->size = 0; 141 142 return 0; 143 } 144 145 /** 146 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 147 * @hw: pointer to the HW structure 148 * @mem: ptr to mem struct to fill out 149 * @size: size of memory requested 150 **/ 151 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 152 u32 size) 153 { 154 mem->size = size; 155 mem->va = kzalloc(size, GFP_KERNEL); 156 157 if (!mem->va) 158 return -ENOMEM; 159 160 return 0; 161 } 162 163 /** 164 * i40e_free_virt_mem_d - OS specific memory free for shared code 165 * @hw: pointer to the HW structure 166 * @mem: ptr to mem struct to free 167 **/ 168 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 169 { 170 /* it's ok to kfree a NULL pointer */ 171 kfree(mem->va); 172 mem->va = NULL; 173 mem->size = 0; 174 175 return 0; 176 } 177 178 /** 179 * i40e_get_lump - find a lump of free generic resource 180 * @pf: board private structure 181 * @pile: the pile of resource to search 182 * @needed: the number of items needed 183 * @id: an owner id to stick on the items assigned 184 * 185 * Returns the base item index of the lump, or negative for error 186 * 187 * The search_hint trick and lack of advanced fit-finding only work 188 * because we're highly likely to have all the same size lump requests. 189 * Linear search time and any fragmentation should be minimal. 190 **/ 191 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 192 u16 needed, u16 id) 193 { 194 int ret = -ENOMEM; 195 int i, j; 196 197 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 198 dev_info(&pf->pdev->dev, 199 "param err: pile=%p needed=%d id=0x%04x\n", 200 pile, needed, id); 201 return -EINVAL; 202 } 203 204 /* start the linear search with an imperfect hint */ 205 i = pile->search_hint; 206 while (i < pile->num_entries) { 207 /* skip already allocated entries */ 208 if (pile->list[i] & I40E_PILE_VALID_BIT) { 209 i++; 210 continue; 211 } 212 213 /* do we have enough in this lump? */ 214 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 215 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 216 break; 217 } 218 219 if (j == needed) { 220 /* there was enough, so assign it to the requestor */ 221 for (j = 0; j < needed; j++) 222 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 223 ret = i; 224 pile->search_hint = i + j; 225 break; 226 } 227 228 /* not enough, so skip over it and continue looking */ 229 i += j; 230 } 231 232 return ret; 233 } 234 235 /** 236 * i40e_put_lump - return a lump of generic resource 237 * @pile: the pile of resource to search 238 * @index: the base item index 239 * @id: the owner id of the items assigned 240 * 241 * Returns the count of items in the lump 242 **/ 243 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 244 { 245 int valid_id = (id | I40E_PILE_VALID_BIT); 246 int count = 0; 247 int i; 248 249 if (!pile || index >= pile->num_entries) 250 return -EINVAL; 251 252 for (i = index; 253 i < pile->num_entries && pile->list[i] == valid_id; 254 i++) { 255 pile->list[i] = 0; 256 count++; 257 } 258 259 if (count && index < pile->search_hint) 260 pile->search_hint = index; 261 262 return count; 263 } 264 265 /** 266 * i40e_find_vsi_from_id - searches for the vsi with the given id 267 * @pf - the pf structure to search for the vsi 268 * @id - id of the vsi it is searching for 269 **/ 270 struct i40e_vsi *i40e_find_vsi_from_id(struct i40e_pf *pf, u16 id) 271 { 272 int i; 273 274 for (i = 0; i < pf->num_alloc_vsi; i++) 275 if (pf->vsi[i] && (pf->vsi[i]->id == id)) 276 return pf->vsi[i]; 277 278 return NULL; 279 } 280 281 /** 282 * i40e_service_event_schedule - Schedule the service task to wake up 283 * @pf: board private structure 284 * 285 * If not already scheduled, this puts the task into the work queue 286 **/ 287 void i40e_service_event_schedule(struct i40e_pf *pf) 288 { 289 if (!test_bit(__I40E_DOWN, &pf->state) && 290 !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 291 queue_work(i40e_wq, &pf->service_task); 292 } 293 294 /** 295 * i40e_tx_timeout - Respond to a Tx Hang 296 * @netdev: network interface device structure 297 * 298 * If any port has noticed a Tx timeout, it is likely that the whole 299 * device is munged, not just the one netdev port, so go for the full 300 * reset. 301 **/ 302 static void i40e_tx_timeout(struct net_device *netdev) 303 { 304 struct i40e_netdev_priv *np = netdev_priv(netdev); 305 struct i40e_vsi *vsi = np->vsi; 306 struct i40e_pf *pf = vsi->back; 307 struct i40e_ring *tx_ring = NULL; 308 unsigned int i, hung_queue = 0; 309 u32 head, val; 310 311 pf->tx_timeout_count++; 312 313 /* find the stopped queue the same way the stack does */ 314 for (i = 0; i < netdev->num_tx_queues; i++) { 315 struct netdev_queue *q; 316 unsigned long trans_start; 317 318 q = netdev_get_tx_queue(netdev, i); 319 trans_start = q->trans_start; 320 if (netif_xmit_stopped(q) && 321 time_after(jiffies, 322 (trans_start + netdev->watchdog_timeo))) { 323 hung_queue = i; 324 break; 325 } 326 } 327 328 if (i == netdev->num_tx_queues) { 329 netdev_info(netdev, "tx_timeout: no netdev hung queue found\n"); 330 } else { 331 /* now that we have an index, find the tx_ring struct */ 332 for (i = 0; i < vsi->num_queue_pairs; i++) { 333 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 334 if (hung_queue == 335 vsi->tx_rings[i]->queue_index) { 336 tx_ring = vsi->tx_rings[i]; 337 break; 338 } 339 } 340 } 341 } 342 343 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 344 pf->tx_timeout_recovery_level = 1; /* reset after some time */ 345 else if (time_before(jiffies, 346 (pf->tx_timeout_last_recovery + netdev->watchdog_timeo))) 347 return; /* don't do any new action before the next timeout */ 348 349 if (tx_ring) { 350 head = i40e_get_head(tx_ring); 351 /* Read interrupt register */ 352 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 353 val = rd32(&pf->hw, 354 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 355 tx_ring->vsi->base_vector - 1)); 356 else 357 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 358 359 netdev_info(netdev, "tx_timeout: VSI_seid: %d, Q %d, NTC: 0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x, INT: 0x%x\n", 360 vsi->seid, hung_queue, tx_ring->next_to_clean, 361 head, tx_ring->next_to_use, 362 readl(tx_ring->tail), val); 363 } 364 365 pf->tx_timeout_last_recovery = jiffies; 366 netdev_info(netdev, "tx_timeout recovery level %d, hung_queue %d\n", 367 pf->tx_timeout_recovery_level, hung_queue); 368 369 switch (pf->tx_timeout_recovery_level) { 370 case 1: 371 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 372 break; 373 case 2: 374 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state); 375 break; 376 case 3: 377 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state); 378 break; 379 default: 380 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 381 break; 382 } 383 384 i40e_service_event_schedule(pf); 385 pf->tx_timeout_recovery_level++; 386 } 387 388 /** 389 * i40e_get_vsi_stats_struct - Get System Network Statistics 390 * @vsi: the VSI we care about 391 * 392 * Returns the address of the device statistics structure. 393 * The statistics are actually updated from the service task. 394 **/ 395 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 396 { 397 return &vsi->net_stats; 398 } 399 400 /** 401 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 402 * @netdev: network interface device structure 403 * 404 * Returns the address of the device statistics structure. 405 * The statistics are actually updated from the service task. 406 **/ 407 static void i40e_get_netdev_stats_struct(struct net_device *netdev, 408 struct rtnl_link_stats64 *stats) 409 { 410 struct i40e_netdev_priv *np = netdev_priv(netdev); 411 struct i40e_ring *tx_ring, *rx_ring; 412 struct i40e_vsi *vsi = np->vsi; 413 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 414 int i; 415 416 if (test_bit(__I40E_DOWN, &vsi->state)) 417 return; 418 419 if (!vsi->tx_rings) 420 return; 421 422 rcu_read_lock(); 423 for (i = 0; i < vsi->num_queue_pairs; i++) { 424 u64 bytes, packets; 425 unsigned int start; 426 427 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]); 428 if (!tx_ring) 429 continue; 430 431 do { 432 start = u64_stats_fetch_begin_irq(&tx_ring->syncp); 433 packets = tx_ring->stats.packets; 434 bytes = tx_ring->stats.bytes; 435 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); 436 437 stats->tx_packets += packets; 438 stats->tx_bytes += bytes; 439 rx_ring = &tx_ring[1]; 440 441 do { 442 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 443 packets = rx_ring->stats.packets; 444 bytes = rx_ring->stats.bytes; 445 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 446 447 stats->rx_packets += packets; 448 stats->rx_bytes += bytes; 449 } 450 rcu_read_unlock(); 451 452 /* following stats updated by i40e_watchdog_subtask() */ 453 stats->multicast = vsi_stats->multicast; 454 stats->tx_errors = vsi_stats->tx_errors; 455 stats->tx_dropped = vsi_stats->tx_dropped; 456 stats->rx_errors = vsi_stats->rx_errors; 457 stats->rx_dropped = vsi_stats->rx_dropped; 458 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 459 stats->rx_length_errors = vsi_stats->rx_length_errors; 460 } 461 462 /** 463 * i40e_vsi_reset_stats - Resets all stats of the given vsi 464 * @vsi: the VSI to have its stats reset 465 **/ 466 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 467 { 468 struct rtnl_link_stats64 *ns; 469 int i; 470 471 if (!vsi) 472 return; 473 474 ns = i40e_get_vsi_stats_struct(vsi); 475 memset(ns, 0, sizeof(*ns)); 476 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 477 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 478 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 479 if (vsi->rx_rings && vsi->rx_rings[0]) { 480 for (i = 0; i < vsi->num_queue_pairs; i++) { 481 memset(&vsi->rx_rings[i]->stats, 0, 482 sizeof(vsi->rx_rings[i]->stats)); 483 memset(&vsi->rx_rings[i]->rx_stats, 0, 484 sizeof(vsi->rx_rings[i]->rx_stats)); 485 memset(&vsi->tx_rings[i]->stats, 0, 486 sizeof(vsi->tx_rings[i]->stats)); 487 memset(&vsi->tx_rings[i]->tx_stats, 0, 488 sizeof(vsi->tx_rings[i]->tx_stats)); 489 } 490 } 491 vsi->stat_offsets_loaded = false; 492 } 493 494 /** 495 * i40e_pf_reset_stats - Reset all of the stats for the given PF 496 * @pf: the PF to be reset 497 **/ 498 void i40e_pf_reset_stats(struct i40e_pf *pf) 499 { 500 int i; 501 502 memset(&pf->stats, 0, sizeof(pf->stats)); 503 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 504 pf->stat_offsets_loaded = false; 505 506 for (i = 0; i < I40E_MAX_VEB; i++) { 507 if (pf->veb[i]) { 508 memset(&pf->veb[i]->stats, 0, 509 sizeof(pf->veb[i]->stats)); 510 memset(&pf->veb[i]->stats_offsets, 0, 511 sizeof(pf->veb[i]->stats_offsets)); 512 pf->veb[i]->stat_offsets_loaded = false; 513 } 514 } 515 pf->hw_csum_rx_error = 0; 516 } 517 518 /** 519 * i40e_stat_update48 - read and update a 48 bit stat from the chip 520 * @hw: ptr to the hardware info 521 * @hireg: the high 32 bit reg to read 522 * @loreg: the low 32 bit reg to read 523 * @offset_loaded: has the initial offset been loaded yet 524 * @offset: ptr to current offset value 525 * @stat: ptr to the stat 526 * 527 * Since the device stats are not reset at PFReset, they likely will not 528 * be zeroed when the driver starts. We'll save the first values read 529 * and use them as offsets to be subtracted from the raw values in order 530 * to report stats that count from zero. In the process, we also manage 531 * the potential roll-over. 532 **/ 533 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 534 bool offset_loaded, u64 *offset, u64 *stat) 535 { 536 u64 new_data; 537 538 if (hw->device_id == I40E_DEV_ID_QEMU) { 539 new_data = rd32(hw, loreg); 540 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 541 } else { 542 new_data = rd64(hw, loreg); 543 } 544 if (!offset_loaded) 545 *offset = new_data; 546 if (likely(new_data >= *offset)) 547 *stat = new_data - *offset; 548 else 549 *stat = (new_data + BIT_ULL(48)) - *offset; 550 *stat &= 0xFFFFFFFFFFFFULL; 551 } 552 553 /** 554 * i40e_stat_update32 - read and update a 32 bit stat from the chip 555 * @hw: ptr to the hardware info 556 * @reg: the hw reg to read 557 * @offset_loaded: has the initial offset been loaded yet 558 * @offset: ptr to current offset value 559 * @stat: ptr to the stat 560 **/ 561 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 562 bool offset_loaded, u64 *offset, u64 *stat) 563 { 564 u32 new_data; 565 566 new_data = rd32(hw, reg); 567 if (!offset_loaded) 568 *offset = new_data; 569 if (likely(new_data >= *offset)) 570 *stat = (u32)(new_data - *offset); 571 else 572 *stat = (u32)((new_data + BIT_ULL(32)) - *offset); 573 } 574 575 /** 576 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 577 * @vsi: the VSI to be updated 578 **/ 579 void i40e_update_eth_stats(struct i40e_vsi *vsi) 580 { 581 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 582 struct i40e_pf *pf = vsi->back; 583 struct i40e_hw *hw = &pf->hw; 584 struct i40e_eth_stats *oes; 585 struct i40e_eth_stats *es; /* device's eth stats */ 586 587 es = &vsi->eth_stats; 588 oes = &vsi->eth_stats_offsets; 589 590 /* Gather up the stats that the hw collects */ 591 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 592 vsi->stat_offsets_loaded, 593 &oes->tx_errors, &es->tx_errors); 594 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 595 vsi->stat_offsets_loaded, 596 &oes->rx_discards, &es->rx_discards); 597 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 598 vsi->stat_offsets_loaded, 599 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 600 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 601 vsi->stat_offsets_loaded, 602 &oes->tx_errors, &es->tx_errors); 603 604 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 605 I40E_GLV_GORCL(stat_idx), 606 vsi->stat_offsets_loaded, 607 &oes->rx_bytes, &es->rx_bytes); 608 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 609 I40E_GLV_UPRCL(stat_idx), 610 vsi->stat_offsets_loaded, 611 &oes->rx_unicast, &es->rx_unicast); 612 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 613 I40E_GLV_MPRCL(stat_idx), 614 vsi->stat_offsets_loaded, 615 &oes->rx_multicast, &es->rx_multicast); 616 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 617 I40E_GLV_BPRCL(stat_idx), 618 vsi->stat_offsets_loaded, 619 &oes->rx_broadcast, &es->rx_broadcast); 620 621 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 622 I40E_GLV_GOTCL(stat_idx), 623 vsi->stat_offsets_loaded, 624 &oes->tx_bytes, &es->tx_bytes); 625 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 626 I40E_GLV_UPTCL(stat_idx), 627 vsi->stat_offsets_loaded, 628 &oes->tx_unicast, &es->tx_unicast); 629 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 630 I40E_GLV_MPTCL(stat_idx), 631 vsi->stat_offsets_loaded, 632 &oes->tx_multicast, &es->tx_multicast); 633 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 634 I40E_GLV_BPTCL(stat_idx), 635 vsi->stat_offsets_loaded, 636 &oes->tx_broadcast, &es->tx_broadcast); 637 vsi->stat_offsets_loaded = true; 638 } 639 640 /** 641 * i40e_update_veb_stats - Update Switch component statistics 642 * @veb: the VEB being updated 643 **/ 644 static void i40e_update_veb_stats(struct i40e_veb *veb) 645 { 646 struct i40e_pf *pf = veb->pf; 647 struct i40e_hw *hw = &pf->hw; 648 struct i40e_eth_stats *oes; 649 struct i40e_eth_stats *es; /* device's eth stats */ 650 struct i40e_veb_tc_stats *veb_oes; 651 struct i40e_veb_tc_stats *veb_es; 652 int i, idx = 0; 653 654 idx = veb->stats_idx; 655 es = &veb->stats; 656 oes = &veb->stats_offsets; 657 veb_es = &veb->tc_stats; 658 veb_oes = &veb->tc_stats_offsets; 659 660 /* Gather up the stats that the hw collects */ 661 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 662 veb->stat_offsets_loaded, 663 &oes->tx_discards, &es->tx_discards); 664 if (hw->revision_id > 0) 665 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 666 veb->stat_offsets_loaded, 667 &oes->rx_unknown_protocol, 668 &es->rx_unknown_protocol); 669 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 670 veb->stat_offsets_loaded, 671 &oes->rx_bytes, &es->rx_bytes); 672 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 673 veb->stat_offsets_loaded, 674 &oes->rx_unicast, &es->rx_unicast); 675 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 676 veb->stat_offsets_loaded, 677 &oes->rx_multicast, &es->rx_multicast); 678 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 679 veb->stat_offsets_loaded, 680 &oes->rx_broadcast, &es->rx_broadcast); 681 682 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 683 veb->stat_offsets_loaded, 684 &oes->tx_bytes, &es->tx_bytes); 685 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 686 veb->stat_offsets_loaded, 687 &oes->tx_unicast, &es->tx_unicast); 688 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 689 veb->stat_offsets_loaded, 690 &oes->tx_multicast, &es->tx_multicast); 691 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 692 veb->stat_offsets_loaded, 693 &oes->tx_broadcast, &es->tx_broadcast); 694 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 695 i40e_stat_update48(hw, I40E_GLVEBTC_RPCH(i, idx), 696 I40E_GLVEBTC_RPCL(i, idx), 697 veb->stat_offsets_loaded, 698 &veb_oes->tc_rx_packets[i], 699 &veb_es->tc_rx_packets[i]); 700 i40e_stat_update48(hw, I40E_GLVEBTC_RBCH(i, idx), 701 I40E_GLVEBTC_RBCL(i, idx), 702 veb->stat_offsets_loaded, 703 &veb_oes->tc_rx_bytes[i], 704 &veb_es->tc_rx_bytes[i]); 705 i40e_stat_update48(hw, I40E_GLVEBTC_TPCH(i, idx), 706 I40E_GLVEBTC_TPCL(i, idx), 707 veb->stat_offsets_loaded, 708 &veb_oes->tc_tx_packets[i], 709 &veb_es->tc_tx_packets[i]); 710 i40e_stat_update48(hw, I40E_GLVEBTC_TBCH(i, idx), 711 I40E_GLVEBTC_TBCL(i, idx), 712 veb->stat_offsets_loaded, 713 &veb_oes->tc_tx_bytes[i], 714 &veb_es->tc_tx_bytes[i]); 715 } 716 veb->stat_offsets_loaded = true; 717 } 718 719 /** 720 * i40e_update_vsi_stats - Update the vsi statistics counters. 721 * @vsi: the VSI to be updated 722 * 723 * There are a few instances where we store the same stat in a 724 * couple of different structs. This is partly because we have 725 * the netdev stats that need to be filled out, which is slightly 726 * different from the "eth_stats" defined by the chip and used in 727 * VF communications. We sort it out here. 728 **/ 729 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 730 { 731 struct i40e_pf *pf = vsi->back; 732 struct rtnl_link_stats64 *ons; 733 struct rtnl_link_stats64 *ns; /* netdev stats */ 734 struct i40e_eth_stats *oes; 735 struct i40e_eth_stats *es; /* device's eth stats */ 736 u32 tx_restart, tx_busy; 737 u64 tx_lost_interrupt; 738 struct i40e_ring *p; 739 u32 rx_page, rx_buf; 740 u64 bytes, packets; 741 unsigned int start; 742 u64 tx_linearize; 743 u64 tx_force_wb; 744 u64 rx_p, rx_b; 745 u64 tx_p, tx_b; 746 u16 q; 747 748 if (test_bit(__I40E_DOWN, &vsi->state) || 749 test_bit(__I40E_CONFIG_BUSY, &pf->state)) 750 return; 751 752 ns = i40e_get_vsi_stats_struct(vsi); 753 ons = &vsi->net_stats_offsets; 754 es = &vsi->eth_stats; 755 oes = &vsi->eth_stats_offsets; 756 757 /* Gather up the netdev and vsi stats that the driver collects 758 * on the fly during packet processing 759 */ 760 rx_b = rx_p = 0; 761 tx_b = tx_p = 0; 762 tx_restart = tx_busy = tx_linearize = tx_force_wb = 0; 763 tx_lost_interrupt = 0; 764 rx_page = 0; 765 rx_buf = 0; 766 rcu_read_lock(); 767 for (q = 0; q < vsi->num_queue_pairs; q++) { 768 /* locate Tx ring */ 769 p = ACCESS_ONCE(vsi->tx_rings[q]); 770 771 do { 772 start = u64_stats_fetch_begin_irq(&p->syncp); 773 packets = p->stats.packets; 774 bytes = p->stats.bytes; 775 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 776 tx_b += bytes; 777 tx_p += packets; 778 tx_restart += p->tx_stats.restart_queue; 779 tx_busy += p->tx_stats.tx_busy; 780 tx_linearize += p->tx_stats.tx_linearize; 781 tx_force_wb += p->tx_stats.tx_force_wb; 782 tx_lost_interrupt += p->tx_stats.tx_lost_interrupt; 783 784 /* Rx queue is part of the same block as Tx queue */ 785 p = &p[1]; 786 do { 787 start = u64_stats_fetch_begin_irq(&p->syncp); 788 packets = p->stats.packets; 789 bytes = p->stats.bytes; 790 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 791 rx_b += bytes; 792 rx_p += packets; 793 rx_buf += p->rx_stats.alloc_buff_failed; 794 rx_page += p->rx_stats.alloc_page_failed; 795 } 796 rcu_read_unlock(); 797 vsi->tx_restart = tx_restart; 798 vsi->tx_busy = tx_busy; 799 vsi->tx_linearize = tx_linearize; 800 vsi->tx_force_wb = tx_force_wb; 801 vsi->tx_lost_interrupt = tx_lost_interrupt; 802 vsi->rx_page_failed = rx_page; 803 vsi->rx_buf_failed = rx_buf; 804 805 ns->rx_packets = rx_p; 806 ns->rx_bytes = rx_b; 807 ns->tx_packets = tx_p; 808 ns->tx_bytes = tx_b; 809 810 /* update netdev stats from eth stats */ 811 i40e_update_eth_stats(vsi); 812 ons->tx_errors = oes->tx_errors; 813 ns->tx_errors = es->tx_errors; 814 ons->multicast = oes->rx_multicast; 815 ns->multicast = es->rx_multicast; 816 ons->rx_dropped = oes->rx_discards; 817 ns->rx_dropped = es->rx_discards; 818 ons->tx_dropped = oes->tx_discards; 819 ns->tx_dropped = es->tx_discards; 820 821 /* pull in a couple PF stats if this is the main vsi */ 822 if (vsi == pf->vsi[pf->lan_vsi]) { 823 ns->rx_crc_errors = pf->stats.crc_errors; 824 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 825 ns->rx_length_errors = pf->stats.rx_length_errors; 826 } 827 } 828 829 /** 830 * i40e_update_pf_stats - Update the PF statistics counters. 831 * @pf: the PF to be updated 832 **/ 833 static void i40e_update_pf_stats(struct i40e_pf *pf) 834 { 835 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 836 struct i40e_hw_port_stats *nsd = &pf->stats; 837 struct i40e_hw *hw = &pf->hw; 838 u32 val; 839 int i; 840 841 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 842 I40E_GLPRT_GORCL(hw->port), 843 pf->stat_offsets_loaded, 844 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 845 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 846 I40E_GLPRT_GOTCL(hw->port), 847 pf->stat_offsets_loaded, 848 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 849 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 850 pf->stat_offsets_loaded, 851 &osd->eth.rx_discards, 852 &nsd->eth.rx_discards); 853 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 854 I40E_GLPRT_UPRCL(hw->port), 855 pf->stat_offsets_loaded, 856 &osd->eth.rx_unicast, 857 &nsd->eth.rx_unicast); 858 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 859 I40E_GLPRT_MPRCL(hw->port), 860 pf->stat_offsets_loaded, 861 &osd->eth.rx_multicast, 862 &nsd->eth.rx_multicast); 863 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 864 I40E_GLPRT_BPRCL(hw->port), 865 pf->stat_offsets_loaded, 866 &osd->eth.rx_broadcast, 867 &nsd->eth.rx_broadcast); 868 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 869 I40E_GLPRT_UPTCL(hw->port), 870 pf->stat_offsets_loaded, 871 &osd->eth.tx_unicast, 872 &nsd->eth.tx_unicast); 873 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 874 I40E_GLPRT_MPTCL(hw->port), 875 pf->stat_offsets_loaded, 876 &osd->eth.tx_multicast, 877 &nsd->eth.tx_multicast); 878 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 879 I40E_GLPRT_BPTCL(hw->port), 880 pf->stat_offsets_loaded, 881 &osd->eth.tx_broadcast, 882 &nsd->eth.tx_broadcast); 883 884 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 885 pf->stat_offsets_loaded, 886 &osd->tx_dropped_link_down, 887 &nsd->tx_dropped_link_down); 888 889 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 890 pf->stat_offsets_loaded, 891 &osd->crc_errors, &nsd->crc_errors); 892 893 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 894 pf->stat_offsets_loaded, 895 &osd->illegal_bytes, &nsd->illegal_bytes); 896 897 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 898 pf->stat_offsets_loaded, 899 &osd->mac_local_faults, 900 &nsd->mac_local_faults); 901 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 902 pf->stat_offsets_loaded, 903 &osd->mac_remote_faults, 904 &nsd->mac_remote_faults); 905 906 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 907 pf->stat_offsets_loaded, 908 &osd->rx_length_errors, 909 &nsd->rx_length_errors); 910 911 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 912 pf->stat_offsets_loaded, 913 &osd->link_xon_rx, &nsd->link_xon_rx); 914 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 915 pf->stat_offsets_loaded, 916 &osd->link_xon_tx, &nsd->link_xon_tx); 917 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 918 pf->stat_offsets_loaded, 919 &osd->link_xoff_rx, &nsd->link_xoff_rx); 920 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 921 pf->stat_offsets_loaded, 922 &osd->link_xoff_tx, &nsd->link_xoff_tx); 923 924 for (i = 0; i < 8; i++) { 925 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 926 pf->stat_offsets_loaded, 927 &osd->priority_xoff_rx[i], 928 &nsd->priority_xoff_rx[i]); 929 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 930 pf->stat_offsets_loaded, 931 &osd->priority_xon_rx[i], 932 &nsd->priority_xon_rx[i]); 933 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 934 pf->stat_offsets_loaded, 935 &osd->priority_xon_tx[i], 936 &nsd->priority_xon_tx[i]); 937 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 938 pf->stat_offsets_loaded, 939 &osd->priority_xoff_tx[i], 940 &nsd->priority_xoff_tx[i]); 941 i40e_stat_update32(hw, 942 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 943 pf->stat_offsets_loaded, 944 &osd->priority_xon_2_xoff[i], 945 &nsd->priority_xon_2_xoff[i]); 946 } 947 948 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 949 I40E_GLPRT_PRC64L(hw->port), 950 pf->stat_offsets_loaded, 951 &osd->rx_size_64, &nsd->rx_size_64); 952 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 953 I40E_GLPRT_PRC127L(hw->port), 954 pf->stat_offsets_loaded, 955 &osd->rx_size_127, &nsd->rx_size_127); 956 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 957 I40E_GLPRT_PRC255L(hw->port), 958 pf->stat_offsets_loaded, 959 &osd->rx_size_255, &nsd->rx_size_255); 960 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 961 I40E_GLPRT_PRC511L(hw->port), 962 pf->stat_offsets_loaded, 963 &osd->rx_size_511, &nsd->rx_size_511); 964 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 965 I40E_GLPRT_PRC1023L(hw->port), 966 pf->stat_offsets_loaded, 967 &osd->rx_size_1023, &nsd->rx_size_1023); 968 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 969 I40E_GLPRT_PRC1522L(hw->port), 970 pf->stat_offsets_loaded, 971 &osd->rx_size_1522, &nsd->rx_size_1522); 972 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 973 I40E_GLPRT_PRC9522L(hw->port), 974 pf->stat_offsets_loaded, 975 &osd->rx_size_big, &nsd->rx_size_big); 976 977 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 978 I40E_GLPRT_PTC64L(hw->port), 979 pf->stat_offsets_loaded, 980 &osd->tx_size_64, &nsd->tx_size_64); 981 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 982 I40E_GLPRT_PTC127L(hw->port), 983 pf->stat_offsets_loaded, 984 &osd->tx_size_127, &nsd->tx_size_127); 985 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 986 I40E_GLPRT_PTC255L(hw->port), 987 pf->stat_offsets_loaded, 988 &osd->tx_size_255, &nsd->tx_size_255); 989 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 990 I40E_GLPRT_PTC511L(hw->port), 991 pf->stat_offsets_loaded, 992 &osd->tx_size_511, &nsd->tx_size_511); 993 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 994 I40E_GLPRT_PTC1023L(hw->port), 995 pf->stat_offsets_loaded, 996 &osd->tx_size_1023, &nsd->tx_size_1023); 997 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 998 I40E_GLPRT_PTC1522L(hw->port), 999 pf->stat_offsets_loaded, 1000 &osd->tx_size_1522, &nsd->tx_size_1522); 1001 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 1002 I40E_GLPRT_PTC9522L(hw->port), 1003 pf->stat_offsets_loaded, 1004 &osd->tx_size_big, &nsd->tx_size_big); 1005 1006 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1007 pf->stat_offsets_loaded, 1008 &osd->rx_undersize, &nsd->rx_undersize); 1009 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1010 pf->stat_offsets_loaded, 1011 &osd->rx_fragments, &nsd->rx_fragments); 1012 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1013 pf->stat_offsets_loaded, 1014 &osd->rx_oversize, &nsd->rx_oversize); 1015 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1016 pf->stat_offsets_loaded, 1017 &osd->rx_jabber, &nsd->rx_jabber); 1018 1019 /* FDIR stats */ 1020 i40e_stat_update32(hw, 1021 I40E_GLQF_PCNT(I40E_FD_ATR_STAT_IDX(pf->hw.pf_id)), 1022 pf->stat_offsets_loaded, 1023 &osd->fd_atr_match, &nsd->fd_atr_match); 1024 i40e_stat_update32(hw, 1025 I40E_GLQF_PCNT(I40E_FD_SB_STAT_IDX(pf->hw.pf_id)), 1026 pf->stat_offsets_loaded, 1027 &osd->fd_sb_match, &nsd->fd_sb_match); 1028 i40e_stat_update32(hw, 1029 I40E_GLQF_PCNT(I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id)), 1030 pf->stat_offsets_loaded, 1031 &osd->fd_atr_tunnel_match, &nsd->fd_atr_tunnel_match); 1032 1033 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1034 nsd->tx_lpi_status = 1035 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1036 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1037 nsd->rx_lpi_status = 1038 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1039 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1040 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1041 pf->stat_offsets_loaded, 1042 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1043 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1044 pf->stat_offsets_loaded, 1045 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1046 1047 if (pf->flags & I40E_FLAG_FD_SB_ENABLED && 1048 !(pf->hw_disabled_flags & I40E_FLAG_FD_SB_ENABLED)) 1049 nsd->fd_sb_status = true; 1050 else 1051 nsd->fd_sb_status = false; 1052 1053 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED && 1054 !(pf->hw_disabled_flags & I40E_FLAG_FD_ATR_ENABLED)) 1055 nsd->fd_atr_status = true; 1056 else 1057 nsd->fd_atr_status = false; 1058 1059 pf->stat_offsets_loaded = true; 1060 } 1061 1062 /** 1063 * i40e_update_stats - Update the various statistics counters. 1064 * @vsi: the VSI to be updated 1065 * 1066 * Update the various stats for this VSI and its related entities. 1067 **/ 1068 void i40e_update_stats(struct i40e_vsi *vsi) 1069 { 1070 struct i40e_pf *pf = vsi->back; 1071 1072 if (vsi == pf->vsi[pf->lan_vsi]) 1073 i40e_update_pf_stats(pf); 1074 1075 i40e_update_vsi_stats(vsi); 1076 } 1077 1078 /** 1079 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1080 * @vsi: the VSI to be searched 1081 * @macaddr: the MAC address 1082 * @vlan: the vlan 1083 * 1084 * Returns ptr to the filter object or NULL 1085 **/ 1086 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1087 const u8 *macaddr, s16 vlan) 1088 { 1089 struct i40e_mac_filter *f; 1090 u64 key; 1091 1092 if (!vsi || !macaddr) 1093 return NULL; 1094 1095 key = i40e_addr_to_hkey(macaddr); 1096 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1097 if ((ether_addr_equal(macaddr, f->macaddr)) && 1098 (vlan == f->vlan)) 1099 return f; 1100 } 1101 return NULL; 1102 } 1103 1104 /** 1105 * i40e_find_mac - Find a mac addr in the macvlan filters list 1106 * @vsi: the VSI to be searched 1107 * @macaddr: the MAC address we are searching for 1108 * 1109 * Returns the first filter with the provided MAC address or NULL if 1110 * MAC address was not found 1111 **/ 1112 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, const u8 *macaddr) 1113 { 1114 struct i40e_mac_filter *f; 1115 u64 key; 1116 1117 if (!vsi || !macaddr) 1118 return NULL; 1119 1120 key = i40e_addr_to_hkey(macaddr); 1121 hash_for_each_possible(vsi->mac_filter_hash, f, hlist, key) { 1122 if ((ether_addr_equal(macaddr, f->macaddr))) 1123 return f; 1124 } 1125 return NULL; 1126 } 1127 1128 /** 1129 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1130 * @vsi: the VSI to be searched 1131 * 1132 * Returns true if VSI is in vlan mode or false otherwise 1133 **/ 1134 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1135 { 1136 /* If we have a PVID, always operate in VLAN mode */ 1137 if (vsi->info.pvid) 1138 return true; 1139 1140 /* We need to operate in VLAN mode whenever we have any filters with 1141 * a VLAN other than I40E_VLAN_ALL. We could check the table each 1142 * time, incurring search cost repeatedly. However, we can notice two 1143 * things: 1144 * 1145 * 1) the only place where we can gain a VLAN filter is in 1146 * i40e_add_filter. 1147 * 1148 * 2) the only place where filters are actually removed is in 1149 * i40e_sync_filters_subtask. 1150 * 1151 * Thus, we can simply use a boolean value, has_vlan_filters which we 1152 * will set to true when we add a VLAN filter in i40e_add_filter. Then 1153 * we have to perform the full search after deleting filters in 1154 * i40e_sync_filters_subtask, but we already have to search 1155 * filters here and can perform the check at the same time. This 1156 * results in avoiding embedding a loop for VLAN mode inside another 1157 * loop over all the filters, and should maintain correctness as noted 1158 * above. 1159 */ 1160 return vsi->has_vlan_filter; 1161 } 1162 1163 /** 1164 * i40e_correct_mac_vlan_filters - Correct non-VLAN filters if necessary 1165 * @vsi: the VSI to configure 1166 * @tmp_add_list: list of filters ready to be added 1167 * @tmp_del_list: list of filters ready to be deleted 1168 * @vlan_filters: the number of active VLAN filters 1169 * 1170 * Update VLAN=0 and VLAN=-1 (I40E_VLAN_ANY) filters properly so that they 1171 * behave as expected. If we have any active VLAN filters remaining or about 1172 * to be added then we need to update non-VLAN filters to be marked as VLAN=0 1173 * so that they only match against untagged traffic. If we no longer have any 1174 * active VLAN filters, we need to make all non-VLAN filters marked as VLAN=-1 1175 * so that they match against both tagged and untagged traffic. In this way, 1176 * we ensure that we correctly receive the desired traffic. This ensures that 1177 * when we have an active VLAN we will receive only untagged traffic and 1178 * traffic matching active VLANs. If we have no active VLANs then we will 1179 * operate in non-VLAN mode and receive all traffic, tagged or untagged. 1180 * 1181 * Finally, in a similar fashion, this function also corrects filters when 1182 * there is an active PVID assigned to this VSI. 1183 * 1184 * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. 1185 * 1186 * This function is only expected to be called from within 1187 * i40e_sync_vsi_filters. 1188 * 1189 * NOTE: This function expects to be called while under the 1190 * mac_filter_hash_lock 1191 */ 1192 static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, 1193 struct hlist_head *tmp_add_list, 1194 struct hlist_head *tmp_del_list, 1195 int vlan_filters) 1196 { 1197 s16 pvid = le16_to_cpu(vsi->info.pvid); 1198 struct i40e_mac_filter *f, *add_head; 1199 struct i40e_new_mac_filter *new; 1200 struct hlist_node *h; 1201 int bkt, new_vlan; 1202 1203 /* To determine if a particular filter needs to be replaced we 1204 * have the three following conditions: 1205 * 1206 * a) if we have a PVID assigned, then all filters which are 1207 * not marked as VLAN=PVID must be replaced with filters that 1208 * are. 1209 * b) otherwise, if we have any active VLANS, all filters 1210 * which are marked as VLAN=-1 must be replaced with 1211 * filters marked as VLAN=0 1212 * c) finally, if we do not have any active VLANS, all filters 1213 * which are marked as VLAN=0 must be replaced with filters 1214 * marked as VLAN=-1 1215 */ 1216 1217 /* Update the filters about to be added in place */ 1218 hlist_for_each_entry(new, tmp_add_list, hlist) { 1219 if (pvid && new->f->vlan != pvid) 1220 new->f->vlan = pvid; 1221 else if (vlan_filters && new->f->vlan == I40E_VLAN_ANY) 1222 new->f->vlan = 0; 1223 else if (!vlan_filters && new->f->vlan == 0) 1224 new->f->vlan = I40E_VLAN_ANY; 1225 } 1226 1227 /* Update the remaining active filters */ 1228 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1229 /* Combine the checks for whether a filter needs to be changed 1230 * and then determine the new VLAN inside the if block, in 1231 * order to avoid duplicating code for adding the new filter 1232 * then deleting the old filter. 1233 */ 1234 if ((pvid && f->vlan != pvid) || 1235 (vlan_filters && f->vlan == I40E_VLAN_ANY) || 1236 (!vlan_filters && f->vlan == 0)) { 1237 /* Determine the new vlan we will be adding */ 1238 if (pvid) 1239 new_vlan = pvid; 1240 else if (vlan_filters) 1241 new_vlan = 0; 1242 else 1243 new_vlan = I40E_VLAN_ANY; 1244 1245 /* Create the new filter */ 1246 add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); 1247 if (!add_head) 1248 return -ENOMEM; 1249 1250 /* Create a temporary i40e_new_mac_filter */ 1251 new = kzalloc(sizeof(*new), GFP_ATOMIC); 1252 if (!new) 1253 return -ENOMEM; 1254 1255 new->f = add_head; 1256 new->state = add_head->state; 1257 1258 /* Add the new filter to the tmp list */ 1259 hlist_add_head(&new->hlist, tmp_add_list); 1260 1261 /* Put the original filter into the delete list */ 1262 f->state = I40E_FILTER_REMOVE; 1263 hash_del(&f->hlist); 1264 hlist_add_head(&f->hlist, tmp_del_list); 1265 } 1266 } 1267 1268 vsi->has_vlan_filter = !!vlan_filters; 1269 1270 return 0; 1271 } 1272 1273 /** 1274 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1275 * @vsi: the PF Main VSI - inappropriate for any other VSI 1276 * @macaddr: the MAC address 1277 * 1278 * Remove whatever filter the firmware set up so the driver can manage 1279 * its own filtering intelligently. 1280 **/ 1281 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1282 { 1283 struct i40e_aqc_remove_macvlan_element_data element; 1284 struct i40e_pf *pf = vsi->back; 1285 1286 /* Only appropriate for the PF main VSI */ 1287 if (vsi->type != I40E_VSI_MAIN) 1288 return; 1289 1290 memset(&element, 0, sizeof(element)); 1291 ether_addr_copy(element.mac_addr, macaddr); 1292 element.vlan_tag = 0; 1293 /* Ignore error returns, some firmware does it this way... */ 1294 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1295 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1296 1297 memset(&element, 0, sizeof(element)); 1298 ether_addr_copy(element.mac_addr, macaddr); 1299 element.vlan_tag = 0; 1300 /* ...and some firmware does it this way. */ 1301 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1302 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1303 i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1304 } 1305 1306 /** 1307 * i40e_add_filter - Add a mac/vlan filter to the VSI 1308 * @vsi: the VSI to be searched 1309 * @macaddr: the MAC address 1310 * @vlan: the vlan 1311 * 1312 * Returns ptr to the filter object or NULL when no memory available. 1313 * 1314 * NOTE: This function is expected to be called with mac_filter_hash_lock 1315 * being held. 1316 **/ 1317 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1318 const u8 *macaddr, s16 vlan) 1319 { 1320 struct i40e_mac_filter *f; 1321 u64 key; 1322 1323 if (!vsi || !macaddr) 1324 return NULL; 1325 1326 f = i40e_find_filter(vsi, macaddr, vlan); 1327 if (!f) { 1328 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1329 if (!f) 1330 return NULL; 1331 1332 /* Update the boolean indicating if we need to function in 1333 * VLAN mode. 1334 */ 1335 if (vlan >= 0) 1336 vsi->has_vlan_filter = true; 1337 1338 ether_addr_copy(f->macaddr, macaddr); 1339 f->vlan = vlan; 1340 /* If we're in overflow promisc mode, set the state directly 1341 * to failed, so we don't bother to try sending the filter 1342 * to the hardware. 1343 */ 1344 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state)) 1345 f->state = I40E_FILTER_FAILED; 1346 else 1347 f->state = I40E_FILTER_NEW; 1348 INIT_HLIST_NODE(&f->hlist); 1349 1350 key = i40e_addr_to_hkey(macaddr); 1351 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1352 1353 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1354 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1355 } 1356 1357 /* If we're asked to add a filter that has been marked for removal, it 1358 * is safe to simply restore it to active state. __i40e_del_filter 1359 * will have simply deleted any filters which were previously marked 1360 * NEW or FAILED, so if it is currently marked REMOVE it must have 1361 * previously been ACTIVE. Since we haven't yet run the sync filters 1362 * task, just restore this filter to the ACTIVE state so that the 1363 * sync task leaves it in place 1364 */ 1365 if (f->state == I40E_FILTER_REMOVE) 1366 f->state = I40E_FILTER_ACTIVE; 1367 1368 return f; 1369 } 1370 1371 /** 1372 * __i40e_del_filter - Remove a specific filter from the VSI 1373 * @vsi: VSI to remove from 1374 * @f: the filter to remove from the list 1375 * 1376 * This function should be called instead of i40e_del_filter only if you know 1377 * the exact filter you will remove already, such as via i40e_find_filter or 1378 * i40e_find_mac. 1379 * 1380 * NOTE: This function is expected to be called with mac_filter_hash_lock 1381 * being held. 1382 * ANOTHER NOTE: This function MUST be called from within the context of 1383 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1384 * instead of list_for_each_entry(). 1385 **/ 1386 void __i40e_del_filter(struct i40e_vsi *vsi, struct i40e_mac_filter *f) 1387 { 1388 if (!f) 1389 return; 1390 1391 /* If the filter was never added to firmware then we can just delete it 1392 * directly and we don't want to set the status to remove or else an 1393 * admin queue command will unnecessarily fire. 1394 */ 1395 if ((f->state == I40E_FILTER_FAILED) || 1396 (f->state == I40E_FILTER_NEW)) { 1397 hash_del(&f->hlist); 1398 kfree(f); 1399 } else { 1400 f->state = I40E_FILTER_REMOVE; 1401 } 1402 1403 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1404 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1405 } 1406 1407 /** 1408 * i40e_del_filter - Remove a MAC/VLAN filter from the VSI 1409 * @vsi: the VSI to be searched 1410 * @macaddr: the MAC address 1411 * @vlan: the VLAN 1412 * 1413 * NOTE: This function is expected to be called with mac_filter_hash_lock 1414 * being held. 1415 * ANOTHER NOTE: This function MUST be called from within the context of 1416 * the "safe" variants of any list iterators, e.g. list_for_each_entry_safe() 1417 * instead of list_for_each_entry(). 1418 **/ 1419 void i40e_del_filter(struct i40e_vsi *vsi, const u8 *macaddr, s16 vlan) 1420 { 1421 struct i40e_mac_filter *f; 1422 1423 if (!vsi || !macaddr) 1424 return; 1425 1426 f = i40e_find_filter(vsi, macaddr, vlan); 1427 __i40e_del_filter(vsi, f); 1428 } 1429 1430 /** 1431 * i40e_add_mac_filter - Add a MAC filter for all active VLANs 1432 * @vsi: the VSI to be searched 1433 * @macaddr: the mac address to be filtered 1434 * 1435 * If we're not in VLAN mode, just add the filter to I40E_VLAN_ANY. Otherwise, 1436 * go through all the macvlan filters and add a macvlan filter for each 1437 * unique vlan that already exists. If a PVID has been assigned, instead only 1438 * add the macaddr to that VLAN. 1439 * 1440 * Returns last filter added on success, else NULL 1441 **/ 1442 struct i40e_mac_filter *i40e_add_mac_filter(struct i40e_vsi *vsi, 1443 const u8 *macaddr) 1444 { 1445 struct i40e_mac_filter *f, *add = NULL; 1446 struct hlist_node *h; 1447 int bkt; 1448 1449 if (vsi->info.pvid) 1450 return i40e_add_filter(vsi, macaddr, 1451 le16_to_cpu(vsi->info.pvid)); 1452 1453 if (!i40e_is_vsi_in_vlan(vsi)) 1454 return i40e_add_filter(vsi, macaddr, I40E_VLAN_ANY); 1455 1456 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1457 if (f->state == I40E_FILTER_REMOVE) 1458 continue; 1459 add = i40e_add_filter(vsi, macaddr, f->vlan); 1460 if (!add) 1461 return NULL; 1462 } 1463 1464 return add; 1465 } 1466 1467 /** 1468 * i40e_del_mac_filter - Remove a MAC filter from all VLANs 1469 * @vsi: the VSI to be searched 1470 * @macaddr: the mac address to be removed 1471 * 1472 * Removes a given MAC address from a VSI regardless of what VLAN it has been 1473 * associated with. 1474 * 1475 * Returns 0 for success, or error 1476 **/ 1477 int i40e_del_mac_filter(struct i40e_vsi *vsi, const u8 *macaddr) 1478 { 1479 struct i40e_mac_filter *f; 1480 struct hlist_node *h; 1481 bool found = false; 1482 int bkt; 1483 1484 WARN(!spin_is_locked(&vsi->mac_filter_hash_lock), 1485 "Missing mac_filter_hash_lock\n"); 1486 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 1487 if (ether_addr_equal(macaddr, f->macaddr)) { 1488 __i40e_del_filter(vsi, f); 1489 found = true; 1490 } 1491 } 1492 1493 if (found) 1494 return 0; 1495 else 1496 return -ENOENT; 1497 } 1498 1499 /** 1500 * i40e_set_mac - NDO callback to set mac address 1501 * @netdev: network interface device structure 1502 * @p: pointer to an address structure 1503 * 1504 * Returns 0 on success, negative on failure 1505 **/ 1506 static int i40e_set_mac(struct net_device *netdev, void *p) 1507 { 1508 struct i40e_netdev_priv *np = netdev_priv(netdev); 1509 struct i40e_vsi *vsi = np->vsi; 1510 struct i40e_pf *pf = vsi->back; 1511 struct i40e_hw *hw = &pf->hw; 1512 struct sockaddr *addr = p; 1513 1514 if (!is_valid_ether_addr(addr->sa_data)) 1515 return -EADDRNOTAVAIL; 1516 1517 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) { 1518 netdev_info(netdev, "already using mac address %pM\n", 1519 addr->sa_data); 1520 return 0; 1521 } 1522 1523 if (test_bit(__I40E_DOWN, &vsi->back->state) || 1524 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 1525 return -EADDRNOTAVAIL; 1526 1527 if (ether_addr_equal(hw->mac.addr, addr->sa_data)) 1528 netdev_info(netdev, "returning to hw mac address %pM\n", 1529 hw->mac.addr); 1530 else 1531 netdev_info(netdev, "set new mac address %pM\n", addr->sa_data); 1532 1533 spin_lock_bh(&vsi->mac_filter_hash_lock); 1534 i40e_del_mac_filter(vsi, netdev->dev_addr); 1535 i40e_add_mac_filter(vsi, addr->sa_data); 1536 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1537 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1538 if (vsi->type == I40E_VSI_MAIN) { 1539 i40e_status ret; 1540 1541 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1542 I40E_AQC_WRITE_TYPE_LAA_WOL, 1543 addr->sa_data, NULL); 1544 if (ret) 1545 netdev_info(netdev, "Ignoring error from firmware on LAA update, status %s, AQ ret %s\n", 1546 i40e_stat_str(hw, ret), 1547 i40e_aq_str(hw, hw->aq.asq_last_status)); 1548 } 1549 1550 /* schedule our worker thread which will take care of 1551 * applying the new filter changes 1552 */ 1553 i40e_service_event_schedule(vsi->back); 1554 return 0; 1555 } 1556 1557 /** 1558 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1559 * @vsi: the VSI being setup 1560 * @ctxt: VSI context structure 1561 * @enabled_tc: Enabled TCs bitmap 1562 * @is_add: True if called before Add VSI 1563 * 1564 * Setup VSI queue mapping for enabled traffic classes. 1565 **/ 1566 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1567 struct i40e_vsi_context *ctxt, 1568 u8 enabled_tc, 1569 bool is_add) 1570 { 1571 struct i40e_pf *pf = vsi->back; 1572 u16 sections = 0; 1573 u8 netdev_tc = 0; 1574 u16 numtc = 0; 1575 u16 qcount; 1576 u8 offset; 1577 u16 qmap; 1578 int i; 1579 u16 num_tc_qps = 0; 1580 1581 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1582 offset = 0; 1583 1584 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1585 /* Find numtc from enabled TC bitmap */ 1586 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1587 if (enabled_tc & BIT(i)) /* TC is enabled */ 1588 numtc++; 1589 } 1590 if (!numtc) { 1591 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1592 numtc = 1; 1593 } 1594 } else { 1595 /* At least TC0 is enabled in case of non-DCB case */ 1596 numtc = 1; 1597 } 1598 1599 vsi->tc_config.numtc = numtc; 1600 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1601 /* Number of queues per enabled TC */ 1602 qcount = vsi->alloc_queue_pairs; 1603 1604 num_tc_qps = qcount / numtc; 1605 num_tc_qps = min_t(int, num_tc_qps, i40e_pf_get_max_q_per_tc(pf)); 1606 1607 /* Setup queue offset/count for all TCs for given VSI */ 1608 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1609 /* See if the given TC is enabled for the given VSI */ 1610 if (vsi->tc_config.enabled_tc & BIT(i)) { 1611 /* TC is enabled */ 1612 int pow, num_qps; 1613 1614 switch (vsi->type) { 1615 case I40E_VSI_MAIN: 1616 qcount = min_t(int, pf->alloc_rss_size, 1617 num_tc_qps); 1618 break; 1619 case I40E_VSI_FDIR: 1620 case I40E_VSI_SRIOV: 1621 case I40E_VSI_VMDQ2: 1622 default: 1623 qcount = num_tc_qps; 1624 WARN_ON(i != 0); 1625 break; 1626 } 1627 vsi->tc_config.tc_info[i].qoffset = offset; 1628 vsi->tc_config.tc_info[i].qcount = qcount; 1629 1630 /* find the next higher power-of-2 of num queue pairs */ 1631 num_qps = qcount; 1632 pow = 0; 1633 while (num_qps && (BIT_ULL(pow) < qcount)) { 1634 pow++; 1635 num_qps >>= 1; 1636 } 1637 1638 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1639 qmap = 1640 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1641 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1642 1643 offset += qcount; 1644 } else { 1645 /* TC is not enabled so set the offset to 1646 * default queue and allocate one queue 1647 * for the given TC. 1648 */ 1649 vsi->tc_config.tc_info[i].qoffset = 0; 1650 vsi->tc_config.tc_info[i].qcount = 1; 1651 vsi->tc_config.tc_info[i].netdev_tc = 0; 1652 1653 qmap = 0; 1654 } 1655 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1656 } 1657 1658 /* Set actual Tx/Rx queue pairs */ 1659 vsi->num_queue_pairs = offset; 1660 if ((vsi->type == I40E_VSI_MAIN) && (numtc == 1)) { 1661 if (vsi->req_queue_pairs > 0) 1662 vsi->num_queue_pairs = vsi->req_queue_pairs; 1663 else if (pf->flags & I40E_FLAG_MSIX_ENABLED) 1664 vsi->num_queue_pairs = pf->num_lan_msix; 1665 } 1666 1667 /* Scheduler section valid can only be set for ADD VSI */ 1668 if (is_add) { 1669 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1670 1671 ctxt->info.up_enable_bits = enabled_tc; 1672 } 1673 if (vsi->type == I40E_VSI_SRIOV) { 1674 ctxt->info.mapping_flags |= 1675 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1676 for (i = 0; i < vsi->num_queue_pairs; i++) 1677 ctxt->info.queue_mapping[i] = 1678 cpu_to_le16(vsi->base_queue + i); 1679 } else { 1680 ctxt->info.mapping_flags |= 1681 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1682 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1683 } 1684 ctxt->info.valid_sections |= cpu_to_le16(sections); 1685 } 1686 1687 /** 1688 * i40e_addr_sync - Callback for dev_(mc|uc)_sync to add address 1689 * @netdev: the netdevice 1690 * @addr: address to add 1691 * 1692 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call 1693 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1694 */ 1695 static int i40e_addr_sync(struct net_device *netdev, const u8 *addr) 1696 { 1697 struct i40e_netdev_priv *np = netdev_priv(netdev); 1698 struct i40e_vsi *vsi = np->vsi; 1699 1700 if (i40e_add_mac_filter(vsi, addr)) 1701 return 0; 1702 else 1703 return -ENOMEM; 1704 } 1705 1706 /** 1707 * i40e_addr_unsync - Callback for dev_(mc|uc)_sync to remove address 1708 * @netdev: the netdevice 1709 * @addr: address to add 1710 * 1711 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call 1712 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock. 1713 */ 1714 static int i40e_addr_unsync(struct net_device *netdev, const u8 *addr) 1715 { 1716 struct i40e_netdev_priv *np = netdev_priv(netdev); 1717 struct i40e_vsi *vsi = np->vsi; 1718 1719 i40e_del_mac_filter(vsi, addr); 1720 1721 return 0; 1722 } 1723 1724 /** 1725 * i40e_set_rx_mode - NDO callback to set the netdev filters 1726 * @netdev: network interface device structure 1727 **/ 1728 static void i40e_set_rx_mode(struct net_device *netdev) 1729 { 1730 struct i40e_netdev_priv *np = netdev_priv(netdev); 1731 struct i40e_vsi *vsi = np->vsi; 1732 1733 spin_lock_bh(&vsi->mac_filter_hash_lock); 1734 1735 __dev_uc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1736 __dev_mc_sync(netdev, i40e_addr_sync, i40e_addr_unsync); 1737 1738 spin_unlock_bh(&vsi->mac_filter_hash_lock); 1739 1740 /* check for other flag changes */ 1741 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1742 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1743 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1744 } 1745 1746 /* schedule our worker thread which will take care of 1747 * applying the new filter changes 1748 */ 1749 i40e_service_event_schedule(vsi->back); 1750 } 1751 1752 /** 1753 * i40e_undo_del_filter_entries - Undo the changes made to MAC filter entries 1754 * @vsi: Pointer to VSI struct 1755 * @from: Pointer to list which contains MAC filter entries - changes to 1756 * those entries needs to be undone. 1757 * 1758 * MAC filter entries from this list were slated for deletion. 1759 **/ 1760 static void i40e_undo_del_filter_entries(struct i40e_vsi *vsi, 1761 struct hlist_head *from) 1762 { 1763 struct i40e_mac_filter *f; 1764 struct hlist_node *h; 1765 1766 hlist_for_each_entry_safe(f, h, from, hlist) { 1767 u64 key = i40e_addr_to_hkey(f->macaddr); 1768 1769 /* Move the element back into MAC filter list*/ 1770 hlist_del(&f->hlist); 1771 hash_add(vsi->mac_filter_hash, &f->hlist, key); 1772 } 1773 } 1774 1775 /** 1776 * i40e_undo_add_filter_entries - Undo the changes made to MAC filter entries 1777 * @vsi: Pointer to vsi struct 1778 * @from: Pointer to list which contains MAC filter entries - changes to 1779 * those entries needs to be undone. 1780 * 1781 * MAC filter entries from this list were slated for addition. 1782 **/ 1783 static void i40e_undo_add_filter_entries(struct i40e_vsi *vsi, 1784 struct hlist_head *from) 1785 { 1786 struct i40e_new_mac_filter *new; 1787 struct hlist_node *h; 1788 1789 hlist_for_each_entry_safe(new, h, from, hlist) { 1790 /* We can simply free the wrapper structure */ 1791 hlist_del(&new->hlist); 1792 kfree(new); 1793 } 1794 } 1795 1796 /** 1797 * i40e_next_entry - Get the next non-broadcast filter from a list 1798 * @next: pointer to filter in list 1799 * 1800 * Returns the next non-broadcast filter in the list. Required so that we 1801 * ignore broadcast filters within the list, since these are not handled via 1802 * the normal firmware update path. 1803 */ 1804 static 1805 struct i40e_new_mac_filter *i40e_next_filter(struct i40e_new_mac_filter *next) 1806 { 1807 hlist_for_each_entry_continue(next, hlist) { 1808 if (!is_broadcast_ether_addr(next->f->macaddr)) 1809 return next; 1810 } 1811 1812 return NULL; 1813 } 1814 1815 /** 1816 * i40e_update_filter_state - Update filter state based on return data 1817 * from firmware 1818 * @count: Number of filters added 1819 * @add_list: return data from fw 1820 * @head: pointer to first filter in current batch 1821 * 1822 * MAC filter entries from list were slated to be added to device. Returns 1823 * number of successful filters. Note that 0 does NOT mean success! 1824 **/ 1825 static int 1826 i40e_update_filter_state(int count, 1827 struct i40e_aqc_add_macvlan_element_data *add_list, 1828 struct i40e_new_mac_filter *add_head) 1829 { 1830 int retval = 0; 1831 int i; 1832 1833 for (i = 0; i < count; i++) { 1834 /* Always check status of each filter. We don't need to check 1835 * the firmware return status because we pre-set the filter 1836 * status to I40E_AQC_MM_ERR_NO_RES when sending the filter 1837 * request to the adminq. Thus, if it no longer matches then 1838 * we know the filter is active. 1839 */ 1840 if (add_list[i].match_method == I40E_AQC_MM_ERR_NO_RES) { 1841 add_head->state = I40E_FILTER_FAILED; 1842 } else { 1843 add_head->state = I40E_FILTER_ACTIVE; 1844 retval++; 1845 } 1846 1847 add_head = i40e_next_filter(add_head); 1848 if (!add_head) 1849 break; 1850 } 1851 1852 return retval; 1853 } 1854 1855 /** 1856 * i40e_aqc_del_filters - Request firmware to delete a set of filters 1857 * @vsi: ptr to the VSI 1858 * @vsi_name: name to display in messages 1859 * @list: the list of filters to send to firmware 1860 * @num_del: the number of filters to delete 1861 * @retval: Set to -EIO on failure to delete 1862 * 1863 * Send a request to firmware via AdminQ to delete a set of filters. Uses 1864 * *retval instead of a return value so that success does not force ret_val to 1865 * be set to 0. This ensures that a sequence of calls to this function 1866 * preserve the previous value of *retval on successful delete. 1867 */ 1868 static 1869 void i40e_aqc_del_filters(struct i40e_vsi *vsi, const char *vsi_name, 1870 struct i40e_aqc_remove_macvlan_element_data *list, 1871 int num_del, int *retval) 1872 { 1873 struct i40e_hw *hw = &vsi->back->hw; 1874 i40e_status aq_ret; 1875 int aq_err; 1876 1877 aq_ret = i40e_aq_remove_macvlan(hw, vsi->seid, list, num_del, NULL); 1878 aq_err = hw->aq.asq_last_status; 1879 1880 /* Explicitly ignore and do not report when firmware returns ENOENT */ 1881 if (aq_ret && !(aq_err == I40E_AQ_RC_ENOENT)) { 1882 *retval = -EIO; 1883 dev_info(&vsi->back->pdev->dev, 1884 "ignoring delete macvlan error on %s, err %s, aq_err %s\n", 1885 vsi_name, i40e_stat_str(hw, aq_ret), 1886 i40e_aq_str(hw, aq_err)); 1887 } 1888 } 1889 1890 /** 1891 * i40e_aqc_add_filters - Request firmware to add a set of filters 1892 * @vsi: ptr to the VSI 1893 * @vsi_name: name to display in messages 1894 * @list: the list of filters to send to firmware 1895 * @add_head: Position in the add hlist 1896 * @num_add: the number of filters to add 1897 * @promisc_change: set to true on exit if promiscuous mode was forced on 1898 * 1899 * Send a request to firmware via AdminQ to add a chunk of filters. Will set 1900 * promisc_changed to true if the firmware has run out of space for more 1901 * filters. 1902 */ 1903 static 1904 void i40e_aqc_add_filters(struct i40e_vsi *vsi, const char *vsi_name, 1905 struct i40e_aqc_add_macvlan_element_data *list, 1906 struct i40e_new_mac_filter *add_head, 1907 int num_add, bool *promisc_changed) 1908 { 1909 struct i40e_hw *hw = &vsi->back->hw; 1910 int aq_err, fcnt; 1911 1912 i40e_aq_add_macvlan(hw, vsi->seid, list, num_add, NULL); 1913 aq_err = hw->aq.asq_last_status; 1914 fcnt = i40e_update_filter_state(num_add, list, add_head); 1915 1916 if (fcnt != num_add) { 1917 *promisc_changed = true; 1918 set_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 1919 dev_warn(&vsi->back->pdev->dev, 1920 "Error %s adding RX filters on %s, promiscuous mode forced on\n", 1921 i40e_aq_str(hw, aq_err), 1922 vsi_name); 1923 } 1924 } 1925 1926 /** 1927 * i40e_aqc_broadcast_filter - Set promiscuous broadcast flags 1928 * @vsi: pointer to the VSI 1929 * @f: filter data 1930 * 1931 * This function sets or clears the promiscuous broadcast flags for VLAN 1932 * filters in order to properly receive broadcast frames. Assumes that only 1933 * broadcast filters are passed. 1934 * 1935 * Returns status indicating success or failure; 1936 **/ 1937 static i40e_status 1938 i40e_aqc_broadcast_filter(struct i40e_vsi *vsi, const char *vsi_name, 1939 struct i40e_mac_filter *f) 1940 { 1941 bool enable = f->state == I40E_FILTER_NEW; 1942 struct i40e_hw *hw = &vsi->back->hw; 1943 i40e_status aq_ret; 1944 1945 if (f->vlan == I40E_VLAN_ANY) { 1946 aq_ret = i40e_aq_set_vsi_broadcast(hw, 1947 vsi->seid, 1948 enable, 1949 NULL); 1950 } else { 1951 aq_ret = i40e_aq_set_vsi_bc_promisc_on_vlan(hw, 1952 vsi->seid, 1953 enable, 1954 f->vlan, 1955 NULL); 1956 } 1957 1958 if (aq_ret) 1959 dev_warn(&vsi->back->pdev->dev, 1960 "Error %s setting broadcast promiscuous mode on %s\n", 1961 i40e_aq_str(hw, hw->aq.asq_last_status), 1962 vsi_name); 1963 1964 return aq_ret; 1965 } 1966 1967 /** 1968 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 1969 * @vsi: ptr to the VSI 1970 * 1971 * Push any outstanding VSI filter changes through the AdminQ. 1972 * 1973 * Returns 0 or error value 1974 **/ 1975 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 1976 { 1977 struct hlist_head tmp_add_list, tmp_del_list; 1978 struct i40e_mac_filter *f; 1979 struct i40e_new_mac_filter *new, *add_head = NULL; 1980 struct i40e_hw *hw = &vsi->back->hw; 1981 unsigned int failed_filters = 0; 1982 unsigned int vlan_filters = 0; 1983 bool promisc_changed = false; 1984 char vsi_name[16] = "PF"; 1985 int filter_list_len = 0; 1986 i40e_status aq_ret = 0; 1987 u32 changed_flags = 0; 1988 struct hlist_node *h; 1989 struct i40e_pf *pf; 1990 int num_add = 0; 1991 int num_del = 0; 1992 int retval = 0; 1993 u16 cmd_flags; 1994 int list_size; 1995 int bkt; 1996 1997 /* empty array typed pointers, kcalloc later */ 1998 struct i40e_aqc_add_macvlan_element_data *add_list; 1999 struct i40e_aqc_remove_macvlan_element_data *del_list; 2000 2001 while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state)) 2002 usleep_range(1000, 2000); 2003 pf = vsi->back; 2004 2005 if (vsi->netdev) { 2006 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 2007 vsi->current_netdev_flags = vsi->netdev->flags; 2008 } 2009 2010 INIT_HLIST_HEAD(&tmp_add_list); 2011 INIT_HLIST_HEAD(&tmp_del_list); 2012 2013 if (vsi->type == I40E_VSI_SRIOV) 2014 snprintf(vsi_name, sizeof(vsi_name) - 1, "VF %d", vsi->vf_id); 2015 else if (vsi->type != I40E_VSI_MAIN) 2016 snprintf(vsi_name, sizeof(vsi_name) - 1, "vsi %d", vsi->seid); 2017 2018 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 2019 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 2020 2021 spin_lock_bh(&vsi->mac_filter_hash_lock); 2022 /* Create a list of filters to delete. */ 2023 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2024 if (f->state == I40E_FILTER_REMOVE) { 2025 /* Move the element into temporary del_list */ 2026 hash_del(&f->hlist); 2027 hlist_add_head(&f->hlist, &tmp_del_list); 2028 2029 /* Avoid counting removed filters */ 2030 continue; 2031 } 2032 if (f->state == I40E_FILTER_NEW) { 2033 /* Create a temporary i40e_new_mac_filter */ 2034 new = kzalloc(sizeof(*new), GFP_ATOMIC); 2035 if (!new) 2036 goto err_no_memory_locked; 2037 2038 /* Store pointer to the real filter */ 2039 new->f = f; 2040 new->state = f->state; 2041 2042 /* Add it to the hash list */ 2043 hlist_add_head(&new->hlist, &tmp_add_list); 2044 } 2045 2046 /* Count the number of active (current and new) VLAN 2047 * filters we have now. Does not count filters which 2048 * are marked for deletion. 2049 */ 2050 if (f->vlan > 0) 2051 vlan_filters++; 2052 } 2053 2054 retval = i40e_correct_mac_vlan_filters(vsi, 2055 &tmp_add_list, 2056 &tmp_del_list, 2057 vlan_filters); 2058 if (retval) 2059 goto err_no_memory_locked; 2060 2061 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2062 } 2063 2064 /* Now process 'del_list' outside the lock */ 2065 if (!hlist_empty(&tmp_del_list)) { 2066 filter_list_len = hw->aq.asq_buf_size / 2067 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2068 list_size = filter_list_len * 2069 sizeof(struct i40e_aqc_remove_macvlan_element_data); 2070 del_list = kzalloc(list_size, GFP_ATOMIC); 2071 if (!del_list) 2072 goto err_no_memory; 2073 2074 hlist_for_each_entry_safe(f, h, &tmp_del_list, hlist) { 2075 cmd_flags = 0; 2076 2077 /* handle broadcast filters by updating the broadcast 2078 * promiscuous flag and release filter list. 2079 */ 2080 if (is_broadcast_ether_addr(f->macaddr)) { 2081 i40e_aqc_broadcast_filter(vsi, vsi_name, f); 2082 2083 hlist_del(&f->hlist); 2084 kfree(f); 2085 continue; 2086 } 2087 2088 /* add to delete list */ 2089 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 2090 if (f->vlan == I40E_VLAN_ANY) { 2091 del_list[num_del].vlan_tag = 0; 2092 cmd_flags |= I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 2093 } else { 2094 del_list[num_del].vlan_tag = 2095 cpu_to_le16((u16)(f->vlan)); 2096 } 2097 2098 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 2099 del_list[num_del].flags = cmd_flags; 2100 num_del++; 2101 2102 /* flush a full buffer */ 2103 if (num_del == filter_list_len) { 2104 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2105 num_del, &retval); 2106 memset(del_list, 0, list_size); 2107 num_del = 0; 2108 } 2109 /* Release memory for MAC filter entries which were 2110 * synced up with HW. 2111 */ 2112 hlist_del(&f->hlist); 2113 kfree(f); 2114 } 2115 2116 if (num_del) { 2117 i40e_aqc_del_filters(vsi, vsi_name, del_list, 2118 num_del, &retval); 2119 } 2120 2121 kfree(del_list); 2122 del_list = NULL; 2123 } 2124 2125 if (!hlist_empty(&tmp_add_list)) { 2126 /* Do all the adds now. */ 2127 filter_list_len = hw->aq.asq_buf_size / 2128 sizeof(struct i40e_aqc_add_macvlan_element_data); 2129 list_size = filter_list_len * 2130 sizeof(struct i40e_aqc_add_macvlan_element_data); 2131 add_list = kzalloc(list_size, GFP_ATOMIC); 2132 if (!add_list) 2133 goto err_no_memory; 2134 2135 num_add = 0; 2136 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2137 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, 2138 &vsi->state)) { 2139 new->state = I40E_FILTER_FAILED; 2140 continue; 2141 } 2142 2143 /* handle broadcast filters by updating the broadcast 2144 * promiscuous flag instead of adding a MAC filter. 2145 */ 2146 if (is_broadcast_ether_addr(new->f->macaddr)) { 2147 if (i40e_aqc_broadcast_filter(vsi, vsi_name, 2148 new->f)) 2149 new->state = I40E_FILTER_FAILED; 2150 else 2151 new->state = I40E_FILTER_ACTIVE; 2152 continue; 2153 } 2154 2155 /* add to add array */ 2156 if (num_add == 0) 2157 add_head = new; 2158 cmd_flags = 0; 2159 ether_addr_copy(add_list[num_add].mac_addr, 2160 new->f->macaddr); 2161 if (new->f->vlan == I40E_VLAN_ANY) { 2162 add_list[num_add].vlan_tag = 0; 2163 cmd_flags |= I40E_AQC_MACVLAN_ADD_IGNORE_VLAN; 2164 } else { 2165 add_list[num_add].vlan_tag = 2166 cpu_to_le16((u16)(new->f->vlan)); 2167 } 2168 add_list[num_add].queue_number = 0; 2169 /* set invalid match method for later detection */ 2170 add_list[num_add].match_method = I40E_AQC_MM_ERR_NO_RES; 2171 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 2172 add_list[num_add].flags = cpu_to_le16(cmd_flags); 2173 num_add++; 2174 2175 /* flush a full buffer */ 2176 if (num_add == filter_list_len) { 2177 i40e_aqc_add_filters(vsi, vsi_name, add_list, 2178 add_head, num_add, 2179 &promisc_changed); 2180 memset(add_list, 0, list_size); 2181 num_add = 0; 2182 } 2183 } 2184 if (num_add) { 2185 i40e_aqc_add_filters(vsi, vsi_name, add_list, add_head, 2186 num_add, &promisc_changed); 2187 } 2188 /* Now move all of the filters from the temp add list back to 2189 * the VSI's list. 2190 */ 2191 spin_lock_bh(&vsi->mac_filter_hash_lock); 2192 hlist_for_each_entry_safe(new, h, &tmp_add_list, hlist) { 2193 /* Only update the state if we're still NEW */ 2194 if (new->f->state == I40E_FILTER_NEW) 2195 new->f->state = new->state; 2196 hlist_del(&new->hlist); 2197 kfree(new); 2198 } 2199 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2200 kfree(add_list); 2201 add_list = NULL; 2202 } 2203 2204 /* Determine the number of active and failed filters. */ 2205 spin_lock_bh(&vsi->mac_filter_hash_lock); 2206 vsi->active_filters = 0; 2207 hash_for_each(vsi->mac_filter_hash, bkt, f, hlist) { 2208 if (f->state == I40E_FILTER_ACTIVE) 2209 vsi->active_filters++; 2210 else if (f->state == I40E_FILTER_FAILED) 2211 failed_filters++; 2212 } 2213 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2214 2215 /* If promiscuous mode has changed, we need to calculate a new 2216 * threshold for when we are safe to exit 2217 */ 2218 if (promisc_changed) 2219 vsi->promisc_threshold = (vsi->active_filters * 3) / 4; 2220 2221 /* Check if we are able to exit overflow promiscuous mode. We can 2222 * safely exit if we didn't just enter, we no longer have any failed 2223 * filters, and we have reduced filters below the threshold value. 2224 */ 2225 if (test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state) && 2226 !promisc_changed && !failed_filters && 2227 (vsi->active_filters < vsi->promisc_threshold)) { 2228 dev_info(&pf->pdev->dev, 2229 "filter logjam cleared on %s, leaving overflow promiscuous mode\n", 2230 vsi_name); 2231 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 2232 promisc_changed = true; 2233 vsi->promisc_threshold = 0; 2234 } 2235 2236 /* if the VF is not trusted do not do promisc */ 2237 if ((vsi->type == I40E_VSI_SRIOV) && !pf->vf[vsi->vf_id].trusted) { 2238 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 2239 goto out; 2240 } 2241 2242 /* check for changes in promiscuous modes */ 2243 if (changed_flags & IFF_ALLMULTI) { 2244 bool cur_multipromisc; 2245 2246 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 2247 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 2248 vsi->seid, 2249 cur_multipromisc, 2250 NULL); 2251 if (aq_ret) { 2252 retval = i40e_aq_rc_to_posix(aq_ret, 2253 hw->aq.asq_last_status); 2254 dev_info(&pf->pdev->dev, 2255 "set multi promisc failed on %s, err %s aq_err %s\n", 2256 vsi_name, 2257 i40e_stat_str(hw, aq_ret), 2258 i40e_aq_str(hw, hw->aq.asq_last_status)); 2259 } 2260 } 2261 if ((changed_flags & IFF_PROMISC) || 2262 (promisc_changed && 2263 test_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state))) { 2264 bool cur_promisc; 2265 2266 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 2267 test_bit(__I40E_FILTER_OVERFLOW_PROMISC, 2268 &vsi->state)); 2269 if ((vsi->type == I40E_VSI_MAIN) && 2270 (pf->lan_veb != I40E_NO_VEB) && 2271 !(pf->flags & I40E_FLAG_MFP_ENABLED)) { 2272 /* set defport ON for Main VSI instead of true promisc 2273 * this way we will get all unicast/multicast and VLAN 2274 * promisc behavior but will not get VF or VMDq traffic 2275 * replicated on the Main VSI. 2276 */ 2277 if (pf->cur_promisc != cur_promisc) { 2278 pf->cur_promisc = cur_promisc; 2279 if (cur_promisc) 2280 aq_ret = 2281 i40e_aq_set_default_vsi(hw, 2282 vsi->seid, 2283 NULL); 2284 else 2285 aq_ret = 2286 i40e_aq_clear_default_vsi(hw, 2287 vsi->seid, 2288 NULL); 2289 if (aq_ret) { 2290 retval = i40e_aq_rc_to_posix(aq_ret, 2291 hw->aq.asq_last_status); 2292 dev_info(&pf->pdev->dev, 2293 "Set default VSI failed on %s, err %s, aq_err %s\n", 2294 vsi_name, 2295 i40e_stat_str(hw, aq_ret), 2296 i40e_aq_str(hw, 2297 hw->aq.asq_last_status)); 2298 } 2299 } 2300 } else { 2301 aq_ret = i40e_aq_set_vsi_unicast_promiscuous( 2302 hw, 2303 vsi->seid, 2304 cur_promisc, NULL, 2305 true); 2306 if (aq_ret) { 2307 retval = 2308 i40e_aq_rc_to_posix(aq_ret, 2309 hw->aq.asq_last_status); 2310 dev_info(&pf->pdev->dev, 2311 "set unicast promisc failed on %s, err %s, aq_err %s\n", 2312 vsi_name, 2313 i40e_stat_str(hw, aq_ret), 2314 i40e_aq_str(hw, 2315 hw->aq.asq_last_status)); 2316 } 2317 aq_ret = i40e_aq_set_vsi_multicast_promiscuous( 2318 hw, 2319 vsi->seid, 2320 cur_promisc, NULL); 2321 if (aq_ret) { 2322 retval = 2323 i40e_aq_rc_to_posix(aq_ret, 2324 hw->aq.asq_last_status); 2325 dev_info(&pf->pdev->dev, 2326 "set multicast promisc failed on %s, err %s, aq_err %s\n", 2327 vsi_name, 2328 i40e_stat_str(hw, aq_ret), 2329 i40e_aq_str(hw, 2330 hw->aq.asq_last_status)); 2331 } 2332 } 2333 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw, 2334 vsi->seid, 2335 cur_promisc, NULL); 2336 if (aq_ret) { 2337 retval = i40e_aq_rc_to_posix(aq_ret, 2338 pf->hw.aq.asq_last_status); 2339 dev_info(&pf->pdev->dev, 2340 "set brdcast promisc failed, err %s, aq_err %s\n", 2341 i40e_stat_str(hw, aq_ret), 2342 i40e_aq_str(hw, 2343 hw->aq.asq_last_status)); 2344 } 2345 } 2346 out: 2347 /* if something went wrong then set the changed flag so we try again */ 2348 if (retval) 2349 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2350 2351 clear_bit(__I40E_CONFIG_BUSY, &vsi->state); 2352 return retval; 2353 2354 err_no_memory: 2355 /* Restore elements on the temporary add and delete lists */ 2356 spin_lock_bh(&vsi->mac_filter_hash_lock); 2357 err_no_memory_locked: 2358 i40e_undo_del_filter_entries(vsi, &tmp_del_list); 2359 i40e_undo_add_filter_entries(vsi, &tmp_add_list); 2360 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2361 2362 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 2363 clear_bit(__I40E_CONFIG_BUSY, &vsi->state); 2364 return -ENOMEM; 2365 } 2366 2367 /** 2368 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 2369 * @pf: board private structure 2370 **/ 2371 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 2372 { 2373 int v; 2374 2375 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC)) 2376 return; 2377 pf->flags &= ~I40E_FLAG_FILTER_SYNC; 2378 2379 for (v = 0; v < pf->num_alloc_vsi; v++) { 2380 if (pf->vsi[v] && 2381 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) { 2382 int ret = i40e_sync_vsi_filters(pf->vsi[v]); 2383 2384 if (ret) { 2385 /* come back and try again later */ 2386 pf->flags |= I40E_FLAG_FILTER_SYNC; 2387 break; 2388 } 2389 } 2390 } 2391 } 2392 2393 /** 2394 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 2395 * @netdev: network interface device structure 2396 * @new_mtu: new value for maximum frame size 2397 * 2398 * Returns 0 on success, negative on failure 2399 **/ 2400 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 2401 { 2402 struct i40e_netdev_priv *np = netdev_priv(netdev); 2403 struct i40e_vsi *vsi = np->vsi; 2404 struct i40e_pf *pf = vsi->back; 2405 2406 netdev_info(netdev, "changing MTU from %d to %d\n", 2407 netdev->mtu, new_mtu); 2408 netdev->mtu = new_mtu; 2409 if (netif_running(netdev)) 2410 i40e_vsi_reinit_locked(vsi); 2411 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 2412 I40E_FLAG_CLIENT_L2_CHANGE); 2413 return 0; 2414 } 2415 2416 /** 2417 * i40e_ioctl - Access the hwtstamp interface 2418 * @netdev: network interface device structure 2419 * @ifr: interface request data 2420 * @cmd: ioctl command 2421 **/ 2422 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 2423 { 2424 struct i40e_netdev_priv *np = netdev_priv(netdev); 2425 struct i40e_pf *pf = np->vsi->back; 2426 2427 switch (cmd) { 2428 case SIOCGHWTSTAMP: 2429 return i40e_ptp_get_ts_config(pf, ifr); 2430 case SIOCSHWTSTAMP: 2431 return i40e_ptp_set_ts_config(pf, ifr); 2432 default: 2433 return -EOPNOTSUPP; 2434 } 2435 } 2436 2437 /** 2438 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 2439 * @vsi: the vsi being adjusted 2440 **/ 2441 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 2442 { 2443 struct i40e_vsi_context ctxt; 2444 i40e_status ret; 2445 2446 if ((vsi->info.valid_sections & 2447 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2448 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 2449 return; /* already enabled */ 2450 2451 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2452 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2453 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 2454 2455 ctxt.seid = vsi->seid; 2456 ctxt.info = vsi->info; 2457 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2458 if (ret) { 2459 dev_info(&vsi->back->pdev->dev, 2460 "update vlan stripping failed, err %s aq_err %s\n", 2461 i40e_stat_str(&vsi->back->hw, ret), 2462 i40e_aq_str(&vsi->back->hw, 2463 vsi->back->hw.aq.asq_last_status)); 2464 } 2465 } 2466 2467 /** 2468 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 2469 * @vsi: the vsi being adjusted 2470 **/ 2471 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 2472 { 2473 struct i40e_vsi_context ctxt; 2474 i40e_status ret; 2475 2476 if ((vsi->info.valid_sections & 2477 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 2478 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 2479 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 2480 return; /* already disabled */ 2481 2482 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2483 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 2484 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 2485 2486 ctxt.seid = vsi->seid; 2487 ctxt.info = vsi->info; 2488 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2489 if (ret) { 2490 dev_info(&vsi->back->pdev->dev, 2491 "update vlan stripping failed, err %s aq_err %s\n", 2492 i40e_stat_str(&vsi->back->hw, ret), 2493 i40e_aq_str(&vsi->back->hw, 2494 vsi->back->hw.aq.asq_last_status)); 2495 } 2496 } 2497 2498 /** 2499 * i40e_vlan_rx_register - Setup or shutdown vlan offload 2500 * @netdev: network interface to be adjusted 2501 * @features: netdev features to test if VLAN offload is enabled or not 2502 **/ 2503 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features) 2504 { 2505 struct i40e_netdev_priv *np = netdev_priv(netdev); 2506 struct i40e_vsi *vsi = np->vsi; 2507 2508 if (features & NETIF_F_HW_VLAN_CTAG_RX) 2509 i40e_vlan_stripping_enable(vsi); 2510 else 2511 i40e_vlan_stripping_disable(vsi); 2512 } 2513 2514 /** 2515 * i40e_add_vlan_all_mac - Add a MAC/VLAN filter for each existing MAC address 2516 * @vsi: the vsi being configured 2517 * @vid: vlan id to be added (0 = untagged only , -1 = any) 2518 * 2519 * This is a helper function for adding a new MAC/VLAN filter with the 2520 * specified VLAN for each existing MAC address already in the hash table. 2521 * This function does *not* perform any accounting to update filters based on 2522 * VLAN mode. 2523 * 2524 * NOTE: this function expects to be called while under the 2525 * mac_filter_hash_lock 2526 **/ 2527 int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2528 { 2529 struct i40e_mac_filter *f, *add_f; 2530 struct hlist_node *h; 2531 int bkt; 2532 2533 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2534 if (f->state == I40E_FILTER_REMOVE) 2535 continue; 2536 add_f = i40e_add_filter(vsi, f->macaddr, vid); 2537 if (!add_f) { 2538 dev_info(&vsi->back->pdev->dev, 2539 "Could not add vlan filter %d for %pM\n", 2540 vid, f->macaddr); 2541 return -ENOMEM; 2542 } 2543 } 2544 2545 return 0; 2546 } 2547 2548 /** 2549 * i40e_vsi_add_vlan - Add VSI membership for given VLAN 2550 * @vsi: the VSI being configured 2551 * @vid: VLAN id to be added 2552 **/ 2553 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, u16 vid) 2554 { 2555 int err; 2556 2557 if (!vid || vsi->info.pvid) 2558 return -EINVAL; 2559 2560 /* Locked once because all functions invoked below iterates list*/ 2561 spin_lock_bh(&vsi->mac_filter_hash_lock); 2562 err = i40e_add_vlan_all_mac(vsi, vid); 2563 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2564 if (err) 2565 return err; 2566 2567 /* schedule our worker thread which will take care of 2568 * applying the new filter changes 2569 */ 2570 i40e_service_event_schedule(vsi->back); 2571 return 0; 2572 } 2573 2574 /** 2575 * i40e_rm_vlan_all_mac - Remove MAC/VLAN pair for all MAC with the given VLAN 2576 * @vsi: the vsi being configured 2577 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 2578 * 2579 * This function should be used to remove all VLAN filters which match the 2580 * given VID. It does not schedule the service event and does not take the 2581 * mac_filter_hash_lock so it may be combined with other operations under 2582 * a single invocation of the mac_filter_hash_lock. 2583 * 2584 * NOTE: this function expects to be called while under the 2585 * mac_filter_hash_lock 2586 */ 2587 void i40e_rm_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) 2588 { 2589 struct i40e_mac_filter *f; 2590 struct hlist_node *h; 2591 int bkt; 2592 2593 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 2594 if (f->vlan == vid) 2595 __i40e_del_filter(vsi, f); 2596 } 2597 } 2598 2599 /** 2600 * i40e_vsi_kill_vlan - Remove VSI membership for given VLAN 2601 * @vsi: the VSI being configured 2602 * @vid: VLAN id to be removed 2603 **/ 2604 void i40e_vsi_kill_vlan(struct i40e_vsi *vsi, u16 vid) 2605 { 2606 if (!vid || vsi->info.pvid) 2607 return; 2608 2609 spin_lock_bh(&vsi->mac_filter_hash_lock); 2610 i40e_rm_vlan_all_mac(vsi, vid); 2611 spin_unlock_bh(&vsi->mac_filter_hash_lock); 2612 2613 /* schedule our worker thread which will take care of 2614 * applying the new filter changes 2615 */ 2616 i40e_service_event_schedule(vsi->back); 2617 } 2618 2619 /** 2620 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2621 * @netdev: network interface to be adjusted 2622 * @vid: vlan id to be added 2623 * 2624 * net_device_ops implementation for adding vlan ids 2625 **/ 2626 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2627 __always_unused __be16 proto, u16 vid) 2628 { 2629 struct i40e_netdev_priv *np = netdev_priv(netdev); 2630 struct i40e_vsi *vsi = np->vsi; 2631 int ret = 0; 2632 2633 if (vid >= VLAN_N_VID) 2634 return -EINVAL; 2635 2636 /* If the network stack called us with vid = 0 then 2637 * it is asking to receive priority tagged packets with 2638 * vlan id 0. Our HW receives them by default when configured 2639 * to receive untagged packets so there is no need to add an 2640 * extra filter for vlan 0 tagged packets. 2641 */ 2642 if (vid) 2643 ret = i40e_vsi_add_vlan(vsi, vid); 2644 2645 if (!ret) 2646 set_bit(vid, vsi->active_vlans); 2647 2648 return ret; 2649 } 2650 2651 /** 2652 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2653 * @netdev: network interface to be adjusted 2654 * @vid: vlan id to be removed 2655 * 2656 * net_device_ops implementation for removing vlan ids 2657 **/ 2658 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2659 __always_unused __be16 proto, u16 vid) 2660 { 2661 struct i40e_netdev_priv *np = netdev_priv(netdev); 2662 struct i40e_vsi *vsi = np->vsi; 2663 2664 /* return code is ignored as there is nothing a user 2665 * can do about failure to remove and a log message was 2666 * already printed from the other function 2667 */ 2668 i40e_vsi_kill_vlan(vsi, vid); 2669 2670 clear_bit(vid, vsi->active_vlans); 2671 2672 return 0; 2673 } 2674 2675 /** 2676 * i40e_macaddr_init - explicitly write the mac address filters 2677 * 2678 * @vsi: pointer to the vsi 2679 * @macaddr: the MAC address 2680 * 2681 * This is needed when the macaddr has been obtained by other 2682 * means than the default, e.g., from Open Firmware or IDPROM. 2683 * Returns 0 on success, negative on failure 2684 **/ 2685 static int i40e_macaddr_init(struct i40e_vsi *vsi, u8 *macaddr) 2686 { 2687 int ret; 2688 struct i40e_aqc_add_macvlan_element_data element; 2689 2690 ret = i40e_aq_mac_address_write(&vsi->back->hw, 2691 I40E_AQC_WRITE_TYPE_LAA_WOL, 2692 macaddr, NULL); 2693 if (ret) { 2694 dev_info(&vsi->back->pdev->dev, 2695 "Addr change for VSI failed: %d\n", ret); 2696 return -EADDRNOTAVAIL; 2697 } 2698 2699 memset(&element, 0, sizeof(element)); 2700 ether_addr_copy(element.mac_addr, macaddr); 2701 element.flags = cpu_to_le16(I40E_AQC_MACVLAN_ADD_PERFECT_MATCH); 2702 ret = i40e_aq_add_macvlan(&vsi->back->hw, vsi->seid, &element, 1, NULL); 2703 if (ret) { 2704 dev_info(&vsi->back->pdev->dev, 2705 "add filter failed err %s aq_err %s\n", 2706 i40e_stat_str(&vsi->back->hw, ret), 2707 i40e_aq_str(&vsi->back->hw, 2708 vsi->back->hw.aq.asq_last_status)); 2709 } 2710 return ret; 2711 } 2712 2713 /** 2714 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2715 * @vsi: the vsi being brought back up 2716 **/ 2717 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2718 { 2719 u16 vid; 2720 2721 if (!vsi->netdev) 2722 return; 2723 2724 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features); 2725 2726 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2727 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), 2728 vid); 2729 } 2730 2731 /** 2732 * i40e_vsi_add_pvid - Add pvid for the VSI 2733 * @vsi: the vsi being adjusted 2734 * @vid: the vlan id to set as a PVID 2735 **/ 2736 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2737 { 2738 struct i40e_vsi_context ctxt; 2739 i40e_status ret; 2740 2741 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2742 vsi->info.pvid = cpu_to_le16(vid); 2743 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2744 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2745 I40E_AQ_VSI_PVLAN_EMOD_STR; 2746 2747 ctxt.seid = vsi->seid; 2748 ctxt.info = vsi->info; 2749 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2750 if (ret) { 2751 dev_info(&vsi->back->pdev->dev, 2752 "add pvid failed, err %s aq_err %s\n", 2753 i40e_stat_str(&vsi->back->hw, ret), 2754 i40e_aq_str(&vsi->back->hw, 2755 vsi->back->hw.aq.asq_last_status)); 2756 return -ENOENT; 2757 } 2758 2759 return 0; 2760 } 2761 2762 /** 2763 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2764 * @vsi: the vsi being adjusted 2765 * 2766 * Just use the vlan_rx_register() service to put it back to normal 2767 **/ 2768 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2769 { 2770 i40e_vlan_stripping_disable(vsi); 2771 2772 vsi->info.pvid = 0; 2773 } 2774 2775 /** 2776 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2777 * @vsi: ptr to the VSI 2778 * 2779 * If this function returns with an error, then it's possible one or 2780 * more of the rings is populated (while the rest are not). It is the 2781 * callers duty to clean those orphaned rings. 2782 * 2783 * Return 0 on success, negative on failure 2784 **/ 2785 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2786 { 2787 int i, err = 0; 2788 2789 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2790 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2791 2792 return err; 2793 } 2794 2795 /** 2796 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 2797 * @vsi: ptr to the VSI 2798 * 2799 * Free VSI's transmit software resources 2800 **/ 2801 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 2802 { 2803 int i; 2804 2805 if (!vsi->tx_rings) 2806 return; 2807 2808 for (i = 0; i < vsi->num_queue_pairs; i++) 2809 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 2810 i40e_free_tx_resources(vsi->tx_rings[i]); 2811 } 2812 2813 /** 2814 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 2815 * @vsi: ptr to the VSI 2816 * 2817 * If this function returns with an error, then it's possible one or 2818 * more of the rings is populated (while the rest are not). It is the 2819 * callers duty to clean those orphaned rings. 2820 * 2821 * Return 0 on success, negative on failure 2822 **/ 2823 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 2824 { 2825 int i, err = 0; 2826 2827 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2828 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 2829 return err; 2830 } 2831 2832 /** 2833 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 2834 * @vsi: ptr to the VSI 2835 * 2836 * Free all receive software resources 2837 **/ 2838 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 2839 { 2840 int i; 2841 2842 if (!vsi->rx_rings) 2843 return; 2844 2845 for (i = 0; i < vsi->num_queue_pairs; i++) 2846 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 2847 i40e_free_rx_resources(vsi->rx_rings[i]); 2848 } 2849 2850 /** 2851 * i40e_config_xps_tx_ring - Configure XPS for a Tx ring 2852 * @ring: The Tx ring to configure 2853 * 2854 * This enables/disables XPS for a given Tx descriptor ring 2855 * based on the TCs enabled for the VSI that ring belongs to. 2856 **/ 2857 static void i40e_config_xps_tx_ring(struct i40e_ring *ring) 2858 { 2859 struct i40e_vsi *vsi = ring->vsi; 2860 cpumask_var_t mask; 2861 2862 if (!ring->q_vector || !ring->netdev) 2863 return; 2864 2865 /* Single TC mode enable XPS */ 2866 if (vsi->tc_config.numtc <= 1) { 2867 if (!test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) 2868 netif_set_xps_queue(ring->netdev, 2869 &ring->q_vector->affinity_mask, 2870 ring->queue_index); 2871 } else if (alloc_cpumask_var(&mask, GFP_KERNEL)) { 2872 /* Disable XPS to allow selection based on TC */ 2873 bitmap_zero(cpumask_bits(mask), nr_cpumask_bits); 2874 netif_set_xps_queue(ring->netdev, mask, ring->queue_index); 2875 free_cpumask_var(mask); 2876 } 2877 2878 /* schedule our worker thread which will take care of 2879 * applying the new filter changes 2880 */ 2881 i40e_service_event_schedule(vsi->back); 2882 } 2883 2884 /** 2885 * i40e_configure_tx_ring - Configure a transmit ring context and rest 2886 * @ring: The Tx ring to configure 2887 * 2888 * Configure the Tx descriptor ring in the HMC context. 2889 **/ 2890 static int i40e_configure_tx_ring(struct i40e_ring *ring) 2891 { 2892 struct i40e_vsi *vsi = ring->vsi; 2893 u16 pf_q = vsi->base_queue + ring->queue_index; 2894 struct i40e_hw *hw = &vsi->back->hw; 2895 struct i40e_hmc_obj_txq tx_ctx; 2896 i40e_status err = 0; 2897 u32 qtx_ctl = 0; 2898 2899 /* some ATR related tx ring init */ 2900 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 2901 ring->atr_sample_rate = vsi->back->atr_sample_rate; 2902 ring->atr_count = 0; 2903 } else { 2904 ring->atr_sample_rate = 0; 2905 } 2906 2907 /* configure XPS */ 2908 i40e_config_xps_tx_ring(ring); 2909 2910 /* clear the context structure first */ 2911 memset(&tx_ctx, 0, sizeof(tx_ctx)); 2912 2913 tx_ctx.new_context = 1; 2914 tx_ctx.base = (ring->dma / 128); 2915 tx_ctx.qlen = ring->count; 2916 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 2917 I40E_FLAG_FD_ATR_ENABLED)); 2918 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 2919 /* FDIR VSI tx ring can still use RS bit and writebacks */ 2920 if (vsi->type != I40E_VSI_FDIR) 2921 tx_ctx.head_wb_ena = 1; 2922 tx_ctx.head_wb_addr = ring->dma + 2923 (ring->count * sizeof(struct i40e_tx_desc)); 2924 2925 /* As part of VSI creation/update, FW allocates certain 2926 * Tx arbitration queue sets for each TC enabled for 2927 * the VSI. The FW returns the handles to these queue 2928 * sets as part of the response buffer to Add VSI, 2929 * Update VSI, etc. AQ commands. It is expected that 2930 * these queue set handles be associated with the Tx 2931 * queues by the driver as part of the TX queue context 2932 * initialization. This has to be done regardless of 2933 * DCB as by default everything is mapped to TC0. 2934 */ 2935 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 2936 tx_ctx.rdylist_act = 0; 2937 2938 /* clear the context in the HMC */ 2939 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 2940 if (err) { 2941 dev_info(&vsi->back->pdev->dev, 2942 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 2943 ring->queue_index, pf_q, err); 2944 return -ENOMEM; 2945 } 2946 2947 /* set the context in the HMC */ 2948 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 2949 if (err) { 2950 dev_info(&vsi->back->pdev->dev, 2951 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 2952 ring->queue_index, pf_q, err); 2953 return -ENOMEM; 2954 } 2955 2956 /* Now associate this queue with this PCI function */ 2957 if (vsi->type == I40E_VSI_VMDQ2) { 2958 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 2959 qtx_ctl |= ((vsi->id) << I40E_QTX_CTL_VFVM_INDX_SHIFT) & 2960 I40E_QTX_CTL_VFVM_INDX_MASK; 2961 } else { 2962 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 2963 } 2964 2965 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2966 I40E_QTX_CTL_PF_INDX_MASK); 2967 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 2968 i40e_flush(hw); 2969 2970 /* cache tail off for easier writes later */ 2971 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 2972 2973 return 0; 2974 } 2975 2976 /** 2977 * i40e_configure_rx_ring - Configure a receive ring context 2978 * @ring: The Rx ring to configure 2979 * 2980 * Configure the Rx descriptor ring in the HMC context. 2981 **/ 2982 static int i40e_configure_rx_ring(struct i40e_ring *ring) 2983 { 2984 struct i40e_vsi *vsi = ring->vsi; 2985 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 2986 u16 pf_q = vsi->base_queue + ring->queue_index; 2987 struct i40e_hw *hw = &vsi->back->hw; 2988 struct i40e_hmc_obj_rxq rx_ctx; 2989 i40e_status err = 0; 2990 2991 ring->state = 0; 2992 2993 /* clear the context structure first */ 2994 memset(&rx_ctx, 0, sizeof(rx_ctx)); 2995 2996 ring->rx_buf_len = vsi->rx_buf_len; 2997 2998 rx_ctx.dbuff = DIV_ROUND_UP(ring->rx_buf_len, 2999 BIT_ULL(I40E_RXQ_CTX_DBUFF_SHIFT)); 3000 3001 rx_ctx.base = (ring->dma / 128); 3002 rx_ctx.qlen = ring->count; 3003 3004 /* use 32 byte descriptors */ 3005 rx_ctx.dsize = 1; 3006 3007 /* descriptor type is always zero 3008 * rx_ctx.dtype = 0; 3009 */ 3010 rx_ctx.hsplit_0 = 0; 3011 3012 rx_ctx.rxmax = min_t(u16, vsi->max_frame, chain_len * ring->rx_buf_len); 3013 if (hw->revision_id == 0) 3014 rx_ctx.lrxqthresh = 0; 3015 else 3016 rx_ctx.lrxqthresh = 2; 3017 rx_ctx.crcstrip = 1; 3018 rx_ctx.l2tsel = 1; 3019 /* this controls whether VLAN is stripped from inner headers */ 3020 rx_ctx.showiv = 0; 3021 /* set the prefena field to 1 because the manual says to */ 3022 rx_ctx.prefena = 1; 3023 3024 /* clear the context in the HMC */ 3025 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 3026 if (err) { 3027 dev_info(&vsi->back->pdev->dev, 3028 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3029 ring->queue_index, pf_q, err); 3030 return -ENOMEM; 3031 } 3032 3033 /* set the context in the HMC */ 3034 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 3035 if (err) { 3036 dev_info(&vsi->back->pdev->dev, 3037 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 3038 ring->queue_index, pf_q, err); 3039 return -ENOMEM; 3040 } 3041 3042 /* cache tail for quicker writes, and clear the reg before use */ 3043 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 3044 writel(0, ring->tail); 3045 3046 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 3047 3048 return 0; 3049 } 3050 3051 /** 3052 * i40e_vsi_configure_tx - Configure the VSI for Tx 3053 * @vsi: VSI structure describing this set of rings and resources 3054 * 3055 * Configure the Tx VSI for operation. 3056 **/ 3057 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 3058 { 3059 int err = 0; 3060 u16 i; 3061 3062 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 3063 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 3064 3065 return err; 3066 } 3067 3068 /** 3069 * i40e_vsi_configure_rx - Configure the VSI for Rx 3070 * @vsi: the VSI being configured 3071 * 3072 * Configure the Rx VSI for operation. 3073 **/ 3074 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 3075 { 3076 int err = 0; 3077 u16 i; 3078 3079 if (!vsi->netdev || (vsi->back->flags & I40E_FLAG_LEGACY_RX)) { 3080 vsi->max_frame = I40E_MAX_RXBUFFER; 3081 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3082 #if (PAGE_SIZE < 8192) 3083 } else if (vsi->netdev->mtu <= ETH_DATA_LEN) { 3084 vsi->max_frame = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3085 vsi->rx_buf_len = I40E_RXBUFFER_1536 - NET_IP_ALIGN; 3086 #endif 3087 } else { 3088 vsi->max_frame = I40E_MAX_RXBUFFER; 3089 vsi->rx_buf_len = I40E_RXBUFFER_2048; 3090 } 3091 3092 /* set up individual rings */ 3093 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 3094 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 3095 3096 return err; 3097 } 3098 3099 /** 3100 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 3101 * @vsi: ptr to the VSI 3102 **/ 3103 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 3104 { 3105 struct i40e_ring *tx_ring, *rx_ring; 3106 u16 qoffset, qcount; 3107 int i, n; 3108 3109 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 3110 /* Reset the TC information */ 3111 for (i = 0; i < vsi->num_queue_pairs; i++) { 3112 rx_ring = vsi->rx_rings[i]; 3113 tx_ring = vsi->tx_rings[i]; 3114 rx_ring->dcb_tc = 0; 3115 tx_ring->dcb_tc = 0; 3116 } 3117 } 3118 3119 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 3120 if (!(vsi->tc_config.enabled_tc & BIT_ULL(n))) 3121 continue; 3122 3123 qoffset = vsi->tc_config.tc_info[n].qoffset; 3124 qcount = vsi->tc_config.tc_info[n].qcount; 3125 for (i = qoffset; i < (qoffset + qcount); i++) { 3126 rx_ring = vsi->rx_rings[i]; 3127 tx_ring = vsi->tx_rings[i]; 3128 rx_ring->dcb_tc = n; 3129 tx_ring->dcb_tc = n; 3130 } 3131 } 3132 } 3133 3134 /** 3135 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 3136 * @vsi: ptr to the VSI 3137 **/ 3138 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 3139 { 3140 struct i40e_pf *pf = vsi->back; 3141 int err; 3142 3143 if (vsi->netdev) 3144 i40e_set_rx_mode(vsi->netdev); 3145 3146 if (!!(pf->flags & I40E_FLAG_PF_MAC)) { 3147 err = i40e_macaddr_init(vsi, pf->hw.mac.addr); 3148 if (err) { 3149 dev_warn(&pf->pdev->dev, 3150 "could not set up macaddr; err %d\n", err); 3151 } 3152 } 3153 } 3154 3155 /** 3156 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 3157 * @vsi: Pointer to the targeted VSI 3158 * 3159 * This function replays the hlist on the hw where all the SB Flow Director 3160 * filters were saved. 3161 **/ 3162 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 3163 { 3164 struct i40e_fdir_filter *filter; 3165 struct i40e_pf *pf = vsi->back; 3166 struct hlist_node *node; 3167 3168 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 3169 return; 3170 3171 /* Reset FDir counters as we're replaying all existing filters */ 3172 pf->fd_tcp4_filter_cnt = 0; 3173 pf->fd_udp4_filter_cnt = 0; 3174 pf->fd_sctp4_filter_cnt = 0; 3175 pf->fd_ip4_filter_cnt = 0; 3176 3177 hlist_for_each_entry_safe(filter, node, 3178 &pf->fdir_filter_list, fdir_node) { 3179 i40e_add_del_fdir(vsi, filter, true); 3180 } 3181 } 3182 3183 /** 3184 * i40e_vsi_configure - Set up the VSI for action 3185 * @vsi: the VSI being configured 3186 **/ 3187 static int i40e_vsi_configure(struct i40e_vsi *vsi) 3188 { 3189 int err; 3190 3191 i40e_set_vsi_rx_mode(vsi); 3192 i40e_restore_vlan(vsi); 3193 i40e_vsi_config_dcb_rings(vsi); 3194 err = i40e_vsi_configure_tx(vsi); 3195 if (!err) 3196 err = i40e_vsi_configure_rx(vsi); 3197 3198 return err; 3199 } 3200 3201 /** 3202 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 3203 * @vsi: the VSI being configured 3204 **/ 3205 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 3206 { 3207 struct i40e_pf *pf = vsi->back; 3208 struct i40e_hw *hw = &pf->hw; 3209 u16 vector; 3210 int i, q; 3211 u32 qp; 3212 3213 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 3214 * and PFINT_LNKLSTn registers, e.g.: 3215 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 3216 */ 3217 qp = vsi->base_queue; 3218 vector = vsi->base_vector; 3219 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 3220 struct i40e_q_vector *q_vector = vsi->q_vectors[i]; 3221 3222 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3223 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[i]->rx_itr_setting); 3224 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3225 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 3226 q_vector->rx.itr); 3227 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[i]->tx_itr_setting); 3228 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3229 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 3230 q_vector->tx.itr); 3231 wr32(hw, I40E_PFINT_RATEN(vector - 1), 3232 i40e_intrl_usec_to_reg(vsi->int_rate_limit)); 3233 3234 /* Linked list for the queuepairs assigned to this vector */ 3235 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 3236 for (q = 0; q < q_vector->num_ringpairs; q++) { 3237 u32 val; 3238 3239 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3240 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3241 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 3242 (qp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 3243 (I40E_QUEUE_TYPE_TX 3244 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 3245 3246 wr32(hw, I40E_QINT_RQCTL(qp), val); 3247 3248 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3249 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3250 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 3251 ((qp+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)| 3252 (I40E_QUEUE_TYPE_RX 3253 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3254 3255 /* Terminate the linked list */ 3256 if (q == (q_vector->num_ringpairs - 1)) 3257 val |= (I40E_QUEUE_END_OF_LIST 3258 << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3259 3260 wr32(hw, I40E_QINT_TQCTL(qp), val); 3261 qp++; 3262 } 3263 } 3264 3265 i40e_flush(hw); 3266 } 3267 3268 /** 3269 * i40e_enable_misc_int_causes - enable the non-queue interrupts 3270 * @hw: ptr to the hardware info 3271 **/ 3272 static void i40e_enable_misc_int_causes(struct i40e_pf *pf) 3273 { 3274 struct i40e_hw *hw = &pf->hw; 3275 u32 val; 3276 3277 /* clear things first */ 3278 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 3279 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 3280 3281 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 3282 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 3283 I40E_PFINT_ICR0_ENA_GRST_MASK | 3284 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 3285 I40E_PFINT_ICR0_ENA_GPIO_MASK | 3286 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 3287 I40E_PFINT_ICR0_ENA_VFLR_MASK | 3288 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3289 3290 if (pf->flags & I40E_FLAG_IWARP_ENABLED) 3291 val |= I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3292 3293 if (pf->flags & I40E_FLAG_PTP) 3294 val |= I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3295 3296 wr32(hw, I40E_PFINT_ICR0_ENA, val); 3297 3298 /* SW_ITR_IDX = 0, but don't change INTENA */ 3299 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 3300 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 3301 3302 /* OTHER_ITR_IDX = 0 */ 3303 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 3304 } 3305 3306 /** 3307 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 3308 * @vsi: the VSI being configured 3309 **/ 3310 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 3311 { 3312 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3313 struct i40e_pf *pf = vsi->back; 3314 struct i40e_hw *hw = &pf->hw; 3315 u32 val; 3316 3317 /* set the ITR configuration */ 3318 q_vector->itr_countdown = ITR_COUNTDOWN_START; 3319 q_vector->rx.itr = ITR_TO_REG(vsi->rx_rings[0]->rx_itr_setting); 3320 q_vector->rx.latency_range = I40E_LOW_LATENCY; 3321 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 3322 q_vector->tx.itr = ITR_TO_REG(vsi->tx_rings[0]->tx_itr_setting); 3323 q_vector->tx.latency_range = I40E_LOW_LATENCY; 3324 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 3325 3326 i40e_enable_misc_int_causes(pf); 3327 3328 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 3329 wr32(hw, I40E_PFINT_LNKLST0, 0); 3330 3331 /* Associate the queue pair to the vector and enable the queue int */ 3332 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3333 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 3334 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 3335 3336 wr32(hw, I40E_QINT_RQCTL(0), val); 3337 3338 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3339 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 3340 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 3341 3342 wr32(hw, I40E_QINT_TQCTL(0), val); 3343 i40e_flush(hw); 3344 } 3345 3346 /** 3347 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 3348 * @pf: board private structure 3349 **/ 3350 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 3351 { 3352 struct i40e_hw *hw = &pf->hw; 3353 3354 wr32(hw, I40E_PFINT_DYN_CTL0, 3355 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 3356 i40e_flush(hw); 3357 } 3358 3359 /** 3360 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 3361 * @pf: board private structure 3362 * @clearpba: true when all pending interrupt events should be cleared 3363 **/ 3364 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf, bool clearpba) 3365 { 3366 struct i40e_hw *hw = &pf->hw; 3367 u32 val; 3368 3369 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 3370 (clearpba ? I40E_PFINT_DYN_CTL0_CLEARPBA_MASK : 0) | 3371 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 3372 3373 wr32(hw, I40E_PFINT_DYN_CTL0, val); 3374 i40e_flush(hw); 3375 } 3376 3377 /** 3378 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 3379 * @irq: interrupt number 3380 * @data: pointer to a q_vector 3381 **/ 3382 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 3383 { 3384 struct i40e_q_vector *q_vector = data; 3385 3386 if (!q_vector->tx.ring && !q_vector->rx.ring) 3387 return IRQ_HANDLED; 3388 3389 napi_schedule_irqoff(&q_vector->napi); 3390 3391 return IRQ_HANDLED; 3392 } 3393 3394 /** 3395 * i40e_irq_affinity_notify - Callback for affinity changes 3396 * @notify: context as to what irq was changed 3397 * @mask: the new affinity mask 3398 * 3399 * This is a callback function used by the irq_set_affinity_notifier function 3400 * so that we may register to receive changes to the irq affinity masks. 3401 **/ 3402 static void i40e_irq_affinity_notify(struct irq_affinity_notify *notify, 3403 const cpumask_t *mask) 3404 { 3405 struct i40e_q_vector *q_vector = 3406 container_of(notify, struct i40e_q_vector, affinity_notify); 3407 3408 q_vector->affinity_mask = *mask; 3409 } 3410 3411 /** 3412 * i40e_irq_affinity_release - Callback for affinity notifier release 3413 * @ref: internal core kernel usage 3414 * 3415 * This is a callback function used by the irq_set_affinity_notifier function 3416 * to inform the current notification subscriber that they will no longer 3417 * receive notifications. 3418 **/ 3419 static void i40e_irq_affinity_release(struct kref *ref) {} 3420 3421 /** 3422 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 3423 * @vsi: the VSI being configured 3424 * @basename: name for the vector 3425 * 3426 * Allocates MSI-X vectors and requests interrupts from the kernel. 3427 **/ 3428 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 3429 { 3430 int q_vectors = vsi->num_q_vectors; 3431 struct i40e_pf *pf = vsi->back; 3432 int base = vsi->base_vector; 3433 int rx_int_idx = 0; 3434 int tx_int_idx = 0; 3435 int vector, err; 3436 int irq_num; 3437 3438 for (vector = 0; vector < q_vectors; vector++) { 3439 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 3440 3441 irq_num = pf->msix_entries[base + vector].vector; 3442 3443 if (q_vector->tx.ring && q_vector->rx.ring) { 3444 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3445 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 3446 tx_int_idx++; 3447 } else if (q_vector->rx.ring) { 3448 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3449 "%s-%s-%d", basename, "rx", rx_int_idx++); 3450 } else if (q_vector->tx.ring) { 3451 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 3452 "%s-%s-%d", basename, "tx", tx_int_idx++); 3453 } else { 3454 /* skip this unused q_vector */ 3455 continue; 3456 } 3457 err = request_irq(irq_num, 3458 vsi->irq_handler, 3459 0, 3460 q_vector->name, 3461 q_vector); 3462 if (err) { 3463 dev_info(&pf->pdev->dev, 3464 "MSIX request_irq failed, error: %d\n", err); 3465 goto free_queue_irqs; 3466 } 3467 3468 /* register for affinity change notifications */ 3469 q_vector->affinity_notify.notify = i40e_irq_affinity_notify; 3470 q_vector->affinity_notify.release = i40e_irq_affinity_release; 3471 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify); 3472 /* assign the mask for this irq */ 3473 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask); 3474 } 3475 3476 vsi->irqs_ready = true; 3477 return 0; 3478 3479 free_queue_irqs: 3480 while (vector) { 3481 vector--; 3482 irq_num = pf->msix_entries[base + vector].vector; 3483 irq_set_affinity_notifier(irq_num, NULL); 3484 irq_set_affinity_hint(irq_num, NULL); 3485 free_irq(irq_num, &vsi->q_vectors[vector]); 3486 } 3487 return err; 3488 } 3489 3490 /** 3491 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 3492 * @vsi: the VSI being un-configured 3493 **/ 3494 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 3495 { 3496 struct i40e_pf *pf = vsi->back; 3497 struct i40e_hw *hw = &pf->hw; 3498 int base = vsi->base_vector; 3499 int i; 3500 3501 for (i = 0; i < vsi->num_queue_pairs; i++) { 3502 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0); 3503 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0); 3504 } 3505 3506 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3507 for (i = vsi->base_vector; 3508 i < (vsi->num_q_vectors + vsi->base_vector); i++) 3509 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 3510 3511 i40e_flush(hw); 3512 for (i = 0; i < vsi->num_q_vectors; i++) 3513 synchronize_irq(pf->msix_entries[i + base].vector); 3514 } else { 3515 /* Legacy and MSI mode - this stops all interrupt handling */ 3516 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 3517 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 3518 i40e_flush(hw); 3519 synchronize_irq(pf->pdev->irq); 3520 } 3521 } 3522 3523 /** 3524 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 3525 * @vsi: the VSI being configured 3526 **/ 3527 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 3528 { 3529 struct i40e_pf *pf = vsi->back; 3530 int i; 3531 3532 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3533 for (i = 0; i < vsi->num_q_vectors; i++) 3534 i40e_irq_dynamic_enable(vsi, i); 3535 } else { 3536 i40e_irq_dynamic_enable_icr0(pf, true); 3537 } 3538 3539 i40e_flush(&pf->hw); 3540 return 0; 3541 } 3542 3543 /** 3544 * i40e_stop_misc_vector - Stop the vector that handles non-queue events 3545 * @pf: board private structure 3546 **/ 3547 static void i40e_stop_misc_vector(struct i40e_pf *pf) 3548 { 3549 /* Disable ICR 0 */ 3550 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 3551 i40e_flush(&pf->hw); 3552 } 3553 3554 /** 3555 * i40e_intr - MSI/Legacy and non-queue interrupt handler 3556 * @irq: interrupt number 3557 * @data: pointer to a q_vector 3558 * 3559 * This is the handler used for all MSI/Legacy interrupts, and deals 3560 * with both queue and non-queue interrupts. This is also used in 3561 * MSIX mode to handle the non-queue interrupts. 3562 **/ 3563 static irqreturn_t i40e_intr(int irq, void *data) 3564 { 3565 struct i40e_pf *pf = (struct i40e_pf *)data; 3566 struct i40e_hw *hw = &pf->hw; 3567 irqreturn_t ret = IRQ_NONE; 3568 u32 icr0, icr0_remaining; 3569 u32 val, ena_mask; 3570 3571 icr0 = rd32(hw, I40E_PFINT_ICR0); 3572 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 3573 3574 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 3575 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 3576 goto enable_intr; 3577 3578 /* if interrupt but no bits showing, must be SWINT */ 3579 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 3580 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 3581 pf->sw_int_count++; 3582 3583 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 3584 (ena_mask & I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK)) { 3585 ena_mask &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3586 icr0 &= ~I40E_PFINT_ICR0_ENA_PE_CRITERR_MASK; 3587 dev_dbg(&pf->pdev->dev, "cleared PE_CRITERR\n"); 3588 } 3589 3590 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 3591 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 3592 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 3593 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 3594 3595 /* We do not have a way to disarm Queue causes while leaving 3596 * interrupt enabled for all other causes, ideally 3597 * interrupt should be disabled while we are in NAPI but 3598 * this is not a performance path and napi_schedule() 3599 * can deal with rescheduling. 3600 */ 3601 if (!test_bit(__I40E_DOWN, &pf->state)) 3602 napi_schedule_irqoff(&q_vector->napi); 3603 } 3604 3605 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 3606 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 3607 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state); 3608 i40e_debug(&pf->hw, I40E_DEBUG_NVM, "AdminQ event\n"); 3609 } 3610 3611 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 3612 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 3613 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state); 3614 } 3615 3616 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 3617 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 3618 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state); 3619 } 3620 3621 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 3622 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 3623 set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state); 3624 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 3625 val = rd32(hw, I40E_GLGEN_RSTAT); 3626 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 3627 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 3628 if (val == I40E_RESET_CORER) { 3629 pf->corer_count++; 3630 } else if (val == I40E_RESET_GLOBR) { 3631 pf->globr_count++; 3632 } else if (val == I40E_RESET_EMPR) { 3633 pf->empr_count++; 3634 set_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state); 3635 } 3636 } 3637 3638 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 3639 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 3640 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 3641 dev_info(&pf->pdev->dev, "HMC error info 0x%x, HMC error data 0x%x\n", 3642 rd32(hw, I40E_PFHMC_ERRORINFO), 3643 rd32(hw, I40E_PFHMC_ERRORDATA)); 3644 } 3645 3646 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 3647 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 3648 3649 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 3650 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 3651 i40e_ptp_tx_hwtstamp(pf); 3652 } 3653 } 3654 3655 /* If a critical error is pending we have no choice but to reset the 3656 * device. 3657 * Report and mask out any remaining unexpected interrupts. 3658 */ 3659 icr0_remaining = icr0 & ena_mask; 3660 if (icr0_remaining) { 3661 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3662 icr0_remaining); 3663 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3664 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3665 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3666 dev_info(&pf->pdev->dev, "device will be reset\n"); 3667 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 3668 i40e_service_event_schedule(pf); 3669 } 3670 ena_mask &= ~icr0_remaining; 3671 } 3672 ret = IRQ_HANDLED; 3673 3674 enable_intr: 3675 /* re-enable interrupt causes */ 3676 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3677 if (!test_bit(__I40E_DOWN, &pf->state)) { 3678 i40e_service_event_schedule(pf); 3679 i40e_irq_dynamic_enable_icr0(pf, false); 3680 } 3681 3682 return ret; 3683 } 3684 3685 /** 3686 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3687 * @tx_ring: tx ring to clean 3688 * @budget: how many cleans we're allowed 3689 * 3690 * Returns true if there's any budget left (e.g. the clean is finished) 3691 **/ 3692 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3693 { 3694 struct i40e_vsi *vsi = tx_ring->vsi; 3695 u16 i = tx_ring->next_to_clean; 3696 struct i40e_tx_buffer *tx_buf; 3697 struct i40e_tx_desc *tx_desc; 3698 3699 tx_buf = &tx_ring->tx_bi[i]; 3700 tx_desc = I40E_TX_DESC(tx_ring, i); 3701 i -= tx_ring->count; 3702 3703 do { 3704 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3705 3706 /* if next_to_watch is not set then there is no work pending */ 3707 if (!eop_desc) 3708 break; 3709 3710 /* prevent any other reads prior to eop_desc */ 3711 read_barrier_depends(); 3712 3713 /* if the descriptor isn't done, no work yet to do */ 3714 if (!(eop_desc->cmd_type_offset_bsz & 3715 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3716 break; 3717 3718 /* clear next_to_watch to prevent false hangs */ 3719 tx_buf->next_to_watch = NULL; 3720 3721 tx_desc->buffer_addr = 0; 3722 tx_desc->cmd_type_offset_bsz = 0; 3723 /* move past filter desc */ 3724 tx_buf++; 3725 tx_desc++; 3726 i++; 3727 if (unlikely(!i)) { 3728 i -= tx_ring->count; 3729 tx_buf = tx_ring->tx_bi; 3730 tx_desc = I40E_TX_DESC(tx_ring, 0); 3731 } 3732 /* unmap skb header data */ 3733 dma_unmap_single(tx_ring->dev, 3734 dma_unmap_addr(tx_buf, dma), 3735 dma_unmap_len(tx_buf, len), 3736 DMA_TO_DEVICE); 3737 if (tx_buf->tx_flags & I40E_TX_FLAGS_FD_SB) 3738 kfree(tx_buf->raw_buf); 3739 3740 tx_buf->raw_buf = NULL; 3741 tx_buf->tx_flags = 0; 3742 tx_buf->next_to_watch = NULL; 3743 dma_unmap_len_set(tx_buf, len, 0); 3744 tx_desc->buffer_addr = 0; 3745 tx_desc->cmd_type_offset_bsz = 0; 3746 3747 /* move us past the eop_desc for start of next FD desc */ 3748 tx_buf++; 3749 tx_desc++; 3750 i++; 3751 if (unlikely(!i)) { 3752 i -= tx_ring->count; 3753 tx_buf = tx_ring->tx_bi; 3754 tx_desc = I40E_TX_DESC(tx_ring, 0); 3755 } 3756 3757 /* update budget accounting */ 3758 budget--; 3759 } while (likely(budget)); 3760 3761 i += tx_ring->count; 3762 tx_ring->next_to_clean = i; 3763 3764 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) 3765 i40e_irq_dynamic_enable(vsi, tx_ring->q_vector->v_idx); 3766 3767 return budget > 0; 3768 } 3769 3770 /** 3771 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 3772 * @irq: interrupt number 3773 * @data: pointer to a q_vector 3774 **/ 3775 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 3776 { 3777 struct i40e_q_vector *q_vector = data; 3778 struct i40e_vsi *vsi; 3779 3780 if (!q_vector->tx.ring) 3781 return IRQ_HANDLED; 3782 3783 vsi = q_vector->tx.ring->vsi; 3784 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 3785 3786 return IRQ_HANDLED; 3787 } 3788 3789 /** 3790 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 3791 * @vsi: the VSI being configured 3792 * @v_idx: vector index 3793 * @qp_idx: queue pair index 3794 **/ 3795 static void i40e_map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 3796 { 3797 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 3798 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 3799 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 3800 3801 tx_ring->q_vector = q_vector; 3802 tx_ring->next = q_vector->tx.ring; 3803 q_vector->tx.ring = tx_ring; 3804 q_vector->tx.count++; 3805 3806 rx_ring->q_vector = q_vector; 3807 rx_ring->next = q_vector->rx.ring; 3808 q_vector->rx.ring = rx_ring; 3809 q_vector->rx.count++; 3810 } 3811 3812 /** 3813 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 3814 * @vsi: the VSI being configured 3815 * 3816 * This function maps descriptor rings to the queue-specific vectors 3817 * we were allotted through the MSI-X enabling code. Ideally, we'd have 3818 * one vector per queue pair, but on a constrained vector budget, we 3819 * group the queue pairs as "efficiently" as possible. 3820 **/ 3821 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 3822 { 3823 int qp_remaining = vsi->num_queue_pairs; 3824 int q_vectors = vsi->num_q_vectors; 3825 int num_ringpairs; 3826 int v_start = 0; 3827 int qp_idx = 0; 3828 3829 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 3830 * group them so there are multiple queues per vector. 3831 * It is also important to go through all the vectors available to be 3832 * sure that if we don't use all the vectors, that the remaining vectors 3833 * are cleared. This is especially important when decreasing the 3834 * number of queues in use. 3835 */ 3836 for (; v_start < q_vectors; v_start++) { 3837 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 3838 3839 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 3840 3841 q_vector->num_ringpairs = num_ringpairs; 3842 3843 q_vector->rx.count = 0; 3844 q_vector->tx.count = 0; 3845 q_vector->rx.ring = NULL; 3846 q_vector->tx.ring = NULL; 3847 3848 while (num_ringpairs--) { 3849 i40e_map_vector_to_qp(vsi, v_start, qp_idx); 3850 qp_idx++; 3851 qp_remaining--; 3852 } 3853 } 3854 } 3855 3856 /** 3857 * i40e_vsi_request_irq - Request IRQ from the OS 3858 * @vsi: the VSI being configured 3859 * @basename: name for the vector 3860 **/ 3861 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 3862 { 3863 struct i40e_pf *pf = vsi->back; 3864 int err; 3865 3866 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 3867 err = i40e_vsi_request_irq_msix(vsi, basename); 3868 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 3869 err = request_irq(pf->pdev->irq, i40e_intr, 0, 3870 pf->int_name, pf); 3871 else 3872 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 3873 pf->int_name, pf); 3874 3875 if (err) 3876 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 3877 3878 return err; 3879 } 3880 3881 #ifdef CONFIG_NET_POLL_CONTROLLER 3882 /** 3883 * i40e_netpoll - A Polling 'interrupt' handler 3884 * @netdev: network interface device structure 3885 * 3886 * This is used by netconsole to send skbs without having to re-enable 3887 * interrupts. It's not called while the normal interrupt routine is executing. 3888 **/ 3889 static void i40e_netpoll(struct net_device *netdev) 3890 { 3891 struct i40e_netdev_priv *np = netdev_priv(netdev); 3892 struct i40e_vsi *vsi = np->vsi; 3893 struct i40e_pf *pf = vsi->back; 3894 int i; 3895 3896 /* if interface is down do nothing */ 3897 if (test_bit(__I40E_DOWN, &vsi->state)) 3898 return; 3899 3900 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3901 for (i = 0; i < vsi->num_q_vectors; i++) 3902 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 3903 } else { 3904 i40e_intr(pf->pdev->irq, netdev); 3905 } 3906 } 3907 #endif 3908 3909 /** 3910 * i40e_pf_txq_wait - Wait for a PF's Tx queue to be enabled or disabled 3911 * @pf: the PF being configured 3912 * @pf_q: the PF queue 3913 * @enable: enable or disable state of the queue 3914 * 3915 * This routine will wait for the given Tx queue of the PF to reach the 3916 * enabled or disabled state. 3917 * Returns -ETIMEDOUT in case of failing to reach the requested state after 3918 * multiple retries; else will return 0 in case of success. 3919 **/ 3920 static int i40e_pf_txq_wait(struct i40e_pf *pf, int pf_q, bool enable) 3921 { 3922 int i; 3923 u32 tx_reg; 3924 3925 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 3926 tx_reg = rd32(&pf->hw, I40E_QTX_ENA(pf_q)); 3927 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3928 break; 3929 3930 usleep_range(10, 20); 3931 } 3932 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 3933 return -ETIMEDOUT; 3934 3935 return 0; 3936 } 3937 3938 /** 3939 * i40e_vsi_control_tx - Start or stop a VSI's rings 3940 * @vsi: the VSI being configured 3941 * @enable: start or stop the rings 3942 **/ 3943 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 3944 { 3945 struct i40e_pf *pf = vsi->back; 3946 struct i40e_hw *hw = &pf->hw; 3947 int i, j, pf_q, ret = 0; 3948 u32 tx_reg; 3949 3950 pf_q = vsi->base_queue; 3951 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 3952 3953 /* warn the TX unit of coming changes */ 3954 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 3955 if (!enable) 3956 usleep_range(10, 20); 3957 3958 for (j = 0; j < 50; j++) { 3959 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 3960 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 3961 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 3962 break; 3963 usleep_range(1000, 2000); 3964 } 3965 /* Skip if the queue is already in the requested state */ 3966 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3967 continue; 3968 3969 /* turn on/off the queue */ 3970 if (enable) { 3971 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 3972 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 3973 } else { 3974 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 3975 } 3976 3977 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 3978 /* No waiting for the Tx queue to disable */ 3979 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state)) 3980 continue; 3981 3982 /* wait for the change to finish */ 3983 ret = i40e_pf_txq_wait(pf, pf_q, enable); 3984 if (ret) { 3985 dev_info(&pf->pdev->dev, 3986 "VSI seid %d Tx ring %d %sable timeout\n", 3987 vsi->seid, pf_q, (enable ? "en" : "dis")); 3988 break; 3989 } 3990 } 3991 3992 return ret; 3993 } 3994 3995 /** 3996 * i40e_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled 3997 * @pf: the PF being configured 3998 * @pf_q: the PF queue 3999 * @enable: enable or disable state of the queue 4000 * 4001 * This routine will wait for the given Rx queue of the PF to reach the 4002 * enabled or disabled state. 4003 * Returns -ETIMEDOUT in case of failing to reach the requested state after 4004 * multiple retries; else will return 0 in case of success. 4005 **/ 4006 static int i40e_pf_rxq_wait(struct i40e_pf *pf, int pf_q, bool enable) 4007 { 4008 int i; 4009 u32 rx_reg; 4010 4011 for (i = 0; i < I40E_QUEUE_WAIT_RETRY_LIMIT; i++) { 4012 rx_reg = rd32(&pf->hw, I40E_QRX_ENA(pf_q)); 4013 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4014 break; 4015 4016 usleep_range(10, 20); 4017 } 4018 if (i >= I40E_QUEUE_WAIT_RETRY_LIMIT) 4019 return -ETIMEDOUT; 4020 4021 return 0; 4022 } 4023 4024 /** 4025 * i40e_vsi_control_rx - Start or stop a VSI's rings 4026 * @vsi: the VSI being configured 4027 * @enable: start or stop the rings 4028 **/ 4029 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 4030 { 4031 struct i40e_pf *pf = vsi->back; 4032 struct i40e_hw *hw = &pf->hw; 4033 int i, j, pf_q, ret = 0; 4034 u32 rx_reg; 4035 4036 pf_q = vsi->base_queue; 4037 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4038 for (j = 0; j < 50; j++) { 4039 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 4040 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 4041 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 4042 break; 4043 usleep_range(1000, 2000); 4044 } 4045 4046 /* Skip if the queue is already in the requested state */ 4047 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 4048 continue; 4049 4050 /* turn on/off the queue */ 4051 if (enable) 4052 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 4053 else 4054 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 4055 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 4056 /* No waiting for the Tx queue to disable */ 4057 if (!enable && test_bit(__I40E_PORT_TX_SUSPENDED, &pf->state)) 4058 continue; 4059 4060 /* wait for the change to finish */ 4061 ret = i40e_pf_rxq_wait(pf, pf_q, enable); 4062 if (ret) { 4063 dev_info(&pf->pdev->dev, 4064 "VSI seid %d Rx ring %d %sable timeout\n", 4065 vsi->seid, pf_q, (enable ? "en" : "dis")); 4066 break; 4067 } 4068 } 4069 4070 /* Due to HW errata, on Rx disable only, the register can indicate done 4071 * before it really is. Needs 50ms to be sure 4072 */ 4073 if (!enable) 4074 mdelay(50); 4075 4076 return ret; 4077 } 4078 4079 /** 4080 * i40e_vsi_start_rings - Start a VSI's rings 4081 * @vsi: the VSI being configured 4082 **/ 4083 int i40e_vsi_start_rings(struct i40e_vsi *vsi) 4084 { 4085 int ret = 0; 4086 4087 /* do rx first for enable and last for disable */ 4088 ret = i40e_vsi_control_rx(vsi, true); 4089 if (ret) 4090 return ret; 4091 ret = i40e_vsi_control_tx(vsi, true); 4092 4093 return ret; 4094 } 4095 4096 /** 4097 * i40e_vsi_stop_rings - Stop a VSI's rings 4098 * @vsi: the VSI being configured 4099 **/ 4100 void i40e_vsi_stop_rings(struct i40e_vsi *vsi) 4101 { 4102 /* do rx first for enable and last for disable 4103 * Ignore return value, we need to shutdown whatever we can 4104 */ 4105 i40e_vsi_control_tx(vsi, false); 4106 i40e_vsi_control_rx(vsi, false); 4107 } 4108 4109 /** 4110 * i40e_vsi_free_irq - Free the irq association with the OS 4111 * @vsi: the VSI being configured 4112 **/ 4113 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 4114 { 4115 struct i40e_pf *pf = vsi->back; 4116 struct i40e_hw *hw = &pf->hw; 4117 int base = vsi->base_vector; 4118 u32 val, qp; 4119 int i; 4120 4121 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4122 if (!vsi->q_vectors) 4123 return; 4124 4125 if (!vsi->irqs_ready) 4126 return; 4127 4128 vsi->irqs_ready = false; 4129 for (i = 0; i < vsi->num_q_vectors; i++) { 4130 int irq_num; 4131 u16 vector; 4132 4133 vector = i + base; 4134 irq_num = pf->msix_entries[vector].vector; 4135 4136 /* free only the irqs that were actually requested */ 4137 if (!vsi->q_vectors[i] || 4138 !vsi->q_vectors[i]->num_ringpairs) 4139 continue; 4140 4141 /* clear the affinity notifier in the IRQ descriptor */ 4142 irq_set_affinity_notifier(irq_num, NULL); 4143 /* clear the affinity_mask in the IRQ descriptor */ 4144 irq_set_affinity_hint(irq_num, NULL); 4145 synchronize_irq(irq_num); 4146 free_irq(irq_num, vsi->q_vectors[i]); 4147 4148 /* Tear down the interrupt queue link list 4149 * 4150 * We know that they come in pairs and always 4151 * the Rx first, then the Tx. To clear the 4152 * link list, stick the EOL value into the 4153 * next_q field of the registers. 4154 */ 4155 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 4156 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4157 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4158 val |= I40E_QUEUE_END_OF_LIST 4159 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4160 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 4161 4162 while (qp != I40E_QUEUE_END_OF_LIST) { 4163 u32 next; 4164 4165 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4166 4167 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4168 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4169 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4170 I40E_QINT_RQCTL_INTEVENT_MASK); 4171 4172 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4173 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4174 4175 wr32(hw, I40E_QINT_RQCTL(qp), val); 4176 4177 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4178 4179 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 4180 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 4181 4182 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4183 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4184 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4185 I40E_QINT_TQCTL_INTEVENT_MASK); 4186 4187 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4188 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4189 4190 wr32(hw, I40E_QINT_TQCTL(qp), val); 4191 qp = next; 4192 } 4193 } 4194 } else { 4195 free_irq(pf->pdev->irq, pf); 4196 4197 val = rd32(hw, I40E_PFINT_LNKLST0); 4198 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 4199 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 4200 val |= I40E_QUEUE_END_OF_LIST 4201 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 4202 wr32(hw, I40E_PFINT_LNKLST0, val); 4203 4204 val = rd32(hw, I40E_QINT_RQCTL(qp)); 4205 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 4206 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 4207 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 4208 I40E_QINT_RQCTL_INTEVENT_MASK); 4209 4210 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 4211 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 4212 4213 wr32(hw, I40E_QINT_RQCTL(qp), val); 4214 4215 val = rd32(hw, I40E_QINT_TQCTL(qp)); 4216 4217 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 4218 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 4219 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 4220 I40E_QINT_TQCTL_INTEVENT_MASK); 4221 4222 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 4223 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 4224 4225 wr32(hw, I40E_QINT_TQCTL(qp), val); 4226 } 4227 } 4228 4229 /** 4230 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 4231 * @vsi: the VSI being configured 4232 * @v_idx: Index of vector to be freed 4233 * 4234 * This function frees the memory allocated to the q_vector. In addition if 4235 * NAPI is enabled it will delete any references to the NAPI struct prior 4236 * to freeing the q_vector. 4237 **/ 4238 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 4239 { 4240 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 4241 struct i40e_ring *ring; 4242 4243 if (!q_vector) 4244 return; 4245 4246 /* disassociate q_vector from rings */ 4247 i40e_for_each_ring(ring, q_vector->tx) 4248 ring->q_vector = NULL; 4249 4250 i40e_for_each_ring(ring, q_vector->rx) 4251 ring->q_vector = NULL; 4252 4253 /* only VSI w/ an associated netdev is set up w/ NAPI */ 4254 if (vsi->netdev) 4255 netif_napi_del(&q_vector->napi); 4256 4257 vsi->q_vectors[v_idx] = NULL; 4258 4259 kfree_rcu(q_vector, rcu); 4260 } 4261 4262 /** 4263 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 4264 * @vsi: the VSI being un-configured 4265 * 4266 * This frees the memory allocated to the q_vectors and 4267 * deletes references to the NAPI struct. 4268 **/ 4269 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 4270 { 4271 int v_idx; 4272 4273 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 4274 i40e_free_q_vector(vsi, v_idx); 4275 } 4276 4277 /** 4278 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 4279 * @pf: board private structure 4280 **/ 4281 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 4282 { 4283 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 4284 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 4285 pci_disable_msix(pf->pdev); 4286 kfree(pf->msix_entries); 4287 pf->msix_entries = NULL; 4288 kfree(pf->irq_pile); 4289 pf->irq_pile = NULL; 4290 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 4291 pci_disable_msi(pf->pdev); 4292 } 4293 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 4294 } 4295 4296 /** 4297 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 4298 * @pf: board private structure 4299 * 4300 * We go through and clear interrupt specific resources and reset the structure 4301 * to pre-load conditions 4302 **/ 4303 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 4304 { 4305 int i; 4306 4307 i40e_stop_misc_vector(pf); 4308 if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { 4309 synchronize_irq(pf->msix_entries[0].vector); 4310 free_irq(pf->msix_entries[0].vector, pf); 4311 } 4312 4313 i40e_put_lump(pf->irq_pile, pf->iwarp_base_vector, 4314 I40E_IWARP_IRQ_PILE_ID); 4315 4316 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 4317 for (i = 0; i < pf->num_alloc_vsi; i++) 4318 if (pf->vsi[i]) 4319 i40e_vsi_free_q_vectors(pf->vsi[i]); 4320 i40e_reset_interrupt_capability(pf); 4321 } 4322 4323 /** 4324 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 4325 * @vsi: the VSI being configured 4326 **/ 4327 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 4328 { 4329 int q_idx; 4330 4331 if (!vsi->netdev) 4332 return; 4333 4334 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4335 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4336 4337 if (q_vector->rx.ring || q_vector->tx.ring) 4338 napi_enable(&q_vector->napi); 4339 } 4340 } 4341 4342 /** 4343 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 4344 * @vsi: the VSI being configured 4345 **/ 4346 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 4347 { 4348 int q_idx; 4349 4350 if (!vsi->netdev) 4351 return; 4352 4353 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) { 4354 struct i40e_q_vector *q_vector = vsi->q_vectors[q_idx]; 4355 4356 if (q_vector->rx.ring || q_vector->tx.ring) 4357 napi_disable(&q_vector->napi); 4358 } 4359 } 4360 4361 /** 4362 * i40e_vsi_close - Shut down a VSI 4363 * @vsi: the vsi to be quelled 4364 **/ 4365 static void i40e_vsi_close(struct i40e_vsi *vsi) 4366 { 4367 struct i40e_pf *pf = vsi->back; 4368 if (!test_and_set_bit(__I40E_DOWN, &vsi->state)) 4369 i40e_down(vsi); 4370 i40e_vsi_free_irq(vsi); 4371 i40e_vsi_free_tx_resources(vsi); 4372 i40e_vsi_free_rx_resources(vsi); 4373 vsi->current_netdev_flags = 0; 4374 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 4375 if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 4376 pf->flags |= I40E_FLAG_CLIENT_RESET; 4377 } 4378 4379 /** 4380 * i40e_quiesce_vsi - Pause a given VSI 4381 * @vsi: the VSI being paused 4382 **/ 4383 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 4384 { 4385 if (test_bit(__I40E_DOWN, &vsi->state)) 4386 return; 4387 4388 set_bit(__I40E_NEEDS_RESTART, &vsi->state); 4389 if (vsi->netdev && netif_running(vsi->netdev)) 4390 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 4391 else 4392 i40e_vsi_close(vsi); 4393 } 4394 4395 /** 4396 * i40e_unquiesce_vsi - Resume a given VSI 4397 * @vsi: the VSI being resumed 4398 **/ 4399 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 4400 { 4401 if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state)) 4402 return; 4403 4404 clear_bit(__I40E_NEEDS_RESTART, &vsi->state); 4405 if (vsi->netdev && netif_running(vsi->netdev)) 4406 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 4407 else 4408 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 4409 } 4410 4411 /** 4412 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 4413 * @pf: the PF 4414 **/ 4415 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 4416 { 4417 int v; 4418 4419 for (v = 0; v < pf->num_alloc_vsi; v++) { 4420 if (pf->vsi[v]) 4421 i40e_quiesce_vsi(pf->vsi[v]); 4422 } 4423 } 4424 4425 /** 4426 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 4427 * @pf: the PF 4428 **/ 4429 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 4430 { 4431 int v; 4432 4433 for (v = 0; v < pf->num_alloc_vsi; v++) { 4434 if (pf->vsi[v]) 4435 i40e_unquiesce_vsi(pf->vsi[v]); 4436 } 4437 } 4438 4439 #ifdef CONFIG_I40E_DCB 4440 /** 4441 * i40e_vsi_wait_queues_disabled - Wait for VSI's queues to be disabled 4442 * @vsi: the VSI being configured 4443 * 4444 * This function waits for the given VSI's queues to be disabled. 4445 **/ 4446 static int i40e_vsi_wait_queues_disabled(struct i40e_vsi *vsi) 4447 { 4448 struct i40e_pf *pf = vsi->back; 4449 int i, pf_q, ret; 4450 4451 pf_q = vsi->base_queue; 4452 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4453 /* Check and wait for the disable status of the queue */ 4454 ret = i40e_pf_txq_wait(pf, pf_q, false); 4455 if (ret) { 4456 dev_info(&pf->pdev->dev, 4457 "VSI seid %d Tx ring %d disable timeout\n", 4458 vsi->seid, pf_q); 4459 return ret; 4460 } 4461 } 4462 4463 pf_q = vsi->base_queue; 4464 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 4465 /* Check and wait for the disable status of the queue */ 4466 ret = i40e_pf_rxq_wait(pf, pf_q, false); 4467 if (ret) { 4468 dev_info(&pf->pdev->dev, 4469 "VSI seid %d Rx ring %d disable timeout\n", 4470 vsi->seid, pf_q); 4471 return ret; 4472 } 4473 } 4474 4475 return 0; 4476 } 4477 4478 /** 4479 * i40e_pf_wait_queues_disabled - Wait for all queues of PF VSIs to be disabled 4480 * @pf: the PF 4481 * 4482 * This function waits for the queues to be in disabled state for all the 4483 * VSIs that are managed by this PF. 4484 **/ 4485 static int i40e_pf_wait_queues_disabled(struct i40e_pf *pf) 4486 { 4487 int v, ret = 0; 4488 4489 for (v = 0; v < pf->hw.func_caps.num_vsis; v++) { 4490 if (pf->vsi[v]) { 4491 ret = i40e_vsi_wait_queues_disabled(pf->vsi[v]); 4492 if (ret) 4493 break; 4494 } 4495 } 4496 4497 return ret; 4498 } 4499 4500 #endif 4501 4502 /** 4503 * i40e_detect_recover_hung_queue - Function to detect and recover hung_queue 4504 * @q_idx: TX queue number 4505 * @vsi: Pointer to VSI struct 4506 * 4507 * This function checks specified queue for given VSI. Detects hung condition. 4508 * Sets hung bit since it is two step process. Before next run of service task 4509 * if napi_poll runs, it reset 'hung' bit for respective q_vector. If not, 4510 * hung condition remain unchanged and during subsequent run, this function 4511 * issues SW interrupt to recover from hung condition. 4512 **/ 4513 static void i40e_detect_recover_hung_queue(int q_idx, struct i40e_vsi *vsi) 4514 { 4515 struct i40e_ring *tx_ring = NULL; 4516 struct i40e_pf *pf; 4517 u32 head, val, tx_pending_hw; 4518 int i; 4519 4520 pf = vsi->back; 4521 4522 /* now that we have an index, find the tx_ring struct */ 4523 for (i = 0; i < vsi->num_queue_pairs; i++) { 4524 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) { 4525 if (q_idx == vsi->tx_rings[i]->queue_index) { 4526 tx_ring = vsi->tx_rings[i]; 4527 break; 4528 } 4529 } 4530 } 4531 4532 if (!tx_ring) 4533 return; 4534 4535 /* Read interrupt register */ 4536 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4537 val = rd32(&pf->hw, 4538 I40E_PFINT_DYN_CTLN(tx_ring->q_vector->v_idx + 4539 tx_ring->vsi->base_vector - 1)); 4540 else 4541 val = rd32(&pf->hw, I40E_PFINT_DYN_CTL0); 4542 4543 head = i40e_get_head(tx_ring); 4544 4545 tx_pending_hw = i40e_get_tx_pending(tx_ring, false); 4546 4547 /* HW is done executing descriptors, updated HEAD write back, 4548 * but SW hasn't processed those descriptors. If interrupt is 4549 * not generated from this point ON, it could result into 4550 * dev_watchdog detecting timeout on those netdev_queue, 4551 * hence proactively trigger SW interrupt. 4552 */ 4553 if (tx_pending_hw && (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) { 4554 /* NAPI Poll didn't run and clear since it was set */ 4555 if (test_and_clear_bit(I40E_Q_VECTOR_HUNG_DETECT, 4556 &tx_ring->q_vector->hung_detected)) { 4557 netdev_info(vsi->netdev, "VSI_seid %d, Hung TX queue %d, tx_pending_hw: %d, NTC:0x%x, HWB: 0x%x, NTU: 0x%x, TAIL: 0x%x\n", 4558 vsi->seid, q_idx, tx_pending_hw, 4559 tx_ring->next_to_clean, head, 4560 tx_ring->next_to_use, 4561 readl(tx_ring->tail)); 4562 netdev_info(vsi->netdev, "VSI_seid %d, Issuing force_wb for TX queue %d, Interrupt Reg: 0x%x\n", 4563 vsi->seid, q_idx, val); 4564 i40e_force_wb(vsi, tx_ring->q_vector); 4565 } else { 4566 /* First Chance - detected possible hung */ 4567 set_bit(I40E_Q_VECTOR_HUNG_DETECT, 4568 &tx_ring->q_vector->hung_detected); 4569 } 4570 } 4571 4572 /* This is the case where we have interrupts missing, 4573 * so the tx_pending in HW will most likely be 0, but we 4574 * will have tx_pending in SW since the WB happened but the 4575 * interrupt got lost. 4576 */ 4577 if ((!tx_pending_hw) && i40e_get_tx_pending(tx_ring, true) && 4578 (!(val & I40E_PFINT_DYN_CTLN_INTENA_MASK))) { 4579 local_bh_disable(); 4580 if (napi_reschedule(&tx_ring->q_vector->napi)) 4581 tx_ring->tx_stats.tx_lost_interrupt++; 4582 local_bh_enable(); 4583 } 4584 } 4585 4586 /** 4587 * i40e_detect_recover_hung - Function to detect and recover hung_queues 4588 * @pf: pointer to PF struct 4589 * 4590 * LAN VSI has netdev and netdev has TX queues. This function is to check 4591 * each of those TX queues if they are hung, trigger recovery by issuing 4592 * SW interrupt. 4593 **/ 4594 static void i40e_detect_recover_hung(struct i40e_pf *pf) 4595 { 4596 struct net_device *netdev; 4597 struct i40e_vsi *vsi; 4598 int i; 4599 4600 /* Only for LAN VSI */ 4601 vsi = pf->vsi[pf->lan_vsi]; 4602 4603 if (!vsi) 4604 return; 4605 4606 /* Make sure, VSI state is not DOWN/RECOVERY_PENDING */ 4607 if (test_bit(__I40E_DOWN, &vsi->back->state) || 4608 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 4609 return; 4610 4611 /* Make sure type is MAIN VSI */ 4612 if (vsi->type != I40E_VSI_MAIN) 4613 return; 4614 4615 netdev = vsi->netdev; 4616 if (!netdev) 4617 return; 4618 4619 /* Bail out if netif_carrier is not OK */ 4620 if (!netif_carrier_ok(netdev)) 4621 return; 4622 4623 /* Go thru' TX queues for netdev */ 4624 for (i = 0; i < netdev->num_tx_queues; i++) { 4625 struct netdev_queue *q; 4626 4627 q = netdev_get_tx_queue(netdev, i); 4628 if (q) 4629 i40e_detect_recover_hung_queue(i, vsi); 4630 } 4631 } 4632 4633 /** 4634 * i40e_get_iscsi_tc_map - Return TC map for iSCSI APP 4635 * @pf: pointer to PF 4636 * 4637 * Get TC map for ISCSI PF type that will include iSCSI TC 4638 * and LAN TC. 4639 **/ 4640 static u8 i40e_get_iscsi_tc_map(struct i40e_pf *pf) 4641 { 4642 struct i40e_dcb_app_priority_table app; 4643 struct i40e_hw *hw = &pf->hw; 4644 u8 enabled_tc = 1; /* TC0 is always enabled */ 4645 u8 tc, i; 4646 /* Get the iSCSI APP TLV */ 4647 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4648 4649 for (i = 0; i < dcbcfg->numapps; i++) { 4650 app = dcbcfg->app[i]; 4651 if (app.selector == I40E_APP_SEL_TCPIP && 4652 app.protocolid == I40E_APP_PROTOID_ISCSI) { 4653 tc = dcbcfg->etscfg.prioritytable[app.priority]; 4654 enabled_tc |= BIT(tc); 4655 break; 4656 } 4657 } 4658 4659 return enabled_tc; 4660 } 4661 4662 /** 4663 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 4664 * @dcbcfg: the corresponding DCBx configuration structure 4665 * 4666 * Return the number of TCs from given DCBx configuration 4667 **/ 4668 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 4669 { 4670 int i, tc_unused = 0; 4671 u8 num_tc = 0; 4672 u8 ret = 0; 4673 4674 /* Scan the ETS Config Priority Table to find 4675 * traffic class enabled for a given priority 4676 * and create a bitmask of enabled TCs 4677 */ 4678 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) 4679 num_tc |= BIT(dcbcfg->etscfg.prioritytable[i]); 4680 4681 /* Now scan the bitmask to check for 4682 * contiguous TCs starting with TC0 4683 */ 4684 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4685 if (num_tc & BIT(i)) { 4686 if (!tc_unused) { 4687 ret++; 4688 } else { 4689 pr_err("Non-contiguous TC - Disabling DCB\n"); 4690 return 1; 4691 } 4692 } else { 4693 tc_unused = 1; 4694 } 4695 } 4696 4697 /* There is always at least TC0 */ 4698 if (!ret) 4699 ret = 1; 4700 4701 return ret; 4702 } 4703 4704 /** 4705 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 4706 * @dcbcfg: the corresponding DCBx configuration structure 4707 * 4708 * Query the current DCB configuration and return the number of 4709 * traffic classes enabled from the given DCBX config 4710 **/ 4711 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 4712 { 4713 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 4714 u8 enabled_tc = 1; 4715 u8 i; 4716 4717 for (i = 0; i < num_tc; i++) 4718 enabled_tc |= BIT(i); 4719 4720 return enabled_tc; 4721 } 4722 4723 /** 4724 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 4725 * @pf: PF being queried 4726 * 4727 * Return number of traffic classes enabled for the given PF 4728 **/ 4729 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 4730 { 4731 struct i40e_hw *hw = &pf->hw; 4732 u8 i, enabled_tc = 1; 4733 u8 num_tc = 0; 4734 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4735 4736 /* If DCB is not enabled then always in single TC */ 4737 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4738 return 1; 4739 4740 /* SFP mode will be enabled for all TCs on port */ 4741 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4742 return i40e_dcb_get_num_tc(dcbcfg); 4743 4744 /* MFP mode return count of enabled TCs for this PF */ 4745 if (pf->hw.func_caps.iscsi) 4746 enabled_tc = i40e_get_iscsi_tc_map(pf); 4747 else 4748 return 1; /* Only TC0 */ 4749 4750 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4751 if (enabled_tc & BIT(i)) 4752 num_tc++; 4753 } 4754 return num_tc; 4755 } 4756 4757 /** 4758 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 4759 * @pf: PF being queried 4760 * 4761 * Return a bitmap for enabled traffic classes for this PF. 4762 **/ 4763 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 4764 { 4765 /* If DCB is not enabled for this PF then just return default TC */ 4766 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 4767 return I40E_DEFAULT_TRAFFIC_CLASS; 4768 4769 /* SFP mode we want PF to be enabled for all TCs */ 4770 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 4771 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 4772 4773 /* MFP enabled and iSCSI PF type */ 4774 if (pf->hw.func_caps.iscsi) 4775 return i40e_get_iscsi_tc_map(pf); 4776 else 4777 return I40E_DEFAULT_TRAFFIC_CLASS; 4778 } 4779 4780 /** 4781 * i40e_vsi_get_bw_info - Query VSI BW Information 4782 * @vsi: the VSI being queried 4783 * 4784 * Returns 0 on success, negative value on failure 4785 **/ 4786 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 4787 { 4788 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 4789 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 4790 struct i40e_pf *pf = vsi->back; 4791 struct i40e_hw *hw = &pf->hw; 4792 i40e_status ret; 4793 u32 tc_bw_max; 4794 int i; 4795 4796 /* Get the VSI level BW configuration */ 4797 ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 4798 if (ret) { 4799 dev_info(&pf->pdev->dev, 4800 "couldn't get PF vsi bw config, err %s aq_err %s\n", 4801 i40e_stat_str(&pf->hw, ret), 4802 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4803 return -EINVAL; 4804 } 4805 4806 /* Get the VSI level BW configuration per TC */ 4807 ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 4808 NULL); 4809 if (ret) { 4810 dev_info(&pf->pdev->dev, 4811 "couldn't get PF vsi ets bw config, err %s aq_err %s\n", 4812 i40e_stat_str(&pf->hw, ret), 4813 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 4814 return -EINVAL; 4815 } 4816 4817 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 4818 dev_info(&pf->pdev->dev, 4819 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 4820 bw_config.tc_valid_bits, 4821 bw_ets_config.tc_valid_bits); 4822 /* Still continuing */ 4823 } 4824 4825 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 4826 vsi->bw_max_quanta = bw_config.max_bw; 4827 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 4828 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 4829 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4830 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 4831 vsi->bw_ets_limit_credits[i] = 4832 le16_to_cpu(bw_ets_config.credits[i]); 4833 /* 3 bits out of 4 for each TC */ 4834 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 4835 } 4836 4837 return 0; 4838 } 4839 4840 /** 4841 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 4842 * @vsi: the VSI being configured 4843 * @enabled_tc: TC bitmap 4844 * @bw_credits: BW shared credits per TC 4845 * 4846 * Returns 0 on success, negative value on failure 4847 **/ 4848 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 4849 u8 *bw_share) 4850 { 4851 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 4852 i40e_status ret; 4853 int i; 4854 4855 bw_data.tc_valid_bits = enabled_tc; 4856 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4857 bw_data.tc_bw_credits[i] = bw_share[i]; 4858 4859 ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 4860 NULL); 4861 if (ret) { 4862 dev_info(&vsi->back->pdev->dev, 4863 "AQ command Config VSI BW allocation per TC failed = %d\n", 4864 vsi->back->hw.aq.asq_last_status); 4865 return -EINVAL; 4866 } 4867 4868 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 4869 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 4870 4871 return 0; 4872 } 4873 4874 /** 4875 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 4876 * @vsi: the VSI being configured 4877 * @enabled_tc: TC map to be enabled 4878 * 4879 **/ 4880 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 4881 { 4882 struct net_device *netdev = vsi->netdev; 4883 struct i40e_pf *pf = vsi->back; 4884 struct i40e_hw *hw = &pf->hw; 4885 u8 netdev_tc = 0; 4886 int i; 4887 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 4888 4889 if (!netdev) 4890 return; 4891 4892 if (!enabled_tc) { 4893 netdev_reset_tc(netdev); 4894 return; 4895 } 4896 4897 /* Set up actual enabled TCs on the VSI */ 4898 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 4899 return; 4900 4901 /* set per TC queues for the VSI */ 4902 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4903 /* Only set TC queues for enabled tcs 4904 * 4905 * e.g. For a VSI that has TC0 and TC3 enabled the 4906 * enabled_tc bitmap would be 0x00001001; the driver 4907 * will set the numtc for netdev as 2 that will be 4908 * referenced by the netdev layer as TC 0 and 1. 4909 */ 4910 if (vsi->tc_config.enabled_tc & BIT(i)) 4911 netdev_set_tc_queue(netdev, 4912 vsi->tc_config.tc_info[i].netdev_tc, 4913 vsi->tc_config.tc_info[i].qcount, 4914 vsi->tc_config.tc_info[i].qoffset); 4915 } 4916 4917 /* Assign UP2TC map for the VSI */ 4918 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 4919 /* Get the actual TC# for the UP */ 4920 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 4921 /* Get the mapped netdev TC# for the UP */ 4922 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 4923 netdev_set_prio_tc_map(netdev, i, netdev_tc); 4924 } 4925 } 4926 4927 /** 4928 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 4929 * @vsi: the VSI being configured 4930 * @ctxt: the ctxt buffer returned from AQ VSI update param command 4931 **/ 4932 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 4933 struct i40e_vsi_context *ctxt) 4934 { 4935 /* copy just the sections touched not the entire info 4936 * since not all sections are valid as returned by 4937 * update vsi params 4938 */ 4939 vsi->info.mapping_flags = ctxt->info.mapping_flags; 4940 memcpy(&vsi->info.queue_mapping, 4941 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 4942 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 4943 sizeof(vsi->info.tc_mapping)); 4944 } 4945 4946 /** 4947 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 4948 * @vsi: VSI to be configured 4949 * @enabled_tc: TC bitmap 4950 * 4951 * This configures a particular VSI for TCs that are mapped to the 4952 * given TC bitmap. It uses default bandwidth share for TCs across 4953 * VSIs to configure TC for a particular VSI. 4954 * 4955 * NOTE: 4956 * It is expected that the VSI queues have been quisced before calling 4957 * this function. 4958 **/ 4959 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 4960 { 4961 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 4962 struct i40e_vsi_context ctxt; 4963 int ret = 0; 4964 int i; 4965 4966 /* Check if enabled_tc is same as existing or new TCs */ 4967 if (vsi->tc_config.enabled_tc == enabled_tc) 4968 return ret; 4969 4970 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 4971 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4972 if (enabled_tc & BIT(i)) 4973 bw_share[i] = 1; 4974 } 4975 4976 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 4977 if (ret) { 4978 dev_info(&vsi->back->pdev->dev, 4979 "Failed configuring TC map %d for VSI %d\n", 4980 enabled_tc, vsi->seid); 4981 goto out; 4982 } 4983 4984 /* Update Queue Pairs Mapping for currently enabled UPs */ 4985 ctxt.seid = vsi->seid; 4986 ctxt.pf_num = vsi->back->hw.pf_id; 4987 ctxt.vf_num = 0; 4988 ctxt.uplink_seid = vsi->uplink_seid; 4989 ctxt.info = vsi->info; 4990 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 4991 4992 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 4993 ctxt.info.valid_sections |= 4994 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 4995 ctxt.info.queueing_opt_flags |= I40E_AQ_VSI_QUE_OPT_TCP_ENA; 4996 } 4997 4998 /* Update the VSI after updating the VSI queue-mapping information */ 4999 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 5000 if (ret) { 5001 dev_info(&vsi->back->pdev->dev, 5002 "Update vsi tc config failed, err %s aq_err %s\n", 5003 i40e_stat_str(&vsi->back->hw, ret), 5004 i40e_aq_str(&vsi->back->hw, 5005 vsi->back->hw.aq.asq_last_status)); 5006 goto out; 5007 } 5008 /* update the local VSI info with updated queue map */ 5009 i40e_vsi_update_queue_map(vsi, &ctxt); 5010 vsi->info.valid_sections = 0; 5011 5012 /* Update current VSI BW information */ 5013 ret = i40e_vsi_get_bw_info(vsi); 5014 if (ret) { 5015 dev_info(&vsi->back->pdev->dev, 5016 "Failed updating vsi bw info, err %s aq_err %s\n", 5017 i40e_stat_str(&vsi->back->hw, ret), 5018 i40e_aq_str(&vsi->back->hw, 5019 vsi->back->hw.aq.asq_last_status)); 5020 goto out; 5021 } 5022 5023 /* Update the netdev TC setup */ 5024 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 5025 out: 5026 return ret; 5027 } 5028 5029 /** 5030 * i40e_veb_config_tc - Configure TCs for given VEB 5031 * @veb: given VEB 5032 * @enabled_tc: TC bitmap 5033 * 5034 * Configures given TC bitmap for VEB (switching) element 5035 **/ 5036 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 5037 { 5038 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 5039 struct i40e_pf *pf = veb->pf; 5040 int ret = 0; 5041 int i; 5042 5043 /* No TCs or already enabled TCs just return */ 5044 if (!enabled_tc || veb->enabled_tc == enabled_tc) 5045 return ret; 5046 5047 bw_data.tc_valid_bits = enabled_tc; 5048 /* bw_data.absolute_credits is not set (relative) */ 5049 5050 /* Enable ETS TCs with equal BW Share for now */ 5051 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 5052 if (enabled_tc & BIT(i)) 5053 bw_data.tc_bw_share_credits[i] = 1; 5054 } 5055 5056 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 5057 &bw_data, NULL); 5058 if (ret) { 5059 dev_info(&pf->pdev->dev, 5060 "VEB bw config failed, err %s aq_err %s\n", 5061 i40e_stat_str(&pf->hw, ret), 5062 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5063 goto out; 5064 } 5065 5066 /* Update the BW information */ 5067 ret = i40e_veb_get_bw_info(veb); 5068 if (ret) { 5069 dev_info(&pf->pdev->dev, 5070 "Failed getting veb bw config, err %s aq_err %s\n", 5071 i40e_stat_str(&pf->hw, ret), 5072 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5073 } 5074 5075 out: 5076 return ret; 5077 } 5078 5079 #ifdef CONFIG_I40E_DCB 5080 /** 5081 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 5082 * @pf: PF struct 5083 * 5084 * Reconfigure VEB/VSIs on a given PF; it is assumed that 5085 * the caller would've quiesce all the VSIs before calling 5086 * this function 5087 **/ 5088 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 5089 { 5090 u8 tc_map = 0; 5091 int ret; 5092 u8 v; 5093 5094 /* Enable the TCs available on PF to all VEBs */ 5095 tc_map = i40e_pf_get_tc_map(pf); 5096 for (v = 0; v < I40E_MAX_VEB; v++) { 5097 if (!pf->veb[v]) 5098 continue; 5099 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 5100 if (ret) { 5101 dev_info(&pf->pdev->dev, 5102 "Failed configuring TC for VEB seid=%d\n", 5103 pf->veb[v]->seid); 5104 /* Will try to configure as many components */ 5105 } 5106 } 5107 5108 /* Update each VSI */ 5109 for (v = 0; v < pf->num_alloc_vsi; v++) { 5110 if (!pf->vsi[v]) 5111 continue; 5112 5113 /* - Enable all TCs for the LAN VSI 5114 * - For all others keep them at TC0 for now 5115 */ 5116 if (v == pf->lan_vsi) 5117 tc_map = i40e_pf_get_tc_map(pf); 5118 else 5119 tc_map = I40E_DEFAULT_TRAFFIC_CLASS; 5120 5121 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 5122 if (ret) { 5123 dev_info(&pf->pdev->dev, 5124 "Failed configuring TC for VSI seid=%d\n", 5125 pf->vsi[v]->seid); 5126 /* Will try to configure as many components */ 5127 } else { 5128 /* Re-configure VSI vectors based on updated TC map */ 5129 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 5130 if (pf->vsi[v]->netdev) 5131 i40e_dcbnl_set_all(pf->vsi[v]); 5132 } 5133 } 5134 } 5135 5136 /** 5137 * i40e_resume_port_tx - Resume port Tx 5138 * @pf: PF struct 5139 * 5140 * Resume a port's Tx and issue a PF reset in case of failure to 5141 * resume. 5142 **/ 5143 static int i40e_resume_port_tx(struct i40e_pf *pf) 5144 { 5145 struct i40e_hw *hw = &pf->hw; 5146 int ret; 5147 5148 ret = i40e_aq_resume_port_tx(hw, NULL); 5149 if (ret) { 5150 dev_info(&pf->pdev->dev, 5151 "Resume Port Tx failed, err %s aq_err %s\n", 5152 i40e_stat_str(&pf->hw, ret), 5153 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5154 /* Schedule PF reset to recover */ 5155 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 5156 i40e_service_event_schedule(pf); 5157 } 5158 5159 return ret; 5160 } 5161 5162 /** 5163 * i40e_init_pf_dcb - Initialize DCB configuration 5164 * @pf: PF being configured 5165 * 5166 * Query the current DCB configuration and cache it 5167 * in the hardware structure 5168 **/ 5169 static int i40e_init_pf_dcb(struct i40e_pf *pf) 5170 { 5171 struct i40e_hw *hw = &pf->hw; 5172 int err = 0; 5173 5174 /* Do not enable DCB for SW1 and SW2 images even if the FW is capable */ 5175 if (pf->flags & I40E_FLAG_NO_DCB_SUPPORT) 5176 goto out; 5177 5178 /* Get the initial DCB configuration */ 5179 err = i40e_init_dcb(hw); 5180 if (!err) { 5181 /* Device/Function is not DCBX capable */ 5182 if ((!hw->func_caps.dcb) || 5183 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 5184 dev_info(&pf->pdev->dev, 5185 "DCBX offload is not supported or is disabled for this PF.\n"); 5186 } else { 5187 /* When status is not DISABLED then DCBX in FW */ 5188 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 5189 DCB_CAP_DCBX_VER_IEEE; 5190 5191 pf->flags |= I40E_FLAG_DCB_CAPABLE; 5192 /* Enable DCB tagging only when more than one TC 5193 * or explicitly disable if only one TC 5194 */ 5195 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5196 pf->flags |= I40E_FLAG_DCB_ENABLED; 5197 else 5198 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5199 dev_dbg(&pf->pdev->dev, 5200 "DCBX offload is supported for this PF.\n"); 5201 } 5202 } else { 5203 dev_info(&pf->pdev->dev, 5204 "Query for DCB configuration failed, err %s aq_err %s\n", 5205 i40e_stat_str(&pf->hw, err), 5206 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5207 } 5208 5209 out: 5210 return err; 5211 } 5212 #endif /* CONFIG_I40E_DCB */ 5213 #define SPEED_SIZE 14 5214 #define FC_SIZE 8 5215 /** 5216 * i40e_print_link_message - print link up or down 5217 * @vsi: the VSI for which link needs a message 5218 */ 5219 void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 5220 { 5221 enum i40e_aq_link_speed new_speed; 5222 char *speed = "Unknown"; 5223 char *fc = "Unknown"; 5224 char *fec = ""; 5225 char *an = ""; 5226 5227 new_speed = vsi->back->hw.phy.link_info.link_speed; 5228 5229 if ((vsi->current_isup == isup) && (vsi->current_speed == new_speed)) 5230 return; 5231 vsi->current_isup = isup; 5232 vsi->current_speed = new_speed; 5233 if (!isup) { 5234 netdev_info(vsi->netdev, "NIC Link is Down\n"); 5235 return; 5236 } 5237 5238 /* Warn user if link speed on NPAR enabled partition is not at 5239 * least 10GB 5240 */ 5241 if (vsi->back->hw.func_caps.npar_enable && 5242 (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_1GB || 5243 vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_100MB)) 5244 netdev_warn(vsi->netdev, 5245 "The partition detected link speed that is less than 10Gbps\n"); 5246 5247 switch (vsi->back->hw.phy.link_info.link_speed) { 5248 case I40E_LINK_SPEED_40GB: 5249 speed = "40 G"; 5250 break; 5251 case I40E_LINK_SPEED_20GB: 5252 speed = "20 G"; 5253 break; 5254 case I40E_LINK_SPEED_25GB: 5255 speed = "25 G"; 5256 break; 5257 case I40E_LINK_SPEED_10GB: 5258 speed = "10 G"; 5259 break; 5260 case I40E_LINK_SPEED_1GB: 5261 speed = "1000 M"; 5262 break; 5263 case I40E_LINK_SPEED_100MB: 5264 speed = "100 M"; 5265 break; 5266 default: 5267 break; 5268 } 5269 5270 switch (vsi->back->hw.fc.current_mode) { 5271 case I40E_FC_FULL: 5272 fc = "RX/TX"; 5273 break; 5274 case I40E_FC_TX_PAUSE: 5275 fc = "TX"; 5276 break; 5277 case I40E_FC_RX_PAUSE: 5278 fc = "RX"; 5279 break; 5280 default: 5281 fc = "None"; 5282 break; 5283 } 5284 5285 if (vsi->back->hw.phy.link_info.link_speed == I40E_LINK_SPEED_25GB) { 5286 fec = ", FEC: None"; 5287 an = ", Autoneg: False"; 5288 5289 if (vsi->back->hw.phy.link_info.an_info & I40E_AQ_AN_COMPLETED) 5290 an = ", Autoneg: True"; 5291 5292 if (vsi->back->hw.phy.link_info.fec_info & 5293 I40E_AQ_CONFIG_FEC_KR_ENA) 5294 fec = ", FEC: CL74 FC-FEC/BASE-R"; 5295 else if (vsi->back->hw.phy.link_info.fec_info & 5296 I40E_AQ_CONFIG_FEC_RS_ENA) 5297 fec = ", FEC: CL108 RS-FEC"; 5298 } 5299 5300 netdev_info(vsi->netdev, "NIC Link is Up, %sbps Full Duplex%s%s, Flow Control: %s\n", 5301 speed, fec, an, fc); 5302 } 5303 5304 /** 5305 * i40e_up_complete - Finish the last steps of bringing up a connection 5306 * @vsi: the VSI being configured 5307 **/ 5308 static int i40e_up_complete(struct i40e_vsi *vsi) 5309 { 5310 struct i40e_pf *pf = vsi->back; 5311 int err; 5312 5313 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 5314 i40e_vsi_configure_msix(vsi); 5315 else 5316 i40e_configure_msi_and_legacy(vsi); 5317 5318 /* start rings */ 5319 err = i40e_vsi_start_rings(vsi); 5320 if (err) 5321 return err; 5322 5323 clear_bit(__I40E_DOWN, &vsi->state); 5324 i40e_napi_enable_all(vsi); 5325 i40e_vsi_enable_irq(vsi); 5326 5327 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 5328 (vsi->netdev)) { 5329 i40e_print_link_message(vsi, true); 5330 netif_tx_start_all_queues(vsi->netdev); 5331 netif_carrier_on(vsi->netdev); 5332 } else if (vsi->netdev) { 5333 i40e_print_link_message(vsi, false); 5334 /* need to check for qualified module here*/ 5335 if ((pf->hw.phy.link_info.link_info & 5336 I40E_AQ_MEDIA_AVAILABLE) && 5337 (!(pf->hw.phy.link_info.an_info & 5338 I40E_AQ_QUALIFIED_MODULE))) 5339 netdev_err(vsi->netdev, 5340 "the driver failed to link because an unqualified module was detected."); 5341 } 5342 5343 /* replay FDIR SB filters */ 5344 if (vsi->type == I40E_VSI_FDIR) { 5345 /* reset fd counters */ 5346 pf->fd_add_err = 0; 5347 pf->fd_atr_cnt = 0; 5348 i40e_fdir_filter_restore(vsi); 5349 } 5350 5351 /* On the next run of the service_task, notify any clients of the new 5352 * opened netdev 5353 */ 5354 pf->flags |= I40E_FLAG_SERVICE_CLIENT_REQUESTED; 5355 i40e_service_event_schedule(pf); 5356 5357 return 0; 5358 } 5359 5360 /** 5361 * i40e_vsi_reinit_locked - Reset the VSI 5362 * @vsi: the VSI being configured 5363 * 5364 * Rebuild the ring structs after some configuration 5365 * has changed, e.g. MTU size. 5366 **/ 5367 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 5368 { 5369 struct i40e_pf *pf = vsi->back; 5370 5371 WARN_ON(in_interrupt()); 5372 while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state)) 5373 usleep_range(1000, 2000); 5374 i40e_down(vsi); 5375 5376 i40e_up(vsi); 5377 clear_bit(__I40E_CONFIG_BUSY, &pf->state); 5378 } 5379 5380 /** 5381 * i40e_up - Bring the connection back up after being down 5382 * @vsi: the VSI being configured 5383 **/ 5384 int i40e_up(struct i40e_vsi *vsi) 5385 { 5386 int err; 5387 5388 err = i40e_vsi_configure(vsi); 5389 if (!err) 5390 err = i40e_up_complete(vsi); 5391 5392 return err; 5393 } 5394 5395 /** 5396 * i40e_down - Shutdown the connection processing 5397 * @vsi: the VSI being stopped 5398 **/ 5399 void i40e_down(struct i40e_vsi *vsi) 5400 { 5401 int i; 5402 5403 /* It is assumed that the caller of this function 5404 * sets the vsi->state __I40E_DOWN bit. 5405 */ 5406 if (vsi->netdev) { 5407 netif_carrier_off(vsi->netdev); 5408 netif_tx_disable(vsi->netdev); 5409 } 5410 i40e_vsi_disable_irq(vsi); 5411 i40e_vsi_stop_rings(vsi); 5412 i40e_napi_disable_all(vsi); 5413 5414 for (i = 0; i < vsi->num_queue_pairs; i++) { 5415 i40e_clean_tx_ring(vsi->tx_rings[i]); 5416 i40e_clean_rx_ring(vsi->rx_rings[i]); 5417 } 5418 5419 } 5420 5421 /** 5422 * i40e_setup_tc - configure multiple traffic classes 5423 * @netdev: net device to configure 5424 * @tc: number of traffic classes to enable 5425 **/ 5426 static int i40e_setup_tc(struct net_device *netdev, u8 tc) 5427 { 5428 struct i40e_netdev_priv *np = netdev_priv(netdev); 5429 struct i40e_vsi *vsi = np->vsi; 5430 struct i40e_pf *pf = vsi->back; 5431 u8 enabled_tc = 0; 5432 int ret = -EINVAL; 5433 int i; 5434 5435 /* Check if DCB enabled to continue */ 5436 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 5437 netdev_info(netdev, "DCB is not enabled for adapter\n"); 5438 goto exit; 5439 } 5440 5441 /* Check if MFP enabled */ 5442 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 5443 netdev_info(netdev, "Configuring TC not supported in MFP mode\n"); 5444 goto exit; 5445 } 5446 5447 /* Check whether tc count is within enabled limit */ 5448 if (tc > i40e_pf_get_num_tc(pf)) { 5449 netdev_info(netdev, "TC count greater than enabled on link for adapter\n"); 5450 goto exit; 5451 } 5452 5453 /* Generate TC map for number of tc requested */ 5454 for (i = 0; i < tc; i++) 5455 enabled_tc |= BIT(i); 5456 5457 /* Requesting same TC configuration as already enabled */ 5458 if (enabled_tc == vsi->tc_config.enabled_tc) 5459 return 0; 5460 5461 /* Quiesce VSI queues */ 5462 i40e_quiesce_vsi(vsi); 5463 5464 /* Configure VSI for enabled TCs */ 5465 ret = i40e_vsi_config_tc(vsi, enabled_tc); 5466 if (ret) { 5467 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 5468 vsi->seid); 5469 goto exit; 5470 } 5471 5472 /* Unquiesce VSI */ 5473 i40e_unquiesce_vsi(vsi); 5474 5475 exit: 5476 return ret; 5477 } 5478 5479 static int __i40e_setup_tc(struct net_device *netdev, u32 handle, __be16 proto, 5480 struct tc_to_netdev *tc) 5481 { 5482 if (tc->type != TC_SETUP_MQPRIO) 5483 return -EINVAL; 5484 5485 tc->mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 5486 5487 return i40e_setup_tc(netdev, tc->mqprio->num_tc); 5488 } 5489 5490 /** 5491 * i40e_open - Called when a network interface is made active 5492 * @netdev: network interface device structure 5493 * 5494 * The open entry point is called when a network interface is made 5495 * active by the system (IFF_UP). At this point all resources needed 5496 * for transmit and receive operations are allocated, the interrupt 5497 * handler is registered with the OS, the netdev watchdog subtask is 5498 * enabled, and the stack is notified that the interface is ready. 5499 * 5500 * Returns 0 on success, negative value on failure 5501 **/ 5502 int i40e_open(struct net_device *netdev) 5503 { 5504 struct i40e_netdev_priv *np = netdev_priv(netdev); 5505 struct i40e_vsi *vsi = np->vsi; 5506 struct i40e_pf *pf = vsi->back; 5507 int err; 5508 5509 /* disallow open during test or if eeprom is broken */ 5510 if (test_bit(__I40E_TESTING, &pf->state) || 5511 test_bit(__I40E_BAD_EEPROM, &pf->state)) 5512 return -EBUSY; 5513 5514 netif_carrier_off(netdev); 5515 5516 err = i40e_vsi_open(vsi); 5517 if (err) 5518 return err; 5519 5520 /* configure global TSO hardware offload settings */ 5521 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 5522 TCP_FLAG_FIN) >> 16); 5523 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 5524 TCP_FLAG_FIN | 5525 TCP_FLAG_CWR) >> 16); 5526 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 5527 5528 udp_tunnel_get_rx_info(netdev); 5529 5530 return 0; 5531 } 5532 5533 /** 5534 * i40e_vsi_open - 5535 * @vsi: the VSI to open 5536 * 5537 * Finish initialization of the VSI. 5538 * 5539 * Returns 0 on success, negative value on failure 5540 **/ 5541 int i40e_vsi_open(struct i40e_vsi *vsi) 5542 { 5543 struct i40e_pf *pf = vsi->back; 5544 char int_name[I40E_INT_NAME_STR_LEN]; 5545 int err; 5546 5547 /* allocate descriptors */ 5548 err = i40e_vsi_setup_tx_resources(vsi); 5549 if (err) 5550 goto err_setup_tx; 5551 err = i40e_vsi_setup_rx_resources(vsi); 5552 if (err) 5553 goto err_setup_rx; 5554 5555 err = i40e_vsi_configure(vsi); 5556 if (err) 5557 goto err_setup_rx; 5558 5559 if (vsi->netdev) { 5560 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 5561 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 5562 err = i40e_vsi_request_irq(vsi, int_name); 5563 if (err) 5564 goto err_setup_rx; 5565 5566 /* Notify the stack of the actual queue counts. */ 5567 err = netif_set_real_num_tx_queues(vsi->netdev, 5568 vsi->num_queue_pairs); 5569 if (err) 5570 goto err_set_queues; 5571 5572 err = netif_set_real_num_rx_queues(vsi->netdev, 5573 vsi->num_queue_pairs); 5574 if (err) 5575 goto err_set_queues; 5576 5577 } else if (vsi->type == I40E_VSI_FDIR) { 5578 snprintf(int_name, sizeof(int_name) - 1, "%s-%s:fdir", 5579 dev_driver_string(&pf->pdev->dev), 5580 dev_name(&pf->pdev->dev)); 5581 err = i40e_vsi_request_irq(vsi, int_name); 5582 5583 } else { 5584 err = -EINVAL; 5585 goto err_setup_rx; 5586 } 5587 5588 err = i40e_up_complete(vsi); 5589 if (err) 5590 goto err_up_complete; 5591 5592 return 0; 5593 5594 err_up_complete: 5595 i40e_down(vsi); 5596 err_set_queues: 5597 i40e_vsi_free_irq(vsi); 5598 err_setup_rx: 5599 i40e_vsi_free_rx_resources(vsi); 5600 err_setup_tx: 5601 i40e_vsi_free_tx_resources(vsi); 5602 if (vsi == pf->vsi[pf->lan_vsi]) 5603 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 5604 5605 return err; 5606 } 5607 5608 /** 5609 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 5610 * @pf: Pointer to PF 5611 * 5612 * This function destroys the hlist where all the Flow Director 5613 * filters were saved. 5614 **/ 5615 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 5616 { 5617 struct i40e_fdir_filter *filter; 5618 struct i40e_flex_pit *pit_entry, *tmp; 5619 struct hlist_node *node2; 5620 5621 hlist_for_each_entry_safe(filter, node2, 5622 &pf->fdir_filter_list, fdir_node) { 5623 hlist_del(&filter->fdir_node); 5624 kfree(filter); 5625 } 5626 5627 list_for_each_entry_safe(pit_entry, tmp, &pf->l3_flex_pit_list, list) { 5628 list_del(&pit_entry->list); 5629 kfree(pit_entry); 5630 } 5631 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 5632 5633 list_for_each_entry_safe(pit_entry, tmp, &pf->l4_flex_pit_list, list) { 5634 list_del(&pit_entry->list); 5635 kfree(pit_entry); 5636 } 5637 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 5638 5639 pf->fdir_pf_active_filters = 0; 5640 pf->fd_tcp4_filter_cnt = 0; 5641 pf->fd_udp4_filter_cnt = 0; 5642 pf->fd_sctp4_filter_cnt = 0; 5643 pf->fd_ip4_filter_cnt = 0; 5644 5645 /* Reprogram the default input set for TCP/IPv4 */ 5646 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_TCP, 5647 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5648 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5649 5650 /* Reprogram the default input set for UDP/IPv4 */ 5651 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_UDP, 5652 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5653 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5654 5655 /* Reprogram the default input set for SCTP/IPv4 */ 5656 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_SCTP, 5657 I40E_L3_SRC_MASK | I40E_L3_DST_MASK | 5658 I40E_L4_SRC_MASK | I40E_L4_DST_MASK); 5659 5660 /* Reprogram the default input set for Other/IPv4 */ 5661 i40e_write_fd_input_set(pf, I40E_FILTER_PCTYPE_NONF_IPV4_OTHER, 5662 I40E_L3_SRC_MASK | I40E_L3_DST_MASK); 5663 } 5664 5665 /** 5666 * i40e_close - Disables a network interface 5667 * @netdev: network interface device structure 5668 * 5669 * The close entry point is called when an interface is de-activated 5670 * by the OS. The hardware is still under the driver's control, but 5671 * this netdev interface is disabled. 5672 * 5673 * Returns 0, this is not allowed to fail 5674 **/ 5675 int i40e_close(struct net_device *netdev) 5676 { 5677 struct i40e_netdev_priv *np = netdev_priv(netdev); 5678 struct i40e_vsi *vsi = np->vsi; 5679 5680 i40e_vsi_close(vsi); 5681 5682 return 0; 5683 } 5684 5685 /** 5686 * i40e_do_reset - Start a PF or Core Reset sequence 5687 * @pf: board private structure 5688 * @reset_flags: which reset is requested 5689 * 5690 * The essential difference in resets is that the PF Reset 5691 * doesn't clear the packet buffers, doesn't reset the PE 5692 * firmware, and doesn't bother the other PFs on the chip. 5693 **/ 5694 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags) 5695 { 5696 u32 val; 5697 5698 WARN_ON(in_interrupt()); 5699 5700 5701 /* do the biggest reset indicated */ 5702 if (reset_flags & BIT_ULL(__I40E_GLOBAL_RESET_REQUESTED)) { 5703 5704 /* Request a Global Reset 5705 * 5706 * This will start the chip's countdown to the actual full 5707 * chip reset event, and a warning interrupt to be sent 5708 * to all PFs, including the requestor. Our handler 5709 * for the warning interrupt will deal with the shutdown 5710 * and recovery of the switch setup. 5711 */ 5712 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 5713 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5714 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 5715 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5716 5717 } else if (reset_flags & BIT_ULL(__I40E_CORE_RESET_REQUESTED)) { 5718 5719 /* Request a Core Reset 5720 * 5721 * Same as Global Reset, except does *not* include the MAC/PHY 5722 */ 5723 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 5724 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 5725 val |= I40E_GLGEN_RTRIG_CORER_MASK; 5726 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 5727 i40e_flush(&pf->hw); 5728 5729 } else if (reset_flags & BIT_ULL(__I40E_PF_RESET_REQUESTED)) { 5730 5731 /* Request a PF Reset 5732 * 5733 * Resets only the PF-specific registers 5734 * 5735 * This goes directly to the tear-down and rebuild of 5736 * the switch, since we need to do all the recovery as 5737 * for the Core Reset. 5738 */ 5739 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 5740 i40e_handle_reset_warning(pf); 5741 5742 } else if (reset_flags & BIT_ULL(__I40E_REINIT_REQUESTED)) { 5743 int v; 5744 5745 /* Find the VSI(s) that requested a re-init */ 5746 dev_info(&pf->pdev->dev, 5747 "VSI reinit requested\n"); 5748 for (v = 0; v < pf->num_alloc_vsi; v++) { 5749 struct i40e_vsi *vsi = pf->vsi[v]; 5750 5751 if (vsi != NULL && 5752 test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) { 5753 i40e_vsi_reinit_locked(pf->vsi[v]); 5754 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state); 5755 } 5756 } 5757 } else if (reset_flags & BIT_ULL(__I40E_DOWN_REQUESTED)) { 5758 int v; 5759 5760 /* Find the VSI(s) that needs to be brought down */ 5761 dev_info(&pf->pdev->dev, "VSI down requested\n"); 5762 for (v = 0; v < pf->num_alloc_vsi; v++) { 5763 struct i40e_vsi *vsi = pf->vsi[v]; 5764 5765 if (vsi != NULL && 5766 test_bit(__I40E_DOWN_REQUESTED, &vsi->state)) { 5767 set_bit(__I40E_DOWN, &vsi->state); 5768 i40e_down(vsi); 5769 clear_bit(__I40E_DOWN_REQUESTED, &vsi->state); 5770 } 5771 } 5772 } else { 5773 dev_info(&pf->pdev->dev, 5774 "bad reset request 0x%08x\n", reset_flags); 5775 } 5776 } 5777 5778 #ifdef CONFIG_I40E_DCB 5779 /** 5780 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 5781 * @pf: board private structure 5782 * @old_cfg: current DCB config 5783 * @new_cfg: new DCB config 5784 **/ 5785 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 5786 struct i40e_dcbx_config *old_cfg, 5787 struct i40e_dcbx_config *new_cfg) 5788 { 5789 bool need_reconfig = false; 5790 5791 /* Check if ETS configuration has changed */ 5792 if (memcmp(&new_cfg->etscfg, 5793 &old_cfg->etscfg, 5794 sizeof(new_cfg->etscfg))) { 5795 /* If Priority Table has changed reconfig is needed */ 5796 if (memcmp(&new_cfg->etscfg.prioritytable, 5797 &old_cfg->etscfg.prioritytable, 5798 sizeof(new_cfg->etscfg.prioritytable))) { 5799 need_reconfig = true; 5800 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 5801 } 5802 5803 if (memcmp(&new_cfg->etscfg.tcbwtable, 5804 &old_cfg->etscfg.tcbwtable, 5805 sizeof(new_cfg->etscfg.tcbwtable))) 5806 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 5807 5808 if (memcmp(&new_cfg->etscfg.tsatable, 5809 &old_cfg->etscfg.tsatable, 5810 sizeof(new_cfg->etscfg.tsatable))) 5811 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 5812 } 5813 5814 /* Check if PFC configuration has changed */ 5815 if (memcmp(&new_cfg->pfc, 5816 &old_cfg->pfc, 5817 sizeof(new_cfg->pfc))) { 5818 need_reconfig = true; 5819 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 5820 } 5821 5822 /* Check if APP Table has changed */ 5823 if (memcmp(&new_cfg->app, 5824 &old_cfg->app, 5825 sizeof(new_cfg->app))) { 5826 need_reconfig = true; 5827 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 5828 } 5829 5830 dev_dbg(&pf->pdev->dev, "dcb need_reconfig=%d\n", need_reconfig); 5831 return need_reconfig; 5832 } 5833 5834 /** 5835 * i40e_handle_lldp_event - Handle LLDP Change MIB event 5836 * @pf: board private structure 5837 * @e: event info posted on ARQ 5838 **/ 5839 static int i40e_handle_lldp_event(struct i40e_pf *pf, 5840 struct i40e_arq_event_info *e) 5841 { 5842 struct i40e_aqc_lldp_get_mib *mib = 5843 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 5844 struct i40e_hw *hw = &pf->hw; 5845 struct i40e_dcbx_config tmp_dcbx_cfg; 5846 bool need_reconfig = false; 5847 int ret = 0; 5848 u8 type; 5849 5850 /* Not DCB capable or capability disabled */ 5851 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 5852 return ret; 5853 5854 /* Ignore if event is not for Nearest Bridge */ 5855 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 5856 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 5857 dev_dbg(&pf->pdev->dev, "LLDP event mib bridge type 0x%x\n", type); 5858 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 5859 return ret; 5860 5861 /* Check MIB Type and return if event for Remote MIB update */ 5862 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 5863 dev_dbg(&pf->pdev->dev, 5864 "LLDP event mib type %s\n", type ? "remote" : "local"); 5865 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 5866 /* Update the remote cached instance and return */ 5867 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 5868 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 5869 &hw->remote_dcbx_config); 5870 goto exit; 5871 } 5872 5873 /* Store the old configuration */ 5874 tmp_dcbx_cfg = hw->local_dcbx_config; 5875 5876 /* Reset the old DCBx configuration data */ 5877 memset(&hw->local_dcbx_config, 0, sizeof(hw->local_dcbx_config)); 5878 /* Get updated DCBX data from firmware */ 5879 ret = i40e_get_dcb_config(&pf->hw); 5880 if (ret) { 5881 dev_info(&pf->pdev->dev, 5882 "Failed querying DCB configuration data from firmware, err %s aq_err %s\n", 5883 i40e_stat_str(&pf->hw, ret), 5884 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 5885 goto exit; 5886 } 5887 5888 /* No change detected in DCBX configs */ 5889 if (!memcmp(&tmp_dcbx_cfg, &hw->local_dcbx_config, 5890 sizeof(tmp_dcbx_cfg))) { 5891 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 5892 goto exit; 5893 } 5894 5895 need_reconfig = i40e_dcb_need_reconfig(pf, &tmp_dcbx_cfg, 5896 &hw->local_dcbx_config); 5897 5898 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg, &hw->local_dcbx_config); 5899 5900 if (!need_reconfig) 5901 goto exit; 5902 5903 /* Enable DCB tagging only when more than one TC */ 5904 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 5905 pf->flags |= I40E_FLAG_DCB_ENABLED; 5906 else 5907 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 5908 5909 set_bit(__I40E_PORT_TX_SUSPENDED, &pf->state); 5910 /* Reconfiguration needed quiesce all VSIs */ 5911 i40e_pf_quiesce_all_vsi(pf); 5912 5913 /* Changes in configuration update VEB/VSI */ 5914 i40e_dcb_reconfigure(pf); 5915 5916 ret = i40e_resume_port_tx(pf); 5917 5918 clear_bit(__I40E_PORT_TX_SUSPENDED, &pf->state); 5919 /* In case of error no point in resuming VSIs */ 5920 if (ret) 5921 goto exit; 5922 5923 /* Wait for the PF's queues to be disabled */ 5924 ret = i40e_pf_wait_queues_disabled(pf); 5925 if (ret) { 5926 /* Schedule PF reset to recover */ 5927 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 5928 i40e_service_event_schedule(pf); 5929 } else { 5930 i40e_pf_unquiesce_all_vsi(pf); 5931 pf->flags |= (I40E_FLAG_SERVICE_CLIENT_REQUESTED | 5932 I40E_FLAG_CLIENT_L2_CHANGE); 5933 } 5934 5935 exit: 5936 return ret; 5937 } 5938 #endif /* CONFIG_I40E_DCB */ 5939 5940 /** 5941 * i40e_do_reset_safe - Protected reset path for userland calls. 5942 * @pf: board private structure 5943 * @reset_flags: which reset is requested 5944 * 5945 **/ 5946 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 5947 { 5948 rtnl_lock(); 5949 i40e_do_reset(pf, reset_flags); 5950 rtnl_unlock(); 5951 } 5952 5953 /** 5954 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 5955 * @pf: board private structure 5956 * @e: event info posted on ARQ 5957 * 5958 * Handler for LAN Queue Overflow Event generated by the firmware for PF 5959 * and VF queues 5960 **/ 5961 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 5962 struct i40e_arq_event_info *e) 5963 { 5964 struct i40e_aqc_lan_overflow *data = 5965 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 5966 u32 queue = le32_to_cpu(data->prtdcb_rupto); 5967 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 5968 struct i40e_hw *hw = &pf->hw; 5969 struct i40e_vf *vf; 5970 u16 vf_id; 5971 5972 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 5973 queue, qtx_ctl); 5974 5975 /* Queue belongs to VF, find the VF and issue VF reset */ 5976 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 5977 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 5978 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 5979 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 5980 vf_id -= hw->func_caps.vf_base_id; 5981 vf = &pf->vf[vf_id]; 5982 i40e_vc_notify_vf_reset(vf); 5983 /* Allow VF to process pending reset notification */ 5984 msleep(20); 5985 i40e_reset_vf(vf, false); 5986 } 5987 } 5988 5989 /** 5990 * i40e_get_cur_guaranteed_fd_count - Get the consumed guaranteed FD filters 5991 * @pf: board private structure 5992 **/ 5993 u32 i40e_get_cur_guaranteed_fd_count(struct i40e_pf *pf) 5994 { 5995 u32 val, fcnt_prog; 5996 5997 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 5998 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK); 5999 return fcnt_prog; 6000 } 6001 6002 /** 6003 * i40e_get_current_fd_count - Get total FD filters programmed for this PF 6004 * @pf: board private structure 6005 **/ 6006 u32 i40e_get_current_fd_count(struct i40e_pf *pf) 6007 { 6008 u32 val, fcnt_prog; 6009 6010 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 6011 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 6012 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 6013 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 6014 return fcnt_prog; 6015 } 6016 6017 /** 6018 * i40e_get_global_fd_count - Get total FD filters programmed on device 6019 * @pf: board private structure 6020 **/ 6021 u32 i40e_get_global_fd_count(struct i40e_pf *pf) 6022 { 6023 u32 val, fcnt_prog; 6024 6025 val = rd32(&pf->hw, I40E_GLQF_FDCNT_0); 6026 fcnt_prog = (val & I40E_GLQF_FDCNT_0_GUARANT_CNT_MASK) + 6027 ((val & I40E_GLQF_FDCNT_0_BESTCNT_MASK) >> 6028 I40E_GLQF_FDCNT_0_BESTCNT_SHIFT); 6029 return fcnt_prog; 6030 } 6031 6032 /** 6033 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 6034 * @pf: board private structure 6035 **/ 6036 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 6037 { 6038 struct i40e_fdir_filter *filter; 6039 u32 fcnt_prog, fcnt_avail; 6040 struct hlist_node *node; 6041 6042 if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state)) 6043 return; 6044 6045 /* Check if, FD SB or ATR was auto disabled and if there is enough room 6046 * to re-enable 6047 */ 6048 fcnt_prog = i40e_get_global_fd_count(pf); 6049 fcnt_avail = pf->fdir_pf_filter_count; 6050 if ((fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) || 6051 (pf->fd_add_err == 0) || 6052 (i40e_get_current_atr_cnt(pf) < pf->fd_atr_cnt)) { 6053 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 6054 (pf->hw_disabled_flags & I40E_FLAG_FD_SB_ENABLED)) { 6055 pf->hw_disabled_flags &= ~I40E_FLAG_FD_SB_ENABLED; 6056 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6057 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 6058 } 6059 } 6060 6061 /* Wait for some more space to be available to turn on ATR. We also 6062 * must check that no existing ntuple rules for TCP are in effect 6063 */ 6064 if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) { 6065 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 6066 (pf->hw_disabled_flags & I40E_FLAG_FD_ATR_ENABLED) && 6067 (pf->fd_tcp4_filter_cnt == 0)) { 6068 pf->hw_disabled_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 6069 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6070 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table and there are no conflicting ntuple rules\n"); 6071 } 6072 } 6073 6074 /* if hw had a problem adding a filter, delete it */ 6075 if (pf->fd_inv > 0) { 6076 hlist_for_each_entry_safe(filter, node, 6077 &pf->fdir_filter_list, fdir_node) { 6078 if (filter->fd_id == pf->fd_inv) { 6079 hlist_del(&filter->fdir_node); 6080 kfree(filter); 6081 pf->fdir_pf_active_filters--; 6082 } 6083 } 6084 } 6085 } 6086 6087 #define I40E_MIN_FD_FLUSH_INTERVAL 10 6088 #define I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE 30 6089 /** 6090 * i40e_fdir_flush_and_replay - Function to flush all FD filters and replay SB 6091 * @pf: board private structure 6092 **/ 6093 static void i40e_fdir_flush_and_replay(struct i40e_pf *pf) 6094 { 6095 unsigned long min_flush_time; 6096 int flush_wait_retry = 50; 6097 bool disable_atr = false; 6098 int fd_room; 6099 int reg; 6100 6101 if (!time_after(jiffies, pf->fd_flush_timestamp + 6102 (I40E_MIN_FD_FLUSH_INTERVAL * HZ))) 6103 return; 6104 6105 /* If the flush is happening too quick and we have mostly SB rules we 6106 * should not re-enable ATR for some time. 6107 */ 6108 min_flush_time = pf->fd_flush_timestamp + 6109 (I40E_MIN_FD_FLUSH_SB_ATR_UNSTABLE * HZ); 6110 fd_room = pf->fdir_pf_filter_count - pf->fdir_pf_active_filters; 6111 6112 if (!(time_after(jiffies, min_flush_time)) && 6113 (fd_room < I40E_FDIR_BUFFER_HEAD_ROOM_FOR_ATR)) { 6114 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6115 dev_info(&pf->pdev->dev, "ATR disabled, not enough FD filter space.\n"); 6116 disable_atr = true; 6117 } 6118 6119 pf->fd_flush_timestamp = jiffies; 6120 pf->hw_disabled_flags |= I40E_FLAG_FD_ATR_ENABLED; 6121 /* flush all filters */ 6122 wr32(&pf->hw, I40E_PFQF_CTL_1, 6123 I40E_PFQF_CTL_1_CLEARFDTABLE_MASK); 6124 i40e_flush(&pf->hw); 6125 pf->fd_flush_cnt++; 6126 pf->fd_add_err = 0; 6127 do { 6128 /* Check FD flush status every 5-6msec */ 6129 usleep_range(5000, 6000); 6130 reg = rd32(&pf->hw, I40E_PFQF_CTL_1); 6131 if (!(reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK)) 6132 break; 6133 } while (flush_wait_retry--); 6134 if (reg & I40E_PFQF_CTL_1_CLEARFDTABLE_MASK) { 6135 dev_warn(&pf->pdev->dev, "FD table did not flush, needs more time\n"); 6136 } else { 6137 /* replay sideband filters */ 6138 i40e_fdir_filter_restore(pf->vsi[pf->lan_vsi]); 6139 if (!disable_atr && !pf->fd_tcp4_filter_cnt) 6140 pf->hw_disabled_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 6141 clear_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state); 6142 if (I40E_DEBUG_FD & pf->hw.debug_mask) 6143 dev_info(&pf->pdev->dev, "FD Filter table flushed and FD-SB replayed.\n"); 6144 } 6145 } 6146 6147 /** 6148 * i40e_get_current_atr_count - Get the count of total FD ATR filters programmed 6149 * @pf: board private structure 6150 **/ 6151 u32 i40e_get_current_atr_cnt(struct i40e_pf *pf) 6152 { 6153 return i40e_get_current_fd_count(pf) - pf->fdir_pf_active_filters; 6154 } 6155 6156 /* We can see up to 256 filter programming desc in transit if the filters are 6157 * being applied really fast; before we see the first 6158 * filter miss error on Rx queue 0. Accumulating enough error messages before 6159 * reacting will make sure we don't cause flush too often. 6160 */ 6161 #define I40E_MAX_FD_PROGRAM_ERROR 256 6162 6163 /** 6164 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 6165 * @pf: board private structure 6166 **/ 6167 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 6168 { 6169 6170 /* if interface is down do nothing */ 6171 if (test_bit(__I40E_DOWN, &pf->state)) 6172 return; 6173 6174 if (test_bit(__I40E_FD_FLUSH_REQUESTED, &pf->state)) 6175 i40e_fdir_flush_and_replay(pf); 6176 6177 i40e_fdir_check_and_reenable(pf); 6178 6179 } 6180 6181 /** 6182 * i40e_vsi_link_event - notify VSI of a link event 6183 * @vsi: vsi to be notified 6184 * @link_up: link up or down 6185 **/ 6186 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 6187 { 6188 if (!vsi || test_bit(__I40E_DOWN, &vsi->state)) 6189 return; 6190 6191 switch (vsi->type) { 6192 case I40E_VSI_MAIN: 6193 if (!vsi->netdev || !vsi->netdev_registered) 6194 break; 6195 6196 if (link_up) { 6197 netif_carrier_on(vsi->netdev); 6198 netif_tx_wake_all_queues(vsi->netdev); 6199 } else { 6200 netif_carrier_off(vsi->netdev); 6201 netif_tx_stop_all_queues(vsi->netdev); 6202 } 6203 break; 6204 6205 case I40E_VSI_SRIOV: 6206 case I40E_VSI_VMDQ2: 6207 case I40E_VSI_CTRL: 6208 case I40E_VSI_IWARP: 6209 case I40E_VSI_MIRROR: 6210 default: 6211 /* there is no notification for other VSIs */ 6212 break; 6213 } 6214 } 6215 6216 /** 6217 * i40e_veb_link_event - notify elements on the veb of a link event 6218 * @veb: veb to be notified 6219 * @link_up: link up or down 6220 **/ 6221 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 6222 { 6223 struct i40e_pf *pf; 6224 int i; 6225 6226 if (!veb || !veb->pf) 6227 return; 6228 pf = veb->pf; 6229 6230 /* depth first... */ 6231 for (i = 0; i < I40E_MAX_VEB; i++) 6232 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 6233 i40e_veb_link_event(pf->veb[i], link_up); 6234 6235 /* ... now the local VSIs */ 6236 for (i = 0; i < pf->num_alloc_vsi; i++) 6237 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 6238 i40e_vsi_link_event(pf->vsi[i], link_up); 6239 } 6240 6241 /** 6242 * i40e_link_event - Update netif_carrier status 6243 * @pf: board private structure 6244 **/ 6245 static void i40e_link_event(struct i40e_pf *pf) 6246 { 6247 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6248 u8 new_link_speed, old_link_speed; 6249 i40e_status status; 6250 bool new_link, old_link; 6251 6252 /* save off old link status information */ 6253 pf->hw.phy.link_info_old = pf->hw.phy.link_info; 6254 6255 /* set this to force the get_link_status call to refresh state */ 6256 pf->hw.phy.get_link_info = true; 6257 6258 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 6259 6260 status = i40e_get_link_status(&pf->hw, &new_link); 6261 6262 /* On success, disable temp link polling */ 6263 if (status == I40E_SUCCESS) { 6264 if (pf->flags & I40E_FLAG_TEMP_LINK_POLLING) 6265 pf->flags &= ~I40E_FLAG_TEMP_LINK_POLLING; 6266 } else { 6267 /* Enable link polling temporarily until i40e_get_link_status 6268 * returns I40E_SUCCESS 6269 */ 6270 pf->flags |= I40E_FLAG_TEMP_LINK_POLLING; 6271 dev_dbg(&pf->pdev->dev, "couldn't get link state, status: %d\n", 6272 status); 6273 return; 6274 } 6275 6276 old_link_speed = pf->hw.phy.link_info_old.link_speed; 6277 new_link_speed = pf->hw.phy.link_info.link_speed; 6278 6279 if (new_link == old_link && 6280 new_link_speed == old_link_speed && 6281 (test_bit(__I40E_DOWN, &vsi->state) || 6282 new_link == netif_carrier_ok(vsi->netdev))) 6283 return; 6284 6285 if (!test_bit(__I40E_DOWN, &vsi->state)) 6286 i40e_print_link_message(vsi, new_link); 6287 6288 /* Notify the base of the switch tree connected to 6289 * the link. Floating VEBs are not notified. 6290 */ 6291 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 6292 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 6293 else 6294 i40e_vsi_link_event(vsi, new_link); 6295 6296 if (pf->vf) 6297 i40e_vc_notify_link_state(pf); 6298 6299 if (pf->flags & I40E_FLAG_PTP) 6300 i40e_ptp_set_increment(pf); 6301 } 6302 6303 /** 6304 * i40e_watchdog_subtask - periodic checks not using event driven response 6305 * @pf: board private structure 6306 **/ 6307 static void i40e_watchdog_subtask(struct i40e_pf *pf) 6308 { 6309 int i; 6310 6311 /* if interface is down do nothing */ 6312 if (test_bit(__I40E_DOWN, &pf->state) || 6313 test_bit(__I40E_CONFIG_BUSY, &pf->state)) 6314 return; 6315 6316 /* make sure we don't do these things too often */ 6317 if (time_before(jiffies, (pf->service_timer_previous + 6318 pf->service_timer_period))) 6319 return; 6320 pf->service_timer_previous = jiffies; 6321 6322 if ((pf->flags & I40E_FLAG_LINK_POLLING_ENABLED) || 6323 (pf->flags & I40E_FLAG_TEMP_LINK_POLLING)) 6324 i40e_link_event(pf); 6325 6326 /* Update the stats for active netdevs so the network stack 6327 * can look at updated numbers whenever it cares to 6328 */ 6329 for (i = 0; i < pf->num_alloc_vsi; i++) 6330 if (pf->vsi[i] && pf->vsi[i]->netdev) 6331 i40e_update_stats(pf->vsi[i]); 6332 6333 if (pf->flags & I40E_FLAG_VEB_STATS_ENABLED) { 6334 /* Update the stats for the active switching components */ 6335 for (i = 0; i < I40E_MAX_VEB; i++) 6336 if (pf->veb[i]) 6337 i40e_update_veb_stats(pf->veb[i]); 6338 } 6339 6340 i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]); 6341 } 6342 6343 /** 6344 * i40e_reset_subtask - Set up for resetting the device and driver 6345 * @pf: board private structure 6346 **/ 6347 static void i40e_reset_subtask(struct i40e_pf *pf) 6348 { 6349 u32 reset_flags = 0; 6350 6351 rtnl_lock(); 6352 if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) { 6353 reset_flags |= BIT(__I40E_REINIT_REQUESTED); 6354 clear_bit(__I40E_REINIT_REQUESTED, &pf->state); 6355 } 6356 if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) { 6357 reset_flags |= BIT(__I40E_PF_RESET_REQUESTED); 6358 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 6359 } 6360 if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) { 6361 reset_flags |= BIT(__I40E_CORE_RESET_REQUESTED); 6362 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state); 6363 } 6364 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) { 6365 reset_flags |= BIT(__I40E_GLOBAL_RESET_REQUESTED); 6366 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state); 6367 } 6368 if (test_bit(__I40E_DOWN_REQUESTED, &pf->state)) { 6369 reset_flags |= BIT(__I40E_DOWN_REQUESTED); 6370 clear_bit(__I40E_DOWN_REQUESTED, &pf->state); 6371 } 6372 6373 /* If there's a recovery already waiting, it takes 6374 * precedence before starting a new reset sequence. 6375 */ 6376 if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) { 6377 i40e_handle_reset_warning(pf); 6378 goto unlock; 6379 } 6380 6381 /* If we're already down or resetting, just bail */ 6382 if (reset_flags && 6383 !test_bit(__I40E_DOWN, &pf->state) && 6384 !test_bit(__I40E_CONFIG_BUSY, &pf->state)) 6385 i40e_do_reset(pf, reset_flags); 6386 6387 unlock: 6388 rtnl_unlock(); 6389 } 6390 6391 /** 6392 * i40e_handle_link_event - Handle link event 6393 * @pf: board private structure 6394 * @e: event info posted on ARQ 6395 **/ 6396 static void i40e_handle_link_event(struct i40e_pf *pf, 6397 struct i40e_arq_event_info *e) 6398 { 6399 struct i40e_aqc_get_link_status *status = 6400 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 6401 6402 /* Do a new status request to re-enable LSE reporting 6403 * and load new status information into the hw struct 6404 * This completely ignores any state information 6405 * in the ARQ event info, instead choosing to always 6406 * issue the AQ update link status command. 6407 */ 6408 i40e_link_event(pf); 6409 6410 /* check for unqualified module, if link is down */ 6411 if ((status->link_info & I40E_AQ_MEDIA_AVAILABLE) && 6412 (!(status->an_info & I40E_AQ_QUALIFIED_MODULE)) && 6413 (!(status->link_info & I40E_AQ_LINK_UP))) 6414 dev_err(&pf->pdev->dev, 6415 "The driver failed to link because an unqualified module was detected.\n"); 6416 } 6417 6418 /** 6419 * i40e_clean_adminq_subtask - Clean the AdminQ rings 6420 * @pf: board private structure 6421 **/ 6422 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 6423 { 6424 struct i40e_arq_event_info event; 6425 struct i40e_hw *hw = &pf->hw; 6426 u16 pending, i = 0; 6427 i40e_status ret; 6428 u16 opcode; 6429 u32 oldval; 6430 u32 val; 6431 6432 /* Do not run clean AQ when PF reset fails */ 6433 if (test_bit(__I40E_RESET_FAILED, &pf->state)) 6434 return; 6435 6436 /* check for error indications */ 6437 val = rd32(&pf->hw, pf->hw.aq.arq.len); 6438 oldval = val; 6439 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 6440 if (hw->debug_mask & I40E_DEBUG_AQ) 6441 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 6442 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 6443 } 6444 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 6445 if (hw->debug_mask & I40E_DEBUG_AQ) 6446 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 6447 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 6448 pf->arq_overflows++; 6449 } 6450 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 6451 if (hw->debug_mask & I40E_DEBUG_AQ) 6452 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 6453 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 6454 } 6455 if (oldval != val) 6456 wr32(&pf->hw, pf->hw.aq.arq.len, val); 6457 6458 val = rd32(&pf->hw, pf->hw.aq.asq.len); 6459 oldval = val; 6460 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 6461 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6462 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 6463 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 6464 } 6465 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 6466 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6467 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 6468 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 6469 } 6470 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 6471 if (pf->hw.debug_mask & I40E_DEBUG_AQ) 6472 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 6473 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 6474 } 6475 if (oldval != val) 6476 wr32(&pf->hw, pf->hw.aq.asq.len, val); 6477 6478 event.buf_len = I40E_MAX_AQ_BUF_SIZE; 6479 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL); 6480 if (!event.msg_buf) 6481 return; 6482 6483 do { 6484 ret = i40e_clean_arq_element(hw, &event, &pending); 6485 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) 6486 break; 6487 else if (ret) { 6488 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 6489 break; 6490 } 6491 6492 opcode = le16_to_cpu(event.desc.opcode); 6493 switch (opcode) { 6494 6495 case i40e_aqc_opc_get_link_status: 6496 i40e_handle_link_event(pf, &event); 6497 break; 6498 case i40e_aqc_opc_send_msg_to_pf: 6499 ret = i40e_vc_process_vf_msg(pf, 6500 le16_to_cpu(event.desc.retval), 6501 le32_to_cpu(event.desc.cookie_high), 6502 le32_to_cpu(event.desc.cookie_low), 6503 event.msg_buf, 6504 event.msg_len); 6505 break; 6506 case i40e_aqc_opc_lldp_update_mib: 6507 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 6508 #ifdef CONFIG_I40E_DCB 6509 rtnl_lock(); 6510 ret = i40e_handle_lldp_event(pf, &event); 6511 rtnl_unlock(); 6512 #endif /* CONFIG_I40E_DCB */ 6513 break; 6514 case i40e_aqc_opc_event_lan_overflow: 6515 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 6516 i40e_handle_lan_overflow_event(pf, &event); 6517 break; 6518 case i40e_aqc_opc_send_msg_to_peer: 6519 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 6520 break; 6521 case i40e_aqc_opc_nvm_erase: 6522 case i40e_aqc_opc_nvm_update: 6523 case i40e_aqc_opc_oem_post_update: 6524 i40e_debug(&pf->hw, I40E_DEBUG_NVM, 6525 "ARQ NVM operation 0x%04x completed\n", 6526 opcode); 6527 break; 6528 default: 6529 dev_info(&pf->pdev->dev, 6530 "ARQ: Unknown event 0x%04x ignored\n", 6531 opcode); 6532 break; 6533 } 6534 } while (i++ < pf->adminq_work_limit); 6535 6536 if (i < pf->adminq_work_limit) 6537 clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state); 6538 6539 /* re-enable Admin queue interrupt cause */ 6540 val = rd32(hw, I40E_PFINT_ICR0_ENA); 6541 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 6542 wr32(hw, I40E_PFINT_ICR0_ENA, val); 6543 i40e_flush(hw); 6544 6545 kfree(event.msg_buf); 6546 } 6547 6548 /** 6549 * i40e_verify_eeprom - make sure eeprom is good to use 6550 * @pf: board private structure 6551 **/ 6552 static void i40e_verify_eeprom(struct i40e_pf *pf) 6553 { 6554 int err; 6555 6556 err = i40e_diag_eeprom_test(&pf->hw); 6557 if (err) { 6558 /* retry in case of garbage read */ 6559 err = i40e_diag_eeprom_test(&pf->hw); 6560 if (err) { 6561 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 6562 err); 6563 set_bit(__I40E_BAD_EEPROM, &pf->state); 6564 } 6565 } 6566 6567 if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) { 6568 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 6569 clear_bit(__I40E_BAD_EEPROM, &pf->state); 6570 } 6571 } 6572 6573 /** 6574 * i40e_enable_pf_switch_lb 6575 * @pf: pointer to the PF structure 6576 * 6577 * enable switch loop back or die - no point in a return value 6578 **/ 6579 static void i40e_enable_pf_switch_lb(struct i40e_pf *pf) 6580 { 6581 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6582 struct i40e_vsi_context ctxt; 6583 int ret; 6584 6585 ctxt.seid = pf->main_vsi_seid; 6586 ctxt.pf_num = pf->hw.pf_id; 6587 ctxt.vf_num = 0; 6588 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6589 if (ret) { 6590 dev_info(&pf->pdev->dev, 6591 "couldn't get PF vsi config, err %s aq_err %s\n", 6592 i40e_stat_str(&pf->hw, ret), 6593 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6594 return; 6595 } 6596 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6597 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6598 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6599 6600 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6601 if (ret) { 6602 dev_info(&pf->pdev->dev, 6603 "update vsi switch failed, err %s aq_err %s\n", 6604 i40e_stat_str(&pf->hw, ret), 6605 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6606 } 6607 } 6608 6609 /** 6610 * i40e_disable_pf_switch_lb 6611 * @pf: pointer to the PF structure 6612 * 6613 * disable switch loop back or die - no point in a return value 6614 **/ 6615 static void i40e_disable_pf_switch_lb(struct i40e_pf *pf) 6616 { 6617 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 6618 struct i40e_vsi_context ctxt; 6619 int ret; 6620 6621 ctxt.seid = pf->main_vsi_seid; 6622 ctxt.pf_num = pf->hw.pf_id; 6623 ctxt.vf_num = 0; 6624 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 6625 if (ret) { 6626 dev_info(&pf->pdev->dev, 6627 "couldn't get PF vsi config, err %s aq_err %s\n", 6628 i40e_stat_str(&pf->hw, ret), 6629 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6630 return; 6631 } 6632 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 6633 ctxt.info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 6634 ctxt.info.switch_id &= ~cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 6635 6636 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 6637 if (ret) { 6638 dev_info(&pf->pdev->dev, 6639 "update vsi switch failed, err %s aq_err %s\n", 6640 i40e_stat_str(&pf->hw, ret), 6641 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6642 } 6643 } 6644 6645 /** 6646 * i40e_config_bridge_mode - Configure the HW bridge mode 6647 * @veb: pointer to the bridge instance 6648 * 6649 * Configure the loop back mode for the LAN VSI that is downlink to the 6650 * specified HW bridge instance. It is expected this function is called 6651 * when a new HW bridge is instantiated. 6652 **/ 6653 static void i40e_config_bridge_mode(struct i40e_veb *veb) 6654 { 6655 struct i40e_pf *pf = veb->pf; 6656 6657 if (pf->hw.debug_mask & I40E_DEBUG_LAN) 6658 dev_info(&pf->pdev->dev, "enabling bridge mode: %s\n", 6659 veb->bridge_mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 6660 if (veb->bridge_mode & BRIDGE_MODE_VEPA) 6661 i40e_disable_pf_switch_lb(pf); 6662 else 6663 i40e_enable_pf_switch_lb(pf); 6664 } 6665 6666 /** 6667 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 6668 * @veb: pointer to the VEB instance 6669 * 6670 * This is a recursive function that first builds the attached VSIs then 6671 * recurses in to build the next layer of VEB. We track the connections 6672 * through our own index numbers because the seid's from the HW could 6673 * change across the reset. 6674 **/ 6675 static int i40e_reconstitute_veb(struct i40e_veb *veb) 6676 { 6677 struct i40e_vsi *ctl_vsi = NULL; 6678 struct i40e_pf *pf = veb->pf; 6679 int v, veb_idx; 6680 int ret; 6681 6682 /* build VSI that owns this VEB, temporarily attached to base VEB */ 6683 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 6684 if (pf->vsi[v] && 6685 pf->vsi[v]->veb_idx == veb->idx && 6686 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 6687 ctl_vsi = pf->vsi[v]; 6688 break; 6689 } 6690 } 6691 if (!ctl_vsi) { 6692 dev_info(&pf->pdev->dev, 6693 "missing owner VSI for veb_idx %d\n", veb->idx); 6694 ret = -ENOENT; 6695 goto end_reconstitute; 6696 } 6697 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 6698 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 6699 ret = i40e_add_vsi(ctl_vsi); 6700 if (ret) { 6701 dev_info(&pf->pdev->dev, 6702 "rebuild of veb_idx %d owner VSI failed: %d\n", 6703 veb->idx, ret); 6704 goto end_reconstitute; 6705 } 6706 i40e_vsi_reset_stats(ctl_vsi); 6707 6708 /* create the VEB in the switch and move the VSI onto the VEB */ 6709 ret = i40e_add_veb(veb, ctl_vsi); 6710 if (ret) 6711 goto end_reconstitute; 6712 6713 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 6714 veb->bridge_mode = BRIDGE_MODE_VEB; 6715 else 6716 veb->bridge_mode = BRIDGE_MODE_VEPA; 6717 i40e_config_bridge_mode(veb); 6718 6719 /* create the remaining VSIs attached to this VEB */ 6720 for (v = 0; v < pf->num_alloc_vsi; v++) { 6721 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 6722 continue; 6723 6724 if (pf->vsi[v]->veb_idx == veb->idx) { 6725 struct i40e_vsi *vsi = pf->vsi[v]; 6726 6727 vsi->uplink_seid = veb->seid; 6728 ret = i40e_add_vsi(vsi); 6729 if (ret) { 6730 dev_info(&pf->pdev->dev, 6731 "rebuild of vsi_idx %d failed: %d\n", 6732 v, ret); 6733 goto end_reconstitute; 6734 } 6735 i40e_vsi_reset_stats(vsi); 6736 } 6737 } 6738 6739 /* create any VEBs attached to this VEB - RECURSION */ 6740 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 6741 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 6742 pf->veb[veb_idx]->uplink_seid = veb->seid; 6743 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 6744 if (ret) 6745 break; 6746 } 6747 } 6748 6749 end_reconstitute: 6750 return ret; 6751 } 6752 6753 /** 6754 * i40e_get_capabilities - get info about the HW 6755 * @pf: the PF struct 6756 **/ 6757 static int i40e_get_capabilities(struct i40e_pf *pf) 6758 { 6759 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 6760 u16 data_size; 6761 int buf_len; 6762 int err; 6763 6764 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 6765 do { 6766 cap_buf = kzalloc(buf_len, GFP_KERNEL); 6767 if (!cap_buf) 6768 return -ENOMEM; 6769 6770 /* this loads the data into the hw struct for us */ 6771 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 6772 &data_size, 6773 i40e_aqc_opc_list_func_capabilities, 6774 NULL); 6775 /* data loaded, buffer no longer needed */ 6776 kfree(cap_buf); 6777 6778 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 6779 /* retry with a larger buffer */ 6780 buf_len = data_size; 6781 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 6782 dev_info(&pf->pdev->dev, 6783 "capability discovery failed, err %s aq_err %s\n", 6784 i40e_stat_str(&pf->hw, err), 6785 i40e_aq_str(&pf->hw, 6786 pf->hw.aq.asq_last_status)); 6787 return -ENODEV; 6788 } 6789 } while (err); 6790 6791 if (pf->hw.debug_mask & I40E_DEBUG_USER) 6792 dev_info(&pf->pdev->dev, 6793 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 6794 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 6795 pf->hw.func_caps.num_msix_vectors, 6796 pf->hw.func_caps.num_msix_vectors_vf, 6797 pf->hw.func_caps.fd_filters_guaranteed, 6798 pf->hw.func_caps.fd_filters_best_effort, 6799 pf->hw.func_caps.num_tx_qp, 6800 pf->hw.func_caps.num_vsis); 6801 6802 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 6803 + pf->hw.func_caps.num_vfs) 6804 if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) { 6805 dev_info(&pf->pdev->dev, 6806 "got num_vsis %d, setting num_vsis to %d\n", 6807 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 6808 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 6809 } 6810 6811 return 0; 6812 } 6813 6814 static int i40e_vsi_clear(struct i40e_vsi *vsi); 6815 6816 /** 6817 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 6818 * @pf: board private structure 6819 **/ 6820 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 6821 { 6822 struct i40e_vsi *vsi; 6823 6824 /* quick workaround for an NVM issue that leaves a critical register 6825 * uninitialized 6826 */ 6827 if (!rd32(&pf->hw, I40E_GLQF_HKEY(0))) { 6828 static const u32 hkey[] = { 6829 0xe640d33f, 0xcdfe98ab, 0x73fa7161, 0x0d7a7d36, 6830 0xeacb7d61, 0xaa4f05b6, 0x9c5c89ed, 0xfc425ddb, 6831 0xa4654832, 0xfc7461d4, 0x8f827619, 0xf5c63c21, 6832 0x95b3a76d}; 6833 int i; 6834 6835 for (i = 0; i <= I40E_GLQF_HKEY_MAX_INDEX; i++) 6836 wr32(&pf->hw, I40E_GLQF_HKEY(i), hkey[i]); 6837 } 6838 6839 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 6840 return; 6841 6842 /* find existing VSI and see if it needs configuring */ 6843 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6844 6845 /* create a new VSI if none exists */ 6846 if (!vsi) { 6847 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 6848 pf->vsi[pf->lan_vsi]->seid, 0); 6849 if (!vsi) { 6850 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 6851 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 6852 return; 6853 } 6854 } 6855 6856 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 6857 } 6858 6859 /** 6860 * i40e_fdir_teardown - release the Flow Director resources 6861 * @pf: board private structure 6862 **/ 6863 static void i40e_fdir_teardown(struct i40e_pf *pf) 6864 { 6865 struct i40e_vsi *vsi; 6866 6867 i40e_fdir_filter_exit(pf); 6868 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 6869 if (vsi) 6870 i40e_vsi_release(vsi); 6871 } 6872 6873 /** 6874 * i40e_prep_for_reset - prep for the core to reset 6875 * @pf: board private structure 6876 * 6877 * Close up the VFs and other things in prep for PF Reset. 6878 **/ 6879 static void i40e_prep_for_reset(struct i40e_pf *pf) 6880 { 6881 struct i40e_hw *hw = &pf->hw; 6882 i40e_status ret = 0; 6883 u32 v; 6884 6885 clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state); 6886 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 6887 return; 6888 if (i40e_check_asq_alive(&pf->hw)) 6889 i40e_vc_notify_reset(pf); 6890 6891 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 6892 6893 /* quiesce the VSIs and their queues that are not already DOWN */ 6894 i40e_pf_quiesce_all_vsi(pf); 6895 6896 for (v = 0; v < pf->num_alloc_vsi; v++) { 6897 if (pf->vsi[v]) 6898 pf->vsi[v]->seid = 0; 6899 } 6900 6901 i40e_shutdown_adminq(&pf->hw); 6902 6903 /* call shutdown HMC */ 6904 if (hw->hmc.hmc_obj) { 6905 ret = i40e_shutdown_lan_hmc(hw); 6906 if (ret) 6907 dev_warn(&pf->pdev->dev, 6908 "shutdown_lan_hmc failed: %d\n", ret); 6909 } 6910 } 6911 6912 /** 6913 * i40e_send_version - update firmware with driver version 6914 * @pf: PF struct 6915 */ 6916 static void i40e_send_version(struct i40e_pf *pf) 6917 { 6918 struct i40e_driver_version dv; 6919 6920 dv.major_version = DRV_VERSION_MAJOR; 6921 dv.minor_version = DRV_VERSION_MINOR; 6922 dv.build_version = DRV_VERSION_BUILD; 6923 dv.subbuild_version = 0; 6924 strlcpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 6925 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 6926 } 6927 6928 /** 6929 * i40e_reset_and_rebuild - reset and rebuild using a saved config 6930 * @pf: board private structure 6931 * @reinit: if the Main VSI needs to re-initialized. 6932 **/ 6933 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit) 6934 { 6935 struct i40e_hw *hw = &pf->hw; 6936 u8 set_fc_aq_fail = 0; 6937 i40e_status ret; 6938 u32 val; 6939 u32 v; 6940 6941 /* Now we wait for GRST to settle out. 6942 * We don't have to delete the VEBs or VSIs from the hw switch 6943 * because the reset will make them disappear. 6944 */ 6945 ret = i40e_pf_reset(hw); 6946 if (ret) { 6947 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 6948 set_bit(__I40E_RESET_FAILED, &pf->state); 6949 goto clear_recovery; 6950 } 6951 pf->pfr_count++; 6952 6953 if (test_bit(__I40E_DOWN, &pf->state)) 6954 goto clear_recovery; 6955 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 6956 6957 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 6958 ret = i40e_init_adminq(&pf->hw); 6959 if (ret) { 6960 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, err %s aq_err %s\n", 6961 i40e_stat_str(&pf->hw, ret), 6962 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 6963 goto clear_recovery; 6964 } 6965 6966 /* re-verify the eeprom if we just had an EMP reset */ 6967 if (test_and_clear_bit(__I40E_EMP_RESET_INTR_RECEIVED, &pf->state)) 6968 i40e_verify_eeprom(pf); 6969 6970 i40e_clear_pxe_mode(hw); 6971 ret = i40e_get_capabilities(pf); 6972 if (ret) 6973 goto end_core_reset; 6974 6975 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 6976 hw->func_caps.num_rx_qp, 0, 0); 6977 if (ret) { 6978 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 6979 goto end_core_reset; 6980 } 6981 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 6982 if (ret) { 6983 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 6984 goto end_core_reset; 6985 } 6986 6987 #ifdef CONFIG_I40E_DCB 6988 ret = i40e_init_pf_dcb(pf); 6989 if (ret) { 6990 dev_info(&pf->pdev->dev, "DCB init failed %d, disabled\n", ret); 6991 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 6992 /* Continue without DCB enabled */ 6993 } 6994 #endif /* CONFIG_I40E_DCB */ 6995 /* do basic switch setup */ 6996 ret = i40e_setup_pf_switch(pf, reinit); 6997 if (ret) 6998 goto end_core_reset; 6999 7000 /* The driver only wants link up/down and module qualification 7001 * reports from firmware. Note the negative logic. 7002 */ 7003 ret = i40e_aq_set_phy_int_mask(&pf->hw, 7004 ~(I40E_AQ_EVENT_LINK_UPDOWN | 7005 I40E_AQ_EVENT_MEDIA_NA | 7006 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 7007 if (ret) 7008 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 7009 i40e_stat_str(&pf->hw, ret), 7010 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7011 7012 /* make sure our flow control settings are restored */ 7013 ret = i40e_set_fc(&pf->hw, &set_fc_aq_fail, true); 7014 if (ret) 7015 dev_dbg(&pf->pdev->dev, "setting flow control: ret = %s last_status = %s\n", 7016 i40e_stat_str(&pf->hw, ret), 7017 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 7018 7019 /* Rebuild the VSIs and VEBs that existed before reset. 7020 * They are still in our local switch element arrays, so only 7021 * need to rebuild the switch model in the HW. 7022 * 7023 * If there were VEBs but the reconstitution failed, we'll try 7024 * try to recover minimal use by getting the basic PF VSI working. 7025 */ 7026 if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) { 7027 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 7028 /* find the one VEB connected to the MAC, and find orphans */ 7029 for (v = 0; v < I40E_MAX_VEB; v++) { 7030 if (!pf->veb[v]) 7031 continue; 7032 7033 if (pf->veb[v]->uplink_seid == pf->mac_seid || 7034 pf->veb[v]->uplink_seid == 0) { 7035 ret = i40e_reconstitute_veb(pf->veb[v]); 7036 7037 if (!ret) 7038 continue; 7039 7040 /* If Main VEB failed, we're in deep doodoo, 7041 * so give up rebuilding the switch and set up 7042 * for minimal rebuild of PF VSI. 7043 * If orphan failed, we'll report the error 7044 * but try to keep going. 7045 */ 7046 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 7047 dev_info(&pf->pdev->dev, 7048 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 7049 ret); 7050 pf->vsi[pf->lan_vsi]->uplink_seid 7051 = pf->mac_seid; 7052 break; 7053 } else if (pf->veb[v]->uplink_seid == 0) { 7054 dev_info(&pf->pdev->dev, 7055 "rebuild of orphan VEB failed: %d\n", 7056 ret); 7057 } 7058 } 7059 } 7060 } 7061 7062 if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) { 7063 dev_dbg(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 7064 /* no VEB, so rebuild only the Main VSI */ 7065 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]); 7066 if (ret) { 7067 dev_info(&pf->pdev->dev, 7068 "rebuild of Main VSI failed: %d\n", ret); 7069 goto end_core_reset; 7070 } 7071 } 7072 7073 /* Reconfigure hardware for allowing smaller MSS in the case 7074 * of TSO, so that we avoid the MDD being fired and causing 7075 * a reset in the case of small MSS+TSO. 7076 */ 7077 #define I40E_REG_MSS 0x000E64DC 7078 #define I40E_REG_MSS_MIN_MASK 0x3FF0000 7079 #define I40E_64BYTE_MSS 0x400000 7080 val = rd32(hw, I40E_REG_MSS); 7081 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 7082 val &= ~I40E_REG_MSS_MIN_MASK; 7083 val |= I40E_64BYTE_MSS; 7084 wr32(hw, I40E_REG_MSS, val); 7085 } 7086 7087 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) { 7088 msleep(75); 7089 ret = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 7090 if (ret) 7091 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 7092 i40e_stat_str(&pf->hw, ret), 7093 i40e_aq_str(&pf->hw, 7094 pf->hw.aq.asq_last_status)); 7095 } 7096 /* reinit the misc interrupt */ 7097 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7098 ret = i40e_setup_misc_vector(pf); 7099 7100 /* Add a filter to drop all Flow control frames from any VSI from being 7101 * transmitted. By doing so we stop a malicious VF from sending out 7102 * PAUSE or PFC frames and potentially controlling traffic for other 7103 * PF/VF VSIs. 7104 * The FW can still send Flow control frames if enabled. 7105 */ 7106 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 7107 pf->main_vsi_seid); 7108 7109 /* restart the VSIs that were rebuilt and running before the reset */ 7110 i40e_pf_unquiesce_all_vsi(pf); 7111 7112 if (pf->num_alloc_vfs) { 7113 for (v = 0; v < pf->num_alloc_vfs; v++) 7114 i40e_reset_vf(&pf->vf[v], true); 7115 } 7116 7117 /* tell the firmware that we're starting */ 7118 i40e_send_version(pf); 7119 7120 end_core_reset: 7121 clear_bit(__I40E_RESET_FAILED, &pf->state); 7122 clear_recovery: 7123 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state); 7124 } 7125 7126 /** 7127 * i40e_handle_reset_warning - prep for the PF to reset, reset and rebuild 7128 * @pf: board private structure 7129 * 7130 * Close up the VFs and other things in prep for a Core Reset, 7131 * then get ready to rebuild the world. 7132 **/ 7133 static void i40e_handle_reset_warning(struct i40e_pf *pf) 7134 { 7135 i40e_prep_for_reset(pf); 7136 i40e_reset_and_rebuild(pf, false); 7137 } 7138 7139 /** 7140 * i40e_handle_mdd_event 7141 * @pf: pointer to the PF structure 7142 * 7143 * Called from the MDD irq handler to identify possibly malicious vfs 7144 **/ 7145 static void i40e_handle_mdd_event(struct i40e_pf *pf) 7146 { 7147 struct i40e_hw *hw = &pf->hw; 7148 bool mdd_detected = false; 7149 bool pf_mdd_detected = false; 7150 struct i40e_vf *vf; 7151 u32 reg; 7152 int i; 7153 7154 if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)) 7155 return; 7156 7157 /* find what triggered the MDD event */ 7158 reg = rd32(hw, I40E_GL_MDET_TX); 7159 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 7160 u8 pf_num = (reg & I40E_GL_MDET_TX_PF_NUM_MASK) >> 7161 I40E_GL_MDET_TX_PF_NUM_SHIFT; 7162 u16 vf_num = (reg & I40E_GL_MDET_TX_VF_NUM_MASK) >> 7163 I40E_GL_MDET_TX_VF_NUM_SHIFT; 7164 u8 event = (reg & I40E_GL_MDET_TX_EVENT_MASK) >> 7165 I40E_GL_MDET_TX_EVENT_SHIFT; 7166 u16 queue = ((reg & I40E_GL_MDET_TX_QUEUE_MASK) >> 7167 I40E_GL_MDET_TX_QUEUE_SHIFT) - 7168 pf->hw.func_caps.base_queue; 7169 if (netif_msg_tx_err(pf)) 7170 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on TX queue %d PF number 0x%02x VF number 0x%02x\n", 7171 event, queue, pf_num, vf_num); 7172 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 7173 mdd_detected = true; 7174 } 7175 reg = rd32(hw, I40E_GL_MDET_RX); 7176 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 7177 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) >> 7178 I40E_GL_MDET_RX_FUNCTION_SHIFT; 7179 u8 event = (reg & I40E_GL_MDET_RX_EVENT_MASK) >> 7180 I40E_GL_MDET_RX_EVENT_SHIFT; 7181 u16 queue = ((reg & I40E_GL_MDET_RX_QUEUE_MASK) >> 7182 I40E_GL_MDET_RX_QUEUE_SHIFT) - 7183 pf->hw.func_caps.base_queue; 7184 if (netif_msg_rx_err(pf)) 7185 dev_info(&pf->pdev->dev, "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 7186 event, queue, func); 7187 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 7188 mdd_detected = true; 7189 } 7190 7191 if (mdd_detected) { 7192 reg = rd32(hw, I40E_PF_MDET_TX); 7193 if (reg & I40E_PF_MDET_TX_VALID_MASK) { 7194 wr32(hw, I40E_PF_MDET_TX, 0xFFFF); 7195 dev_info(&pf->pdev->dev, "TX driver issue detected, PF reset issued\n"); 7196 pf_mdd_detected = true; 7197 } 7198 reg = rd32(hw, I40E_PF_MDET_RX); 7199 if (reg & I40E_PF_MDET_RX_VALID_MASK) { 7200 wr32(hw, I40E_PF_MDET_RX, 0xFFFF); 7201 dev_info(&pf->pdev->dev, "RX driver issue detected, PF reset issued\n"); 7202 pf_mdd_detected = true; 7203 } 7204 /* Queue belongs to the PF, initiate a reset */ 7205 if (pf_mdd_detected) { 7206 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 7207 i40e_service_event_schedule(pf); 7208 } 7209 } 7210 7211 /* see if one of the VFs needs its hand slapped */ 7212 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 7213 vf = &(pf->vf[i]); 7214 reg = rd32(hw, I40E_VP_MDET_TX(i)); 7215 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 7216 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 7217 vf->num_mdd_events++; 7218 dev_info(&pf->pdev->dev, "TX driver issue detected on VF %d\n", 7219 i); 7220 } 7221 7222 reg = rd32(hw, I40E_VP_MDET_RX(i)); 7223 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 7224 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 7225 vf->num_mdd_events++; 7226 dev_info(&pf->pdev->dev, "RX driver issue detected on VF %d\n", 7227 i); 7228 } 7229 7230 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 7231 dev_info(&pf->pdev->dev, 7232 "Too many MDD events on VF %d, disabled\n", i); 7233 dev_info(&pf->pdev->dev, 7234 "Use PF Control I/F to re-enable the VF\n"); 7235 set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states); 7236 } 7237 } 7238 7239 /* re-enable mdd interrupt cause */ 7240 clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state); 7241 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 7242 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 7243 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 7244 i40e_flush(hw); 7245 } 7246 7247 /** 7248 * i40e_sync_udp_filters_subtask - Sync the VSI filter list with HW 7249 * @pf: board private structure 7250 **/ 7251 static void i40e_sync_udp_filters_subtask(struct i40e_pf *pf) 7252 { 7253 struct i40e_hw *hw = &pf->hw; 7254 i40e_status ret; 7255 u16 port; 7256 int i; 7257 7258 if (!(pf->flags & I40E_FLAG_UDP_FILTER_SYNC)) 7259 return; 7260 7261 pf->flags &= ~I40E_FLAG_UDP_FILTER_SYNC; 7262 7263 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 7264 if (pf->pending_udp_bitmap & BIT_ULL(i)) { 7265 pf->pending_udp_bitmap &= ~BIT_ULL(i); 7266 port = pf->udp_ports[i].index; 7267 if (port) 7268 ret = i40e_aq_add_udp_tunnel(hw, port, 7269 pf->udp_ports[i].type, 7270 NULL, NULL); 7271 else 7272 ret = i40e_aq_del_udp_tunnel(hw, i, NULL); 7273 7274 if (ret) { 7275 dev_dbg(&pf->pdev->dev, 7276 "%s %s port %d, index %d failed, err %s aq_err %s\n", 7277 pf->udp_ports[i].type ? "vxlan" : "geneve", 7278 port ? "add" : "delete", 7279 port, i, 7280 i40e_stat_str(&pf->hw, ret), 7281 i40e_aq_str(&pf->hw, 7282 pf->hw.aq.asq_last_status)); 7283 pf->udp_ports[i].index = 0; 7284 } 7285 } 7286 } 7287 } 7288 7289 /** 7290 * i40e_service_task - Run the driver's async subtasks 7291 * @work: pointer to work_struct containing our data 7292 **/ 7293 static void i40e_service_task(struct work_struct *work) 7294 { 7295 struct i40e_pf *pf = container_of(work, 7296 struct i40e_pf, 7297 service_task); 7298 unsigned long start_time = jiffies; 7299 7300 /* don't bother with service tasks if a reset is in progress */ 7301 if (test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) { 7302 return; 7303 } 7304 7305 if (test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state)) 7306 return; 7307 7308 i40e_detect_recover_hung(pf); 7309 i40e_sync_filters_subtask(pf); 7310 i40e_reset_subtask(pf); 7311 i40e_handle_mdd_event(pf); 7312 i40e_vc_process_vflr_event(pf); 7313 i40e_watchdog_subtask(pf); 7314 i40e_fdir_reinit_subtask(pf); 7315 if (pf->flags & I40E_FLAG_CLIENT_RESET) { 7316 /* Client subtask will reopen next time through. */ 7317 i40e_notify_client_of_netdev_close(pf->vsi[pf->lan_vsi], true); 7318 pf->flags &= ~I40E_FLAG_CLIENT_RESET; 7319 } else { 7320 i40e_client_subtask(pf); 7321 if (pf->flags & I40E_FLAG_CLIENT_L2_CHANGE) { 7322 i40e_notify_client_of_l2_param_changes( 7323 pf->vsi[pf->lan_vsi]); 7324 pf->flags &= ~I40E_FLAG_CLIENT_L2_CHANGE; 7325 } 7326 } 7327 i40e_sync_filters_subtask(pf); 7328 i40e_sync_udp_filters_subtask(pf); 7329 i40e_clean_adminq_subtask(pf); 7330 7331 /* flush memory to make sure state is correct before next watchdog */ 7332 smp_mb__before_atomic(); 7333 clear_bit(__I40E_SERVICE_SCHED, &pf->state); 7334 7335 /* If the tasks have taken longer than one timer cycle or there 7336 * is more work to be done, reschedule the service task now 7337 * rather than wait for the timer to tick again. 7338 */ 7339 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 7340 test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state) || 7341 test_bit(__I40E_MDD_EVENT_PENDING, &pf->state) || 7342 test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state)) 7343 i40e_service_event_schedule(pf); 7344 } 7345 7346 /** 7347 * i40e_service_timer - timer callback 7348 * @data: pointer to PF struct 7349 **/ 7350 static void i40e_service_timer(unsigned long data) 7351 { 7352 struct i40e_pf *pf = (struct i40e_pf *)data; 7353 7354 mod_timer(&pf->service_timer, 7355 round_jiffies(jiffies + pf->service_timer_period)); 7356 i40e_service_event_schedule(pf); 7357 } 7358 7359 /** 7360 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 7361 * @vsi: the VSI being configured 7362 **/ 7363 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 7364 { 7365 struct i40e_pf *pf = vsi->back; 7366 7367 switch (vsi->type) { 7368 case I40E_VSI_MAIN: 7369 vsi->alloc_queue_pairs = pf->num_lan_qps; 7370 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7371 I40E_REQ_DESCRIPTOR_MULTIPLE); 7372 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7373 vsi->num_q_vectors = pf->num_lan_msix; 7374 else 7375 vsi->num_q_vectors = 1; 7376 7377 break; 7378 7379 case I40E_VSI_FDIR: 7380 vsi->alloc_queue_pairs = 1; 7381 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 7382 I40E_REQ_DESCRIPTOR_MULTIPLE); 7383 vsi->num_q_vectors = pf->num_fdsb_msix; 7384 break; 7385 7386 case I40E_VSI_VMDQ2: 7387 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 7388 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7389 I40E_REQ_DESCRIPTOR_MULTIPLE); 7390 vsi->num_q_vectors = pf->num_vmdq_msix; 7391 break; 7392 7393 case I40E_VSI_SRIOV: 7394 vsi->alloc_queue_pairs = pf->num_vf_qps; 7395 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 7396 I40E_REQ_DESCRIPTOR_MULTIPLE); 7397 break; 7398 7399 default: 7400 WARN_ON(1); 7401 return -ENODATA; 7402 } 7403 7404 return 0; 7405 } 7406 7407 /** 7408 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 7409 * @type: VSI pointer 7410 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 7411 * 7412 * On error: returns error code (negative) 7413 * On success: returns 0 7414 **/ 7415 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 7416 { 7417 int size; 7418 int ret = 0; 7419 7420 /* allocate memory for both Tx and Rx ring pointers */ 7421 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2; 7422 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 7423 if (!vsi->tx_rings) 7424 return -ENOMEM; 7425 vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs]; 7426 7427 if (alloc_qvectors) { 7428 /* allocate memory for q_vector pointers */ 7429 size = sizeof(struct i40e_q_vector *) * vsi->num_q_vectors; 7430 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 7431 if (!vsi->q_vectors) { 7432 ret = -ENOMEM; 7433 goto err_vectors; 7434 } 7435 } 7436 return ret; 7437 7438 err_vectors: 7439 kfree(vsi->tx_rings); 7440 return ret; 7441 } 7442 7443 /** 7444 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 7445 * @pf: board private structure 7446 * @type: type of VSI 7447 * 7448 * On error: returns error code (negative) 7449 * On success: returns vsi index in PF (positive) 7450 **/ 7451 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 7452 { 7453 int ret = -ENODEV; 7454 struct i40e_vsi *vsi; 7455 int vsi_idx; 7456 int i; 7457 7458 /* Need to protect the allocation of the VSIs at the PF level */ 7459 mutex_lock(&pf->switch_mutex); 7460 7461 /* VSI list may be fragmented if VSI creation/destruction has 7462 * been happening. We can afford to do a quick scan to look 7463 * for any free VSIs in the list. 7464 * 7465 * find next empty vsi slot, looping back around if necessary 7466 */ 7467 i = pf->next_vsi; 7468 while (i < pf->num_alloc_vsi && pf->vsi[i]) 7469 i++; 7470 if (i >= pf->num_alloc_vsi) { 7471 i = 0; 7472 while (i < pf->next_vsi && pf->vsi[i]) 7473 i++; 7474 } 7475 7476 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 7477 vsi_idx = i; /* Found one! */ 7478 } else { 7479 ret = -ENODEV; 7480 goto unlock_pf; /* out of VSI slots! */ 7481 } 7482 pf->next_vsi = ++i; 7483 7484 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 7485 if (!vsi) { 7486 ret = -ENOMEM; 7487 goto unlock_pf; 7488 } 7489 vsi->type = type; 7490 vsi->back = pf; 7491 set_bit(__I40E_DOWN, &vsi->state); 7492 vsi->flags = 0; 7493 vsi->idx = vsi_idx; 7494 vsi->int_rate_limit = 0; 7495 vsi->rss_table_size = (vsi->type == I40E_VSI_MAIN) ? 7496 pf->rss_table_size : 64; 7497 vsi->netdev_registered = false; 7498 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 7499 hash_init(vsi->mac_filter_hash); 7500 vsi->irqs_ready = false; 7501 7502 ret = i40e_set_num_rings_in_vsi(vsi); 7503 if (ret) 7504 goto err_rings; 7505 7506 ret = i40e_vsi_alloc_arrays(vsi, true); 7507 if (ret) 7508 goto err_rings; 7509 7510 /* Setup default MSIX irq handler for VSI */ 7511 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 7512 7513 /* Initialize VSI lock */ 7514 spin_lock_init(&vsi->mac_filter_hash_lock); 7515 pf->vsi[vsi_idx] = vsi; 7516 ret = vsi_idx; 7517 goto unlock_pf; 7518 7519 err_rings: 7520 pf->next_vsi = i - 1; 7521 kfree(vsi); 7522 unlock_pf: 7523 mutex_unlock(&pf->switch_mutex); 7524 return ret; 7525 } 7526 7527 /** 7528 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 7529 * @type: VSI pointer 7530 * @free_qvectors: a bool to specify if q_vectors need to be freed. 7531 * 7532 * On error: returns error code (negative) 7533 * On success: returns 0 7534 **/ 7535 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 7536 { 7537 /* free the ring and vector containers */ 7538 if (free_qvectors) { 7539 kfree(vsi->q_vectors); 7540 vsi->q_vectors = NULL; 7541 } 7542 kfree(vsi->tx_rings); 7543 vsi->tx_rings = NULL; 7544 vsi->rx_rings = NULL; 7545 } 7546 7547 /** 7548 * i40e_clear_rss_config_user - clear the user configured RSS hash keys 7549 * and lookup table 7550 * @vsi: Pointer to VSI structure 7551 */ 7552 static void i40e_clear_rss_config_user(struct i40e_vsi *vsi) 7553 { 7554 if (!vsi) 7555 return; 7556 7557 kfree(vsi->rss_hkey_user); 7558 vsi->rss_hkey_user = NULL; 7559 7560 kfree(vsi->rss_lut_user); 7561 vsi->rss_lut_user = NULL; 7562 } 7563 7564 /** 7565 * i40e_vsi_clear - Deallocate the VSI provided 7566 * @vsi: the VSI being un-configured 7567 **/ 7568 static int i40e_vsi_clear(struct i40e_vsi *vsi) 7569 { 7570 struct i40e_pf *pf; 7571 7572 if (!vsi) 7573 return 0; 7574 7575 if (!vsi->back) 7576 goto free_vsi; 7577 pf = vsi->back; 7578 7579 mutex_lock(&pf->switch_mutex); 7580 if (!pf->vsi[vsi->idx]) { 7581 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n", 7582 vsi->idx, vsi->idx, vsi, vsi->type); 7583 goto unlock_vsi; 7584 } 7585 7586 if (pf->vsi[vsi->idx] != vsi) { 7587 dev_err(&pf->pdev->dev, 7588 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n", 7589 pf->vsi[vsi->idx]->idx, 7590 pf->vsi[vsi->idx], 7591 pf->vsi[vsi->idx]->type, 7592 vsi->idx, vsi, vsi->type); 7593 goto unlock_vsi; 7594 } 7595 7596 /* updates the PF for this cleared vsi */ 7597 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 7598 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 7599 7600 i40e_vsi_free_arrays(vsi, true); 7601 i40e_clear_rss_config_user(vsi); 7602 7603 pf->vsi[vsi->idx] = NULL; 7604 if (vsi->idx < pf->next_vsi) 7605 pf->next_vsi = vsi->idx; 7606 7607 unlock_vsi: 7608 mutex_unlock(&pf->switch_mutex); 7609 free_vsi: 7610 kfree(vsi); 7611 7612 return 0; 7613 } 7614 7615 /** 7616 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 7617 * @vsi: the VSI being cleaned 7618 **/ 7619 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 7620 { 7621 int i; 7622 7623 if (vsi->tx_rings && vsi->tx_rings[0]) { 7624 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7625 kfree_rcu(vsi->tx_rings[i], rcu); 7626 vsi->tx_rings[i] = NULL; 7627 vsi->rx_rings[i] = NULL; 7628 } 7629 } 7630 } 7631 7632 /** 7633 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 7634 * @vsi: the VSI being configured 7635 **/ 7636 static int i40e_alloc_rings(struct i40e_vsi *vsi) 7637 { 7638 struct i40e_ring *tx_ring, *rx_ring; 7639 struct i40e_pf *pf = vsi->back; 7640 int i; 7641 7642 /* Set basic values in the rings to be used later during open() */ 7643 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 7644 /* allocate space for both Tx and Rx in one shot */ 7645 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL); 7646 if (!tx_ring) 7647 goto err_out; 7648 7649 tx_ring->queue_index = i; 7650 tx_ring->reg_idx = vsi->base_queue + i; 7651 tx_ring->ring_active = false; 7652 tx_ring->vsi = vsi; 7653 tx_ring->netdev = vsi->netdev; 7654 tx_ring->dev = &pf->pdev->dev; 7655 tx_ring->count = vsi->num_desc; 7656 tx_ring->size = 0; 7657 tx_ring->dcb_tc = 0; 7658 if (vsi->back->flags & I40E_FLAG_WB_ON_ITR_CAPABLE) 7659 tx_ring->flags = I40E_TXR_FLAGS_WB_ON_ITR; 7660 tx_ring->tx_itr_setting = pf->tx_itr_default; 7661 vsi->tx_rings[i] = tx_ring; 7662 7663 rx_ring = &tx_ring[1]; 7664 rx_ring->queue_index = i; 7665 rx_ring->reg_idx = vsi->base_queue + i; 7666 rx_ring->ring_active = false; 7667 rx_ring->vsi = vsi; 7668 rx_ring->netdev = vsi->netdev; 7669 rx_ring->dev = &pf->pdev->dev; 7670 rx_ring->count = vsi->num_desc; 7671 rx_ring->size = 0; 7672 rx_ring->dcb_tc = 0; 7673 rx_ring->rx_itr_setting = pf->rx_itr_default; 7674 vsi->rx_rings[i] = rx_ring; 7675 } 7676 7677 return 0; 7678 7679 err_out: 7680 i40e_vsi_clear_rings(vsi); 7681 return -ENOMEM; 7682 } 7683 7684 /** 7685 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 7686 * @pf: board private structure 7687 * @vectors: the number of MSI-X vectors to request 7688 * 7689 * Returns the number of vectors reserved, or error 7690 **/ 7691 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 7692 { 7693 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 7694 I40E_MIN_MSIX, vectors); 7695 if (vectors < 0) { 7696 dev_info(&pf->pdev->dev, 7697 "MSI-X vector reservation failed: %d\n", vectors); 7698 vectors = 0; 7699 } 7700 7701 return vectors; 7702 } 7703 7704 /** 7705 * i40e_init_msix - Setup the MSIX capability 7706 * @pf: board private structure 7707 * 7708 * Work with the OS to set up the MSIX vectors needed. 7709 * 7710 * Returns the number of vectors reserved or negative on failure 7711 **/ 7712 static int i40e_init_msix(struct i40e_pf *pf) 7713 { 7714 struct i40e_hw *hw = &pf->hw; 7715 int cpus, extra_vectors; 7716 int vectors_left; 7717 int v_budget, i; 7718 int v_actual; 7719 int iwarp_requested = 0; 7720 7721 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 7722 return -ENODEV; 7723 7724 /* The number of vectors we'll request will be comprised of: 7725 * - Add 1 for "other" cause for Admin Queue events, etc. 7726 * - The number of LAN queue pairs 7727 * - Queues being used for RSS. 7728 * We don't need as many as max_rss_size vectors. 7729 * use rss_size instead in the calculation since that 7730 * is governed by number of cpus in the system. 7731 * - assumes symmetric Tx/Rx pairing 7732 * - The number of VMDq pairs 7733 * - The CPU count within the NUMA node if iWARP is enabled 7734 * Once we count this up, try the request. 7735 * 7736 * If we can't get what we want, we'll simplify to nearly nothing 7737 * and try again. If that still fails, we punt. 7738 */ 7739 vectors_left = hw->func_caps.num_msix_vectors; 7740 v_budget = 0; 7741 7742 /* reserve one vector for miscellaneous handler */ 7743 if (vectors_left) { 7744 v_budget++; 7745 vectors_left--; 7746 } 7747 7748 /* reserve some vectors for the main PF traffic queues. Initially we 7749 * only reserve at most 50% of the available vectors, in the case that 7750 * the number of online CPUs is large. This ensures that we can enable 7751 * extra features as well. Once we've enabled the other features, we 7752 * will use any remaining vectors to reach as close as we can to the 7753 * number of online CPUs. 7754 */ 7755 cpus = num_online_cpus(); 7756 pf->num_lan_msix = min_t(int, cpus, vectors_left / 2); 7757 vectors_left -= pf->num_lan_msix; 7758 7759 /* reserve one vector for sideband flow director */ 7760 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 7761 if (vectors_left) { 7762 pf->num_fdsb_msix = 1; 7763 v_budget++; 7764 vectors_left--; 7765 } else { 7766 pf->num_fdsb_msix = 0; 7767 } 7768 } 7769 7770 /* can we reserve enough for iWARP? */ 7771 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7772 iwarp_requested = pf->num_iwarp_msix; 7773 7774 if (!vectors_left) 7775 pf->num_iwarp_msix = 0; 7776 else if (vectors_left < pf->num_iwarp_msix) 7777 pf->num_iwarp_msix = 1; 7778 v_budget += pf->num_iwarp_msix; 7779 vectors_left -= pf->num_iwarp_msix; 7780 } 7781 7782 /* any vectors left over go for VMDq support */ 7783 if (pf->flags & I40E_FLAG_VMDQ_ENABLED) { 7784 int vmdq_vecs_wanted = pf->num_vmdq_vsis * pf->num_vmdq_qps; 7785 int vmdq_vecs = min_t(int, vectors_left, vmdq_vecs_wanted); 7786 7787 if (!vectors_left) { 7788 pf->num_vmdq_msix = 0; 7789 pf->num_vmdq_qps = 0; 7790 } else { 7791 /* if we're short on vectors for what's desired, we limit 7792 * the queues per vmdq. If this is still more than are 7793 * available, the user will need to change the number of 7794 * queues/vectors used by the PF later with the ethtool 7795 * channels command 7796 */ 7797 if (vmdq_vecs < vmdq_vecs_wanted) 7798 pf->num_vmdq_qps = 1; 7799 pf->num_vmdq_msix = pf->num_vmdq_qps; 7800 7801 v_budget += vmdq_vecs; 7802 vectors_left -= vmdq_vecs; 7803 } 7804 } 7805 7806 /* On systems with a large number of SMP cores, we previously limited 7807 * the number of vectors for num_lan_msix to be at most 50% of the 7808 * available vectors, to allow for other features. Now, we add back 7809 * the remaining vectors. However, we ensure that the total 7810 * num_lan_msix will not exceed num_online_cpus(). To do this, we 7811 * calculate the number of vectors we can add without going over the 7812 * cap of CPUs. For systems with a small number of CPUs this will be 7813 * zero. 7814 */ 7815 extra_vectors = min_t(int, cpus - pf->num_lan_msix, vectors_left); 7816 pf->num_lan_msix += extra_vectors; 7817 vectors_left -= extra_vectors; 7818 7819 WARN(vectors_left < 0, 7820 "Calculation of remaining vectors underflowed. This is an accounting bug when determining total MSI-X vectors.\n"); 7821 7822 v_budget += pf->num_lan_msix; 7823 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 7824 GFP_KERNEL); 7825 if (!pf->msix_entries) 7826 return -ENOMEM; 7827 7828 for (i = 0; i < v_budget; i++) 7829 pf->msix_entries[i].entry = i; 7830 v_actual = i40e_reserve_msix_vectors(pf, v_budget); 7831 7832 if (v_actual < I40E_MIN_MSIX) { 7833 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 7834 kfree(pf->msix_entries); 7835 pf->msix_entries = NULL; 7836 pci_disable_msix(pf->pdev); 7837 return -ENODEV; 7838 7839 } else if (v_actual == I40E_MIN_MSIX) { 7840 /* Adjust for minimal MSIX use */ 7841 pf->num_vmdq_vsis = 0; 7842 pf->num_vmdq_qps = 0; 7843 pf->num_lan_qps = 1; 7844 pf->num_lan_msix = 1; 7845 7846 } else if (!vectors_left) { 7847 /* If we have limited resources, we will start with no vectors 7848 * for the special features and then allocate vectors to some 7849 * of these features based on the policy and at the end disable 7850 * the features that did not get any vectors. 7851 */ 7852 int vec; 7853 7854 dev_info(&pf->pdev->dev, 7855 "MSI-X vector limit reached, attempting to redistribute vectors\n"); 7856 /* reserve the misc vector */ 7857 vec = v_actual - 1; 7858 7859 /* Scale vector usage down */ 7860 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 7861 pf->num_vmdq_vsis = 1; 7862 pf->num_vmdq_qps = 1; 7863 7864 /* partition out the remaining vectors */ 7865 switch (vec) { 7866 case 2: 7867 pf->num_lan_msix = 1; 7868 break; 7869 case 3: 7870 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7871 pf->num_lan_msix = 1; 7872 pf->num_iwarp_msix = 1; 7873 } else { 7874 pf->num_lan_msix = 2; 7875 } 7876 break; 7877 default: 7878 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 7879 pf->num_iwarp_msix = min_t(int, (vec / 3), 7880 iwarp_requested); 7881 pf->num_vmdq_vsis = min_t(int, (vec / 3), 7882 I40E_DEFAULT_NUM_VMDQ_VSI); 7883 } else { 7884 pf->num_vmdq_vsis = min_t(int, (vec / 2), 7885 I40E_DEFAULT_NUM_VMDQ_VSI); 7886 } 7887 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 7888 pf->num_fdsb_msix = 1; 7889 vec--; 7890 } 7891 pf->num_lan_msix = min_t(int, 7892 (vec - (pf->num_iwarp_msix + pf->num_vmdq_vsis)), 7893 pf->num_lan_msix); 7894 pf->num_lan_qps = pf->num_lan_msix; 7895 break; 7896 } 7897 } 7898 7899 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 7900 (pf->num_fdsb_msix == 0)) { 7901 dev_info(&pf->pdev->dev, "Sideband Flowdir disabled, not enough MSI-X vectors\n"); 7902 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 7903 } 7904 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 7905 (pf->num_vmdq_msix == 0)) { 7906 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 7907 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 7908 } 7909 7910 if ((pf->flags & I40E_FLAG_IWARP_ENABLED) && 7911 (pf->num_iwarp_msix == 0)) { 7912 dev_info(&pf->pdev->dev, "IWARP disabled, not enough MSI-X vectors\n"); 7913 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 7914 } 7915 i40e_debug(&pf->hw, I40E_DEBUG_INIT, 7916 "MSI-X vector distribution: PF %d, VMDq %d, FDSB %d, iWARP %d\n", 7917 pf->num_lan_msix, 7918 pf->num_vmdq_msix * pf->num_vmdq_vsis, 7919 pf->num_fdsb_msix, 7920 pf->num_iwarp_msix); 7921 7922 return v_actual; 7923 } 7924 7925 /** 7926 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 7927 * @vsi: the VSI being configured 7928 * @v_idx: index of the vector in the vsi struct 7929 * @cpu: cpu to be used on affinity_mask 7930 * 7931 * We allocate one q_vector. If allocation fails we return -ENOMEM. 7932 **/ 7933 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx, int cpu) 7934 { 7935 struct i40e_q_vector *q_vector; 7936 7937 /* allocate q_vector */ 7938 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 7939 if (!q_vector) 7940 return -ENOMEM; 7941 7942 q_vector->vsi = vsi; 7943 q_vector->v_idx = v_idx; 7944 cpumask_set_cpu(cpu, &q_vector->affinity_mask); 7945 7946 if (vsi->netdev) 7947 netif_napi_add(vsi->netdev, &q_vector->napi, 7948 i40e_napi_poll, NAPI_POLL_WEIGHT); 7949 7950 q_vector->rx.latency_range = I40E_LOW_LATENCY; 7951 q_vector->tx.latency_range = I40E_LOW_LATENCY; 7952 7953 /* tie q_vector and vsi together */ 7954 vsi->q_vectors[v_idx] = q_vector; 7955 7956 return 0; 7957 } 7958 7959 /** 7960 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 7961 * @vsi: the VSI being configured 7962 * 7963 * We allocate one q_vector per queue interrupt. If allocation fails we 7964 * return -ENOMEM. 7965 **/ 7966 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 7967 { 7968 struct i40e_pf *pf = vsi->back; 7969 int err, v_idx, num_q_vectors, current_cpu; 7970 7971 /* if not MSIX, give the one vector only to the LAN VSI */ 7972 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 7973 num_q_vectors = vsi->num_q_vectors; 7974 else if (vsi == pf->vsi[pf->lan_vsi]) 7975 num_q_vectors = 1; 7976 else 7977 return -EINVAL; 7978 7979 current_cpu = cpumask_first(cpu_online_mask); 7980 7981 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 7982 err = i40e_vsi_alloc_q_vector(vsi, v_idx, current_cpu); 7983 if (err) 7984 goto err_out; 7985 current_cpu = cpumask_next(current_cpu, cpu_online_mask); 7986 if (unlikely(current_cpu >= nr_cpu_ids)) 7987 current_cpu = cpumask_first(cpu_online_mask); 7988 } 7989 7990 return 0; 7991 7992 err_out: 7993 while (v_idx--) 7994 i40e_free_q_vector(vsi, v_idx); 7995 7996 return err; 7997 } 7998 7999 /** 8000 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 8001 * @pf: board private structure to initialize 8002 **/ 8003 static int i40e_init_interrupt_scheme(struct i40e_pf *pf) 8004 { 8005 int vectors = 0; 8006 ssize_t size; 8007 8008 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 8009 vectors = i40e_init_msix(pf); 8010 if (vectors < 0) { 8011 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 8012 I40E_FLAG_IWARP_ENABLED | 8013 I40E_FLAG_RSS_ENABLED | 8014 I40E_FLAG_DCB_CAPABLE | 8015 I40E_FLAG_DCB_ENABLED | 8016 I40E_FLAG_SRIOV_ENABLED | 8017 I40E_FLAG_FD_SB_ENABLED | 8018 I40E_FLAG_FD_ATR_ENABLED | 8019 I40E_FLAG_VMDQ_ENABLED); 8020 8021 /* rework the queue expectations without MSIX */ 8022 i40e_determine_queue_usage(pf); 8023 } 8024 } 8025 8026 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 8027 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 8028 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 8029 vectors = pci_enable_msi(pf->pdev); 8030 if (vectors < 0) { 8031 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", 8032 vectors); 8033 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 8034 } 8035 vectors = 1; /* one MSI or Legacy vector */ 8036 } 8037 8038 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 8039 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 8040 8041 /* set up vector assignment tracking */ 8042 size = sizeof(struct i40e_lump_tracking) + (sizeof(u16) * vectors); 8043 pf->irq_pile = kzalloc(size, GFP_KERNEL); 8044 if (!pf->irq_pile) { 8045 dev_err(&pf->pdev->dev, "error allocating irq_pile memory\n"); 8046 return -ENOMEM; 8047 } 8048 pf->irq_pile->num_entries = vectors; 8049 pf->irq_pile->search_hint = 0; 8050 8051 /* track first vector for misc interrupts, ignore return */ 8052 (void)i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT - 1); 8053 8054 return 0; 8055 } 8056 8057 /** 8058 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 8059 * @pf: board private structure 8060 * 8061 * This sets up the handler for MSIX 0, which is used to manage the 8062 * non-queue interrupts, e.g. AdminQ and errors. This is not used 8063 * when in MSI or Legacy interrupt mode. 8064 **/ 8065 static int i40e_setup_misc_vector(struct i40e_pf *pf) 8066 { 8067 struct i40e_hw *hw = &pf->hw; 8068 int err = 0; 8069 8070 /* Only request the irq if this is the first time through, and 8071 * not when we're rebuilding after a Reset 8072 */ 8073 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) { 8074 err = request_irq(pf->msix_entries[0].vector, 8075 i40e_intr, 0, pf->int_name, pf); 8076 if (err) { 8077 dev_info(&pf->pdev->dev, 8078 "request_irq for %s failed: %d\n", 8079 pf->int_name, err); 8080 return -EFAULT; 8081 } 8082 } 8083 8084 i40e_enable_misc_int_causes(pf); 8085 8086 /* associate no queues to the misc vector */ 8087 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 8088 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 8089 8090 i40e_flush(hw); 8091 8092 i40e_irq_dynamic_enable_icr0(pf, true); 8093 8094 return err; 8095 } 8096 8097 /** 8098 * i40e_config_rss_aq - Prepare for RSS using AQ commands 8099 * @vsi: vsi structure 8100 * @seed: RSS hash seed 8101 **/ 8102 static int i40e_config_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8103 u8 *lut, u16 lut_size) 8104 { 8105 struct i40e_pf *pf = vsi->back; 8106 struct i40e_hw *hw = &pf->hw; 8107 int ret = 0; 8108 8109 if (seed) { 8110 struct i40e_aqc_get_set_rss_key_data *seed_dw = 8111 (struct i40e_aqc_get_set_rss_key_data *)seed; 8112 ret = i40e_aq_set_rss_key(hw, vsi->id, seed_dw); 8113 if (ret) { 8114 dev_info(&pf->pdev->dev, 8115 "Cannot set RSS key, err %s aq_err %s\n", 8116 i40e_stat_str(hw, ret), 8117 i40e_aq_str(hw, hw->aq.asq_last_status)); 8118 return ret; 8119 } 8120 } 8121 if (lut) { 8122 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8123 8124 ret = i40e_aq_set_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8125 if (ret) { 8126 dev_info(&pf->pdev->dev, 8127 "Cannot set RSS lut, err %s aq_err %s\n", 8128 i40e_stat_str(hw, ret), 8129 i40e_aq_str(hw, hw->aq.asq_last_status)); 8130 return ret; 8131 } 8132 } 8133 return ret; 8134 } 8135 8136 /** 8137 * i40e_get_rss_aq - Get RSS keys and lut by using AQ commands 8138 * @vsi: Pointer to vsi structure 8139 * @seed: Buffter to store the hash keys 8140 * @lut: Buffer to store the lookup table entries 8141 * @lut_size: Size of buffer to store the lookup table entries 8142 * 8143 * Return 0 on success, negative on failure 8144 */ 8145 static int i40e_get_rss_aq(struct i40e_vsi *vsi, const u8 *seed, 8146 u8 *lut, u16 lut_size) 8147 { 8148 struct i40e_pf *pf = vsi->back; 8149 struct i40e_hw *hw = &pf->hw; 8150 int ret = 0; 8151 8152 if (seed) { 8153 ret = i40e_aq_get_rss_key(hw, vsi->id, 8154 (struct i40e_aqc_get_set_rss_key_data *)seed); 8155 if (ret) { 8156 dev_info(&pf->pdev->dev, 8157 "Cannot get RSS key, err %s aq_err %s\n", 8158 i40e_stat_str(&pf->hw, ret), 8159 i40e_aq_str(&pf->hw, 8160 pf->hw.aq.asq_last_status)); 8161 return ret; 8162 } 8163 } 8164 8165 if (lut) { 8166 bool pf_lut = vsi->type == I40E_VSI_MAIN ? true : false; 8167 8168 ret = i40e_aq_get_rss_lut(hw, vsi->id, pf_lut, lut, lut_size); 8169 if (ret) { 8170 dev_info(&pf->pdev->dev, 8171 "Cannot get RSS lut, err %s aq_err %s\n", 8172 i40e_stat_str(&pf->hw, ret), 8173 i40e_aq_str(&pf->hw, 8174 pf->hw.aq.asq_last_status)); 8175 return ret; 8176 } 8177 } 8178 8179 return ret; 8180 } 8181 8182 /** 8183 * i40e_vsi_config_rss - Prepare for VSI(VMDq) RSS if used 8184 * @vsi: VSI structure 8185 **/ 8186 static int i40e_vsi_config_rss(struct i40e_vsi *vsi) 8187 { 8188 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8189 struct i40e_pf *pf = vsi->back; 8190 u8 *lut; 8191 int ret; 8192 8193 if (!(pf->flags & I40E_FLAG_RSS_AQ_CAPABLE)) 8194 return 0; 8195 8196 if (!vsi->rss_size) 8197 vsi->rss_size = min_t(int, pf->alloc_rss_size, 8198 vsi->num_queue_pairs); 8199 if (!vsi->rss_size) 8200 return -EINVAL; 8201 8202 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8203 if (!lut) 8204 return -ENOMEM; 8205 /* Use the user configured hash keys and lookup table if there is one, 8206 * otherwise use default 8207 */ 8208 if (vsi->rss_lut_user) 8209 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8210 else 8211 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8212 if (vsi->rss_hkey_user) 8213 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8214 else 8215 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8216 ret = i40e_config_rss_aq(vsi, seed, lut, vsi->rss_table_size); 8217 kfree(lut); 8218 8219 return ret; 8220 } 8221 8222 /** 8223 * i40e_config_rss_reg - Configure RSS keys and lut by writing registers 8224 * @vsi: Pointer to vsi structure 8225 * @seed: RSS hash seed 8226 * @lut: Lookup table 8227 * @lut_size: Lookup table size 8228 * 8229 * Returns 0 on success, negative on failure 8230 **/ 8231 static int i40e_config_rss_reg(struct i40e_vsi *vsi, const u8 *seed, 8232 const u8 *lut, u16 lut_size) 8233 { 8234 struct i40e_pf *pf = vsi->back; 8235 struct i40e_hw *hw = &pf->hw; 8236 u16 vf_id = vsi->vf_id; 8237 u8 i; 8238 8239 /* Fill out hash function seed */ 8240 if (seed) { 8241 u32 *seed_dw = (u32 *)seed; 8242 8243 if (vsi->type == I40E_VSI_MAIN) { 8244 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8245 wr32(hw, I40E_PFQF_HKEY(i), seed_dw[i]); 8246 } else if (vsi->type == I40E_VSI_SRIOV) { 8247 for (i = 0; i <= I40E_VFQF_HKEY1_MAX_INDEX; i++) 8248 wr32(hw, I40E_VFQF_HKEY1(i, vf_id), seed_dw[i]); 8249 } else { 8250 dev_err(&pf->pdev->dev, "Cannot set RSS seed - invalid VSI type\n"); 8251 } 8252 } 8253 8254 if (lut) { 8255 u32 *lut_dw = (u32 *)lut; 8256 8257 if (vsi->type == I40E_VSI_MAIN) { 8258 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8259 return -EINVAL; 8260 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8261 wr32(hw, I40E_PFQF_HLUT(i), lut_dw[i]); 8262 } else if (vsi->type == I40E_VSI_SRIOV) { 8263 if (lut_size != I40E_VF_HLUT_ARRAY_SIZE) 8264 return -EINVAL; 8265 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8266 wr32(hw, I40E_VFQF_HLUT1(i, vf_id), lut_dw[i]); 8267 } else { 8268 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8269 } 8270 } 8271 i40e_flush(hw); 8272 8273 return 0; 8274 } 8275 8276 /** 8277 * i40e_get_rss_reg - Get the RSS keys and lut by reading registers 8278 * @vsi: Pointer to VSI structure 8279 * @seed: Buffer to store the keys 8280 * @lut: Buffer to store the lookup table entries 8281 * @lut_size: Size of buffer to store the lookup table entries 8282 * 8283 * Returns 0 on success, negative on failure 8284 */ 8285 static int i40e_get_rss_reg(struct i40e_vsi *vsi, u8 *seed, 8286 u8 *lut, u16 lut_size) 8287 { 8288 struct i40e_pf *pf = vsi->back; 8289 struct i40e_hw *hw = &pf->hw; 8290 u16 i; 8291 8292 if (seed) { 8293 u32 *seed_dw = (u32 *)seed; 8294 8295 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 8296 seed_dw[i] = i40e_read_rx_ctl(hw, I40E_PFQF_HKEY(i)); 8297 } 8298 if (lut) { 8299 u32 *lut_dw = (u32 *)lut; 8300 8301 if (lut_size != I40E_HLUT_ARRAY_SIZE) 8302 return -EINVAL; 8303 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8304 lut_dw[i] = rd32(hw, I40E_PFQF_HLUT(i)); 8305 } 8306 8307 return 0; 8308 } 8309 8310 /** 8311 * i40e_config_rss - Configure RSS keys and lut 8312 * @vsi: Pointer to VSI structure 8313 * @seed: RSS hash seed 8314 * @lut: Lookup table 8315 * @lut_size: Lookup table size 8316 * 8317 * Returns 0 on success, negative on failure 8318 */ 8319 int i40e_config_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8320 { 8321 struct i40e_pf *pf = vsi->back; 8322 8323 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) 8324 return i40e_config_rss_aq(vsi, seed, lut, lut_size); 8325 else 8326 return i40e_config_rss_reg(vsi, seed, lut, lut_size); 8327 } 8328 8329 /** 8330 * i40e_get_rss - Get RSS keys and lut 8331 * @vsi: Pointer to VSI structure 8332 * @seed: Buffer to store the keys 8333 * @lut: Buffer to store the lookup table entries 8334 * lut_size: Size of buffer to store the lookup table entries 8335 * 8336 * Returns 0 on success, negative on failure 8337 */ 8338 int i40e_get_rss(struct i40e_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size) 8339 { 8340 struct i40e_pf *pf = vsi->back; 8341 8342 if (pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) 8343 return i40e_get_rss_aq(vsi, seed, lut, lut_size); 8344 else 8345 return i40e_get_rss_reg(vsi, seed, lut, lut_size); 8346 } 8347 8348 /** 8349 * i40e_fill_rss_lut - Fill the RSS lookup table with default values 8350 * @pf: Pointer to board private structure 8351 * @lut: Lookup table 8352 * @rss_table_size: Lookup table size 8353 * @rss_size: Range of queue number for hashing 8354 */ 8355 void i40e_fill_rss_lut(struct i40e_pf *pf, u8 *lut, 8356 u16 rss_table_size, u16 rss_size) 8357 { 8358 u16 i; 8359 8360 for (i = 0; i < rss_table_size; i++) 8361 lut[i] = i % rss_size; 8362 } 8363 8364 /** 8365 * i40e_pf_config_rss - Prepare for RSS if used 8366 * @pf: board private structure 8367 **/ 8368 static int i40e_pf_config_rss(struct i40e_pf *pf) 8369 { 8370 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8371 u8 seed[I40E_HKEY_ARRAY_SIZE]; 8372 u8 *lut; 8373 struct i40e_hw *hw = &pf->hw; 8374 u32 reg_val; 8375 u64 hena; 8376 int ret; 8377 8378 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 8379 hena = (u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(0)) | 8380 ((u64)i40e_read_rx_ctl(hw, I40E_PFQF_HENA(1)) << 32); 8381 hena |= i40e_pf_get_default_rss_hena(pf); 8382 8383 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), (u32)hena); 8384 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 8385 8386 /* Determine the RSS table size based on the hardware capabilities */ 8387 reg_val = i40e_read_rx_ctl(hw, I40E_PFQF_CTL_0); 8388 reg_val = (pf->rss_table_size == 512) ? 8389 (reg_val | I40E_PFQF_CTL_0_HASHLUTSIZE_512) : 8390 (reg_val & ~I40E_PFQF_CTL_0_HASHLUTSIZE_512); 8391 i40e_write_rx_ctl(hw, I40E_PFQF_CTL_0, reg_val); 8392 8393 /* Determine the RSS size of the VSI */ 8394 if (!vsi->rss_size) { 8395 u16 qcount; 8396 8397 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 8398 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 8399 } 8400 if (!vsi->rss_size) 8401 return -EINVAL; 8402 8403 lut = kzalloc(vsi->rss_table_size, GFP_KERNEL); 8404 if (!lut) 8405 return -ENOMEM; 8406 8407 /* Use user configured lut if there is one, otherwise use default */ 8408 if (vsi->rss_lut_user) 8409 memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size); 8410 else 8411 i40e_fill_rss_lut(pf, lut, vsi->rss_table_size, vsi->rss_size); 8412 8413 /* Use user configured hash key if there is one, otherwise 8414 * use default. 8415 */ 8416 if (vsi->rss_hkey_user) 8417 memcpy(seed, vsi->rss_hkey_user, I40E_HKEY_ARRAY_SIZE); 8418 else 8419 netdev_rss_key_fill((void *)seed, I40E_HKEY_ARRAY_SIZE); 8420 ret = i40e_config_rss(vsi, seed, lut, vsi->rss_table_size); 8421 kfree(lut); 8422 8423 return ret; 8424 } 8425 8426 /** 8427 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 8428 * @pf: board private structure 8429 * @queue_count: the requested queue count for rss. 8430 * 8431 * returns 0 if rss is not enabled, if enabled returns the final rss queue 8432 * count which may be different from the requested queue count. 8433 **/ 8434 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 8435 { 8436 struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; 8437 int new_rss_size; 8438 8439 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 8440 return 0; 8441 8442 new_rss_size = min_t(int, queue_count, pf->rss_size_max); 8443 8444 if (queue_count != vsi->num_queue_pairs) { 8445 u16 qcount; 8446 8447 vsi->req_queue_pairs = queue_count; 8448 i40e_prep_for_reset(pf); 8449 8450 pf->alloc_rss_size = new_rss_size; 8451 8452 i40e_reset_and_rebuild(pf, true); 8453 8454 /* Discard the user configured hash keys and lut, if less 8455 * queues are enabled. 8456 */ 8457 if (queue_count < vsi->rss_size) { 8458 i40e_clear_rss_config_user(vsi); 8459 dev_dbg(&pf->pdev->dev, 8460 "discard user configured hash keys and lut\n"); 8461 } 8462 8463 /* Reset vsi->rss_size, as number of enabled queues changed */ 8464 qcount = vsi->num_queue_pairs / vsi->tc_config.numtc; 8465 vsi->rss_size = min_t(int, pf->alloc_rss_size, qcount); 8466 8467 i40e_pf_config_rss(pf); 8468 } 8469 dev_info(&pf->pdev->dev, "User requested queue count/HW max RSS count: %d/%d\n", 8470 vsi->req_queue_pairs, pf->rss_size_max); 8471 return pf->alloc_rss_size; 8472 } 8473 8474 /** 8475 * i40e_get_npar_bw_setting - Retrieve BW settings for this PF partition 8476 * @pf: board private structure 8477 **/ 8478 i40e_status i40e_get_npar_bw_setting(struct i40e_pf *pf) 8479 { 8480 i40e_status status; 8481 bool min_valid, max_valid; 8482 u32 max_bw, min_bw; 8483 8484 status = i40e_read_bw_from_alt_ram(&pf->hw, &max_bw, &min_bw, 8485 &min_valid, &max_valid); 8486 8487 if (!status) { 8488 if (min_valid) 8489 pf->npar_min_bw = min_bw; 8490 if (max_valid) 8491 pf->npar_max_bw = max_bw; 8492 } 8493 8494 return status; 8495 } 8496 8497 /** 8498 * i40e_set_npar_bw_setting - Set BW settings for this PF partition 8499 * @pf: board private structure 8500 **/ 8501 i40e_status i40e_set_npar_bw_setting(struct i40e_pf *pf) 8502 { 8503 struct i40e_aqc_configure_partition_bw_data bw_data; 8504 i40e_status status; 8505 8506 /* Set the valid bit for this PF */ 8507 bw_data.pf_valid_bits = cpu_to_le16(BIT(pf->hw.pf_id)); 8508 bw_data.max_bw[pf->hw.pf_id] = pf->npar_max_bw & I40E_ALT_BW_VALUE_MASK; 8509 bw_data.min_bw[pf->hw.pf_id] = pf->npar_min_bw & I40E_ALT_BW_VALUE_MASK; 8510 8511 /* Set the new bandwidths */ 8512 status = i40e_aq_configure_partition_bw(&pf->hw, &bw_data, NULL); 8513 8514 return status; 8515 } 8516 8517 /** 8518 * i40e_commit_npar_bw_setting - Commit BW settings for this PF partition 8519 * @pf: board private structure 8520 **/ 8521 i40e_status i40e_commit_npar_bw_setting(struct i40e_pf *pf) 8522 { 8523 /* Commit temporary BW setting to permanent NVM image */ 8524 enum i40e_admin_queue_err last_aq_status; 8525 i40e_status ret; 8526 u16 nvm_word; 8527 8528 if (pf->hw.partition_id != 1) { 8529 dev_info(&pf->pdev->dev, 8530 "Commit BW only works on partition 1! This is partition %d", 8531 pf->hw.partition_id); 8532 ret = I40E_NOT_SUPPORTED; 8533 goto bw_commit_out; 8534 } 8535 8536 /* Acquire NVM for read access */ 8537 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_READ); 8538 last_aq_status = pf->hw.aq.asq_last_status; 8539 if (ret) { 8540 dev_info(&pf->pdev->dev, 8541 "Cannot acquire NVM for read access, err %s aq_err %s\n", 8542 i40e_stat_str(&pf->hw, ret), 8543 i40e_aq_str(&pf->hw, last_aq_status)); 8544 goto bw_commit_out; 8545 } 8546 8547 /* Read word 0x10 of NVM - SW compatibility word 1 */ 8548 ret = i40e_aq_read_nvm(&pf->hw, 8549 I40E_SR_NVM_CONTROL_WORD, 8550 0x10, sizeof(nvm_word), &nvm_word, 8551 false, NULL); 8552 /* Save off last admin queue command status before releasing 8553 * the NVM 8554 */ 8555 last_aq_status = pf->hw.aq.asq_last_status; 8556 i40e_release_nvm(&pf->hw); 8557 if (ret) { 8558 dev_info(&pf->pdev->dev, "NVM read error, err %s aq_err %s\n", 8559 i40e_stat_str(&pf->hw, ret), 8560 i40e_aq_str(&pf->hw, last_aq_status)); 8561 goto bw_commit_out; 8562 } 8563 8564 /* Wait a bit for NVM release to complete */ 8565 msleep(50); 8566 8567 /* Acquire NVM for write access */ 8568 ret = i40e_acquire_nvm(&pf->hw, I40E_RESOURCE_WRITE); 8569 last_aq_status = pf->hw.aq.asq_last_status; 8570 if (ret) { 8571 dev_info(&pf->pdev->dev, 8572 "Cannot acquire NVM for write access, err %s aq_err %s\n", 8573 i40e_stat_str(&pf->hw, ret), 8574 i40e_aq_str(&pf->hw, last_aq_status)); 8575 goto bw_commit_out; 8576 } 8577 /* Write it back out unchanged to initiate update NVM, 8578 * which will force a write of the shadow (alt) RAM to 8579 * the NVM - thus storing the bandwidth values permanently. 8580 */ 8581 ret = i40e_aq_update_nvm(&pf->hw, 8582 I40E_SR_NVM_CONTROL_WORD, 8583 0x10, sizeof(nvm_word), 8584 &nvm_word, true, NULL); 8585 /* Save off last admin queue command status before releasing 8586 * the NVM 8587 */ 8588 last_aq_status = pf->hw.aq.asq_last_status; 8589 i40e_release_nvm(&pf->hw); 8590 if (ret) 8591 dev_info(&pf->pdev->dev, 8592 "BW settings NOT SAVED, err %s aq_err %s\n", 8593 i40e_stat_str(&pf->hw, ret), 8594 i40e_aq_str(&pf->hw, last_aq_status)); 8595 bw_commit_out: 8596 8597 return ret; 8598 } 8599 8600 /** 8601 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 8602 * @pf: board private structure to initialize 8603 * 8604 * i40e_sw_init initializes the Adapter private data structure. 8605 * Fields are initialized based on PCI device information and 8606 * OS network device settings (MTU size). 8607 **/ 8608 static int i40e_sw_init(struct i40e_pf *pf) 8609 { 8610 int err = 0; 8611 int size; 8612 8613 /* Set default capability flags */ 8614 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 8615 I40E_FLAG_MSI_ENABLED | 8616 I40E_FLAG_MSIX_ENABLED; 8617 8618 /* Set default ITR */ 8619 pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF; 8620 pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF; 8621 8622 /* Depending on PF configurations, it is possible that the RSS 8623 * maximum might end up larger than the available queues 8624 */ 8625 pf->rss_size_max = BIT(pf->hw.func_caps.rss_table_entry_width); 8626 pf->alloc_rss_size = 1; 8627 pf->rss_table_size = pf->hw.func_caps.rss_table_size; 8628 pf->rss_size_max = min_t(int, pf->rss_size_max, 8629 pf->hw.func_caps.num_tx_qp); 8630 if (pf->hw.func_caps.rss) { 8631 pf->flags |= I40E_FLAG_RSS_ENABLED; 8632 pf->alloc_rss_size = min_t(int, pf->rss_size_max, 8633 num_online_cpus()); 8634 } 8635 8636 /* MFP mode enabled */ 8637 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.flex10_enable) { 8638 pf->flags |= I40E_FLAG_MFP_ENABLED; 8639 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 8640 if (i40e_get_npar_bw_setting(pf)) 8641 dev_warn(&pf->pdev->dev, 8642 "Could not get NPAR bw settings\n"); 8643 else 8644 dev_info(&pf->pdev->dev, 8645 "Min BW = %8.8x, Max BW = %8.8x\n", 8646 pf->npar_min_bw, pf->npar_max_bw); 8647 } 8648 8649 /* FW/NVM is not yet fixed in this regard */ 8650 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 8651 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 8652 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 8653 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 8654 if (pf->flags & I40E_FLAG_MFP_ENABLED && 8655 pf->hw.num_partitions > 1) 8656 dev_info(&pf->pdev->dev, 8657 "Flow Director Sideband mode Disabled in MFP mode\n"); 8658 else 8659 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8660 pf->fdir_pf_filter_count = 8661 pf->hw.func_caps.fd_filters_guaranteed; 8662 pf->hw.fdir_shared_filter_count = 8663 pf->hw.func_caps.fd_filters_best_effort; 8664 } 8665 8666 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8667 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 33)) || 8668 (pf->hw.aq.fw_maj_ver < 4))) { 8669 pf->flags |= I40E_FLAG_RESTART_AUTONEG; 8670 /* No DCB support for FW < v4.33 */ 8671 pf->flags |= I40E_FLAG_NO_DCB_SUPPORT; 8672 } 8673 8674 /* Disable FW LLDP if FW < v4.3 */ 8675 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8676 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver < 3)) || 8677 (pf->hw.aq.fw_maj_ver < 4))) 8678 pf->flags |= I40E_FLAG_STOP_FW_LLDP; 8679 8680 /* Use the FW Set LLDP MIB API if FW > v4.40 */ 8681 if ((pf->hw.mac.type == I40E_MAC_XL710) && 8682 (((pf->hw.aq.fw_maj_ver == 4) && (pf->hw.aq.fw_min_ver >= 40)) || 8683 (pf->hw.aq.fw_maj_ver >= 5))) 8684 pf->flags |= I40E_FLAG_USE_SET_LLDP_MIB; 8685 8686 if (pf->hw.func_caps.vmdq) { 8687 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 8688 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 8689 pf->num_vmdq_qps = i40e_default_queues_per_vmdq(pf); 8690 } 8691 8692 if (pf->hw.func_caps.iwarp) { 8693 pf->flags |= I40E_FLAG_IWARP_ENABLED; 8694 /* IWARP needs one extra vector for CQP just like MISC.*/ 8695 pf->num_iwarp_msix = (int)num_online_cpus() + 1; 8696 } 8697 8698 #ifdef CONFIG_PCI_IOV 8699 if (pf->hw.func_caps.num_vfs && pf->hw.partition_id == 1) { 8700 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 8701 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 8702 pf->num_req_vfs = min_t(int, 8703 pf->hw.func_caps.num_vfs, 8704 I40E_MAX_VF_COUNT); 8705 } 8706 #endif /* CONFIG_PCI_IOV */ 8707 if (pf->hw.mac.type == I40E_MAC_X722) { 8708 pf->flags |= I40E_FLAG_RSS_AQ_CAPABLE 8709 | I40E_FLAG_128_QP_RSS_CAPABLE 8710 | I40E_FLAG_HW_ATR_EVICT_CAPABLE 8711 | I40E_FLAG_OUTER_UDP_CSUM_CAPABLE 8712 | I40E_FLAG_WB_ON_ITR_CAPABLE 8713 | I40E_FLAG_MULTIPLE_TCP_UDP_RSS_PCTYPE 8714 | I40E_FLAG_NO_PCI_LINK_CHECK 8715 | I40E_FLAG_USE_SET_LLDP_MIB 8716 | I40E_FLAG_GENEVE_OFFLOAD_CAPABLE 8717 | I40E_FLAG_PTP_L4_CAPABLE 8718 | I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE; 8719 } else if ((pf->hw.aq.api_maj_ver > 1) || 8720 ((pf->hw.aq.api_maj_ver == 1) && 8721 (pf->hw.aq.api_min_ver > 4))) { 8722 /* Supported in FW API version higher than 1.4 */ 8723 pf->flags |= I40E_FLAG_GENEVE_OFFLOAD_CAPABLE; 8724 pf->hw_disabled_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE; 8725 } else { 8726 pf->hw_disabled_flags = I40E_FLAG_HW_ATR_EVICT_CAPABLE; 8727 } 8728 8729 pf->eeprom_version = 0xDEAD; 8730 pf->lan_veb = I40E_NO_VEB; 8731 pf->lan_vsi = I40E_NO_VSI; 8732 8733 /* By default FW has this off for performance reasons */ 8734 pf->flags &= ~I40E_FLAG_VEB_STATS_ENABLED; 8735 8736 /* set up queue assignment tracking */ 8737 size = sizeof(struct i40e_lump_tracking) 8738 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 8739 pf->qp_pile = kzalloc(size, GFP_KERNEL); 8740 if (!pf->qp_pile) { 8741 err = -ENOMEM; 8742 goto sw_init_done; 8743 } 8744 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 8745 pf->qp_pile->search_hint = 0; 8746 8747 pf->tx_timeout_recovery_level = 1; 8748 8749 mutex_init(&pf->switch_mutex); 8750 8751 /* If NPAR is enabled nudge the Tx scheduler */ 8752 if (pf->hw.func_caps.npar_enable && (!i40e_get_npar_bw_setting(pf))) 8753 i40e_set_npar_bw_setting(pf); 8754 8755 sw_init_done: 8756 return err; 8757 } 8758 8759 /** 8760 * i40e_set_ntuple - set the ntuple feature flag and take action 8761 * @pf: board private structure to initialize 8762 * @features: the feature set that the stack is suggesting 8763 * 8764 * returns a bool to indicate if reset needs to happen 8765 **/ 8766 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 8767 { 8768 bool need_reset = false; 8769 8770 /* Check if Flow Director n-tuple support was enabled or disabled. If 8771 * the state changed, we need to reset. 8772 */ 8773 if (features & NETIF_F_NTUPLE) { 8774 /* Enable filters and mark for reset */ 8775 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 8776 need_reset = true; 8777 /* enable FD_SB only if there is MSI-X vector */ 8778 if (pf->num_fdsb_msix > 0) 8779 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 8780 } else { 8781 /* turn off filters, mark for reset and clear SW filter list */ 8782 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 8783 need_reset = true; 8784 i40e_fdir_filter_exit(pf); 8785 } 8786 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8787 pf->hw_disabled_flags &= ~I40E_FLAG_FD_SB_ENABLED; 8788 /* reset fd counters */ 8789 pf->fd_add_err = 0; 8790 pf->fd_atr_cnt = 0; 8791 /* if ATR was auto disabled it can be re-enabled. */ 8792 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 8793 (pf->hw_disabled_flags & I40E_FLAG_FD_ATR_ENABLED)) { 8794 pf->hw_disabled_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 8795 if (I40E_DEBUG_FD & pf->hw.debug_mask) 8796 dev_info(&pf->pdev->dev, "ATR re-enabled.\n"); 8797 } 8798 } 8799 return need_reset; 8800 } 8801 8802 /** 8803 * i40e_clear_rss_lut - clear the rx hash lookup table 8804 * @vsi: the VSI being configured 8805 **/ 8806 static void i40e_clear_rss_lut(struct i40e_vsi *vsi) 8807 { 8808 struct i40e_pf *pf = vsi->back; 8809 struct i40e_hw *hw = &pf->hw; 8810 u16 vf_id = vsi->vf_id; 8811 u8 i; 8812 8813 if (vsi->type == I40E_VSI_MAIN) { 8814 for (i = 0; i <= I40E_PFQF_HLUT_MAX_INDEX; i++) 8815 wr32(hw, I40E_PFQF_HLUT(i), 0); 8816 } else if (vsi->type == I40E_VSI_SRIOV) { 8817 for (i = 0; i <= I40E_VFQF_HLUT_MAX_INDEX; i++) 8818 i40e_write_rx_ctl(hw, I40E_VFQF_HLUT1(i, vf_id), 0); 8819 } else { 8820 dev_err(&pf->pdev->dev, "Cannot set RSS LUT - invalid VSI type\n"); 8821 } 8822 } 8823 8824 /** 8825 * i40e_set_features - set the netdev feature flags 8826 * @netdev: ptr to the netdev being adjusted 8827 * @features: the feature set that the stack is suggesting 8828 **/ 8829 static int i40e_set_features(struct net_device *netdev, 8830 netdev_features_t features) 8831 { 8832 struct i40e_netdev_priv *np = netdev_priv(netdev); 8833 struct i40e_vsi *vsi = np->vsi; 8834 struct i40e_pf *pf = vsi->back; 8835 bool need_reset; 8836 8837 if (features & NETIF_F_RXHASH && !(netdev->features & NETIF_F_RXHASH)) 8838 i40e_pf_config_rss(pf); 8839 else if (!(features & NETIF_F_RXHASH) && 8840 netdev->features & NETIF_F_RXHASH) 8841 i40e_clear_rss_lut(vsi); 8842 8843 if (features & NETIF_F_HW_VLAN_CTAG_RX) 8844 i40e_vlan_stripping_enable(vsi); 8845 else 8846 i40e_vlan_stripping_disable(vsi); 8847 8848 need_reset = i40e_set_ntuple(pf, features); 8849 8850 if (need_reset) 8851 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 8852 8853 return 0; 8854 } 8855 8856 /** 8857 * i40e_get_udp_port_idx - Lookup a possibly offloaded for Rx UDP port 8858 * @pf: board private structure 8859 * @port: The UDP port to look up 8860 * 8861 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 8862 **/ 8863 static u8 i40e_get_udp_port_idx(struct i40e_pf *pf, u16 port) 8864 { 8865 u8 i; 8866 8867 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 8868 if (pf->udp_ports[i].index == port) 8869 return i; 8870 } 8871 8872 return i; 8873 } 8874 8875 /** 8876 * i40e_udp_tunnel_add - Get notifications about UDP tunnel ports that come up 8877 * @netdev: This physical port's netdev 8878 * @ti: Tunnel endpoint information 8879 **/ 8880 static void i40e_udp_tunnel_add(struct net_device *netdev, 8881 struct udp_tunnel_info *ti) 8882 { 8883 struct i40e_netdev_priv *np = netdev_priv(netdev); 8884 struct i40e_vsi *vsi = np->vsi; 8885 struct i40e_pf *pf = vsi->back; 8886 u16 port = ntohs(ti->port); 8887 u8 next_idx; 8888 u8 idx; 8889 8890 idx = i40e_get_udp_port_idx(pf, port); 8891 8892 /* Check if port already exists */ 8893 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 8894 netdev_info(netdev, "port %d already offloaded\n", port); 8895 return; 8896 } 8897 8898 /* Now check if there is space to add the new port */ 8899 next_idx = i40e_get_udp_port_idx(pf, 0); 8900 8901 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 8902 netdev_info(netdev, "maximum number of offloaded UDP ports reached, not adding port %d\n", 8903 port); 8904 return; 8905 } 8906 8907 switch (ti->type) { 8908 case UDP_TUNNEL_TYPE_VXLAN: 8909 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_VXLAN; 8910 break; 8911 case UDP_TUNNEL_TYPE_GENEVE: 8912 if (!(pf->flags & I40E_FLAG_GENEVE_OFFLOAD_CAPABLE)) 8913 return; 8914 pf->udp_ports[next_idx].type = I40E_AQC_TUNNEL_TYPE_NGE; 8915 break; 8916 default: 8917 return; 8918 } 8919 8920 /* New port: add it and mark its index in the bitmap */ 8921 pf->udp_ports[next_idx].index = port; 8922 pf->pending_udp_bitmap |= BIT_ULL(next_idx); 8923 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 8924 } 8925 8926 /** 8927 * i40e_udp_tunnel_del - Get notifications about UDP tunnel ports that go away 8928 * @netdev: This physical port's netdev 8929 * @ti: Tunnel endpoint information 8930 **/ 8931 static void i40e_udp_tunnel_del(struct net_device *netdev, 8932 struct udp_tunnel_info *ti) 8933 { 8934 struct i40e_netdev_priv *np = netdev_priv(netdev); 8935 struct i40e_vsi *vsi = np->vsi; 8936 struct i40e_pf *pf = vsi->back; 8937 u16 port = ntohs(ti->port); 8938 u8 idx; 8939 8940 idx = i40e_get_udp_port_idx(pf, port); 8941 8942 /* Check if port already exists */ 8943 if (idx >= I40E_MAX_PF_UDP_OFFLOAD_PORTS) 8944 goto not_found; 8945 8946 switch (ti->type) { 8947 case UDP_TUNNEL_TYPE_VXLAN: 8948 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_VXLAN) 8949 goto not_found; 8950 break; 8951 case UDP_TUNNEL_TYPE_GENEVE: 8952 if (pf->udp_ports[idx].type != I40E_AQC_TUNNEL_TYPE_NGE) 8953 goto not_found; 8954 break; 8955 default: 8956 goto not_found; 8957 } 8958 8959 /* if port exists, set it to 0 (mark for deletion) 8960 * and make it pending 8961 */ 8962 pf->udp_ports[idx].index = 0; 8963 pf->pending_udp_bitmap |= BIT_ULL(idx); 8964 pf->flags |= I40E_FLAG_UDP_FILTER_SYNC; 8965 8966 return; 8967 not_found: 8968 netdev_warn(netdev, "UDP port %d was not found, not deleting\n", 8969 port); 8970 } 8971 8972 static int i40e_get_phys_port_id(struct net_device *netdev, 8973 struct netdev_phys_item_id *ppid) 8974 { 8975 struct i40e_netdev_priv *np = netdev_priv(netdev); 8976 struct i40e_pf *pf = np->vsi->back; 8977 struct i40e_hw *hw = &pf->hw; 8978 8979 if (!(pf->flags & I40E_FLAG_PORT_ID_VALID)) 8980 return -EOPNOTSUPP; 8981 8982 ppid->id_len = min_t(int, sizeof(hw->mac.port_addr), sizeof(ppid->id)); 8983 memcpy(ppid->id, hw->mac.port_addr, ppid->id_len); 8984 8985 return 0; 8986 } 8987 8988 /** 8989 * i40e_ndo_fdb_add - add an entry to the hardware database 8990 * @ndm: the input from the stack 8991 * @tb: pointer to array of nladdr (unused) 8992 * @dev: the net device pointer 8993 * @addr: the MAC address entry being added 8994 * @flags: instructions from stack about fdb operation 8995 */ 8996 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 8997 struct net_device *dev, 8998 const unsigned char *addr, u16 vid, 8999 u16 flags) 9000 { 9001 struct i40e_netdev_priv *np = netdev_priv(dev); 9002 struct i40e_pf *pf = np->vsi->back; 9003 int err = 0; 9004 9005 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 9006 return -EOPNOTSUPP; 9007 9008 if (vid) { 9009 pr_info("%s: vlans aren't supported yet for dev_uc|mc_add()\n", dev->name); 9010 return -EINVAL; 9011 } 9012 9013 /* Hardware does not support aging addresses so if a 9014 * ndm_state is given only allow permanent addresses 9015 */ 9016 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 9017 netdev_info(dev, "FDB only supports static addresses\n"); 9018 return -EINVAL; 9019 } 9020 9021 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 9022 err = dev_uc_add_excl(dev, addr); 9023 else if (is_multicast_ether_addr(addr)) 9024 err = dev_mc_add_excl(dev, addr); 9025 else 9026 err = -EINVAL; 9027 9028 /* Only return duplicate errors if NLM_F_EXCL is set */ 9029 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 9030 err = 0; 9031 9032 return err; 9033 } 9034 9035 /** 9036 * i40e_ndo_bridge_setlink - Set the hardware bridge mode 9037 * @dev: the netdev being configured 9038 * @nlh: RTNL message 9039 * 9040 * Inserts a new hardware bridge if not already created and 9041 * enables the bridging mode requested (VEB or VEPA). If the 9042 * hardware bridge has already been inserted and the request 9043 * is to change the mode then that requires a PF reset to 9044 * allow rebuild of the components with required hardware 9045 * bridge mode enabled. 9046 **/ 9047 static int i40e_ndo_bridge_setlink(struct net_device *dev, 9048 struct nlmsghdr *nlh, 9049 u16 flags) 9050 { 9051 struct i40e_netdev_priv *np = netdev_priv(dev); 9052 struct i40e_vsi *vsi = np->vsi; 9053 struct i40e_pf *pf = vsi->back; 9054 struct i40e_veb *veb = NULL; 9055 struct nlattr *attr, *br_spec; 9056 int i, rem; 9057 9058 /* Only for PF VSI for now */ 9059 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9060 return -EOPNOTSUPP; 9061 9062 /* Find the HW bridge for PF VSI */ 9063 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9064 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9065 veb = pf->veb[i]; 9066 } 9067 9068 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 9069 9070 nla_for_each_nested(attr, br_spec, rem) { 9071 __u16 mode; 9072 9073 if (nla_type(attr) != IFLA_BRIDGE_MODE) 9074 continue; 9075 9076 mode = nla_get_u16(attr); 9077 if ((mode != BRIDGE_MODE_VEPA) && 9078 (mode != BRIDGE_MODE_VEB)) 9079 return -EINVAL; 9080 9081 /* Insert a new HW bridge */ 9082 if (!veb) { 9083 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 9084 vsi->tc_config.enabled_tc); 9085 if (veb) { 9086 veb->bridge_mode = mode; 9087 i40e_config_bridge_mode(veb); 9088 } else { 9089 /* No Bridge HW offload available */ 9090 return -ENOENT; 9091 } 9092 break; 9093 } else if (mode != veb->bridge_mode) { 9094 /* Existing HW bridge but different mode needs reset */ 9095 veb->bridge_mode = mode; 9096 /* TODO: If no VFs or VMDq VSIs, disallow VEB mode */ 9097 if (mode == BRIDGE_MODE_VEB) 9098 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 9099 else 9100 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 9101 i40e_do_reset(pf, BIT_ULL(__I40E_PF_RESET_REQUESTED)); 9102 break; 9103 } 9104 } 9105 9106 return 0; 9107 } 9108 9109 /** 9110 * i40e_ndo_bridge_getlink - Get the hardware bridge mode 9111 * @skb: skb buff 9112 * @pid: process id 9113 * @seq: RTNL message seq # 9114 * @dev: the netdev being configured 9115 * @filter_mask: unused 9116 * @nlflags: netlink flags passed in 9117 * 9118 * Return the mode in which the hardware bridge is operating in 9119 * i.e VEB or VEPA. 9120 **/ 9121 static int i40e_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 9122 struct net_device *dev, 9123 u32 __always_unused filter_mask, 9124 int nlflags) 9125 { 9126 struct i40e_netdev_priv *np = netdev_priv(dev); 9127 struct i40e_vsi *vsi = np->vsi; 9128 struct i40e_pf *pf = vsi->back; 9129 struct i40e_veb *veb = NULL; 9130 int i; 9131 9132 /* Only for PF VSI for now */ 9133 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) 9134 return -EOPNOTSUPP; 9135 9136 /* Find the HW bridge for the PF VSI */ 9137 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9138 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9139 veb = pf->veb[i]; 9140 } 9141 9142 if (!veb) 9143 return 0; 9144 9145 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, veb->bridge_mode, 9146 0, 0, nlflags, filter_mask, NULL); 9147 } 9148 9149 /** 9150 * i40e_features_check - Validate encapsulated packet conforms to limits 9151 * @skb: skb buff 9152 * @dev: This physical port's netdev 9153 * @features: Offload features that the stack believes apply 9154 **/ 9155 static netdev_features_t i40e_features_check(struct sk_buff *skb, 9156 struct net_device *dev, 9157 netdev_features_t features) 9158 { 9159 size_t len; 9160 9161 /* No point in doing any of this if neither checksum nor GSO are 9162 * being requested for this frame. We can rule out both by just 9163 * checking for CHECKSUM_PARTIAL 9164 */ 9165 if (skb->ip_summed != CHECKSUM_PARTIAL) 9166 return features; 9167 9168 /* We cannot support GSO if the MSS is going to be less than 9169 * 64 bytes. If it is then we need to drop support for GSO. 9170 */ 9171 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64)) 9172 features &= ~NETIF_F_GSO_MASK; 9173 9174 /* MACLEN can support at most 63 words */ 9175 len = skb_network_header(skb) - skb->data; 9176 if (len & ~(63 * 2)) 9177 goto out_err; 9178 9179 /* IPLEN and EIPLEN can support at most 127 dwords */ 9180 len = skb_transport_header(skb) - skb_network_header(skb); 9181 if (len & ~(127 * 4)) 9182 goto out_err; 9183 9184 if (skb->encapsulation) { 9185 /* L4TUNLEN can support 127 words */ 9186 len = skb_inner_network_header(skb) - skb_transport_header(skb); 9187 if (len & ~(127 * 2)) 9188 goto out_err; 9189 9190 /* IPLEN can support at most 127 dwords */ 9191 len = skb_inner_transport_header(skb) - 9192 skb_inner_network_header(skb); 9193 if (len & ~(127 * 4)) 9194 goto out_err; 9195 } 9196 9197 /* No need to validate L4LEN as TCP is the only protocol with a 9198 * a flexible value and we support all possible values supported 9199 * by TCP, which is at most 15 dwords 9200 */ 9201 9202 return features; 9203 out_err: 9204 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 9205 } 9206 9207 static const struct net_device_ops i40e_netdev_ops = { 9208 .ndo_open = i40e_open, 9209 .ndo_stop = i40e_close, 9210 .ndo_start_xmit = i40e_lan_xmit_frame, 9211 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 9212 .ndo_set_rx_mode = i40e_set_rx_mode, 9213 .ndo_validate_addr = eth_validate_addr, 9214 .ndo_set_mac_address = i40e_set_mac, 9215 .ndo_change_mtu = i40e_change_mtu, 9216 .ndo_do_ioctl = i40e_ioctl, 9217 .ndo_tx_timeout = i40e_tx_timeout, 9218 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 9219 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 9220 #ifdef CONFIG_NET_POLL_CONTROLLER 9221 .ndo_poll_controller = i40e_netpoll, 9222 #endif 9223 .ndo_setup_tc = __i40e_setup_tc, 9224 .ndo_set_features = i40e_set_features, 9225 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 9226 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 9227 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 9228 .ndo_get_vf_config = i40e_ndo_get_vf_config, 9229 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 9230 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofchk, 9231 .ndo_set_vf_trust = i40e_ndo_set_vf_trust, 9232 .ndo_udp_tunnel_add = i40e_udp_tunnel_add, 9233 .ndo_udp_tunnel_del = i40e_udp_tunnel_del, 9234 .ndo_get_phys_port_id = i40e_get_phys_port_id, 9235 .ndo_fdb_add = i40e_ndo_fdb_add, 9236 .ndo_features_check = i40e_features_check, 9237 .ndo_bridge_getlink = i40e_ndo_bridge_getlink, 9238 .ndo_bridge_setlink = i40e_ndo_bridge_setlink, 9239 }; 9240 9241 /** 9242 * i40e_config_netdev - Setup the netdev flags 9243 * @vsi: the VSI being configured 9244 * 9245 * Returns 0 on success, negative value on failure 9246 **/ 9247 static int i40e_config_netdev(struct i40e_vsi *vsi) 9248 { 9249 struct i40e_pf *pf = vsi->back; 9250 struct i40e_hw *hw = &pf->hw; 9251 struct i40e_netdev_priv *np; 9252 struct net_device *netdev; 9253 u8 broadcast[ETH_ALEN]; 9254 u8 mac_addr[ETH_ALEN]; 9255 int etherdev_size; 9256 9257 etherdev_size = sizeof(struct i40e_netdev_priv); 9258 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 9259 if (!netdev) 9260 return -ENOMEM; 9261 9262 vsi->netdev = netdev; 9263 np = netdev_priv(netdev); 9264 np->vsi = vsi; 9265 9266 netdev->hw_enc_features |= NETIF_F_SG | 9267 NETIF_F_IP_CSUM | 9268 NETIF_F_IPV6_CSUM | 9269 NETIF_F_HIGHDMA | 9270 NETIF_F_SOFT_FEATURES | 9271 NETIF_F_TSO | 9272 NETIF_F_TSO_ECN | 9273 NETIF_F_TSO6 | 9274 NETIF_F_GSO_GRE | 9275 NETIF_F_GSO_GRE_CSUM | 9276 NETIF_F_GSO_IPXIP4 | 9277 NETIF_F_GSO_IPXIP6 | 9278 NETIF_F_GSO_UDP_TUNNEL | 9279 NETIF_F_GSO_UDP_TUNNEL_CSUM | 9280 NETIF_F_GSO_PARTIAL | 9281 NETIF_F_SCTP_CRC | 9282 NETIF_F_RXHASH | 9283 NETIF_F_RXCSUM | 9284 0; 9285 9286 if (!(pf->flags & I40E_FLAG_OUTER_UDP_CSUM_CAPABLE)) 9287 netdev->gso_partial_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM; 9288 9289 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM; 9290 9291 /* record features VLANs can make use of */ 9292 netdev->vlan_features |= netdev->hw_enc_features | 9293 NETIF_F_TSO_MANGLEID; 9294 9295 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 9296 netdev->hw_features |= NETIF_F_NTUPLE; 9297 9298 netdev->hw_features |= netdev->hw_enc_features | 9299 NETIF_F_HW_VLAN_CTAG_TX | 9300 NETIF_F_HW_VLAN_CTAG_RX; 9301 9302 netdev->features |= netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER; 9303 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 9304 9305 if (vsi->type == I40E_VSI_MAIN) { 9306 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 9307 ether_addr_copy(mac_addr, hw->mac.perm_addr); 9308 /* The following steps are necessary to properly keep track of 9309 * MAC-VLAN filters loaded into firmware - first we remove 9310 * filter that is automatically generated by firmware and then 9311 * add new filter both to the driver hash table and firmware. 9312 */ 9313 i40e_rm_default_mac_filter(vsi, mac_addr); 9314 spin_lock_bh(&vsi->mac_filter_hash_lock); 9315 i40e_add_mac_filter(vsi, mac_addr); 9316 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9317 } else { 9318 /* relate the VSI_VMDQ name to the VSI_MAIN name */ 9319 snprintf(netdev->name, IFNAMSIZ, "%sv%%d", 9320 pf->vsi[pf->lan_vsi]->netdev->name); 9321 random_ether_addr(mac_addr); 9322 9323 spin_lock_bh(&vsi->mac_filter_hash_lock); 9324 i40e_add_mac_filter(vsi, mac_addr); 9325 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9326 } 9327 9328 /* Add the broadcast filter so that we initially will receive 9329 * broadcast packets. Note that when a new VLAN is first added the 9330 * driver will convert all filters marked I40E_VLAN_ANY into VLAN 9331 * specific filters as part of transitioning into "vlan" operation. 9332 * When more VLANs are added, the driver will copy each existing MAC 9333 * filter and add it for the new VLAN. 9334 * 9335 * Broadcast filters are handled specially by 9336 * i40e_sync_filters_subtask, as the driver must to set the broadcast 9337 * promiscuous bit instead of adding this directly as a MAC/VLAN 9338 * filter. The subtask will update the correct broadcast promiscuous 9339 * bits as VLANs become active or inactive. 9340 */ 9341 eth_broadcast_addr(broadcast); 9342 spin_lock_bh(&vsi->mac_filter_hash_lock); 9343 i40e_add_mac_filter(vsi, broadcast); 9344 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9345 9346 ether_addr_copy(netdev->dev_addr, mac_addr); 9347 ether_addr_copy(netdev->perm_addr, mac_addr); 9348 9349 netdev->priv_flags |= IFF_UNICAST_FLT; 9350 netdev->priv_flags |= IFF_SUPP_NOFCS; 9351 /* Setup netdev TC information */ 9352 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 9353 9354 netdev->netdev_ops = &i40e_netdev_ops; 9355 netdev->watchdog_timeo = 5 * HZ; 9356 i40e_set_ethtool_ops(netdev); 9357 9358 /* MTU range: 68 - 9706 */ 9359 netdev->min_mtu = ETH_MIN_MTU; 9360 netdev->max_mtu = I40E_MAX_RXBUFFER - 9361 (ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN); 9362 9363 return 0; 9364 } 9365 9366 /** 9367 * i40e_vsi_delete - Delete a VSI from the switch 9368 * @vsi: the VSI being removed 9369 * 9370 * Returns 0 on success, negative value on failure 9371 **/ 9372 static void i40e_vsi_delete(struct i40e_vsi *vsi) 9373 { 9374 /* remove default VSI is not allowed */ 9375 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 9376 return; 9377 9378 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 9379 } 9380 9381 /** 9382 * i40e_is_vsi_uplink_mode_veb - Check if the VSI's uplink bridge mode is VEB 9383 * @vsi: the VSI being queried 9384 * 9385 * Returns 1 if HW bridge mode is VEB and return 0 in case of VEPA mode 9386 **/ 9387 int i40e_is_vsi_uplink_mode_veb(struct i40e_vsi *vsi) 9388 { 9389 struct i40e_veb *veb; 9390 struct i40e_pf *pf = vsi->back; 9391 9392 /* Uplink is not a bridge so default to VEB */ 9393 if (vsi->veb_idx == I40E_NO_VEB) 9394 return 1; 9395 9396 veb = pf->veb[vsi->veb_idx]; 9397 if (!veb) { 9398 dev_info(&pf->pdev->dev, 9399 "There is no veb associated with the bridge\n"); 9400 return -ENOENT; 9401 } 9402 9403 /* Uplink is a bridge in VEPA mode */ 9404 if (veb->bridge_mode & BRIDGE_MODE_VEPA) { 9405 return 0; 9406 } else { 9407 /* Uplink is a bridge in VEB mode */ 9408 return 1; 9409 } 9410 9411 /* VEPA is now default bridge, so return 0 */ 9412 return 0; 9413 } 9414 9415 /** 9416 * i40e_add_vsi - Add a VSI to the switch 9417 * @vsi: the VSI being configured 9418 * 9419 * This initializes a VSI context depending on the VSI type to be added and 9420 * passes it down to the add_vsi aq command. 9421 **/ 9422 static int i40e_add_vsi(struct i40e_vsi *vsi) 9423 { 9424 int ret = -ENODEV; 9425 struct i40e_pf *pf = vsi->back; 9426 struct i40e_hw *hw = &pf->hw; 9427 struct i40e_vsi_context ctxt; 9428 struct i40e_mac_filter *f; 9429 struct hlist_node *h; 9430 int bkt; 9431 9432 u8 enabled_tc = 0x1; /* TC0 enabled */ 9433 int f_count = 0; 9434 9435 memset(&ctxt, 0, sizeof(ctxt)); 9436 switch (vsi->type) { 9437 case I40E_VSI_MAIN: 9438 /* The PF's main VSI is already setup as part of the 9439 * device initialization, so we'll not bother with 9440 * the add_vsi call, but we will retrieve the current 9441 * VSI context. 9442 */ 9443 ctxt.seid = pf->main_vsi_seid; 9444 ctxt.pf_num = pf->hw.pf_id; 9445 ctxt.vf_num = 0; 9446 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 9447 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9448 if (ret) { 9449 dev_info(&pf->pdev->dev, 9450 "couldn't get PF vsi config, err %s aq_err %s\n", 9451 i40e_stat_str(&pf->hw, ret), 9452 i40e_aq_str(&pf->hw, 9453 pf->hw.aq.asq_last_status)); 9454 return -ENOENT; 9455 } 9456 vsi->info = ctxt.info; 9457 vsi->info.valid_sections = 0; 9458 9459 vsi->seid = ctxt.seid; 9460 vsi->id = ctxt.vsi_number; 9461 9462 enabled_tc = i40e_pf_get_tc_map(pf); 9463 9464 /* MFP mode setup queue map and update VSI */ 9465 if ((pf->flags & I40E_FLAG_MFP_ENABLED) && 9466 !(pf->hw.func_caps.iscsi)) { /* NIC type PF */ 9467 memset(&ctxt, 0, sizeof(ctxt)); 9468 ctxt.seid = pf->main_vsi_seid; 9469 ctxt.pf_num = pf->hw.pf_id; 9470 ctxt.vf_num = 0; 9471 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 9472 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 9473 if (ret) { 9474 dev_info(&pf->pdev->dev, 9475 "update vsi failed, err %s aq_err %s\n", 9476 i40e_stat_str(&pf->hw, ret), 9477 i40e_aq_str(&pf->hw, 9478 pf->hw.aq.asq_last_status)); 9479 ret = -ENOENT; 9480 goto err; 9481 } 9482 /* update the local VSI info queue map */ 9483 i40e_vsi_update_queue_map(vsi, &ctxt); 9484 vsi->info.valid_sections = 0; 9485 } else { 9486 /* Default/Main VSI is only enabled for TC0 9487 * reconfigure it to enable all TCs that are 9488 * available on the port in SFP mode. 9489 * For MFP case the iSCSI PF would use this 9490 * flow to enable LAN+iSCSI TC. 9491 */ 9492 ret = i40e_vsi_config_tc(vsi, enabled_tc); 9493 if (ret) { 9494 dev_info(&pf->pdev->dev, 9495 "failed to configure TCs for main VSI tc_map 0x%08x, err %s aq_err %s\n", 9496 enabled_tc, 9497 i40e_stat_str(&pf->hw, ret), 9498 i40e_aq_str(&pf->hw, 9499 pf->hw.aq.asq_last_status)); 9500 ret = -ENOENT; 9501 } 9502 } 9503 break; 9504 9505 case I40E_VSI_FDIR: 9506 ctxt.pf_num = hw->pf_id; 9507 ctxt.vf_num = 0; 9508 ctxt.uplink_seid = vsi->uplink_seid; 9509 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9510 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 9511 if ((pf->flags & I40E_FLAG_VEB_MODE_ENABLED) && 9512 (i40e_is_vsi_uplink_mode_veb(vsi))) { 9513 ctxt.info.valid_sections |= 9514 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9515 ctxt.info.switch_id = 9516 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9517 } 9518 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9519 break; 9520 9521 case I40E_VSI_VMDQ2: 9522 ctxt.pf_num = hw->pf_id; 9523 ctxt.vf_num = 0; 9524 ctxt.uplink_seid = vsi->uplink_seid; 9525 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9526 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 9527 9528 /* This VSI is connected to VEB so the switch_id 9529 * should be set to zero by default. 9530 */ 9531 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9532 ctxt.info.valid_sections |= 9533 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9534 ctxt.info.switch_id = 9535 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9536 } 9537 9538 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9539 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9540 break; 9541 9542 case I40E_VSI_SRIOV: 9543 ctxt.pf_num = hw->pf_id; 9544 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 9545 ctxt.uplink_seid = vsi->uplink_seid; 9546 ctxt.connection_type = I40E_AQ_VSI_CONN_TYPE_NORMAL; 9547 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 9548 9549 /* This VSI is connected to VEB so the switch_id 9550 * should be set to zero by default. 9551 */ 9552 if (i40e_is_vsi_uplink_mode_veb(vsi)) { 9553 ctxt.info.valid_sections |= 9554 cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 9555 ctxt.info.switch_id = 9556 cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 9557 } 9558 9559 if (vsi->back->flags & I40E_FLAG_IWARP_ENABLED) { 9560 ctxt.info.valid_sections |= 9561 cpu_to_le16(I40E_AQ_VSI_PROP_QUEUE_OPT_VALID); 9562 ctxt.info.queueing_opt_flags |= 9563 (I40E_AQ_VSI_QUE_OPT_TCP_ENA | 9564 I40E_AQ_VSI_QUE_OPT_RSS_LUT_VSI); 9565 } 9566 9567 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 9568 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 9569 if (pf->vf[vsi->vf_id].spoofchk) { 9570 ctxt.info.valid_sections |= 9571 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 9572 ctxt.info.sec_flags |= 9573 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 9574 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 9575 } 9576 /* Setup the VSI tx/rx queue map for TC0 only for now */ 9577 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 9578 break; 9579 9580 case I40E_VSI_IWARP: 9581 /* send down message to iWARP */ 9582 break; 9583 9584 default: 9585 return -ENODEV; 9586 } 9587 9588 if (vsi->type != I40E_VSI_MAIN) { 9589 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 9590 if (ret) { 9591 dev_info(&vsi->back->pdev->dev, 9592 "add vsi failed, err %s aq_err %s\n", 9593 i40e_stat_str(&pf->hw, ret), 9594 i40e_aq_str(&pf->hw, 9595 pf->hw.aq.asq_last_status)); 9596 ret = -ENOENT; 9597 goto err; 9598 } 9599 vsi->info = ctxt.info; 9600 vsi->info.valid_sections = 0; 9601 vsi->seid = ctxt.seid; 9602 vsi->id = ctxt.vsi_number; 9603 } 9604 9605 vsi->active_filters = 0; 9606 clear_bit(__I40E_FILTER_OVERFLOW_PROMISC, &vsi->state); 9607 spin_lock_bh(&vsi->mac_filter_hash_lock); 9608 /* If macvlan filters already exist, force them to get loaded */ 9609 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { 9610 f->state = I40E_FILTER_NEW; 9611 f_count++; 9612 } 9613 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9614 9615 if (f_count) { 9616 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 9617 pf->flags |= I40E_FLAG_FILTER_SYNC; 9618 } 9619 9620 /* Update VSI BW information */ 9621 ret = i40e_vsi_get_bw_info(vsi); 9622 if (ret) { 9623 dev_info(&pf->pdev->dev, 9624 "couldn't get vsi bw info, err %s aq_err %s\n", 9625 i40e_stat_str(&pf->hw, ret), 9626 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 9627 /* VSI is already added so not tearing that up */ 9628 ret = 0; 9629 } 9630 9631 err: 9632 return ret; 9633 } 9634 9635 /** 9636 * i40e_vsi_release - Delete a VSI and free its resources 9637 * @vsi: the VSI being removed 9638 * 9639 * Returns 0 on success or < 0 on error 9640 **/ 9641 int i40e_vsi_release(struct i40e_vsi *vsi) 9642 { 9643 struct i40e_mac_filter *f; 9644 struct hlist_node *h; 9645 struct i40e_veb *veb = NULL; 9646 struct i40e_pf *pf; 9647 u16 uplink_seid; 9648 int i, n, bkt; 9649 9650 pf = vsi->back; 9651 9652 /* release of a VEB-owner or last VSI is not allowed */ 9653 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 9654 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 9655 vsi->seid, vsi->uplink_seid); 9656 return -ENODEV; 9657 } 9658 if (vsi == pf->vsi[pf->lan_vsi] && 9659 !test_bit(__I40E_DOWN, &pf->state)) { 9660 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 9661 return -ENODEV; 9662 } 9663 9664 uplink_seid = vsi->uplink_seid; 9665 if (vsi->type != I40E_VSI_SRIOV) { 9666 if (vsi->netdev_registered) { 9667 vsi->netdev_registered = false; 9668 if (vsi->netdev) { 9669 /* results in a call to i40e_close() */ 9670 unregister_netdev(vsi->netdev); 9671 } 9672 } else { 9673 i40e_vsi_close(vsi); 9674 } 9675 i40e_vsi_disable_irq(vsi); 9676 } 9677 9678 spin_lock_bh(&vsi->mac_filter_hash_lock); 9679 9680 /* clear the sync flag on all filters */ 9681 if (vsi->netdev) { 9682 __dev_uc_unsync(vsi->netdev, NULL); 9683 __dev_mc_unsync(vsi->netdev, NULL); 9684 } 9685 9686 /* make sure any remaining filters are marked for deletion */ 9687 hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) 9688 __i40e_del_filter(vsi, f); 9689 9690 spin_unlock_bh(&vsi->mac_filter_hash_lock); 9691 9692 i40e_sync_vsi_filters(vsi); 9693 9694 i40e_vsi_delete(vsi); 9695 i40e_vsi_free_q_vectors(vsi); 9696 if (vsi->netdev) { 9697 free_netdev(vsi->netdev); 9698 vsi->netdev = NULL; 9699 } 9700 i40e_vsi_clear_rings(vsi); 9701 i40e_vsi_clear(vsi); 9702 9703 /* If this was the last thing on the VEB, except for the 9704 * controlling VSI, remove the VEB, which puts the controlling 9705 * VSI onto the next level down in the switch. 9706 * 9707 * Well, okay, there's one more exception here: don't remove 9708 * the orphan VEBs yet. We'll wait for an explicit remove request 9709 * from up the network stack. 9710 */ 9711 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 9712 if (pf->vsi[i] && 9713 pf->vsi[i]->uplink_seid == uplink_seid && 9714 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 9715 n++; /* count the VSIs */ 9716 } 9717 } 9718 for (i = 0; i < I40E_MAX_VEB; i++) { 9719 if (!pf->veb[i]) 9720 continue; 9721 if (pf->veb[i]->uplink_seid == uplink_seid) 9722 n++; /* count the VEBs */ 9723 if (pf->veb[i]->seid == uplink_seid) 9724 veb = pf->veb[i]; 9725 } 9726 if (n == 0 && veb && veb->uplink_seid != 0) 9727 i40e_veb_release(veb); 9728 9729 return 0; 9730 } 9731 9732 /** 9733 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 9734 * @vsi: ptr to the VSI 9735 * 9736 * This should only be called after i40e_vsi_mem_alloc() which allocates the 9737 * corresponding SW VSI structure and initializes num_queue_pairs for the 9738 * newly allocated VSI. 9739 * 9740 * Returns 0 on success or negative on failure 9741 **/ 9742 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 9743 { 9744 int ret = -ENOENT; 9745 struct i40e_pf *pf = vsi->back; 9746 9747 if (vsi->q_vectors[0]) { 9748 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 9749 vsi->seid); 9750 return -EEXIST; 9751 } 9752 9753 if (vsi->base_vector) { 9754 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 9755 vsi->seid, vsi->base_vector); 9756 return -EEXIST; 9757 } 9758 9759 ret = i40e_vsi_alloc_q_vectors(vsi); 9760 if (ret) { 9761 dev_info(&pf->pdev->dev, 9762 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 9763 vsi->num_q_vectors, vsi->seid, ret); 9764 vsi->num_q_vectors = 0; 9765 goto vector_setup_out; 9766 } 9767 9768 /* In Legacy mode, we do not have to get any other vector since we 9769 * piggyback on the misc/ICR0 for queue interrupts. 9770 */ 9771 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 9772 return ret; 9773 if (vsi->num_q_vectors) 9774 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 9775 vsi->num_q_vectors, vsi->idx); 9776 if (vsi->base_vector < 0) { 9777 dev_info(&pf->pdev->dev, 9778 "failed to get tracking for %d vectors for VSI %d, err=%d\n", 9779 vsi->num_q_vectors, vsi->seid, vsi->base_vector); 9780 i40e_vsi_free_q_vectors(vsi); 9781 ret = -ENOENT; 9782 goto vector_setup_out; 9783 } 9784 9785 vector_setup_out: 9786 return ret; 9787 } 9788 9789 /** 9790 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 9791 * @vsi: pointer to the vsi. 9792 * 9793 * This re-allocates a vsi's queue resources. 9794 * 9795 * Returns pointer to the successfully allocated and configured VSI sw struct 9796 * on success, otherwise returns NULL on failure. 9797 **/ 9798 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 9799 { 9800 struct i40e_pf *pf; 9801 u8 enabled_tc; 9802 int ret; 9803 9804 if (!vsi) 9805 return NULL; 9806 9807 pf = vsi->back; 9808 9809 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 9810 i40e_vsi_clear_rings(vsi); 9811 9812 i40e_vsi_free_arrays(vsi, false); 9813 i40e_set_num_rings_in_vsi(vsi); 9814 ret = i40e_vsi_alloc_arrays(vsi, false); 9815 if (ret) 9816 goto err_vsi; 9817 9818 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx); 9819 if (ret < 0) { 9820 dev_info(&pf->pdev->dev, 9821 "failed to get tracking for %d queues for VSI %d err %d\n", 9822 vsi->alloc_queue_pairs, vsi->seid, ret); 9823 goto err_vsi; 9824 } 9825 vsi->base_queue = ret; 9826 9827 /* Update the FW view of the VSI. Force a reset of TC and queue 9828 * layout configurations. 9829 */ 9830 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 9831 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 9832 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 9833 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 9834 if (vsi->type == I40E_VSI_MAIN) 9835 i40e_rm_default_mac_filter(vsi, pf->hw.mac.perm_addr); 9836 9837 /* assign it some queues */ 9838 ret = i40e_alloc_rings(vsi); 9839 if (ret) 9840 goto err_rings; 9841 9842 /* map all of the rings to the q_vectors */ 9843 i40e_vsi_map_rings_to_vectors(vsi); 9844 return vsi; 9845 9846 err_rings: 9847 i40e_vsi_free_q_vectors(vsi); 9848 if (vsi->netdev_registered) { 9849 vsi->netdev_registered = false; 9850 unregister_netdev(vsi->netdev); 9851 free_netdev(vsi->netdev); 9852 vsi->netdev = NULL; 9853 } 9854 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 9855 err_vsi: 9856 i40e_vsi_clear(vsi); 9857 return NULL; 9858 } 9859 9860 /** 9861 * i40e_vsi_setup - Set up a VSI by a given type 9862 * @pf: board private structure 9863 * @type: VSI type 9864 * @uplink_seid: the switch element to link to 9865 * @param1: usage depends upon VSI type. For VF types, indicates VF id 9866 * 9867 * This allocates the sw VSI structure and its queue resources, then add a VSI 9868 * to the identified VEB. 9869 * 9870 * Returns pointer to the successfully allocated and configure VSI sw struct on 9871 * success, otherwise returns NULL on failure. 9872 **/ 9873 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 9874 u16 uplink_seid, u32 param1) 9875 { 9876 struct i40e_vsi *vsi = NULL; 9877 struct i40e_veb *veb = NULL; 9878 int ret, i; 9879 int v_idx; 9880 9881 /* The requested uplink_seid must be either 9882 * - the PF's port seid 9883 * no VEB is needed because this is the PF 9884 * or this is a Flow Director special case VSI 9885 * - seid of an existing VEB 9886 * - seid of a VSI that owns an existing VEB 9887 * - seid of a VSI that doesn't own a VEB 9888 * a new VEB is created and the VSI becomes the owner 9889 * - seid of the PF VSI, which is what creates the first VEB 9890 * this is a special case of the previous 9891 * 9892 * Find which uplink_seid we were given and create a new VEB if needed 9893 */ 9894 for (i = 0; i < I40E_MAX_VEB; i++) { 9895 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 9896 veb = pf->veb[i]; 9897 break; 9898 } 9899 } 9900 9901 if (!veb && uplink_seid != pf->mac_seid) { 9902 9903 for (i = 0; i < pf->num_alloc_vsi; i++) { 9904 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 9905 vsi = pf->vsi[i]; 9906 break; 9907 } 9908 } 9909 if (!vsi) { 9910 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 9911 uplink_seid); 9912 return NULL; 9913 } 9914 9915 if (vsi->uplink_seid == pf->mac_seid) 9916 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 9917 vsi->tc_config.enabled_tc); 9918 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 9919 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 9920 vsi->tc_config.enabled_tc); 9921 if (veb) { 9922 if (vsi->seid != pf->vsi[pf->lan_vsi]->seid) { 9923 dev_info(&vsi->back->pdev->dev, 9924 "New VSI creation error, uplink seid of LAN VSI expected.\n"); 9925 return NULL; 9926 } 9927 /* We come up by default in VEPA mode if SRIOV is not 9928 * already enabled, in which case we can't force VEPA 9929 * mode. 9930 */ 9931 if (!(pf->flags & I40E_FLAG_VEB_MODE_ENABLED)) { 9932 veb->bridge_mode = BRIDGE_MODE_VEPA; 9933 pf->flags &= ~I40E_FLAG_VEB_MODE_ENABLED; 9934 } 9935 i40e_config_bridge_mode(veb); 9936 } 9937 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 9938 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 9939 veb = pf->veb[i]; 9940 } 9941 if (!veb) { 9942 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 9943 return NULL; 9944 } 9945 9946 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 9947 uplink_seid = veb->seid; 9948 } 9949 9950 /* get vsi sw struct */ 9951 v_idx = i40e_vsi_mem_alloc(pf, type); 9952 if (v_idx < 0) 9953 goto err_alloc; 9954 vsi = pf->vsi[v_idx]; 9955 if (!vsi) 9956 goto err_alloc; 9957 vsi->type = type; 9958 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 9959 9960 if (type == I40E_VSI_MAIN) 9961 pf->lan_vsi = v_idx; 9962 else if (type == I40E_VSI_SRIOV) 9963 vsi->vf_id = param1; 9964 /* assign it some queues */ 9965 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, 9966 vsi->idx); 9967 if (ret < 0) { 9968 dev_info(&pf->pdev->dev, 9969 "failed to get tracking for %d queues for VSI %d err=%d\n", 9970 vsi->alloc_queue_pairs, vsi->seid, ret); 9971 goto err_vsi; 9972 } 9973 vsi->base_queue = ret; 9974 9975 /* get a VSI from the hardware */ 9976 vsi->uplink_seid = uplink_seid; 9977 ret = i40e_add_vsi(vsi); 9978 if (ret) 9979 goto err_vsi; 9980 9981 switch (vsi->type) { 9982 /* setup the netdev if needed */ 9983 case I40E_VSI_MAIN: 9984 /* Apply relevant filters if a platform-specific mac 9985 * address was selected. 9986 */ 9987 if (!!(pf->flags & I40E_FLAG_PF_MAC)) { 9988 ret = i40e_macaddr_init(vsi, pf->hw.mac.addr); 9989 if (ret) { 9990 dev_warn(&pf->pdev->dev, 9991 "could not set up macaddr; err %d\n", 9992 ret); 9993 } 9994 } 9995 case I40E_VSI_VMDQ2: 9996 ret = i40e_config_netdev(vsi); 9997 if (ret) 9998 goto err_netdev; 9999 ret = register_netdev(vsi->netdev); 10000 if (ret) 10001 goto err_netdev; 10002 vsi->netdev_registered = true; 10003 netif_carrier_off(vsi->netdev); 10004 #ifdef CONFIG_I40E_DCB 10005 /* Setup DCB netlink interface */ 10006 i40e_dcbnl_setup(vsi); 10007 #endif /* CONFIG_I40E_DCB */ 10008 /* fall through */ 10009 10010 case I40E_VSI_FDIR: 10011 /* set up vectors and rings if needed */ 10012 ret = i40e_vsi_setup_vectors(vsi); 10013 if (ret) 10014 goto err_msix; 10015 10016 ret = i40e_alloc_rings(vsi); 10017 if (ret) 10018 goto err_rings; 10019 10020 /* map all of the rings to the q_vectors */ 10021 i40e_vsi_map_rings_to_vectors(vsi); 10022 10023 i40e_vsi_reset_stats(vsi); 10024 break; 10025 10026 default: 10027 /* no netdev or rings for the other VSI types */ 10028 break; 10029 } 10030 10031 if ((pf->flags & I40E_FLAG_RSS_AQ_CAPABLE) && 10032 (vsi->type == I40E_VSI_VMDQ2)) { 10033 ret = i40e_vsi_config_rss(vsi); 10034 } 10035 return vsi; 10036 10037 err_rings: 10038 i40e_vsi_free_q_vectors(vsi); 10039 err_msix: 10040 if (vsi->netdev_registered) { 10041 vsi->netdev_registered = false; 10042 unregister_netdev(vsi->netdev); 10043 free_netdev(vsi->netdev); 10044 vsi->netdev = NULL; 10045 } 10046 err_netdev: 10047 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 10048 err_vsi: 10049 i40e_vsi_clear(vsi); 10050 err_alloc: 10051 return NULL; 10052 } 10053 10054 /** 10055 * i40e_veb_get_bw_info - Query VEB BW information 10056 * @veb: the veb to query 10057 * 10058 * Query the Tx scheduler BW configuration data for given VEB 10059 **/ 10060 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 10061 { 10062 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 10063 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 10064 struct i40e_pf *pf = veb->pf; 10065 struct i40e_hw *hw = &pf->hw; 10066 u32 tc_bw_max; 10067 int ret = 0; 10068 int i; 10069 10070 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 10071 &bw_data, NULL); 10072 if (ret) { 10073 dev_info(&pf->pdev->dev, 10074 "query veb bw config failed, err %s aq_err %s\n", 10075 i40e_stat_str(&pf->hw, ret), 10076 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 10077 goto out; 10078 } 10079 10080 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 10081 &ets_data, NULL); 10082 if (ret) { 10083 dev_info(&pf->pdev->dev, 10084 "query veb bw ets config failed, err %s aq_err %s\n", 10085 i40e_stat_str(&pf->hw, ret), 10086 i40e_aq_str(&pf->hw, hw->aq.asq_last_status)); 10087 goto out; 10088 } 10089 10090 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 10091 veb->bw_max_quanta = ets_data.tc_bw_max; 10092 veb->is_abs_credits = bw_data.absolute_credits_enable; 10093 veb->enabled_tc = ets_data.tc_valid_bits; 10094 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 10095 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 10096 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 10097 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 10098 veb->bw_tc_limit_credits[i] = 10099 le16_to_cpu(bw_data.tc_bw_limits[i]); 10100 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 10101 } 10102 10103 out: 10104 return ret; 10105 } 10106 10107 /** 10108 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 10109 * @pf: board private structure 10110 * 10111 * On error: returns error code (negative) 10112 * On success: returns vsi index in PF (positive) 10113 **/ 10114 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 10115 { 10116 int ret = -ENOENT; 10117 struct i40e_veb *veb; 10118 int i; 10119 10120 /* Need to protect the allocation of switch elements at the PF level */ 10121 mutex_lock(&pf->switch_mutex); 10122 10123 /* VEB list may be fragmented if VEB creation/destruction has 10124 * been happening. We can afford to do a quick scan to look 10125 * for any free slots in the list. 10126 * 10127 * find next empty veb slot, looping back around if necessary 10128 */ 10129 i = 0; 10130 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 10131 i++; 10132 if (i >= I40E_MAX_VEB) { 10133 ret = -ENOMEM; 10134 goto err_alloc_veb; /* out of VEB slots! */ 10135 } 10136 10137 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 10138 if (!veb) { 10139 ret = -ENOMEM; 10140 goto err_alloc_veb; 10141 } 10142 veb->pf = pf; 10143 veb->idx = i; 10144 veb->enabled_tc = 1; 10145 10146 pf->veb[i] = veb; 10147 ret = i; 10148 err_alloc_veb: 10149 mutex_unlock(&pf->switch_mutex); 10150 return ret; 10151 } 10152 10153 /** 10154 * i40e_switch_branch_release - Delete a branch of the switch tree 10155 * @branch: where to start deleting 10156 * 10157 * This uses recursion to find the tips of the branch to be 10158 * removed, deleting until we get back to and can delete this VEB. 10159 **/ 10160 static void i40e_switch_branch_release(struct i40e_veb *branch) 10161 { 10162 struct i40e_pf *pf = branch->pf; 10163 u16 branch_seid = branch->seid; 10164 u16 veb_idx = branch->idx; 10165 int i; 10166 10167 /* release any VEBs on this VEB - RECURSION */ 10168 for (i = 0; i < I40E_MAX_VEB; i++) { 10169 if (!pf->veb[i]) 10170 continue; 10171 if (pf->veb[i]->uplink_seid == branch->seid) 10172 i40e_switch_branch_release(pf->veb[i]); 10173 } 10174 10175 /* Release the VSIs on this VEB, but not the owner VSI. 10176 * 10177 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 10178 * the VEB itself, so don't use (*branch) after this loop. 10179 */ 10180 for (i = 0; i < pf->num_alloc_vsi; i++) { 10181 if (!pf->vsi[i]) 10182 continue; 10183 if (pf->vsi[i]->uplink_seid == branch_seid && 10184 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 10185 i40e_vsi_release(pf->vsi[i]); 10186 } 10187 } 10188 10189 /* There's one corner case where the VEB might not have been 10190 * removed, so double check it here and remove it if needed. 10191 * This case happens if the veb was created from the debugfs 10192 * commands and no VSIs were added to it. 10193 */ 10194 if (pf->veb[veb_idx]) 10195 i40e_veb_release(pf->veb[veb_idx]); 10196 } 10197 10198 /** 10199 * i40e_veb_clear - remove veb struct 10200 * @veb: the veb to remove 10201 **/ 10202 static void i40e_veb_clear(struct i40e_veb *veb) 10203 { 10204 if (!veb) 10205 return; 10206 10207 if (veb->pf) { 10208 struct i40e_pf *pf = veb->pf; 10209 10210 mutex_lock(&pf->switch_mutex); 10211 if (pf->veb[veb->idx] == veb) 10212 pf->veb[veb->idx] = NULL; 10213 mutex_unlock(&pf->switch_mutex); 10214 } 10215 10216 kfree(veb); 10217 } 10218 10219 /** 10220 * i40e_veb_release - Delete a VEB and free its resources 10221 * @veb: the VEB being removed 10222 **/ 10223 void i40e_veb_release(struct i40e_veb *veb) 10224 { 10225 struct i40e_vsi *vsi = NULL; 10226 struct i40e_pf *pf; 10227 int i, n = 0; 10228 10229 pf = veb->pf; 10230 10231 /* find the remaining VSI and check for extras */ 10232 for (i = 0; i < pf->num_alloc_vsi; i++) { 10233 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 10234 n++; 10235 vsi = pf->vsi[i]; 10236 } 10237 } 10238 if (n != 1) { 10239 dev_info(&pf->pdev->dev, 10240 "can't remove VEB %d with %d VSIs left\n", 10241 veb->seid, n); 10242 return; 10243 } 10244 10245 /* move the remaining VSI to uplink veb */ 10246 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 10247 if (veb->uplink_seid) { 10248 vsi->uplink_seid = veb->uplink_seid; 10249 if (veb->uplink_seid == pf->mac_seid) 10250 vsi->veb_idx = I40E_NO_VEB; 10251 else 10252 vsi->veb_idx = veb->veb_idx; 10253 } else { 10254 /* floating VEB */ 10255 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 10256 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 10257 } 10258 10259 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10260 i40e_veb_clear(veb); 10261 } 10262 10263 /** 10264 * i40e_add_veb - create the VEB in the switch 10265 * @veb: the VEB to be instantiated 10266 * @vsi: the controlling VSI 10267 **/ 10268 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 10269 { 10270 struct i40e_pf *pf = veb->pf; 10271 bool enable_stats = !!(pf->flags & I40E_FLAG_VEB_STATS_ENABLED); 10272 int ret; 10273 10274 ret = i40e_aq_add_veb(&pf->hw, veb->uplink_seid, vsi->seid, 10275 veb->enabled_tc, false, 10276 &veb->seid, enable_stats, NULL); 10277 10278 /* get a VEB from the hardware */ 10279 if (ret) { 10280 dev_info(&pf->pdev->dev, 10281 "couldn't add VEB, err %s aq_err %s\n", 10282 i40e_stat_str(&pf->hw, ret), 10283 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10284 return -EPERM; 10285 } 10286 10287 /* get statistics counter */ 10288 ret = i40e_aq_get_veb_parameters(&pf->hw, veb->seid, NULL, NULL, 10289 &veb->stats_idx, NULL, NULL, NULL); 10290 if (ret) { 10291 dev_info(&pf->pdev->dev, 10292 "couldn't get VEB statistics idx, err %s aq_err %s\n", 10293 i40e_stat_str(&pf->hw, ret), 10294 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10295 return -EPERM; 10296 } 10297 ret = i40e_veb_get_bw_info(veb); 10298 if (ret) { 10299 dev_info(&pf->pdev->dev, 10300 "couldn't get VEB bw info, err %s aq_err %s\n", 10301 i40e_stat_str(&pf->hw, ret), 10302 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10303 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 10304 return -ENOENT; 10305 } 10306 10307 vsi->uplink_seid = veb->seid; 10308 vsi->veb_idx = veb->idx; 10309 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 10310 10311 return 0; 10312 } 10313 10314 /** 10315 * i40e_veb_setup - Set up a VEB 10316 * @pf: board private structure 10317 * @flags: VEB setup flags 10318 * @uplink_seid: the switch element to link to 10319 * @vsi_seid: the initial VSI seid 10320 * @enabled_tc: Enabled TC bit-map 10321 * 10322 * This allocates the sw VEB structure and links it into the switch 10323 * It is possible and legal for this to be a duplicate of an already 10324 * existing VEB. It is also possible for both uplink and vsi seids 10325 * to be zero, in order to create a floating VEB. 10326 * 10327 * Returns pointer to the successfully allocated VEB sw struct on 10328 * success, otherwise returns NULL on failure. 10329 **/ 10330 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 10331 u16 uplink_seid, u16 vsi_seid, 10332 u8 enabled_tc) 10333 { 10334 struct i40e_veb *veb, *uplink_veb = NULL; 10335 int vsi_idx, veb_idx; 10336 int ret; 10337 10338 /* if one seid is 0, the other must be 0 to create a floating relay */ 10339 if ((uplink_seid == 0 || vsi_seid == 0) && 10340 (uplink_seid + vsi_seid != 0)) { 10341 dev_info(&pf->pdev->dev, 10342 "one, not both seid's are 0: uplink=%d vsi=%d\n", 10343 uplink_seid, vsi_seid); 10344 return NULL; 10345 } 10346 10347 /* make sure there is such a vsi and uplink */ 10348 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 10349 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 10350 break; 10351 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 10352 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 10353 vsi_seid); 10354 return NULL; 10355 } 10356 10357 if (uplink_seid && uplink_seid != pf->mac_seid) { 10358 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 10359 if (pf->veb[veb_idx] && 10360 pf->veb[veb_idx]->seid == uplink_seid) { 10361 uplink_veb = pf->veb[veb_idx]; 10362 break; 10363 } 10364 } 10365 if (!uplink_veb) { 10366 dev_info(&pf->pdev->dev, 10367 "uplink seid %d not found\n", uplink_seid); 10368 return NULL; 10369 } 10370 } 10371 10372 /* get veb sw struct */ 10373 veb_idx = i40e_veb_mem_alloc(pf); 10374 if (veb_idx < 0) 10375 goto err_alloc; 10376 veb = pf->veb[veb_idx]; 10377 veb->flags = flags; 10378 veb->uplink_seid = uplink_seid; 10379 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 10380 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 10381 10382 /* create the VEB in the switch */ 10383 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 10384 if (ret) 10385 goto err_veb; 10386 if (vsi_idx == pf->lan_vsi) 10387 pf->lan_veb = veb->idx; 10388 10389 return veb; 10390 10391 err_veb: 10392 i40e_veb_clear(veb); 10393 err_alloc: 10394 return NULL; 10395 } 10396 10397 /** 10398 * i40e_setup_pf_switch_element - set PF vars based on switch type 10399 * @pf: board private structure 10400 * @ele: element we are building info from 10401 * @num_reported: total number of elements 10402 * @printconfig: should we print the contents 10403 * 10404 * helper function to assist in extracting a few useful SEID values. 10405 **/ 10406 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 10407 struct i40e_aqc_switch_config_element_resp *ele, 10408 u16 num_reported, bool printconfig) 10409 { 10410 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 10411 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 10412 u8 element_type = ele->element_type; 10413 u16 seid = le16_to_cpu(ele->seid); 10414 10415 if (printconfig) 10416 dev_info(&pf->pdev->dev, 10417 "type=%d seid=%d uplink=%d downlink=%d\n", 10418 element_type, seid, uplink_seid, downlink_seid); 10419 10420 switch (element_type) { 10421 case I40E_SWITCH_ELEMENT_TYPE_MAC: 10422 pf->mac_seid = seid; 10423 break; 10424 case I40E_SWITCH_ELEMENT_TYPE_VEB: 10425 /* Main VEB? */ 10426 if (uplink_seid != pf->mac_seid) 10427 break; 10428 if (pf->lan_veb == I40E_NO_VEB) { 10429 int v; 10430 10431 /* find existing or else empty VEB */ 10432 for (v = 0; v < I40E_MAX_VEB; v++) { 10433 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 10434 pf->lan_veb = v; 10435 break; 10436 } 10437 } 10438 if (pf->lan_veb == I40E_NO_VEB) { 10439 v = i40e_veb_mem_alloc(pf); 10440 if (v < 0) 10441 break; 10442 pf->lan_veb = v; 10443 } 10444 } 10445 10446 pf->veb[pf->lan_veb]->seid = seid; 10447 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 10448 pf->veb[pf->lan_veb]->pf = pf; 10449 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 10450 break; 10451 case I40E_SWITCH_ELEMENT_TYPE_VSI: 10452 if (num_reported != 1) 10453 break; 10454 /* This is immediately after a reset so we can assume this is 10455 * the PF's VSI 10456 */ 10457 pf->mac_seid = uplink_seid; 10458 pf->pf_seid = downlink_seid; 10459 pf->main_vsi_seid = seid; 10460 if (printconfig) 10461 dev_info(&pf->pdev->dev, 10462 "pf_seid=%d main_vsi_seid=%d\n", 10463 pf->pf_seid, pf->main_vsi_seid); 10464 break; 10465 case I40E_SWITCH_ELEMENT_TYPE_PF: 10466 case I40E_SWITCH_ELEMENT_TYPE_VF: 10467 case I40E_SWITCH_ELEMENT_TYPE_EMP: 10468 case I40E_SWITCH_ELEMENT_TYPE_BMC: 10469 case I40E_SWITCH_ELEMENT_TYPE_PE: 10470 case I40E_SWITCH_ELEMENT_TYPE_PA: 10471 /* ignore these for now */ 10472 break; 10473 default: 10474 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 10475 element_type, seid); 10476 break; 10477 } 10478 } 10479 10480 /** 10481 * i40e_fetch_switch_configuration - Get switch config from firmware 10482 * @pf: board private structure 10483 * @printconfig: should we print the contents 10484 * 10485 * Get the current switch configuration from the device and 10486 * extract a few useful SEID values. 10487 **/ 10488 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 10489 { 10490 struct i40e_aqc_get_switch_config_resp *sw_config; 10491 u16 next_seid = 0; 10492 int ret = 0; 10493 u8 *aq_buf; 10494 int i; 10495 10496 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 10497 if (!aq_buf) 10498 return -ENOMEM; 10499 10500 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 10501 do { 10502 u16 num_reported, num_total; 10503 10504 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 10505 I40E_AQ_LARGE_BUF, 10506 &next_seid, NULL); 10507 if (ret) { 10508 dev_info(&pf->pdev->dev, 10509 "get switch config failed err %s aq_err %s\n", 10510 i40e_stat_str(&pf->hw, ret), 10511 i40e_aq_str(&pf->hw, 10512 pf->hw.aq.asq_last_status)); 10513 kfree(aq_buf); 10514 return -ENOENT; 10515 } 10516 10517 num_reported = le16_to_cpu(sw_config->header.num_reported); 10518 num_total = le16_to_cpu(sw_config->header.num_total); 10519 10520 if (printconfig) 10521 dev_info(&pf->pdev->dev, 10522 "header: %d reported %d total\n", 10523 num_reported, num_total); 10524 10525 for (i = 0; i < num_reported; i++) { 10526 struct i40e_aqc_switch_config_element_resp *ele = 10527 &sw_config->element[i]; 10528 10529 i40e_setup_pf_switch_element(pf, ele, num_reported, 10530 printconfig); 10531 } 10532 } while (next_seid != 0); 10533 10534 kfree(aq_buf); 10535 return ret; 10536 } 10537 10538 /** 10539 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 10540 * @pf: board private structure 10541 * @reinit: if the Main VSI needs to re-initialized. 10542 * 10543 * Returns 0 on success, negative value on failure 10544 **/ 10545 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 10546 { 10547 u16 flags = 0; 10548 int ret; 10549 10550 /* find out what's out there already */ 10551 ret = i40e_fetch_switch_configuration(pf, false); 10552 if (ret) { 10553 dev_info(&pf->pdev->dev, 10554 "couldn't fetch switch config, err %s aq_err %s\n", 10555 i40e_stat_str(&pf->hw, ret), 10556 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 10557 return ret; 10558 } 10559 i40e_pf_reset_stats(pf); 10560 10561 /* set the switch config bit for the whole device to 10562 * support limited promisc or true promisc 10563 * when user requests promisc. The default is limited 10564 * promisc. 10565 */ 10566 10567 if ((pf->hw.pf_id == 0) && 10568 !(pf->flags & I40E_FLAG_TRUE_PROMISC_SUPPORT)) 10569 flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 10570 10571 if (pf->hw.pf_id == 0) { 10572 u16 valid_flags; 10573 10574 valid_flags = I40E_AQ_SET_SWITCH_CFG_PROMISC; 10575 ret = i40e_aq_set_switch_config(&pf->hw, flags, valid_flags, 10576 NULL); 10577 if (ret && pf->hw.aq.asq_last_status != I40E_AQ_RC_ESRCH) { 10578 dev_info(&pf->pdev->dev, 10579 "couldn't set switch config bits, err %s aq_err %s\n", 10580 i40e_stat_str(&pf->hw, ret), 10581 i40e_aq_str(&pf->hw, 10582 pf->hw.aq.asq_last_status)); 10583 /* not a fatal problem, just keep going */ 10584 } 10585 } 10586 10587 /* first time setup */ 10588 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 10589 struct i40e_vsi *vsi = NULL; 10590 u16 uplink_seid; 10591 10592 /* Set up the PF VSI associated with the PF's main VSI 10593 * that is already in the HW switch 10594 */ 10595 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 10596 uplink_seid = pf->veb[pf->lan_veb]->seid; 10597 else 10598 uplink_seid = pf->mac_seid; 10599 if (pf->lan_vsi == I40E_NO_VSI) 10600 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 10601 else if (reinit) 10602 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 10603 if (!vsi) { 10604 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 10605 i40e_fdir_teardown(pf); 10606 return -EAGAIN; 10607 } 10608 } else { 10609 /* force a reset of TC and queue layout configurations */ 10610 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 10611 10612 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 10613 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 10614 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 10615 } 10616 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 10617 10618 i40e_fdir_sb_setup(pf); 10619 10620 /* Setup static PF queue filter control settings */ 10621 ret = i40e_setup_pf_filter_control(pf); 10622 if (ret) { 10623 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 10624 ret); 10625 /* Failure here should not stop continuing other steps */ 10626 } 10627 10628 /* enable RSS in the HW, even for only one queue, as the stack can use 10629 * the hash 10630 */ 10631 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 10632 i40e_pf_config_rss(pf); 10633 10634 /* fill in link information and enable LSE reporting */ 10635 i40e_link_event(pf); 10636 10637 /* Initialize user-specific link properties */ 10638 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 10639 I40E_AQ_AN_COMPLETED) ? true : false); 10640 10641 i40e_ptp_init(pf); 10642 10643 return ret; 10644 } 10645 10646 /** 10647 * i40e_determine_queue_usage - Work out queue distribution 10648 * @pf: board private structure 10649 **/ 10650 static void i40e_determine_queue_usage(struct i40e_pf *pf) 10651 { 10652 int queues_left; 10653 10654 pf->num_lan_qps = 0; 10655 10656 /* Find the max queues to be put into basic use. We'll always be 10657 * using TC0, whether or not DCB is running, and TC0 will get the 10658 * big RSS set. 10659 */ 10660 queues_left = pf->hw.func_caps.num_tx_qp; 10661 10662 if ((queues_left == 1) || 10663 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 10664 /* one qp for PF, no queues for anything else */ 10665 queues_left = 0; 10666 pf->alloc_rss_size = pf->num_lan_qps = 1; 10667 10668 /* make sure all the fancies are disabled */ 10669 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 10670 I40E_FLAG_IWARP_ENABLED | 10671 I40E_FLAG_FD_SB_ENABLED | 10672 I40E_FLAG_FD_ATR_ENABLED | 10673 I40E_FLAG_DCB_CAPABLE | 10674 I40E_FLAG_DCB_ENABLED | 10675 I40E_FLAG_SRIOV_ENABLED | 10676 I40E_FLAG_VMDQ_ENABLED); 10677 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 10678 I40E_FLAG_FD_SB_ENABLED | 10679 I40E_FLAG_FD_ATR_ENABLED | 10680 I40E_FLAG_DCB_CAPABLE))) { 10681 /* one qp for PF */ 10682 pf->alloc_rss_size = pf->num_lan_qps = 1; 10683 queues_left -= pf->num_lan_qps; 10684 10685 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 10686 I40E_FLAG_IWARP_ENABLED | 10687 I40E_FLAG_FD_SB_ENABLED | 10688 I40E_FLAG_FD_ATR_ENABLED | 10689 I40E_FLAG_DCB_ENABLED | 10690 I40E_FLAG_VMDQ_ENABLED); 10691 } else { 10692 /* Not enough queues for all TCs */ 10693 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 10694 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 10695 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | 10696 I40E_FLAG_DCB_ENABLED); 10697 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 10698 } 10699 pf->num_lan_qps = max_t(int, pf->rss_size_max, 10700 num_online_cpus()); 10701 pf->num_lan_qps = min_t(int, pf->num_lan_qps, 10702 pf->hw.func_caps.num_tx_qp); 10703 10704 queues_left -= pf->num_lan_qps; 10705 } 10706 10707 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10708 if (queues_left > 1) { 10709 queues_left -= 1; /* save 1 queue for FD */ 10710 } else { 10711 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 10712 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 10713 } 10714 } 10715 10716 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 10717 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 10718 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 10719 (queues_left / pf->num_vf_qps)); 10720 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 10721 } 10722 10723 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 10724 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 10725 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 10726 (queues_left / pf->num_vmdq_qps)); 10727 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 10728 } 10729 10730 pf->queues_left = queues_left; 10731 dev_dbg(&pf->pdev->dev, 10732 "qs_avail=%d FD SB=%d lan_qs=%d lan_tc0=%d vf=%d*%d vmdq=%d*%d, remaining=%d\n", 10733 pf->hw.func_caps.num_tx_qp, 10734 !!(pf->flags & I40E_FLAG_FD_SB_ENABLED), 10735 pf->num_lan_qps, pf->alloc_rss_size, pf->num_req_vfs, 10736 pf->num_vf_qps, pf->num_vmdq_vsis, pf->num_vmdq_qps, 10737 queues_left); 10738 } 10739 10740 /** 10741 * i40e_setup_pf_filter_control - Setup PF static filter control 10742 * @pf: PF to be setup 10743 * 10744 * i40e_setup_pf_filter_control sets up a PF's initial filter control 10745 * settings. If PE/FCoE are enabled then it will also set the per PF 10746 * based filter sizes required for them. It also enables Flow director, 10747 * ethertype and macvlan type filter settings for the pf. 10748 * 10749 * Returns 0 on success, negative on failure 10750 **/ 10751 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 10752 { 10753 struct i40e_filter_control_settings *settings = &pf->filter_settings; 10754 10755 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 10756 10757 /* Flow Director is enabled */ 10758 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 10759 settings->enable_fdir = true; 10760 10761 /* Ethtype and MACVLAN filters enabled for PF */ 10762 settings->enable_ethtype = true; 10763 settings->enable_macvlan = true; 10764 10765 if (i40e_set_filter_control(&pf->hw, settings)) 10766 return -ENOENT; 10767 10768 return 0; 10769 } 10770 10771 #define INFO_STRING_LEN 255 10772 #define REMAIN(__x) (INFO_STRING_LEN - (__x)) 10773 static void i40e_print_features(struct i40e_pf *pf) 10774 { 10775 struct i40e_hw *hw = &pf->hw; 10776 char *buf; 10777 int i; 10778 10779 buf = kmalloc(INFO_STRING_LEN, GFP_KERNEL); 10780 if (!buf) 10781 return; 10782 10783 i = snprintf(buf, INFO_STRING_LEN, "Features: PF-id[%d]", hw->pf_id); 10784 #ifdef CONFIG_PCI_IOV 10785 i += snprintf(&buf[i], REMAIN(i), " VFs: %d", pf->num_req_vfs); 10786 #endif 10787 i += snprintf(&buf[i], REMAIN(i), " VSIs: %d QP: %d", 10788 pf->hw.func_caps.num_vsis, 10789 pf->vsi[pf->lan_vsi]->num_queue_pairs); 10790 if (pf->flags & I40E_FLAG_RSS_ENABLED) 10791 i += snprintf(&buf[i], REMAIN(i), " RSS"); 10792 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 10793 i += snprintf(&buf[i], REMAIN(i), " FD_ATR"); 10794 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 10795 i += snprintf(&buf[i], REMAIN(i), " FD_SB"); 10796 i += snprintf(&buf[i], REMAIN(i), " NTUPLE"); 10797 } 10798 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 10799 i += snprintf(&buf[i], REMAIN(i), " DCB"); 10800 i += snprintf(&buf[i], REMAIN(i), " VxLAN"); 10801 i += snprintf(&buf[i], REMAIN(i), " Geneve"); 10802 if (pf->flags & I40E_FLAG_PTP) 10803 i += snprintf(&buf[i], REMAIN(i), " PTP"); 10804 if (pf->flags & I40E_FLAG_VEB_MODE_ENABLED) 10805 i += snprintf(&buf[i], REMAIN(i), " VEB"); 10806 else 10807 i += snprintf(&buf[i], REMAIN(i), " VEPA"); 10808 10809 dev_info(&pf->pdev->dev, "%s\n", buf); 10810 kfree(buf); 10811 WARN_ON(i > INFO_STRING_LEN); 10812 } 10813 10814 /** 10815 * i40e_get_platform_mac_addr - get platform-specific MAC address 10816 * 10817 * @pdev: PCI device information struct 10818 * @pf: board private structure 10819 * 10820 * Look up the MAC address in Open Firmware on systems that support it, 10821 * and use IDPROM on SPARC if no OF address is found. On return, the 10822 * I40E_FLAG_PF_MAC will be wset in pf->flags if a platform-specific value 10823 * has been selected. 10824 **/ 10825 static void i40e_get_platform_mac_addr(struct pci_dev *pdev, struct i40e_pf *pf) 10826 { 10827 pf->flags &= ~I40E_FLAG_PF_MAC; 10828 if (!eth_platform_get_mac_address(&pdev->dev, pf->hw.mac.addr)) 10829 pf->flags |= I40E_FLAG_PF_MAC; 10830 } 10831 10832 /** 10833 * i40e_probe - Device initialization routine 10834 * @pdev: PCI device information struct 10835 * @ent: entry in i40e_pci_tbl 10836 * 10837 * i40e_probe initializes a PF identified by a pci_dev structure. 10838 * The OS initialization, configuring of the PF private structure, 10839 * and a hardware reset occur. 10840 * 10841 * Returns 0 on success, negative on failure 10842 **/ 10843 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 10844 { 10845 struct i40e_aq_get_phy_abilities_resp abilities; 10846 struct i40e_pf *pf; 10847 struct i40e_hw *hw; 10848 static u16 pfs_found; 10849 u16 wol_nvm_bits; 10850 u16 link_status; 10851 int err; 10852 u32 val; 10853 u32 i; 10854 u8 set_fc_aq_fail; 10855 10856 err = pci_enable_device_mem(pdev); 10857 if (err) 10858 return err; 10859 10860 /* set up for high or low dma */ 10861 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 10862 if (err) { 10863 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 10864 if (err) { 10865 dev_err(&pdev->dev, 10866 "DMA configuration failed: 0x%x\n", err); 10867 goto err_dma; 10868 } 10869 } 10870 10871 /* set up pci connections */ 10872 err = pci_request_mem_regions(pdev, i40e_driver_name); 10873 if (err) { 10874 dev_info(&pdev->dev, 10875 "pci_request_selected_regions failed %d\n", err); 10876 goto err_pci_reg; 10877 } 10878 10879 pci_enable_pcie_error_reporting(pdev); 10880 pci_set_master(pdev); 10881 10882 /* Now that we have a PCI connection, we need to do the 10883 * low level device setup. This is primarily setting up 10884 * the Admin Queue structures and then querying for the 10885 * device's current profile information. 10886 */ 10887 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 10888 if (!pf) { 10889 err = -ENOMEM; 10890 goto err_pf_alloc; 10891 } 10892 pf->next_vsi = 0; 10893 pf->pdev = pdev; 10894 set_bit(__I40E_DOWN, &pf->state); 10895 10896 hw = &pf->hw; 10897 hw->back = pf; 10898 10899 pf->ioremap_len = min_t(int, pci_resource_len(pdev, 0), 10900 I40E_MAX_CSR_SPACE); 10901 10902 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), pf->ioremap_len); 10903 if (!hw->hw_addr) { 10904 err = -EIO; 10905 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 10906 (unsigned int)pci_resource_start(pdev, 0), 10907 pf->ioremap_len, err); 10908 goto err_ioremap; 10909 } 10910 hw->vendor_id = pdev->vendor; 10911 hw->device_id = pdev->device; 10912 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 10913 hw->subsystem_vendor_id = pdev->subsystem_vendor; 10914 hw->subsystem_device_id = pdev->subsystem_device; 10915 hw->bus.device = PCI_SLOT(pdev->devfn); 10916 hw->bus.func = PCI_FUNC(pdev->devfn); 10917 hw->bus.bus_id = pdev->bus->number; 10918 pf->instance = pfs_found; 10919 10920 INIT_LIST_HEAD(&pf->l3_flex_pit_list); 10921 INIT_LIST_HEAD(&pf->l4_flex_pit_list); 10922 10923 /* set up the locks for the AQ, do this only once in probe 10924 * and destroy them only once in remove 10925 */ 10926 mutex_init(&hw->aq.asq_mutex); 10927 mutex_init(&hw->aq.arq_mutex); 10928 10929 pf->msg_enable = netif_msg_init(debug, 10930 NETIF_MSG_DRV | 10931 NETIF_MSG_PROBE | 10932 NETIF_MSG_LINK); 10933 if (debug < -1) 10934 pf->hw.debug_mask = debug; 10935 10936 /* do a special CORER for clearing PXE mode once at init */ 10937 if (hw->revision_id == 0 && 10938 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 10939 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 10940 i40e_flush(hw); 10941 msleep(200); 10942 pf->corer_count++; 10943 10944 i40e_clear_pxe_mode(hw); 10945 } 10946 10947 /* Reset here to make sure all is clean and to define PF 'n' */ 10948 i40e_clear_hw(hw); 10949 err = i40e_pf_reset(hw); 10950 if (err) { 10951 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 10952 goto err_pf_reset; 10953 } 10954 pf->pfr_count++; 10955 10956 hw->aq.num_arq_entries = I40E_AQ_LEN; 10957 hw->aq.num_asq_entries = I40E_AQ_LEN; 10958 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 10959 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 10960 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 10961 10962 snprintf(pf->int_name, sizeof(pf->int_name) - 1, 10963 "%s-%s:misc", 10964 dev_driver_string(&pf->pdev->dev), dev_name(&pdev->dev)); 10965 10966 err = i40e_init_shared_code(hw); 10967 if (err) { 10968 dev_warn(&pdev->dev, "unidentified MAC or BLANK NVM: %d\n", 10969 err); 10970 goto err_pf_reset; 10971 } 10972 10973 /* set up a default setting for link flow control */ 10974 pf->hw.fc.requested_mode = I40E_FC_NONE; 10975 10976 err = i40e_init_adminq(hw); 10977 if (err) { 10978 if (err == I40E_ERR_FIRMWARE_API_VERSION) 10979 dev_info(&pdev->dev, 10980 "The driver for the device stopped because the NVM image is newer than expected. You must install the most recent version of the network driver.\n"); 10981 else 10982 dev_info(&pdev->dev, 10983 "The driver for the device stopped because the device firmware failed to init. Try updating your NVM image.\n"); 10984 10985 goto err_pf_reset; 10986 } 10987 10988 /* provide nvm, fw, api versions */ 10989 dev_info(&pdev->dev, "fw %d.%d.%05d api %d.%d nvm %s\n", 10990 hw->aq.fw_maj_ver, hw->aq.fw_min_ver, hw->aq.fw_build, 10991 hw->aq.api_maj_ver, hw->aq.api_min_ver, 10992 i40e_nvm_version_str(hw)); 10993 10994 if (hw->aq.api_maj_ver == I40E_FW_API_VERSION_MAJOR && 10995 hw->aq.api_min_ver > I40E_FW_API_VERSION_MINOR) 10996 dev_info(&pdev->dev, 10997 "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); 10998 else if (hw->aq.api_maj_ver < I40E_FW_API_VERSION_MAJOR || 10999 hw->aq.api_min_ver < (I40E_FW_API_VERSION_MINOR - 1)) 11000 dev_info(&pdev->dev, 11001 "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); 11002 11003 i40e_verify_eeprom(pf); 11004 11005 /* Rev 0 hardware was never productized */ 11006 if (hw->revision_id < 1) 11007 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 11008 11009 i40e_clear_pxe_mode(hw); 11010 err = i40e_get_capabilities(pf); 11011 if (err) 11012 goto err_adminq_setup; 11013 11014 err = i40e_sw_init(pf); 11015 if (err) { 11016 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 11017 goto err_sw_init; 11018 } 11019 11020 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 11021 hw->func_caps.num_rx_qp, 0, 0); 11022 if (err) { 11023 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 11024 goto err_init_lan_hmc; 11025 } 11026 11027 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 11028 if (err) { 11029 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 11030 err = -ENOENT; 11031 goto err_configure_lan_hmc; 11032 } 11033 11034 /* Disable LLDP for NICs that have firmware versions lower than v4.3. 11035 * Ignore error return codes because if it was already disabled via 11036 * hardware settings this will fail 11037 */ 11038 if (pf->flags & I40E_FLAG_STOP_FW_LLDP) { 11039 dev_info(&pdev->dev, "Stopping firmware LLDP agent.\n"); 11040 i40e_aq_stop_lldp(hw, true, NULL); 11041 } 11042 11043 i40e_get_mac_addr(hw, hw->mac.addr); 11044 /* allow a platform config to override the HW addr */ 11045 i40e_get_platform_mac_addr(pdev, pf); 11046 if (!is_valid_ether_addr(hw->mac.addr)) { 11047 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 11048 err = -EIO; 11049 goto err_mac_addr; 11050 } 11051 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 11052 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 11053 i40e_get_port_mac_addr(hw, hw->mac.port_addr); 11054 if (is_valid_ether_addr(hw->mac.port_addr)) 11055 pf->flags |= I40E_FLAG_PORT_ID_VALID; 11056 11057 pci_set_drvdata(pdev, pf); 11058 pci_save_state(pdev); 11059 #ifdef CONFIG_I40E_DCB 11060 err = i40e_init_pf_dcb(pf); 11061 if (err) { 11062 dev_info(&pdev->dev, "DCB init failed %d, disabled\n", err); 11063 pf->flags &= ~(I40E_FLAG_DCB_CAPABLE | I40E_FLAG_DCB_ENABLED); 11064 /* Continue without DCB enabled */ 11065 } 11066 #endif /* CONFIG_I40E_DCB */ 11067 11068 /* set up periodic task facility */ 11069 setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf); 11070 pf->service_timer_period = HZ; 11071 11072 INIT_WORK(&pf->service_task, i40e_service_task); 11073 clear_bit(__I40E_SERVICE_SCHED, &pf->state); 11074 pf->flags |= I40E_FLAG_NEED_LINK_UPDATE; 11075 11076 /* NVM bit on means WoL disabled for the port */ 11077 i40e_read_nvm_word(hw, I40E_SR_NVM_WAKE_ON_LAN, &wol_nvm_bits); 11078 if (BIT (hw->port) & wol_nvm_bits || hw->partition_id != 1) 11079 pf->wol_en = false; 11080 else 11081 pf->wol_en = true; 11082 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 11083 11084 /* set up the main switch operations */ 11085 i40e_determine_queue_usage(pf); 11086 err = i40e_init_interrupt_scheme(pf); 11087 if (err) 11088 goto err_switch_setup; 11089 11090 /* The number of VSIs reported by the FW is the minimum guaranteed 11091 * to us; HW supports far more and we share the remaining pool with 11092 * the other PFs. We allocate space for more than the guarantee with 11093 * the understanding that we might not get them all later. 11094 */ 11095 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 11096 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 11097 else 11098 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 11099 11100 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 11101 pf->vsi = kcalloc(pf->num_alloc_vsi, sizeof(struct i40e_vsi *), 11102 GFP_KERNEL); 11103 if (!pf->vsi) { 11104 err = -ENOMEM; 11105 goto err_switch_setup; 11106 } 11107 11108 #ifdef CONFIG_PCI_IOV 11109 /* prep for VF support */ 11110 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11111 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11112 !test_bit(__I40E_BAD_EEPROM, &pf->state)) { 11113 if (pci_num_vf(pdev)) 11114 pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; 11115 } 11116 #endif 11117 err = i40e_setup_pf_switch(pf, false); 11118 if (err) { 11119 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 11120 goto err_vsis; 11121 } 11122 11123 /* Make sure flow control is set according to current settings */ 11124 err = i40e_set_fc(hw, &set_fc_aq_fail, true); 11125 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_GET) 11126 dev_dbg(&pf->pdev->dev, 11127 "Set fc with err %s aq_err %s on get_phy_cap\n", 11128 i40e_stat_str(hw, err), 11129 i40e_aq_str(hw, hw->aq.asq_last_status)); 11130 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_SET) 11131 dev_dbg(&pf->pdev->dev, 11132 "Set fc with err %s aq_err %s on set_phy_config\n", 11133 i40e_stat_str(hw, err), 11134 i40e_aq_str(hw, hw->aq.asq_last_status)); 11135 if (set_fc_aq_fail & I40E_SET_FC_AQ_FAIL_UPDATE) 11136 dev_dbg(&pf->pdev->dev, 11137 "Set fc with err %s aq_err %s on get_link_info\n", 11138 i40e_stat_str(hw, err), 11139 i40e_aq_str(hw, hw->aq.asq_last_status)); 11140 11141 /* if FDIR VSI was set up, start it now */ 11142 for (i = 0; i < pf->num_alloc_vsi; i++) { 11143 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 11144 i40e_vsi_open(pf->vsi[i]); 11145 break; 11146 } 11147 } 11148 11149 /* The driver only wants link up/down and module qualification 11150 * reports from firmware. Note the negative logic. 11151 */ 11152 err = i40e_aq_set_phy_int_mask(&pf->hw, 11153 ~(I40E_AQ_EVENT_LINK_UPDOWN | 11154 I40E_AQ_EVENT_MEDIA_NA | 11155 I40E_AQ_EVENT_MODULE_QUAL_FAIL), NULL); 11156 if (err) 11157 dev_info(&pf->pdev->dev, "set phy mask fail, err %s aq_err %s\n", 11158 i40e_stat_str(&pf->hw, err), 11159 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11160 11161 /* Reconfigure hardware for allowing smaller MSS in the case 11162 * of TSO, so that we avoid the MDD being fired and causing 11163 * a reset in the case of small MSS+TSO. 11164 */ 11165 val = rd32(hw, I40E_REG_MSS); 11166 if ((val & I40E_REG_MSS_MIN_MASK) > I40E_64BYTE_MSS) { 11167 val &= ~I40E_REG_MSS_MIN_MASK; 11168 val |= I40E_64BYTE_MSS; 11169 wr32(hw, I40E_REG_MSS, val); 11170 } 11171 11172 if (pf->flags & I40E_FLAG_RESTART_AUTONEG) { 11173 msleep(75); 11174 err = i40e_aq_set_link_restart_an(&pf->hw, true, NULL); 11175 if (err) 11176 dev_info(&pf->pdev->dev, "link restart failed, err %s aq_err %s\n", 11177 i40e_stat_str(&pf->hw, err), 11178 i40e_aq_str(&pf->hw, 11179 pf->hw.aq.asq_last_status)); 11180 } 11181 /* The main driver is (mostly) up and happy. We need to set this state 11182 * before setting up the misc vector or we get a race and the vector 11183 * ends up disabled forever. 11184 */ 11185 clear_bit(__I40E_DOWN, &pf->state); 11186 11187 /* In case of MSIX we are going to setup the misc vector right here 11188 * to handle admin queue events etc. In case of legacy and MSI 11189 * the misc functionality and queue processing is combined in 11190 * the same vector and that gets setup at open. 11191 */ 11192 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 11193 err = i40e_setup_misc_vector(pf); 11194 if (err) { 11195 dev_info(&pdev->dev, 11196 "setup of misc vector failed: %d\n", err); 11197 goto err_vsis; 11198 } 11199 } 11200 11201 #ifdef CONFIG_PCI_IOV 11202 /* prep for VF support */ 11203 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 11204 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 11205 !test_bit(__I40E_BAD_EEPROM, &pf->state)) { 11206 /* disable link interrupts for VFs */ 11207 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 11208 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 11209 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 11210 i40e_flush(hw); 11211 11212 if (pci_num_vf(pdev)) { 11213 dev_info(&pdev->dev, 11214 "Active VFs found, allocating resources.\n"); 11215 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 11216 if (err) 11217 dev_info(&pdev->dev, 11218 "Error %d allocating resources for existing VFs\n", 11219 err); 11220 } 11221 } 11222 #endif /* CONFIG_PCI_IOV */ 11223 11224 if (pf->flags & I40E_FLAG_IWARP_ENABLED) { 11225 pf->iwarp_base_vector = i40e_get_lump(pf, pf->irq_pile, 11226 pf->num_iwarp_msix, 11227 I40E_IWARP_IRQ_PILE_ID); 11228 if (pf->iwarp_base_vector < 0) { 11229 dev_info(&pdev->dev, 11230 "failed to get tracking for %d vectors for IWARP err=%d\n", 11231 pf->num_iwarp_msix, pf->iwarp_base_vector); 11232 pf->flags &= ~I40E_FLAG_IWARP_ENABLED; 11233 } 11234 } 11235 11236 i40e_dbg_pf_init(pf); 11237 11238 /* tell the firmware that we're starting */ 11239 i40e_send_version(pf); 11240 11241 /* since everything's happy, start the service_task timer */ 11242 mod_timer(&pf->service_timer, 11243 round_jiffies(jiffies + pf->service_timer_period)); 11244 11245 /* add this PF to client device list and launch a client service task */ 11246 err = i40e_lan_add_device(pf); 11247 if (err) 11248 dev_info(&pdev->dev, "Failed to add PF to client API service list: %d\n", 11249 err); 11250 11251 #define PCI_SPEED_SIZE 8 11252 #define PCI_WIDTH_SIZE 8 11253 /* Devices on the IOSF bus do not have this information 11254 * and will report PCI Gen 1 x 1 by default so don't bother 11255 * checking them. 11256 */ 11257 if (!(pf->flags & I40E_FLAG_NO_PCI_LINK_CHECK)) { 11258 char speed[PCI_SPEED_SIZE] = "Unknown"; 11259 char width[PCI_WIDTH_SIZE] = "Unknown"; 11260 11261 /* Get the negotiated link width and speed from PCI config 11262 * space 11263 */ 11264 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, 11265 &link_status); 11266 11267 i40e_set_pci_config_data(hw, link_status); 11268 11269 switch (hw->bus.speed) { 11270 case i40e_bus_speed_8000: 11271 strncpy(speed, "8.0", PCI_SPEED_SIZE); break; 11272 case i40e_bus_speed_5000: 11273 strncpy(speed, "5.0", PCI_SPEED_SIZE); break; 11274 case i40e_bus_speed_2500: 11275 strncpy(speed, "2.5", PCI_SPEED_SIZE); break; 11276 default: 11277 break; 11278 } 11279 switch (hw->bus.width) { 11280 case i40e_bus_width_pcie_x8: 11281 strncpy(width, "8", PCI_WIDTH_SIZE); break; 11282 case i40e_bus_width_pcie_x4: 11283 strncpy(width, "4", PCI_WIDTH_SIZE); break; 11284 case i40e_bus_width_pcie_x2: 11285 strncpy(width, "2", PCI_WIDTH_SIZE); break; 11286 case i40e_bus_width_pcie_x1: 11287 strncpy(width, "1", PCI_WIDTH_SIZE); break; 11288 default: 11289 break; 11290 } 11291 11292 dev_info(&pdev->dev, "PCI-Express: Speed %sGT/s Width x%s\n", 11293 speed, width); 11294 11295 if (hw->bus.width < i40e_bus_width_pcie_x8 || 11296 hw->bus.speed < i40e_bus_speed_8000) { 11297 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 11298 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 11299 } 11300 } 11301 11302 /* get the requested speeds from the fw */ 11303 err = i40e_aq_get_phy_capabilities(hw, false, false, &abilities, NULL); 11304 if (err) 11305 dev_dbg(&pf->pdev->dev, "get requested speeds ret = %s last_status = %s\n", 11306 i40e_stat_str(&pf->hw, err), 11307 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11308 pf->hw.phy.link_info.requested_speeds = abilities.link_speed; 11309 11310 /* get the supported phy types from the fw */ 11311 err = i40e_aq_get_phy_capabilities(hw, false, true, &abilities, NULL); 11312 if (err) 11313 dev_dbg(&pf->pdev->dev, "get supported phy types ret = %s last_status = %s\n", 11314 i40e_stat_str(&pf->hw, err), 11315 i40e_aq_str(&pf->hw, pf->hw.aq.asq_last_status)); 11316 11317 /* Add a filter to drop all Flow control frames from any VSI from being 11318 * transmitted. By doing so we stop a malicious VF from sending out 11319 * PAUSE or PFC frames and potentially controlling traffic for other 11320 * PF/VF VSIs. 11321 * The FW can still send Flow control frames if enabled. 11322 */ 11323 i40e_add_filter_to_drop_tx_flow_control_frames(&pf->hw, 11324 pf->main_vsi_seid); 11325 11326 if ((pf->hw.device_id == I40E_DEV_ID_10G_BASE_T) || 11327 (pf->hw.device_id == I40E_DEV_ID_10G_BASE_T4)) 11328 pf->flags |= I40E_FLAG_PHY_CONTROLS_LEDS; 11329 if (pf->hw.device_id == I40E_DEV_ID_SFP_I_X722) 11330 pf->flags |= I40E_FLAG_HAVE_CRT_RETIMER; 11331 /* print a string summarizing features */ 11332 i40e_print_features(pf); 11333 11334 return 0; 11335 11336 /* Unwind what we've done if something failed in the setup */ 11337 err_vsis: 11338 set_bit(__I40E_DOWN, &pf->state); 11339 i40e_clear_interrupt_scheme(pf); 11340 kfree(pf->vsi); 11341 err_switch_setup: 11342 i40e_reset_interrupt_capability(pf); 11343 del_timer_sync(&pf->service_timer); 11344 err_mac_addr: 11345 err_configure_lan_hmc: 11346 (void)i40e_shutdown_lan_hmc(hw); 11347 err_init_lan_hmc: 11348 kfree(pf->qp_pile); 11349 err_sw_init: 11350 err_adminq_setup: 11351 err_pf_reset: 11352 iounmap(hw->hw_addr); 11353 err_ioremap: 11354 kfree(pf); 11355 err_pf_alloc: 11356 pci_disable_pcie_error_reporting(pdev); 11357 pci_release_mem_regions(pdev); 11358 err_pci_reg: 11359 err_dma: 11360 pci_disable_device(pdev); 11361 return err; 11362 } 11363 11364 /** 11365 * i40e_remove - Device removal routine 11366 * @pdev: PCI device information struct 11367 * 11368 * i40e_remove is called by the PCI subsystem to alert the driver 11369 * that is should release a PCI device. This could be caused by a 11370 * Hot-Plug event, or because the driver is going to be removed from 11371 * memory. 11372 **/ 11373 static void i40e_remove(struct pci_dev *pdev) 11374 { 11375 struct i40e_pf *pf = pci_get_drvdata(pdev); 11376 struct i40e_hw *hw = &pf->hw; 11377 i40e_status ret_code; 11378 int i; 11379 11380 i40e_dbg_pf_exit(pf); 11381 11382 i40e_ptp_stop(pf); 11383 11384 /* Disable RSS in hw */ 11385 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(0), 0); 11386 i40e_write_rx_ctl(hw, I40E_PFQF_HENA(1), 0); 11387 11388 /* no more scheduling of any task */ 11389 set_bit(__I40E_SUSPENDED, &pf->state); 11390 set_bit(__I40E_DOWN, &pf->state); 11391 if (pf->service_timer.data) 11392 del_timer_sync(&pf->service_timer); 11393 if (pf->service_task.func) 11394 cancel_work_sync(&pf->service_task); 11395 11396 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 11397 i40e_free_vfs(pf); 11398 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 11399 } 11400 11401 i40e_fdir_teardown(pf); 11402 11403 /* If there is a switch structure or any orphans, remove them. 11404 * This will leave only the PF's VSI remaining. 11405 */ 11406 for (i = 0; i < I40E_MAX_VEB; i++) { 11407 if (!pf->veb[i]) 11408 continue; 11409 11410 if (pf->veb[i]->uplink_seid == pf->mac_seid || 11411 pf->veb[i]->uplink_seid == 0) 11412 i40e_switch_branch_release(pf->veb[i]); 11413 } 11414 11415 /* Now we can shutdown the PF's VSI, just before we kill 11416 * adminq and hmc. 11417 */ 11418 if (pf->vsi[pf->lan_vsi]) 11419 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 11420 11421 /* remove attached clients */ 11422 ret_code = i40e_lan_del_device(pf); 11423 if (ret_code) { 11424 dev_warn(&pdev->dev, "Failed to delete client device: %d\n", 11425 ret_code); 11426 } 11427 11428 /* shutdown and destroy the HMC */ 11429 if (hw->hmc.hmc_obj) { 11430 ret_code = i40e_shutdown_lan_hmc(hw); 11431 if (ret_code) 11432 dev_warn(&pdev->dev, 11433 "Failed to destroy the HMC resources: %d\n", 11434 ret_code); 11435 } 11436 11437 /* shutdown the adminq */ 11438 i40e_shutdown_adminq(hw); 11439 11440 /* destroy the locks only once, here */ 11441 mutex_destroy(&hw->aq.arq_mutex); 11442 mutex_destroy(&hw->aq.asq_mutex); 11443 11444 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 11445 i40e_clear_interrupt_scheme(pf); 11446 for (i = 0; i < pf->num_alloc_vsi; i++) { 11447 if (pf->vsi[i]) { 11448 i40e_vsi_clear_rings(pf->vsi[i]); 11449 i40e_vsi_clear(pf->vsi[i]); 11450 pf->vsi[i] = NULL; 11451 } 11452 } 11453 11454 for (i = 0; i < I40E_MAX_VEB; i++) { 11455 kfree(pf->veb[i]); 11456 pf->veb[i] = NULL; 11457 } 11458 11459 kfree(pf->qp_pile); 11460 kfree(pf->vsi); 11461 11462 iounmap(hw->hw_addr); 11463 kfree(pf); 11464 pci_release_mem_regions(pdev); 11465 11466 pci_disable_pcie_error_reporting(pdev); 11467 pci_disable_device(pdev); 11468 } 11469 11470 /** 11471 * i40e_pci_error_detected - warning that something funky happened in PCI land 11472 * @pdev: PCI device information struct 11473 * 11474 * Called to warn that something happened and the error handling steps 11475 * are in progress. Allows the driver to quiesce things, be ready for 11476 * remediation. 11477 **/ 11478 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 11479 enum pci_channel_state error) 11480 { 11481 struct i40e_pf *pf = pci_get_drvdata(pdev); 11482 11483 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 11484 11485 if (!pf) { 11486 dev_info(&pdev->dev, 11487 "Cannot recover - error happened during device probe\n"); 11488 return PCI_ERS_RESULT_DISCONNECT; 11489 } 11490 11491 /* shutdown all operations */ 11492 if (!test_bit(__I40E_SUSPENDED, &pf->state)) { 11493 rtnl_lock(); 11494 i40e_prep_for_reset(pf); 11495 rtnl_unlock(); 11496 } 11497 11498 /* Request a slot reset */ 11499 return PCI_ERS_RESULT_NEED_RESET; 11500 } 11501 11502 /** 11503 * i40e_pci_error_slot_reset - a PCI slot reset just happened 11504 * @pdev: PCI device information struct 11505 * 11506 * Called to find if the driver can work with the device now that 11507 * the pci slot has been reset. If a basic connection seems good 11508 * (registers are readable and have sane content) then return a 11509 * happy little PCI_ERS_RESULT_xxx. 11510 **/ 11511 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 11512 { 11513 struct i40e_pf *pf = pci_get_drvdata(pdev); 11514 pci_ers_result_t result; 11515 int err; 11516 u32 reg; 11517 11518 dev_dbg(&pdev->dev, "%s\n", __func__); 11519 if (pci_enable_device_mem(pdev)) { 11520 dev_info(&pdev->dev, 11521 "Cannot re-enable PCI device after reset.\n"); 11522 result = PCI_ERS_RESULT_DISCONNECT; 11523 } else { 11524 pci_set_master(pdev); 11525 pci_restore_state(pdev); 11526 pci_save_state(pdev); 11527 pci_wake_from_d3(pdev, false); 11528 11529 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 11530 if (reg == 0) 11531 result = PCI_ERS_RESULT_RECOVERED; 11532 else 11533 result = PCI_ERS_RESULT_DISCONNECT; 11534 } 11535 11536 err = pci_cleanup_aer_uncorrect_error_status(pdev); 11537 if (err) { 11538 dev_info(&pdev->dev, 11539 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 11540 err); 11541 /* non-fatal, continue */ 11542 } 11543 11544 return result; 11545 } 11546 11547 /** 11548 * i40e_pci_error_resume - restart operations after PCI error recovery 11549 * @pdev: PCI device information struct 11550 * 11551 * Called to allow the driver to bring things back up after PCI error 11552 * and/or reset recovery has finished. 11553 **/ 11554 static void i40e_pci_error_resume(struct pci_dev *pdev) 11555 { 11556 struct i40e_pf *pf = pci_get_drvdata(pdev); 11557 11558 dev_dbg(&pdev->dev, "%s\n", __func__); 11559 if (test_bit(__I40E_SUSPENDED, &pf->state)) 11560 return; 11561 11562 rtnl_lock(); 11563 i40e_handle_reset_warning(pf); 11564 rtnl_unlock(); 11565 } 11566 11567 /** 11568 * i40e_enable_mc_magic_wake - enable multicast magic packet wake up 11569 * using the mac_address_write admin q function 11570 * @pf: pointer to i40e_pf struct 11571 **/ 11572 static void i40e_enable_mc_magic_wake(struct i40e_pf *pf) 11573 { 11574 struct i40e_hw *hw = &pf->hw; 11575 i40e_status ret; 11576 u8 mac_addr[6]; 11577 u16 flags = 0; 11578 11579 /* Get current MAC address in case it's an LAA */ 11580 if (pf->vsi[pf->lan_vsi] && pf->vsi[pf->lan_vsi]->netdev) { 11581 ether_addr_copy(mac_addr, 11582 pf->vsi[pf->lan_vsi]->netdev->dev_addr); 11583 } else { 11584 dev_err(&pf->pdev->dev, 11585 "Failed to retrieve MAC address; using default\n"); 11586 ether_addr_copy(mac_addr, hw->mac.addr); 11587 } 11588 11589 /* The FW expects the mac address write cmd to first be called with 11590 * one of these flags before calling it again with the multicast 11591 * enable flags. 11592 */ 11593 flags = I40E_AQC_WRITE_TYPE_LAA_WOL; 11594 11595 if (hw->func_caps.flex10_enable && hw->partition_id != 1) 11596 flags = I40E_AQC_WRITE_TYPE_LAA_ONLY; 11597 11598 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 11599 if (ret) { 11600 dev_err(&pf->pdev->dev, 11601 "Failed to update MAC address registers; cannot enable Multicast Magic packet wake up"); 11602 return; 11603 } 11604 11605 flags = I40E_AQC_MC_MAG_EN 11606 | I40E_AQC_WOL_PRESERVE_ON_PFR 11607 | I40E_AQC_WRITE_TYPE_UPDATE_MC_MAG; 11608 ret = i40e_aq_mac_address_write(hw, flags, mac_addr, NULL); 11609 if (ret) 11610 dev_err(&pf->pdev->dev, 11611 "Failed to enable Multicast Magic Packet wake up\n"); 11612 } 11613 11614 /** 11615 * i40e_shutdown - PCI callback for shutting down 11616 * @pdev: PCI device information struct 11617 **/ 11618 static void i40e_shutdown(struct pci_dev *pdev) 11619 { 11620 struct i40e_pf *pf = pci_get_drvdata(pdev); 11621 struct i40e_hw *hw = &pf->hw; 11622 11623 set_bit(__I40E_SUSPENDED, &pf->state); 11624 set_bit(__I40E_DOWN, &pf->state); 11625 rtnl_lock(); 11626 i40e_prep_for_reset(pf); 11627 rtnl_unlock(); 11628 11629 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11630 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11631 11632 del_timer_sync(&pf->service_timer); 11633 cancel_work_sync(&pf->service_task); 11634 i40e_fdir_teardown(pf); 11635 11636 if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE)) 11637 i40e_enable_mc_magic_wake(pf); 11638 11639 rtnl_lock(); 11640 i40e_prep_for_reset(pf); 11641 rtnl_unlock(); 11642 11643 wr32(hw, I40E_PFPM_APM, 11644 (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11645 wr32(hw, I40E_PFPM_WUFC, 11646 (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11647 11648 i40e_clear_interrupt_scheme(pf); 11649 11650 if (system_state == SYSTEM_POWER_OFF) { 11651 pci_wake_from_d3(pdev, pf->wol_en); 11652 pci_set_power_state(pdev, PCI_D3hot); 11653 } 11654 } 11655 11656 #ifdef CONFIG_PM 11657 /** 11658 * i40e_suspend - PCI callback for moving to D3 11659 * @pdev: PCI device information struct 11660 **/ 11661 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state) 11662 { 11663 struct i40e_pf *pf = pci_get_drvdata(pdev); 11664 struct i40e_hw *hw = &pf->hw; 11665 int retval = 0; 11666 11667 set_bit(__I40E_SUSPENDED, &pf->state); 11668 set_bit(__I40E_DOWN, &pf->state); 11669 11670 if (pf->wol_en && (pf->flags & I40E_FLAG_WOL_MC_MAGIC_PKT_WAKE)) 11671 i40e_enable_mc_magic_wake(pf); 11672 11673 rtnl_lock(); 11674 i40e_prep_for_reset(pf); 11675 rtnl_unlock(); 11676 11677 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 11678 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 11679 11680 i40e_stop_misc_vector(pf); 11681 11682 retval = pci_save_state(pdev); 11683 if (retval) 11684 return retval; 11685 11686 pci_wake_from_d3(pdev, pf->wol_en); 11687 pci_set_power_state(pdev, PCI_D3hot); 11688 11689 return retval; 11690 } 11691 11692 /** 11693 * i40e_resume - PCI callback for waking up from D3 11694 * @pdev: PCI device information struct 11695 **/ 11696 static int i40e_resume(struct pci_dev *pdev) 11697 { 11698 struct i40e_pf *pf = pci_get_drvdata(pdev); 11699 u32 err; 11700 11701 pci_set_power_state(pdev, PCI_D0); 11702 pci_restore_state(pdev); 11703 /* pci_restore_state() clears dev->state_saves, so 11704 * call pci_save_state() again to restore it. 11705 */ 11706 pci_save_state(pdev); 11707 11708 err = pci_enable_device_mem(pdev); 11709 if (err) { 11710 dev_err(&pdev->dev, "Cannot enable PCI device from suspend\n"); 11711 return err; 11712 } 11713 pci_set_master(pdev); 11714 11715 /* no wakeup events while running */ 11716 pci_wake_from_d3(pdev, false); 11717 11718 /* handling the reset will rebuild the device state */ 11719 if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) { 11720 clear_bit(__I40E_DOWN, &pf->state); 11721 rtnl_lock(); 11722 i40e_reset_and_rebuild(pf, false); 11723 rtnl_unlock(); 11724 } 11725 11726 return 0; 11727 } 11728 11729 #endif 11730 static const struct pci_error_handlers i40e_err_handler = { 11731 .error_detected = i40e_pci_error_detected, 11732 .slot_reset = i40e_pci_error_slot_reset, 11733 .resume = i40e_pci_error_resume, 11734 }; 11735 11736 static struct pci_driver i40e_driver = { 11737 .name = i40e_driver_name, 11738 .id_table = i40e_pci_tbl, 11739 .probe = i40e_probe, 11740 .remove = i40e_remove, 11741 #ifdef CONFIG_PM 11742 .suspend = i40e_suspend, 11743 .resume = i40e_resume, 11744 #endif 11745 .shutdown = i40e_shutdown, 11746 .err_handler = &i40e_err_handler, 11747 .sriov_configure = i40e_pci_sriov_configure, 11748 }; 11749 11750 /** 11751 * i40e_init_module - Driver registration routine 11752 * 11753 * i40e_init_module is the first routine called when the driver is 11754 * loaded. All it does is register with the PCI subsystem. 11755 **/ 11756 static int __init i40e_init_module(void) 11757 { 11758 pr_info("%s: %s - version %s\n", i40e_driver_name, 11759 i40e_driver_string, i40e_driver_version_str); 11760 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 11761 11762 /* we will see if single thread per module is enough for now, 11763 * it can't be any worse than using the system workqueue which 11764 * was already single threaded 11765 */ 11766 i40e_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1, 11767 i40e_driver_name); 11768 if (!i40e_wq) { 11769 pr_err("%s: Failed to create workqueue\n", i40e_driver_name); 11770 return -ENOMEM; 11771 } 11772 11773 i40e_dbg_init(); 11774 return pci_register_driver(&i40e_driver); 11775 } 11776 module_init(i40e_init_module); 11777 11778 /** 11779 * i40e_exit_module - Driver exit cleanup routine 11780 * 11781 * i40e_exit_module is called just before the driver is removed 11782 * from memory. 11783 **/ 11784 static void __exit i40e_exit_module(void) 11785 { 11786 pci_unregister_driver(&i40e_driver); 11787 destroy_workqueue(i40e_wq); 11788 i40e_dbg_exit(); 11789 } 11790 module_exit(i40e_exit_module); 11791