1 /******************************************************************************* 2 * 3 * Intel Ethernet Controller XL710 Family Linux Driver 4 * Copyright(c) 2013 - 2014 Intel Corporation. 5 * 6 * This program is free software; you can redistribute it and/or modify it 7 * under the terms and conditions of the GNU General Public License, 8 * version 2, as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope it will be useful, but WITHOUT 11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 13 * more details. 14 * 15 * You should have received a copy of the GNU General Public License along 16 * with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 * The full GNU General Public License is included in this distribution in 19 * the file called "COPYING". 20 * 21 * Contact Information: 22 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net> 23 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497 24 * 25 ******************************************************************************/ 26 27 /* Local includes */ 28 #include "i40e.h" 29 #include "i40e_diag.h" 30 #ifdef CONFIG_I40E_VXLAN 31 #include <net/vxlan.h> 32 #endif 33 34 const char i40e_driver_name[] = "i40e"; 35 static const char i40e_driver_string[] = 36 "Intel(R) Ethernet Connection XL710 Network Driver"; 37 38 #define DRV_KERN "-k" 39 40 #define DRV_VERSION_MAJOR 0 41 #define DRV_VERSION_MINOR 4 42 #define DRV_VERSION_BUILD 10 43 #define DRV_VERSION __stringify(DRV_VERSION_MAJOR) "." \ 44 __stringify(DRV_VERSION_MINOR) "." \ 45 __stringify(DRV_VERSION_BUILD) DRV_KERN 46 const char i40e_driver_version_str[] = DRV_VERSION; 47 static const char i40e_copyright[] = "Copyright (c) 2013 - 2014 Intel Corporation."; 48 49 /* a bit of forward declarations */ 50 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi); 51 static void i40e_handle_reset_warning(struct i40e_pf *pf); 52 static int i40e_add_vsi(struct i40e_vsi *vsi); 53 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi); 54 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit); 55 static int i40e_setup_misc_vector(struct i40e_pf *pf); 56 static void i40e_determine_queue_usage(struct i40e_pf *pf); 57 static int i40e_setup_pf_filter_control(struct i40e_pf *pf); 58 static void i40e_fdir_sb_setup(struct i40e_pf *pf); 59 static int i40e_veb_get_bw_info(struct i40e_veb *veb); 60 61 /* i40e_pci_tbl - PCI Device ID Table 62 * 63 * Last entry must be all 0s 64 * 65 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID, 66 * Class, Class Mask, private data (not used) } 67 */ 68 static DEFINE_PCI_DEVICE_TABLE(i40e_pci_tbl) = { 69 {PCI_VDEVICE(INTEL, I40E_DEV_ID_SFP_XL710), 0}, 70 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QEMU), 0}, 71 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_A), 0}, 72 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_B), 0}, 73 {PCI_VDEVICE(INTEL, I40E_DEV_ID_KX_C), 0}, 74 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_A), 0}, 75 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_B), 0}, 76 {PCI_VDEVICE(INTEL, I40E_DEV_ID_QSFP_C), 0}, 77 /* required last entry */ 78 {0, } 79 }; 80 MODULE_DEVICE_TABLE(pci, i40e_pci_tbl); 81 82 #define I40E_MAX_VF_COUNT 128 83 static int debug = -1; 84 module_param(debug, int, 0); 85 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); 86 87 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>"); 88 MODULE_DESCRIPTION("Intel(R) Ethernet Connection XL710 Network Driver"); 89 MODULE_LICENSE("GPL"); 90 MODULE_VERSION(DRV_VERSION); 91 92 /** 93 * i40e_allocate_dma_mem_d - OS specific memory alloc for shared code 94 * @hw: pointer to the HW structure 95 * @mem: ptr to mem struct to fill out 96 * @size: size of memory requested 97 * @alignment: what to align the allocation to 98 **/ 99 int i40e_allocate_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem, 100 u64 size, u32 alignment) 101 { 102 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 103 104 mem->size = ALIGN(size, alignment); 105 mem->va = dma_zalloc_coherent(&pf->pdev->dev, mem->size, 106 &mem->pa, GFP_KERNEL); 107 if (!mem->va) 108 return -ENOMEM; 109 110 return 0; 111 } 112 113 /** 114 * i40e_free_dma_mem_d - OS specific memory free for shared code 115 * @hw: pointer to the HW structure 116 * @mem: ptr to mem struct to free 117 **/ 118 int i40e_free_dma_mem_d(struct i40e_hw *hw, struct i40e_dma_mem *mem) 119 { 120 struct i40e_pf *pf = (struct i40e_pf *)hw->back; 121 122 dma_free_coherent(&pf->pdev->dev, mem->size, mem->va, mem->pa); 123 mem->va = NULL; 124 mem->pa = 0; 125 mem->size = 0; 126 127 return 0; 128 } 129 130 /** 131 * i40e_allocate_virt_mem_d - OS specific memory alloc for shared code 132 * @hw: pointer to the HW structure 133 * @mem: ptr to mem struct to fill out 134 * @size: size of memory requested 135 **/ 136 int i40e_allocate_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem, 137 u32 size) 138 { 139 mem->size = size; 140 mem->va = kzalloc(size, GFP_KERNEL); 141 142 if (!mem->va) 143 return -ENOMEM; 144 145 return 0; 146 } 147 148 /** 149 * i40e_free_virt_mem_d - OS specific memory free for shared code 150 * @hw: pointer to the HW structure 151 * @mem: ptr to mem struct to free 152 **/ 153 int i40e_free_virt_mem_d(struct i40e_hw *hw, struct i40e_virt_mem *mem) 154 { 155 /* it's ok to kfree a NULL pointer */ 156 kfree(mem->va); 157 mem->va = NULL; 158 mem->size = 0; 159 160 return 0; 161 } 162 163 /** 164 * i40e_get_lump - find a lump of free generic resource 165 * @pf: board private structure 166 * @pile: the pile of resource to search 167 * @needed: the number of items needed 168 * @id: an owner id to stick on the items assigned 169 * 170 * Returns the base item index of the lump, or negative for error 171 * 172 * The search_hint trick and lack of advanced fit-finding only work 173 * because we're highly likely to have all the same size lump requests. 174 * Linear search time and any fragmentation should be minimal. 175 **/ 176 static int i40e_get_lump(struct i40e_pf *pf, struct i40e_lump_tracking *pile, 177 u16 needed, u16 id) 178 { 179 int ret = -ENOMEM; 180 int i, j; 181 182 if (!pile || needed == 0 || id >= I40E_PILE_VALID_BIT) { 183 dev_info(&pf->pdev->dev, 184 "param err: pile=%p needed=%d id=0x%04x\n", 185 pile, needed, id); 186 return -EINVAL; 187 } 188 189 /* start the linear search with an imperfect hint */ 190 i = pile->search_hint; 191 while (i < pile->num_entries) { 192 /* skip already allocated entries */ 193 if (pile->list[i] & I40E_PILE_VALID_BIT) { 194 i++; 195 continue; 196 } 197 198 /* do we have enough in this lump? */ 199 for (j = 0; (j < needed) && ((i+j) < pile->num_entries); j++) { 200 if (pile->list[i+j] & I40E_PILE_VALID_BIT) 201 break; 202 } 203 204 if (j == needed) { 205 /* there was enough, so assign it to the requestor */ 206 for (j = 0; j < needed; j++) 207 pile->list[i+j] = id | I40E_PILE_VALID_BIT; 208 ret = i; 209 pile->search_hint = i + j; 210 break; 211 } else { 212 /* not enough, so skip over it and continue looking */ 213 i += j; 214 } 215 } 216 217 return ret; 218 } 219 220 /** 221 * i40e_put_lump - return a lump of generic resource 222 * @pile: the pile of resource to search 223 * @index: the base item index 224 * @id: the owner id of the items assigned 225 * 226 * Returns the count of items in the lump 227 **/ 228 static int i40e_put_lump(struct i40e_lump_tracking *pile, u16 index, u16 id) 229 { 230 int valid_id = (id | I40E_PILE_VALID_BIT); 231 int count = 0; 232 int i; 233 234 if (!pile || index >= pile->num_entries) 235 return -EINVAL; 236 237 for (i = index; 238 i < pile->num_entries && pile->list[i] == valid_id; 239 i++) { 240 pile->list[i] = 0; 241 count++; 242 } 243 244 if (count && index < pile->search_hint) 245 pile->search_hint = index; 246 247 return count; 248 } 249 250 /** 251 * i40e_service_event_schedule - Schedule the service task to wake up 252 * @pf: board private structure 253 * 254 * If not already scheduled, this puts the task into the work queue 255 **/ 256 static void i40e_service_event_schedule(struct i40e_pf *pf) 257 { 258 if (!test_bit(__I40E_DOWN, &pf->state) && 259 !test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state) && 260 !test_and_set_bit(__I40E_SERVICE_SCHED, &pf->state)) 261 schedule_work(&pf->service_task); 262 } 263 264 /** 265 * i40e_tx_timeout - Respond to a Tx Hang 266 * @netdev: network interface device structure 267 * 268 * If any port has noticed a Tx timeout, it is likely that the whole 269 * device is munged, not just the one netdev port, so go for the full 270 * reset. 271 **/ 272 static void i40e_tx_timeout(struct net_device *netdev) 273 { 274 struct i40e_netdev_priv *np = netdev_priv(netdev); 275 struct i40e_vsi *vsi = np->vsi; 276 struct i40e_pf *pf = vsi->back; 277 278 pf->tx_timeout_count++; 279 280 if (time_after(jiffies, (pf->tx_timeout_last_recovery + HZ*20))) 281 pf->tx_timeout_recovery_level = 0; 282 pf->tx_timeout_last_recovery = jiffies; 283 netdev_info(netdev, "tx_timeout recovery level %d\n", 284 pf->tx_timeout_recovery_level); 285 286 switch (pf->tx_timeout_recovery_level) { 287 case 0: 288 /* disable and re-enable queues for the VSI */ 289 if (in_interrupt()) { 290 set_bit(__I40E_REINIT_REQUESTED, &pf->state); 291 set_bit(__I40E_REINIT_REQUESTED, &vsi->state); 292 } else { 293 i40e_vsi_reinit_locked(vsi); 294 } 295 break; 296 case 1: 297 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 298 break; 299 case 2: 300 set_bit(__I40E_CORE_RESET_REQUESTED, &pf->state); 301 break; 302 case 3: 303 set_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state); 304 break; 305 default: 306 netdev_err(netdev, "tx_timeout recovery unsuccessful\n"); 307 set_bit(__I40E_DOWN, &vsi->state); 308 i40e_down(vsi); 309 break; 310 } 311 i40e_service_event_schedule(pf); 312 pf->tx_timeout_recovery_level++; 313 } 314 315 /** 316 * i40e_release_rx_desc - Store the new tail and head values 317 * @rx_ring: ring to bump 318 * @val: new head index 319 **/ 320 static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val) 321 { 322 rx_ring->next_to_use = val; 323 324 /* Force memory writes to complete before letting h/w 325 * know there are new descriptors to fetch. (Only 326 * applicable for weak-ordered memory model archs, 327 * such as IA-64). 328 */ 329 wmb(); 330 writel(val, rx_ring->tail); 331 } 332 333 /** 334 * i40e_get_vsi_stats_struct - Get System Network Statistics 335 * @vsi: the VSI we care about 336 * 337 * Returns the address of the device statistics structure. 338 * The statistics are actually updated from the service task. 339 **/ 340 struct rtnl_link_stats64 *i40e_get_vsi_stats_struct(struct i40e_vsi *vsi) 341 { 342 return &vsi->net_stats; 343 } 344 345 /** 346 * i40e_get_netdev_stats_struct - Get statistics for netdev interface 347 * @netdev: network interface device structure 348 * 349 * Returns the address of the device statistics structure. 350 * The statistics are actually updated from the service task. 351 **/ 352 static struct rtnl_link_stats64 *i40e_get_netdev_stats_struct( 353 struct net_device *netdev, 354 struct rtnl_link_stats64 *stats) 355 { 356 struct i40e_netdev_priv *np = netdev_priv(netdev); 357 struct i40e_ring *tx_ring, *rx_ring; 358 struct i40e_vsi *vsi = np->vsi; 359 struct rtnl_link_stats64 *vsi_stats = i40e_get_vsi_stats_struct(vsi); 360 int i; 361 362 if (test_bit(__I40E_DOWN, &vsi->state)) 363 return stats; 364 365 if (!vsi->tx_rings) 366 return stats; 367 368 rcu_read_lock(); 369 for (i = 0; i < vsi->num_queue_pairs; i++) { 370 u64 bytes, packets; 371 unsigned int start; 372 373 tx_ring = ACCESS_ONCE(vsi->tx_rings[i]); 374 if (!tx_ring) 375 continue; 376 377 do { 378 start = u64_stats_fetch_begin_irq(&tx_ring->syncp); 379 packets = tx_ring->stats.packets; 380 bytes = tx_ring->stats.bytes; 381 } while (u64_stats_fetch_retry_irq(&tx_ring->syncp, start)); 382 383 stats->tx_packets += packets; 384 stats->tx_bytes += bytes; 385 rx_ring = &tx_ring[1]; 386 387 do { 388 start = u64_stats_fetch_begin_irq(&rx_ring->syncp); 389 packets = rx_ring->stats.packets; 390 bytes = rx_ring->stats.bytes; 391 } while (u64_stats_fetch_retry_irq(&rx_ring->syncp, start)); 392 393 stats->rx_packets += packets; 394 stats->rx_bytes += bytes; 395 } 396 rcu_read_unlock(); 397 398 /* following stats updated by i40e_watchdog_subtask() */ 399 stats->multicast = vsi_stats->multicast; 400 stats->tx_errors = vsi_stats->tx_errors; 401 stats->tx_dropped = vsi_stats->tx_dropped; 402 stats->rx_errors = vsi_stats->rx_errors; 403 stats->rx_crc_errors = vsi_stats->rx_crc_errors; 404 stats->rx_length_errors = vsi_stats->rx_length_errors; 405 406 return stats; 407 } 408 409 /** 410 * i40e_vsi_reset_stats - Resets all stats of the given vsi 411 * @vsi: the VSI to have its stats reset 412 **/ 413 void i40e_vsi_reset_stats(struct i40e_vsi *vsi) 414 { 415 struct rtnl_link_stats64 *ns; 416 int i; 417 418 if (!vsi) 419 return; 420 421 ns = i40e_get_vsi_stats_struct(vsi); 422 memset(ns, 0, sizeof(*ns)); 423 memset(&vsi->net_stats_offsets, 0, sizeof(vsi->net_stats_offsets)); 424 memset(&vsi->eth_stats, 0, sizeof(vsi->eth_stats)); 425 memset(&vsi->eth_stats_offsets, 0, sizeof(vsi->eth_stats_offsets)); 426 if (vsi->rx_rings && vsi->rx_rings[0]) { 427 for (i = 0; i < vsi->num_queue_pairs; i++) { 428 memset(&vsi->rx_rings[i]->stats, 0 , 429 sizeof(vsi->rx_rings[i]->stats)); 430 memset(&vsi->rx_rings[i]->rx_stats, 0 , 431 sizeof(vsi->rx_rings[i]->rx_stats)); 432 memset(&vsi->tx_rings[i]->stats, 0 , 433 sizeof(vsi->tx_rings[i]->stats)); 434 memset(&vsi->tx_rings[i]->tx_stats, 0, 435 sizeof(vsi->tx_rings[i]->tx_stats)); 436 } 437 } 438 vsi->stat_offsets_loaded = false; 439 } 440 441 /** 442 * i40e_pf_reset_stats - Reset all of the stats for the given pf 443 * @pf: the PF to be reset 444 **/ 445 void i40e_pf_reset_stats(struct i40e_pf *pf) 446 { 447 memset(&pf->stats, 0, sizeof(pf->stats)); 448 memset(&pf->stats_offsets, 0, sizeof(pf->stats_offsets)); 449 pf->stat_offsets_loaded = false; 450 } 451 452 /** 453 * i40e_stat_update48 - read and update a 48 bit stat from the chip 454 * @hw: ptr to the hardware info 455 * @hireg: the high 32 bit reg to read 456 * @loreg: the low 32 bit reg to read 457 * @offset_loaded: has the initial offset been loaded yet 458 * @offset: ptr to current offset value 459 * @stat: ptr to the stat 460 * 461 * Since the device stats are not reset at PFReset, they likely will not 462 * be zeroed when the driver starts. We'll save the first values read 463 * and use them as offsets to be subtracted from the raw values in order 464 * to report stats that count from zero. In the process, we also manage 465 * the potential roll-over. 466 **/ 467 static void i40e_stat_update48(struct i40e_hw *hw, u32 hireg, u32 loreg, 468 bool offset_loaded, u64 *offset, u64 *stat) 469 { 470 u64 new_data; 471 472 if (hw->device_id == I40E_DEV_ID_QEMU) { 473 new_data = rd32(hw, loreg); 474 new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32; 475 } else { 476 new_data = rd64(hw, loreg); 477 } 478 if (!offset_loaded) 479 *offset = new_data; 480 if (likely(new_data >= *offset)) 481 *stat = new_data - *offset; 482 else 483 *stat = (new_data + ((u64)1 << 48)) - *offset; 484 *stat &= 0xFFFFFFFFFFFFULL; 485 } 486 487 /** 488 * i40e_stat_update32 - read and update a 32 bit stat from the chip 489 * @hw: ptr to the hardware info 490 * @reg: the hw reg to read 491 * @offset_loaded: has the initial offset been loaded yet 492 * @offset: ptr to current offset value 493 * @stat: ptr to the stat 494 **/ 495 static void i40e_stat_update32(struct i40e_hw *hw, u32 reg, 496 bool offset_loaded, u64 *offset, u64 *stat) 497 { 498 u32 new_data; 499 500 new_data = rd32(hw, reg); 501 if (!offset_loaded) 502 *offset = new_data; 503 if (likely(new_data >= *offset)) 504 *stat = (u32)(new_data - *offset); 505 else 506 *stat = (u32)((new_data + ((u64)1 << 32)) - *offset); 507 } 508 509 /** 510 * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. 511 * @vsi: the VSI to be updated 512 **/ 513 void i40e_update_eth_stats(struct i40e_vsi *vsi) 514 { 515 int stat_idx = le16_to_cpu(vsi->info.stat_counter_idx); 516 struct i40e_pf *pf = vsi->back; 517 struct i40e_hw *hw = &pf->hw; 518 struct i40e_eth_stats *oes; 519 struct i40e_eth_stats *es; /* device's eth stats */ 520 521 es = &vsi->eth_stats; 522 oes = &vsi->eth_stats_offsets; 523 524 /* Gather up the stats that the hw collects */ 525 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 526 vsi->stat_offsets_loaded, 527 &oes->tx_errors, &es->tx_errors); 528 i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), 529 vsi->stat_offsets_loaded, 530 &oes->rx_discards, &es->rx_discards); 531 i40e_stat_update32(hw, I40E_GLV_RUPP(stat_idx), 532 vsi->stat_offsets_loaded, 533 &oes->rx_unknown_protocol, &es->rx_unknown_protocol); 534 i40e_stat_update32(hw, I40E_GLV_TEPC(stat_idx), 535 vsi->stat_offsets_loaded, 536 &oes->tx_errors, &es->tx_errors); 537 538 i40e_stat_update48(hw, I40E_GLV_GORCH(stat_idx), 539 I40E_GLV_GORCL(stat_idx), 540 vsi->stat_offsets_loaded, 541 &oes->rx_bytes, &es->rx_bytes); 542 i40e_stat_update48(hw, I40E_GLV_UPRCH(stat_idx), 543 I40E_GLV_UPRCL(stat_idx), 544 vsi->stat_offsets_loaded, 545 &oes->rx_unicast, &es->rx_unicast); 546 i40e_stat_update48(hw, I40E_GLV_MPRCH(stat_idx), 547 I40E_GLV_MPRCL(stat_idx), 548 vsi->stat_offsets_loaded, 549 &oes->rx_multicast, &es->rx_multicast); 550 i40e_stat_update48(hw, I40E_GLV_BPRCH(stat_idx), 551 I40E_GLV_BPRCL(stat_idx), 552 vsi->stat_offsets_loaded, 553 &oes->rx_broadcast, &es->rx_broadcast); 554 555 i40e_stat_update48(hw, I40E_GLV_GOTCH(stat_idx), 556 I40E_GLV_GOTCL(stat_idx), 557 vsi->stat_offsets_loaded, 558 &oes->tx_bytes, &es->tx_bytes); 559 i40e_stat_update48(hw, I40E_GLV_UPTCH(stat_idx), 560 I40E_GLV_UPTCL(stat_idx), 561 vsi->stat_offsets_loaded, 562 &oes->tx_unicast, &es->tx_unicast); 563 i40e_stat_update48(hw, I40E_GLV_MPTCH(stat_idx), 564 I40E_GLV_MPTCL(stat_idx), 565 vsi->stat_offsets_loaded, 566 &oes->tx_multicast, &es->tx_multicast); 567 i40e_stat_update48(hw, I40E_GLV_BPTCH(stat_idx), 568 I40E_GLV_BPTCL(stat_idx), 569 vsi->stat_offsets_loaded, 570 &oes->tx_broadcast, &es->tx_broadcast); 571 vsi->stat_offsets_loaded = true; 572 } 573 574 /** 575 * i40e_update_veb_stats - Update Switch component statistics 576 * @veb: the VEB being updated 577 **/ 578 static void i40e_update_veb_stats(struct i40e_veb *veb) 579 { 580 struct i40e_pf *pf = veb->pf; 581 struct i40e_hw *hw = &pf->hw; 582 struct i40e_eth_stats *oes; 583 struct i40e_eth_stats *es; /* device's eth stats */ 584 int idx = 0; 585 586 idx = veb->stats_idx; 587 es = &veb->stats; 588 oes = &veb->stats_offsets; 589 590 /* Gather up the stats that the hw collects */ 591 i40e_stat_update32(hw, I40E_GLSW_TDPC(idx), 592 veb->stat_offsets_loaded, 593 &oes->tx_discards, &es->tx_discards); 594 if (hw->revision_id > 0) 595 i40e_stat_update32(hw, I40E_GLSW_RUPP(idx), 596 veb->stat_offsets_loaded, 597 &oes->rx_unknown_protocol, 598 &es->rx_unknown_protocol); 599 i40e_stat_update48(hw, I40E_GLSW_GORCH(idx), I40E_GLSW_GORCL(idx), 600 veb->stat_offsets_loaded, 601 &oes->rx_bytes, &es->rx_bytes); 602 i40e_stat_update48(hw, I40E_GLSW_UPRCH(idx), I40E_GLSW_UPRCL(idx), 603 veb->stat_offsets_loaded, 604 &oes->rx_unicast, &es->rx_unicast); 605 i40e_stat_update48(hw, I40E_GLSW_MPRCH(idx), I40E_GLSW_MPRCL(idx), 606 veb->stat_offsets_loaded, 607 &oes->rx_multicast, &es->rx_multicast); 608 i40e_stat_update48(hw, I40E_GLSW_BPRCH(idx), I40E_GLSW_BPRCL(idx), 609 veb->stat_offsets_loaded, 610 &oes->rx_broadcast, &es->rx_broadcast); 611 612 i40e_stat_update48(hw, I40E_GLSW_GOTCH(idx), I40E_GLSW_GOTCL(idx), 613 veb->stat_offsets_loaded, 614 &oes->tx_bytes, &es->tx_bytes); 615 i40e_stat_update48(hw, I40E_GLSW_UPTCH(idx), I40E_GLSW_UPTCL(idx), 616 veb->stat_offsets_loaded, 617 &oes->tx_unicast, &es->tx_unicast); 618 i40e_stat_update48(hw, I40E_GLSW_MPTCH(idx), I40E_GLSW_MPTCL(idx), 619 veb->stat_offsets_loaded, 620 &oes->tx_multicast, &es->tx_multicast); 621 i40e_stat_update48(hw, I40E_GLSW_BPTCH(idx), I40E_GLSW_BPTCL(idx), 622 veb->stat_offsets_loaded, 623 &oes->tx_broadcast, &es->tx_broadcast); 624 veb->stat_offsets_loaded = true; 625 } 626 627 /** 628 * i40e_update_link_xoff_rx - Update XOFF received in link flow control mode 629 * @pf: the corresponding PF 630 * 631 * Update the Rx XOFF counter (PAUSE frames) in link flow control mode 632 **/ 633 static void i40e_update_link_xoff_rx(struct i40e_pf *pf) 634 { 635 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 636 struct i40e_hw_port_stats *nsd = &pf->stats; 637 struct i40e_hw *hw = &pf->hw; 638 u64 xoff = 0; 639 u16 i, v; 640 641 if ((hw->fc.current_mode != I40E_FC_FULL) && 642 (hw->fc.current_mode != I40E_FC_RX_PAUSE)) 643 return; 644 645 xoff = nsd->link_xoff_rx; 646 i40e_stat_update32(hw, I40E_GLPRT_LXOFFRXC(hw->port), 647 pf->stat_offsets_loaded, 648 &osd->link_xoff_rx, &nsd->link_xoff_rx); 649 650 /* No new LFC xoff rx */ 651 if (!(nsd->link_xoff_rx - xoff)) 652 return; 653 654 /* Clear the __I40E_HANG_CHECK_ARMED bit for all Tx rings */ 655 for (v = 0; v < pf->num_alloc_vsi; v++) { 656 struct i40e_vsi *vsi = pf->vsi[v]; 657 658 if (!vsi || !vsi->tx_rings[0]) 659 continue; 660 661 for (i = 0; i < vsi->num_queue_pairs; i++) { 662 struct i40e_ring *ring = vsi->tx_rings[i]; 663 clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state); 664 } 665 } 666 } 667 668 /** 669 * i40e_update_prio_xoff_rx - Update XOFF received in PFC mode 670 * @pf: the corresponding PF 671 * 672 * Update the Rx XOFF counter (PAUSE frames) in PFC mode 673 **/ 674 static void i40e_update_prio_xoff_rx(struct i40e_pf *pf) 675 { 676 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 677 struct i40e_hw_port_stats *nsd = &pf->stats; 678 bool xoff[I40E_MAX_TRAFFIC_CLASS] = {false}; 679 struct i40e_dcbx_config *dcb_cfg; 680 struct i40e_hw *hw = &pf->hw; 681 u16 i, v; 682 u8 tc; 683 684 dcb_cfg = &hw->local_dcbx_config; 685 686 /* See if DCB enabled with PFC TC */ 687 if (!(pf->flags & I40E_FLAG_DCB_ENABLED) || 688 !(dcb_cfg->pfc.pfcenable)) { 689 i40e_update_link_xoff_rx(pf); 690 return; 691 } 692 693 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 694 u64 prio_xoff = nsd->priority_xoff_rx[i]; 695 i40e_stat_update32(hw, I40E_GLPRT_PXOFFRXC(hw->port, i), 696 pf->stat_offsets_loaded, 697 &osd->priority_xoff_rx[i], 698 &nsd->priority_xoff_rx[i]); 699 700 /* No new PFC xoff rx */ 701 if (!(nsd->priority_xoff_rx[i] - prio_xoff)) 702 continue; 703 /* Get the TC for given priority */ 704 tc = dcb_cfg->etscfg.prioritytable[i]; 705 xoff[tc] = true; 706 } 707 708 /* Clear the __I40E_HANG_CHECK_ARMED bit for Tx rings */ 709 for (v = 0; v < pf->num_alloc_vsi; v++) { 710 struct i40e_vsi *vsi = pf->vsi[v]; 711 712 if (!vsi || !vsi->tx_rings[0]) 713 continue; 714 715 for (i = 0; i < vsi->num_queue_pairs; i++) { 716 struct i40e_ring *ring = vsi->tx_rings[i]; 717 718 tc = ring->dcb_tc; 719 if (xoff[tc]) 720 clear_bit(__I40E_HANG_CHECK_ARMED, 721 &ring->state); 722 } 723 } 724 } 725 726 /** 727 * i40e_update_vsi_stats - Update the vsi statistics counters. 728 * @vsi: the VSI to be updated 729 * 730 * There are a few instances where we store the same stat in a 731 * couple of different structs. This is partly because we have 732 * the netdev stats that need to be filled out, which is slightly 733 * different from the "eth_stats" defined by the chip and used in 734 * VF communications. We sort it out here. 735 **/ 736 static void i40e_update_vsi_stats(struct i40e_vsi *vsi) 737 { 738 struct i40e_pf *pf = vsi->back; 739 struct rtnl_link_stats64 *ons; 740 struct rtnl_link_stats64 *ns; /* netdev stats */ 741 struct i40e_eth_stats *oes; 742 struct i40e_eth_stats *es; /* device's eth stats */ 743 u32 tx_restart, tx_busy; 744 u32 rx_page, rx_buf; 745 u64 rx_p, rx_b; 746 u64 tx_p, tx_b; 747 u16 q; 748 749 if (test_bit(__I40E_DOWN, &vsi->state) || 750 test_bit(__I40E_CONFIG_BUSY, &pf->state)) 751 return; 752 753 ns = i40e_get_vsi_stats_struct(vsi); 754 ons = &vsi->net_stats_offsets; 755 es = &vsi->eth_stats; 756 oes = &vsi->eth_stats_offsets; 757 758 /* Gather up the netdev and vsi stats that the driver collects 759 * on the fly during packet processing 760 */ 761 rx_b = rx_p = 0; 762 tx_b = tx_p = 0; 763 tx_restart = tx_busy = 0; 764 rx_page = 0; 765 rx_buf = 0; 766 rcu_read_lock(); 767 for (q = 0; q < vsi->num_queue_pairs; q++) { 768 struct i40e_ring *p; 769 u64 bytes, packets; 770 unsigned int start; 771 772 /* locate Tx ring */ 773 p = ACCESS_ONCE(vsi->tx_rings[q]); 774 775 do { 776 start = u64_stats_fetch_begin_irq(&p->syncp); 777 packets = p->stats.packets; 778 bytes = p->stats.bytes; 779 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 780 tx_b += bytes; 781 tx_p += packets; 782 tx_restart += p->tx_stats.restart_queue; 783 tx_busy += p->tx_stats.tx_busy; 784 785 /* Rx queue is part of the same block as Tx queue */ 786 p = &p[1]; 787 do { 788 start = u64_stats_fetch_begin_irq(&p->syncp); 789 packets = p->stats.packets; 790 bytes = p->stats.bytes; 791 } while (u64_stats_fetch_retry_irq(&p->syncp, start)); 792 rx_b += bytes; 793 rx_p += packets; 794 rx_buf += p->rx_stats.alloc_buff_failed; 795 rx_page += p->rx_stats.alloc_page_failed; 796 } 797 rcu_read_unlock(); 798 vsi->tx_restart = tx_restart; 799 vsi->tx_busy = tx_busy; 800 vsi->rx_page_failed = rx_page; 801 vsi->rx_buf_failed = rx_buf; 802 803 ns->rx_packets = rx_p; 804 ns->rx_bytes = rx_b; 805 ns->tx_packets = tx_p; 806 ns->tx_bytes = tx_b; 807 808 /* update netdev stats from eth stats */ 809 i40e_update_eth_stats(vsi); 810 ons->tx_errors = oes->tx_errors; 811 ns->tx_errors = es->tx_errors; 812 ons->multicast = oes->rx_multicast; 813 ns->multicast = es->rx_multicast; 814 ons->rx_dropped = oes->rx_discards; 815 ns->rx_dropped = es->rx_discards; 816 ons->tx_dropped = oes->tx_discards; 817 ns->tx_dropped = es->tx_discards; 818 819 /* pull in a couple PF stats if this is the main vsi */ 820 if (vsi == pf->vsi[pf->lan_vsi]) { 821 ns->rx_crc_errors = pf->stats.crc_errors; 822 ns->rx_errors = pf->stats.crc_errors + pf->stats.illegal_bytes; 823 ns->rx_length_errors = pf->stats.rx_length_errors; 824 } 825 } 826 827 /** 828 * i40e_update_pf_stats - Update the pf statistics counters. 829 * @pf: the PF to be updated 830 **/ 831 static void i40e_update_pf_stats(struct i40e_pf *pf) 832 { 833 struct i40e_hw_port_stats *osd = &pf->stats_offsets; 834 struct i40e_hw_port_stats *nsd = &pf->stats; 835 struct i40e_hw *hw = &pf->hw; 836 u32 val; 837 int i; 838 839 i40e_stat_update48(hw, I40E_GLPRT_GORCH(hw->port), 840 I40E_GLPRT_GORCL(hw->port), 841 pf->stat_offsets_loaded, 842 &osd->eth.rx_bytes, &nsd->eth.rx_bytes); 843 i40e_stat_update48(hw, I40E_GLPRT_GOTCH(hw->port), 844 I40E_GLPRT_GOTCL(hw->port), 845 pf->stat_offsets_loaded, 846 &osd->eth.tx_bytes, &nsd->eth.tx_bytes); 847 i40e_stat_update32(hw, I40E_GLPRT_RDPC(hw->port), 848 pf->stat_offsets_loaded, 849 &osd->eth.rx_discards, 850 &nsd->eth.rx_discards); 851 i40e_stat_update32(hw, I40E_GLPRT_TDPC(hw->port), 852 pf->stat_offsets_loaded, 853 &osd->eth.tx_discards, 854 &nsd->eth.tx_discards); 855 856 i40e_stat_update48(hw, I40E_GLPRT_UPRCH(hw->port), 857 I40E_GLPRT_UPRCL(hw->port), 858 pf->stat_offsets_loaded, 859 &osd->eth.rx_unicast, 860 &nsd->eth.rx_unicast); 861 i40e_stat_update48(hw, I40E_GLPRT_MPRCH(hw->port), 862 I40E_GLPRT_MPRCL(hw->port), 863 pf->stat_offsets_loaded, 864 &osd->eth.rx_multicast, 865 &nsd->eth.rx_multicast); 866 i40e_stat_update48(hw, I40E_GLPRT_BPRCH(hw->port), 867 I40E_GLPRT_BPRCL(hw->port), 868 pf->stat_offsets_loaded, 869 &osd->eth.rx_broadcast, 870 &nsd->eth.rx_broadcast); 871 i40e_stat_update48(hw, I40E_GLPRT_UPTCH(hw->port), 872 I40E_GLPRT_UPTCL(hw->port), 873 pf->stat_offsets_loaded, 874 &osd->eth.tx_unicast, 875 &nsd->eth.tx_unicast); 876 i40e_stat_update48(hw, I40E_GLPRT_MPTCH(hw->port), 877 I40E_GLPRT_MPTCL(hw->port), 878 pf->stat_offsets_loaded, 879 &osd->eth.tx_multicast, 880 &nsd->eth.tx_multicast); 881 i40e_stat_update48(hw, I40E_GLPRT_BPTCH(hw->port), 882 I40E_GLPRT_BPTCL(hw->port), 883 pf->stat_offsets_loaded, 884 &osd->eth.tx_broadcast, 885 &nsd->eth.tx_broadcast); 886 887 i40e_stat_update32(hw, I40E_GLPRT_TDOLD(hw->port), 888 pf->stat_offsets_loaded, 889 &osd->tx_dropped_link_down, 890 &nsd->tx_dropped_link_down); 891 892 i40e_stat_update32(hw, I40E_GLPRT_CRCERRS(hw->port), 893 pf->stat_offsets_loaded, 894 &osd->crc_errors, &nsd->crc_errors); 895 896 i40e_stat_update32(hw, I40E_GLPRT_ILLERRC(hw->port), 897 pf->stat_offsets_loaded, 898 &osd->illegal_bytes, &nsd->illegal_bytes); 899 900 i40e_stat_update32(hw, I40E_GLPRT_MLFC(hw->port), 901 pf->stat_offsets_loaded, 902 &osd->mac_local_faults, 903 &nsd->mac_local_faults); 904 i40e_stat_update32(hw, I40E_GLPRT_MRFC(hw->port), 905 pf->stat_offsets_loaded, 906 &osd->mac_remote_faults, 907 &nsd->mac_remote_faults); 908 909 i40e_stat_update32(hw, I40E_GLPRT_RLEC(hw->port), 910 pf->stat_offsets_loaded, 911 &osd->rx_length_errors, 912 &nsd->rx_length_errors); 913 914 i40e_stat_update32(hw, I40E_GLPRT_LXONRXC(hw->port), 915 pf->stat_offsets_loaded, 916 &osd->link_xon_rx, &nsd->link_xon_rx); 917 i40e_stat_update32(hw, I40E_GLPRT_LXONTXC(hw->port), 918 pf->stat_offsets_loaded, 919 &osd->link_xon_tx, &nsd->link_xon_tx); 920 i40e_update_prio_xoff_rx(pf); /* handles I40E_GLPRT_LXOFFRXC */ 921 i40e_stat_update32(hw, I40E_GLPRT_LXOFFTXC(hw->port), 922 pf->stat_offsets_loaded, 923 &osd->link_xoff_tx, &nsd->link_xoff_tx); 924 925 for (i = 0; i < 8; i++) { 926 i40e_stat_update32(hw, I40E_GLPRT_PXONRXC(hw->port, i), 927 pf->stat_offsets_loaded, 928 &osd->priority_xon_rx[i], 929 &nsd->priority_xon_rx[i]); 930 i40e_stat_update32(hw, I40E_GLPRT_PXONTXC(hw->port, i), 931 pf->stat_offsets_loaded, 932 &osd->priority_xon_tx[i], 933 &nsd->priority_xon_tx[i]); 934 i40e_stat_update32(hw, I40E_GLPRT_PXOFFTXC(hw->port, i), 935 pf->stat_offsets_loaded, 936 &osd->priority_xoff_tx[i], 937 &nsd->priority_xoff_tx[i]); 938 i40e_stat_update32(hw, 939 I40E_GLPRT_RXON2OFFCNT(hw->port, i), 940 pf->stat_offsets_loaded, 941 &osd->priority_xon_2_xoff[i], 942 &nsd->priority_xon_2_xoff[i]); 943 } 944 945 i40e_stat_update48(hw, I40E_GLPRT_PRC64H(hw->port), 946 I40E_GLPRT_PRC64L(hw->port), 947 pf->stat_offsets_loaded, 948 &osd->rx_size_64, &nsd->rx_size_64); 949 i40e_stat_update48(hw, I40E_GLPRT_PRC127H(hw->port), 950 I40E_GLPRT_PRC127L(hw->port), 951 pf->stat_offsets_loaded, 952 &osd->rx_size_127, &nsd->rx_size_127); 953 i40e_stat_update48(hw, I40E_GLPRT_PRC255H(hw->port), 954 I40E_GLPRT_PRC255L(hw->port), 955 pf->stat_offsets_loaded, 956 &osd->rx_size_255, &nsd->rx_size_255); 957 i40e_stat_update48(hw, I40E_GLPRT_PRC511H(hw->port), 958 I40E_GLPRT_PRC511L(hw->port), 959 pf->stat_offsets_loaded, 960 &osd->rx_size_511, &nsd->rx_size_511); 961 i40e_stat_update48(hw, I40E_GLPRT_PRC1023H(hw->port), 962 I40E_GLPRT_PRC1023L(hw->port), 963 pf->stat_offsets_loaded, 964 &osd->rx_size_1023, &nsd->rx_size_1023); 965 i40e_stat_update48(hw, I40E_GLPRT_PRC1522H(hw->port), 966 I40E_GLPRT_PRC1522L(hw->port), 967 pf->stat_offsets_loaded, 968 &osd->rx_size_1522, &nsd->rx_size_1522); 969 i40e_stat_update48(hw, I40E_GLPRT_PRC9522H(hw->port), 970 I40E_GLPRT_PRC9522L(hw->port), 971 pf->stat_offsets_loaded, 972 &osd->rx_size_big, &nsd->rx_size_big); 973 974 i40e_stat_update48(hw, I40E_GLPRT_PTC64H(hw->port), 975 I40E_GLPRT_PTC64L(hw->port), 976 pf->stat_offsets_loaded, 977 &osd->tx_size_64, &nsd->tx_size_64); 978 i40e_stat_update48(hw, I40E_GLPRT_PTC127H(hw->port), 979 I40E_GLPRT_PTC127L(hw->port), 980 pf->stat_offsets_loaded, 981 &osd->tx_size_127, &nsd->tx_size_127); 982 i40e_stat_update48(hw, I40E_GLPRT_PTC255H(hw->port), 983 I40E_GLPRT_PTC255L(hw->port), 984 pf->stat_offsets_loaded, 985 &osd->tx_size_255, &nsd->tx_size_255); 986 i40e_stat_update48(hw, I40E_GLPRT_PTC511H(hw->port), 987 I40E_GLPRT_PTC511L(hw->port), 988 pf->stat_offsets_loaded, 989 &osd->tx_size_511, &nsd->tx_size_511); 990 i40e_stat_update48(hw, I40E_GLPRT_PTC1023H(hw->port), 991 I40E_GLPRT_PTC1023L(hw->port), 992 pf->stat_offsets_loaded, 993 &osd->tx_size_1023, &nsd->tx_size_1023); 994 i40e_stat_update48(hw, I40E_GLPRT_PTC1522H(hw->port), 995 I40E_GLPRT_PTC1522L(hw->port), 996 pf->stat_offsets_loaded, 997 &osd->tx_size_1522, &nsd->tx_size_1522); 998 i40e_stat_update48(hw, I40E_GLPRT_PTC9522H(hw->port), 999 I40E_GLPRT_PTC9522L(hw->port), 1000 pf->stat_offsets_loaded, 1001 &osd->tx_size_big, &nsd->tx_size_big); 1002 1003 i40e_stat_update32(hw, I40E_GLPRT_RUC(hw->port), 1004 pf->stat_offsets_loaded, 1005 &osd->rx_undersize, &nsd->rx_undersize); 1006 i40e_stat_update32(hw, I40E_GLPRT_RFC(hw->port), 1007 pf->stat_offsets_loaded, 1008 &osd->rx_fragments, &nsd->rx_fragments); 1009 i40e_stat_update32(hw, I40E_GLPRT_ROC(hw->port), 1010 pf->stat_offsets_loaded, 1011 &osd->rx_oversize, &nsd->rx_oversize); 1012 i40e_stat_update32(hw, I40E_GLPRT_RJC(hw->port), 1013 pf->stat_offsets_loaded, 1014 &osd->rx_jabber, &nsd->rx_jabber); 1015 1016 /* FDIR stats */ 1017 i40e_stat_update32(hw, I40E_GLQF_PCNT(pf->fd_atr_cnt_idx), 1018 pf->stat_offsets_loaded, 1019 &osd->fd_atr_match, &nsd->fd_atr_match); 1020 i40e_stat_update32(hw, I40E_GLQF_PCNT(pf->fd_sb_cnt_idx), 1021 pf->stat_offsets_loaded, 1022 &osd->fd_sb_match, &nsd->fd_sb_match); 1023 1024 val = rd32(hw, I40E_PRTPM_EEE_STAT); 1025 nsd->tx_lpi_status = 1026 (val & I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_MASK) >> 1027 I40E_PRTPM_EEE_STAT_TX_LPI_STATUS_SHIFT; 1028 nsd->rx_lpi_status = 1029 (val & I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_MASK) >> 1030 I40E_PRTPM_EEE_STAT_RX_LPI_STATUS_SHIFT; 1031 i40e_stat_update32(hw, I40E_PRTPM_TLPIC, 1032 pf->stat_offsets_loaded, 1033 &osd->tx_lpi_count, &nsd->tx_lpi_count); 1034 i40e_stat_update32(hw, I40E_PRTPM_RLPIC, 1035 pf->stat_offsets_loaded, 1036 &osd->rx_lpi_count, &nsd->rx_lpi_count); 1037 1038 pf->stat_offsets_loaded = true; 1039 } 1040 1041 /** 1042 * i40e_update_stats - Update the various statistics counters. 1043 * @vsi: the VSI to be updated 1044 * 1045 * Update the various stats for this VSI and its related entities. 1046 **/ 1047 void i40e_update_stats(struct i40e_vsi *vsi) 1048 { 1049 struct i40e_pf *pf = vsi->back; 1050 1051 if (vsi == pf->vsi[pf->lan_vsi]) 1052 i40e_update_pf_stats(pf); 1053 1054 i40e_update_vsi_stats(vsi); 1055 } 1056 1057 /** 1058 * i40e_find_filter - Search VSI filter list for specific mac/vlan filter 1059 * @vsi: the VSI to be searched 1060 * @macaddr: the MAC address 1061 * @vlan: the vlan 1062 * @is_vf: make sure its a vf filter, else doesn't matter 1063 * @is_netdev: make sure its a netdev filter, else doesn't matter 1064 * 1065 * Returns ptr to the filter object or NULL 1066 **/ 1067 static struct i40e_mac_filter *i40e_find_filter(struct i40e_vsi *vsi, 1068 u8 *macaddr, s16 vlan, 1069 bool is_vf, bool is_netdev) 1070 { 1071 struct i40e_mac_filter *f; 1072 1073 if (!vsi || !macaddr) 1074 return NULL; 1075 1076 list_for_each_entry(f, &vsi->mac_filter_list, list) { 1077 if ((ether_addr_equal(macaddr, f->macaddr)) && 1078 (vlan == f->vlan) && 1079 (!is_vf || f->is_vf) && 1080 (!is_netdev || f->is_netdev)) 1081 return f; 1082 } 1083 return NULL; 1084 } 1085 1086 /** 1087 * i40e_find_mac - Find a mac addr in the macvlan filters list 1088 * @vsi: the VSI to be searched 1089 * @macaddr: the MAC address we are searching for 1090 * @is_vf: make sure its a vf filter, else doesn't matter 1091 * @is_netdev: make sure its a netdev filter, else doesn't matter 1092 * 1093 * Returns the first filter with the provided MAC address or NULL if 1094 * MAC address was not found 1095 **/ 1096 struct i40e_mac_filter *i40e_find_mac(struct i40e_vsi *vsi, u8 *macaddr, 1097 bool is_vf, bool is_netdev) 1098 { 1099 struct i40e_mac_filter *f; 1100 1101 if (!vsi || !macaddr) 1102 return NULL; 1103 1104 list_for_each_entry(f, &vsi->mac_filter_list, list) { 1105 if ((ether_addr_equal(macaddr, f->macaddr)) && 1106 (!is_vf || f->is_vf) && 1107 (!is_netdev || f->is_netdev)) 1108 return f; 1109 } 1110 return NULL; 1111 } 1112 1113 /** 1114 * i40e_is_vsi_in_vlan - Check if VSI is in vlan mode 1115 * @vsi: the VSI to be searched 1116 * 1117 * Returns true if VSI is in vlan mode or false otherwise 1118 **/ 1119 bool i40e_is_vsi_in_vlan(struct i40e_vsi *vsi) 1120 { 1121 struct i40e_mac_filter *f; 1122 1123 /* Only -1 for all the filters denotes not in vlan mode 1124 * so we have to go through all the list in order to make sure 1125 */ 1126 list_for_each_entry(f, &vsi->mac_filter_list, list) { 1127 if (f->vlan >= 0) 1128 return true; 1129 } 1130 1131 return false; 1132 } 1133 1134 /** 1135 * i40e_put_mac_in_vlan - Make macvlan filters from macaddrs and vlans 1136 * @vsi: the VSI to be searched 1137 * @macaddr: the mac address to be filtered 1138 * @is_vf: true if it is a vf 1139 * @is_netdev: true if it is a netdev 1140 * 1141 * Goes through all the macvlan filters and adds a 1142 * macvlan filter for each unique vlan that already exists 1143 * 1144 * Returns first filter found on success, else NULL 1145 **/ 1146 struct i40e_mac_filter *i40e_put_mac_in_vlan(struct i40e_vsi *vsi, u8 *macaddr, 1147 bool is_vf, bool is_netdev) 1148 { 1149 struct i40e_mac_filter *f; 1150 1151 list_for_each_entry(f, &vsi->mac_filter_list, list) { 1152 if (!i40e_find_filter(vsi, macaddr, f->vlan, 1153 is_vf, is_netdev)) { 1154 if (!i40e_add_filter(vsi, macaddr, f->vlan, 1155 is_vf, is_netdev)) 1156 return NULL; 1157 } 1158 } 1159 1160 return list_first_entry_or_null(&vsi->mac_filter_list, 1161 struct i40e_mac_filter, list); 1162 } 1163 1164 /** 1165 * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM 1166 * @vsi: the PF Main VSI - inappropriate for any other VSI 1167 * @macaddr: the MAC address 1168 **/ 1169 static void i40e_rm_default_mac_filter(struct i40e_vsi *vsi, u8 *macaddr) 1170 { 1171 struct i40e_aqc_remove_macvlan_element_data element; 1172 struct i40e_pf *pf = vsi->back; 1173 i40e_status aq_ret; 1174 1175 /* Only appropriate for the PF main VSI */ 1176 if (vsi->type != I40E_VSI_MAIN) 1177 return; 1178 1179 ether_addr_copy(element.mac_addr, macaddr); 1180 element.vlan_tag = 0; 1181 element.flags = I40E_AQC_MACVLAN_DEL_PERFECT_MATCH | 1182 I40E_AQC_MACVLAN_DEL_IGNORE_VLAN; 1183 aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid, &element, 1, NULL); 1184 if (aq_ret) 1185 dev_err(&pf->pdev->dev, "Could not remove default MAC-VLAN\n"); 1186 } 1187 1188 /** 1189 * i40e_add_filter - Add a mac/vlan filter to the VSI 1190 * @vsi: the VSI to be searched 1191 * @macaddr: the MAC address 1192 * @vlan: the vlan 1193 * @is_vf: make sure its a vf filter, else doesn't matter 1194 * @is_netdev: make sure its a netdev filter, else doesn't matter 1195 * 1196 * Returns ptr to the filter object or NULL when no memory available. 1197 **/ 1198 struct i40e_mac_filter *i40e_add_filter(struct i40e_vsi *vsi, 1199 u8 *macaddr, s16 vlan, 1200 bool is_vf, bool is_netdev) 1201 { 1202 struct i40e_mac_filter *f; 1203 1204 if (!vsi || !macaddr) 1205 return NULL; 1206 1207 f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev); 1208 if (!f) { 1209 f = kzalloc(sizeof(*f), GFP_ATOMIC); 1210 if (!f) 1211 goto add_filter_out; 1212 1213 ether_addr_copy(f->macaddr, macaddr); 1214 f->vlan = vlan; 1215 f->changed = true; 1216 1217 INIT_LIST_HEAD(&f->list); 1218 list_add(&f->list, &vsi->mac_filter_list); 1219 } 1220 1221 /* increment counter and add a new flag if needed */ 1222 if (is_vf) { 1223 if (!f->is_vf) { 1224 f->is_vf = true; 1225 f->counter++; 1226 } 1227 } else if (is_netdev) { 1228 if (!f->is_netdev) { 1229 f->is_netdev = true; 1230 f->counter++; 1231 } 1232 } else { 1233 f->counter++; 1234 } 1235 1236 /* changed tells sync_filters_subtask to 1237 * push the filter down to the firmware 1238 */ 1239 if (f->changed) { 1240 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1241 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1242 } 1243 1244 add_filter_out: 1245 return f; 1246 } 1247 1248 /** 1249 * i40e_del_filter - Remove a mac/vlan filter from the VSI 1250 * @vsi: the VSI to be searched 1251 * @macaddr: the MAC address 1252 * @vlan: the vlan 1253 * @is_vf: make sure it's a vf filter, else doesn't matter 1254 * @is_netdev: make sure it's a netdev filter, else doesn't matter 1255 **/ 1256 void i40e_del_filter(struct i40e_vsi *vsi, 1257 u8 *macaddr, s16 vlan, 1258 bool is_vf, bool is_netdev) 1259 { 1260 struct i40e_mac_filter *f; 1261 1262 if (!vsi || !macaddr) 1263 return; 1264 1265 f = i40e_find_filter(vsi, macaddr, vlan, is_vf, is_netdev); 1266 if (!f || f->counter == 0) 1267 return; 1268 1269 if (is_vf) { 1270 if (f->is_vf) { 1271 f->is_vf = false; 1272 f->counter--; 1273 } 1274 } else if (is_netdev) { 1275 if (f->is_netdev) { 1276 f->is_netdev = false; 1277 f->counter--; 1278 } 1279 } else { 1280 /* make sure we don't remove a filter in use by vf or netdev */ 1281 int min_f = 0; 1282 min_f += (f->is_vf ? 1 : 0); 1283 min_f += (f->is_netdev ? 1 : 0); 1284 1285 if (f->counter > min_f) 1286 f->counter--; 1287 } 1288 1289 /* counter == 0 tells sync_filters_subtask to 1290 * remove the filter from the firmware's list 1291 */ 1292 if (f->counter == 0) { 1293 f->changed = true; 1294 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1295 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1296 } 1297 } 1298 1299 /** 1300 * i40e_set_mac - NDO callback to set mac address 1301 * @netdev: network interface device structure 1302 * @p: pointer to an address structure 1303 * 1304 * Returns 0 on success, negative on failure 1305 **/ 1306 static int i40e_set_mac(struct net_device *netdev, void *p) 1307 { 1308 struct i40e_netdev_priv *np = netdev_priv(netdev); 1309 struct i40e_vsi *vsi = np->vsi; 1310 struct sockaddr *addr = p; 1311 struct i40e_mac_filter *f; 1312 1313 if (!is_valid_ether_addr(addr->sa_data)) 1314 return -EADDRNOTAVAIL; 1315 1316 netdev_info(netdev, "set mac address=%pM\n", addr->sa_data); 1317 1318 if (ether_addr_equal(netdev->dev_addr, addr->sa_data)) 1319 return 0; 1320 1321 if (test_bit(__I40E_DOWN, &vsi->back->state) || 1322 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 1323 return -EADDRNOTAVAIL; 1324 1325 if (vsi->type == I40E_VSI_MAIN) { 1326 i40e_status ret; 1327 ret = i40e_aq_mac_address_write(&vsi->back->hw, 1328 I40E_AQC_WRITE_TYPE_LAA_ONLY, 1329 addr->sa_data, NULL); 1330 if (ret) { 1331 netdev_info(netdev, 1332 "Addr change for Main VSI failed: %d\n", 1333 ret); 1334 return -EADDRNOTAVAIL; 1335 } 1336 1337 ether_addr_copy(vsi->back->hw.mac.addr, addr->sa_data); 1338 } 1339 1340 /* In order to be sure to not drop any packets, add the new address 1341 * then delete the old one. 1342 */ 1343 f = i40e_add_filter(vsi, addr->sa_data, I40E_VLAN_ANY, false, false); 1344 if (!f) 1345 return -ENOMEM; 1346 1347 i40e_sync_vsi_filters(vsi); 1348 i40e_del_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY, false, false); 1349 i40e_sync_vsi_filters(vsi); 1350 1351 ether_addr_copy(netdev->dev_addr, addr->sa_data); 1352 1353 return 0; 1354 } 1355 1356 /** 1357 * i40e_vsi_setup_queue_map - Setup a VSI queue map based on enabled_tc 1358 * @vsi: the VSI being setup 1359 * @ctxt: VSI context structure 1360 * @enabled_tc: Enabled TCs bitmap 1361 * @is_add: True if called before Add VSI 1362 * 1363 * Setup VSI queue mapping for enabled traffic classes. 1364 **/ 1365 static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, 1366 struct i40e_vsi_context *ctxt, 1367 u8 enabled_tc, 1368 bool is_add) 1369 { 1370 struct i40e_pf *pf = vsi->back; 1371 u16 sections = 0; 1372 u8 netdev_tc = 0; 1373 u16 numtc = 0; 1374 u16 qcount; 1375 u8 offset; 1376 u16 qmap; 1377 int i; 1378 u16 num_tc_qps = 0; 1379 1380 sections = I40E_AQ_VSI_PROP_QUEUE_MAP_VALID; 1381 offset = 0; 1382 1383 if (enabled_tc && (vsi->back->flags & I40E_FLAG_DCB_ENABLED)) { 1384 /* Find numtc from enabled TC bitmap */ 1385 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1386 if (enabled_tc & (1 << i)) /* TC is enabled */ 1387 numtc++; 1388 } 1389 if (!numtc) { 1390 dev_warn(&pf->pdev->dev, "DCB is enabled but no TC enabled, forcing TC0\n"); 1391 numtc = 1; 1392 } 1393 } else { 1394 /* At least TC0 is enabled in case of non-DCB case */ 1395 numtc = 1; 1396 } 1397 1398 vsi->tc_config.numtc = numtc; 1399 vsi->tc_config.enabled_tc = enabled_tc ? enabled_tc : 1; 1400 /* Number of queues per enabled TC */ 1401 num_tc_qps = vsi->alloc_queue_pairs/numtc; 1402 num_tc_qps = min_t(int, num_tc_qps, I40E_MAX_QUEUES_PER_TC); 1403 1404 /* Setup queue offset/count for all TCs for given VSI */ 1405 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 1406 /* See if the given TC is enabled for the given VSI */ 1407 if (vsi->tc_config.enabled_tc & (1 << i)) { /* TC is enabled */ 1408 int pow, num_qps; 1409 1410 switch (vsi->type) { 1411 case I40E_VSI_MAIN: 1412 qcount = min_t(int, pf->rss_size, num_tc_qps); 1413 break; 1414 case I40E_VSI_FDIR: 1415 case I40E_VSI_SRIOV: 1416 case I40E_VSI_VMDQ2: 1417 default: 1418 qcount = num_tc_qps; 1419 WARN_ON(i != 0); 1420 break; 1421 } 1422 vsi->tc_config.tc_info[i].qoffset = offset; 1423 vsi->tc_config.tc_info[i].qcount = qcount; 1424 1425 /* find the power-of-2 of the number of queue pairs */ 1426 num_qps = qcount; 1427 pow = 0; 1428 while (num_qps && ((1 << pow) < qcount)) { 1429 pow++; 1430 num_qps >>= 1; 1431 } 1432 1433 vsi->tc_config.tc_info[i].netdev_tc = netdev_tc++; 1434 qmap = 1435 (offset << I40E_AQ_VSI_TC_QUE_OFFSET_SHIFT) | 1436 (pow << I40E_AQ_VSI_TC_QUE_NUMBER_SHIFT); 1437 1438 offset += qcount; 1439 } else { 1440 /* TC is not enabled so set the offset to 1441 * default queue and allocate one queue 1442 * for the given TC. 1443 */ 1444 vsi->tc_config.tc_info[i].qoffset = 0; 1445 vsi->tc_config.tc_info[i].qcount = 1; 1446 vsi->tc_config.tc_info[i].netdev_tc = 0; 1447 1448 qmap = 0; 1449 } 1450 ctxt->info.tc_mapping[i] = cpu_to_le16(qmap); 1451 } 1452 1453 /* Set actual Tx/Rx queue pairs */ 1454 vsi->num_queue_pairs = offset; 1455 1456 /* Scheduler section valid can only be set for ADD VSI */ 1457 if (is_add) { 1458 sections |= I40E_AQ_VSI_PROP_SCHED_VALID; 1459 1460 ctxt->info.up_enable_bits = enabled_tc; 1461 } 1462 if (vsi->type == I40E_VSI_SRIOV) { 1463 ctxt->info.mapping_flags |= 1464 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_NONCONTIG); 1465 for (i = 0; i < vsi->num_queue_pairs; i++) 1466 ctxt->info.queue_mapping[i] = 1467 cpu_to_le16(vsi->base_queue + i); 1468 } else { 1469 ctxt->info.mapping_flags |= 1470 cpu_to_le16(I40E_AQ_VSI_QUE_MAP_CONTIG); 1471 ctxt->info.queue_mapping[0] = cpu_to_le16(vsi->base_queue); 1472 } 1473 ctxt->info.valid_sections |= cpu_to_le16(sections); 1474 } 1475 1476 /** 1477 * i40e_set_rx_mode - NDO callback to set the netdev filters 1478 * @netdev: network interface device structure 1479 **/ 1480 static void i40e_set_rx_mode(struct net_device *netdev) 1481 { 1482 struct i40e_netdev_priv *np = netdev_priv(netdev); 1483 struct i40e_mac_filter *f, *ftmp; 1484 struct i40e_vsi *vsi = np->vsi; 1485 struct netdev_hw_addr *uca; 1486 struct netdev_hw_addr *mca; 1487 struct netdev_hw_addr *ha; 1488 1489 /* add addr if not already in the filter list */ 1490 netdev_for_each_uc_addr(uca, netdev) { 1491 if (!i40e_find_mac(vsi, uca->addr, false, true)) { 1492 if (i40e_is_vsi_in_vlan(vsi)) 1493 i40e_put_mac_in_vlan(vsi, uca->addr, 1494 false, true); 1495 else 1496 i40e_add_filter(vsi, uca->addr, I40E_VLAN_ANY, 1497 false, true); 1498 } 1499 } 1500 1501 netdev_for_each_mc_addr(mca, netdev) { 1502 if (!i40e_find_mac(vsi, mca->addr, false, true)) { 1503 if (i40e_is_vsi_in_vlan(vsi)) 1504 i40e_put_mac_in_vlan(vsi, mca->addr, 1505 false, true); 1506 else 1507 i40e_add_filter(vsi, mca->addr, I40E_VLAN_ANY, 1508 false, true); 1509 } 1510 } 1511 1512 /* remove filter if not in netdev list */ 1513 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) { 1514 bool found = false; 1515 1516 if (!f->is_netdev) 1517 continue; 1518 1519 if (is_multicast_ether_addr(f->macaddr)) { 1520 netdev_for_each_mc_addr(mca, netdev) { 1521 if (ether_addr_equal(mca->addr, f->macaddr)) { 1522 found = true; 1523 break; 1524 } 1525 } 1526 } else { 1527 netdev_for_each_uc_addr(uca, netdev) { 1528 if (ether_addr_equal(uca->addr, f->macaddr)) { 1529 found = true; 1530 break; 1531 } 1532 } 1533 1534 for_each_dev_addr(netdev, ha) { 1535 if (ether_addr_equal(ha->addr, f->macaddr)) { 1536 found = true; 1537 break; 1538 } 1539 } 1540 } 1541 if (!found) 1542 i40e_del_filter( 1543 vsi, f->macaddr, I40E_VLAN_ANY, false, true); 1544 } 1545 1546 /* check for other flag changes */ 1547 if (vsi->current_netdev_flags != vsi->netdev->flags) { 1548 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 1549 vsi->back->flags |= I40E_FLAG_FILTER_SYNC; 1550 } 1551 } 1552 1553 /** 1554 * i40e_sync_vsi_filters - Update the VSI filter list to the HW 1555 * @vsi: ptr to the VSI 1556 * 1557 * Push any outstanding VSI filter changes through the AdminQ. 1558 * 1559 * Returns 0 or error value 1560 **/ 1561 int i40e_sync_vsi_filters(struct i40e_vsi *vsi) 1562 { 1563 struct i40e_mac_filter *f, *ftmp; 1564 bool promisc_forced_on = false; 1565 bool add_happened = false; 1566 int filter_list_len = 0; 1567 u32 changed_flags = 0; 1568 i40e_status aq_ret = 0; 1569 struct i40e_pf *pf; 1570 int num_add = 0; 1571 int num_del = 0; 1572 u16 cmd_flags; 1573 1574 /* empty array typed pointers, kcalloc later */ 1575 struct i40e_aqc_add_macvlan_element_data *add_list; 1576 struct i40e_aqc_remove_macvlan_element_data *del_list; 1577 1578 while (test_and_set_bit(__I40E_CONFIG_BUSY, &vsi->state)) 1579 usleep_range(1000, 2000); 1580 pf = vsi->back; 1581 1582 if (vsi->netdev) { 1583 changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags; 1584 vsi->current_netdev_flags = vsi->netdev->flags; 1585 } 1586 1587 if (vsi->flags & I40E_VSI_FLAG_FILTER_CHANGED) { 1588 vsi->flags &= ~I40E_VSI_FLAG_FILTER_CHANGED; 1589 1590 filter_list_len = pf->hw.aq.asq_buf_size / 1591 sizeof(struct i40e_aqc_remove_macvlan_element_data); 1592 del_list = kcalloc(filter_list_len, 1593 sizeof(struct i40e_aqc_remove_macvlan_element_data), 1594 GFP_KERNEL); 1595 if (!del_list) 1596 return -ENOMEM; 1597 1598 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) { 1599 if (!f->changed) 1600 continue; 1601 1602 if (f->counter != 0) 1603 continue; 1604 f->changed = false; 1605 cmd_flags = 0; 1606 1607 /* add to delete list */ 1608 ether_addr_copy(del_list[num_del].mac_addr, f->macaddr); 1609 del_list[num_del].vlan_tag = 1610 cpu_to_le16((u16)(f->vlan == 1611 I40E_VLAN_ANY ? 0 : f->vlan)); 1612 1613 cmd_flags |= I40E_AQC_MACVLAN_DEL_PERFECT_MATCH; 1614 del_list[num_del].flags = cmd_flags; 1615 num_del++; 1616 1617 /* unlink from filter list */ 1618 list_del(&f->list); 1619 kfree(f); 1620 1621 /* flush a full buffer */ 1622 if (num_del == filter_list_len) { 1623 aq_ret = i40e_aq_remove_macvlan(&pf->hw, 1624 vsi->seid, del_list, num_del, 1625 NULL); 1626 num_del = 0; 1627 memset(del_list, 0, sizeof(*del_list)); 1628 1629 if (aq_ret && 1630 pf->hw.aq.asq_last_status != 1631 I40E_AQ_RC_ENOENT) 1632 dev_info(&pf->pdev->dev, 1633 "ignoring delete macvlan error, err %d, aq_err %d while flushing a full buffer\n", 1634 aq_ret, 1635 pf->hw.aq.asq_last_status); 1636 } 1637 } 1638 if (num_del) { 1639 aq_ret = i40e_aq_remove_macvlan(&pf->hw, vsi->seid, 1640 del_list, num_del, NULL); 1641 num_del = 0; 1642 1643 if (aq_ret && 1644 pf->hw.aq.asq_last_status != I40E_AQ_RC_ENOENT) 1645 dev_info(&pf->pdev->dev, 1646 "ignoring delete macvlan error, err %d, aq_err %d\n", 1647 aq_ret, pf->hw.aq.asq_last_status); 1648 } 1649 1650 kfree(del_list); 1651 del_list = NULL; 1652 1653 /* do all the adds now */ 1654 filter_list_len = pf->hw.aq.asq_buf_size / 1655 sizeof(struct i40e_aqc_add_macvlan_element_data), 1656 add_list = kcalloc(filter_list_len, 1657 sizeof(struct i40e_aqc_add_macvlan_element_data), 1658 GFP_KERNEL); 1659 if (!add_list) 1660 return -ENOMEM; 1661 1662 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) { 1663 if (!f->changed) 1664 continue; 1665 1666 if (f->counter == 0) 1667 continue; 1668 f->changed = false; 1669 add_happened = true; 1670 cmd_flags = 0; 1671 1672 /* add to add array */ 1673 ether_addr_copy(add_list[num_add].mac_addr, f->macaddr); 1674 add_list[num_add].vlan_tag = 1675 cpu_to_le16( 1676 (u16)(f->vlan == I40E_VLAN_ANY ? 0 : f->vlan)); 1677 add_list[num_add].queue_number = 0; 1678 1679 cmd_flags |= I40E_AQC_MACVLAN_ADD_PERFECT_MATCH; 1680 add_list[num_add].flags = cpu_to_le16(cmd_flags); 1681 num_add++; 1682 1683 /* flush a full buffer */ 1684 if (num_add == filter_list_len) { 1685 aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid, 1686 add_list, num_add, 1687 NULL); 1688 num_add = 0; 1689 1690 if (aq_ret) 1691 break; 1692 memset(add_list, 0, sizeof(*add_list)); 1693 } 1694 } 1695 if (num_add) { 1696 aq_ret = i40e_aq_add_macvlan(&pf->hw, vsi->seid, 1697 add_list, num_add, NULL); 1698 num_add = 0; 1699 } 1700 kfree(add_list); 1701 add_list = NULL; 1702 1703 if (add_happened && (!aq_ret)) { 1704 /* do nothing */; 1705 } else if (add_happened && (aq_ret)) { 1706 dev_info(&pf->pdev->dev, 1707 "add filter failed, err %d, aq_err %d\n", 1708 aq_ret, pf->hw.aq.asq_last_status); 1709 if ((pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOSPC) && 1710 !test_bit(__I40E_FILTER_OVERFLOW_PROMISC, 1711 &vsi->state)) { 1712 promisc_forced_on = true; 1713 set_bit(__I40E_FILTER_OVERFLOW_PROMISC, 1714 &vsi->state); 1715 dev_info(&pf->pdev->dev, "promiscuous mode forced on\n"); 1716 } 1717 } 1718 } 1719 1720 /* check for changes in promiscuous modes */ 1721 if (changed_flags & IFF_ALLMULTI) { 1722 bool cur_multipromisc; 1723 cur_multipromisc = !!(vsi->current_netdev_flags & IFF_ALLMULTI); 1724 aq_ret = i40e_aq_set_vsi_multicast_promiscuous(&vsi->back->hw, 1725 vsi->seid, 1726 cur_multipromisc, 1727 NULL); 1728 if (aq_ret) 1729 dev_info(&pf->pdev->dev, 1730 "set multi promisc failed, err %d, aq_err %d\n", 1731 aq_ret, pf->hw.aq.asq_last_status); 1732 } 1733 if ((changed_flags & IFF_PROMISC) || promisc_forced_on) { 1734 bool cur_promisc; 1735 cur_promisc = (!!(vsi->current_netdev_flags & IFF_PROMISC) || 1736 test_bit(__I40E_FILTER_OVERFLOW_PROMISC, 1737 &vsi->state)); 1738 aq_ret = i40e_aq_set_vsi_unicast_promiscuous(&vsi->back->hw, 1739 vsi->seid, 1740 cur_promisc, NULL); 1741 if (aq_ret) 1742 dev_info(&pf->pdev->dev, 1743 "set uni promisc failed, err %d, aq_err %d\n", 1744 aq_ret, pf->hw.aq.asq_last_status); 1745 aq_ret = i40e_aq_set_vsi_broadcast(&vsi->back->hw, 1746 vsi->seid, 1747 cur_promisc, NULL); 1748 if (aq_ret) 1749 dev_info(&pf->pdev->dev, 1750 "set brdcast promisc failed, err %d, aq_err %d\n", 1751 aq_ret, pf->hw.aq.asq_last_status); 1752 } 1753 1754 clear_bit(__I40E_CONFIG_BUSY, &vsi->state); 1755 return 0; 1756 } 1757 1758 /** 1759 * i40e_sync_filters_subtask - Sync the VSI filter list with HW 1760 * @pf: board private structure 1761 **/ 1762 static void i40e_sync_filters_subtask(struct i40e_pf *pf) 1763 { 1764 int v; 1765 1766 if (!pf || !(pf->flags & I40E_FLAG_FILTER_SYNC)) 1767 return; 1768 pf->flags &= ~I40E_FLAG_FILTER_SYNC; 1769 1770 for (v = 0; v < pf->num_alloc_vsi; v++) { 1771 if (pf->vsi[v] && 1772 (pf->vsi[v]->flags & I40E_VSI_FLAG_FILTER_CHANGED)) 1773 i40e_sync_vsi_filters(pf->vsi[v]); 1774 } 1775 } 1776 1777 /** 1778 * i40e_change_mtu - NDO callback to change the Maximum Transfer Unit 1779 * @netdev: network interface device structure 1780 * @new_mtu: new value for maximum frame size 1781 * 1782 * Returns 0 on success, negative on failure 1783 **/ 1784 static int i40e_change_mtu(struct net_device *netdev, int new_mtu) 1785 { 1786 struct i40e_netdev_priv *np = netdev_priv(netdev); 1787 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 1788 struct i40e_vsi *vsi = np->vsi; 1789 1790 /* MTU < 68 is an error and causes problems on some kernels */ 1791 if ((new_mtu < 68) || (max_frame > I40E_MAX_RXBUFFER)) 1792 return -EINVAL; 1793 1794 netdev_info(netdev, "changing MTU from %d to %d\n", 1795 netdev->mtu, new_mtu); 1796 netdev->mtu = new_mtu; 1797 if (netif_running(netdev)) 1798 i40e_vsi_reinit_locked(vsi); 1799 1800 return 0; 1801 } 1802 1803 /** 1804 * i40e_ioctl - Access the hwtstamp interface 1805 * @netdev: network interface device structure 1806 * @ifr: interface request data 1807 * @cmd: ioctl command 1808 **/ 1809 int i40e_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) 1810 { 1811 struct i40e_netdev_priv *np = netdev_priv(netdev); 1812 struct i40e_pf *pf = np->vsi->back; 1813 1814 switch (cmd) { 1815 case SIOCGHWTSTAMP: 1816 return i40e_ptp_get_ts_config(pf, ifr); 1817 case SIOCSHWTSTAMP: 1818 return i40e_ptp_set_ts_config(pf, ifr); 1819 default: 1820 return -EOPNOTSUPP; 1821 } 1822 } 1823 1824 /** 1825 * i40e_vlan_stripping_enable - Turn on vlan stripping for the VSI 1826 * @vsi: the vsi being adjusted 1827 **/ 1828 void i40e_vlan_stripping_enable(struct i40e_vsi *vsi) 1829 { 1830 struct i40e_vsi_context ctxt; 1831 i40e_status ret; 1832 1833 if ((vsi->info.valid_sections & 1834 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 1835 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_MODE_MASK) == 0)) 1836 return; /* already enabled */ 1837 1838 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 1839 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 1840 I40E_AQ_VSI_PVLAN_EMOD_STR_BOTH; 1841 1842 ctxt.seid = vsi->seid; 1843 memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); 1844 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 1845 if (ret) { 1846 dev_info(&vsi->back->pdev->dev, 1847 "%s: update vsi failed, aq_err=%d\n", 1848 __func__, vsi->back->hw.aq.asq_last_status); 1849 } 1850 } 1851 1852 /** 1853 * i40e_vlan_stripping_disable - Turn off vlan stripping for the VSI 1854 * @vsi: the vsi being adjusted 1855 **/ 1856 void i40e_vlan_stripping_disable(struct i40e_vsi *vsi) 1857 { 1858 struct i40e_vsi_context ctxt; 1859 i40e_status ret; 1860 1861 if ((vsi->info.valid_sections & 1862 cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID)) && 1863 ((vsi->info.port_vlan_flags & I40E_AQ_VSI_PVLAN_EMOD_MASK) == 1864 I40E_AQ_VSI_PVLAN_EMOD_MASK)) 1865 return; /* already disabled */ 1866 1867 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 1868 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_ALL | 1869 I40E_AQ_VSI_PVLAN_EMOD_NOTHING; 1870 1871 ctxt.seid = vsi->seid; 1872 memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); 1873 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 1874 if (ret) { 1875 dev_info(&vsi->back->pdev->dev, 1876 "%s: update vsi failed, aq_err=%d\n", 1877 __func__, vsi->back->hw.aq.asq_last_status); 1878 } 1879 } 1880 1881 /** 1882 * i40e_vlan_rx_register - Setup or shutdown vlan offload 1883 * @netdev: network interface to be adjusted 1884 * @features: netdev features to test if VLAN offload is enabled or not 1885 **/ 1886 static void i40e_vlan_rx_register(struct net_device *netdev, u32 features) 1887 { 1888 struct i40e_netdev_priv *np = netdev_priv(netdev); 1889 struct i40e_vsi *vsi = np->vsi; 1890 1891 if (features & NETIF_F_HW_VLAN_CTAG_RX) 1892 i40e_vlan_stripping_enable(vsi); 1893 else 1894 i40e_vlan_stripping_disable(vsi); 1895 } 1896 1897 /** 1898 * i40e_vsi_add_vlan - Add vsi membership for given vlan 1899 * @vsi: the vsi being configured 1900 * @vid: vlan id to be added (0 = untagged only , -1 = any) 1901 **/ 1902 int i40e_vsi_add_vlan(struct i40e_vsi *vsi, s16 vid) 1903 { 1904 struct i40e_mac_filter *f, *add_f; 1905 bool is_netdev, is_vf; 1906 1907 is_vf = (vsi->type == I40E_VSI_SRIOV); 1908 is_netdev = !!(vsi->netdev); 1909 1910 if (is_netdev) { 1911 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, vid, 1912 is_vf, is_netdev); 1913 if (!add_f) { 1914 dev_info(&vsi->back->pdev->dev, 1915 "Could not add vlan filter %d for %pM\n", 1916 vid, vsi->netdev->dev_addr); 1917 return -ENOMEM; 1918 } 1919 } 1920 1921 list_for_each_entry(f, &vsi->mac_filter_list, list) { 1922 add_f = i40e_add_filter(vsi, f->macaddr, vid, is_vf, is_netdev); 1923 if (!add_f) { 1924 dev_info(&vsi->back->pdev->dev, 1925 "Could not add vlan filter %d for %pM\n", 1926 vid, f->macaddr); 1927 return -ENOMEM; 1928 } 1929 } 1930 1931 /* Now if we add a vlan tag, make sure to check if it is the first 1932 * tag (i.e. a "tag" -1 does exist) and if so replace the -1 "tag" 1933 * with 0, so we now accept untagged and specified tagged traffic 1934 * (and not any taged and untagged) 1935 */ 1936 if (vid > 0) { 1937 if (is_netdev && i40e_find_filter(vsi, vsi->netdev->dev_addr, 1938 I40E_VLAN_ANY, 1939 is_vf, is_netdev)) { 1940 i40e_del_filter(vsi, vsi->netdev->dev_addr, 1941 I40E_VLAN_ANY, is_vf, is_netdev); 1942 add_f = i40e_add_filter(vsi, vsi->netdev->dev_addr, 0, 1943 is_vf, is_netdev); 1944 if (!add_f) { 1945 dev_info(&vsi->back->pdev->dev, 1946 "Could not add filter 0 for %pM\n", 1947 vsi->netdev->dev_addr); 1948 return -ENOMEM; 1949 } 1950 } 1951 } 1952 1953 /* Do not assume that I40E_VLAN_ANY should be reset to VLAN 0 */ 1954 if (vid > 0 && !vsi->info.pvid) { 1955 list_for_each_entry(f, &vsi->mac_filter_list, list) { 1956 if (i40e_find_filter(vsi, f->macaddr, I40E_VLAN_ANY, 1957 is_vf, is_netdev)) { 1958 i40e_del_filter(vsi, f->macaddr, I40E_VLAN_ANY, 1959 is_vf, is_netdev); 1960 add_f = i40e_add_filter(vsi, f->macaddr, 1961 0, is_vf, is_netdev); 1962 if (!add_f) { 1963 dev_info(&vsi->back->pdev->dev, 1964 "Could not add filter 0 for %pM\n", 1965 f->macaddr); 1966 return -ENOMEM; 1967 } 1968 } 1969 } 1970 } 1971 1972 if (test_bit(__I40E_DOWN, &vsi->back->state) || 1973 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 1974 return 0; 1975 1976 return i40e_sync_vsi_filters(vsi); 1977 } 1978 1979 /** 1980 * i40e_vsi_kill_vlan - Remove vsi membership for given vlan 1981 * @vsi: the vsi being configured 1982 * @vid: vlan id to be removed (0 = untagged only , -1 = any) 1983 * 1984 * Return: 0 on success or negative otherwise 1985 **/ 1986 int i40e_vsi_kill_vlan(struct i40e_vsi *vsi, s16 vid) 1987 { 1988 struct net_device *netdev = vsi->netdev; 1989 struct i40e_mac_filter *f, *add_f; 1990 bool is_vf, is_netdev; 1991 int filter_count = 0; 1992 1993 is_vf = (vsi->type == I40E_VSI_SRIOV); 1994 is_netdev = !!(netdev); 1995 1996 if (is_netdev) 1997 i40e_del_filter(vsi, netdev->dev_addr, vid, is_vf, is_netdev); 1998 1999 list_for_each_entry(f, &vsi->mac_filter_list, list) 2000 i40e_del_filter(vsi, f->macaddr, vid, is_vf, is_netdev); 2001 2002 /* go through all the filters for this VSI and if there is only 2003 * vid == 0 it means there are no other filters, so vid 0 must 2004 * be replaced with -1. This signifies that we should from now 2005 * on accept any traffic (with any tag present, or untagged) 2006 */ 2007 list_for_each_entry(f, &vsi->mac_filter_list, list) { 2008 if (is_netdev) { 2009 if (f->vlan && 2010 ether_addr_equal(netdev->dev_addr, f->macaddr)) 2011 filter_count++; 2012 } 2013 2014 if (f->vlan) 2015 filter_count++; 2016 } 2017 2018 if (!filter_count && is_netdev) { 2019 i40e_del_filter(vsi, netdev->dev_addr, 0, is_vf, is_netdev); 2020 f = i40e_add_filter(vsi, netdev->dev_addr, I40E_VLAN_ANY, 2021 is_vf, is_netdev); 2022 if (!f) { 2023 dev_info(&vsi->back->pdev->dev, 2024 "Could not add filter %d for %pM\n", 2025 I40E_VLAN_ANY, netdev->dev_addr); 2026 return -ENOMEM; 2027 } 2028 } 2029 2030 if (!filter_count) { 2031 list_for_each_entry(f, &vsi->mac_filter_list, list) { 2032 i40e_del_filter(vsi, f->macaddr, 0, is_vf, is_netdev); 2033 add_f = i40e_add_filter(vsi, f->macaddr, I40E_VLAN_ANY, 2034 is_vf, is_netdev); 2035 if (!add_f) { 2036 dev_info(&vsi->back->pdev->dev, 2037 "Could not add filter %d for %pM\n", 2038 I40E_VLAN_ANY, f->macaddr); 2039 return -ENOMEM; 2040 } 2041 } 2042 } 2043 2044 if (test_bit(__I40E_DOWN, &vsi->back->state) || 2045 test_bit(__I40E_RESET_RECOVERY_PENDING, &vsi->back->state)) 2046 return 0; 2047 2048 return i40e_sync_vsi_filters(vsi); 2049 } 2050 2051 /** 2052 * i40e_vlan_rx_add_vid - Add a vlan id filter to HW offload 2053 * @netdev: network interface to be adjusted 2054 * @vid: vlan id to be added 2055 * 2056 * net_device_ops implementation for adding vlan ids 2057 **/ 2058 static int i40e_vlan_rx_add_vid(struct net_device *netdev, 2059 __always_unused __be16 proto, u16 vid) 2060 { 2061 struct i40e_netdev_priv *np = netdev_priv(netdev); 2062 struct i40e_vsi *vsi = np->vsi; 2063 int ret = 0; 2064 2065 if (vid > 4095) 2066 return -EINVAL; 2067 2068 netdev_info(netdev, "adding %pM vid=%d\n", netdev->dev_addr, vid); 2069 2070 /* If the network stack called us with vid = 0 then 2071 * it is asking to receive priority tagged packets with 2072 * vlan id 0. Our HW receives them by default when configured 2073 * to receive untagged packets so there is no need to add an 2074 * extra filter for vlan 0 tagged packets. 2075 */ 2076 if (vid) 2077 ret = i40e_vsi_add_vlan(vsi, vid); 2078 2079 if (!ret && (vid < VLAN_N_VID)) 2080 set_bit(vid, vsi->active_vlans); 2081 2082 return ret; 2083 } 2084 2085 /** 2086 * i40e_vlan_rx_kill_vid - Remove a vlan id filter from HW offload 2087 * @netdev: network interface to be adjusted 2088 * @vid: vlan id to be removed 2089 * 2090 * net_device_ops implementation for removing vlan ids 2091 **/ 2092 static int i40e_vlan_rx_kill_vid(struct net_device *netdev, 2093 __always_unused __be16 proto, u16 vid) 2094 { 2095 struct i40e_netdev_priv *np = netdev_priv(netdev); 2096 struct i40e_vsi *vsi = np->vsi; 2097 2098 netdev_info(netdev, "removing %pM vid=%d\n", netdev->dev_addr, vid); 2099 2100 /* return code is ignored as there is nothing a user 2101 * can do about failure to remove and a log message was 2102 * already printed from the other function 2103 */ 2104 i40e_vsi_kill_vlan(vsi, vid); 2105 2106 clear_bit(vid, vsi->active_vlans); 2107 2108 return 0; 2109 } 2110 2111 /** 2112 * i40e_restore_vlan - Reinstate vlans when vsi/netdev comes back up 2113 * @vsi: the vsi being brought back up 2114 **/ 2115 static void i40e_restore_vlan(struct i40e_vsi *vsi) 2116 { 2117 u16 vid; 2118 2119 if (!vsi->netdev) 2120 return; 2121 2122 i40e_vlan_rx_register(vsi->netdev, vsi->netdev->features); 2123 2124 for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) 2125 i40e_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), 2126 vid); 2127 } 2128 2129 /** 2130 * i40e_vsi_add_pvid - Add pvid for the VSI 2131 * @vsi: the vsi being adjusted 2132 * @vid: the vlan id to set as a PVID 2133 **/ 2134 int i40e_vsi_add_pvid(struct i40e_vsi *vsi, u16 vid) 2135 { 2136 struct i40e_vsi_context ctxt; 2137 i40e_status aq_ret; 2138 2139 vsi->info.valid_sections = cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 2140 vsi->info.pvid = cpu_to_le16(vid); 2141 vsi->info.port_vlan_flags = I40E_AQ_VSI_PVLAN_MODE_TAGGED | 2142 I40E_AQ_VSI_PVLAN_INSERT_PVID | 2143 I40E_AQ_VSI_PVLAN_EMOD_STR; 2144 2145 ctxt.seid = vsi->seid; 2146 memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); 2147 aq_ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 2148 if (aq_ret) { 2149 dev_info(&vsi->back->pdev->dev, 2150 "%s: update vsi failed, aq_err=%d\n", 2151 __func__, vsi->back->hw.aq.asq_last_status); 2152 return -ENOENT; 2153 } 2154 2155 return 0; 2156 } 2157 2158 /** 2159 * i40e_vsi_remove_pvid - Remove the pvid from the VSI 2160 * @vsi: the vsi being adjusted 2161 * 2162 * Just use the vlan_rx_register() service to put it back to normal 2163 **/ 2164 void i40e_vsi_remove_pvid(struct i40e_vsi *vsi) 2165 { 2166 i40e_vlan_stripping_disable(vsi); 2167 2168 vsi->info.pvid = 0; 2169 } 2170 2171 /** 2172 * i40e_vsi_setup_tx_resources - Allocate VSI Tx queue resources 2173 * @vsi: ptr to the VSI 2174 * 2175 * If this function returns with an error, then it's possible one or 2176 * more of the rings is populated (while the rest are not). It is the 2177 * callers duty to clean those orphaned rings. 2178 * 2179 * Return 0 on success, negative on failure 2180 **/ 2181 static int i40e_vsi_setup_tx_resources(struct i40e_vsi *vsi) 2182 { 2183 int i, err = 0; 2184 2185 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2186 err = i40e_setup_tx_descriptors(vsi->tx_rings[i]); 2187 2188 return err; 2189 } 2190 2191 /** 2192 * i40e_vsi_free_tx_resources - Free Tx resources for VSI queues 2193 * @vsi: ptr to the VSI 2194 * 2195 * Free VSI's transmit software resources 2196 **/ 2197 static void i40e_vsi_free_tx_resources(struct i40e_vsi *vsi) 2198 { 2199 int i; 2200 2201 if (!vsi->tx_rings) 2202 return; 2203 2204 for (i = 0; i < vsi->num_queue_pairs; i++) 2205 if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc) 2206 i40e_free_tx_resources(vsi->tx_rings[i]); 2207 } 2208 2209 /** 2210 * i40e_vsi_setup_rx_resources - Allocate VSI queues Rx resources 2211 * @vsi: ptr to the VSI 2212 * 2213 * If this function returns with an error, then it's possible one or 2214 * more of the rings is populated (while the rest are not). It is the 2215 * callers duty to clean those orphaned rings. 2216 * 2217 * Return 0 on success, negative on failure 2218 **/ 2219 static int i40e_vsi_setup_rx_resources(struct i40e_vsi *vsi) 2220 { 2221 int i, err = 0; 2222 2223 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2224 err = i40e_setup_rx_descriptors(vsi->rx_rings[i]); 2225 return err; 2226 } 2227 2228 /** 2229 * i40e_vsi_free_rx_resources - Free Rx Resources for VSI queues 2230 * @vsi: ptr to the VSI 2231 * 2232 * Free all receive software resources 2233 **/ 2234 static void i40e_vsi_free_rx_resources(struct i40e_vsi *vsi) 2235 { 2236 int i; 2237 2238 if (!vsi->rx_rings) 2239 return; 2240 2241 for (i = 0; i < vsi->num_queue_pairs; i++) 2242 if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc) 2243 i40e_free_rx_resources(vsi->rx_rings[i]); 2244 } 2245 2246 /** 2247 * i40e_configure_tx_ring - Configure a transmit ring context and rest 2248 * @ring: The Tx ring to configure 2249 * 2250 * Configure the Tx descriptor ring in the HMC context. 2251 **/ 2252 static int i40e_configure_tx_ring(struct i40e_ring *ring) 2253 { 2254 struct i40e_vsi *vsi = ring->vsi; 2255 u16 pf_q = vsi->base_queue + ring->queue_index; 2256 struct i40e_hw *hw = &vsi->back->hw; 2257 struct i40e_hmc_obj_txq tx_ctx; 2258 i40e_status err = 0; 2259 u32 qtx_ctl = 0; 2260 2261 /* some ATR related tx ring init */ 2262 if (vsi->back->flags & I40E_FLAG_FD_ATR_ENABLED) { 2263 ring->atr_sample_rate = vsi->back->atr_sample_rate; 2264 ring->atr_count = 0; 2265 } else { 2266 ring->atr_sample_rate = 0; 2267 } 2268 2269 /* initialize XPS */ 2270 if (ring->q_vector && ring->netdev && 2271 vsi->tc_config.numtc <= 1 && 2272 !test_and_set_bit(__I40E_TX_XPS_INIT_DONE, &ring->state)) 2273 netif_set_xps_queue(ring->netdev, 2274 &ring->q_vector->affinity_mask, 2275 ring->queue_index); 2276 2277 /* clear the context structure first */ 2278 memset(&tx_ctx, 0, sizeof(tx_ctx)); 2279 2280 tx_ctx.new_context = 1; 2281 tx_ctx.base = (ring->dma / 128); 2282 tx_ctx.qlen = ring->count; 2283 tx_ctx.fd_ena = !!(vsi->back->flags & (I40E_FLAG_FD_SB_ENABLED | 2284 I40E_FLAG_FD_ATR_ENABLED)); 2285 tx_ctx.timesync_ena = !!(vsi->back->flags & I40E_FLAG_PTP); 2286 /* FDIR VSI tx ring can still use RS bit and writebacks */ 2287 if (vsi->type != I40E_VSI_FDIR) 2288 tx_ctx.head_wb_ena = 1; 2289 tx_ctx.head_wb_addr = ring->dma + 2290 (ring->count * sizeof(struct i40e_tx_desc)); 2291 2292 /* As part of VSI creation/update, FW allocates certain 2293 * Tx arbitration queue sets for each TC enabled for 2294 * the VSI. The FW returns the handles to these queue 2295 * sets as part of the response buffer to Add VSI, 2296 * Update VSI, etc. AQ commands. It is expected that 2297 * these queue set handles be associated with the Tx 2298 * queues by the driver as part of the TX queue context 2299 * initialization. This has to be done regardless of 2300 * DCB as by default everything is mapped to TC0. 2301 */ 2302 tx_ctx.rdylist = le16_to_cpu(vsi->info.qs_handle[ring->dcb_tc]); 2303 tx_ctx.rdylist_act = 0; 2304 2305 /* clear the context in the HMC */ 2306 err = i40e_clear_lan_tx_queue_context(hw, pf_q); 2307 if (err) { 2308 dev_info(&vsi->back->pdev->dev, 2309 "Failed to clear LAN Tx queue context on Tx ring %d (pf_q %d), error: %d\n", 2310 ring->queue_index, pf_q, err); 2311 return -ENOMEM; 2312 } 2313 2314 /* set the context in the HMC */ 2315 err = i40e_set_lan_tx_queue_context(hw, pf_q, &tx_ctx); 2316 if (err) { 2317 dev_info(&vsi->back->pdev->dev, 2318 "Failed to set LAN Tx queue context on Tx ring %d (pf_q %d, error: %d\n", 2319 ring->queue_index, pf_q, err); 2320 return -ENOMEM; 2321 } 2322 2323 /* Now associate this queue with this PCI function */ 2324 if (vsi->type == I40E_VSI_VMDQ2) 2325 qtx_ctl = I40E_QTX_CTL_VM_QUEUE; 2326 else 2327 qtx_ctl = I40E_QTX_CTL_PF_QUEUE; 2328 qtx_ctl |= ((hw->pf_id << I40E_QTX_CTL_PF_INDX_SHIFT) & 2329 I40E_QTX_CTL_PF_INDX_MASK); 2330 wr32(hw, I40E_QTX_CTL(pf_q), qtx_ctl); 2331 i40e_flush(hw); 2332 2333 clear_bit(__I40E_HANG_CHECK_ARMED, &ring->state); 2334 2335 /* cache tail off for easier writes later */ 2336 ring->tail = hw->hw_addr + I40E_QTX_TAIL(pf_q); 2337 2338 return 0; 2339 } 2340 2341 /** 2342 * i40e_configure_rx_ring - Configure a receive ring context 2343 * @ring: The Rx ring to configure 2344 * 2345 * Configure the Rx descriptor ring in the HMC context. 2346 **/ 2347 static int i40e_configure_rx_ring(struct i40e_ring *ring) 2348 { 2349 struct i40e_vsi *vsi = ring->vsi; 2350 u32 chain_len = vsi->back->hw.func_caps.rx_buf_chain_len; 2351 u16 pf_q = vsi->base_queue + ring->queue_index; 2352 struct i40e_hw *hw = &vsi->back->hw; 2353 struct i40e_hmc_obj_rxq rx_ctx; 2354 i40e_status err = 0; 2355 2356 ring->state = 0; 2357 2358 /* clear the context structure first */ 2359 memset(&rx_ctx, 0, sizeof(rx_ctx)); 2360 2361 ring->rx_buf_len = vsi->rx_buf_len; 2362 ring->rx_hdr_len = vsi->rx_hdr_len; 2363 2364 rx_ctx.dbuff = ring->rx_buf_len >> I40E_RXQ_CTX_DBUFF_SHIFT; 2365 rx_ctx.hbuff = ring->rx_hdr_len >> I40E_RXQ_CTX_HBUFF_SHIFT; 2366 2367 rx_ctx.base = (ring->dma / 128); 2368 rx_ctx.qlen = ring->count; 2369 2370 if (vsi->back->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) { 2371 set_ring_16byte_desc_enabled(ring); 2372 rx_ctx.dsize = 0; 2373 } else { 2374 rx_ctx.dsize = 1; 2375 } 2376 2377 rx_ctx.dtype = vsi->dtype; 2378 if (vsi->dtype) { 2379 set_ring_ps_enabled(ring); 2380 rx_ctx.hsplit_0 = I40E_RX_SPLIT_L2 | 2381 I40E_RX_SPLIT_IP | 2382 I40E_RX_SPLIT_TCP_UDP | 2383 I40E_RX_SPLIT_SCTP; 2384 } else { 2385 rx_ctx.hsplit_0 = 0; 2386 } 2387 2388 rx_ctx.rxmax = min_t(u16, vsi->max_frame, 2389 (chain_len * ring->rx_buf_len)); 2390 rx_ctx.tphrdesc_ena = 1; 2391 rx_ctx.tphwdesc_ena = 1; 2392 rx_ctx.tphdata_ena = 1; 2393 rx_ctx.tphhead_ena = 1; 2394 if (hw->revision_id == 0) 2395 rx_ctx.lrxqthresh = 0; 2396 else 2397 rx_ctx.lrxqthresh = 2; 2398 rx_ctx.crcstrip = 1; 2399 rx_ctx.l2tsel = 1; 2400 rx_ctx.showiv = 1; 2401 /* set the prefena field to 1 because the manual says to */ 2402 rx_ctx.prefena = 1; 2403 2404 /* clear the context in the HMC */ 2405 err = i40e_clear_lan_rx_queue_context(hw, pf_q); 2406 if (err) { 2407 dev_info(&vsi->back->pdev->dev, 2408 "Failed to clear LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 2409 ring->queue_index, pf_q, err); 2410 return -ENOMEM; 2411 } 2412 2413 /* set the context in the HMC */ 2414 err = i40e_set_lan_rx_queue_context(hw, pf_q, &rx_ctx); 2415 if (err) { 2416 dev_info(&vsi->back->pdev->dev, 2417 "Failed to set LAN Rx queue context on Rx ring %d (pf_q %d), error: %d\n", 2418 ring->queue_index, pf_q, err); 2419 return -ENOMEM; 2420 } 2421 2422 /* cache tail for quicker writes, and clear the reg before use */ 2423 ring->tail = hw->hw_addr + I40E_QRX_TAIL(pf_q); 2424 writel(0, ring->tail); 2425 2426 i40e_alloc_rx_buffers(ring, I40E_DESC_UNUSED(ring)); 2427 2428 return 0; 2429 } 2430 2431 /** 2432 * i40e_vsi_configure_tx - Configure the VSI for Tx 2433 * @vsi: VSI structure describing this set of rings and resources 2434 * 2435 * Configure the Tx VSI for operation. 2436 **/ 2437 static int i40e_vsi_configure_tx(struct i40e_vsi *vsi) 2438 { 2439 int err = 0; 2440 u16 i; 2441 2442 for (i = 0; (i < vsi->num_queue_pairs) && !err; i++) 2443 err = i40e_configure_tx_ring(vsi->tx_rings[i]); 2444 2445 return err; 2446 } 2447 2448 /** 2449 * i40e_vsi_configure_rx - Configure the VSI for Rx 2450 * @vsi: the VSI being configured 2451 * 2452 * Configure the Rx VSI for operation. 2453 **/ 2454 static int i40e_vsi_configure_rx(struct i40e_vsi *vsi) 2455 { 2456 int err = 0; 2457 u16 i; 2458 2459 if (vsi->netdev && (vsi->netdev->mtu > ETH_DATA_LEN)) 2460 vsi->max_frame = vsi->netdev->mtu + ETH_HLEN 2461 + ETH_FCS_LEN + VLAN_HLEN; 2462 else 2463 vsi->max_frame = I40E_RXBUFFER_2048; 2464 2465 /* figure out correct receive buffer length */ 2466 switch (vsi->back->flags & (I40E_FLAG_RX_1BUF_ENABLED | 2467 I40E_FLAG_RX_PS_ENABLED)) { 2468 case I40E_FLAG_RX_1BUF_ENABLED: 2469 vsi->rx_hdr_len = 0; 2470 vsi->rx_buf_len = vsi->max_frame; 2471 vsi->dtype = I40E_RX_DTYPE_NO_SPLIT; 2472 break; 2473 case I40E_FLAG_RX_PS_ENABLED: 2474 vsi->rx_hdr_len = I40E_RX_HDR_SIZE; 2475 vsi->rx_buf_len = I40E_RXBUFFER_2048; 2476 vsi->dtype = I40E_RX_DTYPE_HEADER_SPLIT; 2477 break; 2478 default: 2479 vsi->rx_hdr_len = I40E_RX_HDR_SIZE; 2480 vsi->rx_buf_len = I40E_RXBUFFER_2048; 2481 vsi->dtype = I40E_RX_DTYPE_SPLIT_ALWAYS; 2482 break; 2483 } 2484 2485 /* round up for the chip's needs */ 2486 vsi->rx_hdr_len = ALIGN(vsi->rx_hdr_len, 2487 (1 << I40E_RXQ_CTX_HBUFF_SHIFT)); 2488 vsi->rx_buf_len = ALIGN(vsi->rx_buf_len, 2489 (1 << I40E_RXQ_CTX_DBUFF_SHIFT)); 2490 2491 /* set up individual rings */ 2492 for (i = 0; i < vsi->num_queue_pairs && !err; i++) 2493 err = i40e_configure_rx_ring(vsi->rx_rings[i]); 2494 2495 return err; 2496 } 2497 2498 /** 2499 * i40e_vsi_config_dcb_rings - Update rings to reflect DCB TC 2500 * @vsi: ptr to the VSI 2501 **/ 2502 static void i40e_vsi_config_dcb_rings(struct i40e_vsi *vsi) 2503 { 2504 struct i40e_ring *tx_ring, *rx_ring; 2505 u16 qoffset, qcount; 2506 int i, n; 2507 2508 if (!(vsi->back->flags & I40E_FLAG_DCB_ENABLED)) 2509 return; 2510 2511 for (n = 0; n < I40E_MAX_TRAFFIC_CLASS; n++) { 2512 if (!(vsi->tc_config.enabled_tc & (1 << n))) 2513 continue; 2514 2515 qoffset = vsi->tc_config.tc_info[n].qoffset; 2516 qcount = vsi->tc_config.tc_info[n].qcount; 2517 for (i = qoffset; i < (qoffset + qcount); i++) { 2518 rx_ring = vsi->rx_rings[i]; 2519 tx_ring = vsi->tx_rings[i]; 2520 rx_ring->dcb_tc = n; 2521 tx_ring->dcb_tc = n; 2522 } 2523 } 2524 } 2525 2526 /** 2527 * i40e_set_vsi_rx_mode - Call set_rx_mode on a VSI 2528 * @vsi: ptr to the VSI 2529 **/ 2530 static void i40e_set_vsi_rx_mode(struct i40e_vsi *vsi) 2531 { 2532 if (vsi->netdev) 2533 i40e_set_rx_mode(vsi->netdev); 2534 } 2535 2536 /** 2537 * i40e_fdir_filter_restore - Restore the Sideband Flow Director filters 2538 * @vsi: Pointer to the targeted VSI 2539 * 2540 * This function replays the hlist on the hw where all the SB Flow Director 2541 * filters were saved. 2542 **/ 2543 static void i40e_fdir_filter_restore(struct i40e_vsi *vsi) 2544 { 2545 struct i40e_fdir_filter *filter; 2546 struct i40e_pf *pf = vsi->back; 2547 struct hlist_node *node; 2548 2549 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 2550 return; 2551 2552 hlist_for_each_entry_safe(filter, node, 2553 &pf->fdir_filter_list, fdir_node) { 2554 i40e_add_del_fdir(vsi, filter, true); 2555 } 2556 } 2557 2558 /** 2559 * i40e_vsi_configure - Set up the VSI for action 2560 * @vsi: the VSI being configured 2561 **/ 2562 static int i40e_vsi_configure(struct i40e_vsi *vsi) 2563 { 2564 int err; 2565 2566 i40e_set_vsi_rx_mode(vsi); 2567 i40e_restore_vlan(vsi); 2568 i40e_vsi_config_dcb_rings(vsi); 2569 err = i40e_vsi_configure_tx(vsi); 2570 if (!err) 2571 err = i40e_vsi_configure_rx(vsi); 2572 2573 return err; 2574 } 2575 2576 /** 2577 * i40e_vsi_configure_msix - MSIX mode Interrupt Config in the HW 2578 * @vsi: the VSI being configured 2579 **/ 2580 static void i40e_vsi_configure_msix(struct i40e_vsi *vsi) 2581 { 2582 struct i40e_pf *pf = vsi->back; 2583 struct i40e_q_vector *q_vector; 2584 struct i40e_hw *hw = &pf->hw; 2585 u16 vector; 2586 int i, q; 2587 u32 val; 2588 u32 qp; 2589 2590 /* The interrupt indexing is offset by 1 in the PFINT_ITRn 2591 * and PFINT_LNKLSTn registers, e.g.: 2592 * PFINT_ITRn[0..n-1] gets msix-1..msix-n (qpair interrupts) 2593 */ 2594 qp = vsi->base_queue; 2595 vector = vsi->base_vector; 2596 for (i = 0; i < vsi->num_q_vectors; i++, vector++) { 2597 q_vector = vsi->q_vectors[i]; 2598 q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting); 2599 q_vector->rx.latency_range = I40E_LOW_LATENCY; 2600 wr32(hw, I40E_PFINT_ITRN(I40E_RX_ITR, vector - 1), 2601 q_vector->rx.itr); 2602 q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting); 2603 q_vector->tx.latency_range = I40E_LOW_LATENCY; 2604 wr32(hw, I40E_PFINT_ITRN(I40E_TX_ITR, vector - 1), 2605 q_vector->tx.itr); 2606 2607 /* Linked list for the queuepairs assigned to this vector */ 2608 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), qp); 2609 for (q = 0; q < q_vector->num_ringpairs; q++) { 2610 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 2611 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 2612 (vector << I40E_QINT_RQCTL_MSIX_INDX_SHIFT) | 2613 (qp << I40E_QINT_RQCTL_NEXTQ_INDX_SHIFT)| 2614 (I40E_QUEUE_TYPE_TX 2615 << I40E_QINT_RQCTL_NEXTQ_TYPE_SHIFT); 2616 2617 wr32(hw, I40E_QINT_RQCTL(qp), val); 2618 2619 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 2620 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 2621 (vector << I40E_QINT_TQCTL_MSIX_INDX_SHIFT) | 2622 ((qp+1) << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT)| 2623 (I40E_QUEUE_TYPE_RX 2624 << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 2625 2626 /* Terminate the linked list */ 2627 if (q == (q_vector->num_ringpairs - 1)) 2628 val |= (I40E_QUEUE_END_OF_LIST 2629 << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 2630 2631 wr32(hw, I40E_QINT_TQCTL(qp), val); 2632 qp++; 2633 } 2634 } 2635 2636 i40e_flush(hw); 2637 } 2638 2639 /** 2640 * i40e_enable_misc_int_causes - enable the non-queue interrupts 2641 * @hw: ptr to the hardware info 2642 **/ 2643 static void i40e_enable_misc_int_causes(struct i40e_hw *hw) 2644 { 2645 u32 val; 2646 2647 /* clear things first */ 2648 wr32(hw, I40E_PFINT_ICR0_ENA, 0); /* disable all */ 2649 rd32(hw, I40E_PFINT_ICR0); /* read to clear */ 2650 2651 val = I40E_PFINT_ICR0_ENA_ECC_ERR_MASK | 2652 I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK | 2653 I40E_PFINT_ICR0_ENA_GRST_MASK | 2654 I40E_PFINT_ICR0_ENA_PCI_EXCEPTION_MASK | 2655 I40E_PFINT_ICR0_ENA_GPIO_MASK | 2656 I40E_PFINT_ICR0_ENA_TIMESYNC_MASK | 2657 I40E_PFINT_ICR0_ENA_HMC_ERR_MASK | 2658 I40E_PFINT_ICR0_ENA_VFLR_MASK | 2659 I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 2660 2661 wr32(hw, I40E_PFINT_ICR0_ENA, val); 2662 2663 /* SW_ITR_IDX = 0, but don't change INTENA */ 2664 wr32(hw, I40E_PFINT_DYN_CTL0, I40E_PFINT_DYN_CTL0_SW_ITR_INDX_MASK | 2665 I40E_PFINT_DYN_CTL0_INTENA_MSK_MASK); 2666 2667 /* OTHER_ITR_IDX = 0 */ 2668 wr32(hw, I40E_PFINT_STAT_CTL0, 0); 2669 } 2670 2671 /** 2672 * i40e_configure_msi_and_legacy - Legacy mode interrupt config in the HW 2673 * @vsi: the VSI being configured 2674 **/ 2675 static void i40e_configure_msi_and_legacy(struct i40e_vsi *vsi) 2676 { 2677 struct i40e_q_vector *q_vector = vsi->q_vectors[0]; 2678 struct i40e_pf *pf = vsi->back; 2679 struct i40e_hw *hw = &pf->hw; 2680 u32 val; 2681 2682 /* set the ITR configuration */ 2683 q_vector->rx.itr = ITR_TO_REG(vsi->rx_itr_setting); 2684 q_vector->rx.latency_range = I40E_LOW_LATENCY; 2685 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), q_vector->rx.itr); 2686 q_vector->tx.itr = ITR_TO_REG(vsi->tx_itr_setting); 2687 q_vector->tx.latency_range = I40E_LOW_LATENCY; 2688 wr32(hw, I40E_PFINT_ITR0(I40E_TX_ITR), q_vector->tx.itr); 2689 2690 i40e_enable_misc_int_causes(hw); 2691 2692 /* FIRSTQ_INDX = 0, FIRSTQ_TYPE = 0 (rx) */ 2693 wr32(hw, I40E_PFINT_LNKLST0, 0); 2694 2695 /* Associate the queue pair to the vector and enable the queue int */ 2696 val = I40E_QINT_RQCTL_CAUSE_ENA_MASK | 2697 (I40E_RX_ITR << I40E_QINT_RQCTL_ITR_INDX_SHIFT) | 2698 (I40E_QUEUE_TYPE_TX << I40E_QINT_TQCTL_NEXTQ_TYPE_SHIFT); 2699 2700 wr32(hw, I40E_QINT_RQCTL(0), val); 2701 2702 val = I40E_QINT_TQCTL_CAUSE_ENA_MASK | 2703 (I40E_TX_ITR << I40E_QINT_TQCTL_ITR_INDX_SHIFT) | 2704 (I40E_QUEUE_END_OF_LIST << I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT); 2705 2706 wr32(hw, I40E_QINT_TQCTL(0), val); 2707 i40e_flush(hw); 2708 } 2709 2710 /** 2711 * i40e_irq_dynamic_disable_icr0 - Disable default interrupt generation for icr0 2712 * @pf: board private structure 2713 **/ 2714 void i40e_irq_dynamic_disable_icr0(struct i40e_pf *pf) 2715 { 2716 struct i40e_hw *hw = &pf->hw; 2717 2718 wr32(hw, I40E_PFINT_DYN_CTL0, 2719 I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 2720 i40e_flush(hw); 2721 } 2722 2723 /** 2724 * i40e_irq_dynamic_enable_icr0 - Enable default interrupt generation for icr0 2725 * @pf: board private structure 2726 **/ 2727 void i40e_irq_dynamic_enable_icr0(struct i40e_pf *pf) 2728 { 2729 struct i40e_hw *hw = &pf->hw; 2730 u32 val; 2731 2732 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 2733 I40E_PFINT_DYN_CTL0_CLEARPBA_MASK | 2734 (I40E_ITR_NONE << I40E_PFINT_DYN_CTL0_ITR_INDX_SHIFT); 2735 2736 wr32(hw, I40E_PFINT_DYN_CTL0, val); 2737 i40e_flush(hw); 2738 } 2739 2740 /** 2741 * i40e_irq_dynamic_enable - Enable default interrupt generation settings 2742 * @vsi: pointer to a vsi 2743 * @vector: enable a particular Hw Interrupt vector 2744 **/ 2745 void i40e_irq_dynamic_enable(struct i40e_vsi *vsi, int vector) 2746 { 2747 struct i40e_pf *pf = vsi->back; 2748 struct i40e_hw *hw = &pf->hw; 2749 u32 val; 2750 2751 val = I40E_PFINT_DYN_CTLN_INTENA_MASK | 2752 I40E_PFINT_DYN_CTLN_CLEARPBA_MASK | 2753 (I40E_ITR_NONE << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT); 2754 wr32(hw, I40E_PFINT_DYN_CTLN(vector - 1), val); 2755 /* skip the flush */ 2756 } 2757 2758 /** 2759 * i40e_msix_clean_rings - MSIX mode Interrupt Handler 2760 * @irq: interrupt number 2761 * @data: pointer to a q_vector 2762 **/ 2763 static irqreturn_t i40e_msix_clean_rings(int irq, void *data) 2764 { 2765 struct i40e_q_vector *q_vector = data; 2766 2767 if (!q_vector->tx.ring && !q_vector->rx.ring) 2768 return IRQ_HANDLED; 2769 2770 napi_schedule(&q_vector->napi); 2771 2772 return IRQ_HANDLED; 2773 } 2774 2775 /** 2776 * i40e_vsi_request_irq_msix - Initialize MSI-X interrupts 2777 * @vsi: the VSI being configured 2778 * @basename: name for the vector 2779 * 2780 * Allocates MSI-X vectors and requests interrupts from the kernel. 2781 **/ 2782 static int i40e_vsi_request_irq_msix(struct i40e_vsi *vsi, char *basename) 2783 { 2784 int q_vectors = vsi->num_q_vectors; 2785 struct i40e_pf *pf = vsi->back; 2786 int base = vsi->base_vector; 2787 int rx_int_idx = 0; 2788 int tx_int_idx = 0; 2789 int vector, err; 2790 2791 for (vector = 0; vector < q_vectors; vector++) { 2792 struct i40e_q_vector *q_vector = vsi->q_vectors[vector]; 2793 2794 if (q_vector->tx.ring && q_vector->rx.ring) { 2795 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2796 "%s-%s-%d", basename, "TxRx", rx_int_idx++); 2797 tx_int_idx++; 2798 } else if (q_vector->rx.ring) { 2799 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2800 "%s-%s-%d", basename, "rx", rx_int_idx++); 2801 } else if (q_vector->tx.ring) { 2802 snprintf(q_vector->name, sizeof(q_vector->name) - 1, 2803 "%s-%s-%d", basename, "tx", tx_int_idx++); 2804 } else { 2805 /* skip this unused q_vector */ 2806 continue; 2807 } 2808 err = request_irq(pf->msix_entries[base + vector].vector, 2809 vsi->irq_handler, 2810 0, 2811 q_vector->name, 2812 q_vector); 2813 if (err) { 2814 dev_info(&pf->pdev->dev, 2815 "%s: request_irq failed, error: %d\n", 2816 __func__, err); 2817 goto free_queue_irqs; 2818 } 2819 /* assign the mask for this irq */ 2820 irq_set_affinity_hint(pf->msix_entries[base + vector].vector, 2821 &q_vector->affinity_mask); 2822 } 2823 2824 vsi->irqs_ready = true; 2825 return 0; 2826 2827 free_queue_irqs: 2828 while (vector) { 2829 vector--; 2830 irq_set_affinity_hint(pf->msix_entries[base + vector].vector, 2831 NULL); 2832 free_irq(pf->msix_entries[base + vector].vector, 2833 &(vsi->q_vectors[vector])); 2834 } 2835 return err; 2836 } 2837 2838 /** 2839 * i40e_vsi_disable_irq - Mask off queue interrupt generation on the VSI 2840 * @vsi: the VSI being un-configured 2841 **/ 2842 static void i40e_vsi_disable_irq(struct i40e_vsi *vsi) 2843 { 2844 struct i40e_pf *pf = vsi->back; 2845 struct i40e_hw *hw = &pf->hw; 2846 int base = vsi->base_vector; 2847 int i; 2848 2849 for (i = 0; i < vsi->num_queue_pairs; i++) { 2850 wr32(hw, I40E_QINT_TQCTL(vsi->tx_rings[i]->reg_idx), 0); 2851 wr32(hw, I40E_QINT_RQCTL(vsi->rx_rings[i]->reg_idx), 0); 2852 } 2853 2854 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 2855 for (i = vsi->base_vector; 2856 i < (vsi->num_q_vectors + vsi->base_vector); i++) 2857 wr32(hw, I40E_PFINT_DYN_CTLN(i - 1), 0); 2858 2859 i40e_flush(hw); 2860 for (i = 0; i < vsi->num_q_vectors; i++) 2861 synchronize_irq(pf->msix_entries[i + base].vector); 2862 } else { 2863 /* Legacy and MSI mode - this stops all interrupt handling */ 2864 wr32(hw, I40E_PFINT_ICR0_ENA, 0); 2865 wr32(hw, I40E_PFINT_DYN_CTL0, 0); 2866 i40e_flush(hw); 2867 synchronize_irq(pf->pdev->irq); 2868 } 2869 } 2870 2871 /** 2872 * i40e_vsi_enable_irq - Enable IRQ for the given VSI 2873 * @vsi: the VSI being configured 2874 **/ 2875 static int i40e_vsi_enable_irq(struct i40e_vsi *vsi) 2876 { 2877 struct i40e_pf *pf = vsi->back; 2878 int i; 2879 2880 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 2881 for (i = vsi->base_vector; 2882 i < (vsi->num_q_vectors + vsi->base_vector); i++) 2883 i40e_irq_dynamic_enable(vsi, i); 2884 } else { 2885 i40e_irq_dynamic_enable_icr0(pf); 2886 } 2887 2888 i40e_flush(&pf->hw); 2889 return 0; 2890 } 2891 2892 /** 2893 * i40e_stop_misc_vector - Stop the vector that handles non-queue events 2894 * @pf: board private structure 2895 **/ 2896 static void i40e_stop_misc_vector(struct i40e_pf *pf) 2897 { 2898 /* Disable ICR 0 */ 2899 wr32(&pf->hw, I40E_PFINT_ICR0_ENA, 0); 2900 i40e_flush(&pf->hw); 2901 } 2902 2903 /** 2904 * i40e_intr - MSI/Legacy and non-queue interrupt handler 2905 * @irq: interrupt number 2906 * @data: pointer to a q_vector 2907 * 2908 * This is the handler used for all MSI/Legacy interrupts, and deals 2909 * with both queue and non-queue interrupts. This is also used in 2910 * MSIX mode to handle the non-queue interrupts. 2911 **/ 2912 static irqreturn_t i40e_intr(int irq, void *data) 2913 { 2914 struct i40e_pf *pf = (struct i40e_pf *)data; 2915 struct i40e_hw *hw = &pf->hw; 2916 irqreturn_t ret = IRQ_NONE; 2917 u32 icr0, icr0_remaining; 2918 u32 val, ena_mask; 2919 2920 icr0 = rd32(hw, I40E_PFINT_ICR0); 2921 ena_mask = rd32(hw, I40E_PFINT_ICR0_ENA); 2922 2923 /* if sharing a legacy IRQ, we might get called w/o an intr pending */ 2924 if ((icr0 & I40E_PFINT_ICR0_INTEVENT_MASK) == 0) 2925 goto enable_intr; 2926 2927 /* if interrupt but no bits showing, must be SWINT */ 2928 if (((icr0 & ~I40E_PFINT_ICR0_INTEVENT_MASK) == 0) || 2929 (icr0 & I40E_PFINT_ICR0_SWINT_MASK)) 2930 pf->sw_int_count++; 2931 2932 /* only q0 is used in MSI/Legacy mode, and none are used in MSIX */ 2933 if (icr0 & I40E_PFINT_ICR0_QUEUE_0_MASK) { 2934 2935 /* temporarily disable queue cause for NAPI processing */ 2936 u32 qval = rd32(hw, I40E_QINT_RQCTL(0)); 2937 qval &= ~I40E_QINT_RQCTL_CAUSE_ENA_MASK; 2938 wr32(hw, I40E_QINT_RQCTL(0), qval); 2939 2940 qval = rd32(hw, I40E_QINT_TQCTL(0)); 2941 qval &= ~I40E_QINT_TQCTL_CAUSE_ENA_MASK; 2942 wr32(hw, I40E_QINT_TQCTL(0), qval); 2943 2944 if (!test_bit(__I40E_DOWN, &pf->state)) 2945 napi_schedule(&pf->vsi[pf->lan_vsi]->q_vectors[0]->napi); 2946 } 2947 2948 if (icr0 & I40E_PFINT_ICR0_ADMINQ_MASK) { 2949 ena_mask &= ~I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 2950 set_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state); 2951 } 2952 2953 if (icr0 & I40E_PFINT_ICR0_MAL_DETECT_MASK) { 2954 ena_mask &= ~I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 2955 set_bit(__I40E_MDD_EVENT_PENDING, &pf->state); 2956 } 2957 2958 if (icr0 & I40E_PFINT_ICR0_VFLR_MASK) { 2959 ena_mask &= ~I40E_PFINT_ICR0_ENA_VFLR_MASK; 2960 set_bit(__I40E_VFLR_EVENT_PENDING, &pf->state); 2961 } 2962 2963 if (icr0 & I40E_PFINT_ICR0_GRST_MASK) { 2964 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 2965 set_bit(__I40E_RESET_INTR_RECEIVED, &pf->state); 2966 ena_mask &= ~I40E_PFINT_ICR0_ENA_GRST_MASK; 2967 val = rd32(hw, I40E_GLGEN_RSTAT); 2968 val = (val & I40E_GLGEN_RSTAT_RESET_TYPE_MASK) 2969 >> I40E_GLGEN_RSTAT_RESET_TYPE_SHIFT; 2970 if (val == I40E_RESET_CORER) { 2971 pf->corer_count++; 2972 } else if (val == I40E_RESET_GLOBR) { 2973 pf->globr_count++; 2974 } else if (val == I40E_RESET_EMPR) { 2975 pf->empr_count++; 2976 set_bit(__I40E_EMP_RESET_REQUESTED, &pf->state); 2977 } 2978 } 2979 2980 if (icr0 & I40E_PFINT_ICR0_HMC_ERR_MASK) { 2981 icr0 &= ~I40E_PFINT_ICR0_HMC_ERR_MASK; 2982 dev_info(&pf->pdev->dev, "HMC error interrupt\n"); 2983 } 2984 2985 if (icr0 & I40E_PFINT_ICR0_TIMESYNC_MASK) { 2986 u32 prttsyn_stat = rd32(hw, I40E_PRTTSYN_STAT_0); 2987 2988 if (prttsyn_stat & I40E_PRTTSYN_STAT_0_TXTIME_MASK) { 2989 icr0 &= ~I40E_PFINT_ICR0_ENA_TIMESYNC_MASK; 2990 i40e_ptp_tx_hwtstamp(pf); 2991 } 2992 } 2993 2994 /* If a critical error is pending we have no choice but to reset the 2995 * device. 2996 * Report and mask out any remaining unexpected interrupts. 2997 */ 2998 icr0_remaining = icr0 & ena_mask; 2999 if (icr0_remaining) { 3000 dev_info(&pf->pdev->dev, "unhandled interrupt icr0=0x%08x\n", 3001 icr0_remaining); 3002 if ((icr0_remaining & I40E_PFINT_ICR0_PE_CRITERR_MASK) || 3003 (icr0_remaining & I40E_PFINT_ICR0_PCI_EXCEPTION_MASK) || 3004 (icr0_remaining & I40E_PFINT_ICR0_ECC_ERR_MASK)) { 3005 dev_info(&pf->pdev->dev, "device will be reset\n"); 3006 set_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 3007 i40e_service_event_schedule(pf); 3008 } 3009 ena_mask &= ~icr0_remaining; 3010 } 3011 ret = IRQ_HANDLED; 3012 3013 enable_intr: 3014 /* re-enable interrupt causes */ 3015 wr32(hw, I40E_PFINT_ICR0_ENA, ena_mask); 3016 if (!test_bit(__I40E_DOWN, &pf->state)) { 3017 i40e_service_event_schedule(pf); 3018 i40e_irq_dynamic_enable_icr0(pf); 3019 } 3020 3021 return ret; 3022 } 3023 3024 /** 3025 * i40e_clean_fdir_tx_irq - Reclaim resources after transmit completes 3026 * @tx_ring: tx ring to clean 3027 * @budget: how many cleans we're allowed 3028 * 3029 * Returns true if there's any budget left (e.g. the clean is finished) 3030 **/ 3031 static bool i40e_clean_fdir_tx_irq(struct i40e_ring *tx_ring, int budget) 3032 { 3033 struct i40e_vsi *vsi = tx_ring->vsi; 3034 u16 i = tx_ring->next_to_clean; 3035 struct i40e_tx_buffer *tx_buf; 3036 struct i40e_tx_desc *tx_desc; 3037 3038 tx_buf = &tx_ring->tx_bi[i]; 3039 tx_desc = I40E_TX_DESC(tx_ring, i); 3040 i -= tx_ring->count; 3041 3042 do { 3043 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 3044 3045 /* if next_to_watch is not set then there is no work pending */ 3046 if (!eop_desc) 3047 break; 3048 3049 /* prevent any other reads prior to eop_desc */ 3050 read_barrier_depends(); 3051 3052 /* if the descriptor isn't done, no work yet to do */ 3053 if (!(eop_desc->cmd_type_offset_bsz & 3054 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE))) 3055 break; 3056 3057 /* clear next_to_watch to prevent false hangs */ 3058 tx_buf->next_to_watch = NULL; 3059 3060 /* unmap skb header data */ 3061 dma_unmap_single(tx_ring->dev, 3062 dma_unmap_addr(tx_buf, dma), 3063 dma_unmap_len(tx_buf, len), 3064 DMA_TO_DEVICE); 3065 3066 dma_unmap_len_set(tx_buf, len, 0); 3067 3068 3069 /* move to the next desc and buffer to clean */ 3070 tx_buf++; 3071 tx_desc++; 3072 i++; 3073 if (unlikely(!i)) { 3074 i -= tx_ring->count; 3075 tx_buf = tx_ring->tx_bi; 3076 tx_desc = I40E_TX_DESC(tx_ring, 0); 3077 } 3078 3079 /* update budget accounting */ 3080 budget--; 3081 } while (likely(budget)); 3082 3083 i += tx_ring->count; 3084 tx_ring->next_to_clean = i; 3085 3086 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 3087 i40e_irq_dynamic_enable(vsi, 3088 tx_ring->q_vector->v_idx + vsi->base_vector); 3089 } 3090 return budget > 0; 3091 } 3092 3093 /** 3094 * i40e_fdir_clean_ring - Interrupt Handler for FDIR SB ring 3095 * @irq: interrupt number 3096 * @data: pointer to a q_vector 3097 **/ 3098 static irqreturn_t i40e_fdir_clean_ring(int irq, void *data) 3099 { 3100 struct i40e_q_vector *q_vector = data; 3101 struct i40e_vsi *vsi; 3102 3103 if (!q_vector->tx.ring) 3104 return IRQ_HANDLED; 3105 3106 vsi = q_vector->tx.ring->vsi; 3107 i40e_clean_fdir_tx_irq(q_vector->tx.ring, vsi->work_limit); 3108 3109 return IRQ_HANDLED; 3110 } 3111 3112 /** 3113 * i40e_map_vector_to_qp - Assigns the queue pair to the vector 3114 * @vsi: the VSI being configured 3115 * @v_idx: vector index 3116 * @qp_idx: queue pair index 3117 **/ 3118 static void map_vector_to_qp(struct i40e_vsi *vsi, int v_idx, int qp_idx) 3119 { 3120 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 3121 struct i40e_ring *tx_ring = vsi->tx_rings[qp_idx]; 3122 struct i40e_ring *rx_ring = vsi->rx_rings[qp_idx]; 3123 3124 tx_ring->q_vector = q_vector; 3125 tx_ring->next = q_vector->tx.ring; 3126 q_vector->tx.ring = tx_ring; 3127 q_vector->tx.count++; 3128 3129 rx_ring->q_vector = q_vector; 3130 rx_ring->next = q_vector->rx.ring; 3131 q_vector->rx.ring = rx_ring; 3132 q_vector->rx.count++; 3133 } 3134 3135 /** 3136 * i40e_vsi_map_rings_to_vectors - Maps descriptor rings to vectors 3137 * @vsi: the VSI being configured 3138 * 3139 * This function maps descriptor rings to the queue-specific vectors 3140 * we were allotted through the MSI-X enabling code. Ideally, we'd have 3141 * one vector per queue pair, but on a constrained vector budget, we 3142 * group the queue pairs as "efficiently" as possible. 3143 **/ 3144 static void i40e_vsi_map_rings_to_vectors(struct i40e_vsi *vsi) 3145 { 3146 int qp_remaining = vsi->num_queue_pairs; 3147 int q_vectors = vsi->num_q_vectors; 3148 int num_ringpairs; 3149 int v_start = 0; 3150 int qp_idx = 0; 3151 3152 /* If we don't have enough vectors for a 1-to-1 mapping, we'll have to 3153 * group them so there are multiple queues per vector. 3154 */ 3155 for (; v_start < q_vectors && qp_remaining; v_start++) { 3156 struct i40e_q_vector *q_vector = vsi->q_vectors[v_start]; 3157 3158 num_ringpairs = DIV_ROUND_UP(qp_remaining, q_vectors - v_start); 3159 3160 q_vector->num_ringpairs = num_ringpairs; 3161 3162 q_vector->rx.count = 0; 3163 q_vector->tx.count = 0; 3164 q_vector->rx.ring = NULL; 3165 q_vector->tx.ring = NULL; 3166 3167 while (num_ringpairs--) { 3168 map_vector_to_qp(vsi, v_start, qp_idx); 3169 qp_idx++; 3170 qp_remaining--; 3171 } 3172 } 3173 } 3174 3175 /** 3176 * i40e_vsi_request_irq - Request IRQ from the OS 3177 * @vsi: the VSI being configured 3178 * @basename: name for the vector 3179 **/ 3180 static int i40e_vsi_request_irq(struct i40e_vsi *vsi, char *basename) 3181 { 3182 struct i40e_pf *pf = vsi->back; 3183 int err; 3184 3185 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 3186 err = i40e_vsi_request_irq_msix(vsi, basename); 3187 else if (pf->flags & I40E_FLAG_MSI_ENABLED) 3188 err = request_irq(pf->pdev->irq, i40e_intr, 0, 3189 pf->misc_int_name, pf); 3190 else 3191 err = request_irq(pf->pdev->irq, i40e_intr, IRQF_SHARED, 3192 pf->misc_int_name, pf); 3193 3194 if (err) 3195 dev_info(&pf->pdev->dev, "request_irq failed, Error %d\n", err); 3196 3197 return err; 3198 } 3199 3200 #ifdef CONFIG_NET_POLL_CONTROLLER 3201 /** 3202 * i40e_netpoll - A Polling 'interrupt'handler 3203 * @netdev: network interface device structure 3204 * 3205 * This is used by netconsole to send skbs without having to re-enable 3206 * interrupts. It's not called while the normal interrupt routine is executing. 3207 **/ 3208 static void i40e_netpoll(struct net_device *netdev) 3209 { 3210 struct i40e_netdev_priv *np = netdev_priv(netdev); 3211 struct i40e_vsi *vsi = np->vsi; 3212 struct i40e_pf *pf = vsi->back; 3213 int i; 3214 3215 /* if interface is down do nothing */ 3216 if (test_bit(__I40E_DOWN, &vsi->state)) 3217 return; 3218 3219 pf->flags |= I40E_FLAG_IN_NETPOLL; 3220 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3221 for (i = 0; i < vsi->num_q_vectors; i++) 3222 i40e_msix_clean_rings(0, vsi->q_vectors[i]); 3223 } else { 3224 i40e_intr(pf->pdev->irq, netdev); 3225 } 3226 pf->flags &= ~I40E_FLAG_IN_NETPOLL; 3227 } 3228 #endif 3229 3230 /** 3231 * i40e_vsi_control_tx - Start or stop a VSI's rings 3232 * @vsi: the VSI being configured 3233 * @enable: start or stop the rings 3234 **/ 3235 static int i40e_vsi_control_tx(struct i40e_vsi *vsi, bool enable) 3236 { 3237 struct i40e_pf *pf = vsi->back; 3238 struct i40e_hw *hw = &pf->hw; 3239 int i, j, pf_q; 3240 u32 tx_reg; 3241 3242 pf_q = vsi->base_queue; 3243 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 3244 3245 /* warn the TX unit of coming changes */ 3246 i40e_pre_tx_queue_cfg(&pf->hw, pf_q, enable); 3247 if (!enable) 3248 udelay(10); 3249 3250 for (j = 0; j < 50; j++) { 3251 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 3252 if (((tx_reg >> I40E_QTX_ENA_QENA_REQ_SHIFT) & 1) == 3253 ((tx_reg >> I40E_QTX_ENA_QENA_STAT_SHIFT) & 1)) 3254 break; 3255 usleep_range(1000, 2000); 3256 } 3257 /* Skip if the queue is already in the requested state */ 3258 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3259 continue; 3260 3261 /* turn on/off the queue */ 3262 if (enable) { 3263 wr32(hw, I40E_QTX_HEAD(pf_q), 0); 3264 tx_reg |= I40E_QTX_ENA_QENA_REQ_MASK; 3265 } else { 3266 tx_reg &= ~I40E_QTX_ENA_QENA_REQ_MASK; 3267 } 3268 3269 wr32(hw, I40E_QTX_ENA(pf_q), tx_reg); 3270 3271 /* wait for the change to finish */ 3272 for (j = 0; j < 10; j++) { 3273 tx_reg = rd32(hw, I40E_QTX_ENA(pf_q)); 3274 if (enable == !!(tx_reg & I40E_QTX_ENA_QENA_STAT_MASK)) 3275 break; 3276 3277 udelay(10); 3278 } 3279 if (j >= 10) { 3280 dev_info(&pf->pdev->dev, "Tx ring %d %sable timeout\n", 3281 pf_q, (enable ? "en" : "dis")); 3282 return -ETIMEDOUT; 3283 } 3284 } 3285 3286 if (hw->revision_id == 0) 3287 mdelay(50); 3288 3289 return 0; 3290 } 3291 3292 /** 3293 * i40e_vsi_control_rx - Start or stop a VSI's rings 3294 * @vsi: the VSI being configured 3295 * @enable: start or stop the rings 3296 **/ 3297 static int i40e_vsi_control_rx(struct i40e_vsi *vsi, bool enable) 3298 { 3299 struct i40e_pf *pf = vsi->back; 3300 struct i40e_hw *hw = &pf->hw; 3301 int i, j, pf_q; 3302 u32 rx_reg; 3303 3304 pf_q = vsi->base_queue; 3305 for (i = 0; i < vsi->num_queue_pairs; i++, pf_q++) { 3306 for (j = 0; j < 50; j++) { 3307 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 3308 if (((rx_reg >> I40E_QRX_ENA_QENA_REQ_SHIFT) & 1) == 3309 ((rx_reg >> I40E_QRX_ENA_QENA_STAT_SHIFT) & 1)) 3310 break; 3311 usleep_range(1000, 2000); 3312 } 3313 3314 /* Skip if the queue is already in the requested state */ 3315 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 3316 continue; 3317 3318 /* turn on/off the queue */ 3319 if (enable) 3320 rx_reg |= I40E_QRX_ENA_QENA_REQ_MASK; 3321 else 3322 rx_reg &= ~I40E_QRX_ENA_QENA_REQ_MASK; 3323 wr32(hw, I40E_QRX_ENA(pf_q), rx_reg); 3324 3325 /* wait for the change to finish */ 3326 for (j = 0; j < 10; j++) { 3327 rx_reg = rd32(hw, I40E_QRX_ENA(pf_q)); 3328 3329 if (enable == !!(rx_reg & I40E_QRX_ENA_QENA_STAT_MASK)) 3330 break; 3331 3332 udelay(10); 3333 } 3334 if (j >= 10) { 3335 dev_info(&pf->pdev->dev, "Rx ring %d %sable timeout\n", 3336 pf_q, (enable ? "en" : "dis")); 3337 return -ETIMEDOUT; 3338 } 3339 } 3340 3341 return 0; 3342 } 3343 3344 /** 3345 * i40e_vsi_control_rings - Start or stop a VSI's rings 3346 * @vsi: the VSI being configured 3347 * @enable: start or stop the rings 3348 **/ 3349 int i40e_vsi_control_rings(struct i40e_vsi *vsi, bool request) 3350 { 3351 int ret = 0; 3352 3353 /* do rx first for enable and last for disable */ 3354 if (request) { 3355 ret = i40e_vsi_control_rx(vsi, request); 3356 if (ret) 3357 return ret; 3358 ret = i40e_vsi_control_tx(vsi, request); 3359 } else { 3360 /* Ignore return value, we need to shutdown whatever we can */ 3361 i40e_vsi_control_tx(vsi, request); 3362 i40e_vsi_control_rx(vsi, request); 3363 } 3364 3365 return ret; 3366 } 3367 3368 /** 3369 * i40e_vsi_free_irq - Free the irq association with the OS 3370 * @vsi: the VSI being configured 3371 **/ 3372 static void i40e_vsi_free_irq(struct i40e_vsi *vsi) 3373 { 3374 struct i40e_pf *pf = vsi->back; 3375 struct i40e_hw *hw = &pf->hw; 3376 int base = vsi->base_vector; 3377 u32 val, qp; 3378 int i; 3379 3380 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3381 if (!vsi->q_vectors) 3382 return; 3383 3384 if (!vsi->irqs_ready) 3385 return; 3386 3387 vsi->irqs_ready = false; 3388 for (i = 0; i < vsi->num_q_vectors; i++) { 3389 u16 vector = i + base; 3390 3391 /* free only the irqs that were actually requested */ 3392 if (!vsi->q_vectors[i] || 3393 !vsi->q_vectors[i]->num_ringpairs) 3394 continue; 3395 3396 /* clear the affinity_mask in the IRQ descriptor */ 3397 irq_set_affinity_hint(pf->msix_entries[vector].vector, 3398 NULL); 3399 free_irq(pf->msix_entries[vector].vector, 3400 vsi->q_vectors[i]); 3401 3402 /* Tear down the interrupt queue link list 3403 * 3404 * We know that they come in pairs and always 3405 * the Rx first, then the Tx. To clear the 3406 * link list, stick the EOL value into the 3407 * next_q field of the registers. 3408 */ 3409 val = rd32(hw, I40E_PFINT_LNKLSTN(vector - 1)); 3410 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 3411 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 3412 val |= I40E_QUEUE_END_OF_LIST 3413 << I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 3414 wr32(hw, I40E_PFINT_LNKLSTN(vector - 1), val); 3415 3416 while (qp != I40E_QUEUE_END_OF_LIST) { 3417 u32 next; 3418 3419 val = rd32(hw, I40E_QINT_RQCTL(qp)); 3420 3421 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 3422 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 3423 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3424 I40E_QINT_RQCTL_INTEVENT_MASK); 3425 3426 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 3427 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 3428 3429 wr32(hw, I40E_QINT_RQCTL(qp), val); 3430 3431 val = rd32(hw, I40E_QINT_TQCTL(qp)); 3432 3433 next = (val & I40E_QINT_TQCTL_NEXTQ_INDX_MASK) 3434 >> I40E_QINT_TQCTL_NEXTQ_INDX_SHIFT; 3435 3436 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 3437 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 3438 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3439 I40E_QINT_TQCTL_INTEVENT_MASK); 3440 3441 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 3442 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 3443 3444 wr32(hw, I40E_QINT_TQCTL(qp), val); 3445 qp = next; 3446 } 3447 } 3448 } else { 3449 free_irq(pf->pdev->irq, pf); 3450 3451 val = rd32(hw, I40E_PFINT_LNKLST0); 3452 qp = (val & I40E_PFINT_LNKLSTN_FIRSTQ_INDX_MASK) 3453 >> I40E_PFINT_LNKLSTN_FIRSTQ_INDX_SHIFT; 3454 val |= I40E_QUEUE_END_OF_LIST 3455 << I40E_PFINT_LNKLST0_FIRSTQ_INDX_SHIFT; 3456 wr32(hw, I40E_PFINT_LNKLST0, val); 3457 3458 val = rd32(hw, I40E_QINT_RQCTL(qp)); 3459 val &= ~(I40E_QINT_RQCTL_MSIX_INDX_MASK | 3460 I40E_QINT_RQCTL_MSIX0_INDX_MASK | 3461 I40E_QINT_RQCTL_CAUSE_ENA_MASK | 3462 I40E_QINT_RQCTL_INTEVENT_MASK); 3463 3464 val |= (I40E_QINT_RQCTL_ITR_INDX_MASK | 3465 I40E_QINT_RQCTL_NEXTQ_INDX_MASK); 3466 3467 wr32(hw, I40E_QINT_RQCTL(qp), val); 3468 3469 val = rd32(hw, I40E_QINT_TQCTL(qp)); 3470 3471 val &= ~(I40E_QINT_TQCTL_MSIX_INDX_MASK | 3472 I40E_QINT_TQCTL_MSIX0_INDX_MASK | 3473 I40E_QINT_TQCTL_CAUSE_ENA_MASK | 3474 I40E_QINT_TQCTL_INTEVENT_MASK); 3475 3476 val |= (I40E_QINT_TQCTL_ITR_INDX_MASK | 3477 I40E_QINT_TQCTL_NEXTQ_INDX_MASK); 3478 3479 wr32(hw, I40E_QINT_TQCTL(qp), val); 3480 } 3481 } 3482 3483 /** 3484 * i40e_free_q_vector - Free memory allocated for specific interrupt vector 3485 * @vsi: the VSI being configured 3486 * @v_idx: Index of vector to be freed 3487 * 3488 * This function frees the memory allocated to the q_vector. In addition if 3489 * NAPI is enabled it will delete any references to the NAPI struct prior 3490 * to freeing the q_vector. 3491 **/ 3492 static void i40e_free_q_vector(struct i40e_vsi *vsi, int v_idx) 3493 { 3494 struct i40e_q_vector *q_vector = vsi->q_vectors[v_idx]; 3495 struct i40e_ring *ring; 3496 3497 if (!q_vector) 3498 return; 3499 3500 /* disassociate q_vector from rings */ 3501 i40e_for_each_ring(ring, q_vector->tx) 3502 ring->q_vector = NULL; 3503 3504 i40e_for_each_ring(ring, q_vector->rx) 3505 ring->q_vector = NULL; 3506 3507 /* only VSI w/ an associated netdev is set up w/ NAPI */ 3508 if (vsi->netdev) 3509 netif_napi_del(&q_vector->napi); 3510 3511 vsi->q_vectors[v_idx] = NULL; 3512 3513 kfree_rcu(q_vector, rcu); 3514 } 3515 3516 /** 3517 * i40e_vsi_free_q_vectors - Free memory allocated for interrupt vectors 3518 * @vsi: the VSI being un-configured 3519 * 3520 * This frees the memory allocated to the q_vectors and 3521 * deletes references to the NAPI struct. 3522 **/ 3523 static void i40e_vsi_free_q_vectors(struct i40e_vsi *vsi) 3524 { 3525 int v_idx; 3526 3527 for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++) 3528 i40e_free_q_vector(vsi, v_idx); 3529 } 3530 3531 /** 3532 * i40e_reset_interrupt_capability - Disable interrupt setup in OS 3533 * @pf: board private structure 3534 **/ 3535 static void i40e_reset_interrupt_capability(struct i40e_pf *pf) 3536 { 3537 /* If we're in Legacy mode, the interrupt was cleaned in vsi_close */ 3538 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 3539 pci_disable_msix(pf->pdev); 3540 kfree(pf->msix_entries); 3541 pf->msix_entries = NULL; 3542 } else if (pf->flags & I40E_FLAG_MSI_ENABLED) { 3543 pci_disable_msi(pf->pdev); 3544 } 3545 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED); 3546 } 3547 3548 /** 3549 * i40e_clear_interrupt_scheme - Clear the current interrupt scheme settings 3550 * @pf: board private structure 3551 * 3552 * We go through and clear interrupt specific resources and reset the structure 3553 * to pre-load conditions 3554 **/ 3555 static void i40e_clear_interrupt_scheme(struct i40e_pf *pf) 3556 { 3557 int i; 3558 3559 i40e_put_lump(pf->irq_pile, 0, I40E_PILE_VALID_BIT-1); 3560 for (i = 0; i < pf->num_alloc_vsi; i++) 3561 if (pf->vsi[i]) 3562 i40e_vsi_free_q_vectors(pf->vsi[i]); 3563 i40e_reset_interrupt_capability(pf); 3564 } 3565 3566 /** 3567 * i40e_napi_enable_all - Enable NAPI for all q_vectors in the VSI 3568 * @vsi: the VSI being configured 3569 **/ 3570 static void i40e_napi_enable_all(struct i40e_vsi *vsi) 3571 { 3572 int q_idx; 3573 3574 if (!vsi->netdev) 3575 return; 3576 3577 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) 3578 napi_enable(&vsi->q_vectors[q_idx]->napi); 3579 } 3580 3581 /** 3582 * i40e_napi_disable_all - Disable NAPI for all q_vectors in the VSI 3583 * @vsi: the VSI being configured 3584 **/ 3585 static void i40e_napi_disable_all(struct i40e_vsi *vsi) 3586 { 3587 int q_idx; 3588 3589 if (!vsi->netdev) 3590 return; 3591 3592 for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++) 3593 napi_disable(&vsi->q_vectors[q_idx]->napi); 3594 } 3595 3596 /** 3597 * i40e_vsi_close - Shut down a VSI 3598 * @vsi: the vsi to be quelled 3599 **/ 3600 static void i40e_vsi_close(struct i40e_vsi *vsi) 3601 { 3602 if (!test_and_set_bit(__I40E_DOWN, &vsi->state)) 3603 i40e_down(vsi); 3604 i40e_vsi_free_irq(vsi); 3605 i40e_vsi_free_tx_resources(vsi); 3606 i40e_vsi_free_rx_resources(vsi); 3607 } 3608 3609 /** 3610 * i40e_quiesce_vsi - Pause a given VSI 3611 * @vsi: the VSI being paused 3612 **/ 3613 static void i40e_quiesce_vsi(struct i40e_vsi *vsi) 3614 { 3615 if (test_bit(__I40E_DOWN, &vsi->state)) 3616 return; 3617 3618 set_bit(__I40E_NEEDS_RESTART, &vsi->state); 3619 if (vsi->netdev && netif_running(vsi->netdev)) { 3620 vsi->netdev->netdev_ops->ndo_stop(vsi->netdev); 3621 } else { 3622 i40e_vsi_close(vsi); 3623 } 3624 } 3625 3626 /** 3627 * i40e_unquiesce_vsi - Resume a given VSI 3628 * @vsi: the VSI being resumed 3629 **/ 3630 static void i40e_unquiesce_vsi(struct i40e_vsi *vsi) 3631 { 3632 if (!test_bit(__I40E_NEEDS_RESTART, &vsi->state)) 3633 return; 3634 3635 clear_bit(__I40E_NEEDS_RESTART, &vsi->state); 3636 if (vsi->netdev && netif_running(vsi->netdev)) 3637 vsi->netdev->netdev_ops->ndo_open(vsi->netdev); 3638 else 3639 i40e_vsi_open(vsi); /* this clears the DOWN bit */ 3640 } 3641 3642 /** 3643 * i40e_pf_quiesce_all_vsi - Pause all VSIs on a PF 3644 * @pf: the PF 3645 **/ 3646 static void i40e_pf_quiesce_all_vsi(struct i40e_pf *pf) 3647 { 3648 int v; 3649 3650 for (v = 0; v < pf->num_alloc_vsi; v++) { 3651 if (pf->vsi[v]) 3652 i40e_quiesce_vsi(pf->vsi[v]); 3653 } 3654 } 3655 3656 /** 3657 * i40e_pf_unquiesce_all_vsi - Resume all VSIs on a PF 3658 * @pf: the PF 3659 **/ 3660 static void i40e_pf_unquiesce_all_vsi(struct i40e_pf *pf) 3661 { 3662 int v; 3663 3664 for (v = 0; v < pf->num_alloc_vsi; v++) { 3665 if (pf->vsi[v]) 3666 i40e_unquiesce_vsi(pf->vsi[v]); 3667 } 3668 } 3669 3670 /** 3671 * i40e_dcb_get_num_tc - Get the number of TCs from DCBx config 3672 * @dcbcfg: the corresponding DCBx configuration structure 3673 * 3674 * Return the number of TCs from given DCBx configuration 3675 **/ 3676 static u8 i40e_dcb_get_num_tc(struct i40e_dcbx_config *dcbcfg) 3677 { 3678 u8 num_tc = 0; 3679 int i; 3680 3681 /* Scan the ETS Config Priority Table to find 3682 * traffic class enabled for a given priority 3683 * and use the traffic class index to get the 3684 * number of traffic classes enabled 3685 */ 3686 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 3687 if (dcbcfg->etscfg.prioritytable[i] > num_tc) 3688 num_tc = dcbcfg->etscfg.prioritytable[i]; 3689 } 3690 3691 /* Traffic class index starts from zero so 3692 * increment to return the actual count 3693 */ 3694 return num_tc + 1; 3695 } 3696 3697 /** 3698 * i40e_dcb_get_enabled_tc - Get enabled traffic classes 3699 * @dcbcfg: the corresponding DCBx configuration structure 3700 * 3701 * Query the current DCB configuration and return the number of 3702 * traffic classes enabled from the given DCBX config 3703 **/ 3704 static u8 i40e_dcb_get_enabled_tc(struct i40e_dcbx_config *dcbcfg) 3705 { 3706 u8 num_tc = i40e_dcb_get_num_tc(dcbcfg); 3707 u8 enabled_tc = 1; 3708 u8 i; 3709 3710 for (i = 0; i < num_tc; i++) 3711 enabled_tc |= 1 << i; 3712 3713 return enabled_tc; 3714 } 3715 3716 /** 3717 * i40e_pf_get_num_tc - Get enabled traffic classes for PF 3718 * @pf: PF being queried 3719 * 3720 * Return number of traffic classes enabled for the given PF 3721 **/ 3722 static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) 3723 { 3724 struct i40e_hw *hw = &pf->hw; 3725 u8 i, enabled_tc; 3726 u8 num_tc = 0; 3727 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 3728 3729 /* If DCB is not enabled then always in single TC */ 3730 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 3731 return 1; 3732 3733 /* MFP mode return count of enabled TCs for this PF */ 3734 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 3735 enabled_tc = pf->hw.func_caps.enabled_tcmap; 3736 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 3737 if (enabled_tc & (1 << i)) 3738 num_tc++; 3739 } 3740 return num_tc; 3741 } 3742 3743 /* SFP mode will be enabled for all TCs on port */ 3744 return i40e_dcb_get_num_tc(dcbcfg); 3745 } 3746 3747 /** 3748 * i40e_pf_get_default_tc - Get bitmap for first enabled TC 3749 * @pf: PF being queried 3750 * 3751 * Return a bitmap for first enabled traffic class for this PF. 3752 **/ 3753 static u8 i40e_pf_get_default_tc(struct i40e_pf *pf) 3754 { 3755 u8 enabled_tc = pf->hw.func_caps.enabled_tcmap; 3756 u8 i = 0; 3757 3758 if (!enabled_tc) 3759 return 0x1; /* TC0 */ 3760 3761 /* Find the first enabled TC */ 3762 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 3763 if (enabled_tc & (1 << i)) 3764 break; 3765 } 3766 3767 return 1 << i; 3768 } 3769 3770 /** 3771 * i40e_pf_get_pf_tc_map - Get bitmap for enabled traffic classes 3772 * @pf: PF being queried 3773 * 3774 * Return a bitmap for enabled traffic classes for this PF. 3775 **/ 3776 static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) 3777 { 3778 /* If DCB is not enabled for this PF then just return default TC */ 3779 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) 3780 return i40e_pf_get_default_tc(pf); 3781 3782 /* MFP mode will have enabled TCs set by FW */ 3783 if (pf->flags & I40E_FLAG_MFP_ENABLED) 3784 return pf->hw.func_caps.enabled_tcmap; 3785 3786 /* SFP mode we want PF to be enabled for all TCs */ 3787 return i40e_dcb_get_enabled_tc(&pf->hw.local_dcbx_config); 3788 } 3789 3790 /** 3791 * i40e_vsi_get_bw_info - Query VSI BW Information 3792 * @vsi: the VSI being queried 3793 * 3794 * Returns 0 on success, negative value on failure 3795 **/ 3796 static int i40e_vsi_get_bw_info(struct i40e_vsi *vsi) 3797 { 3798 struct i40e_aqc_query_vsi_ets_sla_config_resp bw_ets_config = {0}; 3799 struct i40e_aqc_query_vsi_bw_config_resp bw_config = {0}; 3800 struct i40e_pf *pf = vsi->back; 3801 struct i40e_hw *hw = &pf->hw; 3802 i40e_status aq_ret; 3803 u32 tc_bw_max; 3804 int i; 3805 3806 /* Get the VSI level BW configuration */ 3807 aq_ret = i40e_aq_query_vsi_bw_config(hw, vsi->seid, &bw_config, NULL); 3808 if (aq_ret) { 3809 dev_info(&pf->pdev->dev, 3810 "couldn't get pf vsi bw config, err %d, aq_err %d\n", 3811 aq_ret, pf->hw.aq.asq_last_status); 3812 return -EINVAL; 3813 } 3814 3815 /* Get the VSI level BW configuration per TC */ 3816 aq_ret = i40e_aq_query_vsi_ets_sla_config(hw, vsi->seid, &bw_ets_config, 3817 NULL); 3818 if (aq_ret) { 3819 dev_info(&pf->pdev->dev, 3820 "couldn't get pf vsi ets bw config, err %d, aq_err %d\n", 3821 aq_ret, pf->hw.aq.asq_last_status); 3822 return -EINVAL; 3823 } 3824 3825 if (bw_config.tc_valid_bits != bw_ets_config.tc_valid_bits) { 3826 dev_info(&pf->pdev->dev, 3827 "Enabled TCs mismatch from querying VSI BW info 0x%08x 0x%08x\n", 3828 bw_config.tc_valid_bits, 3829 bw_ets_config.tc_valid_bits); 3830 /* Still continuing */ 3831 } 3832 3833 vsi->bw_limit = le16_to_cpu(bw_config.port_bw_limit); 3834 vsi->bw_max_quanta = bw_config.max_bw; 3835 tc_bw_max = le16_to_cpu(bw_ets_config.tc_bw_max[0]) | 3836 (le16_to_cpu(bw_ets_config.tc_bw_max[1]) << 16); 3837 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 3838 vsi->bw_ets_share_credits[i] = bw_ets_config.share_credits[i]; 3839 vsi->bw_ets_limit_credits[i] = 3840 le16_to_cpu(bw_ets_config.credits[i]); 3841 /* 3 bits out of 4 for each TC */ 3842 vsi->bw_ets_max_quanta[i] = (u8)((tc_bw_max >> (i*4)) & 0x7); 3843 } 3844 3845 return 0; 3846 } 3847 3848 /** 3849 * i40e_vsi_configure_bw_alloc - Configure VSI BW allocation per TC 3850 * @vsi: the VSI being configured 3851 * @enabled_tc: TC bitmap 3852 * @bw_credits: BW shared credits per TC 3853 * 3854 * Returns 0 on success, negative value on failure 3855 **/ 3856 static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, 3857 u8 *bw_share) 3858 { 3859 struct i40e_aqc_configure_vsi_tc_bw_data bw_data; 3860 i40e_status aq_ret; 3861 int i; 3862 3863 bw_data.tc_valid_bits = enabled_tc; 3864 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 3865 bw_data.tc_bw_credits[i] = bw_share[i]; 3866 3867 aq_ret = i40e_aq_config_vsi_tc_bw(&vsi->back->hw, vsi->seid, &bw_data, 3868 NULL); 3869 if (aq_ret) { 3870 dev_info(&vsi->back->pdev->dev, 3871 "AQ command Config VSI BW allocation per TC failed = %d\n", 3872 vsi->back->hw.aq.asq_last_status); 3873 return -EINVAL; 3874 } 3875 3876 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) 3877 vsi->info.qs_handle[i] = bw_data.qs_handles[i]; 3878 3879 return 0; 3880 } 3881 3882 /** 3883 * i40e_vsi_config_netdev_tc - Setup the netdev TC configuration 3884 * @vsi: the VSI being configured 3885 * @enabled_tc: TC map to be enabled 3886 * 3887 **/ 3888 static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) 3889 { 3890 struct net_device *netdev = vsi->netdev; 3891 struct i40e_pf *pf = vsi->back; 3892 struct i40e_hw *hw = &pf->hw; 3893 u8 netdev_tc = 0; 3894 int i; 3895 struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; 3896 3897 if (!netdev) 3898 return; 3899 3900 if (!enabled_tc) { 3901 netdev_reset_tc(netdev); 3902 return; 3903 } 3904 3905 /* Set up actual enabled TCs on the VSI */ 3906 if (netdev_set_num_tc(netdev, vsi->tc_config.numtc)) 3907 return; 3908 3909 /* set per TC queues for the VSI */ 3910 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 3911 /* Only set TC queues for enabled tcs 3912 * 3913 * e.g. For a VSI that has TC0 and TC3 enabled the 3914 * enabled_tc bitmap would be 0x00001001; the driver 3915 * will set the numtc for netdev as 2 that will be 3916 * referenced by the netdev layer as TC 0 and 1. 3917 */ 3918 if (vsi->tc_config.enabled_tc & (1 << i)) 3919 netdev_set_tc_queue(netdev, 3920 vsi->tc_config.tc_info[i].netdev_tc, 3921 vsi->tc_config.tc_info[i].qcount, 3922 vsi->tc_config.tc_info[i].qoffset); 3923 } 3924 3925 /* Assign UP2TC map for the VSI */ 3926 for (i = 0; i < I40E_MAX_USER_PRIORITY; i++) { 3927 /* Get the actual TC# for the UP */ 3928 u8 ets_tc = dcbcfg->etscfg.prioritytable[i]; 3929 /* Get the mapped netdev TC# for the UP */ 3930 netdev_tc = vsi->tc_config.tc_info[ets_tc].netdev_tc; 3931 netdev_set_prio_tc_map(netdev, i, netdev_tc); 3932 } 3933 } 3934 3935 /** 3936 * i40e_vsi_update_queue_map - Update our copy of VSi info with new queue map 3937 * @vsi: the VSI being configured 3938 * @ctxt: the ctxt buffer returned from AQ VSI update param command 3939 **/ 3940 static void i40e_vsi_update_queue_map(struct i40e_vsi *vsi, 3941 struct i40e_vsi_context *ctxt) 3942 { 3943 /* copy just the sections touched not the entire info 3944 * since not all sections are valid as returned by 3945 * update vsi params 3946 */ 3947 vsi->info.mapping_flags = ctxt->info.mapping_flags; 3948 memcpy(&vsi->info.queue_mapping, 3949 &ctxt->info.queue_mapping, sizeof(vsi->info.queue_mapping)); 3950 memcpy(&vsi->info.tc_mapping, ctxt->info.tc_mapping, 3951 sizeof(vsi->info.tc_mapping)); 3952 } 3953 3954 /** 3955 * i40e_vsi_config_tc - Configure VSI Tx Scheduler for given TC map 3956 * @vsi: VSI to be configured 3957 * @enabled_tc: TC bitmap 3958 * 3959 * This configures a particular VSI for TCs that are mapped to the 3960 * given TC bitmap. It uses default bandwidth share for TCs across 3961 * VSIs to configure TC for a particular VSI. 3962 * 3963 * NOTE: 3964 * It is expected that the VSI queues have been quisced before calling 3965 * this function. 3966 **/ 3967 static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) 3968 { 3969 u8 bw_share[I40E_MAX_TRAFFIC_CLASS] = {0}; 3970 struct i40e_vsi_context ctxt; 3971 int ret = 0; 3972 int i; 3973 3974 /* Check if enabled_tc is same as existing or new TCs */ 3975 if (vsi->tc_config.enabled_tc == enabled_tc) 3976 return ret; 3977 3978 /* Enable ETS TCs with equal BW Share for now across all VSIs */ 3979 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 3980 if (enabled_tc & (1 << i)) 3981 bw_share[i] = 1; 3982 } 3983 3984 ret = i40e_vsi_configure_bw_alloc(vsi, enabled_tc, bw_share); 3985 if (ret) { 3986 dev_info(&vsi->back->pdev->dev, 3987 "Failed configuring TC map %d for VSI %d\n", 3988 enabled_tc, vsi->seid); 3989 goto out; 3990 } 3991 3992 /* Update Queue Pairs Mapping for currently enabled UPs */ 3993 ctxt.seid = vsi->seid; 3994 ctxt.pf_num = vsi->back->hw.pf_id; 3995 ctxt.vf_num = 0; 3996 ctxt.uplink_seid = vsi->uplink_seid; 3997 memcpy(&ctxt.info, &vsi->info, sizeof(vsi->info)); 3998 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 3999 4000 /* Update the VSI after updating the VSI queue-mapping information */ 4001 ret = i40e_aq_update_vsi_params(&vsi->back->hw, &ctxt, NULL); 4002 if (ret) { 4003 dev_info(&vsi->back->pdev->dev, 4004 "update vsi failed, aq_err=%d\n", 4005 vsi->back->hw.aq.asq_last_status); 4006 goto out; 4007 } 4008 /* update the local VSI info with updated queue map */ 4009 i40e_vsi_update_queue_map(vsi, &ctxt); 4010 vsi->info.valid_sections = 0; 4011 4012 /* Update current VSI BW information */ 4013 ret = i40e_vsi_get_bw_info(vsi); 4014 if (ret) { 4015 dev_info(&vsi->back->pdev->dev, 4016 "Failed updating vsi bw info, aq_err=%d\n", 4017 vsi->back->hw.aq.asq_last_status); 4018 goto out; 4019 } 4020 4021 /* Update the netdev TC setup */ 4022 i40e_vsi_config_netdev_tc(vsi, enabled_tc); 4023 out: 4024 return ret; 4025 } 4026 4027 /** 4028 * i40e_veb_config_tc - Configure TCs for given VEB 4029 * @veb: given VEB 4030 * @enabled_tc: TC bitmap 4031 * 4032 * Configures given TC bitmap for VEB (switching) element 4033 **/ 4034 int i40e_veb_config_tc(struct i40e_veb *veb, u8 enabled_tc) 4035 { 4036 struct i40e_aqc_configure_switching_comp_bw_config_data bw_data = {0}; 4037 struct i40e_pf *pf = veb->pf; 4038 int ret = 0; 4039 int i; 4040 4041 /* No TCs or already enabled TCs just return */ 4042 if (!enabled_tc || veb->enabled_tc == enabled_tc) 4043 return ret; 4044 4045 bw_data.tc_valid_bits = enabled_tc; 4046 /* bw_data.absolute_credits is not set (relative) */ 4047 4048 /* Enable ETS TCs with equal BW Share for now */ 4049 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 4050 if (enabled_tc & (1 << i)) 4051 bw_data.tc_bw_share_credits[i] = 1; 4052 } 4053 4054 ret = i40e_aq_config_switch_comp_bw_config(&pf->hw, veb->seid, 4055 &bw_data, NULL); 4056 if (ret) { 4057 dev_info(&pf->pdev->dev, 4058 "veb bw config failed, aq_err=%d\n", 4059 pf->hw.aq.asq_last_status); 4060 goto out; 4061 } 4062 4063 /* Update the BW information */ 4064 ret = i40e_veb_get_bw_info(veb); 4065 if (ret) { 4066 dev_info(&pf->pdev->dev, 4067 "Failed getting veb bw config, aq_err=%d\n", 4068 pf->hw.aq.asq_last_status); 4069 } 4070 4071 out: 4072 return ret; 4073 } 4074 4075 #ifdef CONFIG_I40E_DCB 4076 /** 4077 * i40e_dcb_reconfigure - Reconfigure all VEBs and VSIs 4078 * @pf: PF struct 4079 * 4080 * Reconfigure VEB/VSIs on a given PF; it is assumed that 4081 * the caller would've quiesce all the VSIs before calling 4082 * this function 4083 **/ 4084 static void i40e_dcb_reconfigure(struct i40e_pf *pf) 4085 { 4086 u8 tc_map = 0; 4087 int ret; 4088 u8 v; 4089 4090 /* Enable the TCs available on PF to all VEBs */ 4091 tc_map = i40e_pf_get_tc_map(pf); 4092 for (v = 0; v < I40E_MAX_VEB; v++) { 4093 if (!pf->veb[v]) 4094 continue; 4095 ret = i40e_veb_config_tc(pf->veb[v], tc_map); 4096 if (ret) { 4097 dev_info(&pf->pdev->dev, 4098 "Failed configuring TC for VEB seid=%d\n", 4099 pf->veb[v]->seid); 4100 /* Will try to configure as many components */ 4101 } 4102 } 4103 4104 /* Update each VSI */ 4105 for (v = 0; v < pf->num_alloc_vsi; v++) { 4106 if (!pf->vsi[v]) 4107 continue; 4108 4109 /* - Enable all TCs for the LAN VSI 4110 * - For all others keep them at TC0 for now 4111 */ 4112 if (v == pf->lan_vsi) 4113 tc_map = i40e_pf_get_tc_map(pf); 4114 else 4115 tc_map = i40e_pf_get_default_tc(pf); 4116 4117 ret = i40e_vsi_config_tc(pf->vsi[v], tc_map); 4118 if (ret) { 4119 dev_info(&pf->pdev->dev, 4120 "Failed configuring TC for VSI seid=%d\n", 4121 pf->vsi[v]->seid); 4122 /* Will try to configure as many components */ 4123 } else { 4124 /* Re-configure VSI vectors based on updated TC map */ 4125 i40e_vsi_map_rings_to_vectors(pf->vsi[v]); 4126 if (pf->vsi[v]->netdev) 4127 i40e_dcbnl_set_all(pf->vsi[v]); 4128 } 4129 } 4130 } 4131 4132 /** 4133 * i40e_init_pf_dcb - Initialize DCB configuration 4134 * @pf: PF being configured 4135 * 4136 * Query the current DCB configuration and cache it 4137 * in the hardware structure 4138 **/ 4139 static int i40e_init_pf_dcb(struct i40e_pf *pf) 4140 { 4141 struct i40e_hw *hw = &pf->hw; 4142 int err = 0; 4143 4144 if (pf->hw.func_caps.npar_enable) 4145 goto out; 4146 4147 /* Get the initial DCB configuration */ 4148 err = i40e_init_dcb(hw); 4149 if (!err) { 4150 /* Device/Function is not DCBX capable */ 4151 if ((!hw->func_caps.dcb) || 4152 (hw->dcbx_status == I40E_DCBX_STATUS_DISABLED)) { 4153 dev_info(&pf->pdev->dev, 4154 "DCBX offload is not supported or is disabled for this PF.\n"); 4155 4156 if (pf->flags & I40E_FLAG_MFP_ENABLED) 4157 goto out; 4158 4159 } else { 4160 /* When status is not DISABLED then DCBX in FW */ 4161 pf->dcbx_cap = DCB_CAP_DCBX_LLD_MANAGED | 4162 DCB_CAP_DCBX_VER_IEEE; 4163 4164 pf->flags |= I40E_FLAG_DCB_CAPABLE; 4165 /* Enable DCB tagging only when more than one TC */ 4166 if (i40e_dcb_get_num_tc(&hw->local_dcbx_config) > 1) 4167 pf->flags |= I40E_FLAG_DCB_ENABLED; 4168 } 4169 } else { 4170 dev_info(&pf->pdev->dev, "AQ Querying DCB configuration failed: %d\n", 4171 pf->hw.aq.asq_last_status); 4172 } 4173 4174 out: 4175 return err; 4176 } 4177 #endif /* CONFIG_I40E_DCB */ 4178 #define SPEED_SIZE 14 4179 #define FC_SIZE 8 4180 /** 4181 * i40e_print_link_message - print link up or down 4182 * @vsi: the VSI for which link needs a message 4183 */ 4184 static void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) 4185 { 4186 char speed[SPEED_SIZE] = "Unknown"; 4187 char fc[FC_SIZE] = "RX/TX"; 4188 4189 if (!isup) { 4190 netdev_info(vsi->netdev, "NIC Link is Down\n"); 4191 return; 4192 } 4193 4194 switch (vsi->back->hw.phy.link_info.link_speed) { 4195 case I40E_LINK_SPEED_40GB: 4196 strncpy(speed, "40 Gbps", SPEED_SIZE); 4197 break; 4198 case I40E_LINK_SPEED_10GB: 4199 strncpy(speed, "10 Gbps", SPEED_SIZE); 4200 break; 4201 case I40E_LINK_SPEED_1GB: 4202 strncpy(speed, "1000 Mbps", SPEED_SIZE); 4203 break; 4204 default: 4205 break; 4206 } 4207 4208 switch (vsi->back->hw.fc.current_mode) { 4209 case I40E_FC_FULL: 4210 strncpy(fc, "RX/TX", FC_SIZE); 4211 break; 4212 case I40E_FC_TX_PAUSE: 4213 strncpy(fc, "TX", FC_SIZE); 4214 break; 4215 case I40E_FC_RX_PAUSE: 4216 strncpy(fc, "RX", FC_SIZE); 4217 break; 4218 default: 4219 strncpy(fc, "None", FC_SIZE); 4220 break; 4221 } 4222 4223 netdev_info(vsi->netdev, "NIC Link is Up %s Full Duplex, Flow Control: %s\n", 4224 speed, fc); 4225 } 4226 4227 /** 4228 * i40e_up_complete - Finish the last steps of bringing up a connection 4229 * @vsi: the VSI being configured 4230 **/ 4231 static int i40e_up_complete(struct i40e_vsi *vsi) 4232 { 4233 struct i40e_pf *pf = vsi->back; 4234 int err; 4235 4236 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 4237 i40e_vsi_configure_msix(vsi); 4238 else 4239 i40e_configure_msi_and_legacy(vsi); 4240 4241 /* start rings */ 4242 err = i40e_vsi_control_rings(vsi, true); 4243 if (err) 4244 return err; 4245 4246 clear_bit(__I40E_DOWN, &vsi->state); 4247 i40e_napi_enable_all(vsi); 4248 i40e_vsi_enable_irq(vsi); 4249 4250 if ((pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP) && 4251 (vsi->netdev)) { 4252 i40e_print_link_message(vsi, true); 4253 netif_tx_start_all_queues(vsi->netdev); 4254 netif_carrier_on(vsi->netdev); 4255 } else if (vsi->netdev) { 4256 i40e_print_link_message(vsi, false); 4257 } 4258 4259 /* replay FDIR SB filters */ 4260 if (vsi->type == I40E_VSI_FDIR) 4261 i40e_fdir_filter_restore(vsi); 4262 i40e_service_event_schedule(pf); 4263 4264 return 0; 4265 } 4266 4267 /** 4268 * i40e_vsi_reinit_locked - Reset the VSI 4269 * @vsi: the VSI being configured 4270 * 4271 * Rebuild the ring structs after some configuration 4272 * has changed, e.g. MTU size. 4273 **/ 4274 static void i40e_vsi_reinit_locked(struct i40e_vsi *vsi) 4275 { 4276 struct i40e_pf *pf = vsi->back; 4277 4278 WARN_ON(in_interrupt()); 4279 while (test_and_set_bit(__I40E_CONFIG_BUSY, &pf->state)) 4280 usleep_range(1000, 2000); 4281 i40e_down(vsi); 4282 4283 /* Give a VF some time to respond to the reset. The 4284 * two second wait is based upon the watchdog cycle in 4285 * the VF driver. 4286 */ 4287 if (vsi->type == I40E_VSI_SRIOV) 4288 msleep(2000); 4289 i40e_up(vsi); 4290 clear_bit(__I40E_CONFIG_BUSY, &pf->state); 4291 } 4292 4293 /** 4294 * i40e_up - Bring the connection back up after being down 4295 * @vsi: the VSI being configured 4296 **/ 4297 int i40e_up(struct i40e_vsi *vsi) 4298 { 4299 int err; 4300 4301 err = i40e_vsi_configure(vsi); 4302 if (!err) 4303 err = i40e_up_complete(vsi); 4304 4305 return err; 4306 } 4307 4308 /** 4309 * i40e_down - Shutdown the connection processing 4310 * @vsi: the VSI being stopped 4311 **/ 4312 void i40e_down(struct i40e_vsi *vsi) 4313 { 4314 int i; 4315 4316 /* It is assumed that the caller of this function 4317 * sets the vsi->state __I40E_DOWN bit. 4318 */ 4319 if (vsi->netdev) { 4320 netif_carrier_off(vsi->netdev); 4321 netif_tx_disable(vsi->netdev); 4322 } 4323 i40e_vsi_disable_irq(vsi); 4324 i40e_vsi_control_rings(vsi, false); 4325 i40e_napi_disable_all(vsi); 4326 4327 for (i = 0; i < vsi->num_queue_pairs; i++) { 4328 i40e_clean_tx_ring(vsi->tx_rings[i]); 4329 i40e_clean_rx_ring(vsi->rx_rings[i]); 4330 } 4331 } 4332 4333 /** 4334 * i40e_setup_tc - configure multiple traffic classes 4335 * @netdev: net device to configure 4336 * @tc: number of traffic classes to enable 4337 **/ 4338 static int i40e_setup_tc(struct net_device *netdev, u8 tc) 4339 { 4340 struct i40e_netdev_priv *np = netdev_priv(netdev); 4341 struct i40e_vsi *vsi = np->vsi; 4342 struct i40e_pf *pf = vsi->back; 4343 u8 enabled_tc = 0; 4344 int ret = -EINVAL; 4345 int i; 4346 4347 /* Check if DCB enabled to continue */ 4348 if (!(pf->flags & I40E_FLAG_DCB_ENABLED)) { 4349 netdev_info(netdev, "DCB is not enabled for adapter\n"); 4350 goto exit; 4351 } 4352 4353 /* Check if MFP enabled */ 4354 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 4355 netdev_info(netdev, "Configuring TC not supported in MFP mode\n"); 4356 goto exit; 4357 } 4358 4359 /* Check whether tc count is within enabled limit */ 4360 if (tc > i40e_pf_get_num_tc(pf)) { 4361 netdev_info(netdev, "TC count greater than enabled on link for adapter\n"); 4362 goto exit; 4363 } 4364 4365 /* Generate TC map for number of tc requested */ 4366 for (i = 0; i < tc; i++) 4367 enabled_tc |= (1 << i); 4368 4369 /* Requesting same TC configuration as already enabled */ 4370 if (enabled_tc == vsi->tc_config.enabled_tc) 4371 return 0; 4372 4373 /* Quiesce VSI queues */ 4374 i40e_quiesce_vsi(vsi); 4375 4376 /* Configure VSI for enabled TCs */ 4377 ret = i40e_vsi_config_tc(vsi, enabled_tc); 4378 if (ret) { 4379 netdev_info(netdev, "Failed configuring TC for VSI seid=%d\n", 4380 vsi->seid); 4381 goto exit; 4382 } 4383 4384 /* Unquiesce VSI */ 4385 i40e_unquiesce_vsi(vsi); 4386 4387 exit: 4388 return ret; 4389 } 4390 4391 /** 4392 * i40e_open - Called when a network interface is made active 4393 * @netdev: network interface device structure 4394 * 4395 * The open entry point is called when a network interface is made 4396 * active by the system (IFF_UP). At this point all resources needed 4397 * for transmit and receive operations are allocated, the interrupt 4398 * handler is registered with the OS, the netdev watchdog subtask is 4399 * enabled, and the stack is notified that the interface is ready. 4400 * 4401 * Returns 0 on success, negative value on failure 4402 **/ 4403 static int i40e_open(struct net_device *netdev) 4404 { 4405 struct i40e_netdev_priv *np = netdev_priv(netdev); 4406 struct i40e_vsi *vsi = np->vsi; 4407 struct i40e_pf *pf = vsi->back; 4408 int err; 4409 4410 /* disallow open during test or if eeprom is broken */ 4411 if (test_bit(__I40E_TESTING, &pf->state) || 4412 test_bit(__I40E_BAD_EEPROM, &pf->state)) 4413 return -EBUSY; 4414 4415 netif_carrier_off(netdev); 4416 4417 err = i40e_vsi_open(vsi); 4418 if (err) 4419 return err; 4420 4421 /* configure global TSO hardware offload settings */ 4422 wr32(&pf->hw, I40E_GLLAN_TSOMSK_F, be32_to_cpu(TCP_FLAG_PSH | 4423 TCP_FLAG_FIN) >> 16); 4424 wr32(&pf->hw, I40E_GLLAN_TSOMSK_M, be32_to_cpu(TCP_FLAG_PSH | 4425 TCP_FLAG_FIN | 4426 TCP_FLAG_CWR) >> 16); 4427 wr32(&pf->hw, I40E_GLLAN_TSOMSK_L, be32_to_cpu(TCP_FLAG_CWR) >> 16); 4428 4429 #ifdef CONFIG_I40E_VXLAN 4430 vxlan_get_rx_port(netdev); 4431 #endif 4432 4433 return 0; 4434 } 4435 4436 /** 4437 * i40e_vsi_open - 4438 * @vsi: the VSI to open 4439 * 4440 * Finish initialization of the VSI. 4441 * 4442 * Returns 0 on success, negative value on failure 4443 **/ 4444 int i40e_vsi_open(struct i40e_vsi *vsi) 4445 { 4446 struct i40e_pf *pf = vsi->back; 4447 char int_name[IFNAMSIZ]; 4448 int err; 4449 4450 /* allocate descriptors */ 4451 err = i40e_vsi_setup_tx_resources(vsi); 4452 if (err) 4453 goto err_setup_tx; 4454 err = i40e_vsi_setup_rx_resources(vsi); 4455 if (err) 4456 goto err_setup_rx; 4457 4458 err = i40e_vsi_configure(vsi); 4459 if (err) 4460 goto err_setup_rx; 4461 4462 if (vsi->netdev) { 4463 snprintf(int_name, sizeof(int_name) - 1, "%s-%s", 4464 dev_driver_string(&pf->pdev->dev), vsi->netdev->name); 4465 err = i40e_vsi_request_irq(vsi, int_name); 4466 if (err) 4467 goto err_setup_rx; 4468 4469 /* Notify the stack of the actual queue counts. */ 4470 err = netif_set_real_num_tx_queues(vsi->netdev, 4471 vsi->num_queue_pairs); 4472 if (err) 4473 goto err_set_queues; 4474 4475 err = netif_set_real_num_rx_queues(vsi->netdev, 4476 vsi->num_queue_pairs); 4477 if (err) 4478 goto err_set_queues; 4479 4480 } else if (vsi->type == I40E_VSI_FDIR) { 4481 snprintf(int_name, sizeof(int_name) - 1, "%s-fdir", 4482 dev_driver_string(&pf->pdev->dev)); 4483 err = i40e_vsi_request_irq(vsi, int_name); 4484 } else { 4485 err = -EINVAL; 4486 goto err_setup_rx; 4487 } 4488 4489 err = i40e_up_complete(vsi); 4490 if (err) 4491 goto err_up_complete; 4492 4493 return 0; 4494 4495 err_up_complete: 4496 i40e_down(vsi); 4497 err_set_queues: 4498 i40e_vsi_free_irq(vsi); 4499 err_setup_rx: 4500 i40e_vsi_free_rx_resources(vsi); 4501 err_setup_tx: 4502 i40e_vsi_free_tx_resources(vsi); 4503 if (vsi == pf->vsi[pf->lan_vsi]) 4504 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED)); 4505 4506 return err; 4507 } 4508 4509 /** 4510 * i40e_fdir_filter_exit - Cleans up the Flow Director accounting 4511 * @pf: Pointer to pf 4512 * 4513 * This function destroys the hlist where all the Flow Director 4514 * filters were saved. 4515 **/ 4516 static void i40e_fdir_filter_exit(struct i40e_pf *pf) 4517 { 4518 struct i40e_fdir_filter *filter; 4519 struct hlist_node *node2; 4520 4521 hlist_for_each_entry_safe(filter, node2, 4522 &pf->fdir_filter_list, fdir_node) { 4523 hlist_del(&filter->fdir_node); 4524 kfree(filter); 4525 } 4526 pf->fdir_pf_active_filters = 0; 4527 } 4528 4529 /** 4530 * i40e_close - Disables a network interface 4531 * @netdev: network interface device structure 4532 * 4533 * The close entry point is called when an interface is de-activated 4534 * by the OS. The hardware is still under the driver's control, but 4535 * this netdev interface is disabled. 4536 * 4537 * Returns 0, this is not allowed to fail 4538 **/ 4539 static int i40e_close(struct net_device *netdev) 4540 { 4541 struct i40e_netdev_priv *np = netdev_priv(netdev); 4542 struct i40e_vsi *vsi = np->vsi; 4543 4544 i40e_vsi_close(vsi); 4545 4546 return 0; 4547 } 4548 4549 /** 4550 * i40e_do_reset - Start a PF or Core Reset sequence 4551 * @pf: board private structure 4552 * @reset_flags: which reset is requested 4553 * 4554 * The essential difference in resets is that the PF Reset 4555 * doesn't clear the packet buffers, doesn't reset the PE 4556 * firmware, and doesn't bother the other PFs on the chip. 4557 **/ 4558 void i40e_do_reset(struct i40e_pf *pf, u32 reset_flags) 4559 { 4560 u32 val; 4561 4562 WARN_ON(in_interrupt()); 4563 4564 if (i40e_check_asq_alive(&pf->hw)) 4565 i40e_vc_notify_reset(pf); 4566 4567 /* do the biggest reset indicated */ 4568 if (reset_flags & (1 << __I40E_GLOBAL_RESET_REQUESTED)) { 4569 4570 /* Request a Global Reset 4571 * 4572 * This will start the chip's countdown to the actual full 4573 * chip reset event, and a warning interrupt to be sent 4574 * to all PFs, including the requestor. Our handler 4575 * for the warning interrupt will deal with the shutdown 4576 * and recovery of the switch setup. 4577 */ 4578 dev_dbg(&pf->pdev->dev, "GlobalR requested\n"); 4579 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 4580 val |= I40E_GLGEN_RTRIG_GLOBR_MASK; 4581 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 4582 4583 } else if (reset_flags & (1 << __I40E_CORE_RESET_REQUESTED)) { 4584 4585 /* Request a Core Reset 4586 * 4587 * Same as Global Reset, except does *not* include the MAC/PHY 4588 */ 4589 dev_dbg(&pf->pdev->dev, "CoreR requested\n"); 4590 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 4591 val |= I40E_GLGEN_RTRIG_CORER_MASK; 4592 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 4593 i40e_flush(&pf->hw); 4594 4595 } else if (reset_flags & (1 << __I40E_EMP_RESET_REQUESTED)) { 4596 4597 /* Request a Firmware Reset 4598 * 4599 * Same as Global reset, plus restarting the 4600 * embedded firmware engine. 4601 */ 4602 /* enable EMP Reset */ 4603 val = rd32(&pf->hw, I40E_GLGEN_RSTENA_EMP); 4604 val |= I40E_GLGEN_RSTENA_EMP_EMP_RST_ENA_MASK; 4605 wr32(&pf->hw, I40E_GLGEN_RSTENA_EMP, val); 4606 4607 /* force the reset */ 4608 val = rd32(&pf->hw, I40E_GLGEN_RTRIG); 4609 val |= I40E_GLGEN_RTRIG_EMPFWR_MASK; 4610 wr32(&pf->hw, I40E_GLGEN_RTRIG, val); 4611 i40e_flush(&pf->hw); 4612 4613 } else if (reset_flags & (1 << __I40E_PF_RESET_REQUESTED)) { 4614 4615 /* Request a PF Reset 4616 * 4617 * Resets only the PF-specific registers 4618 * 4619 * This goes directly to the tear-down and rebuild of 4620 * the switch, since we need to do all the recovery as 4621 * for the Core Reset. 4622 */ 4623 dev_dbg(&pf->pdev->dev, "PFR requested\n"); 4624 i40e_handle_reset_warning(pf); 4625 4626 } else if (reset_flags & (1 << __I40E_REINIT_REQUESTED)) { 4627 int v; 4628 4629 /* Find the VSI(s) that requested a re-init */ 4630 dev_info(&pf->pdev->dev, 4631 "VSI reinit requested\n"); 4632 for (v = 0; v < pf->num_alloc_vsi; v++) { 4633 struct i40e_vsi *vsi = pf->vsi[v]; 4634 if (vsi != NULL && 4635 test_bit(__I40E_REINIT_REQUESTED, &vsi->state)) { 4636 i40e_vsi_reinit_locked(pf->vsi[v]); 4637 clear_bit(__I40E_REINIT_REQUESTED, &vsi->state); 4638 } 4639 } 4640 4641 /* no further action needed, so return now */ 4642 return; 4643 } else { 4644 dev_info(&pf->pdev->dev, 4645 "bad reset request 0x%08x\n", reset_flags); 4646 return; 4647 } 4648 } 4649 4650 #ifdef CONFIG_I40E_DCB 4651 /** 4652 * i40e_dcb_need_reconfig - Check if DCB needs reconfig 4653 * @pf: board private structure 4654 * @old_cfg: current DCB config 4655 * @new_cfg: new DCB config 4656 **/ 4657 bool i40e_dcb_need_reconfig(struct i40e_pf *pf, 4658 struct i40e_dcbx_config *old_cfg, 4659 struct i40e_dcbx_config *new_cfg) 4660 { 4661 bool need_reconfig = false; 4662 4663 /* Check if ETS configuration has changed */ 4664 if (memcmp(&new_cfg->etscfg, 4665 &old_cfg->etscfg, 4666 sizeof(new_cfg->etscfg))) { 4667 /* If Priority Table has changed reconfig is needed */ 4668 if (memcmp(&new_cfg->etscfg.prioritytable, 4669 &old_cfg->etscfg.prioritytable, 4670 sizeof(new_cfg->etscfg.prioritytable))) { 4671 need_reconfig = true; 4672 dev_dbg(&pf->pdev->dev, "ETS UP2TC changed.\n"); 4673 } 4674 4675 if (memcmp(&new_cfg->etscfg.tcbwtable, 4676 &old_cfg->etscfg.tcbwtable, 4677 sizeof(new_cfg->etscfg.tcbwtable))) 4678 dev_dbg(&pf->pdev->dev, "ETS TC BW Table changed.\n"); 4679 4680 if (memcmp(&new_cfg->etscfg.tsatable, 4681 &old_cfg->etscfg.tsatable, 4682 sizeof(new_cfg->etscfg.tsatable))) 4683 dev_dbg(&pf->pdev->dev, "ETS TSA Table changed.\n"); 4684 } 4685 4686 /* Check if PFC configuration has changed */ 4687 if (memcmp(&new_cfg->pfc, 4688 &old_cfg->pfc, 4689 sizeof(new_cfg->pfc))) { 4690 need_reconfig = true; 4691 dev_dbg(&pf->pdev->dev, "PFC config change detected.\n"); 4692 } 4693 4694 /* Check if APP Table has changed */ 4695 if (memcmp(&new_cfg->app, 4696 &old_cfg->app, 4697 sizeof(new_cfg->app))) { 4698 need_reconfig = true; 4699 dev_dbg(&pf->pdev->dev, "APP Table change detected.\n"); 4700 } 4701 4702 return need_reconfig; 4703 } 4704 4705 /** 4706 * i40e_handle_lldp_event - Handle LLDP Change MIB event 4707 * @pf: board private structure 4708 * @e: event info posted on ARQ 4709 **/ 4710 static int i40e_handle_lldp_event(struct i40e_pf *pf, 4711 struct i40e_arq_event_info *e) 4712 { 4713 struct i40e_aqc_lldp_get_mib *mib = 4714 (struct i40e_aqc_lldp_get_mib *)&e->desc.params.raw; 4715 struct i40e_hw *hw = &pf->hw; 4716 struct i40e_dcbx_config *dcbx_cfg = &hw->local_dcbx_config; 4717 struct i40e_dcbx_config tmp_dcbx_cfg; 4718 bool need_reconfig = false; 4719 int ret = 0; 4720 u8 type; 4721 4722 /* Not DCB capable or capability disabled */ 4723 if (!(pf->flags & I40E_FLAG_DCB_CAPABLE)) 4724 return ret; 4725 4726 /* Ignore if event is not for Nearest Bridge */ 4727 type = ((mib->type >> I40E_AQ_LLDP_BRIDGE_TYPE_SHIFT) 4728 & I40E_AQ_LLDP_BRIDGE_TYPE_MASK); 4729 if (type != I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE) 4730 return ret; 4731 4732 /* Check MIB Type and return if event for Remote MIB update */ 4733 type = mib->type & I40E_AQ_LLDP_MIB_TYPE_MASK; 4734 if (type == I40E_AQ_LLDP_MIB_REMOTE) { 4735 /* Update the remote cached instance and return */ 4736 ret = i40e_aq_get_dcb_config(hw, I40E_AQ_LLDP_MIB_REMOTE, 4737 I40E_AQ_LLDP_BRIDGE_TYPE_NEAREST_BRIDGE, 4738 &hw->remote_dcbx_config); 4739 goto exit; 4740 } 4741 4742 /* Convert/store the DCBX data from LLDPDU temporarily */ 4743 memset(&tmp_dcbx_cfg, 0, sizeof(tmp_dcbx_cfg)); 4744 ret = i40e_lldp_to_dcb_config(e->msg_buf, &tmp_dcbx_cfg); 4745 if (ret) { 4746 /* Error in LLDPDU parsing return */ 4747 dev_info(&pf->pdev->dev, "Failed parsing LLDPDU from event buffer\n"); 4748 goto exit; 4749 } 4750 4751 /* No change detected in DCBX configs */ 4752 if (!memcmp(&tmp_dcbx_cfg, dcbx_cfg, sizeof(tmp_dcbx_cfg))) { 4753 dev_dbg(&pf->pdev->dev, "No change detected in DCBX configuration.\n"); 4754 goto exit; 4755 } 4756 4757 need_reconfig = i40e_dcb_need_reconfig(pf, dcbx_cfg, &tmp_dcbx_cfg); 4758 4759 i40e_dcbnl_flush_apps(pf, &tmp_dcbx_cfg); 4760 4761 /* Overwrite the new configuration */ 4762 *dcbx_cfg = tmp_dcbx_cfg; 4763 4764 if (!need_reconfig) 4765 goto exit; 4766 4767 /* Enable DCB tagging only when more than one TC */ 4768 if (i40e_dcb_get_num_tc(dcbx_cfg) > 1) 4769 pf->flags |= I40E_FLAG_DCB_ENABLED; 4770 else 4771 pf->flags &= ~I40E_FLAG_DCB_ENABLED; 4772 4773 /* Reconfiguration needed quiesce all VSIs */ 4774 i40e_pf_quiesce_all_vsi(pf); 4775 4776 /* Changes in configuration update VEB/VSI */ 4777 i40e_dcb_reconfigure(pf); 4778 4779 i40e_pf_unquiesce_all_vsi(pf); 4780 exit: 4781 return ret; 4782 } 4783 #endif /* CONFIG_I40E_DCB */ 4784 4785 /** 4786 * i40e_do_reset_safe - Protected reset path for userland calls. 4787 * @pf: board private structure 4788 * @reset_flags: which reset is requested 4789 * 4790 **/ 4791 void i40e_do_reset_safe(struct i40e_pf *pf, u32 reset_flags) 4792 { 4793 rtnl_lock(); 4794 i40e_do_reset(pf, reset_flags); 4795 rtnl_unlock(); 4796 } 4797 4798 /** 4799 * i40e_handle_lan_overflow_event - Handler for LAN queue overflow event 4800 * @pf: board private structure 4801 * @e: event info posted on ARQ 4802 * 4803 * Handler for LAN Queue Overflow Event generated by the firmware for PF 4804 * and VF queues 4805 **/ 4806 static void i40e_handle_lan_overflow_event(struct i40e_pf *pf, 4807 struct i40e_arq_event_info *e) 4808 { 4809 struct i40e_aqc_lan_overflow *data = 4810 (struct i40e_aqc_lan_overflow *)&e->desc.params.raw; 4811 u32 queue = le32_to_cpu(data->prtdcb_rupto); 4812 u32 qtx_ctl = le32_to_cpu(data->otx_ctl); 4813 struct i40e_hw *hw = &pf->hw; 4814 struct i40e_vf *vf; 4815 u16 vf_id; 4816 4817 dev_dbg(&pf->pdev->dev, "overflow Rx Queue Number = %d QTX_CTL=0x%08x\n", 4818 queue, qtx_ctl); 4819 4820 /* Queue belongs to VF, find the VF and issue VF reset */ 4821 if (((qtx_ctl & I40E_QTX_CTL_PFVF_Q_MASK) 4822 >> I40E_QTX_CTL_PFVF_Q_SHIFT) == I40E_QTX_CTL_VF_QUEUE) { 4823 vf_id = (u16)((qtx_ctl & I40E_QTX_CTL_VFVM_INDX_MASK) 4824 >> I40E_QTX_CTL_VFVM_INDX_SHIFT); 4825 vf_id -= hw->func_caps.vf_base_id; 4826 vf = &pf->vf[vf_id]; 4827 i40e_vc_notify_vf_reset(vf); 4828 /* Allow VF to process pending reset notification */ 4829 msleep(20); 4830 i40e_reset_vf(vf, false); 4831 } 4832 } 4833 4834 /** 4835 * i40e_service_event_complete - Finish up the service event 4836 * @pf: board private structure 4837 **/ 4838 static void i40e_service_event_complete(struct i40e_pf *pf) 4839 { 4840 BUG_ON(!test_bit(__I40E_SERVICE_SCHED, &pf->state)); 4841 4842 /* flush memory to make sure state is correct before next watchog */ 4843 smp_mb__before_atomic(); 4844 clear_bit(__I40E_SERVICE_SCHED, &pf->state); 4845 } 4846 4847 /** 4848 * i40e_get_current_fd_count - Get the count of FD filters programmed in the HW 4849 * @pf: board private structure 4850 **/ 4851 int i40e_get_current_fd_count(struct i40e_pf *pf) 4852 { 4853 int val, fcnt_prog; 4854 val = rd32(&pf->hw, I40E_PFQF_FDSTAT); 4855 fcnt_prog = (val & I40E_PFQF_FDSTAT_GUARANT_CNT_MASK) + 4856 ((val & I40E_PFQF_FDSTAT_BEST_CNT_MASK) >> 4857 I40E_PFQF_FDSTAT_BEST_CNT_SHIFT); 4858 return fcnt_prog; 4859 } 4860 4861 /** 4862 * i40e_fdir_check_and_reenable - Function to reenabe FD ATR or SB if disabled 4863 * @pf: board private structure 4864 **/ 4865 void i40e_fdir_check_and_reenable(struct i40e_pf *pf) 4866 { 4867 u32 fcnt_prog, fcnt_avail; 4868 4869 /* Check if, FD SB or ATR was auto disabled and if there is enough room 4870 * to re-enable 4871 */ 4872 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 4873 (pf->flags & I40E_FLAG_FD_SB_ENABLED)) 4874 return; 4875 fcnt_prog = i40e_get_current_fd_count(pf); 4876 fcnt_avail = i40e_get_fd_cnt_all(pf); 4877 if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM)) { 4878 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 4879 (pf->auto_disable_flags & I40E_FLAG_FD_SB_ENABLED)) { 4880 pf->auto_disable_flags &= ~I40E_FLAG_FD_SB_ENABLED; 4881 dev_info(&pf->pdev->dev, "FD Sideband/ntuple is being enabled since we have space in the table now\n"); 4882 } 4883 } 4884 /* Wait for some more space to be available to turn on ATR */ 4885 if (fcnt_prog < (fcnt_avail - I40E_FDIR_BUFFER_HEAD_ROOM * 2)) { 4886 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 4887 (pf->auto_disable_flags & I40E_FLAG_FD_ATR_ENABLED)) { 4888 pf->auto_disable_flags &= ~I40E_FLAG_FD_ATR_ENABLED; 4889 dev_info(&pf->pdev->dev, "ATR is being enabled since we have space in the table now\n"); 4890 } 4891 } 4892 } 4893 4894 /** 4895 * i40e_fdir_reinit_subtask - Worker thread to reinit FDIR filter table 4896 * @pf: board private structure 4897 **/ 4898 static void i40e_fdir_reinit_subtask(struct i40e_pf *pf) 4899 { 4900 if (!(pf->flags & I40E_FLAG_FDIR_REQUIRES_REINIT)) 4901 return; 4902 4903 /* if interface is down do nothing */ 4904 if (test_bit(__I40E_DOWN, &pf->state)) 4905 return; 4906 i40e_fdir_check_and_reenable(pf); 4907 4908 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 4909 (pf->flags & I40E_FLAG_FD_SB_ENABLED)) 4910 pf->flags &= ~I40E_FLAG_FDIR_REQUIRES_REINIT; 4911 } 4912 4913 /** 4914 * i40e_vsi_link_event - notify VSI of a link event 4915 * @vsi: vsi to be notified 4916 * @link_up: link up or down 4917 **/ 4918 static void i40e_vsi_link_event(struct i40e_vsi *vsi, bool link_up) 4919 { 4920 if (!vsi) 4921 return; 4922 4923 switch (vsi->type) { 4924 case I40E_VSI_MAIN: 4925 if (!vsi->netdev || !vsi->netdev_registered) 4926 break; 4927 4928 if (link_up) { 4929 netif_carrier_on(vsi->netdev); 4930 netif_tx_wake_all_queues(vsi->netdev); 4931 } else { 4932 netif_carrier_off(vsi->netdev); 4933 netif_tx_stop_all_queues(vsi->netdev); 4934 } 4935 break; 4936 4937 case I40E_VSI_SRIOV: 4938 break; 4939 4940 case I40E_VSI_VMDQ2: 4941 case I40E_VSI_CTRL: 4942 case I40E_VSI_MIRROR: 4943 default: 4944 /* there is no notification for other VSIs */ 4945 break; 4946 } 4947 } 4948 4949 /** 4950 * i40e_veb_link_event - notify elements on the veb of a link event 4951 * @veb: veb to be notified 4952 * @link_up: link up or down 4953 **/ 4954 static void i40e_veb_link_event(struct i40e_veb *veb, bool link_up) 4955 { 4956 struct i40e_pf *pf; 4957 int i; 4958 4959 if (!veb || !veb->pf) 4960 return; 4961 pf = veb->pf; 4962 4963 /* depth first... */ 4964 for (i = 0; i < I40E_MAX_VEB; i++) 4965 if (pf->veb[i] && (pf->veb[i]->uplink_seid == veb->seid)) 4966 i40e_veb_link_event(pf->veb[i], link_up); 4967 4968 /* ... now the local VSIs */ 4969 for (i = 0; i < pf->num_alloc_vsi; i++) 4970 if (pf->vsi[i] && (pf->vsi[i]->uplink_seid == veb->seid)) 4971 i40e_vsi_link_event(pf->vsi[i], link_up); 4972 } 4973 4974 /** 4975 * i40e_link_event - Update netif_carrier status 4976 * @pf: board private structure 4977 **/ 4978 static void i40e_link_event(struct i40e_pf *pf) 4979 { 4980 bool new_link, old_link; 4981 4982 new_link = (pf->hw.phy.link_info.link_info & I40E_AQ_LINK_UP); 4983 old_link = (pf->hw.phy.link_info_old.link_info & I40E_AQ_LINK_UP); 4984 4985 if (new_link == old_link) 4986 return; 4987 if (!test_bit(__I40E_DOWN, &pf->vsi[pf->lan_vsi]->state)) 4988 i40e_print_link_message(pf->vsi[pf->lan_vsi], new_link); 4989 4990 /* Notify the base of the switch tree connected to 4991 * the link. Floating VEBs are not notified. 4992 */ 4993 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 4994 i40e_veb_link_event(pf->veb[pf->lan_veb], new_link); 4995 else 4996 i40e_vsi_link_event(pf->vsi[pf->lan_vsi], new_link); 4997 4998 if (pf->vf) 4999 i40e_vc_notify_link_state(pf); 5000 5001 if (pf->flags & I40E_FLAG_PTP) 5002 i40e_ptp_set_increment(pf); 5003 } 5004 5005 /** 5006 * i40e_check_hang_subtask - Check for hung queues and dropped interrupts 5007 * @pf: board private structure 5008 * 5009 * Set the per-queue flags to request a check for stuck queues in the irq 5010 * clean functions, then force interrupts to be sure the irq clean is called. 5011 **/ 5012 static void i40e_check_hang_subtask(struct i40e_pf *pf) 5013 { 5014 int i, v; 5015 5016 /* If we're down or resetting, just bail */ 5017 if (test_bit(__I40E_CONFIG_BUSY, &pf->state)) 5018 return; 5019 5020 /* for each VSI/netdev 5021 * for each Tx queue 5022 * set the check flag 5023 * for each q_vector 5024 * force an interrupt 5025 */ 5026 for (v = 0; v < pf->num_alloc_vsi; v++) { 5027 struct i40e_vsi *vsi = pf->vsi[v]; 5028 int armed = 0; 5029 5030 if (!pf->vsi[v] || 5031 test_bit(__I40E_DOWN, &vsi->state) || 5032 (vsi->netdev && !netif_carrier_ok(vsi->netdev))) 5033 continue; 5034 5035 for (i = 0; i < vsi->num_queue_pairs; i++) { 5036 set_check_for_tx_hang(vsi->tx_rings[i]); 5037 if (test_bit(__I40E_HANG_CHECK_ARMED, 5038 &vsi->tx_rings[i]->state)) 5039 armed++; 5040 } 5041 5042 if (armed) { 5043 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 5044 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, 5045 (I40E_PFINT_DYN_CTL0_INTENA_MASK | 5046 I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK)); 5047 } else { 5048 u16 vec = vsi->base_vector - 1; 5049 u32 val = (I40E_PFINT_DYN_CTLN_INTENA_MASK | 5050 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK); 5051 for (i = 0; i < vsi->num_q_vectors; i++, vec++) 5052 wr32(&vsi->back->hw, 5053 I40E_PFINT_DYN_CTLN(vec), val); 5054 } 5055 i40e_flush(&vsi->back->hw); 5056 } 5057 } 5058 } 5059 5060 /** 5061 * i40e_watchdog_subtask - Check and bring link up 5062 * @pf: board private structure 5063 **/ 5064 static void i40e_watchdog_subtask(struct i40e_pf *pf) 5065 { 5066 int i; 5067 5068 /* if interface is down do nothing */ 5069 if (test_bit(__I40E_DOWN, &pf->state) || 5070 test_bit(__I40E_CONFIG_BUSY, &pf->state)) 5071 return; 5072 5073 /* Update the stats for active netdevs so the network stack 5074 * can look at updated numbers whenever it cares to 5075 */ 5076 for (i = 0; i < pf->num_alloc_vsi; i++) 5077 if (pf->vsi[i] && pf->vsi[i]->netdev) 5078 i40e_update_stats(pf->vsi[i]); 5079 5080 /* Update the stats for the active switching components */ 5081 for (i = 0; i < I40E_MAX_VEB; i++) 5082 if (pf->veb[i]) 5083 i40e_update_veb_stats(pf->veb[i]); 5084 5085 i40e_ptp_rx_hang(pf->vsi[pf->lan_vsi]); 5086 } 5087 5088 /** 5089 * i40e_reset_subtask - Set up for resetting the device and driver 5090 * @pf: board private structure 5091 **/ 5092 static void i40e_reset_subtask(struct i40e_pf *pf) 5093 { 5094 u32 reset_flags = 0; 5095 5096 rtnl_lock(); 5097 if (test_bit(__I40E_REINIT_REQUESTED, &pf->state)) { 5098 reset_flags |= (1 << __I40E_REINIT_REQUESTED); 5099 clear_bit(__I40E_REINIT_REQUESTED, &pf->state); 5100 } 5101 if (test_bit(__I40E_PF_RESET_REQUESTED, &pf->state)) { 5102 reset_flags |= (1 << __I40E_PF_RESET_REQUESTED); 5103 clear_bit(__I40E_PF_RESET_REQUESTED, &pf->state); 5104 } 5105 if (test_bit(__I40E_CORE_RESET_REQUESTED, &pf->state)) { 5106 reset_flags |= (1 << __I40E_CORE_RESET_REQUESTED); 5107 clear_bit(__I40E_CORE_RESET_REQUESTED, &pf->state); 5108 } 5109 if (test_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state)) { 5110 reset_flags |= (1 << __I40E_GLOBAL_RESET_REQUESTED); 5111 clear_bit(__I40E_GLOBAL_RESET_REQUESTED, &pf->state); 5112 } 5113 5114 /* If there's a recovery already waiting, it takes 5115 * precedence before starting a new reset sequence. 5116 */ 5117 if (test_bit(__I40E_RESET_INTR_RECEIVED, &pf->state)) { 5118 i40e_handle_reset_warning(pf); 5119 goto unlock; 5120 } 5121 5122 /* If we're already down or resetting, just bail */ 5123 if (reset_flags && 5124 !test_bit(__I40E_DOWN, &pf->state) && 5125 !test_bit(__I40E_CONFIG_BUSY, &pf->state)) 5126 i40e_do_reset(pf, reset_flags); 5127 5128 unlock: 5129 rtnl_unlock(); 5130 } 5131 5132 /** 5133 * i40e_handle_link_event - Handle link event 5134 * @pf: board private structure 5135 * @e: event info posted on ARQ 5136 **/ 5137 static void i40e_handle_link_event(struct i40e_pf *pf, 5138 struct i40e_arq_event_info *e) 5139 { 5140 struct i40e_hw *hw = &pf->hw; 5141 struct i40e_aqc_get_link_status *status = 5142 (struct i40e_aqc_get_link_status *)&e->desc.params.raw; 5143 struct i40e_link_status *hw_link_info = &hw->phy.link_info; 5144 5145 /* save off old link status information */ 5146 memcpy(&pf->hw.phy.link_info_old, hw_link_info, 5147 sizeof(pf->hw.phy.link_info_old)); 5148 5149 /* update link status */ 5150 hw_link_info->phy_type = (enum i40e_aq_phy_type)status->phy_type; 5151 hw_link_info->link_speed = (enum i40e_aq_link_speed)status->link_speed; 5152 hw_link_info->link_info = status->link_info; 5153 hw_link_info->an_info = status->an_info; 5154 hw_link_info->ext_info = status->ext_info; 5155 hw_link_info->lse_enable = 5156 le16_to_cpu(status->command_flags) & 5157 I40E_AQ_LSE_ENABLE; 5158 5159 /* process the event */ 5160 i40e_link_event(pf); 5161 5162 /* Do a new status request to re-enable LSE reporting 5163 * and load new status information into the hw struct, 5164 * then see if the status changed while processing the 5165 * initial event. 5166 */ 5167 i40e_aq_get_link_info(&pf->hw, true, NULL, NULL); 5168 i40e_link_event(pf); 5169 } 5170 5171 /** 5172 * i40e_clean_adminq_subtask - Clean the AdminQ rings 5173 * @pf: board private structure 5174 **/ 5175 static void i40e_clean_adminq_subtask(struct i40e_pf *pf) 5176 { 5177 struct i40e_arq_event_info event; 5178 struct i40e_hw *hw = &pf->hw; 5179 u16 pending, i = 0; 5180 i40e_status ret; 5181 u16 opcode; 5182 u32 oldval; 5183 u32 val; 5184 5185 if (!test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state)) 5186 return; 5187 5188 /* check for error indications */ 5189 val = rd32(&pf->hw, pf->hw.aq.arq.len); 5190 oldval = val; 5191 if (val & I40E_PF_ARQLEN_ARQVFE_MASK) { 5192 dev_info(&pf->pdev->dev, "ARQ VF Error detected\n"); 5193 val &= ~I40E_PF_ARQLEN_ARQVFE_MASK; 5194 } 5195 if (val & I40E_PF_ARQLEN_ARQOVFL_MASK) { 5196 dev_info(&pf->pdev->dev, "ARQ Overflow Error detected\n"); 5197 val &= ~I40E_PF_ARQLEN_ARQOVFL_MASK; 5198 } 5199 if (val & I40E_PF_ARQLEN_ARQCRIT_MASK) { 5200 dev_info(&pf->pdev->dev, "ARQ Critical Error detected\n"); 5201 val &= ~I40E_PF_ARQLEN_ARQCRIT_MASK; 5202 } 5203 if (oldval != val) 5204 wr32(&pf->hw, pf->hw.aq.arq.len, val); 5205 5206 val = rd32(&pf->hw, pf->hw.aq.asq.len); 5207 oldval = val; 5208 if (val & I40E_PF_ATQLEN_ATQVFE_MASK) { 5209 dev_info(&pf->pdev->dev, "ASQ VF Error detected\n"); 5210 val &= ~I40E_PF_ATQLEN_ATQVFE_MASK; 5211 } 5212 if (val & I40E_PF_ATQLEN_ATQOVFL_MASK) { 5213 dev_info(&pf->pdev->dev, "ASQ Overflow Error detected\n"); 5214 val &= ~I40E_PF_ATQLEN_ATQOVFL_MASK; 5215 } 5216 if (val & I40E_PF_ATQLEN_ATQCRIT_MASK) { 5217 dev_info(&pf->pdev->dev, "ASQ Critical Error detected\n"); 5218 val &= ~I40E_PF_ATQLEN_ATQCRIT_MASK; 5219 } 5220 if (oldval != val) 5221 wr32(&pf->hw, pf->hw.aq.asq.len, val); 5222 5223 event.msg_size = I40E_MAX_AQ_BUF_SIZE; 5224 event.msg_buf = kzalloc(event.msg_size, GFP_KERNEL); 5225 if (!event.msg_buf) 5226 return; 5227 5228 do { 5229 event.msg_size = I40E_MAX_AQ_BUF_SIZE; /* reinit each time */ 5230 ret = i40e_clean_arq_element(hw, &event, &pending); 5231 if (ret == I40E_ERR_ADMIN_QUEUE_NO_WORK) { 5232 dev_info(&pf->pdev->dev, "No ARQ event found\n"); 5233 break; 5234 } else if (ret) { 5235 dev_info(&pf->pdev->dev, "ARQ event error %d\n", ret); 5236 break; 5237 } 5238 5239 opcode = le16_to_cpu(event.desc.opcode); 5240 switch (opcode) { 5241 5242 case i40e_aqc_opc_get_link_status: 5243 i40e_handle_link_event(pf, &event); 5244 break; 5245 case i40e_aqc_opc_send_msg_to_pf: 5246 ret = i40e_vc_process_vf_msg(pf, 5247 le16_to_cpu(event.desc.retval), 5248 le32_to_cpu(event.desc.cookie_high), 5249 le32_to_cpu(event.desc.cookie_low), 5250 event.msg_buf, 5251 event.msg_size); 5252 break; 5253 case i40e_aqc_opc_lldp_update_mib: 5254 dev_dbg(&pf->pdev->dev, "ARQ: Update LLDP MIB event received\n"); 5255 #ifdef CONFIG_I40E_DCB 5256 rtnl_lock(); 5257 ret = i40e_handle_lldp_event(pf, &event); 5258 rtnl_unlock(); 5259 #endif /* CONFIG_I40E_DCB */ 5260 break; 5261 case i40e_aqc_opc_event_lan_overflow: 5262 dev_dbg(&pf->pdev->dev, "ARQ LAN queue overflow event received\n"); 5263 i40e_handle_lan_overflow_event(pf, &event); 5264 break; 5265 case i40e_aqc_opc_send_msg_to_peer: 5266 dev_info(&pf->pdev->dev, "ARQ: Msg from other pf\n"); 5267 break; 5268 default: 5269 dev_info(&pf->pdev->dev, 5270 "ARQ Error: Unknown event 0x%04x received\n", 5271 opcode); 5272 break; 5273 } 5274 } while (pending && (i++ < pf->adminq_work_limit)); 5275 5276 clear_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state); 5277 /* re-enable Admin queue interrupt cause */ 5278 val = rd32(hw, I40E_PFINT_ICR0_ENA); 5279 val |= I40E_PFINT_ICR0_ENA_ADMINQ_MASK; 5280 wr32(hw, I40E_PFINT_ICR0_ENA, val); 5281 i40e_flush(hw); 5282 5283 kfree(event.msg_buf); 5284 } 5285 5286 /** 5287 * i40e_verify_eeprom - make sure eeprom is good to use 5288 * @pf: board private structure 5289 **/ 5290 static void i40e_verify_eeprom(struct i40e_pf *pf) 5291 { 5292 int err; 5293 5294 err = i40e_diag_eeprom_test(&pf->hw); 5295 if (err) { 5296 /* retry in case of garbage read */ 5297 err = i40e_diag_eeprom_test(&pf->hw); 5298 if (err) { 5299 dev_info(&pf->pdev->dev, "eeprom check failed (%d), Tx/Rx traffic disabled\n", 5300 err); 5301 set_bit(__I40E_BAD_EEPROM, &pf->state); 5302 } 5303 } 5304 5305 if (!err && test_bit(__I40E_BAD_EEPROM, &pf->state)) { 5306 dev_info(&pf->pdev->dev, "eeprom check passed, Tx/Rx traffic enabled\n"); 5307 clear_bit(__I40E_BAD_EEPROM, &pf->state); 5308 } 5309 } 5310 5311 /** 5312 * i40e_reconstitute_veb - rebuild the VEB and anything connected to it 5313 * @veb: pointer to the VEB instance 5314 * 5315 * This is a recursive function that first builds the attached VSIs then 5316 * recurses in to build the next layer of VEB. We track the connections 5317 * through our own index numbers because the seid's from the HW could 5318 * change across the reset. 5319 **/ 5320 static int i40e_reconstitute_veb(struct i40e_veb *veb) 5321 { 5322 struct i40e_vsi *ctl_vsi = NULL; 5323 struct i40e_pf *pf = veb->pf; 5324 int v, veb_idx; 5325 int ret; 5326 5327 /* build VSI that owns this VEB, temporarily attached to base VEB */ 5328 for (v = 0; v < pf->num_alloc_vsi && !ctl_vsi; v++) { 5329 if (pf->vsi[v] && 5330 pf->vsi[v]->veb_idx == veb->idx && 5331 pf->vsi[v]->flags & I40E_VSI_FLAG_VEB_OWNER) { 5332 ctl_vsi = pf->vsi[v]; 5333 break; 5334 } 5335 } 5336 if (!ctl_vsi) { 5337 dev_info(&pf->pdev->dev, 5338 "missing owner VSI for veb_idx %d\n", veb->idx); 5339 ret = -ENOENT; 5340 goto end_reconstitute; 5341 } 5342 if (ctl_vsi != pf->vsi[pf->lan_vsi]) 5343 ctl_vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 5344 ret = i40e_add_vsi(ctl_vsi); 5345 if (ret) { 5346 dev_info(&pf->pdev->dev, 5347 "rebuild of owner VSI failed: %d\n", ret); 5348 goto end_reconstitute; 5349 } 5350 i40e_vsi_reset_stats(ctl_vsi); 5351 5352 /* create the VEB in the switch and move the VSI onto the VEB */ 5353 ret = i40e_add_veb(veb, ctl_vsi); 5354 if (ret) 5355 goto end_reconstitute; 5356 5357 /* create the remaining VSIs attached to this VEB */ 5358 for (v = 0; v < pf->num_alloc_vsi; v++) { 5359 if (!pf->vsi[v] || pf->vsi[v] == ctl_vsi) 5360 continue; 5361 5362 if (pf->vsi[v]->veb_idx == veb->idx) { 5363 struct i40e_vsi *vsi = pf->vsi[v]; 5364 vsi->uplink_seid = veb->seid; 5365 ret = i40e_add_vsi(vsi); 5366 if (ret) { 5367 dev_info(&pf->pdev->dev, 5368 "rebuild of vsi_idx %d failed: %d\n", 5369 v, ret); 5370 goto end_reconstitute; 5371 } 5372 i40e_vsi_reset_stats(vsi); 5373 } 5374 } 5375 5376 /* create any VEBs attached to this VEB - RECURSION */ 5377 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 5378 if (pf->veb[veb_idx] && pf->veb[veb_idx]->veb_idx == veb->idx) { 5379 pf->veb[veb_idx]->uplink_seid = veb->seid; 5380 ret = i40e_reconstitute_veb(pf->veb[veb_idx]); 5381 if (ret) 5382 break; 5383 } 5384 } 5385 5386 end_reconstitute: 5387 return ret; 5388 } 5389 5390 /** 5391 * i40e_get_capabilities - get info about the HW 5392 * @pf: the PF struct 5393 **/ 5394 static int i40e_get_capabilities(struct i40e_pf *pf) 5395 { 5396 struct i40e_aqc_list_capabilities_element_resp *cap_buf; 5397 u16 data_size; 5398 int buf_len; 5399 int err; 5400 5401 buf_len = 40 * sizeof(struct i40e_aqc_list_capabilities_element_resp); 5402 do { 5403 cap_buf = kzalloc(buf_len, GFP_KERNEL); 5404 if (!cap_buf) 5405 return -ENOMEM; 5406 5407 /* this loads the data into the hw struct for us */ 5408 err = i40e_aq_discover_capabilities(&pf->hw, cap_buf, buf_len, 5409 &data_size, 5410 i40e_aqc_opc_list_func_capabilities, 5411 NULL); 5412 /* data loaded, buffer no longer needed */ 5413 kfree(cap_buf); 5414 5415 if (pf->hw.aq.asq_last_status == I40E_AQ_RC_ENOMEM) { 5416 /* retry with a larger buffer */ 5417 buf_len = data_size; 5418 } else if (pf->hw.aq.asq_last_status != I40E_AQ_RC_OK) { 5419 dev_info(&pf->pdev->dev, 5420 "capability discovery failed: aq=%d\n", 5421 pf->hw.aq.asq_last_status); 5422 return -ENODEV; 5423 } 5424 } while (err); 5425 5426 if (((pf->hw.aq.fw_maj_ver == 2) && (pf->hw.aq.fw_min_ver < 22)) || 5427 (pf->hw.aq.fw_maj_ver < 2)) { 5428 pf->hw.func_caps.num_msix_vectors++; 5429 pf->hw.func_caps.num_msix_vectors_vf++; 5430 } 5431 5432 if (pf->hw.debug_mask & I40E_DEBUG_USER) 5433 dev_info(&pf->pdev->dev, 5434 "pf=%d, num_vfs=%d, msix_pf=%d, msix_vf=%d, fd_g=%d, fd_b=%d, pf_max_q=%d num_vsi=%d\n", 5435 pf->hw.pf_id, pf->hw.func_caps.num_vfs, 5436 pf->hw.func_caps.num_msix_vectors, 5437 pf->hw.func_caps.num_msix_vectors_vf, 5438 pf->hw.func_caps.fd_filters_guaranteed, 5439 pf->hw.func_caps.fd_filters_best_effort, 5440 pf->hw.func_caps.num_tx_qp, 5441 pf->hw.func_caps.num_vsis); 5442 5443 #define DEF_NUM_VSI (1 + (pf->hw.func_caps.fcoe ? 1 : 0) \ 5444 + pf->hw.func_caps.num_vfs) 5445 if (pf->hw.revision_id == 0 && (DEF_NUM_VSI > pf->hw.func_caps.num_vsis)) { 5446 dev_info(&pf->pdev->dev, 5447 "got num_vsis %d, setting num_vsis to %d\n", 5448 pf->hw.func_caps.num_vsis, DEF_NUM_VSI); 5449 pf->hw.func_caps.num_vsis = DEF_NUM_VSI; 5450 } 5451 5452 return 0; 5453 } 5454 5455 static int i40e_vsi_clear(struct i40e_vsi *vsi); 5456 5457 /** 5458 * i40e_fdir_sb_setup - initialize the Flow Director resources for Sideband 5459 * @pf: board private structure 5460 **/ 5461 static void i40e_fdir_sb_setup(struct i40e_pf *pf) 5462 { 5463 struct i40e_vsi *vsi; 5464 int i; 5465 5466 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 5467 return; 5468 5469 /* find existing VSI and see if it needs configuring */ 5470 vsi = NULL; 5471 for (i = 0; i < pf->num_alloc_vsi; i++) { 5472 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 5473 vsi = pf->vsi[i]; 5474 break; 5475 } 5476 } 5477 5478 /* create a new VSI if none exists */ 5479 if (!vsi) { 5480 vsi = i40e_vsi_setup(pf, I40E_VSI_FDIR, 5481 pf->vsi[pf->lan_vsi]->seid, 0); 5482 if (!vsi) { 5483 dev_info(&pf->pdev->dev, "Couldn't create FDir VSI\n"); 5484 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 5485 return; 5486 } 5487 } 5488 5489 i40e_vsi_setup_irqhandler(vsi, i40e_fdir_clean_ring); 5490 } 5491 5492 /** 5493 * i40e_fdir_teardown - release the Flow Director resources 5494 * @pf: board private structure 5495 **/ 5496 static void i40e_fdir_teardown(struct i40e_pf *pf) 5497 { 5498 int i; 5499 5500 i40e_fdir_filter_exit(pf); 5501 for (i = 0; i < pf->num_alloc_vsi; i++) { 5502 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 5503 i40e_vsi_release(pf->vsi[i]); 5504 break; 5505 } 5506 } 5507 } 5508 5509 /** 5510 * i40e_prep_for_reset - prep for the core to reset 5511 * @pf: board private structure 5512 * 5513 * Close up the VFs and other things in prep for pf Reset. 5514 **/ 5515 static int i40e_prep_for_reset(struct i40e_pf *pf) 5516 { 5517 struct i40e_hw *hw = &pf->hw; 5518 i40e_status ret = 0; 5519 u32 v; 5520 5521 clear_bit(__I40E_RESET_INTR_RECEIVED, &pf->state); 5522 if (test_and_set_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) 5523 return 0; 5524 5525 dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n"); 5526 5527 /* quiesce the VSIs and their queues that are not already DOWN */ 5528 i40e_pf_quiesce_all_vsi(pf); 5529 5530 for (v = 0; v < pf->num_alloc_vsi; v++) { 5531 if (pf->vsi[v]) 5532 pf->vsi[v]->seid = 0; 5533 } 5534 5535 i40e_shutdown_adminq(&pf->hw); 5536 5537 /* call shutdown HMC */ 5538 if (hw->hmc.hmc_obj) { 5539 ret = i40e_shutdown_lan_hmc(hw); 5540 if (ret) { 5541 dev_warn(&pf->pdev->dev, 5542 "shutdown_lan_hmc failed: %d\n", ret); 5543 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state); 5544 } 5545 } 5546 return ret; 5547 } 5548 5549 /** 5550 * i40e_send_version - update firmware with driver version 5551 * @pf: PF struct 5552 */ 5553 static void i40e_send_version(struct i40e_pf *pf) 5554 { 5555 struct i40e_driver_version dv; 5556 5557 dv.major_version = DRV_VERSION_MAJOR; 5558 dv.minor_version = DRV_VERSION_MINOR; 5559 dv.build_version = DRV_VERSION_BUILD; 5560 dv.subbuild_version = 0; 5561 strncpy(dv.driver_string, DRV_VERSION, sizeof(dv.driver_string)); 5562 i40e_aq_send_driver_version(&pf->hw, &dv, NULL); 5563 } 5564 5565 /** 5566 * i40e_reset_and_rebuild - reset and rebuild using a saved config 5567 * @pf: board private structure 5568 * @reinit: if the Main VSI needs to re-initialized. 5569 **/ 5570 static void i40e_reset_and_rebuild(struct i40e_pf *pf, bool reinit) 5571 { 5572 struct i40e_hw *hw = &pf->hw; 5573 i40e_status ret; 5574 u32 v; 5575 5576 /* Now we wait for GRST to settle out. 5577 * We don't have to delete the VEBs or VSIs from the hw switch 5578 * because the reset will make them disappear. 5579 */ 5580 ret = i40e_pf_reset(hw); 5581 if (ret) { 5582 dev_info(&pf->pdev->dev, "PF reset failed, %d\n", ret); 5583 goto end_core_reset; 5584 } 5585 pf->pfr_count++; 5586 5587 if (test_bit(__I40E_DOWN, &pf->state)) 5588 goto end_core_reset; 5589 dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); 5590 5591 /* rebuild the basics for the AdminQ, HMC, and initial HW switch */ 5592 ret = i40e_init_adminq(&pf->hw); 5593 if (ret) { 5594 dev_info(&pf->pdev->dev, "Rebuild AdminQ failed, %d\n", ret); 5595 goto end_core_reset; 5596 } 5597 5598 /* re-verify the eeprom if we just had an EMP reset */ 5599 if (test_bit(__I40E_EMP_RESET_REQUESTED, &pf->state)) { 5600 clear_bit(__I40E_EMP_RESET_REQUESTED, &pf->state); 5601 i40e_verify_eeprom(pf); 5602 } 5603 5604 i40e_clear_pxe_mode(hw); 5605 ret = i40e_get_capabilities(pf); 5606 if (ret) { 5607 dev_info(&pf->pdev->dev, "i40e_get_capabilities failed, %d\n", 5608 ret); 5609 goto end_core_reset; 5610 } 5611 5612 ret = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 5613 hw->func_caps.num_rx_qp, 5614 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num); 5615 if (ret) { 5616 dev_info(&pf->pdev->dev, "init_lan_hmc failed: %d\n", ret); 5617 goto end_core_reset; 5618 } 5619 ret = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 5620 if (ret) { 5621 dev_info(&pf->pdev->dev, "configure_lan_hmc failed: %d\n", ret); 5622 goto end_core_reset; 5623 } 5624 5625 #ifdef CONFIG_I40E_DCB 5626 ret = i40e_init_pf_dcb(pf); 5627 if (ret) { 5628 dev_info(&pf->pdev->dev, "init_pf_dcb failed: %d\n", ret); 5629 goto end_core_reset; 5630 } 5631 #endif /* CONFIG_I40E_DCB */ 5632 5633 /* do basic switch setup */ 5634 ret = i40e_setup_pf_switch(pf, reinit); 5635 if (ret) 5636 goto end_core_reset; 5637 5638 /* Rebuild the VSIs and VEBs that existed before reset. 5639 * They are still in our local switch element arrays, so only 5640 * need to rebuild the switch model in the HW. 5641 * 5642 * If there were VEBs but the reconstitution failed, we'll try 5643 * try to recover minimal use by getting the basic PF VSI working. 5644 */ 5645 if (pf->vsi[pf->lan_vsi]->uplink_seid != pf->mac_seid) { 5646 dev_dbg(&pf->pdev->dev, "attempting to rebuild switch\n"); 5647 /* find the one VEB connected to the MAC, and find orphans */ 5648 for (v = 0; v < I40E_MAX_VEB; v++) { 5649 if (!pf->veb[v]) 5650 continue; 5651 5652 if (pf->veb[v]->uplink_seid == pf->mac_seid || 5653 pf->veb[v]->uplink_seid == 0) { 5654 ret = i40e_reconstitute_veb(pf->veb[v]); 5655 5656 if (!ret) 5657 continue; 5658 5659 /* If Main VEB failed, we're in deep doodoo, 5660 * so give up rebuilding the switch and set up 5661 * for minimal rebuild of PF VSI. 5662 * If orphan failed, we'll report the error 5663 * but try to keep going. 5664 */ 5665 if (pf->veb[v]->uplink_seid == pf->mac_seid) { 5666 dev_info(&pf->pdev->dev, 5667 "rebuild of switch failed: %d, will try to set up simple PF connection\n", 5668 ret); 5669 pf->vsi[pf->lan_vsi]->uplink_seid 5670 = pf->mac_seid; 5671 break; 5672 } else if (pf->veb[v]->uplink_seid == 0) { 5673 dev_info(&pf->pdev->dev, 5674 "rebuild of orphan VEB failed: %d\n", 5675 ret); 5676 } 5677 } 5678 } 5679 } 5680 5681 if (pf->vsi[pf->lan_vsi]->uplink_seid == pf->mac_seid) { 5682 dev_info(&pf->pdev->dev, "attempting to rebuild PF VSI\n"); 5683 /* no VEB, so rebuild only the Main VSI */ 5684 ret = i40e_add_vsi(pf->vsi[pf->lan_vsi]); 5685 if (ret) { 5686 dev_info(&pf->pdev->dev, 5687 "rebuild of Main VSI failed: %d\n", ret); 5688 goto end_core_reset; 5689 } 5690 } 5691 5692 /* reinit the misc interrupt */ 5693 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 5694 ret = i40e_setup_misc_vector(pf); 5695 5696 /* restart the VSIs that were rebuilt and running before the reset */ 5697 i40e_pf_unquiesce_all_vsi(pf); 5698 5699 if (pf->num_alloc_vfs) { 5700 for (v = 0; v < pf->num_alloc_vfs; v++) 5701 i40e_reset_vf(&pf->vf[v], true); 5702 } 5703 5704 /* tell the firmware that we're starting */ 5705 i40e_send_version(pf); 5706 5707 end_core_reset: 5708 clear_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state); 5709 } 5710 5711 /** 5712 * i40e_handle_reset_warning - prep for the pf to reset, reset and rebuild 5713 * @pf: board private structure 5714 * 5715 * Close up the VFs and other things in prep for a Core Reset, 5716 * then get ready to rebuild the world. 5717 **/ 5718 static void i40e_handle_reset_warning(struct i40e_pf *pf) 5719 { 5720 i40e_status ret; 5721 5722 ret = i40e_prep_for_reset(pf); 5723 if (!ret) 5724 i40e_reset_and_rebuild(pf, false); 5725 } 5726 5727 /** 5728 * i40e_handle_mdd_event 5729 * @pf: pointer to the pf structure 5730 * 5731 * Called from the MDD irq handler to identify possibly malicious vfs 5732 **/ 5733 static void i40e_handle_mdd_event(struct i40e_pf *pf) 5734 { 5735 struct i40e_hw *hw = &pf->hw; 5736 bool mdd_detected = false; 5737 struct i40e_vf *vf; 5738 u32 reg; 5739 int i; 5740 5741 if (!test_bit(__I40E_MDD_EVENT_PENDING, &pf->state)) 5742 return; 5743 5744 /* find what triggered the MDD event */ 5745 reg = rd32(hw, I40E_GL_MDET_TX); 5746 if (reg & I40E_GL_MDET_TX_VALID_MASK) { 5747 u8 func = (reg & I40E_GL_MDET_TX_FUNCTION_MASK) 5748 >> I40E_GL_MDET_TX_FUNCTION_SHIFT; 5749 u8 event = (reg & I40E_GL_MDET_TX_EVENT_SHIFT) 5750 >> I40E_GL_MDET_TX_EVENT_SHIFT; 5751 u8 queue = (reg & I40E_GL_MDET_TX_QUEUE_MASK) 5752 >> I40E_GL_MDET_TX_QUEUE_SHIFT; 5753 dev_info(&pf->pdev->dev, 5754 "Malicious Driver Detection event 0x%02x on TX queue %d of function 0x%02x\n", 5755 event, queue, func); 5756 wr32(hw, I40E_GL_MDET_TX, 0xffffffff); 5757 mdd_detected = true; 5758 } 5759 reg = rd32(hw, I40E_GL_MDET_RX); 5760 if (reg & I40E_GL_MDET_RX_VALID_MASK) { 5761 u8 func = (reg & I40E_GL_MDET_RX_FUNCTION_MASK) 5762 >> I40E_GL_MDET_RX_FUNCTION_SHIFT; 5763 u8 event = (reg & I40E_GL_MDET_RX_EVENT_SHIFT) 5764 >> I40E_GL_MDET_RX_EVENT_SHIFT; 5765 u8 queue = (reg & I40E_GL_MDET_RX_QUEUE_MASK) 5766 >> I40E_GL_MDET_RX_QUEUE_SHIFT; 5767 dev_info(&pf->pdev->dev, 5768 "Malicious Driver Detection event 0x%02x on RX queue %d of function 0x%02x\n", 5769 event, queue, func); 5770 wr32(hw, I40E_GL_MDET_RX, 0xffffffff); 5771 mdd_detected = true; 5772 } 5773 5774 /* see if one of the VFs needs its hand slapped */ 5775 for (i = 0; i < pf->num_alloc_vfs && mdd_detected; i++) { 5776 vf = &(pf->vf[i]); 5777 reg = rd32(hw, I40E_VP_MDET_TX(i)); 5778 if (reg & I40E_VP_MDET_TX_VALID_MASK) { 5779 wr32(hw, I40E_VP_MDET_TX(i), 0xFFFF); 5780 vf->num_mdd_events++; 5781 dev_info(&pf->pdev->dev, "MDD TX event on VF %d\n", i); 5782 } 5783 5784 reg = rd32(hw, I40E_VP_MDET_RX(i)); 5785 if (reg & I40E_VP_MDET_RX_VALID_MASK) { 5786 wr32(hw, I40E_VP_MDET_RX(i), 0xFFFF); 5787 vf->num_mdd_events++; 5788 dev_info(&pf->pdev->dev, "MDD RX event on VF %d\n", i); 5789 } 5790 5791 if (vf->num_mdd_events > I40E_DEFAULT_NUM_MDD_EVENTS_ALLOWED) { 5792 dev_info(&pf->pdev->dev, 5793 "Too many MDD events on VF %d, disabled\n", i); 5794 dev_info(&pf->pdev->dev, 5795 "Use PF Control I/F to re-enable the VF\n"); 5796 set_bit(I40E_VF_STAT_DISABLED, &vf->vf_states); 5797 } 5798 } 5799 5800 /* re-enable mdd interrupt cause */ 5801 clear_bit(__I40E_MDD_EVENT_PENDING, &pf->state); 5802 reg = rd32(hw, I40E_PFINT_ICR0_ENA); 5803 reg |= I40E_PFINT_ICR0_ENA_MAL_DETECT_MASK; 5804 wr32(hw, I40E_PFINT_ICR0_ENA, reg); 5805 i40e_flush(hw); 5806 } 5807 5808 #ifdef CONFIG_I40E_VXLAN 5809 /** 5810 * i40e_sync_vxlan_filters_subtask - Sync the VSI filter list with HW 5811 * @pf: board private structure 5812 **/ 5813 static void i40e_sync_vxlan_filters_subtask(struct i40e_pf *pf) 5814 { 5815 struct i40e_hw *hw = &pf->hw; 5816 i40e_status ret; 5817 u8 filter_index; 5818 __be16 port; 5819 int i; 5820 5821 if (!(pf->flags & I40E_FLAG_VXLAN_FILTER_SYNC)) 5822 return; 5823 5824 pf->flags &= ~I40E_FLAG_VXLAN_FILTER_SYNC; 5825 5826 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 5827 if (pf->pending_vxlan_bitmap & (1 << i)) { 5828 pf->pending_vxlan_bitmap &= ~(1 << i); 5829 port = pf->vxlan_ports[i]; 5830 ret = port ? 5831 i40e_aq_add_udp_tunnel(hw, ntohs(port), 5832 I40E_AQC_TUNNEL_TYPE_VXLAN, 5833 &filter_index, NULL) 5834 : i40e_aq_del_udp_tunnel(hw, i, NULL); 5835 5836 if (ret) { 5837 dev_info(&pf->pdev->dev, "Failed to execute AQ command for %s port %d with index %d\n", 5838 port ? "adding" : "deleting", 5839 ntohs(port), port ? i : i); 5840 5841 pf->vxlan_ports[i] = 0; 5842 } else { 5843 dev_info(&pf->pdev->dev, "%s port %d with AQ command with index %d\n", 5844 port ? "Added" : "Deleted", 5845 ntohs(port), port ? i : filter_index); 5846 } 5847 } 5848 } 5849 } 5850 5851 #endif 5852 /** 5853 * i40e_service_task - Run the driver's async subtasks 5854 * @work: pointer to work_struct containing our data 5855 **/ 5856 static void i40e_service_task(struct work_struct *work) 5857 { 5858 struct i40e_pf *pf = container_of(work, 5859 struct i40e_pf, 5860 service_task); 5861 unsigned long start_time = jiffies; 5862 5863 i40e_reset_subtask(pf); 5864 i40e_handle_mdd_event(pf); 5865 i40e_vc_process_vflr_event(pf); 5866 i40e_watchdog_subtask(pf); 5867 i40e_fdir_reinit_subtask(pf); 5868 i40e_check_hang_subtask(pf); 5869 i40e_sync_filters_subtask(pf); 5870 #ifdef CONFIG_I40E_VXLAN 5871 i40e_sync_vxlan_filters_subtask(pf); 5872 #endif 5873 i40e_clean_adminq_subtask(pf); 5874 5875 i40e_service_event_complete(pf); 5876 5877 /* If the tasks have taken longer than one timer cycle or there 5878 * is more work to be done, reschedule the service task now 5879 * rather than wait for the timer to tick again. 5880 */ 5881 if (time_after(jiffies, (start_time + pf->service_timer_period)) || 5882 test_bit(__I40E_ADMINQ_EVENT_PENDING, &pf->state) || 5883 test_bit(__I40E_MDD_EVENT_PENDING, &pf->state) || 5884 test_bit(__I40E_VFLR_EVENT_PENDING, &pf->state)) 5885 i40e_service_event_schedule(pf); 5886 } 5887 5888 /** 5889 * i40e_service_timer - timer callback 5890 * @data: pointer to PF struct 5891 **/ 5892 static void i40e_service_timer(unsigned long data) 5893 { 5894 struct i40e_pf *pf = (struct i40e_pf *)data; 5895 5896 mod_timer(&pf->service_timer, 5897 round_jiffies(jiffies + pf->service_timer_period)); 5898 i40e_service_event_schedule(pf); 5899 } 5900 5901 /** 5902 * i40e_set_num_rings_in_vsi - Determine number of rings in the VSI 5903 * @vsi: the VSI being configured 5904 **/ 5905 static int i40e_set_num_rings_in_vsi(struct i40e_vsi *vsi) 5906 { 5907 struct i40e_pf *pf = vsi->back; 5908 5909 switch (vsi->type) { 5910 case I40E_VSI_MAIN: 5911 vsi->alloc_queue_pairs = pf->num_lan_qps; 5912 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 5913 I40E_REQ_DESCRIPTOR_MULTIPLE); 5914 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 5915 vsi->num_q_vectors = pf->num_lan_msix; 5916 else 5917 vsi->num_q_vectors = 1; 5918 5919 break; 5920 5921 case I40E_VSI_FDIR: 5922 vsi->alloc_queue_pairs = 1; 5923 vsi->num_desc = ALIGN(I40E_FDIR_RING_COUNT, 5924 I40E_REQ_DESCRIPTOR_MULTIPLE); 5925 vsi->num_q_vectors = 1; 5926 break; 5927 5928 case I40E_VSI_VMDQ2: 5929 vsi->alloc_queue_pairs = pf->num_vmdq_qps; 5930 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 5931 I40E_REQ_DESCRIPTOR_MULTIPLE); 5932 vsi->num_q_vectors = pf->num_vmdq_msix; 5933 break; 5934 5935 case I40E_VSI_SRIOV: 5936 vsi->alloc_queue_pairs = pf->num_vf_qps; 5937 vsi->num_desc = ALIGN(I40E_DEFAULT_NUM_DESCRIPTORS, 5938 I40E_REQ_DESCRIPTOR_MULTIPLE); 5939 break; 5940 5941 default: 5942 WARN_ON(1); 5943 return -ENODATA; 5944 } 5945 5946 return 0; 5947 } 5948 5949 /** 5950 * i40e_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi 5951 * @type: VSI pointer 5952 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated. 5953 * 5954 * On error: returns error code (negative) 5955 * On success: returns 0 5956 **/ 5957 static int i40e_vsi_alloc_arrays(struct i40e_vsi *vsi, bool alloc_qvectors) 5958 { 5959 int size; 5960 int ret = 0; 5961 5962 /* allocate memory for both Tx and Rx ring pointers */ 5963 size = sizeof(struct i40e_ring *) * vsi->alloc_queue_pairs * 2; 5964 vsi->tx_rings = kzalloc(size, GFP_KERNEL); 5965 if (!vsi->tx_rings) 5966 return -ENOMEM; 5967 vsi->rx_rings = &vsi->tx_rings[vsi->alloc_queue_pairs]; 5968 5969 if (alloc_qvectors) { 5970 /* allocate memory for q_vector pointers */ 5971 size = sizeof(struct i40e_q_vectors *) * vsi->num_q_vectors; 5972 vsi->q_vectors = kzalloc(size, GFP_KERNEL); 5973 if (!vsi->q_vectors) { 5974 ret = -ENOMEM; 5975 goto err_vectors; 5976 } 5977 } 5978 return ret; 5979 5980 err_vectors: 5981 kfree(vsi->tx_rings); 5982 return ret; 5983 } 5984 5985 /** 5986 * i40e_vsi_mem_alloc - Allocates the next available struct vsi in the PF 5987 * @pf: board private structure 5988 * @type: type of VSI 5989 * 5990 * On error: returns error code (negative) 5991 * On success: returns vsi index in PF (positive) 5992 **/ 5993 static int i40e_vsi_mem_alloc(struct i40e_pf *pf, enum i40e_vsi_type type) 5994 { 5995 int ret = -ENODEV; 5996 struct i40e_vsi *vsi; 5997 int vsi_idx; 5998 int i; 5999 6000 /* Need to protect the allocation of the VSIs at the PF level */ 6001 mutex_lock(&pf->switch_mutex); 6002 6003 /* VSI list may be fragmented if VSI creation/destruction has 6004 * been happening. We can afford to do a quick scan to look 6005 * for any free VSIs in the list. 6006 * 6007 * find next empty vsi slot, looping back around if necessary 6008 */ 6009 i = pf->next_vsi; 6010 while (i < pf->num_alloc_vsi && pf->vsi[i]) 6011 i++; 6012 if (i >= pf->num_alloc_vsi) { 6013 i = 0; 6014 while (i < pf->next_vsi && pf->vsi[i]) 6015 i++; 6016 } 6017 6018 if (i < pf->num_alloc_vsi && !pf->vsi[i]) { 6019 vsi_idx = i; /* Found one! */ 6020 } else { 6021 ret = -ENODEV; 6022 goto unlock_pf; /* out of VSI slots! */ 6023 } 6024 pf->next_vsi = ++i; 6025 6026 vsi = kzalloc(sizeof(*vsi), GFP_KERNEL); 6027 if (!vsi) { 6028 ret = -ENOMEM; 6029 goto unlock_pf; 6030 } 6031 vsi->type = type; 6032 vsi->back = pf; 6033 set_bit(__I40E_DOWN, &vsi->state); 6034 vsi->flags = 0; 6035 vsi->idx = vsi_idx; 6036 vsi->rx_itr_setting = pf->rx_itr_default; 6037 vsi->tx_itr_setting = pf->tx_itr_default; 6038 vsi->netdev_registered = false; 6039 vsi->work_limit = I40E_DEFAULT_IRQ_WORK; 6040 INIT_LIST_HEAD(&vsi->mac_filter_list); 6041 vsi->irqs_ready = false; 6042 6043 ret = i40e_set_num_rings_in_vsi(vsi); 6044 if (ret) 6045 goto err_rings; 6046 6047 ret = i40e_vsi_alloc_arrays(vsi, true); 6048 if (ret) 6049 goto err_rings; 6050 6051 /* Setup default MSIX irq handler for VSI */ 6052 i40e_vsi_setup_irqhandler(vsi, i40e_msix_clean_rings); 6053 6054 pf->vsi[vsi_idx] = vsi; 6055 ret = vsi_idx; 6056 goto unlock_pf; 6057 6058 err_rings: 6059 pf->next_vsi = i - 1; 6060 kfree(vsi); 6061 unlock_pf: 6062 mutex_unlock(&pf->switch_mutex); 6063 return ret; 6064 } 6065 6066 /** 6067 * i40e_vsi_free_arrays - Free queue and vector pointer arrays for the VSI 6068 * @type: VSI pointer 6069 * @free_qvectors: a bool to specify if q_vectors need to be freed. 6070 * 6071 * On error: returns error code (negative) 6072 * On success: returns 0 6073 **/ 6074 static void i40e_vsi_free_arrays(struct i40e_vsi *vsi, bool free_qvectors) 6075 { 6076 /* free the ring and vector containers */ 6077 if (free_qvectors) { 6078 kfree(vsi->q_vectors); 6079 vsi->q_vectors = NULL; 6080 } 6081 kfree(vsi->tx_rings); 6082 vsi->tx_rings = NULL; 6083 vsi->rx_rings = NULL; 6084 } 6085 6086 /** 6087 * i40e_vsi_clear - Deallocate the VSI provided 6088 * @vsi: the VSI being un-configured 6089 **/ 6090 static int i40e_vsi_clear(struct i40e_vsi *vsi) 6091 { 6092 struct i40e_pf *pf; 6093 6094 if (!vsi) 6095 return 0; 6096 6097 if (!vsi->back) 6098 goto free_vsi; 6099 pf = vsi->back; 6100 6101 mutex_lock(&pf->switch_mutex); 6102 if (!pf->vsi[vsi->idx]) { 6103 dev_err(&pf->pdev->dev, "pf->vsi[%d] is NULL, just free vsi[%d](%p,type %d)\n", 6104 vsi->idx, vsi->idx, vsi, vsi->type); 6105 goto unlock_vsi; 6106 } 6107 6108 if (pf->vsi[vsi->idx] != vsi) { 6109 dev_err(&pf->pdev->dev, 6110 "pf->vsi[%d](%p, type %d) != vsi[%d](%p,type %d): no free!\n", 6111 pf->vsi[vsi->idx]->idx, 6112 pf->vsi[vsi->idx], 6113 pf->vsi[vsi->idx]->type, 6114 vsi->idx, vsi, vsi->type); 6115 goto unlock_vsi; 6116 } 6117 6118 /* updates the pf for this cleared vsi */ 6119 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 6120 i40e_put_lump(pf->irq_pile, vsi->base_vector, vsi->idx); 6121 6122 i40e_vsi_free_arrays(vsi, true); 6123 6124 pf->vsi[vsi->idx] = NULL; 6125 if (vsi->idx < pf->next_vsi) 6126 pf->next_vsi = vsi->idx; 6127 6128 unlock_vsi: 6129 mutex_unlock(&pf->switch_mutex); 6130 free_vsi: 6131 kfree(vsi); 6132 6133 return 0; 6134 } 6135 6136 /** 6137 * i40e_vsi_clear_rings - Deallocates the Rx and Tx rings for the provided VSI 6138 * @vsi: the VSI being cleaned 6139 **/ 6140 static void i40e_vsi_clear_rings(struct i40e_vsi *vsi) 6141 { 6142 int i; 6143 6144 if (vsi->tx_rings && vsi->tx_rings[0]) { 6145 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 6146 kfree_rcu(vsi->tx_rings[i], rcu); 6147 vsi->tx_rings[i] = NULL; 6148 vsi->rx_rings[i] = NULL; 6149 } 6150 } 6151 } 6152 6153 /** 6154 * i40e_alloc_rings - Allocates the Rx and Tx rings for the provided VSI 6155 * @vsi: the VSI being configured 6156 **/ 6157 static int i40e_alloc_rings(struct i40e_vsi *vsi) 6158 { 6159 struct i40e_ring *tx_ring, *rx_ring; 6160 struct i40e_pf *pf = vsi->back; 6161 int i; 6162 6163 /* Set basic values in the rings to be used later during open() */ 6164 for (i = 0; i < vsi->alloc_queue_pairs; i++) { 6165 /* allocate space for both Tx and Rx in one shot */ 6166 tx_ring = kzalloc(sizeof(struct i40e_ring) * 2, GFP_KERNEL); 6167 if (!tx_ring) 6168 goto err_out; 6169 6170 tx_ring->queue_index = i; 6171 tx_ring->reg_idx = vsi->base_queue + i; 6172 tx_ring->ring_active = false; 6173 tx_ring->vsi = vsi; 6174 tx_ring->netdev = vsi->netdev; 6175 tx_ring->dev = &pf->pdev->dev; 6176 tx_ring->count = vsi->num_desc; 6177 tx_ring->size = 0; 6178 tx_ring->dcb_tc = 0; 6179 vsi->tx_rings[i] = tx_ring; 6180 6181 rx_ring = &tx_ring[1]; 6182 rx_ring->queue_index = i; 6183 rx_ring->reg_idx = vsi->base_queue + i; 6184 rx_ring->ring_active = false; 6185 rx_ring->vsi = vsi; 6186 rx_ring->netdev = vsi->netdev; 6187 rx_ring->dev = &pf->pdev->dev; 6188 rx_ring->count = vsi->num_desc; 6189 rx_ring->size = 0; 6190 rx_ring->dcb_tc = 0; 6191 if (pf->flags & I40E_FLAG_16BYTE_RX_DESC_ENABLED) 6192 set_ring_16byte_desc_enabled(rx_ring); 6193 else 6194 clear_ring_16byte_desc_enabled(rx_ring); 6195 vsi->rx_rings[i] = rx_ring; 6196 } 6197 6198 return 0; 6199 6200 err_out: 6201 i40e_vsi_clear_rings(vsi); 6202 return -ENOMEM; 6203 } 6204 6205 /** 6206 * i40e_reserve_msix_vectors - Reserve MSI-X vectors in the kernel 6207 * @pf: board private structure 6208 * @vectors: the number of MSI-X vectors to request 6209 * 6210 * Returns the number of vectors reserved, or error 6211 **/ 6212 static int i40e_reserve_msix_vectors(struct i40e_pf *pf, int vectors) 6213 { 6214 vectors = pci_enable_msix_range(pf->pdev, pf->msix_entries, 6215 I40E_MIN_MSIX, vectors); 6216 if (vectors < 0) { 6217 dev_info(&pf->pdev->dev, 6218 "MSI-X vector reservation failed: %d\n", vectors); 6219 vectors = 0; 6220 } 6221 6222 return vectors; 6223 } 6224 6225 /** 6226 * i40e_init_msix - Setup the MSIX capability 6227 * @pf: board private structure 6228 * 6229 * Work with the OS to set up the MSIX vectors needed. 6230 * 6231 * Returns 0 on success, negative on failure 6232 **/ 6233 static int i40e_init_msix(struct i40e_pf *pf) 6234 { 6235 i40e_status err = 0; 6236 struct i40e_hw *hw = &pf->hw; 6237 int v_budget, i; 6238 int vec; 6239 6240 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED)) 6241 return -ENODEV; 6242 6243 /* The number of vectors we'll request will be comprised of: 6244 * - Add 1 for "other" cause for Admin Queue events, etc. 6245 * - The number of LAN queue pairs 6246 * - Queues being used for RSS. 6247 * We don't need as many as max_rss_size vectors. 6248 * use rss_size instead in the calculation since that 6249 * is governed by number of cpus in the system. 6250 * - assumes symmetric Tx/Rx pairing 6251 * - The number of VMDq pairs 6252 * Once we count this up, try the request. 6253 * 6254 * If we can't get what we want, we'll simplify to nearly nothing 6255 * and try again. If that still fails, we punt. 6256 */ 6257 pf->num_lan_msix = pf->num_lan_qps - (pf->rss_size_max - pf->rss_size); 6258 pf->num_vmdq_msix = pf->num_vmdq_qps; 6259 v_budget = 1 + pf->num_lan_msix; 6260 v_budget += (pf->num_vmdq_vsis * pf->num_vmdq_msix); 6261 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) 6262 v_budget++; 6263 6264 /* Scale down if necessary, and the rings will share vectors */ 6265 v_budget = min_t(int, v_budget, hw->func_caps.num_msix_vectors); 6266 6267 pf->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry), 6268 GFP_KERNEL); 6269 if (!pf->msix_entries) 6270 return -ENOMEM; 6271 6272 for (i = 0; i < v_budget; i++) 6273 pf->msix_entries[i].entry = i; 6274 vec = i40e_reserve_msix_vectors(pf, v_budget); 6275 6276 if (vec != v_budget) { 6277 /* If we have limited resources, we will start with no vectors 6278 * for the special features and then allocate vectors to some 6279 * of these features based on the policy and at the end disable 6280 * the features that did not get any vectors. 6281 */ 6282 pf->num_vmdq_msix = 0; 6283 } 6284 6285 if (vec < I40E_MIN_MSIX) { 6286 pf->flags &= ~I40E_FLAG_MSIX_ENABLED; 6287 kfree(pf->msix_entries); 6288 pf->msix_entries = NULL; 6289 return -ENODEV; 6290 6291 } else if (vec == I40E_MIN_MSIX) { 6292 /* Adjust for minimal MSIX use */ 6293 pf->num_vmdq_vsis = 0; 6294 pf->num_vmdq_qps = 0; 6295 pf->num_lan_qps = 1; 6296 pf->num_lan_msix = 1; 6297 6298 } else if (vec != v_budget) { 6299 /* reserve the misc vector */ 6300 vec--; 6301 6302 /* Scale vector usage down */ 6303 pf->num_vmdq_msix = 1; /* force VMDqs to only one vector */ 6304 pf->num_vmdq_vsis = 1; 6305 6306 /* partition out the remaining vectors */ 6307 switch (vec) { 6308 case 2: 6309 pf->num_lan_msix = 1; 6310 break; 6311 case 3: 6312 pf->num_lan_msix = 2; 6313 break; 6314 default: 6315 pf->num_lan_msix = min_t(int, (vec / 2), 6316 pf->num_lan_qps); 6317 pf->num_vmdq_vsis = min_t(int, (vec - pf->num_lan_msix), 6318 I40E_DEFAULT_NUM_VMDQ_VSI); 6319 break; 6320 } 6321 } 6322 6323 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 6324 (pf->num_vmdq_msix == 0)) { 6325 dev_info(&pf->pdev->dev, "VMDq disabled, not enough MSI-X vectors\n"); 6326 pf->flags &= ~I40E_FLAG_VMDQ_ENABLED; 6327 } 6328 return err; 6329 } 6330 6331 /** 6332 * i40e_vsi_alloc_q_vector - Allocate memory for a single interrupt vector 6333 * @vsi: the VSI being configured 6334 * @v_idx: index of the vector in the vsi struct 6335 * 6336 * We allocate one q_vector. If allocation fails we return -ENOMEM. 6337 **/ 6338 static int i40e_vsi_alloc_q_vector(struct i40e_vsi *vsi, int v_idx) 6339 { 6340 struct i40e_q_vector *q_vector; 6341 6342 /* allocate q_vector */ 6343 q_vector = kzalloc(sizeof(struct i40e_q_vector), GFP_KERNEL); 6344 if (!q_vector) 6345 return -ENOMEM; 6346 6347 q_vector->vsi = vsi; 6348 q_vector->v_idx = v_idx; 6349 cpumask_set_cpu(v_idx, &q_vector->affinity_mask); 6350 if (vsi->netdev) 6351 netif_napi_add(vsi->netdev, &q_vector->napi, 6352 i40e_napi_poll, NAPI_POLL_WEIGHT); 6353 6354 q_vector->rx.latency_range = I40E_LOW_LATENCY; 6355 q_vector->tx.latency_range = I40E_LOW_LATENCY; 6356 6357 /* tie q_vector and vsi together */ 6358 vsi->q_vectors[v_idx] = q_vector; 6359 6360 return 0; 6361 } 6362 6363 /** 6364 * i40e_vsi_alloc_q_vectors - Allocate memory for interrupt vectors 6365 * @vsi: the VSI being configured 6366 * 6367 * We allocate one q_vector per queue interrupt. If allocation fails we 6368 * return -ENOMEM. 6369 **/ 6370 static int i40e_vsi_alloc_q_vectors(struct i40e_vsi *vsi) 6371 { 6372 struct i40e_pf *pf = vsi->back; 6373 int v_idx, num_q_vectors; 6374 int err; 6375 6376 /* if not MSIX, give the one vector only to the LAN VSI */ 6377 if (pf->flags & I40E_FLAG_MSIX_ENABLED) 6378 num_q_vectors = vsi->num_q_vectors; 6379 else if (vsi == pf->vsi[pf->lan_vsi]) 6380 num_q_vectors = 1; 6381 else 6382 return -EINVAL; 6383 6384 for (v_idx = 0; v_idx < num_q_vectors; v_idx++) { 6385 err = i40e_vsi_alloc_q_vector(vsi, v_idx); 6386 if (err) 6387 goto err_out; 6388 } 6389 6390 return 0; 6391 6392 err_out: 6393 while (v_idx--) 6394 i40e_free_q_vector(vsi, v_idx); 6395 6396 return err; 6397 } 6398 6399 /** 6400 * i40e_init_interrupt_scheme - Determine proper interrupt scheme 6401 * @pf: board private structure to initialize 6402 **/ 6403 static void i40e_init_interrupt_scheme(struct i40e_pf *pf) 6404 { 6405 int err = 0; 6406 6407 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 6408 err = i40e_init_msix(pf); 6409 if (err) { 6410 pf->flags &= ~(I40E_FLAG_MSIX_ENABLED | 6411 I40E_FLAG_RSS_ENABLED | 6412 I40E_FLAG_DCB_CAPABLE | 6413 I40E_FLAG_SRIOV_ENABLED | 6414 I40E_FLAG_FD_SB_ENABLED | 6415 I40E_FLAG_FD_ATR_ENABLED | 6416 I40E_FLAG_VMDQ_ENABLED); 6417 6418 /* rework the queue expectations without MSIX */ 6419 i40e_determine_queue_usage(pf); 6420 } 6421 } 6422 6423 if (!(pf->flags & I40E_FLAG_MSIX_ENABLED) && 6424 (pf->flags & I40E_FLAG_MSI_ENABLED)) { 6425 dev_info(&pf->pdev->dev, "MSI-X not available, trying MSI\n"); 6426 err = pci_enable_msi(pf->pdev); 6427 if (err) { 6428 dev_info(&pf->pdev->dev, "MSI init failed - %d\n", err); 6429 pf->flags &= ~I40E_FLAG_MSI_ENABLED; 6430 } 6431 } 6432 6433 if (!(pf->flags & (I40E_FLAG_MSIX_ENABLED | I40E_FLAG_MSI_ENABLED))) 6434 dev_info(&pf->pdev->dev, "MSI-X and MSI not available, falling back to Legacy IRQ\n"); 6435 6436 /* track first vector for misc interrupts */ 6437 err = i40e_get_lump(pf, pf->irq_pile, 1, I40E_PILE_VALID_BIT-1); 6438 } 6439 6440 /** 6441 * i40e_setup_misc_vector - Setup the misc vector to handle non queue events 6442 * @pf: board private structure 6443 * 6444 * This sets up the handler for MSIX 0, which is used to manage the 6445 * non-queue interrupts, e.g. AdminQ and errors. This is not used 6446 * when in MSI or Legacy interrupt mode. 6447 **/ 6448 static int i40e_setup_misc_vector(struct i40e_pf *pf) 6449 { 6450 struct i40e_hw *hw = &pf->hw; 6451 int err = 0; 6452 6453 /* Only request the irq if this is the first time through, and 6454 * not when we're rebuilding after a Reset 6455 */ 6456 if (!test_bit(__I40E_RESET_RECOVERY_PENDING, &pf->state)) { 6457 err = request_irq(pf->msix_entries[0].vector, 6458 i40e_intr, 0, pf->misc_int_name, pf); 6459 if (err) { 6460 dev_info(&pf->pdev->dev, 6461 "request_irq for %s failed: %d\n", 6462 pf->misc_int_name, err); 6463 return -EFAULT; 6464 } 6465 } 6466 6467 i40e_enable_misc_int_causes(hw); 6468 6469 /* associate no queues to the misc vector */ 6470 wr32(hw, I40E_PFINT_LNKLST0, I40E_QUEUE_END_OF_LIST); 6471 wr32(hw, I40E_PFINT_ITR0(I40E_RX_ITR), I40E_ITR_8K); 6472 6473 i40e_flush(hw); 6474 6475 i40e_irq_dynamic_enable_icr0(pf); 6476 6477 return err; 6478 } 6479 6480 /** 6481 * i40e_config_rss - Prepare for RSS if used 6482 * @pf: board private structure 6483 **/ 6484 static int i40e_config_rss(struct i40e_pf *pf) 6485 { 6486 /* Set of random keys generated using kernel random number generator */ 6487 static const u32 seed[I40E_PFQF_HKEY_MAX_INDEX + 1] = {0x41b01687, 6488 0x183cfd8c, 0xce880440, 0x580cbc3c, 0x35897377, 6489 0x328b25e1, 0x4fa98922, 0xb7d90c14, 0xd5bad70d, 6490 0xcd15a2c1, 0xe8580225, 0x4a1e9d11, 0xfe5731be}; 6491 struct i40e_hw *hw = &pf->hw; 6492 u32 lut = 0; 6493 int i, j; 6494 u64 hena; 6495 6496 /* Fill out hash function seed */ 6497 for (i = 0; i <= I40E_PFQF_HKEY_MAX_INDEX; i++) 6498 wr32(hw, I40E_PFQF_HKEY(i), seed[i]); 6499 6500 /* By default we enable TCP/UDP with IPv4/IPv6 ptypes */ 6501 hena = (u64)rd32(hw, I40E_PFQF_HENA(0)) | 6502 ((u64)rd32(hw, I40E_PFQF_HENA(1)) << 32); 6503 hena |= I40E_DEFAULT_RSS_HENA; 6504 wr32(hw, I40E_PFQF_HENA(0), (u32)hena); 6505 wr32(hw, I40E_PFQF_HENA(1), (u32)(hena >> 32)); 6506 6507 /* Populate the LUT with max no. of queues in round robin fashion */ 6508 for (i = 0, j = 0; i < pf->hw.func_caps.rss_table_size; i++, j++) { 6509 6510 /* The assumption is that lan qp count will be the highest 6511 * qp count for any PF VSI that needs RSS. 6512 * If multiple VSIs need RSS support, all the qp counts 6513 * for those VSIs should be a power of 2 for RSS to work. 6514 * If LAN VSI is the only consumer for RSS then this requirement 6515 * is not necessary. 6516 */ 6517 if (j == pf->rss_size) 6518 j = 0; 6519 /* lut = 4-byte sliding window of 4 lut entries */ 6520 lut = (lut << 8) | (j & 6521 ((0x1 << pf->hw.func_caps.rss_table_entry_width) - 1)); 6522 /* On i = 3, we have 4 entries in lut; write to the register */ 6523 if ((i & 3) == 3) 6524 wr32(hw, I40E_PFQF_HLUT(i >> 2), lut); 6525 } 6526 i40e_flush(hw); 6527 6528 return 0; 6529 } 6530 6531 /** 6532 * i40e_reconfig_rss_queues - change number of queues for rss and rebuild 6533 * @pf: board private structure 6534 * @queue_count: the requested queue count for rss. 6535 * 6536 * returns 0 if rss is not enabled, if enabled returns the final rss queue 6537 * count which may be different from the requested queue count. 6538 **/ 6539 int i40e_reconfig_rss_queues(struct i40e_pf *pf, int queue_count) 6540 { 6541 if (!(pf->flags & I40E_FLAG_RSS_ENABLED)) 6542 return 0; 6543 6544 queue_count = min_t(int, queue_count, pf->rss_size_max); 6545 6546 if (queue_count != pf->rss_size) { 6547 i40e_prep_for_reset(pf); 6548 6549 pf->rss_size = queue_count; 6550 6551 i40e_reset_and_rebuild(pf, true); 6552 i40e_config_rss(pf); 6553 } 6554 dev_info(&pf->pdev->dev, "RSS count: %d\n", pf->rss_size); 6555 return pf->rss_size; 6556 } 6557 6558 /** 6559 * i40e_sw_init - Initialize general software structures (struct i40e_pf) 6560 * @pf: board private structure to initialize 6561 * 6562 * i40e_sw_init initializes the Adapter private data structure. 6563 * Fields are initialized based on PCI device information and 6564 * OS network device settings (MTU size). 6565 **/ 6566 static int i40e_sw_init(struct i40e_pf *pf) 6567 { 6568 int err = 0; 6569 int size; 6570 6571 pf->msg_enable = netif_msg_init(I40E_DEFAULT_MSG_ENABLE, 6572 (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)); 6573 pf->hw.debug_mask = pf->msg_enable | I40E_DEBUG_DIAG; 6574 if (debug != -1 && debug != I40E_DEFAULT_MSG_ENABLE) { 6575 if (I40E_DEBUG_USER & debug) 6576 pf->hw.debug_mask = debug; 6577 pf->msg_enable = netif_msg_init((debug & ~I40E_DEBUG_USER), 6578 I40E_DEFAULT_MSG_ENABLE); 6579 } 6580 6581 /* Set default capability flags */ 6582 pf->flags = I40E_FLAG_RX_CSUM_ENABLED | 6583 I40E_FLAG_MSI_ENABLED | 6584 I40E_FLAG_MSIX_ENABLED | 6585 I40E_FLAG_RX_1BUF_ENABLED; 6586 6587 /* Set default ITR */ 6588 pf->rx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_RX_DEF; 6589 pf->tx_itr_default = I40E_ITR_DYNAMIC | I40E_ITR_TX_DEF; 6590 6591 /* Depending on PF configurations, it is possible that the RSS 6592 * maximum might end up larger than the available queues 6593 */ 6594 pf->rss_size_max = 0x1 << pf->hw.func_caps.rss_table_entry_width; 6595 pf->rss_size_max = min_t(int, pf->rss_size_max, 6596 pf->hw.func_caps.num_tx_qp); 6597 if (pf->hw.func_caps.rss) { 6598 pf->flags |= I40E_FLAG_RSS_ENABLED; 6599 pf->rss_size = min_t(int, pf->rss_size_max, num_online_cpus()); 6600 } else { 6601 pf->rss_size = 1; 6602 } 6603 6604 /* MFP mode enabled */ 6605 if (pf->hw.func_caps.npar_enable || pf->hw.func_caps.mfp_mode_1) { 6606 pf->flags |= I40E_FLAG_MFP_ENABLED; 6607 dev_info(&pf->pdev->dev, "MFP mode Enabled\n"); 6608 } 6609 6610 /* FW/NVM is not yet fixed in this regard */ 6611 if ((pf->hw.func_caps.fd_filters_guaranteed > 0) || 6612 (pf->hw.func_caps.fd_filters_best_effort > 0)) { 6613 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 6614 pf->atr_sample_rate = I40E_DEFAULT_ATR_SAMPLE_RATE; 6615 /* Setup a counter for fd_atr per pf */ 6616 pf->fd_atr_cnt_idx = I40E_FD_ATR_STAT_IDX(pf->hw.pf_id); 6617 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) { 6618 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 6619 /* Setup a counter for fd_sb per pf */ 6620 pf->fd_sb_cnt_idx = I40E_FD_SB_STAT_IDX(pf->hw.pf_id); 6621 } else { 6622 dev_info(&pf->pdev->dev, 6623 "Flow Director Sideband mode Disabled in MFP mode\n"); 6624 } 6625 pf->fdir_pf_filter_count = 6626 pf->hw.func_caps.fd_filters_guaranteed; 6627 pf->hw.fdir_shared_filter_count = 6628 pf->hw.func_caps.fd_filters_best_effort; 6629 } 6630 6631 if (pf->hw.func_caps.vmdq) { 6632 pf->flags |= I40E_FLAG_VMDQ_ENABLED; 6633 pf->num_vmdq_vsis = I40E_DEFAULT_NUM_VMDQ_VSI; 6634 pf->num_vmdq_qps = I40E_DEFAULT_QUEUES_PER_VMDQ; 6635 } 6636 6637 #ifdef CONFIG_PCI_IOV 6638 if (pf->hw.func_caps.num_vfs) { 6639 pf->num_vf_qps = I40E_DEFAULT_QUEUES_PER_VF; 6640 pf->flags |= I40E_FLAG_SRIOV_ENABLED; 6641 pf->num_req_vfs = min_t(int, 6642 pf->hw.func_caps.num_vfs, 6643 I40E_MAX_VF_COUNT); 6644 } 6645 #endif /* CONFIG_PCI_IOV */ 6646 pf->eeprom_version = 0xDEAD; 6647 pf->lan_veb = I40E_NO_VEB; 6648 pf->lan_vsi = I40E_NO_VSI; 6649 6650 /* set up queue assignment tracking */ 6651 size = sizeof(struct i40e_lump_tracking) 6652 + (sizeof(u16) * pf->hw.func_caps.num_tx_qp); 6653 pf->qp_pile = kzalloc(size, GFP_KERNEL); 6654 if (!pf->qp_pile) { 6655 err = -ENOMEM; 6656 goto sw_init_done; 6657 } 6658 pf->qp_pile->num_entries = pf->hw.func_caps.num_tx_qp; 6659 pf->qp_pile->search_hint = 0; 6660 6661 /* set up vector assignment tracking */ 6662 size = sizeof(struct i40e_lump_tracking) 6663 + (sizeof(u16) * pf->hw.func_caps.num_msix_vectors); 6664 pf->irq_pile = kzalloc(size, GFP_KERNEL); 6665 if (!pf->irq_pile) { 6666 kfree(pf->qp_pile); 6667 err = -ENOMEM; 6668 goto sw_init_done; 6669 } 6670 pf->irq_pile->num_entries = pf->hw.func_caps.num_msix_vectors; 6671 pf->irq_pile->search_hint = 0; 6672 6673 mutex_init(&pf->switch_mutex); 6674 6675 sw_init_done: 6676 return err; 6677 } 6678 6679 /** 6680 * i40e_set_ntuple - set the ntuple feature flag and take action 6681 * @pf: board private structure to initialize 6682 * @features: the feature set that the stack is suggesting 6683 * 6684 * returns a bool to indicate if reset needs to happen 6685 **/ 6686 bool i40e_set_ntuple(struct i40e_pf *pf, netdev_features_t features) 6687 { 6688 bool need_reset = false; 6689 6690 /* Check if Flow Director n-tuple support was enabled or disabled. If 6691 * the state changed, we need to reset. 6692 */ 6693 if (features & NETIF_F_NTUPLE) { 6694 /* Enable filters and mark for reset */ 6695 if (!(pf->flags & I40E_FLAG_FD_SB_ENABLED)) 6696 need_reset = true; 6697 pf->flags |= I40E_FLAG_FD_SB_ENABLED; 6698 } else { 6699 /* turn off filters, mark for reset and clear SW filter list */ 6700 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 6701 need_reset = true; 6702 i40e_fdir_filter_exit(pf); 6703 } 6704 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 6705 /* if ATR was disabled it can be re-enabled. */ 6706 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED)) 6707 pf->flags |= I40E_FLAG_FD_ATR_ENABLED; 6708 } 6709 return need_reset; 6710 } 6711 6712 /** 6713 * i40e_set_features - set the netdev feature flags 6714 * @netdev: ptr to the netdev being adjusted 6715 * @features: the feature set that the stack is suggesting 6716 **/ 6717 static int i40e_set_features(struct net_device *netdev, 6718 netdev_features_t features) 6719 { 6720 struct i40e_netdev_priv *np = netdev_priv(netdev); 6721 struct i40e_vsi *vsi = np->vsi; 6722 struct i40e_pf *pf = vsi->back; 6723 bool need_reset; 6724 6725 if (features & NETIF_F_HW_VLAN_CTAG_RX) 6726 i40e_vlan_stripping_enable(vsi); 6727 else 6728 i40e_vlan_stripping_disable(vsi); 6729 6730 need_reset = i40e_set_ntuple(pf, features); 6731 6732 if (need_reset) 6733 i40e_do_reset(pf, (1 << __I40E_PF_RESET_REQUESTED)); 6734 6735 return 0; 6736 } 6737 6738 #ifdef CONFIG_I40E_VXLAN 6739 /** 6740 * i40e_get_vxlan_port_idx - Lookup a possibly offloaded for Rx UDP port 6741 * @pf: board private structure 6742 * @port: The UDP port to look up 6743 * 6744 * Returns the index number or I40E_MAX_PF_UDP_OFFLOAD_PORTS if port not found 6745 **/ 6746 static u8 i40e_get_vxlan_port_idx(struct i40e_pf *pf, __be16 port) 6747 { 6748 u8 i; 6749 6750 for (i = 0; i < I40E_MAX_PF_UDP_OFFLOAD_PORTS; i++) { 6751 if (pf->vxlan_ports[i] == port) 6752 return i; 6753 } 6754 6755 return i; 6756 } 6757 6758 /** 6759 * i40e_add_vxlan_port - Get notifications about VXLAN ports that come up 6760 * @netdev: This physical port's netdev 6761 * @sa_family: Socket Family that VXLAN is notifying us about 6762 * @port: New UDP port number that VXLAN started listening to 6763 **/ 6764 static void i40e_add_vxlan_port(struct net_device *netdev, 6765 sa_family_t sa_family, __be16 port) 6766 { 6767 struct i40e_netdev_priv *np = netdev_priv(netdev); 6768 struct i40e_vsi *vsi = np->vsi; 6769 struct i40e_pf *pf = vsi->back; 6770 u8 next_idx; 6771 u8 idx; 6772 6773 if (sa_family == AF_INET6) 6774 return; 6775 6776 idx = i40e_get_vxlan_port_idx(pf, port); 6777 6778 /* Check if port already exists */ 6779 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 6780 netdev_info(netdev, "Port %d already offloaded\n", ntohs(port)); 6781 return; 6782 } 6783 6784 /* Now check if there is space to add the new port */ 6785 next_idx = i40e_get_vxlan_port_idx(pf, 0); 6786 6787 if (next_idx == I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 6788 netdev_info(netdev, "Maximum number of UDP ports reached, not adding port %d\n", 6789 ntohs(port)); 6790 return; 6791 } 6792 6793 /* New port: add it and mark its index in the bitmap */ 6794 pf->vxlan_ports[next_idx] = port; 6795 pf->pending_vxlan_bitmap |= (1 << next_idx); 6796 6797 pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC; 6798 } 6799 6800 /** 6801 * i40e_del_vxlan_port - Get notifications about VXLAN ports that go away 6802 * @netdev: This physical port's netdev 6803 * @sa_family: Socket Family that VXLAN is notifying us about 6804 * @port: UDP port number that VXLAN stopped listening to 6805 **/ 6806 static void i40e_del_vxlan_port(struct net_device *netdev, 6807 sa_family_t sa_family, __be16 port) 6808 { 6809 struct i40e_netdev_priv *np = netdev_priv(netdev); 6810 struct i40e_vsi *vsi = np->vsi; 6811 struct i40e_pf *pf = vsi->back; 6812 u8 idx; 6813 6814 if (sa_family == AF_INET6) 6815 return; 6816 6817 idx = i40e_get_vxlan_port_idx(pf, port); 6818 6819 /* Check if port already exists */ 6820 if (idx < I40E_MAX_PF_UDP_OFFLOAD_PORTS) { 6821 /* if port exists, set it to 0 (mark for deletion) 6822 * and make it pending 6823 */ 6824 pf->vxlan_ports[idx] = 0; 6825 6826 pf->pending_vxlan_bitmap |= (1 << idx); 6827 6828 pf->flags |= I40E_FLAG_VXLAN_FILTER_SYNC; 6829 } else { 6830 netdev_warn(netdev, "Port %d was not found, not deleting\n", 6831 ntohs(port)); 6832 } 6833 } 6834 6835 #endif 6836 #ifdef HAVE_FDB_OPS 6837 #ifdef USE_CONST_DEV_UC_CHAR 6838 static int i40e_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], 6839 struct net_device *dev, 6840 const unsigned char *addr, 6841 u16 flags) 6842 #else 6843 static int i40e_ndo_fdb_add(struct ndmsg *ndm, 6844 struct net_device *dev, 6845 unsigned char *addr, 6846 u16 flags) 6847 #endif 6848 { 6849 struct i40e_netdev_priv *np = netdev_priv(dev); 6850 struct i40e_pf *pf = np->vsi->back; 6851 int err = 0; 6852 6853 if (!(pf->flags & I40E_FLAG_SRIOV_ENABLED)) 6854 return -EOPNOTSUPP; 6855 6856 /* Hardware does not support aging addresses so if a 6857 * ndm_state is given only allow permanent addresses 6858 */ 6859 if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) { 6860 netdev_info(dev, "FDB only supports static addresses\n"); 6861 return -EINVAL; 6862 } 6863 6864 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) 6865 err = dev_uc_add_excl(dev, addr); 6866 else if (is_multicast_ether_addr(addr)) 6867 err = dev_mc_add_excl(dev, addr); 6868 else 6869 err = -EINVAL; 6870 6871 /* Only return duplicate errors if NLM_F_EXCL is set */ 6872 if (err == -EEXIST && !(flags & NLM_F_EXCL)) 6873 err = 0; 6874 6875 return err; 6876 } 6877 6878 #ifndef USE_DEFAULT_FDB_DEL_DUMP 6879 #ifdef USE_CONST_DEV_UC_CHAR 6880 static int i40e_ndo_fdb_del(struct ndmsg *ndm, 6881 struct net_device *dev, 6882 const unsigned char *addr) 6883 #else 6884 static int i40e_ndo_fdb_del(struct ndmsg *ndm, 6885 struct net_device *dev, 6886 unsigned char *addr) 6887 #endif 6888 { 6889 struct i40e_netdev_priv *np = netdev_priv(dev); 6890 struct i40e_pf *pf = np->vsi->back; 6891 int err = -EOPNOTSUPP; 6892 6893 if (ndm->ndm_state & NUD_PERMANENT) { 6894 netdev_info(dev, "FDB only supports static addresses\n"); 6895 return -EINVAL; 6896 } 6897 6898 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 6899 if (is_unicast_ether_addr(addr)) 6900 err = dev_uc_del(dev, addr); 6901 else if (is_multicast_ether_addr(addr)) 6902 err = dev_mc_del(dev, addr); 6903 else 6904 err = -EINVAL; 6905 } 6906 6907 return err; 6908 } 6909 6910 static int i40e_ndo_fdb_dump(struct sk_buff *skb, 6911 struct netlink_callback *cb, 6912 struct net_device *dev, 6913 int idx) 6914 { 6915 struct i40e_netdev_priv *np = netdev_priv(dev); 6916 struct i40e_pf *pf = np->vsi->back; 6917 6918 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) 6919 idx = ndo_dflt_fdb_dump(skb, cb, dev, idx); 6920 6921 return idx; 6922 } 6923 6924 #endif /* USE_DEFAULT_FDB_DEL_DUMP */ 6925 #endif /* HAVE_FDB_OPS */ 6926 static const struct net_device_ops i40e_netdev_ops = { 6927 .ndo_open = i40e_open, 6928 .ndo_stop = i40e_close, 6929 .ndo_start_xmit = i40e_lan_xmit_frame, 6930 .ndo_get_stats64 = i40e_get_netdev_stats_struct, 6931 .ndo_set_rx_mode = i40e_set_rx_mode, 6932 .ndo_validate_addr = eth_validate_addr, 6933 .ndo_set_mac_address = i40e_set_mac, 6934 .ndo_change_mtu = i40e_change_mtu, 6935 .ndo_do_ioctl = i40e_ioctl, 6936 .ndo_tx_timeout = i40e_tx_timeout, 6937 .ndo_vlan_rx_add_vid = i40e_vlan_rx_add_vid, 6938 .ndo_vlan_rx_kill_vid = i40e_vlan_rx_kill_vid, 6939 #ifdef CONFIG_NET_POLL_CONTROLLER 6940 .ndo_poll_controller = i40e_netpoll, 6941 #endif 6942 .ndo_setup_tc = i40e_setup_tc, 6943 .ndo_set_features = i40e_set_features, 6944 .ndo_set_vf_mac = i40e_ndo_set_vf_mac, 6945 .ndo_set_vf_vlan = i40e_ndo_set_vf_port_vlan, 6946 .ndo_set_vf_rate = i40e_ndo_set_vf_bw, 6947 .ndo_get_vf_config = i40e_ndo_get_vf_config, 6948 .ndo_set_vf_link_state = i40e_ndo_set_vf_link_state, 6949 .ndo_set_vf_spoofchk = i40e_ndo_set_vf_spoofck, 6950 #ifdef CONFIG_I40E_VXLAN 6951 .ndo_add_vxlan_port = i40e_add_vxlan_port, 6952 .ndo_del_vxlan_port = i40e_del_vxlan_port, 6953 #endif 6954 #ifdef HAVE_FDB_OPS 6955 .ndo_fdb_add = i40e_ndo_fdb_add, 6956 #ifndef USE_DEFAULT_FDB_DEL_DUMP 6957 .ndo_fdb_del = i40e_ndo_fdb_del, 6958 .ndo_fdb_dump = i40e_ndo_fdb_dump, 6959 #endif 6960 #endif 6961 }; 6962 6963 /** 6964 * i40e_config_netdev - Setup the netdev flags 6965 * @vsi: the VSI being configured 6966 * 6967 * Returns 0 on success, negative value on failure 6968 **/ 6969 static int i40e_config_netdev(struct i40e_vsi *vsi) 6970 { 6971 u8 brdcast[ETH_ALEN] = {0xff, 0xff, 0xff, 0xff, 0xff, 0xff}; 6972 struct i40e_pf *pf = vsi->back; 6973 struct i40e_hw *hw = &pf->hw; 6974 struct i40e_netdev_priv *np; 6975 struct net_device *netdev; 6976 u8 mac_addr[ETH_ALEN]; 6977 int etherdev_size; 6978 6979 etherdev_size = sizeof(struct i40e_netdev_priv); 6980 netdev = alloc_etherdev_mq(etherdev_size, vsi->alloc_queue_pairs); 6981 if (!netdev) 6982 return -ENOMEM; 6983 6984 vsi->netdev = netdev; 6985 np = netdev_priv(netdev); 6986 np->vsi = vsi; 6987 6988 netdev->hw_enc_features |= NETIF_F_IP_CSUM | 6989 NETIF_F_GSO_UDP_TUNNEL | 6990 NETIF_F_TSO; 6991 6992 netdev->features = NETIF_F_SG | 6993 NETIF_F_IP_CSUM | 6994 NETIF_F_SCTP_CSUM | 6995 NETIF_F_HIGHDMA | 6996 NETIF_F_GSO_UDP_TUNNEL | 6997 NETIF_F_HW_VLAN_CTAG_TX | 6998 NETIF_F_HW_VLAN_CTAG_RX | 6999 NETIF_F_HW_VLAN_CTAG_FILTER | 7000 NETIF_F_IPV6_CSUM | 7001 NETIF_F_TSO | 7002 NETIF_F_TSO_ECN | 7003 NETIF_F_TSO6 | 7004 NETIF_F_RXCSUM | 7005 NETIF_F_RXHASH | 7006 0; 7007 7008 if (!(pf->flags & I40E_FLAG_MFP_ENABLED)) 7009 netdev->features |= NETIF_F_NTUPLE; 7010 7011 /* copy netdev features into list of user selectable features */ 7012 netdev->hw_features |= netdev->features; 7013 7014 if (vsi->type == I40E_VSI_MAIN) { 7015 SET_NETDEV_DEV(netdev, &pf->pdev->dev); 7016 ether_addr_copy(mac_addr, hw->mac.perm_addr); 7017 /* The following two steps are necessary to prevent reception 7018 * of tagged packets - by default the NVM loads a MAC-VLAN 7019 * filter that will accept any tagged packet. This is to 7020 * prevent that during normal operations until a specific 7021 * VLAN tag filter has been set. 7022 */ 7023 i40e_rm_default_mac_filter(vsi, mac_addr); 7024 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, true); 7025 } else { 7026 /* relate the VSI_VMDQ name to the VSI_MAIN name */ 7027 snprintf(netdev->name, IFNAMSIZ, "%sv%%d", 7028 pf->vsi[pf->lan_vsi]->netdev->name); 7029 random_ether_addr(mac_addr); 7030 i40e_add_filter(vsi, mac_addr, I40E_VLAN_ANY, false, false); 7031 } 7032 i40e_add_filter(vsi, brdcast, I40E_VLAN_ANY, false, false); 7033 7034 ether_addr_copy(netdev->dev_addr, mac_addr); 7035 ether_addr_copy(netdev->perm_addr, mac_addr); 7036 /* vlan gets same features (except vlan offload) 7037 * after any tweaks for specific VSI types 7038 */ 7039 netdev->vlan_features = netdev->features & ~(NETIF_F_HW_VLAN_CTAG_TX | 7040 NETIF_F_HW_VLAN_CTAG_RX | 7041 NETIF_F_HW_VLAN_CTAG_FILTER); 7042 netdev->priv_flags |= IFF_UNICAST_FLT; 7043 netdev->priv_flags |= IFF_SUPP_NOFCS; 7044 /* Setup netdev TC information */ 7045 i40e_vsi_config_netdev_tc(vsi, vsi->tc_config.enabled_tc); 7046 7047 netdev->netdev_ops = &i40e_netdev_ops; 7048 netdev->watchdog_timeo = 5 * HZ; 7049 i40e_set_ethtool_ops(netdev); 7050 7051 return 0; 7052 } 7053 7054 /** 7055 * i40e_vsi_delete - Delete a VSI from the switch 7056 * @vsi: the VSI being removed 7057 * 7058 * Returns 0 on success, negative value on failure 7059 **/ 7060 static void i40e_vsi_delete(struct i40e_vsi *vsi) 7061 { 7062 /* remove default VSI is not allowed */ 7063 if (vsi == vsi->back->vsi[vsi->back->lan_vsi]) 7064 return; 7065 7066 i40e_aq_delete_element(&vsi->back->hw, vsi->seid, NULL); 7067 } 7068 7069 /** 7070 * i40e_add_vsi - Add a VSI to the switch 7071 * @vsi: the VSI being configured 7072 * 7073 * This initializes a VSI context depending on the VSI type to be added and 7074 * passes it down to the add_vsi aq command. 7075 **/ 7076 static int i40e_add_vsi(struct i40e_vsi *vsi) 7077 { 7078 int ret = -ENODEV; 7079 struct i40e_mac_filter *f, *ftmp; 7080 struct i40e_pf *pf = vsi->back; 7081 struct i40e_hw *hw = &pf->hw; 7082 struct i40e_vsi_context ctxt; 7083 u8 enabled_tc = 0x1; /* TC0 enabled */ 7084 int f_count = 0; 7085 7086 memset(&ctxt, 0, sizeof(ctxt)); 7087 switch (vsi->type) { 7088 case I40E_VSI_MAIN: 7089 /* The PF's main VSI is already setup as part of the 7090 * device initialization, so we'll not bother with 7091 * the add_vsi call, but we will retrieve the current 7092 * VSI context. 7093 */ 7094 ctxt.seid = pf->main_vsi_seid; 7095 ctxt.pf_num = pf->hw.pf_id; 7096 ctxt.vf_num = 0; 7097 ret = i40e_aq_get_vsi_params(&pf->hw, &ctxt, NULL); 7098 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 7099 if (ret) { 7100 dev_info(&pf->pdev->dev, 7101 "couldn't get pf vsi config, err %d, aq_err %d\n", 7102 ret, pf->hw.aq.asq_last_status); 7103 return -ENOENT; 7104 } 7105 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info)); 7106 vsi->info.valid_sections = 0; 7107 7108 vsi->seid = ctxt.seid; 7109 vsi->id = ctxt.vsi_number; 7110 7111 enabled_tc = i40e_pf_get_tc_map(pf); 7112 7113 /* MFP mode setup queue map and update VSI */ 7114 if (pf->flags & I40E_FLAG_MFP_ENABLED) { 7115 memset(&ctxt, 0, sizeof(ctxt)); 7116 ctxt.seid = pf->main_vsi_seid; 7117 ctxt.pf_num = pf->hw.pf_id; 7118 ctxt.vf_num = 0; 7119 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, false); 7120 ret = i40e_aq_update_vsi_params(hw, &ctxt, NULL); 7121 if (ret) { 7122 dev_info(&pf->pdev->dev, 7123 "update vsi failed, aq_err=%d\n", 7124 pf->hw.aq.asq_last_status); 7125 ret = -ENOENT; 7126 goto err; 7127 } 7128 /* update the local VSI info queue map */ 7129 i40e_vsi_update_queue_map(vsi, &ctxt); 7130 vsi->info.valid_sections = 0; 7131 } else { 7132 /* Default/Main VSI is only enabled for TC0 7133 * reconfigure it to enable all TCs that are 7134 * available on the port in SFP mode. 7135 */ 7136 ret = i40e_vsi_config_tc(vsi, enabled_tc); 7137 if (ret) { 7138 dev_info(&pf->pdev->dev, 7139 "failed to configure TCs for main VSI tc_map 0x%08x, err %d, aq_err %d\n", 7140 enabled_tc, ret, 7141 pf->hw.aq.asq_last_status); 7142 ret = -ENOENT; 7143 } 7144 } 7145 break; 7146 7147 case I40E_VSI_FDIR: 7148 ctxt.pf_num = hw->pf_id; 7149 ctxt.vf_num = 0; 7150 ctxt.uplink_seid = vsi->uplink_seid; 7151 ctxt.connection_type = 0x1; /* regular data port */ 7152 ctxt.flags = I40E_AQ_VSI_TYPE_PF; 7153 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 7154 break; 7155 7156 case I40E_VSI_VMDQ2: 7157 ctxt.pf_num = hw->pf_id; 7158 ctxt.vf_num = 0; 7159 ctxt.uplink_seid = vsi->uplink_seid; 7160 ctxt.connection_type = 0x1; /* regular data port */ 7161 ctxt.flags = I40E_AQ_VSI_TYPE_VMDQ2; 7162 7163 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 7164 7165 /* This VSI is connected to VEB so the switch_id 7166 * should be set to zero by default. 7167 */ 7168 ctxt.info.switch_id = 0; 7169 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_LOCAL_LB); 7170 ctxt.info.switch_id |= cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 7171 7172 /* Setup the VSI tx/rx queue map for TC0 only for now */ 7173 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 7174 break; 7175 7176 case I40E_VSI_SRIOV: 7177 ctxt.pf_num = hw->pf_id; 7178 ctxt.vf_num = vsi->vf_id + hw->func_caps.vf_base_id; 7179 ctxt.uplink_seid = vsi->uplink_seid; 7180 ctxt.connection_type = 0x1; /* regular data port */ 7181 ctxt.flags = I40E_AQ_VSI_TYPE_VF; 7182 7183 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_SWITCH_VALID); 7184 7185 /* This VSI is connected to VEB so the switch_id 7186 * should be set to zero by default. 7187 */ 7188 ctxt.info.switch_id = cpu_to_le16(I40E_AQ_VSI_SW_ID_FLAG_ALLOW_LB); 7189 7190 ctxt.info.valid_sections |= cpu_to_le16(I40E_AQ_VSI_PROP_VLAN_VALID); 7191 ctxt.info.port_vlan_flags |= I40E_AQ_VSI_PVLAN_MODE_ALL; 7192 if (pf->vf[vsi->vf_id].spoofchk) { 7193 ctxt.info.valid_sections |= 7194 cpu_to_le16(I40E_AQ_VSI_PROP_SECURITY_VALID); 7195 ctxt.info.sec_flags |= 7196 (I40E_AQ_VSI_SEC_FLAG_ENABLE_VLAN_CHK | 7197 I40E_AQ_VSI_SEC_FLAG_ENABLE_MAC_CHK); 7198 } 7199 /* Setup the VSI tx/rx queue map for TC0 only for now */ 7200 i40e_vsi_setup_queue_map(vsi, &ctxt, enabled_tc, true); 7201 break; 7202 7203 default: 7204 return -ENODEV; 7205 } 7206 7207 if (vsi->type != I40E_VSI_MAIN) { 7208 ret = i40e_aq_add_vsi(hw, &ctxt, NULL); 7209 if (ret) { 7210 dev_info(&vsi->back->pdev->dev, 7211 "add vsi failed, aq_err=%d\n", 7212 vsi->back->hw.aq.asq_last_status); 7213 ret = -ENOENT; 7214 goto err; 7215 } 7216 memcpy(&vsi->info, &ctxt.info, sizeof(ctxt.info)); 7217 vsi->info.valid_sections = 0; 7218 vsi->seid = ctxt.seid; 7219 vsi->id = ctxt.vsi_number; 7220 } 7221 7222 /* If macvlan filters already exist, force them to get loaded */ 7223 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) { 7224 f->changed = true; 7225 f_count++; 7226 } 7227 if (f_count) { 7228 vsi->flags |= I40E_VSI_FLAG_FILTER_CHANGED; 7229 pf->flags |= I40E_FLAG_FILTER_SYNC; 7230 } 7231 7232 /* Update VSI BW information */ 7233 ret = i40e_vsi_get_bw_info(vsi); 7234 if (ret) { 7235 dev_info(&pf->pdev->dev, 7236 "couldn't get vsi bw info, err %d, aq_err %d\n", 7237 ret, pf->hw.aq.asq_last_status); 7238 /* VSI is already added so not tearing that up */ 7239 ret = 0; 7240 } 7241 7242 err: 7243 return ret; 7244 } 7245 7246 /** 7247 * i40e_vsi_release - Delete a VSI and free its resources 7248 * @vsi: the VSI being removed 7249 * 7250 * Returns 0 on success or < 0 on error 7251 **/ 7252 int i40e_vsi_release(struct i40e_vsi *vsi) 7253 { 7254 struct i40e_mac_filter *f, *ftmp; 7255 struct i40e_veb *veb = NULL; 7256 struct i40e_pf *pf; 7257 u16 uplink_seid; 7258 int i, n; 7259 7260 pf = vsi->back; 7261 7262 /* release of a VEB-owner or last VSI is not allowed */ 7263 if (vsi->flags & I40E_VSI_FLAG_VEB_OWNER) { 7264 dev_info(&pf->pdev->dev, "VSI %d has existing VEB %d\n", 7265 vsi->seid, vsi->uplink_seid); 7266 return -ENODEV; 7267 } 7268 if (vsi == pf->vsi[pf->lan_vsi] && 7269 !test_bit(__I40E_DOWN, &pf->state)) { 7270 dev_info(&pf->pdev->dev, "Can't remove PF VSI\n"); 7271 return -ENODEV; 7272 } 7273 7274 uplink_seid = vsi->uplink_seid; 7275 if (vsi->type != I40E_VSI_SRIOV) { 7276 if (vsi->netdev_registered) { 7277 vsi->netdev_registered = false; 7278 if (vsi->netdev) { 7279 /* results in a call to i40e_close() */ 7280 unregister_netdev(vsi->netdev); 7281 } 7282 } else { 7283 i40e_vsi_close(vsi); 7284 } 7285 i40e_vsi_disable_irq(vsi); 7286 } 7287 7288 list_for_each_entry_safe(f, ftmp, &vsi->mac_filter_list, list) 7289 i40e_del_filter(vsi, f->macaddr, f->vlan, 7290 f->is_vf, f->is_netdev); 7291 i40e_sync_vsi_filters(vsi); 7292 7293 i40e_vsi_delete(vsi); 7294 i40e_vsi_free_q_vectors(vsi); 7295 if (vsi->netdev) { 7296 free_netdev(vsi->netdev); 7297 vsi->netdev = NULL; 7298 } 7299 i40e_vsi_clear_rings(vsi); 7300 i40e_vsi_clear(vsi); 7301 7302 /* If this was the last thing on the VEB, except for the 7303 * controlling VSI, remove the VEB, which puts the controlling 7304 * VSI onto the next level down in the switch. 7305 * 7306 * Well, okay, there's one more exception here: don't remove 7307 * the orphan VEBs yet. We'll wait for an explicit remove request 7308 * from up the network stack. 7309 */ 7310 for (n = 0, i = 0; i < pf->num_alloc_vsi; i++) { 7311 if (pf->vsi[i] && 7312 pf->vsi[i]->uplink_seid == uplink_seid && 7313 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 7314 n++; /* count the VSIs */ 7315 } 7316 } 7317 for (i = 0; i < I40E_MAX_VEB; i++) { 7318 if (!pf->veb[i]) 7319 continue; 7320 if (pf->veb[i]->uplink_seid == uplink_seid) 7321 n++; /* count the VEBs */ 7322 if (pf->veb[i]->seid == uplink_seid) 7323 veb = pf->veb[i]; 7324 } 7325 if (n == 0 && veb && veb->uplink_seid != 0) 7326 i40e_veb_release(veb); 7327 7328 return 0; 7329 } 7330 7331 /** 7332 * i40e_vsi_setup_vectors - Set up the q_vectors for the given VSI 7333 * @vsi: ptr to the VSI 7334 * 7335 * This should only be called after i40e_vsi_mem_alloc() which allocates the 7336 * corresponding SW VSI structure and initializes num_queue_pairs for the 7337 * newly allocated VSI. 7338 * 7339 * Returns 0 on success or negative on failure 7340 **/ 7341 static int i40e_vsi_setup_vectors(struct i40e_vsi *vsi) 7342 { 7343 int ret = -ENOENT; 7344 struct i40e_pf *pf = vsi->back; 7345 7346 if (vsi->q_vectors[0]) { 7347 dev_info(&pf->pdev->dev, "VSI %d has existing q_vectors\n", 7348 vsi->seid); 7349 return -EEXIST; 7350 } 7351 7352 if (vsi->base_vector) { 7353 dev_info(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n", 7354 vsi->seid, vsi->base_vector); 7355 return -EEXIST; 7356 } 7357 7358 ret = i40e_vsi_alloc_q_vectors(vsi); 7359 if (ret) { 7360 dev_info(&pf->pdev->dev, 7361 "failed to allocate %d q_vector for VSI %d, ret=%d\n", 7362 vsi->num_q_vectors, vsi->seid, ret); 7363 vsi->num_q_vectors = 0; 7364 goto vector_setup_out; 7365 } 7366 7367 if (vsi->num_q_vectors) 7368 vsi->base_vector = i40e_get_lump(pf, pf->irq_pile, 7369 vsi->num_q_vectors, vsi->idx); 7370 if (vsi->base_vector < 0) { 7371 dev_info(&pf->pdev->dev, 7372 "failed to get queue tracking for VSI %d, err=%d\n", 7373 vsi->seid, vsi->base_vector); 7374 i40e_vsi_free_q_vectors(vsi); 7375 ret = -ENOENT; 7376 goto vector_setup_out; 7377 } 7378 7379 vector_setup_out: 7380 return ret; 7381 } 7382 7383 /** 7384 * i40e_vsi_reinit_setup - return and reallocate resources for a VSI 7385 * @vsi: pointer to the vsi. 7386 * 7387 * This re-allocates a vsi's queue resources. 7388 * 7389 * Returns pointer to the successfully allocated and configured VSI sw struct 7390 * on success, otherwise returns NULL on failure. 7391 **/ 7392 static struct i40e_vsi *i40e_vsi_reinit_setup(struct i40e_vsi *vsi) 7393 { 7394 struct i40e_pf *pf = vsi->back; 7395 u8 enabled_tc; 7396 int ret; 7397 7398 i40e_put_lump(pf->qp_pile, vsi->base_queue, vsi->idx); 7399 i40e_vsi_clear_rings(vsi); 7400 7401 i40e_vsi_free_arrays(vsi, false); 7402 i40e_set_num_rings_in_vsi(vsi); 7403 ret = i40e_vsi_alloc_arrays(vsi, false); 7404 if (ret) 7405 goto err_vsi; 7406 7407 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, vsi->idx); 7408 if (ret < 0) { 7409 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n", 7410 vsi->seid, ret); 7411 goto err_vsi; 7412 } 7413 vsi->base_queue = ret; 7414 7415 /* Update the FW view of the VSI. Force a reset of TC and queue 7416 * layout configurations. 7417 */ 7418 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 7419 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 7420 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 7421 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 7422 7423 /* assign it some queues */ 7424 ret = i40e_alloc_rings(vsi); 7425 if (ret) 7426 goto err_rings; 7427 7428 /* map all of the rings to the q_vectors */ 7429 i40e_vsi_map_rings_to_vectors(vsi); 7430 return vsi; 7431 7432 err_rings: 7433 i40e_vsi_free_q_vectors(vsi); 7434 if (vsi->netdev_registered) { 7435 vsi->netdev_registered = false; 7436 unregister_netdev(vsi->netdev); 7437 free_netdev(vsi->netdev); 7438 vsi->netdev = NULL; 7439 } 7440 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 7441 err_vsi: 7442 i40e_vsi_clear(vsi); 7443 return NULL; 7444 } 7445 7446 /** 7447 * i40e_vsi_setup - Set up a VSI by a given type 7448 * @pf: board private structure 7449 * @type: VSI type 7450 * @uplink_seid: the switch element to link to 7451 * @param1: usage depends upon VSI type. For VF types, indicates VF id 7452 * 7453 * This allocates the sw VSI structure and its queue resources, then add a VSI 7454 * to the identified VEB. 7455 * 7456 * Returns pointer to the successfully allocated and configure VSI sw struct on 7457 * success, otherwise returns NULL on failure. 7458 **/ 7459 struct i40e_vsi *i40e_vsi_setup(struct i40e_pf *pf, u8 type, 7460 u16 uplink_seid, u32 param1) 7461 { 7462 struct i40e_vsi *vsi = NULL; 7463 struct i40e_veb *veb = NULL; 7464 int ret, i; 7465 int v_idx; 7466 7467 /* The requested uplink_seid must be either 7468 * - the PF's port seid 7469 * no VEB is needed because this is the PF 7470 * or this is a Flow Director special case VSI 7471 * - seid of an existing VEB 7472 * - seid of a VSI that owns an existing VEB 7473 * - seid of a VSI that doesn't own a VEB 7474 * a new VEB is created and the VSI becomes the owner 7475 * - seid of the PF VSI, which is what creates the first VEB 7476 * this is a special case of the previous 7477 * 7478 * Find which uplink_seid we were given and create a new VEB if needed 7479 */ 7480 for (i = 0; i < I40E_MAX_VEB; i++) { 7481 if (pf->veb[i] && pf->veb[i]->seid == uplink_seid) { 7482 veb = pf->veb[i]; 7483 break; 7484 } 7485 } 7486 7487 if (!veb && uplink_seid != pf->mac_seid) { 7488 7489 for (i = 0; i < pf->num_alloc_vsi; i++) { 7490 if (pf->vsi[i] && pf->vsi[i]->seid == uplink_seid) { 7491 vsi = pf->vsi[i]; 7492 break; 7493 } 7494 } 7495 if (!vsi) { 7496 dev_info(&pf->pdev->dev, "no such uplink_seid %d\n", 7497 uplink_seid); 7498 return NULL; 7499 } 7500 7501 if (vsi->uplink_seid == pf->mac_seid) 7502 veb = i40e_veb_setup(pf, 0, pf->mac_seid, vsi->seid, 7503 vsi->tc_config.enabled_tc); 7504 else if ((vsi->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) 7505 veb = i40e_veb_setup(pf, 0, vsi->uplink_seid, vsi->seid, 7506 vsi->tc_config.enabled_tc); 7507 7508 for (i = 0; i < I40E_MAX_VEB && !veb; i++) { 7509 if (pf->veb[i] && pf->veb[i]->seid == vsi->uplink_seid) 7510 veb = pf->veb[i]; 7511 } 7512 if (!veb) { 7513 dev_info(&pf->pdev->dev, "couldn't add VEB\n"); 7514 return NULL; 7515 } 7516 7517 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 7518 uplink_seid = veb->seid; 7519 } 7520 7521 /* get vsi sw struct */ 7522 v_idx = i40e_vsi_mem_alloc(pf, type); 7523 if (v_idx < 0) 7524 goto err_alloc; 7525 vsi = pf->vsi[v_idx]; 7526 if (!vsi) 7527 goto err_alloc; 7528 vsi->type = type; 7529 vsi->veb_idx = (veb ? veb->idx : I40E_NO_VEB); 7530 7531 if (type == I40E_VSI_MAIN) 7532 pf->lan_vsi = v_idx; 7533 else if (type == I40E_VSI_SRIOV) 7534 vsi->vf_id = param1; 7535 /* assign it some queues */ 7536 ret = i40e_get_lump(pf, pf->qp_pile, vsi->alloc_queue_pairs, 7537 vsi->idx); 7538 if (ret < 0) { 7539 dev_info(&pf->pdev->dev, "VSI %d get_lump failed %d\n", 7540 vsi->seid, ret); 7541 goto err_vsi; 7542 } 7543 vsi->base_queue = ret; 7544 7545 /* get a VSI from the hardware */ 7546 vsi->uplink_seid = uplink_seid; 7547 ret = i40e_add_vsi(vsi); 7548 if (ret) 7549 goto err_vsi; 7550 7551 switch (vsi->type) { 7552 /* setup the netdev if needed */ 7553 case I40E_VSI_MAIN: 7554 case I40E_VSI_VMDQ2: 7555 ret = i40e_config_netdev(vsi); 7556 if (ret) 7557 goto err_netdev; 7558 ret = register_netdev(vsi->netdev); 7559 if (ret) 7560 goto err_netdev; 7561 vsi->netdev_registered = true; 7562 netif_carrier_off(vsi->netdev); 7563 #ifdef CONFIG_I40E_DCB 7564 /* Setup DCB netlink interface */ 7565 i40e_dcbnl_setup(vsi); 7566 #endif /* CONFIG_I40E_DCB */ 7567 /* fall through */ 7568 7569 case I40E_VSI_FDIR: 7570 /* set up vectors and rings if needed */ 7571 ret = i40e_vsi_setup_vectors(vsi); 7572 if (ret) 7573 goto err_msix; 7574 7575 ret = i40e_alloc_rings(vsi); 7576 if (ret) 7577 goto err_rings; 7578 7579 /* map all of the rings to the q_vectors */ 7580 i40e_vsi_map_rings_to_vectors(vsi); 7581 7582 i40e_vsi_reset_stats(vsi); 7583 break; 7584 7585 default: 7586 /* no netdev or rings for the other VSI types */ 7587 break; 7588 } 7589 7590 return vsi; 7591 7592 err_rings: 7593 i40e_vsi_free_q_vectors(vsi); 7594 err_msix: 7595 if (vsi->netdev_registered) { 7596 vsi->netdev_registered = false; 7597 unregister_netdev(vsi->netdev); 7598 free_netdev(vsi->netdev); 7599 vsi->netdev = NULL; 7600 } 7601 err_netdev: 7602 i40e_aq_delete_element(&pf->hw, vsi->seid, NULL); 7603 err_vsi: 7604 i40e_vsi_clear(vsi); 7605 err_alloc: 7606 return NULL; 7607 } 7608 7609 /** 7610 * i40e_veb_get_bw_info - Query VEB BW information 7611 * @veb: the veb to query 7612 * 7613 * Query the Tx scheduler BW configuration data for given VEB 7614 **/ 7615 static int i40e_veb_get_bw_info(struct i40e_veb *veb) 7616 { 7617 struct i40e_aqc_query_switching_comp_ets_config_resp ets_data; 7618 struct i40e_aqc_query_switching_comp_bw_config_resp bw_data; 7619 struct i40e_pf *pf = veb->pf; 7620 struct i40e_hw *hw = &pf->hw; 7621 u32 tc_bw_max; 7622 int ret = 0; 7623 int i; 7624 7625 ret = i40e_aq_query_switch_comp_bw_config(hw, veb->seid, 7626 &bw_data, NULL); 7627 if (ret) { 7628 dev_info(&pf->pdev->dev, 7629 "query veb bw config failed, aq_err=%d\n", 7630 hw->aq.asq_last_status); 7631 goto out; 7632 } 7633 7634 ret = i40e_aq_query_switch_comp_ets_config(hw, veb->seid, 7635 &ets_data, NULL); 7636 if (ret) { 7637 dev_info(&pf->pdev->dev, 7638 "query veb bw ets config failed, aq_err=%d\n", 7639 hw->aq.asq_last_status); 7640 goto out; 7641 } 7642 7643 veb->bw_limit = le16_to_cpu(ets_data.port_bw_limit); 7644 veb->bw_max_quanta = ets_data.tc_bw_max; 7645 veb->is_abs_credits = bw_data.absolute_credits_enable; 7646 tc_bw_max = le16_to_cpu(bw_data.tc_bw_max[0]) | 7647 (le16_to_cpu(bw_data.tc_bw_max[1]) << 16); 7648 for (i = 0; i < I40E_MAX_TRAFFIC_CLASS; i++) { 7649 veb->bw_tc_share_credits[i] = bw_data.tc_bw_share_credits[i]; 7650 veb->bw_tc_limit_credits[i] = 7651 le16_to_cpu(bw_data.tc_bw_limits[i]); 7652 veb->bw_tc_max_quanta[i] = ((tc_bw_max >> (i*4)) & 0x7); 7653 } 7654 7655 out: 7656 return ret; 7657 } 7658 7659 /** 7660 * i40e_veb_mem_alloc - Allocates the next available struct veb in the PF 7661 * @pf: board private structure 7662 * 7663 * On error: returns error code (negative) 7664 * On success: returns vsi index in PF (positive) 7665 **/ 7666 static int i40e_veb_mem_alloc(struct i40e_pf *pf) 7667 { 7668 int ret = -ENOENT; 7669 struct i40e_veb *veb; 7670 int i; 7671 7672 /* Need to protect the allocation of switch elements at the PF level */ 7673 mutex_lock(&pf->switch_mutex); 7674 7675 /* VEB list may be fragmented if VEB creation/destruction has 7676 * been happening. We can afford to do a quick scan to look 7677 * for any free slots in the list. 7678 * 7679 * find next empty veb slot, looping back around if necessary 7680 */ 7681 i = 0; 7682 while ((i < I40E_MAX_VEB) && (pf->veb[i] != NULL)) 7683 i++; 7684 if (i >= I40E_MAX_VEB) { 7685 ret = -ENOMEM; 7686 goto err_alloc_veb; /* out of VEB slots! */ 7687 } 7688 7689 veb = kzalloc(sizeof(*veb), GFP_KERNEL); 7690 if (!veb) { 7691 ret = -ENOMEM; 7692 goto err_alloc_veb; 7693 } 7694 veb->pf = pf; 7695 veb->idx = i; 7696 veb->enabled_tc = 1; 7697 7698 pf->veb[i] = veb; 7699 ret = i; 7700 err_alloc_veb: 7701 mutex_unlock(&pf->switch_mutex); 7702 return ret; 7703 } 7704 7705 /** 7706 * i40e_switch_branch_release - Delete a branch of the switch tree 7707 * @branch: where to start deleting 7708 * 7709 * This uses recursion to find the tips of the branch to be 7710 * removed, deleting until we get back to and can delete this VEB. 7711 **/ 7712 static void i40e_switch_branch_release(struct i40e_veb *branch) 7713 { 7714 struct i40e_pf *pf = branch->pf; 7715 u16 branch_seid = branch->seid; 7716 u16 veb_idx = branch->idx; 7717 int i; 7718 7719 /* release any VEBs on this VEB - RECURSION */ 7720 for (i = 0; i < I40E_MAX_VEB; i++) { 7721 if (!pf->veb[i]) 7722 continue; 7723 if (pf->veb[i]->uplink_seid == branch->seid) 7724 i40e_switch_branch_release(pf->veb[i]); 7725 } 7726 7727 /* Release the VSIs on this VEB, but not the owner VSI. 7728 * 7729 * NOTE: Removing the last VSI on a VEB has the SIDE EFFECT of removing 7730 * the VEB itself, so don't use (*branch) after this loop. 7731 */ 7732 for (i = 0; i < pf->num_alloc_vsi; i++) { 7733 if (!pf->vsi[i]) 7734 continue; 7735 if (pf->vsi[i]->uplink_seid == branch_seid && 7736 (pf->vsi[i]->flags & I40E_VSI_FLAG_VEB_OWNER) == 0) { 7737 i40e_vsi_release(pf->vsi[i]); 7738 } 7739 } 7740 7741 /* There's one corner case where the VEB might not have been 7742 * removed, so double check it here and remove it if needed. 7743 * This case happens if the veb was created from the debugfs 7744 * commands and no VSIs were added to it. 7745 */ 7746 if (pf->veb[veb_idx]) 7747 i40e_veb_release(pf->veb[veb_idx]); 7748 } 7749 7750 /** 7751 * i40e_veb_clear - remove veb struct 7752 * @veb: the veb to remove 7753 **/ 7754 static void i40e_veb_clear(struct i40e_veb *veb) 7755 { 7756 if (!veb) 7757 return; 7758 7759 if (veb->pf) { 7760 struct i40e_pf *pf = veb->pf; 7761 7762 mutex_lock(&pf->switch_mutex); 7763 if (pf->veb[veb->idx] == veb) 7764 pf->veb[veb->idx] = NULL; 7765 mutex_unlock(&pf->switch_mutex); 7766 } 7767 7768 kfree(veb); 7769 } 7770 7771 /** 7772 * i40e_veb_release - Delete a VEB and free its resources 7773 * @veb: the VEB being removed 7774 **/ 7775 void i40e_veb_release(struct i40e_veb *veb) 7776 { 7777 struct i40e_vsi *vsi = NULL; 7778 struct i40e_pf *pf; 7779 int i, n = 0; 7780 7781 pf = veb->pf; 7782 7783 /* find the remaining VSI and check for extras */ 7784 for (i = 0; i < pf->num_alloc_vsi; i++) { 7785 if (pf->vsi[i] && pf->vsi[i]->uplink_seid == veb->seid) { 7786 n++; 7787 vsi = pf->vsi[i]; 7788 } 7789 } 7790 if (n != 1) { 7791 dev_info(&pf->pdev->dev, 7792 "can't remove VEB %d with %d VSIs left\n", 7793 veb->seid, n); 7794 return; 7795 } 7796 7797 /* move the remaining VSI to uplink veb */ 7798 vsi->flags &= ~I40E_VSI_FLAG_VEB_OWNER; 7799 if (veb->uplink_seid) { 7800 vsi->uplink_seid = veb->uplink_seid; 7801 if (veb->uplink_seid == pf->mac_seid) 7802 vsi->veb_idx = I40E_NO_VEB; 7803 else 7804 vsi->veb_idx = veb->veb_idx; 7805 } else { 7806 /* floating VEB */ 7807 vsi->uplink_seid = pf->vsi[pf->lan_vsi]->uplink_seid; 7808 vsi->veb_idx = pf->vsi[pf->lan_vsi]->veb_idx; 7809 } 7810 7811 i40e_aq_delete_element(&pf->hw, veb->seid, NULL); 7812 i40e_veb_clear(veb); 7813 } 7814 7815 /** 7816 * i40e_add_veb - create the VEB in the switch 7817 * @veb: the VEB to be instantiated 7818 * @vsi: the controlling VSI 7819 **/ 7820 static int i40e_add_veb(struct i40e_veb *veb, struct i40e_vsi *vsi) 7821 { 7822 bool is_default = false; 7823 bool is_cloud = false; 7824 int ret; 7825 7826 /* get a VEB from the hardware */ 7827 ret = i40e_aq_add_veb(&veb->pf->hw, veb->uplink_seid, vsi->seid, 7828 veb->enabled_tc, is_default, 7829 is_cloud, &veb->seid, NULL); 7830 if (ret) { 7831 dev_info(&veb->pf->pdev->dev, 7832 "couldn't add VEB, err %d, aq_err %d\n", 7833 ret, veb->pf->hw.aq.asq_last_status); 7834 return -EPERM; 7835 } 7836 7837 /* get statistics counter */ 7838 ret = i40e_aq_get_veb_parameters(&veb->pf->hw, veb->seid, NULL, NULL, 7839 &veb->stats_idx, NULL, NULL, NULL); 7840 if (ret) { 7841 dev_info(&veb->pf->pdev->dev, 7842 "couldn't get VEB statistics idx, err %d, aq_err %d\n", 7843 ret, veb->pf->hw.aq.asq_last_status); 7844 return -EPERM; 7845 } 7846 ret = i40e_veb_get_bw_info(veb); 7847 if (ret) { 7848 dev_info(&veb->pf->pdev->dev, 7849 "couldn't get VEB bw info, err %d, aq_err %d\n", 7850 ret, veb->pf->hw.aq.asq_last_status); 7851 i40e_aq_delete_element(&veb->pf->hw, veb->seid, NULL); 7852 return -ENOENT; 7853 } 7854 7855 vsi->uplink_seid = veb->seid; 7856 vsi->veb_idx = veb->idx; 7857 vsi->flags |= I40E_VSI_FLAG_VEB_OWNER; 7858 7859 return 0; 7860 } 7861 7862 /** 7863 * i40e_veb_setup - Set up a VEB 7864 * @pf: board private structure 7865 * @flags: VEB setup flags 7866 * @uplink_seid: the switch element to link to 7867 * @vsi_seid: the initial VSI seid 7868 * @enabled_tc: Enabled TC bit-map 7869 * 7870 * This allocates the sw VEB structure and links it into the switch 7871 * It is possible and legal for this to be a duplicate of an already 7872 * existing VEB. It is also possible for both uplink and vsi seids 7873 * to be zero, in order to create a floating VEB. 7874 * 7875 * Returns pointer to the successfully allocated VEB sw struct on 7876 * success, otherwise returns NULL on failure. 7877 **/ 7878 struct i40e_veb *i40e_veb_setup(struct i40e_pf *pf, u16 flags, 7879 u16 uplink_seid, u16 vsi_seid, 7880 u8 enabled_tc) 7881 { 7882 struct i40e_veb *veb, *uplink_veb = NULL; 7883 int vsi_idx, veb_idx; 7884 int ret; 7885 7886 /* if one seid is 0, the other must be 0 to create a floating relay */ 7887 if ((uplink_seid == 0 || vsi_seid == 0) && 7888 (uplink_seid + vsi_seid != 0)) { 7889 dev_info(&pf->pdev->dev, 7890 "one, not both seid's are 0: uplink=%d vsi=%d\n", 7891 uplink_seid, vsi_seid); 7892 return NULL; 7893 } 7894 7895 /* make sure there is such a vsi and uplink */ 7896 for (vsi_idx = 0; vsi_idx < pf->num_alloc_vsi; vsi_idx++) 7897 if (pf->vsi[vsi_idx] && pf->vsi[vsi_idx]->seid == vsi_seid) 7898 break; 7899 if (vsi_idx >= pf->num_alloc_vsi && vsi_seid != 0) { 7900 dev_info(&pf->pdev->dev, "vsi seid %d not found\n", 7901 vsi_seid); 7902 return NULL; 7903 } 7904 7905 if (uplink_seid && uplink_seid != pf->mac_seid) { 7906 for (veb_idx = 0; veb_idx < I40E_MAX_VEB; veb_idx++) { 7907 if (pf->veb[veb_idx] && 7908 pf->veb[veb_idx]->seid == uplink_seid) { 7909 uplink_veb = pf->veb[veb_idx]; 7910 break; 7911 } 7912 } 7913 if (!uplink_veb) { 7914 dev_info(&pf->pdev->dev, 7915 "uplink seid %d not found\n", uplink_seid); 7916 return NULL; 7917 } 7918 } 7919 7920 /* get veb sw struct */ 7921 veb_idx = i40e_veb_mem_alloc(pf); 7922 if (veb_idx < 0) 7923 goto err_alloc; 7924 veb = pf->veb[veb_idx]; 7925 veb->flags = flags; 7926 veb->uplink_seid = uplink_seid; 7927 veb->veb_idx = (uplink_veb ? uplink_veb->idx : I40E_NO_VEB); 7928 veb->enabled_tc = (enabled_tc ? enabled_tc : 0x1); 7929 7930 /* create the VEB in the switch */ 7931 ret = i40e_add_veb(veb, pf->vsi[vsi_idx]); 7932 if (ret) 7933 goto err_veb; 7934 if (vsi_idx == pf->lan_vsi) 7935 pf->lan_veb = veb->idx; 7936 7937 return veb; 7938 7939 err_veb: 7940 i40e_veb_clear(veb); 7941 err_alloc: 7942 return NULL; 7943 } 7944 7945 /** 7946 * i40e_setup_pf_switch_element - set pf vars based on switch type 7947 * @pf: board private structure 7948 * @ele: element we are building info from 7949 * @num_reported: total number of elements 7950 * @printconfig: should we print the contents 7951 * 7952 * helper function to assist in extracting a few useful SEID values. 7953 **/ 7954 static void i40e_setup_pf_switch_element(struct i40e_pf *pf, 7955 struct i40e_aqc_switch_config_element_resp *ele, 7956 u16 num_reported, bool printconfig) 7957 { 7958 u16 downlink_seid = le16_to_cpu(ele->downlink_seid); 7959 u16 uplink_seid = le16_to_cpu(ele->uplink_seid); 7960 u8 element_type = ele->element_type; 7961 u16 seid = le16_to_cpu(ele->seid); 7962 7963 if (printconfig) 7964 dev_info(&pf->pdev->dev, 7965 "type=%d seid=%d uplink=%d downlink=%d\n", 7966 element_type, seid, uplink_seid, downlink_seid); 7967 7968 switch (element_type) { 7969 case I40E_SWITCH_ELEMENT_TYPE_MAC: 7970 pf->mac_seid = seid; 7971 break; 7972 case I40E_SWITCH_ELEMENT_TYPE_VEB: 7973 /* Main VEB? */ 7974 if (uplink_seid != pf->mac_seid) 7975 break; 7976 if (pf->lan_veb == I40E_NO_VEB) { 7977 int v; 7978 7979 /* find existing or else empty VEB */ 7980 for (v = 0; v < I40E_MAX_VEB; v++) { 7981 if (pf->veb[v] && (pf->veb[v]->seid == seid)) { 7982 pf->lan_veb = v; 7983 break; 7984 } 7985 } 7986 if (pf->lan_veb == I40E_NO_VEB) { 7987 v = i40e_veb_mem_alloc(pf); 7988 if (v < 0) 7989 break; 7990 pf->lan_veb = v; 7991 } 7992 } 7993 7994 pf->veb[pf->lan_veb]->seid = seid; 7995 pf->veb[pf->lan_veb]->uplink_seid = pf->mac_seid; 7996 pf->veb[pf->lan_veb]->pf = pf; 7997 pf->veb[pf->lan_veb]->veb_idx = I40E_NO_VEB; 7998 break; 7999 case I40E_SWITCH_ELEMENT_TYPE_VSI: 8000 if (num_reported != 1) 8001 break; 8002 /* This is immediately after a reset so we can assume this is 8003 * the PF's VSI 8004 */ 8005 pf->mac_seid = uplink_seid; 8006 pf->pf_seid = downlink_seid; 8007 pf->main_vsi_seid = seid; 8008 if (printconfig) 8009 dev_info(&pf->pdev->dev, 8010 "pf_seid=%d main_vsi_seid=%d\n", 8011 pf->pf_seid, pf->main_vsi_seid); 8012 break; 8013 case I40E_SWITCH_ELEMENT_TYPE_PF: 8014 case I40E_SWITCH_ELEMENT_TYPE_VF: 8015 case I40E_SWITCH_ELEMENT_TYPE_EMP: 8016 case I40E_SWITCH_ELEMENT_TYPE_BMC: 8017 case I40E_SWITCH_ELEMENT_TYPE_PE: 8018 case I40E_SWITCH_ELEMENT_TYPE_PA: 8019 /* ignore these for now */ 8020 break; 8021 default: 8022 dev_info(&pf->pdev->dev, "unknown element type=%d seid=%d\n", 8023 element_type, seid); 8024 break; 8025 } 8026 } 8027 8028 /** 8029 * i40e_fetch_switch_configuration - Get switch config from firmware 8030 * @pf: board private structure 8031 * @printconfig: should we print the contents 8032 * 8033 * Get the current switch configuration from the device and 8034 * extract a few useful SEID values. 8035 **/ 8036 int i40e_fetch_switch_configuration(struct i40e_pf *pf, bool printconfig) 8037 { 8038 struct i40e_aqc_get_switch_config_resp *sw_config; 8039 u16 next_seid = 0; 8040 int ret = 0; 8041 u8 *aq_buf; 8042 int i; 8043 8044 aq_buf = kzalloc(I40E_AQ_LARGE_BUF, GFP_KERNEL); 8045 if (!aq_buf) 8046 return -ENOMEM; 8047 8048 sw_config = (struct i40e_aqc_get_switch_config_resp *)aq_buf; 8049 do { 8050 u16 num_reported, num_total; 8051 8052 ret = i40e_aq_get_switch_config(&pf->hw, sw_config, 8053 I40E_AQ_LARGE_BUF, 8054 &next_seid, NULL); 8055 if (ret) { 8056 dev_info(&pf->pdev->dev, 8057 "get switch config failed %d aq_err=%x\n", 8058 ret, pf->hw.aq.asq_last_status); 8059 kfree(aq_buf); 8060 return -ENOENT; 8061 } 8062 8063 num_reported = le16_to_cpu(sw_config->header.num_reported); 8064 num_total = le16_to_cpu(sw_config->header.num_total); 8065 8066 if (printconfig) 8067 dev_info(&pf->pdev->dev, 8068 "header: %d reported %d total\n", 8069 num_reported, num_total); 8070 8071 for (i = 0; i < num_reported; i++) { 8072 struct i40e_aqc_switch_config_element_resp *ele = 8073 &sw_config->element[i]; 8074 8075 i40e_setup_pf_switch_element(pf, ele, num_reported, 8076 printconfig); 8077 } 8078 } while (next_seid != 0); 8079 8080 kfree(aq_buf); 8081 return ret; 8082 } 8083 8084 /** 8085 * i40e_setup_pf_switch - Setup the HW switch on startup or after reset 8086 * @pf: board private structure 8087 * @reinit: if the Main VSI needs to re-initialized. 8088 * 8089 * Returns 0 on success, negative value on failure 8090 **/ 8091 static int i40e_setup_pf_switch(struct i40e_pf *pf, bool reinit) 8092 { 8093 u32 rxfc = 0, txfc = 0, rxfc_reg; 8094 int ret; 8095 8096 /* find out what's out there already */ 8097 ret = i40e_fetch_switch_configuration(pf, false); 8098 if (ret) { 8099 dev_info(&pf->pdev->dev, 8100 "couldn't fetch switch config, err %d, aq_err %d\n", 8101 ret, pf->hw.aq.asq_last_status); 8102 return ret; 8103 } 8104 i40e_pf_reset_stats(pf); 8105 8106 /* first time setup */ 8107 if (pf->lan_vsi == I40E_NO_VSI || reinit) { 8108 struct i40e_vsi *vsi = NULL; 8109 u16 uplink_seid; 8110 8111 /* Set up the PF VSI associated with the PF's main VSI 8112 * that is already in the HW switch 8113 */ 8114 if (pf->lan_veb != I40E_NO_VEB && pf->veb[pf->lan_veb]) 8115 uplink_seid = pf->veb[pf->lan_veb]->seid; 8116 else 8117 uplink_seid = pf->mac_seid; 8118 if (pf->lan_vsi == I40E_NO_VSI) 8119 vsi = i40e_vsi_setup(pf, I40E_VSI_MAIN, uplink_seid, 0); 8120 else if (reinit) 8121 vsi = i40e_vsi_reinit_setup(pf->vsi[pf->lan_vsi]); 8122 if (!vsi) { 8123 dev_info(&pf->pdev->dev, "setup of MAIN VSI failed\n"); 8124 i40e_fdir_teardown(pf); 8125 return -EAGAIN; 8126 } 8127 } else { 8128 /* force a reset of TC and queue layout configurations */ 8129 u8 enabled_tc = pf->vsi[pf->lan_vsi]->tc_config.enabled_tc; 8130 pf->vsi[pf->lan_vsi]->tc_config.enabled_tc = 0; 8131 pf->vsi[pf->lan_vsi]->seid = pf->main_vsi_seid; 8132 i40e_vsi_config_tc(pf->vsi[pf->lan_vsi], enabled_tc); 8133 } 8134 i40e_vlan_stripping_disable(pf->vsi[pf->lan_vsi]); 8135 8136 i40e_fdir_sb_setup(pf); 8137 8138 /* Setup static PF queue filter control settings */ 8139 ret = i40e_setup_pf_filter_control(pf); 8140 if (ret) { 8141 dev_info(&pf->pdev->dev, "setup_pf_filter_control failed: %d\n", 8142 ret); 8143 /* Failure here should not stop continuing other steps */ 8144 } 8145 8146 /* enable RSS in the HW, even for only one queue, as the stack can use 8147 * the hash 8148 */ 8149 if ((pf->flags & I40E_FLAG_RSS_ENABLED)) 8150 i40e_config_rss(pf); 8151 8152 /* fill in link information and enable LSE reporting */ 8153 i40e_aq_get_link_info(&pf->hw, true, NULL, NULL); 8154 i40e_link_event(pf); 8155 8156 /* Initialize user-specific link properties */ 8157 pf->fc_autoneg_status = ((pf->hw.phy.link_info.an_info & 8158 I40E_AQ_AN_COMPLETED) ? true : false); 8159 /* requested_mode is set in probe or by ethtool */ 8160 if (!pf->fc_autoneg_status) 8161 goto no_autoneg; 8162 8163 if ((pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) && 8164 (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX)) 8165 pf->hw.fc.current_mode = I40E_FC_FULL; 8166 else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_TX) 8167 pf->hw.fc.current_mode = I40E_FC_TX_PAUSE; 8168 else if (pf->hw.phy.link_info.an_info & I40E_AQ_LINK_PAUSE_RX) 8169 pf->hw.fc.current_mode = I40E_FC_RX_PAUSE; 8170 else 8171 pf->hw.fc.current_mode = I40E_FC_NONE; 8172 8173 /* sync the flow control settings with the auto-neg values */ 8174 switch (pf->hw.fc.current_mode) { 8175 case I40E_FC_FULL: 8176 txfc = 1; 8177 rxfc = 1; 8178 break; 8179 case I40E_FC_TX_PAUSE: 8180 txfc = 1; 8181 rxfc = 0; 8182 break; 8183 case I40E_FC_RX_PAUSE: 8184 txfc = 0; 8185 rxfc = 1; 8186 break; 8187 case I40E_FC_NONE: 8188 case I40E_FC_DEFAULT: 8189 txfc = 0; 8190 rxfc = 0; 8191 break; 8192 case I40E_FC_PFC: 8193 /* TBD */ 8194 break; 8195 /* no default case, we have to handle all possibilities here */ 8196 } 8197 8198 wr32(&pf->hw, I40E_PRTDCB_FCCFG, txfc << I40E_PRTDCB_FCCFG_TFCE_SHIFT); 8199 8200 rxfc_reg = rd32(&pf->hw, I40E_PRTDCB_MFLCN) & 8201 ~I40E_PRTDCB_MFLCN_RFCE_MASK; 8202 rxfc_reg |= (rxfc << I40E_PRTDCB_MFLCN_RFCE_SHIFT); 8203 8204 wr32(&pf->hw, I40E_PRTDCB_MFLCN, rxfc_reg); 8205 8206 goto fc_complete; 8207 8208 no_autoneg: 8209 /* disable L2 flow control, user can turn it on if they wish */ 8210 wr32(&pf->hw, I40E_PRTDCB_FCCFG, 0); 8211 wr32(&pf->hw, I40E_PRTDCB_MFLCN, rd32(&pf->hw, I40E_PRTDCB_MFLCN) & 8212 ~I40E_PRTDCB_MFLCN_RFCE_MASK); 8213 8214 fc_complete: 8215 i40e_ptp_init(pf); 8216 8217 return ret; 8218 } 8219 8220 /** 8221 * i40e_determine_queue_usage - Work out queue distribution 8222 * @pf: board private structure 8223 **/ 8224 static void i40e_determine_queue_usage(struct i40e_pf *pf) 8225 { 8226 int queues_left; 8227 8228 pf->num_lan_qps = 0; 8229 8230 /* Find the max queues to be put into basic use. We'll always be 8231 * using TC0, whether or not DCB is running, and TC0 will get the 8232 * big RSS set. 8233 */ 8234 queues_left = pf->hw.func_caps.num_tx_qp; 8235 8236 if ((queues_left == 1) || 8237 !(pf->flags & I40E_FLAG_MSIX_ENABLED)) { 8238 /* one qp for PF, no queues for anything else */ 8239 queues_left = 0; 8240 pf->rss_size = pf->num_lan_qps = 1; 8241 8242 /* make sure all the fancies are disabled */ 8243 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 8244 I40E_FLAG_FD_SB_ENABLED | 8245 I40E_FLAG_FD_ATR_ENABLED | 8246 I40E_FLAG_DCB_CAPABLE | 8247 I40E_FLAG_SRIOV_ENABLED | 8248 I40E_FLAG_VMDQ_ENABLED); 8249 } else if (!(pf->flags & (I40E_FLAG_RSS_ENABLED | 8250 I40E_FLAG_FD_SB_ENABLED | 8251 I40E_FLAG_FD_ATR_ENABLED | 8252 I40E_FLAG_DCB_CAPABLE))) { 8253 /* one qp for PF */ 8254 pf->rss_size = pf->num_lan_qps = 1; 8255 queues_left -= pf->num_lan_qps; 8256 8257 pf->flags &= ~(I40E_FLAG_RSS_ENABLED | 8258 I40E_FLAG_FD_SB_ENABLED | 8259 I40E_FLAG_FD_ATR_ENABLED | 8260 I40E_FLAG_DCB_ENABLED | 8261 I40E_FLAG_VMDQ_ENABLED); 8262 } else { 8263 /* Not enough queues for all TCs */ 8264 if ((pf->flags & I40E_FLAG_DCB_CAPABLE) && 8265 (queues_left < I40E_MAX_TRAFFIC_CLASS)) { 8266 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 8267 dev_info(&pf->pdev->dev, "not enough queues for DCB. DCB is disabled.\n"); 8268 } 8269 pf->num_lan_qps = pf->rss_size_max; 8270 queues_left -= pf->num_lan_qps; 8271 } 8272 8273 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 8274 if (queues_left > 1) { 8275 queues_left -= 1; /* save 1 queue for FD */ 8276 } else { 8277 pf->flags &= ~I40E_FLAG_FD_SB_ENABLED; 8278 dev_info(&pf->pdev->dev, "not enough queues for Flow Director. Flow Director feature is disabled\n"); 8279 } 8280 } 8281 8282 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 8283 pf->num_vf_qps && pf->num_req_vfs && queues_left) { 8284 pf->num_req_vfs = min_t(int, pf->num_req_vfs, 8285 (queues_left / pf->num_vf_qps)); 8286 queues_left -= (pf->num_req_vfs * pf->num_vf_qps); 8287 } 8288 8289 if ((pf->flags & I40E_FLAG_VMDQ_ENABLED) && 8290 pf->num_vmdq_vsis && pf->num_vmdq_qps && queues_left) { 8291 pf->num_vmdq_vsis = min_t(int, pf->num_vmdq_vsis, 8292 (queues_left / pf->num_vmdq_qps)); 8293 queues_left -= (pf->num_vmdq_vsis * pf->num_vmdq_qps); 8294 } 8295 8296 pf->queues_left = queues_left; 8297 } 8298 8299 /** 8300 * i40e_setup_pf_filter_control - Setup PF static filter control 8301 * @pf: PF to be setup 8302 * 8303 * i40e_setup_pf_filter_control sets up a pf's initial filter control 8304 * settings. If PE/FCoE are enabled then it will also set the per PF 8305 * based filter sizes required for them. It also enables Flow director, 8306 * ethertype and macvlan type filter settings for the pf. 8307 * 8308 * Returns 0 on success, negative on failure 8309 **/ 8310 static int i40e_setup_pf_filter_control(struct i40e_pf *pf) 8311 { 8312 struct i40e_filter_control_settings *settings = &pf->filter_settings; 8313 8314 settings->hash_lut_size = I40E_HASH_LUT_SIZE_128; 8315 8316 /* Flow Director is enabled */ 8317 if (pf->flags & (I40E_FLAG_FD_SB_ENABLED | I40E_FLAG_FD_ATR_ENABLED)) 8318 settings->enable_fdir = true; 8319 8320 /* Ethtype and MACVLAN filters enabled for PF */ 8321 settings->enable_ethtype = true; 8322 settings->enable_macvlan = true; 8323 8324 if (i40e_set_filter_control(&pf->hw, settings)) 8325 return -ENOENT; 8326 8327 return 0; 8328 } 8329 8330 #define INFO_STRING_LEN 255 8331 static void i40e_print_features(struct i40e_pf *pf) 8332 { 8333 struct i40e_hw *hw = &pf->hw; 8334 char *buf, *string; 8335 8336 string = kzalloc(INFO_STRING_LEN, GFP_KERNEL); 8337 if (!string) { 8338 dev_err(&pf->pdev->dev, "Features string allocation failed\n"); 8339 return; 8340 } 8341 8342 buf = string; 8343 8344 buf += sprintf(string, "Features: PF-id[%d] ", hw->pf_id); 8345 #ifdef CONFIG_PCI_IOV 8346 buf += sprintf(buf, "VFs: %d ", pf->num_req_vfs); 8347 #endif 8348 buf += sprintf(buf, "VSIs: %d QP: %d ", pf->hw.func_caps.num_vsis, 8349 pf->vsi[pf->lan_vsi]->num_queue_pairs); 8350 8351 if (pf->flags & I40E_FLAG_RSS_ENABLED) 8352 buf += sprintf(buf, "RSS "); 8353 if (pf->flags & I40E_FLAG_FD_ATR_ENABLED) 8354 buf += sprintf(buf, "FD_ATR "); 8355 if (pf->flags & I40E_FLAG_FD_SB_ENABLED) { 8356 buf += sprintf(buf, "FD_SB "); 8357 buf += sprintf(buf, "NTUPLE "); 8358 } 8359 if (pf->flags & I40E_FLAG_DCB_CAPABLE) 8360 buf += sprintf(buf, "DCB "); 8361 if (pf->flags & I40E_FLAG_PTP) 8362 buf += sprintf(buf, "PTP "); 8363 8364 BUG_ON(buf > (string + INFO_STRING_LEN)); 8365 dev_info(&pf->pdev->dev, "%s\n", string); 8366 kfree(string); 8367 } 8368 8369 /** 8370 * i40e_probe - Device initialization routine 8371 * @pdev: PCI device information struct 8372 * @ent: entry in i40e_pci_tbl 8373 * 8374 * i40e_probe initializes a pf identified by a pci_dev structure. 8375 * The OS initialization, configuring of the pf private structure, 8376 * and a hardware reset occur. 8377 * 8378 * Returns 0 on success, negative on failure 8379 **/ 8380 static int i40e_probe(struct pci_dev *pdev, const struct pci_device_id *ent) 8381 { 8382 struct i40e_pf *pf; 8383 struct i40e_hw *hw; 8384 static u16 pfs_found; 8385 u16 link_status; 8386 int err = 0; 8387 u32 len; 8388 u32 i; 8389 8390 err = pci_enable_device_mem(pdev); 8391 if (err) 8392 return err; 8393 8394 /* set up for high or low dma */ 8395 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 8396 if (err) { 8397 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 8398 if (err) { 8399 dev_err(&pdev->dev, 8400 "DMA configuration failed: 0x%x\n", err); 8401 goto err_dma; 8402 } 8403 } 8404 8405 /* set up pci connections */ 8406 err = pci_request_selected_regions(pdev, pci_select_bars(pdev, 8407 IORESOURCE_MEM), i40e_driver_name); 8408 if (err) { 8409 dev_info(&pdev->dev, 8410 "pci_request_selected_regions failed %d\n", err); 8411 goto err_pci_reg; 8412 } 8413 8414 pci_enable_pcie_error_reporting(pdev); 8415 pci_set_master(pdev); 8416 8417 /* Now that we have a PCI connection, we need to do the 8418 * low level device setup. This is primarily setting up 8419 * the Admin Queue structures and then querying for the 8420 * device's current profile information. 8421 */ 8422 pf = kzalloc(sizeof(*pf), GFP_KERNEL); 8423 if (!pf) { 8424 err = -ENOMEM; 8425 goto err_pf_alloc; 8426 } 8427 pf->next_vsi = 0; 8428 pf->pdev = pdev; 8429 set_bit(__I40E_DOWN, &pf->state); 8430 8431 hw = &pf->hw; 8432 hw->back = pf; 8433 hw->hw_addr = ioremap(pci_resource_start(pdev, 0), 8434 pci_resource_len(pdev, 0)); 8435 if (!hw->hw_addr) { 8436 err = -EIO; 8437 dev_info(&pdev->dev, "ioremap(0x%04x, 0x%04x) failed: 0x%x\n", 8438 (unsigned int)pci_resource_start(pdev, 0), 8439 (unsigned int)pci_resource_len(pdev, 0), err); 8440 goto err_ioremap; 8441 } 8442 hw->vendor_id = pdev->vendor; 8443 hw->device_id = pdev->device; 8444 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id); 8445 hw->subsystem_vendor_id = pdev->subsystem_vendor; 8446 hw->subsystem_device_id = pdev->subsystem_device; 8447 hw->bus.device = PCI_SLOT(pdev->devfn); 8448 hw->bus.func = PCI_FUNC(pdev->devfn); 8449 pf->instance = pfs_found; 8450 8451 /* do a special CORER for clearing PXE mode once at init */ 8452 if (hw->revision_id == 0 && 8453 (rd32(hw, I40E_GLLAN_RCTL_0) & I40E_GLLAN_RCTL_0_PXE_MODE_MASK)) { 8454 wr32(hw, I40E_GLGEN_RTRIG, I40E_GLGEN_RTRIG_CORER_MASK); 8455 i40e_flush(hw); 8456 msleep(200); 8457 pf->corer_count++; 8458 8459 i40e_clear_pxe_mode(hw); 8460 } 8461 8462 /* Reset here to make sure all is clean and to define PF 'n' */ 8463 err = i40e_pf_reset(hw); 8464 if (err) { 8465 dev_info(&pdev->dev, "Initial pf_reset failed: %d\n", err); 8466 goto err_pf_reset; 8467 } 8468 pf->pfr_count++; 8469 8470 hw->aq.num_arq_entries = I40E_AQ_LEN; 8471 hw->aq.num_asq_entries = I40E_AQ_LEN; 8472 hw->aq.arq_buf_size = I40E_MAX_AQ_BUF_SIZE; 8473 hw->aq.asq_buf_size = I40E_MAX_AQ_BUF_SIZE; 8474 pf->adminq_work_limit = I40E_AQ_WORK_LIMIT; 8475 snprintf(pf->misc_int_name, sizeof(pf->misc_int_name) - 1, 8476 "%s-pf%d:misc", 8477 dev_driver_string(&pf->pdev->dev), pf->hw.pf_id); 8478 8479 err = i40e_init_shared_code(hw); 8480 if (err) { 8481 dev_info(&pdev->dev, "init_shared_code failed: %d\n", err); 8482 goto err_pf_reset; 8483 } 8484 8485 /* set up a default setting for link flow control */ 8486 pf->hw.fc.requested_mode = I40E_FC_NONE; 8487 8488 err = i40e_init_adminq(hw); 8489 dev_info(&pdev->dev, "%s\n", i40e_fw_version_str(hw)); 8490 if (err) { 8491 dev_info(&pdev->dev, 8492 "init_adminq failed: %d expecting API %02x.%02x\n", 8493 err, 8494 I40E_FW_API_VERSION_MAJOR, I40E_FW_API_VERSION_MINOR); 8495 goto err_pf_reset; 8496 } 8497 8498 i40e_verify_eeprom(pf); 8499 8500 /* Rev 0 hardware was never productized */ 8501 if (hw->revision_id < 1) 8502 dev_warn(&pdev->dev, "This device is a pre-production adapter/LOM. Please be aware there may be issues with your hardware. If you are experiencing problems please contact your Intel or hardware representative who provided you with this hardware.\n"); 8503 8504 i40e_clear_pxe_mode(hw); 8505 err = i40e_get_capabilities(pf); 8506 if (err) 8507 goto err_adminq_setup; 8508 8509 err = i40e_sw_init(pf); 8510 if (err) { 8511 dev_info(&pdev->dev, "sw_init failed: %d\n", err); 8512 goto err_sw_init; 8513 } 8514 8515 err = i40e_init_lan_hmc(hw, hw->func_caps.num_tx_qp, 8516 hw->func_caps.num_rx_qp, 8517 pf->fcoe_hmc_cntx_num, pf->fcoe_hmc_filt_num); 8518 if (err) { 8519 dev_info(&pdev->dev, "init_lan_hmc failed: %d\n", err); 8520 goto err_init_lan_hmc; 8521 } 8522 8523 err = i40e_configure_lan_hmc(hw, I40E_HMC_MODEL_DIRECT_ONLY); 8524 if (err) { 8525 dev_info(&pdev->dev, "configure_lan_hmc failed: %d\n", err); 8526 err = -ENOENT; 8527 goto err_configure_lan_hmc; 8528 } 8529 8530 i40e_get_mac_addr(hw, hw->mac.addr); 8531 if (!is_valid_ether_addr(hw->mac.addr)) { 8532 dev_info(&pdev->dev, "invalid MAC address %pM\n", hw->mac.addr); 8533 err = -EIO; 8534 goto err_mac_addr; 8535 } 8536 dev_info(&pdev->dev, "MAC address: %pM\n", hw->mac.addr); 8537 ether_addr_copy(hw->mac.perm_addr, hw->mac.addr); 8538 8539 pci_set_drvdata(pdev, pf); 8540 pci_save_state(pdev); 8541 #ifdef CONFIG_I40E_DCB 8542 err = i40e_init_pf_dcb(pf); 8543 if (err) { 8544 dev_info(&pdev->dev, "init_pf_dcb failed: %d\n", err); 8545 pf->flags &= ~I40E_FLAG_DCB_CAPABLE; 8546 /* Continue without DCB enabled */ 8547 } 8548 #endif /* CONFIG_I40E_DCB */ 8549 8550 /* set up periodic task facility */ 8551 setup_timer(&pf->service_timer, i40e_service_timer, (unsigned long)pf); 8552 pf->service_timer_period = HZ; 8553 8554 INIT_WORK(&pf->service_task, i40e_service_task); 8555 clear_bit(__I40E_SERVICE_SCHED, &pf->state); 8556 pf->flags |= I40E_FLAG_NEED_LINK_UPDATE; 8557 pf->link_check_timeout = jiffies; 8558 8559 /* WoL defaults to disabled */ 8560 pf->wol_en = false; 8561 device_set_wakeup_enable(&pf->pdev->dev, pf->wol_en); 8562 8563 /* set up the main switch operations */ 8564 i40e_determine_queue_usage(pf); 8565 i40e_init_interrupt_scheme(pf); 8566 8567 /* The number of VSIs reported by the FW is the minimum guaranteed 8568 * to us; HW supports far more and we share the remaining pool with 8569 * the other PFs. We allocate space for more than the guarantee with 8570 * the understanding that we might not get them all later. 8571 */ 8572 if (pf->hw.func_caps.num_vsis < I40E_MIN_VSI_ALLOC) 8573 pf->num_alloc_vsi = I40E_MIN_VSI_ALLOC; 8574 else 8575 pf->num_alloc_vsi = pf->hw.func_caps.num_vsis; 8576 8577 /* Set up the *vsi struct and our local tracking of the MAIN PF vsi. */ 8578 len = sizeof(struct i40e_vsi *) * pf->num_alloc_vsi; 8579 pf->vsi = kzalloc(len, GFP_KERNEL); 8580 if (!pf->vsi) { 8581 err = -ENOMEM; 8582 goto err_switch_setup; 8583 } 8584 8585 err = i40e_setup_pf_switch(pf, false); 8586 if (err) { 8587 dev_info(&pdev->dev, "setup_pf_switch failed: %d\n", err); 8588 goto err_vsis; 8589 } 8590 /* if FDIR VSI was set up, start it now */ 8591 for (i = 0; i < pf->num_alloc_vsi; i++) { 8592 if (pf->vsi[i] && pf->vsi[i]->type == I40E_VSI_FDIR) { 8593 i40e_vsi_open(pf->vsi[i]); 8594 break; 8595 } 8596 } 8597 8598 /* The main driver is (mostly) up and happy. We need to set this state 8599 * before setting up the misc vector or we get a race and the vector 8600 * ends up disabled forever. 8601 */ 8602 clear_bit(__I40E_DOWN, &pf->state); 8603 8604 /* In case of MSIX we are going to setup the misc vector right here 8605 * to handle admin queue events etc. In case of legacy and MSI 8606 * the misc functionality and queue processing is combined in 8607 * the same vector and that gets setup at open. 8608 */ 8609 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 8610 err = i40e_setup_misc_vector(pf); 8611 if (err) { 8612 dev_info(&pdev->dev, 8613 "setup of misc vector failed: %d\n", err); 8614 goto err_vsis; 8615 } 8616 } 8617 8618 #ifdef CONFIG_PCI_IOV 8619 /* prep for VF support */ 8620 if ((pf->flags & I40E_FLAG_SRIOV_ENABLED) && 8621 (pf->flags & I40E_FLAG_MSIX_ENABLED) && 8622 !test_bit(__I40E_BAD_EEPROM, &pf->state)) { 8623 u32 val; 8624 8625 /* disable link interrupts for VFs */ 8626 val = rd32(hw, I40E_PFGEN_PORTMDIO_NUM); 8627 val &= ~I40E_PFGEN_PORTMDIO_NUM_VFLINK_STAT_ENA_MASK; 8628 wr32(hw, I40E_PFGEN_PORTMDIO_NUM, val); 8629 i40e_flush(hw); 8630 8631 if (pci_num_vf(pdev)) { 8632 dev_info(&pdev->dev, 8633 "Active VFs found, allocating resources.\n"); 8634 err = i40e_alloc_vfs(pf, pci_num_vf(pdev)); 8635 if (err) 8636 dev_info(&pdev->dev, 8637 "Error %d allocating resources for existing VFs\n", 8638 err); 8639 } 8640 } 8641 #endif /* CONFIG_PCI_IOV */ 8642 8643 pfs_found++; 8644 8645 i40e_dbg_pf_init(pf); 8646 8647 /* tell the firmware that we're starting */ 8648 i40e_send_version(pf); 8649 8650 /* since everything's happy, start the service_task timer */ 8651 mod_timer(&pf->service_timer, 8652 round_jiffies(jiffies + pf->service_timer_period)); 8653 8654 /* Get the negotiated link width and speed from PCI config space */ 8655 pcie_capability_read_word(pf->pdev, PCI_EXP_LNKSTA, &link_status); 8656 8657 i40e_set_pci_config_data(hw, link_status); 8658 8659 dev_info(&pdev->dev, "PCI-Express: %s %s\n", 8660 (hw->bus.speed == i40e_bus_speed_8000 ? "Speed 8.0GT/s" : 8661 hw->bus.speed == i40e_bus_speed_5000 ? "Speed 5.0GT/s" : 8662 hw->bus.speed == i40e_bus_speed_2500 ? "Speed 2.5GT/s" : 8663 "Unknown"), 8664 (hw->bus.width == i40e_bus_width_pcie_x8 ? "Width x8" : 8665 hw->bus.width == i40e_bus_width_pcie_x4 ? "Width x4" : 8666 hw->bus.width == i40e_bus_width_pcie_x2 ? "Width x2" : 8667 hw->bus.width == i40e_bus_width_pcie_x1 ? "Width x1" : 8668 "Unknown")); 8669 8670 if (hw->bus.width < i40e_bus_width_pcie_x8 || 8671 hw->bus.speed < i40e_bus_speed_8000) { 8672 dev_warn(&pdev->dev, "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n"); 8673 dev_warn(&pdev->dev, "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n"); 8674 } 8675 8676 /* print a string summarizing features */ 8677 i40e_print_features(pf); 8678 8679 return 0; 8680 8681 /* Unwind what we've done if something failed in the setup */ 8682 err_vsis: 8683 set_bit(__I40E_DOWN, &pf->state); 8684 i40e_clear_interrupt_scheme(pf); 8685 kfree(pf->vsi); 8686 err_switch_setup: 8687 i40e_reset_interrupt_capability(pf); 8688 del_timer_sync(&pf->service_timer); 8689 err_mac_addr: 8690 err_configure_lan_hmc: 8691 (void)i40e_shutdown_lan_hmc(hw); 8692 err_init_lan_hmc: 8693 kfree(pf->qp_pile); 8694 kfree(pf->irq_pile); 8695 err_sw_init: 8696 err_adminq_setup: 8697 (void)i40e_shutdown_adminq(hw); 8698 err_pf_reset: 8699 iounmap(hw->hw_addr); 8700 err_ioremap: 8701 kfree(pf); 8702 err_pf_alloc: 8703 pci_disable_pcie_error_reporting(pdev); 8704 pci_release_selected_regions(pdev, 8705 pci_select_bars(pdev, IORESOURCE_MEM)); 8706 err_pci_reg: 8707 err_dma: 8708 pci_disable_device(pdev); 8709 return err; 8710 } 8711 8712 /** 8713 * i40e_remove - Device removal routine 8714 * @pdev: PCI device information struct 8715 * 8716 * i40e_remove is called by the PCI subsystem to alert the driver 8717 * that is should release a PCI device. This could be caused by a 8718 * Hot-Plug event, or because the driver is going to be removed from 8719 * memory. 8720 **/ 8721 static void i40e_remove(struct pci_dev *pdev) 8722 { 8723 struct i40e_pf *pf = pci_get_drvdata(pdev); 8724 i40e_status ret_code; 8725 u32 reg; 8726 int i; 8727 8728 i40e_dbg_pf_exit(pf); 8729 8730 i40e_ptp_stop(pf); 8731 8732 /* no more scheduling of any task */ 8733 set_bit(__I40E_DOWN, &pf->state); 8734 del_timer_sync(&pf->service_timer); 8735 cancel_work_sync(&pf->service_task); 8736 8737 if (pf->flags & I40E_FLAG_SRIOV_ENABLED) { 8738 i40e_free_vfs(pf); 8739 pf->flags &= ~I40E_FLAG_SRIOV_ENABLED; 8740 } 8741 8742 i40e_fdir_teardown(pf); 8743 8744 /* If there is a switch structure or any orphans, remove them. 8745 * This will leave only the PF's VSI remaining. 8746 */ 8747 for (i = 0; i < I40E_MAX_VEB; i++) { 8748 if (!pf->veb[i]) 8749 continue; 8750 8751 if (pf->veb[i]->uplink_seid == pf->mac_seid || 8752 pf->veb[i]->uplink_seid == 0) 8753 i40e_switch_branch_release(pf->veb[i]); 8754 } 8755 8756 /* Now we can shutdown the PF's VSI, just before we kill 8757 * adminq and hmc. 8758 */ 8759 if (pf->vsi[pf->lan_vsi]) 8760 i40e_vsi_release(pf->vsi[pf->lan_vsi]); 8761 8762 i40e_stop_misc_vector(pf); 8763 if (pf->flags & I40E_FLAG_MSIX_ENABLED) { 8764 synchronize_irq(pf->msix_entries[0].vector); 8765 free_irq(pf->msix_entries[0].vector, pf); 8766 } 8767 8768 /* shutdown and destroy the HMC */ 8769 if (pf->hw.hmc.hmc_obj) { 8770 ret_code = i40e_shutdown_lan_hmc(&pf->hw); 8771 if (ret_code) 8772 dev_warn(&pdev->dev, 8773 "Failed to destroy the HMC resources: %d\n", 8774 ret_code); 8775 } 8776 8777 /* shutdown the adminq */ 8778 ret_code = i40e_shutdown_adminq(&pf->hw); 8779 if (ret_code) 8780 dev_warn(&pdev->dev, 8781 "Failed to destroy the Admin Queue resources: %d\n", 8782 ret_code); 8783 8784 /* Clear all dynamic memory lists of rings, q_vectors, and VSIs */ 8785 i40e_clear_interrupt_scheme(pf); 8786 for (i = 0; i < pf->num_alloc_vsi; i++) { 8787 if (pf->vsi[i]) { 8788 i40e_vsi_clear_rings(pf->vsi[i]); 8789 i40e_vsi_clear(pf->vsi[i]); 8790 pf->vsi[i] = NULL; 8791 } 8792 } 8793 8794 for (i = 0; i < I40E_MAX_VEB; i++) { 8795 kfree(pf->veb[i]); 8796 pf->veb[i] = NULL; 8797 } 8798 8799 kfree(pf->qp_pile); 8800 kfree(pf->irq_pile); 8801 kfree(pf->vsi); 8802 8803 /* force a PF reset to clean anything leftover */ 8804 reg = rd32(&pf->hw, I40E_PFGEN_CTRL); 8805 wr32(&pf->hw, I40E_PFGEN_CTRL, (reg | I40E_PFGEN_CTRL_PFSWR_MASK)); 8806 i40e_flush(&pf->hw); 8807 8808 iounmap(pf->hw.hw_addr); 8809 kfree(pf); 8810 pci_release_selected_regions(pdev, 8811 pci_select_bars(pdev, IORESOURCE_MEM)); 8812 8813 pci_disable_pcie_error_reporting(pdev); 8814 pci_disable_device(pdev); 8815 } 8816 8817 /** 8818 * i40e_pci_error_detected - warning that something funky happened in PCI land 8819 * @pdev: PCI device information struct 8820 * 8821 * Called to warn that something happened and the error handling steps 8822 * are in progress. Allows the driver to quiesce things, be ready for 8823 * remediation. 8824 **/ 8825 static pci_ers_result_t i40e_pci_error_detected(struct pci_dev *pdev, 8826 enum pci_channel_state error) 8827 { 8828 struct i40e_pf *pf = pci_get_drvdata(pdev); 8829 8830 dev_info(&pdev->dev, "%s: error %d\n", __func__, error); 8831 8832 /* shutdown all operations */ 8833 if (!test_bit(__I40E_SUSPENDED, &pf->state)) { 8834 rtnl_lock(); 8835 i40e_prep_for_reset(pf); 8836 rtnl_unlock(); 8837 } 8838 8839 /* Request a slot reset */ 8840 return PCI_ERS_RESULT_NEED_RESET; 8841 } 8842 8843 /** 8844 * i40e_pci_error_slot_reset - a PCI slot reset just happened 8845 * @pdev: PCI device information struct 8846 * 8847 * Called to find if the driver can work with the device now that 8848 * the pci slot has been reset. If a basic connection seems good 8849 * (registers are readable and have sane content) then return a 8850 * happy little PCI_ERS_RESULT_xxx. 8851 **/ 8852 static pci_ers_result_t i40e_pci_error_slot_reset(struct pci_dev *pdev) 8853 { 8854 struct i40e_pf *pf = pci_get_drvdata(pdev); 8855 pci_ers_result_t result; 8856 int err; 8857 u32 reg; 8858 8859 dev_info(&pdev->dev, "%s\n", __func__); 8860 if (pci_enable_device_mem(pdev)) { 8861 dev_info(&pdev->dev, 8862 "Cannot re-enable PCI device after reset.\n"); 8863 result = PCI_ERS_RESULT_DISCONNECT; 8864 } else { 8865 pci_set_master(pdev); 8866 pci_restore_state(pdev); 8867 pci_save_state(pdev); 8868 pci_wake_from_d3(pdev, false); 8869 8870 reg = rd32(&pf->hw, I40E_GLGEN_RTRIG); 8871 if (reg == 0) 8872 result = PCI_ERS_RESULT_RECOVERED; 8873 else 8874 result = PCI_ERS_RESULT_DISCONNECT; 8875 } 8876 8877 err = pci_cleanup_aer_uncorrect_error_status(pdev); 8878 if (err) { 8879 dev_info(&pdev->dev, 8880 "pci_cleanup_aer_uncorrect_error_status failed 0x%0x\n", 8881 err); 8882 /* non-fatal, continue */ 8883 } 8884 8885 return result; 8886 } 8887 8888 /** 8889 * i40e_pci_error_resume - restart operations after PCI error recovery 8890 * @pdev: PCI device information struct 8891 * 8892 * Called to allow the driver to bring things back up after PCI error 8893 * and/or reset recovery has finished. 8894 **/ 8895 static void i40e_pci_error_resume(struct pci_dev *pdev) 8896 { 8897 struct i40e_pf *pf = pci_get_drvdata(pdev); 8898 8899 dev_info(&pdev->dev, "%s\n", __func__); 8900 if (test_bit(__I40E_SUSPENDED, &pf->state)) 8901 return; 8902 8903 rtnl_lock(); 8904 i40e_handle_reset_warning(pf); 8905 rtnl_lock(); 8906 } 8907 8908 /** 8909 * i40e_shutdown - PCI callback for shutting down 8910 * @pdev: PCI device information struct 8911 **/ 8912 static void i40e_shutdown(struct pci_dev *pdev) 8913 { 8914 struct i40e_pf *pf = pci_get_drvdata(pdev); 8915 struct i40e_hw *hw = &pf->hw; 8916 8917 set_bit(__I40E_SUSPENDED, &pf->state); 8918 set_bit(__I40E_DOWN, &pf->state); 8919 rtnl_lock(); 8920 i40e_prep_for_reset(pf); 8921 rtnl_unlock(); 8922 8923 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 8924 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 8925 8926 if (system_state == SYSTEM_POWER_OFF) { 8927 pci_wake_from_d3(pdev, pf->wol_en); 8928 pci_set_power_state(pdev, PCI_D3hot); 8929 } 8930 } 8931 8932 #ifdef CONFIG_PM 8933 /** 8934 * i40e_suspend - PCI callback for moving to D3 8935 * @pdev: PCI device information struct 8936 **/ 8937 static int i40e_suspend(struct pci_dev *pdev, pm_message_t state) 8938 { 8939 struct i40e_pf *pf = pci_get_drvdata(pdev); 8940 struct i40e_hw *hw = &pf->hw; 8941 8942 set_bit(__I40E_SUSPENDED, &pf->state); 8943 set_bit(__I40E_DOWN, &pf->state); 8944 rtnl_lock(); 8945 i40e_prep_for_reset(pf); 8946 rtnl_unlock(); 8947 8948 wr32(hw, I40E_PFPM_APM, (pf->wol_en ? I40E_PFPM_APM_APME_MASK : 0)); 8949 wr32(hw, I40E_PFPM_WUFC, (pf->wol_en ? I40E_PFPM_WUFC_MAG_MASK : 0)); 8950 8951 pci_wake_from_d3(pdev, pf->wol_en); 8952 pci_set_power_state(pdev, PCI_D3hot); 8953 8954 return 0; 8955 } 8956 8957 /** 8958 * i40e_resume - PCI callback for waking up from D3 8959 * @pdev: PCI device information struct 8960 **/ 8961 static int i40e_resume(struct pci_dev *pdev) 8962 { 8963 struct i40e_pf *pf = pci_get_drvdata(pdev); 8964 u32 err; 8965 8966 pci_set_power_state(pdev, PCI_D0); 8967 pci_restore_state(pdev); 8968 /* pci_restore_state() clears dev->state_saves, so 8969 * call pci_save_state() again to restore it. 8970 */ 8971 pci_save_state(pdev); 8972 8973 err = pci_enable_device_mem(pdev); 8974 if (err) { 8975 dev_err(&pdev->dev, 8976 "%s: Cannot enable PCI device from suspend\n", 8977 __func__); 8978 return err; 8979 } 8980 pci_set_master(pdev); 8981 8982 /* no wakeup events while running */ 8983 pci_wake_from_d3(pdev, false); 8984 8985 /* handling the reset will rebuild the device state */ 8986 if (test_and_clear_bit(__I40E_SUSPENDED, &pf->state)) { 8987 clear_bit(__I40E_DOWN, &pf->state); 8988 rtnl_lock(); 8989 i40e_reset_and_rebuild(pf, false); 8990 rtnl_unlock(); 8991 } 8992 8993 return 0; 8994 } 8995 8996 #endif 8997 static const struct pci_error_handlers i40e_err_handler = { 8998 .error_detected = i40e_pci_error_detected, 8999 .slot_reset = i40e_pci_error_slot_reset, 9000 .resume = i40e_pci_error_resume, 9001 }; 9002 9003 static struct pci_driver i40e_driver = { 9004 .name = i40e_driver_name, 9005 .id_table = i40e_pci_tbl, 9006 .probe = i40e_probe, 9007 .remove = i40e_remove, 9008 #ifdef CONFIG_PM 9009 .suspend = i40e_suspend, 9010 .resume = i40e_resume, 9011 #endif 9012 .shutdown = i40e_shutdown, 9013 .err_handler = &i40e_err_handler, 9014 .sriov_configure = i40e_pci_sriov_configure, 9015 }; 9016 9017 /** 9018 * i40e_init_module - Driver registration routine 9019 * 9020 * i40e_init_module is the first routine called when the driver is 9021 * loaded. All it does is register with the PCI subsystem. 9022 **/ 9023 static int __init i40e_init_module(void) 9024 { 9025 pr_info("%s: %s - version %s\n", i40e_driver_name, 9026 i40e_driver_string, i40e_driver_version_str); 9027 pr_info("%s: %s\n", i40e_driver_name, i40e_copyright); 9028 i40e_dbg_init(); 9029 return pci_register_driver(&i40e_driver); 9030 } 9031 module_init(i40e_init_module); 9032 9033 /** 9034 * i40e_exit_module - Driver exit cleanup routine 9035 * 9036 * i40e_exit_module is called just before the driver is removed 9037 * from memory. 9038 **/ 9039 static void __exit i40e_exit_module(void) 9040 { 9041 pci_unregister_driver(&i40e_driver); 9042 i40e_dbg_exit(); 9043 } 9044 module_exit(i40e_exit_module); 9045