1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2005 - 2016 Broadcom 4 * All rights reserved. 5 * 6 * Contact Information: 7 * linux-drivers@emulex.com 8 * 9 * Emulex 10 * 3333 Susan Street 11 * Costa Mesa, CA 92626 12 */ 13 14 #include <linux/prefetch.h> 15 #include <linux/module.h> 16 #include "be.h" 17 #include "be_cmds.h" 18 #include <asm/div64.h> 19 #include <linux/aer.h> 20 #include <linux/if_bridge.h> 21 #include <net/busy_poll.h> 22 #include <net/vxlan.h> 23 24 MODULE_DESCRIPTION(DRV_DESC); 25 MODULE_AUTHOR("Emulex Corporation"); 26 MODULE_LICENSE("GPL"); 27 28 /* num_vfs module param is obsolete. 29 * Use sysfs method to enable/disable VFs. 30 */ 31 static unsigned int num_vfs; 32 module_param(num_vfs, uint, 0444); 33 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize"); 34 35 static ushort rx_frag_size = 2048; 36 module_param(rx_frag_size, ushort, 0444); 37 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data."); 38 39 /* Per-module error detection/recovery workq shared across all functions. 40 * Each function schedules its own work request on this shared workq. 41 */ 42 static struct workqueue_struct *be_err_recovery_workq; 43 44 static const struct pci_device_id be_dev_ids[] = { 45 #ifdef CONFIG_BE2NET_BE2 46 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 47 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 48 #endif /* CONFIG_BE2NET_BE2 */ 49 #ifdef CONFIG_BE2NET_BE3 50 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 51 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 52 #endif /* CONFIG_BE2NET_BE3 */ 53 #ifdef CONFIG_BE2NET_LANCER 54 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)}, 55 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)}, 56 #endif /* CONFIG_BE2NET_LANCER */ 57 #ifdef CONFIG_BE2NET_SKYHAWK 58 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)}, 59 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)}, 60 #endif /* CONFIG_BE2NET_SKYHAWK */ 61 { 0 } 62 }; 63 MODULE_DEVICE_TABLE(pci, be_dev_ids); 64 65 /* Workqueue used by all functions for defering cmd calls to the adapter */ 66 static struct workqueue_struct *be_wq; 67 68 /* UE Status Low CSR */ 69 static const char * const ue_status_low_desc[] = { 70 "CEV", 71 "CTX", 72 "DBUF", 73 "ERX", 74 "Host", 75 "MPU", 76 "NDMA", 77 "PTC ", 78 "RDMA ", 79 "RXF ", 80 "RXIPS ", 81 "RXULP0 ", 82 "RXULP1 ", 83 "RXULP2 ", 84 "TIM ", 85 "TPOST ", 86 "TPRE ", 87 "TXIPS ", 88 "TXULP0 ", 89 "TXULP1 ", 90 "UC ", 91 "WDMA ", 92 "TXULP2 ", 93 "HOST1 ", 94 "P0_OB_LINK ", 95 "P1_OB_LINK ", 96 "HOST_GPIO ", 97 "MBOX ", 98 "ERX2 ", 99 "SPARE ", 100 "JTAG ", 101 "MPU_INTPEND " 102 }; 103 104 /* UE Status High CSR */ 105 static const char * const ue_status_hi_desc[] = { 106 "LPCMEMHOST", 107 "MGMT_MAC", 108 "PCS0ONLINE", 109 "MPU_IRAM", 110 "PCS1ONLINE", 111 "PCTL0", 112 "PCTL1", 113 "PMEM", 114 "RR", 115 "TXPB", 116 "RXPP", 117 "XAUI", 118 "TXP", 119 "ARM", 120 "IPC", 121 "HOST2", 122 "HOST3", 123 "HOST4", 124 "HOST5", 125 "HOST6", 126 "HOST7", 127 "ECRC", 128 "Poison TLP", 129 "NETC", 130 "PERIPH", 131 "LLTXULP", 132 "D2P", 133 "RCON", 134 "LDMA", 135 "LLTXP", 136 "LLTXPB", 137 "Unknown" 138 }; 139 140 #define BE_VF_IF_EN_FLAGS (BE_IF_FLAGS_UNTAGGED | \ 141 BE_IF_FLAGS_BROADCAST | \ 142 BE_IF_FLAGS_MULTICAST | \ 143 BE_IF_FLAGS_PASS_L3L4_ERRORS) 144 145 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q) 146 { 147 struct be_dma_mem *mem = &q->dma_mem; 148 149 if (mem->va) { 150 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 151 mem->dma); 152 mem->va = NULL; 153 } 154 } 155 156 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q, 157 u16 len, u16 entry_size) 158 { 159 struct be_dma_mem *mem = &q->dma_mem; 160 161 memset(q, 0, sizeof(*q)); 162 q->len = len; 163 q->entry_size = entry_size; 164 mem->size = len * entry_size; 165 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size, 166 &mem->dma, GFP_KERNEL); 167 if (!mem->va) 168 return -ENOMEM; 169 return 0; 170 } 171 172 static void be_reg_intr_set(struct be_adapter *adapter, bool enable) 173 { 174 u32 reg, enabled; 175 176 pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, 177 ®); 178 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 179 180 if (!enabled && enable) 181 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 182 else if (enabled && !enable) 183 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 184 else 185 return; 186 187 pci_write_config_dword(adapter->pdev, 188 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg); 189 } 190 191 static void be_intr_set(struct be_adapter *adapter, bool enable) 192 { 193 int status = 0; 194 195 /* On lancer interrupts can't be controlled via this register */ 196 if (lancer_chip(adapter)) 197 return; 198 199 if (be_check_error(adapter, BE_ERROR_EEH)) 200 return; 201 202 status = be_cmd_intr_set(adapter, enable); 203 if (status) 204 be_reg_intr_set(adapter, enable); 205 } 206 207 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted) 208 { 209 u32 val = 0; 210 211 if (be_check_error(adapter, BE_ERROR_HW)) 212 return; 213 214 val |= qid & DB_RQ_RING_ID_MASK; 215 val |= posted << DB_RQ_NUM_POSTED_SHIFT; 216 217 wmb(); 218 iowrite32(val, adapter->db + DB_RQ_OFFSET); 219 } 220 221 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo, 222 u16 posted) 223 { 224 u32 val = 0; 225 226 if (be_check_error(adapter, BE_ERROR_HW)) 227 return; 228 229 val |= txo->q.id & DB_TXULP_RING_ID_MASK; 230 val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT; 231 232 wmb(); 233 iowrite32(val, adapter->db + txo->db_offset); 234 } 235 236 static void be_eq_notify(struct be_adapter *adapter, u16 qid, 237 bool arm, bool clear_int, u16 num_popped, 238 u32 eq_delay_mult_enc) 239 { 240 u32 val = 0; 241 242 val |= qid & DB_EQ_RING_ID_MASK; 243 val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT); 244 245 if (be_check_error(adapter, BE_ERROR_HW)) 246 return; 247 248 if (arm) 249 val |= 1 << DB_EQ_REARM_SHIFT; 250 if (clear_int) 251 val |= 1 << DB_EQ_CLR_SHIFT; 252 val |= 1 << DB_EQ_EVNT_SHIFT; 253 val |= num_popped << DB_EQ_NUM_POPPED_SHIFT; 254 val |= eq_delay_mult_enc << DB_EQ_R2I_DLY_SHIFT; 255 iowrite32(val, adapter->db + DB_EQ_OFFSET); 256 } 257 258 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped) 259 { 260 u32 val = 0; 261 262 val |= qid & DB_CQ_RING_ID_MASK; 263 val |= ((qid & DB_CQ_RING_ID_EXT_MASK) << 264 DB_CQ_RING_ID_EXT_MASK_SHIFT); 265 266 if (be_check_error(adapter, BE_ERROR_HW)) 267 return; 268 269 if (arm) 270 val |= 1 << DB_CQ_REARM_SHIFT; 271 val |= num_popped << DB_CQ_NUM_POPPED_SHIFT; 272 iowrite32(val, adapter->db + DB_CQ_OFFSET); 273 } 274 275 static int be_dev_mac_add(struct be_adapter *adapter, const u8 *mac) 276 { 277 int i; 278 279 /* Check if mac has already been added as part of uc-list */ 280 for (i = 0; i < adapter->uc_macs; i++) { 281 if (ether_addr_equal(adapter->uc_list[i].mac, mac)) { 282 /* mac already added, skip addition */ 283 adapter->pmac_id[0] = adapter->pmac_id[i + 1]; 284 return 0; 285 } 286 } 287 288 return be_cmd_pmac_add(adapter, mac, adapter->if_handle, 289 &adapter->pmac_id[0], 0); 290 } 291 292 static void be_dev_mac_del(struct be_adapter *adapter, int pmac_id) 293 { 294 int i; 295 296 /* Skip deletion if the programmed mac is 297 * being used in uc-list 298 */ 299 for (i = 0; i < adapter->uc_macs; i++) { 300 if (adapter->pmac_id[i + 1] == pmac_id) 301 return; 302 } 303 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 304 } 305 306 static int be_mac_addr_set(struct net_device *netdev, void *p) 307 { 308 struct be_adapter *adapter = netdev_priv(netdev); 309 struct device *dev = &adapter->pdev->dev; 310 struct sockaddr *addr = p; 311 int status; 312 u8 mac[ETH_ALEN]; 313 u32 old_pmac_id = adapter->pmac_id[0]; 314 315 if (!is_valid_ether_addr(addr->sa_data)) 316 return -EADDRNOTAVAIL; 317 318 /* Proceed further only if, User provided MAC is different 319 * from active MAC 320 */ 321 if (ether_addr_equal(addr->sa_data, adapter->dev_mac)) 322 return 0; 323 324 /* BE3 VFs without FILTMGMT privilege are not allowed to set its MAC 325 * address 326 */ 327 if (BEx_chip(adapter) && be_virtfn(adapter) && 328 !check_privilege(adapter, BE_PRIV_FILTMGMT)) 329 return -EPERM; 330 331 /* if device is not running, copy MAC to netdev->dev_addr */ 332 if (!netif_running(netdev)) 333 goto done; 334 335 /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT 336 * privilege or if PF did not provision the new MAC address. 337 * On BE3, this cmd will always fail if the VF doesn't have the 338 * FILTMGMT privilege. This failure is OK, only if the PF programmed 339 * the MAC for the VF. 340 */ 341 mutex_lock(&adapter->rx_filter_lock); 342 status = be_dev_mac_add(adapter, (u8 *)addr->sa_data); 343 if (!status) { 344 345 /* Delete the old programmed MAC. This call may fail if the 346 * old MAC was already deleted by the PF driver. 347 */ 348 if (adapter->pmac_id[0] != old_pmac_id) 349 be_dev_mac_del(adapter, old_pmac_id); 350 } 351 352 mutex_unlock(&adapter->rx_filter_lock); 353 /* Decide if the new MAC is successfully activated only after 354 * querying the FW 355 */ 356 status = be_cmd_get_active_mac(adapter, adapter->pmac_id[0], mac, 357 adapter->if_handle, true, 0); 358 if (status) 359 goto err; 360 361 /* The MAC change did not happen, either due to lack of privilege 362 * or PF didn't pre-provision. 363 */ 364 if (!ether_addr_equal(addr->sa_data, mac)) { 365 status = -EPERM; 366 goto err; 367 } 368 369 /* Remember currently programmed MAC */ 370 ether_addr_copy(adapter->dev_mac, addr->sa_data); 371 done: 372 eth_hw_addr_set(netdev, addr->sa_data); 373 dev_info(dev, "MAC address changed to %pM\n", addr->sa_data); 374 return 0; 375 err: 376 dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data); 377 return status; 378 } 379 380 /* BE2 supports only v0 cmd */ 381 static void *hw_stats_from_cmd(struct be_adapter *adapter) 382 { 383 if (BE2_chip(adapter)) { 384 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va; 385 386 return &cmd->hw_stats; 387 } else if (BE3_chip(adapter)) { 388 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va; 389 390 return &cmd->hw_stats; 391 } else { 392 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va; 393 394 return &cmd->hw_stats; 395 } 396 } 397 398 /* BE2 supports only v0 cmd */ 399 static void *be_erx_stats_from_cmd(struct be_adapter *adapter) 400 { 401 if (BE2_chip(adapter)) { 402 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 403 404 return &hw_stats->erx; 405 } else if (BE3_chip(adapter)) { 406 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 407 408 return &hw_stats->erx; 409 } else { 410 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 411 412 return &hw_stats->erx; 413 } 414 } 415 416 static void populate_be_v0_stats(struct be_adapter *adapter) 417 { 418 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 419 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 420 struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf; 421 struct be_port_rxf_stats_v0 *port_stats = 422 &rxf_stats->port[adapter->port_num]; 423 struct be_drv_stats *drvs = &adapter->drv_stats; 424 425 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 426 drvs->rx_pause_frames = port_stats->rx_pause_frames; 427 drvs->rx_crc_errors = port_stats->rx_crc_errors; 428 drvs->rx_control_frames = port_stats->rx_control_frames; 429 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 430 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 431 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 432 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 433 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 434 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 435 drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow; 436 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 437 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 438 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 439 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 440 drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow; 441 drvs->rx_dropped_header_too_small = 442 port_stats->rx_dropped_header_too_small; 443 drvs->rx_address_filtered = 444 port_stats->rx_address_filtered + 445 port_stats->rx_vlan_filtered; 446 drvs->rx_alignment_symbol_errors = 447 port_stats->rx_alignment_symbol_errors; 448 449 drvs->tx_pauseframes = port_stats->tx_pauseframes; 450 drvs->tx_controlframes = port_stats->tx_controlframes; 451 452 if (adapter->port_num) 453 drvs->jabber_events = rxf_stats->port1_jabber_events; 454 else 455 drvs->jabber_events = rxf_stats->port0_jabber_events; 456 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 457 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 458 drvs->forwarded_packets = rxf_stats->forwarded_packets; 459 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 460 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 461 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 462 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 463 } 464 465 static void populate_be_v1_stats(struct be_adapter *adapter) 466 { 467 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 468 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 469 struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf; 470 struct be_port_rxf_stats_v1 *port_stats = 471 &rxf_stats->port[adapter->port_num]; 472 struct be_drv_stats *drvs = &adapter->drv_stats; 473 474 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 475 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 476 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 477 drvs->rx_pause_frames = port_stats->rx_pause_frames; 478 drvs->rx_crc_errors = port_stats->rx_crc_errors; 479 drvs->rx_control_frames = port_stats->rx_control_frames; 480 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 481 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 482 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 483 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 484 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 485 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 486 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 487 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 488 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 489 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 490 drvs->rx_dropped_header_too_small = 491 port_stats->rx_dropped_header_too_small; 492 drvs->rx_input_fifo_overflow_drop = 493 port_stats->rx_input_fifo_overflow_drop; 494 drvs->rx_address_filtered = port_stats->rx_address_filtered; 495 drvs->rx_alignment_symbol_errors = 496 port_stats->rx_alignment_symbol_errors; 497 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 498 drvs->tx_pauseframes = port_stats->tx_pauseframes; 499 drvs->tx_controlframes = port_stats->tx_controlframes; 500 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 501 drvs->jabber_events = port_stats->jabber_events; 502 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 503 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 504 drvs->forwarded_packets = rxf_stats->forwarded_packets; 505 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 506 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 507 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 508 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 509 } 510 511 static void populate_be_v2_stats(struct be_adapter *adapter) 512 { 513 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 514 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 515 struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf; 516 struct be_port_rxf_stats_v2 *port_stats = 517 &rxf_stats->port[adapter->port_num]; 518 struct be_drv_stats *drvs = &adapter->drv_stats; 519 520 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 521 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 522 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 523 drvs->rx_pause_frames = port_stats->rx_pause_frames; 524 drvs->rx_crc_errors = port_stats->rx_crc_errors; 525 drvs->rx_control_frames = port_stats->rx_control_frames; 526 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 527 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 528 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 529 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 530 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 531 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 532 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 533 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 534 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 535 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 536 drvs->rx_dropped_header_too_small = 537 port_stats->rx_dropped_header_too_small; 538 drvs->rx_input_fifo_overflow_drop = 539 port_stats->rx_input_fifo_overflow_drop; 540 drvs->rx_address_filtered = port_stats->rx_address_filtered; 541 drvs->rx_alignment_symbol_errors = 542 port_stats->rx_alignment_symbol_errors; 543 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 544 drvs->tx_pauseframes = port_stats->tx_pauseframes; 545 drvs->tx_controlframes = port_stats->tx_controlframes; 546 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 547 drvs->jabber_events = port_stats->jabber_events; 548 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 549 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 550 drvs->forwarded_packets = rxf_stats->forwarded_packets; 551 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 552 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 553 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 554 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 555 if (be_roce_supported(adapter)) { 556 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd; 557 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd; 558 drvs->rx_roce_frames = port_stats->roce_frames_received; 559 drvs->roce_drops_crc = port_stats->roce_drops_crc; 560 drvs->roce_drops_payload_len = 561 port_stats->roce_drops_payload_len; 562 } 563 } 564 565 static void populate_lancer_stats(struct be_adapter *adapter) 566 { 567 struct be_drv_stats *drvs = &adapter->drv_stats; 568 struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter); 569 570 be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats)); 571 drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo; 572 drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo; 573 drvs->rx_control_frames = pport_stats->rx_control_frames_lo; 574 drvs->rx_in_range_errors = pport_stats->rx_in_range_errors; 575 drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo; 576 drvs->rx_dropped_runt = pport_stats->rx_dropped_runt; 577 drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors; 578 drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors; 579 drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors; 580 drvs->rx_dropped_tcp_length = 581 pport_stats->rx_dropped_invalid_tcp_length; 582 drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small; 583 drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short; 584 drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors; 585 drvs->rx_dropped_header_too_small = 586 pport_stats->rx_dropped_header_too_small; 587 drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 588 drvs->rx_address_filtered = 589 pport_stats->rx_address_filtered + 590 pport_stats->rx_vlan_filtered; 591 drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo; 592 drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 593 drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo; 594 drvs->tx_controlframes = pport_stats->tx_control_frames_lo; 595 drvs->jabber_events = pport_stats->rx_jabbers; 596 drvs->forwarded_packets = pport_stats->num_forwards_lo; 597 drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo; 598 drvs->rx_drops_too_many_frags = 599 pport_stats->rx_drops_too_many_frags_lo; 600 } 601 602 static void accumulate_16bit_val(u32 *acc, u16 val) 603 { 604 #define lo(x) (x & 0xFFFF) 605 #define hi(x) (x & 0xFFFF0000) 606 bool wrapped = val < lo(*acc); 607 u32 newacc = hi(*acc) + val; 608 609 if (wrapped) 610 newacc += 65536; 611 WRITE_ONCE(*acc, newacc); 612 } 613 614 static void populate_erx_stats(struct be_adapter *adapter, 615 struct be_rx_obj *rxo, u32 erx_stat) 616 { 617 if (!BEx_chip(adapter)) 618 rx_stats(rxo)->rx_drops_no_frags = erx_stat; 619 else 620 /* below erx HW counter can actually wrap around after 621 * 65535. Driver accumulates a 32-bit value 622 */ 623 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags, 624 (u16)erx_stat); 625 } 626 627 void be_parse_stats(struct be_adapter *adapter) 628 { 629 struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter); 630 struct be_rx_obj *rxo; 631 int i; 632 u32 erx_stat; 633 634 if (lancer_chip(adapter)) { 635 populate_lancer_stats(adapter); 636 } else { 637 if (BE2_chip(adapter)) 638 populate_be_v0_stats(adapter); 639 else if (BE3_chip(adapter)) 640 /* for BE3 */ 641 populate_be_v1_stats(adapter); 642 else 643 populate_be_v2_stats(adapter); 644 645 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */ 646 for_all_rx_queues(adapter, rxo, i) { 647 erx_stat = erx->rx_drops_no_fragments[rxo->q.id]; 648 populate_erx_stats(adapter, rxo, erx_stat); 649 } 650 } 651 } 652 653 static void be_get_stats64(struct net_device *netdev, 654 struct rtnl_link_stats64 *stats) 655 { 656 struct be_adapter *adapter = netdev_priv(netdev); 657 struct be_drv_stats *drvs = &adapter->drv_stats; 658 struct be_rx_obj *rxo; 659 struct be_tx_obj *txo; 660 u64 pkts, bytes; 661 unsigned int start; 662 int i; 663 664 for_all_rx_queues(adapter, rxo, i) { 665 const struct be_rx_stats *rx_stats = rx_stats(rxo); 666 667 do { 668 start = u64_stats_fetch_begin_irq(&rx_stats->sync); 669 pkts = rx_stats(rxo)->rx_pkts; 670 bytes = rx_stats(rxo)->rx_bytes; 671 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start)); 672 stats->rx_packets += pkts; 673 stats->rx_bytes += bytes; 674 stats->multicast += rx_stats(rxo)->rx_mcast_pkts; 675 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs + 676 rx_stats(rxo)->rx_drops_no_frags; 677 } 678 679 for_all_tx_queues(adapter, txo, i) { 680 const struct be_tx_stats *tx_stats = tx_stats(txo); 681 682 do { 683 start = u64_stats_fetch_begin_irq(&tx_stats->sync); 684 pkts = tx_stats(txo)->tx_pkts; 685 bytes = tx_stats(txo)->tx_bytes; 686 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start)); 687 stats->tx_packets += pkts; 688 stats->tx_bytes += bytes; 689 } 690 691 /* bad pkts received */ 692 stats->rx_errors = drvs->rx_crc_errors + 693 drvs->rx_alignment_symbol_errors + 694 drvs->rx_in_range_errors + 695 drvs->rx_out_range_errors + 696 drvs->rx_frame_too_long + 697 drvs->rx_dropped_too_small + 698 drvs->rx_dropped_too_short + 699 drvs->rx_dropped_header_too_small + 700 drvs->rx_dropped_tcp_length + 701 drvs->rx_dropped_runt; 702 703 /* detailed rx errors */ 704 stats->rx_length_errors = drvs->rx_in_range_errors + 705 drvs->rx_out_range_errors + 706 drvs->rx_frame_too_long; 707 708 stats->rx_crc_errors = drvs->rx_crc_errors; 709 710 /* frame alignment errors */ 711 stats->rx_frame_errors = drvs->rx_alignment_symbol_errors; 712 713 /* receiver fifo overrun */ 714 /* drops_no_pbuf is no per i/f, it's per BE card */ 715 stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop + 716 drvs->rx_input_fifo_overflow_drop + 717 drvs->rx_drops_no_pbuf; 718 } 719 720 void be_link_status_update(struct be_adapter *adapter, u8 link_status) 721 { 722 struct net_device *netdev = adapter->netdev; 723 724 if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) { 725 netif_carrier_off(netdev); 726 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT; 727 } 728 729 if (link_status) 730 netif_carrier_on(netdev); 731 else 732 netif_carrier_off(netdev); 733 734 netdev_info(netdev, "Link is %s\n", link_status ? "Up" : "Down"); 735 } 736 737 static int be_gso_hdr_len(struct sk_buff *skb) 738 { 739 if (skb->encapsulation) 740 return skb_inner_transport_offset(skb) + 741 inner_tcp_hdrlen(skb); 742 return skb_transport_offset(skb) + tcp_hdrlen(skb); 743 } 744 745 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb) 746 { 747 struct be_tx_stats *stats = tx_stats(txo); 748 u32 tx_pkts = skb_shinfo(skb)->gso_segs ? : 1; 749 /* Account for headers which get duplicated in TSO pkt */ 750 u32 dup_hdr_len = tx_pkts > 1 ? be_gso_hdr_len(skb) * (tx_pkts - 1) : 0; 751 752 u64_stats_update_begin(&stats->sync); 753 stats->tx_reqs++; 754 stats->tx_bytes += skb->len + dup_hdr_len; 755 stats->tx_pkts += tx_pkts; 756 if (skb->encapsulation && skb->ip_summed == CHECKSUM_PARTIAL) 757 stats->tx_vxlan_offload_pkts += tx_pkts; 758 u64_stats_update_end(&stats->sync); 759 } 760 761 /* Returns number of WRBs needed for the skb */ 762 static u32 skb_wrb_cnt(struct sk_buff *skb) 763 { 764 /* +1 for the header wrb */ 765 return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags; 766 } 767 768 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len) 769 { 770 wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr)); 771 wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr)); 772 wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK); 773 wrb->rsvd0 = 0; 774 } 775 776 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb 777 * to avoid the swap and shift/mask operations in wrb_fill(). 778 */ 779 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb) 780 { 781 wrb->frag_pa_hi = 0; 782 wrb->frag_pa_lo = 0; 783 wrb->frag_len = 0; 784 wrb->rsvd0 = 0; 785 } 786 787 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter, 788 struct sk_buff *skb) 789 { 790 u8 vlan_prio; 791 u16 vlan_tag; 792 793 vlan_tag = skb_vlan_tag_get(skb); 794 vlan_prio = skb_vlan_tag_get_prio(skb); 795 /* If vlan priority provided by OS is NOT in available bmap */ 796 if (!(adapter->vlan_prio_bmap & (1 << vlan_prio))) 797 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) | 798 adapter->recommended_prio_bits; 799 800 return vlan_tag; 801 } 802 803 /* Used only for IP tunnel packets */ 804 static u16 skb_inner_ip_proto(struct sk_buff *skb) 805 { 806 return (inner_ip_hdr(skb)->version == 4) ? 807 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr; 808 } 809 810 static u16 skb_ip_proto(struct sk_buff *skb) 811 { 812 return (ip_hdr(skb)->version == 4) ? 813 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr; 814 } 815 816 static inline bool be_is_txq_full(struct be_tx_obj *txo) 817 { 818 return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len; 819 } 820 821 static inline bool be_can_txq_wake(struct be_tx_obj *txo) 822 { 823 return atomic_read(&txo->q.used) < txo->q.len / 2; 824 } 825 826 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo) 827 { 828 return atomic_read(&txo->q.used) > txo->pend_wrb_cnt; 829 } 830 831 static void be_get_wrb_params_from_skb(struct be_adapter *adapter, 832 struct sk_buff *skb, 833 struct be_wrb_params *wrb_params) 834 { 835 u16 proto; 836 837 if (skb_is_gso(skb)) { 838 BE_WRB_F_SET(wrb_params->features, LSO, 1); 839 wrb_params->lso_mss = skb_shinfo(skb)->gso_size; 840 if (skb_is_gso_v6(skb) && !lancer_chip(adapter)) 841 BE_WRB_F_SET(wrb_params->features, LSO6, 1); 842 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 843 if (skb->encapsulation) { 844 BE_WRB_F_SET(wrb_params->features, IPCS, 1); 845 proto = skb_inner_ip_proto(skb); 846 } else { 847 proto = skb_ip_proto(skb); 848 } 849 if (proto == IPPROTO_TCP) 850 BE_WRB_F_SET(wrb_params->features, TCPCS, 1); 851 else if (proto == IPPROTO_UDP) 852 BE_WRB_F_SET(wrb_params->features, UDPCS, 1); 853 } 854 855 if (skb_vlan_tag_present(skb)) { 856 BE_WRB_F_SET(wrb_params->features, VLAN, 1); 857 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb); 858 } 859 860 BE_WRB_F_SET(wrb_params->features, CRC, 1); 861 } 862 863 static void wrb_fill_hdr(struct be_adapter *adapter, 864 struct be_eth_hdr_wrb *hdr, 865 struct be_wrb_params *wrb_params, 866 struct sk_buff *skb) 867 { 868 memset(hdr, 0, sizeof(*hdr)); 869 870 SET_TX_WRB_HDR_BITS(crc, hdr, 871 BE_WRB_F_GET(wrb_params->features, CRC)); 872 SET_TX_WRB_HDR_BITS(ipcs, hdr, 873 BE_WRB_F_GET(wrb_params->features, IPCS)); 874 SET_TX_WRB_HDR_BITS(tcpcs, hdr, 875 BE_WRB_F_GET(wrb_params->features, TCPCS)); 876 SET_TX_WRB_HDR_BITS(udpcs, hdr, 877 BE_WRB_F_GET(wrb_params->features, UDPCS)); 878 879 SET_TX_WRB_HDR_BITS(lso, hdr, 880 BE_WRB_F_GET(wrb_params->features, LSO)); 881 SET_TX_WRB_HDR_BITS(lso6, hdr, 882 BE_WRB_F_GET(wrb_params->features, LSO6)); 883 SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss); 884 885 /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this 886 * hack is not needed, the evt bit is set while ringing DB. 887 */ 888 SET_TX_WRB_HDR_BITS(event, hdr, 889 BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW)); 890 SET_TX_WRB_HDR_BITS(vlan, hdr, 891 BE_WRB_F_GET(wrb_params->features, VLAN)); 892 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag); 893 894 SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb)); 895 SET_TX_WRB_HDR_BITS(len, hdr, skb->len); 896 SET_TX_WRB_HDR_BITS(mgmt, hdr, 897 BE_WRB_F_GET(wrb_params->features, OS2BMC)); 898 } 899 900 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb, 901 bool unmap_single) 902 { 903 dma_addr_t dma; 904 u32 frag_len = le32_to_cpu(wrb->frag_len); 905 906 907 dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 | 908 (u64)le32_to_cpu(wrb->frag_pa_lo); 909 if (frag_len) { 910 if (unmap_single) 911 dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE); 912 else 913 dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE); 914 } 915 } 916 917 /* Grab a WRB header for xmit */ 918 static u32 be_tx_get_wrb_hdr(struct be_tx_obj *txo) 919 { 920 u32 head = txo->q.head; 921 922 queue_head_inc(&txo->q); 923 return head; 924 } 925 926 /* Set up the WRB header for xmit */ 927 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter, 928 struct be_tx_obj *txo, 929 struct be_wrb_params *wrb_params, 930 struct sk_buff *skb, u16 head) 931 { 932 u32 num_frags = skb_wrb_cnt(skb); 933 struct be_queue_info *txq = &txo->q; 934 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head); 935 936 wrb_fill_hdr(adapter, hdr, wrb_params, skb); 937 be_dws_cpu_to_le(hdr, sizeof(*hdr)); 938 939 BUG_ON(txo->sent_skb_list[head]); 940 txo->sent_skb_list[head] = skb; 941 txo->last_req_hdr = head; 942 atomic_add(num_frags, &txq->used); 943 txo->last_req_wrb_cnt = num_frags; 944 txo->pend_wrb_cnt += num_frags; 945 } 946 947 /* Setup a WRB fragment (buffer descriptor) for xmit */ 948 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr, 949 int len) 950 { 951 struct be_eth_wrb *wrb; 952 struct be_queue_info *txq = &txo->q; 953 954 wrb = queue_head_node(txq); 955 wrb_fill(wrb, busaddr, len); 956 queue_head_inc(txq); 957 } 958 959 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine 960 * was invoked. The producer index is restored to the previous packet and the 961 * WRBs of the current packet are unmapped. Invoked to handle tx setup errors. 962 */ 963 static void be_xmit_restore(struct be_adapter *adapter, 964 struct be_tx_obj *txo, u32 head, bool map_single, 965 u32 copied) 966 { 967 struct device *dev; 968 struct be_eth_wrb *wrb; 969 struct be_queue_info *txq = &txo->q; 970 971 dev = &adapter->pdev->dev; 972 txq->head = head; 973 974 /* skip the first wrb (hdr); it's not mapped */ 975 queue_head_inc(txq); 976 while (copied) { 977 wrb = queue_head_node(txq); 978 unmap_tx_frag(dev, wrb, map_single); 979 map_single = false; 980 copied -= le32_to_cpu(wrb->frag_len); 981 queue_head_inc(txq); 982 } 983 984 txq->head = head; 985 } 986 987 /* Enqueue the given packet for transmit. This routine allocates WRBs for the 988 * packet, dma maps the packet buffers and sets up the WRBs. Returns the number 989 * of WRBs used up by the packet. 990 */ 991 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo, 992 struct sk_buff *skb, 993 struct be_wrb_params *wrb_params) 994 { 995 u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb); 996 struct device *dev = &adapter->pdev->dev; 997 bool map_single = false; 998 u32 head; 999 dma_addr_t busaddr; 1000 int len; 1001 1002 head = be_tx_get_wrb_hdr(txo); 1003 1004 if (skb->len > skb->data_len) { 1005 len = skb_headlen(skb); 1006 1007 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE); 1008 if (dma_mapping_error(dev, busaddr)) 1009 goto dma_err; 1010 map_single = true; 1011 be_tx_setup_wrb_frag(txo, busaddr, len); 1012 copied += len; 1013 } 1014 1015 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 1016 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 1017 len = skb_frag_size(frag); 1018 1019 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE); 1020 if (dma_mapping_error(dev, busaddr)) 1021 goto dma_err; 1022 be_tx_setup_wrb_frag(txo, busaddr, len); 1023 copied += len; 1024 } 1025 1026 be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head); 1027 1028 be_tx_stats_update(txo, skb); 1029 return wrb_cnt; 1030 1031 dma_err: 1032 adapter->drv_stats.dma_map_errors++; 1033 be_xmit_restore(adapter, txo, head, map_single, copied); 1034 return 0; 1035 } 1036 1037 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter) 1038 { 1039 return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD; 1040 } 1041 1042 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter, 1043 struct sk_buff *skb, 1044 struct be_wrb_params 1045 *wrb_params) 1046 { 1047 bool insert_vlan = false; 1048 u16 vlan_tag = 0; 1049 1050 skb = skb_share_check(skb, GFP_ATOMIC); 1051 if (unlikely(!skb)) 1052 return skb; 1053 1054 if (skb_vlan_tag_present(skb)) { 1055 vlan_tag = be_get_tx_vlan_tag(adapter, skb); 1056 insert_vlan = true; 1057 } 1058 1059 if (qnq_async_evt_rcvd(adapter) && adapter->pvid) { 1060 if (!insert_vlan) { 1061 vlan_tag = adapter->pvid; 1062 insert_vlan = true; 1063 } 1064 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to 1065 * skip VLAN insertion 1066 */ 1067 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1068 } 1069 1070 if (insert_vlan) { 1071 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 1072 vlan_tag); 1073 if (unlikely(!skb)) 1074 return skb; 1075 __vlan_hwaccel_clear_tag(skb); 1076 } 1077 1078 /* Insert the outer VLAN, if any */ 1079 if (adapter->qnq_vid) { 1080 vlan_tag = adapter->qnq_vid; 1081 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 1082 vlan_tag); 1083 if (unlikely(!skb)) 1084 return skb; 1085 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1086 } 1087 1088 return skb; 1089 } 1090 1091 static bool be_ipv6_exthdr_check(struct sk_buff *skb) 1092 { 1093 struct ethhdr *eh = (struct ethhdr *)skb->data; 1094 u16 offset = ETH_HLEN; 1095 1096 if (eh->h_proto == htons(ETH_P_IPV6)) { 1097 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset); 1098 1099 offset += sizeof(struct ipv6hdr); 1100 if (ip6h->nexthdr != NEXTHDR_TCP && 1101 ip6h->nexthdr != NEXTHDR_UDP) { 1102 struct ipv6_opt_hdr *ehdr = 1103 (struct ipv6_opt_hdr *)(skb->data + offset); 1104 1105 /* offending pkt: 2nd byte following IPv6 hdr is 0xff */ 1106 if (ehdr->hdrlen == 0xff) 1107 return true; 1108 } 1109 } 1110 return false; 1111 } 1112 1113 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb) 1114 { 1115 return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid; 1116 } 1117 1118 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb) 1119 { 1120 return BE3_chip(adapter) && be_ipv6_exthdr_check(skb); 1121 } 1122 1123 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter, 1124 struct sk_buff *skb, 1125 struct be_wrb_params 1126 *wrb_params) 1127 { 1128 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 1129 unsigned int eth_hdr_len; 1130 struct iphdr *ip; 1131 1132 /* For padded packets, BE HW modifies tot_len field in IP header 1133 * incorrecly when VLAN tag is inserted by HW. 1134 * For padded packets, Lancer computes incorrect checksum. 1135 */ 1136 eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ? 1137 VLAN_ETH_HLEN : ETH_HLEN; 1138 if (skb->len <= 60 && 1139 (lancer_chip(adapter) || skb_vlan_tag_present(skb)) && 1140 is_ipv4_pkt(skb)) { 1141 ip = (struct iphdr *)ip_hdr(skb); 1142 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len)); 1143 } 1144 1145 /* If vlan tag is already inlined in the packet, skip HW VLAN 1146 * tagging in pvid-tagging mode 1147 */ 1148 if (be_pvid_tagging_enabled(adapter) && 1149 veh->h_vlan_proto == htons(ETH_P_8021Q)) 1150 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1151 1152 /* HW has a bug wherein it will calculate CSUM for VLAN 1153 * pkts even though it is disabled. 1154 * Manually insert VLAN in pkt. 1155 */ 1156 if (skb->ip_summed != CHECKSUM_PARTIAL && 1157 skb_vlan_tag_present(skb)) { 1158 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1159 if (unlikely(!skb)) 1160 goto err; 1161 } 1162 1163 /* HW may lockup when VLAN HW tagging is requested on 1164 * certain ipv6 packets. Drop such pkts if the HW workaround to 1165 * skip HW tagging is not enabled by FW. 1166 */ 1167 if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) && 1168 (adapter->pvid || adapter->qnq_vid) && 1169 !qnq_async_evt_rcvd(adapter))) 1170 goto tx_drop; 1171 1172 /* Manual VLAN tag insertion to prevent: 1173 * ASIC lockup when the ASIC inserts VLAN tag into 1174 * certain ipv6 packets. Insert VLAN tags in driver, 1175 * and set event, completion, vlan bits accordingly 1176 * in the Tx WRB. 1177 */ 1178 if (be_ipv6_tx_stall_chk(adapter, skb) && 1179 be_vlan_tag_tx_chk(adapter, skb)) { 1180 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1181 if (unlikely(!skb)) 1182 goto err; 1183 } 1184 1185 return skb; 1186 tx_drop: 1187 dev_kfree_skb_any(skb); 1188 err: 1189 return NULL; 1190 } 1191 1192 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter, 1193 struct sk_buff *skb, 1194 struct be_wrb_params *wrb_params) 1195 { 1196 int err; 1197 1198 /* Lancer, SH and BE3 in SRIOV mode have a bug wherein 1199 * packets that are 32b or less may cause a transmit stall 1200 * on that port. The workaround is to pad such packets 1201 * (len <= 32 bytes) to a minimum length of 36b. 1202 */ 1203 if (skb->len <= 32) { 1204 if (skb_put_padto(skb, 36)) 1205 return NULL; 1206 } 1207 1208 if (BEx_chip(adapter) || lancer_chip(adapter)) { 1209 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params); 1210 if (!skb) 1211 return NULL; 1212 } 1213 1214 /* The stack can send us skbs with length greater than 1215 * what the HW can handle. Trim the extra bytes. 1216 */ 1217 WARN_ON_ONCE(skb->len > BE_MAX_GSO_SIZE); 1218 err = pskb_trim(skb, BE_MAX_GSO_SIZE); 1219 WARN_ON(err); 1220 1221 return skb; 1222 } 1223 1224 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo) 1225 { 1226 struct be_queue_info *txq = &txo->q; 1227 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr); 1228 1229 /* Mark the last request eventable if it hasn't been marked already */ 1230 if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT))) 1231 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL); 1232 1233 /* compose a dummy wrb if there are odd set of wrbs to notify */ 1234 if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) { 1235 wrb_fill_dummy(queue_head_node(txq)); 1236 queue_head_inc(txq); 1237 atomic_inc(&txq->used); 1238 txo->pend_wrb_cnt++; 1239 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK << 1240 TX_HDR_WRB_NUM_SHIFT); 1241 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) << 1242 TX_HDR_WRB_NUM_SHIFT); 1243 } 1244 be_txq_notify(adapter, txo, txo->pend_wrb_cnt); 1245 txo->pend_wrb_cnt = 0; 1246 } 1247 1248 /* OS2BMC related */ 1249 1250 #define DHCP_CLIENT_PORT 68 1251 #define DHCP_SERVER_PORT 67 1252 #define NET_BIOS_PORT1 137 1253 #define NET_BIOS_PORT2 138 1254 #define DHCPV6_RAS_PORT 547 1255 1256 #define is_mc_allowed_on_bmc(adapter, eh) \ 1257 (!is_multicast_filt_enabled(adapter) && \ 1258 is_multicast_ether_addr(eh->h_dest) && \ 1259 !is_broadcast_ether_addr(eh->h_dest)) 1260 1261 #define is_bc_allowed_on_bmc(adapter, eh) \ 1262 (!is_broadcast_filt_enabled(adapter) && \ 1263 is_broadcast_ether_addr(eh->h_dest)) 1264 1265 #define is_arp_allowed_on_bmc(adapter, skb) \ 1266 (is_arp(skb) && is_arp_filt_enabled(adapter)) 1267 1268 #define is_arp(skb) (skb->protocol == htons(ETH_P_ARP)) 1269 1270 #define is_arp_filt_enabled(adapter) \ 1271 (adapter->bmc_filt_mask & (BMC_FILT_BROADCAST_ARP)) 1272 1273 #define is_dhcp_client_filt_enabled(adapter) \ 1274 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_CLIENT) 1275 1276 #define is_dhcp_srvr_filt_enabled(adapter) \ 1277 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_DHCP_SERVER) 1278 1279 #define is_nbios_filt_enabled(adapter) \ 1280 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST_NET_BIOS) 1281 1282 #define is_ipv6_na_filt_enabled(adapter) \ 1283 (adapter->bmc_filt_mask & \ 1284 BMC_FILT_MULTICAST_IPV6_NEIGH_ADVER) 1285 1286 #define is_ipv6_ra_filt_enabled(adapter) \ 1287 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RA) 1288 1289 #define is_ipv6_ras_filt_enabled(adapter) \ 1290 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST_IPV6_RAS) 1291 1292 #define is_broadcast_filt_enabled(adapter) \ 1293 (adapter->bmc_filt_mask & BMC_FILT_BROADCAST) 1294 1295 #define is_multicast_filt_enabled(adapter) \ 1296 (adapter->bmc_filt_mask & BMC_FILT_MULTICAST) 1297 1298 static bool be_send_pkt_to_bmc(struct be_adapter *adapter, 1299 struct sk_buff **skb) 1300 { 1301 struct ethhdr *eh = (struct ethhdr *)(*skb)->data; 1302 bool os2bmc = false; 1303 1304 if (!be_is_os2bmc_enabled(adapter)) 1305 goto done; 1306 1307 if (!is_multicast_ether_addr(eh->h_dest)) 1308 goto done; 1309 1310 if (is_mc_allowed_on_bmc(adapter, eh) || 1311 is_bc_allowed_on_bmc(adapter, eh) || 1312 is_arp_allowed_on_bmc(adapter, (*skb))) { 1313 os2bmc = true; 1314 goto done; 1315 } 1316 1317 if ((*skb)->protocol == htons(ETH_P_IPV6)) { 1318 struct ipv6hdr *hdr = ipv6_hdr((*skb)); 1319 u8 nexthdr = hdr->nexthdr; 1320 1321 if (nexthdr == IPPROTO_ICMPV6) { 1322 struct icmp6hdr *icmp6 = icmp6_hdr((*skb)); 1323 1324 switch (icmp6->icmp6_type) { 1325 case NDISC_ROUTER_ADVERTISEMENT: 1326 os2bmc = is_ipv6_ra_filt_enabled(adapter); 1327 goto done; 1328 case NDISC_NEIGHBOUR_ADVERTISEMENT: 1329 os2bmc = is_ipv6_na_filt_enabled(adapter); 1330 goto done; 1331 default: 1332 break; 1333 } 1334 } 1335 } 1336 1337 if (is_udp_pkt((*skb))) { 1338 struct udphdr *udp = udp_hdr((*skb)); 1339 1340 switch (ntohs(udp->dest)) { 1341 case DHCP_CLIENT_PORT: 1342 os2bmc = is_dhcp_client_filt_enabled(adapter); 1343 goto done; 1344 case DHCP_SERVER_PORT: 1345 os2bmc = is_dhcp_srvr_filt_enabled(adapter); 1346 goto done; 1347 case NET_BIOS_PORT1: 1348 case NET_BIOS_PORT2: 1349 os2bmc = is_nbios_filt_enabled(adapter); 1350 goto done; 1351 case DHCPV6_RAS_PORT: 1352 os2bmc = is_ipv6_ras_filt_enabled(adapter); 1353 goto done; 1354 default: 1355 break; 1356 } 1357 } 1358 done: 1359 /* For packets over a vlan, which are destined 1360 * to BMC, asic expects the vlan to be inline in the packet. 1361 */ 1362 if (os2bmc) 1363 *skb = be_insert_vlan_in_pkt(adapter, *skb, NULL); 1364 1365 return os2bmc; 1366 } 1367 1368 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev) 1369 { 1370 struct be_adapter *adapter = netdev_priv(netdev); 1371 u16 q_idx = skb_get_queue_mapping(skb); 1372 struct be_tx_obj *txo = &adapter->tx_obj[q_idx]; 1373 struct be_wrb_params wrb_params = { 0 }; 1374 bool flush = !netdev_xmit_more(); 1375 u16 wrb_cnt; 1376 1377 skb = be_xmit_workarounds(adapter, skb, &wrb_params); 1378 if (unlikely(!skb)) 1379 goto drop; 1380 1381 be_get_wrb_params_from_skb(adapter, skb, &wrb_params); 1382 1383 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1384 if (unlikely(!wrb_cnt)) { 1385 dev_kfree_skb_any(skb); 1386 goto drop; 1387 } 1388 1389 /* if os2bmc is enabled and if the pkt is destined to bmc, 1390 * enqueue the pkt a 2nd time with mgmt bit set. 1391 */ 1392 if (be_send_pkt_to_bmc(adapter, &skb)) { 1393 BE_WRB_F_SET(wrb_params.features, OS2BMC, 1); 1394 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1395 if (unlikely(!wrb_cnt)) 1396 goto drop; 1397 else 1398 skb_get(skb); 1399 } 1400 1401 if (be_is_txq_full(txo)) { 1402 netif_stop_subqueue(netdev, q_idx); 1403 tx_stats(txo)->tx_stops++; 1404 } 1405 1406 if (flush || __netif_subqueue_stopped(netdev, q_idx)) 1407 be_xmit_flush(adapter, txo); 1408 1409 return NETDEV_TX_OK; 1410 drop: 1411 tx_stats(txo)->tx_drv_drops++; 1412 /* Flush the already enqueued tx requests */ 1413 if (flush && txo->pend_wrb_cnt) 1414 be_xmit_flush(adapter, txo); 1415 1416 return NETDEV_TX_OK; 1417 } 1418 1419 static void be_tx_timeout(struct net_device *netdev, unsigned int txqueue) 1420 { 1421 struct be_adapter *adapter = netdev_priv(netdev); 1422 struct device *dev = &adapter->pdev->dev; 1423 struct be_tx_obj *txo; 1424 struct sk_buff *skb; 1425 struct tcphdr *tcphdr; 1426 struct udphdr *udphdr; 1427 u32 *entry; 1428 int status; 1429 int i, j; 1430 1431 for_all_tx_queues(adapter, txo, i) { 1432 dev_info(dev, "TXQ Dump: %d H: %d T: %d used: %d, qid: 0x%x\n", 1433 i, txo->q.head, txo->q.tail, 1434 atomic_read(&txo->q.used), txo->q.id); 1435 1436 entry = txo->q.dma_mem.va; 1437 for (j = 0; j < TX_Q_LEN * 4; j += 4) { 1438 if (entry[j] != 0 || entry[j + 1] != 0 || 1439 entry[j + 2] != 0 || entry[j + 3] != 0) { 1440 dev_info(dev, "Entry %d 0x%x 0x%x 0x%x 0x%x\n", 1441 j, entry[j], entry[j + 1], 1442 entry[j + 2], entry[j + 3]); 1443 } 1444 } 1445 1446 entry = txo->cq.dma_mem.va; 1447 dev_info(dev, "TXCQ Dump: %d H: %d T: %d used: %d\n", 1448 i, txo->cq.head, txo->cq.tail, 1449 atomic_read(&txo->cq.used)); 1450 for (j = 0; j < TX_CQ_LEN * 4; j += 4) { 1451 if (entry[j] != 0 || entry[j + 1] != 0 || 1452 entry[j + 2] != 0 || entry[j + 3] != 0) { 1453 dev_info(dev, "Entry %d 0x%x 0x%x 0x%x 0x%x\n", 1454 j, entry[j], entry[j + 1], 1455 entry[j + 2], entry[j + 3]); 1456 } 1457 } 1458 1459 for (j = 0; j < TX_Q_LEN; j++) { 1460 if (txo->sent_skb_list[j]) { 1461 skb = txo->sent_skb_list[j]; 1462 if (ip_hdr(skb)->protocol == IPPROTO_TCP) { 1463 tcphdr = tcp_hdr(skb); 1464 dev_info(dev, "TCP source port %d\n", 1465 ntohs(tcphdr->source)); 1466 dev_info(dev, "TCP dest port %d\n", 1467 ntohs(tcphdr->dest)); 1468 dev_info(dev, "TCP sequence num %d\n", 1469 ntohs(tcphdr->seq)); 1470 dev_info(dev, "TCP ack_seq %d\n", 1471 ntohs(tcphdr->ack_seq)); 1472 } else if (ip_hdr(skb)->protocol == 1473 IPPROTO_UDP) { 1474 udphdr = udp_hdr(skb); 1475 dev_info(dev, "UDP source port %d\n", 1476 ntohs(udphdr->source)); 1477 dev_info(dev, "UDP dest port %d\n", 1478 ntohs(udphdr->dest)); 1479 } 1480 dev_info(dev, "skb[%d] %p len %d proto 0x%x\n", 1481 j, skb, skb->len, skb->protocol); 1482 } 1483 } 1484 } 1485 1486 if (lancer_chip(adapter)) { 1487 dev_info(dev, "Initiating reset due to tx timeout\n"); 1488 dev_info(dev, "Resetting adapter\n"); 1489 status = lancer_physdev_ctrl(adapter, 1490 PHYSDEV_CONTROL_FW_RESET_MASK); 1491 if (status) 1492 dev_err(dev, "Reset failed .. Reboot server\n"); 1493 } 1494 } 1495 1496 static inline bool be_in_all_promisc(struct be_adapter *adapter) 1497 { 1498 return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) == 1499 BE_IF_FLAGS_ALL_PROMISCUOUS; 1500 } 1501 1502 static int be_set_vlan_promisc(struct be_adapter *adapter) 1503 { 1504 struct device *dev = &adapter->pdev->dev; 1505 int status; 1506 1507 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) 1508 return 0; 1509 1510 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON); 1511 if (!status) { 1512 dev_info(dev, "Enabled VLAN promiscuous mode\n"); 1513 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS; 1514 } else { 1515 dev_err(dev, "Failed to enable VLAN promiscuous mode\n"); 1516 } 1517 return status; 1518 } 1519 1520 static int be_clear_vlan_promisc(struct be_adapter *adapter) 1521 { 1522 struct device *dev = &adapter->pdev->dev; 1523 int status; 1524 1525 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF); 1526 if (!status) { 1527 dev_info(dev, "Disabling VLAN promiscuous mode\n"); 1528 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 1529 } 1530 return status; 1531 } 1532 1533 /* 1534 * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE. 1535 * If the user configures more, place BE in vlan promiscuous mode. 1536 */ 1537 static int be_vid_config(struct be_adapter *adapter) 1538 { 1539 struct device *dev = &adapter->pdev->dev; 1540 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1541 u16 num = 0, i = 0; 1542 int status = 0; 1543 1544 /* No need to change the VLAN state if the I/F is in promiscuous */ 1545 if (adapter->netdev->flags & IFF_PROMISC) 1546 return 0; 1547 1548 if (adapter->vlans_added > be_max_vlans(adapter)) 1549 return be_set_vlan_promisc(adapter); 1550 1551 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 1552 status = be_clear_vlan_promisc(adapter); 1553 if (status) 1554 return status; 1555 } 1556 /* Construct VLAN Table to give to HW */ 1557 for_each_set_bit(i, adapter->vids, VLAN_N_VID) 1558 vids[num++] = cpu_to_le16(i); 1559 1560 status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0); 1561 if (status) { 1562 dev_err(dev, "Setting HW VLAN filtering failed\n"); 1563 /* Set to VLAN promisc mode as setting VLAN filter failed */ 1564 if (addl_status(status) == MCC_ADDL_STATUS_INSUFFICIENT_VLANS || 1565 addl_status(status) == 1566 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES) 1567 return be_set_vlan_promisc(adapter); 1568 } 1569 return status; 1570 } 1571 1572 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 1573 { 1574 struct be_adapter *adapter = netdev_priv(netdev); 1575 int status = 0; 1576 1577 mutex_lock(&adapter->rx_filter_lock); 1578 1579 /* Packets with VID 0 are always received by Lancer by default */ 1580 if (lancer_chip(adapter) && vid == 0) 1581 goto done; 1582 1583 if (test_bit(vid, adapter->vids)) 1584 goto done; 1585 1586 set_bit(vid, adapter->vids); 1587 adapter->vlans_added++; 1588 1589 status = be_vid_config(adapter); 1590 done: 1591 mutex_unlock(&adapter->rx_filter_lock); 1592 return status; 1593 } 1594 1595 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid) 1596 { 1597 struct be_adapter *adapter = netdev_priv(netdev); 1598 int status = 0; 1599 1600 mutex_lock(&adapter->rx_filter_lock); 1601 1602 /* Packets with VID 0 are always received by Lancer by default */ 1603 if (lancer_chip(adapter) && vid == 0) 1604 goto done; 1605 1606 if (!test_bit(vid, adapter->vids)) 1607 goto done; 1608 1609 clear_bit(vid, adapter->vids); 1610 adapter->vlans_added--; 1611 1612 status = be_vid_config(adapter); 1613 done: 1614 mutex_unlock(&adapter->rx_filter_lock); 1615 return status; 1616 } 1617 1618 static void be_set_all_promisc(struct be_adapter *adapter) 1619 { 1620 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON); 1621 adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS; 1622 } 1623 1624 static void be_set_mc_promisc(struct be_adapter *adapter) 1625 { 1626 int status; 1627 1628 if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) 1629 return; 1630 1631 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON); 1632 if (!status) 1633 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS; 1634 } 1635 1636 static void be_set_uc_promisc(struct be_adapter *adapter) 1637 { 1638 int status; 1639 1640 if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) 1641 return; 1642 1643 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, ON); 1644 if (!status) 1645 adapter->if_flags |= BE_IF_FLAGS_PROMISCUOUS; 1646 } 1647 1648 static void be_clear_uc_promisc(struct be_adapter *adapter) 1649 { 1650 int status; 1651 1652 if (!(adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS)) 1653 return; 1654 1655 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_PROMISCUOUS, OFF); 1656 if (!status) 1657 adapter->if_flags &= ~BE_IF_FLAGS_PROMISCUOUS; 1658 } 1659 1660 /* The below 2 functions are the callback args for __dev_mc_sync/dev_uc_sync(). 1661 * We use a single callback function for both sync and unsync. We really don't 1662 * add/remove addresses through this callback. But, we use it to detect changes 1663 * to the uc/mc lists. The entire uc/mc list is programmed in be_set_rx_mode(). 1664 */ 1665 static int be_uc_list_update(struct net_device *netdev, 1666 const unsigned char *addr) 1667 { 1668 struct be_adapter *adapter = netdev_priv(netdev); 1669 1670 adapter->update_uc_list = true; 1671 return 0; 1672 } 1673 1674 static int be_mc_list_update(struct net_device *netdev, 1675 const unsigned char *addr) 1676 { 1677 struct be_adapter *adapter = netdev_priv(netdev); 1678 1679 adapter->update_mc_list = true; 1680 return 0; 1681 } 1682 1683 static void be_set_mc_list(struct be_adapter *adapter) 1684 { 1685 struct net_device *netdev = adapter->netdev; 1686 struct netdev_hw_addr *ha; 1687 bool mc_promisc = false; 1688 int status; 1689 1690 netif_addr_lock_bh(netdev); 1691 __dev_mc_sync(netdev, be_mc_list_update, be_mc_list_update); 1692 1693 if (netdev->flags & IFF_PROMISC) { 1694 adapter->update_mc_list = false; 1695 } else if (netdev->flags & IFF_ALLMULTI || 1696 netdev_mc_count(netdev) > be_max_mc(adapter)) { 1697 /* Enable multicast promisc if num configured exceeds 1698 * what we support 1699 */ 1700 mc_promisc = true; 1701 adapter->update_mc_list = false; 1702 } else if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) { 1703 /* Update mc-list unconditionally if the iface was previously 1704 * in mc-promisc mode and now is out of that mode. 1705 */ 1706 adapter->update_mc_list = true; 1707 } 1708 1709 if (adapter->update_mc_list) { 1710 int i = 0; 1711 1712 /* cache the mc-list in adapter */ 1713 netdev_for_each_mc_addr(ha, netdev) { 1714 ether_addr_copy(adapter->mc_list[i].mac, ha->addr); 1715 i++; 1716 } 1717 adapter->mc_count = netdev_mc_count(netdev); 1718 } 1719 netif_addr_unlock_bh(netdev); 1720 1721 if (mc_promisc) { 1722 be_set_mc_promisc(adapter); 1723 } else if (adapter->update_mc_list) { 1724 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON); 1725 if (!status) 1726 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS; 1727 else 1728 be_set_mc_promisc(adapter); 1729 1730 adapter->update_mc_list = false; 1731 } 1732 } 1733 1734 static void be_clear_mc_list(struct be_adapter *adapter) 1735 { 1736 struct net_device *netdev = adapter->netdev; 1737 1738 __dev_mc_unsync(netdev, NULL); 1739 be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, OFF); 1740 adapter->mc_count = 0; 1741 } 1742 1743 static int be_uc_mac_add(struct be_adapter *adapter, int uc_idx) 1744 { 1745 if (ether_addr_equal(adapter->uc_list[uc_idx].mac, adapter->dev_mac)) { 1746 adapter->pmac_id[uc_idx + 1] = adapter->pmac_id[0]; 1747 return 0; 1748 } 1749 1750 return be_cmd_pmac_add(adapter, adapter->uc_list[uc_idx].mac, 1751 adapter->if_handle, 1752 &adapter->pmac_id[uc_idx + 1], 0); 1753 } 1754 1755 static void be_uc_mac_del(struct be_adapter *adapter, int pmac_id) 1756 { 1757 if (pmac_id == adapter->pmac_id[0]) 1758 return; 1759 1760 be_cmd_pmac_del(adapter, adapter->if_handle, pmac_id, 0); 1761 } 1762 1763 static void be_set_uc_list(struct be_adapter *adapter) 1764 { 1765 struct net_device *netdev = adapter->netdev; 1766 struct netdev_hw_addr *ha; 1767 bool uc_promisc = false; 1768 int curr_uc_macs = 0, i; 1769 1770 netif_addr_lock_bh(netdev); 1771 __dev_uc_sync(netdev, be_uc_list_update, be_uc_list_update); 1772 1773 if (netdev->flags & IFF_PROMISC) { 1774 adapter->update_uc_list = false; 1775 } else if (netdev_uc_count(netdev) > (be_max_uc(adapter) - 1)) { 1776 uc_promisc = true; 1777 adapter->update_uc_list = false; 1778 } else if (adapter->if_flags & BE_IF_FLAGS_PROMISCUOUS) { 1779 /* Update uc-list unconditionally if the iface was previously 1780 * in uc-promisc mode and now is out of that mode. 1781 */ 1782 adapter->update_uc_list = true; 1783 } 1784 1785 if (adapter->update_uc_list) { 1786 /* cache the uc-list in adapter array */ 1787 i = 0; 1788 netdev_for_each_uc_addr(ha, netdev) { 1789 ether_addr_copy(adapter->uc_list[i].mac, ha->addr); 1790 i++; 1791 } 1792 curr_uc_macs = netdev_uc_count(netdev); 1793 } 1794 netif_addr_unlock_bh(netdev); 1795 1796 if (uc_promisc) { 1797 be_set_uc_promisc(adapter); 1798 } else if (adapter->update_uc_list) { 1799 be_clear_uc_promisc(adapter); 1800 1801 for (i = 0; i < adapter->uc_macs; i++) 1802 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]); 1803 1804 for (i = 0; i < curr_uc_macs; i++) 1805 be_uc_mac_add(adapter, i); 1806 adapter->uc_macs = curr_uc_macs; 1807 adapter->update_uc_list = false; 1808 } 1809 } 1810 1811 static void be_clear_uc_list(struct be_adapter *adapter) 1812 { 1813 struct net_device *netdev = adapter->netdev; 1814 int i; 1815 1816 __dev_uc_unsync(netdev, NULL); 1817 for (i = 0; i < adapter->uc_macs; i++) 1818 be_uc_mac_del(adapter, adapter->pmac_id[i + 1]); 1819 1820 adapter->uc_macs = 0; 1821 } 1822 1823 static void __be_set_rx_mode(struct be_adapter *adapter) 1824 { 1825 struct net_device *netdev = adapter->netdev; 1826 1827 mutex_lock(&adapter->rx_filter_lock); 1828 1829 if (netdev->flags & IFF_PROMISC) { 1830 if (!be_in_all_promisc(adapter)) 1831 be_set_all_promisc(adapter); 1832 } else if (be_in_all_promisc(adapter)) { 1833 /* We need to re-program the vlan-list or clear 1834 * vlan-promisc mode (if needed) when the interface 1835 * comes out of promisc mode. 1836 */ 1837 be_vid_config(adapter); 1838 } 1839 1840 be_set_uc_list(adapter); 1841 be_set_mc_list(adapter); 1842 1843 mutex_unlock(&adapter->rx_filter_lock); 1844 } 1845 1846 static void be_work_set_rx_mode(struct work_struct *work) 1847 { 1848 struct be_cmd_work *cmd_work = 1849 container_of(work, struct be_cmd_work, work); 1850 1851 __be_set_rx_mode(cmd_work->adapter); 1852 kfree(cmd_work); 1853 } 1854 1855 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 1856 { 1857 struct be_adapter *adapter = netdev_priv(netdev); 1858 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1859 int status; 1860 1861 if (!sriov_enabled(adapter)) 1862 return -EPERM; 1863 1864 if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs) 1865 return -EINVAL; 1866 1867 /* Proceed further only if user provided MAC is different 1868 * from active MAC 1869 */ 1870 if (ether_addr_equal(mac, vf_cfg->mac_addr)) 1871 return 0; 1872 1873 if (BEx_chip(adapter)) { 1874 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id, 1875 vf + 1); 1876 1877 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle, 1878 &vf_cfg->pmac_id, vf + 1); 1879 } else { 1880 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 1881 vf + 1); 1882 } 1883 1884 if (status) { 1885 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x", 1886 mac, vf, status); 1887 return be_cmd_status(status); 1888 } 1889 1890 ether_addr_copy(vf_cfg->mac_addr, mac); 1891 1892 return 0; 1893 } 1894 1895 static int be_get_vf_config(struct net_device *netdev, int vf, 1896 struct ifla_vf_info *vi) 1897 { 1898 struct be_adapter *adapter = netdev_priv(netdev); 1899 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1900 1901 if (!sriov_enabled(adapter)) 1902 return -EPERM; 1903 1904 if (vf >= adapter->num_vfs) 1905 return -EINVAL; 1906 1907 vi->vf = vf; 1908 vi->max_tx_rate = vf_cfg->tx_rate; 1909 vi->min_tx_rate = 0; 1910 vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK; 1911 vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT; 1912 memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN); 1913 vi->linkstate = adapter->vf_cfg[vf].plink_tracking; 1914 vi->spoofchk = adapter->vf_cfg[vf].spoofchk; 1915 1916 return 0; 1917 } 1918 1919 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan) 1920 { 1921 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1922 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1923 int vf_if_id = vf_cfg->if_handle; 1924 int status; 1925 1926 /* Enable Transparent VLAN Tagging */ 1927 status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0, 0); 1928 if (status) 1929 return status; 1930 1931 /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */ 1932 vids[0] = 0; 1933 status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1); 1934 if (!status) 1935 dev_info(&adapter->pdev->dev, 1936 "Cleared guest VLANs on VF%d", vf); 1937 1938 /* After TVT is enabled, disallow VFs to program VLAN filters */ 1939 if (vf_cfg->privileges & BE_PRIV_FILTMGMT) { 1940 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges & 1941 ~BE_PRIV_FILTMGMT, vf + 1); 1942 if (!status) 1943 vf_cfg->privileges &= ~BE_PRIV_FILTMGMT; 1944 } 1945 return 0; 1946 } 1947 1948 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf) 1949 { 1950 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1951 struct device *dev = &adapter->pdev->dev; 1952 int status; 1953 1954 /* Reset Transparent VLAN Tagging. */ 1955 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1, 1956 vf_cfg->if_handle, 0, 0); 1957 if (status) 1958 return status; 1959 1960 /* Allow VFs to program VLAN filtering */ 1961 if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 1962 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges | 1963 BE_PRIV_FILTMGMT, vf + 1); 1964 if (!status) { 1965 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 1966 dev_info(dev, "VF%d: FILTMGMT priv enabled", vf); 1967 } 1968 } 1969 1970 dev_info(dev, 1971 "Disable/re-enable i/f in VM to clear Transparent VLAN tag"); 1972 return 0; 1973 } 1974 1975 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos, 1976 __be16 vlan_proto) 1977 { 1978 struct be_adapter *adapter = netdev_priv(netdev); 1979 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1980 int status; 1981 1982 if (!sriov_enabled(adapter)) 1983 return -EPERM; 1984 1985 if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7) 1986 return -EINVAL; 1987 1988 if (vlan_proto != htons(ETH_P_8021Q)) 1989 return -EPROTONOSUPPORT; 1990 1991 if (vlan || qos) { 1992 vlan |= qos << VLAN_PRIO_SHIFT; 1993 status = be_set_vf_tvt(adapter, vf, vlan); 1994 } else { 1995 status = be_clear_vf_tvt(adapter, vf); 1996 } 1997 1998 if (status) { 1999 dev_err(&adapter->pdev->dev, 2000 "VLAN %d config on VF %d failed : %#x\n", vlan, vf, 2001 status); 2002 return be_cmd_status(status); 2003 } 2004 2005 vf_cfg->vlan_tag = vlan; 2006 return 0; 2007 } 2008 2009 static int be_set_vf_tx_rate(struct net_device *netdev, int vf, 2010 int min_tx_rate, int max_tx_rate) 2011 { 2012 struct be_adapter *adapter = netdev_priv(netdev); 2013 struct device *dev = &adapter->pdev->dev; 2014 int percent_rate, status = 0; 2015 u16 link_speed = 0; 2016 u8 link_status; 2017 2018 if (!sriov_enabled(adapter)) 2019 return -EPERM; 2020 2021 if (vf >= adapter->num_vfs) 2022 return -EINVAL; 2023 2024 if (min_tx_rate) 2025 return -EINVAL; 2026 2027 if (!max_tx_rate) 2028 goto config_qos; 2029 2030 status = be_cmd_link_status_query(adapter, &link_speed, 2031 &link_status, 0); 2032 if (status) 2033 goto err; 2034 2035 if (!link_status) { 2036 dev_err(dev, "TX-rate setting not allowed when link is down\n"); 2037 status = -ENETDOWN; 2038 goto err; 2039 } 2040 2041 if (max_tx_rate < 100 || max_tx_rate > link_speed) { 2042 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n", 2043 link_speed); 2044 status = -EINVAL; 2045 goto err; 2046 } 2047 2048 /* On Skyhawk the QOS setting must be done only as a % value */ 2049 percent_rate = link_speed / 100; 2050 if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) { 2051 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n", 2052 percent_rate); 2053 status = -EINVAL; 2054 goto err; 2055 } 2056 2057 config_qos: 2058 status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1); 2059 if (status) 2060 goto err; 2061 2062 adapter->vf_cfg[vf].tx_rate = max_tx_rate; 2063 return 0; 2064 2065 err: 2066 dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n", 2067 max_tx_rate, vf); 2068 return be_cmd_status(status); 2069 } 2070 2071 static int be_set_vf_link_state(struct net_device *netdev, int vf, 2072 int link_state) 2073 { 2074 struct be_adapter *adapter = netdev_priv(netdev); 2075 int status; 2076 2077 if (!sriov_enabled(adapter)) 2078 return -EPERM; 2079 2080 if (vf >= adapter->num_vfs) 2081 return -EINVAL; 2082 2083 status = be_cmd_set_logical_link_config(adapter, link_state, vf+1); 2084 if (status) { 2085 dev_err(&adapter->pdev->dev, 2086 "Link state change on VF %d failed: %#x\n", vf, status); 2087 return be_cmd_status(status); 2088 } 2089 2090 adapter->vf_cfg[vf].plink_tracking = link_state; 2091 2092 return 0; 2093 } 2094 2095 static int be_set_vf_spoofchk(struct net_device *netdev, int vf, bool enable) 2096 { 2097 struct be_adapter *adapter = netdev_priv(netdev); 2098 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 2099 u8 spoofchk; 2100 int status; 2101 2102 if (!sriov_enabled(adapter)) 2103 return -EPERM; 2104 2105 if (vf >= adapter->num_vfs) 2106 return -EINVAL; 2107 2108 if (BEx_chip(adapter)) 2109 return -EOPNOTSUPP; 2110 2111 if (enable == vf_cfg->spoofchk) 2112 return 0; 2113 2114 spoofchk = enable ? ENABLE_MAC_SPOOFCHK : DISABLE_MAC_SPOOFCHK; 2115 2116 status = be_cmd_set_hsw_config(adapter, 0, vf + 1, vf_cfg->if_handle, 2117 0, spoofchk); 2118 if (status) { 2119 dev_err(&adapter->pdev->dev, 2120 "Spoofchk change on VF %d failed: %#x\n", vf, status); 2121 return be_cmd_status(status); 2122 } 2123 2124 vf_cfg->spoofchk = enable; 2125 return 0; 2126 } 2127 2128 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts, 2129 ulong now) 2130 { 2131 aic->rx_pkts_prev = rx_pkts; 2132 aic->tx_reqs_prev = tx_pkts; 2133 aic->jiffies = now; 2134 } 2135 2136 static int be_get_new_eqd(struct be_eq_obj *eqo) 2137 { 2138 struct be_adapter *adapter = eqo->adapter; 2139 int eqd, start; 2140 struct be_aic_obj *aic; 2141 struct be_rx_obj *rxo; 2142 struct be_tx_obj *txo; 2143 u64 rx_pkts = 0, tx_pkts = 0; 2144 ulong now; 2145 u32 pps, delta; 2146 int i; 2147 2148 aic = &adapter->aic_obj[eqo->idx]; 2149 if (!adapter->aic_enabled) { 2150 if (aic->jiffies) 2151 aic->jiffies = 0; 2152 eqd = aic->et_eqd; 2153 return eqd; 2154 } 2155 2156 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 2157 do { 2158 start = u64_stats_fetch_begin_irq(&rxo->stats.sync); 2159 rx_pkts += rxo->stats.rx_pkts; 2160 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start)); 2161 } 2162 2163 for_all_tx_queues_on_eq(adapter, eqo, txo, i) { 2164 do { 2165 start = u64_stats_fetch_begin_irq(&txo->stats.sync); 2166 tx_pkts += txo->stats.tx_reqs; 2167 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start)); 2168 } 2169 2170 /* Skip, if wrapped around or first calculation */ 2171 now = jiffies; 2172 if (!aic->jiffies || time_before(now, aic->jiffies) || 2173 rx_pkts < aic->rx_pkts_prev || 2174 tx_pkts < aic->tx_reqs_prev) { 2175 be_aic_update(aic, rx_pkts, tx_pkts, now); 2176 return aic->prev_eqd; 2177 } 2178 2179 delta = jiffies_to_msecs(now - aic->jiffies); 2180 if (delta == 0) 2181 return aic->prev_eqd; 2182 2183 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) + 2184 (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta); 2185 eqd = (pps / 15000) << 2; 2186 2187 if (eqd < 8) 2188 eqd = 0; 2189 eqd = min_t(u32, eqd, aic->max_eqd); 2190 eqd = max_t(u32, eqd, aic->min_eqd); 2191 2192 be_aic_update(aic, rx_pkts, tx_pkts, now); 2193 2194 return eqd; 2195 } 2196 2197 /* For Skyhawk-R only */ 2198 static u32 be_get_eq_delay_mult_enc(struct be_eq_obj *eqo) 2199 { 2200 struct be_adapter *adapter = eqo->adapter; 2201 struct be_aic_obj *aic = &adapter->aic_obj[eqo->idx]; 2202 ulong now = jiffies; 2203 int eqd; 2204 u32 mult_enc; 2205 2206 if (!adapter->aic_enabled) 2207 return 0; 2208 2209 if (jiffies_to_msecs(now - aic->jiffies) < 1) 2210 eqd = aic->prev_eqd; 2211 else 2212 eqd = be_get_new_eqd(eqo); 2213 2214 if (eqd > 100) 2215 mult_enc = R2I_DLY_ENC_1; 2216 else if (eqd > 60) 2217 mult_enc = R2I_DLY_ENC_2; 2218 else if (eqd > 20) 2219 mult_enc = R2I_DLY_ENC_3; 2220 else 2221 mult_enc = R2I_DLY_ENC_0; 2222 2223 aic->prev_eqd = eqd; 2224 2225 return mult_enc; 2226 } 2227 2228 void be_eqd_update(struct be_adapter *adapter, bool force_update) 2229 { 2230 struct be_set_eqd set_eqd[MAX_EVT_QS]; 2231 struct be_aic_obj *aic; 2232 struct be_eq_obj *eqo; 2233 int i, num = 0, eqd; 2234 2235 for_all_evt_queues(adapter, eqo, i) { 2236 aic = &adapter->aic_obj[eqo->idx]; 2237 eqd = be_get_new_eqd(eqo); 2238 if (force_update || eqd != aic->prev_eqd) { 2239 set_eqd[num].delay_multiplier = (eqd * 65)/100; 2240 set_eqd[num].eq_id = eqo->q.id; 2241 aic->prev_eqd = eqd; 2242 num++; 2243 } 2244 } 2245 2246 if (num) 2247 be_cmd_modify_eqd(adapter, set_eqd, num); 2248 } 2249 2250 static void be_rx_stats_update(struct be_rx_obj *rxo, 2251 struct be_rx_compl_info *rxcp) 2252 { 2253 struct be_rx_stats *stats = rx_stats(rxo); 2254 2255 u64_stats_update_begin(&stats->sync); 2256 stats->rx_compl++; 2257 stats->rx_bytes += rxcp->pkt_size; 2258 stats->rx_pkts++; 2259 if (rxcp->tunneled) 2260 stats->rx_vxlan_offload_pkts++; 2261 if (rxcp->pkt_type == BE_MULTICAST_PACKET) 2262 stats->rx_mcast_pkts++; 2263 if (rxcp->err) 2264 stats->rx_compl_err++; 2265 u64_stats_update_end(&stats->sync); 2266 } 2267 2268 static inline bool csum_passed(struct be_rx_compl_info *rxcp) 2269 { 2270 /* L4 checksum is not reliable for non TCP/UDP packets. 2271 * Also ignore ipcksm for ipv6 pkts 2272 */ 2273 return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum && 2274 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err; 2275 } 2276 2277 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo) 2278 { 2279 struct be_adapter *adapter = rxo->adapter; 2280 struct be_rx_page_info *rx_page_info; 2281 struct be_queue_info *rxq = &rxo->q; 2282 u32 frag_idx = rxq->tail; 2283 2284 rx_page_info = &rxo->page_info_tbl[frag_idx]; 2285 BUG_ON(!rx_page_info->page); 2286 2287 if (rx_page_info->last_frag) { 2288 dma_unmap_page(&adapter->pdev->dev, 2289 dma_unmap_addr(rx_page_info, bus), 2290 adapter->big_page_size, DMA_FROM_DEVICE); 2291 rx_page_info->last_frag = false; 2292 } else { 2293 dma_sync_single_for_cpu(&adapter->pdev->dev, 2294 dma_unmap_addr(rx_page_info, bus), 2295 rx_frag_size, DMA_FROM_DEVICE); 2296 } 2297 2298 queue_tail_inc(rxq); 2299 atomic_dec(&rxq->used); 2300 return rx_page_info; 2301 } 2302 2303 /* Throwaway the data in the Rx completion */ 2304 static void be_rx_compl_discard(struct be_rx_obj *rxo, 2305 struct be_rx_compl_info *rxcp) 2306 { 2307 struct be_rx_page_info *page_info; 2308 u16 i, num_rcvd = rxcp->num_rcvd; 2309 2310 for (i = 0; i < num_rcvd; i++) { 2311 page_info = get_rx_page_info(rxo); 2312 put_page(page_info->page); 2313 memset(page_info, 0, sizeof(*page_info)); 2314 } 2315 } 2316 2317 /* 2318 * skb_fill_rx_data forms a complete skb for an ether frame 2319 * indicated by rxcp. 2320 */ 2321 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb, 2322 struct be_rx_compl_info *rxcp) 2323 { 2324 struct be_rx_page_info *page_info; 2325 u16 i, j; 2326 u16 hdr_len, curr_frag_len, remaining; 2327 u8 *start; 2328 2329 page_info = get_rx_page_info(rxo); 2330 start = page_address(page_info->page) + page_info->page_offset; 2331 prefetch(start); 2332 2333 /* Copy data in the first descriptor of this completion */ 2334 curr_frag_len = min(rxcp->pkt_size, rx_frag_size); 2335 2336 skb->len = curr_frag_len; 2337 if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */ 2338 memcpy(skb->data, start, curr_frag_len); 2339 /* Complete packet has now been moved to data */ 2340 put_page(page_info->page); 2341 skb->data_len = 0; 2342 skb->tail += curr_frag_len; 2343 } else { 2344 hdr_len = ETH_HLEN; 2345 memcpy(skb->data, start, hdr_len); 2346 skb_shinfo(skb)->nr_frags = 1; 2347 skb_frag_set_page(skb, 0, page_info->page); 2348 skb_frag_off_set(&skb_shinfo(skb)->frags[0], 2349 page_info->page_offset + hdr_len); 2350 skb_frag_size_set(&skb_shinfo(skb)->frags[0], 2351 curr_frag_len - hdr_len); 2352 skb->data_len = curr_frag_len - hdr_len; 2353 skb->truesize += rx_frag_size; 2354 skb->tail += hdr_len; 2355 } 2356 page_info->page = NULL; 2357 2358 if (rxcp->pkt_size <= rx_frag_size) { 2359 BUG_ON(rxcp->num_rcvd != 1); 2360 return; 2361 } 2362 2363 /* More frags present for this completion */ 2364 remaining = rxcp->pkt_size - curr_frag_len; 2365 for (i = 1, j = 0; i < rxcp->num_rcvd; i++) { 2366 page_info = get_rx_page_info(rxo); 2367 curr_frag_len = min(remaining, rx_frag_size); 2368 2369 /* Coalesce all frags from the same physical page in one slot */ 2370 if (page_info->page_offset == 0) { 2371 /* Fresh page */ 2372 j++; 2373 skb_frag_set_page(skb, j, page_info->page); 2374 skb_frag_off_set(&skb_shinfo(skb)->frags[j], 2375 page_info->page_offset); 2376 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 2377 skb_shinfo(skb)->nr_frags++; 2378 } else { 2379 put_page(page_info->page); 2380 } 2381 2382 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 2383 skb->len += curr_frag_len; 2384 skb->data_len += curr_frag_len; 2385 skb->truesize += rx_frag_size; 2386 remaining -= curr_frag_len; 2387 page_info->page = NULL; 2388 } 2389 BUG_ON(j > MAX_SKB_FRAGS); 2390 } 2391 2392 /* Process the RX completion indicated by rxcp when GRO is disabled */ 2393 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi, 2394 struct be_rx_compl_info *rxcp) 2395 { 2396 struct be_adapter *adapter = rxo->adapter; 2397 struct net_device *netdev = adapter->netdev; 2398 struct sk_buff *skb; 2399 2400 skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE); 2401 if (unlikely(!skb)) { 2402 rx_stats(rxo)->rx_drops_no_skbs++; 2403 be_rx_compl_discard(rxo, rxcp); 2404 return; 2405 } 2406 2407 skb_fill_rx_data(rxo, skb, rxcp); 2408 2409 if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp))) 2410 skb->ip_summed = CHECKSUM_UNNECESSARY; 2411 else 2412 skb_checksum_none_assert(skb); 2413 2414 skb->protocol = eth_type_trans(skb, netdev); 2415 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 2416 if (netdev->features & NETIF_F_RXHASH) 2417 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 2418 2419 skb->csum_level = rxcp->tunneled; 2420 skb_mark_napi_id(skb, napi); 2421 2422 if (rxcp->vlanf) 2423 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 2424 2425 netif_receive_skb(skb); 2426 } 2427 2428 /* Process the RX completion indicated by rxcp when GRO is enabled */ 2429 static void be_rx_compl_process_gro(struct be_rx_obj *rxo, 2430 struct napi_struct *napi, 2431 struct be_rx_compl_info *rxcp) 2432 { 2433 struct be_adapter *adapter = rxo->adapter; 2434 struct be_rx_page_info *page_info; 2435 struct sk_buff *skb = NULL; 2436 u16 remaining, curr_frag_len; 2437 u16 i, j; 2438 2439 skb = napi_get_frags(napi); 2440 if (!skb) { 2441 be_rx_compl_discard(rxo, rxcp); 2442 return; 2443 } 2444 2445 remaining = rxcp->pkt_size; 2446 for (i = 0, j = -1; i < rxcp->num_rcvd; i++) { 2447 page_info = get_rx_page_info(rxo); 2448 2449 curr_frag_len = min(remaining, rx_frag_size); 2450 2451 /* Coalesce all frags from the same physical page in one slot */ 2452 if (i == 0 || page_info->page_offset == 0) { 2453 /* First frag or Fresh page */ 2454 j++; 2455 skb_frag_set_page(skb, j, page_info->page); 2456 skb_frag_off_set(&skb_shinfo(skb)->frags[j], 2457 page_info->page_offset); 2458 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 2459 } else { 2460 put_page(page_info->page); 2461 } 2462 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 2463 skb->truesize += rx_frag_size; 2464 remaining -= curr_frag_len; 2465 memset(page_info, 0, sizeof(*page_info)); 2466 } 2467 BUG_ON(j > MAX_SKB_FRAGS); 2468 2469 skb_shinfo(skb)->nr_frags = j + 1; 2470 skb->len = rxcp->pkt_size; 2471 skb->data_len = rxcp->pkt_size; 2472 skb->ip_summed = CHECKSUM_UNNECESSARY; 2473 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 2474 if (adapter->netdev->features & NETIF_F_RXHASH) 2475 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 2476 2477 skb->csum_level = rxcp->tunneled; 2478 2479 if (rxcp->vlanf) 2480 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 2481 2482 napi_gro_frags(napi); 2483 } 2484 2485 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl, 2486 struct be_rx_compl_info *rxcp) 2487 { 2488 rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl); 2489 rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl); 2490 rxcp->err = GET_RX_COMPL_V1_BITS(err, compl); 2491 rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl); 2492 rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl); 2493 rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl); 2494 rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl); 2495 rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl); 2496 rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl); 2497 rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl); 2498 rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl); 2499 if (rxcp->vlanf) { 2500 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl); 2501 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl); 2502 } 2503 rxcp->port = GET_RX_COMPL_V1_BITS(port, compl); 2504 rxcp->tunneled = 2505 GET_RX_COMPL_V1_BITS(tunneled, compl); 2506 } 2507 2508 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl, 2509 struct be_rx_compl_info *rxcp) 2510 { 2511 rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl); 2512 rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl); 2513 rxcp->err = GET_RX_COMPL_V0_BITS(err, compl); 2514 rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl); 2515 rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl); 2516 rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl); 2517 rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl); 2518 rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl); 2519 rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl); 2520 rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl); 2521 rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl); 2522 if (rxcp->vlanf) { 2523 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl); 2524 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl); 2525 } 2526 rxcp->port = GET_RX_COMPL_V0_BITS(port, compl); 2527 rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl); 2528 } 2529 2530 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo) 2531 { 2532 struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq); 2533 struct be_rx_compl_info *rxcp = &rxo->rxcp; 2534 struct be_adapter *adapter = rxo->adapter; 2535 2536 /* For checking the valid bit it is Ok to use either definition as the 2537 * valid bit is at the same position in both v0 and v1 Rx compl */ 2538 if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0) 2539 return NULL; 2540 2541 rmb(); 2542 be_dws_le_to_cpu(compl, sizeof(*compl)); 2543 2544 if (adapter->be3_native) 2545 be_parse_rx_compl_v1(compl, rxcp); 2546 else 2547 be_parse_rx_compl_v0(compl, rxcp); 2548 2549 if (rxcp->ip_frag) 2550 rxcp->l4_csum = 0; 2551 2552 if (rxcp->vlanf) { 2553 /* In QNQ modes, if qnq bit is not set, then the packet was 2554 * tagged only with the transparent outer vlan-tag and must 2555 * not be treated as a vlan packet by host 2556 */ 2557 if (be_is_qnq_mode(adapter) && !rxcp->qnq) 2558 rxcp->vlanf = 0; 2559 2560 if (!lancer_chip(adapter)) 2561 rxcp->vlan_tag = swab16(rxcp->vlan_tag); 2562 2563 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) && 2564 !test_bit(rxcp->vlan_tag, adapter->vids)) 2565 rxcp->vlanf = 0; 2566 } 2567 2568 /* As the compl has been parsed, reset it; we wont touch it again */ 2569 compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0; 2570 2571 queue_tail_inc(&rxo->cq); 2572 return rxcp; 2573 } 2574 2575 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp) 2576 { 2577 u32 order = get_order(size); 2578 2579 if (order > 0) 2580 gfp |= __GFP_COMP; 2581 return alloc_pages(gfp, order); 2582 } 2583 2584 /* 2585 * Allocate a page, split it to fragments of size rx_frag_size and post as 2586 * receive buffers to BE 2587 */ 2588 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed) 2589 { 2590 struct be_adapter *adapter = rxo->adapter; 2591 struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL; 2592 struct be_queue_info *rxq = &rxo->q; 2593 struct page *pagep = NULL; 2594 struct device *dev = &adapter->pdev->dev; 2595 struct be_eth_rx_d *rxd; 2596 u64 page_dmaaddr = 0, frag_dmaaddr; 2597 u32 posted, page_offset = 0, notify = 0; 2598 2599 page_info = &rxo->page_info_tbl[rxq->head]; 2600 for (posted = 0; posted < frags_needed && !page_info->page; posted++) { 2601 if (!pagep) { 2602 pagep = be_alloc_pages(adapter->big_page_size, gfp); 2603 if (unlikely(!pagep)) { 2604 rx_stats(rxo)->rx_post_fail++; 2605 break; 2606 } 2607 page_dmaaddr = dma_map_page(dev, pagep, 0, 2608 adapter->big_page_size, 2609 DMA_FROM_DEVICE); 2610 if (dma_mapping_error(dev, page_dmaaddr)) { 2611 put_page(pagep); 2612 pagep = NULL; 2613 adapter->drv_stats.dma_map_errors++; 2614 break; 2615 } 2616 page_offset = 0; 2617 } else { 2618 get_page(pagep); 2619 page_offset += rx_frag_size; 2620 } 2621 page_info->page_offset = page_offset; 2622 page_info->page = pagep; 2623 2624 rxd = queue_head_node(rxq); 2625 frag_dmaaddr = page_dmaaddr + page_info->page_offset; 2626 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF); 2627 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr)); 2628 2629 /* Any space left in the current big page for another frag? */ 2630 if ((page_offset + rx_frag_size + rx_frag_size) > 2631 adapter->big_page_size) { 2632 pagep = NULL; 2633 page_info->last_frag = true; 2634 dma_unmap_addr_set(page_info, bus, page_dmaaddr); 2635 } else { 2636 dma_unmap_addr_set(page_info, bus, frag_dmaaddr); 2637 } 2638 2639 prev_page_info = page_info; 2640 queue_head_inc(rxq); 2641 page_info = &rxo->page_info_tbl[rxq->head]; 2642 } 2643 2644 /* Mark the last frag of a page when we break out of the above loop 2645 * with no more slots available in the RXQ 2646 */ 2647 if (pagep) { 2648 prev_page_info->last_frag = true; 2649 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr); 2650 } 2651 2652 if (posted) { 2653 atomic_add(posted, &rxq->used); 2654 if (rxo->rx_post_starved) 2655 rxo->rx_post_starved = false; 2656 do { 2657 notify = min(MAX_NUM_POST_ERX_DB, posted); 2658 be_rxq_notify(adapter, rxq->id, notify); 2659 posted -= notify; 2660 } while (posted); 2661 } else if (atomic_read(&rxq->used) == 0) { 2662 /* Let be_worker replenish when memory is available */ 2663 rxo->rx_post_starved = true; 2664 } 2665 } 2666 2667 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status) 2668 { 2669 switch (status) { 2670 case BE_TX_COMP_HDR_PARSE_ERR: 2671 tx_stats(txo)->tx_hdr_parse_err++; 2672 break; 2673 case BE_TX_COMP_NDMA_ERR: 2674 tx_stats(txo)->tx_dma_err++; 2675 break; 2676 case BE_TX_COMP_ACL_ERR: 2677 tx_stats(txo)->tx_spoof_check_err++; 2678 break; 2679 } 2680 } 2681 2682 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status) 2683 { 2684 switch (status) { 2685 case LANCER_TX_COMP_LSO_ERR: 2686 tx_stats(txo)->tx_tso_err++; 2687 break; 2688 case LANCER_TX_COMP_HSW_DROP_MAC_ERR: 2689 case LANCER_TX_COMP_HSW_DROP_VLAN_ERR: 2690 tx_stats(txo)->tx_spoof_check_err++; 2691 break; 2692 case LANCER_TX_COMP_QINQ_ERR: 2693 tx_stats(txo)->tx_qinq_err++; 2694 break; 2695 case LANCER_TX_COMP_PARITY_ERR: 2696 tx_stats(txo)->tx_internal_parity_err++; 2697 break; 2698 case LANCER_TX_COMP_DMA_ERR: 2699 tx_stats(txo)->tx_dma_err++; 2700 break; 2701 case LANCER_TX_COMP_SGE_ERR: 2702 tx_stats(txo)->tx_sge_err++; 2703 break; 2704 } 2705 } 2706 2707 static struct be_tx_compl_info *be_tx_compl_get(struct be_adapter *adapter, 2708 struct be_tx_obj *txo) 2709 { 2710 struct be_queue_info *tx_cq = &txo->cq; 2711 struct be_tx_compl_info *txcp = &txo->txcp; 2712 struct be_eth_tx_compl *compl = queue_tail_node(tx_cq); 2713 2714 if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0) 2715 return NULL; 2716 2717 /* Ensure load ordering of valid bit dword and other dwords below */ 2718 rmb(); 2719 be_dws_le_to_cpu(compl, sizeof(*compl)); 2720 2721 txcp->status = GET_TX_COMPL_BITS(status, compl); 2722 txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl); 2723 2724 if (txcp->status) { 2725 if (lancer_chip(adapter)) { 2726 lancer_update_tx_err(txo, txcp->status); 2727 /* Reset the adapter incase of TSO, 2728 * SGE or Parity error 2729 */ 2730 if (txcp->status == LANCER_TX_COMP_LSO_ERR || 2731 txcp->status == LANCER_TX_COMP_PARITY_ERR || 2732 txcp->status == LANCER_TX_COMP_SGE_ERR) 2733 be_set_error(adapter, BE_ERROR_TX); 2734 } else { 2735 be_update_tx_err(txo, txcp->status); 2736 } 2737 } 2738 2739 if (be_check_error(adapter, BE_ERROR_TX)) 2740 return NULL; 2741 2742 compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0; 2743 queue_tail_inc(tx_cq); 2744 return txcp; 2745 } 2746 2747 static u16 be_tx_compl_process(struct be_adapter *adapter, 2748 struct be_tx_obj *txo, u16 last_index) 2749 { 2750 struct sk_buff **sent_skbs = txo->sent_skb_list; 2751 struct be_queue_info *txq = &txo->q; 2752 struct sk_buff *skb = NULL; 2753 bool unmap_skb_hdr = false; 2754 struct be_eth_wrb *wrb; 2755 u16 num_wrbs = 0; 2756 u32 frag_index; 2757 2758 do { 2759 if (sent_skbs[txq->tail]) { 2760 /* Free skb from prev req */ 2761 if (skb) 2762 dev_consume_skb_any(skb); 2763 skb = sent_skbs[txq->tail]; 2764 sent_skbs[txq->tail] = NULL; 2765 queue_tail_inc(txq); /* skip hdr wrb */ 2766 num_wrbs++; 2767 unmap_skb_hdr = true; 2768 } 2769 wrb = queue_tail_node(txq); 2770 frag_index = txq->tail; 2771 unmap_tx_frag(&adapter->pdev->dev, wrb, 2772 (unmap_skb_hdr && skb_headlen(skb))); 2773 unmap_skb_hdr = false; 2774 queue_tail_inc(txq); 2775 num_wrbs++; 2776 } while (frag_index != last_index); 2777 dev_consume_skb_any(skb); 2778 2779 return num_wrbs; 2780 } 2781 2782 /* Return the number of events in the event queue */ 2783 static inline int events_get(struct be_eq_obj *eqo) 2784 { 2785 struct be_eq_entry *eqe; 2786 int num = 0; 2787 2788 do { 2789 eqe = queue_tail_node(&eqo->q); 2790 if (eqe->evt == 0) 2791 break; 2792 2793 rmb(); 2794 eqe->evt = 0; 2795 num++; 2796 queue_tail_inc(&eqo->q); 2797 } while (true); 2798 2799 return num; 2800 } 2801 2802 /* Leaves the EQ is disarmed state */ 2803 static void be_eq_clean(struct be_eq_obj *eqo) 2804 { 2805 int num = events_get(eqo); 2806 2807 be_eq_notify(eqo->adapter, eqo->q.id, false, true, num, 0); 2808 } 2809 2810 /* Free posted rx buffers that were not used */ 2811 static void be_rxq_clean(struct be_rx_obj *rxo) 2812 { 2813 struct be_queue_info *rxq = &rxo->q; 2814 struct be_rx_page_info *page_info; 2815 2816 while (atomic_read(&rxq->used) > 0) { 2817 page_info = get_rx_page_info(rxo); 2818 put_page(page_info->page); 2819 memset(page_info, 0, sizeof(*page_info)); 2820 } 2821 BUG_ON(atomic_read(&rxq->used)); 2822 rxq->tail = 0; 2823 rxq->head = 0; 2824 } 2825 2826 static void be_rx_cq_clean(struct be_rx_obj *rxo) 2827 { 2828 struct be_queue_info *rx_cq = &rxo->cq; 2829 struct be_rx_compl_info *rxcp; 2830 struct be_adapter *adapter = rxo->adapter; 2831 int flush_wait = 0; 2832 2833 /* Consume pending rx completions. 2834 * Wait for the flush completion (identified by zero num_rcvd) 2835 * to arrive. Notify CQ even when there are no more CQ entries 2836 * for HW to flush partially coalesced CQ entries. 2837 * In Lancer, there is no need to wait for flush compl. 2838 */ 2839 for (;;) { 2840 rxcp = be_rx_compl_get(rxo); 2841 if (!rxcp) { 2842 if (lancer_chip(adapter)) 2843 break; 2844 2845 if (flush_wait++ > 50 || 2846 be_check_error(adapter, 2847 BE_ERROR_HW)) { 2848 dev_warn(&adapter->pdev->dev, 2849 "did not receive flush compl\n"); 2850 break; 2851 } 2852 be_cq_notify(adapter, rx_cq->id, true, 0); 2853 mdelay(1); 2854 } else { 2855 be_rx_compl_discard(rxo, rxcp); 2856 be_cq_notify(adapter, rx_cq->id, false, 1); 2857 if (rxcp->num_rcvd == 0) 2858 break; 2859 } 2860 } 2861 2862 /* After cleanup, leave the CQ in unarmed state */ 2863 be_cq_notify(adapter, rx_cq->id, false, 0); 2864 } 2865 2866 static void be_tx_compl_clean(struct be_adapter *adapter) 2867 { 2868 struct device *dev = &adapter->pdev->dev; 2869 u16 cmpl = 0, timeo = 0, num_wrbs = 0; 2870 struct be_tx_compl_info *txcp; 2871 struct be_queue_info *txq; 2872 u32 end_idx, notified_idx; 2873 struct be_tx_obj *txo; 2874 int i, pending_txqs; 2875 2876 /* Stop polling for compls when HW has been silent for 10ms */ 2877 do { 2878 pending_txqs = adapter->num_tx_qs; 2879 2880 for_all_tx_queues(adapter, txo, i) { 2881 cmpl = 0; 2882 num_wrbs = 0; 2883 txq = &txo->q; 2884 while ((txcp = be_tx_compl_get(adapter, txo))) { 2885 num_wrbs += 2886 be_tx_compl_process(adapter, txo, 2887 txcp->end_index); 2888 cmpl++; 2889 } 2890 if (cmpl) { 2891 be_cq_notify(adapter, txo->cq.id, false, cmpl); 2892 atomic_sub(num_wrbs, &txq->used); 2893 timeo = 0; 2894 } 2895 if (!be_is_tx_compl_pending(txo)) 2896 pending_txqs--; 2897 } 2898 2899 if (pending_txqs == 0 || ++timeo > 10 || 2900 be_check_error(adapter, BE_ERROR_HW)) 2901 break; 2902 2903 mdelay(1); 2904 } while (true); 2905 2906 /* Free enqueued TX that was never notified to HW */ 2907 for_all_tx_queues(adapter, txo, i) { 2908 txq = &txo->q; 2909 2910 if (atomic_read(&txq->used)) { 2911 dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n", 2912 i, atomic_read(&txq->used)); 2913 notified_idx = txq->tail; 2914 end_idx = txq->tail; 2915 index_adv(&end_idx, atomic_read(&txq->used) - 1, 2916 txq->len); 2917 /* Use the tx-compl process logic to handle requests 2918 * that were not sent to the HW. 2919 */ 2920 num_wrbs = be_tx_compl_process(adapter, txo, end_idx); 2921 atomic_sub(num_wrbs, &txq->used); 2922 BUG_ON(atomic_read(&txq->used)); 2923 txo->pend_wrb_cnt = 0; 2924 /* Since hw was never notified of these requests, 2925 * reset TXQ indices 2926 */ 2927 txq->head = notified_idx; 2928 txq->tail = notified_idx; 2929 } 2930 } 2931 } 2932 2933 static void be_evt_queues_destroy(struct be_adapter *adapter) 2934 { 2935 struct be_eq_obj *eqo; 2936 int i; 2937 2938 for_all_evt_queues(adapter, eqo, i) { 2939 if (eqo->q.created) { 2940 be_eq_clean(eqo); 2941 be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ); 2942 netif_napi_del(&eqo->napi); 2943 free_cpumask_var(eqo->affinity_mask); 2944 } 2945 be_queue_free(adapter, &eqo->q); 2946 } 2947 } 2948 2949 static int be_evt_queues_create(struct be_adapter *adapter) 2950 { 2951 struct be_queue_info *eq; 2952 struct be_eq_obj *eqo; 2953 struct be_aic_obj *aic; 2954 int i, rc; 2955 2956 /* need enough EQs to service both RX and TX queues */ 2957 adapter->num_evt_qs = min_t(u16, num_irqs(adapter), 2958 max(adapter->cfg_num_rx_irqs, 2959 adapter->cfg_num_tx_irqs)); 2960 2961 adapter->aic_enabled = true; 2962 2963 for_all_evt_queues(adapter, eqo, i) { 2964 int numa_node = dev_to_node(&adapter->pdev->dev); 2965 2966 aic = &adapter->aic_obj[i]; 2967 eqo->adapter = adapter; 2968 eqo->idx = i; 2969 aic->max_eqd = BE_MAX_EQD; 2970 2971 eq = &eqo->q; 2972 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN, 2973 sizeof(struct be_eq_entry)); 2974 if (rc) 2975 return rc; 2976 2977 rc = be_cmd_eq_create(adapter, eqo); 2978 if (rc) 2979 return rc; 2980 2981 if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL)) 2982 return -ENOMEM; 2983 cpumask_set_cpu(cpumask_local_spread(i, numa_node), 2984 eqo->affinity_mask); 2985 netif_napi_add(adapter->netdev, &eqo->napi, be_poll, 2986 BE_NAPI_WEIGHT); 2987 } 2988 return 0; 2989 } 2990 2991 static void be_mcc_queues_destroy(struct be_adapter *adapter) 2992 { 2993 struct be_queue_info *q; 2994 2995 q = &adapter->mcc_obj.q; 2996 if (q->created) 2997 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ); 2998 be_queue_free(adapter, q); 2999 3000 q = &adapter->mcc_obj.cq; 3001 if (q->created) 3002 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 3003 be_queue_free(adapter, q); 3004 } 3005 3006 /* Must be called only after TX qs are created as MCC shares TX EQ */ 3007 static int be_mcc_queues_create(struct be_adapter *adapter) 3008 { 3009 struct be_queue_info *q, *cq; 3010 3011 cq = &adapter->mcc_obj.cq; 3012 if (be_queue_alloc(adapter, cq, MCC_CQ_LEN, 3013 sizeof(struct be_mcc_compl))) 3014 goto err; 3015 3016 /* Use the default EQ for MCC completions */ 3017 if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0)) 3018 goto mcc_cq_free; 3019 3020 q = &adapter->mcc_obj.q; 3021 if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 3022 goto mcc_cq_destroy; 3023 3024 if (be_cmd_mccq_create(adapter, q, cq)) 3025 goto mcc_q_free; 3026 3027 return 0; 3028 3029 mcc_q_free: 3030 be_queue_free(adapter, q); 3031 mcc_cq_destroy: 3032 be_cmd_q_destroy(adapter, cq, QTYPE_CQ); 3033 mcc_cq_free: 3034 be_queue_free(adapter, cq); 3035 err: 3036 return -1; 3037 } 3038 3039 static void be_tx_queues_destroy(struct be_adapter *adapter) 3040 { 3041 struct be_queue_info *q; 3042 struct be_tx_obj *txo; 3043 u8 i; 3044 3045 for_all_tx_queues(adapter, txo, i) { 3046 q = &txo->q; 3047 if (q->created) 3048 be_cmd_q_destroy(adapter, q, QTYPE_TXQ); 3049 be_queue_free(adapter, q); 3050 3051 q = &txo->cq; 3052 if (q->created) 3053 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 3054 be_queue_free(adapter, q); 3055 } 3056 } 3057 3058 static int be_tx_qs_create(struct be_adapter *adapter) 3059 { 3060 struct be_queue_info *cq; 3061 struct be_tx_obj *txo; 3062 struct be_eq_obj *eqo; 3063 int status, i; 3064 3065 adapter->num_tx_qs = min(adapter->num_evt_qs, adapter->cfg_num_tx_irqs); 3066 3067 for_all_tx_queues(adapter, txo, i) { 3068 cq = &txo->cq; 3069 status = be_queue_alloc(adapter, cq, TX_CQ_LEN, 3070 sizeof(struct be_eth_tx_compl)); 3071 if (status) 3072 return status; 3073 3074 u64_stats_init(&txo->stats.sync); 3075 u64_stats_init(&txo->stats.sync_compl); 3076 3077 /* If num_evt_qs is less than num_tx_qs, then more than 3078 * one txq share an eq 3079 */ 3080 eqo = &adapter->eq_obj[i % adapter->num_evt_qs]; 3081 status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3); 3082 if (status) 3083 return status; 3084 3085 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN, 3086 sizeof(struct be_eth_wrb)); 3087 if (status) 3088 return status; 3089 3090 status = be_cmd_txq_create(adapter, txo); 3091 if (status) 3092 return status; 3093 3094 netif_set_xps_queue(adapter->netdev, eqo->affinity_mask, 3095 eqo->idx); 3096 } 3097 3098 dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n", 3099 adapter->num_tx_qs); 3100 return 0; 3101 } 3102 3103 static void be_rx_cqs_destroy(struct be_adapter *adapter) 3104 { 3105 struct be_queue_info *q; 3106 struct be_rx_obj *rxo; 3107 int i; 3108 3109 for_all_rx_queues(adapter, rxo, i) { 3110 q = &rxo->cq; 3111 if (q->created) 3112 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 3113 be_queue_free(adapter, q); 3114 } 3115 } 3116 3117 static int be_rx_cqs_create(struct be_adapter *adapter) 3118 { 3119 struct be_queue_info *eq, *cq; 3120 struct be_rx_obj *rxo; 3121 int rc, i; 3122 3123 adapter->num_rss_qs = 3124 min(adapter->num_evt_qs, adapter->cfg_num_rx_irqs); 3125 3126 /* We'll use RSS only if atleast 2 RSS rings are supported. */ 3127 if (adapter->num_rss_qs < 2) 3128 adapter->num_rss_qs = 0; 3129 3130 adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq; 3131 3132 /* When the interface is not capable of RSS rings (and there is no 3133 * need to create a default RXQ) we'll still need one RXQ 3134 */ 3135 if (adapter->num_rx_qs == 0) 3136 adapter->num_rx_qs = 1; 3137 3138 adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE; 3139 for_all_rx_queues(adapter, rxo, i) { 3140 rxo->adapter = adapter; 3141 cq = &rxo->cq; 3142 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN, 3143 sizeof(struct be_eth_rx_compl)); 3144 if (rc) 3145 return rc; 3146 3147 u64_stats_init(&rxo->stats.sync); 3148 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; 3149 rc = be_cmd_cq_create(adapter, cq, eq, false, 3); 3150 if (rc) 3151 return rc; 3152 } 3153 3154 dev_info(&adapter->pdev->dev, 3155 "created %d RX queue(s)\n", adapter->num_rx_qs); 3156 return 0; 3157 } 3158 3159 static irqreturn_t be_intx(int irq, void *dev) 3160 { 3161 struct be_eq_obj *eqo = dev; 3162 struct be_adapter *adapter = eqo->adapter; 3163 int num_evts = 0; 3164 3165 /* IRQ is not expected when NAPI is scheduled as the EQ 3166 * will not be armed. 3167 * But, this can happen on Lancer INTx where it takes 3168 * a while to de-assert INTx or in BE2 where occasionaly 3169 * an interrupt may be raised even when EQ is unarmed. 3170 * If NAPI is already scheduled, then counting & notifying 3171 * events will orphan them. 3172 */ 3173 if (napi_schedule_prep(&eqo->napi)) { 3174 num_evts = events_get(eqo); 3175 __napi_schedule(&eqo->napi); 3176 if (num_evts) 3177 eqo->spurious_intr = 0; 3178 } 3179 be_eq_notify(adapter, eqo->q.id, false, true, num_evts, 0); 3180 3181 /* Return IRQ_HANDLED only for the the first spurious intr 3182 * after a valid intr to stop the kernel from branding 3183 * this irq as a bad one! 3184 */ 3185 if (num_evts || eqo->spurious_intr++ == 0) 3186 return IRQ_HANDLED; 3187 else 3188 return IRQ_NONE; 3189 } 3190 3191 static irqreturn_t be_msix(int irq, void *dev) 3192 { 3193 struct be_eq_obj *eqo = dev; 3194 3195 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0); 3196 napi_schedule(&eqo->napi); 3197 return IRQ_HANDLED; 3198 } 3199 3200 static inline bool do_gro(struct be_rx_compl_info *rxcp) 3201 { 3202 return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false; 3203 } 3204 3205 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi, 3206 int budget) 3207 { 3208 struct be_adapter *adapter = rxo->adapter; 3209 struct be_queue_info *rx_cq = &rxo->cq; 3210 struct be_rx_compl_info *rxcp; 3211 u32 work_done; 3212 u32 frags_consumed = 0; 3213 3214 for (work_done = 0; work_done < budget; work_done++) { 3215 rxcp = be_rx_compl_get(rxo); 3216 if (!rxcp) 3217 break; 3218 3219 /* Is it a flush compl that has no data */ 3220 if (unlikely(rxcp->num_rcvd == 0)) 3221 goto loop_continue; 3222 3223 /* Discard compl with partial DMA Lancer B0 */ 3224 if (unlikely(!rxcp->pkt_size)) { 3225 be_rx_compl_discard(rxo, rxcp); 3226 goto loop_continue; 3227 } 3228 3229 /* On BE drop pkts that arrive due to imperfect filtering in 3230 * promiscuous mode on some skews 3231 */ 3232 if (unlikely(rxcp->port != adapter->port_num && 3233 !lancer_chip(adapter))) { 3234 be_rx_compl_discard(rxo, rxcp); 3235 goto loop_continue; 3236 } 3237 3238 if (do_gro(rxcp)) 3239 be_rx_compl_process_gro(rxo, napi, rxcp); 3240 else 3241 be_rx_compl_process(rxo, napi, rxcp); 3242 3243 loop_continue: 3244 frags_consumed += rxcp->num_rcvd; 3245 be_rx_stats_update(rxo, rxcp); 3246 } 3247 3248 if (work_done) { 3249 be_cq_notify(adapter, rx_cq->id, true, work_done); 3250 3251 /* When an rx-obj gets into post_starved state, just 3252 * let be_worker do the posting. 3253 */ 3254 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM && 3255 !rxo->rx_post_starved) 3256 be_post_rx_frags(rxo, GFP_ATOMIC, 3257 max_t(u32, MAX_RX_POST, 3258 frags_consumed)); 3259 } 3260 3261 return work_done; 3262 } 3263 3264 3265 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo, 3266 int idx) 3267 { 3268 int num_wrbs = 0, work_done = 0; 3269 struct be_tx_compl_info *txcp; 3270 3271 while ((txcp = be_tx_compl_get(adapter, txo))) { 3272 num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index); 3273 work_done++; 3274 } 3275 3276 if (work_done) { 3277 be_cq_notify(adapter, txo->cq.id, true, work_done); 3278 atomic_sub(num_wrbs, &txo->q.used); 3279 3280 /* As Tx wrbs have been freed up, wake up netdev queue 3281 * if it was stopped due to lack of tx wrbs. */ 3282 if (__netif_subqueue_stopped(adapter->netdev, idx) && 3283 be_can_txq_wake(txo)) { 3284 netif_wake_subqueue(adapter->netdev, idx); 3285 } 3286 3287 u64_stats_update_begin(&tx_stats(txo)->sync_compl); 3288 tx_stats(txo)->tx_compl += work_done; 3289 u64_stats_update_end(&tx_stats(txo)->sync_compl); 3290 } 3291 } 3292 3293 int be_poll(struct napi_struct *napi, int budget) 3294 { 3295 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 3296 struct be_adapter *adapter = eqo->adapter; 3297 int max_work = 0, work, i, num_evts; 3298 struct be_rx_obj *rxo; 3299 struct be_tx_obj *txo; 3300 u32 mult_enc = 0; 3301 3302 num_evts = events_get(eqo); 3303 3304 for_all_tx_queues_on_eq(adapter, eqo, txo, i) 3305 be_process_tx(adapter, txo, i); 3306 3307 /* This loop will iterate twice for EQ0 in which 3308 * completions of the last RXQ (default one) are also processed 3309 * For other EQs the loop iterates only once 3310 */ 3311 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 3312 work = be_process_rx(rxo, napi, budget); 3313 max_work = max(work, max_work); 3314 } 3315 3316 if (is_mcc_eqo(eqo)) 3317 be_process_mcc(adapter); 3318 3319 if (max_work < budget) { 3320 napi_complete_done(napi, max_work); 3321 3322 /* Skyhawk EQ_DB has a provision to set the rearm to interrupt 3323 * delay via a delay multiplier encoding value 3324 */ 3325 if (skyhawk_chip(adapter)) 3326 mult_enc = be_get_eq_delay_mult_enc(eqo); 3327 3328 be_eq_notify(adapter, eqo->q.id, true, false, num_evts, 3329 mult_enc); 3330 } else { 3331 /* As we'll continue in polling mode, count and clear events */ 3332 be_eq_notify(adapter, eqo->q.id, false, false, num_evts, 0); 3333 } 3334 return max_work; 3335 } 3336 3337 void be_detect_error(struct be_adapter *adapter) 3338 { 3339 u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0; 3340 u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 3341 struct device *dev = &adapter->pdev->dev; 3342 u16 val; 3343 u32 i; 3344 3345 if (be_check_error(adapter, BE_ERROR_HW)) 3346 return; 3347 3348 if (lancer_chip(adapter)) { 3349 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 3350 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 3351 be_set_error(adapter, BE_ERROR_UE); 3352 sliport_err1 = ioread32(adapter->db + 3353 SLIPORT_ERROR1_OFFSET); 3354 sliport_err2 = ioread32(adapter->db + 3355 SLIPORT_ERROR2_OFFSET); 3356 /* Do not log error messages if its a FW reset */ 3357 if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 && 3358 sliport_err2 == SLIPORT_ERROR_FW_RESET2) { 3359 dev_info(dev, "Reset is in progress\n"); 3360 } else { 3361 dev_err(dev, "Error detected in the card\n"); 3362 dev_err(dev, "ERR: sliport status 0x%x\n", 3363 sliport_status); 3364 dev_err(dev, "ERR: sliport error1 0x%x\n", 3365 sliport_err1); 3366 dev_err(dev, "ERR: sliport error2 0x%x\n", 3367 sliport_err2); 3368 } 3369 } 3370 } else { 3371 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW); 3372 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH); 3373 ue_lo_mask = ioread32(adapter->pcicfg + 3374 PCICFG_UE_STATUS_LOW_MASK); 3375 ue_hi_mask = ioread32(adapter->pcicfg + 3376 PCICFG_UE_STATUS_HI_MASK); 3377 3378 ue_lo = (ue_lo & ~ue_lo_mask); 3379 ue_hi = (ue_hi & ~ue_hi_mask); 3380 3381 if (ue_lo || ue_hi) { 3382 /* On certain platforms BE3 hardware can indicate 3383 * spurious UEs. In case of a UE in the chip, 3384 * the POST register correctly reports either a 3385 * FAT_LOG_START state (FW is currently dumping 3386 * FAT log data) or a ARMFW_UE state. Check for the 3387 * above states to ascertain if the UE is valid or not. 3388 */ 3389 if (BE3_chip(adapter)) { 3390 val = be_POST_stage_get(adapter); 3391 if ((val & POST_STAGE_FAT_LOG_START) 3392 != POST_STAGE_FAT_LOG_START && 3393 (val & POST_STAGE_ARMFW_UE) 3394 != POST_STAGE_ARMFW_UE && 3395 (val & POST_STAGE_RECOVERABLE_ERR) 3396 != POST_STAGE_RECOVERABLE_ERR) 3397 return; 3398 } 3399 3400 dev_err(dev, "Error detected in the adapter"); 3401 be_set_error(adapter, BE_ERROR_UE); 3402 3403 for (i = 0; ue_lo; ue_lo >>= 1, i++) { 3404 if (ue_lo & 1) 3405 dev_err(dev, "UE: %s bit set\n", 3406 ue_status_low_desc[i]); 3407 } 3408 for (i = 0; ue_hi; ue_hi >>= 1, i++) { 3409 if (ue_hi & 1) 3410 dev_err(dev, "UE: %s bit set\n", 3411 ue_status_hi_desc[i]); 3412 } 3413 } 3414 } 3415 } 3416 3417 static void be_msix_disable(struct be_adapter *adapter) 3418 { 3419 if (msix_enabled(adapter)) { 3420 pci_disable_msix(adapter->pdev); 3421 adapter->num_msix_vec = 0; 3422 adapter->num_msix_roce_vec = 0; 3423 } 3424 } 3425 3426 static int be_msix_enable(struct be_adapter *adapter) 3427 { 3428 unsigned int i, max_roce_eqs; 3429 struct device *dev = &adapter->pdev->dev; 3430 int num_vec; 3431 3432 /* If RoCE is supported, program the max number of vectors that 3433 * could be used for NIC and RoCE, else, just program the number 3434 * we'll use initially. 3435 */ 3436 if (be_roce_supported(adapter)) { 3437 max_roce_eqs = 3438 be_max_func_eqs(adapter) - be_max_nic_eqs(adapter); 3439 max_roce_eqs = min(max_roce_eqs, num_online_cpus()); 3440 num_vec = be_max_any_irqs(adapter) + max_roce_eqs; 3441 } else { 3442 num_vec = max(adapter->cfg_num_rx_irqs, 3443 adapter->cfg_num_tx_irqs); 3444 } 3445 3446 for (i = 0; i < num_vec; i++) 3447 adapter->msix_entries[i].entry = i; 3448 3449 num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 3450 MIN_MSIX_VECTORS, num_vec); 3451 if (num_vec < 0) 3452 goto fail; 3453 3454 if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) { 3455 adapter->num_msix_roce_vec = num_vec / 2; 3456 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n", 3457 adapter->num_msix_roce_vec); 3458 } 3459 3460 adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec; 3461 3462 dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n", 3463 adapter->num_msix_vec); 3464 return 0; 3465 3466 fail: 3467 dev_warn(dev, "MSIx enable failed\n"); 3468 3469 /* INTx is not supported in VFs, so fail probe if enable_msix fails */ 3470 if (be_virtfn(adapter)) 3471 return num_vec; 3472 return 0; 3473 } 3474 3475 static inline int be_msix_vec_get(struct be_adapter *adapter, 3476 struct be_eq_obj *eqo) 3477 { 3478 return adapter->msix_entries[eqo->msix_idx].vector; 3479 } 3480 3481 static int be_msix_register(struct be_adapter *adapter) 3482 { 3483 struct net_device *netdev = adapter->netdev; 3484 struct be_eq_obj *eqo; 3485 int status, i, vec; 3486 3487 for_all_evt_queues(adapter, eqo, i) { 3488 sprintf(eqo->desc, "%s-q%d", netdev->name, i); 3489 vec = be_msix_vec_get(adapter, eqo); 3490 status = request_irq(vec, be_msix, 0, eqo->desc, eqo); 3491 if (status) 3492 goto err_msix; 3493 3494 irq_update_affinity_hint(vec, eqo->affinity_mask); 3495 } 3496 3497 return 0; 3498 err_msix: 3499 for (i--; i >= 0; i--) { 3500 eqo = &adapter->eq_obj[i]; 3501 free_irq(be_msix_vec_get(adapter, eqo), eqo); 3502 } 3503 dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n", 3504 status); 3505 be_msix_disable(adapter); 3506 return status; 3507 } 3508 3509 static int be_irq_register(struct be_adapter *adapter) 3510 { 3511 struct net_device *netdev = adapter->netdev; 3512 int status; 3513 3514 if (msix_enabled(adapter)) { 3515 status = be_msix_register(adapter); 3516 if (status == 0) 3517 goto done; 3518 /* INTx is not supported for VF */ 3519 if (be_virtfn(adapter)) 3520 return status; 3521 } 3522 3523 /* INTx: only the first EQ is used */ 3524 netdev->irq = adapter->pdev->irq; 3525 status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name, 3526 &adapter->eq_obj[0]); 3527 if (status) { 3528 dev_err(&adapter->pdev->dev, 3529 "INTx request IRQ failed - err %d\n", status); 3530 return status; 3531 } 3532 done: 3533 adapter->isr_registered = true; 3534 return 0; 3535 } 3536 3537 static void be_irq_unregister(struct be_adapter *adapter) 3538 { 3539 struct net_device *netdev = adapter->netdev; 3540 struct be_eq_obj *eqo; 3541 int i, vec; 3542 3543 if (!adapter->isr_registered) 3544 return; 3545 3546 /* INTx */ 3547 if (!msix_enabled(adapter)) { 3548 free_irq(netdev->irq, &adapter->eq_obj[0]); 3549 goto done; 3550 } 3551 3552 /* MSIx */ 3553 for_all_evt_queues(adapter, eqo, i) { 3554 vec = be_msix_vec_get(adapter, eqo); 3555 irq_update_affinity_hint(vec, NULL); 3556 free_irq(vec, eqo); 3557 } 3558 3559 done: 3560 adapter->isr_registered = false; 3561 } 3562 3563 static void be_rx_qs_destroy(struct be_adapter *adapter) 3564 { 3565 struct rss_info *rss = &adapter->rss_info; 3566 struct be_queue_info *q; 3567 struct be_rx_obj *rxo; 3568 int i; 3569 3570 for_all_rx_queues(adapter, rxo, i) { 3571 q = &rxo->q; 3572 if (q->created) { 3573 /* If RXQs are destroyed while in an "out of buffer" 3574 * state, there is a possibility of an HW stall on 3575 * Lancer. So, post 64 buffers to each queue to relieve 3576 * the "out of buffer" condition. 3577 * Make sure there's space in the RXQ before posting. 3578 */ 3579 if (lancer_chip(adapter)) { 3580 be_rx_cq_clean(rxo); 3581 if (atomic_read(&q->used) == 0) 3582 be_post_rx_frags(rxo, GFP_KERNEL, 3583 MAX_RX_POST); 3584 } 3585 3586 be_cmd_rxq_destroy(adapter, q); 3587 be_rx_cq_clean(rxo); 3588 be_rxq_clean(rxo); 3589 } 3590 be_queue_free(adapter, q); 3591 } 3592 3593 if (rss->rss_flags) { 3594 rss->rss_flags = RSS_ENABLE_NONE; 3595 be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3596 128, rss->rss_hkey); 3597 } 3598 } 3599 3600 static void be_disable_if_filters(struct be_adapter *adapter) 3601 { 3602 /* Don't delete MAC on BE3 VFs without FILTMGMT privilege */ 3603 if (!BEx_chip(adapter) || !be_virtfn(adapter) || 3604 check_privilege(adapter, BE_PRIV_FILTMGMT)) { 3605 be_dev_mac_del(adapter, adapter->pmac_id[0]); 3606 eth_zero_addr(adapter->dev_mac); 3607 } 3608 3609 be_clear_uc_list(adapter); 3610 be_clear_mc_list(adapter); 3611 3612 /* The IFACE flags are enabled in the open path and cleared 3613 * in the close path. When a VF gets detached from the host and 3614 * assigned to a VM the following happens: 3615 * - VF's IFACE flags get cleared in the detach path 3616 * - IFACE create is issued by the VF in the attach path 3617 * Due to a bug in the BE3/Skyhawk-R FW 3618 * (Lancer FW doesn't have the bug), the IFACE capability flags 3619 * specified along with the IFACE create cmd issued by a VF are not 3620 * honoured by FW. As a consequence, if a *new* driver 3621 * (that enables/disables IFACE flags in open/close) 3622 * is loaded in the host and an *old* driver is * used by a VM/VF, 3623 * the IFACE gets created *without* the needed flags. 3624 * To avoid this, disable RX-filter flags only for Lancer. 3625 */ 3626 if (lancer_chip(adapter)) { 3627 be_cmd_rx_filter(adapter, BE_IF_ALL_FILT_FLAGS, OFF); 3628 adapter->if_flags &= ~BE_IF_ALL_FILT_FLAGS; 3629 } 3630 } 3631 3632 static int be_close(struct net_device *netdev) 3633 { 3634 struct be_adapter *adapter = netdev_priv(netdev); 3635 struct be_eq_obj *eqo; 3636 int i; 3637 3638 /* This protection is needed as be_close() may be called even when the 3639 * adapter is in cleared state (after eeh perm failure) 3640 */ 3641 if (!(adapter->flags & BE_FLAGS_SETUP_DONE)) 3642 return 0; 3643 3644 /* Before attempting cleanup ensure all the pending cmds in the 3645 * config_wq have finished execution 3646 */ 3647 flush_workqueue(be_wq); 3648 3649 be_disable_if_filters(adapter); 3650 3651 if (adapter->flags & BE_FLAGS_NAPI_ENABLED) { 3652 for_all_evt_queues(adapter, eqo, i) { 3653 napi_disable(&eqo->napi); 3654 } 3655 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED; 3656 } 3657 3658 be_async_mcc_disable(adapter); 3659 3660 /* Wait for all pending tx completions to arrive so that 3661 * all tx skbs are freed. 3662 */ 3663 netif_tx_disable(netdev); 3664 be_tx_compl_clean(adapter); 3665 3666 be_rx_qs_destroy(adapter); 3667 3668 for_all_evt_queues(adapter, eqo, i) { 3669 if (msix_enabled(adapter)) 3670 synchronize_irq(be_msix_vec_get(adapter, eqo)); 3671 else 3672 synchronize_irq(netdev->irq); 3673 be_eq_clean(eqo); 3674 } 3675 3676 be_irq_unregister(adapter); 3677 3678 return 0; 3679 } 3680 3681 static int be_rx_qs_create(struct be_adapter *adapter) 3682 { 3683 struct rss_info *rss = &adapter->rss_info; 3684 u8 rss_key[RSS_HASH_KEY_LEN]; 3685 struct be_rx_obj *rxo; 3686 int rc, i, j; 3687 3688 for_all_rx_queues(adapter, rxo, i) { 3689 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN, 3690 sizeof(struct be_eth_rx_d)); 3691 if (rc) 3692 return rc; 3693 } 3694 3695 if (adapter->need_def_rxq || !adapter->num_rss_qs) { 3696 rxo = default_rxo(adapter); 3697 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3698 rx_frag_size, adapter->if_handle, 3699 false, &rxo->rss_id); 3700 if (rc) 3701 return rc; 3702 } 3703 3704 for_all_rss_queues(adapter, rxo, i) { 3705 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3706 rx_frag_size, adapter->if_handle, 3707 true, &rxo->rss_id); 3708 if (rc) 3709 return rc; 3710 } 3711 3712 if (be_multi_rxq(adapter)) { 3713 for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) { 3714 for_all_rss_queues(adapter, rxo, i) { 3715 if ((j + i) >= RSS_INDIR_TABLE_LEN) 3716 break; 3717 rss->rsstable[j + i] = rxo->rss_id; 3718 rss->rss_queue[j + i] = i; 3719 } 3720 } 3721 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 | 3722 RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6; 3723 3724 if (!BEx_chip(adapter)) 3725 rss->rss_flags |= RSS_ENABLE_UDP_IPV4 | 3726 RSS_ENABLE_UDP_IPV6; 3727 3728 netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN); 3729 rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3730 RSS_INDIR_TABLE_LEN, rss_key); 3731 if (rc) { 3732 rss->rss_flags = RSS_ENABLE_NONE; 3733 return rc; 3734 } 3735 3736 memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN); 3737 } else { 3738 /* Disable RSS, if only default RX Q is created */ 3739 rss->rss_flags = RSS_ENABLE_NONE; 3740 } 3741 3742 3743 /* Post 1 less than RXQ-len to avoid head being equal to tail, 3744 * which is a queue empty condition 3745 */ 3746 for_all_rx_queues(adapter, rxo, i) 3747 be_post_rx_frags(rxo, GFP_KERNEL, RX_Q_LEN - 1); 3748 3749 return 0; 3750 } 3751 3752 static int be_enable_if_filters(struct be_adapter *adapter) 3753 { 3754 int status; 3755 3756 status = be_cmd_rx_filter(adapter, BE_IF_FILT_FLAGS_BASIC, ON); 3757 if (status) 3758 return status; 3759 3760 /* Normally this condition usually true as the ->dev_mac is zeroed. 3761 * But on BE3 VFs the initial MAC is pre-programmed by PF and 3762 * subsequent be_dev_mac_add() can fail (after fresh boot) 3763 */ 3764 if (!ether_addr_equal(adapter->dev_mac, adapter->netdev->dev_addr)) { 3765 int old_pmac_id = -1; 3766 3767 /* Remember old programmed MAC if any - can happen on BE3 VF */ 3768 if (!is_zero_ether_addr(adapter->dev_mac)) 3769 old_pmac_id = adapter->pmac_id[0]; 3770 3771 status = be_dev_mac_add(adapter, adapter->netdev->dev_addr); 3772 if (status) 3773 return status; 3774 3775 /* Delete the old programmed MAC as we successfully programmed 3776 * a new MAC 3777 */ 3778 if (old_pmac_id >= 0 && old_pmac_id != adapter->pmac_id[0]) 3779 be_dev_mac_del(adapter, old_pmac_id); 3780 3781 ether_addr_copy(adapter->dev_mac, adapter->netdev->dev_addr); 3782 } 3783 3784 if (adapter->vlans_added) 3785 be_vid_config(adapter); 3786 3787 __be_set_rx_mode(adapter); 3788 3789 return 0; 3790 } 3791 3792 static int be_open(struct net_device *netdev) 3793 { 3794 struct be_adapter *adapter = netdev_priv(netdev); 3795 struct be_eq_obj *eqo; 3796 struct be_rx_obj *rxo; 3797 struct be_tx_obj *txo; 3798 u8 link_status; 3799 int status, i; 3800 3801 status = be_rx_qs_create(adapter); 3802 if (status) 3803 goto err; 3804 3805 status = be_enable_if_filters(adapter); 3806 if (status) 3807 goto err; 3808 3809 status = be_irq_register(adapter); 3810 if (status) 3811 goto err; 3812 3813 for_all_rx_queues(adapter, rxo, i) 3814 be_cq_notify(adapter, rxo->cq.id, true, 0); 3815 3816 for_all_tx_queues(adapter, txo, i) 3817 be_cq_notify(adapter, txo->cq.id, true, 0); 3818 3819 be_async_mcc_enable(adapter); 3820 3821 for_all_evt_queues(adapter, eqo, i) { 3822 napi_enable(&eqo->napi); 3823 be_eq_notify(adapter, eqo->q.id, true, true, 0, 0); 3824 } 3825 adapter->flags |= BE_FLAGS_NAPI_ENABLED; 3826 3827 status = be_cmd_link_status_query(adapter, NULL, &link_status, 0); 3828 if (!status) 3829 be_link_status_update(adapter, link_status); 3830 3831 netif_tx_start_all_queues(netdev); 3832 3833 udp_tunnel_nic_reset_ntf(netdev); 3834 3835 return 0; 3836 err: 3837 be_close(adapter->netdev); 3838 return -EIO; 3839 } 3840 3841 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac) 3842 { 3843 u32 addr; 3844 3845 addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0); 3846 3847 mac[5] = (u8)(addr & 0xFF); 3848 mac[4] = (u8)((addr >> 8) & 0xFF); 3849 mac[3] = (u8)((addr >> 16) & 0xFF); 3850 /* Use the OUI from the current MAC address */ 3851 memcpy(mac, adapter->netdev->dev_addr, 3); 3852 } 3853 3854 /* 3855 * Generate a seed MAC address from the PF MAC Address using jhash. 3856 * MAC Address for VFs are assigned incrementally starting from the seed. 3857 * These addresses are programmed in the ASIC by the PF and the VF driver 3858 * queries for the MAC address during its probe. 3859 */ 3860 static int be_vf_eth_addr_config(struct be_adapter *adapter) 3861 { 3862 u32 vf; 3863 int status = 0; 3864 u8 mac[ETH_ALEN]; 3865 struct be_vf_cfg *vf_cfg; 3866 3867 be_vf_eth_addr_generate(adapter, mac); 3868 3869 for_all_vfs(adapter, vf_cfg, vf) { 3870 if (BEx_chip(adapter)) 3871 status = be_cmd_pmac_add(adapter, mac, 3872 vf_cfg->if_handle, 3873 &vf_cfg->pmac_id, vf + 1); 3874 else 3875 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 3876 vf + 1); 3877 3878 if (status) 3879 dev_err(&adapter->pdev->dev, 3880 "Mac address assignment failed for VF %d\n", 3881 vf); 3882 else 3883 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3884 3885 mac[5] += 1; 3886 } 3887 return status; 3888 } 3889 3890 static int be_vfs_mac_query(struct be_adapter *adapter) 3891 { 3892 int status, vf; 3893 u8 mac[ETH_ALEN]; 3894 struct be_vf_cfg *vf_cfg; 3895 3896 for_all_vfs(adapter, vf_cfg, vf) { 3897 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id, 3898 mac, vf_cfg->if_handle, 3899 false, vf+1); 3900 if (status) 3901 return status; 3902 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3903 } 3904 return 0; 3905 } 3906 3907 static void be_vf_clear(struct be_adapter *adapter) 3908 { 3909 struct be_vf_cfg *vf_cfg; 3910 u32 vf; 3911 3912 if (pci_vfs_assigned(adapter->pdev)) { 3913 dev_warn(&adapter->pdev->dev, 3914 "VFs are assigned to VMs: not disabling VFs\n"); 3915 goto done; 3916 } 3917 3918 pci_disable_sriov(adapter->pdev); 3919 3920 for_all_vfs(adapter, vf_cfg, vf) { 3921 if (BEx_chip(adapter)) 3922 be_cmd_pmac_del(adapter, vf_cfg->if_handle, 3923 vf_cfg->pmac_id, vf + 1); 3924 else 3925 be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle, 3926 vf + 1); 3927 3928 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1); 3929 } 3930 3931 if (BE3_chip(adapter)) 3932 be_cmd_set_hsw_config(adapter, 0, 0, 3933 adapter->if_handle, 3934 PORT_FWD_TYPE_PASSTHRU, 0); 3935 done: 3936 kfree(adapter->vf_cfg); 3937 adapter->num_vfs = 0; 3938 adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED; 3939 } 3940 3941 static void be_clear_queues(struct be_adapter *adapter) 3942 { 3943 be_mcc_queues_destroy(adapter); 3944 be_rx_cqs_destroy(adapter); 3945 be_tx_queues_destroy(adapter); 3946 be_evt_queues_destroy(adapter); 3947 } 3948 3949 static void be_cancel_worker(struct be_adapter *adapter) 3950 { 3951 if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) { 3952 cancel_delayed_work_sync(&adapter->work); 3953 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED; 3954 } 3955 } 3956 3957 static void be_cancel_err_detection(struct be_adapter *adapter) 3958 { 3959 struct be_error_recovery *err_rec = &adapter->error_recovery; 3960 3961 if (!be_err_recovery_workq) 3962 return; 3963 3964 if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) { 3965 cancel_delayed_work_sync(&err_rec->err_detection_work); 3966 adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED; 3967 } 3968 } 3969 3970 /* VxLAN offload Notes: 3971 * 3972 * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't 3973 * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload 3974 * is expected to work across all types of IP tunnels once exported. Skyhawk 3975 * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN 3976 * offloads in hw_enc_features only when a VxLAN port is added. If other (non 3977 * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for 3978 * those other tunnels are unexported on the fly through ndo_features_check(). 3979 */ 3980 static int be_vxlan_set_port(struct net_device *netdev, unsigned int table, 3981 unsigned int entry, struct udp_tunnel_info *ti) 3982 { 3983 struct be_adapter *adapter = netdev_priv(netdev); 3984 struct device *dev = &adapter->pdev->dev; 3985 int status; 3986 3987 status = be_cmd_manage_iface(adapter, adapter->if_handle, 3988 OP_CONVERT_NORMAL_TO_TUNNEL); 3989 if (status) { 3990 dev_warn(dev, "Failed to convert normal interface to tunnel\n"); 3991 return status; 3992 } 3993 adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS; 3994 3995 status = be_cmd_set_vxlan_port(adapter, ti->port); 3996 if (status) { 3997 dev_warn(dev, "Failed to add VxLAN port\n"); 3998 return status; 3999 } 4000 adapter->vxlan_port = ti->port; 4001 4002 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4003 NETIF_F_TSO | NETIF_F_TSO6 | 4004 NETIF_F_GSO_UDP_TUNNEL; 4005 4006 dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n", 4007 be16_to_cpu(ti->port)); 4008 return 0; 4009 } 4010 4011 static int be_vxlan_unset_port(struct net_device *netdev, unsigned int table, 4012 unsigned int entry, struct udp_tunnel_info *ti) 4013 { 4014 struct be_adapter *adapter = netdev_priv(netdev); 4015 4016 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) 4017 be_cmd_manage_iface(adapter, adapter->if_handle, 4018 OP_CONVERT_TUNNEL_TO_NORMAL); 4019 4020 if (adapter->vxlan_port) 4021 be_cmd_set_vxlan_port(adapter, 0); 4022 4023 adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS; 4024 adapter->vxlan_port = 0; 4025 4026 netdev->hw_enc_features = 0; 4027 return 0; 4028 } 4029 4030 static const struct udp_tunnel_nic_info be_udp_tunnels = { 4031 .set_port = be_vxlan_set_port, 4032 .unset_port = be_vxlan_unset_port, 4033 .flags = UDP_TUNNEL_NIC_INFO_MAY_SLEEP | 4034 UDP_TUNNEL_NIC_INFO_OPEN_ONLY, 4035 .tables = { 4036 { .n_entries = 1, .tunnel_types = UDP_TUNNEL_TYPE_VXLAN, }, 4037 }, 4038 }; 4039 4040 static void be_calculate_vf_res(struct be_adapter *adapter, u16 num_vfs, 4041 struct be_resources *vft_res) 4042 { 4043 struct be_resources res = adapter->pool_res; 4044 u32 vf_if_cap_flags = res.vf_if_cap_flags; 4045 struct be_resources res_mod = {0}; 4046 u16 num_vf_qs = 1; 4047 4048 /* Distribute the queue resources among the PF and it's VFs */ 4049 if (num_vfs) { 4050 /* Divide the rx queues evenly among the VFs and the PF, capped 4051 * at VF-EQ-count. Any remainder queues belong to the PF. 4052 */ 4053 num_vf_qs = min(SH_VF_MAX_NIC_EQS, 4054 res.max_rss_qs / (num_vfs + 1)); 4055 4056 /* Skyhawk-R chip supports only MAX_PORT_RSS_TABLES 4057 * RSS Tables per port. Provide RSS on VFs, only if number of 4058 * VFs requested is less than it's PF Pool's RSS Tables limit. 4059 */ 4060 if (num_vfs >= be_max_pf_pool_rss_tables(adapter)) 4061 num_vf_qs = 1; 4062 } 4063 4064 /* Resource with fields set to all '1's by GET_PROFILE_CONFIG cmd, 4065 * which are modifiable using SET_PROFILE_CONFIG cmd. 4066 */ 4067 be_cmd_get_profile_config(adapter, &res_mod, NULL, ACTIVE_PROFILE_TYPE, 4068 RESOURCE_MODIFIABLE, 0); 4069 4070 /* If RSS IFACE capability flags are modifiable for a VF, set the 4071 * capability flag as valid and set RSS and DEFQ_RSS IFACE flags if 4072 * more than 1 RSSQ is available for a VF. 4073 * Otherwise, provision only 1 queue pair for VF. 4074 */ 4075 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_RSS) { 4076 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT); 4077 if (num_vf_qs > 1) { 4078 vf_if_cap_flags |= BE_IF_FLAGS_RSS; 4079 if (res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS) 4080 vf_if_cap_flags |= BE_IF_FLAGS_DEFQ_RSS; 4081 } else { 4082 vf_if_cap_flags &= ~(BE_IF_FLAGS_RSS | 4083 BE_IF_FLAGS_DEFQ_RSS); 4084 } 4085 } else { 4086 num_vf_qs = 1; 4087 } 4088 4089 if (res_mod.vf_if_cap_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 4090 vft_res->flags |= BIT(IF_CAPS_FLAGS_VALID_SHIFT); 4091 vf_if_cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 4092 } 4093 4094 vft_res->vf_if_cap_flags = vf_if_cap_flags; 4095 vft_res->max_rx_qs = num_vf_qs; 4096 vft_res->max_rss_qs = num_vf_qs; 4097 vft_res->max_tx_qs = res.max_tx_qs / (num_vfs + 1); 4098 vft_res->max_cq_count = res.max_cq_count / (num_vfs + 1); 4099 4100 /* Distribute unicast MACs, VLANs, IFACE count and MCCQ count equally 4101 * among the PF and it's VFs, if the fields are changeable 4102 */ 4103 if (res_mod.max_uc_mac == FIELD_MODIFIABLE) 4104 vft_res->max_uc_mac = res.max_uc_mac / (num_vfs + 1); 4105 4106 if (res_mod.max_vlans == FIELD_MODIFIABLE) 4107 vft_res->max_vlans = res.max_vlans / (num_vfs + 1); 4108 4109 if (res_mod.max_iface_count == FIELD_MODIFIABLE) 4110 vft_res->max_iface_count = res.max_iface_count / (num_vfs + 1); 4111 4112 if (res_mod.max_mcc_count == FIELD_MODIFIABLE) 4113 vft_res->max_mcc_count = res.max_mcc_count / (num_vfs + 1); 4114 } 4115 4116 static void be_if_destroy(struct be_adapter *adapter) 4117 { 4118 be_cmd_if_destroy(adapter, adapter->if_handle, 0); 4119 4120 kfree(adapter->pmac_id); 4121 adapter->pmac_id = NULL; 4122 4123 kfree(adapter->mc_list); 4124 adapter->mc_list = NULL; 4125 4126 kfree(adapter->uc_list); 4127 adapter->uc_list = NULL; 4128 } 4129 4130 static int be_clear(struct be_adapter *adapter) 4131 { 4132 struct pci_dev *pdev = adapter->pdev; 4133 struct be_resources vft_res = {0}; 4134 4135 be_cancel_worker(adapter); 4136 4137 flush_workqueue(be_wq); 4138 4139 if (sriov_enabled(adapter)) 4140 be_vf_clear(adapter); 4141 4142 /* Re-configure FW to distribute resources evenly across max-supported 4143 * number of VFs, only when VFs are not already enabled. 4144 */ 4145 if (skyhawk_chip(adapter) && be_physfn(adapter) && 4146 !pci_vfs_assigned(pdev)) { 4147 be_calculate_vf_res(adapter, 4148 pci_sriov_get_totalvfs(pdev), 4149 &vft_res); 4150 be_cmd_set_sriov_config(adapter, adapter->pool_res, 4151 pci_sriov_get_totalvfs(pdev), 4152 &vft_res); 4153 } 4154 4155 be_vxlan_unset_port(adapter->netdev, 0, 0, NULL); 4156 4157 be_if_destroy(adapter); 4158 4159 be_clear_queues(adapter); 4160 4161 be_msix_disable(adapter); 4162 adapter->flags &= ~BE_FLAGS_SETUP_DONE; 4163 return 0; 4164 } 4165 4166 static int be_vfs_if_create(struct be_adapter *adapter) 4167 { 4168 struct be_resources res = {0}; 4169 u32 cap_flags, en_flags, vf; 4170 struct be_vf_cfg *vf_cfg; 4171 int status; 4172 4173 /* If a FW profile exists, then cap_flags are updated */ 4174 cap_flags = BE_VF_IF_EN_FLAGS; 4175 4176 for_all_vfs(adapter, vf_cfg, vf) { 4177 if (!BE3_chip(adapter)) { 4178 status = be_cmd_get_profile_config(adapter, &res, NULL, 4179 ACTIVE_PROFILE_TYPE, 4180 RESOURCE_LIMITS, 4181 vf + 1); 4182 if (!status) { 4183 cap_flags = res.if_cap_flags; 4184 /* Prevent VFs from enabling VLAN promiscuous 4185 * mode 4186 */ 4187 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 4188 } 4189 } 4190 4191 /* PF should enable IF flags during proxy if_create call */ 4192 en_flags = cap_flags & BE_VF_IF_EN_FLAGS; 4193 status = be_cmd_if_create(adapter, cap_flags, en_flags, 4194 &vf_cfg->if_handle, vf + 1); 4195 if (status) 4196 return status; 4197 } 4198 4199 return 0; 4200 } 4201 4202 static int be_vf_setup_init(struct be_adapter *adapter) 4203 { 4204 struct be_vf_cfg *vf_cfg; 4205 int vf; 4206 4207 adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg), 4208 GFP_KERNEL); 4209 if (!adapter->vf_cfg) 4210 return -ENOMEM; 4211 4212 for_all_vfs(adapter, vf_cfg, vf) { 4213 vf_cfg->if_handle = -1; 4214 vf_cfg->pmac_id = -1; 4215 } 4216 return 0; 4217 } 4218 4219 static int be_vf_setup(struct be_adapter *adapter) 4220 { 4221 struct device *dev = &adapter->pdev->dev; 4222 struct be_vf_cfg *vf_cfg; 4223 int status, old_vfs, vf; 4224 bool spoofchk; 4225 4226 old_vfs = pci_num_vf(adapter->pdev); 4227 4228 status = be_vf_setup_init(adapter); 4229 if (status) 4230 goto err; 4231 4232 if (old_vfs) { 4233 for_all_vfs(adapter, vf_cfg, vf) { 4234 status = be_cmd_get_if_id(adapter, vf_cfg, vf); 4235 if (status) 4236 goto err; 4237 } 4238 4239 status = be_vfs_mac_query(adapter); 4240 if (status) 4241 goto err; 4242 } else { 4243 status = be_vfs_if_create(adapter); 4244 if (status) 4245 goto err; 4246 4247 status = be_vf_eth_addr_config(adapter); 4248 if (status) 4249 goto err; 4250 } 4251 4252 for_all_vfs(adapter, vf_cfg, vf) { 4253 /* Allow VFs to programs MAC/VLAN filters */ 4254 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges, 4255 vf + 1); 4256 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 4257 status = be_cmd_set_fn_privileges(adapter, 4258 vf_cfg->privileges | 4259 BE_PRIV_FILTMGMT, 4260 vf + 1); 4261 if (!status) { 4262 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 4263 dev_info(dev, "VF%d has FILTMGMT privilege\n", 4264 vf); 4265 } 4266 } 4267 4268 /* Allow full available bandwidth */ 4269 if (!old_vfs) 4270 be_cmd_config_qos(adapter, 0, 0, vf + 1); 4271 4272 status = be_cmd_get_hsw_config(adapter, NULL, vf + 1, 4273 vf_cfg->if_handle, NULL, 4274 &spoofchk); 4275 if (!status) 4276 vf_cfg->spoofchk = spoofchk; 4277 4278 if (!old_vfs) { 4279 be_cmd_enable_vf(adapter, vf + 1); 4280 be_cmd_set_logical_link_config(adapter, 4281 IFLA_VF_LINK_STATE_AUTO, 4282 vf+1); 4283 } 4284 } 4285 4286 if (!old_vfs) { 4287 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs); 4288 if (status) { 4289 dev_err(dev, "SRIOV enable failed\n"); 4290 adapter->num_vfs = 0; 4291 goto err; 4292 } 4293 } 4294 4295 if (BE3_chip(adapter)) { 4296 /* On BE3, enable VEB only when SRIOV is enabled */ 4297 status = be_cmd_set_hsw_config(adapter, 0, 0, 4298 adapter->if_handle, 4299 PORT_FWD_TYPE_VEB, 0); 4300 if (status) 4301 goto err; 4302 } 4303 4304 adapter->flags |= BE_FLAGS_SRIOV_ENABLED; 4305 return 0; 4306 err: 4307 dev_err(dev, "VF setup failed\n"); 4308 be_vf_clear(adapter); 4309 return status; 4310 } 4311 4312 /* Converting function_mode bits on BE3 to SH mc_type enums */ 4313 4314 static u8 be_convert_mc_type(u32 function_mode) 4315 { 4316 if (function_mode & VNIC_MODE && function_mode & QNQ_MODE) 4317 return vNIC1; 4318 else if (function_mode & QNQ_MODE) 4319 return FLEX10; 4320 else if (function_mode & VNIC_MODE) 4321 return vNIC2; 4322 else if (function_mode & UMC_ENABLED) 4323 return UMC; 4324 else 4325 return MC_NONE; 4326 } 4327 4328 /* On BE2/BE3 FW does not suggest the supported limits */ 4329 static void BEx_get_resources(struct be_adapter *adapter, 4330 struct be_resources *res) 4331 { 4332 bool use_sriov = adapter->num_vfs ? 1 : 0; 4333 4334 if (be_physfn(adapter)) 4335 res->max_uc_mac = BE_UC_PMAC_COUNT; 4336 else 4337 res->max_uc_mac = BE_VF_UC_PMAC_COUNT; 4338 4339 adapter->mc_type = be_convert_mc_type(adapter->function_mode); 4340 4341 if (be_is_mc(adapter)) { 4342 /* Assuming that there are 4 channels per port, 4343 * when multi-channel is enabled 4344 */ 4345 if (be_is_qnq_mode(adapter)) 4346 res->max_vlans = BE_NUM_VLANS_SUPPORTED/8; 4347 else 4348 /* In a non-qnq multichannel mode, the pvid 4349 * takes up one vlan entry 4350 */ 4351 res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1; 4352 } else { 4353 res->max_vlans = BE_NUM_VLANS_SUPPORTED; 4354 } 4355 4356 res->max_mcast_mac = BE_MAX_MC; 4357 4358 /* 1) For BE3 1Gb ports, FW does not support multiple TXQs 4359 * 2) Create multiple TX rings on a BE3-R multi-channel interface 4360 * *only* if it is RSS-capable. 4361 */ 4362 if (BE2_chip(adapter) || use_sriov || (adapter->port_num > 1) || 4363 be_virtfn(adapter) || 4364 (be_is_mc(adapter) && 4365 !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) { 4366 res->max_tx_qs = 1; 4367 } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) { 4368 struct be_resources super_nic_res = {0}; 4369 4370 /* On a SuperNIC profile, the driver needs to use the 4371 * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits 4372 */ 4373 be_cmd_get_profile_config(adapter, &super_nic_res, NULL, 4374 ACTIVE_PROFILE_TYPE, RESOURCE_LIMITS, 4375 0); 4376 /* Some old versions of BE3 FW don't report max_tx_qs value */ 4377 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS; 4378 } else { 4379 res->max_tx_qs = BE3_MAX_TX_QS; 4380 } 4381 4382 if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) && 4383 !use_sriov && be_physfn(adapter)) 4384 res->max_rss_qs = (adapter->be3_native) ? 4385 BE3_MAX_RSS_QS : BE2_MAX_RSS_QS; 4386 res->max_rx_qs = res->max_rss_qs + 1; 4387 4388 if (be_physfn(adapter)) 4389 res->max_evt_qs = (be_max_vfs(adapter) > 0) ? 4390 BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS; 4391 else 4392 res->max_evt_qs = 1; 4393 4394 res->if_cap_flags = BE_IF_CAP_FLAGS_WANT; 4395 res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS; 4396 if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS)) 4397 res->if_cap_flags &= ~BE_IF_FLAGS_RSS; 4398 } 4399 4400 static void be_setup_init(struct be_adapter *adapter) 4401 { 4402 adapter->vlan_prio_bmap = 0xff; 4403 adapter->phy.link_speed = -1; 4404 adapter->if_handle = -1; 4405 adapter->be3_native = false; 4406 adapter->if_flags = 0; 4407 adapter->phy_state = BE_UNKNOWN_PHY_STATE; 4408 if (be_physfn(adapter)) 4409 adapter->cmd_privileges = MAX_PRIVILEGES; 4410 else 4411 adapter->cmd_privileges = MIN_PRIVILEGES; 4412 } 4413 4414 /* HW supports only MAX_PORT_RSS_TABLES RSS Policy Tables per port. 4415 * However, this HW limitation is not exposed to the host via any SLI cmd. 4416 * As a result, in the case of SRIOV and in particular multi-partition configs 4417 * the driver needs to calcuate a proportional share of RSS Tables per PF-pool 4418 * for distribution between the VFs. This self-imposed limit will determine the 4419 * no: of VFs for which RSS can be enabled. 4420 */ 4421 static void be_calculate_pf_pool_rss_tables(struct be_adapter *adapter) 4422 { 4423 struct be_port_resources port_res = {0}; 4424 u8 rss_tables_on_port; 4425 u16 max_vfs = be_max_vfs(adapter); 4426 4427 be_cmd_get_profile_config(adapter, NULL, &port_res, SAVED_PROFILE_TYPE, 4428 RESOURCE_LIMITS, 0); 4429 4430 rss_tables_on_port = MAX_PORT_RSS_TABLES - port_res.nic_pfs; 4431 4432 /* Each PF Pool's RSS Tables limit = 4433 * PF's Max VFs / Total_Max_VFs on Port * RSS Tables on Port 4434 */ 4435 adapter->pool_res.max_rss_tables = 4436 max_vfs * rss_tables_on_port / port_res.max_vfs; 4437 } 4438 4439 static int be_get_sriov_config(struct be_adapter *adapter) 4440 { 4441 struct be_resources res = {0}; 4442 int max_vfs, old_vfs; 4443 4444 be_cmd_get_profile_config(adapter, &res, NULL, ACTIVE_PROFILE_TYPE, 4445 RESOURCE_LIMITS, 0); 4446 4447 /* Some old versions of BE3 FW don't report max_vfs value */ 4448 if (BE3_chip(adapter) && !res.max_vfs) { 4449 max_vfs = pci_sriov_get_totalvfs(adapter->pdev); 4450 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0; 4451 } 4452 4453 adapter->pool_res = res; 4454 4455 /* If during previous unload of the driver, the VFs were not disabled, 4456 * then we cannot rely on the PF POOL limits for the TotalVFs value. 4457 * Instead use the TotalVFs value stored in the pci-dev struct. 4458 */ 4459 old_vfs = pci_num_vf(adapter->pdev); 4460 if (old_vfs) { 4461 dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n", 4462 old_vfs); 4463 4464 adapter->pool_res.max_vfs = 4465 pci_sriov_get_totalvfs(adapter->pdev); 4466 adapter->num_vfs = old_vfs; 4467 } 4468 4469 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 4470 be_calculate_pf_pool_rss_tables(adapter); 4471 dev_info(&adapter->pdev->dev, 4472 "RSS can be enabled for all VFs if num_vfs <= %d\n", 4473 be_max_pf_pool_rss_tables(adapter)); 4474 } 4475 return 0; 4476 } 4477 4478 static void be_alloc_sriov_res(struct be_adapter *adapter) 4479 { 4480 int old_vfs = pci_num_vf(adapter->pdev); 4481 struct be_resources vft_res = {0}; 4482 int status; 4483 4484 be_get_sriov_config(adapter); 4485 4486 if (!old_vfs) 4487 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter)); 4488 4489 /* When the HW is in SRIOV capable configuration, the PF-pool 4490 * resources are given to PF during driver load, if there are no 4491 * old VFs. This facility is not available in BE3 FW. 4492 * Also, this is done by FW in Lancer chip. 4493 */ 4494 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 4495 be_calculate_vf_res(adapter, 0, &vft_res); 4496 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0, 4497 &vft_res); 4498 if (status) 4499 dev_err(&adapter->pdev->dev, 4500 "Failed to optimize SRIOV resources\n"); 4501 } 4502 } 4503 4504 static int be_get_resources(struct be_adapter *adapter) 4505 { 4506 struct device *dev = &adapter->pdev->dev; 4507 struct be_resources res = {0}; 4508 int status; 4509 4510 /* For Lancer, SH etc read per-function resource limits from FW. 4511 * GET_FUNC_CONFIG returns per function guaranteed limits. 4512 * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits 4513 */ 4514 if (BEx_chip(adapter)) { 4515 BEx_get_resources(adapter, &res); 4516 } else { 4517 status = be_cmd_get_func_config(adapter, &res); 4518 if (status) 4519 return status; 4520 4521 /* If a deafault RXQ must be created, we'll use up one RSSQ*/ 4522 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs && 4523 !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)) 4524 res.max_rss_qs -= 1; 4525 } 4526 4527 /* If RoCE is supported stash away half the EQs for RoCE */ 4528 res.max_nic_evt_qs = be_roce_supported(adapter) ? 4529 res.max_evt_qs / 2 : res.max_evt_qs; 4530 adapter->res = res; 4531 4532 /* If FW supports RSS default queue, then skip creating non-RSS 4533 * queue for non-IP traffic. 4534 */ 4535 adapter->need_def_rxq = (be_if_cap_flags(adapter) & 4536 BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1; 4537 4538 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n", 4539 be_max_txqs(adapter), be_max_rxqs(adapter), 4540 be_max_rss(adapter), be_max_nic_eqs(adapter), 4541 be_max_vfs(adapter)); 4542 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n", 4543 be_max_uc(adapter), be_max_mc(adapter), 4544 be_max_vlans(adapter)); 4545 4546 /* Ensure RX and TX queues are created in pairs at init time */ 4547 adapter->cfg_num_rx_irqs = 4548 min_t(u16, netif_get_num_default_rss_queues(), 4549 be_max_qp_irqs(adapter)); 4550 adapter->cfg_num_tx_irqs = adapter->cfg_num_rx_irqs; 4551 return 0; 4552 } 4553 4554 static int be_get_config(struct be_adapter *adapter) 4555 { 4556 int status, level; 4557 u16 profile_id; 4558 4559 status = be_cmd_get_cntl_attributes(adapter); 4560 if (status) 4561 return status; 4562 4563 status = be_cmd_query_fw_cfg(adapter); 4564 if (status) 4565 return status; 4566 4567 if (!lancer_chip(adapter) && be_physfn(adapter)) 4568 be_cmd_get_fat_dump_len(adapter, &adapter->fat_dump_len); 4569 4570 if (BEx_chip(adapter)) { 4571 level = be_cmd_get_fw_log_level(adapter); 4572 adapter->msg_enable = 4573 level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0; 4574 } 4575 4576 be_cmd_get_acpi_wol_cap(adapter); 4577 pci_enable_wake(adapter->pdev, PCI_D3hot, adapter->wol_en); 4578 pci_enable_wake(adapter->pdev, PCI_D3cold, adapter->wol_en); 4579 4580 be_cmd_query_port_name(adapter); 4581 4582 if (be_physfn(adapter)) { 4583 status = be_cmd_get_active_profile(adapter, &profile_id); 4584 if (!status) 4585 dev_info(&adapter->pdev->dev, 4586 "Using profile 0x%x\n", profile_id); 4587 } 4588 4589 return 0; 4590 } 4591 4592 static int be_mac_setup(struct be_adapter *adapter) 4593 { 4594 u8 mac[ETH_ALEN]; 4595 int status; 4596 4597 if (is_zero_ether_addr(adapter->netdev->dev_addr)) { 4598 status = be_cmd_get_perm_mac(adapter, mac); 4599 if (status) 4600 return status; 4601 4602 eth_hw_addr_set(adapter->netdev, mac); 4603 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN); 4604 4605 /* Initial MAC for BE3 VFs is already programmed by PF */ 4606 if (BEx_chip(adapter) && be_virtfn(adapter)) 4607 memcpy(adapter->dev_mac, mac, ETH_ALEN); 4608 } 4609 4610 return 0; 4611 } 4612 4613 static void be_schedule_worker(struct be_adapter *adapter) 4614 { 4615 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000)); 4616 adapter->flags |= BE_FLAGS_WORKER_SCHEDULED; 4617 } 4618 4619 static void be_destroy_err_recovery_workq(void) 4620 { 4621 if (!be_err_recovery_workq) 4622 return; 4623 4624 destroy_workqueue(be_err_recovery_workq); 4625 be_err_recovery_workq = NULL; 4626 } 4627 4628 static void be_schedule_err_detection(struct be_adapter *adapter, u32 delay) 4629 { 4630 struct be_error_recovery *err_rec = &adapter->error_recovery; 4631 4632 if (!be_err_recovery_workq) 4633 return; 4634 4635 queue_delayed_work(be_err_recovery_workq, &err_rec->err_detection_work, 4636 msecs_to_jiffies(delay)); 4637 adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED; 4638 } 4639 4640 static int be_setup_queues(struct be_adapter *adapter) 4641 { 4642 struct net_device *netdev = adapter->netdev; 4643 int status; 4644 4645 status = be_evt_queues_create(adapter); 4646 if (status) 4647 goto err; 4648 4649 status = be_tx_qs_create(adapter); 4650 if (status) 4651 goto err; 4652 4653 status = be_rx_cqs_create(adapter); 4654 if (status) 4655 goto err; 4656 4657 status = be_mcc_queues_create(adapter); 4658 if (status) 4659 goto err; 4660 4661 status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs); 4662 if (status) 4663 goto err; 4664 4665 status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs); 4666 if (status) 4667 goto err; 4668 4669 return 0; 4670 err: 4671 dev_err(&adapter->pdev->dev, "queue_setup failed\n"); 4672 return status; 4673 } 4674 4675 static int be_if_create(struct be_adapter *adapter) 4676 { 4677 u32 en_flags = BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS; 4678 u32 cap_flags = be_if_cap_flags(adapter); 4679 4680 /* alloc required memory for other filtering fields */ 4681 adapter->pmac_id = kcalloc(be_max_uc(adapter), 4682 sizeof(*adapter->pmac_id), GFP_KERNEL); 4683 if (!adapter->pmac_id) 4684 return -ENOMEM; 4685 4686 adapter->mc_list = kcalloc(be_max_mc(adapter), 4687 sizeof(*adapter->mc_list), GFP_KERNEL); 4688 if (!adapter->mc_list) 4689 return -ENOMEM; 4690 4691 adapter->uc_list = kcalloc(be_max_uc(adapter), 4692 sizeof(*adapter->uc_list), GFP_KERNEL); 4693 if (!adapter->uc_list) 4694 return -ENOMEM; 4695 4696 if (adapter->cfg_num_rx_irqs == 1) 4697 cap_flags &= ~(BE_IF_FLAGS_DEFQ_RSS | BE_IF_FLAGS_RSS); 4698 4699 en_flags &= cap_flags; 4700 /* will enable all the needed filter flags in be_open() */ 4701 return be_cmd_if_create(adapter, be_if_cap_flags(adapter), en_flags, 4702 &adapter->if_handle, 0); 4703 } 4704 4705 int be_update_queues(struct be_adapter *adapter) 4706 { 4707 struct net_device *netdev = adapter->netdev; 4708 int status; 4709 4710 if (netif_running(netdev)) { 4711 /* be_tx_timeout() must not run concurrently with this 4712 * function, synchronize with an already-running dev_watchdog 4713 */ 4714 netif_tx_lock_bh(netdev); 4715 /* device cannot transmit now, avoid dev_watchdog timeouts */ 4716 netif_carrier_off(netdev); 4717 netif_tx_unlock_bh(netdev); 4718 4719 be_close(netdev); 4720 } 4721 4722 be_cancel_worker(adapter); 4723 4724 /* If any vectors have been shared with RoCE we cannot re-program 4725 * the MSIx table. 4726 */ 4727 if (!adapter->num_msix_roce_vec) 4728 be_msix_disable(adapter); 4729 4730 be_clear_queues(adapter); 4731 status = be_cmd_if_destroy(adapter, adapter->if_handle, 0); 4732 if (status) 4733 return status; 4734 4735 if (!msix_enabled(adapter)) { 4736 status = be_msix_enable(adapter); 4737 if (status) 4738 return status; 4739 } 4740 4741 status = be_if_create(adapter); 4742 if (status) 4743 return status; 4744 4745 status = be_setup_queues(adapter); 4746 if (status) 4747 return status; 4748 4749 be_schedule_worker(adapter); 4750 4751 /* The IF was destroyed and re-created. We need to clear 4752 * all promiscuous flags valid for the destroyed IF. 4753 * Without this promisc mode is not restored during 4754 * be_open() because the driver thinks that it is 4755 * already enabled in HW. 4756 */ 4757 adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS; 4758 4759 if (netif_running(netdev)) 4760 status = be_open(netdev); 4761 4762 return status; 4763 } 4764 4765 static inline int fw_major_num(const char *fw_ver) 4766 { 4767 int fw_major = 0, i; 4768 4769 i = sscanf(fw_ver, "%d.", &fw_major); 4770 if (i != 1) 4771 return 0; 4772 4773 return fw_major; 4774 } 4775 4776 /* If it is error recovery, FLR the PF 4777 * Else if any VFs are already enabled don't FLR the PF 4778 */ 4779 static bool be_reset_required(struct be_adapter *adapter) 4780 { 4781 if (be_error_recovering(adapter)) 4782 return true; 4783 else 4784 return pci_num_vf(adapter->pdev) == 0; 4785 } 4786 4787 /* Wait for the FW to be ready and perform the required initialization */ 4788 static int be_func_init(struct be_adapter *adapter) 4789 { 4790 int status; 4791 4792 status = be_fw_wait_ready(adapter); 4793 if (status) 4794 return status; 4795 4796 /* FW is now ready; clear errors to allow cmds/doorbell */ 4797 be_clear_error(adapter, BE_CLEAR_ALL); 4798 4799 if (be_reset_required(adapter)) { 4800 status = be_cmd_reset_function(adapter); 4801 if (status) 4802 return status; 4803 4804 /* Wait for interrupts to quiesce after an FLR */ 4805 msleep(100); 4806 } 4807 4808 /* Tell FW we're ready to fire cmds */ 4809 status = be_cmd_fw_init(adapter); 4810 if (status) 4811 return status; 4812 4813 /* Allow interrupts for other ULPs running on NIC function */ 4814 be_intr_set(adapter, true); 4815 4816 return 0; 4817 } 4818 4819 static int be_setup(struct be_adapter *adapter) 4820 { 4821 struct device *dev = &adapter->pdev->dev; 4822 int status; 4823 4824 status = be_func_init(adapter); 4825 if (status) 4826 return status; 4827 4828 be_setup_init(adapter); 4829 4830 if (!lancer_chip(adapter)) 4831 be_cmd_req_native_mode(adapter); 4832 4833 /* invoke this cmd first to get pf_num and vf_num which are needed 4834 * for issuing profile related cmds 4835 */ 4836 if (!BEx_chip(adapter)) { 4837 status = be_cmd_get_func_config(adapter, NULL); 4838 if (status) 4839 return status; 4840 } 4841 4842 status = be_get_config(adapter); 4843 if (status) 4844 goto err; 4845 4846 if (!BE2_chip(adapter) && be_physfn(adapter)) 4847 be_alloc_sriov_res(adapter); 4848 4849 status = be_get_resources(adapter); 4850 if (status) 4851 goto err; 4852 4853 status = be_msix_enable(adapter); 4854 if (status) 4855 goto err; 4856 4857 /* will enable all the needed filter flags in be_open() */ 4858 status = be_if_create(adapter); 4859 if (status) 4860 goto err; 4861 4862 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 4863 rtnl_lock(); 4864 status = be_setup_queues(adapter); 4865 rtnl_unlock(); 4866 if (status) 4867 goto err; 4868 4869 be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0); 4870 4871 status = be_mac_setup(adapter); 4872 if (status) 4873 goto err; 4874 4875 be_cmd_get_fw_ver(adapter); 4876 dev_info(dev, "FW version is %s\n", adapter->fw_ver); 4877 4878 if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) { 4879 dev_err(dev, "Firmware on card is old(%s), IRQs may not work", 4880 adapter->fw_ver); 4881 dev_err(dev, "Please upgrade firmware to version >= 4.0\n"); 4882 } 4883 4884 status = be_cmd_set_flow_control(adapter, adapter->tx_fc, 4885 adapter->rx_fc); 4886 if (status) 4887 be_cmd_get_flow_control(adapter, &adapter->tx_fc, 4888 &adapter->rx_fc); 4889 4890 dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n", 4891 adapter->tx_fc, adapter->rx_fc); 4892 4893 if (be_physfn(adapter)) 4894 be_cmd_set_logical_link_config(adapter, 4895 IFLA_VF_LINK_STATE_AUTO, 0); 4896 4897 /* BE3 EVB echoes broadcast/multicast packets back to PF's vport 4898 * confusing a linux bridge or OVS that it might be connected to. 4899 * Set the EVB to PASSTHRU mode which effectively disables the EVB 4900 * when SRIOV is not enabled. 4901 */ 4902 if (BE3_chip(adapter)) 4903 be_cmd_set_hsw_config(adapter, 0, 0, adapter->if_handle, 4904 PORT_FWD_TYPE_PASSTHRU, 0); 4905 4906 if (adapter->num_vfs) 4907 be_vf_setup(adapter); 4908 4909 status = be_cmd_get_phy_info(adapter); 4910 if (!status && be_pause_supported(adapter)) 4911 adapter->phy.fc_autoneg = 1; 4912 4913 if (be_physfn(adapter) && !lancer_chip(adapter)) 4914 be_cmd_set_features(adapter); 4915 4916 be_schedule_worker(adapter); 4917 adapter->flags |= BE_FLAGS_SETUP_DONE; 4918 return 0; 4919 err: 4920 be_clear(adapter); 4921 return status; 4922 } 4923 4924 #ifdef CONFIG_NET_POLL_CONTROLLER 4925 static void be_netpoll(struct net_device *netdev) 4926 { 4927 struct be_adapter *adapter = netdev_priv(netdev); 4928 struct be_eq_obj *eqo; 4929 int i; 4930 4931 for_all_evt_queues(adapter, eqo, i) { 4932 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0, 0); 4933 napi_schedule(&eqo->napi); 4934 } 4935 } 4936 #endif 4937 4938 int be_load_fw(struct be_adapter *adapter, u8 *fw_file) 4939 { 4940 const struct firmware *fw; 4941 int status; 4942 4943 if (!netif_running(adapter->netdev)) { 4944 dev_err(&adapter->pdev->dev, 4945 "Firmware load not allowed (interface is down)\n"); 4946 return -ENETDOWN; 4947 } 4948 4949 status = request_firmware(&fw, fw_file, &adapter->pdev->dev); 4950 if (status) 4951 goto fw_exit; 4952 4953 dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file); 4954 4955 if (lancer_chip(adapter)) 4956 status = lancer_fw_download(adapter, fw); 4957 else 4958 status = be_fw_download(adapter, fw); 4959 4960 if (!status) 4961 be_cmd_get_fw_ver(adapter); 4962 4963 fw_exit: 4964 release_firmware(fw); 4965 return status; 4966 } 4967 4968 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4969 u16 flags, struct netlink_ext_ack *extack) 4970 { 4971 struct be_adapter *adapter = netdev_priv(dev); 4972 struct nlattr *attr, *br_spec; 4973 int rem; 4974 int status = 0; 4975 u16 mode = 0; 4976 4977 if (!sriov_enabled(adapter)) 4978 return -EOPNOTSUPP; 4979 4980 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4981 if (!br_spec) 4982 return -EINVAL; 4983 4984 nla_for_each_nested(attr, br_spec, rem) { 4985 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4986 continue; 4987 4988 if (nla_len(attr) < sizeof(mode)) 4989 return -EINVAL; 4990 4991 mode = nla_get_u16(attr); 4992 if (BE3_chip(adapter) && mode == BRIDGE_MODE_VEPA) 4993 return -EOPNOTSUPP; 4994 4995 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 4996 return -EINVAL; 4997 4998 status = be_cmd_set_hsw_config(adapter, 0, 0, 4999 adapter->if_handle, 5000 mode == BRIDGE_MODE_VEPA ? 5001 PORT_FWD_TYPE_VEPA : 5002 PORT_FWD_TYPE_VEB, 0); 5003 if (status) 5004 goto err; 5005 5006 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n", 5007 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 5008 5009 return status; 5010 } 5011 err: 5012 dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n", 5013 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 5014 5015 return status; 5016 } 5017 5018 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 5019 struct net_device *dev, u32 filter_mask, 5020 int nlflags) 5021 { 5022 struct be_adapter *adapter = netdev_priv(dev); 5023 int status = 0; 5024 u8 hsw_mode; 5025 5026 /* BE and Lancer chips support VEB mode only */ 5027 if (BEx_chip(adapter) || lancer_chip(adapter)) { 5028 /* VEB is disabled in non-SR-IOV profiles on BE3/Lancer */ 5029 if (!pci_sriov_get_totalvfs(adapter->pdev)) 5030 return 0; 5031 hsw_mode = PORT_FWD_TYPE_VEB; 5032 } else { 5033 status = be_cmd_get_hsw_config(adapter, NULL, 0, 5034 adapter->if_handle, &hsw_mode, 5035 NULL); 5036 if (status) 5037 return 0; 5038 5039 if (hsw_mode == PORT_FWD_TYPE_PASSTHRU) 5040 return 0; 5041 } 5042 5043 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 5044 hsw_mode == PORT_FWD_TYPE_VEPA ? 5045 BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB, 5046 0, 0, nlflags, filter_mask, NULL); 5047 } 5048 5049 static struct be_cmd_work *be_alloc_work(struct be_adapter *adapter, 5050 void (*func)(struct work_struct *)) 5051 { 5052 struct be_cmd_work *work; 5053 5054 work = kzalloc(sizeof(*work), GFP_ATOMIC); 5055 if (!work) { 5056 dev_err(&adapter->pdev->dev, 5057 "be_work memory allocation failed\n"); 5058 return NULL; 5059 } 5060 5061 INIT_WORK(&work->work, func); 5062 work->adapter = adapter; 5063 return work; 5064 } 5065 5066 static netdev_features_t be_features_check(struct sk_buff *skb, 5067 struct net_device *dev, 5068 netdev_features_t features) 5069 { 5070 struct be_adapter *adapter = netdev_priv(dev); 5071 u8 l4_hdr = 0; 5072 5073 if (skb_is_gso(skb)) { 5074 /* IPv6 TSO requests with extension hdrs are a problem 5075 * to Lancer and BE3 HW. Disable TSO6 feature. 5076 */ 5077 if (!skyhawk_chip(adapter) && is_ipv6_ext_hdr(skb)) 5078 features &= ~NETIF_F_TSO6; 5079 5080 /* Lancer cannot handle the packet with MSS less than 256. 5081 * Also it can't handle a TSO packet with a single segment 5082 * Disable the GSO support in such cases 5083 */ 5084 if (lancer_chip(adapter) && 5085 (skb_shinfo(skb)->gso_size < 256 || 5086 skb_shinfo(skb)->gso_segs == 1)) 5087 features &= ~NETIF_F_GSO_MASK; 5088 } 5089 5090 /* The code below restricts offload features for some tunneled and 5091 * Q-in-Q packets. 5092 * Offload features for normal (non tunnel) packets are unchanged. 5093 */ 5094 features = vlan_features_check(skb, features); 5095 if (!skb->encapsulation || 5096 !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)) 5097 return features; 5098 5099 /* It's an encapsulated packet and VxLAN offloads are enabled. We 5100 * should disable tunnel offload features if it's not a VxLAN packet, 5101 * as tunnel offloads have been enabled only for VxLAN. This is done to 5102 * allow other tunneled traffic like GRE work fine while VxLAN 5103 * offloads are configured in Skyhawk-R. 5104 */ 5105 switch (vlan_get_protocol(skb)) { 5106 case htons(ETH_P_IP): 5107 l4_hdr = ip_hdr(skb)->protocol; 5108 break; 5109 case htons(ETH_P_IPV6): 5110 l4_hdr = ipv6_hdr(skb)->nexthdr; 5111 break; 5112 default: 5113 return features; 5114 } 5115 5116 if (l4_hdr != IPPROTO_UDP || 5117 skb->inner_protocol_type != ENCAP_TYPE_ETHER || 5118 skb->inner_protocol != htons(ETH_P_TEB) || 5119 skb_inner_mac_header(skb) - skb_transport_header(skb) != 5120 sizeof(struct udphdr) + sizeof(struct vxlanhdr) || 5121 !adapter->vxlan_port || 5122 udp_hdr(skb)->dest != adapter->vxlan_port) 5123 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 5124 5125 return features; 5126 } 5127 5128 static int be_get_phys_port_id(struct net_device *dev, 5129 struct netdev_phys_item_id *ppid) 5130 { 5131 int i, id_len = CNTL_SERIAL_NUM_WORDS * CNTL_SERIAL_NUM_WORD_SZ + 1; 5132 struct be_adapter *adapter = netdev_priv(dev); 5133 u8 *id; 5134 5135 if (MAX_PHYS_ITEM_ID_LEN < id_len) 5136 return -ENOSPC; 5137 5138 ppid->id[0] = adapter->hba_port_num + 1; 5139 id = &ppid->id[1]; 5140 for (i = CNTL_SERIAL_NUM_WORDS - 1; i >= 0; 5141 i--, id += CNTL_SERIAL_NUM_WORD_SZ) 5142 memcpy(id, &adapter->serial_num[i], CNTL_SERIAL_NUM_WORD_SZ); 5143 5144 ppid->id_len = id_len; 5145 5146 return 0; 5147 } 5148 5149 static void be_set_rx_mode(struct net_device *dev) 5150 { 5151 struct be_adapter *adapter = netdev_priv(dev); 5152 struct be_cmd_work *work; 5153 5154 work = be_alloc_work(adapter, be_work_set_rx_mode); 5155 if (work) 5156 queue_work(be_wq, &work->work); 5157 } 5158 5159 static const struct net_device_ops be_netdev_ops = { 5160 .ndo_open = be_open, 5161 .ndo_stop = be_close, 5162 .ndo_start_xmit = be_xmit, 5163 .ndo_set_rx_mode = be_set_rx_mode, 5164 .ndo_set_mac_address = be_mac_addr_set, 5165 .ndo_get_stats64 = be_get_stats64, 5166 .ndo_validate_addr = eth_validate_addr, 5167 .ndo_vlan_rx_add_vid = be_vlan_add_vid, 5168 .ndo_vlan_rx_kill_vid = be_vlan_rem_vid, 5169 .ndo_set_vf_mac = be_set_vf_mac, 5170 .ndo_set_vf_vlan = be_set_vf_vlan, 5171 .ndo_set_vf_rate = be_set_vf_tx_rate, 5172 .ndo_get_vf_config = be_get_vf_config, 5173 .ndo_set_vf_link_state = be_set_vf_link_state, 5174 .ndo_set_vf_spoofchk = be_set_vf_spoofchk, 5175 .ndo_tx_timeout = be_tx_timeout, 5176 #ifdef CONFIG_NET_POLL_CONTROLLER 5177 .ndo_poll_controller = be_netpoll, 5178 #endif 5179 .ndo_bridge_setlink = be_ndo_bridge_setlink, 5180 .ndo_bridge_getlink = be_ndo_bridge_getlink, 5181 .ndo_features_check = be_features_check, 5182 .ndo_get_phys_port_id = be_get_phys_port_id, 5183 }; 5184 5185 static void be_netdev_init(struct net_device *netdev) 5186 { 5187 struct be_adapter *adapter = netdev_priv(netdev); 5188 5189 netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5190 NETIF_F_GSO_UDP_TUNNEL | 5191 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | 5192 NETIF_F_HW_VLAN_CTAG_TX; 5193 if ((be_if_cap_flags(adapter) & BE_IF_FLAGS_RSS)) 5194 netdev->hw_features |= NETIF_F_RXHASH; 5195 5196 netdev->features |= netdev->hw_features | 5197 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER | 5198 NETIF_F_HIGHDMA; 5199 5200 netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5201 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 5202 5203 netdev->priv_flags |= IFF_UNICAST_FLT; 5204 5205 netdev->flags |= IFF_MULTICAST; 5206 5207 netif_set_gso_max_size(netdev, BE_MAX_GSO_SIZE - ETH_HLEN); 5208 5209 netdev->netdev_ops = &be_netdev_ops; 5210 5211 netdev->ethtool_ops = &be_ethtool_ops; 5212 5213 if (!lancer_chip(adapter) && !BEx_chip(adapter) && !be_is_mc(adapter)) 5214 netdev->udp_tunnel_nic_info = &be_udp_tunnels; 5215 5216 /* MTU range: 256 - 9000 */ 5217 netdev->min_mtu = BE_MIN_MTU; 5218 netdev->max_mtu = BE_MAX_MTU; 5219 } 5220 5221 static void be_cleanup(struct be_adapter *adapter) 5222 { 5223 struct net_device *netdev = adapter->netdev; 5224 5225 rtnl_lock(); 5226 netif_device_detach(netdev); 5227 if (netif_running(netdev)) 5228 be_close(netdev); 5229 rtnl_unlock(); 5230 5231 be_clear(adapter); 5232 } 5233 5234 static int be_resume(struct be_adapter *adapter) 5235 { 5236 struct net_device *netdev = adapter->netdev; 5237 int status; 5238 5239 status = be_setup(adapter); 5240 if (status) 5241 return status; 5242 5243 rtnl_lock(); 5244 if (netif_running(netdev)) 5245 status = be_open(netdev); 5246 rtnl_unlock(); 5247 5248 if (status) 5249 return status; 5250 5251 netif_device_attach(netdev); 5252 5253 return 0; 5254 } 5255 5256 static void be_soft_reset(struct be_adapter *adapter) 5257 { 5258 u32 val; 5259 5260 dev_info(&adapter->pdev->dev, "Initiating chip soft reset\n"); 5261 val = ioread32(adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET); 5262 val |= SLIPORT_SOFTRESET_SR_MASK; 5263 iowrite32(val, adapter->pcicfg + SLIPORT_SOFTRESET_OFFSET); 5264 } 5265 5266 static bool be_err_is_recoverable(struct be_adapter *adapter) 5267 { 5268 struct be_error_recovery *err_rec = &adapter->error_recovery; 5269 unsigned long initial_idle_time = 5270 msecs_to_jiffies(ERR_RECOVERY_IDLE_TIME); 5271 unsigned long recovery_interval = 5272 msecs_to_jiffies(ERR_RECOVERY_INTERVAL); 5273 u16 ue_err_code; 5274 u32 val; 5275 5276 val = be_POST_stage_get(adapter); 5277 if ((val & POST_STAGE_RECOVERABLE_ERR) != POST_STAGE_RECOVERABLE_ERR) 5278 return false; 5279 ue_err_code = val & POST_ERR_RECOVERY_CODE_MASK; 5280 if (ue_err_code == 0) 5281 return false; 5282 5283 dev_err(&adapter->pdev->dev, "Recoverable HW error code: 0x%x\n", 5284 ue_err_code); 5285 5286 if (time_before_eq(jiffies - err_rec->probe_time, initial_idle_time)) { 5287 dev_err(&adapter->pdev->dev, 5288 "Cannot recover within %lu sec from driver load\n", 5289 jiffies_to_msecs(initial_idle_time) / MSEC_PER_SEC); 5290 return false; 5291 } 5292 5293 if (err_rec->last_recovery_time && time_before_eq( 5294 jiffies - err_rec->last_recovery_time, recovery_interval)) { 5295 dev_err(&adapter->pdev->dev, 5296 "Cannot recover within %lu sec from last recovery\n", 5297 jiffies_to_msecs(recovery_interval) / MSEC_PER_SEC); 5298 return false; 5299 } 5300 5301 if (ue_err_code == err_rec->last_err_code) { 5302 dev_err(&adapter->pdev->dev, 5303 "Cannot recover from a consecutive TPE error\n"); 5304 return false; 5305 } 5306 5307 err_rec->last_recovery_time = jiffies; 5308 err_rec->last_err_code = ue_err_code; 5309 return true; 5310 } 5311 5312 static int be_tpe_recover(struct be_adapter *adapter) 5313 { 5314 struct be_error_recovery *err_rec = &adapter->error_recovery; 5315 int status = -EAGAIN; 5316 u32 val; 5317 5318 switch (err_rec->recovery_state) { 5319 case ERR_RECOVERY_ST_NONE: 5320 err_rec->recovery_state = ERR_RECOVERY_ST_DETECT; 5321 err_rec->resched_delay = ERR_RECOVERY_UE_DETECT_DURATION; 5322 break; 5323 5324 case ERR_RECOVERY_ST_DETECT: 5325 val = be_POST_stage_get(adapter); 5326 if ((val & POST_STAGE_RECOVERABLE_ERR) != 5327 POST_STAGE_RECOVERABLE_ERR) { 5328 dev_err(&adapter->pdev->dev, 5329 "Unrecoverable HW error detected: 0x%x\n", val); 5330 status = -EINVAL; 5331 err_rec->resched_delay = 0; 5332 break; 5333 } 5334 5335 dev_err(&adapter->pdev->dev, "Recoverable HW error detected\n"); 5336 5337 /* Only PF0 initiates Chip Soft Reset. But PF0 must wait UE2SR 5338 * milliseconds before it checks for final error status in 5339 * SLIPORT_SEMAPHORE to determine if recovery criteria is met. 5340 * If it does, then PF0 initiates a Soft Reset. 5341 */ 5342 if (adapter->pf_num == 0) { 5343 err_rec->recovery_state = ERR_RECOVERY_ST_RESET; 5344 err_rec->resched_delay = err_rec->ue_to_reset_time - 5345 ERR_RECOVERY_UE_DETECT_DURATION; 5346 break; 5347 } 5348 5349 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL; 5350 err_rec->resched_delay = err_rec->ue_to_poll_time - 5351 ERR_RECOVERY_UE_DETECT_DURATION; 5352 break; 5353 5354 case ERR_RECOVERY_ST_RESET: 5355 if (!be_err_is_recoverable(adapter)) { 5356 dev_err(&adapter->pdev->dev, 5357 "Failed to meet recovery criteria\n"); 5358 status = -EIO; 5359 err_rec->resched_delay = 0; 5360 break; 5361 } 5362 be_soft_reset(adapter); 5363 err_rec->recovery_state = ERR_RECOVERY_ST_PRE_POLL; 5364 err_rec->resched_delay = err_rec->ue_to_poll_time - 5365 err_rec->ue_to_reset_time; 5366 break; 5367 5368 case ERR_RECOVERY_ST_PRE_POLL: 5369 err_rec->recovery_state = ERR_RECOVERY_ST_REINIT; 5370 err_rec->resched_delay = 0; 5371 status = 0; /* done */ 5372 break; 5373 5374 default: 5375 status = -EINVAL; 5376 err_rec->resched_delay = 0; 5377 break; 5378 } 5379 5380 return status; 5381 } 5382 5383 static int be_err_recover(struct be_adapter *adapter) 5384 { 5385 int status; 5386 5387 if (!lancer_chip(adapter)) { 5388 if (!adapter->error_recovery.recovery_supported || 5389 adapter->priv_flags & BE_DISABLE_TPE_RECOVERY) 5390 return -EIO; 5391 status = be_tpe_recover(adapter); 5392 if (status) 5393 goto err; 5394 } 5395 5396 /* Wait for adapter to reach quiescent state before 5397 * destroying queues 5398 */ 5399 status = be_fw_wait_ready(adapter); 5400 if (status) 5401 goto err; 5402 5403 adapter->flags |= BE_FLAGS_TRY_RECOVERY; 5404 5405 be_cleanup(adapter); 5406 5407 status = be_resume(adapter); 5408 if (status) 5409 goto err; 5410 5411 adapter->flags &= ~BE_FLAGS_TRY_RECOVERY; 5412 5413 err: 5414 return status; 5415 } 5416 5417 static void be_err_detection_task(struct work_struct *work) 5418 { 5419 struct be_error_recovery *err_rec = 5420 container_of(work, struct be_error_recovery, 5421 err_detection_work.work); 5422 struct be_adapter *adapter = 5423 container_of(err_rec, struct be_adapter, 5424 error_recovery); 5425 u32 resched_delay = ERR_RECOVERY_DETECTION_DELAY; 5426 struct device *dev = &adapter->pdev->dev; 5427 int recovery_status; 5428 5429 be_detect_error(adapter); 5430 if (!be_check_error(adapter, BE_ERROR_HW)) 5431 goto reschedule_task; 5432 5433 recovery_status = be_err_recover(adapter); 5434 if (!recovery_status) { 5435 err_rec->recovery_retries = 0; 5436 err_rec->recovery_state = ERR_RECOVERY_ST_NONE; 5437 dev_info(dev, "Adapter recovery successful\n"); 5438 goto reschedule_task; 5439 } else if (!lancer_chip(adapter) && err_rec->resched_delay) { 5440 /* BEx/SH recovery state machine */ 5441 if (adapter->pf_num == 0 && 5442 err_rec->recovery_state > ERR_RECOVERY_ST_DETECT) 5443 dev_err(&adapter->pdev->dev, 5444 "Adapter recovery in progress\n"); 5445 resched_delay = err_rec->resched_delay; 5446 goto reschedule_task; 5447 } else if (lancer_chip(adapter) && be_virtfn(adapter)) { 5448 /* For VFs, check if PF have allocated resources 5449 * every second. 5450 */ 5451 dev_err(dev, "Re-trying adapter recovery\n"); 5452 goto reschedule_task; 5453 } else if (lancer_chip(adapter) && err_rec->recovery_retries++ < 5454 ERR_RECOVERY_MAX_RETRY_COUNT) { 5455 /* In case of another error during recovery, it takes 30 sec 5456 * for adapter to come out of error. Retry error recovery after 5457 * this time interval. 5458 */ 5459 dev_err(&adapter->pdev->dev, "Re-trying adapter recovery\n"); 5460 resched_delay = ERR_RECOVERY_RETRY_DELAY; 5461 goto reschedule_task; 5462 } else { 5463 dev_err(dev, "Adapter recovery failed\n"); 5464 dev_err(dev, "Please reboot server to recover\n"); 5465 } 5466 5467 return; 5468 5469 reschedule_task: 5470 be_schedule_err_detection(adapter, resched_delay); 5471 } 5472 5473 static void be_log_sfp_info(struct be_adapter *adapter) 5474 { 5475 int status; 5476 5477 status = be_cmd_query_sfp_info(adapter); 5478 if (!status) { 5479 dev_err(&adapter->pdev->dev, 5480 "Port %c: %s Vendor: %s part no: %s", 5481 adapter->port_name, 5482 be_misconfig_evt_port_state[adapter->phy_state], 5483 adapter->phy.vendor_name, 5484 adapter->phy.vendor_pn); 5485 } 5486 adapter->flags &= ~BE_FLAGS_PHY_MISCONFIGURED; 5487 } 5488 5489 static void be_worker(struct work_struct *work) 5490 { 5491 struct be_adapter *adapter = 5492 container_of(work, struct be_adapter, work.work); 5493 struct be_rx_obj *rxo; 5494 int i; 5495 5496 if (be_physfn(adapter) && 5497 MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0) 5498 be_cmd_get_die_temperature(adapter); 5499 5500 /* when interrupts are not yet enabled, just reap any pending 5501 * mcc completions 5502 */ 5503 if (!netif_running(adapter->netdev)) { 5504 local_bh_disable(); 5505 be_process_mcc(adapter); 5506 local_bh_enable(); 5507 goto reschedule; 5508 } 5509 5510 if (!adapter->stats_cmd_sent) { 5511 if (lancer_chip(adapter)) 5512 lancer_cmd_get_pport_stats(adapter, 5513 &adapter->stats_cmd); 5514 else 5515 be_cmd_get_stats(adapter, &adapter->stats_cmd); 5516 } 5517 5518 for_all_rx_queues(adapter, rxo, i) { 5519 /* Replenish RX-queues starved due to memory 5520 * allocation failures. 5521 */ 5522 if (rxo->rx_post_starved) 5523 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST); 5524 } 5525 5526 /* EQ-delay update for Skyhawk is done while notifying EQ */ 5527 if (!skyhawk_chip(adapter)) 5528 be_eqd_update(adapter, false); 5529 5530 if (adapter->flags & BE_FLAGS_PHY_MISCONFIGURED) 5531 be_log_sfp_info(adapter); 5532 5533 reschedule: 5534 adapter->work_counter++; 5535 queue_delayed_work(be_wq, &adapter->work, msecs_to_jiffies(1000)); 5536 } 5537 5538 static void be_unmap_pci_bars(struct be_adapter *adapter) 5539 { 5540 if (adapter->csr) 5541 pci_iounmap(adapter->pdev, adapter->csr); 5542 if (adapter->db) 5543 pci_iounmap(adapter->pdev, adapter->db); 5544 if (adapter->pcicfg && adapter->pcicfg_mapped) 5545 pci_iounmap(adapter->pdev, adapter->pcicfg); 5546 } 5547 5548 static int db_bar(struct be_adapter *adapter) 5549 { 5550 if (lancer_chip(adapter) || be_virtfn(adapter)) 5551 return 0; 5552 else 5553 return 4; 5554 } 5555 5556 static int be_roce_map_pci_bars(struct be_adapter *adapter) 5557 { 5558 if (skyhawk_chip(adapter)) { 5559 adapter->roce_db.size = 4096; 5560 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev, 5561 db_bar(adapter)); 5562 adapter->roce_db.total_size = pci_resource_len(adapter->pdev, 5563 db_bar(adapter)); 5564 } 5565 return 0; 5566 } 5567 5568 static int be_map_pci_bars(struct be_adapter *adapter) 5569 { 5570 struct pci_dev *pdev = adapter->pdev; 5571 u8 __iomem *addr; 5572 u32 sli_intf; 5573 5574 pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf); 5575 adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >> 5576 SLI_INTF_FAMILY_SHIFT; 5577 adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0; 5578 5579 if (BEx_chip(adapter) && be_physfn(adapter)) { 5580 adapter->csr = pci_iomap(pdev, 2, 0); 5581 if (!adapter->csr) 5582 return -ENOMEM; 5583 } 5584 5585 addr = pci_iomap(pdev, db_bar(adapter), 0); 5586 if (!addr) 5587 goto pci_map_err; 5588 adapter->db = addr; 5589 5590 if (skyhawk_chip(adapter) || BEx_chip(adapter)) { 5591 if (be_physfn(adapter)) { 5592 /* PCICFG is the 2nd BAR in BE2 */ 5593 addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0); 5594 if (!addr) 5595 goto pci_map_err; 5596 adapter->pcicfg = addr; 5597 adapter->pcicfg_mapped = true; 5598 } else { 5599 adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET; 5600 adapter->pcicfg_mapped = false; 5601 } 5602 } 5603 5604 be_roce_map_pci_bars(adapter); 5605 return 0; 5606 5607 pci_map_err: 5608 dev_err(&pdev->dev, "Error in mapping PCI BARs\n"); 5609 be_unmap_pci_bars(adapter); 5610 return -ENOMEM; 5611 } 5612 5613 static void be_drv_cleanup(struct be_adapter *adapter) 5614 { 5615 struct be_dma_mem *mem = &adapter->mbox_mem_alloced; 5616 struct device *dev = &adapter->pdev->dev; 5617 5618 if (mem->va) 5619 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5620 5621 mem = &adapter->rx_filter; 5622 if (mem->va) 5623 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5624 5625 mem = &adapter->stats_cmd; 5626 if (mem->va) 5627 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5628 } 5629 5630 /* Allocate and initialize various fields in be_adapter struct */ 5631 static int be_drv_init(struct be_adapter *adapter) 5632 { 5633 struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced; 5634 struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem; 5635 struct be_dma_mem *rx_filter = &adapter->rx_filter; 5636 struct be_dma_mem *stats_cmd = &adapter->stats_cmd; 5637 struct device *dev = &adapter->pdev->dev; 5638 int status = 0; 5639 5640 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 5641 mbox_mem_alloc->va = dma_alloc_coherent(dev, mbox_mem_alloc->size, 5642 &mbox_mem_alloc->dma, 5643 GFP_KERNEL); 5644 if (!mbox_mem_alloc->va) 5645 return -ENOMEM; 5646 5647 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 5648 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 5649 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 5650 5651 rx_filter->size = sizeof(struct be_cmd_req_rx_filter); 5652 rx_filter->va = dma_alloc_coherent(dev, rx_filter->size, 5653 &rx_filter->dma, GFP_KERNEL); 5654 if (!rx_filter->va) { 5655 status = -ENOMEM; 5656 goto free_mbox; 5657 } 5658 5659 if (lancer_chip(adapter)) 5660 stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats); 5661 else if (BE2_chip(adapter)) 5662 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0); 5663 else if (BE3_chip(adapter)) 5664 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1); 5665 else 5666 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2); 5667 stats_cmd->va = dma_alloc_coherent(dev, stats_cmd->size, 5668 &stats_cmd->dma, GFP_KERNEL); 5669 if (!stats_cmd->va) { 5670 status = -ENOMEM; 5671 goto free_rx_filter; 5672 } 5673 5674 mutex_init(&adapter->mbox_lock); 5675 mutex_init(&adapter->mcc_lock); 5676 mutex_init(&adapter->rx_filter_lock); 5677 spin_lock_init(&adapter->mcc_cq_lock); 5678 init_completion(&adapter->et_cmd_compl); 5679 5680 pci_save_state(adapter->pdev); 5681 5682 INIT_DELAYED_WORK(&adapter->work, be_worker); 5683 5684 adapter->error_recovery.recovery_state = ERR_RECOVERY_ST_NONE; 5685 adapter->error_recovery.resched_delay = 0; 5686 INIT_DELAYED_WORK(&adapter->error_recovery.err_detection_work, 5687 be_err_detection_task); 5688 5689 adapter->rx_fc = true; 5690 adapter->tx_fc = true; 5691 5692 /* Must be a power of 2 or else MODULO will BUG_ON */ 5693 adapter->be_get_temp_freq = 64; 5694 5695 return 0; 5696 5697 free_rx_filter: 5698 dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma); 5699 free_mbox: 5700 dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va, 5701 mbox_mem_alloc->dma); 5702 return status; 5703 } 5704 5705 static void be_remove(struct pci_dev *pdev) 5706 { 5707 struct be_adapter *adapter = pci_get_drvdata(pdev); 5708 5709 if (!adapter) 5710 return; 5711 5712 be_roce_dev_remove(adapter); 5713 be_intr_set(adapter, false); 5714 5715 be_cancel_err_detection(adapter); 5716 5717 unregister_netdev(adapter->netdev); 5718 5719 be_clear(adapter); 5720 5721 if (!pci_vfs_assigned(adapter->pdev)) 5722 be_cmd_reset_function(adapter); 5723 5724 /* tell fw we're done with firing cmds */ 5725 be_cmd_fw_clean(adapter); 5726 5727 be_unmap_pci_bars(adapter); 5728 be_drv_cleanup(adapter); 5729 5730 pci_disable_pcie_error_reporting(pdev); 5731 5732 pci_release_regions(pdev); 5733 pci_disable_device(pdev); 5734 5735 free_netdev(adapter->netdev); 5736 } 5737 5738 static ssize_t be_hwmon_show_temp(struct device *dev, 5739 struct device_attribute *dev_attr, 5740 char *buf) 5741 { 5742 struct be_adapter *adapter = dev_get_drvdata(dev); 5743 5744 /* Unit: millidegree Celsius */ 5745 if (adapter->hwmon_info.be_on_die_temp == BE_INVALID_DIE_TEMP) 5746 return -EIO; 5747 else 5748 return sprintf(buf, "%u\n", 5749 adapter->hwmon_info.be_on_die_temp * 1000); 5750 } 5751 5752 static SENSOR_DEVICE_ATTR(temp1_input, 0444, 5753 be_hwmon_show_temp, NULL, 1); 5754 5755 static struct attribute *be_hwmon_attrs[] = { 5756 &sensor_dev_attr_temp1_input.dev_attr.attr, 5757 NULL 5758 }; 5759 5760 ATTRIBUTE_GROUPS(be_hwmon); 5761 5762 static char *mc_name(struct be_adapter *adapter) 5763 { 5764 char *str = ""; /* default */ 5765 5766 switch (adapter->mc_type) { 5767 case UMC: 5768 str = "UMC"; 5769 break; 5770 case FLEX10: 5771 str = "FLEX10"; 5772 break; 5773 case vNIC1: 5774 str = "vNIC-1"; 5775 break; 5776 case nPAR: 5777 str = "nPAR"; 5778 break; 5779 case UFP: 5780 str = "UFP"; 5781 break; 5782 case vNIC2: 5783 str = "vNIC-2"; 5784 break; 5785 default: 5786 str = ""; 5787 } 5788 5789 return str; 5790 } 5791 5792 static inline char *func_name(struct be_adapter *adapter) 5793 { 5794 return be_physfn(adapter) ? "PF" : "VF"; 5795 } 5796 5797 static inline char *nic_name(struct pci_dev *pdev) 5798 { 5799 switch (pdev->device) { 5800 case OC_DEVICE_ID1: 5801 return OC_NAME; 5802 case OC_DEVICE_ID2: 5803 return OC_NAME_BE; 5804 case OC_DEVICE_ID3: 5805 case OC_DEVICE_ID4: 5806 return OC_NAME_LANCER; 5807 case BE_DEVICE_ID2: 5808 return BE3_NAME; 5809 case OC_DEVICE_ID5: 5810 case OC_DEVICE_ID6: 5811 return OC_NAME_SH; 5812 default: 5813 return BE_NAME; 5814 } 5815 } 5816 5817 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id) 5818 { 5819 struct be_adapter *adapter; 5820 struct net_device *netdev; 5821 int status = 0; 5822 5823 status = pci_enable_device(pdev); 5824 if (status) 5825 goto do_none; 5826 5827 status = pci_request_regions(pdev, DRV_NAME); 5828 if (status) 5829 goto disable_dev; 5830 pci_set_master(pdev); 5831 5832 netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS); 5833 if (!netdev) { 5834 status = -ENOMEM; 5835 goto rel_reg; 5836 } 5837 adapter = netdev_priv(netdev); 5838 adapter->pdev = pdev; 5839 pci_set_drvdata(pdev, adapter); 5840 adapter->netdev = netdev; 5841 SET_NETDEV_DEV(netdev, &pdev->dev); 5842 5843 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 5844 if (status) { 5845 dev_err(&pdev->dev, "Could not set PCI DMA Mask\n"); 5846 goto free_netdev; 5847 } 5848 5849 status = pci_enable_pcie_error_reporting(pdev); 5850 if (!status) 5851 dev_info(&pdev->dev, "PCIe error reporting enabled\n"); 5852 5853 status = be_map_pci_bars(adapter); 5854 if (status) 5855 goto free_netdev; 5856 5857 status = be_drv_init(adapter); 5858 if (status) 5859 goto unmap_bars; 5860 5861 status = be_setup(adapter); 5862 if (status) 5863 goto drv_cleanup; 5864 5865 be_netdev_init(netdev); 5866 status = register_netdev(netdev); 5867 if (status != 0) 5868 goto unsetup; 5869 5870 be_roce_dev_add(adapter); 5871 5872 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 5873 adapter->error_recovery.probe_time = jiffies; 5874 5875 /* On Die temperature not supported for VF. */ 5876 if (be_physfn(adapter) && IS_ENABLED(CONFIG_BE2NET_HWMON)) { 5877 adapter->hwmon_info.hwmon_dev = 5878 devm_hwmon_device_register_with_groups(&pdev->dev, 5879 DRV_NAME, 5880 adapter, 5881 be_hwmon_groups); 5882 adapter->hwmon_info.be_on_die_temp = BE_INVALID_DIE_TEMP; 5883 } 5884 5885 dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev), 5886 func_name(adapter), mc_name(adapter), adapter->port_name); 5887 5888 return 0; 5889 5890 unsetup: 5891 be_clear(adapter); 5892 drv_cleanup: 5893 be_drv_cleanup(adapter); 5894 unmap_bars: 5895 be_unmap_pci_bars(adapter); 5896 free_netdev: 5897 pci_disable_pcie_error_reporting(pdev); 5898 free_netdev(netdev); 5899 rel_reg: 5900 pci_release_regions(pdev); 5901 disable_dev: 5902 pci_disable_device(pdev); 5903 do_none: 5904 dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev)); 5905 return status; 5906 } 5907 5908 static int __maybe_unused be_suspend(struct device *dev_d) 5909 { 5910 struct be_adapter *adapter = dev_get_drvdata(dev_d); 5911 5912 be_intr_set(adapter, false); 5913 be_cancel_err_detection(adapter); 5914 5915 be_cleanup(adapter); 5916 5917 return 0; 5918 } 5919 5920 static int __maybe_unused be_pci_resume(struct device *dev_d) 5921 { 5922 struct be_adapter *adapter = dev_get_drvdata(dev_d); 5923 int status = 0; 5924 5925 status = be_resume(adapter); 5926 if (status) 5927 return status; 5928 5929 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 5930 5931 return 0; 5932 } 5933 5934 /* 5935 * An FLR will stop BE from DMAing any data. 5936 */ 5937 static void be_shutdown(struct pci_dev *pdev) 5938 { 5939 struct be_adapter *adapter = pci_get_drvdata(pdev); 5940 5941 if (!adapter) 5942 return; 5943 5944 be_roce_dev_shutdown(adapter); 5945 cancel_delayed_work_sync(&adapter->work); 5946 be_cancel_err_detection(adapter); 5947 5948 netif_device_detach(adapter->netdev); 5949 5950 be_cmd_reset_function(adapter); 5951 5952 pci_disable_device(pdev); 5953 } 5954 5955 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev, 5956 pci_channel_state_t state) 5957 { 5958 struct be_adapter *adapter = pci_get_drvdata(pdev); 5959 5960 dev_err(&adapter->pdev->dev, "EEH error detected\n"); 5961 5962 be_roce_dev_remove(adapter); 5963 5964 if (!be_check_error(adapter, BE_ERROR_EEH)) { 5965 be_set_error(adapter, BE_ERROR_EEH); 5966 5967 be_cancel_err_detection(adapter); 5968 5969 be_cleanup(adapter); 5970 } 5971 5972 if (state == pci_channel_io_perm_failure) 5973 return PCI_ERS_RESULT_DISCONNECT; 5974 5975 pci_disable_device(pdev); 5976 5977 /* The error could cause the FW to trigger a flash debug dump. 5978 * Resetting the card while flash dump is in progress 5979 * can cause it not to recover; wait for it to finish. 5980 * Wait only for first function as it is needed only once per 5981 * adapter. 5982 */ 5983 if (pdev->devfn == 0) 5984 ssleep(30); 5985 5986 return PCI_ERS_RESULT_NEED_RESET; 5987 } 5988 5989 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev) 5990 { 5991 struct be_adapter *adapter = pci_get_drvdata(pdev); 5992 int status; 5993 5994 dev_info(&adapter->pdev->dev, "EEH reset\n"); 5995 5996 status = pci_enable_device(pdev); 5997 if (status) 5998 return PCI_ERS_RESULT_DISCONNECT; 5999 6000 pci_set_master(pdev); 6001 pci_restore_state(pdev); 6002 6003 /* Check if card is ok and fw is ready */ 6004 dev_info(&adapter->pdev->dev, 6005 "Waiting for FW to be ready after EEH reset\n"); 6006 status = be_fw_wait_ready(adapter); 6007 if (status) 6008 return PCI_ERS_RESULT_DISCONNECT; 6009 6010 be_clear_error(adapter, BE_CLEAR_ALL); 6011 return PCI_ERS_RESULT_RECOVERED; 6012 } 6013 6014 static void be_eeh_resume(struct pci_dev *pdev) 6015 { 6016 int status = 0; 6017 struct be_adapter *adapter = pci_get_drvdata(pdev); 6018 6019 dev_info(&adapter->pdev->dev, "EEH resume\n"); 6020 6021 pci_save_state(pdev); 6022 6023 status = be_resume(adapter); 6024 if (status) 6025 goto err; 6026 6027 be_roce_dev_add(adapter); 6028 6029 be_schedule_err_detection(adapter, ERR_DETECTION_DELAY); 6030 return; 6031 err: 6032 dev_err(&adapter->pdev->dev, "EEH resume failed\n"); 6033 } 6034 6035 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) 6036 { 6037 struct be_adapter *adapter = pci_get_drvdata(pdev); 6038 struct be_resources vft_res = {0}; 6039 int status; 6040 6041 if (!num_vfs) 6042 be_vf_clear(adapter); 6043 6044 adapter->num_vfs = num_vfs; 6045 6046 if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) { 6047 dev_warn(&pdev->dev, 6048 "Cannot disable VFs while they are assigned\n"); 6049 return -EBUSY; 6050 } 6051 6052 /* When the HW is in SRIOV capable configuration, the PF-pool resources 6053 * are equally distributed across the max-number of VFs. The user may 6054 * request only a subset of the max-vfs to be enabled. 6055 * Based on num_vfs, redistribute the resources across num_vfs so that 6056 * each VF will have access to more number of resources. 6057 * This facility is not available in BE3 FW. 6058 * Also, this is done by FW in Lancer chip. 6059 */ 6060 if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) { 6061 be_calculate_vf_res(adapter, adapter->num_vfs, 6062 &vft_res); 6063 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 6064 adapter->num_vfs, &vft_res); 6065 if (status) 6066 dev_err(&pdev->dev, 6067 "Failed to optimize SR-IOV resources\n"); 6068 } 6069 6070 status = be_get_resources(adapter); 6071 if (status) 6072 return be_cmd_status(status); 6073 6074 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 6075 rtnl_lock(); 6076 status = be_update_queues(adapter); 6077 rtnl_unlock(); 6078 if (status) 6079 return be_cmd_status(status); 6080 6081 if (adapter->num_vfs) 6082 status = be_vf_setup(adapter); 6083 6084 if (!status) 6085 return adapter->num_vfs; 6086 6087 return 0; 6088 } 6089 6090 static const struct pci_error_handlers be_eeh_handlers = { 6091 .error_detected = be_eeh_err_detected, 6092 .slot_reset = be_eeh_reset, 6093 .resume = be_eeh_resume, 6094 }; 6095 6096 static SIMPLE_DEV_PM_OPS(be_pci_pm_ops, be_suspend, be_pci_resume); 6097 6098 static struct pci_driver be_driver = { 6099 .name = DRV_NAME, 6100 .id_table = be_dev_ids, 6101 .probe = be_probe, 6102 .remove = be_remove, 6103 .driver.pm = &be_pci_pm_ops, 6104 .shutdown = be_shutdown, 6105 .sriov_configure = be_pci_sriov_configure, 6106 .err_handler = &be_eeh_handlers 6107 }; 6108 6109 static int __init be_init_module(void) 6110 { 6111 int status; 6112 6113 if (rx_frag_size != 8192 && rx_frag_size != 4096 && 6114 rx_frag_size != 2048) { 6115 printk(KERN_WARNING DRV_NAME 6116 " : Module param rx_frag_size must be 2048/4096/8192." 6117 " Using 2048\n"); 6118 rx_frag_size = 2048; 6119 } 6120 6121 if (num_vfs > 0) { 6122 pr_info(DRV_NAME " : Module param num_vfs is obsolete."); 6123 pr_info(DRV_NAME " : Use sysfs method to enable VFs\n"); 6124 } 6125 6126 be_wq = create_singlethread_workqueue("be_wq"); 6127 if (!be_wq) { 6128 pr_warn(DRV_NAME "workqueue creation failed\n"); 6129 return -1; 6130 } 6131 6132 be_err_recovery_workq = 6133 create_singlethread_workqueue("be_err_recover"); 6134 if (!be_err_recovery_workq) 6135 pr_warn(DRV_NAME "Could not create error recovery workqueue\n"); 6136 6137 status = pci_register_driver(&be_driver); 6138 if (status) { 6139 destroy_workqueue(be_wq); 6140 be_destroy_err_recovery_workq(); 6141 } 6142 return status; 6143 } 6144 module_init(be_init_module); 6145 6146 static void __exit be_exit_module(void) 6147 { 6148 pci_unregister_driver(&be_driver); 6149 6150 be_destroy_err_recovery_workq(); 6151 6152 if (be_wq) 6153 destroy_workqueue(be_wq); 6154 } 6155 module_exit(be_exit_module); 6156