1 /* 2 * Copyright (C) 2005 - 2014 Emulex 3 * All rights reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License version 2 7 * as published by the Free Software Foundation. The full GNU General 8 * Public License is included in this distribution in the file called COPYING. 9 * 10 * Contact Information: 11 * linux-drivers@emulex.com 12 * 13 * Emulex 14 * 3333 Susan Street 15 * Costa Mesa, CA 92626 16 */ 17 18 #include <linux/prefetch.h> 19 #include <linux/module.h> 20 #include "be.h" 21 #include "be_cmds.h" 22 #include <asm/div64.h> 23 #include <linux/aer.h> 24 #include <linux/if_bridge.h> 25 #include <net/busy_poll.h> 26 #include <net/vxlan.h> 27 28 MODULE_VERSION(DRV_VER); 29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER); 30 MODULE_AUTHOR("Emulex Corporation"); 31 MODULE_LICENSE("GPL"); 32 33 /* num_vfs module param is obsolete. 34 * Use sysfs method to enable/disable VFs. 35 */ 36 static unsigned int num_vfs; 37 module_param(num_vfs, uint, S_IRUGO); 38 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize"); 39 40 static ushort rx_frag_size = 2048; 41 module_param(rx_frag_size, ushort, S_IRUGO); 42 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data."); 43 44 static const struct pci_device_id be_dev_ids[] = { 45 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 46 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 47 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 48 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 49 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)}, 50 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)}, 51 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)}, 52 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)}, 53 { 0 } 54 }; 55 MODULE_DEVICE_TABLE(pci, be_dev_ids); 56 /* UE Status Low CSR */ 57 static const char * const ue_status_low_desc[] = { 58 "CEV", 59 "CTX", 60 "DBUF", 61 "ERX", 62 "Host", 63 "MPU", 64 "NDMA", 65 "PTC ", 66 "RDMA ", 67 "RXF ", 68 "RXIPS ", 69 "RXULP0 ", 70 "RXULP1 ", 71 "RXULP2 ", 72 "TIM ", 73 "TPOST ", 74 "TPRE ", 75 "TXIPS ", 76 "TXULP0 ", 77 "TXULP1 ", 78 "UC ", 79 "WDMA ", 80 "TXULP2 ", 81 "HOST1 ", 82 "P0_OB_LINK ", 83 "P1_OB_LINK ", 84 "HOST_GPIO ", 85 "MBOX ", 86 "ERX2 ", 87 "SPARE ", 88 "JTAG ", 89 "MPU_INTPEND " 90 }; 91 92 /* UE Status High CSR */ 93 static const char * const ue_status_hi_desc[] = { 94 "LPCMEMHOST", 95 "MGMT_MAC", 96 "PCS0ONLINE", 97 "MPU_IRAM", 98 "PCS1ONLINE", 99 "PCTL0", 100 "PCTL1", 101 "PMEM", 102 "RR", 103 "TXPB", 104 "RXPP", 105 "XAUI", 106 "TXP", 107 "ARM", 108 "IPC", 109 "HOST2", 110 "HOST3", 111 "HOST4", 112 "HOST5", 113 "HOST6", 114 "HOST7", 115 "ECRC", 116 "Poison TLP", 117 "NETC", 118 "PERIPH", 119 "LLTXULP", 120 "D2P", 121 "RCON", 122 "LDMA", 123 "LLTXP", 124 "LLTXPB", 125 "Unknown" 126 }; 127 128 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q) 129 { 130 struct be_dma_mem *mem = &q->dma_mem; 131 132 if (mem->va) { 133 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 134 mem->dma); 135 mem->va = NULL; 136 } 137 } 138 139 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q, 140 u16 len, u16 entry_size) 141 { 142 struct be_dma_mem *mem = &q->dma_mem; 143 144 memset(q, 0, sizeof(*q)); 145 q->len = len; 146 q->entry_size = entry_size; 147 mem->size = len * entry_size; 148 mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma, 149 GFP_KERNEL); 150 if (!mem->va) 151 return -ENOMEM; 152 return 0; 153 } 154 155 static void be_reg_intr_set(struct be_adapter *adapter, bool enable) 156 { 157 u32 reg, enabled; 158 159 pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, 160 ®); 161 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 162 163 if (!enabled && enable) 164 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 165 else if (enabled && !enable) 166 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 167 else 168 return; 169 170 pci_write_config_dword(adapter->pdev, 171 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg); 172 } 173 174 static void be_intr_set(struct be_adapter *adapter, bool enable) 175 { 176 int status = 0; 177 178 /* On lancer interrupts can't be controlled via this register */ 179 if (lancer_chip(adapter)) 180 return; 181 182 if (adapter->eeh_error) 183 return; 184 185 status = be_cmd_intr_set(adapter, enable); 186 if (status) 187 be_reg_intr_set(adapter, enable); 188 } 189 190 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted) 191 { 192 u32 val = 0; 193 194 val |= qid & DB_RQ_RING_ID_MASK; 195 val |= posted << DB_RQ_NUM_POSTED_SHIFT; 196 197 wmb(); 198 iowrite32(val, adapter->db + DB_RQ_OFFSET); 199 } 200 201 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo, 202 u16 posted) 203 { 204 u32 val = 0; 205 206 val |= txo->q.id & DB_TXULP_RING_ID_MASK; 207 val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT; 208 209 wmb(); 210 iowrite32(val, adapter->db + txo->db_offset); 211 } 212 213 static void be_eq_notify(struct be_adapter *adapter, u16 qid, 214 bool arm, bool clear_int, u16 num_popped) 215 { 216 u32 val = 0; 217 218 val |= qid & DB_EQ_RING_ID_MASK; 219 val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT); 220 221 if (adapter->eeh_error) 222 return; 223 224 if (arm) 225 val |= 1 << DB_EQ_REARM_SHIFT; 226 if (clear_int) 227 val |= 1 << DB_EQ_CLR_SHIFT; 228 val |= 1 << DB_EQ_EVNT_SHIFT; 229 val |= num_popped << DB_EQ_NUM_POPPED_SHIFT; 230 iowrite32(val, adapter->db + DB_EQ_OFFSET); 231 } 232 233 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped) 234 { 235 u32 val = 0; 236 237 val |= qid & DB_CQ_RING_ID_MASK; 238 val |= ((qid & DB_CQ_RING_ID_EXT_MASK) << 239 DB_CQ_RING_ID_EXT_MASK_SHIFT); 240 241 if (adapter->eeh_error) 242 return; 243 244 if (arm) 245 val |= 1 << DB_CQ_REARM_SHIFT; 246 val |= num_popped << DB_CQ_NUM_POPPED_SHIFT; 247 iowrite32(val, adapter->db + DB_CQ_OFFSET); 248 } 249 250 static int be_mac_addr_set(struct net_device *netdev, void *p) 251 { 252 struct be_adapter *adapter = netdev_priv(netdev); 253 struct device *dev = &adapter->pdev->dev; 254 struct sockaddr *addr = p; 255 int status; 256 u8 mac[ETH_ALEN]; 257 u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0; 258 259 if (!is_valid_ether_addr(addr->sa_data)) 260 return -EADDRNOTAVAIL; 261 262 /* Proceed further only if, User provided MAC is different 263 * from active MAC 264 */ 265 if (ether_addr_equal(addr->sa_data, netdev->dev_addr)) 266 return 0; 267 268 /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT 269 * privilege or if PF did not provision the new MAC address. 270 * On BE3, this cmd will always fail if the VF doesn't have the 271 * FILTMGMT privilege. This failure is OK, only if the PF programmed 272 * the MAC for the VF. 273 */ 274 status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data, 275 adapter->if_handle, &adapter->pmac_id[0], 0); 276 if (!status) { 277 curr_pmac_id = adapter->pmac_id[0]; 278 279 /* Delete the old programmed MAC. This call may fail if the 280 * old MAC was already deleted by the PF driver. 281 */ 282 if (adapter->pmac_id[0] != old_pmac_id) 283 be_cmd_pmac_del(adapter, adapter->if_handle, 284 old_pmac_id, 0); 285 } 286 287 /* Decide if the new MAC is successfully activated only after 288 * querying the FW 289 */ 290 status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac, 291 adapter->if_handle, true, 0); 292 if (status) 293 goto err; 294 295 /* The MAC change did not happen, either due to lack of privilege 296 * or PF didn't pre-provision. 297 */ 298 if (!ether_addr_equal(addr->sa_data, mac)) { 299 status = -EPERM; 300 goto err; 301 } 302 303 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 304 dev_info(dev, "MAC address changed to %pM\n", mac); 305 return 0; 306 err: 307 dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data); 308 return status; 309 } 310 311 /* BE2 supports only v0 cmd */ 312 static void *hw_stats_from_cmd(struct be_adapter *adapter) 313 { 314 if (BE2_chip(adapter)) { 315 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va; 316 317 return &cmd->hw_stats; 318 } else if (BE3_chip(adapter)) { 319 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va; 320 321 return &cmd->hw_stats; 322 } else { 323 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va; 324 325 return &cmd->hw_stats; 326 } 327 } 328 329 /* BE2 supports only v0 cmd */ 330 static void *be_erx_stats_from_cmd(struct be_adapter *adapter) 331 { 332 if (BE2_chip(adapter)) { 333 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 334 335 return &hw_stats->erx; 336 } else if (BE3_chip(adapter)) { 337 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 338 339 return &hw_stats->erx; 340 } else { 341 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 342 343 return &hw_stats->erx; 344 } 345 } 346 347 static void populate_be_v0_stats(struct be_adapter *adapter) 348 { 349 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 350 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 351 struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf; 352 struct be_port_rxf_stats_v0 *port_stats = 353 &rxf_stats->port[adapter->port_num]; 354 struct be_drv_stats *drvs = &adapter->drv_stats; 355 356 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 357 drvs->rx_pause_frames = port_stats->rx_pause_frames; 358 drvs->rx_crc_errors = port_stats->rx_crc_errors; 359 drvs->rx_control_frames = port_stats->rx_control_frames; 360 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 361 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 362 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 363 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 364 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 365 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 366 drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow; 367 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 368 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 369 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 370 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 371 drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow; 372 drvs->rx_dropped_header_too_small = 373 port_stats->rx_dropped_header_too_small; 374 drvs->rx_address_filtered = 375 port_stats->rx_address_filtered + 376 port_stats->rx_vlan_filtered; 377 drvs->rx_alignment_symbol_errors = 378 port_stats->rx_alignment_symbol_errors; 379 380 drvs->tx_pauseframes = port_stats->tx_pauseframes; 381 drvs->tx_controlframes = port_stats->tx_controlframes; 382 383 if (adapter->port_num) 384 drvs->jabber_events = rxf_stats->port1_jabber_events; 385 else 386 drvs->jabber_events = rxf_stats->port0_jabber_events; 387 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 388 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 389 drvs->forwarded_packets = rxf_stats->forwarded_packets; 390 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 391 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 392 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 393 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 394 } 395 396 static void populate_be_v1_stats(struct be_adapter *adapter) 397 { 398 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 399 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 400 struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf; 401 struct be_port_rxf_stats_v1 *port_stats = 402 &rxf_stats->port[adapter->port_num]; 403 struct be_drv_stats *drvs = &adapter->drv_stats; 404 405 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 406 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 407 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 408 drvs->rx_pause_frames = port_stats->rx_pause_frames; 409 drvs->rx_crc_errors = port_stats->rx_crc_errors; 410 drvs->rx_control_frames = port_stats->rx_control_frames; 411 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 412 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 413 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 414 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 415 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 416 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 417 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 418 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 419 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 420 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 421 drvs->rx_dropped_header_too_small = 422 port_stats->rx_dropped_header_too_small; 423 drvs->rx_input_fifo_overflow_drop = 424 port_stats->rx_input_fifo_overflow_drop; 425 drvs->rx_address_filtered = port_stats->rx_address_filtered; 426 drvs->rx_alignment_symbol_errors = 427 port_stats->rx_alignment_symbol_errors; 428 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 429 drvs->tx_pauseframes = port_stats->tx_pauseframes; 430 drvs->tx_controlframes = port_stats->tx_controlframes; 431 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 432 drvs->jabber_events = port_stats->jabber_events; 433 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 434 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 435 drvs->forwarded_packets = rxf_stats->forwarded_packets; 436 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 437 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 438 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 439 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 440 } 441 442 static void populate_be_v2_stats(struct be_adapter *adapter) 443 { 444 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 445 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 446 struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf; 447 struct be_port_rxf_stats_v2 *port_stats = 448 &rxf_stats->port[adapter->port_num]; 449 struct be_drv_stats *drvs = &adapter->drv_stats; 450 451 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 452 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 453 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 454 drvs->rx_pause_frames = port_stats->rx_pause_frames; 455 drvs->rx_crc_errors = port_stats->rx_crc_errors; 456 drvs->rx_control_frames = port_stats->rx_control_frames; 457 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 458 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 459 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 460 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 461 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 462 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 463 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 464 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 465 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 466 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 467 drvs->rx_dropped_header_too_small = 468 port_stats->rx_dropped_header_too_small; 469 drvs->rx_input_fifo_overflow_drop = 470 port_stats->rx_input_fifo_overflow_drop; 471 drvs->rx_address_filtered = port_stats->rx_address_filtered; 472 drvs->rx_alignment_symbol_errors = 473 port_stats->rx_alignment_symbol_errors; 474 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 475 drvs->tx_pauseframes = port_stats->tx_pauseframes; 476 drvs->tx_controlframes = port_stats->tx_controlframes; 477 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 478 drvs->jabber_events = port_stats->jabber_events; 479 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 480 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 481 drvs->forwarded_packets = rxf_stats->forwarded_packets; 482 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 483 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 484 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 485 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 486 if (be_roce_supported(adapter)) { 487 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd; 488 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd; 489 drvs->rx_roce_frames = port_stats->roce_frames_received; 490 drvs->roce_drops_crc = port_stats->roce_drops_crc; 491 drvs->roce_drops_payload_len = 492 port_stats->roce_drops_payload_len; 493 } 494 } 495 496 static void populate_lancer_stats(struct be_adapter *adapter) 497 { 498 struct be_drv_stats *drvs = &adapter->drv_stats; 499 struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter); 500 501 be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats)); 502 drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo; 503 drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo; 504 drvs->rx_control_frames = pport_stats->rx_control_frames_lo; 505 drvs->rx_in_range_errors = pport_stats->rx_in_range_errors; 506 drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo; 507 drvs->rx_dropped_runt = pport_stats->rx_dropped_runt; 508 drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors; 509 drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors; 510 drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors; 511 drvs->rx_dropped_tcp_length = 512 pport_stats->rx_dropped_invalid_tcp_length; 513 drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small; 514 drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short; 515 drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors; 516 drvs->rx_dropped_header_too_small = 517 pport_stats->rx_dropped_header_too_small; 518 drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 519 drvs->rx_address_filtered = 520 pport_stats->rx_address_filtered + 521 pport_stats->rx_vlan_filtered; 522 drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo; 523 drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 524 drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo; 525 drvs->tx_controlframes = pport_stats->tx_control_frames_lo; 526 drvs->jabber_events = pport_stats->rx_jabbers; 527 drvs->forwarded_packets = pport_stats->num_forwards_lo; 528 drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo; 529 drvs->rx_drops_too_many_frags = 530 pport_stats->rx_drops_too_many_frags_lo; 531 } 532 533 static void accumulate_16bit_val(u32 *acc, u16 val) 534 { 535 #define lo(x) (x & 0xFFFF) 536 #define hi(x) (x & 0xFFFF0000) 537 bool wrapped = val < lo(*acc); 538 u32 newacc = hi(*acc) + val; 539 540 if (wrapped) 541 newacc += 65536; 542 ACCESS_ONCE(*acc) = newacc; 543 } 544 545 static void populate_erx_stats(struct be_adapter *adapter, 546 struct be_rx_obj *rxo, u32 erx_stat) 547 { 548 if (!BEx_chip(adapter)) 549 rx_stats(rxo)->rx_drops_no_frags = erx_stat; 550 else 551 /* below erx HW counter can actually wrap around after 552 * 65535. Driver accumulates a 32-bit value 553 */ 554 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags, 555 (u16)erx_stat); 556 } 557 558 void be_parse_stats(struct be_adapter *adapter) 559 { 560 struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter); 561 struct be_rx_obj *rxo; 562 int i; 563 u32 erx_stat; 564 565 if (lancer_chip(adapter)) { 566 populate_lancer_stats(adapter); 567 } else { 568 if (BE2_chip(adapter)) 569 populate_be_v0_stats(adapter); 570 else if (BE3_chip(adapter)) 571 /* for BE3 */ 572 populate_be_v1_stats(adapter); 573 else 574 populate_be_v2_stats(adapter); 575 576 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */ 577 for_all_rx_queues(adapter, rxo, i) { 578 erx_stat = erx->rx_drops_no_fragments[rxo->q.id]; 579 populate_erx_stats(adapter, rxo, erx_stat); 580 } 581 } 582 } 583 584 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev, 585 struct rtnl_link_stats64 *stats) 586 { 587 struct be_adapter *adapter = netdev_priv(netdev); 588 struct be_drv_stats *drvs = &adapter->drv_stats; 589 struct be_rx_obj *rxo; 590 struct be_tx_obj *txo; 591 u64 pkts, bytes; 592 unsigned int start; 593 int i; 594 595 for_all_rx_queues(adapter, rxo, i) { 596 const struct be_rx_stats *rx_stats = rx_stats(rxo); 597 598 do { 599 start = u64_stats_fetch_begin_irq(&rx_stats->sync); 600 pkts = rx_stats(rxo)->rx_pkts; 601 bytes = rx_stats(rxo)->rx_bytes; 602 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start)); 603 stats->rx_packets += pkts; 604 stats->rx_bytes += bytes; 605 stats->multicast += rx_stats(rxo)->rx_mcast_pkts; 606 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs + 607 rx_stats(rxo)->rx_drops_no_frags; 608 } 609 610 for_all_tx_queues(adapter, txo, i) { 611 const struct be_tx_stats *tx_stats = tx_stats(txo); 612 613 do { 614 start = u64_stats_fetch_begin_irq(&tx_stats->sync); 615 pkts = tx_stats(txo)->tx_pkts; 616 bytes = tx_stats(txo)->tx_bytes; 617 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start)); 618 stats->tx_packets += pkts; 619 stats->tx_bytes += bytes; 620 } 621 622 /* bad pkts received */ 623 stats->rx_errors = drvs->rx_crc_errors + 624 drvs->rx_alignment_symbol_errors + 625 drvs->rx_in_range_errors + 626 drvs->rx_out_range_errors + 627 drvs->rx_frame_too_long + 628 drvs->rx_dropped_too_small + 629 drvs->rx_dropped_too_short + 630 drvs->rx_dropped_header_too_small + 631 drvs->rx_dropped_tcp_length + 632 drvs->rx_dropped_runt; 633 634 /* detailed rx errors */ 635 stats->rx_length_errors = drvs->rx_in_range_errors + 636 drvs->rx_out_range_errors + 637 drvs->rx_frame_too_long; 638 639 stats->rx_crc_errors = drvs->rx_crc_errors; 640 641 /* frame alignment errors */ 642 stats->rx_frame_errors = drvs->rx_alignment_symbol_errors; 643 644 /* receiver fifo overrun */ 645 /* drops_no_pbuf is no per i/f, it's per BE card */ 646 stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop + 647 drvs->rx_input_fifo_overflow_drop + 648 drvs->rx_drops_no_pbuf; 649 return stats; 650 } 651 652 void be_link_status_update(struct be_adapter *adapter, u8 link_status) 653 { 654 struct net_device *netdev = adapter->netdev; 655 656 if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) { 657 netif_carrier_off(netdev); 658 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT; 659 } 660 661 if (link_status) 662 netif_carrier_on(netdev); 663 else 664 netif_carrier_off(netdev); 665 } 666 667 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb) 668 { 669 struct be_tx_stats *stats = tx_stats(txo); 670 671 u64_stats_update_begin(&stats->sync); 672 stats->tx_reqs++; 673 stats->tx_bytes += skb->len; 674 stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1); 675 u64_stats_update_end(&stats->sync); 676 } 677 678 /* Returns number of WRBs needed for the skb */ 679 static u32 skb_wrb_cnt(struct sk_buff *skb) 680 { 681 /* +1 for the header wrb */ 682 return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags; 683 } 684 685 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len) 686 { 687 wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr)); 688 wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr)); 689 wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK); 690 wrb->rsvd0 = 0; 691 } 692 693 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb 694 * to avoid the swap and shift/mask operations in wrb_fill(). 695 */ 696 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb) 697 { 698 wrb->frag_pa_hi = 0; 699 wrb->frag_pa_lo = 0; 700 wrb->frag_len = 0; 701 wrb->rsvd0 = 0; 702 } 703 704 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter, 705 struct sk_buff *skb) 706 { 707 u8 vlan_prio; 708 u16 vlan_tag; 709 710 vlan_tag = skb_vlan_tag_get(skb); 711 vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 712 /* If vlan priority provided by OS is NOT in available bmap */ 713 if (!(adapter->vlan_prio_bmap & (1 << vlan_prio))) 714 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) | 715 adapter->recommended_prio; 716 717 return vlan_tag; 718 } 719 720 /* Used only for IP tunnel packets */ 721 static u16 skb_inner_ip_proto(struct sk_buff *skb) 722 { 723 return (inner_ip_hdr(skb)->version == 4) ? 724 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr; 725 } 726 727 static u16 skb_ip_proto(struct sk_buff *skb) 728 { 729 return (ip_hdr(skb)->version == 4) ? 730 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr; 731 } 732 733 static inline bool be_is_txq_full(struct be_tx_obj *txo) 734 { 735 return atomic_read(&txo->q.used) + BE_MAX_TX_FRAG_COUNT >= txo->q.len; 736 } 737 738 static inline bool be_can_txq_wake(struct be_tx_obj *txo) 739 { 740 return atomic_read(&txo->q.used) < txo->q.len / 2; 741 } 742 743 static inline bool be_is_tx_compl_pending(struct be_tx_obj *txo) 744 { 745 return atomic_read(&txo->q.used) > txo->pend_wrb_cnt; 746 } 747 748 static void be_get_wrb_params_from_skb(struct be_adapter *adapter, 749 struct sk_buff *skb, 750 struct be_wrb_params *wrb_params) 751 { 752 u16 proto; 753 754 if (skb_is_gso(skb)) { 755 BE_WRB_F_SET(wrb_params->features, LSO, 1); 756 wrb_params->lso_mss = skb_shinfo(skb)->gso_size; 757 if (skb_is_gso_v6(skb) && !lancer_chip(adapter)) 758 BE_WRB_F_SET(wrb_params->features, LSO6, 1); 759 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 760 if (skb->encapsulation) { 761 BE_WRB_F_SET(wrb_params->features, IPCS, 1); 762 proto = skb_inner_ip_proto(skb); 763 } else { 764 proto = skb_ip_proto(skb); 765 } 766 if (proto == IPPROTO_TCP) 767 BE_WRB_F_SET(wrb_params->features, TCPCS, 1); 768 else if (proto == IPPROTO_UDP) 769 BE_WRB_F_SET(wrb_params->features, UDPCS, 1); 770 } 771 772 if (skb_vlan_tag_present(skb)) { 773 BE_WRB_F_SET(wrb_params->features, VLAN, 1); 774 wrb_params->vlan_tag = be_get_tx_vlan_tag(adapter, skb); 775 } 776 777 BE_WRB_F_SET(wrb_params->features, CRC, 1); 778 } 779 780 static void wrb_fill_hdr(struct be_adapter *adapter, 781 struct be_eth_hdr_wrb *hdr, 782 struct be_wrb_params *wrb_params, 783 struct sk_buff *skb) 784 { 785 memset(hdr, 0, sizeof(*hdr)); 786 787 SET_TX_WRB_HDR_BITS(crc, hdr, 788 BE_WRB_F_GET(wrb_params->features, CRC)); 789 SET_TX_WRB_HDR_BITS(ipcs, hdr, 790 BE_WRB_F_GET(wrb_params->features, IPCS)); 791 SET_TX_WRB_HDR_BITS(tcpcs, hdr, 792 BE_WRB_F_GET(wrb_params->features, TCPCS)); 793 SET_TX_WRB_HDR_BITS(udpcs, hdr, 794 BE_WRB_F_GET(wrb_params->features, UDPCS)); 795 796 SET_TX_WRB_HDR_BITS(lso, hdr, 797 BE_WRB_F_GET(wrb_params->features, LSO)); 798 SET_TX_WRB_HDR_BITS(lso6, hdr, 799 BE_WRB_F_GET(wrb_params->features, LSO6)); 800 SET_TX_WRB_HDR_BITS(lso_mss, hdr, wrb_params->lso_mss); 801 802 /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0. When this 803 * hack is not needed, the evt bit is set while ringing DB. 804 */ 805 SET_TX_WRB_HDR_BITS(event, hdr, 806 BE_WRB_F_GET(wrb_params->features, VLAN_SKIP_HW)); 807 SET_TX_WRB_HDR_BITS(vlan, hdr, 808 BE_WRB_F_GET(wrb_params->features, VLAN)); 809 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, wrb_params->vlan_tag); 810 811 SET_TX_WRB_HDR_BITS(num_wrb, hdr, skb_wrb_cnt(skb)); 812 SET_TX_WRB_HDR_BITS(len, hdr, skb->len); 813 } 814 815 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb, 816 bool unmap_single) 817 { 818 dma_addr_t dma; 819 u32 frag_len = le32_to_cpu(wrb->frag_len); 820 821 822 dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 | 823 (u64)le32_to_cpu(wrb->frag_pa_lo); 824 if (frag_len) { 825 if (unmap_single) 826 dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE); 827 else 828 dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE); 829 } 830 } 831 832 /* Grab a WRB header for xmit */ 833 static u16 be_tx_get_wrb_hdr(struct be_tx_obj *txo) 834 { 835 u16 head = txo->q.head; 836 837 queue_head_inc(&txo->q); 838 return head; 839 } 840 841 /* Set up the WRB header for xmit */ 842 static void be_tx_setup_wrb_hdr(struct be_adapter *adapter, 843 struct be_tx_obj *txo, 844 struct be_wrb_params *wrb_params, 845 struct sk_buff *skb, u16 head) 846 { 847 u32 num_frags = skb_wrb_cnt(skb); 848 struct be_queue_info *txq = &txo->q; 849 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, head); 850 851 wrb_fill_hdr(adapter, hdr, wrb_params, skb); 852 be_dws_cpu_to_le(hdr, sizeof(*hdr)); 853 854 BUG_ON(txo->sent_skb_list[head]); 855 txo->sent_skb_list[head] = skb; 856 txo->last_req_hdr = head; 857 atomic_add(num_frags, &txq->used); 858 txo->last_req_wrb_cnt = num_frags; 859 txo->pend_wrb_cnt += num_frags; 860 } 861 862 /* Setup a WRB fragment (buffer descriptor) for xmit */ 863 static void be_tx_setup_wrb_frag(struct be_tx_obj *txo, dma_addr_t busaddr, 864 int len) 865 { 866 struct be_eth_wrb *wrb; 867 struct be_queue_info *txq = &txo->q; 868 869 wrb = queue_head_node(txq); 870 wrb_fill(wrb, busaddr, len); 871 queue_head_inc(txq); 872 } 873 874 /* Bring the queue back to the state it was in before be_xmit_enqueue() routine 875 * was invoked. The producer index is restored to the previous packet and the 876 * WRBs of the current packet are unmapped. Invoked to handle tx setup errors. 877 */ 878 static void be_xmit_restore(struct be_adapter *adapter, 879 struct be_tx_obj *txo, u16 head, bool map_single, 880 u32 copied) 881 { 882 struct device *dev; 883 struct be_eth_wrb *wrb; 884 struct be_queue_info *txq = &txo->q; 885 886 dev = &adapter->pdev->dev; 887 txq->head = head; 888 889 /* skip the first wrb (hdr); it's not mapped */ 890 queue_head_inc(txq); 891 while (copied) { 892 wrb = queue_head_node(txq); 893 unmap_tx_frag(dev, wrb, map_single); 894 map_single = false; 895 copied -= le32_to_cpu(wrb->frag_len); 896 queue_head_inc(txq); 897 } 898 899 txq->head = head; 900 } 901 902 /* Enqueue the given packet for transmit. This routine allocates WRBs for the 903 * packet, dma maps the packet buffers and sets up the WRBs. Returns the number 904 * of WRBs used up by the packet. 905 */ 906 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo, 907 struct sk_buff *skb, 908 struct be_wrb_params *wrb_params) 909 { 910 u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb); 911 struct device *dev = &adapter->pdev->dev; 912 struct be_queue_info *txq = &txo->q; 913 bool map_single = false; 914 u16 head = txq->head; 915 dma_addr_t busaddr; 916 int len; 917 918 head = be_tx_get_wrb_hdr(txo); 919 920 if (skb->len > skb->data_len) { 921 len = skb_headlen(skb); 922 923 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE); 924 if (dma_mapping_error(dev, busaddr)) 925 goto dma_err; 926 map_single = true; 927 be_tx_setup_wrb_frag(txo, busaddr, len); 928 copied += len; 929 } 930 931 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 932 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; 933 len = skb_frag_size(frag); 934 935 busaddr = skb_frag_dma_map(dev, frag, 0, len, DMA_TO_DEVICE); 936 if (dma_mapping_error(dev, busaddr)) 937 goto dma_err; 938 be_tx_setup_wrb_frag(txo, busaddr, len); 939 copied += len; 940 } 941 942 be_tx_setup_wrb_hdr(adapter, txo, wrb_params, skb, head); 943 944 be_tx_stats_update(txo, skb); 945 return wrb_cnt; 946 947 dma_err: 948 adapter->drv_stats.dma_map_errors++; 949 be_xmit_restore(adapter, txo, head, map_single, copied); 950 return 0; 951 } 952 953 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter) 954 { 955 return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD; 956 } 957 958 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter, 959 struct sk_buff *skb, 960 struct be_wrb_params 961 *wrb_params) 962 { 963 u16 vlan_tag = 0; 964 965 skb = skb_share_check(skb, GFP_ATOMIC); 966 if (unlikely(!skb)) 967 return skb; 968 969 if (skb_vlan_tag_present(skb)) 970 vlan_tag = be_get_tx_vlan_tag(adapter, skb); 971 972 if (qnq_async_evt_rcvd(adapter) && adapter->pvid) { 973 if (!vlan_tag) 974 vlan_tag = adapter->pvid; 975 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to 976 * skip VLAN insertion 977 */ 978 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 979 } 980 981 if (vlan_tag) { 982 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 983 vlan_tag); 984 if (unlikely(!skb)) 985 return skb; 986 skb->vlan_tci = 0; 987 } 988 989 /* Insert the outer VLAN, if any */ 990 if (adapter->qnq_vid) { 991 vlan_tag = adapter->qnq_vid; 992 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 993 vlan_tag); 994 if (unlikely(!skb)) 995 return skb; 996 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 997 } 998 999 return skb; 1000 } 1001 1002 static bool be_ipv6_exthdr_check(struct sk_buff *skb) 1003 { 1004 struct ethhdr *eh = (struct ethhdr *)skb->data; 1005 u16 offset = ETH_HLEN; 1006 1007 if (eh->h_proto == htons(ETH_P_IPV6)) { 1008 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset); 1009 1010 offset += sizeof(struct ipv6hdr); 1011 if (ip6h->nexthdr != NEXTHDR_TCP && 1012 ip6h->nexthdr != NEXTHDR_UDP) { 1013 struct ipv6_opt_hdr *ehdr = 1014 (struct ipv6_opt_hdr *)(skb->data + offset); 1015 1016 /* offending pkt: 2nd byte following IPv6 hdr is 0xff */ 1017 if (ehdr->hdrlen == 0xff) 1018 return true; 1019 } 1020 } 1021 return false; 1022 } 1023 1024 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb) 1025 { 1026 return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid; 1027 } 1028 1029 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb) 1030 { 1031 return BE3_chip(adapter) && be_ipv6_exthdr_check(skb); 1032 } 1033 1034 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter, 1035 struct sk_buff *skb, 1036 struct be_wrb_params 1037 *wrb_params) 1038 { 1039 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 1040 unsigned int eth_hdr_len; 1041 struct iphdr *ip; 1042 1043 /* For padded packets, BE HW modifies tot_len field in IP header 1044 * incorrecly when VLAN tag is inserted by HW. 1045 * For padded packets, Lancer computes incorrect checksum. 1046 */ 1047 eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ? 1048 VLAN_ETH_HLEN : ETH_HLEN; 1049 if (skb->len <= 60 && 1050 (lancer_chip(adapter) || skb_vlan_tag_present(skb)) && 1051 is_ipv4_pkt(skb)) { 1052 ip = (struct iphdr *)ip_hdr(skb); 1053 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len)); 1054 } 1055 1056 /* If vlan tag is already inlined in the packet, skip HW VLAN 1057 * tagging in pvid-tagging mode 1058 */ 1059 if (be_pvid_tagging_enabled(adapter) && 1060 veh->h_vlan_proto == htons(ETH_P_8021Q)) 1061 BE_WRB_F_SET(wrb_params->features, VLAN_SKIP_HW, 1); 1062 1063 /* HW has a bug wherein it will calculate CSUM for VLAN 1064 * pkts even though it is disabled. 1065 * Manually insert VLAN in pkt. 1066 */ 1067 if (skb->ip_summed != CHECKSUM_PARTIAL && 1068 skb_vlan_tag_present(skb)) { 1069 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1070 if (unlikely(!skb)) 1071 goto err; 1072 } 1073 1074 /* HW may lockup when VLAN HW tagging is requested on 1075 * certain ipv6 packets. Drop such pkts if the HW workaround to 1076 * skip HW tagging is not enabled by FW. 1077 */ 1078 if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) && 1079 (adapter->pvid || adapter->qnq_vid) && 1080 !qnq_async_evt_rcvd(adapter))) 1081 goto tx_drop; 1082 1083 /* Manual VLAN tag insertion to prevent: 1084 * ASIC lockup when the ASIC inserts VLAN tag into 1085 * certain ipv6 packets. Insert VLAN tags in driver, 1086 * and set event, completion, vlan bits accordingly 1087 * in the Tx WRB. 1088 */ 1089 if (be_ipv6_tx_stall_chk(adapter, skb) && 1090 be_vlan_tag_tx_chk(adapter, skb)) { 1091 skb = be_insert_vlan_in_pkt(adapter, skb, wrb_params); 1092 if (unlikely(!skb)) 1093 goto err; 1094 } 1095 1096 return skb; 1097 tx_drop: 1098 dev_kfree_skb_any(skb); 1099 err: 1100 return NULL; 1101 } 1102 1103 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter, 1104 struct sk_buff *skb, 1105 struct be_wrb_params *wrb_params) 1106 { 1107 /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or 1108 * less may cause a transmit stall on that port. So the work-around is 1109 * to pad short packets (<= 32 bytes) to a 36-byte length. 1110 */ 1111 if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) { 1112 if (skb_put_padto(skb, 36)) 1113 return NULL; 1114 } 1115 1116 if (BEx_chip(adapter) || lancer_chip(adapter)) { 1117 skb = be_lancer_xmit_workarounds(adapter, skb, wrb_params); 1118 if (!skb) 1119 return NULL; 1120 } 1121 1122 return skb; 1123 } 1124 1125 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo) 1126 { 1127 struct be_queue_info *txq = &txo->q; 1128 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr); 1129 1130 /* Mark the last request eventable if it hasn't been marked already */ 1131 if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT))) 1132 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL); 1133 1134 /* compose a dummy wrb if there are odd set of wrbs to notify */ 1135 if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) { 1136 wrb_fill_dummy(queue_head_node(txq)); 1137 queue_head_inc(txq); 1138 atomic_inc(&txq->used); 1139 txo->pend_wrb_cnt++; 1140 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK << 1141 TX_HDR_WRB_NUM_SHIFT); 1142 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) << 1143 TX_HDR_WRB_NUM_SHIFT); 1144 } 1145 be_txq_notify(adapter, txo, txo->pend_wrb_cnt); 1146 txo->pend_wrb_cnt = 0; 1147 } 1148 1149 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev) 1150 { 1151 struct be_adapter *adapter = netdev_priv(netdev); 1152 u16 q_idx = skb_get_queue_mapping(skb); 1153 struct be_tx_obj *txo = &adapter->tx_obj[q_idx]; 1154 struct be_wrb_params wrb_params = { 0 }; 1155 bool flush = !skb->xmit_more; 1156 u16 wrb_cnt; 1157 1158 skb = be_xmit_workarounds(adapter, skb, &wrb_params); 1159 if (unlikely(!skb)) 1160 goto drop; 1161 1162 be_get_wrb_params_from_skb(adapter, skb, &wrb_params); 1163 1164 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, &wrb_params); 1165 if (unlikely(!wrb_cnt)) { 1166 dev_kfree_skb_any(skb); 1167 goto drop; 1168 } 1169 1170 if (be_is_txq_full(txo)) { 1171 netif_stop_subqueue(netdev, q_idx); 1172 tx_stats(txo)->tx_stops++; 1173 } 1174 1175 if (flush || __netif_subqueue_stopped(netdev, q_idx)) 1176 be_xmit_flush(adapter, txo); 1177 1178 return NETDEV_TX_OK; 1179 drop: 1180 tx_stats(txo)->tx_drv_drops++; 1181 /* Flush the already enqueued tx requests */ 1182 if (flush && txo->pend_wrb_cnt) 1183 be_xmit_flush(adapter, txo); 1184 1185 return NETDEV_TX_OK; 1186 } 1187 1188 static int be_change_mtu(struct net_device *netdev, int new_mtu) 1189 { 1190 struct be_adapter *adapter = netdev_priv(netdev); 1191 struct device *dev = &adapter->pdev->dev; 1192 1193 if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) { 1194 dev_info(dev, "MTU must be between %d and %d bytes\n", 1195 BE_MIN_MTU, BE_MAX_MTU); 1196 return -EINVAL; 1197 } 1198 1199 dev_info(dev, "MTU changed from %d to %d bytes\n", 1200 netdev->mtu, new_mtu); 1201 netdev->mtu = new_mtu; 1202 return 0; 1203 } 1204 1205 static inline bool be_in_all_promisc(struct be_adapter *adapter) 1206 { 1207 return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) == 1208 BE_IF_FLAGS_ALL_PROMISCUOUS; 1209 } 1210 1211 static int be_set_vlan_promisc(struct be_adapter *adapter) 1212 { 1213 struct device *dev = &adapter->pdev->dev; 1214 int status; 1215 1216 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) 1217 return 0; 1218 1219 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON); 1220 if (!status) { 1221 dev_info(dev, "Enabled VLAN promiscuous mode\n"); 1222 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS; 1223 } else { 1224 dev_err(dev, "Failed to enable VLAN promiscuous mode\n"); 1225 } 1226 return status; 1227 } 1228 1229 static int be_clear_vlan_promisc(struct be_adapter *adapter) 1230 { 1231 struct device *dev = &adapter->pdev->dev; 1232 int status; 1233 1234 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF); 1235 if (!status) { 1236 dev_info(dev, "Disabling VLAN promiscuous mode\n"); 1237 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 1238 } 1239 return status; 1240 } 1241 1242 /* 1243 * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE. 1244 * If the user configures more, place BE in vlan promiscuous mode. 1245 */ 1246 static int be_vid_config(struct be_adapter *adapter) 1247 { 1248 struct device *dev = &adapter->pdev->dev; 1249 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1250 u16 num = 0, i = 0; 1251 int status = 0; 1252 1253 /* No need to further configure vids if in promiscuous mode */ 1254 if (be_in_all_promisc(adapter)) 1255 return 0; 1256 1257 if (adapter->vlans_added > be_max_vlans(adapter)) 1258 return be_set_vlan_promisc(adapter); 1259 1260 /* Construct VLAN Table to give to HW */ 1261 for_each_set_bit(i, adapter->vids, VLAN_N_VID) 1262 vids[num++] = cpu_to_le16(i); 1263 1264 status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num, 0); 1265 if (status) { 1266 dev_err(dev, "Setting HW VLAN filtering failed\n"); 1267 /* Set to VLAN promisc mode as setting VLAN filter failed */ 1268 if (addl_status(status) == 1269 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES) 1270 return be_set_vlan_promisc(adapter); 1271 } else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 1272 status = be_clear_vlan_promisc(adapter); 1273 } 1274 return status; 1275 } 1276 1277 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 1278 { 1279 struct be_adapter *adapter = netdev_priv(netdev); 1280 int status = 0; 1281 1282 /* Packets with VID 0 are always received by Lancer by default */ 1283 if (lancer_chip(adapter) && vid == 0) 1284 return status; 1285 1286 if (test_bit(vid, adapter->vids)) 1287 return status; 1288 1289 set_bit(vid, adapter->vids); 1290 adapter->vlans_added++; 1291 1292 status = be_vid_config(adapter); 1293 if (status) { 1294 adapter->vlans_added--; 1295 clear_bit(vid, adapter->vids); 1296 } 1297 1298 return status; 1299 } 1300 1301 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid) 1302 { 1303 struct be_adapter *adapter = netdev_priv(netdev); 1304 1305 /* Packets with VID 0 are always received by Lancer by default */ 1306 if (lancer_chip(adapter) && vid == 0) 1307 return 0; 1308 1309 clear_bit(vid, adapter->vids); 1310 adapter->vlans_added--; 1311 1312 return be_vid_config(adapter); 1313 } 1314 1315 static void be_clear_all_promisc(struct be_adapter *adapter) 1316 { 1317 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF); 1318 adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS; 1319 } 1320 1321 static void be_set_all_promisc(struct be_adapter *adapter) 1322 { 1323 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON); 1324 adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS; 1325 } 1326 1327 static void be_set_mc_promisc(struct be_adapter *adapter) 1328 { 1329 int status; 1330 1331 if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) 1332 return; 1333 1334 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON); 1335 if (!status) 1336 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS; 1337 } 1338 1339 static void be_set_mc_list(struct be_adapter *adapter) 1340 { 1341 int status; 1342 1343 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON); 1344 if (!status) 1345 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS; 1346 else 1347 be_set_mc_promisc(adapter); 1348 } 1349 1350 static void be_set_uc_list(struct be_adapter *adapter) 1351 { 1352 struct netdev_hw_addr *ha; 1353 int i = 1; /* First slot is claimed by the Primary MAC */ 1354 1355 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) 1356 be_cmd_pmac_del(adapter, adapter->if_handle, 1357 adapter->pmac_id[i], 0); 1358 1359 if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) { 1360 be_set_all_promisc(adapter); 1361 return; 1362 } 1363 1364 netdev_for_each_uc_addr(ha, adapter->netdev) { 1365 adapter->uc_macs++; /* First slot is for Primary MAC */ 1366 be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle, 1367 &adapter->pmac_id[adapter->uc_macs], 0); 1368 } 1369 } 1370 1371 static void be_clear_uc_list(struct be_adapter *adapter) 1372 { 1373 int i; 1374 1375 for (i = 1; i < (adapter->uc_macs + 1); i++) 1376 be_cmd_pmac_del(adapter, adapter->if_handle, 1377 adapter->pmac_id[i], 0); 1378 adapter->uc_macs = 0; 1379 } 1380 1381 static void be_set_rx_mode(struct net_device *netdev) 1382 { 1383 struct be_adapter *adapter = netdev_priv(netdev); 1384 1385 if (netdev->flags & IFF_PROMISC) { 1386 be_set_all_promisc(adapter); 1387 return; 1388 } 1389 1390 /* Interface was previously in promiscuous mode; disable it */ 1391 if (be_in_all_promisc(adapter)) { 1392 be_clear_all_promisc(adapter); 1393 if (adapter->vlans_added) 1394 be_vid_config(adapter); 1395 } 1396 1397 /* Enable multicast promisc if num configured exceeds what we support */ 1398 if (netdev->flags & IFF_ALLMULTI || 1399 netdev_mc_count(netdev) > be_max_mc(adapter)) { 1400 be_set_mc_promisc(adapter); 1401 return; 1402 } 1403 1404 if (netdev_uc_count(netdev) != adapter->uc_macs) 1405 be_set_uc_list(adapter); 1406 1407 be_set_mc_list(adapter); 1408 } 1409 1410 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 1411 { 1412 struct be_adapter *adapter = netdev_priv(netdev); 1413 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1414 int status; 1415 1416 if (!sriov_enabled(adapter)) 1417 return -EPERM; 1418 1419 if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs) 1420 return -EINVAL; 1421 1422 /* Proceed further only if user provided MAC is different 1423 * from active MAC 1424 */ 1425 if (ether_addr_equal(mac, vf_cfg->mac_addr)) 1426 return 0; 1427 1428 if (BEx_chip(adapter)) { 1429 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id, 1430 vf + 1); 1431 1432 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle, 1433 &vf_cfg->pmac_id, vf + 1); 1434 } else { 1435 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 1436 vf + 1); 1437 } 1438 1439 if (status) { 1440 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x", 1441 mac, vf, status); 1442 return be_cmd_status(status); 1443 } 1444 1445 ether_addr_copy(vf_cfg->mac_addr, mac); 1446 1447 return 0; 1448 } 1449 1450 static int be_get_vf_config(struct net_device *netdev, int vf, 1451 struct ifla_vf_info *vi) 1452 { 1453 struct be_adapter *adapter = netdev_priv(netdev); 1454 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1455 1456 if (!sriov_enabled(adapter)) 1457 return -EPERM; 1458 1459 if (vf >= adapter->num_vfs) 1460 return -EINVAL; 1461 1462 vi->vf = vf; 1463 vi->max_tx_rate = vf_cfg->tx_rate; 1464 vi->min_tx_rate = 0; 1465 vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK; 1466 vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT; 1467 memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN); 1468 vi->linkstate = adapter->vf_cfg[vf].plink_tracking; 1469 1470 return 0; 1471 } 1472 1473 static int be_set_vf_tvt(struct be_adapter *adapter, int vf, u16 vlan) 1474 { 1475 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1476 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1477 int vf_if_id = vf_cfg->if_handle; 1478 int status; 1479 1480 /* Enable Transparent VLAN Tagging */ 1481 status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, vf_if_id, 0); 1482 if (status) 1483 return status; 1484 1485 /* Clear pre-programmed VLAN filters on VF if any, if TVT is enabled */ 1486 vids[0] = 0; 1487 status = be_cmd_vlan_config(adapter, vf_if_id, vids, 1, vf + 1); 1488 if (!status) 1489 dev_info(&adapter->pdev->dev, 1490 "Cleared guest VLANs on VF%d", vf); 1491 1492 /* After TVT is enabled, disallow VFs to program VLAN filters */ 1493 if (vf_cfg->privileges & BE_PRIV_FILTMGMT) { 1494 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges & 1495 ~BE_PRIV_FILTMGMT, vf + 1); 1496 if (!status) 1497 vf_cfg->privileges &= ~BE_PRIV_FILTMGMT; 1498 } 1499 return 0; 1500 } 1501 1502 static int be_clear_vf_tvt(struct be_adapter *adapter, int vf) 1503 { 1504 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1505 struct device *dev = &adapter->pdev->dev; 1506 int status; 1507 1508 /* Reset Transparent VLAN Tagging. */ 1509 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, vf + 1, 1510 vf_cfg->if_handle, 0); 1511 if (status) 1512 return status; 1513 1514 /* Allow VFs to program VLAN filtering */ 1515 if (!(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 1516 status = be_cmd_set_fn_privileges(adapter, vf_cfg->privileges | 1517 BE_PRIV_FILTMGMT, vf + 1); 1518 if (!status) { 1519 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 1520 dev_info(dev, "VF%d: FILTMGMT priv enabled", vf); 1521 } 1522 } 1523 1524 dev_info(dev, 1525 "Disable/re-enable i/f in VM to clear Transparent VLAN tag"); 1526 return 0; 1527 } 1528 1529 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos) 1530 { 1531 struct be_adapter *adapter = netdev_priv(netdev); 1532 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1533 int status; 1534 1535 if (!sriov_enabled(adapter)) 1536 return -EPERM; 1537 1538 if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7) 1539 return -EINVAL; 1540 1541 if (vlan || qos) { 1542 vlan |= qos << VLAN_PRIO_SHIFT; 1543 status = be_set_vf_tvt(adapter, vf, vlan); 1544 } else { 1545 status = be_clear_vf_tvt(adapter, vf); 1546 } 1547 1548 if (status) { 1549 dev_err(&adapter->pdev->dev, 1550 "VLAN %d config on VF %d failed : %#x\n", vlan, vf, 1551 status); 1552 return be_cmd_status(status); 1553 } 1554 1555 vf_cfg->vlan_tag = vlan; 1556 return 0; 1557 } 1558 1559 static int be_set_vf_tx_rate(struct net_device *netdev, int vf, 1560 int min_tx_rate, int max_tx_rate) 1561 { 1562 struct be_adapter *adapter = netdev_priv(netdev); 1563 struct device *dev = &adapter->pdev->dev; 1564 int percent_rate, status = 0; 1565 u16 link_speed = 0; 1566 u8 link_status; 1567 1568 if (!sriov_enabled(adapter)) 1569 return -EPERM; 1570 1571 if (vf >= adapter->num_vfs) 1572 return -EINVAL; 1573 1574 if (min_tx_rate) 1575 return -EINVAL; 1576 1577 if (!max_tx_rate) 1578 goto config_qos; 1579 1580 status = be_cmd_link_status_query(adapter, &link_speed, 1581 &link_status, 0); 1582 if (status) 1583 goto err; 1584 1585 if (!link_status) { 1586 dev_err(dev, "TX-rate setting not allowed when link is down\n"); 1587 status = -ENETDOWN; 1588 goto err; 1589 } 1590 1591 if (max_tx_rate < 100 || max_tx_rate > link_speed) { 1592 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n", 1593 link_speed); 1594 status = -EINVAL; 1595 goto err; 1596 } 1597 1598 /* On Skyhawk the QOS setting must be done only as a % value */ 1599 percent_rate = link_speed / 100; 1600 if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) { 1601 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n", 1602 percent_rate); 1603 status = -EINVAL; 1604 goto err; 1605 } 1606 1607 config_qos: 1608 status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1); 1609 if (status) 1610 goto err; 1611 1612 adapter->vf_cfg[vf].tx_rate = max_tx_rate; 1613 return 0; 1614 1615 err: 1616 dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n", 1617 max_tx_rate, vf); 1618 return be_cmd_status(status); 1619 } 1620 1621 static int be_set_vf_link_state(struct net_device *netdev, int vf, 1622 int link_state) 1623 { 1624 struct be_adapter *adapter = netdev_priv(netdev); 1625 int status; 1626 1627 if (!sriov_enabled(adapter)) 1628 return -EPERM; 1629 1630 if (vf >= adapter->num_vfs) 1631 return -EINVAL; 1632 1633 status = be_cmd_set_logical_link_config(adapter, link_state, vf+1); 1634 if (status) { 1635 dev_err(&adapter->pdev->dev, 1636 "Link state change on VF %d failed: %#x\n", vf, status); 1637 return be_cmd_status(status); 1638 } 1639 1640 adapter->vf_cfg[vf].plink_tracking = link_state; 1641 1642 return 0; 1643 } 1644 1645 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts, 1646 ulong now) 1647 { 1648 aic->rx_pkts_prev = rx_pkts; 1649 aic->tx_reqs_prev = tx_pkts; 1650 aic->jiffies = now; 1651 } 1652 1653 static void be_eqd_update(struct be_adapter *adapter) 1654 { 1655 struct be_set_eqd set_eqd[MAX_EVT_QS]; 1656 int eqd, i, num = 0, start; 1657 struct be_aic_obj *aic; 1658 struct be_eq_obj *eqo; 1659 struct be_rx_obj *rxo; 1660 struct be_tx_obj *txo; 1661 u64 rx_pkts, tx_pkts; 1662 ulong now; 1663 u32 pps, delta; 1664 1665 for_all_evt_queues(adapter, eqo, i) { 1666 aic = &adapter->aic_obj[eqo->idx]; 1667 if (!aic->enable) { 1668 if (aic->jiffies) 1669 aic->jiffies = 0; 1670 eqd = aic->et_eqd; 1671 goto modify_eqd; 1672 } 1673 1674 rxo = &adapter->rx_obj[eqo->idx]; 1675 do { 1676 start = u64_stats_fetch_begin_irq(&rxo->stats.sync); 1677 rx_pkts = rxo->stats.rx_pkts; 1678 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start)); 1679 1680 txo = &adapter->tx_obj[eqo->idx]; 1681 do { 1682 start = u64_stats_fetch_begin_irq(&txo->stats.sync); 1683 tx_pkts = txo->stats.tx_reqs; 1684 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start)); 1685 1686 /* Skip, if wrapped around or first calculation */ 1687 now = jiffies; 1688 if (!aic->jiffies || time_before(now, aic->jiffies) || 1689 rx_pkts < aic->rx_pkts_prev || 1690 tx_pkts < aic->tx_reqs_prev) { 1691 be_aic_update(aic, rx_pkts, tx_pkts, now); 1692 continue; 1693 } 1694 1695 delta = jiffies_to_msecs(now - aic->jiffies); 1696 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) + 1697 (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta); 1698 eqd = (pps / 15000) << 2; 1699 1700 if (eqd < 8) 1701 eqd = 0; 1702 eqd = min_t(u32, eqd, aic->max_eqd); 1703 eqd = max_t(u32, eqd, aic->min_eqd); 1704 1705 be_aic_update(aic, rx_pkts, tx_pkts, now); 1706 modify_eqd: 1707 if (eqd != aic->prev_eqd) { 1708 set_eqd[num].delay_multiplier = (eqd * 65)/100; 1709 set_eqd[num].eq_id = eqo->q.id; 1710 aic->prev_eqd = eqd; 1711 num++; 1712 } 1713 } 1714 1715 if (num) 1716 be_cmd_modify_eqd(adapter, set_eqd, num); 1717 } 1718 1719 static void be_rx_stats_update(struct be_rx_obj *rxo, 1720 struct be_rx_compl_info *rxcp) 1721 { 1722 struct be_rx_stats *stats = rx_stats(rxo); 1723 1724 u64_stats_update_begin(&stats->sync); 1725 stats->rx_compl++; 1726 stats->rx_bytes += rxcp->pkt_size; 1727 stats->rx_pkts++; 1728 if (rxcp->pkt_type == BE_MULTICAST_PACKET) 1729 stats->rx_mcast_pkts++; 1730 if (rxcp->err) 1731 stats->rx_compl_err++; 1732 u64_stats_update_end(&stats->sync); 1733 } 1734 1735 static inline bool csum_passed(struct be_rx_compl_info *rxcp) 1736 { 1737 /* L4 checksum is not reliable for non TCP/UDP packets. 1738 * Also ignore ipcksm for ipv6 pkts 1739 */ 1740 return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum && 1741 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err; 1742 } 1743 1744 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo) 1745 { 1746 struct be_adapter *adapter = rxo->adapter; 1747 struct be_rx_page_info *rx_page_info; 1748 struct be_queue_info *rxq = &rxo->q; 1749 u16 frag_idx = rxq->tail; 1750 1751 rx_page_info = &rxo->page_info_tbl[frag_idx]; 1752 BUG_ON(!rx_page_info->page); 1753 1754 if (rx_page_info->last_frag) { 1755 dma_unmap_page(&adapter->pdev->dev, 1756 dma_unmap_addr(rx_page_info, bus), 1757 adapter->big_page_size, DMA_FROM_DEVICE); 1758 rx_page_info->last_frag = false; 1759 } else { 1760 dma_sync_single_for_cpu(&adapter->pdev->dev, 1761 dma_unmap_addr(rx_page_info, bus), 1762 rx_frag_size, DMA_FROM_DEVICE); 1763 } 1764 1765 queue_tail_inc(rxq); 1766 atomic_dec(&rxq->used); 1767 return rx_page_info; 1768 } 1769 1770 /* Throwaway the data in the Rx completion */ 1771 static void be_rx_compl_discard(struct be_rx_obj *rxo, 1772 struct be_rx_compl_info *rxcp) 1773 { 1774 struct be_rx_page_info *page_info; 1775 u16 i, num_rcvd = rxcp->num_rcvd; 1776 1777 for (i = 0; i < num_rcvd; i++) { 1778 page_info = get_rx_page_info(rxo); 1779 put_page(page_info->page); 1780 memset(page_info, 0, sizeof(*page_info)); 1781 } 1782 } 1783 1784 /* 1785 * skb_fill_rx_data forms a complete skb for an ether frame 1786 * indicated by rxcp. 1787 */ 1788 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb, 1789 struct be_rx_compl_info *rxcp) 1790 { 1791 struct be_rx_page_info *page_info; 1792 u16 i, j; 1793 u16 hdr_len, curr_frag_len, remaining; 1794 u8 *start; 1795 1796 page_info = get_rx_page_info(rxo); 1797 start = page_address(page_info->page) + page_info->page_offset; 1798 prefetch(start); 1799 1800 /* Copy data in the first descriptor of this completion */ 1801 curr_frag_len = min(rxcp->pkt_size, rx_frag_size); 1802 1803 skb->len = curr_frag_len; 1804 if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */ 1805 memcpy(skb->data, start, curr_frag_len); 1806 /* Complete packet has now been moved to data */ 1807 put_page(page_info->page); 1808 skb->data_len = 0; 1809 skb->tail += curr_frag_len; 1810 } else { 1811 hdr_len = ETH_HLEN; 1812 memcpy(skb->data, start, hdr_len); 1813 skb_shinfo(skb)->nr_frags = 1; 1814 skb_frag_set_page(skb, 0, page_info->page); 1815 skb_shinfo(skb)->frags[0].page_offset = 1816 page_info->page_offset + hdr_len; 1817 skb_frag_size_set(&skb_shinfo(skb)->frags[0], 1818 curr_frag_len - hdr_len); 1819 skb->data_len = curr_frag_len - hdr_len; 1820 skb->truesize += rx_frag_size; 1821 skb->tail += hdr_len; 1822 } 1823 page_info->page = NULL; 1824 1825 if (rxcp->pkt_size <= rx_frag_size) { 1826 BUG_ON(rxcp->num_rcvd != 1); 1827 return; 1828 } 1829 1830 /* More frags present for this completion */ 1831 remaining = rxcp->pkt_size - curr_frag_len; 1832 for (i = 1, j = 0; i < rxcp->num_rcvd; i++) { 1833 page_info = get_rx_page_info(rxo); 1834 curr_frag_len = min(remaining, rx_frag_size); 1835 1836 /* Coalesce all frags from the same physical page in one slot */ 1837 if (page_info->page_offset == 0) { 1838 /* Fresh page */ 1839 j++; 1840 skb_frag_set_page(skb, j, page_info->page); 1841 skb_shinfo(skb)->frags[j].page_offset = 1842 page_info->page_offset; 1843 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 1844 skb_shinfo(skb)->nr_frags++; 1845 } else { 1846 put_page(page_info->page); 1847 } 1848 1849 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 1850 skb->len += curr_frag_len; 1851 skb->data_len += curr_frag_len; 1852 skb->truesize += rx_frag_size; 1853 remaining -= curr_frag_len; 1854 page_info->page = NULL; 1855 } 1856 BUG_ON(j > MAX_SKB_FRAGS); 1857 } 1858 1859 /* Process the RX completion indicated by rxcp when GRO is disabled */ 1860 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi, 1861 struct be_rx_compl_info *rxcp) 1862 { 1863 struct be_adapter *adapter = rxo->adapter; 1864 struct net_device *netdev = adapter->netdev; 1865 struct sk_buff *skb; 1866 1867 skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE); 1868 if (unlikely(!skb)) { 1869 rx_stats(rxo)->rx_drops_no_skbs++; 1870 be_rx_compl_discard(rxo, rxcp); 1871 return; 1872 } 1873 1874 skb_fill_rx_data(rxo, skb, rxcp); 1875 1876 if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp))) 1877 skb->ip_summed = CHECKSUM_UNNECESSARY; 1878 else 1879 skb_checksum_none_assert(skb); 1880 1881 skb->protocol = eth_type_trans(skb, netdev); 1882 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 1883 if (netdev->features & NETIF_F_RXHASH) 1884 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 1885 1886 skb->csum_level = rxcp->tunneled; 1887 skb_mark_napi_id(skb, napi); 1888 1889 if (rxcp->vlanf) 1890 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 1891 1892 netif_receive_skb(skb); 1893 } 1894 1895 /* Process the RX completion indicated by rxcp when GRO is enabled */ 1896 static void be_rx_compl_process_gro(struct be_rx_obj *rxo, 1897 struct napi_struct *napi, 1898 struct be_rx_compl_info *rxcp) 1899 { 1900 struct be_adapter *adapter = rxo->adapter; 1901 struct be_rx_page_info *page_info; 1902 struct sk_buff *skb = NULL; 1903 u16 remaining, curr_frag_len; 1904 u16 i, j; 1905 1906 skb = napi_get_frags(napi); 1907 if (!skb) { 1908 be_rx_compl_discard(rxo, rxcp); 1909 return; 1910 } 1911 1912 remaining = rxcp->pkt_size; 1913 for (i = 0, j = -1; i < rxcp->num_rcvd; i++) { 1914 page_info = get_rx_page_info(rxo); 1915 1916 curr_frag_len = min(remaining, rx_frag_size); 1917 1918 /* Coalesce all frags from the same physical page in one slot */ 1919 if (i == 0 || page_info->page_offset == 0) { 1920 /* First frag or Fresh page */ 1921 j++; 1922 skb_frag_set_page(skb, j, page_info->page); 1923 skb_shinfo(skb)->frags[j].page_offset = 1924 page_info->page_offset; 1925 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 1926 } else { 1927 put_page(page_info->page); 1928 } 1929 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 1930 skb->truesize += rx_frag_size; 1931 remaining -= curr_frag_len; 1932 memset(page_info, 0, sizeof(*page_info)); 1933 } 1934 BUG_ON(j > MAX_SKB_FRAGS); 1935 1936 skb_shinfo(skb)->nr_frags = j + 1; 1937 skb->len = rxcp->pkt_size; 1938 skb->data_len = rxcp->pkt_size; 1939 skb->ip_summed = CHECKSUM_UNNECESSARY; 1940 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 1941 if (adapter->netdev->features & NETIF_F_RXHASH) 1942 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 1943 1944 skb->csum_level = rxcp->tunneled; 1945 skb_mark_napi_id(skb, napi); 1946 1947 if (rxcp->vlanf) 1948 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 1949 1950 napi_gro_frags(napi); 1951 } 1952 1953 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl, 1954 struct be_rx_compl_info *rxcp) 1955 { 1956 rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl); 1957 rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl); 1958 rxcp->err = GET_RX_COMPL_V1_BITS(err, compl); 1959 rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl); 1960 rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl); 1961 rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl); 1962 rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl); 1963 rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl); 1964 rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl); 1965 rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl); 1966 rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl); 1967 if (rxcp->vlanf) { 1968 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl); 1969 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl); 1970 } 1971 rxcp->port = GET_RX_COMPL_V1_BITS(port, compl); 1972 rxcp->tunneled = 1973 GET_RX_COMPL_V1_BITS(tunneled, compl); 1974 } 1975 1976 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl, 1977 struct be_rx_compl_info *rxcp) 1978 { 1979 rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl); 1980 rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl); 1981 rxcp->err = GET_RX_COMPL_V0_BITS(err, compl); 1982 rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl); 1983 rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl); 1984 rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl); 1985 rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl); 1986 rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl); 1987 rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl); 1988 rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl); 1989 rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl); 1990 if (rxcp->vlanf) { 1991 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl); 1992 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl); 1993 } 1994 rxcp->port = GET_RX_COMPL_V0_BITS(port, compl); 1995 rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl); 1996 } 1997 1998 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo) 1999 { 2000 struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq); 2001 struct be_rx_compl_info *rxcp = &rxo->rxcp; 2002 struct be_adapter *adapter = rxo->adapter; 2003 2004 /* For checking the valid bit it is Ok to use either definition as the 2005 * valid bit is at the same position in both v0 and v1 Rx compl */ 2006 if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0) 2007 return NULL; 2008 2009 rmb(); 2010 be_dws_le_to_cpu(compl, sizeof(*compl)); 2011 2012 if (adapter->be3_native) 2013 be_parse_rx_compl_v1(compl, rxcp); 2014 else 2015 be_parse_rx_compl_v0(compl, rxcp); 2016 2017 if (rxcp->ip_frag) 2018 rxcp->l4_csum = 0; 2019 2020 if (rxcp->vlanf) { 2021 /* In QNQ modes, if qnq bit is not set, then the packet was 2022 * tagged only with the transparent outer vlan-tag and must 2023 * not be treated as a vlan packet by host 2024 */ 2025 if (be_is_qnq_mode(adapter) && !rxcp->qnq) 2026 rxcp->vlanf = 0; 2027 2028 if (!lancer_chip(adapter)) 2029 rxcp->vlan_tag = swab16(rxcp->vlan_tag); 2030 2031 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) && 2032 !test_bit(rxcp->vlan_tag, adapter->vids)) 2033 rxcp->vlanf = 0; 2034 } 2035 2036 /* As the compl has been parsed, reset it; we wont touch it again */ 2037 compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0; 2038 2039 queue_tail_inc(&rxo->cq); 2040 return rxcp; 2041 } 2042 2043 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp) 2044 { 2045 u32 order = get_order(size); 2046 2047 if (order > 0) 2048 gfp |= __GFP_COMP; 2049 return alloc_pages(gfp, order); 2050 } 2051 2052 /* 2053 * Allocate a page, split it to fragments of size rx_frag_size and post as 2054 * receive buffers to BE 2055 */ 2056 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed) 2057 { 2058 struct be_adapter *adapter = rxo->adapter; 2059 struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL; 2060 struct be_queue_info *rxq = &rxo->q; 2061 struct page *pagep = NULL; 2062 struct device *dev = &adapter->pdev->dev; 2063 struct be_eth_rx_d *rxd; 2064 u64 page_dmaaddr = 0, frag_dmaaddr; 2065 u32 posted, page_offset = 0, notify = 0; 2066 2067 page_info = &rxo->page_info_tbl[rxq->head]; 2068 for (posted = 0; posted < frags_needed && !page_info->page; posted++) { 2069 if (!pagep) { 2070 pagep = be_alloc_pages(adapter->big_page_size, gfp); 2071 if (unlikely(!pagep)) { 2072 rx_stats(rxo)->rx_post_fail++; 2073 break; 2074 } 2075 page_dmaaddr = dma_map_page(dev, pagep, 0, 2076 adapter->big_page_size, 2077 DMA_FROM_DEVICE); 2078 if (dma_mapping_error(dev, page_dmaaddr)) { 2079 put_page(pagep); 2080 pagep = NULL; 2081 adapter->drv_stats.dma_map_errors++; 2082 break; 2083 } 2084 page_offset = 0; 2085 } else { 2086 get_page(pagep); 2087 page_offset += rx_frag_size; 2088 } 2089 page_info->page_offset = page_offset; 2090 page_info->page = pagep; 2091 2092 rxd = queue_head_node(rxq); 2093 frag_dmaaddr = page_dmaaddr + page_info->page_offset; 2094 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF); 2095 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr)); 2096 2097 /* Any space left in the current big page for another frag? */ 2098 if ((page_offset + rx_frag_size + rx_frag_size) > 2099 adapter->big_page_size) { 2100 pagep = NULL; 2101 page_info->last_frag = true; 2102 dma_unmap_addr_set(page_info, bus, page_dmaaddr); 2103 } else { 2104 dma_unmap_addr_set(page_info, bus, frag_dmaaddr); 2105 } 2106 2107 prev_page_info = page_info; 2108 queue_head_inc(rxq); 2109 page_info = &rxo->page_info_tbl[rxq->head]; 2110 } 2111 2112 /* Mark the last frag of a page when we break out of the above loop 2113 * with no more slots available in the RXQ 2114 */ 2115 if (pagep) { 2116 prev_page_info->last_frag = true; 2117 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr); 2118 } 2119 2120 if (posted) { 2121 atomic_add(posted, &rxq->used); 2122 if (rxo->rx_post_starved) 2123 rxo->rx_post_starved = false; 2124 do { 2125 notify = min(MAX_NUM_POST_ERX_DB, posted); 2126 be_rxq_notify(adapter, rxq->id, notify); 2127 posted -= notify; 2128 } while (posted); 2129 } else if (atomic_read(&rxq->used) == 0) { 2130 /* Let be_worker replenish when memory is available */ 2131 rxo->rx_post_starved = true; 2132 } 2133 } 2134 2135 static struct be_tx_compl_info *be_tx_compl_get(struct be_tx_obj *txo) 2136 { 2137 struct be_queue_info *tx_cq = &txo->cq; 2138 struct be_tx_compl_info *txcp = &txo->txcp; 2139 struct be_eth_tx_compl *compl = queue_tail_node(tx_cq); 2140 2141 if (compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0) 2142 return NULL; 2143 2144 /* Ensure load ordering of valid bit dword and other dwords below */ 2145 rmb(); 2146 be_dws_le_to_cpu(compl, sizeof(*compl)); 2147 2148 txcp->status = GET_TX_COMPL_BITS(status, compl); 2149 txcp->end_index = GET_TX_COMPL_BITS(wrb_index, compl); 2150 2151 compl->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0; 2152 queue_tail_inc(tx_cq); 2153 return txcp; 2154 } 2155 2156 static u16 be_tx_compl_process(struct be_adapter *adapter, 2157 struct be_tx_obj *txo, u16 last_index) 2158 { 2159 struct sk_buff **sent_skbs = txo->sent_skb_list; 2160 struct be_queue_info *txq = &txo->q; 2161 u16 frag_index, num_wrbs = 0; 2162 struct sk_buff *skb = NULL; 2163 bool unmap_skb_hdr = false; 2164 struct be_eth_wrb *wrb; 2165 2166 do { 2167 if (sent_skbs[txq->tail]) { 2168 /* Free skb from prev req */ 2169 if (skb) 2170 dev_consume_skb_any(skb); 2171 skb = sent_skbs[txq->tail]; 2172 sent_skbs[txq->tail] = NULL; 2173 queue_tail_inc(txq); /* skip hdr wrb */ 2174 num_wrbs++; 2175 unmap_skb_hdr = true; 2176 } 2177 wrb = queue_tail_node(txq); 2178 frag_index = txq->tail; 2179 unmap_tx_frag(&adapter->pdev->dev, wrb, 2180 (unmap_skb_hdr && skb_headlen(skb))); 2181 unmap_skb_hdr = false; 2182 queue_tail_inc(txq); 2183 num_wrbs++; 2184 } while (frag_index != last_index); 2185 dev_consume_skb_any(skb); 2186 2187 return num_wrbs; 2188 } 2189 2190 /* Return the number of events in the event queue */ 2191 static inline int events_get(struct be_eq_obj *eqo) 2192 { 2193 struct be_eq_entry *eqe; 2194 int num = 0; 2195 2196 do { 2197 eqe = queue_tail_node(&eqo->q); 2198 if (eqe->evt == 0) 2199 break; 2200 2201 rmb(); 2202 eqe->evt = 0; 2203 num++; 2204 queue_tail_inc(&eqo->q); 2205 } while (true); 2206 2207 return num; 2208 } 2209 2210 /* Leaves the EQ is disarmed state */ 2211 static void be_eq_clean(struct be_eq_obj *eqo) 2212 { 2213 int num = events_get(eqo); 2214 2215 be_eq_notify(eqo->adapter, eqo->q.id, false, true, num); 2216 } 2217 2218 static void be_rx_cq_clean(struct be_rx_obj *rxo) 2219 { 2220 struct be_rx_page_info *page_info; 2221 struct be_queue_info *rxq = &rxo->q; 2222 struct be_queue_info *rx_cq = &rxo->cq; 2223 struct be_rx_compl_info *rxcp; 2224 struct be_adapter *adapter = rxo->adapter; 2225 int flush_wait = 0; 2226 2227 /* Consume pending rx completions. 2228 * Wait for the flush completion (identified by zero num_rcvd) 2229 * to arrive. Notify CQ even when there are no more CQ entries 2230 * for HW to flush partially coalesced CQ entries. 2231 * In Lancer, there is no need to wait for flush compl. 2232 */ 2233 for (;;) { 2234 rxcp = be_rx_compl_get(rxo); 2235 if (!rxcp) { 2236 if (lancer_chip(adapter)) 2237 break; 2238 2239 if (flush_wait++ > 10 || be_hw_error(adapter)) { 2240 dev_warn(&adapter->pdev->dev, 2241 "did not receive flush compl\n"); 2242 break; 2243 } 2244 be_cq_notify(adapter, rx_cq->id, true, 0); 2245 mdelay(1); 2246 } else { 2247 be_rx_compl_discard(rxo, rxcp); 2248 be_cq_notify(adapter, rx_cq->id, false, 1); 2249 if (rxcp->num_rcvd == 0) 2250 break; 2251 } 2252 } 2253 2254 /* After cleanup, leave the CQ in unarmed state */ 2255 be_cq_notify(adapter, rx_cq->id, false, 0); 2256 2257 /* Then free posted rx buffers that were not used */ 2258 while (atomic_read(&rxq->used) > 0) { 2259 page_info = get_rx_page_info(rxo); 2260 put_page(page_info->page); 2261 memset(page_info, 0, sizeof(*page_info)); 2262 } 2263 BUG_ON(atomic_read(&rxq->used)); 2264 rxq->tail = 0; 2265 rxq->head = 0; 2266 } 2267 2268 static void be_tx_compl_clean(struct be_adapter *adapter) 2269 { 2270 u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0; 2271 struct device *dev = &adapter->pdev->dev; 2272 struct be_tx_compl_info *txcp; 2273 struct be_queue_info *txq; 2274 struct be_tx_obj *txo; 2275 int i, pending_txqs; 2276 2277 /* Stop polling for compls when HW has been silent for 10ms */ 2278 do { 2279 pending_txqs = adapter->num_tx_qs; 2280 2281 for_all_tx_queues(adapter, txo, i) { 2282 cmpl = 0; 2283 num_wrbs = 0; 2284 txq = &txo->q; 2285 while ((txcp = be_tx_compl_get(txo))) { 2286 num_wrbs += 2287 be_tx_compl_process(adapter, txo, 2288 txcp->end_index); 2289 cmpl++; 2290 } 2291 if (cmpl) { 2292 be_cq_notify(adapter, txo->cq.id, false, cmpl); 2293 atomic_sub(num_wrbs, &txq->used); 2294 timeo = 0; 2295 } 2296 if (!be_is_tx_compl_pending(txo)) 2297 pending_txqs--; 2298 } 2299 2300 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter)) 2301 break; 2302 2303 mdelay(1); 2304 } while (true); 2305 2306 /* Free enqueued TX that was never notified to HW */ 2307 for_all_tx_queues(adapter, txo, i) { 2308 txq = &txo->q; 2309 2310 if (atomic_read(&txq->used)) { 2311 dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n", 2312 i, atomic_read(&txq->used)); 2313 notified_idx = txq->tail; 2314 end_idx = txq->tail; 2315 index_adv(&end_idx, atomic_read(&txq->used) - 1, 2316 txq->len); 2317 /* Use the tx-compl process logic to handle requests 2318 * that were not sent to the HW. 2319 */ 2320 num_wrbs = be_tx_compl_process(adapter, txo, end_idx); 2321 atomic_sub(num_wrbs, &txq->used); 2322 BUG_ON(atomic_read(&txq->used)); 2323 txo->pend_wrb_cnt = 0; 2324 /* Since hw was never notified of these requests, 2325 * reset TXQ indices 2326 */ 2327 txq->head = notified_idx; 2328 txq->tail = notified_idx; 2329 } 2330 } 2331 } 2332 2333 static void be_evt_queues_destroy(struct be_adapter *adapter) 2334 { 2335 struct be_eq_obj *eqo; 2336 int i; 2337 2338 for_all_evt_queues(adapter, eqo, i) { 2339 if (eqo->q.created) { 2340 be_eq_clean(eqo); 2341 be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ); 2342 napi_hash_del(&eqo->napi); 2343 netif_napi_del(&eqo->napi); 2344 } 2345 free_cpumask_var(eqo->affinity_mask); 2346 be_queue_free(adapter, &eqo->q); 2347 } 2348 } 2349 2350 static int be_evt_queues_create(struct be_adapter *adapter) 2351 { 2352 struct be_queue_info *eq; 2353 struct be_eq_obj *eqo; 2354 struct be_aic_obj *aic; 2355 int i, rc; 2356 2357 adapter->num_evt_qs = min_t(u16, num_irqs(adapter), 2358 adapter->cfg_num_qs); 2359 2360 for_all_evt_queues(adapter, eqo, i) { 2361 if (!zalloc_cpumask_var(&eqo->affinity_mask, GFP_KERNEL)) 2362 return -ENOMEM; 2363 cpumask_set_cpu_local_first(i, dev_to_node(&adapter->pdev->dev), 2364 eqo->affinity_mask); 2365 2366 netif_napi_add(adapter->netdev, &eqo->napi, be_poll, 2367 BE_NAPI_WEIGHT); 2368 napi_hash_add(&eqo->napi); 2369 aic = &adapter->aic_obj[i]; 2370 eqo->adapter = adapter; 2371 eqo->idx = i; 2372 aic->max_eqd = BE_MAX_EQD; 2373 aic->enable = true; 2374 2375 eq = &eqo->q; 2376 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN, 2377 sizeof(struct be_eq_entry)); 2378 if (rc) 2379 return rc; 2380 2381 rc = be_cmd_eq_create(adapter, eqo); 2382 if (rc) 2383 return rc; 2384 } 2385 return 0; 2386 } 2387 2388 static void be_mcc_queues_destroy(struct be_adapter *adapter) 2389 { 2390 struct be_queue_info *q; 2391 2392 q = &adapter->mcc_obj.q; 2393 if (q->created) 2394 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ); 2395 be_queue_free(adapter, q); 2396 2397 q = &adapter->mcc_obj.cq; 2398 if (q->created) 2399 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2400 be_queue_free(adapter, q); 2401 } 2402 2403 /* Must be called only after TX qs are created as MCC shares TX EQ */ 2404 static int be_mcc_queues_create(struct be_adapter *adapter) 2405 { 2406 struct be_queue_info *q, *cq; 2407 2408 cq = &adapter->mcc_obj.cq; 2409 if (be_queue_alloc(adapter, cq, MCC_CQ_LEN, 2410 sizeof(struct be_mcc_compl))) 2411 goto err; 2412 2413 /* Use the default EQ for MCC completions */ 2414 if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0)) 2415 goto mcc_cq_free; 2416 2417 q = &adapter->mcc_obj.q; 2418 if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 2419 goto mcc_cq_destroy; 2420 2421 if (be_cmd_mccq_create(adapter, q, cq)) 2422 goto mcc_q_free; 2423 2424 return 0; 2425 2426 mcc_q_free: 2427 be_queue_free(adapter, q); 2428 mcc_cq_destroy: 2429 be_cmd_q_destroy(adapter, cq, QTYPE_CQ); 2430 mcc_cq_free: 2431 be_queue_free(adapter, cq); 2432 err: 2433 return -1; 2434 } 2435 2436 static void be_tx_queues_destroy(struct be_adapter *adapter) 2437 { 2438 struct be_queue_info *q; 2439 struct be_tx_obj *txo; 2440 u8 i; 2441 2442 for_all_tx_queues(adapter, txo, i) { 2443 q = &txo->q; 2444 if (q->created) 2445 be_cmd_q_destroy(adapter, q, QTYPE_TXQ); 2446 be_queue_free(adapter, q); 2447 2448 q = &txo->cq; 2449 if (q->created) 2450 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2451 be_queue_free(adapter, q); 2452 } 2453 } 2454 2455 static int be_tx_qs_create(struct be_adapter *adapter) 2456 { 2457 struct be_queue_info *cq; 2458 struct be_tx_obj *txo; 2459 struct be_eq_obj *eqo; 2460 int status, i; 2461 2462 adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter)); 2463 2464 for_all_tx_queues(adapter, txo, i) { 2465 cq = &txo->cq; 2466 status = be_queue_alloc(adapter, cq, TX_CQ_LEN, 2467 sizeof(struct be_eth_tx_compl)); 2468 if (status) 2469 return status; 2470 2471 u64_stats_init(&txo->stats.sync); 2472 u64_stats_init(&txo->stats.sync_compl); 2473 2474 /* If num_evt_qs is less than num_tx_qs, then more than 2475 * one txq share an eq 2476 */ 2477 eqo = &adapter->eq_obj[i % adapter->num_evt_qs]; 2478 status = be_cmd_cq_create(adapter, cq, &eqo->q, false, 3); 2479 if (status) 2480 return status; 2481 2482 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN, 2483 sizeof(struct be_eth_wrb)); 2484 if (status) 2485 return status; 2486 2487 status = be_cmd_txq_create(adapter, txo); 2488 if (status) 2489 return status; 2490 2491 netif_set_xps_queue(adapter->netdev, eqo->affinity_mask, 2492 eqo->idx); 2493 } 2494 2495 dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n", 2496 adapter->num_tx_qs); 2497 return 0; 2498 } 2499 2500 static void be_rx_cqs_destroy(struct be_adapter *adapter) 2501 { 2502 struct be_queue_info *q; 2503 struct be_rx_obj *rxo; 2504 int i; 2505 2506 for_all_rx_queues(adapter, rxo, i) { 2507 q = &rxo->cq; 2508 if (q->created) 2509 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2510 be_queue_free(adapter, q); 2511 } 2512 } 2513 2514 static int be_rx_cqs_create(struct be_adapter *adapter) 2515 { 2516 struct be_queue_info *eq, *cq; 2517 struct be_rx_obj *rxo; 2518 int rc, i; 2519 2520 /* We can create as many RSS rings as there are EQs. */ 2521 adapter->num_rss_qs = adapter->num_evt_qs; 2522 2523 /* We'll use RSS only if atleast 2 RSS rings are supported. */ 2524 if (adapter->num_rss_qs <= 1) 2525 adapter->num_rss_qs = 0; 2526 2527 adapter->num_rx_qs = adapter->num_rss_qs + adapter->need_def_rxq; 2528 2529 /* When the interface is not capable of RSS rings (and there is no 2530 * need to create a default RXQ) we'll still need one RXQ 2531 */ 2532 if (adapter->num_rx_qs == 0) 2533 adapter->num_rx_qs = 1; 2534 2535 adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE; 2536 for_all_rx_queues(adapter, rxo, i) { 2537 rxo->adapter = adapter; 2538 cq = &rxo->cq; 2539 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN, 2540 sizeof(struct be_eth_rx_compl)); 2541 if (rc) 2542 return rc; 2543 2544 u64_stats_init(&rxo->stats.sync); 2545 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; 2546 rc = be_cmd_cq_create(adapter, cq, eq, false, 3); 2547 if (rc) 2548 return rc; 2549 } 2550 2551 dev_info(&adapter->pdev->dev, 2552 "created %d RX queue(s)\n", adapter->num_rx_qs); 2553 return 0; 2554 } 2555 2556 static irqreturn_t be_intx(int irq, void *dev) 2557 { 2558 struct be_eq_obj *eqo = dev; 2559 struct be_adapter *adapter = eqo->adapter; 2560 int num_evts = 0; 2561 2562 /* IRQ is not expected when NAPI is scheduled as the EQ 2563 * will not be armed. 2564 * But, this can happen on Lancer INTx where it takes 2565 * a while to de-assert INTx or in BE2 where occasionaly 2566 * an interrupt may be raised even when EQ is unarmed. 2567 * If NAPI is already scheduled, then counting & notifying 2568 * events will orphan them. 2569 */ 2570 if (napi_schedule_prep(&eqo->napi)) { 2571 num_evts = events_get(eqo); 2572 __napi_schedule(&eqo->napi); 2573 if (num_evts) 2574 eqo->spurious_intr = 0; 2575 } 2576 be_eq_notify(adapter, eqo->q.id, false, true, num_evts); 2577 2578 /* Return IRQ_HANDLED only for the the first spurious intr 2579 * after a valid intr to stop the kernel from branding 2580 * this irq as a bad one! 2581 */ 2582 if (num_evts || eqo->spurious_intr++ == 0) 2583 return IRQ_HANDLED; 2584 else 2585 return IRQ_NONE; 2586 } 2587 2588 static irqreturn_t be_msix(int irq, void *dev) 2589 { 2590 struct be_eq_obj *eqo = dev; 2591 2592 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0); 2593 napi_schedule(&eqo->napi); 2594 return IRQ_HANDLED; 2595 } 2596 2597 static inline bool do_gro(struct be_rx_compl_info *rxcp) 2598 { 2599 return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false; 2600 } 2601 2602 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi, 2603 int budget, int polling) 2604 { 2605 struct be_adapter *adapter = rxo->adapter; 2606 struct be_queue_info *rx_cq = &rxo->cq; 2607 struct be_rx_compl_info *rxcp; 2608 u32 work_done; 2609 u32 frags_consumed = 0; 2610 2611 for (work_done = 0; work_done < budget; work_done++) { 2612 rxcp = be_rx_compl_get(rxo); 2613 if (!rxcp) 2614 break; 2615 2616 /* Is it a flush compl that has no data */ 2617 if (unlikely(rxcp->num_rcvd == 0)) 2618 goto loop_continue; 2619 2620 /* Discard compl with partial DMA Lancer B0 */ 2621 if (unlikely(!rxcp->pkt_size)) { 2622 be_rx_compl_discard(rxo, rxcp); 2623 goto loop_continue; 2624 } 2625 2626 /* On BE drop pkts that arrive due to imperfect filtering in 2627 * promiscuous mode on some skews 2628 */ 2629 if (unlikely(rxcp->port != adapter->port_num && 2630 !lancer_chip(adapter))) { 2631 be_rx_compl_discard(rxo, rxcp); 2632 goto loop_continue; 2633 } 2634 2635 /* Don't do gro when we're busy_polling */ 2636 if (do_gro(rxcp) && polling != BUSY_POLLING) 2637 be_rx_compl_process_gro(rxo, napi, rxcp); 2638 else 2639 be_rx_compl_process(rxo, napi, rxcp); 2640 2641 loop_continue: 2642 frags_consumed += rxcp->num_rcvd; 2643 be_rx_stats_update(rxo, rxcp); 2644 } 2645 2646 if (work_done) { 2647 be_cq_notify(adapter, rx_cq->id, true, work_done); 2648 2649 /* When an rx-obj gets into post_starved state, just 2650 * let be_worker do the posting. 2651 */ 2652 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM && 2653 !rxo->rx_post_starved) 2654 be_post_rx_frags(rxo, GFP_ATOMIC, 2655 max_t(u32, MAX_RX_POST, 2656 frags_consumed)); 2657 } 2658 2659 return work_done; 2660 } 2661 2662 static inline void be_update_tx_err(struct be_tx_obj *txo, u8 status) 2663 { 2664 switch (status) { 2665 case BE_TX_COMP_HDR_PARSE_ERR: 2666 tx_stats(txo)->tx_hdr_parse_err++; 2667 break; 2668 case BE_TX_COMP_NDMA_ERR: 2669 tx_stats(txo)->tx_dma_err++; 2670 break; 2671 case BE_TX_COMP_ACL_ERR: 2672 tx_stats(txo)->tx_spoof_check_err++; 2673 break; 2674 } 2675 } 2676 2677 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u8 status) 2678 { 2679 switch (status) { 2680 case LANCER_TX_COMP_LSO_ERR: 2681 tx_stats(txo)->tx_tso_err++; 2682 break; 2683 case LANCER_TX_COMP_HSW_DROP_MAC_ERR: 2684 case LANCER_TX_COMP_HSW_DROP_VLAN_ERR: 2685 tx_stats(txo)->tx_spoof_check_err++; 2686 break; 2687 case LANCER_TX_COMP_QINQ_ERR: 2688 tx_stats(txo)->tx_qinq_err++; 2689 break; 2690 case LANCER_TX_COMP_PARITY_ERR: 2691 tx_stats(txo)->tx_internal_parity_err++; 2692 break; 2693 case LANCER_TX_COMP_DMA_ERR: 2694 tx_stats(txo)->tx_dma_err++; 2695 break; 2696 } 2697 } 2698 2699 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo, 2700 int idx) 2701 { 2702 int num_wrbs = 0, work_done = 0; 2703 struct be_tx_compl_info *txcp; 2704 2705 while ((txcp = be_tx_compl_get(txo))) { 2706 num_wrbs += be_tx_compl_process(adapter, txo, txcp->end_index); 2707 work_done++; 2708 2709 if (txcp->status) { 2710 if (lancer_chip(adapter)) 2711 lancer_update_tx_err(txo, txcp->status); 2712 else 2713 be_update_tx_err(txo, txcp->status); 2714 } 2715 } 2716 2717 if (work_done) { 2718 be_cq_notify(adapter, txo->cq.id, true, work_done); 2719 atomic_sub(num_wrbs, &txo->q.used); 2720 2721 /* As Tx wrbs have been freed up, wake up netdev queue 2722 * if it was stopped due to lack of tx wrbs. */ 2723 if (__netif_subqueue_stopped(adapter->netdev, idx) && 2724 be_can_txq_wake(txo)) { 2725 netif_wake_subqueue(adapter->netdev, idx); 2726 } 2727 2728 u64_stats_update_begin(&tx_stats(txo)->sync_compl); 2729 tx_stats(txo)->tx_compl += work_done; 2730 u64_stats_update_end(&tx_stats(txo)->sync_compl); 2731 } 2732 } 2733 2734 #ifdef CONFIG_NET_RX_BUSY_POLL 2735 static inline bool be_lock_napi(struct be_eq_obj *eqo) 2736 { 2737 bool status = true; 2738 2739 spin_lock(&eqo->lock); /* BH is already disabled */ 2740 if (eqo->state & BE_EQ_LOCKED) { 2741 WARN_ON(eqo->state & BE_EQ_NAPI); 2742 eqo->state |= BE_EQ_NAPI_YIELD; 2743 status = false; 2744 } else { 2745 eqo->state = BE_EQ_NAPI; 2746 } 2747 spin_unlock(&eqo->lock); 2748 return status; 2749 } 2750 2751 static inline void be_unlock_napi(struct be_eq_obj *eqo) 2752 { 2753 spin_lock(&eqo->lock); /* BH is already disabled */ 2754 2755 WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD)); 2756 eqo->state = BE_EQ_IDLE; 2757 2758 spin_unlock(&eqo->lock); 2759 } 2760 2761 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo) 2762 { 2763 bool status = true; 2764 2765 spin_lock_bh(&eqo->lock); 2766 if (eqo->state & BE_EQ_LOCKED) { 2767 eqo->state |= BE_EQ_POLL_YIELD; 2768 status = false; 2769 } else { 2770 eqo->state |= BE_EQ_POLL; 2771 } 2772 spin_unlock_bh(&eqo->lock); 2773 return status; 2774 } 2775 2776 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo) 2777 { 2778 spin_lock_bh(&eqo->lock); 2779 2780 WARN_ON(eqo->state & (BE_EQ_NAPI)); 2781 eqo->state = BE_EQ_IDLE; 2782 2783 spin_unlock_bh(&eqo->lock); 2784 } 2785 2786 static inline void be_enable_busy_poll(struct be_eq_obj *eqo) 2787 { 2788 spin_lock_init(&eqo->lock); 2789 eqo->state = BE_EQ_IDLE; 2790 } 2791 2792 static inline void be_disable_busy_poll(struct be_eq_obj *eqo) 2793 { 2794 local_bh_disable(); 2795 2796 /* It's enough to just acquire napi lock on the eqo to stop 2797 * be_busy_poll() from processing any queueus. 2798 */ 2799 while (!be_lock_napi(eqo)) 2800 mdelay(1); 2801 2802 local_bh_enable(); 2803 } 2804 2805 #else /* CONFIG_NET_RX_BUSY_POLL */ 2806 2807 static inline bool be_lock_napi(struct be_eq_obj *eqo) 2808 { 2809 return true; 2810 } 2811 2812 static inline void be_unlock_napi(struct be_eq_obj *eqo) 2813 { 2814 } 2815 2816 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo) 2817 { 2818 return false; 2819 } 2820 2821 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo) 2822 { 2823 } 2824 2825 static inline void be_enable_busy_poll(struct be_eq_obj *eqo) 2826 { 2827 } 2828 2829 static inline void be_disable_busy_poll(struct be_eq_obj *eqo) 2830 { 2831 } 2832 #endif /* CONFIG_NET_RX_BUSY_POLL */ 2833 2834 int be_poll(struct napi_struct *napi, int budget) 2835 { 2836 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 2837 struct be_adapter *adapter = eqo->adapter; 2838 int max_work = 0, work, i, num_evts; 2839 struct be_rx_obj *rxo; 2840 struct be_tx_obj *txo; 2841 2842 num_evts = events_get(eqo); 2843 2844 for_all_tx_queues_on_eq(adapter, eqo, txo, i) 2845 be_process_tx(adapter, txo, i); 2846 2847 if (be_lock_napi(eqo)) { 2848 /* This loop will iterate twice for EQ0 in which 2849 * completions of the last RXQ (default one) are also processed 2850 * For other EQs the loop iterates only once 2851 */ 2852 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 2853 work = be_process_rx(rxo, napi, budget, NAPI_POLLING); 2854 max_work = max(work, max_work); 2855 } 2856 be_unlock_napi(eqo); 2857 } else { 2858 max_work = budget; 2859 } 2860 2861 if (is_mcc_eqo(eqo)) 2862 be_process_mcc(adapter); 2863 2864 if (max_work < budget) { 2865 napi_complete(napi); 2866 be_eq_notify(adapter, eqo->q.id, true, false, num_evts); 2867 } else { 2868 /* As we'll continue in polling mode, count and clear events */ 2869 be_eq_notify(adapter, eqo->q.id, false, false, num_evts); 2870 } 2871 return max_work; 2872 } 2873 2874 #ifdef CONFIG_NET_RX_BUSY_POLL 2875 static int be_busy_poll(struct napi_struct *napi) 2876 { 2877 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 2878 struct be_adapter *adapter = eqo->adapter; 2879 struct be_rx_obj *rxo; 2880 int i, work = 0; 2881 2882 if (!be_lock_busy_poll(eqo)) 2883 return LL_FLUSH_BUSY; 2884 2885 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 2886 work = be_process_rx(rxo, napi, 4, BUSY_POLLING); 2887 if (work) 2888 break; 2889 } 2890 2891 be_unlock_busy_poll(eqo); 2892 return work; 2893 } 2894 #endif 2895 2896 void be_detect_error(struct be_adapter *adapter) 2897 { 2898 u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0; 2899 u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2900 u32 i; 2901 bool error_detected = false; 2902 struct device *dev = &adapter->pdev->dev; 2903 struct net_device *netdev = adapter->netdev; 2904 2905 if (be_hw_error(adapter)) 2906 return; 2907 2908 if (lancer_chip(adapter)) { 2909 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 2910 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2911 sliport_err1 = ioread32(adapter->db + 2912 SLIPORT_ERROR1_OFFSET); 2913 sliport_err2 = ioread32(adapter->db + 2914 SLIPORT_ERROR2_OFFSET); 2915 adapter->hw_error = true; 2916 error_detected = true; 2917 /* Do not log error messages if its a FW reset */ 2918 if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 && 2919 sliport_err2 == SLIPORT_ERROR_FW_RESET2) { 2920 dev_info(dev, "Firmware update in progress\n"); 2921 } else { 2922 dev_err(dev, "Error detected in the card\n"); 2923 dev_err(dev, "ERR: sliport status 0x%x\n", 2924 sliport_status); 2925 dev_err(dev, "ERR: sliport error1 0x%x\n", 2926 sliport_err1); 2927 dev_err(dev, "ERR: sliport error2 0x%x\n", 2928 sliport_err2); 2929 } 2930 } 2931 } else { 2932 ue_lo = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_LOW); 2933 ue_hi = ioread32(adapter->pcicfg + PCICFG_UE_STATUS_HIGH); 2934 ue_lo_mask = ioread32(adapter->pcicfg + 2935 PCICFG_UE_STATUS_LOW_MASK); 2936 ue_hi_mask = ioread32(adapter->pcicfg + 2937 PCICFG_UE_STATUS_HI_MASK); 2938 2939 ue_lo = (ue_lo & ~ue_lo_mask); 2940 ue_hi = (ue_hi & ~ue_hi_mask); 2941 2942 /* On certain platforms BE hardware can indicate spurious UEs. 2943 * Allow HW to stop working completely in case of a real UE. 2944 * Hence not setting the hw_error for UE detection. 2945 */ 2946 2947 if (ue_lo || ue_hi) { 2948 error_detected = true; 2949 dev_err(dev, 2950 "Unrecoverable Error detected in the adapter"); 2951 dev_err(dev, "Please reboot server to recover"); 2952 if (skyhawk_chip(adapter)) 2953 adapter->hw_error = true; 2954 for (i = 0; ue_lo; ue_lo >>= 1, i++) { 2955 if (ue_lo & 1) 2956 dev_err(dev, "UE: %s bit set\n", 2957 ue_status_low_desc[i]); 2958 } 2959 for (i = 0; ue_hi; ue_hi >>= 1, i++) { 2960 if (ue_hi & 1) 2961 dev_err(dev, "UE: %s bit set\n", 2962 ue_status_hi_desc[i]); 2963 } 2964 } 2965 } 2966 if (error_detected) 2967 netif_carrier_off(netdev); 2968 } 2969 2970 static void be_msix_disable(struct be_adapter *adapter) 2971 { 2972 if (msix_enabled(adapter)) { 2973 pci_disable_msix(adapter->pdev); 2974 adapter->num_msix_vec = 0; 2975 adapter->num_msix_roce_vec = 0; 2976 } 2977 } 2978 2979 static int be_msix_enable(struct be_adapter *adapter) 2980 { 2981 int i, num_vec; 2982 struct device *dev = &adapter->pdev->dev; 2983 2984 /* If RoCE is supported, program the max number of NIC vectors that 2985 * may be configured via set-channels, along with vectors needed for 2986 * RoCe. Else, just program the number we'll use initially. 2987 */ 2988 if (be_roce_supported(adapter)) 2989 num_vec = min_t(int, 2 * be_max_eqs(adapter), 2990 2 * num_online_cpus()); 2991 else 2992 num_vec = adapter->cfg_num_qs; 2993 2994 for (i = 0; i < num_vec; i++) 2995 adapter->msix_entries[i].entry = i; 2996 2997 num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2998 MIN_MSIX_VECTORS, num_vec); 2999 if (num_vec < 0) 3000 goto fail; 3001 3002 if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) { 3003 adapter->num_msix_roce_vec = num_vec / 2; 3004 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n", 3005 adapter->num_msix_roce_vec); 3006 } 3007 3008 adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec; 3009 3010 dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n", 3011 adapter->num_msix_vec); 3012 return 0; 3013 3014 fail: 3015 dev_warn(dev, "MSIx enable failed\n"); 3016 3017 /* INTx is not supported in VFs, so fail probe if enable_msix fails */ 3018 if (!be_physfn(adapter)) 3019 return num_vec; 3020 return 0; 3021 } 3022 3023 static inline int be_msix_vec_get(struct be_adapter *adapter, 3024 struct be_eq_obj *eqo) 3025 { 3026 return adapter->msix_entries[eqo->msix_idx].vector; 3027 } 3028 3029 static int be_msix_register(struct be_adapter *adapter) 3030 { 3031 struct net_device *netdev = adapter->netdev; 3032 struct be_eq_obj *eqo; 3033 int status, i, vec; 3034 3035 for_all_evt_queues(adapter, eqo, i) { 3036 sprintf(eqo->desc, "%s-q%d", netdev->name, i); 3037 vec = be_msix_vec_get(adapter, eqo); 3038 status = request_irq(vec, be_msix, 0, eqo->desc, eqo); 3039 if (status) 3040 goto err_msix; 3041 3042 irq_set_affinity_hint(vec, eqo->affinity_mask); 3043 } 3044 3045 return 0; 3046 err_msix: 3047 for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--) 3048 free_irq(be_msix_vec_get(adapter, eqo), eqo); 3049 dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n", 3050 status); 3051 be_msix_disable(adapter); 3052 return status; 3053 } 3054 3055 static int be_irq_register(struct be_adapter *adapter) 3056 { 3057 struct net_device *netdev = adapter->netdev; 3058 int status; 3059 3060 if (msix_enabled(adapter)) { 3061 status = be_msix_register(adapter); 3062 if (status == 0) 3063 goto done; 3064 /* INTx is not supported for VF */ 3065 if (!be_physfn(adapter)) 3066 return status; 3067 } 3068 3069 /* INTx: only the first EQ is used */ 3070 netdev->irq = adapter->pdev->irq; 3071 status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name, 3072 &adapter->eq_obj[0]); 3073 if (status) { 3074 dev_err(&adapter->pdev->dev, 3075 "INTx request IRQ failed - err %d\n", status); 3076 return status; 3077 } 3078 done: 3079 adapter->isr_registered = true; 3080 return 0; 3081 } 3082 3083 static void be_irq_unregister(struct be_adapter *adapter) 3084 { 3085 struct net_device *netdev = adapter->netdev; 3086 struct be_eq_obj *eqo; 3087 int i, vec; 3088 3089 if (!adapter->isr_registered) 3090 return; 3091 3092 /* INTx */ 3093 if (!msix_enabled(adapter)) { 3094 free_irq(netdev->irq, &adapter->eq_obj[0]); 3095 goto done; 3096 } 3097 3098 /* MSIx */ 3099 for_all_evt_queues(adapter, eqo, i) { 3100 vec = be_msix_vec_get(adapter, eqo); 3101 irq_set_affinity_hint(vec, NULL); 3102 free_irq(vec, eqo); 3103 } 3104 3105 done: 3106 adapter->isr_registered = false; 3107 } 3108 3109 static void be_rx_qs_destroy(struct be_adapter *adapter) 3110 { 3111 struct be_queue_info *q; 3112 struct be_rx_obj *rxo; 3113 int i; 3114 3115 for_all_rx_queues(adapter, rxo, i) { 3116 q = &rxo->q; 3117 if (q->created) { 3118 be_cmd_rxq_destroy(adapter, q); 3119 be_rx_cq_clean(rxo); 3120 } 3121 be_queue_free(adapter, q); 3122 } 3123 } 3124 3125 static int be_close(struct net_device *netdev) 3126 { 3127 struct be_adapter *adapter = netdev_priv(netdev); 3128 struct be_eq_obj *eqo; 3129 int i; 3130 3131 /* This protection is needed as be_close() may be called even when the 3132 * adapter is in cleared state (after eeh perm failure) 3133 */ 3134 if (!(adapter->flags & BE_FLAGS_SETUP_DONE)) 3135 return 0; 3136 3137 be_roce_dev_close(adapter); 3138 3139 if (adapter->flags & BE_FLAGS_NAPI_ENABLED) { 3140 for_all_evt_queues(adapter, eqo, i) { 3141 napi_disable(&eqo->napi); 3142 be_disable_busy_poll(eqo); 3143 } 3144 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED; 3145 } 3146 3147 be_async_mcc_disable(adapter); 3148 3149 /* Wait for all pending tx completions to arrive so that 3150 * all tx skbs are freed. 3151 */ 3152 netif_tx_disable(netdev); 3153 be_tx_compl_clean(adapter); 3154 3155 be_rx_qs_destroy(adapter); 3156 be_clear_uc_list(adapter); 3157 3158 for_all_evt_queues(adapter, eqo, i) { 3159 if (msix_enabled(adapter)) 3160 synchronize_irq(be_msix_vec_get(adapter, eqo)); 3161 else 3162 synchronize_irq(netdev->irq); 3163 be_eq_clean(eqo); 3164 } 3165 3166 be_irq_unregister(adapter); 3167 3168 return 0; 3169 } 3170 3171 static int be_rx_qs_create(struct be_adapter *adapter) 3172 { 3173 struct rss_info *rss = &adapter->rss_info; 3174 u8 rss_key[RSS_HASH_KEY_LEN]; 3175 struct be_rx_obj *rxo; 3176 int rc, i, j; 3177 3178 for_all_rx_queues(adapter, rxo, i) { 3179 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN, 3180 sizeof(struct be_eth_rx_d)); 3181 if (rc) 3182 return rc; 3183 } 3184 3185 if (adapter->need_def_rxq || !adapter->num_rss_qs) { 3186 rxo = default_rxo(adapter); 3187 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3188 rx_frag_size, adapter->if_handle, 3189 false, &rxo->rss_id); 3190 if (rc) 3191 return rc; 3192 } 3193 3194 for_all_rss_queues(adapter, rxo, i) { 3195 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3196 rx_frag_size, adapter->if_handle, 3197 true, &rxo->rss_id); 3198 if (rc) 3199 return rc; 3200 } 3201 3202 if (be_multi_rxq(adapter)) { 3203 for (j = 0; j < RSS_INDIR_TABLE_LEN; j += adapter->num_rss_qs) { 3204 for_all_rss_queues(adapter, rxo, i) { 3205 if ((j + i) >= RSS_INDIR_TABLE_LEN) 3206 break; 3207 rss->rsstable[j + i] = rxo->rss_id; 3208 rss->rss_queue[j + i] = i; 3209 } 3210 } 3211 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 | 3212 RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6; 3213 3214 if (!BEx_chip(adapter)) 3215 rss->rss_flags |= RSS_ENABLE_UDP_IPV4 | 3216 RSS_ENABLE_UDP_IPV6; 3217 } else { 3218 /* Disable RSS, if only default RX Q is created */ 3219 rss->rss_flags = RSS_ENABLE_NONE; 3220 } 3221 3222 netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN); 3223 rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3224 128, rss_key); 3225 if (rc) { 3226 rss->rss_flags = RSS_ENABLE_NONE; 3227 return rc; 3228 } 3229 3230 memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN); 3231 3232 /* First time posting */ 3233 for_all_rx_queues(adapter, rxo, i) 3234 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST); 3235 return 0; 3236 } 3237 3238 static int be_open(struct net_device *netdev) 3239 { 3240 struct be_adapter *adapter = netdev_priv(netdev); 3241 struct be_eq_obj *eqo; 3242 struct be_rx_obj *rxo; 3243 struct be_tx_obj *txo; 3244 u8 link_status; 3245 int status, i; 3246 3247 status = be_rx_qs_create(adapter); 3248 if (status) 3249 goto err; 3250 3251 status = be_irq_register(adapter); 3252 if (status) 3253 goto err; 3254 3255 for_all_rx_queues(adapter, rxo, i) 3256 be_cq_notify(adapter, rxo->cq.id, true, 0); 3257 3258 for_all_tx_queues(adapter, txo, i) 3259 be_cq_notify(adapter, txo->cq.id, true, 0); 3260 3261 be_async_mcc_enable(adapter); 3262 3263 for_all_evt_queues(adapter, eqo, i) { 3264 napi_enable(&eqo->napi); 3265 be_enable_busy_poll(eqo); 3266 be_eq_notify(adapter, eqo->q.id, true, true, 0); 3267 } 3268 adapter->flags |= BE_FLAGS_NAPI_ENABLED; 3269 3270 status = be_cmd_link_status_query(adapter, NULL, &link_status, 0); 3271 if (!status) 3272 be_link_status_update(adapter, link_status); 3273 3274 netif_tx_start_all_queues(netdev); 3275 be_roce_dev_open(adapter); 3276 3277 #ifdef CONFIG_BE2NET_VXLAN 3278 if (skyhawk_chip(adapter)) 3279 vxlan_get_rx_port(netdev); 3280 #endif 3281 3282 return 0; 3283 err: 3284 be_close(adapter->netdev); 3285 return -EIO; 3286 } 3287 3288 static int be_setup_wol(struct be_adapter *adapter, bool enable) 3289 { 3290 struct be_dma_mem cmd; 3291 int status = 0; 3292 u8 mac[ETH_ALEN]; 3293 3294 eth_zero_addr(mac); 3295 3296 cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config); 3297 cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma, 3298 GFP_KERNEL); 3299 if (!cmd.va) 3300 return -ENOMEM; 3301 3302 if (enable) { 3303 status = pci_write_config_dword(adapter->pdev, 3304 PCICFG_PM_CONTROL_OFFSET, 3305 PCICFG_PM_CONTROL_MASK); 3306 if (status) { 3307 dev_err(&adapter->pdev->dev, 3308 "Could not enable Wake-on-lan\n"); 3309 dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, 3310 cmd.dma); 3311 return status; 3312 } 3313 status = be_cmd_enable_magic_wol(adapter, 3314 adapter->netdev->dev_addr, 3315 &cmd); 3316 pci_enable_wake(adapter->pdev, PCI_D3hot, 1); 3317 pci_enable_wake(adapter->pdev, PCI_D3cold, 1); 3318 } else { 3319 status = be_cmd_enable_magic_wol(adapter, mac, &cmd); 3320 pci_enable_wake(adapter->pdev, PCI_D3hot, 0); 3321 pci_enable_wake(adapter->pdev, PCI_D3cold, 0); 3322 } 3323 3324 dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma); 3325 return status; 3326 } 3327 3328 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac) 3329 { 3330 u32 addr; 3331 3332 addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0); 3333 3334 mac[5] = (u8)(addr & 0xFF); 3335 mac[4] = (u8)((addr >> 8) & 0xFF); 3336 mac[3] = (u8)((addr >> 16) & 0xFF); 3337 /* Use the OUI from the current MAC address */ 3338 memcpy(mac, adapter->netdev->dev_addr, 3); 3339 } 3340 3341 /* 3342 * Generate a seed MAC address from the PF MAC Address using jhash. 3343 * MAC Address for VFs are assigned incrementally starting from the seed. 3344 * These addresses are programmed in the ASIC by the PF and the VF driver 3345 * queries for the MAC address during its probe. 3346 */ 3347 static int be_vf_eth_addr_config(struct be_adapter *adapter) 3348 { 3349 u32 vf; 3350 int status = 0; 3351 u8 mac[ETH_ALEN]; 3352 struct be_vf_cfg *vf_cfg; 3353 3354 be_vf_eth_addr_generate(adapter, mac); 3355 3356 for_all_vfs(adapter, vf_cfg, vf) { 3357 if (BEx_chip(adapter)) 3358 status = be_cmd_pmac_add(adapter, mac, 3359 vf_cfg->if_handle, 3360 &vf_cfg->pmac_id, vf + 1); 3361 else 3362 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 3363 vf + 1); 3364 3365 if (status) 3366 dev_err(&adapter->pdev->dev, 3367 "Mac address assignment failed for VF %d\n", 3368 vf); 3369 else 3370 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3371 3372 mac[5] += 1; 3373 } 3374 return status; 3375 } 3376 3377 static int be_vfs_mac_query(struct be_adapter *adapter) 3378 { 3379 int status, vf; 3380 u8 mac[ETH_ALEN]; 3381 struct be_vf_cfg *vf_cfg; 3382 3383 for_all_vfs(adapter, vf_cfg, vf) { 3384 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id, 3385 mac, vf_cfg->if_handle, 3386 false, vf+1); 3387 if (status) 3388 return status; 3389 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3390 } 3391 return 0; 3392 } 3393 3394 static void be_vf_clear(struct be_adapter *adapter) 3395 { 3396 struct be_vf_cfg *vf_cfg; 3397 u32 vf; 3398 3399 if (pci_vfs_assigned(adapter->pdev)) { 3400 dev_warn(&adapter->pdev->dev, 3401 "VFs are assigned to VMs: not disabling VFs\n"); 3402 goto done; 3403 } 3404 3405 pci_disable_sriov(adapter->pdev); 3406 3407 for_all_vfs(adapter, vf_cfg, vf) { 3408 if (BEx_chip(adapter)) 3409 be_cmd_pmac_del(adapter, vf_cfg->if_handle, 3410 vf_cfg->pmac_id, vf + 1); 3411 else 3412 be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle, 3413 vf + 1); 3414 3415 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1); 3416 } 3417 done: 3418 kfree(adapter->vf_cfg); 3419 adapter->num_vfs = 0; 3420 adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED; 3421 } 3422 3423 static void be_clear_queues(struct be_adapter *adapter) 3424 { 3425 be_mcc_queues_destroy(adapter); 3426 be_rx_cqs_destroy(adapter); 3427 be_tx_queues_destroy(adapter); 3428 be_evt_queues_destroy(adapter); 3429 } 3430 3431 static void be_cancel_worker(struct be_adapter *adapter) 3432 { 3433 if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) { 3434 cancel_delayed_work_sync(&adapter->work); 3435 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED; 3436 } 3437 } 3438 3439 static void be_cancel_err_detection(struct be_adapter *adapter) 3440 { 3441 if (adapter->flags & BE_FLAGS_ERR_DETECTION_SCHEDULED) { 3442 cancel_delayed_work_sync(&adapter->be_err_detection_work); 3443 adapter->flags &= ~BE_FLAGS_ERR_DETECTION_SCHEDULED; 3444 } 3445 } 3446 3447 static void be_mac_clear(struct be_adapter *adapter) 3448 { 3449 if (adapter->pmac_id) { 3450 be_cmd_pmac_del(adapter, adapter->if_handle, 3451 adapter->pmac_id[0], 0); 3452 kfree(adapter->pmac_id); 3453 adapter->pmac_id = NULL; 3454 } 3455 } 3456 3457 #ifdef CONFIG_BE2NET_VXLAN 3458 static void be_disable_vxlan_offloads(struct be_adapter *adapter) 3459 { 3460 struct net_device *netdev = adapter->netdev; 3461 3462 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) 3463 be_cmd_manage_iface(adapter, adapter->if_handle, 3464 OP_CONVERT_TUNNEL_TO_NORMAL); 3465 3466 if (adapter->vxlan_port) 3467 be_cmd_set_vxlan_port(adapter, 0); 3468 3469 adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS; 3470 adapter->vxlan_port = 0; 3471 3472 netdev->hw_enc_features = 0; 3473 netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL); 3474 netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL); 3475 } 3476 #endif 3477 3478 static u16 be_calculate_vf_qs(struct be_adapter *adapter, u16 num_vfs) 3479 { 3480 struct be_resources res = adapter->pool_res; 3481 u16 num_vf_qs = 1; 3482 3483 /* Distribute the queue resources equally among the PF and it's VFs 3484 * Do not distribute queue resources in multi-channel configuration. 3485 */ 3486 if (num_vfs && !be_is_mc(adapter)) { 3487 /* If number of VFs requested is 8 less than max supported, 3488 * assign 8 queue pairs to the PF and divide the remaining 3489 * resources evenly among the VFs 3490 */ 3491 if (num_vfs < (be_max_vfs(adapter) - 8)) 3492 num_vf_qs = (res.max_rss_qs - 8) / num_vfs; 3493 else 3494 num_vf_qs = res.max_rss_qs / num_vfs; 3495 3496 /* Skyhawk-R chip supports only MAX_RSS_IFACES RSS capable 3497 * interfaces per port. Provide RSS on VFs, only if number 3498 * of VFs requested is less than MAX_RSS_IFACES limit. 3499 */ 3500 if (num_vfs >= MAX_RSS_IFACES) 3501 num_vf_qs = 1; 3502 } 3503 return num_vf_qs; 3504 } 3505 3506 static int be_clear(struct be_adapter *adapter) 3507 { 3508 struct pci_dev *pdev = adapter->pdev; 3509 u16 num_vf_qs; 3510 3511 be_cancel_worker(adapter); 3512 3513 if (sriov_enabled(adapter)) 3514 be_vf_clear(adapter); 3515 3516 /* Re-configure FW to distribute resources evenly across max-supported 3517 * number of VFs, only when VFs are not already enabled. 3518 */ 3519 if (skyhawk_chip(adapter) && be_physfn(adapter) && 3520 !pci_vfs_assigned(pdev)) { 3521 num_vf_qs = be_calculate_vf_qs(adapter, 3522 pci_sriov_get_totalvfs(pdev)); 3523 be_cmd_set_sriov_config(adapter, adapter->pool_res, 3524 pci_sriov_get_totalvfs(pdev), 3525 num_vf_qs); 3526 } 3527 3528 #ifdef CONFIG_BE2NET_VXLAN 3529 be_disable_vxlan_offloads(adapter); 3530 #endif 3531 /* delete the primary mac along with the uc-mac list */ 3532 be_mac_clear(adapter); 3533 3534 be_cmd_if_destroy(adapter, adapter->if_handle, 0); 3535 3536 be_clear_queues(adapter); 3537 3538 be_msix_disable(adapter); 3539 adapter->flags &= ~BE_FLAGS_SETUP_DONE; 3540 return 0; 3541 } 3542 3543 static int be_if_create(struct be_adapter *adapter, u32 *if_handle, 3544 u32 cap_flags, u32 vf) 3545 { 3546 u32 en_flags; 3547 3548 en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST | 3549 BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS | 3550 BE_IF_FLAGS_RSS | BE_IF_FLAGS_DEFQ_RSS; 3551 3552 en_flags &= cap_flags; 3553 3554 return be_cmd_if_create(adapter, cap_flags, en_flags, if_handle, vf); 3555 } 3556 3557 static int be_vfs_if_create(struct be_adapter *adapter) 3558 { 3559 struct be_resources res = {0}; 3560 struct be_vf_cfg *vf_cfg; 3561 u32 cap_flags, vf; 3562 int status; 3563 3564 /* If a FW profile exists, then cap_flags are updated */ 3565 cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST | 3566 BE_IF_FLAGS_MULTICAST; 3567 3568 for_all_vfs(adapter, vf_cfg, vf) { 3569 if (!BE3_chip(adapter)) { 3570 status = be_cmd_get_profile_config(adapter, &res, 3571 RESOURCE_LIMITS, 3572 vf + 1); 3573 if (!status) { 3574 cap_flags = res.if_cap_flags; 3575 /* Prevent VFs from enabling VLAN promiscuous 3576 * mode 3577 */ 3578 cap_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 3579 } 3580 } 3581 3582 status = be_if_create(adapter, &vf_cfg->if_handle, 3583 cap_flags, vf + 1); 3584 if (status) 3585 return status; 3586 } 3587 3588 return 0; 3589 } 3590 3591 static int be_vf_setup_init(struct be_adapter *adapter) 3592 { 3593 struct be_vf_cfg *vf_cfg; 3594 int vf; 3595 3596 adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg), 3597 GFP_KERNEL); 3598 if (!adapter->vf_cfg) 3599 return -ENOMEM; 3600 3601 for_all_vfs(adapter, vf_cfg, vf) { 3602 vf_cfg->if_handle = -1; 3603 vf_cfg->pmac_id = -1; 3604 } 3605 return 0; 3606 } 3607 3608 static int be_vf_setup(struct be_adapter *adapter) 3609 { 3610 struct device *dev = &adapter->pdev->dev; 3611 struct be_vf_cfg *vf_cfg; 3612 int status, old_vfs, vf; 3613 3614 old_vfs = pci_num_vf(adapter->pdev); 3615 3616 status = be_vf_setup_init(adapter); 3617 if (status) 3618 goto err; 3619 3620 if (old_vfs) { 3621 for_all_vfs(adapter, vf_cfg, vf) { 3622 status = be_cmd_get_if_id(adapter, vf_cfg, vf); 3623 if (status) 3624 goto err; 3625 } 3626 3627 status = be_vfs_mac_query(adapter); 3628 if (status) 3629 goto err; 3630 } else { 3631 status = be_vfs_if_create(adapter); 3632 if (status) 3633 goto err; 3634 3635 status = be_vf_eth_addr_config(adapter); 3636 if (status) 3637 goto err; 3638 } 3639 3640 for_all_vfs(adapter, vf_cfg, vf) { 3641 /* Allow VFs to programs MAC/VLAN filters */ 3642 status = be_cmd_get_fn_privileges(adapter, &vf_cfg->privileges, 3643 vf + 1); 3644 if (!status && !(vf_cfg->privileges & BE_PRIV_FILTMGMT)) { 3645 status = be_cmd_set_fn_privileges(adapter, 3646 vf_cfg->privileges | 3647 BE_PRIV_FILTMGMT, 3648 vf + 1); 3649 if (!status) { 3650 vf_cfg->privileges |= BE_PRIV_FILTMGMT; 3651 dev_info(dev, "VF%d has FILTMGMT privilege\n", 3652 vf); 3653 } 3654 } 3655 3656 /* Allow full available bandwidth */ 3657 if (!old_vfs) 3658 be_cmd_config_qos(adapter, 0, 0, vf + 1); 3659 3660 if (!old_vfs) { 3661 be_cmd_enable_vf(adapter, vf + 1); 3662 be_cmd_set_logical_link_config(adapter, 3663 IFLA_VF_LINK_STATE_AUTO, 3664 vf+1); 3665 } 3666 } 3667 3668 if (!old_vfs) { 3669 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs); 3670 if (status) { 3671 dev_err(dev, "SRIOV enable failed\n"); 3672 adapter->num_vfs = 0; 3673 goto err; 3674 } 3675 } 3676 3677 adapter->flags |= BE_FLAGS_SRIOV_ENABLED; 3678 return 0; 3679 err: 3680 dev_err(dev, "VF setup failed\n"); 3681 be_vf_clear(adapter); 3682 return status; 3683 } 3684 3685 /* Converting function_mode bits on BE3 to SH mc_type enums */ 3686 3687 static u8 be_convert_mc_type(u32 function_mode) 3688 { 3689 if (function_mode & VNIC_MODE && function_mode & QNQ_MODE) 3690 return vNIC1; 3691 else if (function_mode & QNQ_MODE) 3692 return FLEX10; 3693 else if (function_mode & VNIC_MODE) 3694 return vNIC2; 3695 else if (function_mode & UMC_ENABLED) 3696 return UMC; 3697 else 3698 return MC_NONE; 3699 } 3700 3701 /* On BE2/BE3 FW does not suggest the supported limits */ 3702 static void BEx_get_resources(struct be_adapter *adapter, 3703 struct be_resources *res) 3704 { 3705 bool use_sriov = adapter->num_vfs ? 1 : 0; 3706 3707 if (be_physfn(adapter)) 3708 res->max_uc_mac = BE_UC_PMAC_COUNT; 3709 else 3710 res->max_uc_mac = BE_VF_UC_PMAC_COUNT; 3711 3712 adapter->mc_type = be_convert_mc_type(adapter->function_mode); 3713 3714 if (be_is_mc(adapter)) { 3715 /* Assuming that there are 4 channels per port, 3716 * when multi-channel is enabled 3717 */ 3718 if (be_is_qnq_mode(adapter)) 3719 res->max_vlans = BE_NUM_VLANS_SUPPORTED/8; 3720 else 3721 /* In a non-qnq multichannel mode, the pvid 3722 * takes up one vlan entry 3723 */ 3724 res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1; 3725 } else { 3726 res->max_vlans = BE_NUM_VLANS_SUPPORTED; 3727 } 3728 3729 res->max_mcast_mac = BE_MAX_MC; 3730 3731 /* 1) For BE3 1Gb ports, FW does not support multiple TXQs 3732 * 2) Create multiple TX rings on a BE3-R multi-channel interface 3733 * *only* if it is RSS-capable. 3734 */ 3735 if (BE2_chip(adapter) || use_sriov || (adapter->port_num > 1) || 3736 !be_physfn(adapter) || (be_is_mc(adapter) && 3737 !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) { 3738 res->max_tx_qs = 1; 3739 } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) { 3740 struct be_resources super_nic_res = {0}; 3741 3742 /* On a SuperNIC profile, the driver needs to use the 3743 * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits 3744 */ 3745 be_cmd_get_profile_config(adapter, &super_nic_res, 3746 RESOURCE_LIMITS, 0); 3747 /* Some old versions of BE3 FW don't report max_tx_qs value */ 3748 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS; 3749 } else { 3750 res->max_tx_qs = BE3_MAX_TX_QS; 3751 } 3752 3753 if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) && 3754 !use_sriov && be_physfn(adapter)) 3755 res->max_rss_qs = (adapter->be3_native) ? 3756 BE3_MAX_RSS_QS : BE2_MAX_RSS_QS; 3757 res->max_rx_qs = res->max_rss_qs + 1; 3758 3759 if (be_physfn(adapter)) 3760 res->max_evt_qs = (be_max_vfs(adapter) > 0) ? 3761 BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS; 3762 else 3763 res->max_evt_qs = 1; 3764 3765 res->if_cap_flags = BE_IF_CAP_FLAGS_WANT; 3766 res->if_cap_flags &= ~BE_IF_FLAGS_DEFQ_RSS; 3767 if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS)) 3768 res->if_cap_flags &= ~BE_IF_FLAGS_RSS; 3769 } 3770 3771 static void be_setup_init(struct be_adapter *adapter) 3772 { 3773 adapter->vlan_prio_bmap = 0xff; 3774 adapter->phy.link_speed = -1; 3775 adapter->if_handle = -1; 3776 adapter->be3_native = false; 3777 adapter->if_flags = 0; 3778 if (be_physfn(adapter)) 3779 adapter->cmd_privileges = MAX_PRIVILEGES; 3780 else 3781 adapter->cmd_privileges = MIN_PRIVILEGES; 3782 } 3783 3784 static int be_get_sriov_config(struct be_adapter *adapter) 3785 { 3786 struct be_resources res = {0}; 3787 int max_vfs, old_vfs; 3788 3789 be_cmd_get_profile_config(adapter, &res, RESOURCE_LIMITS, 0); 3790 3791 /* Some old versions of BE3 FW don't report max_vfs value */ 3792 if (BE3_chip(adapter) && !res.max_vfs) { 3793 max_vfs = pci_sriov_get_totalvfs(adapter->pdev); 3794 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0; 3795 } 3796 3797 adapter->pool_res = res; 3798 3799 /* If during previous unload of the driver, the VFs were not disabled, 3800 * then we cannot rely on the PF POOL limits for the TotalVFs value. 3801 * Instead use the TotalVFs value stored in the pci-dev struct. 3802 */ 3803 old_vfs = pci_num_vf(adapter->pdev); 3804 if (old_vfs) { 3805 dev_info(&adapter->pdev->dev, "%d VFs are already enabled\n", 3806 old_vfs); 3807 3808 adapter->pool_res.max_vfs = 3809 pci_sriov_get_totalvfs(adapter->pdev); 3810 adapter->num_vfs = old_vfs; 3811 } 3812 3813 return 0; 3814 } 3815 3816 static void be_alloc_sriov_res(struct be_adapter *adapter) 3817 { 3818 int old_vfs = pci_num_vf(adapter->pdev); 3819 u16 num_vf_qs; 3820 int status; 3821 3822 be_get_sriov_config(adapter); 3823 3824 if (!old_vfs) 3825 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter)); 3826 3827 /* When the HW is in SRIOV capable configuration, the PF-pool 3828 * resources are given to PF during driver load, if there are no 3829 * old VFs. This facility is not available in BE3 FW. 3830 * Also, this is done by FW in Lancer chip. 3831 */ 3832 if (skyhawk_chip(adapter) && be_max_vfs(adapter) && !old_vfs) { 3833 num_vf_qs = be_calculate_vf_qs(adapter, 0); 3834 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 0, 3835 num_vf_qs); 3836 if (status) 3837 dev_err(&adapter->pdev->dev, 3838 "Failed to optimize SRIOV resources\n"); 3839 } 3840 } 3841 3842 static int be_get_resources(struct be_adapter *adapter) 3843 { 3844 struct device *dev = &adapter->pdev->dev; 3845 struct be_resources res = {0}; 3846 int status; 3847 3848 if (BEx_chip(adapter)) { 3849 BEx_get_resources(adapter, &res); 3850 adapter->res = res; 3851 } 3852 3853 /* For Lancer, SH etc read per-function resource limits from FW. 3854 * GET_FUNC_CONFIG returns per function guaranteed limits. 3855 * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits 3856 */ 3857 if (!BEx_chip(adapter)) { 3858 status = be_cmd_get_func_config(adapter, &res); 3859 if (status) 3860 return status; 3861 3862 /* If a deafault RXQ must be created, we'll use up one RSSQ*/ 3863 if (res.max_rss_qs && res.max_rss_qs == res.max_rx_qs && 3864 !(res.if_cap_flags & BE_IF_FLAGS_DEFQ_RSS)) 3865 res.max_rss_qs -= 1; 3866 3867 /* If RoCE may be enabled stash away half the EQs for RoCE */ 3868 if (be_roce_supported(adapter)) 3869 res.max_evt_qs /= 2; 3870 adapter->res = res; 3871 } 3872 3873 /* If FW supports RSS default queue, then skip creating non-RSS 3874 * queue for non-IP traffic. 3875 */ 3876 adapter->need_def_rxq = (be_if_cap_flags(adapter) & 3877 BE_IF_FLAGS_DEFQ_RSS) ? 0 : 1; 3878 3879 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n", 3880 be_max_txqs(adapter), be_max_rxqs(adapter), 3881 be_max_rss(adapter), be_max_eqs(adapter), 3882 be_max_vfs(adapter)); 3883 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n", 3884 be_max_uc(adapter), be_max_mc(adapter), 3885 be_max_vlans(adapter)); 3886 3887 /* Sanitize cfg_num_qs based on HW and platform limits */ 3888 adapter->cfg_num_qs = min_t(u16, netif_get_num_default_rss_queues(), 3889 be_max_qs(adapter)); 3890 return 0; 3891 } 3892 3893 static int be_get_config(struct be_adapter *adapter) 3894 { 3895 int status, level; 3896 u16 profile_id; 3897 3898 status = be_cmd_get_cntl_attributes(adapter); 3899 if (status) 3900 return status; 3901 3902 status = be_cmd_query_fw_cfg(adapter); 3903 if (status) 3904 return status; 3905 3906 if (BEx_chip(adapter)) { 3907 level = be_cmd_get_fw_log_level(adapter); 3908 adapter->msg_enable = 3909 level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0; 3910 } 3911 3912 be_cmd_get_acpi_wol_cap(adapter); 3913 3914 be_cmd_query_port_name(adapter); 3915 3916 if (be_physfn(adapter)) { 3917 status = be_cmd_get_active_profile(adapter, &profile_id); 3918 if (!status) 3919 dev_info(&adapter->pdev->dev, 3920 "Using profile 0x%x\n", profile_id); 3921 } 3922 3923 status = be_get_resources(adapter); 3924 if (status) 3925 return status; 3926 3927 adapter->pmac_id = kcalloc(be_max_uc(adapter), 3928 sizeof(*adapter->pmac_id), GFP_KERNEL); 3929 if (!adapter->pmac_id) 3930 return -ENOMEM; 3931 3932 return 0; 3933 } 3934 3935 static int be_mac_setup(struct be_adapter *adapter) 3936 { 3937 u8 mac[ETH_ALEN]; 3938 int status; 3939 3940 if (is_zero_ether_addr(adapter->netdev->dev_addr)) { 3941 status = be_cmd_get_perm_mac(adapter, mac); 3942 if (status) 3943 return status; 3944 3945 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN); 3946 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN); 3947 } else { 3948 /* Maybe the HW was reset; dev_addr must be re-programmed */ 3949 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN); 3950 } 3951 3952 /* For BE3-R VFs, the PF programs the initial MAC address */ 3953 if (!(BEx_chip(adapter) && be_virtfn(adapter))) 3954 be_cmd_pmac_add(adapter, mac, adapter->if_handle, 3955 &adapter->pmac_id[0], 0); 3956 return 0; 3957 } 3958 3959 static void be_schedule_worker(struct be_adapter *adapter) 3960 { 3961 schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000)); 3962 adapter->flags |= BE_FLAGS_WORKER_SCHEDULED; 3963 } 3964 3965 static void be_schedule_err_detection(struct be_adapter *adapter) 3966 { 3967 schedule_delayed_work(&adapter->be_err_detection_work, 3968 msecs_to_jiffies(1000)); 3969 adapter->flags |= BE_FLAGS_ERR_DETECTION_SCHEDULED; 3970 } 3971 3972 static int be_setup_queues(struct be_adapter *adapter) 3973 { 3974 struct net_device *netdev = adapter->netdev; 3975 int status; 3976 3977 status = be_evt_queues_create(adapter); 3978 if (status) 3979 goto err; 3980 3981 status = be_tx_qs_create(adapter); 3982 if (status) 3983 goto err; 3984 3985 status = be_rx_cqs_create(adapter); 3986 if (status) 3987 goto err; 3988 3989 status = be_mcc_queues_create(adapter); 3990 if (status) 3991 goto err; 3992 3993 status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs); 3994 if (status) 3995 goto err; 3996 3997 status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs); 3998 if (status) 3999 goto err; 4000 4001 return 0; 4002 err: 4003 dev_err(&adapter->pdev->dev, "queue_setup failed\n"); 4004 return status; 4005 } 4006 4007 int be_update_queues(struct be_adapter *adapter) 4008 { 4009 struct net_device *netdev = adapter->netdev; 4010 int status; 4011 4012 if (netif_running(netdev)) 4013 be_close(netdev); 4014 4015 be_cancel_worker(adapter); 4016 4017 /* If any vectors have been shared with RoCE we cannot re-program 4018 * the MSIx table. 4019 */ 4020 if (!adapter->num_msix_roce_vec) 4021 be_msix_disable(adapter); 4022 4023 be_clear_queues(adapter); 4024 4025 if (!msix_enabled(adapter)) { 4026 status = be_msix_enable(adapter); 4027 if (status) 4028 return status; 4029 } 4030 4031 status = be_setup_queues(adapter); 4032 if (status) 4033 return status; 4034 4035 be_schedule_worker(adapter); 4036 4037 if (netif_running(netdev)) 4038 status = be_open(netdev); 4039 4040 return status; 4041 } 4042 4043 static inline int fw_major_num(const char *fw_ver) 4044 { 4045 int fw_major = 0, i; 4046 4047 i = sscanf(fw_ver, "%d.", &fw_major); 4048 if (i != 1) 4049 return 0; 4050 4051 return fw_major; 4052 } 4053 4054 /* If any VFs are already enabled don't FLR the PF */ 4055 static bool be_reset_required(struct be_adapter *adapter) 4056 { 4057 return pci_num_vf(adapter->pdev) ? false : true; 4058 } 4059 4060 /* Wait for the FW to be ready and perform the required initialization */ 4061 static int be_func_init(struct be_adapter *adapter) 4062 { 4063 int status; 4064 4065 status = be_fw_wait_ready(adapter); 4066 if (status) 4067 return status; 4068 4069 if (be_reset_required(adapter)) { 4070 status = be_cmd_reset_function(adapter); 4071 if (status) 4072 return status; 4073 4074 /* Wait for interrupts to quiesce after an FLR */ 4075 msleep(100); 4076 4077 /* We can clear all errors when function reset succeeds */ 4078 be_clear_all_error(adapter); 4079 } 4080 4081 /* Tell FW we're ready to fire cmds */ 4082 status = be_cmd_fw_init(adapter); 4083 if (status) 4084 return status; 4085 4086 /* Allow interrupts for other ULPs running on NIC function */ 4087 be_intr_set(adapter, true); 4088 4089 return 0; 4090 } 4091 4092 static int be_setup(struct be_adapter *adapter) 4093 { 4094 struct device *dev = &adapter->pdev->dev; 4095 int status; 4096 4097 status = be_func_init(adapter); 4098 if (status) 4099 return status; 4100 4101 be_setup_init(adapter); 4102 4103 if (!lancer_chip(adapter)) 4104 be_cmd_req_native_mode(adapter); 4105 4106 if (!BE2_chip(adapter) && be_physfn(adapter)) 4107 be_alloc_sriov_res(adapter); 4108 4109 status = be_get_config(adapter); 4110 if (status) 4111 goto err; 4112 4113 status = be_msix_enable(adapter); 4114 if (status) 4115 goto err; 4116 4117 status = be_if_create(adapter, &adapter->if_handle, 4118 be_if_cap_flags(adapter), 0); 4119 if (status) 4120 goto err; 4121 4122 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 4123 rtnl_lock(); 4124 status = be_setup_queues(adapter); 4125 rtnl_unlock(); 4126 if (status) 4127 goto err; 4128 4129 be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0); 4130 4131 status = be_mac_setup(adapter); 4132 if (status) 4133 goto err; 4134 4135 be_cmd_get_fw_ver(adapter); 4136 dev_info(dev, "FW version is %s\n", adapter->fw_ver); 4137 4138 if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) { 4139 dev_err(dev, "Firmware on card is old(%s), IRQs may not work", 4140 adapter->fw_ver); 4141 dev_err(dev, "Please upgrade firmware to version >= 4.0\n"); 4142 } 4143 4144 if (adapter->vlans_added) 4145 be_vid_config(adapter); 4146 4147 be_set_rx_mode(adapter->netdev); 4148 4149 status = be_cmd_set_flow_control(adapter, adapter->tx_fc, 4150 adapter->rx_fc); 4151 if (status) 4152 be_cmd_get_flow_control(adapter, &adapter->tx_fc, 4153 &adapter->rx_fc); 4154 4155 dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n", 4156 adapter->tx_fc, adapter->rx_fc); 4157 4158 if (be_physfn(adapter)) 4159 be_cmd_set_logical_link_config(adapter, 4160 IFLA_VF_LINK_STATE_AUTO, 0); 4161 4162 if (adapter->num_vfs) 4163 be_vf_setup(adapter); 4164 4165 status = be_cmd_get_phy_info(adapter); 4166 if (!status && be_pause_supported(adapter)) 4167 adapter->phy.fc_autoneg = 1; 4168 4169 be_schedule_worker(adapter); 4170 adapter->flags |= BE_FLAGS_SETUP_DONE; 4171 return 0; 4172 err: 4173 be_clear(adapter); 4174 return status; 4175 } 4176 4177 #ifdef CONFIG_NET_POLL_CONTROLLER 4178 static void be_netpoll(struct net_device *netdev) 4179 { 4180 struct be_adapter *adapter = netdev_priv(netdev); 4181 struct be_eq_obj *eqo; 4182 int i; 4183 4184 for_all_evt_queues(adapter, eqo, i) { 4185 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0); 4186 napi_schedule(&eqo->napi); 4187 } 4188 } 4189 #endif 4190 4191 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "}; 4192 4193 static bool phy_flashing_required(struct be_adapter *adapter) 4194 { 4195 return (adapter->phy.phy_type == PHY_TYPE_TN_8022 && 4196 adapter->phy.interface_type == PHY_TYPE_BASET_10GB); 4197 } 4198 4199 static bool is_comp_in_ufi(struct be_adapter *adapter, 4200 struct flash_section_info *fsec, int type) 4201 { 4202 int i = 0, img_type = 0; 4203 struct flash_section_info_g2 *fsec_g2 = NULL; 4204 4205 if (BE2_chip(adapter)) 4206 fsec_g2 = (struct flash_section_info_g2 *)fsec; 4207 4208 for (i = 0; i < MAX_FLASH_COMP; i++) { 4209 if (fsec_g2) 4210 img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type); 4211 else 4212 img_type = le32_to_cpu(fsec->fsec_entry[i].type); 4213 4214 if (img_type == type) 4215 return true; 4216 } 4217 return false; 4218 4219 } 4220 4221 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter, 4222 int header_size, 4223 const struct firmware *fw) 4224 { 4225 struct flash_section_info *fsec = NULL; 4226 const u8 *p = fw->data; 4227 4228 p += header_size; 4229 while (p < (fw->data + fw->size)) { 4230 fsec = (struct flash_section_info *)p; 4231 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie))) 4232 return fsec; 4233 p += 32; 4234 } 4235 return NULL; 4236 } 4237 4238 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p, 4239 u32 img_offset, u32 img_size, int hdr_size, 4240 u16 img_optype, bool *crc_match) 4241 { 4242 u32 crc_offset; 4243 int status; 4244 u8 crc[4]; 4245 4246 status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_offset, 4247 img_size - 4); 4248 if (status) 4249 return status; 4250 4251 crc_offset = hdr_size + img_offset + img_size - 4; 4252 4253 /* Skip flashing, if crc of flashed region matches */ 4254 if (!memcmp(crc, p + crc_offset, 4)) 4255 *crc_match = true; 4256 else 4257 *crc_match = false; 4258 4259 return status; 4260 } 4261 4262 static int be_flash(struct be_adapter *adapter, const u8 *img, 4263 struct be_dma_mem *flash_cmd, int optype, int img_size, 4264 u32 img_offset) 4265 { 4266 u32 flash_op, num_bytes, total_bytes = img_size, bytes_sent = 0; 4267 struct be_cmd_write_flashrom *req = flash_cmd->va; 4268 int status; 4269 4270 while (total_bytes) { 4271 num_bytes = min_t(u32, 32*1024, total_bytes); 4272 4273 total_bytes -= num_bytes; 4274 4275 if (!total_bytes) { 4276 if (optype == OPTYPE_PHY_FW) 4277 flash_op = FLASHROM_OPER_PHY_FLASH; 4278 else 4279 flash_op = FLASHROM_OPER_FLASH; 4280 } else { 4281 if (optype == OPTYPE_PHY_FW) 4282 flash_op = FLASHROM_OPER_PHY_SAVE; 4283 else 4284 flash_op = FLASHROM_OPER_SAVE; 4285 } 4286 4287 memcpy(req->data_buf, img, num_bytes); 4288 img += num_bytes; 4289 status = be_cmd_write_flashrom(adapter, flash_cmd, optype, 4290 flash_op, img_offset + 4291 bytes_sent, num_bytes); 4292 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST && 4293 optype == OPTYPE_PHY_FW) 4294 break; 4295 else if (status) 4296 return status; 4297 4298 bytes_sent += num_bytes; 4299 } 4300 return 0; 4301 } 4302 4303 /* For BE2, BE3 and BE3-R */ 4304 static int be_flash_BEx(struct be_adapter *adapter, 4305 const struct firmware *fw, 4306 struct be_dma_mem *flash_cmd, int num_of_images) 4307 { 4308 int img_hdrs_size = (num_of_images * sizeof(struct image_hdr)); 4309 struct device *dev = &adapter->pdev->dev; 4310 struct flash_section_info *fsec = NULL; 4311 int status, i, filehdr_size, num_comp; 4312 const struct flash_comp *pflashcomp; 4313 bool crc_match; 4314 const u8 *p; 4315 4316 struct flash_comp gen3_flash_types[] = { 4317 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE, 4318 FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI}, 4319 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT, 4320 FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE}, 4321 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS, 4322 FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI}, 4323 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS, 4324 FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE}, 4325 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS, 4326 FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE}, 4327 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP, 4328 FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI}, 4329 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE, 4330 FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE}, 4331 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP, 4332 FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE}, 4333 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW, 4334 FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI}, 4335 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW, 4336 FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY} 4337 }; 4338 4339 struct flash_comp gen2_flash_types[] = { 4340 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE, 4341 FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI}, 4342 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT, 4343 FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE}, 4344 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS, 4345 FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI}, 4346 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS, 4347 FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE}, 4348 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS, 4349 FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE}, 4350 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP, 4351 FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI}, 4352 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE, 4353 FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE}, 4354 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP, 4355 FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE} 4356 }; 4357 4358 if (BE3_chip(adapter)) { 4359 pflashcomp = gen3_flash_types; 4360 filehdr_size = sizeof(struct flash_file_hdr_g3); 4361 num_comp = ARRAY_SIZE(gen3_flash_types); 4362 } else { 4363 pflashcomp = gen2_flash_types; 4364 filehdr_size = sizeof(struct flash_file_hdr_g2); 4365 num_comp = ARRAY_SIZE(gen2_flash_types); 4366 img_hdrs_size = 0; 4367 } 4368 4369 /* Get flash section info*/ 4370 fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw); 4371 if (!fsec) { 4372 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n"); 4373 return -1; 4374 } 4375 for (i = 0; i < num_comp; i++) { 4376 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type)) 4377 continue; 4378 4379 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) && 4380 memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0) 4381 continue; 4382 4383 if (pflashcomp[i].optype == OPTYPE_PHY_FW && 4384 !phy_flashing_required(adapter)) 4385 continue; 4386 4387 if (pflashcomp[i].optype == OPTYPE_REDBOOT) { 4388 status = be_check_flash_crc(adapter, fw->data, 4389 pflashcomp[i].offset, 4390 pflashcomp[i].size, 4391 filehdr_size + 4392 img_hdrs_size, 4393 OPTYPE_REDBOOT, &crc_match); 4394 if (status) { 4395 dev_err(dev, 4396 "Could not get CRC for 0x%x region\n", 4397 pflashcomp[i].optype); 4398 continue; 4399 } 4400 4401 if (crc_match) 4402 continue; 4403 } 4404 4405 p = fw->data + filehdr_size + pflashcomp[i].offset + 4406 img_hdrs_size; 4407 if (p + pflashcomp[i].size > fw->data + fw->size) 4408 return -1; 4409 4410 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype, 4411 pflashcomp[i].size, 0); 4412 if (status) { 4413 dev_err(dev, "Flashing section type 0x%x failed\n", 4414 pflashcomp[i].img_type); 4415 return status; 4416 } 4417 } 4418 return 0; 4419 } 4420 4421 static u16 be_get_img_optype(struct flash_section_entry fsec_entry) 4422 { 4423 u32 img_type = le32_to_cpu(fsec_entry.type); 4424 u16 img_optype = le16_to_cpu(fsec_entry.optype); 4425 4426 if (img_optype != 0xFFFF) 4427 return img_optype; 4428 4429 switch (img_type) { 4430 case IMAGE_FIRMWARE_iSCSI: 4431 img_optype = OPTYPE_ISCSI_ACTIVE; 4432 break; 4433 case IMAGE_BOOT_CODE: 4434 img_optype = OPTYPE_REDBOOT; 4435 break; 4436 case IMAGE_OPTION_ROM_ISCSI: 4437 img_optype = OPTYPE_BIOS; 4438 break; 4439 case IMAGE_OPTION_ROM_PXE: 4440 img_optype = OPTYPE_PXE_BIOS; 4441 break; 4442 case IMAGE_OPTION_ROM_FCoE: 4443 img_optype = OPTYPE_FCOE_BIOS; 4444 break; 4445 case IMAGE_FIRMWARE_BACKUP_iSCSI: 4446 img_optype = OPTYPE_ISCSI_BACKUP; 4447 break; 4448 case IMAGE_NCSI: 4449 img_optype = OPTYPE_NCSI_FW; 4450 break; 4451 case IMAGE_FLASHISM_JUMPVECTOR: 4452 img_optype = OPTYPE_FLASHISM_JUMPVECTOR; 4453 break; 4454 case IMAGE_FIRMWARE_PHY: 4455 img_optype = OPTYPE_SH_PHY_FW; 4456 break; 4457 case IMAGE_REDBOOT_DIR: 4458 img_optype = OPTYPE_REDBOOT_DIR; 4459 break; 4460 case IMAGE_REDBOOT_CONFIG: 4461 img_optype = OPTYPE_REDBOOT_CONFIG; 4462 break; 4463 case IMAGE_UFI_DIR: 4464 img_optype = OPTYPE_UFI_DIR; 4465 break; 4466 default: 4467 break; 4468 } 4469 4470 return img_optype; 4471 } 4472 4473 static int be_flash_skyhawk(struct be_adapter *adapter, 4474 const struct firmware *fw, 4475 struct be_dma_mem *flash_cmd, int num_of_images) 4476 { 4477 int img_hdrs_size = num_of_images * sizeof(struct image_hdr); 4478 bool crc_match, old_fw_img, flash_offset_support = true; 4479 struct device *dev = &adapter->pdev->dev; 4480 struct flash_section_info *fsec = NULL; 4481 u32 img_offset, img_size, img_type; 4482 u16 img_optype, flash_optype; 4483 int status, i, filehdr_size; 4484 const u8 *p; 4485 4486 filehdr_size = sizeof(struct flash_file_hdr_g3); 4487 fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw); 4488 if (!fsec) { 4489 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n"); 4490 return -EINVAL; 4491 } 4492 4493 retry_flash: 4494 for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) { 4495 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset); 4496 img_size = le32_to_cpu(fsec->fsec_entry[i].pad_size); 4497 img_type = le32_to_cpu(fsec->fsec_entry[i].type); 4498 img_optype = be_get_img_optype(fsec->fsec_entry[i]); 4499 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF; 4500 4501 if (img_optype == 0xFFFF) 4502 continue; 4503 4504 if (flash_offset_support) 4505 flash_optype = OPTYPE_OFFSET_SPECIFIED; 4506 else 4507 flash_optype = img_optype; 4508 4509 /* Don't bother verifying CRC if an old FW image is being 4510 * flashed 4511 */ 4512 if (old_fw_img) 4513 goto flash; 4514 4515 status = be_check_flash_crc(adapter, fw->data, img_offset, 4516 img_size, filehdr_size + 4517 img_hdrs_size, flash_optype, 4518 &crc_match); 4519 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST || 4520 base_status(status) == MCC_STATUS_ILLEGAL_FIELD) { 4521 /* The current FW image on the card does not support 4522 * OFFSET based flashing. Retry using older mechanism 4523 * of OPTYPE based flashing 4524 */ 4525 if (flash_optype == OPTYPE_OFFSET_SPECIFIED) { 4526 flash_offset_support = false; 4527 goto retry_flash; 4528 } 4529 4530 /* The current FW image on the card does not recognize 4531 * the new FLASH op_type. The FW download is partially 4532 * complete. Reboot the server now to enable FW image 4533 * to recognize the new FLASH op_type. To complete the 4534 * remaining process, download the same FW again after 4535 * the reboot. 4536 */ 4537 dev_err(dev, "Flash incomplete. Reset the server\n"); 4538 dev_err(dev, "Download FW image again after reset\n"); 4539 return -EAGAIN; 4540 } else if (status) { 4541 dev_err(dev, "Could not get CRC for 0x%x region\n", 4542 img_optype); 4543 return -EFAULT; 4544 } 4545 4546 if (crc_match) 4547 continue; 4548 4549 flash: 4550 p = fw->data + filehdr_size + img_offset + img_hdrs_size; 4551 if (p + img_size > fw->data + fw->size) 4552 return -1; 4553 4554 status = be_flash(adapter, p, flash_cmd, flash_optype, img_size, 4555 img_offset); 4556 4557 /* The current FW image on the card does not support OFFSET 4558 * based flashing. Retry using older mechanism of OPTYPE based 4559 * flashing 4560 */ 4561 if (base_status(status) == MCC_STATUS_ILLEGAL_FIELD && 4562 flash_optype == OPTYPE_OFFSET_SPECIFIED) { 4563 flash_offset_support = false; 4564 goto retry_flash; 4565 } 4566 4567 /* For old FW images ignore ILLEGAL_FIELD error or errors on 4568 * UFI_DIR region 4569 */ 4570 if (old_fw_img && 4571 (base_status(status) == MCC_STATUS_ILLEGAL_FIELD || 4572 (img_optype == OPTYPE_UFI_DIR && 4573 base_status(status) == MCC_STATUS_FAILED))) { 4574 continue; 4575 } else if (status) { 4576 dev_err(dev, "Flashing section type 0x%x failed\n", 4577 img_type); 4578 return -EFAULT; 4579 } 4580 } 4581 return 0; 4582 } 4583 4584 static int lancer_fw_download(struct be_adapter *adapter, 4585 const struct firmware *fw) 4586 { 4587 #define LANCER_FW_DOWNLOAD_CHUNK (32 * 1024) 4588 #define LANCER_FW_DOWNLOAD_LOCATION "/prg" 4589 struct device *dev = &adapter->pdev->dev; 4590 struct be_dma_mem flash_cmd; 4591 const u8 *data_ptr = NULL; 4592 u8 *dest_image_ptr = NULL; 4593 size_t image_size = 0; 4594 u32 chunk_size = 0; 4595 u32 data_written = 0; 4596 u32 offset = 0; 4597 int status = 0; 4598 u8 add_status = 0; 4599 u8 change_status; 4600 4601 if (!IS_ALIGNED(fw->size, sizeof(u32))) { 4602 dev_err(dev, "FW image size should be multiple of 4\n"); 4603 return -EINVAL; 4604 } 4605 4606 flash_cmd.size = sizeof(struct lancer_cmd_req_write_object) 4607 + LANCER_FW_DOWNLOAD_CHUNK; 4608 flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, 4609 &flash_cmd.dma, GFP_KERNEL); 4610 if (!flash_cmd.va) 4611 return -ENOMEM; 4612 4613 dest_image_ptr = flash_cmd.va + 4614 sizeof(struct lancer_cmd_req_write_object); 4615 image_size = fw->size; 4616 data_ptr = fw->data; 4617 4618 while (image_size) { 4619 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK); 4620 4621 /* Copy the image chunk content. */ 4622 memcpy(dest_image_ptr, data_ptr, chunk_size); 4623 4624 status = lancer_cmd_write_object(adapter, &flash_cmd, 4625 chunk_size, offset, 4626 LANCER_FW_DOWNLOAD_LOCATION, 4627 &data_written, &change_status, 4628 &add_status); 4629 if (status) 4630 break; 4631 4632 offset += data_written; 4633 data_ptr += data_written; 4634 image_size -= data_written; 4635 } 4636 4637 if (!status) { 4638 /* Commit the FW written */ 4639 status = lancer_cmd_write_object(adapter, &flash_cmd, 4640 0, offset, 4641 LANCER_FW_DOWNLOAD_LOCATION, 4642 &data_written, &change_status, 4643 &add_status); 4644 } 4645 4646 dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma); 4647 if (status) { 4648 dev_err(dev, "Firmware load error\n"); 4649 return be_cmd_status(status); 4650 } 4651 4652 dev_info(dev, "Firmware flashed successfully\n"); 4653 4654 if (change_status == LANCER_FW_RESET_NEEDED) { 4655 dev_info(dev, "Resetting adapter to activate new FW\n"); 4656 status = lancer_physdev_ctrl(adapter, 4657 PHYSDEV_CONTROL_FW_RESET_MASK); 4658 if (status) { 4659 dev_err(dev, "Adapter busy, could not reset FW\n"); 4660 dev_err(dev, "Reboot server to activate new FW\n"); 4661 } 4662 } else if (change_status != LANCER_NO_RESET_NEEDED) { 4663 dev_info(dev, "Reboot server to activate new FW\n"); 4664 } 4665 4666 return 0; 4667 } 4668 4669 #define BE2_UFI 2 4670 #define BE3_UFI 3 4671 #define BE3R_UFI 10 4672 #define SH_UFI 4 4673 #define SH_P2_UFI 11 4674 4675 static int be_get_ufi_type(struct be_adapter *adapter, 4676 struct flash_file_hdr_g3 *fhdr) 4677 { 4678 if (!fhdr) { 4679 dev_err(&adapter->pdev->dev, "Invalid FW UFI file"); 4680 return -1; 4681 } 4682 4683 /* First letter of the build version is used to identify 4684 * which chip this image file is meant for. 4685 */ 4686 switch (fhdr->build[0]) { 4687 case BLD_STR_UFI_TYPE_SH: 4688 return (fhdr->asic_type_rev == ASIC_REV_P2) ? SH_P2_UFI : 4689 SH_UFI; 4690 case BLD_STR_UFI_TYPE_BE3: 4691 return (fhdr->asic_type_rev == ASIC_REV_B0) ? BE3R_UFI : 4692 BE3_UFI; 4693 case BLD_STR_UFI_TYPE_BE2: 4694 return BE2_UFI; 4695 default: 4696 return -1; 4697 } 4698 } 4699 4700 /* Check if the flash image file is compatible with the adapter that 4701 * is being flashed. 4702 * BE3 chips with asic-rev B0 must be flashed only with BE3R_UFI type. 4703 * Skyhawk chips with asic-rev P2 must be flashed only with SH_P2_UFI type. 4704 */ 4705 static bool be_check_ufi_compatibility(struct be_adapter *adapter, 4706 struct flash_file_hdr_g3 *fhdr) 4707 { 4708 int ufi_type = be_get_ufi_type(adapter, fhdr); 4709 4710 switch (ufi_type) { 4711 case SH_P2_UFI: 4712 return skyhawk_chip(adapter); 4713 case SH_UFI: 4714 return (skyhawk_chip(adapter) && 4715 adapter->asic_rev < ASIC_REV_P2); 4716 case BE3R_UFI: 4717 return BE3_chip(adapter); 4718 case BE3_UFI: 4719 return (BE3_chip(adapter) && adapter->asic_rev < ASIC_REV_B0); 4720 case BE2_UFI: 4721 return BE2_chip(adapter); 4722 default: 4723 return false; 4724 } 4725 } 4726 4727 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw) 4728 { 4729 struct device *dev = &adapter->pdev->dev; 4730 struct flash_file_hdr_g3 *fhdr3; 4731 struct image_hdr *img_hdr_ptr; 4732 int status = 0, i, num_imgs; 4733 struct be_dma_mem flash_cmd; 4734 4735 fhdr3 = (struct flash_file_hdr_g3 *)fw->data; 4736 if (!be_check_ufi_compatibility(adapter, fhdr3)) { 4737 dev_err(dev, "Flash image is not compatible with adapter\n"); 4738 return -EINVAL; 4739 } 4740 4741 flash_cmd.size = sizeof(struct be_cmd_write_flashrom); 4742 flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, &flash_cmd.dma, 4743 GFP_KERNEL); 4744 if (!flash_cmd.va) 4745 return -ENOMEM; 4746 4747 num_imgs = le32_to_cpu(fhdr3->num_imgs); 4748 for (i = 0; i < num_imgs; i++) { 4749 img_hdr_ptr = (struct image_hdr *)(fw->data + 4750 (sizeof(struct flash_file_hdr_g3) + 4751 i * sizeof(struct image_hdr))); 4752 if (!BE2_chip(adapter) && 4753 le32_to_cpu(img_hdr_ptr->imageid) != 1) 4754 continue; 4755 4756 if (skyhawk_chip(adapter)) 4757 status = be_flash_skyhawk(adapter, fw, &flash_cmd, 4758 num_imgs); 4759 else 4760 status = be_flash_BEx(adapter, fw, &flash_cmd, 4761 num_imgs); 4762 } 4763 4764 dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma); 4765 if (!status) 4766 dev_info(dev, "Firmware flashed successfully\n"); 4767 4768 return status; 4769 } 4770 4771 int be_load_fw(struct be_adapter *adapter, u8 *fw_file) 4772 { 4773 const struct firmware *fw; 4774 int status; 4775 4776 if (!netif_running(adapter->netdev)) { 4777 dev_err(&adapter->pdev->dev, 4778 "Firmware load not allowed (interface is down)\n"); 4779 return -ENETDOWN; 4780 } 4781 4782 status = request_firmware(&fw, fw_file, &adapter->pdev->dev); 4783 if (status) 4784 goto fw_exit; 4785 4786 dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file); 4787 4788 if (lancer_chip(adapter)) 4789 status = lancer_fw_download(adapter, fw); 4790 else 4791 status = be_fw_download(adapter, fw); 4792 4793 if (!status) 4794 be_cmd_get_fw_ver(adapter); 4795 4796 fw_exit: 4797 release_firmware(fw); 4798 return status; 4799 } 4800 4801 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4802 u16 flags) 4803 { 4804 struct be_adapter *adapter = netdev_priv(dev); 4805 struct nlattr *attr, *br_spec; 4806 int rem; 4807 int status = 0; 4808 u16 mode = 0; 4809 4810 if (!sriov_enabled(adapter)) 4811 return -EOPNOTSUPP; 4812 4813 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4814 if (!br_spec) 4815 return -EINVAL; 4816 4817 nla_for_each_nested(attr, br_spec, rem) { 4818 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4819 continue; 4820 4821 if (nla_len(attr) < sizeof(mode)) 4822 return -EINVAL; 4823 4824 mode = nla_get_u16(attr); 4825 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 4826 return -EINVAL; 4827 4828 status = be_cmd_set_hsw_config(adapter, 0, 0, 4829 adapter->if_handle, 4830 mode == BRIDGE_MODE_VEPA ? 4831 PORT_FWD_TYPE_VEPA : 4832 PORT_FWD_TYPE_VEB); 4833 if (status) 4834 goto err; 4835 4836 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n", 4837 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4838 4839 return status; 4840 } 4841 err: 4842 dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n", 4843 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4844 4845 return status; 4846 } 4847 4848 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4849 struct net_device *dev, u32 filter_mask, 4850 int nlflags) 4851 { 4852 struct be_adapter *adapter = netdev_priv(dev); 4853 int status = 0; 4854 u8 hsw_mode; 4855 4856 if (!sriov_enabled(adapter)) 4857 return 0; 4858 4859 /* BE and Lancer chips support VEB mode only */ 4860 if (BEx_chip(adapter) || lancer_chip(adapter)) { 4861 hsw_mode = PORT_FWD_TYPE_VEB; 4862 } else { 4863 status = be_cmd_get_hsw_config(adapter, NULL, 0, 4864 adapter->if_handle, &hsw_mode); 4865 if (status) 4866 return 0; 4867 } 4868 4869 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4870 hsw_mode == PORT_FWD_TYPE_VEPA ? 4871 BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB, 4872 0, 0, nlflags); 4873 } 4874 4875 #ifdef CONFIG_BE2NET_VXLAN 4876 /* VxLAN offload Notes: 4877 * 4878 * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't 4879 * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload 4880 * is expected to work across all types of IP tunnels once exported. Skyhawk 4881 * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN 4882 * offloads in hw_enc_features only when a VxLAN port is added. If other (non 4883 * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for 4884 * those other tunnels are unexported on the fly through ndo_features_check(). 4885 * 4886 * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack 4887 * adds more than one port, disable offloads and don't re-enable them again 4888 * until after all the tunnels are removed. 4889 */ 4890 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family, 4891 __be16 port) 4892 { 4893 struct be_adapter *adapter = netdev_priv(netdev); 4894 struct device *dev = &adapter->pdev->dev; 4895 int status; 4896 4897 if (lancer_chip(adapter) || BEx_chip(adapter)) 4898 return; 4899 4900 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) { 4901 dev_info(dev, 4902 "Only one UDP port supported for VxLAN offloads\n"); 4903 dev_info(dev, "Disabling VxLAN offloads\n"); 4904 adapter->vxlan_port_count++; 4905 goto err; 4906 } 4907 4908 if (adapter->vxlan_port_count++ >= 1) 4909 return; 4910 4911 status = be_cmd_manage_iface(adapter, adapter->if_handle, 4912 OP_CONVERT_NORMAL_TO_TUNNEL); 4913 if (status) { 4914 dev_warn(dev, "Failed to convert normal interface to tunnel\n"); 4915 goto err; 4916 } 4917 4918 status = be_cmd_set_vxlan_port(adapter, port); 4919 if (status) { 4920 dev_warn(dev, "Failed to add VxLAN port\n"); 4921 goto err; 4922 } 4923 adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS; 4924 adapter->vxlan_port = port; 4925 4926 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4927 NETIF_F_TSO | NETIF_F_TSO6 | 4928 NETIF_F_GSO_UDP_TUNNEL; 4929 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL; 4930 netdev->features |= NETIF_F_GSO_UDP_TUNNEL; 4931 4932 dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n", 4933 be16_to_cpu(port)); 4934 return; 4935 err: 4936 be_disable_vxlan_offloads(adapter); 4937 } 4938 4939 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family, 4940 __be16 port) 4941 { 4942 struct be_adapter *adapter = netdev_priv(netdev); 4943 4944 if (lancer_chip(adapter) || BEx_chip(adapter)) 4945 return; 4946 4947 if (adapter->vxlan_port != port) 4948 goto done; 4949 4950 be_disable_vxlan_offloads(adapter); 4951 4952 dev_info(&adapter->pdev->dev, 4953 "Disabled VxLAN offloads for UDP port %d\n", 4954 be16_to_cpu(port)); 4955 done: 4956 adapter->vxlan_port_count--; 4957 } 4958 4959 static netdev_features_t be_features_check(struct sk_buff *skb, 4960 struct net_device *dev, 4961 netdev_features_t features) 4962 { 4963 struct be_adapter *adapter = netdev_priv(dev); 4964 u8 l4_hdr = 0; 4965 4966 /* The code below restricts offload features for some tunneled packets. 4967 * Offload features for normal (non tunnel) packets are unchanged. 4968 */ 4969 if (!skb->encapsulation || 4970 !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)) 4971 return features; 4972 4973 /* It's an encapsulated packet and VxLAN offloads are enabled. We 4974 * should disable tunnel offload features if it's not a VxLAN packet, 4975 * as tunnel offloads have been enabled only for VxLAN. This is done to 4976 * allow other tunneled traffic like GRE work fine while VxLAN 4977 * offloads are configured in Skyhawk-R. 4978 */ 4979 switch (vlan_get_protocol(skb)) { 4980 case htons(ETH_P_IP): 4981 l4_hdr = ip_hdr(skb)->protocol; 4982 break; 4983 case htons(ETH_P_IPV6): 4984 l4_hdr = ipv6_hdr(skb)->nexthdr; 4985 break; 4986 default: 4987 return features; 4988 } 4989 4990 if (l4_hdr != IPPROTO_UDP || 4991 skb->inner_protocol_type != ENCAP_TYPE_ETHER || 4992 skb->inner_protocol != htons(ETH_P_TEB) || 4993 skb_inner_mac_header(skb) - skb_transport_header(skb) != 4994 sizeof(struct udphdr) + sizeof(struct vxlanhdr)) 4995 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK); 4996 4997 return features; 4998 } 4999 #endif 5000 5001 static const struct net_device_ops be_netdev_ops = { 5002 .ndo_open = be_open, 5003 .ndo_stop = be_close, 5004 .ndo_start_xmit = be_xmit, 5005 .ndo_set_rx_mode = be_set_rx_mode, 5006 .ndo_set_mac_address = be_mac_addr_set, 5007 .ndo_change_mtu = be_change_mtu, 5008 .ndo_get_stats64 = be_get_stats64, 5009 .ndo_validate_addr = eth_validate_addr, 5010 .ndo_vlan_rx_add_vid = be_vlan_add_vid, 5011 .ndo_vlan_rx_kill_vid = be_vlan_rem_vid, 5012 .ndo_set_vf_mac = be_set_vf_mac, 5013 .ndo_set_vf_vlan = be_set_vf_vlan, 5014 .ndo_set_vf_rate = be_set_vf_tx_rate, 5015 .ndo_get_vf_config = be_get_vf_config, 5016 .ndo_set_vf_link_state = be_set_vf_link_state, 5017 #ifdef CONFIG_NET_POLL_CONTROLLER 5018 .ndo_poll_controller = be_netpoll, 5019 #endif 5020 .ndo_bridge_setlink = be_ndo_bridge_setlink, 5021 .ndo_bridge_getlink = be_ndo_bridge_getlink, 5022 #ifdef CONFIG_NET_RX_BUSY_POLL 5023 .ndo_busy_poll = be_busy_poll, 5024 #endif 5025 #ifdef CONFIG_BE2NET_VXLAN 5026 .ndo_add_vxlan_port = be_add_vxlan_port, 5027 .ndo_del_vxlan_port = be_del_vxlan_port, 5028 .ndo_features_check = be_features_check, 5029 #endif 5030 }; 5031 5032 static void be_netdev_init(struct net_device *netdev) 5033 { 5034 struct be_adapter *adapter = netdev_priv(netdev); 5035 5036 netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5037 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | 5038 NETIF_F_HW_VLAN_CTAG_TX; 5039 if (be_multi_rxq(adapter)) 5040 netdev->hw_features |= NETIF_F_RXHASH; 5041 5042 netdev->features |= netdev->hw_features | 5043 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER; 5044 5045 netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 5046 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 5047 5048 netdev->priv_flags |= IFF_UNICAST_FLT; 5049 5050 netdev->flags |= IFF_MULTICAST; 5051 5052 netif_set_gso_max_size(netdev, 65535 - ETH_HLEN); 5053 5054 netdev->netdev_ops = &be_netdev_ops; 5055 5056 netdev->ethtool_ops = &be_ethtool_ops; 5057 } 5058 5059 static void be_cleanup(struct be_adapter *adapter) 5060 { 5061 struct net_device *netdev = adapter->netdev; 5062 5063 rtnl_lock(); 5064 netif_device_detach(netdev); 5065 if (netif_running(netdev)) 5066 be_close(netdev); 5067 rtnl_unlock(); 5068 5069 be_clear(adapter); 5070 } 5071 5072 static int be_resume(struct be_adapter *adapter) 5073 { 5074 struct net_device *netdev = adapter->netdev; 5075 int status; 5076 5077 status = be_setup(adapter); 5078 if (status) 5079 return status; 5080 5081 if (netif_running(netdev)) { 5082 status = be_open(netdev); 5083 if (status) 5084 return status; 5085 } 5086 5087 netif_device_attach(netdev); 5088 5089 return 0; 5090 } 5091 5092 static int be_err_recover(struct be_adapter *adapter) 5093 { 5094 struct device *dev = &adapter->pdev->dev; 5095 int status; 5096 5097 status = be_resume(adapter); 5098 if (status) 5099 goto err; 5100 5101 dev_info(dev, "Adapter recovery successful\n"); 5102 return 0; 5103 err: 5104 if (be_physfn(adapter)) 5105 dev_err(dev, "Adapter recovery failed\n"); 5106 else 5107 dev_err(dev, "Re-trying adapter recovery\n"); 5108 5109 return status; 5110 } 5111 5112 static void be_err_detection_task(struct work_struct *work) 5113 { 5114 struct be_adapter *adapter = 5115 container_of(work, struct be_adapter, 5116 be_err_detection_work.work); 5117 int status = 0; 5118 5119 be_detect_error(adapter); 5120 5121 if (adapter->hw_error) { 5122 be_cleanup(adapter); 5123 5124 /* As of now error recovery support is in Lancer only */ 5125 if (lancer_chip(adapter)) 5126 status = be_err_recover(adapter); 5127 } 5128 5129 /* Always attempt recovery on VFs */ 5130 if (!status || be_virtfn(adapter)) 5131 be_schedule_err_detection(adapter); 5132 } 5133 5134 static void be_log_sfp_info(struct be_adapter *adapter) 5135 { 5136 int status; 5137 5138 status = be_cmd_query_sfp_info(adapter); 5139 if (!status) { 5140 dev_err(&adapter->pdev->dev, 5141 "Unqualified SFP+ detected on %c from %s part no: %s", 5142 adapter->port_name, adapter->phy.vendor_name, 5143 adapter->phy.vendor_pn); 5144 } 5145 adapter->flags &= ~BE_FLAGS_EVT_INCOMPATIBLE_SFP; 5146 } 5147 5148 static void be_worker(struct work_struct *work) 5149 { 5150 struct be_adapter *adapter = 5151 container_of(work, struct be_adapter, work.work); 5152 struct be_rx_obj *rxo; 5153 int i; 5154 5155 /* when interrupts are not yet enabled, just reap any pending 5156 * mcc completions 5157 */ 5158 if (!netif_running(adapter->netdev)) { 5159 local_bh_disable(); 5160 be_process_mcc(adapter); 5161 local_bh_enable(); 5162 goto reschedule; 5163 } 5164 5165 if (!adapter->stats_cmd_sent) { 5166 if (lancer_chip(adapter)) 5167 lancer_cmd_get_pport_stats(adapter, 5168 &adapter->stats_cmd); 5169 else 5170 be_cmd_get_stats(adapter, &adapter->stats_cmd); 5171 } 5172 5173 if (be_physfn(adapter) && 5174 MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0) 5175 be_cmd_get_die_temperature(adapter); 5176 5177 for_all_rx_queues(adapter, rxo, i) { 5178 /* Replenish RX-queues starved due to memory 5179 * allocation failures. 5180 */ 5181 if (rxo->rx_post_starved) 5182 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST); 5183 } 5184 5185 be_eqd_update(adapter); 5186 5187 if (adapter->flags & BE_FLAGS_EVT_INCOMPATIBLE_SFP) 5188 be_log_sfp_info(adapter); 5189 5190 reschedule: 5191 adapter->work_counter++; 5192 schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000)); 5193 } 5194 5195 static void be_unmap_pci_bars(struct be_adapter *adapter) 5196 { 5197 if (adapter->csr) 5198 pci_iounmap(adapter->pdev, adapter->csr); 5199 if (adapter->db) 5200 pci_iounmap(adapter->pdev, adapter->db); 5201 } 5202 5203 static int db_bar(struct be_adapter *adapter) 5204 { 5205 if (lancer_chip(adapter) || !be_physfn(adapter)) 5206 return 0; 5207 else 5208 return 4; 5209 } 5210 5211 static int be_roce_map_pci_bars(struct be_adapter *adapter) 5212 { 5213 if (skyhawk_chip(adapter)) { 5214 adapter->roce_db.size = 4096; 5215 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev, 5216 db_bar(adapter)); 5217 adapter->roce_db.total_size = pci_resource_len(adapter->pdev, 5218 db_bar(adapter)); 5219 } 5220 return 0; 5221 } 5222 5223 static int be_map_pci_bars(struct be_adapter *adapter) 5224 { 5225 struct pci_dev *pdev = adapter->pdev; 5226 u8 __iomem *addr; 5227 u32 sli_intf; 5228 5229 pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf); 5230 adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >> 5231 SLI_INTF_FAMILY_SHIFT; 5232 adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0; 5233 5234 if (BEx_chip(adapter) && be_physfn(adapter)) { 5235 adapter->csr = pci_iomap(pdev, 2, 0); 5236 if (!adapter->csr) 5237 return -ENOMEM; 5238 } 5239 5240 addr = pci_iomap(pdev, db_bar(adapter), 0); 5241 if (!addr) 5242 goto pci_map_err; 5243 adapter->db = addr; 5244 5245 if (skyhawk_chip(adapter) || BEx_chip(adapter)) { 5246 if (be_physfn(adapter)) { 5247 /* PCICFG is the 2nd BAR in BE2 */ 5248 addr = pci_iomap(pdev, BE2_chip(adapter) ? 1 : 0, 0); 5249 if (!addr) 5250 goto pci_map_err; 5251 adapter->pcicfg = addr; 5252 } else { 5253 adapter->pcicfg = adapter->db + SRIOV_VF_PCICFG_OFFSET; 5254 } 5255 } 5256 5257 be_roce_map_pci_bars(adapter); 5258 return 0; 5259 5260 pci_map_err: 5261 dev_err(&pdev->dev, "Error in mapping PCI BARs\n"); 5262 be_unmap_pci_bars(adapter); 5263 return -ENOMEM; 5264 } 5265 5266 static void be_drv_cleanup(struct be_adapter *adapter) 5267 { 5268 struct be_dma_mem *mem = &adapter->mbox_mem_alloced; 5269 struct device *dev = &adapter->pdev->dev; 5270 5271 if (mem->va) 5272 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5273 5274 mem = &adapter->rx_filter; 5275 if (mem->va) 5276 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5277 5278 mem = &adapter->stats_cmd; 5279 if (mem->va) 5280 dma_free_coherent(dev, mem->size, mem->va, mem->dma); 5281 } 5282 5283 /* Allocate and initialize various fields in be_adapter struct */ 5284 static int be_drv_init(struct be_adapter *adapter) 5285 { 5286 struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced; 5287 struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem; 5288 struct be_dma_mem *rx_filter = &adapter->rx_filter; 5289 struct be_dma_mem *stats_cmd = &adapter->stats_cmd; 5290 struct device *dev = &adapter->pdev->dev; 5291 int status = 0; 5292 5293 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 5294 mbox_mem_alloc->va = dma_alloc_coherent(dev, mbox_mem_alloc->size, 5295 &mbox_mem_alloc->dma, 5296 GFP_KERNEL); 5297 if (!mbox_mem_alloc->va) 5298 return -ENOMEM; 5299 5300 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 5301 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 5302 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 5303 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); 5304 5305 rx_filter->size = sizeof(struct be_cmd_req_rx_filter); 5306 rx_filter->va = dma_zalloc_coherent(dev, rx_filter->size, 5307 &rx_filter->dma, GFP_KERNEL); 5308 if (!rx_filter->va) { 5309 status = -ENOMEM; 5310 goto free_mbox; 5311 } 5312 5313 if (lancer_chip(adapter)) 5314 stats_cmd->size = sizeof(struct lancer_cmd_req_pport_stats); 5315 else if (BE2_chip(adapter)) 5316 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v0); 5317 else if (BE3_chip(adapter)) 5318 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v1); 5319 else 5320 stats_cmd->size = sizeof(struct be_cmd_req_get_stats_v2); 5321 stats_cmd->va = dma_zalloc_coherent(dev, stats_cmd->size, 5322 &stats_cmd->dma, GFP_KERNEL); 5323 if (!stats_cmd->va) { 5324 status = -ENOMEM; 5325 goto free_rx_filter; 5326 } 5327 5328 mutex_init(&adapter->mbox_lock); 5329 spin_lock_init(&adapter->mcc_lock); 5330 spin_lock_init(&adapter->mcc_cq_lock); 5331 init_completion(&adapter->et_cmd_compl); 5332 5333 pci_save_state(adapter->pdev); 5334 5335 INIT_DELAYED_WORK(&adapter->work, be_worker); 5336 INIT_DELAYED_WORK(&adapter->be_err_detection_work, 5337 be_err_detection_task); 5338 5339 adapter->rx_fc = true; 5340 adapter->tx_fc = true; 5341 5342 /* Must be a power of 2 or else MODULO will BUG_ON */ 5343 adapter->be_get_temp_freq = 64; 5344 5345 return 0; 5346 5347 free_rx_filter: 5348 dma_free_coherent(dev, rx_filter->size, rx_filter->va, rx_filter->dma); 5349 free_mbox: 5350 dma_free_coherent(dev, mbox_mem_alloc->size, mbox_mem_alloc->va, 5351 mbox_mem_alloc->dma); 5352 return status; 5353 } 5354 5355 static void be_remove(struct pci_dev *pdev) 5356 { 5357 struct be_adapter *adapter = pci_get_drvdata(pdev); 5358 5359 if (!adapter) 5360 return; 5361 5362 be_roce_dev_remove(adapter); 5363 be_intr_set(adapter, false); 5364 5365 be_cancel_err_detection(adapter); 5366 5367 unregister_netdev(adapter->netdev); 5368 5369 be_clear(adapter); 5370 5371 /* tell fw we're done with firing cmds */ 5372 be_cmd_fw_clean(adapter); 5373 5374 be_unmap_pci_bars(adapter); 5375 be_drv_cleanup(adapter); 5376 5377 pci_disable_pcie_error_reporting(pdev); 5378 5379 pci_release_regions(pdev); 5380 pci_disable_device(pdev); 5381 5382 free_netdev(adapter->netdev); 5383 } 5384 5385 static char *mc_name(struct be_adapter *adapter) 5386 { 5387 char *str = ""; /* default */ 5388 5389 switch (adapter->mc_type) { 5390 case UMC: 5391 str = "UMC"; 5392 break; 5393 case FLEX10: 5394 str = "FLEX10"; 5395 break; 5396 case vNIC1: 5397 str = "vNIC-1"; 5398 break; 5399 case nPAR: 5400 str = "nPAR"; 5401 break; 5402 case UFP: 5403 str = "UFP"; 5404 break; 5405 case vNIC2: 5406 str = "vNIC-2"; 5407 break; 5408 default: 5409 str = ""; 5410 } 5411 5412 return str; 5413 } 5414 5415 static inline char *func_name(struct be_adapter *adapter) 5416 { 5417 return be_physfn(adapter) ? "PF" : "VF"; 5418 } 5419 5420 static inline char *nic_name(struct pci_dev *pdev) 5421 { 5422 switch (pdev->device) { 5423 case OC_DEVICE_ID1: 5424 return OC_NAME; 5425 case OC_DEVICE_ID2: 5426 return OC_NAME_BE; 5427 case OC_DEVICE_ID3: 5428 case OC_DEVICE_ID4: 5429 return OC_NAME_LANCER; 5430 case BE_DEVICE_ID2: 5431 return BE3_NAME; 5432 case OC_DEVICE_ID5: 5433 case OC_DEVICE_ID6: 5434 return OC_NAME_SH; 5435 default: 5436 return BE_NAME; 5437 } 5438 } 5439 5440 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id) 5441 { 5442 struct be_adapter *adapter; 5443 struct net_device *netdev; 5444 int status = 0; 5445 5446 dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER); 5447 5448 status = pci_enable_device(pdev); 5449 if (status) 5450 goto do_none; 5451 5452 status = pci_request_regions(pdev, DRV_NAME); 5453 if (status) 5454 goto disable_dev; 5455 pci_set_master(pdev); 5456 5457 netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS); 5458 if (!netdev) { 5459 status = -ENOMEM; 5460 goto rel_reg; 5461 } 5462 adapter = netdev_priv(netdev); 5463 adapter->pdev = pdev; 5464 pci_set_drvdata(pdev, adapter); 5465 adapter->netdev = netdev; 5466 SET_NETDEV_DEV(netdev, &pdev->dev); 5467 5468 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 5469 if (!status) { 5470 netdev->features |= NETIF_F_HIGHDMA; 5471 } else { 5472 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 5473 if (status) { 5474 dev_err(&pdev->dev, "Could not set PCI DMA Mask\n"); 5475 goto free_netdev; 5476 } 5477 } 5478 5479 status = pci_enable_pcie_error_reporting(pdev); 5480 if (!status) 5481 dev_info(&pdev->dev, "PCIe error reporting enabled\n"); 5482 5483 status = be_map_pci_bars(adapter); 5484 if (status) 5485 goto free_netdev; 5486 5487 status = be_drv_init(adapter); 5488 if (status) 5489 goto unmap_bars; 5490 5491 status = be_setup(adapter); 5492 if (status) 5493 goto drv_cleanup; 5494 5495 be_netdev_init(netdev); 5496 status = register_netdev(netdev); 5497 if (status != 0) 5498 goto unsetup; 5499 5500 be_roce_dev_add(adapter); 5501 5502 be_schedule_err_detection(adapter); 5503 5504 dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev), 5505 func_name(adapter), mc_name(adapter), adapter->port_name); 5506 5507 return 0; 5508 5509 unsetup: 5510 be_clear(adapter); 5511 drv_cleanup: 5512 be_drv_cleanup(adapter); 5513 unmap_bars: 5514 be_unmap_pci_bars(adapter); 5515 free_netdev: 5516 free_netdev(netdev); 5517 rel_reg: 5518 pci_release_regions(pdev); 5519 disable_dev: 5520 pci_disable_device(pdev); 5521 do_none: 5522 dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev)); 5523 return status; 5524 } 5525 5526 static int be_suspend(struct pci_dev *pdev, pm_message_t state) 5527 { 5528 struct be_adapter *adapter = pci_get_drvdata(pdev); 5529 5530 if (adapter->wol_en) 5531 be_setup_wol(adapter, true); 5532 5533 be_intr_set(adapter, false); 5534 be_cancel_err_detection(adapter); 5535 5536 be_cleanup(adapter); 5537 5538 pci_save_state(pdev); 5539 pci_disable_device(pdev); 5540 pci_set_power_state(pdev, pci_choose_state(pdev, state)); 5541 return 0; 5542 } 5543 5544 static int be_pci_resume(struct pci_dev *pdev) 5545 { 5546 struct be_adapter *adapter = pci_get_drvdata(pdev); 5547 int status = 0; 5548 5549 status = pci_enable_device(pdev); 5550 if (status) 5551 return status; 5552 5553 pci_set_power_state(pdev, PCI_D0); 5554 pci_restore_state(pdev); 5555 5556 status = be_resume(adapter); 5557 if (status) 5558 return status; 5559 5560 be_schedule_err_detection(adapter); 5561 5562 if (adapter->wol_en) 5563 be_setup_wol(adapter, false); 5564 5565 return 0; 5566 } 5567 5568 /* 5569 * An FLR will stop BE from DMAing any data. 5570 */ 5571 static void be_shutdown(struct pci_dev *pdev) 5572 { 5573 struct be_adapter *adapter = pci_get_drvdata(pdev); 5574 5575 if (!adapter) 5576 return; 5577 5578 be_roce_dev_shutdown(adapter); 5579 cancel_delayed_work_sync(&adapter->work); 5580 be_cancel_err_detection(adapter); 5581 5582 netif_device_detach(adapter->netdev); 5583 5584 be_cmd_reset_function(adapter); 5585 5586 pci_disable_device(pdev); 5587 } 5588 5589 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev, 5590 pci_channel_state_t state) 5591 { 5592 struct be_adapter *adapter = pci_get_drvdata(pdev); 5593 5594 dev_err(&adapter->pdev->dev, "EEH error detected\n"); 5595 5596 if (!adapter->eeh_error) { 5597 adapter->eeh_error = true; 5598 5599 be_cancel_err_detection(adapter); 5600 5601 be_cleanup(adapter); 5602 } 5603 5604 if (state == pci_channel_io_perm_failure) 5605 return PCI_ERS_RESULT_DISCONNECT; 5606 5607 pci_disable_device(pdev); 5608 5609 /* The error could cause the FW to trigger a flash debug dump. 5610 * Resetting the card while flash dump is in progress 5611 * can cause it not to recover; wait for it to finish. 5612 * Wait only for first function as it is needed only once per 5613 * adapter. 5614 */ 5615 if (pdev->devfn == 0) 5616 ssleep(30); 5617 5618 return PCI_ERS_RESULT_NEED_RESET; 5619 } 5620 5621 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev) 5622 { 5623 struct be_adapter *adapter = pci_get_drvdata(pdev); 5624 int status; 5625 5626 dev_info(&adapter->pdev->dev, "EEH reset\n"); 5627 5628 status = pci_enable_device(pdev); 5629 if (status) 5630 return PCI_ERS_RESULT_DISCONNECT; 5631 5632 pci_set_master(pdev); 5633 pci_set_power_state(pdev, PCI_D0); 5634 pci_restore_state(pdev); 5635 5636 /* Check if card is ok and fw is ready */ 5637 dev_info(&adapter->pdev->dev, 5638 "Waiting for FW to be ready after EEH reset\n"); 5639 status = be_fw_wait_ready(adapter); 5640 if (status) 5641 return PCI_ERS_RESULT_DISCONNECT; 5642 5643 pci_cleanup_aer_uncorrect_error_status(pdev); 5644 be_clear_all_error(adapter); 5645 return PCI_ERS_RESULT_RECOVERED; 5646 } 5647 5648 static void be_eeh_resume(struct pci_dev *pdev) 5649 { 5650 int status = 0; 5651 struct be_adapter *adapter = pci_get_drvdata(pdev); 5652 5653 dev_info(&adapter->pdev->dev, "EEH resume\n"); 5654 5655 pci_save_state(pdev); 5656 5657 status = be_resume(adapter); 5658 if (status) 5659 goto err; 5660 5661 be_schedule_err_detection(adapter); 5662 return; 5663 err: 5664 dev_err(&adapter->pdev->dev, "EEH resume failed\n"); 5665 } 5666 5667 static int be_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) 5668 { 5669 struct be_adapter *adapter = pci_get_drvdata(pdev); 5670 u16 num_vf_qs; 5671 int status; 5672 5673 if (!num_vfs) 5674 be_vf_clear(adapter); 5675 5676 adapter->num_vfs = num_vfs; 5677 5678 if (adapter->num_vfs == 0 && pci_vfs_assigned(pdev)) { 5679 dev_warn(&pdev->dev, 5680 "Cannot disable VFs while they are assigned\n"); 5681 return -EBUSY; 5682 } 5683 5684 /* When the HW is in SRIOV capable configuration, the PF-pool resources 5685 * are equally distributed across the max-number of VFs. The user may 5686 * request only a subset of the max-vfs to be enabled. 5687 * Based on num_vfs, redistribute the resources across num_vfs so that 5688 * each VF will have access to more number of resources. 5689 * This facility is not available in BE3 FW. 5690 * Also, this is done by FW in Lancer chip. 5691 */ 5692 if (skyhawk_chip(adapter) && !pci_num_vf(pdev)) { 5693 num_vf_qs = be_calculate_vf_qs(adapter, adapter->num_vfs); 5694 status = be_cmd_set_sriov_config(adapter, adapter->pool_res, 5695 adapter->num_vfs, num_vf_qs); 5696 if (status) 5697 dev_err(&pdev->dev, 5698 "Failed to optimize SR-IOV resources\n"); 5699 } 5700 5701 status = be_get_resources(adapter); 5702 if (status) 5703 return be_cmd_status(status); 5704 5705 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 5706 rtnl_lock(); 5707 status = be_update_queues(adapter); 5708 rtnl_unlock(); 5709 if (status) 5710 return be_cmd_status(status); 5711 5712 if (adapter->num_vfs) 5713 status = be_vf_setup(adapter); 5714 5715 if (!status) 5716 return adapter->num_vfs; 5717 5718 return 0; 5719 } 5720 5721 static const struct pci_error_handlers be_eeh_handlers = { 5722 .error_detected = be_eeh_err_detected, 5723 .slot_reset = be_eeh_reset, 5724 .resume = be_eeh_resume, 5725 }; 5726 5727 static struct pci_driver be_driver = { 5728 .name = DRV_NAME, 5729 .id_table = be_dev_ids, 5730 .probe = be_probe, 5731 .remove = be_remove, 5732 .suspend = be_suspend, 5733 .resume = be_pci_resume, 5734 .shutdown = be_shutdown, 5735 .sriov_configure = be_pci_sriov_configure, 5736 .err_handler = &be_eeh_handlers 5737 }; 5738 5739 static int __init be_init_module(void) 5740 { 5741 if (rx_frag_size != 8192 && rx_frag_size != 4096 && 5742 rx_frag_size != 2048) { 5743 printk(KERN_WARNING DRV_NAME 5744 " : Module param rx_frag_size must be 2048/4096/8192." 5745 " Using 2048\n"); 5746 rx_frag_size = 2048; 5747 } 5748 5749 if (num_vfs > 0) { 5750 pr_info(DRV_NAME " : Module param num_vfs is obsolete."); 5751 pr_info(DRV_NAME " : Use sysfs method to enable VFs\n"); 5752 } 5753 5754 return pci_register_driver(&be_driver); 5755 } 5756 module_init(be_init_module); 5757 5758 static void __exit be_exit_module(void) 5759 { 5760 pci_unregister_driver(&be_driver); 5761 } 5762 module_exit(be_exit_module); 5763