1 /* 2 * Copyright (C) 2005 - 2014 Emulex 3 * All rights reserved. 4 * 5 * This program is free software; you can redistribute it and/or 6 * modify it under the terms of the GNU General Public License version 2 7 * as published by the Free Software Foundation. The full GNU General 8 * Public License is included in this distribution in the file called COPYING. 9 * 10 * Contact Information: 11 * linux-drivers@emulex.com 12 * 13 * Emulex 14 * 3333 Susan Street 15 * Costa Mesa, CA 92626 16 */ 17 18 #include <linux/prefetch.h> 19 #include <linux/module.h> 20 #include "be.h" 21 #include "be_cmds.h" 22 #include <asm/div64.h> 23 #include <linux/aer.h> 24 #include <linux/if_bridge.h> 25 #include <net/busy_poll.h> 26 #include <net/vxlan.h> 27 28 MODULE_VERSION(DRV_VER); 29 MODULE_DESCRIPTION(DRV_DESC " " DRV_VER); 30 MODULE_AUTHOR("Emulex Corporation"); 31 MODULE_LICENSE("GPL"); 32 33 static unsigned int num_vfs; 34 module_param(num_vfs, uint, S_IRUGO); 35 MODULE_PARM_DESC(num_vfs, "Number of PCI VFs to initialize"); 36 37 static ushort rx_frag_size = 2048; 38 module_param(rx_frag_size, ushort, S_IRUGO); 39 MODULE_PARM_DESC(rx_frag_size, "Size of a fragment that holds rcvd data."); 40 41 static const struct pci_device_id be_dev_ids[] = { 42 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID1) }, 43 { PCI_DEVICE(BE_VENDOR_ID, BE_DEVICE_ID2) }, 44 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID1) }, 45 { PCI_DEVICE(BE_VENDOR_ID, OC_DEVICE_ID2) }, 46 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID3)}, 47 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID4)}, 48 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID5)}, 49 { PCI_DEVICE(EMULEX_VENDOR_ID, OC_DEVICE_ID6)}, 50 { 0 } 51 }; 52 MODULE_DEVICE_TABLE(pci, be_dev_ids); 53 /* UE Status Low CSR */ 54 static const char * const ue_status_low_desc[] = { 55 "CEV", 56 "CTX", 57 "DBUF", 58 "ERX", 59 "Host", 60 "MPU", 61 "NDMA", 62 "PTC ", 63 "RDMA ", 64 "RXF ", 65 "RXIPS ", 66 "RXULP0 ", 67 "RXULP1 ", 68 "RXULP2 ", 69 "TIM ", 70 "TPOST ", 71 "TPRE ", 72 "TXIPS ", 73 "TXULP0 ", 74 "TXULP1 ", 75 "UC ", 76 "WDMA ", 77 "TXULP2 ", 78 "HOST1 ", 79 "P0_OB_LINK ", 80 "P1_OB_LINK ", 81 "HOST_GPIO ", 82 "MBOX ", 83 "ERX2 ", 84 "SPARE ", 85 "JTAG ", 86 "MPU_INTPEND " 87 }; 88 89 /* UE Status High CSR */ 90 static const char * const ue_status_hi_desc[] = { 91 "LPCMEMHOST", 92 "MGMT_MAC", 93 "PCS0ONLINE", 94 "MPU_IRAM", 95 "PCS1ONLINE", 96 "PCTL0", 97 "PCTL1", 98 "PMEM", 99 "RR", 100 "TXPB", 101 "RXPP", 102 "XAUI", 103 "TXP", 104 "ARM", 105 "IPC", 106 "HOST2", 107 "HOST3", 108 "HOST4", 109 "HOST5", 110 "HOST6", 111 "HOST7", 112 "ECRC", 113 "Poison TLP", 114 "NETC", 115 "PERIPH", 116 "LLTXULP", 117 "D2P", 118 "RCON", 119 "LDMA", 120 "LLTXP", 121 "LLTXPB", 122 "Unknown" 123 }; 124 125 static void be_queue_free(struct be_adapter *adapter, struct be_queue_info *q) 126 { 127 struct be_dma_mem *mem = &q->dma_mem; 128 129 if (mem->va) { 130 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 131 mem->dma); 132 mem->va = NULL; 133 } 134 } 135 136 static int be_queue_alloc(struct be_adapter *adapter, struct be_queue_info *q, 137 u16 len, u16 entry_size) 138 { 139 struct be_dma_mem *mem = &q->dma_mem; 140 141 memset(q, 0, sizeof(*q)); 142 q->len = len; 143 q->entry_size = entry_size; 144 mem->size = len * entry_size; 145 mem->va = dma_zalloc_coherent(&adapter->pdev->dev, mem->size, &mem->dma, 146 GFP_KERNEL); 147 if (!mem->va) 148 return -ENOMEM; 149 return 0; 150 } 151 152 static void be_reg_intr_set(struct be_adapter *adapter, bool enable) 153 { 154 u32 reg, enabled; 155 156 pci_read_config_dword(adapter->pdev, PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, 157 ®); 158 enabled = reg & MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 159 160 if (!enabled && enable) 161 reg |= MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 162 else if (enabled && !enable) 163 reg &= ~MEMBAR_CTRL_INT_CTRL_HOSTINTR_MASK; 164 else 165 return; 166 167 pci_write_config_dword(adapter->pdev, 168 PCICFG_MEMBAR_CTRL_INT_CTRL_OFFSET, reg); 169 } 170 171 static void be_intr_set(struct be_adapter *adapter, bool enable) 172 { 173 int status = 0; 174 175 /* On lancer interrupts can't be controlled via this register */ 176 if (lancer_chip(adapter)) 177 return; 178 179 if (adapter->eeh_error) 180 return; 181 182 status = be_cmd_intr_set(adapter, enable); 183 if (status) 184 be_reg_intr_set(adapter, enable); 185 } 186 187 static void be_rxq_notify(struct be_adapter *adapter, u16 qid, u16 posted) 188 { 189 u32 val = 0; 190 191 val |= qid & DB_RQ_RING_ID_MASK; 192 val |= posted << DB_RQ_NUM_POSTED_SHIFT; 193 194 wmb(); 195 iowrite32(val, adapter->db + DB_RQ_OFFSET); 196 } 197 198 static void be_txq_notify(struct be_adapter *adapter, struct be_tx_obj *txo, 199 u16 posted) 200 { 201 u32 val = 0; 202 203 val |= txo->q.id & DB_TXULP_RING_ID_MASK; 204 val |= (posted & DB_TXULP_NUM_POSTED_MASK) << DB_TXULP_NUM_POSTED_SHIFT; 205 206 wmb(); 207 iowrite32(val, adapter->db + txo->db_offset); 208 } 209 210 static void be_eq_notify(struct be_adapter *adapter, u16 qid, 211 bool arm, bool clear_int, u16 num_popped) 212 { 213 u32 val = 0; 214 215 val |= qid & DB_EQ_RING_ID_MASK; 216 val |= ((qid & DB_EQ_RING_ID_EXT_MASK) << DB_EQ_RING_ID_EXT_MASK_SHIFT); 217 218 if (adapter->eeh_error) 219 return; 220 221 if (arm) 222 val |= 1 << DB_EQ_REARM_SHIFT; 223 if (clear_int) 224 val |= 1 << DB_EQ_CLR_SHIFT; 225 val |= 1 << DB_EQ_EVNT_SHIFT; 226 val |= num_popped << DB_EQ_NUM_POPPED_SHIFT; 227 iowrite32(val, adapter->db + DB_EQ_OFFSET); 228 } 229 230 void be_cq_notify(struct be_adapter *adapter, u16 qid, bool arm, u16 num_popped) 231 { 232 u32 val = 0; 233 234 val |= qid & DB_CQ_RING_ID_MASK; 235 val |= ((qid & DB_CQ_RING_ID_EXT_MASK) << 236 DB_CQ_RING_ID_EXT_MASK_SHIFT); 237 238 if (adapter->eeh_error) 239 return; 240 241 if (arm) 242 val |= 1 << DB_CQ_REARM_SHIFT; 243 val |= num_popped << DB_CQ_NUM_POPPED_SHIFT; 244 iowrite32(val, adapter->db + DB_CQ_OFFSET); 245 } 246 247 static int be_mac_addr_set(struct net_device *netdev, void *p) 248 { 249 struct be_adapter *adapter = netdev_priv(netdev); 250 struct device *dev = &adapter->pdev->dev; 251 struct sockaddr *addr = p; 252 int status; 253 u8 mac[ETH_ALEN]; 254 u32 old_pmac_id = adapter->pmac_id[0], curr_pmac_id = 0; 255 256 if (!is_valid_ether_addr(addr->sa_data)) 257 return -EADDRNOTAVAIL; 258 259 /* Proceed further only if, User provided MAC is different 260 * from active MAC 261 */ 262 if (ether_addr_equal(addr->sa_data, netdev->dev_addr)) 263 return 0; 264 265 /* The PMAC_ADD cmd may fail if the VF doesn't have FILTMGMT 266 * privilege or if PF did not provision the new MAC address. 267 * On BE3, this cmd will always fail if the VF doesn't have the 268 * FILTMGMT privilege. This failure is OK, only if the PF programmed 269 * the MAC for the VF. 270 */ 271 status = be_cmd_pmac_add(adapter, (u8 *)addr->sa_data, 272 adapter->if_handle, &adapter->pmac_id[0], 0); 273 if (!status) { 274 curr_pmac_id = adapter->pmac_id[0]; 275 276 /* Delete the old programmed MAC. This call may fail if the 277 * old MAC was already deleted by the PF driver. 278 */ 279 if (adapter->pmac_id[0] != old_pmac_id) 280 be_cmd_pmac_del(adapter, adapter->if_handle, 281 old_pmac_id, 0); 282 } 283 284 /* Decide if the new MAC is successfully activated only after 285 * querying the FW 286 */ 287 status = be_cmd_get_active_mac(adapter, curr_pmac_id, mac, 288 adapter->if_handle, true, 0); 289 if (status) 290 goto err; 291 292 /* The MAC change did not happen, either due to lack of privilege 293 * or PF didn't pre-provision. 294 */ 295 if (!ether_addr_equal(addr->sa_data, mac)) { 296 status = -EPERM; 297 goto err; 298 } 299 300 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len); 301 dev_info(dev, "MAC address changed to %pM\n", mac); 302 return 0; 303 err: 304 dev_warn(dev, "MAC address change to %pM failed\n", addr->sa_data); 305 return status; 306 } 307 308 /* BE2 supports only v0 cmd */ 309 static void *hw_stats_from_cmd(struct be_adapter *adapter) 310 { 311 if (BE2_chip(adapter)) { 312 struct be_cmd_resp_get_stats_v0 *cmd = adapter->stats_cmd.va; 313 314 return &cmd->hw_stats; 315 } else if (BE3_chip(adapter)) { 316 struct be_cmd_resp_get_stats_v1 *cmd = adapter->stats_cmd.va; 317 318 return &cmd->hw_stats; 319 } else { 320 struct be_cmd_resp_get_stats_v2 *cmd = adapter->stats_cmd.va; 321 322 return &cmd->hw_stats; 323 } 324 } 325 326 /* BE2 supports only v0 cmd */ 327 static void *be_erx_stats_from_cmd(struct be_adapter *adapter) 328 { 329 if (BE2_chip(adapter)) { 330 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 331 332 return &hw_stats->erx; 333 } else if (BE3_chip(adapter)) { 334 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 335 336 return &hw_stats->erx; 337 } else { 338 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 339 340 return &hw_stats->erx; 341 } 342 } 343 344 static void populate_be_v0_stats(struct be_adapter *adapter) 345 { 346 struct be_hw_stats_v0 *hw_stats = hw_stats_from_cmd(adapter); 347 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 348 struct be_rxf_stats_v0 *rxf_stats = &hw_stats->rxf; 349 struct be_port_rxf_stats_v0 *port_stats = 350 &rxf_stats->port[adapter->port_num]; 351 struct be_drv_stats *drvs = &adapter->drv_stats; 352 353 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 354 drvs->rx_pause_frames = port_stats->rx_pause_frames; 355 drvs->rx_crc_errors = port_stats->rx_crc_errors; 356 drvs->rx_control_frames = port_stats->rx_control_frames; 357 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 358 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 359 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 360 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 361 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 362 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 363 drvs->rxpp_fifo_overflow_drop = port_stats->rx_fifo_overflow; 364 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 365 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 366 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 367 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 368 drvs->rx_input_fifo_overflow_drop = port_stats->rx_input_fifo_overflow; 369 drvs->rx_dropped_header_too_small = 370 port_stats->rx_dropped_header_too_small; 371 drvs->rx_address_filtered = 372 port_stats->rx_address_filtered + 373 port_stats->rx_vlan_filtered; 374 drvs->rx_alignment_symbol_errors = 375 port_stats->rx_alignment_symbol_errors; 376 377 drvs->tx_pauseframes = port_stats->tx_pauseframes; 378 drvs->tx_controlframes = port_stats->tx_controlframes; 379 380 if (adapter->port_num) 381 drvs->jabber_events = rxf_stats->port1_jabber_events; 382 else 383 drvs->jabber_events = rxf_stats->port0_jabber_events; 384 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 385 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 386 drvs->forwarded_packets = rxf_stats->forwarded_packets; 387 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 388 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 389 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 390 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 391 } 392 393 static void populate_be_v1_stats(struct be_adapter *adapter) 394 { 395 struct be_hw_stats_v1 *hw_stats = hw_stats_from_cmd(adapter); 396 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 397 struct be_rxf_stats_v1 *rxf_stats = &hw_stats->rxf; 398 struct be_port_rxf_stats_v1 *port_stats = 399 &rxf_stats->port[adapter->port_num]; 400 struct be_drv_stats *drvs = &adapter->drv_stats; 401 402 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 403 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 404 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 405 drvs->rx_pause_frames = port_stats->rx_pause_frames; 406 drvs->rx_crc_errors = port_stats->rx_crc_errors; 407 drvs->rx_control_frames = port_stats->rx_control_frames; 408 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 409 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 410 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 411 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 412 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 413 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 414 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 415 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 416 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 417 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 418 drvs->rx_dropped_header_too_small = 419 port_stats->rx_dropped_header_too_small; 420 drvs->rx_input_fifo_overflow_drop = 421 port_stats->rx_input_fifo_overflow_drop; 422 drvs->rx_address_filtered = port_stats->rx_address_filtered; 423 drvs->rx_alignment_symbol_errors = 424 port_stats->rx_alignment_symbol_errors; 425 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 426 drvs->tx_pauseframes = port_stats->tx_pauseframes; 427 drvs->tx_controlframes = port_stats->tx_controlframes; 428 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 429 drvs->jabber_events = port_stats->jabber_events; 430 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 431 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 432 drvs->forwarded_packets = rxf_stats->forwarded_packets; 433 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 434 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 435 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 436 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 437 } 438 439 static void populate_be_v2_stats(struct be_adapter *adapter) 440 { 441 struct be_hw_stats_v2 *hw_stats = hw_stats_from_cmd(adapter); 442 struct be_pmem_stats *pmem_sts = &hw_stats->pmem; 443 struct be_rxf_stats_v2 *rxf_stats = &hw_stats->rxf; 444 struct be_port_rxf_stats_v2 *port_stats = 445 &rxf_stats->port[adapter->port_num]; 446 struct be_drv_stats *drvs = &adapter->drv_stats; 447 448 be_dws_le_to_cpu(hw_stats, sizeof(*hw_stats)); 449 drvs->pmem_fifo_overflow_drop = port_stats->pmem_fifo_overflow_drop; 450 drvs->rx_priority_pause_frames = port_stats->rx_priority_pause_frames; 451 drvs->rx_pause_frames = port_stats->rx_pause_frames; 452 drvs->rx_crc_errors = port_stats->rx_crc_errors; 453 drvs->rx_control_frames = port_stats->rx_control_frames; 454 drvs->rx_in_range_errors = port_stats->rx_in_range_errors; 455 drvs->rx_frame_too_long = port_stats->rx_frame_too_long; 456 drvs->rx_dropped_runt = port_stats->rx_dropped_runt; 457 drvs->rx_ip_checksum_errs = port_stats->rx_ip_checksum_errs; 458 drvs->rx_tcp_checksum_errs = port_stats->rx_tcp_checksum_errs; 459 drvs->rx_udp_checksum_errs = port_stats->rx_udp_checksum_errs; 460 drvs->rx_dropped_tcp_length = port_stats->rx_dropped_tcp_length; 461 drvs->rx_dropped_too_small = port_stats->rx_dropped_too_small; 462 drvs->rx_dropped_too_short = port_stats->rx_dropped_too_short; 463 drvs->rx_out_range_errors = port_stats->rx_out_range_errors; 464 drvs->rx_dropped_header_too_small = 465 port_stats->rx_dropped_header_too_small; 466 drvs->rx_input_fifo_overflow_drop = 467 port_stats->rx_input_fifo_overflow_drop; 468 drvs->rx_address_filtered = port_stats->rx_address_filtered; 469 drvs->rx_alignment_symbol_errors = 470 port_stats->rx_alignment_symbol_errors; 471 drvs->rxpp_fifo_overflow_drop = port_stats->rxpp_fifo_overflow_drop; 472 drvs->tx_pauseframes = port_stats->tx_pauseframes; 473 drvs->tx_controlframes = port_stats->tx_controlframes; 474 drvs->tx_priority_pauseframes = port_stats->tx_priority_pauseframes; 475 drvs->jabber_events = port_stats->jabber_events; 476 drvs->rx_drops_no_pbuf = rxf_stats->rx_drops_no_pbuf; 477 drvs->rx_drops_no_erx_descr = rxf_stats->rx_drops_no_erx_descr; 478 drvs->forwarded_packets = rxf_stats->forwarded_packets; 479 drvs->rx_drops_mtu = rxf_stats->rx_drops_mtu; 480 drvs->rx_drops_no_tpre_descr = rxf_stats->rx_drops_no_tpre_descr; 481 drvs->rx_drops_too_many_frags = rxf_stats->rx_drops_too_many_frags; 482 adapter->drv_stats.eth_red_drops = pmem_sts->eth_red_drops; 483 if (be_roce_supported(adapter)) { 484 drvs->rx_roce_bytes_lsd = port_stats->roce_bytes_received_lsd; 485 drvs->rx_roce_bytes_msd = port_stats->roce_bytes_received_msd; 486 drvs->rx_roce_frames = port_stats->roce_frames_received; 487 drvs->roce_drops_crc = port_stats->roce_drops_crc; 488 drvs->roce_drops_payload_len = 489 port_stats->roce_drops_payload_len; 490 } 491 } 492 493 static void populate_lancer_stats(struct be_adapter *adapter) 494 { 495 struct be_drv_stats *drvs = &adapter->drv_stats; 496 struct lancer_pport_stats *pport_stats = pport_stats_from_cmd(adapter); 497 498 be_dws_le_to_cpu(pport_stats, sizeof(*pport_stats)); 499 drvs->rx_pause_frames = pport_stats->rx_pause_frames_lo; 500 drvs->rx_crc_errors = pport_stats->rx_crc_errors_lo; 501 drvs->rx_control_frames = pport_stats->rx_control_frames_lo; 502 drvs->rx_in_range_errors = pport_stats->rx_in_range_errors; 503 drvs->rx_frame_too_long = pport_stats->rx_frames_too_long_lo; 504 drvs->rx_dropped_runt = pport_stats->rx_dropped_runt; 505 drvs->rx_ip_checksum_errs = pport_stats->rx_ip_checksum_errors; 506 drvs->rx_tcp_checksum_errs = pport_stats->rx_tcp_checksum_errors; 507 drvs->rx_udp_checksum_errs = pport_stats->rx_udp_checksum_errors; 508 drvs->rx_dropped_tcp_length = 509 pport_stats->rx_dropped_invalid_tcp_length; 510 drvs->rx_dropped_too_small = pport_stats->rx_dropped_too_small; 511 drvs->rx_dropped_too_short = pport_stats->rx_dropped_too_short; 512 drvs->rx_out_range_errors = pport_stats->rx_out_of_range_errors; 513 drvs->rx_dropped_header_too_small = 514 pport_stats->rx_dropped_header_too_small; 515 drvs->rx_input_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 516 drvs->rx_address_filtered = 517 pport_stats->rx_address_filtered + 518 pport_stats->rx_vlan_filtered; 519 drvs->rx_alignment_symbol_errors = pport_stats->rx_symbol_errors_lo; 520 drvs->rxpp_fifo_overflow_drop = pport_stats->rx_fifo_overflow; 521 drvs->tx_pauseframes = pport_stats->tx_pause_frames_lo; 522 drvs->tx_controlframes = pport_stats->tx_control_frames_lo; 523 drvs->jabber_events = pport_stats->rx_jabbers; 524 drvs->forwarded_packets = pport_stats->num_forwards_lo; 525 drvs->rx_drops_mtu = pport_stats->rx_drops_mtu_lo; 526 drvs->rx_drops_too_many_frags = 527 pport_stats->rx_drops_too_many_frags_lo; 528 } 529 530 static void accumulate_16bit_val(u32 *acc, u16 val) 531 { 532 #define lo(x) (x & 0xFFFF) 533 #define hi(x) (x & 0xFFFF0000) 534 bool wrapped = val < lo(*acc); 535 u32 newacc = hi(*acc) + val; 536 537 if (wrapped) 538 newacc += 65536; 539 ACCESS_ONCE(*acc) = newacc; 540 } 541 542 static void populate_erx_stats(struct be_adapter *adapter, 543 struct be_rx_obj *rxo, u32 erx_stat) 544 { 545 if (!BEx_chip(adapter)) 546 rx_stats(rxo)->rx_drops_no_frags = erx_stat; 547 else 548 /* below erx HW counter can actually wrap around after 549 * 65535. Driver accumulates a 32-bit value 550 */ 551 accumulate_16bit_val(&rx_stats(rxo)->rx_drops_no_frags, 552 (u16)erx_stat); 553 } 554 555 void be_parse_stats(struct be_adapter *adapter) 556 { 557 struct be_erx_stats_v2 *erx = be_erx_stats_from_cmd(adapter); 558 struct be_rx_obj *rxo; 559 int i; 560 u32 erx_stat; 561 562 if (lancer_chip(adapter)) { 563 populate_lancer_stats(adapter); 564 } else { 565 if (BE2_chip(adapter)) 566 populate_be_v0_stats(adapter); 567 else if (BE3_chip(adapter)) 568 /* for BE3 */ 569 populate_be_v1_stats(adapter); 570 else 571 populate_be_v2_stats(adapter); 572 573 /* erx_v2 is longer than v0, v1. use v2 for v0, v1 access */ 574 for_all_rx_queues(adapter, rxo, i) { 575 erx_stat = erx->rx_drops_no_fragments[rxo->q.id]; 576 populate_erx_stats(adapter, rxo, erx_stat); 577 } 578 } 579 } 580 581 static struct rtnl_link_stats64 *be_get_stats64(struct net_device *netdev, 582 struct rtnl_link_stats64 *stats) 583 { 584 struct be_adapter *adapter = netdev_priv(netdev); 585 struct be_drv_stats *drvs = &adapter->drv_stats; 586 struct be_rx_obj *rxo; 587 struct be_tx_obj *txo; 588 u64 pkts, bytes; 589 unsigned int start; 590 int i; 591 592 for_all_rx_queues(adapter, rxo, i) { 593 const struct be_rx_stats *rx_stats = rx_stats(rxo); 594 595 do { 596 start = u64_stats_fetch_begin_irq(&rx_stats->sync); 597 pkts = rx_stats(rxo)->rx_pkts; 598 bytes = rx_stats(rxo)->rx_bytes; 599 } while (u64_stats_fetch_retry_irq(&rx_stats->sync, start)); 600 stats->rx_packets += pkts; 601 stats->rx_bytes += bytes; 602 stats->multicast += rx_stats(rxo)->rx_mcast_pkts; 603 stats->rx_dropped += rx_stats(rxo)->rx_drops_no_skbs + 604 rx_stats(rxo)->rx_drops_no_frags; 605 } 606 607 for_all_tx_queues(adapter, txo, i) { 608 const struct be_tx_stats *tx_stats = tx_stats(txo); 609 610 do { 611 start = u64_stats_fetch_begin_irq(&tx_stats->sync); 612 pkts = tx_stats(txo)->tx_pkts; 613 bytes = tx_stats(txo)->tx_bytes; 614 } while (u64_stats_fetch_retry_irq(&tx_stats->sync, start)); 615 stats->tx_packets += pkts; 616 stats->tx_bytes += bytes; 617 } 618 619 /* bad pkts received */ 620 stats->rx_errors = drvs->rx_crc_errors + 621 drvs->rx_alignment_symbol_errors + 622 drvs->rx_in_range_errors + 623 drvs->rx_out_range_errors + 624 drvs->rx_frame_too_long + 625 drvs->rx_dropped_too_small + 626 drvs->rx_dropped_too_short + 627 drvs->rx_dropped_header_too_small + 628 drvs->rx_dropped_tcp_length + 629 drvs->rx_dropped_runt; 630 631 /* detailed rx errors */ 632 stats->rx_length_errors = drvs->rx_in_range_errors + 633 drvs->rx_out_range_errors + 634 drvs->rx_frame_too_long; 635 636 stats->rx_crc_errors = drvs->rx_crc_errors; 637 638 /* frame alignment errors */ 639 stats->rx_frame_errors = drvs->rx_alignment_symbol_errors; 640 641 /* receiver fifo overrun */ 642 /* drops_no_pbuf is no per i/f, it's per BE card */ 643 stats->rx_fifo_errors = drvs->rxpp_fifo_overflow_drop + 644 drvs->rx_input_fifo_overflow_drop + 645 drvs->rx_drops_no_pbuf; 646 return stats; 647 } 648 649 void be_link_status_update(struct be_adapter *adapter, u8 link_status) 650 { 651 struct net_device *netdev = adapter->netdev; 652 653 if (!(adapter->flags & BE_FLAGS_LINK_STATUS_INIT)) { 654 netif_carrier_off(netdev); 655 adapter->flags |= BE_FLAGS_LINK_STATUS_INIT; 656 } 657 658 if (link_status) 659 netif_carrier_on(netdev); 660 else 661 netif_carrier_off(netdev); 662 } 663 664 static void be_tx_stats_update(struct be_tx_obj *txo, struct sk_buff *skb) 665 { 666 struct be_tx_stats *stats = tx_stats(txo); 667 668 u64_stats_update_begin(&stats->sync); 669 stats->tx_reqs++; 670 stats->tx_bytes += skb->len; 671 stats->tx_pkts += (skb_shinfo(skb)->gso_segs ? : 1); 672 u64_stats_update_end(&stats->sync); 673 } 674 675 /* Returns number of WRBs needed for the skb */ 676 static u32 skb_wrb_cnt(struct sk_buff *skb) 677 { 678 /* +1 for the header wrb */ 679 return 1 + (skb_headlen(skb) ? 1 : 0) + skb_shinfo(skb)->nr_frags; 680 } 681 682 static inline void wrb_fill(struct be_eth_wrb *wrb, u64 addr, int len) 683 { 684 wrb->frag_pa_hi = cpu_to_le32(upper_32_bits(addr)); 685 wrb->frag_pa_lo = cpu_to_le32(lower_32_bits(addr)); 686 wrb->frag_len = cpu_to_le32(len & ETH_WRB_FRAG_LEN_MASK); 687 wrb->rsvd0 = 0; 688 } 689 690 /* A dummy wrb is just all zeros. Using a separate routine for dummy-wrb 691 * to avoid the swap and shift/mask operations in wrb_fill(). 692 */ 693 static inline void wrb_fill_dummy(struct be_eth_wrb *wrb) 694 { 695 wrb->frag_pa_hi = 0; 696 wrb->frag_pa_lo = 0; 697 wrb->frag_len = 0; 698 wrb->rsvd0 = 0; 699 } 700 701 static inline u16 be_get_tx_vlan_tag(struct be_adapter *adapter, 702 struct sk_buff *skb) 703 { 704 u8 vlan_prio; 705 u16 vlan_tag; 706 707 vlan_tag = skb_vlan_tag_get(skb); 708 vlan_prio = (vlan_tag & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 709 /* If vlan priority provided by OS is NOT in available bmap */ 710 if (!(adapter->vlan_prio_bmap & (1 << vlan_prio))) 711 vlan_tag = (vlan_tag & ~VLAN_PRIO_MASK) | 712 adapter->recommended_prio; 713 714 return vlan_tag; 715 } 716 717 /* Used only for IP tunnel packets */ 718 static u16 skb_inner_ip_proto(struct sk_buff *skb) 719 { 720 return (inner_ip_hdr(skb)->version == 4) ? 721 inner_ip_hdr(skb)->protocol : inner_ipv6_hdr(skb)->nexthdr; 722 } 723 724 static u16 skb_ip_proto(struct sk_buff *skb) 725 { 726 return (ip_hdr(skb)->version == 4) ? 727 ip_hdr(skb)->protocol : ipv6_hdr(skb)->nexthdr; 728 } 729 730 static void wrb_fill_hdr(struct be_adapter *adapter, struct be_eth_hdr_wrb *hdr, 731 struct sk_buff *skb, u32 wrb_cnt, u32 len, 732 bool skip_hw_vlan) 733 { 734 u16 vlan_tag, proto; 735 736 memset(hdr, 0, sizeof(*hdr)); 737 738 SET_TX_WRB_HDR_BITS(crc, hdr, 1); 739 740 if (skb_is_gso(skb)) { 741 SET_TX_WRB_HDR_BITS(lso, hdr, 1); 742 SET_TX_WRB_HDR_BITS(lso_mss, hdr, skb_shinfo(skb)->gso_size); 743 if (skb_is_gso_v6(skb) && !lancer_chip(adapter)) 744 SET_TX_WRB_HDR_BITS(lso6, hdr, 1); 745 } else if (skb->ip_summed == CHECKSUM_PARTIAL) { 746 if (skb->encapsulation) { 747 SET_TX_WRB_HDR_BITS(ipcs, hdr, 1); 748 proto = skb_inner_ip_proto(skb); 749 } else { 750 proto = skb_ip_proto(skb); 751 } 752 if (proto == IPPROTO_TCP) 753 SET_TX_WRB_HDR_BITS(tcpcs, hdr, 1); 754 else if (proto == IPPROTO_UDP) 755 SET_TX_WRB_HDR_BITS(udpcs, hdr, 1); 756 } 757 758 if (skb_vlan_tag_present(skb)) { 759 SET_TX_WRB_HDR_BITS(vlan, hdr, 1); 760 vlan_tag = be_get_tx_vlan_tag(adapter, skb); 761 SET_TX_WRB_HDR_BITS(vlan_tag, hdr, vlan_tag); 762 } 763 764 SET_TX_WRB_HDR_BITS(num_wrb, hdr, wrb_cnt); 765 SET_TX_WRB_HDR_BITS(len, hdr, len); 766 767 /* Hack to skip HW VLAN tagging needs evt = 1, compl = 0 768 * When this hack is not needed, the evt bit is set while ringing DB 769 */ 770 if (skip_hw_vlan) 771 SET_TX_WRB_HDR_BITS(event, hdr, 1); 772 } 773 774 static void unmap_tx_frag(struct device *dev, struct be_eth_wrb *wrb, 775 bool unmap_single) 776 { 777 dma_addr_t dma; 778 u32 frag_len = le32_to_cpu(wrb->frag_len); 779 780 781 dma = (u64)le32_to_cpu(wrb->frag_pa_hi) << 32 | 782 (u64)le32_to_cpu(wrb->frag_pa_lo); 783 if (frag_len) { 784 if (unmap_single) 785 dma_unmap_single(dev, dma, frag_len, DMA_TO_DEVICE); 786 else 787 dma_unmap_page(dev, dma, frag_len, DMA_TO_DEVICE); 788 } 789 } 790 791 /* Returns the number of WRBs used up by the skb */ 792 static u32 be_xmit_enqueue(struct be_adapter *adapter, struct be_tx_obj *txo, 793 struct sk_buff *skb, bool skip_hw_vlan) 794 { 795 u32 i, copied = 0, wrb_cnt = skb_wrb_cnt(skb); 796 struct device *dev = &adapter->pdev->dev; 797 struct be_queue_info *txq = &txo->q; 798 struct be_eth_hdr_wrb *hdr; 799 bool map_single = false; 800 struct be_eth_wrb *wrb; 801 dma_addr_t busaddr; 802 u16 head = txq->head; 803 804 hdr = queue_head_node(txq); 805 wrb_fill_hdr(adapter, hdr, skb, wrb_cnt, skb->len, skip_hw_vlan); 806 be_dws_cpu_to_le(hdr, sizeof(*hdr)); 807 808 queue_head_inc(txq); 809 810 if (skb->len > skb->data_len) { 811 int len = skb_headlen(skb); 812 813 busaddr = dma_map_single(dev, skb->data, len, DMA_TO_DEVICE); 814 if (dma_mapping_error(dev, busaddr)) 815 goto dma_err; 816 map_single = true; 817 wrb = queue_head_node(txq); 818 wrb_fill(wrb, busaddr, len); 819 queue_head_inc(txq); 820 copied += len; 821 } 822 823 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 824 const struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[i]; 825 826 busaddr = skb_frag_dma_map(dev, frag, 0, 827 skb_frag_size(frag), DMA_TO_DEVICE); 828 if (dma_mapping_error(dev, busaddr)) 829 goto dma_err; 830 wrb = queue_head_node(txq); 831 wrb_fill(wrb, busaddr, skb_frag_size(frag)); 832 queue_head_inc(txq); 833 copied += skb_frag_size(frag); 834 } 835 836 BUG_ON(txo->sent_skb_list[head]); 837 txo->sent_skb_list[head] = skb; 838 txo->last_req_hdr = head; 839 atomic_add(wrb_cnt, &txq->used); 840 txo->last_req_wrb_cnt = wrb_cnt; 841 txo->pend_wrb_cnt += wrb_cnt; 842 843 be_tx_stats_update(txo, skb); 844 return wrb_cnt; 845 846 dma_err: 847 /* Bring the queue back to the state it was in before this 848 * routine was invoked. 849 */ 850 txq->head = head; 851 /* skip the first wrb (hdr); it's not mapped */ 852 queue_head_inc(txq); 853 while (copied) { 854 wrb = queue_head_node(txq); 855 unmap_tx_frag(dev, wrb, map_single); 856 map_single = false; 857 copied -= le32_to_cpu(wrb->frag_len); 858 adapter->drv_stats.dma_map_errors++; 859 queue_head_inc(txq); 860 } 861 txq->head = head; 862 return 0; 863 } 864 865 static inline int qnq_async_evt_rcvd(struct be_adapter *adapter) 866 { 867 return adapter->flags & BE_FLAGS_QNQ_ASYNC_EVT_RCVD; 868 } 869 870 static struct sk_buff *be_insert_vlan_in_pkt(struct be_adapter *adapter, 871 struct sk_buff *skb, 872 bool *skip_hw_vlan) 873 { 874 u16 vlan_tag = 0; 875 876 skb = skb_share_check(skb, GFP_ATOMIC); 877 if (unlikely(!skb)) 878 return skb; 879 880 if (skb_vlan_tag_present(skb)) 881 vlan_tag = be_get_tx_vlan_tag(adapter, skb); 882 883 if (qnq_async_evt_rcvd(adapter) && adapter->pvid) { 884 if (!vlan_tag) 885 vlan_tag = adapter->pvid; 886 /* f/w workaround to set skip_hw_vlan = 1, informs the F/W to 887 * skip VLAN insertion 888 */ 889 if (skip_hw_vlan) 890 *skip_hw_vlan = true; 891 } 892 893 if (vlan_tag) { 894 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 895 vlan_tag); 896 if (unlikely(!skb)) 897 return skb; 898 skb->vlan_tci = 0; 899 } 900 901 /* Insert the outer VLAN, if any */ 902 if (adapter->qnq_vid) { 903 vlan_tag = adapter->qnq_vid; 904 skb = vlan_insert_tag_set_proto(skb, htons(ETH_P_8021Q), 905 vlan_tag); 906 if (unlikely(!skb)) 907 return skb; 908 if (skip_hw_vlan) 909 *skip_hw_vlan = true; 910 } 911 912 return skb; 913 } 914 915 static bool be_ipv6_exthdr_check(struct sk_buff *skb) 916 { 917 struct ethhdr *eh = (struct ethhdr *)skb->data; 918 u16 offset = ETH_HLEN; 919 920 if (eh->h_proto == htons(ETH_P_IPV6)) { 921 struct ipv6hdr *ip6h = (struct ipv6hdr *)(skb->data + offset); 922 923 offset += sizeof(struct ipv6hdr); 924 if (ip6h->nexthdr != NEXTHDR_TCP && 925 ip6h->nexthdr != NEXTHDR_UDP) { 926 struct ipv6_opt_hdr *ehdr = 927 (struct ipv6_opt_hdr *)(skb->data + offset); 928 929 /* offending pkt: 2nd byte following IPv6 hdr is 0xff */ 930 if (ehdr->hdrlen == 0xff) 931 return true; 932 } 933 } 934 return false; 935 } 936 937 static int be_vlan_tag_tx_chk(struct be_adapter *adapter, struct sk_buff *skb) 938 { 939 return skb_vlan_tag_present(skb) || adapter->pvid || adapter->qnq_vid; 940 } 941 942 static int be_ipv6_tx_stall_chk(struct be_adapter *adapter, struct sk_buff *skb) 943 { 944 return BE3_chip(adapter) && be_ipv6_exthdr_check(skb); 945 } 946 947 static struct sk_buff *be_lancer_xmit_workarounds(struct be_adapter *adapter, 948 struct sk_buff *skb, 949 bool *skip_hw_vlan) 950 { 951 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data; 952 unsigned int eth_hdr_len; 953 struct iphdr *ip; 954 955 /* For padded packets, BE HW modifies tot_len field in IP header 956 * incorrecly when VLAN tag is inserted by HW. 957 * For padded packets, Lancer computes incorrect checksum. 958 */ 959 eth_hdr_len = ntohs(skb->protocol) == ETH_P_8021Q ? 960 VLAN_ETH_HLEN : ETH_HLEN; 961 if (skb->len <= 60 && 962 (lancer_chip(adapter) || skb_vlan_tag_present(skb)) && 963 is_ipv4_pkt(skb)) { 964 ip = (struct iphdr *)ip_hdr(skb); 965 pskb_trim(skb, eth_hdr_len + ntohs(ip->tot_len)); 966 } 967 968 /* If vlan tag is already inlined in the packet, skip HW VLAN 969 * tagging in pvid-tagging mode 970 */ 971 if (be_pvid_tagging_enabled(adapter) && 972 veh->h_vlan_proto == htons(ETH_P_8021Q)) 973 *skip_hw_vlan = true; 974 975 /* HW has a bug wherein it will calculate CSUM for VLAN 976 * pkts even though it is disabled. 977 * Manually insert VLAN in pkt. 978 */ 979 if (skb->ip_summed != CHECKSUM_PARTIAL && 980 skb_vlan_tag_present(skb)) { 981 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan); 982 if (unlikely(!skb)) 983 goto err; 984 } 985 986 /* HW may lockup when VLAN HW tagging is requested on 987 * certain ipv6 packets. Drop such pkts if the HW workaround to 988 * skip HW tagging is not enabled by FW. 989 */ 990 if (unlikely(be_ipv6_tx_stall_chk(adapter, skb) && 991 (adapter->pvid || adapter->qnq_vid) && 992 !qnq_async_evt_rcvd(adapter))) 993 goto tx_drop; 994 995 /* Manual VLAN tag insertion to prevent: 996 * ASIC lockup when the ASIC inserts VLAN tag into 997 * certain ipv6 packets. Insert VLAN tags in driver, 998 * and set event, completion, vlan bits accordingly 999 * in the Tx WRB. 1000 */ 1001 if (be_ipv6_tx_stall_chk(adapter, skb) && 1002 be_vlan_tag_tx_chk(adapter, skb)) { 1003 skb = be_insert_vlan_in_pkt(adapter, skb, skip_hw_vlan); 1004 if (unlikely(!skb)) 1005 goto err; 1006 } 1007 1008 return skb; 1009 tx_drop: 1010 dev_kfree_skb_any(skb); 1011 err: 1012 return NULL; 1013 } 1014 1015 static struct sk_buff *be_xmit_workarounds(struct be_adapter *adapter, 1016 struct sk_buff *skb, 1017 bool *skip_hw_vlan) 1018 { 1019 /* Lancer, SH-R ASICs have a bug wherein Packets that are 32 bytes or 1020 * less may cause a transmit stall on that port. So the work-around is 1021 * to pad short packets (<= 32 bytes) to a 36-byte length. 1022 */ 1023 if (unlikely(!BEx_chip(adapter) && skb->len <= 32)) { 1024 if (skb_put_padto(skb, 36)) 1025 return NULL; 1026 } 1027 1028 if (BEx_chip(adapter) || lancer_chip(adapter)) { 1029 skb = be_lancer_xmit_workarounds(adapter, skb, skip_hw_vlan); 1030 if (!skb) 1031 return NULL; 1032 } 1033 1034 return skb; 1035 } 1036 1037 static void be_xmit_flush(struct be_adapter *adapter, struct be_tx_obj *txo) 1038 { 1039 struct be_queue_info *txq = &txo->q; 1040 struct be_eth_hdr_wrb *hdr = queue_index_node(txq, txo->last_req_hdr); 1041 1042 /* Mark the last request eventable if it hasn't been marked already */ 1043 if (!(hdr->dw[2] & cpu_to_le32(TX_HDR_WRB_EVT))) 1044 hdr->dw[2] |= cpu_to_le32(TX_HDR_WRB_EVT | TX_HDR_WRB_COMPL); 1045 1046 /* compose a dummy wrb if there are odd set of wrbs to notify */ 1047 if (!lancer_chip(adapter) && (txo->pend_wrb_cnt & 1)) { 1048 wrb_fill_dummy(queue_head_node(txq)); 1049 queue_head_inc(txq); 1050 atomic_inc(&txq->used); 1051 txo->pend_wrb_cnt++; 1052 hdr->dw[2] &= ~cpu_to_le32(TX_HDR_WRB_NUM_MASK << 1053 TX_HDR_WRB_NUM_SHIFT); 1054 hdr->dw[2] |= cpu_to_le32((txo->last_req_wrb_cnt + 1) << 1055 TX_HDR_WRB_NUM_SHIFT); 1056 } 1057 be_txq_notify(adapter, txo, txo->pend_wrb_cnt); 1058 txo->pend_wrb_cnt = 0; 1059 } 1060 1061 static netdev_tx_t be_xmit(struct sk_buff *skb, struct net_device *netdev) 1062 { 1063 bool skip_hw_vlan = false, flush = !skb->xmit_more; 1064 struct be_adapter *adapter = netdev_priv(netdev); 1065 u16 q_idx = skb_get_queue_mapping(skb); 1066 struct be_tx_obj *txo = &adapter->tx_obj[q_idx]; 1067 struct be_queue_info *txq = &txo->q; 1068 u16 wrb_cnt; 1069 1070 skb = be_xmit_workarounds(adapter, skb, &skip_hw_vlan); 1071 if (unlikely(!skb)) 1072 goto drop; 1073 1074 wrb_cnt = be_xmit_enqueue(adapter, txo, skb, skip_hw_vlan); 1075 if (unlikely(!wrb_cnt)) { 1076 dev_kfree_skb_any(skb); 1077 goto drop; 1078 } 1079 1080 if ((atomic_read(&txq->used) + BE_MAX_TX_FRAG_COUNT) >= txq->len) { 1081 netif_stop_subqueue(netdev, q_idx); 1082 tx_stats(txo)->tx_stops++; 1083 } 1084 1085 if (flush || __netif_subqueue_stopped(netdev, q_idx)) 1086 be_xmit_flush(adapter, txo); 1087 1088 return NETDEV_TX_OK; 1089 drop: 1090 tx_stats(txo)->tx_drv_drops++; 1091 /* Flush the already enqueued tx requests */ 1092 if (flush && txo->pend_wrb_cnt) 1093 be_xmit_flush(adapter, txo); 1094 1095 return NETDEV_TX_OK; 1096 } 1097 1098 static int be_change_mtu(struct net_device *netdev, int new_mtu) 1099 { 1100 struct be_adapter *adapter = netdev_priv(netdev); 1101 struct device *dev = &adapter->pdev->dev; 1102 1103 if (new_mtu < BE_MIN_MTU || new_mtu > BE_MAX_MTU) { 1104 dev_info(dev, "MTU must be between %d and %d bytes\n", 1105 BE_MIN_MTU, BE_MAX_MTU); 1106 return -EINVAL; 1107 } 1108 1109 dev_info(dev, "MTU changed from %d to %d bytes\n", 1110 netdev->mtu, new_mtu); 1111 netdev->mtu = new_mtu; 1112 return 0; 1113 } 1114 1115 static inline bool be_in_all_promisc(struct be_adapter *adapter) 1116 { 1117 return (adapter->if_flags & BE_IF_FLAGS_ALL_PROMISCUOUS) == 1118 BE_IF_FLAGS_ALL_PROMISCUOUS; 1119 } 1120 1121 static int be_set_vlan_promisc(struct be_adapter *adapter) 1122 { 1123 struct device *dev = &adapter->pdev->dev; 1124 int status; 1125 1126 if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) 1127 return 0; 1128 1129 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, ON); 1130 if (!status) { 1131 dev_info(dev, "Enabled VLAN promiscuous mode\n"); 1132 adapter->if_flags |= BE_IF_FLAGS_VLAN_PROMISCUOUS; 1133 } else { 1134 dev_err(dev, "Failed to enable VLAN promiscuous mode\n"); 1135 } 1136 return status; 1137 } 1138 1139 static int be_clear_vlan_promisc(struct be_adapter *adapter) 1140 { 1141 struct device *dev = &adapter->pdev->dev; 1142 int status; 1143 1144 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_VLAN_PROMISCUOUS, OFF); 1145 if (!status) { 1146 dev_info(dev, "Disabling VLAN promiscuous mode\n"); 1147 adapter->if_flags &= ~BE_IF_FLAGS_VLAN_PROMISCUOUS; 1148 } 1149 return status; 1150 } 1151 1152 /* 1153 * A max of 64 (BE_NUM_VLANS_SUPPORTED) vlans can be configured in BE. 1154 * If the user configures more, place BE in vlan promiscuous mode. 1155 */ 1156 static int be_vid_config(struct be_adapter *adapter) 1157 { 1158 struct device *dev = &adapter->pdev->dev; 1159 u16 vids[BE_NUM_VLANS_SUPPORTED]; 1160 u16 num = 0, i = 0; 1161 int status = 0; 1162 1163 /* No need to further configure vids if in promiscuous mode */ 1164 if (be_in_all_promisc(adapter)) 1165 return 0; 1166 1167 if (adapter->vlans_added > be_max_vlans(adapter)) 1168 return be_set_vlan_promisc(adapter); 1169 1170 /* Construct VLAN Table to give to HW */ 1171 for_each_set_bit(i, adapter->vids, VLAN_N_VID) 1172 vids[num++] = cpu_to_le16(i); 1173 1174 status = be_cmd_vlan_config(adapter, adapter->if_handle, vids, num); 1175 if (status) { 1176 dev_err(dev, "Setting HW VLAN filtering failed\n"); 1177 /* Set to VLAN promisc mode as setting VLAN filter failed */ 1178 if (addl_status(status) == 1179 MCC_ADDL_STATUS_INSUFFICIENT_RESOURCES) 1180 return be_set_vlan_promisc(adapter); 1181 } else if (adapter->if_flags & BE_IF_FLAGS_VLAN_PROMISCUOUS) { 1182 status = be_clear_vlan_promisc(adapter); 1183 } 1184 return status; 1185 } 1186 1187 static int be_vlan_add_vid(struct net_device *netdev, __be16 proto, u16 vid) 1188 { 1189 struct be_adapter *adapter = netdev_priv(netdev); 1190 int status = 0; 1191 1192 /* Packets with VID 0 are always received by Lancer by default */ 1193 if (lancer_chip(adapter) && vid == 0) 1194 return status; 1195 1196 if (test_bit(vid, adapter->vids)) 1197 return status; 1198 1199 set_bit(vid, adapter->vids); 1200 adapter->vlans_added++; 1201 1202 status = be_vid_config(adapter); 1203 if (status) { 1204 adapter->vlans_added--; 1205 clear_bit(vid, adapter->vids); 1206 } 1207 1208 return status; 1209 } 1210 1211 static int be_vlan_rem_vid(struct net_device *netdev, __be16 proto, u16 vid) 1212 { 1213 struct be_adapter *adapter = netdev_priv(netdev); 1214 1215 /* Packets with VID 0 are always received by Lancer by default */ 1216 if (lancer_chip(adapter) && vid == 0) 1217 return 0; 1218 1219 clear_bit(vid, adapter->vids); 1220 adapter->vlans_added--; 1221 1222 return be_vid_config(adapter); 1223 } 1224 1225 static void be_clear_all_promisc(struct be_adapter *adapter) 1226 { 1227 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, OFF); 1228 adapter->if_flags &= ~BE_IF_FLAGS_ALL_PROMISCUOUS; 1229 } 1230 1231 static void be_set_all_promisc(struct be_adapter *adapter) 1232 { 1233 be_cmd_rx_filter(adapter, BE_IF_FLAGS_ALL_PROMISCUOUS, ON); 1234 adapter->if_flags |= BE_IF_FLAGS_ALL_PROMISCUOUS; 1235 } 1236 1237 static void be_set_mc_promisc(struct be_adapter *adapter) 1238 { 1239 int status; 1240 1241 if (adapter->if_flags & BE_IF_FLAGS_MCAST_PROMISCUOUS) 1242 return; 1243 1244 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MCAST_PROMISCUOUS, ON); 1245 if (!status) 1246 adapter->if_flags |= BE_IF_FLAGS_MCAST_PROMISCUOUS; 1247 } 1248 1249 static void be_set_mc_list(struct be_adapter *adapter) 1250 { 1251 int status; 1252 1253 status = be_cmd_rx_filter(adapter, BE_IF_FLAGS_MULTICAST, ON); 1254 if (!status) 1255 adapter->if_flags &= ~BE_IF_FLAGS_MCAST_PROMISCUOUS; 1256 else 1257 be_set_mc_promisc(adapter); 1258 } 1259 1260 static void be_set_uc_list(struct be_adapter *adapter) 1261 { 1262 struct netdev_hw_addr *ha; 1263 int i = 1; /* First slot is claimed by the Primary MAC */ 1264 1265 for (; adapter->uc_macs > 0; adapter->uc_macs--, i++) 1266 be_cmd_pmac_del(adapter, adapter->if_handle, 1267 adapter->pmac_id[i], 0); 1268 1269 if (netdev_uc_count(adapter->netdev) > be_max_uc(adapter)) { 1270 be_set_all_promisc(adapter); 1271 return; 1272 } 1273 1274 netdev_for_each_uc_addr(ha, adapter->netdev) { 1275 adapter->uc_macs++; /* First slot is for Primary MAC */ 1276 be_cmd_pmac_add(adapter, (u8 *)ha->addr, adapter->if_handle, 1277 &adapter->pmac_id[adapter->uc_macs], 0); 1278 } 1279 } 1280 1281 static void be_clear_uc_list(struct be_adapter *adapter) 1282 { 1283 int i; 1284 1285 for (i = 1; i < (adapter->uc_macs + 1); i++) 1286 be_cmd_pmac_del(adapter, adapter->if_handle, 1287 adapter->pmac_id[i], 0); 1288 adapter->uc_macs = 0; 1289 } 1290 1291 static void be_set_rx_mode(struct net_device *netdev) 1292 { 1293 struct be_adapter *adapter = netdev_priv(netdev); 1294 1295 if (netdev->flags & IFF_PROMISC) { 1296 be_set_all_promisc(adapter); 1297 return; 1298 } 1299 1300 /* Interface was previously in promiscuous mode; disable it */ 1301 if (be_in_all_promisc(adapter)) { 1302 be_clear_all_promisc(adapter); 1303 if (adapter->vlans_added) 1304 be_vid_config(adapter); 1305 } 1306 1307 /* Enable multicast promisc if num configured exceeds what we support */ 1308 if (netdev->flags & IFF_ALLMULTI || 1309 netdev_mc_count(netdev) > be_max_mc(adapter)) { 1310 be_set_mc_promisc(adapter); 1311 return; 1312 } 1313 1314 if (netdev_uc_count(netdev) != adapter->uc_macs) 1315 be_set_uc_list(adapter); 1316 1317 be_set_mc_list(adapter); 1318 } 1319 1320 static int be_set_vf_mac(struct net_device *netdev, int vf, u8 *mac) 1321 { 1322 struct be_adapter *adapter = netdev_priv(netdev); 1323 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1324 int status; 1325 1326 if (!sriov_enabled(adapter)) 1327 return -EPERM; 1328 1329 if (!is_valid_ether_addr(mac) || vf >= adapter->num_vfs) 1330 return -EINVAL; 1331 1332 /* Proceed further only if user provided MAC is different 1333 * from active MAC 1334 */ 1335 if (ether_addr_equal(mac, vf_cfg->mac_addr)) 1336 return 0; 1337 1338 if (BEx_chip(adapter)) { 1339 be_cmd_pmac_del(adapter, vf_cfg->if_handle, vf_cfg->pmac_id, 1340 vf + 1); 1341 1342 status = be_cmd_pmac_add(adapter, mac, vf_cfg->if_handle, 1343 &vf_cfg->pmac_id, vf + 1); 1344 } else { 1345 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 1346 vf + 1); 1347 } 1348 1349 if (status) { 1350 dev_err(&adapter->pdev->dev, "MAC %pM set on VF %d Failed: %#x", 1351 mac, vf, status); 1352 return be_cmd_status(status); 1353 } 1354 1355 ether_addr_copy(vf_cfg->mac_addr, mac); 1356 1357 return 0; 1358 } 1359 1360 static int be_get_vf_config(struct net_device *netdev, int vf, 1361 struct ifla_vf_info *vi) 1362 { 1363 struct be_adapter *adapter = netdev_priv(netdev); 1364 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1365 1366 if (!sriov_enabled(adapter)) 1367 return -EPERM; 1368 1369 if (vf >= adapter->num_vfs) 1370 return -EINVAL; 1371 1372 vi->vf = vf; 1373 vi->max_tx_rate = vf_cfg->tx_rate; 1374 vi->min_tx_rate = 0; 1375 vi->vlan = vf_cfg->vlan_tag & VLAN_VID_MASK; 1376 vi->qos = vf_cfg->vlan_tag >> VLAN_PRIO_SHIFT; 1377 memcpy(&vi->mac, vf_cfg->mac_addr, ETH_ALEN); 1378 vi->linkstate = adapter->vf_cfg[vf].plink_tracking; 1379 1380 return 0; 1381 } 1382 1383 static int be_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan, u8 qos) 1384 { 1385 struct be_adapter *adapter = netdev_priv(netdev); 1386 struct be_vf_cfg *vf_cfg = &adapter->vf_cfg[vf]; 1387 int status = 0; 1388 1389 if (!sriov_enabled(adapter)) 1390 return -EPERM; 1391 1392 if (vf >= adapter->num_vfs || vlan > 4095 || qos > 7) 1393 return -EINVAL; 1394 1395 if (vlan || qos) { 1396 vlan |= qos << VLAN_PRIO_SHIFT; 1397 if (vf_cfg->vlan_tag != vlan) 1398 status = be_cmd_set_hsw_config(adapter, vlan, vf + 1, 1399 vf_cfg->if_handle, 0); 1400 } else { 1401 /* Reset Transparent Vlan Tagging. */ 1402 status = be_cmd_set_hsw_config(adapter, BE_RESET_VLAN_TAG_ID, 1403 vf + 1, vf_cfg->if_handle, 0); 1404 } 1405 1406 if (status) { 1407 dev_err(&adapter->pdev->dev, 1408 "VLAN %d config on VF %d failed : %#x\n", vlan, 1409 vf, status); 1410 return be_cmd_status(status); 1411 } 1412 1413 vf_cfg->vlan_tag = vlan; 1414 1415 return 0; 1416 } 1417 1418 static int be_set_vf_tx_rate(struct net_device *netdev, int vf, 1419 int min_tx_rate, int max_tx_rate) 1420 { 1421 struct be_adapter *adapter = netdev_priv(netdev); 1422 struct device *dev = &adapter->pdev->dev; 1423 int percent_rate, status = 0; 1424 u16 link_speed = 0; 1425 u8 link_status; 1426 1427 if (!sriov_enabled(adapter)) 1428 return -EPERM; 1429 1430 if (vf >= adapter->num_vfs) 1431 return -EINVAL; 1432 1433 if (min_tx_rate) 1434 return -EINVAL; 1435 1436 if (!max_tx_rate) 1437 goto config_qos; 1438 1439 status = be_cmd_link_status_query(adapter, &link_speed, 1440 &link_status, 0); 1441 if (status) 1442 goto err; 1443 1444 if (!link_status) { 1445 dev_err(dev, "TX-rate setting not allowed when link is down\n"); 1446 status = -ENETDOWN; 1447 goto err; 1448 } 1449 1450 if (max_tx_rate < 100 || max_tx_rate > link_speed) { 1451 dev_err(dev, "TX-rate must be between 100 and %d Mbps\n", 1452 link_speed); 1453 status = -EINVAL; 1454 goto err; 1455 } 1456 1457 /* On Skyhawk the QOS setting must be done only as a % value */ 1458 percent_rate = link_speed / 100; 1459 if (skyhawk_chip(adapter) && (max_tx_rate % percent_rate)) { 1460 dev_err(dev, "TX-rate must be a multiple of %d Mbps\n", 1461 percent_rate); 1462 status = -EINVAL; 1463 goto err; 1464 } 1465 1466 config_qos: 1467 status = be_cmd_config_qos(adapter, max_tx_rate, link_speed, vf + 1); 1468 if (status) 1469 goto err; 1470 1471 adapter->vf_cfg[vf].tx_rate = max_tx_rate; 1472 return 0; 1473 1474 err: 1475 dev_err(dev, "TX-rate setting of %dMbps on VF%d failed\n", 1476 max_tx_rate, vf); 1477 return be_cmd_status(status); 1478 } 1479 1480 static int be_set_vf_link_state(struct net_device *netdev, int vf, 1481 int link_state) 1482 { 1483 struct be_adapter *adapter = netdev_priv(netdev); 1484 int status; 1485 1486 if (!sriov_enabled(adapter)) 1487 return -EPERM; 1488 1489 if (vf >= adapter->num_vfs) 1490 return -EINVAL; 1491 1492 status = be_cmd_set_logical_link_config(adapter, link_state, vf+1); 1493 if (status) { 1494 dev_err(&adapter->pdev->dev, 1495 "Link state change on VF %d failed: %#x\n", vf, status); 1496 return be_cmd_status(status); 1497 } 1498 1499 adapter->vf_cfg[vf].plink_tracking = link_state; 1500 1501 return 0; 1502 } 1503 1504 static void be_aic_update(struct be_aic_obj *aic, u64 rx_pkts, u64 tx_pkts, 1505 ulong now) 1506 { 1507 aic->rx_pkts_prev = rx_pkts; 1508 aic->tx_reqs_prev = tx_pkts; 1509 aic->jiffies = now; 1510 } 1511 1512 static void be_eqd_update(struct be_adapter *adapter) 1513 { 1514 struct be_set_eqd set_eqd[MAX_EVT_QS]; 1515 int eqd, i, num = 0, start; 1516 struct be_aic_obj *aic; 1517 struct be_eq_obj *eqo; 1518 struct be_rx_obj *rxo; 1519 struct be_tx_obj *txo; 1520 u64 rx_pkts, tx_pkts; 1521 ulong now; 1522 u32 pps, delta; 1523 1524 for_all_evt_queues(adapter, eqo, i) { 1525 aic = &adapter->aic_obj[eqo->idx]; 1526 if (!aic->enable) { 1527 if (aic->jiffies) 1528 aic->jiffies = 0; 1529 eqd = aic->et_eqd; 1530 goto modify_eqd; 1531 } 1532 1533 rxo = &adapter->rx_obj[eqo->idx]; 1534 do { 1535 start = u64_stats_fetch_begin_irq(&rxo->stats.sync); 1536 rx_pkts = rxo->stats.rx_pkts; 1537 } while (u64_stats_fetch_retry_irq(&rxo->stats.sync, start)); 1538 1539 txo = &adapter->tx_obj[eqo->idx]; 1540 do { 1541 start = u64_stats_fetch_begin_irq(&txo->stats.sync); 1542 tx_pkts = txo->stats.tx_reqs; 1543 } while (u64_stats_fetch_retry_irq(&txo->stats.sync, start)); 1544 1545 /* Skip, if wrapped around or first calculation */ 1546 now = jiffies; 1547 if (!aic->jiffies || time_before(now, aic->jiffies) || 1548 rx_pkts < aic->rx_pkts_prev || 1549 tx_pkts < aic->tx_reqs_prev) { 1550 be_aic_update(aic, rx_pkts, tx_pkts, now); 1551 continue; 1552 } 1553 1554 delta = jiffies_to_msecs(now - aic->jiffies); 1555 pps = (((u32)(rx_pkts - aic->rx_pkts_prev) * 1000) / delta) + 1556 (((u32)(tx_pkts - aic->tx_reqs_prev) * 1000) / delta); 1557 eqd = (pps / 15000) << 2; 1558 1559 if (eqd < 8) 1560 eqd = 0; 1561 eqd = min_t(u32, eqd, aic->max_eqd); 1562 eqd = max_t(u32, eqd, aic->min_eqd); 1563 1564 be_aic_update(aic, rx_pkts, tx_pkts, now); 1565 modify_eqd: 1566 if (eqd != aic->prev_eqd) { 1567 set_eqd[num].delay_multiplier = (eqd * 65)/100; 1568 set_eqd[num].eq_id = eqo->q.id; 1569 aic->prev_eqd = eqd; 1570 num++; 1571 } 1572 } 1573 1574 if (num) 1575 be_cmd_modify_eqd(adapter, set_eqd, num); 1576 } 1577 1578 static void be_rx_stats_update(struct be_rx_obj *rxo, 1579 struct be_rx_compl_info *rxcp) 1580 { 1581 struct be_rx_stats *stats = rx_stats(rxo); 1582 1583 u64_stats_update_begin(&stats->sync); 1584 stats->rx_compl++; 1585 stats->rx_bytes += rxcp->pkt_size; 1586 stats->rx_pkts++; 1587 if (rxcp->pkt_type == BE_MULTICAST_PACKET) 1588 stats->rx_mcast_pkts++; 1589 if (rxcp->err) 1590 stats->rx_compl_err++; 1591 u64_stats_update_end(&stats->sync); 1592 } 1593 1594 static inline bool csum_passed(struct be_rx_compl_info *rxcp) 1595 { 1596 /* L4 checksum is not reliable for non TCP/UDP packets. 1597 * Also ignore ipcksm for ipv6 pkts 1598 */ 1599 return (rxcp->tcpf || rxcp->udpf) && rxcp->l4_csum && 1600 (rxcp->ip_csum || rxcp->ipv6) && !rxcp->err; 1601 } 1602 1603 static struct be_rx_page_info *get_rx_page_info(struct be_rx_obj *rxo) 1604 { 1605 struct be_adapter *adapter = rxo->adapter; 1606 struct be_rx_page_info *rx_page_info; 1607 struct be_queue_info *rxq = &rxo->q; 1608 u16 frag_idx = rxq->tail; 1609 1610 rx_page_info = &rxo->page_info_tbl[frag_idx]; 1611 BUG_ON(!rx_page_info->page); 1612 1613 if (rx_page_info->last_frag) { 1614 dma_unmap_page(&adapter->pdev->dev, 1615 dma_unmap_addr(rx_page_info, bus), 1616 adapter->big_page_size, DMA_FROM_DEVICE); 1617 rx_page_info->last_frag = false; 1618 } else { 1619 dma_sync_single_for_cpu(&adapter->pdev->dev, 1620 dma_unmap_addr(rx_page_info, bus), 1621 rx_frag_size, DMA_FROM_DEVICE); 1622 } 1623 1624 queue_tail_inc(rxq); 1625 atomic_dec(&rxq->used); 1626 return rx_page_info; 1627 } 1628 1629 /* Throwaway the data in the Rx completion */ 1630 static void be_rx_compl_discard(struct be_rx_obj *rxo, 1631 struct be_rx_compl_info *rxcp) 1632 { 1633 struct be_rx_page_info *page_info; 1634 u16 i, num_rcvd = rxcp->num_rcvd; 1635 1636 for (i = 0; i < num_rcvd; i++) { 1637 page_info = get_rx_page_info(rxo); 1638 put_page(page_info->page); 1639 memset(page_info, 0, sizeof(*page_info)); 1640 } 1641 } 1642 1643 /* 1644 * skb_fill_rx_data forms a complete skb for an ether frame 1645 * indicated by rxcp. 1646 */ 1647 static void skb_fill_rx_data(struct be_rx_obj *rxo, struct sk_buff *skb, 1648 struct be_rx_compl_info *rxcp) 1649 { 1650 struct be_rx_page_info *page_info; 1651 u16 i, j; 1652 u16 hdr_len, curr_frag_len, remaining; 1653 u8 *start; 1654 1655 page_info = get_rx_page_info(rxo); 1656 start = page_address(page_info->page) + page_info->page_offset; 1657 prefetch(start); 1658 1659 /* Copy data in the first descriptor of this completion */ 1660 curr_frag_len = min(rxcp->pkt_size, rx_frag_size); 1661 1662 skb->len = curr_frag_len; 1663 if (curr_frag_len <= BE_HDR_LEN) { /* tiny packet */ 1664 memcpy(skb->data, start, curr_frag_len); 1665 /* Complete packet has now been moved to data */ 1666 put_page(page_info->page); 1667 skb->data_len = 0; 1668 skb->tail += curr_frag_len; 1669 } else { 1670 hdr_len = ETH_HLEN; 1671 memcpy(skb->data, start, hdr_len); 1672 skb_shinfo(skb)->nr_frags = 1; 1673 skb_frag_set_page(skb, 0, page_info->page); 1674 skb_shinfo(skb)->frags[0].page_offset = 1675 page_info->page_offset + hdr_len; 1676 skb_frag_size_set(&skb_shinfo(skb)->frags[0], 1677 curr_frag_len - hdr_len); 1678 skb->data_len = curr_frag_len - hdr_len; 1679 skb->truesize += rx_frag_size; 1680 skb->tail += hdr_len; 1681 } 1682 page_info->page = NULL; 1683 1684 if (rxcp->pkt_size <= rx_frag_size) { 1685 BUG_ON(rxcp->num_rcvd != 1); 1686 return; 1687 } 1688 1689 /* More frags present for this completion */ 1690 remaining = rxcp->pkt_size - curr_frag_len; 1691 for (i = 1, j = 0; i < rxcp->num_rcvd; i++) { 1692 page_info = get_rx_page_info(rxo); 1693 curr_frag_len = min(remaining, rx_frag_size); 1694 1695 /* Coalesce all frags from the same physical page in one slot */ 1696 if (page_info->page_offset == 0) { 1697 /* Fresh page */ 1698 j++; 1699 skb_frag_set_page(skb, j, page_info->page); 1700 skb_shinfo(skb)->frags[j].page_offset = 1701 page_info->page_offset; 1702 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 1703 skb_shinfo(skb)->nr_frags++; 1704 } else { 1705 put_page(page_info->page); 1706 } 1707 1708 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 1709 skb->len += curr_frag_len; 1710 skb->data_len += curr_frag_len; 1711 skb->truesize += rx_frag_size; 1712 remaining -= curr_frag_len; 1713 page_info->page = NULL; 1714 } 1715 BUG_ON(j > MAX_SKB_FRAGS); 1716 } 1717 1718 /* Process the RX completion indicated by rxcp when GRO is disabled */ 1719 static void be_rx_compl_process(struct be_rx_obj *rxo, struct napi_struct *napi, 1720 struct be_rx_compl_info *rxcp) 1721 { 1722 struct be_adapter *adapter = rxo->adapter; 1723 struct net_device *netdev = adapter->netdev; 1724 struct sk_buff *skb; 1725 1726 skb = netdev_alloc_skb_ip_align(netdev, BE_RX_SKB_ALLOC_SIZE); 1727 if (unlikely(!skb)) { 1728 rx_stats(rxo)->rx_drops_no_skbs++; 1729 be_rx_compl_discard(rxo, rxcp); 1730 return; 1731 } 1732 1733 skb_fill_rx_data(rxo, skb, rxcp); 1734 1735 if (likely((netdev->features & NETIF_F_RXCSUM) && csum_passed(rxcp))) 1736 skb->ip_summed = CHECKSUM_UNNECESSARY; 1737 else 1738 skb_checksum_none_assert(skb); 1739 1740 skb->protocol = eth_type_trans(skb, netdev); 1741 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 1742 if (netdev->features & NETIF_F_RXHASH) 1743 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 1744 1745 skb->csum_level = rxcp->tunneled; 1746 skb_mark_napi_id(skb, napi); 1747 1748 if (rxcp->vlanf) 1749 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 1750 1751 netif_receive_skb(skb); 1752 } 1753 1754 /* Process the RX completion indicated by rxcp when GRO is enabled */ 1755 static void be_rx_compl_process_gro(struct be_rx_obj *rxo, 1756 struct napi_struct *napi, 1757 struct be_rx_compl_info *rxcp) 1758 { 1759 struct be_adapter *adapter = rxo->adapter; 1760 struct be_rx_page_info *page_info; 1761 struct sk_buff *skb = NULL; 1762 u16 remaining, curr_frag_len; 1763 u16 i, j; 1764 1765 skb = napi_get_frags(napi); 1766 if (!skb) { 1767 be_rx_compl_discard(rxo, rxcp); 1768 return; 1769 } 1770 1771 remaining = rxcp->pkt_size; 1772 for (i = 0, j = -1; i < rxcp->num_rcvd; i++) { 1773 page_info = get_rx_page_info(rxo); 1774 1775 curr_frag_len = min(remaining, rx_frag_size); 1776 1777 /* Coalesce all frags from the same physical page in one slot */ 1778 if (i == 0 || page_info->page_offset == 0) { 1779 /* First frag or Fresh page */ 1780 j++; 1781 skb_frag_set_page(skb, j, page_info->page); 1782 skb_shinfo(skb)->frags[j].page_offset = 1783 page_info->page_offset; 1784 skb_frag_size_set(&skb_shinfo(skb)->frags[j], 0); 1785 } else { 1786 put_page(page_info->page); 1787 } 1788 skb_frag_size_add(&skb_shinfo(skb)->frags[j], curr_frag_len); 1789 skb->truesize += rx_frag_size; 1790 remaining -= curr_frag_len; 1791 memset(page_info, 0, sizeof(*page_info)); 1792 } 1793 BUG_ON(j > MAX_SKB_FRAGS); 1794 1795 skb_shinfo(skb)->nr_frags = j + 1; 1796 skb->len = rxcp->pkt_size; 1797 skb->data_len = rxcp->pkt_size; 1798 skb->ip_summed = CHECKSUM_UNNECESSARY; 1799 skb_record_rx_queue(skb, rxo - &adapter->rx_obj[0]); 1800 if (adapter->netdev->features & NETIF_F_RXHASH) 1801 skb_set_hash(skb, rxcp->rss_hash, PKT_HASH_TYPE_L3); 1802 1803 skb->csum_level = rxcp->tunneled; 1804 skb_mark_napi_id(skb, napi); 1805 1806 if (rxcp->vlanf) 1807 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rxcp->vlan_tag); 1808 1809 napi_gro_frags(napi); 1810 } 1811 1812 static void be_parse_rx_compl_v1(struct be_eth_rx_compl *compl, 1813 struct be_rx_compl_info *rxcp) 1814 { 1815 rxcp->pkt_size = GET_RX_COMPL_V1_BITS(pktsize, compl); 1816 rxcp->vlanf = GET_RX_COMPL_V1_BITS(vtp, compl); 1817 rxcp->err = GET_RX_COMPL_V1_BITS(err, compl); 1818 rxcp->tcpf = GET_RX_COMPL_V1_BITS(tcpf, compl); 1819 rxcp->udpf = GET_RX_COMPL_V1_BITS(udpf, compl); 1820 rxcp->ip_csum = GET_RX_COMPL_V1_BITS(ipcksm, compl); 1821 rxcp->l4_csum = GET_RX_COMPL_V1_BITS(l4_cksm, compl); 1822 rxcp->ipv6 = GET_RX_COMPL_V1_BITS(ip_version, compl); 1823 rxcp->num_rcvd = GET_RX_COMPL_V1_BITS(numfrags, compl); 1824 rxcp->pkt_type = GET_RX_COMPL_V1_BITS(cast_enc, compl); 1825 rxcp->rss_hash = GET_RX_COMPL_V1_BITS(rsshash, compl); 1826 if (rxcp->vlanf) { 1827 rxcp->qnq = GET_RX_COMPL_V1_BITS(qnq, compl); 1828 rxcp->vlan_tag = GET_RX_COMPL_V1_BITS(vlan_tag, compl); 1829 } 1830 rxcp->port = GET_RX_COMPL_V1_BITS(port, compl); 1831 rxcp->tunneled = 1832 GET_RX_COMPL_V1_BITS(tunneled, compl); 1833 } 1834 1835 static void be_parse_rx_compl_v0(struct be_eth_rx_compl *compl, 1836 struct be_rx_compl_info *rxcp) 1837 { 1838 rxcp->pkt_size = GET_RX_COMPL_V0_BITS(pktsize, compl); 1839 rxcp->vlanf = GET_RX_COMPL_V0_BITS(vtp, compl); 1840 rxcp->err = GET_RX_COMPL_V0_BITS(err, compl); 1841 rxcp->tcpf = GET_RX_COMPL_V0_BITS(tcpf, compl); 1842 rxcp->udpf = GET_RX_COMPL_V0_BITS(udpf, compl); 1843 rxcp->ip_csum = GET_RX_COMPL_V0_BITS(ipcksm, compl); 1844 rxcp->l4_csum = GET_RX_COMPL_V0_BITS(l4_cksm, compl); 1845 rxcp->ipv6 = GET_RX_COMPL_V0_BITS(ip_version, compl); 1846 rxcp->num_rcvd = GET_RX_COMPL_V0_BITS(numfrags, compl); 1847 rxcp->pkt_type = GET_RX_COMPL_V0_BITS(cast_enc, compl); 1848 rxcp->rss_hash = GET_RX_COMPL_V0_BITS(rsshash, compl); 1849 if (rxcp->vlanf) { 1850 rxcp->qnq = GET_RX_COMPL_V0_BITS(qnq, compl); 1851 rxcp->vlan_tag = GET_RX_COMPL_V0_BITS(vlan_tag, compl); 1852 } 1853 rxcp->port = GET_RX_COMPL_V0_BITS(port, compl); 1854 rxcp->ip_frag = GET_RX_COMPL_V0_BITS(ip_frag, compl); 1855 } 1856 1857 static struct be_rx_compl_info *be_rx_compl_get(struct be_rx_obj *rxo) 1858 { 1859 struct be_eth_rx_compl *compl = queue_tail_node(&rxo->cq); 1860 struct be_rx_compl_info *rxcp = &rxo->rxcp; 1861 struct be_adapter *adapter = rxo->adapter; 1862 1863 /* For checking the valid bit it is Ok to use either definition as the 1864 * valid bit is at the same position in both v0 and v1 Rx compl */ 1865 if (compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] == 0) 1866 return NULL; 1867 1868 rmb(); 1869 be_dws_le_to_cpu(compl, sizeof(*compl)); 1870 1871 if (adapter->be3_native) 1872 be_parse_rx_compl_v1(compl, rxcp); 1873 else 1874 be_parse_rx_compl_v0(compl, rxcp); 1875 1876 if (rxcp->ip_frag) 1877 rxcp->l4_csum = 0; 1878 1879 if (rxcp->vlanf) { 1880 /* In QNQ modes, if qnq bit is not set, then the packet was 1881 * tagged only with the transparent outer vlan-tag and must 1882 * not be treated as a vlan packet by host 1883 */ 1884 if (be_is_qnq_mode(adapter) && !rxcp->qnq) 1885 rxcp->vlanf = 0; 1886 1887 if (!lancer_chip(adapter)) 1888 rxcp->vlan_tag = swab16(rxcp->vlan_tag); 1889 1890 if (adapter->pvid == (rxcp->vlan_tag & VLAN_VID_MASK) && 1891 !test_bit(rxcp->vlan_tag, adapter->vids)) 1892 rxcp->vlanf = 0; 1893 } 1894 1895 /* As the compl has been parsed, reset it; we wont touch it again */ 1896 compl->dw[offsetof(struct amap_eth_rx_compl_v1, valid) / 32] = 0; 1897 1898 queue_tail_inc(&rxo->cq); 1899 return rxcp; 1900 } 1901 1902 static inline struct page *be_alloc_pages(u32 size, gfp_t gfp) 1903 { 1904 u32 order = get_order(size); 1905 1906 if (order > 0) 1907 gfp |= __GFP_COMP; 1908 return alloc_pages(gfp, order); 1909 } 1910 1911 /* 1912 * Allocate a page, split it to fragments of size rx_frag_size and post as 1913 * receive buffers to BE 1914 */ 1915 static void be_post_rx_frags(struct be_rx_obj *rxo, gfp_t gfp, u32 frags_needed) 1916 { 1917 struct be_adapter *adapter = rxo->adapter; 1918 struct be_rx_page_info *page_info = NULL, *prev_page_info = NULL; 1919 struct be_queue_info *rxq = &rxo->q; 1920 struct page *pagep = NULL; 1921 struct device *dev = &adapter->pdev->dev; 1922 struct be_eth_rx_d *rxd; 1923 u64 page_dmaaddr = 0, frag_dmaaddr; 1924 u32 posted, page_offset = 0, notify = 0; 1925 1926 page_info = &rxo->page_info_tbl[rxq->head]; 1927 for (posted = 0; posted < frags_needed && !page_info->page; posted++) { 1928 if (!pagep) { 1929 pagep = be_alloc_pages(adapter->big_page_size, gfp); 1930 if (unlikely(!pagep)) { 1931 rx_stats(rxo)->rx_post_fail++; 1932 break; 1933 } 1934 page_dmaaddr = dma_map_page(dev, pagep, 0, 1935 adapter->big_page_size, 1936 DMA_FROM_DEVICE); 1937 if (dma_mapping_error(dev, page_dmaaddr)) { 1938 put_page(pagep); 1939 pagep = NULL; 1940 adapter->drv_stats.dma_map_errors++; 1941 break; 1942 } 1943 page_offset = 0; 1944 } else { 1945 get_page(pagep); 1946 page_offset += rx_frag_size; 1947 } 1948 page_info->page_offset = page_offset; 1949 page_info->page = pagep; 1950 1951 rxd = queue_head_node(rxq); 1952 frag_dmaaddr = page_dmaaddr + page_info->page_offset; 1953 rxd->fragpa_lo = cpu_to_le32(frag_dmaaddr & 0xFFFFFFFF); 1954 rxd->fragpa_hi = cpu_to_le32(upper_32_bits(frag_dmaaddr)); 1955 1956 /* Any space left in the current big page for another frag? */ 1957 if ((page_offset + rx_frag_size + rx_frag_size) > 1958 adapter->big_page_size) { 1959 pagep = NULL; 1960 page_info->last_frag = true; 1961 dma_unmap_addr_set(page_info, bus, page_dmaaddr); 1962 } else { 1963 dma_unmap_addr_set(page_info, bus, frag_dmaaddr); 1964 } 1965 1966 prev_page_info = page_info; 1967 queue_head_inc(rxq); 1968 page_info = &rxo->page_info_tbl[rxq->head]; 1969 } 1970 1971 /* Mark the last frag of a page when we break out of the above loop 1972 * with no more slots available in the RXQ 1973 */ 1974 if (pagep) { 1975 prev_page_info->last_frag = true; 1976 dma_unmap_addr_set(prev_page_info, bus, page_dmaaddr); 1977 } 1978 1979 if (posted) { 1980 atomic_add(posted, &rxq->used); 1981 if (rxo->rx_post_starved) 1982 rxo->rx_post_starved = false; 1983 do { 1984 notify = min(256u, posted); 1985 be_rxq_notify(adapter, rxq->id, notify); 1986 posted -= notify; 1987 } while (posted); 1988 } else if (atomic_read(&rxq->used) == 0) { 1989 /* Let be_worker replenish when memory is available */ 1990 rxo->rx_post_starved = true; 1991 } 1992 } 1993 1994 static struct be_eth_tx_compl *be_tx_compl_get(struct be_queue_info *tx_cq) 1995 { 1996 struct be_eth_tx_compl *txcp = queue_tail_node(tx_cq); 1997 1998 if (txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] == 0) 1999 return NULL; 2000 2001 rmb(); 2002 be_dws_le_to_cpu(txcp, sizeof(*txcp)); 2003 2004 txcp->dw[offsetof(struct amap_eth_tx_compl, valid) / 32] = 0; 2005 2006 queue_tail_inc(tx_cq); 2007 return txcp; 2008 } 2009 2010 static u16 be_tx_compl_process(struct be_adapter *adapter, 2011 struct be_tx_obj *txo, u16 last_index) 2012 { 2013 struct sk_buff **sent_skbs = txo->sent_skb_list; 2014 struct be_queue_info *txq = &txo->q; 2015 u16 frag_index, num_wrbs = 0; 2016 struct sk_buff *skb = NULL; 2017 bool unmap_skb_hdr = false; 2018 struct be_eth_wrb *wrb; 2019 2020 do { 2021 if (sent_skbs[txq->tail]) { 2022 /* Free skb from prev req */ 2023 if (skb) 2024 dev_consume_skb_any(skb); 2025 skb = sent_skbs[txq->tail]; 2026 sent_skbs[txq->tail] = NULL; 2027 queue_tail_inc(txq); /* skip hdr wrb */ 2028 num_wrbs++; 2029 unmap_skb_hdr = true; 2030 } 2031 wrb = queue_tail_node(txq); 2032 frag_index = txq->tail; 2033 unmap_tx_frag(&adapter->pdev->dev, wrb, 2034 (unmap_skb_hdr && skb_headlen(skb))); 2035 unmap_skb_hdr = false; 2036 queue_tail_inc(txq); 2037 num_wrbs++; 2038 } while (frag_index != last_index); 2039 dev_consume_skb_any(skb); 2040 2041 return num_wrbs; 2042 } 2043 2044 /* Return the number of events in the event queue */ 2045 static inline int events_get(struct be_eq_obj *eqo) 2046 { 2047 struct be_eq_entry *eqe; 2048 int num = 0; 2049 2050 do { 2051 eqe = queue_tail_node(&eqo->q); 2052 if (eqe->evt == 0) 2053 break; 2054 2055 rmb(); 2056 eqe->evt = 0; 2057 num++; 2058 queue_tail_inc(&eqo->q); 2059 } while (true); 2060 2061 return num; 2062 } 2063 2064 /* Leaves the EQ is disarmed state */ 2065 static void be_eq_clean(struct be_eq_obj *eqo) 2066 { 2067 int num = events_get(eqo); 2068 2069 be_eq_notify(eqo->adapter, eqo->q.id, false, true, num); 2070 } 2071 2072 static void be_rx_cq_clean(struct be_rx_obj *rxo) 2073 { 2074 struct be_rx_page_info *page_info; 2075 struct be_queue_info *rxq = &rxo->q; 2076 struct be_queue_info *rx_cq = &rxo->cq; 2077 struct be_rx_compl_info *rxcp; 2078 struct be_adapter *adapter = rxo->adapter; 2079 int flush_wait = 0; 2080 2081 /* Consume pending rx completions. 2082 * Wait for the flush completion (identified by zero num_rcvd) 2083 * to arrive. Notify CQ even when there are no more CQ entries 2084 * for HW to flush partially coalesced CQ entries. 2085 * In Lancer, there is no need to wait for flush compl. 2086 */ 2087 for (;;) { 2088 rxcp = be_rx_compl_get(rxo); 2089 if (!rxcp) { 2090 if (lancer_chip(adapter)) 2091 break; 2092 2093 if (flush_wait++ > 10 || be_hw_error(adapter)) { 2094 dev_warn(&adapter->pdev->dev, 2095 "did not receive flush compl\n"); 2096 break; 2097 } 2098 be_cq_notify(adapter, rx_cq->id, true, 0); 2099 mdelay(1); 2100 } else { 2101 be_rx_compl_discard(rxo, rxcp); 2102 be_cq_notify(adapter, rx_cq->id, false, 1); 2103 if (rxcp->num_rcvd == 0) 2104 break; 2105 } 2106 } 2107 2108 /* After cleanup, leave the CQ in unarmed state */ 2109 be_cq_notify(adapter, rx_cq->id, false, 0); 2110 2111 /* Then free posted rx buffers that were not used */ 2112 while (atomic_read(&rxq->used) > 0) { 2113 page_info = get_rx_page_info(rxo); 2114 put_page(page_info->page); 2115 memset(page_info, 0, sizeof(*page_info)); 2116 } 2117 BUG_ON(atomic_read(&rxq->used)); 2118 rxq->tail = 0; 2119 rxq->head = 0; 2120 } 2121 2122 static void be_tx_compl_clean(struct be_adapter *adapter) 2123 { 2124 u16 end_idx, notified_idx, cmpl = 0, timeo = 0, num_wrbs = 0; 2125 struct device *dev = &adapter->pdev->dev; 2126 struct be_tx_obj *txo; 2127 struct be_queue_info *txq; 2128 struct be_eth_tx_compl *txcp; 2129 int i, pending_txqs; 2130 2131 /* Stop polling for compls when HW has been silent for 10ms */ 2132 do { 2133 pending_txqs = adapter->num_tx_qs; 2134 2135 for_all_tx_queues(adapter, txo, i) { 2136 cmpl = 0; 2137 num_wrbs = 0; 2138 txq = &txo->q; 2139 while ((txcp = be_tx_compl_get(&txo->cq))) { 2140 end_idx = GET_TX_COMPL_BITS(wrb_index, txcp); 2141 num_wrbs += be_tx_compl_process(adapter, txo, 2142 end_idx); 2143 cmpl++; 2144 } 2145 if (cmpl) { 2146 be_cq_notify(adapter, txo->cq.id, false, cmpl); 2147 atomic_sub(num_wrbs, &txq->used); 2148 timeo = 0; 2149 } 2150 if (atomic_read(&txq->used) == txo->pend_wrb_cnt) 2151 pending_txqs--; 2152 } 2153 2154 if (pending_txqs == 0 || ++timeo > 10 || be_hw_error(adapter)) 2155 break; 2156 2157 mdelay(1); 2158 } while (true); 2159 2160 /* Free enqueued TX that was never notified to HW */ 2161 for_all_tx_queues(adapter, txo, i) { 2162 txq = &txo->q; 2163 2164 if (atomic_read(&txq->used)) { 2165 dev_info(dev, "txq%d: cleaning %d pending tx-wrbs\n", 2166 i, atomic_read(&txq->used)); 2167 notified_idx = txq->tail; 2168 end_idx = txq->tail; 2169 index_adv(&end_idx, atomic_read(&txq->used) - 1, 2170 txq->len); 2171 /* Use the tx-compl process logic to handle requests 2172 * that were not sent to the HW. 2173 */ 2174 num_wrbs = be_tx_compl_process(adapter, txo, end_idx); 2175 atomic_sub(num_wrbs, &txq->used); 2176 BUG_ON(atomic_read(&txq->used)); 2177 txo->pend_wrb_cnt = 0; 2178 /* Since hw was never notified of these requests, 2179 * reset TXQ indices 2180 */ 2181 txq->head = notified_idx; 2182 txq->tail = notified_idx; 2183 } 2184 } 2185 } 2186 2187 static void be_evt_queues_destroy(struct be_adapter *adapter) 2188 { 2189 struct be_eq_obj *eqo; 2190 int i; 2191 2192 for_all_evt_queues(adapter, eqo, i) { 2193 if (eqo->q.created) { 2194 be_eq_clean(eqo); 2195 be_cmd_q_destroy(adapter, &eqo->q, QTYPE_EQ); 2196 napi_hash_del(&eqo->napi); 2197 netif_napi_del(&eqo->napi); 2198 } 2199 be_queue_free(adapter, &eqo->q); 2200 } 2201 } 2202 2203 static int be_evt_queues_create(struct be_adapter *adapter) 2204 { 2205 struct be_queue_info *eq; 2206 struct be_eq_obj *eqo; 2207 struct be_aic_obj *aic; 2208 int i, rc; 2209 2210 adapter->num_evt_qs = min_t(u16, num_irqs(adapter), 2211 adapter->cfg_num_qs); 2212 2213 for_all_evt_queues(adapter, eqo, i) { 2214 netif_napi_add(adapter->netdev, &eqo->napi, be_poll, 2215 BE_NAPI_WEIGHT); 2216 napi_hash_add(&eqo->napi); 2217 aic = &adapter->aic_obj[i]; 2218 eqo->adapter = adapter; 2219 eqo->idx = i; 2220 aic->max_eqd = BE_MAX_EQD; 2221 aic->enable = true; 2222 2223 eq = &eqo->q; 2224 rc = be_queue_alloc(adapter, eq, EVNT_Q_LEN, 2225 sizeof(struct be_eq_entry)); 2226 if (rc) 2227 return rc; 2228 2229 rc = be_cmd_eq_create(adapter, eqo); 2230 if (rc) 2231 return rc; 2232 } 2233 return 0; 2234 } 2235 2236 static void be_mcc_queues_destroy(struct be_adapter *adapter) 2237 { 2238 struct be_queue_info *q; 2239 2240 q = &adapter->mcc_obj.q; 2241 if (q->created) 2242 be_cmd_q_destroy(adapter, q, QTYPE_MCCQ); 2243 be_queue_free(adapter, q); 2244 2245 q = &adapter->mcc_obj.cq; 2246 if (q->created) 2247 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2248 be_queue_free(adapter, q); 2249 } 2250 2251 /* Must be called only after TX qs are created as MCC shares TX EQ */ 2252 static int be_mcc_queues_create(struct be_adapter *adapter) 2253 { 2254 struct be_queue_info *q, *cq; 2255 2256 cq = &adapter->mcc_obj.cq; 2257 if (be_queue_alloc(adapter, cq, MCC_CQ_LEN, 2258 sizeof(struct be_mcc_compl))) 2259 goto err; 2260 2261 /* Use the default EQ for MCC completions */ 2262 if (be_cmd_cq_create(adapter, cq, &mcc_eqo(adapter)->q, true, 0)) 2263 goto mcc_cq_free; 2264 2265 q = &adapter->mcc_obj.q; 2266 if (be_queue_alloc(adapter, q, MCC_Q_LEN, sizeof(struct be_mcc_wrb))) 2267 goto mcc_cq_destroy; 2268 2269 if (be_cmd_mccq_create(adapter, q, cq)) 2270 goto mcc_q_free; 2271 2272 return 0; 2273 2274 mcc_q_free: 2275 be_queue_free(adapter, q); 2276 mcc_cq_destroy: 2277 be_cmd_q_destroy(adapter, cq, QTYPE_CQ); 2278 mcc_cq_free: 2279 be_queue_free(adapter, cq); 2280 err: 2281 return -1; 2282 } 2283 2284 static void be_tx_queues_destroy(struct be_adapter *adapter) 2285 { 2286 struct be_queue_info *q; 2287 struct be_tx_obj *txo; 2288 u8 i; 2289 2290 for_all_tx_queues(adapter, txo, i) { 2291 q = &txo->q; 2292 if (q->created) 2293 be_cmd_q_destroy(adapter, q, QTYPE_TXQ); 2294 be_queue_free(adapter, q); 2295 2296 q = &txo->cq; 2297 if (q->created) 2298 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2299 be_queue_free(adapter, q); 2300 } 2301 } 2302 2303 static int be_tx_qs_create(struct be_adapter *adapter) 2304 { 2305 struct be_queue_info *cq, *eq; 2306 struct be_tx_obj *txo; 2307 int status, i; 2308 2309 adapter->num_tx_qs = min(adapter->num_evt_qs, be_max_txqs(adapter)); 2310 2311 for_all_tx_queues(adapter, txo, i) { 2312 cq = &txo->cq; 2313 status = be_queue_alloc(adapter, cq, TX_CQ_LEN, 2314 sizeof(struct be_eth_tx_compl)); 2315 if (status) 2316 return status; 2317 2318 u64_stats_init(&txo->stats.sync); 2319 u64_stats_init(&txo->stats.sync_compl); 2320 2321 /* If num_evt_qs is less than num_tx_qs, then more than 2322 * one txq share an eq 2323 */ 2324 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; 2325 status = be_cmd_cq_create(adapter, cq, eq, false, 3); 2326 if (status) 2327 return status; 2328 2329 status = be_queue_alloc(adapter, &txo->q, TX_Q_LEN, 2330 sizeof(struct be_eth_wrb)); 2331 if (status) 2332 return status; 2333 2334 status = be_cmd_txq_create(adapter, txo); 2335 if (status) 2336 return status; 2337 } 2338 2339 dev_info(&adapter->pdev->dev, "created %d TX queue(s)\n", 2340 adapter->num_tx_qs); 2341 return 0; 2342 } 2343 2344 static void be_rx_cqs_destroy(struct be_adapter *adapter) 2345 { 2346 struct be_queue_info *q; 2347 struct be_rx_obj *rxo; 2348 int i; 2349 2350 for_all_rx_queues(adapter, rxo, i) { 2351 q = &rxo->cq; 2352 if (q->created) 2353 be_cmd_q_destroy(adapter, q, QTYPE_CQ); 2354 be_queue_free(adapter, q); 2355 } 2356 } 2357 2358 static int be_rx_cqs_create(struct be_adapter *adapter) 2359 { 2360 struct be_queue_info *eq, *cq; 2361 struct be_rx_obj *rxo; 2362 int rc, i; 2363 2364 /* We can create as many RSS rings as there are EQs. */ 2365 adapter->num_rx_qs = adapter->num_evt_qs; 2366 2367 /* We'll use RSS only if atleast 2 RSS rings are supported. 2368 * When RSS is used, we'll need a default RXQ for non-IP traffic. 2369 */ 2370 if (adapter->num_rx_qs > 1) 2371 adapter->num_rx_qs++; 2372 2373 adapter->big_page_size = (1 << get_order(rx_frag_size)) * PAGE_SIZE; 2374 for_all_rx_queues(adapter, rxo, i) { 2375 rxo->adapter = adapter; 2376 cq = &rxo->cq; 2377 rc = be_queue_alloc(adapter, cq, RX_CQ_LEN, 2378 sizeof(struct be_eth_rx_compl)); 2379 if (rc) 2380 return rc; 2381 2382 u64_stats_init(&rxo->stats.sync); 2383 eq = &adapter->eq_obj[i % adapter->num_evt_qs].q; 2384 rc = be_cmd_cq_create(adapter, cq, eq, false, 3); 2385 if (rc) 2386 return rc; 2387 } 2388 2389 dev_info(&adapter->pdev->dev, 2390 "created %d RSS queue(s) and 1 default RX queue\n", 2391 adapter->num_rx_qs - 1); 2392 return 0; 2393 } 2394 2395 static irqreturn_t be_intx(int irq, void *dev) 2396 { 2397 struct be_eq_obj *eqo = dev; 2398 struct be_adapter *adapter = eqo->adapter; 2399 int num_evts = 0; 2400 2401 /* IRQ is not expected when NAPI is scheduled as the EQ 2402 * will not be armed. 2403 * But, this can happen on Lancer INTx where it takes 2404 * a while to de-assert INTx or in BE2 where occasionaly 2405 * an interrupt may be raised even when EQ is unarmed. 2406 * If NAPI is already scheduled, then counting & notifying 2407 * events will orphan them. 2408 */ 2409 if (napi_schedule_prep(&eqo->napi)) { 2410 num_evts = events_get(eqo); 2411 __napi_schedule(&eqo->napi); 2412 if (num_evts) 2413 eqo->spurious_intr = 0; 2414 } 2415 be_eq_notify(adapter, eqo->q.id, false, true, num_evts); 2416 2417 /* Return IRQ_HANDLED only for the the first spurious intr 2418 * after a valid intr to stop the kernel from branding 2419 * this irq as a bad one! 2420 */ 2421 if (num_evts || eqo->spurious_intr++ == 0) 2422 return IRQ_HANDLED; 2423 else 2424 return IRQ_NONE; 2425 } 2426 2427 static irqreturn_t be_msix(int irq, void *dev) 2428 { 2429 struct be_eq_obj *eqo = dev; 2430 2431 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0); 2432 napi_schedule(&eqo->napi); 2433 return IRQ_HANDLED; 2434 } 2435 2436 static inline bool do_gro(struct be_rx_compl_info *rxcp) 2437 { 2438 return (rxcp->tcpf && !rxcp->err && rxcp->l4_csum) ? true : false; 2439 } 2440 2441 static int be_process_rx(struct be_rx_obj *rxo, struct napi_struct *napi, 2442 int budget, int polling) 2443 { 2444 struct be_adapter *adapter = rxo->adapter; 2445 struct be_queue_info *rx_cq = &rxo->cq; 2446 struct be_rx_compl_info *rxcp; 2447 u32 work_done; 2448 u32 frags_consumed = 0; 2449 2450 for (work_done = 0; work_done < budget; work_done++) { 2451 rxcp = be_rx_compl_get(rxo); 2452 if (!rxcp) 2453 break; 2454 2455 /* Is it a flush compl that has no data */ 2456 if (unlikely(rxcp->num_rcvd == 0)) 2457 goto loop_continue; 2458 2459 /* Discard compl with partial DMA Lancer B0 */ 2460 if (unlikely(!rxcp->pkt_size)) { 2461 be_rx_compl_discard(rxo, rxcp); 2462 goto loop_continue; 2463 } 2464 2465 /* On BE drop pkts that arrive due to imperfect filtering in 2466 * promiscuous mode on some skews 2467 */ 2468 if (unlikely(rxcp->port != adapter->port_num && 2469 !lancer_chip(adapter))) { 2470 be_rx_compl_discard(rxo, rxcp); 2471 goto loop_continue; 2472 } 2473 2474 /* Don't do gro when we're busy_polling */ 2475 if (do_gro(rxcp) && polling != BUSY_POLLING) 2476 be_rx_compl_process_gro(rxo, napi, rxcp); 2477 else 2478 be_rx_compl_process(rxo, napi, rxcp); 2479 2480 loop_continue: 2481 frags_consumed += rxcp->num_rcvd; 2482 be_rx_stats_update(rxo, rxcp); 2483 } 2484 2485 if (work_done) { 2486 be_cq_notify(adapter, rx_cq->id, true, work_done); 2487 2488 /* When an rx-obj gets into post_starved state, just 2489 * let be_worker do the posting. 2490 */ 2491 if (atomic_read(&rxo->q.used) < RX_FRAGS_REFILL_WM && 2492 !rxo->rx_post_starved) 2493 be_post_rx_frags(rxo, GFP_ATOMIC, 2494 max_t(u32, MAX_RX_POST, 2495 frags_consumed)); 2496 } 2497 2498 return work_done; 2499 } 2500 2501 static inline void be_update_tx_err(struct be_tx_obj *txo, u32 status) 2502 { 2503 switch (status) { 2504 case BE_TX_COMP_HDR_PARSE_ERR: 2505 tx_stats(txo)->tx_hdr_parse_err++; 2506 break; 2507 case BE_TX_COMP_NDMA_ERR: 2508 tx_stats(txo)->tx_dma_err++; 2509 break; 2510 case BE_TX_COMP_ACL_ERR: 2511 tx_stats(txo)->tx_spoof_check_err++; 2512 break; 2513 } 2514 } 2515 2516 static inline void lancer_update_tx_err(struct be_tx_obj *txo, u32 status) 2517 { 2518 switch (status) { 2519 case LANCER_TX_COMP_LSO_ERR: 2520 tx_stats(txo)->tx_tso_err++; 2521 break; 2522 case LANCER_TX_COMP_HSW_DROP_MAC_ERR: 2523 case LANCER_TX_COMP_HSW_DROP_VLAN_ERR: 2524 tx_stats(txo)->tx_spoof_check_err++; 2525 break; 2526 case LANCER_TX_COMP_QINQ_ERR: 2527 tx_stats(txo)->tx_qinq_err++; 2528 break; 2529 case LANCER_TX_COMP_PARITY_ERR: 2530 tx_stats(txo)->tx_internal_parity_err++; 2531 break; 2532 case LANCER_TX_COMP_DMA_ERR: 2533 tx_stats(txo)->tx_dma_err++; 2534 break; 2535 } 2536 } 2537 2538 static void be_process_tx(struct be_adapter *adapter, struct be_tx_obj *txo, 2539 int idx) 2540 { 2541 struct be_eth_tx_compl *txcp; 2542 int num_wrbs = 0, work_done = 0; 2543 u32 compl_status; 2544 u16 last_idx; 2545 2546 while ((txcp = be_tx_compl_get(&txo->cq))) { 2547 last_idx = GET_TX_COMPL_BITS(wrb_index, txcp); 2548 num_wrbs += be_tx_compl_process(adapter, txo, last_idx); 2549 work_done++; 2550 2551 compl_status = GET_TX_COMPL_BITS(status, txcp); 2552 if (compl_status) { 2553 if (lancer_chip(adapter)) 2554 lancer_update_tx_err(txo, compl_status); 2555 else 2556 be_update_tx_err(txo, compl_status); 2557 } 2558 } 2559 2560 if (work_done) { 2561 be_cq_notify(adapter, txo->cq.id, true, work_done); 2562 atomic_sub(num_wrbs, &txo->q.used); 2563 2564 /* As Tx wrbs have been freed up, wake up netdev queue 2565 * if it was stopped due to lack of tx wrbs. */ 2566 if (__netif_subqueue_stopped(adapter->netdev, idx) && 2567 atomic_read(&txo->q.used) < txo->q.len / 2) { 2568 netif_wake_subqueue(adapter->netdev, idx); 2569 } 2570 2571 u64_stats_update_begin(&tx_stats(txo)->sync_compl); 2572 tx_stats(txo)->tx_compl += work_done; 2573 u64_stats_update_end(&tx_stats(txo)->sync_compl); 2574 } 2575 } 2576 2577 #ifdef CONFIG_NET_RX_BUSY_POLL 2578 static inline bool be_lock_napi(struct be_eq_obj *eqo) 2579 { 2580 bool status = true; 2581 2582 spin_lock(&eqo->lock); /* BH is already disabled */ 2583 if (eqo->state & BE_EQ_LOCKED) { 2584 WARN_ON(eqo->state & BE_EQ_NAPI); 2585 eqo->state |= BE_EQ_NAPI_YIELD; 2586 status = false; 2587 } else { 2588 eqo->state = BE_EQ_NAPI; 2589 } 2590 spin_unlock(&eqo->lock); 2591 return status; 2592 } 2593 2594 static inline void be_unlock_napi(struct be_eq_obj *eqo) 2595 { 2596 spin_lock(&eqo->lock); /* BH is already disabled */ 2597 2598 WARN_ON(eqo->state & (BE_EQ_POLL | BE_EQ_NAPI_YIELD)); 2599 eqo->state = BE_EQ_IDLE; 2600 2601 spin_unlock(&eqo->lock); 2602 } 2603 2604 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo) 2605 { 2606 bool status = true; 2607 2608 spin_lock_bh(&eqo->lock); 2609 if (eqo->state & BE_EQ_LOCKED) { 2610 eqo->state |= BE_EQ_POLL_YIELD; 2611 status = false; 2612 } else { 2613 eqo->state |= BE_EQ_POLL; 2614 } 2615 spin_unlock_bh(&eqo->lock); 2616 return status; 2617 } 2618 2619 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo) 2620 { 2621 spin_lock_bh(&eqo->lock); 2622 2623 WARN_ON(eqo->state & (BE_EQ_NAPI)); 2624 eqo->state = BE_EQ_IDLE; 2625 2626 spin_unlock_bh(&eqo->lock); 2627 } 2628 2629 static inline void be_enable_busy_poll(struct be_eq_obj *eqo) 2630 { 2631 spin_lock_init(&eqo->lock); 2632 eqo->state = BE_EQ_IDLE; 2633 } 2634 2635 static inline void be_disable_busy_poll(struct be_eq_obj *eqo) 2636 { 2637 local_bh_disable(); 2638 2639 /* It's enough to just acquire napi lock on the eqo to stop 2640 * be_busy_poll() from processing any queueus. 2641 */ 2642 while (!be_lock_napi(eqo)) 2643 mdelay(1); 2644 2645 local_bh_enable(); 2646 } 2647 2648 #else /* CONFIG_NET_RX_BUSY_POLL */ 2649 2650 static inline bool be_lock_napi(struct be_eq_obj *eqo) 2651 { 2652 return true; 2653 } 2654 2655 static inline void be_unlock_napi(struct be_eq_obj *eqo) 2656 { 2657 } 2658 2659 static inline bool be_lock_busy_poll(struct be_eq_obj *eqo) 2660 { 2661 return false; 2662 } 2663 2664 static inline void be_unlock_busy_poll(struct be_eq_obj *eqo) 2665 { 2666 } 2667 2668 static inline void be_enable_busy_poll(struct be_eq_obj *eqo) 2669 { 2670 } 2671 2672 static inline void be_disable_busy_poll(struct be_eq_obj *eqo) 2673 { 2674 } 2675 #endif /* CONFIG_NET_RX_BUSY_POLL */ 2676 2677 int be_poll(struct napi_struct *napi, int budget) 2678 { 2679 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 2680 struct be_adapter *adapter = eqo->adapter; 2681 int max_work = 0, work, i, num_evts; 2682 struct be_rx_obj *rxo; 2683 struct be_tx_obj *txo; 2684 2685 num_evts = events_get(eqo); 2686 2687 for_all_tx_queues_on_eq(adapter, eqo, txo, i) 2688 be_process_tx(adapter, txo, i); 2689 2690 if (be_lock_napi(eqo)) { 2691 /* This loop will iterate twice for EQ0 in which 2692 * completions of the last RXQ (default one) are also processed 2693 * For other EQs the loop iterates only once 2694 */ 2695 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 2696 work = be_process_rx(rxo, napi, budget, NAPI_POLLING); 2697 max_work = max(work, max_work); 2698 } 2699 be_unlock_napi(eqo); 2700 } else { 2701 max_work = budget; 2702 } 2703 2704 if (is_mcc_eqo(eqo)) 2705 be_process_mcc(adapter); 2706 2707 if (max_work < budget) { 2708 napi_complete(napi); 2709 be_eq_notify(adapter, eqo->q.id, true, false, num_evts); 2710 } else { 2711 /* As we'll continue in polling mode, count and clear events */ 2712 be_eq_notify(adapter, eqo->q.id, false, false, num_evts); 2713 } 2714 return max_work; 2715 } 2716 2717 #ifdef CONFIG_NET_RX_BUSY_POLL 2718 static int be_busy_poll(struct napi_struct *napi) 2719 { 2720 struct be_eq_obj *eqo = container_of(napi, struct be_eq_obj, napi); 2721 struct be_adapter *adapter = eqo->adapter; 2722 struct be_rx_obj *rxo; 2723 int i, work = 0; 2724 2725 if (!be_lock_busy_poll(eqo)) 2726 return LL_FLUSH_BUSY; 2727 2728 for_all_rx_queues_on_eq(adapter, eqo, rxo, i) { 2729 work = be_process_rx(rxo, napi, 4, BUSY_POLLING); 2730 if (work) 2731 break; 2732 } 2733 2734 be_unlock_busy_poll(eqo); 2735 return work; 2736 } 2737 #endif 2738 2739 void be_detect_error(struct be_adapter *adapter) 2740 { 2741 u32 ue_lo = 0, ue_hi = 0, ue_lo_mask = 0, ue_hi_mask = 0; 2742 u32 sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0; 2743 u32 i; 2744 bool error_detected = false; 2745 struct device *dev = &adapter->pdev->dev; 2746 struct net_device *netdev = adapter->netdev; 2747 2748 if (be_hw_error(adapter)) 2749 return; 2750 2751 if (lancer_chip(adapter)) { 2752 sliport_status = ioread32(adapter->db + SLIPORT_STATUS_OFFSET); 2753 if (sliport_status & SLIPORT_STATUS_ERR_MASK) { 2754 sliport_err1 = ioread32(adapter->db + 2755 SLIPORT_ERROR1_OFFSET); 2756 sliport_err2 = ioread32(adapter->db + 2757 SLIPORT_ERROR2_OFFSET); 2758 adapter->hw_error = true; 2759 /* Do not log error messages if its a FW reset */ 2760 if (sliport_err1 == SLIPORT_ERROR_FW_RESET1 && 2761 sliport_err2 == SLIPORT_ERROR_FW_RESET2) { 2762 dev_info(dev, "Firmware update in progress\n"); 2763 } else { 2764 error_detected = true; 2765 dev_err(dev, "Error detected in the card\n"); 2766 dev_err(dev, "ERR: sliport status 0x%x\n", 2767 sliport_status); 2768 dev_err(dev, "ERR: sliport error1 0x%x\n", 2769 sliport_err1); 2770 dev_err(dev, "ERR: sliport error2 0x%x\n", 2771 sliport_err2); 2772 } 2773 } 2774 } else { 2775 pci_read_config_dword(adapter->pdev, 2776 PCICFG_UE_STATUS_LOW, &ue_lo); 2777 pci_read_config_dword(adapter->pdev, 2778 PCICFG_UE_STATUS_HIGH, &ue_hi); 2779 pci_read_config_dword(adapter->pdev, 2780 PCICFG_UE_STATUS_LOW_MASK, &ue_lo_mask); 2781 pci_read_config_dword(adapter->pdev, 2782 PCICFG_UE_STATUS_HI_MASK, &ue_hi_mask); 2783 2784 ue_lo = (ue_lo & ~ue_lo_mask); 2785 ue_hi = (ue_hi & ~ue_hi_mask); 2786 2787 /* On certain platforms BE hardware can indicate spurious UEs. 2788 * Allow HW to stop working completely in case of a real UE. 2789 * Hence not setting the hw_error for UE detection. 2790 */ 2791 2792 if (ue_lo || ue_hi) { 2793 error_detected = true; 2794 dev_err(dev, 2795 "Unrecoverable Error detected in the adapter"); 2796 dev_err(dev, "Please reboot server to recover"); 2797 if (skyhawk_chip(adapter)) 2798 adapter->hw_error = true; 2799 for (i = 0; ue_lo; ue_lo >>= 1, i++) { 2800 if (ue_lo & 1) 2801 dev_err(dev, "UE: %s bit set\n", 2802 ue_status_low_desc[i]); 2803 } 2804 for (i = 0; ue_hi; ue_hi >>= 1, i++) { 2805 if (ue_hi & 1) 2806 dev_err(dev, "UE: %s bit set\n", 2807 ue_status_hi_desc[i]); 2808 } 2809 } 2810 } 2811 if (error_detected) 2812 netif_carrier_off(netdev); 2813 } 2814 2815 static void be_msix_disable(struct be_adapter *adapter) 2816 { 2817 if (msix_enabled(adapter)) { 2818 pci_disable_msix(adapter->pdev); 2819 adapter->num_msix_vec = 0; 2820 adapter->num_msix_roce_vec = 0; 2821 } 2822 } 2823 2824 static int be_msix_enable(struct be_adapter *adapter) 2825 { 2826 int i, num_vec; 2827 struct device *dev = &adapter->pdev->dev; 2828 2829 /* If RoCE is supported, program the max number of NIC vectors that 2830 * may be configured via set-channels, along with vectors needed for 2831 * RoCe. Else, just program the number we'll use initially. 2832 */ 2833 if (be_roce_supported(adapter)) 2834 num_vec = min_t(int, 2 * be_max_eqs(adapter), 2835 2 * num_online_cpus()); 2836 else 2837 num_vec = adapter->cfg_num_qs; 2838 2839 for (i = 0; i < num_vec; i++) 2840 adapter->msix_entries[i].entry = i; 2841 2842 num_vec = pci_enable_msix_range(adapter->pdev, adapter->msix_entries, 2843 MIN_MSIX_VECTORS, num_vec); 2844 if (num_vec < 0) 2845 goto fail; 2846 2847 if (be_roce_supported(adapter) && num_vec > MIN_MSIX_VECTORS) { 2848 adapter->num_msix_roce_vec = num_vec / 2; 2849 dev_info(dev, "enabled %d MSI-x vector(s) for RoCE\n", 2850 adapter->num_msix_roce_vec); 2851 } 2852 2853 adapter->num_msix_vec = num_vec - adapter->num_msix_roce_vec; 2854 2855 dev_info(dev, "enabled %d MSI-x vector(s) for NIC\n", 2856 adapter->num_msix_vec); 2857 return 0; 2858 2859 fail: 2860 dev_warn(dev, "MSIx enable failed\n"); 2861 2862 /* INTx is not supported in VFs, so fail probe if enable_msix fails */ 2863 if (!be_physfn(adapter)) 2864 return num_vec; 2865 return 0; 2866 } 2867 2868 static inline int be_msix_vec_get(struct be_adapter *adapter, 2869 struct be_eq_obj *eqo) 2870 { 2871 return adapter->msix_entries[eqo->msix_idx].vector; 2872 } 2873 2874 static int be_msix_register(struct be_adapter *adapter) 2875 { 2876 struct net_device *netdev = adapter->netdev; 2877 struct be_eq_obj *eqo; 2878 int status, i, vec; 2879 2880 for_all_evt_queues(adapter, eqo, i) { 2881 sprintf(eqo->desc, "%s-q%d", netdev->name, i); 2882 vec = be_msix_vec_get(adapter, eqo); 2883 status = request_irq(vec, be_msix, 0, eqo->desc, eqo); 2884 if (status) 2885 goto err_msix; 2886 } 2887 2888 return 0; 2889 err_msix: 2890 for (i--, eqo = &adapter->eq_obj[i]; i >= 0; i--, eqo--) 2891 free_irq(be_msix_vec_get(adapter, eqo), eqo); 2892 dev_warn(&adapter->pdev->dev, "MSIX Request IRQ failed - err %d\n", 2893 status); 2894 be_msix_disable(adapter); 2895 return status; 2896 } 2897 2898 static int be_irq_register(struct be_adapter *adapter) 2899 { 2900 struct net_device *netdev = adapter->netdev; 2901 int status; 2902 2903 if (msix_enabled(adapter)) { 2904 status = be_msix_register(adapter); 2905 if (status == 0) 2906 goto done; 2907 /* INTx is not supported for VF */ 2908 if (!be_physfn(adapter)) 2909 return status; 2910 } 2911 2912 /* INTx: only the first EQ is used */ 2913 netdev->irq = adapter->pdev->irq; 2914 status = request_irq(netdev->irq, be_intx, IRQF_SHARED, netdev->name, 2915 &adapter->eq_obj[0]); 2916 if (status) { 2917 dev_err(&adapter->pdev->dev, 2918 "INTx request IRQ failed - err %d\n", status); 2919 return status; 2920 } 2921 done: 2922 adapter->isr_registered = true; 2923 return 0; 2924 } 2925 2926 static void be_irq_unregister(struct be_adapter *adapter) 2927 { 2928 struct net_device *netdev = adapter->netdev; 2929 struct be_eq_obj *eqo; 2930 int i; 2931 2932 if (!adapter->isr_registered) 2933 return; 2934 2935 /* INTx */ 2936 if (!msix_enabled(adapter)) { 2937 free_irq(netdev->irq, &adapter->eq_obj[0]); 2938 goto done; 2939 } 2940 2941 /* MSIx */ 2942 for_all_evt_queues(adapter, eqo, i) 2943 free_irq(be_msix_vec_get(adapter, eqo), eqo); 2944 2945 done: 2946 adapter->isr_registered = false; 2947 } 2948 2949 static void be_rx_qs_destroy(struct be_adapter *adapter) 2950 { 2951 struct be_queue_info *q; 2952 struct be_rx_obj *rxo; 2953 int i; 2954 2955 for_all_rx_queues(adapter, rxo, i) { 2956 q = &rxo->q; 2957 if (q->created) { 2958 be_cmd_rxq_destroy(adapter, q); 2959 be_rx_cq_clean(rxo); 2960 } 2961 be_queue_free(adapter, q); 2962 } 2963 } 2964 2965 static int be_close(struct net_device *netdev) 2966 { 2967 struct be_adapter *adapter = netdev_priv(netdev); 2968 struct be_eq_obj *eqo; 2969 int i; 2970 2971 /* This protection is needed as be_close() may be called even when the 2972 * adapter is in cleared state (after eeh perm failure) 2973 */ 2974 if (!(adapter->flags & BE_FLAGS_SETUP_DONE)) 2975 return 0; 2976 2977 be_roce_dev_close(adapter); 2978 2979 if (adapter->flags & BE_FLAGS_NAPI_ENABLED) { 2980 for_all_evt_queues(adapter, eqo, i) { 2981 napi_disable(&eqo->napi); 2982 be_disable_busy_poll(eqo); 2983 } 2984 adapter->flags &= ~BE_FLAGS_NAPI_ENABLED; 2985 } 2986 2987 be_async_mcc_disable(adapter); 2988 2989 /* Wait for all pending tx completions to arrive so that 2990 * all tx skbs are freed. 2991 */ 2992 netif_tx_disable(netdev); 2993 be_tx_compl_clean(adapter); 2994 2995 be_rx_qs_destroy(adapter); 2996 be_clear_uc_list(adapter); 2997 2998 for_all_evt_queues(adapter, eqo, i) { 2999 if (msix_enabled(adapter)) 3000 synchronize_irq(be_msix_vec_get(adapter, eqo)); 3001 else 3002 synchronize_irq(netdev->irq); 3003 be_eq_clean(eqo); 3004 } 3005 3006 be_irq_unregister(adapter); 3007 3008 return 0; 3009 } 3010 3011 static int be_rx_qs_create(struct be_adapter *adapter) 3012 { 3013 struct rss_info *rss = &adapter->rss_info; 3014 u8 rss_key[RSS_HASH_KEY_LEN]; 3015 struct be_rx_obj *rxo; 3016 int rc, i, j; 3017 3018 for_all_rx_queues(adapter, rxo, i) { 3019 rc = be_queue_alloc(adapter, &rxo->q, RX_Q_LEN, 3020 sizeof(struct be_eth_rx_d)); 3021 if (rc) 3022 return rc; 3023 } 3024 3025 /* The FW would like the default RXQ to be created first */ 3026 rxo = default_rxo(adapter); 3027 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, rx_frag_size, 3028 adapter->if_handle, false, &rxo->rss_id); 3029 if (rc) 3030 return rc; 3031 3032 for_all_rss_queues(adapter, rxo, i) { 3033 rc = be_cmd_rxq_create(adapter, &rxo->q, rxo->cq.id, 3034 rx_frag_size, adapter->if_handle, 3035 true, &rxo->rss_id); 3036 if (rc) 3037 return rc; 3038 } 3039 3040 if (be_multi_rxq(adapter)) { 3041 for (j = 0; j < RSS_INDIR_TABLE_LEN; 3042 j += adapter->num_rx_qs - 1) { 3043 for_all_rss_queues(adapter, rxo, i) { 3044 if ((j + i) >= RSS_INDIR_TABLE_LEN) 3045 break; 3046 rss->rsstable[j + i] = rxo->rss_id; 3047 rss->rss_queue[j + i] = i; 3048 } 3049 } 3050 rss->rss_flags = RSS_ENABLE_TCP_IPV4 | RSS_ENABLE_IPV4 | 3051 RSS_ENABLE_TCP_IPV6 | RSS_ENABLE_IPV6; 3052 3053 if (!BEx_chip(adapter)) 3054 rss->rss_flags |= RSS_ENABLE_UDP_IPV4 | 3055 RSS_ENABLE_UDP_IPV6; 3056 } else { 3057 /* Disable RSS, if only default RX Q is created */ 3058 rss->rss_flags = RSS_ENABLE_NONE; 3059 } 3060 3061 netdev_rss_key_fill(rss_key, RSS_HASH_KEY_LEN); 3062 rc = be_cmd_rss_config(adapter, rss->rsstable, rss->rss_flags, 3063 128, rss_key); 3064 if (rc) { 3065 rss->rss_flags = RSS_ENABLE_NONE; 3066 return rc; 3067 } 3068 3069 memcpy(rss->rss_hkey, rss_key, RSS_HASH_KEY_LEN); 3070 3071 /* First time posting */ 3072 for_all_rx_queues(adapter, rxo, i) 3073 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST); 3074 return 0; 3075 } 3076 3077 static int be_open(struct net_device *netdev) 3078 { 3079 struct be_adapter *adapter = netdev_priv(netdev); 3080 struct be_eq_obj *eqo; 3081 struct be_rx_obj *rxo; 3082 struct be_tx_obj *txo; 3083 u8 link_status; 3084 int status, i; 3085 3086 status = be_rx_qs_create(adapter); 3087 if (status) 3088 goto err; 3089 3090 status = be_irq_register(adapter); 3091 if (status) 3092 goto err; 3093 3094 for_all_rx_queues(adapter, rxo, i) 3095 be_cq_notify(adapter, rxo->cq.id, true, 0); 3096 3097 for_all_tx_queues(adapter, txo, i) 3098 be_cq_notify(adapter, txo->cq.id, true, 0); 3099 3100 be_async_mcc_enable(adapter); 3101 3102 for_all_evt_queues(adapter, eqo, i) { 3103 napi_enable(&eqo->napi); 3104 be_enable_busy_poll(eqo); 3105 be_eq_notify(adapter, eqo->q.id, true, true, 0); 3106 } 3107 adapter->flags |= BE_FLAGS_NAPI_ENABLED; 3108 3109 status = be_cmd_link_status_query(adapter, NULL, &link_status, 0); 3110 if (!status) 3111 be_link_status_update(adapter, link_status); 3112 3113 netif_tx_start_all_queues(netdev); 3114 be_roce_dev_open(adapter); 3115 3116 #ifdef CONFIG_BE2NET_VXLAN 3117 if (skyhawk_chip(adapter)) 3118 vxlan_get_rx_port(netdev); 3119 #endif 3120 3121 return 0; 3122 err: 3123 be_close(adapter->netdev); 3124 return -EIO; 3125 } 3126 3127 static int be_setup_wol(struct be_adapter *adapter, bool enable) 3128 { 3129 struct be_dma_mem cmd; 3130 int status = 0; 3131 u8 mac[ETH_ALEN]; 3132 3133 memset(mac, 0, ETH_ALEN); 3134 3135 cmd.size = sizeof(struct be_cmd_req_acpi_wol_magic_config); 3136 cmd.va = dma_zalloc_coherent(&adapter->pdev->dev, cmd.size, &cmd.dma, 3137 GFP_KERNEL); 3138 if (!cmd.va) 3139 return -ENOMEM; 3140 3141 if (enable) { 3142 status = pci_write_config_dword(adapter->pdev, 3143 PCICFG_PM_CONTROL_OFFSET, 3144 PCICFG_PM_CONTROL_MASK); 3145 if (status) { 3146 dev_err(&adapter->pdev->dev, 3147 "Could not enable Wake-on-lan\n"); 3148 dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, 3149 cmd.dma); 3150 return status; 3151 } 3152 status = be_cmd_enable_magic_wol(adapter, 3153 adapter->netdev->dev_addr, 3154 &cmd); 3155 pci_enable_wake(adapter->pdev, PCI_D3hot, 1); 3156 pci_enable_wake(adapter->pdev, PCI_D3cold, 1); 3157 } else { 3158 status = be_cmd_enable_magic_wol(adapter, mac, &cmd); 3159 pci_enable_wake(adapter->pdev, PCI_D3hot, 0); 3160 pci_enable_wake(adapter->pdev, PCI_D3cold, 0); 3161 } 3162 3163 dma_free_coherent(&adapter->pdev->dev, cmd.size, cmd.va, cmd.dma); 3164 return status; 3165 } 3166 3167 static void be_vf_eth_addr_generate(struct be_adapter *adapter, u8 *mac) 3168 { 3169 u32 addr; 3170 3171 addr = jhash(adapter->netdev->dev_addr, ETH_ALEN, 0); 3172 3173 mac[5] = (u8)(addr & 0xFF); 3174 mac[4] = (u8)((addr >> 8) & 0xFF); 3175 mac[3] = (u8)((addr >> 16) & 0xFF); 3176 /* Use the OUI from the current MAC address */ 3177 memcpy(mac, adapter->netdev->dev_addr, 3); 3178 } 3179 3180 /* 3181 * Generate a seed MAC address from the PF MAC Address using jhash. 3182 * MAC Address for VFs are assigned incrementally starting from the seed. 3183 * These addresses are programmed in the ASIC by the PF and the VF driver 3184 * queries for the MAC address during its probe. 3185 */ 3186 static int be_vf_eth_addr_config(struct be_adapter *adapter) 3187 { 3188 u32 vf; 3189 int status = 0; 3190 u8 mac[ETH_ALEN]; 3191 struct be_vf_cfg *vf_cfg; 3192 3193 be_vf_eth_addr_generate(adapter, mac); 3194 3195 for_all_vfs(adapter, vf_cfg, vf) { 3196 if (BEx_chip(adapter)) 3197 status = be_cmd_pmac_add(adapter, mac, 3198 vf_cfg->if_handle, 3199 &vf_cfg->pmac_id, vf + 1); 3200 else 3201 status = be_cmd_set_mac(adapter, mac, vf_cfg->if_handle, 3202 vf + 1); 3203 3204 if (status) 3205 dev_err(&adapter->pdev->dev, 3206 "Mac address assignment failed for VF %d\n", 3207 vf); 3208 else 3209 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3210 3211 mac[5] += 1; 3212 } 3213 return status; 3214 } 3215 3216 static int be_vfs_mac_query(struct be_adapter *adapter) 3217 { 3218 int status, vf; 3219 u8 mac[ETH_ALEN]; 3220 struct be_vf_cfg *vf_cfg; 3221 3222 for_all_vfs(adapter, vf_cfg, vf) { 3223 status = be_cmd_get_active_mac(adapter, vf_cfg->pmac_id, 3224 mac, vf_cfg->if_handle, 3225 false, vf+1); 3226 if (status) 3227 return status; 3228 memcpy(vf_cfg->mac_addr, mac, ETH_ALEN); 3229 } 3230 return 0; 3231 } 3232 3233 static void be_vf_clear(struct be_adapter *adapter) 3234 { 3235 struct be_vf_cfg *vf_cfg; 3236 u32 vf; 3237 3238 if (pci_vfs_assigned(adapter->pdev)) { 3239 dev_warn(&adapter->pdev->dev, 3240 "VFs are assigned to VMs: not disabling VFs\n"); 3241 goto done; 3242 } 3243 3244 pci_disable_sriov(adapter->pdev); 3245 3246 for_all_vfs(adapter, vf_cfg, vf) { 3247 if (BEx_chip(adapter)) 3248 be_cmd_pmac_del(adapter, vf_cfg->if_handle, 3249 vf_cfg->pmac_id, vf + 1); 3250 else 3251 be_cmd_set_mac(adapter, NULL, vf_cfg->if_handle, 3252 vf + 1); 3253 3254 be_cmd_if_destroy(adapter, vf_cfg->if_handle, vf + 1); 3255 } 3256 done: 3257 kfree(adapter->vf_cfg); 3258 adapter->num_vfs = 0; 3259 adapter->flags &= ~BE_FLAGS_SRIOV_ENABLED; 3260 } 3261 3262 static void be_clear_queues(struct be_adapter *adapter) 3263 { 3264 be_mcc_queues_destroy(adapter); 3265 be_rx_cqs_destroy(adapter); 3266 be_tx_queues_destroy(adapter); 3267 be_evt_queues_destroy(adapter); 3268 } 3269 3270 static void be_cancel_worker(struct be_adapter *adapter) 3271 { 3272 if (adapter->flags & BE_FLAGS_WORKER_SCHEDULED) { 3273 cancel_delayed_work_sync(&adapter->work); 3274 adapter->flags &= ~BE_FLAGS_WORKER_SCHEDULED; 3275 } 3276 } 3277 3278 static void be_mac_clear(struct be_adapter *adapter) 3279 { 3280 if (adapter->pmac_id) { 3281 be_cmd_pmac_del(adapter, adapter->if_handle, 3282 adapter->pmac_id[0], 0); 3283 kfree(adapter->pmac_id); 3284 adapter->pmac_id = NULL; 3285 } 3286 } 3287 3288 #ifdef CONFIG_BE2NET_VXLAN 3289 static void be_disable_vxlan_offloads(struct be_adapter *adapter) 3290 { 3291 struct net_device *netdev = adapter->netdev; 3292 3293 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) 3294 be_cmd_manage_iface(adapter, adapter->if_handle, 3295 OP_CONVERT_TUNNEL_TO_NORMAL); 3296 3297 if (adapter->vxlan_port) 3298 be_cmd_set_vxlan_port(adapter, 0); 3299 3300 adapter->flags &= ~BE_FLAGS_VXLAN_OFFLOADS; 3301 adapter->vxlan_port = 0; 3302 3303 netdev->hw_enc_features = 0; 3304 netdev->hw_features &= ~(NETIF_F_GSO_UDP_TUNNEL); 3305 netdev->features &= ~(NETIF_F_GSO_UDP_TUNNEL); 3306 } 3307 #endif 3308 3309 static int be_clear(struct be_adapter *adapter) 3310 { 3311 be_cancel_worker(adapter); 3312 3313 if (sriov_enabled(adapter)) 3314 be_vf_clear(adapter); 3315 3316 /* Re-configure FW to distribute resources evenly across max-supported 3317 * number of VFs, only when VFs are not already enabled. 3318 */ 3319 if (be_physfn(adapter) && !pci_vfs_assigned(adapter->pdev)) 3320 be_cmd_set_sriov_config(adapter, adapter->pool_res, 3321 pci_sriov_get_totalvfs(adapter->pdev)); 3322 3323 #ifdef CONFIG_BE2NET_VXLAN 3324 be_disable_vxlan_offloads(adapter); 3325 #endif 3326 /* delete the primary mac along with the uc-mac list */ 3327 be_mac_clear(adapter); 3328 3329 be_cmd_if_destroy(adapter, adapter->if_handle, 0); 3330 3331 be_clear_queues(adapter); 3332 3333 be_msix_disable(adapter); 3334 adapter->flags &= ~BE_FLAGS_SETUP_DONE; 3335 return 0; 3336 } 3337 3338 static int be_if_create(struct be_adapter *adapter, u32 *if_handle, 3339 u32 cap_flags, u32 vf) 3340 { 3341 u32 en_flags; 3342 int status; 3343 3344 en_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST | 3345 BE_IF_FLAGS_MULTICAST | BE_IF_FLAGS_PASS_L3L4_ERRORS | 3346 BE_IF_FLAGS_RSS; 3347 3348 en_flags &= cap_flags; 3349 3350 status = be_cmd_if_create(adapter, cap_flags, en_flags, 3351 if_handle, vf); 3352 3353 return status; 3354 } 3355 3356 static int be_vfs_if_create(struct be_adapter *adapter) 3357 { 3358 struct be_resources res = {0}; 3359 struct be_vf_cfg *vf_cfg; 3360 u32 cap_flags, vf; 3361 int status; 3362 3363 /* If a FW profile exists, then cap_flags are updated */ 3364 cap_flags = BE_IF_FLAGS_UNTAGGED | BE_IF_FLAGS_BROADCAST | 3365 BE_IF_FLAGS_MULTICAST; 3366 3367 for_all_vfs(adapter, vf_cfg, vf) { 3368 if (!BE3_chip(adapter)) { 3369 status = be_cmd_get_profile_config(adapter, &res, 3370 vf + 1); 3371 if (!status) 3372 cap_flags = res.if_cap_flags; 3373 } 3374 3375 status = be_if_create(adapter, &vf_cfg->if_handle, 3376 cap_flags, vf + 1); 3377 if (status) 3378 return status; 3379 } 3380 3381 return 0; 3382 } 3383 3384 static int be_vf_setup_init(struct be_adapter *adapter) 3385 { 3386 struct be_vf_cfg *vf_cfg; 3387 int vf; 3388 3389 adapter->vf_cfg = kcalloc(adapter->num_vfs, sizeof(*vf_cfg), 3390 GFP_KERNEL); 3391 if (!adapter->vf_cfg) 3392 return -ENOMEM; 3393 3394 for_all_vfs(adapter, vf_cfg, vf) { 3395 vf_cfg->if_handle = -1; 3396 vf_cfg->pmac_id = -1; 3397 } 3398 return 0; 3399 } 3400 3401 static int be_vf_setup(struct be_adapter *adapter) 3402 { 3403 struct device *dev = &adapter->pdev->dev; 3404 struct be_vf_cfg *vf_cfg; 3405 int status, old_vfs, vf; 3406 u32 privileges; 3407 3408 old_vfs = pci_num_vf(adapter->pdev); 3409 3410 status = be_vf_setup_init(adapter); 3411 if (status) 3412 goto err; 3413 3414 if (old_vfs) { 3415 for_all_vfs(adapter, vf_cfg, vf) { 3416 status = be_cmd_get_if_id(adapter, vf_cfg, vf); 3417 if (status) 3418 goto err; 3419 } 3420 3421 status = be_vfs_mac_query(adapter); 3422 if (status) 3423 goto err; 3424 } else { 3425 status = be_vfs_if_create(adapter); 3426 if (status) 3427 goto err; 3428 3429 status = be_vf_eth_addr_config(adapter); 3430 if (status) 3431 goto err; 3432 } 3433 3434 for_all_vfs(adapter, vf_cfg, vf) { 3435 /* Allow VFs to programs MAC/VLAN filters */ 3436 status = be_cmd_get_fn_privileges(adapter, &privileges, vf + 1); 3437 if (!status && !(privileges & BE_PRIV_FILTMGMT)) { 3438 status = be_cmd_set_fn_privileges(adapter, 3439 privileges | 3440 BE_PRIV_FILTMGMT, 3441 vf + 1); 3442 if (!status) 3443 dev_info(dev, "VF%d has FILTMGMT privilege\n", 3444 vf); 3445 } 3446 3447 /* Allow full available bandwidth */ 3448 if (!old_vfs) 3449 be_cmd_config_qos(adapter, 0, 0, vf + 1); 3450 3451 if (!old_vfs) { 3452 be_cmd_enable_vf(adapter, vf + 1); 3453 be_cmd_set_logical_link_config(adapter, 3454 IFLA_VF_LINK_STATE_AUTO, 3455 vf+1); 3456 } 3457 } 3458 3459 if (!old_vfs) { 3460 status = pci_enable_sriov(adapter->pdev, adapter->num_vfs); 3461 if (status) { 3462 dev_err(dev, "SRIOV enable failed\n"); 3463 adapter->num_vfs = 0; 3464 goto err; 3465 } 3466 } 3467 3468 adapter->flags |= BE_FLAGS_SRIOV_ENABLED; 3469 return 0; 3470 err: 3471 dev_err(dev, "VF setup failed\n"); 3472 be_vf_clear(adapter); 3473 return status; 3474 } 3475 3476 /* Converting function_mode bits on BE3 to SH mc_type enums */ 3477 3478 static u8 be_convert_mc_type(u32 function_mode) 3479 { 3480 if (function_mode & VNIC_MODE && function_mode & QNQ_MODE) 3481 return vNIC1; 3482 else if (function_mode & QNQ_MODE) 3483 return FLEX10; 3484 else if (function_mode & VNIC_MODE) 3485 return vNIC2; 3486 else if (function_mode & UMC_ENABLED) 3487 return UMC; 3488 else 3489 return MC_NONE; 3490 } 3491 3492 /* On BE2/BE3 FW does not suggest the supported limits */ 3493 static void BEx_get_resources(struct be_adapter *adapter, 3494 struct be_resources *res) 3495 { 3496 bool use_sriov = adapter->num_vfs ? 1 : 0; 3497 3498 if (be_physfn(adapter)) 3499 res->max_uc_mac = BE_UC_PMAC_COUNT; 3500 else 3501 res->max_uc_mac = BE_VF_UC_PMAC_COUNT; 3502 3503 adapter->mc_type = be_convert_mc_type(adapter->function_mode); 3504 3505 if (be_is_mc(adapter)) { 3506 /* Assuming that there are 4 channels per port, 3507 * when multi-channel is enabled 3508 */ 3509 if (be_is_qnq_mode(adapter)) 3510 res->max_vlans = BE_NUM_VLANS_SUPPORTED/8; 3511 else 3512 /* In a non-qnq multichannel mode, the pvid 3513 * takes up one vlan entry 3514 */ 3515 res->max_vlans = (BE_NUM_VLANS_SUPPORTED / 4) - 1; 3516 } else { 3517 res->max_vlans = BE_NUM_VLANS_SUPPORTED; 3518 } 3519 3520 res->max_mcast_mac = BE_MAX_MC; 3521 3522 /* 1) For BE3 1Gb ports, FW does not support multiple TXQs 3523 * 2) Create multiple TX rings on a BE3-R multi-channel interface 3524 * *only* if it is RSS-capable. 3525 */ 3526 if (BE2_chip(adapter) || use_sriov || (adapter->port_num > 1) || 3527 !be_physfn(adapter) || (be_is_mc(adapter) && 3528 !(adapter->function_caps & BE_FUNCTION_CAPS_RSS))) { 3529 res->max_tx_qs = 1; 3530 } else if (adapter->function_caps & BE_FUNCTION_CAPS_SUPER_NIC) { 3531 struct be_resources super_nic_res = {0}; 3532 3533 /* On a SuperNIC profile, the driver needs to use the 3534 * GET_PROFILE_CONFIG cmd to query the per-function TXQ limits 3535 */ 3536 be_cmd_get_profile_config(adapter, &super_nic_res, 0); 3537 /* Some old versions of BE3 FW don't report max_tx_qs value */ 3538 res->max_tx_qs = super_nic_res.max_tx_qs ? : BE3_MAX_TX_QS; 3539 } else { 3540 res->max_tx_qs = BE3_MAX_TX_QS; 3541 } 3542 3543 if ((adapter->function_caps & BE_FUNCTION_CAPS_RSS) && 3544 !use_sriov && be_physfn(adapter)) 3545 res->max_rss_qs = (adapter->be3_native) ? 3546 BE3_MAX_RSS_QS : BE2_MAX_RSS_QS; 3547 res->max_rx_qs = res->max_rss_qs + 1; 3548 3549 if (be_physfn(adapter)) 3550 res->max_evt_qs = (be_max_vfs(adapter) > 0) ? 3551 BE3_SRIOV_MAX_EVT_QS : BE3_MAX_EVT_QS; 3552 else 3553 res->max_evt_qs = 1; 3554 3555 res->if_cap_flags = BE_IF_CAP_FLAGS_WANT; 3556 if (!(adapter->function_caps & BE_FUNCTION_CAPS_RSS)) 3557 res->if_cap_flags &= ~BE_IF_FLAGS_RSS; 3558 } 3559 3560 static void be_setup_init(struct be_adapter *adapter) 3561 { 3562 adapter->vlan_prio_bmap = 0xff; 3563 adapter->phy.link_speed = -1; 3564 adapter->if_handle = -1; 3565 adapter->be3_native = false; 3566 adapter->if_flags = 0; 3567 if (be_physfn(adapter)) 3568 adapter->cmd_privileges = MAX_PRIVILEGES; 3569 else 3570 adapter->cmd_privileges = MIN_PRIVILEGES; 3571 } 3572 3573 static int be_get_sriov_config(struct be_adapter *adapter) 3574 { 3575 struct device *dev = &adapter->pdev->dev; 3576 struct be_resources res = {0}; 3577 int max_vfs, old_vfs; 3578 3579 /* Some old versions of BE3 FW don't report max_vfs value */ 3580 be_cmd_get_profile_config(adapter, &res, 0); 3581 3582 if (BE3_chip(adapter) && !res.max_vfs) { 3583 max_vfs = pci_sriov_get_totalvfs(adapter->pdev); 3584 res.max_vfs = max_vfs > 0 ? min(MAX_VFS, max_vfs) : 0; 3585 } 3586 3587 adapter->pool_res = res; 3588 3589 if (!be_max_vfs(adapter)) { 3590 if (num_vfs) 3591 dev_warn(dev, "SRIOV is disabled. Ignoring num_vfs\n"); 3592 adapter->num_vfs = 0; 3593 return 0; 3594 } 3595 3596 pci_sriov_set_totalvfs(adapter->pdev, be_max_vfs(adapter)); 3597 3598 /* validate num_vfs module param */ 3599 old_vfs = pci_num_vf(adapter->pdev); 3600 if (old_vfs) { 3601 dev_info(dev, "%d VFs are already enabled\n", old_vfs); 3602 if (old_vfs != num_vfs) 3603 dev_warn(dev, "Ignoring num_vfs=%d setting\n", num_vfs); 3604 adapter->num_vfs = old_vfs; 3605 } else { 3606 if (num_vfs > be_max_vfs(adapter)) { 3607 dev_info(dev, "Resources unavailable to init %d VFs\n", 3608 num_vfs); 3609 dev_info(dev, "Limiting to %d VFs\n", 3610 be_max_vfs(adapter)); 3611 } 3612 adapter->num_vfs = min_t(u16, num_vfs, be_max_vfs(adapter)); 3613 } 3614 3615 return 0; 3616 } 3617 3618 static int be_get_resources(struct be_adapter *adapter) 3619 { 3620 struct device *dev = &adapter->pdev->dev; 3621 struct be_resources res = {0}; 3622 int status; 3623 3624 if (BEx_chip(adapter)) { 3625 BEx_get_resources(adapter, &res); 3626 adapter->res = res; 3627 } 3628 3629 /* For Lancer, SH etc read per-function resource limits from FW. 3630 * GET_FUNC_CONFIG returns per function guaranteed limits. 3631 * GET_PROFILE_CONFIG returns PCI-E related limits PF-pool limits 3632 */ 3633 if (!BEx_chip(adapter)) { 3634 status = be_cmd_get_func_config(adapter, &res); 3635 if (status) 3636 return status; 3637 3638 /* If RoCE may be enabled stash away half the EQs for RoCE */ 3639 if (be_roce_supported(adapter)) 3640 res.max_evt_qs /= 2; 3641 adapter->res = res; 3642 } 3643 3644 dev_info(dev, "Max: txqs %d, rxqs %d, rss %d, eqs %d, vfs %d\n", 3645 be_max_txqs(adapter), be_max_rxqs(adapter), 3646 be_max_rss(adapter), be_max_eqs(adapter), 3647 be_max_vfs(adapter)); 3648 dev_info(dev, "Max: uc-macs %d, mc-macs %d, vlans %d\n", 3649 be_max_uc(adapter), be_max_mc(adapter), 3650 be_max_vlans(adapter)); 3651 3652 return 0; 3653 } 3654 3655 static void be_sriov_config(struct be_adapter *adapter) 3656 { 3657 struct device *dev = &adapter->pdev->dev; 3658 int status; 3659 3660 status = be_get_sriov_config(adapter); 3661 if (status) { 3662 dev_err(dev, "Failed to query SR-IOV configuration\n"); 3663 dev_err(dev, "SR-IOV cannot be enabled\n"); 3664 return; 3665 } 3666 3667 /* When the HW is in SRIOV capable configuration, the PF-pool 3668 * resources are equally distributed across the max-number of 3669 * VFs. The user may request only a subset of the max-vfs to be 3670 * enabled. Based on num_vfs, redistribute the resources across 3671 * num_vfs so that each VF will have access to more number of 3672 * resources. This facility is not available in BE3 FW. 3673 * Also, this is done by FW in Lancer chip. 3674 */ 3675 if (be_max_vfs(adapter) && !pci_num_vf(adapter->pdev)) { 3676 status = be_cmd_set_sriov_config(adapter, 3677 adapter->pool_res, 3678 adapter->num_vfs); 3679 if (status) 3680 dev_err(dev, "Failed to optimize SR-IOV resources\n"); 3681 } 3682 } 3683 3684 static int be_get_config(struct be_adapter *adapter) 3685 { 3686 u16 profile_id; 3687 int status; 3688 3689 status = be_cmd_query_fw_cfg(adapter); 3690 if (status) 3691 return status; 3692 3693 be_cmd_query_port_name(adapter); 3694 3695 if (be_physfn(adapter)) { 3696 status = be_cmd_get_active_profile(adapter, &profile_id); 3697 if (!status) 3698 dev_info(&adapter->pdev->dev, 3699 "Using profile 0x%x\n", profile_id); 3700 } 3701 3702 if (!BE2_chip(adapter) && be_physfn(adapter)) 3703 be_sriov_config(adapter); 3704 3705 status = be_get_resources(adapter); 3706 if (status) 3707 return status; 3708 3709 adapter->pmac_id = kcalloc(be_max_uc(adapter), 3710 sizeof(*adapter->pmac_id), GFP_KERNEL); 3711 if (!adapter->pmac_id) 3712 return -ENOMEM; 3713 3714 /* Sanitize cfg_num_qs based on HW and platform limits */ 3715 adapter->cfg_num_qs = min(adapter->cfg_num_qs, be_max_qs(adapter)); 3716 3717 return 0; 3718 } 3719 3720 static int be_mac_setup(struct be_adapter *adapter) 3721 { 3722 u8 mac[ETH_ALEN]; 3723 int status; 3724 3725 if (is_zero_ether_addr(adapter->netdev->dev_addr)) { 3726 status = be_cmd_get_perm_mac(adapter, mac); 3727 if (status) 3728 return status; 3729 3730 memcpy(adapter->netdev->dev_addr, mac, ETH_ALEN); 3731 memcpy(adapter->netdev->perm_addr, mac, ETH_ALEN); 3732 } else { 3733 /* Maybe the HW was reset; dev_addr must be re-programmed */ 3734 memcpy(mac, adapter->netdev->dev_addr, ETH_ALEN); 3735 } 3736 3737 /* For BE3-R VFs, the PF programs the initial MAC address */ 3738 if (!(BEx_chip(adapter) && be_virtfn(adapter))) 3739 be_cmd_pmac_add(adapter, mac, adapter->if_handle, 3740 &adapter->pmac_id[0], 0); 3741 return 0; 3742 } 3743 3744 static void be_schedule_worker(struct be_adapter *adapter) 3745 { 3746 schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000)); 3747 adapter->flags |= BE_FLAGS_WORKER_SCHEDULED; 3748 } 3749 3750 static int be_setup_queues(struct be_adapter *adapter) 3751 { 3752 struct net_device *netdev = adapter->netdev; 3753 int status; 3754 3755 status = be_evt_queues_create(adapter); 3756 if (status) 3757 goto err; 3758 3759 status = be_tx_qs_create(adapter); 3760 if (status) 3761 goto err; 3762 3763 status = be_rx_cqs_create(adapter); 3764 if (status) 3765 goto err; 3766 3767 status = be_mcc_queues_create(adapter); 3768 if (status) 3769 goto err; 3770 3771 status = netif_set_real_num_rx_queues(netdev, adapter->num_rx_qs); 3772 if (status) 3773 goto err; 3774 3775 status = netif_set_real_num_tx_queues(netdev, adapter->num_tx_qs); 3776 if (status) 3777 goto err; 3778 3779 return 0; 3780 err: 3781 dev_err(&adapter->pdev->dev, "queue_setup failed\n"); 3782 return status; 3783 } 3784 3785 int be_update_queues(struct be_adapter *adapter) 3786 { 3787 struct net_device *netdev = adapter->netdev; 3788 int status; 3789 3790 if (netif_running(netdev)) 3791 be_close(netdev); 3792 3793 be_cancel_worker(adapter); 3794 3795 /* If any vectors have been shared with RoCE we cannot re-program 3796 * the MSIx table. 3797 */ 3798 if (!adapter->num_msix_roce_vec) 3799 be_msix_disable(adapter); 3800 3801 be_clear_queues(adapter); 3802 3803 if (!msix_enabled(adapter)) { 3804 status = be_msix_enable(adapter); 3805 if (status) 3806 return status; 3807 } 3808 3809 status = be_setup_queues(adapter); 3810 if (status) 3811 return status; 3812 3813 be_schedule_worker(adapter); 3814 3815 if (netif_running(netdev)) 3816 status = be_open(netdev); 3817 3818 return status; 3819 } 3820 3821 static inline int fw_major_num(const char *fw_ver) 3822 { 3823 int fw_major = 0, i; 3824 3825 i = sscanf(fw_ver, "%d.", &fw_major); 3826 if (i != 1) 3827 return 0; 3828 3829 return fw_major; 3830 } 3831 3832 static int be_setup(struct be_adapter *adapter) 3833 { 3834 struct device *dev = &adapter->pdev->dev; 3835 int status; 3836 3837 be_setup_init(adapter); 3838 3839 if (!lancer_chip(adapter)) 3840 be_cmd_req_native_mode(adapter); 3841 3842 status = be_get_config(adapter); 3843 if (status) 3844 goto err; 3845 3846 status = be_msix_enable(adapter); 3847 if (status) 3848 goto err; 3849 3850 status = be_if_create(adapter, &adapter->if_handle, 3851 be_if_cap_flags(adapter), 0); 3852 if (status) 3853 goto err; 3854 3855 /* Updating real_num_tx/rx_queues() requires rtnl_lock() */ 3856 rtnl_lock(); 3857 status = be_setup_queues(adapter); 3858 rtnl_unlock(); 3859 if (status) 3860 goto err; 3861 3862 be_cmd_get_fn_privileges(adapter, &adapter->cmd_privileges, 0); 3863 3864 status = be_mac_setup(adapter); 3865 if (status) 3866 goto err; 3867 3868 be_cmd_get_fw_ver(adapter); 3869 dev_info(dev, "FW version is %s\n", adapter->fw_ver); 3870 3871 if (BE2_chip(adapter) && fw_major_num(adapter->fw_ver) < 4) { 3872 dev_err(dev, "Firmware on card is old(%s), IRQs may not work", 3873 adapter->fw_ver); 3874 dev_err(dev, "Please upgrade firmware to version >= 4.0\n"); 3875 } 3876 3877 if (adapter->vlans_added) 3878 be_vid_config(adapter); 3879 3880 be_set_rx_mode(adapter->netdev); 3881 3882 be_cmd_get_acpi_wol_cap(adapter); 3883 3884 status = be_cmd_set_flow_control(adapter, adapter->tx_fc, 3885 adapter->rx_fc); 3886 if (status) 3887 be_cmd_get_flow_control(adapter, &adapter->tx_fc, 3888 &adapter->rx_fc); 3889 3890 dev_info(&adapter->pdev->dev, "HW Flow control - TX:%d RX:%d\n", 3891 adapter->tx_fc, adapter->rx_fc); 3892 3893 if (be_physfn(adapter)) 3894 be_cmd_set_logical_link_config(adapter, 3895 IFLA_VF_LINK_STATE_AUTO, 0); 3896 3897 if (adapter->num_vfs) 3898 be_vf_setup(adapter); 3899 3900 status = be_cmd_get_phy_info(adapter); 3901 if (!status && be_pause_supported(adapter)) 3902 adapter->phy.fc_autoneg = 1; 3903 3904 be_schedule_worker(adapter); 3905 adapter->flags |= BE_FLAGS_SETUP_DONE; 3906 return 0; 3907 err: 3908 be_clear(adapter); 3909 return status; 3910 } 3911 3912 #ifdef CONFIG_NET_POLL_CONTROLLER 3913 static void be_netpoll(struct net_device *netdev) 3914 { 3915 struct be_adapter *adapter = netdev_priv(netdev); 3916 struct be_eq_obj *eqo; 3917 int i; 3918 3919 for_all_evt_queues(adapter, eqo, i) { 3920 be_eq_notify(eqo->adapter, eqo->q.id, false, true, 0); 3921 napi_schedule(&eqo->napi); 3922 } 3923 } 3924 #endif 3925 3926 static char flash_cookie[2][16] = {"*** SE FLAS", "H DIRECTORY *** "}; 3927 3928 static bool phy_flashing_required(struct be_adapter *adapter) 3929 { 3930 return (adapter->phy.phy_type == PHY_TYPE_TN_8022 && 3931 adapter->phy.interface_type == PHY_TYPE_BASET_10GB); 3932 } 3933 3934 static bool is_comp_in_ufi(struct be_adapter *adapter, 3935 struct flash_section_info *fsec, int type) 3936 { 3937 int i = 0, img_type = 0; 3938 struct flash_section_info_g2 *fsec_g2 = NULL; 3939 3940 if (BE2_chip(adapter)) 3941 fsec_g2 = (struct flash_section_info_g2 *)fsec; 3942 3943 for (i = 0; i < MAX_FLASH_COMP; i++) { 3944 if (fsec_g2) 3945 img_type = le32_to_cpu(fsec_g2->fsec_entry[i].type); 3946 else 3947 img_type = le32_to_cpu(fsec->fsec_entry[i].type); 3948 3949 if (img_type == type) 3950 return true; 3951 } 3952 return false; 3953 3954 } 3955 3956 static struct flash_section_info *get_fsec_info(struct be_adapter *adapter, 3957 int header_size, 3958 const struct firmware *fw) 3959 { 3960 struct flash_section_info *fsec = NULL; 3961 const u8 *p = fw->data; 3962 3963 p += header_size; 3964 while (p < (fw->data + fw->size)) { 3965 fsec = (struct flash_section_info *)p; 3966 if (!memcmp(flash_cookie, fsec->cookie, sizeof(flash_cookie))) 3967 return fsec; 3968 p += 32; 3969 } 3970 return NULL; 3971 } 3972 3973 static int be_check_flash_crc(struct be_adapter *adapter, const u8 *p, 3974 u32 img_offset, u32 img_size, int hdr_size, 3975 u16 img_optype, bool *crc_match) 3976 { 3977 u32 crc_offset; 3978 int status; 3979 u8 crc[4]; 3980 3981 status = be_cmd_get_flash_crc(adapter, crc, img_optype, img_offset, 3982 img_size - 4); 3983 if (status) 3984 return status; 3985 3986 crc_offset = hdr_size + img_offset + img_size - 4; 3987 3988 /* Skip flashing, if crc of flashed region matches */ 3989 if (!memcmp(crc, p + crc_offset, 4)) 3990 *crc_match = true; 3991 else 3992 *crc_match = false; 3993 3994 return status; 3995 } 3996 3997 static int be_flash(struct be_adapter *adapter, const u8 *img, 3998 struct be_dma_mem *flash_cmd, int optype, int img_size, 3999 u32 img_offset) 4000 { 4001 u32 flash_op, num_bytes, total_bytes = img_size, bytes_sent = 0; 4002 struct be_cmd_write_flashrom *req = flash_cmd->va; 4003 int status; 4004 4005 while (total_bytes) { 4006 num_bytes = min_t(u32, 32*1024, total_bytes); 4007 4008 total_bytes -= num_bytes; 4009 4010 if (!total_bytes) { 4011 if (optype == OPTYPE_PHY_FW) 4012 flash_op = FLASHROM_OPER_PHY_FLASH; 4013 else 4014 flash_op = FLASHROM_OPER_FLASH; 4015 } else { 4016 if (optype == OPTYPE_PHY_FW) 4017 flash_op = FLASHROM_OPER_PHY_SAVE; 4018 else 4019 flash_op = FLASHROM_OPER_SAVE; 4020 } 4021 4022 memcpy(req->data_buf, img, num_bytes); 4023 img += num_bytes; 4024 status = be_cmd_write_flashrom(adapter, flash_cmd, optype, 4025 flash_op, img_offset + 4026 bytes_sent, num_bytes); 4027 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST && 4028 optype == OPTYPE_PHY_FW) 4029 break; 4030 else if (status) 4031 return status; 4032 4033 bytes_sent += num_bytes; 4034 } 4035 return 0; 4036 } 4037 4038 /* For BE2, BE3 and BE3-R */ 4039 static int be_flash_BEx(struct be_adapter *adapter, 4040 const struct firmware *fw, 4041 struct be_dma_mem *flash_cmd, int num_of_images) 4042 { 4043 int img_hdrs_size = (num_of_images * sizeof(struct image_hdr)); 4044 struct device *dev = &adapter->pdev->dev; 4045 struct flash_section_info *fsec = NULL; 4046 int status, i, filehdr_size, num_comp; 4047 const struct flash_comp *pflashcomp; 4048 bool crc_match; 4049 const u8 *p; 4050 4051 struct flash_comp gen3_flash_types[] = { 4052 { FLASH_iSCSI_PRIMARY_IMAGE_START_g3, OPTYPE_ISCSI_ACTIVE, 4053 FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_iSCSI}, 4054 { FLASH_REDBOOT_START_g3, OPTYPE_REDBOOT, 4055 FLASH_REDBOOT_IMAGE_MAX_SIZE_g3, IMAGE_BOOT_CODE}, 4056 { FLASH_iSCSI_BIOS_START_g3, OPTYPE_BIOS, 4057 FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_ISCSI}, 4058 { FLASH_PXE_BIOS_START_g3, OPTYPE_PXE_BIOS, 4059 FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_PXE}, 4060 { FLASH_FCoE_BIOS_START_g3, OPTYPE_FCOE_BIOS, 4061 FLASH_BIOS_IMAGE_MAX_SIZE_g3, IMAGE_OPTION_ROM_FCoE}, 4062 { FLASH_iSCSI_BACKUP_IMAGE_START_g3, OPTYPE_ISCSI_BACKUP, 4063 FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_iSCSI}, 4064 { FLASH_FCoE_PRIMARY_IMAGE_START_g3, OPTYPE_FCOE_FW_ACTIVE, 4065 FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_FCoE}, 4066 { FLASH_FCoE_BACKUP_IMAGE_START_g3, OPTYPE_FCOE_FW_BACKUP, 4067 FLASH_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_BACKUP_FCoE}, 4068 { FLASH_NCSI_START_g3, OPTYPE_NCSI_FW, 4069 FLASH_NCSI_IMAGE_MAX_SIZE_g3, IMAGE_NCSI}, 4070 { FLASH_PHY_FW_START_g3, OPTYPE_PHY_FW, 4071 FLASH_PHY_FW_IMAGE_MAX_SIZE_g3, IMAGE_FIRMWARE_PHY} 4072 }; 4073 4074 struct flash_comp gen2_flash_types[] = { 4075 { FLASH_iSCSI_PRIMARY_IMAGE_START_g2, OPTYPE_ISCSI_ACTIVE, 4076 FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_iSCSI}, 4077 { FLASH_REDBOOT_START_g2, OPTYPE_REDBOOT, 4078 FLASH_REDBOOT_IMAGE_MAX_SIZE_g2, IMAGE_BOOT_CODE}, 4079 { FLASH_iSCSI_BIOS_START_g2, OPTYPE_BIOS, 4080 FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_ISCSI}, 4081 { FLASH_PXE_BIOS_START_g2, OPTYPE_PXE_BIOS, 4082 FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_PXE}, 4083 { FLASH_FCoE_BIOS_START_g2, OPTYPE_FCOE_BIOS, 4084 FLASH_BIOS_IMAGE_MAX_SIZE_g2, IMAGE_OPTION_ROM_FCoE}, 4085 { FLASH_iSCSI_BACKUP_IMAGE_START_g2, OPTYPE_ISCSI_BACKUP, 4086 FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_iSCSI}, 4087 { FLASH_FCoE_PRIMARY_IMAGE_START_g2, OPTYPE_FCOE_FW_ACTIVE, 4088 FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_FCoE}, 4089 { FLASH_FCoE_BACKUP_IMAGE_START_g2, OPTYPE_FCOE_FW_BACKUP, 4090 FLASH_IMAGE_MAX_SIZE_g2, IMAGE_FIRMWARE_BACKUP_FCoE} 4091 }; 4092 4093 if (BE3_chip(adapter)) { 4094 pflashcomp = gen3_flash_types; 4095 filehdr_size = sizeof(struct flash_file_hdr_g3); 4096 num_comp = ARRAY_SIZE(gen3_flash_types); 4097 } else { 4098 pflashcomp = gen2_flash_types; 4099 filehdr_size = sizeof(struct flash_file_hdr_g2); 4100 num_comp = ARRAY_SIZE(gen2_flash_types); 4101 img_hdrs_size = 0; 4102 } 4103 4104 /* Get flash section info*/ 4105 fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw); 4106 if (!fsec) { 4107 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n"); 4108 return -1; 4109 } 4110 for (i = 0; i < num_comp; i++) { 4111 if (!is_comp_in_ufi(adapter, fsec, pflashcomp[i].img_type)) 4112 continue; 4113 4114 if ((pflashcomp[i].optype == OPTYPE_NCSI_FW) && 4115 memcmp(adapter->fw_ver, "3.102.148.0", 11) < 0) 4116 continue; 4117 4118 if (pflashcomp[i].optype == OPTYPE_PHY_FW && 4119 !phy_flashing_required(adapter)) 4120 continue; 4121 4122 if (pflashcomp[i].optype == OPTYPE_REDBOOT) { 4123 status = be_check_flash_crc(adapter, fw->data, 4124 pflashcomp[i].offset, 4125 pflashcomp[i].size, 4126 filehdr_size + 4127 img_hdrs_size, 4128 OPTYPE_REDBOOT, &crc_match); 4129 if (status) { 4130 dev_err(dev, 4131 "Could not get CRC for 0x%x region\n", 4132 pflashcomp[i].optype); 4133 continue; 4134 } 4135 4136 if (crc_match) 4137 continue; 4138 } 4139 4140 p = fw->data + filehdr_size + pflashcomp[i].offset + 4141 img_hdrs_size; 4142 if (p + pflashcomp[i].size > fw->data + fw->size) 4143 return -1; 4144 4145 status = be_flash(adapter, p, flash_cmd, pflashcomp[i].optype, 4146 pflashcomp[i].size, 0); 4147 if (status) { 4148 dev_err(dev, "Flashing section type 0x%x failed\n", 4149 pflashcomp[i].img_type); 4150 return status; 4151 } 4152 } 4153 return 0; 4154 } 4155 4156 static u16 be_get_img_optype(struct flash_section_entry fsec_entry) 4157 { 4158 u32 img_type = le32_to_cpu(fsec_entry.type); 4159 u16 img_optype = le16_to_cpu(fsec_entry.optype); 4160 4161 if (img_optype != 0xFFFF) 4162 return img_optype; 4163 4164 switch (img_type) { 4165 case IMAGE_FIRMWARE_iSCSI: 4166 img_optype = OPTYPE_ISCSI_ACTIVE; 4167 break; 4168 case IMAGE_BOOT_CODE: 4169 img_optype = OPTYPE_REDBOOT; 4170 break; 4171 case IMAGE_OPTION_ROM_ISCSI: 4172 img_optype = OPTYPE_BIOS; 4173 break; 4174 case IMAGE_OPTION_ROM_PXE: 4175 img_optype = OPTYPE_PXE_BIOS; 4176 break; 4177 case IMAGE_OPTION_ROM_FCoE: 4178 img_optype = OPTYPE_FCOE_BIOS; 4179 break; 4180 case IMAGE_FIRMWARE_BACKUP_iSCSI: 4181 img_optype = OPTYPE_ISCSI_BACKUP; 4182 break; 4183 case IMAGE_NCSI: 4184 img_optype = OPTYPE_NCSI_FW; 4185 break; 4186 case IMAGE_FLASHISM_JUMPVECTOR: 4187 img_optype = OPTYPE_FLASHISM_JUMPVECTOR; 4188 break; 4189 case IMAGE_FIRMWARE_PHY: 4190 img_optype = OPTYPE_SH_PHY_FW; 4191 break; 4192 case IMAGE_REDBOOT_DIR: 4193 img_optype = OPTYPE_REDBOOT_DIR; 4194 break; 4195 case IMAGE_REDBOOT_CONFIG: 4196 img_optype = OPTYPE_REDBOOT_CONFIG; 4197 break; 4198 case IMAGE_UFI_DIR: 4199 img_optype = OPTYPE_UFI_DIR; 4200 break; 4201 default: 4202 break; 4203 } 4204 4205 return img_optype; 4206 } 4207 4208 static int be_flash_skyhawk(struct be_adapter *adapter, 4209 const struct firmware *fw, 4210 struct be_dma_mem *flash_cmd, int num_of_images) 4211 { 4212 int img_hdrs_size = num_of_images * sizeof(struct image_hdr); 4213 bool crc_match, old_fw_img, flash_offset_support = true; 4214 struct device *dev = &adapter->pdev->dev; 4215 struct flash_section_info *fsec = NULL; 4216 u32 img_offset, img_size, img_type; 4217 u16 img_optype, flash_optype; 4218 int status, i, filehdr_size; 4219 const u8 *p; 4220 4221 filehdr_size = sizeof(struct flash_file_hdr_g3); 4222 fsec = get_fsec_info(adapter, filehdr_size + img_hdrs_size, fw); 4223 if (!fsec) { 4224 dev_err(dev, "Invalid Cookie. FW image may be corrupted\n"); 4225 return -EINVAL; 4226 } 4227 4228 retry_flash: 4229 for (i = 0; i < le32_to_cpu(fsec->fsec_hdr.num_images); i++) { 4230 img_offset = le32_to_cpu(fsec->fsec_entry[i].offset); 4231 img_size = le32_to_cpu(fsec->fsec_entry[i].pad_size); 4232 img_type = le32_to_cpu(fsec->fsec_entry[i].type); 4233 img_optype = be_get_img_optype(fsec->fsec_entry[i]); 4234 old_fw_img = fsec->fsec_entry[i].optype == 0xFFFF; 4235 4236 if (img_optype == 0xFFFF) 4237 continue; 4238 4239 if (flash_offset_support) 4240 flash_optype = OPTYPE_OFFSET_SPECIFIED; 4241 else 4242 flash_optype = img_optype; 4243 4244 /* Don't bother verifying CRC if an old FW image is being 4245 * flashed 4246 */ 4247 if (old_fw_img) 4248 goto flash; 4249 4250 status = be_check_flash_crc(adapter, fw->data, img_offset, 4251 img_size, filehdr_size + 4252 img_hdrs_size, flash_optype, 4253 &crc_match); 4254 if (base_status(status) == MCC_STATUS_ILLEGAL_REQUEST || 4255 base_status(status) == MCC_STATUS_ILLEGAL_FIELD) { 4256 /* The current FW image on the card does not support 4257 * OFFSET based flashing. Retry using older mechanism 4258 * of OPTYPE based flashing 4259 */ 4260 if (flash_optype == OPTYPE_OFFSET_SPECIFIED) { 4261 flash_offset_support = false; 4262 goto retry_flash; 4263 } 4264 4265 /* The current FW image on the card does not recognize 4266 * the new FLASH op_type. The FW download is partially 4267 * complete. Reboot the server now to enable FW image 4268 * to recognize the new FLASH op_type. To complete the 4269 * remaining process, download the same FW again after 4270 * the reboot. 4271 */ 4272 dev_err(dev, "Flash incomplete. Reset the server\n"); 4273 dev_err(dev, "Download FW image again after reset\n"); 4274 return -EAGAIN; 4275 } else if (status) { 4276 dev_err(dev, "Could not get CRC for 0x%x region\n", 4277 img_optype); 4278 return -EFAULT; 4279 } 4280 4281 if (crc_match) 4282 continue; 4283 4284 flash: 4285 p = fw->data + filehdr_size + img_offset + img_hdrs_size; 4286 if (p + img_size > fw->data + fw->size) 4287 return -1; 4288 4289 status = be_flash(adapter, p, flash_cmd, flash_optype, img_size, 4290 img_offset); 4291 4292 /* The current FW image on the card does not support OFFSET 4293 * based flashing. Retry using older mechanism of OPTYPE based 4294 * flashing 4295 */ 4296 if (base_status(status) == MCC_STATUS_ILLEGAL_FIELD && 4297 flash_optype == OPTYPE_OFFSET_SPECIFIED) { 4298 flash_offset_support = false; 4299 goto retry_flash; 4300 } 4301 4302 /* For old FW images ignore ILLEGAL_FIELD error or errors on 4303 * UFI_DIR region 4304 */ 4305 if (old_fw_img && 4306 (base_status(status) == MCC_STATUS_ILLEGAL_FIELD || 4307 (img_optype == OPTYPE_UFI_DIR && 4308 base_status(status) == MCC_STATUS_FAILED))) { 4309 continue; 4310 } else if (status) { 4311 dev_err(dev, "Flashing section type 0x%x failed\n", 4312 img_type); 4313 return -EFAULT; 4314 } 4315 } 4316 return 0; 4317 } 4318 4319 static int lancer_fw_download(struct be_adapter *adapter, 4320 const struct firmware *fw) 4321 { 4322 #define LANCER_FW_DOWNLOAD_CHUNK (32 * 1024) 4323 #define LANCER_FW_DOWNLOAD_LOCATION "/prg" 4324 struct device *dev = &adapter->pdev->dev; 4325 struct be_dma_mem flash_cmd; 4326 const u8 *data_ptr = NULL; 4327 u8 *dest_image_ptr = NULL; 4328 size_t image_size = 0; 4329 u32 chunk_size = 0; 4330 u32 data_written = 0; 4331 u32 offset = 0; 4332 int status = 0; 4333 u8 add_status = 0; 4334 u8 change_status; 4335 4336 if (!IS_ALIGNED(fw->size, sizeof(u32))) { 4337 dev_err(dev, "FW image size should be multiple of 4\n"); 4338 return -EINVAL; 4339 } 4340 4341 flash_cmd.size = sizeof(struct lancer_cmd_req_write_object) 4342 + LANCER_FW_DOWNLOAD_CHUNK; 4343 flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, 4344 &flash_cmd.dma, GFP_KERNEL); 4345 if (!flash_cmd.va) 4346 return -ENOMEM; 4347 4348 dest_image_ptr = flash_cmd.va + 4349 sizeof(struct lancer_cmd_req_write_object); 4350 image_size = fw->size; 4351 data_ptr = fw->data; 4352 4353 while (image_size) { 4354 chunk_size = min_t(u32, image_size, LANCER_FW_DOWNLOAD_CHUNK); 4355 4356 /* Copy the image chunk content. */ 4357 memcpy(dest_image_ptr, data_ptr, chunk_size); 4358 4359 status = lancer_cmd_write_object(adapter, &flash_cmd, 4360 chunk_size, offset, 4361 LANCER_FW_DOWNLOAD_LOCATION, 4362 &data_written, &change_status, 4363 &add_status); 4364 if (status) 4365 break; 4366 4367 offset += data_written; 4368 data_ptr += data_written; 4369 image_size -= data_written; 4370 } 4371 4372 if (!status) { 4373 /* Commit the FW written */ 4374 status = lancer_cmd_write_object(adapter, &flash_cmd, 4375 0, offset, 4376 LANCER_FW_DOWNLOAD_LOCATION, 4377 &data_written, &change_status, 4378 &add_status); 4379 } 4380 4381 dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma); 4382 if (status) { 4383 dev_err(dev, "Firmware load error\n"); 4384 return be_cmd_status(status); 4385 } 4386 4387 dev_info(dev, "Firmware flashed successfully\n"); 4388 4389 if (change_status == LANCER_FW_RESET_NEEDED) { 4390 dev_info(dev, "Resetting adapter to activate new FW\n"); 4391 status = lancer_physdev_ctrl(adapter, 4392 PHYSDEV_CONTROL_FW_RESET_MASK); 4393 if (status) { 4394 dev_err(dev, "Adapter busy, could not reset FW\n"); 4395 dev_err(dev, "Reboot server to activate new FW\n"); 4396 } 4397 } else if (change_status != LANCER_NO_RESET_NEEDED) { 4398 dev_info(dev, "Reboot server to activate new FW\n"); 4399 } 4400 4401 return 0; 4402 } 4403 4404 #define BE2_UFI 2 4405 #define BE3_UFI 3 4406 #define BE3R_UFI 10 4407 #define SH_UFI 4 4408 #define SH_P2_UFI 11 4409 4410 static int be_get_ufi_type(struct be_adapter *adapter, 4411 struct flash_file_hdr_g3 *fhdr) 4412 { 4413 if (!fhdr) { 4414 dev_err(&adapter->pdev->dev, "Invalid FW UFI file"); 4415 return -1; 4416 } 4417 4418 /* First letter of the build version is used to identify 4419 * which chip this image file is meant for. 4420 */ 4421 switch (fhdr->build[0]) { 4422 case BLD_STR_UFI_TYPE_SH: 4423 return (fhdr->asic_type_rev == ASIC_REV_P2) ? SH_P2_UFI : 4424 SH_UFI; 4425 case BLD_STR_UFI_TYPE_BE3: 4426 return (fhdr->asic_type_rev == ASIC_REV_B0) ? BE3R_UFI : 4427 BE3_UFI; 4428 case BLD_STR_UFI_TYPE_BE2: 4429 return BE2_UFI; 4430 default: 4431 return -1; 4432 } 4433 } 4434 4435 /* Check if the flash image file is compatible with the adapter that 4436 * is being flashed. 4437 * BE3 chips with asic-rev B0 must be flashed only with BE3R_UFI type. 4438 * Skyhawk chips with asic-rev P2 must be flashed only with SH_P2_UFI type. 4439 */ 4440 static bool be_check_ufi_compatibility(struct be_adapter *adapter, 4441 struct flash_file_hdr_g3 *fhdr) 4442 { 4443 int ufi_type = be_get_ufi_type(adapter, fhdr); 4444 4445 switch (ufi_type) { 4446 case SH_P2_UFI: 4447 return skyhawk_chip(adapter); 4448 case SH_UFI: 4449 return (skyhawk_chip(adapter) && 4450 adapter->asic_rev < ASIC_REV_P2); 4451 case BE3R_UFI: 4452 return BE3_chip(adapter); 4453 case BE3_UFI: 4454 return (BE3_chip(adapter) && adapter->asic_rev < ASIC_REV_B0); 4455 case BE2_UFI: 4456 return BE2_chip(adapter); 4457 default: 4458 return false; 4459 } 4460 } 4461 4462 static int be_fw_download(struct be_adapter *adapter, const struct firmware* fw) 4463 { 4464 struct device *dev = &adapter->pdev->dev; 4465 struct flash_file_hdr_g3 *fhdr3; 4466 struct image_hdr *img_hdr_ptr; 4467 int status = 0, i, num_imgs; 4468 struct be_dma_mem flash_cmd; 4469 4470 fhdr3 = (struct flash_file_hdr_g3 *)fw->data; 4471 if (!be_check_ufi_compatibility(adapter, fhdr3)) { 4472 dev_err(dev, "Flash image is not compatible with adapter\n"); 4473 return -EINVAL; 4474 } 4475 4476 flash_cmd.size = sizeof(struct be_cmd_write_flashrom); 4477 flash_cmd.va = dma_alloc_coherent(dev, flash_cmd.size, &flash_cmd.dma, 4478 GFP_KERNEL); 4479 if (!flash_cmd.va) 4480 return -ENOMEM; 4481 4482 num_imgs = le32_to_cpu(fhdr3->num_imgs); 4483 for (i = 0; i < num_imgs; i++) { 4484 img_hdr_ptr = (struct image_hdr *)(fw->data + 4485 (sizeof(struct flash_file_hdr_g3) + 4486 i * sizeof(struct image_hdr))); 4487 if (!BE2_chip(adapter) && 4488 le32_to_cpu(img_hdr_ptr->imageid) != 1) 4489 continue; 4490 4491 if (skyhawk_chip(adapter)) 4492 status = be_flash_skyhawk(adapter, fw, &flash_cmd, 4493 num_imgs); 4494 else 4495 status = be_flash_BEx(adapter, fw, &flash_cmd, 4496 num_imgs); 4497 } 4498 4499 dma_free_coherent(dev, flash_cmd.size, flash_cmd.va, flash_cmd.dma); 4500 if (!status) 4501 dev_info(dev, "Firmware flashed successfully\n"); 4502 4503 return status; 4504 } 4505 4506 int be_load_fw(struct be_adapter *adapter, u8 *fw_file) 4507 { 4508 const struct firmware *fw; 4509 int status; 4510 4511 if (!netif_running(adapter->netdev)) { 4512 dev_err(&adapter->pdev->dev, 4513 "Firmware load not allowed (interface is down)\n"); 4514 return -ENETDOWN; 4515 } 4516 4517 status = request_firmware(&fw, fw_file, &adapter->pdev->dev); 4518 if (status) 4519 goto fw_exit; 4520 4521 dev_info(&adapter->pdev->dev, "Flashing firmware file %s\n", fw_file); 4522 4523 if (lancer_chip(adapter)) 4524 status = lancer_fw_download(adapter, fw); 4525 else 4526 status = be_fw_download(adapter, fw); 4527 4528 if (!status) 4529 be_cmd_get_fw_ver(adapter); 4530 4531 fw_exit: 4532 release_firmware(fw); 4533 return status; 4534 } 4535 4536 static int be_ndo_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4537 u16 flags) 4538 { 4539 struct be_adapter *adapter = netdev_priv(dev); 4540 struct nlattr *attr, *br_spec; 4541 int rem; 4542 int status = 0; 4543 u16 mode = 0; 4544 4545 if (!sriov_enabled(adapter)) 4546 return -EOPNOTSUPP; 4547 4548 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4549 if (!br_spec) 4550 return -EINVAL; 4551 4552 nla_for_each_nested(attr, br_spec, rem) { 4553 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4554 continue; 4555 4556 if (nla_len(attr) < sizeof(mode)) 4557 return -EINVAL; 4558 4559 mode = nla_get_u16(attr); 4560 if (mode != BRIDGE_MODE_VEPA && mode != BRIDGE_MODE_VEB) 4561 return -EINVAL; 4562 4563 status = be_cmd_set_hsw_config(adapter, 0, 0, 4564 adapter->if_handle, 4565 mode == BRIDGE_MODE_VEPA ? 4566 PORT_FWD_TYPE_VEPA : 4567 PORT_FWD_TYPE_VEB); 4568 if (status) 4569 goto err; 4570 4571 dev_info(&adapter->pdev->dev, "enabled switch mode: %s\n", 4572 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4573 4574 return status; 4575 } 4576 err: 4577 dev_err(&adapter->pdev->dev, "Failed to set switch mode %s\n", 4578 mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); 4579 4580 return status; 4581 } 4582 4583 static int be_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4584 struct net_device *dev, u32 filter_mask) 4585 { 4586 struct be_adapter *adapter = netdev_priv(dev); 4587 int status = 0; 4588 u8 hsw_mode; 4589 4590 if (!sriov_enabled(adapter)) 4591 return 0; 4592 4593 /* BE and Lancer chips support VEB mode only */ 4594 if (BEx_chip(adapter) || lancer_chip(adapter)) { 4595 hsw_mode = PORT_FWD_TYPE_VEB; 4596 } else { 4597 status = be_cmd_get_hsw_config(adapter, NULL, 0, 4598 adapter->if_handle, &hsw_mode); 4599 if (status) 4600 return 0; 4601 } 4602 4603 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4604 hsw_mode == PORT_FWD_TYPE_VEPA ? 4605 BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB, 4606 0, 0); 4607 } 4608 4609 #ifdef CONFIG_BE2NET_VXLAN 4610 /* VxLAN offload Notes: 4611 * 4612 * The stack defines tunnel offload flags (hw_enc_features) for IP and doesn't 4613 * distinguish various types of transports (VxLAN, GRE, NVGRE ..). So, offload 4614 * is expected to work across all types of IP tunnels once exported. Skyhawk 4615 * supports offloads for either VxLAN or NVGRE, exclusively. So we export VxLAN 4616 * offloads in hw_enc_features only when a VxLAN port is added. If other (non 4617 * VxLAN) tunnels are configured while VxLAN offloads are enabled, offloads for 4618 * those other tunnels are unexported on the fly through ndo_features_check(). 4619 * 4620 * Skyhawk supports VxLAN offloads only for one UDP dport. So, if the stack 4621 * adds more than one port, disable offloads and don't re-enable them again 4622 * until after all the tunnels are removed. 4623 */ 4624 static void be_add_vxlan_port(struct net_device *netdev, sa_family_t sa_family, 4625 __be16 port) 4626 { 4627 struct be_adapter *adapter = netdev_priv(netdev); 4628 struct device *dev = &adapter->pdev->dev; 4629 int status; 4630 4631 if (lancer_chip(adapter) || BEx_chip(adapter)) 4632 return; 4633 4634 if (adapter->flags & BE_FLAGS_VXLAN_OFFLOADS) { 4635 dev_info(dev, 4636 "Only one UDP port supported for VxLAN offloads\n"); 4637 dev_info(dev, "Disabling VxLAN offloads\n"); 4638 adapter->vxlan_port_count++; 4639 goto err; 4640 } 4641 4642 if (adapter->vxlan_port_count++ >= 1) 4643 return; 4644 4645 status = be_cmd_manage_iface(adapter, adapter->if_handle, 4646 OP_CONVERT_NORMAL_TO_TUNNEL); 4647 if (status) { 4648 dev_warn(dev, "Failed to convert normal interface to tunnel\n"); 4649 goto err; 4650 } 4651 4652 status = be_cmd_set_vxlan_port(adapter, port); 4653 if (status) { 4654 dev_warn(dev, "Failed to add VxLAN port\n"); 4655 goto err; 4656 } 4657 adapter->flags |= BE_FLAGS_VXLAN_OFFLOADS; 4658 adapter->vxlan_port = port; 4659 4660 netdev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4661 NETIF_F_TSO | NETIF_F_TSO6 | 4662 NETIF_F_GSO_UDP_TUNNEL; 4663 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL; 4664 netdev->features |= NETIF_F_GSO_UDP_TUNNEL; 4665 4666 dev_info(dev, "Enabled VxLAN offloads for UDP port %d\n", 4667 be16_to_cpu(port)); 4668 return; 4669 err: 4670 be_disable_vxlan_offloads(adapter); 4671 } 4672 4673 static void be_del_vxlan_port(struct net_device *netdev, sa_family_t sa_family, 4674 __be16 port) 4675 { 4676 struct be_adapter *adapter = netdev_priv(netdev); 4677 4678 if (lancer_chip(adapter) || BEx_chip(adapter)) 4679 return; 4680 4681 if (adapter->vxlan_port != port) 4682 goto done; 4683 4684 be_disable_vxlan_offloads(adapter); 4685 4686 dev_info(&adapter->pdev->dev, 4687 "Disabled VxLAN offloads for UDP port %d\n", 4688 be16_to_cpu(port)); 4689 done: 4690 adapter->vxlan_port_count--; 4691 } 4692 4693 static netdev_features_t be_features_check(struct sk_buff *skb, 4694 struct net_device *dev, 4695 netdev_features_t features) 4696 { 4697 struct be_adapter *adapter = netdev_priv(dev); 4698 u8 l4_hdr = 0; 4699 4700 /* The code below restricts offload features for some tunneled packets. 4701 * Offload features for normal (non tunnel) packets are unchanged. 4702 */ 4703 if (!skb->encapsulation || 4704 !(adapter->flags & BE_FLAGS_VXLAN_OFFLOADS)) 4705 return features; 4706 4707 /* It's an encapsulated packet and VxLAN offloads are enabled. We 4708 * should disable tunnel offload features if it's not a VxLAN packet, 4709 * as tunnel offloads have been enabled only for VxLAN. This is done to 4710 * allow other tunneled traffic like GRE work fine while VxLAN 4711 * offloads are configured in Skyhawk-R. 4712 */ 4713 switch (vlan_get_protocol(skb)) { 4714 case htons(ETH_P_IP): 4715 l4_hdr = ip_hdr(skb)->protocol; 4716 break; 4717 case htons(ETH_P_IPV6): 4718 l4_hdr = ipv6_hdr(skb)->nexthdr; 4719 break; 4720 default: 4721 return features; 4722 } 4723 4724 if (l4_hdr != IPPROTO_UDP || 4725 skb->inner_protocol_type != ENCAP_TYPE_ETHER || 4726 skb->inner_protocol != htons(ETH_P_TEB) || 4727 skb_inner_mac_header(skb) - skb_transport_header(skb) != 4728 sizeof(struct udphdr) + sizeof(struct vxlanhdr)) 4729 return features & ~(NETIF_F_ALL_CSUM | NETIF_F_GSO_MASK); 4730 4731 return features; 4732 } 4733 #endif 4734 4735 static const struct net_device_ops be_netdev_ops = { 4736 .ndo_open = be_open, 4737 .ndo_stop = be_close, 4738 .ndo_start_xmit = be_xmit, 4739 .ndo_set_rx_mode = be_set_rx_mode, 4740 .ndo_set_mac_address = be_mac_addr_set, 4741 .ndo_change_mtu = be_change_mtu, 4742 .ndo_get_stats64 = be_get_stats64, 4743 .ndo_validate_addr = eth_validate_addr, 4744 .ndo_vlan_rx_add_vid = be_vlan_add_vid, 4745 .ndo_vlan_rx_kill_vid = be_vlan_rem_vid, 4746 .ndo_set_vf_mac = be_set_vf_mac, 4747 .ndo_set_vf_vlan = be_set_vf_vlan, 4748 .ndo_set_vf_rate = be_set_vf_tx_rate, 4749 .ndo_get_vf_config = be_get_vf_config, 4750 .ndo_set_vf_link_state = be_set_vf_link_state, 4751 #ifdef CONFIG_NET_POLL_CONTROLLER 4752 .ndo_poll_controller = be_netpoll, 4753 #endif 4754 .ndo_bridge_setlink = be_ndo_bridge_setlink, 4755 .ndo_bridge_getlink = be_ndo_bridge_getlink, 4756 #ifdef CONFIG_NET_RX_BUSY_POLL 4757 .ndo_busy_poll = be_busy_poll, 4758 #endif 4759 #ifdef CONFIG_BE2NET_VXLAN 4760 .ndo_add_vxlan_port = be_add_vxlan_port, 4761 .ndo_del_vxlan_port = be_del_vxlan_port, 4762 .ndo_features_check = be_features_check, 4763 #endif 4764 }; 4765 4766 static void be_netdev_init(struct net_device *netdev) 4767 { 4768 struct be_adapter *adapter = netdev_priv(netdev); 4769 4770 netdev->hw_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 4771 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | 4772 NETIF_F_HW_VLAN_CTAG_TX; 4773 if (be_multi_rxq(adapter)) 4774 netdev->hw_features |= NETIF_F_RXHASH; 4775 4776 netdev->features |= netdev->hw_features | 4777 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_FILTER; 4778 4779 netdev->vlan_features |= NETIF_F_SG | NETIF_F_TSO | NETIF_F_TSO6 | 4780 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 4781 4782 netdev->priv_flags |= IFF_UNICAST_FLT; 4783 4784 netdev->flags |= IFF_MULTICAST; 4785 4786 netif_set_gso_max_size(netdev, 65535 - ETH_HLEN); 4787 4788 netdev->netdev_ops = &be_netdev_ops; 4789 4790 netdev->ethtool_ops = &be_ethtool_ops; 4791 } 4792 4793 static void be_unmap_pci_bars(struct be_adapter *adapter) 4794 { 4795 if (adapter->csr) 4796 pci_iounmap(adapter->pdev, adapter->csr); 4797 if (adapter->db) 4798 pci_iounmap(adapter->pdev, adapter->db); 4799 } 4800 4801 static int db_bar(struct be_adapter *adapter) 4802 { 4803 if (lancer_chip(adapter) || !be_physfn(adapter)) 4804 return 0; 4805 else 4806 return 4; 4807 } 4808 4809 static int be_roce_map_pci_bars(struct be_adapter *adapter) 4810 { 4811 if (skyhawk_chip(adapter)) { 4812 adapter->roce_db.size = 4096; 4813 adapter->roce_db.io_addr = pci_resource_start(adapter->pdev, 4814 db_bar(adapter)); 4815 adapter->roce_db.total_size = pci_resource_len(adapter->pdev, 4816 db_bar(adapter)); 4817 } 4818 return 0; 4819 } 4820 4821 static int be_map_pci_bars(struct be_adapter *adapter) 4822 { 4823 u8 __iomem *addr; 4824 4825 if (BEx_chip(adapter) && be_physfn(adapter)) { 4826 adapter->csr = pci_iomap(adapter->pdev, 2, 0); 4827 if (!adapter->csr) 4828 return -ENOMEM; 4829 } 4830 4831 addr = pci_iomap(adapter->pdev, db_bar(adapter), 0); 4832 if (!addr) 4833 goto pci_map_err; 4834 adapter->db = addr; 4835 4836 be_roce_map_pci_bars(adapter); 4837 return 0; 4838 4839 pci_map_err: 4840 dev_err(&adapter->pdev->dev, "Error in mapping PCI BARs\n"); 4841 be_unmap_pci_bars(adapter); 4842 return -ENOMEM; 4843 } 4844 4845 static void be_ctrl_cleanup(struct be_adapter *adapter) 4846 { 4847 struct be_dma_mem *mem = &adapter->mbox_mem_alloced; 4848 4849 be_unmap_pci_bars(adapter); 4850 4851 if (mem->va) 4852 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 4853 mem->dma); 4854 4855 mem = &adapter->rx_filter; 4856 if (mem->va) 4857 dma_free_coherent(&adapter->pdev->dev, mem->size, mem->va, 4858 mem->dma); 4859 } 4860 4861 static int be_ctrl_init(struct be_adapter *adapter) 4862 { 4863 struct be_dma_mem *mbox_mem_alloc = &adapter->mbox_mem_alloced; 4864 struct be_dma_mem *mbox_mem_align = &adapter->mbox_mem; 4865 struct be_dma_mem *rx_filter = &adapter->rx_filter; 4866 u32 sli_intf; 4867 int status; 4868 4869 pci_read_config_dword(adapter->pdev, SLI_INTF_REG_OFFSET, &sli_intf); 4870 adapter->sli_family = (sli_intf & SLI_INTF_FAMILY_MASK) >> 4871 SLI_INTF_FAMILY_SHIFT; 4872 adapter->virtfn = (sli_intf & SLI_INTF_FT_MASK) ? 1 : 0; 4873 4874 status = be_map_pci_bars(adapter); 4875 if (status) 4876 goto done; 4877 4878 mbox_mem_alloc->size = sizeof(struct be_mcc_mailbox) + 16; 4879 mbox_mem_alloc->va = dma_alloc_coherent(&adapter->pdev->dev, 4880 mbox_mem_alloc->size, 4881 &mbox_mem_alloc->dma, 4882 GFP_KERNEL); 4883 if (!mbox_mem_alloc->va) { 4884 status = -ENOMEM; 4885 goto unmap_pci_bars; 4886 } 4887 mbox_mem_align->size = sizeof(struct be_mcc_mailbox); 4888 mbox_mem_align->va = PTR_ALIGN(mbox_mem_alloc->va, 16); 4889 mbox_mem_align->dma = PTR_ALIGN(mbox_mem_alloc->dma, 16); 4890 memset(mbox_mem_align->va, 0, sizeof(struct be_mcc_mailbox)); 4891 4892 rx_filter->size = sizeof(struct be_cmd_req_rx_filter); 4893 rx_filter->va = dma_zalloc_coherent(&adapter->pdev->dev, 4894 rx_filter->size, &rx_filter->dma, 4895 GFP_KERNEL); 4896 if (!rx_filter->va) { 4897 status = -ENOMEM; 4898 goto free_mbox; 4899 } 4900 4901 mutex_init(&adapter->mbox_lock); 4902 spin_lock_init(&adapter->mcc_lock); 4903 spin_lock_init(&adapter->mcc_cq_lock); 4904 4905 init_completion(&adapter->et_cmd_compl); 4906 pci_save_state(adapter->pdev); 4907 return 0; 4908 4909 free_mbox: 4910 dma_free_coherent(&adapter->pdev->dev, mbox_mem_alloc->size, 4911 mbox_mem_alloc->va, mbox_mem_alloc->dma); 4912 4913 unmap_pci_bars: 4914 be_unmap_pci_bars(adapter); 4915 4916 done: 4917 return status; 4918 } 4919 4920 static void be_stats_cleanup(struct be_adapter *adapter) 4921 { 4922 struct be_dma_mem *cmd = &adapter->stats_cmd; 4923 4924 if (cmd->va) 4925 dma_free_coherent(&adapter->pdev->dev, cmd->size, 4926 cmd->va, cmd->dma); 4927 } 4928 4929 static int be_stats_init(struct be_adapter *adapter) 4930 { 4931 struct be_dma_mem *cmd = &adapter->stats_cmd; 4932 4933 if (lancer_chip(adapter)) 4934 cmd->size = sizeof(struct lancer_cmd_req_pport_stats); 4935 else if (BE2_chip(adapter)) 4936 cmd->size = sizeof(struct be_cmd_req_get_stats_v0); 4937 else if (BE3_chip(adapter)) 4938 cmd->size = sizeof(struct be_cmd_req_get_stats_v1); 4939 else 4940 /* ALL non-BE ASICs */ 4941 cmd->size = sizeof(struct be_cmd_req_get_stats_v2); 4942 4943 cmd->va = dma_zalloc_coherent(&adapter->pdev->dev, cmd->size, &cmd->dma, 4944 GFP_KERNEL); 4945 if (!cmd->va) 4946 return -ENOMEM; 4947 return 0; 4948 } 4949 4950 static void be_remove(struct pci_dev *pdev) 4951 { 4952 struct be_adapter *adapter = pci_get_drvdata(pdev); 4953 4954 if (!adapter) 4955 return; 4956 4957 be_roce_dev_remove(adapter); 4958 be_intr_set(adapter, false); 4959 4960 cancel_delayed_work_sync(&adapter->func_recovery_work); 4961 4962 unregister_netdev(adapter->netdev); 4963 4964 be_clear(adapter); 4965 4966 /* tell fw we're done with firing cmds */ 4967 be_cmd_fw_clean(adapter); 4968 4969 be_stats_cleanup(adapter); 4970 4971 be_ctrl_cleanup(adapter); 4972 4973 pci_disable_pcie_error_reporting(pdev); 4974 4975 pci_release_regions(pdev); 4976 pci_disable_device(pdev); 4977 4978 free_netdev(adapter->netdev); 4979 } 4980 4981 static int be_get_initial_config(struct be_adapter *adapter) 4982 { 4983 int status, level; 4984 4985 status = be_cmd_get_cntl_attributes(adapter); 4986 if (status) 4987 return status; 4988 4989 /* Must be a power of 2 or else MODULO will BUG_ON */ 4990 adapter->be_get_temp_freq = 64; 4991 4992 if (BEx_chip(adapter)) { 4993 level = be_cmd_get_fw_log_level(adapter); 4994 adapter->msg_enable = 4995 level <= FW_LOG_LEVEL_DEFAULT ? NETIF_MSG_HW : 0; 4996 } 4997 4998 adapter->cfg_num_qs = netif_get_num_default_rss_queues(); 4999 return 0; 5000 } 5001 5002 static int lancer_recover_func(struct be_adapter *adapter) 5003 { 5004 struct device *dev = &adapter->pdev->dev; 5005 int status; 5006 5007 status = lancer_test_and_set_rdy_state(adapter); 5008 if (status) 5009 goto err; 5010 5011 if (netif_running(adapter->netdev)) 5012 be_close(adapter->netdev); 5013 5014 be_clear(adapter); 5015 5016 be_clear_all_error(adapter); 5017 5018 status = be_setup(adapter); 5019 if (status) 5020 goto err; 5021 5022 if (netif_running(adapter->netdev)) { 5023 status = be_open(adapter->netdev); 5024 if (status) 5025 goto err; 5026 } 5027 5028 dev_err(dev, "Adapter recovery successful\n"); 5029 return 0; 5030 err: 5031 if (status == -EAGAIN) 5032 dev_err(dev, "Waiting for resource provisioning\n"); 5033 else 5034 dev_err(dev, "Adapter recovery failed\n"); 5035 5036 return status; 5037 } 5038 5039 static void be_func_recovery_task(struct work_struct *work) 5040 { 5041 struct be_adapter *adapter = 5042 container_of(work, struct be_adapter, func_recovery_work.work); 5043 int status = 0; 5044 5045 be_detect_error(adapter); 5046 5047 if (adapter->hw_error && lancer_chip(adapter)) { 5048 rtnl_lock(); 5049 netif_device_detach(adapter->netdev); 5050 rtnl_unlock(); 5051 5052 status = lancer_recover_func(adapter); 5053 if (!status) 5054 netif_device_attach(adapter->netdev); 5055 } 5056 5057 /* In Lancer, for all errors other than provisioning error (-EAGAIN), 5058 * no need to attempt further recovery. 5059 */ 5060 if (!status || status == -EAGAIN) 5061 schedule_delayed_work(&adapter->func_recovery_work, 5062 msecs_to_jiffies(1000)); 5063 } 5064 5065 static void be_log_sfp_info(struct be_adapter *adapter) 5066 { 5067 int status; 5068 5069 status = be_cmd_query_sfp_info(adapter); 5070 if (!status) { 5071 dev_err(&adapter->pdev->dev, 5072 "Unqualified SFP+ detected on %c from %s part no: %s", 5073 adapter->port_name, adapter->phy.vendor_name, 5074 adapter->phy.vendor_pn); 5075 } 5076 adapter->flags &= ~BE_FLAGS_EVT_INCOMPATIBLE_SFP; 5077 } 5078 5079 static void be_worker(struct work_struct *work) 5080 { 5081 struct be_adapter *adapter = 5082 container_of(work, struct be_adapter, work.work); 5083 struct be_rx_obj *rxo; 5084 int i; 5085 5086 /* when interrupts are not yet enabled, just reap any pending 5087 * mcc completions */ 5088 if (!netif_running(adapter->netdev)) { 5089 local_bh_disable(); 5090 be_process_mcc(adapter); 5091 local_bh_enable(); 5092 goto reschedule; 5093 } 5094 5095 if (!adapter->stats_cmd_sent) { 5096 if (lancer_chip(adapter)) 5097 lancer_cmd_get_pport_stats(adapter, 5098 &adapter->stats_cmd); 5099 else 5100 be_cmd_get_stats(adapter, &adapter->stats_cmd); 5101 } 5102 5103 if (be_physfn(adapter) && 5104 MODULO(adapter->work_counter, adapter->be_get_temp_freq) == 0) 5105 be_cmd_get_die_temperature(adapter); 5106 5107 for_all_rx_queues(adapter, rxo, i) { 5108 /* Replenish RX-queues starved due to memory 5109 * allocation failures. 5110 */ 5111 if (rxo->rx_post_starved) 5112 be_post_rx_frags(rxo, GFP_KERNEL, MAX_RX_POST); 5113 } 5114 5115 be_eqd_update(adapter); 5116 5117 if (adapter->flags & BE_FLAGS_EVT_INCOMPATIBLE_SFP) 5118 be_log_sfp_info(adapter); 5119 5120 reschedule: 5121 adapter->work_counter++; 5122 schedule_delayed_work(&adapter->work, msecs_to_jiffies(1000)); 5123 } 5124 5125 /* If any VFs are already enabled don't FLR the PF */ 5126 static bool be_reset_required(struct be_adapter *adapter) 5127 { 5128 return pci_num_vf(adapter->pdev) ? false : true; 5129 } 5130 5131 static char *mc_name(struct be_adapter *adapter) 5132 { 5133 char *str = ""; /* default */ 5134 5135 switch (adapter->mc_type) { 5136 case UMC: 5137 str = "UMC"; 5138 break; 5139 case FLEX10: 5140 str = "FLEX10"; 5141 break; 5142 case vNIC1: 5143 str = "vNIC-1"; 5144 break; 5145 case nPAR: 5146 str = "nPAR"; 5147 break; 5148 case UFP: 5149 str = "UFP"; 5150 break; 5151 case vNIC2: 5152 str = "vNIC-2"; 5153 break; 5154 default: 5155 str = ""; 5156 } 5157 5158 return str; 5159 } 5160 5161 static inline char *func_name(struct be_adapter *adapter) 5162 { 5163 return be_physfn(adapter) ? "PF" : "VF"; 5164 } 5165 5166 static inline char *nic_name(struct pci_dev *pdev) 5167 { 5168 switch (pdev->device) { 5169 case OC_DEVICE_ID1: 5170 return OC_NAME; 5171 case OC_DEVICE_ID2: 5172 return OC_NAME_BE; 5173 case OC_DEVICE_ID3: 5174 case OC_DEVICE_ID4: 5175 return OC_NAME_LANCER; 5176 case BE_DEVICE_ID2: 5177 return BE3_NAME; 5178 case OC_DEVICE_ID5: 5179 case OC_DEVICE_ID6: 5180 return OC_NAME_SH; 5181 default: 5182 return BE_NAME; 5183 } 5184 } 5185 5186 static int be_probe(struct pci_dev *pdev, const struct pci_device_id *pdev_id) 5187 { 5188 struct be_adapter *adapter; 5189 struct net_device *netdev; 5190 int status = 0; 5191 5192 dev_info(&pdev->dev, "%s version is %s\n", DRV_NAME, DRV_VER); 5193 5194 status = pci_enable_device(pdev); 5195 if (status) 5196 goto do_none; 5197 5198 status = pci_request_regions(pdev, DRV_NAME); 5199 if (status) 5200 goto disable_dev; 5201 pci_set_master(pdev); 5202 5203 netdev = alloc_etherdev_mqs(sizeof(*adapter), MAX_TX_QS, MAX_RX_QS); 5204 if (!netdev) { 5205 status = -ENOMEM; 5206 goto rel_reg; 5207 } 5208 adapter = netdev_priv(netdev); 5209 adapter->pdev = pdev; 5210 pci_set_drvdata(pdev, adapter); 5211 adapter->netdev = netdev; 5212 SET_NETDEV_DEV(netdev, &pdev->dev); 5213 5214 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 5215 if (!status) { 5216 netdev->features |= NETIF_F_HIGHDMA; 5217 } else { 5218 status = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 5219 if (status) { 5220 dev_err(&pdev->dev, "Could not set PCI DMA Mask\n"); 5221 goto free_netdev; 5222 } 5223 } 5224 5225 status = pci_enable_pcie_error_reporting(pdev); 5226 if (!status) 5227 dev_info(&pdev->dev, "PCIe error reporting enabled\n"); 5228 5229 status = be_ctrl_init(adapter); 5230 if (status) 5231 goto free_netdev; 5232 5233 /* sync up with fw's ready state */ 5234 if (be_physfn(adapter)) { 5235 status = be_fw_wait_ready(adapter); 5236 if (status) 5237 goto ctrl_clean; 5238 } 5239 5240 if (be_reset_required(adapter)) { 5241 status = be_cmd_reset_function(adapter); 5242 if (status) 5243 goto ctrl_clean; 5244 5245 /* Wait for interrupts to quiesce after an FLR */ 5246 msleep(100); 5247 } 5248 5249 /* Allow interrupts for other ULPs running on NIC function */ 5250 be_intr_set(adapter, true); 5251 5252 /* tell fw we're ready to fire cmds */ 5253 status = be_cmd_fw_init(adapter); 5254 if (status) 5255 goto ctrl_clean; 5256 5257 status = be_stats_init(adapter); 5258 if (status) 5259 goto ctrl_clean; 5260 5261 status = be_get_initial_config(adapter); 5262 if (status) 5263 goto stats_clean; 5264 5265 INIT_DELAYED_WORK(&adapter->work, be_worker); 5266 INIT_DELAYED_WORK(&adapter->func_recovery_work, be_func_recovery_task); 5267 adapter->rx_fc = true; 5268 adapter->tx_fc = true; 5269 5270 status = be_setup(adapter); 5271 if (status) 5272 goto stats_clean; 5273 5274 be_netdev_init(netdev); 5275 status = register_netdev(netdev); 5276 if (status != 0) 5277 goto unsetup; 5278 5279 be_roce_dev_add(adapter); 5280 5281 schedule_delayed_work(&adapter->func_recovery_work, 5282 msecs_to_jiffies(1000)); 5283 5284 dev_info(&pdev->dev, "%s: %s %s port %c\n", nic_name(pdev), 5285 func_name(adapter), mc_name(adapter), adapter->port_name); 5286 5287 return 0; 5288 5289 unsetup: 5290 be_clear(adapter); 5291 stats_clean: 5292 be_stats_cleanup(adapter); 5293 ctrl_clean: 5294 be_ctrl_cleanup(adapter); 5295 free_netdev: 5296 free_netdev(netdev); 5297 rel_reg: 5298 pci_release_regions(pdev); 5299 disable_dev: 5300 pci_disable_device(pdev); 5301 do_none: 5302 dev_err(&pdev->dev, "%s initialization failed\n", nic_name(pdev)); 5303 return status; 5304 } 5305 5306 static int be_suspend(struct pci_dev *pdev, pm_message_t state) 5307 { 5308 struct be_adapter *adapter = pci_get_drvdata(pdev); 5309 struct net_device *netdev = adapter->netdev; 5310 5311 if (adapter->wol_en) 5312 be_setup_wol(adapter, true); 5313 5314 be_intr_set(adapter, false); 5315 cancel_delayed_work_sync(&adapter->func_recovery_work); 5316 5317 netif_device_detach(netdev); 5318 if (netif_running(netdev)) { 5319 rtnl_lock(); 5320 be_close(netdev); 5321 rtnl_unlock(); 5322 } 5323 be_clear(adapter); 5324 5325 pci_save_state(pdev); 5326 pci_disable_device(pdev); 5327 pci_set_power_state(pdev, pci_choose_state(pdev, state)); 5328 return 0; 5329 } 5330 5331 static int be_resume(struct pci_dev *pdev) 5332 { 5333 int status = 0; 5334 struct be_adapter *adapter = pci_get_drvdata(pdev); 5335 struct net_device *netdev = adapter->netdev; 5336 5337 netif_device_detach(netdev); 5338 5339 status = pci_enable_device(pdev); 5340 if (status) 5341 return status; 5342 5343 pci_set_power_state(pdev, PCI_D0); 5344 pci_restore_state(pdev); 5345 5346 status = be_fw_wait_ready(adapter); 5347 if (status) 5348 return status; 5349 5350 status = be_cmd_reset_function(adapter); 5351 if (status) 5352 return status; 5353 5354 be_intr_set(adapter, true); 5355 /* tell fw we're ready to fire cmds */ 5356 status = be_cmd_fw_init(adapter); 5357 if (status) 5358 return status; 5359 5360 be_setup(adapter); 5361 if (netif_running(netdev)) { 5362 rtnl_lock(); 5363 be_open(netdev); 5364 rtnl_unlock(); 5365 } 5366 5367 schedule_delayed_work(&adapter->func_recovery_work, 5368 msecs_to_jiffies(1000)); 5369 netif_device_attach(netdev); 5370 5371 if (adapter->wol_en) 5372 be_setup_wol(adapter, false); 5373 5374 return 0; 5375 } 5376 5377 /* 5378 * An FLR will stop BE from DMAing any data. 5379 */ 5380 static void be_shutdown(struct pci_dev *pdev) 5381 { 5382 struct be_adapter *adapter = pci_get_drvdata(pdev); 5383 5384 if (!adapter) 5385 return; 5386 5387 be_roce_dev_shutdown(adapter); 5388 cancel_delayed_work_sync(&adapter->work); 5389 cancel_delayed_work_sync(&adapter->func_recovery_work); 5390 5391 netif_device_detach(adapter->netdev); 5392 5393 be_cmd_reset_function(adapter); 5394 5395 pci_disable_device(pdev); 5396 } 5397 5398 static pci_ers_result_t be_eeh_err_detected(struct pci_dev *pdev, 5399 pci_channel_state_t state) 5400 { 5401 struct be_adapter *adapter = pci_get_drvdata(pdev); 5402 struct net_device *netdev = adapter->netdev; 5403 5404 dev_err(&adapter->pdev->dev, "EEH error detected\n"); 5405 5406 if (!adapter->eeh_error) { 5407 adapter->eeh_error = true; 5408 5409 cancel_delayed_work_sync(&adapter->func_recovery_work); 5410 5411 rtnl_lock(); 5412 netif_device_detach(netdev); 5413 if (netif_running(netdev)) 5414 be_close(netdev); 5415 rtnl_unlock(); 5416 5417 be_clear(adapter); 5418 } 5419 5420 if (state == pci_channel_io_perm_failure) 5421 return PCI_ERS_RESULT_DISCONNECT; 5422 5423 pci_disable_device(pdev); 5424 5425 /* The error could cause the FW to trigger a flash debug dump. 5426 * Resetting the card while flash dump is in progress 5427 * can cause it not to recover; wait for it to finish. 5428 * Wait only for first function as it is needed only once per 5429 * adapter. 5430 */ 5431 if (pdev->devfn == 0) 5432 ssleep(30); 5433 5434 return PCI_ERS_RESULT_NEED_RESET; 5435 } 5436 5437 static pci_ers_result_t be_eeh_reset(struct pci_dev *pdev) 5438 { 5439 struct be_adapter *adapter = pci_get_drvdata(pdev); 5440 int status; 5441 5442 dev_info(&adapter->pdev->dev, "EEH reset\n"); 5443 5444 status = pci_enable_device(pdev); 5445 if (status) 5446 return PCI_ERS_RESULT_DISCONNECT; 5447 5448 pci_set_master(pdev); 5449 pci_set_power_state(pdev, PCI_D0); 5450 pci_restore_state(pdev); 5451 5452 /* Check if card is ok and fw is ready */ 5453 dev_info(&adapter->pdev->dev, 5454 "Waiting for FW to be ready after EEH reset\n"); 5455 status = be_fw_wait_ready(adapter); 5456 if (status) 5457 return PCI_ERS_RESULT_DISCONNECT; 5458 5459 pci_cleanup_aer_uncorrect_error_status(pdev); 5460 be_clear_all_error(adapter); 5461 return PCI_ERS_RESULT_RECOVERED; 5462 } 5463 5464 static void be_eeh_resume(struct pci_dev *pdev) 5465 { 5466 int status = 0; 5467 struct be_adapter *adapter = pci_get_drvdata(pdev); 5468 struct net_device *netdev = adapter->netdev; 5469 5470 dev_info(&adapter->pdev->dev, "EEH resume\n"); 5471 5472 pci_save_state(pdev); 5473 5474 status = be_cmd_reset_function(adapter); 5475 if (status) 5476 goto err; 5477 5478 /* On some BE3 FW versions, after a HW reset, 5479 * interrupts will remain disabled for each function. 5480 * So, explicitly enable interrupts 5481 */ 5482 be_intr_set(adapter, true); 5483 5484 /* tell fw we're ready to fire cmds */ 5485 status = be_cmd_fw_init(adapter); 5486 if (status) 5487 goto err; 5488 5489 status = be_setup(adapter); 5490 if (status) 5491 goto err; 5492 5493 if (netif_running(netdev)) { 5494 status = be_open(netdev); 5495 if (status) 5496 goto err; 5497 } 5498 5499 schedule_delayed_work(&adapter->func_recovery_work, 5500 msecs_to_jiffies(1000)); 5501 netif_device_attach(netdev); 5502 return; 5503 err: 5504 dev_err(&adapter->pdev->dev, "EEH resume failed\n"); 5505 } 5506 5507 static const struct pci_error_handlers be_eeh_handlers = { 5508 .error_detected = be_eeh_err_detected, 5509 .slot_reset = be_eeh_reset, 5510 .resume = be_eeh_resume, 5511 }; 5512 5513 static struct pci_driver be_driver = { 5514 .name = DRV_NAME, 5515 .id_table = be_dev_ids, 5516 .probe = be_probe, 5517 .remove = be_remove, 5518 .suspend = be_suspend, 5519 .resume = be_resume, 5520 .shutdown = be_shutdown, 5521 .err_handler = &be_eeh_handlers 5522 }; 5523 5524 static int __init be_init_module(void) 5525 { 5526 if (rx_frag_size != 8192 && rx_frag_size != 4096 && 5527 rx_frag_size != 2048) { 5528 printk(KERN_WARNING DRV_NAME 5529 " : Module param rx_frag_size must be 2048/4096/8192." 5530 " Using 2048\n"); 5531 rx_frag_size = 2048; 5532 } 5533 5534 return pci_register_driver(&be_driver); 5535 } 5536 module_init(be_init_module); 5537 5538 static void __exit be_exit_module(void) 5539 { 5540 pci_unregister_driver(&be_driver); 5541 } 5542 module_exit(be_exit_module); 5543