1 /* 2 * This file is part of the Chelsio T4 Ethernet driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/bitmap.h> 38 #include <linux/crc32.h> 39 #include <linux/ctype.h> 40 #include <linux/debugfs.h> 41 #include <linux/err.h> 42 #include <linux/etherdevice.h> 43 #include <linux/firmware.h> 44 #include <linux/if.h> 45 #include <linux/if_vlan.h> 46 #include <linux/init.h> 47 #include <linux/log2.h> 48 #include <linux/mdio.h> 49 #include <linux/module.h> 50 #include <linux/moduleparam.h> 51 #include <linux/mutex.h> 52 #include <linux/netdevice.h> 53 #include <linux/pci.h> 54 #include <linux/aer.h> 55 #include <linux/rtnetlink.h> 56 #include <linux/sched.h> 57 #include <linux/seq_file.h> 58 #include <linux/sockios.h> 59 #include <linux/vmalloc.h> 60 #include <linux/workqueue.h> 61 #include <net/neighbour.h> 62 #include <net/netevent.h> 63 #include <net/addrconf.h> 64 #include <net/bonding.h> 65 #include <net/addrconf.h> 66 #include <linux/uaccess.h> 67 #include <linux/crash_dump.h> 68 #include <net/udp_tunnel.h> 69 70 #include "cxgb4.h" 71 #include "cxgb4_filter.h" 72 #include "t4_regs.h" 73 #include "t4_values.h" 74 #include "t4_msg.h" 75 #include "t4fw_api.h" 76 #include "t4fw_version.h" 77 #include "cxgb4_dcb.h" 78 #include "cxgb4_debugfs.h" 79 #include "clip_tbl.h" 80 #include "l2t.h" 81 #include "smt.h" 82 #include "sched.h" 83 #include "cxgb4_tc_u32.h" 84 #include "cxgb4_tc_flower.h" 85 #include "cxgb4_ptp.h" 86 #include "cxgb4_cudbg.h" 87 88 char cxgb4_driver_name[] = KBUILD_MODNAME; 89 90 #ifdef DRV_VERSION 91 #undef DRV_VERSION 92 #endif 93 #define DRV_VERSION "2.0.0-ko" 94 const char cxgb4_driver_version[] = DRV_VERSION; 95 #define DRV_DESC "Chelsio T4/T5/T6 Network Driver" 96 97 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ 98 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ 99 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) 100 101 /* Macros needed to support the PCI Device ID Table ... 102 */ 103 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ 104 static const struct pci_device_id cxgb4_pci_tbl[] = { 105 #define CXGB4_UNIFIED_PF 0x4 106 107 #define CH_PCI_DEVICE_ID_FUNCTION CXGB4_UNIFIED_PF 108 109 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is 110 * called for both. 111 */ 112 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0 113 114 #define CH_PCI_ID_TABLE_ENTRY(devid) \ 115 {PCI_VDEVICE(CHELSIO, (devid)), CXGB4_UNIFIED_PF} 116 117 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ 118 { 0, } \ 119 } 120 121 #include "t4_pci_id_tbl.h" 122 123 #define FW4_FNAME "cxgb4/t4fw.bin" 124 #define FW5_FNAME "cxgb4/t5fw.bin" 125 #define FW6_FNAME "cxgb4/t6fw.bin" 126 #define FW4_CFNAME "cxgb4/t4-config.txt" 127 #define FW5_CFNAME "cxgb4/t5-config.txt" 128 #define FW6_CFNAME "cxgb4/t6-config.txt" 129 #define PHY_AQ1202_FIRMWARE "cxgb4/aq1202_fw.cld" 130 #define PHY_BCM84834_FIRMWARE "cxgb4/bcm8483.bin" 131 #define PHY_AQ1202_DEVICEID 0x4409 132 #define PHY_BCM84834_DEVICEID 0x4486 133 134 MODULE_DESCRIPTION(DRV_DESC); 135 MODULE_AUTHOR("Chelsio Communications"); 136 MODULE_LICENSE("Dual BSD/GPL"); 137 MODULE_VERSION(DRV_VERSION); 138 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl); 139 MODULE_FIRMWARE(FW4_FNAME); 140 MODULE_FIRMWARE(FW5_FNAME); 141 MODULE_FIRMWARE(FW6_FNAME); 142 143 /* 144 * The driver uses the best interrupt scheme available on a platform in the 145 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which 146 * of these schemes the driver may consider as follows: 147 * 148 * msi = 2: choose from among all three options 149 * msi = 1: only consider MSI and INTx interrupts 150 * msi = 0: force INTx interrupts 151 */ 152 static int msi = 2; 153 154 module_param(msi, int, 0644); 155 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)"); 156 157 /* 158 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers 159 * offset by 2 bytes in order to have the IP headers line up on 4-byte 160 * boundaries. This is a requirement for many architectures which will throw 161 * a machine check fault if an attempt is made to access one of the 4-byte IP 162 * header fields on a non-4-byte boundary. And it's a major performance issue 163 * even on some architectures which allow it like some implementations of the 164 * x86 ISA. However, some architectures don't mind this and for some very 165 * edge-case performance sensitive applications (like forwarding large volumes 166 * of small packets), setting this DMA offset to 0 will decrease the number of 167 * PCI-E Bus transfers enough to measurably affect performance. 168 */ 169 static int rx_dma_offset = 2; 170 171 /* TX Queue select used to determine what algorithm to use for selecting TX 172 * queue. Select between the kernel provided function (select_queue=0) or user 173 * cxgb_select_queue function (select_queue=1) 174 * 175 * Default: select_queue=0 176 */ 177 static int select_queue; 178 module_param(select_queue, int, 0644); 179 MODULE_PARM_DESC(select_queue, 180 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method."); 181 182 static struct dentry *cxgb4_debugfs_root; 183 184 LIST_HEAD(adapter_list); 185 DEFINE_MUTEX(uld_mutex); 186 187 static void link_report(struct net_device *dev) 188 { 189 if (!netif_carrier_ok(dev)) 190 netdev_info(dev, "link down\n"); 191 else { 192 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" }; 193 194 const char *s; 195 const struct port_info *p = netdev_priv(dev); 196 197 switch (p->link_cfg.speed) { 198 case 100: 199 s = "100Mbps"; 200 break; 201 case 1000: 202 s = "1Gbps"; 203 break; 204 case 10000: 205 s = "10Gbps"; 206 break; 207 case 25000: 208 s = "25Gbps"; 209 break; 210 case 40000: 211 s = "40Gbps"; 212 break; 213 case 100000: 214 s = "100Gbps"; 215 break; 216 default: 217 pr_info("%s: unsupported speed: %d\n", 218 dev->name, p->link_cfg.speed); 219 return; 220 } 221 222 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, 223 fc[p->link_cfg.fc]); 224 } 225 } 226 227 #ifdef CONFIG_CHELSIO_T4_DCB 228 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */ 229 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable) 230 { 231 struct port_info *pi = netdev_priv(dev); 232 struct adapter *adap = pi->adapter; 233 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; 234 int i; 235 236 /* We use a simple mapping of Port TX Queue Index to DCB 237 * Priority when we're enabling DCB. 238 */ 239 for (i = 0; i < pi->nqsets; i++, txq++) { 240 u32 name, value; 241 int err; 242 243 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 244 FW_PARAMS_PARAM_X_V( 245 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) | 246 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); 247 value = enable ? i : 0xffffffff; 248 249 /* Since we can be called while atomic (from "interrupt 250 * level") we need to issue the Set Parameters Commannd 251 * without sleeping (timeout < 0). 252 */ 253 err = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1, 254 &name, &value, 255 -FW_CMD_MAX_TIMEOUT); 256 257 if (err) 258 dev_err(adap->pdev_dev, 259 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n", 260 enable ? "set" : "unset", pi->port_id, i, -err); 261 else 262 txq->dcb_prio = value; 263 } 264 } 265 266 static int cxgb4_dcb_enabled(const struct net_device *dev) 267 { 268 struct port_info *pi = netdev_priv(dev); 269 270 if (!pi->dcb.enabled) 271 return 0; 272 273 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) || 274 (pi->dcb.state == CXGB4_DCB_STATE_HOST)); 275 } 276 #endif /* CONFIG_CHELSIO_T4_DCB */ 277 278 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat) 279 { 280 struct net_device *dev = adapter->port[port_id]; 281 282 /* Skip changes from disabled ports. */ 283 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) { 284 if (link_stat) 285 netif_carrier_on(dev); 286 else { 287 #ifdef CONFIG_CHELSIO_T4_DCB 288 if (cxgb4_dcb_enabled(dev)) { 289 cxgb4_dcb_reset(dev); 290 dcb_tx_queue_prio_enable(dev, false); 291 } 292 #endif /* CONFIG_CHELSIO_T4_DCB */ 293 netif_carrier_off(dev); 294 } 295 296 link_report(dev); 297 } 298 } 299 300 void t4_os_portmod_changed(const struct adapter *adap, int port_id) 301 { 302 static const char *mod_str[] = { 303 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" 304 }; 305 306 const struct net_device *dev = adap->port[port_id]; 307 const struct port_info *pi = netdev_priv(dev); 308 309 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 310 netdev_info(dev, "port module unplugged\n"); 311 else if (pi->mod_type < ARRAY_SIZE(mod_str)) 312 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]); 313 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 314 netdev_info(dev, "%s: unsupported port module inserted\n", 315 dev->name); 316 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 317 netdev_info(dev, "%s: unknown port module inserted\n", 318 dev->name); 319 else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) 320 netdev_info(dev, "%s: transceiver module error\n", dev->name); 321 else 322 netdev_info(dev, "%s: unknown module type %d inserted\n", 323 dev->name, pi->mod_type); 324 } 325 326 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ 327 module_param(dbfifo_int_thresh, int, 0644); 328 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); 329 330 /* 331 * usecs to sleep while draining the dbfifo 332 */ 333 static int dbfifo_drain_delay = 1000; 334 module_param(dbfifo_drain_delay, int, 0644); 335 MODULE_PARM_DESC(dbfifo_drain_delay, 336 "usecs to sleep while draining the dbfifo"); 337 338 static inline int cxgb4_set_addr_hash(struct port_info *pi) 339 { 340 struct adapter *adap = pi->adapter; 341 u64 vec = 0; 342 bool ucast = false; 343 struct hash_mac_addr *entry; 344 345 /* Calculate the hash vector for the updated list and program it */ 346 list_for_each_entry(entry, &adap->mac_hlist, list) { 347 ucast |= is_unicast_ether_addr(entry->addr); 348 vec |= (1ULL << hash_mac_addr(entry->addr)); 349 } 350 return t4_set_addr_hash(adap, adap->mbox, pi->viid, ucast, 351 vec, false); 352 } 353 354 static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr) 355 { 356 struct port_info *pi = netdev_priv(netdev); 357 struct adapter *adap = pi->adapter; 358 int ret; 359 u64 mhash = 0; 360 u64 uhash = 0; 361 bool free = false; 362 bool ucast = is_unicast_ether_addr(mac_addr); 363 const u8 *maclist[1] = {mac_addr}; 364 struct hash_mac_addr *new_entry; 365 366 ret = t4_alloc_mac_filt(adap, adap->mbox, pi->viid, free, 1, maclist, 367 NULL, ucast ? &uhash : &mhash, false); 368 if (ret < 0) 369 goto out; 370 /* if hash != 0, then add the addr to hash addr list 371 * so on the end we will calculate the hash for the 372 * list and program it 373 */ 374 if (uhash || mhash) { 375 new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC); 376 if (!new_entry) 377 return -ENOMEM; 378 ether_addr_copy(new_entry->addr, mac_addr); 379 list_add_tail(&new_entry->list, &adap->mac_hlist); 380 ret = cxgb4_set_addr_hash(pi); 381 } 382 out: 383 return ret < 0 ? ret : 0; 384 } 385 386 static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr) 387 { 388 struct port_info *pi = netdev_priv(netdev); 389 struct adapter *adap = pi->adapter; 390 int ret; 391 const u8 *maclist[1] = {mac_addr}; 392 struct hash_mac_addr *entry, *tmp; 393 394 /* If the MAC address to be removed is in the hash addr 395 * list, delete it from the list and update hash vector 396 */ 397 list_for_each_entry_safe(entry, tmp, &adap->mac_hlist, list) { 398 if (ether_addr_equal(entry->addr, mac_addr)) { 399 list_del(&entry->list); 400 kfree(entry); 401 return cxgb4_set_addr_hash(pi); 402 } 403 } 404 405 ret = t4_free_mac_filt(adap, adap->mbox, pi->viid, 1, maclist, false); 406 return ret < 0 ? -EINVAL : 0; 407 } 408 409 /* 410 * Set Rx properties of a port, such as promiscruity, address filters, and MTU. 411 * If @mtu is -1 it is left unchanged. 412 */ 413 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) 414 { 415 struct port_info *pi = netdev_priv(dev); 416 struct adapter *adapter = pi->adapter; 417 418 __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 419 __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 420 421 return t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, 422 (dev->flags & IFF_PROMISC) ? 1 : 0, 423 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1, 424 sleep_ok); 425 } 426 427 /** 428 * link_start - enable a port 429 * @dev: the port to enable 430 * 431 * Performs the MAC and PHY actions needed to enable a port. 432 */ 433 static int link_start(struct net_device *dev) 434 { 435 int ret; 436 struct port_info *pi = netdev_priv(dev); 437 unsigned int mb = pi->adapter->pf; 438 439 /* 440 * We do not set address filters and promiscuity here, the stack does 441 * that step explicitly. 442 */ 443 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1, 444 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); 445 if (ret == 0) { 446 ret = t4_change_mac(pi->adapter, mb, pi->viid, 447 pi->xact_addr_filt, dev->dev_addr, true, 448 true); 449 if (ret >= 0) { 450 pi->xact_addr_filt = ret; 451 ret = 0; 452 } 453 } 454 if (ret == 0) 455 ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan, 456 &pi->link_cfg); 457 if (ret == 0) { 458 local_bh_disable(); 459 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true, 460 true, CXGB4_DCB_ENABLED); 461 local_bh_enable(); 462 } 463 464 return ret; 465 } 466 467 #ifdef CONFIG_CHELSIO_T4_DCB 468 /* Handle a Data Center Bridging update message from the firmware. */ 469 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd) 470 { 471 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid)); 472 struct net_device *dev = adap->port[adap->chan_map[port]]; 473 int old_dcb_enabled = cxgb4_dcb_enabled(dev); 474 int new_dcb_enabled; 475 476 cxgb4_dcb_handle_fw_update(adap, pcmd); 477 new_dcb_enabled = cxgb4_dcb_enabled(dev); 478 479 /* If the DCB has become enabled or disabled on the port then we're 480 * going to need to set up/tear down DCB Priority parameters for the 481 * TX Queues associated with the port. 482 */ 483 if (new_dcb_enabled != old_dcb_enabled) 484 dcb_tx_queue_prio_enable(dev, new_dcb_enabled); 485 } 486 #endif /* CONFIG_CHELSIO_T4_DCB */ 487 488 /* Response queue handler for the FW event queue. 489 */ 490 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, 491 const struct pkt_gl *gl) 492 { 493 u8 opcode = ((const struct rss_header *)rsp)->opcode; 494 495 rsp++; /* skip RSS header */ 496 497 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. 498 */ 499 if (unlikely(opcode == CPL_FW4_MSG && 500 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) { 501 rsp++; 502 opcode = ((const struct rss_header *)rsp)->opcode; 503 rsp++; 504 if (opcode != CPL_SGE_EGR_UPDATE) { 505 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" 506 , opcode); 507 goto out; 508 } 509 } 510 511 if (likely(opcode == CPL_SGE_EGR_UPDATE)) { 512 const struct cpl_sge_egr_update *p = (void *)rsp; 513 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid)); 514 struct sge_txq *txq; 515 516 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start]; 517 txq->restarts++; 518 if (txq->q_type == CXGB4_TXQ_ETH) { 519 struct sge_eth_txq *eq; 520 521 eq = container_of(txq, struct sge_eth_txq, q); 522 netif_tx_wake_queue(eq->txq); 523 } else { 524 struct sge_uld_txq *oq; 525 526 oq = container_of(txq, struct sge_uld_txq, q); 527 tasklet_schedule(&oq->qresume_tsk); 528 } 529 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { 530 const struct cpl_fw6_msg *p = (void *)rsp; 531 532 #ifdef CONFIG_CHELSIO_T4_DCB 533 const struct fw_port_cmd *pcmd = (const void *)p->data; 534 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid)); 535 unsigned int action = 536 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16)); 537 538 if (cmd == FW_PORT_CMD && 539 (action == FW_PORT_ACTION_GET_PORT_INFO || 540 action == FW_PORT_ACTION_GET_PORT_INFO32)) { 541 int port = FW_PORT_CMD_PORTID_G( 542 be32_to_cpu(pcmd->op_to_portid)); 543 struct net_device *dev; 544 int dcbxdis, state_input; 545 546 dev = q->adap->port[q->adap->chan_map[port]]; 547 dcbxdis = (action == FW_PORT_ACTION_GET_PORT_INFO 548 ? !!(pcmd->u.info.dcbxdis_pkd & 549 FW_PORT_CMD_DCBXDIS_F) 550 : !!(pcmd->u.info32.lstatus32_to_cbllen32 & 551 FW_PORT_CMD_DCBXDIS32_F)); 552 state_input = (dcbxdis 553 ? CXGB4_DCB_INPUT_FW_DISABLED 554 : CXGB4_DCB_INPUT_FW_ENABLED); 555 556 cxgb4_dcb_state_fsm(dev, state_input); 557 } 558 559 if (cmd == FW_PORT_CMD && 560 action == FW_PORT_ACTION_L2_DCB_CFG) 561 dcb_rpl(q->adap, pcmd); 562 else 563 #endif 564 if (p->type == 0) 565 t4_handle_fw_rpl(q->adap, p->data); 566 } else if (opcode == CPL_L2T_WRITE_RPL) { 567 const struct cpl_l2t_write_rpl *p = (void *)rsp; 568 569 do_l2t_write_rpl(q->adap, p); 570 } else if (opcode == CPL_SMT_WRITE_RPL) { 571 const struct cpl_smt_write_rpl *p = (void *)rsp; 572 573 do_smt_write_rpl(q->adap, p); 574 } else if (opcode == CPL_SET_TCB_RPL) { 575 const struct cpl_set_tcb_rpl *p = (void *)rsp; 576 577 filter_rpl(q->adap, p); 578 } else if (opcode == CPL_ACT_OPEN_RPL) { 579 const struct cpl_act_open_rpl *p = (void *)rsp; 580 581 hash_filter_rpl(q->adap, p); 582 } else if (opcode == CPL_ABORT_RPL_RSS) { 583 const struct cpl_abort_rpl_rss *p = (void *)rsp; 584 585 hash_del_filter_rpl(q->adap, p); 586 } else 587 dev_err(q->adap->pdev_dev, 588 "unexpected CPL %#x on FW event queue\n", opcode); 589 out: 590 return 0; 591 } 592 593 static void disable_msi(struct adapter *adapter) 594 { 595 if (adapter->flags & USING_MSIX) { 596 pci_disable_msix(adapter->pdev); 597 adapter->flags &= ~USING_MSIX; 598 } else if (adapter->flags & USING_MSI) { 599 pci_disable_msi(adapter->pdev); 600 adapter->flags &= ~USING_MSI; 601 } 602 } 603 604 /* 605 * Interrupt handler for non-data events used with MSI-X. 606 */ 607 static irqreturn_t t4_nondata_intr(int irq, void *cookie) 608 { 609 struct adapter *adap = cookie; 610 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A)); 611 612 if (v & PFSW_F) { 613 adap->swintr = 1; 614 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v); 615 } 616 if (adap->flags & MASTER_PF) 617 t4_slow_intr_handler(adap); 618 return IRQ_HANDLED; 619 } 620 621 /* 622 * Name the MSI-X interrupts. 623 */ 624 static void name_msix_vecs(struct adapter *adap) 625 { 626 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc); 627 628 /* non-data interrupts */ 629 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name); 630 631 /* FW events */ 632 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", 633 adap->port[0]->name); 634 635 /* Ethernet queues */ 636 for_each_port(adap, j) { 637 struct net_device *d = adap->port[j]; 638 const struct port_info *pi = netdev_priv(d); 639 640 for (i = 0; i < pi->nqsets; i++, msi_idx++) 641 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d", 642 d->name, i); 643 } 644 } 645 646 static int request_msix_queue_irqs(struct adapter *adap) 647 { 648 struct sge *s = &adap->sge; 649 int err, ethqidx; 650 int msi_index = 2; 651 652 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0, 653 adap->msix_info[1].desc, &s->fw_evtq); 654 if (err) 655 return err; 656 657 for_each_ethrxq(s, ethqidx) { 658 err = request_irq(adap->msix_info[msi_index].vec, 659 t4_sge_intr_msix, 0, 660 adap->msix_info[msi_index].desc, 661 &s->ethrxq[ethqidx].rspq); 662 if (err) 663 goto unwind; 664 msi_index++; 665 } 666 return 0; 667 668 unwind: 669 while (--ethqidx >= 0) 670 free_irq(adap->msix_info[--msi_index].vec, 671 &s->ethrxq[ethqidx].rspq); 672 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 673 return err; 674 } 675 676 static void free_msix_queue_irqs(struct adapter *adap) 677 { 678 int i, msi_index = 2; 679 struct sge *s = &adap->sge; 680 681 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 682 for_each_ethrxq(s, i) 683 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq); 684 } 685 686 /** 687 * cxgb4_write_rss - write the RSS table for a given port 688 * @pi: the port 689 * @queues: array of queue indices for RSS 690 * 691 * Sets up the portion of the HW RSS table for the port's VI to distribute 692 * packets to the Rx queues in @queues. 693 * Should never be called before setting up sge eth rx queues 694 */ 695 int cxgb4_write_rss(const struct port_info *pi, const u16 *queues) 696 { 697 u16 *rss; 698 int i, err; 699 struct adapter *adapter = pi->adapter; 700 const struct sge_eth_rxq *rxq; 701 702 rxq = &adapter->sge.ethrxq[pi->first_qset]; 703 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL); 704 if (!rss) 705 return -ENOMEM; 706 707 /* map the queue indices to queue ids */ 708 for (i = 0; i < pi->rss_size; i++, queues++) 709 rss[i] = rxq[*queues].rspq.abs_id; 710 711 err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0, 712 pi->rss_size, rss, pi->rss_size); 713 /* If Tunnel All Lookup isn't specified in the global RSS 714 * Configuration, then we need to specify a default Ingress 715 * Queue for any ingress packets which aren't hashed. We'll 716 * use our first ingress queue ... 717 */ 718 if (!err) 719 err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid, 720 FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F | 721 FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F | 722 FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F | 723 FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F | 724 FW_RSS_VI_CONFIG_CMD_UDPEN_F, 725 rss[0]); 726 kfree(rss); 727 return err; 728 } 729 730 /** 731 * setup_rss - configure RSS 732 * @adap: the adapter 733 * 734 * Sets up RSS for each port. 735 */ 736 static int setup_rss(struct adapter *adap) 737 { 738 int i, j, err; 739 740 for_each_port(adap, i) { 741 const struct port_info *pi = adap2pinfo(adap, i); 742 743 /* Fill default values with equal distribution */ 744 for (j = 0; j < pi->rss_size; j++) 745 pi->rss[j] = j % pi->nqsets; 746 747 err = cxgb4_write_rss(pi, pi->rss); 748 if (err) 749 return err; 750 } 751 return 0; 752 } 753 754 /* 755 * Return the channel of the ingress queue with the given qid. 756 */ 757 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid) 758 { 759 qid -= p->ingr_start; 760 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan; 761 } 762 763 /* 764 * Wait until all NAPI handlers are descheduled. 765 */ 766 static void quiesce_rx(struct adapter *adap) 767 { 768 int i; 769 770 for (i = 0; i < adap->sge.ingr_sz; i++) { 771 struct sge_rspq *q = adap->sge.ingr_map[i]; 772 773 if (q && q->handler) 774 napi_disable(&q->napi); 775 } 776 } 777 778 /* Disable interrupt and napi handler */ 779 static void disable_interrupts(struct adapter *adap) 780 { 781 if (adap->flags & FULL_INIT_DONE) { 782 t4_intr_disable(adap); 783 if (adap->flags & USING_MSIX) { 784 free_msix_queue_irqs(adap); 785 free_irq(adap->msix_info[0].vec, adap); 786 } else { 787 free_irq(adap->pdev->irq, adap); 788 } 789 quiesce_rx(adap); 790 } 791 } 792 793 /* 794 * Enable NAPI scheduling and interrupt generation for all Rx queues. 795 */ 796 static void enable_rx(struct adapter *adap) 797 { 798 int i; 799 800 for (i = 0; i < adap->sge.ingr_sz; i++) { 801 struct sge_rspq *q = adap->sge.ingr_map[i]; 802 803 if (!q) 804 continue; 805 if (q->handler) 806 napi_enable(&q->napi); 807 808 /* 0-increment GTS to start the timer and enable interrupts */ 809 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), 810 SEINTARM_V(q->intr_params) | 811 INGRESSQID_V(q->cntxt_id)); 812 } 813 } 814 815 816 static int setup_fw_sge_queues(struct adapter *adap) 817 { 818 struct sge *s = &adap->sge; 819 int err = 0; 820 821 bitmap_zero(s->starving_fl, s->egr_sz); 822 bitmap_zero(s->txq_maperr, s->egr_sz); 823 824 if (adap->flags & USING_MSIX) 825 adap->msi_idx = 1; /* vector 0 is for non-queue interrupts */ 826 else { 827 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0, 828 NULL, NULL, NULL, -1); 829 if (err) 830 return err; 831 adap->msi_idx = -((int)s->intrq.abs_id + 1); 832 } 833 834 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0], 835 adap->msi_idx, NULL, fwevtq_handler, NULL, -1); 836 if (err) 837 t4_free_sge_resources(adap); 838 return err; 839 } 840 841 /** 842 * setup_sge_queues - configure SGE Tx/Rx/response queues 843 * @adap: the adapter 844 * 845 * Determines how many sets of SGE queues to use and initializes them. 846 * We support multiple queue sets per port if we have MSI-X, otherwise 847 * just one queue set per port. 848 */ 849 static int setup_sge_queues(struct adapter *adap) 850 { 851 int err, i, j; 852 struct sge *s = &adap->sge; 853 struct sge_uld_rxq_info *rxq_info = NULL; 854 unsigned int cmplqid = 0; 855 856 if (is_uld(adap)) 857 rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA]; 858 859 for_each_port(adap, i) { 860 struct net_device *dev = adap->port[i]; 861 struct port_info *pi = netdev_priv(dev); 862 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset]; 863 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset]; 864 865 for (j = 0; j < pi->nqsets; j++, q++) { 866 if (adap->msi_idx > 0) 867 adap->msi_idx++; 868 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, 869 adap->msi_idx, &q->fl, 870 t4_ethrx_handler, 871 NULL, 872 t4_get_tp_ch_map(adap, 873 pi->tx_chan)); 874 if (err) 875 goto freeout; 876 q->rspq.idx = j; 877 memset(&q->stats, 0, sizeof(q->stats)); 878 } 879 for (j = 0; j < pi->nqsets; j++, t++) { 880 err = t4_sge_alloc_eth_txq(adap, t, dev, 881 netdev_get_tx_queue(dev, j), 882 s->fw_evtq.cntxt_id); 883 if (err) 884 goto freeout; 885 } 886 } 887 888 for_each_port(adap, i) { 889 /* Note that cmplqid below is 0 if we don't 890 * have RDMA queues, and that's the right value. 891 */ 892 if (rxq_info) 893 cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id; 894 895 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i], 896 s->fw_evtq.cntxt_id, cmplqid); 897 if (err) 898 goto freeout; 899 } 900 901 if (!is_t4(adap->params.chip)) { 902 err = t4_sge_alloc_eth_txq(adap, &s->ptptxq, adap->port[0], 903 netdev_get_tx_queue(adap->port[0], 0) 904 , s->fw_evtq.cntxt_id); 905 if (err) 906 goto freeout; 907 } 908 909 t4_write_reg(adap, is_t4(adap->params.chip) ? 910 MPS_TRC_RSS_CONTROL_A : 911 MPS_T5_TRC_RSS_CONTROL_A, 912 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) | 913 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id)); 914 return 0; 915 freeout: 916 t4_free_sge_resources(adap); 917 return err; 918 } 919 920 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb, 921 void *accel_priv, select_queue_fallback_t fallback) 922 { 923 int txq; 924 925 #ifdef CONFIG_CHELSIO_T4_DCB 926 /* If a Data Center Bridging has been successfully negotiated on this 927 * link then we'll use the skb's priority to map it to a TX Queue. 928 * The skb's priority is determined via the VLAN Tag Priority Code 929 * Point field. 930 */ 931 if (cxgb4_dcb_enabled(dev) && !is_kdump_kernel()) { 932 u16 vlan_tci; 933 int err; 934 935 err = vlan_get_tag(skb, &vlan_tci); 936 if (unlikely(err)) { 937 if (net_ratelimit()) 938 netdev_warn(dev, 939 "TX Packet without VLAN Tag on DCB Link\n"); 940 txq = 0; 941 } else { 942 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 943 #ifdef CONFIG_CHELSIO_T4_FCOE 944 if (skb->protocol == htons(ETH_P_FCOE)) 945 txq = skb->priority & 0x7; 946 #endif /* CONFIG_CHELSIO_T4_FCOE */ 947 } 948 return txq; 949 } 950 #endif /* CONFIG_CHELSIO_T4_DCB */ 951 952 if (select_queue) { 953 txq = (skb_rx_queue_recorded(skb) 954 ? skb_get_rx_queue(skb) 955 : smp_processor_id()); 956 957 while (unlikely(txq >= dev->real_num_tx_queues)) 958 txq -= dev->real_num_tx_queues; 959 960 return txq; 961 } 962 963 return fallback(dev, skb) % dev->real_num_tx_queues; 964 } 965 966 static int closest_timer(const struct sge *s, int time) 967 { 968 int i, delta, match = 0, min_delta = INT_MAX; 969 970 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 971 delta = time - s->timer_val[i]; 972 if (delta < 0) 973 delta = -delta; 974 if (delta < min_delta) { 975 min_delta = delta; 976 match = i; 977 } 978 } 979 return match; 980 } 981 982 static int closest_thres(const struct sge *s, int thres) 983 { 984 int i, delta, match = 0, min_delta = INT_MAX; 985 986 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 987 delta = thres - s->counter_val[i]; 988 if (delta < 0) 989 delta = -delta; 990 if (delta < min_delta) { 991 min_delta = delta; 992 match = i; 993 } 994 } 995 return match; 996 } 997 998 /** 999 * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters 1000 * @q: the Rx queue 1001 * @us: the hold-off time in us, or 0 to disable timer 1002 * @cnt: the hold-off packet count, or 0 to disable counter 1003 * 1004 * Sets an Rx queue's interrupt hold-off time and packet count. At least 1005 * one of the two needs to be enabled for the queue to generate interrupts. 1006 */ 1007 int cxgb4_set_rspq_intr_params(struct sge_rspq *q, 1008 unsigned int us, unsigned int cnt) 1009 { 1010 struct adapter *adap = q->adap; 1011 1012 if ((us | cnt) == 0) 1013 cnt = 1; 1014 1015 if (cnt) { 1016 int err; 1017 u32 v, new_idx; 1018 1019 new_idx = closest_thres(&adap->sge, cnt); 1020 if (q->desc && q->pktcnt_idx != new_idx) { 1021 /* the queue has already been created, update it */ 1022 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 1023 FW_PARAMS_PARAM_X_V( 1024 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | 1025 FW_PARAMS_PARAM_YZ_V(q->cntxt_id); 1026 err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, 1027 &v, &new_idx); 1028 if (err) 1029 return err; 1030 } 1031 q->pktcnt_idx = new_idx; 1032 } 1033 1034 us = us == 0 ? 6 : closest_timer(&adap->sge, us); 1035 q->intr_params = QINTR_TIMER_IDX_V(us) | QINTR_CNT_EN_V(cnt > 0); 1036 return 0; 1037 } 1038 1039 static int cxgb_set_features(struct net_device *dev, netdev_features_t features) 1040 { 1041 const struct port_info *pi = netdev_priv(dev); 1042 netdev_features_t changed = dev->features ^ features; 1043 int err; 1044 1045 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX)) 1046 return 0; 1047 1048 err = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, -1, 1049 -1, -1, -1, 1050 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true); 1051 if (unlikely(err)) 1052 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX; 1053 return err; 1054 } 1055 1056 static int setup_debugfs(struct adapter *adap) 1057 { 1058 if (IS_ERR_OR_NULL(adap->debugfs_root)) 1059 return -1; 1060 1061 #ifdef CONFIG_DEBUG_FS 1062 t4_setup_debugfs(adap); 1063 #endif 1064 return 0; 1065 } 1066 1067 /* 1068 * upper-layer driver support 1069 */ 1070 1071 /* 1072 * Allocate an active-open TID and set it to the supplied value. 1073 */ 1074 int cxgb4_alloc_atid(struct tid_info *t, void *data) 1075 { 1076 int atid = -1; 1077 1078 spin_lock_bh(&t->atid_lock); 1079 if (t->afree) { 1080 union aopen_entry *p = t->afree; 1081 1082 atid = (p - t->atid_tab) + t->atid_base; 1083 t->afree = p->next; 1084 p->data = data; 1085 t->atids_in_use++; 1086 } 1087 spin_unlock_bh(&t->atid_lock); 1088 return atid; 1089 } 1090 EXPORT_SYMBOL(cxgb4_alloc_atid); 1091 1092 /* 1093 * Release an active-open TID. 1094 */ 1095 void cxgb4_free_atid(struct tid_info *t, unsigned int atid) 1096 { 1097 union aopen_entry *p = &t->atid_tab[atid - t->atid_base]; 1098 1099 spin_lock_bh(&t->atid_lock); 1100 p->next = t->afree; 1101 t->afree = p; 1102 t->atids_in_use--; 1103 spin_unlock_bh(&t->atid_lock); 1104 } 1105 EXPORT_SYMBOL(cxgb4_free_atid); 1106 1107 /* 1108 * Allocate a server TID and set it to the supplied value. 1109 */ 1110 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data) 1111 { 1112 int stid; 1113 1114 spin_lock_bh(&t->stid_lock); 1115 if (family == PF_INET) { 1116 stid = find_first_zero_bit(t->stid_bmap, t->nstids); 1117 if (stid < t->nstids) 1118 __set_bit(stid, t->stid_bmap); 1119 else 1120 stid = -1; 1121 } else { 1122 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 1); 1123 if (stid < 0) 1124 stid = -1; 1125 } 1126 if (stid >= 0) { 1127 t->stid_tab[stid].data = data; 1128 stid += t->stid_base; 1129 /* IPv6 requires max of 520 bits or 16 cells in TCAM 1130 * This is equivalent to 4 TIDs. With CLIP enabled it 1131 * needs 2 TIDs. 1132 */ 1133 if (family == PF_INET6) { 1134 t->stids_in_use += 2; 1135 t->v6_stids_in_use += 2; 1136 } else { 1137 t->stids_in_use++; 1138 } 1139 } 1140 spin_unlock_bh(&t->stid_lock); 1141 return stid; 1142 } 1143 EXPORT_SYMBOL(cxgb4_alloc_stid); 1144 1145 /* Allocate a server filter TID and set it to the supplied value. 1146 */ 1147 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data) 1148 { 1149 int stid; 1150 1151 spin_lock_bh(&t->stid_lock); 1152 if (family == PF_INET) { 1153 stid = find_next_zero_bit(t->stid_bmap, 1154 t->nstids + t->nsftids, t->nstids); 1155 if (stid < (t->nstids + t->nsftids)) 1156 __set_bit(stid, t->stid_bmap); 1157 else 1158 stid = -1; 1159 } else { 1160 stid = -1; 1161 } 1162 if (stid >= 0) { 1163 t->stid_tab[stid].data = data; 1164 stid -= t->nstids; 1165 stid += t->sftid_base; 1166 t->sftids_in_use++; 1167 } 1168 spin_unlock_bh(&t->stid_lock); 1169 return stid; 1170 } 1171 EXPORT_SYMBOL(cxgb4_alloc_sftid); 1172 1173 /* Release a server TID. 1174 */ 1175 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family) 1176 { 1177 /* Is it a server filter TID? */ 1178 if (t->nsftids && (stid >= t->sftid_base)) { 1179 stid -= t->sftid_base; 1180 stid += t->nstids; 1181 } else { 1182 stid -= t->stid_base; 1183 } 1184 1185 spin_lock_bh(&t->stid_lock); 1186 if (family == PF_INET) 1187 __clear_bit(stid, t->stid_bmap); 1188 else 1189 bitmap_release_region(t->stid_bmap, stid, 1); 1190 t->stid_tab[stid].data = NULL; 1191 if (stid < t->nstids) { 1192 if (family == PF_INET6) { 1193 t->stids_in_use -= 2; 1194 t->v6_stids_in_use -= 2; 1195 } else { 1196 t->stids_in_use--; 1197 } 1198 } else { 1199 t->sftids_in_use--; 1200 } 1201 1202 spin_unlock_bh(&t->stid_lock); 1203 } 1204 EXPORT_SYMBOL(cxgb4_free_stid); 1205 1206 /* 1207 * Populate a TID_RELEASE WR. Caller must properly size the skb. 1208 */ 1209 static void mk_tid_release(struct sk_buff *skb, unsigned int chan, 1210 unsigned int tid) 1211 { 1212 struct cpl_tid_release *req; 1213 1214 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); 1215 req = __skb_put(skb, sizeof(*req)); 1216 INIT_TP_WR(req, tid); 1217 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 1218 } 1219 1220 /* 1221 * Queue a TID release request and if necessary schedule a work queue to 1222 * process it. 1223 */ 1224 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, 1225 unsigned int tid) 1226 { 1227 void **p = &t->tid_tab[tid]; 1228 struct adapter *adap = container_of(t, struct adapter, tids); 1229 1230 spin_lock_bh(&adap->tid_release_lock); 1231 *p = adap->tid_release_head; 1232 /* Low 2 bits encode the Tx channel number */ 1233 adap->tid_release_head = (void **)((uintptr_t)p | chan); 1234 if (!adap->tid_release_task_busy) { 1235 adap->tid_release_task_busy = true; 1236 queue_work(adap->workq, &adap->tid_release_task); 1237 } 1238 spin_unlock_bh(&adap->tid_release_lock); 1239 } 1240 1241 /* 1242 * Process the list of pending TID release requests. 1243 */ 1244 static void process_tid_release_list(struct work_struct *work) 1245 { 1246 struct sk_buff *skb; 1247 struct adapter *adap; 1248 1249 adap = container_of(work, struct adapter, tid_release_task); 1250 1251 spin_lock_bh(&adap->tid_release_lock); 1252 while (adap->tid_release_head) { 1253 void **p = adap->tid_release_head; 1254 unsigned int chan = (uintptr_t)p & 3; 1255 p = (void *)p - chan; 1256 1257 adap->tid_release_head = *p; 1258 *p = NULL; 1259 spin_unlock_bh(&adap->tid_release_lock); 1260 1261 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release), 1262 GFP_KERNEL))) 1263 schedule_timeout_uninterruptible(1); 1264 1265 mk_tid_release(skb, chan, p - adap->tids.tid_tab); 1266 t4_ofld_send(adap, skb); 1267 spin_lock_bh(&adap->tid_release_lock); 1268 } 1269 adap->tid_release_task_busy = false; 1270 spin_unlock_bh(&adap->tid_release_lock); 1271 } 1272 1273 /* 1274 * Release a TID and inform HW. If we are unable to allocate the release 1275 * message we defer to a work queue. 1276 */ 1277 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid, 1278 unsigned short family) 1279 { 1280 struct sk_buff *skb; 1281 struct adapter *adap = container_of(t, struct adapter, tids); 1282 1283 WARN_ON(tid >= t->ntids); 1284 1285 if (t->tid_tab[tid]) { 1286 t->tid_tab[tid] = NULL; 1287 atomic_dec(&t->conns_in_use); 1288 if (t->hash_base && (tid >= t->hash_base)) { 1289 if (family == AF_INET6) 1290 atomic_sub(2, &t->hash_tids_in_use); 1291 else 1292 atomic_dec(&t->hash_tids_in_use); 1293 } else { 1294 if (family == AF_INET6) 1295 atomic_sub(2, &t->tids_in_use); 1296 else 1297 atomic_dec(&t->tids_in_use); 1298 } 1299 } 1300 1301 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); 1302 if (likely(skb)) { 1303 mk_tid_release(skb, chan, tid); 1304 t4_ofld_send(adap, skb); 1305 } else 1306 cxgb4_queue_tid_release(t, chan, tid); 1307 } 1308 EXPORT_SYMBOL(cxgb4_remove_tid); 1309 1310 /* 1311 * Allocate and initialize the TID tables. Returns 0 on success. 1312 */ 1313 static int tid_init(struct tid_info *t) 1314 { 1315 struct adapter *adap = container_of(t, struct adapter, tids); 1316 unsigned int max_ftids = t->nftids + t->nsftids; 1317 unsigned int natids = t->natids; 1318 unsigned int stid_bmap_size; 1319 unsigned int ftid_bmap_size; 1320 size_t size; 1321 1322 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids); 1323 ftid_bmap_size = BITS_TO_LONGS(t->nftids); 1324 size = t->ntids * sizeof(*t->tid_tab) + 1325 natids * sizeof(*t->atid_tab) + 1326 t->nstids * sizeof(*t->stid_tab) + 1327 t->nsftids * sizeof(*t->stid_tab) + 1328 stid_bmap_size * sizeof(long) + 1329 max_ftids * sizeof(*t->ftid_tab) + 1330 ftid_bmap_size * sizeof(long); 1331 1332 t->tid_tab = kvzalloc(size, GFP_KERNEL); 1333 if (!t->tid_tab) 1334 return -ENOMEM; 1335 1336 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 1337 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids]; 1338 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids]; 1339 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size]; 1340 t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids]; 1341 spin_lock_init(&t->stid_lock); 1342 spin_lock_init(&t->atid_lock); 1343 spin_lock_init(&t->ftid_lock); 1344 1345 t->stids_in_use = 0; 1346 t->v6_stids_in_use = 0; 1347 t->sftids_in_use = 0; 1348 t->afree = NULL; 1349 t->atids_in_use = 0; 1350 atomic_set(&t->tids_in_use, 0); 1351 atomic_set(&t->conns_in_use, 0); 1352 atomic_set(&t->hash_tids_in_use, 0); 1353 1354 /* Setup the free list for atid_tab and clear the stid bitmap. */ 1355 if (natids) { 1356 while (--natids) 1357 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 1358 t->afree = t->atid_tab; 1359 } 1360 1361 if (is_offload(adap)) { 1362 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids); 1363 /* Reserve stid 0 for T4/T5 adapters */ 1364 if (!t->stid_base && 1365 CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 1366 __set_bit(0, t->stid_bmap); 1367 } 1368 1369 bitmap_zero(t->ftid_bmap, t->nftids); 1370 return 0; 1371 } 1372 1373 /** 1374 * cxgb4_create_server - create an IP server 1375 * @dev: the device 1376 * @stid: the server TID 1377 * @sip: local IP address to bind server to 1378 * @sport: the server's TCP port 1379 * @queue: queue to direct messages from this server to 1380 * 1381 * Create an IP server for the given port and address. 1382 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1383 */ 1384 int cxgb4_create_server(const struct net_device *dev, unsigned int stid, 1385 __be32 sip, __be16 sport, __be16 vlan, 1386 unsigned int queue) 1387 { 1388 unsigned int chan; 1389 struct sk_buff *skb; 1390 struct adapter *adap; 1391 struct cpl_pass_open_req *req; 1392 int ret; 1393 1394 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1395 if (!skb) 1396 return -ENOMEM; 1397 1398 adap = netdev2adap(dev); 1399 req = __skb_put(skb, sizeof(*req)); 1400 INIT_TP_WR(req, 0); 1401 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid)); 1402 req->local_port = sport; 1403 req->peer_port = htons(0); 1404 req->local_ip = sip; 1405 req->peer_ip = htonl(0); 1406 chan = rxq_to_chan(&adap->sge, queue); 1407 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1408 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1409 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1410 ret = t4_mgmt_tx(adap, skb); 1411 return net_xmit_eval(ret); 1412 } 1413 EXPORT_SYMBOL(cxgb4_create_server); 1414 1415 /* cxgb4_create_server6 - create an IPv6 server 1416 * @dev: the device 1417 * @stid: the server TID 1418 * @sip: local IPv6 address to bind server to 1419 * @sport: the server's TCP port 1420 * @queue: queue to direct messages from this server to 1421 * 1422 * Create an IPv6 server for the given port and address. 1423 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1424 */ 1425 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, 1426 const struct in6_addr *sip, __be16 sport, 1427 unsigned int queue) 1428 { 1429 unsigned int chan; 1430 struct sk_buff *skb; 1431 struct adapter *adap; 1432 struct cpl_pass_open_req6 *req; 1433 int ret; 1434 1435 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1436 if (!skb) 1437 return -ENOMEM; 1438 1439 adap = netdev2adap(dev); 1440 req = __skb_put(skb, sizeof(*req)); 1441 INIT_TP_WR(req, 0); 1442 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid)); 1443 req->local_port = sport; 1444 req->peer_port = htons(0); 1445 req->local_ip_hi = *(__be64 *)(sip->s6_addr); 1446 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8); 1447 req->peer_ip_hi = cpu_to_be64(0); 1448 req->peer_ip_lo = cpu_to_be64(0); 1449 chan = rxq_to_chan(&adap->sge, queue); 1450 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1451 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1452 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1453 ret = t4_mgmt_tx(adap, skb); 1454 return net_xmit_eval(ret); 1455 } 1456 EXPORT_SYMBOL(cxgb4_create_server6); 1457 1458 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, 1459 unsigned int queue, bool ipv6) 1460 { 1461 struct sk_buff *skb; 1462 struct adapter *adap; 1463 struct cpl_close_listsvr_req *req; 1464 int ret; 1465 1466 adap = netdev2adap(dev); 1467 1468 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1469 if (!skb) 1470 return -ENOMEM; 1471 1472 req = __skb_put(skb, sizeof(*req)); 1473 INIT_TP_WR(req, 0); 1474 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid)); 1475 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) : 1476 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue)); 1477 ret = t4_mgmt_tx(adap, skb); 1478 return net_xmit_eval(ret); 1479 } 1480 EXPORT_SYMBOL(cxgb4_remove_server); 1481 1482 /** 1483 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU 1484 * @mtus: the HW MTU table 1485 * @mtu: the target MTU 1486 * @idx: index of selected entry in the MTU table 1487 * 1488 * Returns the index and the value in the HW MTU table that is closest to 1489 * but does not exceed @mtu, unless @mtu is smaller than any value in the 1490 * table, in which case that smallest available value is selected. 1491 */ 1492 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, 1493 unsigned int *idx) 1494 { 1495 unsigned int i = 0; 1496 1497 while (i < NMTUS - 1 && mtus[i + 1] <= mtu) 1498 ++i; 1499 if (idx) 1500 *idx = i; 1501 return mtus[i]; 1502 } 1503 EXPORT_SYMBOL(cxgb4_best_mtu); 1504 1505 /** 1506 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned 1507 * @mtus: the HW MTU table 1508 * @header_size: Header Size 1509 * @data_size_max: maximum Data Segment Size 1510 * @data_size_align: desired Data Segment Size Alignment (2^N) 1511 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL) 1512 * 1513 * Similar to cxgb4_best_mtu() but instead of searching the Hardware 1514 * MTU Table based solely on a Maximum MTU parameter, we break that 1515 * parameter up into a Header Size and Maximum Data Segment Size, and 1516 * provide a desired Data Segment Size Alignment. If we find an MTU in 1517 * the Hardware MTU Table which will result in a Data Segment Size with 1518 * the requested alignment _and_ that MTU isn't "too far" from the 1519 * closest MTU, then we'll return that rather than the closest MTU. 1520 */ 1521 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, 1522 unsigned short header_size, 1523 unsigned short data_size_max, 1524 unsigned short data_size_align, 1525 unsigned int *mtu_idxp) 1526 { 1527 unsigned short max_mtu = header_size + data_size_max; 1528 unsigned short data_size_align_mask = data_size_align - 1; 1529 int mtu_idx, aligned_mtu_idx; 1530 1531 /* Scan the MTU Table till we find an MTU which is larger than our 1532 * Maximum MTU or we reach the end of the table. Along the way, 1533 * record the last MTU found, if any, which will result in a Data 1534 * Segment Length matching the requested alignment. 1535 */ 1536 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) { 1537 unsigned short data_size = mtus[mtu_idx] - header_size; 1538 1539 /* If this MTU minus the Header Size would result in a 1540 * Data Segment Size of the desired alignment, remember it. 1541 */ 1542 if ((data_size & data_size_align_mask) == 0) 1543 aligned_mtu_idx = mtu_idx; 1544 1545 /* If we're not at the end of the Hardware MTU Table and the 1546 * next element is larger than our Maximum MTU, drop out of 1547 * the loop. 1548 */ 1549 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu) 1550 break; 1551 } 1552 1553 /* If we fell out of the loop because we ran to the end of the table, 1554 * then we just have to use the last [largest] entry. 1555 */ 1556 if (mtu_idx == NMTUS) 1557 mtu_idx--; 1558 1559 /* If we found an MTU which resulted in the requested Data Segment 1560 * Length alignment and that's "not far" from the largest MTU which is 1561 * less than or equal to the maximum MTU, then use that. 1562 */ 1563 if (aligned_mtu_idx >= 0 && 1564 mtu_idx - aligned_mtu_idx <= 1) 1565 mtu_idx = aligned_mtu_idx; 1566 1567 /* If the caller has passed in an MTU Index pointer, pass the 1568 * MTU Index back. Return the MTU value. 1569 */ 1570 if (mtu_idxp) 1571 *mtu_idxp = mtu_idx; 1572 return mtus[mtu_idx]; 1573 } 1574 EXPORT_SYMBOL(cxgb4_best_aligned_mtu); 1575 1576 /** 1577 * cxgb4_tp_smt_idx - Get the Source Mac Table index for this VI 1578 * @chip: chip type 1579 * @viid: VI id of the given port 1580 * 1581 * Return the SMT index for this VI. 1582 */ 1583 unsigned int cxgb4_tp_smt_idx(enum chip_type chip, unsigned int viid) 1584 { 1585 /* In T4/T5, SMT contains 256 SMAC entries organized in 1586 * 128 rows of 2 entries each. 1587 * In T6, SMT contains 256 SMAC entries in 256 rows. 1588 * TODO: The below code needs to be updated when we add support 1589 * for 256 VFs. 1590 */ 1591 if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) 1592 return ((viid & 0x7f) << 1); 1593 else 1594 return (viid & 0x7f); 1595 } 1596 EXPORT_SYMBOL(cxgb4_tp_smt_idx); 1597 1598 /** 1599 * cxgb4_port_chan - get the HW channel of a port 1600 * @dev: the net device for the port 1601 * 1602 * Return the HW Tx channel of the given port. 1603 */ 1604 unsigned int cxgb4_port_chan(const struct net_device *dev) 1605 { 1606 return netdev2pinfo(dev)->tx_chan; 1607 } 1608 EXPORT_SYMBOL(cxgb4_port_chan); 1609 1610 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo) 1611 { 1612 struct adapter *adap = netdev2adap(dev); 1613 u32 v1, v2, lp_count, hp_count; 1614 1615 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 1616 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 1617 if (is_t4(adap->params.chip)) { 1618 lp_count = LP_COUNT_G(v1); 1619 hp_count = HP_COUNT_G(v1); 1620 } else { 1621 lp_count = LP_COUNT_T5_G(v1); 1622 hp_count = HP_COUNT_T5_G(v2); 1623 } 1624 return lpfifo ? lp_count : hp_count; 1625 } 1626 EXPORT_SYMBOL(cxgb4_dbfifo_count); 1627 1628 /** 1629 * cxgb4_port_viid - get the VI id of a port 1630 * @dev: the net device for the port 1631 * 1632 * Return the VI id of the given port. 1633 */ 1634 unsigned int cxgb4_port_viid(const struct net_device *dev) 1635 { 1636 return netdev2pinfo(dev)->viid; 1637 } 1638 EXPORT_SYMBOL(cxgb4_port_viid); 1639 1640 /** 1641 * cxgb4_port_idx - get the index of a port 1642 * @dev: the net device for the port 1643 * 1644 * Return the index of the given port. 1645 */ 1646 unsigned int cxgb4_port_idx(const struct net_device *dev) 1647 { 1648 return netdev2pinfo(dev)->port_id; 1649 } 1650 EXPORT_SYMBOL(cxgb4_port_idx); 1651 1652 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, 1653 struct tp_tcp_stats *v6) 1654 { 1655 struct adapter *adap = pci_get_drvdata(pdev); 1656 1657 spin_lock(&adap->stats_lock); 1658 t4_tp_get_tcp_stats(adap, v4, v6, false); 1659 spin_unlock(&adap->stats_lock); 1660 } 1661 EXPORT_SYMBOL(cxgb4_get_tcp_stats); 1662 1663 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, 1664 const unsigned int *pgsz_order) 1665 { 1666 struct adapter *adap = netdev2adap(dev); 1667 1668 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask); 1669 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) | 1670 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) | 1671 HPZ3_V(pgsz_order[3])); 1672 } 1673 EXPORT_SYMBOL(cxgb4_iscsi_init); 1674 1675 int cxgb4_flush_eq_cache(struct net_device *dev) 1676 { 1677 struct adapter *adap = netdev2adap(dev); 1678 1679 return t4_sge_ctxt_flush(adap, adap->mbox, CTXT_EGRESS); 1680 } 1681 EXPORT_SYMBOL(cxgb4_flush_eq_cache); 1682 1683 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) 1684 { 1685 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8; 1686 __be64 indices; 1687 int ret; 1688 1689 spin_lock(&adap->win0_lock); 1690 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr, 1691 sizeof(indices), (__be32 *)&indices, 1692 T4_MEMORY_READ); 1693 spin_unlock(&adap->win0_lock); 1694 if (!ret) { 1695 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff; 1696 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff; 1697 } 1698 return ret; 1699 } 1700 1701 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, 1702 u16 size) 1703 { 1704 struct adapter *adap = netdev2adap(dev); 1705 u16 hw_pidx, hw_cidx; 1706 int ret; 1707 1708 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); 1709 if (ret) 1710 goto out; 1711 1712 if (pidx != hw_pidx) { 1713 u16 delta; 1714 u32 val; 1715 1716 if (pidx >= hw_pidx) 1717 delta = pidx - hw_pidx; 1718 else 1719 delta = size - hw_pidx + pidx; 1720 1721 if (is_t4(adap->params.chip)) 1722 val = PIDX_V(delta); 1723 else 1724 val = PIDX_T5_V(delta); 1725 wmb(); 1726 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 1727 QID_V(qid) | val); 1728 } 1729 out: 1730 return ret; 1731 } 1732 EXPORT_SYMBOL(cxgb4_sync_txq_pidx); 1733 1734 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte) 1735 { 1736 struct adapter *adap; 1737 u32 offset, memtype, memaddr; 1738 u32 edc0_size, edc1_size, mc0_size, mc1_size, size; 1739 u32 edc0_end, edc1_end, mc0_end, mc1_end; 1740 int ret; 1741 1742 adap = netdev2adap(dev); 1743 1744 offset = ((stag >> 8) * 32) + adap->vres.stag.start; 1745 1746 /* Figure out where the offset lands in the Memory Type/Address scheme. 1747 * This code assumes that the memory is laid out starting at offset 0 1748 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0 1749 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have 1750 * MC0, and some have both MC0 and MC1. 1751 */ 1752 size = t4_read_reg(adap, MA_EDRAM0_BAR_A); 1753 edc0_size = EDRAM0_SIZE_G(size) << 20; 1754 size = t4_read_reg(adap, MA_EDRAM1_BAR_A); 1755 edc1_size = EDRAM1_SIZE_G(size) << 20; 1756 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); 1757 mc0_size = EXT_MEM0_SIZE_G(size) << 20; 1758 1759 edc0_end = edc0_size; 1760 edc1_end = edc0_end + edc1_size; 1761 mc0_end = edc1_end + mc0_size; 1762 1763 if (offset < edc0_end) { 1764 memtype = MEM_EDC0; 1765 memaddr = offset; 1766 } else if (offset < edc1_end) { 1767 memtype = MEM_EDC1; 1768 memaddr = offset - edc0_end; 1769 } else { 1770 if (offset < mc0_end) { 1771 memtype = MEM_MC0; 1772 memaddr = offset - edc1_end; 1773 } else if (is_t5(adap->params.chip)) { 1774 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); 1775 mc1_size = EXT_MEM1_SIZE_G(size) << 20; 1776 mc1_end = mc0_end + mc1_size; 1777 if (offset < mc1_end) { 1778 memtype = MEM_MC1; 1779 memaddr = offset - mc0_end; 1780 } else { 1781 /* offset beyond the end of any memory */ 1782 goto err; 1783 } 1784 } else { 1785 /* T4/T6 only has a single memory channel */ 1786 goto err; 1787 } 1788 } 1789 1790 spin_lock(&adap->win0_lock); 1791 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ); 1792 spin_unlock(&adap->win0_lock); 1793 return ret; 1794 1795 err: 1796 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n", 1797 stag, offset); 1798 return -EINVAL; 1799 } 1800 EXPORT_SYMBOL(cxgb4_read_tpte); 1801 1802 u64 cxgb4_read_sge_timestamp(struct net_device *dev) 1803 { 1804 u32 hi, lo; 1805 struct adapter *adap; 1806 1807 adap = netdev2adap(dev); 1808 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A); 1809 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A)); 1810 1811 return ((u64)hi << 32) | (u64)lo; 1812 } 1813 EXPORT_SYMBOL(cxgb4_read_sge_timestamp); 1814 1815 int cxgb4_bar2_sge_qregs(struct net_device *dev, 1816 unsigned int qid, 1817 enum cxgb4_bar2_qtype qtype, 1818 int user, 1819 u64 *pbar2_qoffset, 1820 unsigned int *pbar2_qid) 1821 { 1822 return t4_bar2_sge_qregs(netdev2adap(dev), 1823 qid, 1824 (qtype == CXGB4_BAR2_QTYPE_EGRESS 1825 ? T4_BAR2_QTYPE_EGRESS 1826 : T4_BAR2_QTYPE_INGRESS), 1827 user, 1828 pbar2_qoffset, 1829 pbar2_qid); 1830 } 1831 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs); 1832 1833 static struct pci_driver cxgb4_driver; 1834 1835 static void check_neigh_update(struct neighbour *neigh) 1836 { 1837 const struct device *parent; 1838 const struct net_device *netdev = neigh->dev; 1839 1840 if (is_vlan_dev(netdev)) 1841 netdev = vlan_dev_real_dev(netdev); 1842 parent = netdev->dev.parent; 1843 if (parent && parent->driver == &cxgb4_driver.driver) 1844 t4_l2t_update(dev_get_drvdata(parent), neigh); 1845 } 1846 1847 static int netevent_cb(struct notifier_block *nb, unsigned long event, 1848 void *data) 1849 { 1850 switch (event) { 1851 case NETEVENT_NEIGH_UPDATE: 1852 check_neigh_update(data); 1853 break; 1854 case NETEVENT_REDIRECT: 1855 default: 1856 break; 1857 } 1858 return 0; 1859 } 1860 1861 static bool netevent_registered; 1862 static struct notifier_block cxgb4_netevent_nb = { 1863 .notifier_call = netevent_cb 1864 }; 1865 1866 static void drain_db_fifo(struct adapter *adap, int usecs) 1867 { 1868 u32 v1, v2, lp_count, hp_count; 1869 1870 do { 1871 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 1872 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 1873 if (is_t4(adap->params.chip)) { 1874 lp_count = LP_COUNT_G(v1); 1875 hp_count = HP_COUNT_G(v1); 1876 } else { 1877 lp_count = LP_COUNT_T5_G(v1); 1878 hp_count = HP_COUNT_T5_G(v2); 1879 } 1880 1881 if (lp_count == 0 && hp_count == 0) 1882 break; 1883 set_current_state(TASK_UNINTERRUPTIBLE); 1884 schedule_timeout(usecs_to_jiffies(usecs)); 1885 } while (1); 1886 } 1887 1888 static void disable_txq_db(struct sge_txq *q) 1889 { 1890 unsigned long flags; 1891 1892 spin_lock_irqsave(&q->db_lock, flags); 1893 q->db_disabled = 1; 1894 spin_unlock_irqrestore(&q->db_lock, flags); 1895 } 1896 1897 static void enable_txq_db(struct adapter *adap, struct sge_txq *q) 1898 { 1899 spin_lock_irq(&q->db_lock); 1900 if (q->db_pidx_inc) { 1901 /* Make sure that all writes to the TX descriptors 1902 * are committed before we tell HW about them. 1903 */ 1904 wmb(); 1905 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 1906 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc)); 1907 q->db_pidx_inc = 0; 1908 } 1909 q->db_disabled = 0; 1910 spin_unlock_irq(&q->db_lock); 1911 } 1912 1913 static void disable_dbs(struct adapter *adap) 1914 { 1915 int i; 1916 1917 for_each_ethrxq(&adap->sge, i) 1918 disable_txq_db(&adap->sge.ethtxq[i].q); 1919 if (is_offload(adap)) { 1920 struct sge_uld_txq_info *txq_info = 1921 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 1922 1923 if (txq_info) { 1924 for_each_ofldtxq(&adap->sge, i) { 1925 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 1926 1927 disable_txq_db(&txq->q); 1928 } 1929 } 1930 } 1931 for_each_port(adap, i) 1932 disable_txq_db(&adap->sge.ctrlq[i].q); 1933 } 1934 1935 static void enable_dbs(struct adapter *adap) 1936 { 1937 int i; 1938 1939 for_each_ethrxq(&adap->sge, i) 1940 enable_txq_db(adap, &adap->sge.ethtxq[i].q); 1941 if (is_offload(adap)) { 1942 struct sge_uld_txq_info *txq_info = 1943 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 1944 1945 if (txq_info) { 1946 for_each_ofldtxq(&adap->sge, i) { 1947 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 1948 1949 enable_txq_db(adap, &txq->q); 1950 } 1951 } 1952 } 1953 for_each_port(adap, i) 1954 enable_txq_db(adap, &adap->sge.ctrlq[i].q); 1955 } 1956 1957 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) 1958 { 1959 enum cxgb4_uld type = CXGB4_ULD_RDMA; 1960 1961 if (adap->uld && adap->uld[type].handle) 1962 adap->uld[type].control(adap->uld[type].handle, cmd); 1963 } 1964 1965 static void process_db_full(struct work_struct *work) 1966 { 1967 struct adapter *adap; 1968 1969 adap = container_of(work, struct adapter, db_full_task); 1970 1971 drain_db_fifo(adap, dbfifo_drain_delay); 1972 enable_dbs(adap); 1973 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 1974 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 1975 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 1976 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 1977 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F); 1978 else 1979 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 1980 DBFIFO_LP_INT_F, DBFIFO_LP_INT_F); 1981 } 1982 1983 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) 1984 { 1985 u16 hw_pidx, hw_cidx; 1986 int ret; 1987 1988 spin_lock_irq(&q->db_lock); 1989 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); 1990 if (ret) 1991 goto out; 1992 if (q->db_pidx != hw_pidx) { 1993 u16 delta; 1994 u32 val; 1995 1996 if (q->db_pidx >= hw_pidx) 1997 delta = q->db_pidx - hw_pidx; 1998 else 1999 delta = q->size - hw_pidx + q->db_pidx; 2000 2001 if (is_t4(adap->params.chip)) 2002 val = PIDX_V(delta); 2003 else 2004 val = PIDX_T5_V(delta); 2005 wmb(); 2006 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 2007 QID_V(q->cntxt_id) | val); 2008 } 2009 out: 2010 q->db_disabled = 0; 2011 q->db_pidx_inc = 0; 2012 spin_unlock_irq(&q->db_lock); 2013 if (ret) 2014 CH_WARN(adap, "DB drop recovery failed.\n"); 2015 } 2016 2017 static void recover_all_queues(struct adapter *adap) 2018 { 2019 int i; 2020 2021 for_each_ethrxq(&adap->sge, i) 2022 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); 2023 if (is_offload(adap)) { 2024 struct sge_uld_txq_info *txq_info = 2025 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 2026 if (txq_info) { 2027 for_each_ofldtxq(&adap->sge, i) { 2028 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 2029 2030 sync_txq_pidx(adap, &txq->q); 2031 } 2032 } 2033 } 2034 for_each_port(adap, i) 2035 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); 2036 } 2037 2038 static void process_db_drop(struct work_struct *work) 2039 { 2040 struct adapter *adap; 2041 2042 adap = container_of(work, struct adapter, db_drop_task); 2043 2044 if (is_t4(adap->params.chip)) { 2045 drain_db_fifo(adap, dbfifo_drain_delay); 2046 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); 2047 drain_db_fifo(adap, dbfifo_drain_delay); 2048 recover_all_queues(adap); 2049 drain_db_fifo(adap, dbfifo_drain_delay); 2050 enable_dbs(adap); 2051 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 2052 } else if (is_t5(adap->params.chip)) { 2053 u32 dropped_db = t4_read_reg(adap, 0x010ac); 2054 u16 qid = (dropped_db >> 15) & 0x1ffff; 2055 u16 pidx_inc = dropped_db & 0x1fff; 2056 u64 bar2_qoffset; 2057 unsigned int bar2_qid; 2058 int ret; 2059 2060 ret = t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS, 2061 0, &bar2_qoffset, &bar2_qid); 2062 if (ret) 2063 dev_err(adap->pdev_dev, "doorbell drop recovery: " 2064 "qid=%d, pidx_inc=%d\n", qid, pidx_inc); 2065 else 2066 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid), 2067 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL); 2068 2069 /* Re-enable BAR2 WC */ 2070 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15); 2071 } 2072 2073 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 2074 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0); 2075 } 2076 2077 void t4_db_full(struct adapter *adap) 2078 { 2079 if (is_t4(adap->params.chip)) { 2080 disable_dbs(adap); 2081 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2082 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2083 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0); 2084 queue_work(adap->workq, &adap->db_full_task); 2085 } 2086 } 2087 2088 void t4_db_dropped(struct adapter *adap) 2089 { 2090 if (is_t4(adap->params.chip)) { 2091 disable_dbs(adap); 2092 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2093 } 2094 queue_work(adap->workq, &adap->db_drop_task); 2095 } 2096 2097 void t4_register_netevent_notifier(void) 2098 { 2099 if (!netevent_registered) { 2100 register_netevent_notifier(&cxgb4_netevent_nb); 2101 netevent_registered = true; 2102 } 2103 } 2104 2105 static void detach_ulds(struct adapter *adap) 2106 { 2107 unsigned int i; 2108 2109 mutex_lock(&uld_mutex); 2110 list_del(&adap->list_node); 2111 2112 for (i = 0; i < CXGB4_ULD_MAX; i++) 2113 if (adap->uld && adap->uld[i].handle) 2114 adap->uld[i].state_change(adap->uld[i].handle, 2115 CXGB4_STATE_DETACH); 2116 2117 if (netevent_registered && list_empty(&adapter_list)) { 2118 unregister_netevent_notifier(&cxgb4_netevent_nb); 2119 netevent_registered = false; 2120 } 2121 mutex_unlock(&uld_mutex); 2122 } 2123 2124 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state) 2125 { 2126 unsigned int i; 2127 2128 mutex_lock(&uld_mutex); 2129 for (i = 0; i < CXGB4_ULD_MAX; i++) 2130 if (adap->uld && adap->uld[i].handle) 2131 adap->uld[i].state_change(adap->uld[i].handle, 2132 new_state); 2133 mutex_unlock(&uld_mutex); 2134 } 2135 2136 #if IS_ENABLED(CONFIG_IPV6) 2137 static int cxgb4_inet6addr_handler(struct notifier_block *this, 2138 unsigned long event, void *data) 2139 { 2140 struct inet6_ifaddr *ifa = data; 2141 struct net_device *event_dev = ifa->idev->dev; 2142 const struct device *parent = NULL; 2143 #if IS_ENABLED(CONFIG_BONDING) 2144 struct adapter *adap; 2145 #endif 2146 if (is_vlan_dev(event_dev)) 2147 event_dev = vlan_dev_real_dev(event_dev); 2148 #if IS_ENABLED(CONFIG_BONDING) 2149 if (event_dev->flags & IFF_MASTER) { 2150 list_for_each_entry(adap, &adapter_list, list_node) { 2151 switch (event) { 2152 case NETDEV_UP: 2153 cxgb4_clip_get(adap->port[0], 2154 (const u32 *)ifa, 1); 2155 break; 2156 case NETDEV_DOWN: 2157 cxgb4_clip_release(adap->port[0], 2158 (const u32 *)ifa, 1); 2159 break; 2160 default: 2161 break; 2162 } 2163 } 2164 return NOTIFY_OK; 2165 } 2166 #endif 2167 2168 if (event_dev) 2169 parent = event_dev->dev.parent; 2170 2171 if (parent && parent->driver == &cxgb4_driver.driver) { 2172 switch (event) { 2173 case NETDEV_UP: 2174 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1); 2175 break; 2176 case NETDEV_DOWN: 2177 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1); 2178 break; 2179 default: 2180 break; 2181 } 2182 } 2183 return NOTIFY_OK; 2184 } 2185 2186 static bool inet6addr_registered; 2187 static struct notifier_block cxgb4_inet6addr_notifier = { 2188 .notifier_call = cxgb4_inet6addr_handler 2189 }; 2190 2191 static void update_clip(const struct adapter *adap) 2192 { 2193 int i; 2194 struct net_device *dev; 2195 int ret; 2196 2197 rcu_read_lock(); 2198 2199 for (i = 0; i < MAX_NPORTS; i++) { 2200 dev = adap->port[i]; 2201 ret = 0; 2202 2203 if (dev) 2204 ret = cxgb4_update_root_dev_clip(dev); 2205 2206 if (ret < 0) 2207 break; 2208 } 2209 rcu_read_unlock(); 2210 } 2211 #endif /* IS_ENABLED(CONFIG_IPV6) */ 2212 2213 /** 2214 * cxgb_up - enable the adapter 2215 * @adap: adapter being enabled 2216 * 2217 * Called when the first port is enabled, this function performs the 2218 * actions necessary to make an adapter operational, such as completing 2219 * the initialization of HW modules, and enabling interrupts. 2220 * 2221 * Must be called with the rtnl lock held. 2222 */ 2223 static int cxgb_up(struct adapter *adap) 2224 { 2225 int err; 2226 2227 mutex_lock(&uld_mutex); 2228 err = setup_sge_queues(adap); 2229 if (err) 2230 goto rel_lock; 2231 err = setup_rss(adap); 2232 if (err) 2233 goto freeq; 2234 2235 if (adap->flags & USING_MSIX) { 2236 name_msix_vecs(adap); 2237 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0, 2238 adap->msix_info[0].desc, adap); 2239 if (err) 2240 goto irq_err; 2241 err = request_msix_queue_irqs(adap); 2242 if (err) { 2243 free_irq(adap->msix_info[0].vec, adap); 2244 goto irq_err; 2245 } 2246 } else { 2247 err = request_irq(adap->pdev->irq, t4_intr_handler(adap), 2248 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED, 2249 adap->port[0]->name, adap); 2250 if (err) 2251 goto irq_err; 2252 } 2253 2254 enable_rx(adap); 2255 t4_sge_start(adap); 2256 t4_intr_enable(adap); 2257 adap->flags |= FULL_INIT_DONE; 2258 mutex_unlock(&uld_mutex); 2259 2260 notify_ulds(adap, CXGB4_STATE_UP); 2261 #if IS_ENABLED(CONFIG_IPV6) 2262 update_clip(adap); 2263 #endif 2264 /* Initialize hash mac addr list*/ 2265 INIT_LIST_HEAD(&adap->mac_hlist); 2266 return err; 2267 2268 irq_err: 2269 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err); 2270 freeq: 2271 t4_free_sge_resources(adap); 2272 rel_lock: 2273 mutex_unlock(&uld_mutex); 2274 return err; 2275 } 2276 2277 static void cxgb_down(struct adapter *adapter) 2278 { 2279 cancel_work_sync(&adapter->tid_release_task); 2280 cancel_work_sync(&adapter->db_full_task); 2281 cancel_work_sync(&adapter->db_drop_task); 2282 adapter->tid_release_task_busy = false; 2283 adapter->tid_release_head = NULL; 2284 2285 t4_sge_stop(adapter); 2286 t4_free_sge_resources(adapter); 2287 adapter->flags &= ~FULL_INIT_DONE; 2288 } 2289 2290 /* 2291 * net_device operations 2292 */ 2293 static int cxgb_open(struct net_device *dev) 2294 { 2295 int err; 2296 struct port_info *pi = netdev_priv(dev); 2297 struct adapter *adapter = pi->adapter; 2298 2299 netif_carrier_off(dev); 2300 2301 if (!(adapter->flags & FULL_INIT_DONE)) { 2302 err = cxgb_up(adapter); 2303 if (err < 0) 2304 return err; 2305 } 2306 2307 /* It's possible that the basic port information could have 2308 * changed since we first read it. 2309 */ 2310 err = t4_update_port_info(pi); 2311 if (err < 0) 2312 return err; 2313 2314 err = link_start(dev); 2315 if (!err) 2316 netif_tx_start_all_queues(dev); 2317 return err; 2318 } 2319 2320 static int cxgb_close(struct net_device *dev) 2321 { 2322 struct port_info *pi = netdev_priv(dev); 2323 struct adapter *adapter = pi->adapter; 2324 int ret; 2325 2326 netif_tx_stop_all_queues(dev); 2327 netif_carrier_off(dev); 2328 ret = t4_enable_vi(adapter, adapter->pf, pi->viid, false, false); 2329 #ifdef CONFIG_CHELSIO_T4_DCB 2330 cxgb4_dcb_reset(dev); 2331 dcb_tx_queue_prio_enable(dev, false); 2332 #endif 2333 return ret; 2334 } 2335 2336 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, 2337 __be32 sip, __be16 sport, __be16 vlan, 2338 unsigned int queue, unsigned char port, unsigned char mask) 2339 { 2340 int ret; 2341 struct filter_entry *f; 2342 struct adapter *adap; 2343 int i; 2344 u8 *val; 2345 2346 adap = netdev2adap(dev); 2347 2348 /* Adjust stid to correct filter index */ 2349 stid -= adap->tids.sftid_base; 2350 stid += adap->tids.nftids; 2351 2352 /* Check to make sure the filter requested is writable ... 2353 */ 2354 f = &adap->tids.ftid_tab[stid]; 2355 ret = writable_filter(f); 2356 if (ret) 2357 return ret; 2358 2359 /* Clear out any old resources being used by the filter before 2360 * we start constructing the new filter. 2361 */ 2362 if (f->valid) 2363 clear_filter(adap, f); 2364 2365 /* Clear out filter specifications */ 2366 memset(&f->fs, 0, sizeof(struct ch_filter_specification)); 2367 f->fs.val.lport = cpu_to_be16(sport); 2368 f->fs.mask.lport = ~0; 2369 val = (u8 *)&sip; 2370 if ((val[0] | val[1] | val[2] | val[3]) != 0) { 2371 for (i = 0; i < 4; i++) { 2372 f->fs.val.lip[i] = val[i]; 2373 f->fs.mask.lip[i] = ~0; 2374 } 2375 if (adap->params.tp.vlan_pri_map & PORT_F) { 2376 f->fs.val.iport = port; 2377 f->fs.mask.iport = mask; 2378 } 2379 } 2380 2381 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) { 2382 f->fs.val.proto = IPPROTO_TCP; 2383 f->fs.mask.proto = ~0; 2384 } 2385 2386 f->fs.dirsteer = 1; 2387 f->fs.iq = queue; 2388 /* Mark filter as locked */ 2389 f->locked = 1; 2390 f->fs.rpttid = 1; 2391 2392 /* Save the actual tid. We need this to get the corresponding 2393 * filter entry structure in filter_rpl. 2394 */ 2395 f->tid = stid + adap->tids.ftid_base; 2396 ret = set_filter_wr(adap, stid); 2397 if (ret) { 2398 clear_filter(adap, f); 2399 return ret; 2400 } 2401 2402 return 0; 2403 } 2404 EXPORT_SYMBOL(cxgb4_create_server_filter); 2405 2406 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, 2407 unsigned int queue, bool ipv6) 2408 { 2409 struct filter_entry *f; 2410 struct adapter *adap; 2411 2412 adap = netdev2adap(dev); 2413 2414 /* Adjust stid to correct filter index */ 2415 stid -= adap->tids.sftid_base; 2416 stid += adap->tids.nftids; 2417 2418 f = &adap->tids.ftid_tab[stid]; 2419 /* Unlock the filter */ 2420 f->locked = 0; 2421 2422 return delete_filter(adap, stid); 2423 } 2424 EXPORT_SYMBOL(cxgb4_remove_server_filter); 2425 2426 static void cxgb_get_stats(struct net_device *dev, 2427 struct rtnl_link_stats64 *ns) 2428 { 2429 struct port_stats stats; 2430 struct port_info *p = netdev_priv(dev); 2431 struct adapter *adapter = p->adapter; 2432 2433 /* Block retrieving statistics during EEH error 2434 * recovery. Otherwise, the recovery might fail 2435 * and the PCI device will be removed permanently 2436 */ 2437 spin_lock(&adapter->stats_lock); 2438 if (!netif_device_present(dev)) { 2439 spin_unlock(&adapter->stats_lock); 2440 return; 2441 } 2442 t4_get_port_stats_offset(adapter, p->tx_chan, &stats, 2443 &p->stats_base); 2444 spin_unlock(&adapter->stats_lock); 2445 2446 ns->tx_bytes = stats.tx_octets; 2447 ns->tx_packets = stats.tx_frames; 2448 ns->rx_bytes = stats.rx_octets; 2449 ns->rx_packets = stats.rx_frames; 2450 ns->multicast = stats.rx_mcast_frames; 2451 2452 /* detailed rx_errors */ 2453 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long + 2454 stats.rx_runt; 2455 ns->rx_over_errors = 0; 2456 ns->rx_crc_errors = stats.rx_fcs_err; 2457 ns->rx_frame_errors = stats.rx_symbol_err; 2458 ns->rx_dropped = stats.rx_ovflow0 + stats.rx_ovflow1 + 2459 stats.rx_ovflow2 + stats.rx_ovflow3 + 2460 stats.rx_trunc0 + stats.rx_trunc1 + 2461 stats.rx_trunc2 + stats.rx_trunc3; 2462 ns->rx_missed_errors = 0; 2463 2464 /* detailed tx_errors */ 2465 ns->tx_aborted_errors = 0; 2466 ns->tx_carrier_errors = 0; 2467 ns->tx_fifo_errors = 0; 2468 ns->tx_heartbeat_errors = 0; 2469 ns->tx_window_errors = 0; 2470 2471 ns->tx_errors = stats.tx_error_frames; 2472 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err + 2473 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors; 2474 } 2475 2476 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) 2477 { 2478 unsigned int mbox; 2479 int ret = 0, prtad, devad; 2480 struct port_info *pi = netdev_priv(dev); 2481 struct adapter *adapter = pi->adapter; 2482 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data; 2483 2484 switch (cmd) { 2485 case SIOCGMIIPHY: 2486 if (pi->mdio_addr < 0) 2487 return -EOPNOTSUPP; 2488 data->phy_id = pi->mdio_addr; 2489 break; 2490 case SIOCGMIIREG: 2491 case SIOCSMIIREG: 2492 if (mdio_phy_id_is_c45(data->phy_id)) { 2493 prtad = mdio_phy_id_prtad(data->phy_id); 2494 devad = mdio_phy_id_devad(data->phy_id); 2495 } else if (data->phy_id < 32) { 2496 prtad = data->phy_id; 2497 devad = 0; 2498 data->reg_num &= 0x1f; 2499 } else 2500 return -EINVAL; 2501 2502 mbox = pi->adapter->pf; 2503 if (cmd == SIOCGMIIREG) 2504 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad, 2505 data->reg_num, &data->val_out); 2506 else 2507 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad, 2508 data->reg_num, data->val_in); 2509 break; 2510 case SIOCGHWTSTAMP: 2511 return copy_to_user(req->ifr_data, &pi->tstamp_config, 2512 sizeof(pi->tstamp_config)) ? 2513 -EFAULT : 0; 2514 case SIOCSHWTSTAMP: 2515 if (copy_from_user(&pi->tstamp_config, req->ifr_data, 2516 sizeof(pi->tstamp_config))) 2517 return -EFAULT; 2518 2519 if (!is_t4(adapter->params.chip)) { 2520 switch (pi->tstamp_config.tx_type) { 2521 case HWTSTAMP_TX_OFF: 2522 case HWTSTAMP_TX_ON: 2523 break; 2524 default: 2525 return -ERANGE; 2526 } 2527 2528 switch (pi->tstamp_config.rx_filter) { 2529 case HWTSTAMP_FILTER_NONE: 2530 pi->rxtstamp = false; 2531 break; 2532 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 2533 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 2534 cxgb4_ptprx_timestamping(pi, pi->port_id, 2535 PTP_TS_L4); 2536 break; 2537 case HWTSTAMP_FILTER_PTP_V2_EVENT: 2538 cxgb4_ptprx_timestamping(pi, pi->port_id, 2539 PTP_TS_L2_L4); 2540 break; 2541 case HWTSTAMP_FILTER_ALL: 2542 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 2543 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 2544 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 2545 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 2546 pi->rxtstamp = true; 2547 break; 2548 default: 2549 pi->tstamp_config.rx_filter = 2550 HWTSTAMP_FILTER_NONE; 2551 return -ERANGE; 2552 } 2553 2554 if ((pi->tstamp_config.tx_type == HWTSTAMP_TX_OFF) && 2555 (pi->tstamp_config.rx_filter == 2556 HWTSTAMP_FILTER_NONE)) { 2557 if (cxgb4_ptp_txtype(adapter, pi->port_id) >= 0) 2558 pi->ptp_enable = false; 2559 } 2560 2561 if (pi->tstamp_config.rx_filter != 2562 HWTSTAMP_FILTER_NONE) { 2563 if (cxgb4_ptp_redirect_rx_packet(adapter, 2564 pi) >= 0) 2565 pi->ptp_enable = true; 2566 } 2567 } else { 2568 /* For T4 Adapters */ 2569 switch (pi->tstamp_config.rx_filter) { 2570 case HWTSTAMP_FILTER_NONE: 2571 pi->rxtstamp = false; 2572 break; 2573 case HWTSTAMP_FILTER_ALL: 2574 pi->rxtstamp = true; 2575 break; 2576 default: 2577 pi->tstamp_config.rx_filter = 2578 HWTSTAMP_FILTER_NONE; 2579 return -ERANGE; 2580 } 2581 } 2582 return copy_to_user(req->ifr_data, &pi->tstamp_config, 2583 sizeof(pi->tstamp_config)) ? 2584 -EFAULT : 0; 2585 default: 2586 return -EOPNOTSUPP; 2587 } 2588 return ret; 2589 } 2590 2591 static void cxgb_set_rxmode(struct net_device *dev) 2592 { 2593 /* unfortunately we can't return errors to the stack */ 2594 set_rxmode(dev, -1, false); 2595 } 2596 2597 static int cxgb_change_mtu(struct net_device *dev, int new_mtu) 2598 { 2599 int ret; 2600 struct port_info *pi = netdev_priv(dev); 2601 2602 ret = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, new_mtu, -1, 2603 -1, -1, -1, true); 2604 if (!ret) 2605 dev->mtu = new_mtu; 2606 return ret; 2607 } 2608 2609 #ifdef CONFIG_PCI_IOV 2610 static int cxgb4_mgmt_open(struct net_device *dev) 2611 { 2612 /* Turn carrier off since we don't have to transmit anything on this 2613 * interface. 2614 */ 2615 netif_carrier_off(dev); 2616 return 0; 2617 } 2618 2619 /* Fill MAC address that will be assigned by the FW */ 2620 static void cxgb4_mgmt_fill_vf_station_mac_addr(struct adapter *adap) 2621 { 2622 u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN]; 2623 unsigned int i, vf, nvfs; 2624 u16 a, b; 2625 int err; 2626 u8 *na; 2627 2628 adap->params.pci.vpd_cap_addr = pci_find_capability(adap->pdev, 2629 PCI_CAP_ID_VPD); 2630 err = t4_get_raw_vpd_params(adap, &adap->params.vpd); 2631 if (err) 2632 return; 2633 2634 na = adap->params.vpd.na; 2635 for (i = 0; i < ETH_ALEN; i++) 2636 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 2637 hex2val(na[2 * i + 1])); 2638 2639 a = (hw_addr[0] << 8) | hw_addr[1]; 2640 b = (hw_addr[1] << 8) | hw_addr[2]; 2641 a ^= b; 2642 a |= 0x0200; /* locally assigned Ethernet MAC address */ 2643 a &= ~0x0100; /* not a multicast Ethernet MAC address */ 2644 macaddr[0] = a >> 8; 2645 macaddr[1] = a & 0xff; 2646 2647 for (i = 2; i < 5; i++) 2648 macaddr[i] = hw_addr[i + 1]; 2649 2650 for (vf = 0, nvfs = pci_sriov_get_totalvfs(adap->pdev); 2651 vf < nvfs; vf++) { 2652 macaddr[5] = adap->pf * 16 + vf; 2653 ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, macaddr); 2654 } 2655 } 2656 2657 static int cxgb4_mgmt_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 2658 { 2659 struct port_info *pi = netdev_priv(dev); 2660 struct adapter *adap = pi->adapter; 2661 int ret; 2662 2663 /* verify MAC addr is valid */ 2664 if (!is_valid_ether_addr(mac)) { 2665 dev_err(pi->adapter->pdev_dev, 2666 "Invalid Ethernet address %pM for VF %d\n", 2667 mac, vf); 2668 return -EINVAL; 2669 } 2670 2671 dev_info(pi->adapter->pdev_dev, 2672 "Setting MAC %pM on VF %d\n", mac, vf); 2673 ret = t4_set_vf_mac_acl(adap, vf + 1, 1, mac); 2674 if (!ret) 2675 ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac); 2676 return ret; 2677 } 2678 2679 static int cxgb4_mgmt_get_vf_config(struct net_device *dev, 2680 int vf, struct ifla_vf_info *ivi) 2681 { 2682 struct port_info *pi = netdev_priv(dev); 2683 struct adapter *adap = pi->adapter; 2684 2685 if (vf >= adap->num_vfs) 2686 return -EINVAL; 2687 ivi->vf = vf; 2688 ivi->max_tx_rate = adap->vfinfo[vf].tx_rate; 2689 ivi->min_tx_rate = 0; 2690 ether_addr_copy(ivi->mac, adap->vfinfo[vf].vf_mac_addr); 2691 return 0; 2692 } 2693 2694 static int cxgb4_mgmt_get_phys_port_id(struct net_device *dev, 2695 struct netdev_phys_item_id *ppid) 2696 { 2697 struct port_info *pi = netdev_priv(dev); 2698 unsigned int phy_port_id; 2699 2700 phy_port_id = pi->adapter->adap_idx * 10 + pi->port_id; 2701 ppid->id_len = sizeof(phy_port_id); 2702 memcpy(ppid->id, &phy_port_id, ppid->id_len); 2703 return 0; 2704 } 2705 2706 static int cxgb4_mgmt_set_vf_rate(struct net_device *dev, int vf, 2707 int min_tx_rate, int max_tx_rate) 2708 { 2709 struct port_info *pi = netdev_priv(dev); 2710 struct adapter *adap = pi->adapter; 2711 unsigned int link_ok, speed, mtu; 2712 u32 fw_pfvf, fw_class; 2713 int class_id = vf; 2714 int ret; 2715 u16 pktsize; 2716 2717 if (vf >= adap->num_vfs) 2718 return -EINVAL; 2719 2720 if (min_tx_rate) { 2721 dev_err(adap->pdev_dev, 2722 "Min tx rate (%d) (> 0) for VF %d is Invalid.\n", 2723 min_tx_rate, vf); 2724 return -EINVAL; 2725 } 2726 2727 ret = t4_get_link_params(pi, &link_ok, &speed, &mtu); 2728 if (ret != FW_SUCCESS) { 2729 dev_err(adap->pdev_dev, 2730 "Failed to get link information for VF %d\n", vf); 2731 return -EINVAL; 2732 } 2733 2734 if (!link_ok) { 2735 dev_err(adap->pdev_dev, "Link down for VF %d\n", vf); 2736 return -EINVAL; 2737 } 2738 2739 if (max_tx_rate > speed) { 2740 dev_err(adap->pdev_dev, 2741 "Max tx rate %d for VF %d can't be > link-speed %u", 2742 max_tx_rate, vf, speed); 2743 return -EINVAL; 2744 } 2745 2746 pktsize = mtu; 2747 /* subtract ethhdr size and 4 bytes crc since, f/w appends it */ 2748 pktsize = pktsize - sizeof(struct ethhdr) - 4; 2749 /* subtract ipv4 hdr size, tcp hdr size to get typical IPv4 MSS size */ 2750 pktsize = pktsize - sizeof(struct iphdr) - sizeof(struct tcphdr); 2751 /* configure Traffic Class for rate-limiting */ 2752 ret = t4_sched_params(adap, SCHED_CLASS_TYPE_PACKET, 2753 SCHED_CLASS_LEVEL_CL_RL, 2754 SCHED_CLASS_MODE_CLASS, 2755 SCHED_CLASS_RATEUNIT_BITS, 2756 SCHED_CLASS_RATEMODE_ABS, 2757 pi->tx_chan, class_id, 0, 2758 max_tx_rate * 1000, 0, pktsize); 2759 if (ret) { 2760 dev_err(adap->pdev_dev, "Err %d for Traffic Class config\n", 2761 ret); 2762 return -EINVAL; 2763 } 2764 dev_info(adap->pdev_dev, 2765 "Class %d with MSS %u configured with rate %u\n", 2766 class_id, pktsize, max_tx_rate); 2767 2768 /* bind VF to configured Traffic Class */ 2769 fw_pfvf = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | 2770 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH)); 2771 fw_class = class_id; 2772 ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, &fw_pfvf, 2773 &fw_class); 2774 if (ret) { 2775 dev_err(adap->pdev_dev, 2776 "Err %d in binding VF %d to Traffic Class %d\n", 2777 ret, vf, class_id); 2778 return -EINVAL; 2779 } 2780 dev_info(adap->pdev_dev, "PF %d VF %d is bound to Class %d\n", 2781 adap->pf, vf, class_id); 2782 adap->vfinfo[vf].tx_rate = max_tx_rate; 2783 return 0; 2784 } 2785 2786 static int cxgb4_mgmt_set_vf_vlan(struct net_device *dev, int vf, 2787 u16 vlan, u8 qos, __be16 vlan_proto) 2788 { 2789 struct port_info *pi = netdev_priv(dev); 2790 struct adapter *adap = pi->adapter; 2791 int ret; 2792 2793 if (vf >= adap->num_vfs || vlan > 4095 || qos > 7) 2794 return -EINVAL; 2795 2796 if (vlan_proto != htons(ETH_P_8021Q) || qos != 0) 2797 return -EPROTONOSUPPORT; 2798 2799 ret = t4_set_vlan_acl(adap, adap->mbox, vf + 1, vlan); 2800 if (!ret) { 2801 adap->vfinfo[vf].vlan = vlan; 2802 return 0; 2803 } 2804 2805 dev_err(adap->pdev_dev, "Err %d %s VLAN ACL for PF/VF %d/%d\n", 2806 ret, (vlan ? "setting" : "clearing"), adap->pf, vf); 2807 return ret; 2808 } 2809 #endif /* CONFIG_PCI_IOV */ 2810 2811 static int cxgb_set_mac_addr(struct net_device *dev, void *p) 2812 { 2813 int ret; 2814 struct sockaddr *addr = p; 2815 struct port_info *pi = netdev_priv(dev); 2816 2817 if (!is_valid_ether_addr(addr->sa_data)) 2818 return -EADDRNOTAVAIL; 2819 2820 ret = t4_change_mac(pi->adapter, pi->adapter->pf, pi->viid, 2821 pi->xact_addr_filt, addr->sa_data, true, true); 2822 if (ret < 0) 2823 return ret; 2824 2825 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 2826 pi->xact_addr_filt = ret; 2827 return 0; 2828 } 2829 2830 #ifdef CONFIG_NET_POLL_CONTROLLER 2831 static void cxgb_netpoll(struct net_device *dev) 2832 { 2833 struct port_info *pi = netdev_priv(dev); 2834 struct adapter *adap = pi->adapter; 2835 2836 if (adap->flags & USING_MSIX) { 2837 int i; 2838 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset]; 2839 2840 for (i = pi->nqsets; i; i--, rx++) 2841 t4_sge_intr_msix(0, &rx->rspq); 2842 } else 2843 t4_intr_handler(adap)(0, adap); 2844 } 2845 #endif 2846 2847 static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 2848 { 2849 struct port_info *pi = netdev_priv(dev); 2850 struct adapter *adap = pi->adapter; 2851 struct sched_class *e; 2852 struct ch_sched_params p; 2853 struct ch_sched_queue qe; 2854 u32 req_rate; 2855 int err = 0; 2856 2857 if (!can_sched(dev)) 2858 return -ENOTSUPP; 2859 2860 if (index < 0 || index > pi->nqsets - 1) 2861 return -EINVAL; 2862 2863 if (!(adap->flags & FULL_INIT_DONE)) { 2864 dev_err(adap->pdev_dev, 2865 "Failed to rate limit on queue %d. Link Down?\n", 2866 index); 2867 return -EINVAL; 2868 } 2869 2870 /* Convert from Mbps to Kbps */ 2871 req_rate = rate << 10; 2872 2873 /* Max rate is 10 Gbps */ 2874 if (req_rate >= SCHED_MAX_RATE_KBPS) { 2875 dev_err(adap->pdev_dev, 2876 "Invalid rate %u Mbps, Max rate is %u Gbps\n", 2877 rate, SCHED_MAX_RATE_KBPS); 2878 return -ERANGE; 2879 } 2880 2881 /* First unbind the queue from any existing class */ 2882 memset(&qe, 0, sizeof(qe)); 2883 qe.queue = index; 2884 qe.class = SCHED_CLS_NONE; 2885 2886 err = cxgb4_sched_class_unbind(dev, (void *)(&qe), SCHED_QUEUE); 2887 if (err) { 2888 dev_err(adap->pdev_dev, 2889 "Unbinding Queue %d on port %d fail. Err: %d\n", 2890 index, pi->port_id, err); 2891 return err; 2892 } 2893 2894 /* Queue already unbound */ 2895 if (!req_rate) 2896 return 0; 2897 2898 /* Fetch any available unused or matching scheduling class */ 2899 memset(&p, 0, sizeof(p)); 2900 p.type = SCHED_CLASS_TYPE_PACKET; 2901 p.u.params.level = SCHED_CLASS_LEVEL_CL_RL; 2902 p.u.params.mode = SCHED_CLASS_MODE_CLASS; 2903 p.u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS; 2904 p.u.params.ratemode = SCHED_CLASS_RATEMODE_ABS; 2905 p.u.params.channel = pi->tx_chan; 2906 p.u.params.class = SCHED_CLS_NONE; 2907 p.u.params.minrate = 0; 2908 p.u.params.maxrate = req_rate; 2909 p.u.params.weight = 0; 2910 p.u.params.pktsize = dev->mtu; 2911 2912 e = cxgb4_sched_class_alloc(dev, &p); 2913 if (!e) 2914 return -ENOMEM; 2915 2916 /* Bind the queue to a scheduling class */ 2917 memset(&qe, 0, sizeof(qe)); 2918 qe.queue = index; 2919 qe.class = e->idx; 2920 2921 err = cxgb4_sched_class_bind(dev, (void *)(&qe), SCHED_QUEUE); 2922 if (err) 2923 dev_err(adap->pdev_dev, 2924 "Queue rate limiting failed. Err: %d\n", err); 2925 return err; 2926 } 2927 2928 static int cxgb_setup_tc_flower(struct net_device *dev, 2929 struct tc_cls_flower_offload *cls_flower) 2930 { 2931 switch (cls_flower->command) { 2932 case TC_CLSFLOWER_REPLACE: 2933 return cxgb4_tc_flower_replace(dev, cls_flower); 2934 case TC_CLSFLOWER_DESTROY: 2935 return cxgb4_tc_flower_destroy(dev, cls_flower); 2936 case TC_CLSFLOWER_STATS: 2937 return cxgb4_tc_flower_stats(dev, cls_flower); 2938 default: 2939 return -EOPNOTSUPP; 2940 } 2941 } 2942 2943 static int cxgb_setup_tc_cls_u32(struct net_device *dev, 2944 struct tc_cls_u32_offload *cls_u32) 2945 { 2946 switch (cls_u32->command) { 2947 case TC_CLSU32_NEW_KNODE: 2948 case TC_CLSU32_REPLACE_KNODE: 2949 return cxgb4_config_knode(dev, cls_u32); 2950 case TC_CLSU32_DELETE_KNODE: 2951 return cxgb4_delete_knode(dev, cls_u32); 2952 default: 2953 return -EOPNOTSUPP; 2954 } 2955 } 2956 2957 static int cxgb_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 2958 void *cb_priv) 2959 { 2960 struct net_device *dev = cb_priv; 2961 struct port_info *pi = netdev2pinfo(dev); 2962 struct adapter *adap = netdev2adap(dev); 2963 2964 if (!(adap->flags & FULL_INIT_DONE)) { 2965 dev_err(adap->pdev_dev, 2966 "Failed to setup tc on port %d. Link Down?\n", 2967 pi->port_id); 2968 return -EINVAL; 2969 } 2970 2971 if (!tc_cls_can_offload_and_chain0(dev, type_data)) 2972 return -EOPNOTSUPP; 2973 2974 switch (type) { 2975 case TC_SETUP_CLSU32: 2976 return cxgb_setup_tc_cls_u32(dev, type_data); 2977 case TC_SETUP_CLSFLOWER: 2978 return cxgb_setup_tc_flower(dev, type_data); 2979 default: 2980 return -EOPNOTSUPP; 2981 } 2982 } 2983 2984 static int cxgb_setup_tc_block(struct net_device *dev, 2985 struct tc_block_offload *f) 2986 { 2987 struct port_info *pi = netdev2pinfo(dev); 2988 2989 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 2990 return -EOPNOTSUPP; 2991 2992 switch (f->command) { 2993 case TC_BLOCK_BIND: 2994 return tcf_block_cb_register(f->block, cxgb_setup_tc_block_cb, 2995 pi, dev); 2996 case TC_BLOCK_UNBIND: 2997 tcf_block_cb_unregister(f->block, cxgb_setup_tc_block_cb, pi); 2998 return 0; 2999 default: 3000 return -EOPNOTSUPP; 3001 } 3002 } 3003 3004 static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type, 3005 void *type_data) 3006 { 3007 switch (type) { 3008 case TC_SETUP_BLOCK: 3009 return cxgb_setup_tc_block(dev, type_data); 3010 default: 3011 return -EOPNOTSUPP; 3012 } 3013 } 3014 3015 static void cxgb_del_udp_tunnel(struct net_device *netdev, 3016 struct udp_tunnel_info *ti) 3017 { 3018 struct port_info *pi = netdev_priv(netdev); 3019 struct adapter *adapter = pi->adapter; 3020 unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip); 3021 u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 }; 3022 int ret = 0, i; 3023 3024 if (chip_ver < CHELSIO_T6) 3025 return; 3026 3027 switch (ti->type) { 3028 case UDP_TUNNEL_TYPE_VXLAN: 3029 if (!adapter->vxlan_port_cnt || 3030 adapter->vxlan_port != ti->port) 3031 return; /* Invalid VxLAN destination port */ 3032 3033 adapter->vxlan_port_cnt--; 3034 if (adapter->vxlan_port_cnt) 3035 return; 3036 3037 adapter->vxlan_port = 0; 3038 t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, 0); 3039 break; 3040 case UDP_TUNNEL_TYPE_GENEVE: 3041 if (!adapter->geneve_port_cnt || 3042 adapter->geneve_port != ti->port) 3043 return; /* Invalid GENEVE destination port */ 3044 3045 adapter->geneve_port_cnt--; 3046 if (adapter->geneve_port_cnt) 3047 return; 3048 3049 adapter->geneve_port = 0; 3050 t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, 0); 3051 default: 3052 return; 3053 } 3054 3055 /* Matchall mac entries can be deleted only after all tunnel ports 3056 * are brought down or removed. 3057 */ 3058 if (!adapter->rawf_cnt) 3059 return; 3060 for_each_port(adapter, i) { 3061 pi = adap2pinfo(adapter, i); 3062 ret = t4_free_raw_mac_filt(adapter, pi->viid, 3063 match_all_mac, match_all_mac, 3064 adapter->rawf_start + 3065 pi->port_id, 3066 1, pi->port_id, true); 3067 if (ret < 0) { 3068 netdev_info(netdev, "Failed to free mac filter entry, for port %d\n", 3069 i); 3070 return; 3071 } 3072 atomic_dec(&adapter->mps_encap[adapter->rawf_start + 3073 pi->port_id].refcnt); 3074 } 3075 } 3076 3077 static void cxgb_add_udp_tunnel(struct net_device *netdev, 3078 struct udp_tunnel_info *ti) 3079 { 3080 struct port_info *pi = netdev_priv(netdev); 3081 struct adapter *adapter = pi->adapter; 3082 unsigned int chip_ver = CHELSIO_CHIP_VERSION(adapter->params.chip); 3083 u8 match_all_mac[] = { 0, 0, 0, 0, 0, 0 }; 3084 int i, ret; 3085 3086 if (chip_ver < CHELSIO_T6 || !adapter->rawf_cnt) 3087 return; 3088 3089 switch (ti->type) { 3090 case UDP_TUNNEL_TYPE_VXLAN: 3091 /* Callback for adding vxlan port can be called with the same 3092 * port for both IPv4 and IPv6. We should not disable the 3093 * offloading when the same port for both protocols is added 3094 * and later one of them is removed. 3095 */ 3096 if (adapter->vxlan_port_cnt && 3097 adapter->vxlan_port == ti->port) { 3098 adapter->vxlan_port_cnt++; 3099 return; 3100 } 3101 3102 /* We will support only one VxLAN port */ 3103 if (adapter->vxlan_port_cnt) { 3104 netdev_info(netdev, "UDP port %d already offloaded, not adding port %d\n", 3105 be16_to_cpu(adapter->vxlan_port), 3106 be16_to_cpu(ti->port)); 3107 return; 3108 } 3109 3110 adapter->vxlan_port = ti->port; 3111 adapter->vxlan_port_cnt = 1; 3112 3113 t4_write_reg(adapter, MPS_RX_VXLAN_TYPE_A, 3114 VXLAN_V(be16_to_cpu(ti->port)) | VXLAN_EN_F); 3115 break; 3116 case UDP_TUNNEL_TYPE_GENEVE: 3117 if (adapter->geneve_port_cnt && 3118 adapter->geneve_port == ti->port) { 3119 adapter->geneve_port_cnt++; 3120 return; 3121 } 3122 3123 /* We will support only one GENEVE port */ 3124 if (adapter->geneve_port_cnt) { 3125 netdev_info(netdev, "UDP port %d already offloaded, not adding port %d\n", 3126 be16_to_cpu(adapter->geneve_port), 3127 be16_to_cpu(ti->port)); 3128 return; 3129 } 3130 3131 adapter->geneve_port = ti->port; 3132 adapter->geneve_port_cnt = 1; 3133 3134 t4_write_reg(adapter, MPS_RX_GENEVE_TYPE_A, 3135 GENEVE_V(be16_to_cpu(ti->port)) | GENEVE_EN_F); 3136 default: 3137 return; 3138 } 3139 3140 /* Create a 'match all' mac filter entry for inner mac, 3141 * if raw mac interface is supported. Once the linux kernel provides 3142 * driver entry points for adding/deleting the inner mac addresses, 3143 * we will remove this 'match all' entry and fallback to adding 3144 * exact match filters. 3145 */ 3146 for_each_port(adapter, i) { 3147 pi = adap2pinfo(adapter, i); 3148 3149 ret = t4_alloc_raw_mac_filt(adapter, pi->viid, 3150 match_all_mac, 3151 match_all_mac, 3152 adapter->rawf_start + 3153 pi->port_id, 3154 1, pi->port_id, true); 3155 if (ret < 0) { 3156 netdev_info(netdev, "Failed to allocate a mac filter entry, not adding port %d\n", 3157 be16_to_cpu(ti->port)); 3158 cxgb_del_udp_tunnel(netdev, ti); 3159 return; 3160 } 3161 atomic_inc(&adapter->mps_encap[ret].refcnt); 3162 } 3163 } 3164 3165 static netdev_features_t cxgb_features_check(struct sk_buff *skb, 3166 struct net_device *dev, 3167 netdev_features_t features) 3168 { 3169 struct port_info *pi = netdev_priv(dev); 3170 struct adapter *adapter = pi->adapter; 3171 3172 if (CHELSIO_CHIP_VERSION(adapter->params.chip) < CHELSIO_T6) 3173 return features; 3174 3175 /* Check if hw supports offload for this packet */ 3176 if (!skb->encapsulation || cxgb_encap_offload_supported(skb)) 3177 return features; 3178 3179 /* Offload is not supported for this encapsulated packet */ 3180 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3181 } 3182 3183 static netdev_features_t cxgb_fix_features(struct net_device *dev, 3184 netdev_features_t features) 3185 { 3186 /* Disable GRO, if RX_CSUM is disabled */ 3187 if (!(features & NETIF_F_RXCSUM)) 3188 features &= ~NETIF_F_GRO; 3189 3190 return features; 3191 } 3192 3193 static const struct net_device_ops cxgb4_netdev_ops = { 3194 .ndo_open = cxgb_open, 3195 .ndo_stop = cxgb_close, 3196 .ndo_start_xmit = t4_eth_xmit, 3197 .ndo_select_queue = cxgb_select_queue, 3198 .ndo_get_stats64 = cxgb_get_stats, 3199 .ndo_set_rx_mode = cxgb_set_rxmode, 3200 .ndo_set_mac_address = cxgb_set_mac_addr, 3201 .ndo_set_features = cxgb_set_features, 3202 .ndo_validate_addr = eth_validate_addr, 3203 .ndo_do_ioctl = cxgb_ioctl, 3204 .ndo_change_mtu = cxgb_change_mtu, 3205 #ifdef CONFIG_NET_POLL_CONTROLLER 3206 .ndo_poll_controller = cxgb_netpoll, 3207 #endif 3208 #ifdef CONFIG_CHELSIO_T4_FCOE 3209 .ndo_fcoe_enable = cxgb_fcoe_enable, 3210 .ndo_fcoe_disable = cxgb_fcoe_disable, 3211 #endif /* CONFIG_CHELSIO_T4_FCOE */ 3212 .ndo_set_tx_maxrate = cxgb_set_tx_maxrate, 3213 .ndo_setup_tc = cxgb_setup_tc, 3214 .ndo_udp_tunnel_add = cxgb_add_udp_tunnel, 3215 .ndo_udp_tunnel_del = cxgb_del_udp_tunnel, 3216 .ndo_features_check = cxgb_features_check, 3217 .ndo_fix_features = cxgb_fix_features, 3218 }; 3219 3220 #ifdef CONFIG_PCI_IOV 3221 static const struct net_device_ops cxgb4_mgmt_netdev_ops = { 3222 .ndo_open = cxgb4_mgmt_open, 3223 .ndo_set_vf_mac = cxgb4_mgmt_set_vf_mac, 3224 .ndo_get_vf_config = cxgb4_mgmt_get_vf_config, 3225 .ndo_set_vf_rate = cxgb4_mgmt_set_vf_rate, 3226 .ndo_get_phys_port_id = cxgb4_mgmt_get_phys_port_id, 3227 .ndo_set_vf_vlan = cxgb4_mgmt_set_vf_vlan, 3228 }; 3229 #endif 3230 3231 static void cxgb4_mgmt_get_drvinfo(struct net_device *dev, 3232 struct ethtool_drvinfo *info) 3233 { 3234 struct adapter *adapter = netdev2adap(dev); 3235 3236 strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); 3237 strlcpy(info->version, cxgb4_driver_version, 3238 sizeof(info->version)); 3239 strlcpy(info->bus_info, pci_name(adapter->pdev), 3240 sizeof(info->bus_info)); 3241 } 3242 3243 static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = { 3244 .get_drvinfo = cxgb4_mgmt_get_drvinfo, 3245 }; 3246 3247 void t4_fatal_err(struct adapter *adap) 3248 { 3249 int port; 3250 3251 if (pci_channel_offline(adap->pdev)) 3252 return; 3253 3254 /* Disable the SGE since ULDs are going to free resources that 3255 * could be exposed to the adapter. RDMA MWs for example... 3256 */ 3257 t4_shutdown_adapter(adap); 3258 for_each_port(adap, port) { 3259 struct net_device *dev = adap->port[port]; 3260 3261 /* If we get here in very early initialization the network 3262 * devices may not have been set up yet. 3263 */ 3264 if (!dev) 3265 continue; 3266 3267 netif_tx_stop_all_queues(dev); 3268 netif_carrier_off(dev); 3269 } 3270 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n"); 3271 } 3272 3273 static void setup_memwin(struct adapter *adap) 3274 { 3275 u32 nic_win_base = t4_get_util_window(adap); 3276 3277 t4_setup_memwin(adap, nic_win_base, MEMWIN_NIC); 3278 } 3279 3280 static void setup_memwin_rdma(struct adapter *adap) 3281 { 3282 if (adap->vres.ocq.size) { 3283 u32 start; 3284 unsigned int sz_kb; 3285 3286 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2); 3287 start &= PCI_BASE_ADDRESS_MEM_MASK; 3288 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres); 3289 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10; 3290 t4_write_reg(adap, 3291 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3), 3292 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb))); 3293 t4_write_reg(adap, 3294 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3), 3295 adap->vres.ocq.start); 3296 t4_read_reg(adap, 3297 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3)); 3298 } 3299 } 3300 3301 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c) 3302 { 3303 u32 v; 3304 int ret; 3305 3306 /* get device capabilities */ 3307 memset(c, 0, sizeof(*c)); 3308 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3309 FW_CMD_REQUEST_F | FW_CMD_READ_F); 3310 c->cfvalid_to_len16 = htonl(FW_LEN16(*c)); 3311 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), c); 3312 if (ret < 0) 3313 return ret; 3314 3315 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3316 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 3317 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), NULL); 3318 if (ret < 0) 3319 return ret; 3320 3321 ret = t4_config_glbl_rss(adap, adap->pf, 3322 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 3323 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | 3324 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F); 3325 if (ret < 0) 3326 return ret; 3327 3328 ret = t4_cfg_pfvf(adap, adap->mbox, adap->pf, 0, adap->sge.egr_sz, 64, 3329 MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, 3330 FW_CMD_CAP_PF); 3331 if (ret < 0) 3332 return ret; 3333 3334 t4_sge_init(adap); 3335 3336 /* tweak some settings */ 3337 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849); 3338 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12)); 3339 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A); 3340 v = t4_read_reg(adap, TP_PIO_DATA_A); 3341 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F); 3342 3343 /* first 4 Tx modulation queues point to consecutive Tx channels */ 3344 adap->params.tp.tx_modq_map = 0xE4; 3345 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A, 3346 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map)); 3347 3348 /* associate each Tx modulation queue with consecutive Tx channels */ 3349 v = 0x84218421; 3350 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3351 &v, 1, TP_TX_SCHED_HDR_A); 3352 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3353 &v, 1, TP_TX_SCHED_FIFO_A); 3354 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3355 &v, 1, TP_TX_SCHED_PCMD_A); 3356 3357 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */ 3358 if (is_offload(adap)) { 3359 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A, 3360 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3361 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3362 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3363 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 3364 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A, 3365 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3366 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3367 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3368 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 3369 } 3370 3371 /* get basic stuff going */ 3372 return t4_early_init(adap, adap->pf); 3373 } 3374 3375 /* 3376 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. 3377 */ 3378 #define MAX_ATIDS 8192U 3379 3380 /* 3381 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 3382 * 3383 * If the firmware we're dealing with has Configuration File support, then 3384 * we use that to perform all configuration 3385 */ 3386 3387 /* 3388 * Tweak configuration based on module parameters, etc. Most of these have 3389 * defaults assigned to them by Firmware Configuration Files (if we're using 3390 * them) but need to be explicitly set if we're using hard-coded 3391 * initialization. But even in the case of using Firmware Configuration 3392 * Files, we'd like to expose the ability to change these via module 3393 * parameters so these are essentially common tweaks/settings for 3394 * Configuration Files and hard-coded initialization ... 3395 */ 3396 static int adap_init0_tweaks(struct adapter *adapter) 3397 { 3398 /* 3399 * Fix up various Host-Dependent Parameters like Page Size, Cache 3400 * Line Size, etc. The firmware default is for a 4KB Page Size and 3401 * 64B Cache Line Size ... 3402 */ 3403 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES); 3404 3405 /* 3406 * Process module parameters which affect early initialization. 3407 */ 3408 if (rx_dma_offset != 2 && rx_dma_offset != 0) { 3409 dev_err(&adapter->pdev->dev, 3410 "Ignoring illegal rx_dma_offset=%d, using 2\n", 3411 rx_dma_offset); 3412 rx_dma_offset = 2; 3413 } 3414 t4_set_reg_field(adapter, SGE_CONTROL_A, 3415 PKTSHIFT_V(PKTSHIFT_M), 3416 PKTSHIFT_V(rx_dma_offset)); 3417 3418 /* 3419 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux 3420 * adds the pseudo header itself. 3421 */ 3422 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A, 3423 CSUM_HAS_PSEUDO_HDR_F, 0); 3424 3425 return 0; 3426 } 3427 3428 /* 10Gb/s-BT PHY Support. chip-external 10Gb/s-BT PHYs are complex chips 3429 * unto themselves and they contain their own firmware to perform their 3430 * tasks ... 3431 */ 3432 static int phy_aq1202_version(const u8 *phy_fw_data, 3433 size_t phy_fw_size) 3434 { 3435 int offset; 3436 3437 /* At offset 0x8 you're looking for the primary image's 3438 * starting offset which is 3 Bytes wide 3439 * 3440 * At offset 0xa of the primary image, you look for the offset 3441 * of the DRAM segment which is 3 Bytes wide. 3442 * 3443 * The FW version is at offset 0x27e of the DRAM and is 2 Bytes 3444 * wide 3445 */ 3446 #define be16(__p) (((__p)[0] << 8) | (__p)[1]) 3447 #define le16(__p) ((__p)[0] | ((__p)[1] << 8)) 3448 #define le24(__p) (le16(__p) | ((__p)[2] << 16)) 3449 3450 offset = le24(phy_fw_data + 0x8) << 12; 3451 offset = le24(phy_fw_data + offset + 0xa); 3452 return be16(phy_fw_data + offset + 0x27e); 3453 3454 #undef be16 3455 #undef le16 3456 #undef le24 3457 } 3458 3459 static struct info_10gbt_phy_fw { 3460 unsigned int phy_fw_id; /* PCI Device ID */ 3461 char *phy_fw_file; /* /lib/firmware/ PHY Firmware file */ 3462 int (*phy_fw_version)(const u8 *phy_fw_data, size_t phy_fw_size); 3463 int phy_flash; /* Has FLASH for PHY Firmware */ 3464 } phy_info_array[] = { 3465 { 3466 PHY_AQ1202_DEVICEID, 3467 PHY_AQ1202_FIRMWARE, 3468 phy_aq1202_version, 3469 1, 3470 }, 3471 { 3472 PHY_BCM84834_DEVICEID, 3473 PHY_BCM84834_FIRMWARE, 3474 NULL, 3475 0, 3476 }, 3477 { 0, NULL, NULL }, 3478 }; 3479 3480 static struct info_10gbt_phy_fw *find_phy_info(int devid) 3481 { 3482 int i; 3483 3484 for (i = 0; i < ARRAY_SIZE(phy_info_array); i++) { 3485 if (phy_info_array[i].phy_fw_id == devid) 3486 return &phy_info_array[i]; 3487 } 3488 return NULL; 3489 } 3490 3491 /* Handle updating of chip-external 10Gb/s-BT PHY firmware. This needs to 3492 * happen after the FW_RESET_CMD but before the FW_INITIALIZE_CMD. On error 3493 * we return a negative error number. If we transfer new firmware we return 1 3494 * (from t4_load_phy_fw()). If we don't do anything we return 0. 3495 */ 3496 static int adap_init0_phy(struct adapter *adap) 3497 { 3498 const struct firmware *phyf; 3499 int ret; 3500 struct info_10gbt_phy_fw *phy_info; 3501 3502 /* Use the device ID to determine which PHY file to flash. 3503 */ 3504 phy_info = find_phy_info(adap->pdev->device); 3505 if (!phy_info) { 3506 dev_warn(adap->pdev_dev, 3507 "No PHY Firmware file found for this PHY\n"); 3508 return -EOPNOTSUPP; 3509 } 3510 3511 /* If we have a T4 PHY firmware file under /lib/firmware/cxgb4/, then 3512 * use that. The adapter firmware provides us with a memory buffer 3513 * where we can load a PHY firmware file from the host if we want to 3514 * override the PHY firmware File in flash. 3515 */ 3516 ret = request_firmware_direct(&phyf, phy_info->phy_fw_file, 3517 adap->pdev_dev); 3518 if (ret < 0) { 3519 /* For adapters without FLASH attached to PHY for their 3520 * firmware, it's obviously a fatal error if we can't get the 3521 * firmware to the adapter. For adapters with PHY firmware 3522 * FLASH storage, it's worth a warning if we can't find the 3523 * PHY Firmware but we'll neuter the error ... 3524 */ 3525 dev_err(adap->pdev_dev, "unable to find PHY Firmware image " 3526 "/lib/firmware/%s, error %d\n", 3527 phy_info->phy_fw_file, -ret); 3528 if (phy_info->phy_flash) { 3529 int cur_phy_fw_ver = 0; 3530 3531 t4_phy_fw_ver(adap, &cur_phy_fw_ver); 3532 dev_warn(adap->pdev_dev, "continuing with, on-adapter " 3533 "FLASH copy, version %#x\n", cur_phy_fw_ver); 3534 ret = 0; 3535 } 3536 3537 return ret; 3538 } 3539 3540 /* Load PHY Firmware onto adapter. 3541 */ 3542 ret = t4_load_phy_fw(adap, MEMWIN_NIC, &adap->win0_lock, 3543 phy_info->phy_fw_version, 3544 (u8 *)phyf->data, phyf->size); 3545 if (ret < 0) 3546 dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n", 3547 -ret); 3548 else if (ret > 0) { 3549 int new_phy_fw_ver = 0; 3550 3551 if (phy_info->phy_fw_version) 3552 new_phy_fw_ver = phy_info->phy_fw_version(phyf->data, 3553 phyf->size); 3554 dev_info(adap->pdev_dev, "Successfully transferred PHY " 3555 "Firmware /lib/firmware/%s, version %#x\n", 3556 phy_info->phy_fw_file, new_phy_fw_ver); 3557 } 3558 3559 release_firmware(phyf); 3560 3561 return ret; 3562 } 3563 3564 /* 3565 * Attempt to initialize the adapter via a Firmware Configuration File. 3566 */ 3567 static int adap_init0_config(struct adapter *adapter, int reset) 3568 { 3569 struct fw_caps_config_cmd caps_cmd; 3570 const struct firmware *cf; 3571 unsigned long mtype = 0, maddr = 0; 3572 u32 finiver, finicsum, cfcsum; 3573 int ret; 3574 int config_issued = 0; 3575 char *fw_config_file, fw_config_file_path[256]; 3576 char *config_name = NULL; 3577 3578 /* 3579 * Reset device if necessary. 3580 */ 3581 if (reset) { 3582 ret = t4_fw_reset(adapter, adapter->mbox, 3583 PIORSTMODE_F | PIORST_F); 3584 if (ret < 0) 3585 goto bye; 3586 } 3587 3588 /* If this is a 10Gb/s-BT adapter make sure the chip-external 3589 * 10Gb/s-BT PHYs have up-to-date firmware. Note that this step needs 3590 * to be performed after any global adapter RESET above since some 3591 * PHYs only have local RAM copies of the PHY firmware. 3592 */ 3593 if (is_10gbt_device(adapter->pdev->device)) { 3594 ret = adap_init0_phy(adapter); 3595 if (ret < 0) 3596 goto bye; 3597 } 3598 /* 3599 * If we have a T4 configuration file under /lib/firmware/cxgb4/, 3600 * then use that. Otherwise, use the configuration file stored 3601 * in the adapter flash ... 3602 */ 3603 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { 3604 case CHELSIO_T4: 3605 fw_config_file = FW4_CFNAME; 3606 break; 3607 case CHELSIO_T5: 3608 fw_config_file = FW5_CFNAME; 3609 break; 3610 case CHELSIO_T6: 3611 fw_config_file = FW6_CFNAME; 3612 break; 3613 default: 3614 dev_err(adapter->pdev_dev, "Device %d is not supported\n", 3615 adapter->pdev->device); 3616 ret = -EINVAL; 3617 goto bye; 3618 } 3619 3620 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev); 3621 if (ret < 0) { 3622 config_name = "On FLASH"; 3623 mtype = FW_MEMTYPE_CF_FLASH; 3624 maddr = t4_flash_cfg_addr(adapter); 3625 } else { 3626 u32 params[7], val[7]; 3627 3628 sprintf(fw_config_file_path, 3629 "/lib/firmware/%s", fw_config_file); 3630 config_name = fw_config_file_path; 3631 3632 if (cf->size >= FLASH_CFG_MAX_SIZE) 3633 ret = -ENOMEM; 3634 else { 3635 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3636 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 3637 ret = t4_query_params(adapter, adapter->mbox, 3638 adapter->pf, 0, 1, params, val); 3639 if (ret == 0) { 3640 /* 3641 * For t4_memory_rw() below addresses and 3642 * sizes have to be in terms of multiples of 4 3643 * bytes. So, if the Configuration File isn't 3644 * a multiple of 4 bytes in length we'll have 3645 * to write that out separately since we can't 3646 * guarantee that the bytes following the 3647 * residual byte in the buffer returned by 3648 * request_firmware() are zeroed out ... 3649 */ 3650 size_t resid = cf->size & 0x3; 3651 size_t size = cf->size & ~0x3; 3652 __be32 *data = (__be32 *)cf->data; 3653 3654 mtype = FW_PARAMS_PARAM_Y_G(val[0]); 3655 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16; 3656 3657 spin_lock(&adapter->win0_lock); 3658 ret = t4_memory_rw(adapter, 0, mtype, maddr, 3659 size, data, T4_MEMORY_WRITE); 3660 if (ret == 0 && resid != 0) { 3661 union { 3662 __be32 word; 3663 char buf[4]; 3664 } last; 3665 int i; 3666 3667 last.word = data[size >> 2]; 3668 for (i = resid; i < 4; i++) 3669 last.buf[i] = 0; 3670 ret = t4_memory_rw(adapter, 0, mtype, 3671 maddr + size, 3672 4, &last.word, 3673 T4_MEMORY_WRITE); 3674 } 3675 spin_unlock(&adapter->win0_lock); 3676 } 3677 } 3678 3679 release_firmware(cf); 3680 if (ret) 3681 goto bye; 3682 } 3683 3684 /* 3685 * Issue a Capability Configuration command to the firmware to get it 3686 * to parse the Configuration File. We don't use t4_fw_config_file() 3687 * because we want the ability to modify various features after we've 3688 * processed the configuration file ... 3689 */ 3690 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3691 caps_cmd.op_to_write = 3692 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3693 FW_CMD_REQUEST_F | 3694 FW_CMD_READ_F); 3695 caps_cmd.cfvalid_to_len16 = 3696 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F | 3697 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) | 3698 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) | 3699 FW_LEN16(caps_cmd)); 3700 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 3701 &caps_cmd); 3702 3703 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware 3704 * Configuration File in FLASH), our last gasp effort is to use the 3705 * Firmware Configuration File which is embedded in the firmware. A 3706 * very few early versions of the firmware didn't have one embedded 3707 * but we can ignore those. 3708 */ 3709 if (ret == -ENOENT) { 3710 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3711 caps_cmd.op_to_write = 3712 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3713 FW_CMD_REQUEST_F | 3714 FW_CMD_READ_F); 3715 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3716 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, 3717 sizeof(caps_cmd), &caps_cmd); 3718 config_name = "Firmware Default"; 3719 } 3720 3721 config_issued = 1; 3722 if (ret < 0) 3723 goto bye; 3724 3725 finiver = ntohl(caps_cmd.finiver); 3726 finicsum = ntohl(caps_cmd.finicsum); 3727 cfcsum = ntohl(caps_cmd.cfcsum); 3728 if (finicsum != cfcsum) 3729 dev_warn(adapter->pdev_dev, "Configuration File checksum "\ 3730 "mismatch: [fini] csum=%#x, computed csum=%#x\n", 3731 finicsum, cfcsum); 3732 3733 /* 3734 * And now tell the firmware to use the configuration we just loaded. 3735 */ 3736 caps_cmd.op_to_write = 3737 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3738 FW_CMD_REQUEST_F | 3739 FW_CMD_WRITE_F); 3740 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3741 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 3742 NULL); 3743 if (ret < 0) 3744 goto bye; 3745 3746 /* 3747 * Tweak configuration based on system architecture, module 3748 * parameters, etc. 3749 */ 3750 ret = adap_init0_tweaks(adapter); 3751 if (ret < 0) 3752 goto bye; 3753 3754 /* 3755 * And finally tell the firmware to initialize itself using the 3756 * parameters from the Configuration File. 3757 */ 3758 ret = t4_fw_initialize(adapter, adapter->mbox); 3759 if (ret < 0) 3760 goto bye; 3761 3762 /* Emit Firmware Configuration File information and return 3763 * successfully. 3764 */ 3765 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\ 3766 "Configuration File \"%s\", version %#x, computed checksum %#x\n", 3767 config_name, finiver, cfcsum); 3768 return 0; 3769 3770 /* 3771 * Something bad happened. Return the error ... (If the "error" 3772 * is that there's no Configuration File on the adapter we don't 3773 * want to issue a warning since this is fairly common.) 3774 */ 3775 bye: 3776 if (config_issued && ret != -ENOENT) 3777 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n", 3778 config_name, -ret); 3779 return ret; 3780 } 3781 3782 static struct fw_info fw_info_array[] = { 3783 { 3784 .chip = CHELSIO_T4, 3785 .fs_name = FW4_CFNAME, 3786 .fw_mod_name = FW4_FNAME, 3787 .fw_hdr = { 3788 .chip = FW_HDR_CHIP_T4, 3789 .fw_ver = __cpu_to_be32(FW_VERSION(T4)), 3790 .intfver_nic = FW_INTFVER(T4, NIC), 3791 .intfver_vnic = FW_INTFVER(T4, VNIC), 3792 .intfver_ri = FW_INTFVER(T4, RI), 3793 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 3794 .intfver_fcoe = FW_INTFVER(T4, FCOE), 3795 }, 3796 }, { 3797 .chip = CHELSIO_T5, 3798 .fs_name = FW5_CFNAME, 3799 .fw_mod_name = FW5_FNAME, 3800 .fw_hdr = { 3801 .chip = FW_HDR_CHIP_T5, 3802 .fw_ver = __cpu_to_be32(FW_VERSION(T5)), 3803 .intfver_nic = FW_INTFVER(T5, NIC), 3804 .intfver_vnic = FW_INTFVER(T5, VNIC), 3805 .intfver_ri = FW_INTFVER(T5, RI), 3806 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 3807 .intfver_fcoe = FW_INTFVER(T5, FCOE), 3808 }, 3809 }, { 3810 .chip = CHELSIO_T6, 3811 .fs_name = FW6_CFNAME, 3812 .fw_mod_name = FW6_FNAME, 3813 .fw_hdr = { 3814 .chip = FW_HDR_CHIP_T6, 3815 .fw_ver = __cpu_to_be32(FW_VERSION(T6)), 3816 .intfver_nic = FW_INTFVER(T6, NIC), 3817 .intfver_vnic = FW_INTFVER(T6, VNIC), 3818 .intfver_ofld = FW_INTFVER(T6, OFLD), 3819 .intfver_ri = FW_INTFVER(T6, RI), 3820 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), 3821 .intfver_iscsi = FW_INTFVER(T6, ISCSI), 3822 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), 3823 .intfver_fcoe = FW_INTFVER(T6, FCOE), 3824 }, 3825 } 3826 3827 }; 3828 3829 static struct fw_info *find_fw_info(int chip) 3830 { 3831 int i; 3832 3833 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) { 3834 if (fw_info_array[i].chip == chip) 3835 return &fw_info_array[i]; 3836 } 3837 return NULL; 3838 } 3839 3840 /* 3841 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 3842 */ 3843 static int adap_init0(struct adapter *adap) 3844 { 3845 int ret; 3846 u32 v, port_vec; 3847 enum dev_state state; 3848 u32 params[7], val[7]; 3849 struct fw_caps_config_cmd caps_cmd; 3850 int reset = 1; 3851 3852 /* Grab Firmware Device Log parameters as early as possible so we have 3853 * access to it for debugging, etc. 3854 */ 3855 ret = t4_init_devlog_params(adap); 3856 if (ret < 0) 3857 return ret; 3858 3859 /* Contact FW, advertising Master capability */ 3860 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, 3861 is_kdump_kernel() ? MASTER_MUST : MASTER_MAY, &state); 3862 if (ret < 0) { 3863 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n", 3864 ret); 3865 return ret; 3866 } 3867 if (ret == adap->mbox) 3868 adap->flags |= MASTER_PF; 3869 3870 /* 3871 * If we're the Master PF Driver and the device is uninitialized, 3872 * then let's consider upgrading the firmware ... (We always want 3873 * to check the firmware version number in order to A. get it for 3874 * later reporting and B. to warn if the currently loaded firmware 3875 * is excessively mismatched relative to the driver.) 3876 */ 3877 3878 t4_get_version_info(adap); 3879 ret = t4_check_fw_version(adap); 3880 /* If firmware is too old (not supported by driver) force an update. */ 3881 if (ret) 3882 state = DEV_STATE_UNINIT; 3883 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) { 3884 struct fw_info *fw_info; 3885 struct fw_hdr *card_fw; 3886 const struct firmware *fw; 3887 const u8 *fw_data = NULL; 3888 unsigned int fw_size = 0; 3889 3890 /* This is the firmware whose headers the driver was compiled 3891 * against 3892 */ 3893 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip)); 3894 if (fw_info == NULL) { 3895 dev_err(adap->pdev_dev, 3896 "unable to get firmware info for chip %d.\n", 3897 CHELSIO_CHIP_VERSION(adap->params.chip)); 3898 return -EINVAL; 3899 } 3900 3901 /* allocate memory to read the header of the firmware on the 3902 * card 3903 */ 3904 card_fw = kvzalloc(sizeof(*card_fw), GFP_KERNEL); 3905 3906 /* Get FW from from /lib/firmware/ */ 3907 ret = request_firmware(&fw, fw_info->fw_mod_name, 3908 adap->pdev_dev); 3909 if (ret < 0) { 3910 dev_err(adap->pdev_dev, 3911 "unable to load firmware image %s, error %d\n", 3912 fw_info->fw_mod_name, ret); 3913 } else { 3914 fw_data = fw->data; 3915 fw_size = fw->size; 3916 } 3917 3918 /* upgrade FW logic */ 3919 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw, 3920 state, &reset); 3921 3922 /* Cleaning up */ 3923 release_firmware(fw); 3924 kvfree(card_fw); 3925 3926 if (ret < 0) 3927 goto bye; 3928 } 3929 3930 /* 3931 * Grab VPD parameters. This should be done after we establish a 3932 * connection to the firmware since some of the VPD parameters 3933 * (notably the Core Clock frequency) are retrieved via requests to 3934 * the firmware. On the other hand, we need these fairly early on 3935 * so we do this right after getting ahold of the firmware. 3936 */ 3937 ret = t4_get_vpd_params(adap, &adap->params.vpd); 3938 if (ret < 0) 3939 goto bye; 3940 3941 /* 3942 * Find out what ports are available to us. Note that we need to do 3943 * this before calling adap_init0_no_config() since it needs nports 3944 * and portvec ... 3945 */ 3946 v = 3947 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3948 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 3949 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); 3950 if (ret < 0) 3951 goto bye; 3952 3953 adap->params.nports = hweight32(port_vec); 3954 adap->params.portvec = port_vec; 3955 3956 /* If the firmware is initialized already, emit a simply note to that 3957 * effect. Otherwise, it's time to try initializing the adapter. 3958 */ 3959 if (state == DEV_STATE_INIT) { 3960 dev_info(adap->pdev_dev, "Coming up as %s: "\ 3961 "Adapter already initialized\n", 3962 adap->flags & MASTER_PF ? "MASTER" : "SLAVE"); 3963 } else { 3964 dev_info(adap->pdev_dev, "Coming up as MASTER: "\ 3965 "Initializing adapter\n"); 3966 3967 /* Find out whether we're dealing with a version of the 3968 * firmware which has configuration file support. 3969 */ 3970 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3971 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 3972 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, 3973 params, val); 3974 3975 /* If the firmware doesn't support Configuration Files, 3976 * return an error. 3977 */ 3978 if (ret < 0) { 3979 dev_err(adap->pdev_dev, "firmware doesn't support " 3980 "Firmware Configuration Files\n"); 3981 goto bye; 3982 } 3983 3984 /* The firmware provides us with a memory buffer where we can 3985 * load a Configuration File from the host if we want to 3986 * override the Configuration File in flash. 3987 */ 3988 ret = adap_init0_config(adap, reset); 3989 if (ret == -ENOENT) { 3990 dev_err(adap->pdev_dev, "no Configuration File " 3991 "present on adapter.\n"); 3992 goto bye; 3993 } 3994 if (ret < 0) { 3995 dev_err(adap->pdev_dev, "could not initialize " 3996 "adapter, error %d\n", -ret); 3997 goto bye; 3998 } 3999 } 4000 4001 /* Give the SGE code a chance to pull in anything that it needs ... 4002 * Note that this must be called after we retrieve our VPD parameters 4003 * in order to know how to convert core ticks to seconds, etc. 4004 */ 4005 ret = t4_sge_init(adap); 4006 if (ret < 0) 4007 goto bye; 4008 4009 if (is_bypass_device(adap->pdev->device)) 4010 adap->params.bypass = 1; 4011 4012 /* 4013 * Grab some of our basic fundamental operating parameters. 4014 */ 4015 #define FW_PARAM_DEV(param) \ 4016 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \ 4017 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param)) 4018 4019 #define FW_PARAM_PFVF(param) \ 4020 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \ 4021 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \ 4022 FW_PARAMS_PARAM_Y_V(0) | \ 4023 FW_PARAMS_PARAM_Z_V(0) 4024 4025 params[0] = FW_PARAM_PFVF(EQ_START); 4026 params[1] = FW_PARAM_PFVF(L2T_START); 4027 params[2] = FW_PARAM_PFVF(L2T_END); 4028 params[3] = FW_PARAM_PFVF(FILTER_START); 4029 params[4] = FW_PARAM_PFVF(FILTER_END); 4030 params[5] = FW_PARAM_PFVF(IQFLINT_START); 4031 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, val); 4032 if (ret < 0) 4033 goto bye; 4034 adap->sge.egr_start = val[0]; 4035 adap->l2t_start = val[1]; 4036 adap->l2t_end = val[2]; 4037 adap->tids.ftid_base = val[3]; 4038 adap->tids.nftids = val[4] - val[3] + 1; 4039 adap->sge.ingr_start = val[5]; 4040 4041 /* qids (ingress/egress) returned from firmware can be anywhere 4042 * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END. 4043 * Hence driver needs to allocate memory for this range to 4044 * store the queue info. Get the highest IQFLINT/EQ index returned 4045 * in FW_EQ_*_CMD.alloc command. 4046 */ 4047 params[0] = FW_PARAM_PFVF(EQ_END); 4048 params[1] = FW_PARAM_PFVF(IQFLINT_END); 4049 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 4050 if (ret < 0) 4051 goto bye; 4052 adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1; 4053 adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1; 4054 4055 adap->sge.egr_map = kcalloc(adap->sge.egr_sz, 4056 sizeof(*adap->sge.egr_map), GFP_KERNEL); 4057 if (!adap->sge.egr_map) { 4058 ret = -ENOMEM; 4059 goto bye; 4060 } 4061 4062 adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz, 4063 sizeof(*adap->sge.ingr_map), GFP_KERNEL); 4064 if (!adap->sge.ingr_map) { 4065 ret = -ENOMEM; 4066 goto bye; 4067 } 4068 4069 /* Allocate the memory for the vaious egress queue bitmaps 4070 * ie starving_fl, txq_maperr and blocked_fl. 4071 */ 4072 adap->sge.starving_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 4073 sizeof(long), GFP_KERNEL); 4074 if (!adap->sge.starving_fl) { 4075 ret = -ENOMEM; 4076 goto bye; 4077 } 4078 4079 adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 4080 sizeof(long), GFP_KERNEL); 4081 if (!adap->sge.txq_maperr) { 4082 ret = -ENOMEM; 4083 goto bye; 4084 } 4085 4086 #ifdef CONFIG_DEBUG_FS 4087 adap->sge.blocked_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 4088 sizeof(long), GFP_KERNEL); 4089 if (!adap->sge.blocked_fl) { 4090 ret = -ENOMEM; 4091 goto bye; 4092 } 4093 #endif 4094 4095 params[0] = FW_PARAM_PFVF(CLIP_START); 4096 params[1] = FW_PARAM_PFVF(CLIP_END); 4097 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 4098 if (ret < 0) 4099 goto bye; 4100 adap->clipt_start = val[0]; 4101 adap->clipt_end = val[1]; 4102 4103 /* We don't yet have a PARAMs calls to retrieve the number of Traffic 4104 * Classes supported by the hardware/firmware so we hard code it here 4105 * for now. 4106 */ 4107 adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16; 4108 4109 /* query params related to active filter region */ 4110 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START); 4111 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END); 4112 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 4113 /* If Active filter size is set we enable establishing 4114 * offload connection through firmware work request 4115 */ 4116 if ((val[0] != val[1]) && (ret >= 0)) { 4117 adap->flags |= FW_OFLD_CONN; 4118 adap->tids.aftid_base = val[0]; 4119 adap->tids.aftid_end = val[1]; 4120 } 4121 4122 /* If we're running on newer firmware, let it know that we're 4123 * prepared to deal with encapsulated CPL messages. Older 4124 * firmware won't understand this and we'll just get 4125 * unencapsulated messages ... 4126 */ 4127 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 4128 val[0] = 1; 4129 (void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val); 4130 4131 /* 4132 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL 4133 * capability. Earlier versions of the firmware didn't have the 4134 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no 4135 * permission to use ULPTX MEMWRITE DSGL. 4136 */ 4137 if (is_t4(adap->params.chip)) { 4138 adap->params.ulptx_memwrite_dsgl = false; 4139 } else { 4140 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); 4141 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 4142 1, params, val); 4143 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0); 4144 } 4145 4146 /* See if FW supports FW_RI_FR_NSMR_TPTE_WR work request */ 4147 params[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR); 4148 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 4149 1, params, val); 4150 adap->params.fr_nsmr_tpte_wr_support = (ret == 0 && val[0] != 0); 4151 4152 /* See if FW supports FW_FILTER2 work request */ 4153 if (is_t4(adap->params.chip)) { 4154 adap->params.filter2_wr_support = 0; 4155 } else { 4156 params[0] = FW_PARAM_DEV(FILTER2_WR); 4157 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 4158 1, params, val); 4159 adap->params.filter2_wr_support = (ret == 0 && val[0] != 0); 4160 } 4161 4162 /* 4163 * Get device capabilities so we can determine what resources we need 4164 * to manage. 4165 */ 4166 memset(&caps_cmd, 0, sizeof(caps_cmd)); 4167 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4168 FW_CMD_REQUEST_F | FW_CMD_READ_F); 4169 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 4170 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd), 4171 &caps_cmd); 4172 if (ret < 0) 4173 goto bye; 4174 4175 if (caps_cmd.ofldcaps || 4176 (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER))) { 4177 /* query offload-related parameters */ 4178 params[0] = FW_PARAM_DEV(NTID); 4179 params[1] = FW_PARAM_PFVF(SERVER_START); 4180 params[2] = FW_PARAM_PFVF(SERVER_END); 4181 params[3] = FW_PARAM_PFVF(TDDP_START); 4182 params[4] = FW_PARAM_PFVF(TDDP_END); 4183 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 4184 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 4185 params, val); 4186 if (ret < 0) 4187 goto bye; 4188 adap->tids.ntids = val[0]; 4189 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); 4190 adap->tids.stid_base = val[1]; 4191 adap->tids.nstids = val[2] - val[1] + 1; 4192 /* 4193 * Setup server filter region. Divide the available filter 4194 * region into two parts. Regular filters get 1/3rd and server 4195 * filters get 2/3rd part. This is only enabled if workarond 4196 * path is enabled. 4197 * 1. For regular filters. 4198 * 2. Server filter: This are special filters which are used 4199 * to redirect SYN packets to offload queue. 4200 */ 4201 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) { 4202 adap->tids.sftid_base = adap->tids.ftid_base + 4203 DIV_ROUND_UP(adap->tids.nftids, 3); 4204 adap->tids.nsftids = adap->tids.nftids - 4205 DIV_ROUND_UP(adap->tids.nftids, 3); 4206 adap->tids.nftids = adap->tids.sftid_base - 4207 adap->tids.ftid_base; 4208 } 4209 adap->vres.ddp.start = val[3]; 4210 adap->vres.ddp.size = val[4] - val[3] + 1; 4211 adap->params.ofldq_wr_cred = val[5]; 4212 4213 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_HASHFILTER)) { 4214 if (init_hash_filter(adap) < 0) 4215 goto bye; 4216 } else { 4217 adap->params.offload = 1; 4218 adap->num_ofld_uld += 1; 4219 } 4220 } 4221 if (caps_cmd.rdmacaps) { 4222 params[0] = FW_PARAM_PFVF(STAG_START); 4223 params[1] = FW_PARAM_PFVF(STAG_END); 4224 params[2] = FW_PARAM_PFVF(RQ_START); 4225 params[3] = FW_PARAM_PFVF(RQ_END); 4226 params[4] = FW_PARAM_PFVF(PBL_START); 4227 params[5] = FW_PARAM_PFVF(PBL_END); 4228 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 4229 params, val); 4230 if (ret < 0) 4231 goto bye; 4232 adap->vres.stag.start = val[0]; 4233 adap->vres.stag.size = val[1] - val[0] + 1; 4234 adap->vres.rq.start = val[2]; 4235 adap->vres.rq.size = val[3] - val[2] + 1; 4236 adap->vres.pbl.start = val[4]; 4237 adap->vres.pbl.size = val[5] - val[4] + 1; 4238 4239 params[0] = FW_PARAM_PFVF(SQRQ_START); 4240 params[1] = FW_PARAM_PFVF(SQRQ_END); 4241 params[2] = FW_PARAM_PFVF(CQ_START); 4242 params[3] = FW_PARAM_PFVF(CQ_END); 4243 params[4] = FW_PARAM_PFVF(OCQ_START); 4244 params[5] = FW_PARAM_PFVF(OCQ_END); 4245 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, 4246 val); 4247 if (ret < 0) 4248 goto bye; 4249 adap->vres.qp.start = val[0]; 4250 adap->vres.qp.size = val[1] - val[0] + 1; 4251 adap->vres.cq.start = val[2]; 4252 adap->vres.cq.size = val[3] - val[2] + 1; 4253 adap->vres.ocq.start = val[4]; 4254 adap->vres.ocq.size = val[5] - val[4] + 1; 4255 4256 params[0] = FW_PARAM_DEV(MAXORDIRD_QP); 4257 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER); 4258 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, 4259 val); 4260 if (ret < 0) { 4261 adap->params.max_ordird_qp = 8; 4262 adap->params.max_ird_adapter = 32 * adap->tids.ntids; 4263 ret = 0; 4264 } else { 4265 adap->params.max_ordird_qp = val[0]; 4266 adap->params.max_ird_adapter = val[1]; 4267 } 4268 dev_info(adap->pdev_dev, 4269 "max_ordird_qp %d max_ird_adapter %d\n", 4270 adap->params.max_ordird_qp, 4271 adap->params.max_ird_adapter); 4272 adap->num_ofld_uld += 2; 4273 } 4274 if (caps_cmd.iscsicaps) { 4275 params[0] = FW_PARAM_PFVF(ISCSI_START); 4276 params[1] = FW_PARAM_PFVF(ISCSI_END); 4277 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 4278 params, val); 4279 if (ret < 0) 4280 goto bye; 4281 adap->vres.iscsi.start = val[0]; 4282 adap->vres.iscsi.size = val[1] - val[0] + 1; 4283 /* LIO target and cxgb4i initiaitor */ 4284 adap->num_ofld_uld += 2; 4285 } 4286 if (caps_cmd.cryptocaps) { 4287 /* Should query params here...TODO */ 4288 params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE); 4289 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 4290 params, val); 4291 if (ret < 0) { 4292 if (ret != -EINVAL) 4293 goto bye; 4294 } else { 4295 adap->vres.ncrypto_fc = val[0]; 4296 } 4297 adap->params.crypto = ntohs(caps_cmd.cryptocaps); 4298 adap->num_uld += 1; 4299 } 4300 #undef FW_PARAM_PFVF 4301 #undef FW_PARAM_DEV 4302 4303 /* The MTU/MSS Table is initialized by now, so load their values. If 4304 * we're initializing the adapter, then we'll make any modifications 4305 * we want to the MTU/MSS Table and also initialize the congestion 4306 * parameters. 4307 */ 4308 t4_read_mtu_tbl(adap, adap->params.mtus, NULL); 4309 if (state != DEV_STATE_INIT) { 4310 int i; 4311 4312 /* The default MTU Table contains values 1492 and 1500. 4313 * However, for TCP, it's better to have two values which are 4314 * a multiple of 8 +/- 4 bytes apart near this popular MTU. 4315 * This allows us to have a TCP Data Payload which is a 4316 * multiple of 8 regardless of what combination of TCP Options 4317 * are in use (always a multiple of 4 bytes) which is 4318 * important for performance reasons. For instance, if no 4319 * options are in use, then we have a 20-byte IP header and a 4320 * 20-byte TCP header. In this case, a 1500-byte MSS would 4321 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes 4322 * which is not a multiple of 8. So using an MSS of 1488 in 4323 * this case results in a TCP Data Payload of 1448 bytes which 4324 * is a multiple of 8. On the other hand, if 12-byte TCP Time 4325 * Stamps have been negotiated, then an MTU of 1500 bytes 4326 * results in a TCP Data Payload of 1448 bytes which, as 4327 * above, is a multiple of 8 bytes ... 4328 */ 4329 for (i = 0; i < NMTUS; i++) 4330 if (adap->params.mtus[i] == 1492) { 4331 adap->params.mtus[i] = 1488; 4332 break; 4333 } 4334 4335 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 4336 adap->params.b_wnd); 4337 } 4338 t4_init_sge_params(adap); 4339 adap->flags |= FW_OK; 4340 t4_init_tp_params(adap, true); 4341 return 0; 4342 4343 /* 4344 * Something bad happened. If a command timed out or failed with EIO 4345 * FW does not operate within its spec or something catastrophic 4346 * happened to HW/FW, stop issuing commands. 4347 */ 4348 bye: 4349 kfree(adap->sge.egr_map); 4350 kfree(adap->sge.ingr_map); 4351 kfree(adap->sge.starving_fl); 4352 kfree(adap->sge.txq_maperr); 4353 #ifdef CONFIG_DEBUG_FS 4354 kfree(adap->sge.blocked_fl); 4355 #endif 4356 if (ret != -ETIMEDOUT && ret != -EIO) 4357 t4_fw_bye(adap, adap->mbox); 4358 return ret; 4359 } 4360 4361 /* EEH callbacks */ 4362 4363 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev, 4364 pci_channel_state_t state) 4365 { 4366 int i; 4367 struct adapter *adap = pci_get_drvdata(pdev); 4368 4369 if (!adap) 4370 goto out; 4371 4372 rtnl_lock(); 4373 adap->flags &= ~FW_OK; 4374 notify_ulds(adap, CXGB4_STATE_START_RECOVERY); 4375 spin_lock(&adap->stats_lock); 4376 for_each_port(adap, i) { 4377 struct net_device *dev = adap->port[i]; 4378 if (dev) { 4379 netif_device_detach(dev); 4380 netif_carrier_off(dev); 4381 } 4382 } 4383 spin_unlock(&adap->stats_lock); 4384 disable_interrupts(adap); 4385 if (adap->flags & FULL_INIT_DONE) 4386 cxgb_down(adap); 4387 rtnl_unlock(); 4388 if ((adap->flags & DEV_ENABLED)) { 4389 pci_disable_device(pdev); 4390 adap->flags &= ~DEV_ENABLED; 4391 } 4392 out: return state == pci_channel_io_perm_failure ? 4393 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 4394 } 4395 4396 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) 4397 { 4398 int i, ret; 4399 struct fw_caps_config_cmd c; 4400 struct adapter *adap = pci_get_drvdata(pdev); 4401 4402 if (!adap) { 4403 pci_restore_state(pdev); 4404 pci_save_state(pdev); 4405 return PCI_ERS_RESULT_RECOVERED; 4406 } 4407 4408 if (!(adap->flags & DEV_ENABLED)) { 4409 if (pci_enable_device(pdev)) { 4410 dev_err(&pdev->dev, "Cannot reenable PCI " 4411 "device after reset\n"); 4412 return PCI_ERS_RESULT_DISCONNECT; 4413 } 4414 adap->flags |= DEV_ENABLED; 4415 } 4416 4417 pci_set_master(pdev); 4418 pci_restore_state(pdev); 4419 pci_save_state(pdev); 4420 pci_cleanup_aer_uncorrect_error_status(pdev); 4421 4422 if (t4_wait_dev_ready(adap->regs) < 0) 4423 return PCI_ERS_RESULT_DISCONNECT; 4424 if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0) 4425 return PCI_ERS_RESULT_DISCONNECT; 4426 adap->flags |= FW_OK; 4427 if (adap_init1(adap, &c)) 4428 return PCI_ERS_RESULT_DISCONNECT; 4429 4430 for_each_port(adap, i) { 4431 struct port_info *p = adap2pinfo(adap, i); 4432 4433 ret = t4_alloc_vi(adap, adap->mbox, p->tx_chan, adap->pf, 0, 1, 4434 NULL, NULL); 4435 if (ret < 0) 4436 return PCI_ERS_RESULT_DISCONNECT; 4437 p->viid = ret; 4438 p->xact_addr_filt = -1; 4439 } 4440 4441 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 4442 adap->params.b_wnd); 4443 setup_memwin(adap); 4444 if (cxgb_up(adap)) 4445 return PCI_ERS_RESULT_DISCONNECT; 4446 return PCI_ERS_RESULT_RECOVERED; 4447 } 4448 4449 static void eeh_resume(struct pci_dev *pdev) 4450 { 4451 int i; 4452 struct adapter *adap = pci_get_drvdata(pdev); 4453 4454 if (!adap) 4455 return; 4456 4457 rtnl_lock(); 4458 for_each_port(adap, i) { 4459 struct net_device *dev = adap->port[i]; 4460 if (dev) { 4461 if (netif_running(dev)) { 4462 link_start(dev); 4463 cxgb_set_rxmode(dev); 4464 } 4465 netif_device_attach(dev); 4466 } 4467 } 4468 rtnl_unlock(); 4469 } 4470 4471 static const struct pci_error_handlers cxgb4_eeh = { 4472 .error_detected = eeh_err_detected, 4473 .slot_reset = eeh_slot_reset, 4474 .resume = eeh_resume, 4475 }; 4476 4477 /* Return true if the Link Configuration supports "High Speeds" (those greater 4478 * than 1Gb/s). 4479 */ 4480 static inline bool is_x_10g_port(const struct link_config *lc) 4481 { 4482 unsigned int speeds, high_speeds; 4483 4484 speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps)); 4485 high_speeds = speeds & 4486 ~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G); 4487 4488 return high_speeds != 0; 4489 } 4490 4491 /* 4492 * Perform default configuration of DMA queues depending on the number and type 4493 * of ports we found and the number of available CPUs. Most settings can be 4494 * modified by the admin prior to actual use. 4495 */ 4496 static void cfg_queues(struct adapter *adap) 4497 { 4498 struct sge *s = &adap->sge; 4499 int i = 0, n10g = 0, qidx = 0; 4500 #ifndef CONFIG_CHELSIO_T4_DCB 4501 int q10g = 0; 4502 #endif 4503 4504 /* Reduce memory usage in kdump environment, disable all offload. 4505 */ 4506 if (is_kdump_kernel() || (is_uld(adap) && t4_uld_mem_alloc(adap))) { 4507 adap->params.offload = 0; 4508 adap->params.crypto = 0; 4509 } 4510 4511 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg); 4512 #ifdef CONFIG_CHELSIO_T4_DCB 4513 /* For Data Center Bridging support we need to be able to support up 4514 * to 8 Traffic Priorities; each of which will be assigned to its 4515 * own TX Queue in order to prevent Head-Of-Line Blocking. 4516 */ 4517 if (adap->params.nports * 8 > MAX_ETH_QSETS) { 4518 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n", 4519 MAX_ETH_QSETS, adap->params.nports * 8); 4520 BUG_ON(1); 4521 } 4522 4523 for_each_port(adap, i) { 4524 struct port_info *pi = adap2pinfo(adap, i); 4525 4526 pi->first_qset = qidx; 4527 pi->nqsets = is_kdump_kernel() ? 1 : 8; 4528 qidx += pi->nqsets; 4529 } 4530 #else /* !CONFIG_CHELSIO_T4_DCB */ 4531 /* 4532 * We default to 1 queue per non-10G port and up to # of cores queues 4533 * per 10G port. 4534 */ 4535 if (n10g) 4536 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g; 4537 if (q10g > netif_get_num_default_rss_queues()) 4538 q10g = netif_get_num_default_rss_queues(); 4539 4540 if (is_kdump_kernel()) 4541 q10g = 1; 4542 4543 for_each_port(adap, i) { 4544 struct port_info *pi = adap2pinfo(adap, i); 4545 4546 pi->first_qset = qidx; 4547 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1; 4548 qidx += pi->nqsets; 4549 } 4550 #endif /* !CONFIG_CHELSIO_T4_DCB */ 4551 4552 s->ethqsets = qidx; 4553 s->max_ethqsets = qidx; /* MSI-X may lower it later */ 4554 4555 if (is_uld(adap)) { 4556 /* 4557 * For offload we use 1 queue/channel if all ports are up to 1G, 4558 * otherwise we divide all available queues amongst the channels 4559 * capped by the number of available cores. 4560 */ 4561 if (n10g) { 4562 i = min_t(int, MAX_OFLD_QSETS, num_online_cpus()); 4563 s->ofldqsets = roundup(i, adap->params.nports); 4564 } else { 4565 s->ofldqsets = adap->params.nports; 4566 } 4567 } 4568 4569 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { 4570 struct sge_eth_rxq *r = &s->ethrxq[i]; 4571 4572 init_rspq(adap, &r->rspq, 5, 10, 1024, 64); 4573 r->fl.size = 72; 4574 } 4575 4576 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) 4577 s->ethtxq[i].q.size = 1024; 4578 4579 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) 4580 s->ctrlq[i].q.size = 512; 4581 4582 if (!is_t4(adap->params.chip)) 4583 s->ptptxq.q.size = 8; 4584 4585 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64); 4586 init_rspq(adap, &s->intrq, 0, 1, 512, 64); 4587 } 4588 4589 /* 4590 * Reduce the number of Ethernet queues across all ports to at most n. 4591 * n provides at least one queue per port. 4592 */ 4593 static void reduce_ethqs(struct adapter *adap, int n) 4594 { 4595 int i; 4596 struct port_info *pi; 4597 4598 while (n < adap->sge.ethqsets) 4599 for_each_port(adap, i) { 4600 pi = adap2pinfo(adap, i); 4601 if (pi->nqsets > 1) { 4602 pi->nqsets--; 4603 adap->sge.ethqsets--; 4604 if (adap->sge.ethqsets <= n) 4605 break; 4606 } 4607 } 4608 4609 n = 0; 4610 for_each_port(adap, i) { 4611 pi = adap2pinfo(adap, i); 4612 pi->first_qset = n; 4613 n += pi->nqsets; 4614 } 4615 } 4616 4617 static int get_msix_info(struct adapter *adap) 4618 { 4619 struct uld_msix_info *msix_info; 4620 unsigned int max_ingq = 0; 4621 4622 if (is_offload(adap)) 4623 max_ingq += MAX_OFLD_QSETS * adap->num_ofld_uld; 4624 if (is_pci_uld(adap)) 4625 max_ingq += MAX_OFLD_QSETS * adap->num_uld; 4626 4627 if (!max_ingq) 4628 goto out; 4629 4630 msix_info = kcalloc(max_ingq, sizeof(*msix_info), GFP_KERNEL); 4631 if (!msix_info) 4632 return -ENOMEM; 4633 4634 adap->msix_bmap_ulds.msix_bmap = kcalloc(BITS_TO_LONGS(max_ingq), 4635 sizeof(long), GFP_KERNEL); 4636 if (!adap->msix_bmap_ulds.msix_bmap) { 4637 kfree(msix_info); 4638 return -ENOMEM; 4639 } 4640 spin_lock_init(&adap->msix_bmap_ulds.lock); 4641 adap->msix_info_ulds = msix_info; 4642 out: 4643 return 0; 4644 } 4645 4646 static void free_msix_info(struct adapter *adap) 4647 { 4648 if (!(adap->num_uld && adap->num_ofld_uld)) 4649 return; 4650 4651 kfree(adap->msix_info_ulds); 4652 kfree(adap->msix_bmap_ulds.msix_bmap); 4653 } 4654 4655 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */ 4656 #define EXTRA_VECS 2 4657 4658 static int enable_msix(struct adapter *adap) 4659 { 4660 int ofld_need = 0, uld_need = 0; 4661 int i, j, want, need, allocated; 4662 struct sge *s = &adap->sge; 4663 unsigned int nchan = adap->params.nports; 4664 struct msix_entry *entries; 4665 int max_ingq = MAX_INGQ; 4666 4667 if (is_pci_uld(adap)) 4668 max_ingq += (MAX_OFLD_QSETS * adap->num_uld); 4669 if (is_offload(adap)) 4670 max_ingq += (MAX_OFLD_QSETS * adap->num_ofld_uld); 4671 entries = kmalloc(sizeof(*entries) * (max_ingq + 1), 4672 GFP_KERNEL); 4673 if (!entries) 4674 return -ENOMEM; 4675 4676 /* map for msix */ 4677 if (get_msix_info(adap)) { 4678 adap->params.offload = 0; 4679 adap->params.crypto = 0; 4680 } 4681 4682 for (i = 0; i < max_ingq + 1; ++i) 4683 entries[i].entry = i; 4684 4685 want = s->max_ethqsets + EXTRA_VECS; 4686 if (is_offload(adap)) { 4687 want += adap->num_ofld_uld * s->ofldqsets; 4688 ofld_need = adap->num_ofld_uld * nchan; 4689 } 4690 if (is_pci_uld(adap)) { 4691 want += adap->num_uld * s->ofldqsets; 4692 uld_need = adap->num_uld * nchan; 4693 } 4694 #ifdef CONFIG_CHELSIO_T4_DCB 4695 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for 4696 * each port. 4697 */ 4698 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need + uld_need; 4699 #else 4700 need = adap->params.nports + EXTRA_VECS + ofld_need + uld_need; 4701 #endif 4702 allocated = pci_enable_msix_range(adap->pdev, entries, need, want); 4703 if (allocated < 0) { 4704 dev_info(adap->pdev_dev, "not enough MSI-X vectors left," 4705 " not using MSI-X\n"); 4706 kfree(entries); 4707 return allocated; 4708 } 4709 4710 /* Distribute available vectors to the various queue groups. 4711 * Every group gets its minimum requirement and NIC gets top 4712 * priority for leftovers. 4713 */ 4714 i = allocated - EXTRA_VECS - ofld_need - uld_need; 4715 if (i < s->max_ethqsets) { 4716 s->max_ethqsets = i; 4717 if (i < s->ethqsets) 4718 reduce_ethqs(adap, i); 4719 } 4720 if (is_uld(adap)) { 4721 if (allocated < want) 4722 s->nqs_per_uld = nchan; 4723 else 4724 s->nqs_per_uld = s->ofldqsets; 4725 } 4726 4727 for (i = 0; i < (s->max_ethqsets + EXTRA_VECS); ++i) 4728 adap->msix_info[i].vec = entries[i].vector; 4729 if (is_uld(adap)) { 4730 for (j = 0 ; i < allocated; ++i, j++) { 4731 adap->msix_info_ulds[j].vec = entries[i].vector; 4732 adap->msix_info_ulds[j].idx = i; 4733 } 4734 adap->msix_bmap_ulds.mapsize = j; 4735 } 4736 dev_info(adap->pdev_dev, "%d MSI-X vectors allocated, " 4737 "nic %d per uld %d\n", 4738 allocated, s->max_ethqsets, s->nqs_per_uld); 4739 4740 kfree(entries); 4741 return 0; 4742 } 4743 4744 #undef EXTRA_VECS 4745 4746 static int init_rss(struct adapter *adap) 4747 { 4748 unsigned int i; 4749 int err; 4750 4751 err = t4_init_rss_mode(adap, adap->mbox); 4752 if (err) 4753 return err; 4754 4755 for_each_port(adap, i) { 4756 struct port_info *pi = adap2pinfo(adap, i); 4757 4758 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); 4759 if (!pi->rss) 4760 return -ENOMEM; 4761 } 4762 return 0; 4763 } 4764 4765 static int cxgb4_get_pcie_dev_link_caps(struct adapter *adap, 4766 enum pci_bus_speed *speed, 4767 enum pcie_link_width *width) 4768 { 4769 u32 lnkcap1, lnkcap2; 4770 int err1, err2; 4771 4772 #define PCIE_MLW_CAP_SHIFT 4 /* start of MLW mask in link capabilities */ 4773 4774 *speed = PCI_SPEED_UNKNOWN; 4775 *width = PCIE_LNK_WIDTH_UNKNOWN; 4776 4777 err1 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP, 4778 &lnkcap1); 4779 err2 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP2, 4780 &lnkcap2); 4781 if (!err2 && lnkcap2) { /* PCIe r3.0-compliant */ 4782 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) 4783 *speed = PCIE_SPEED_8_0GT; 4784 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) 4785 *speed = PCIE_SPEED_5_0GT; 4786 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) 4787 *speed = PCIE_SPEED_2_5GT; 4788 } 4789 if (!err1) { 4790 *width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT; 4791 if (!lnkcap2) { /* pre-r3.0 */ 4792 if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB) 4793 *speed = PCIE_SPEED_5_0GT; 4794 else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB) 4795 *speed = PCIE_SPEED_2_5GT; 4796 } 4797 } 4798 4799 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 4800 return err1 ? err1 : err2 ? err2 : -EINVAL; 4801 return 0; 4802 } 4803 4804 static void cxgb4_check_pcie_caps(struct adapter *adap) 4805 { 4806 enum pcie_link_width width, width_cap; 4807 enum pci_bus_speed speed, speed_cap; 4808 4809 #define PCIE_SPEED_STR(speed) \ 4810 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \ 4811 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \ 4812 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \ 4813 "Unknown") 4814 4815 if (cxgb4_get_pcie_dev_link_caps(adap, &speed_cap, &width_cap)) { 4816 dev_warn(adap->pdev_dev, 4817 "Unable to determine PCIe device BW capabilities\n"); 4818 return; 4819 } 4820 4821 if (pcie_get_minimum_link(adap->pdev, &speed, &width) || 4822 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { 4823 dev_warn(adap->pdev_dev, 4824 "Unable to determine PCI Express bandwidth.\n"); 4825 return; 4826 } 4827 4828 dev_info(adap->pdev_dev, "PCIe link speed is %s, device supports %s\n", 4829 PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap)); 4830 dev_info(adap->pdev_dev, "PCIe link width is x%d, device supports x%d\n", 4831 width, width_cap); 4832 if (speed < speed_cap || width < width_cap) 4833 dev_info(adap->pdev_dev, 4834 "A slot with more lanes and/or higher speed is " 4835 "suggested for optimal performance.\n"); 4836 } 4837 4838 /* Dump basic information about the adapter */ 4839 static void print_adapter_info(struct adapter *adapter) 4840 { 4841 /* Hardware/Firmware/etc. Version/Revision IDs */ 4842 t4_dump_version_info(adapter); 4843 4844 /* Software/Hardware configuration */ 4845 dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n", 4846 is_offload(adapter) ? "R" : "", 4847 ((adapter->flags & USING_MSIX) ? "MSI-X" : 4848 (adapter->flags & USING_MSI) ? "MSI" : ""), 4849 is_offload(adapter) ? "Offload" : "non-Offload"); 4850 } 4851 4852 static void print_port_info(const struct net_device *dev) 4853 { 4854 char buf[80]; 4855 char *bufp = buf; 4856 const char *spd = ""; 4857 const struct port_info *pi = netdev_priv(dev); 4858 const struct adapter *adap = pi->adapter; 4859 4860 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB) 4861 spd = " 2.5 GT/s"; 4862 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB) 4863 spd = " 5 GT/s"; 4864 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB) 4865 spd = " 8 GT/s"; 4866 4867 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M) 4868 bufp += sprintf(bufp, "100M/"); 4869 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G) 4870 bufp += sprintf(bufp, "1G/"); 4871 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G) 4872 bufp += sprintf(bufp, "10G/"); 4873 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G) 4874 bufp += sprintf(bufp, "25G/"); 4875 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G) 4876 bufp += sprintf(bufp, "40G/"); 4877 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G) 4878 bufp += sprintf(bufp, "50G/"); 4879 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G) 4880 bufp += sprintf(bufp, "100G/"); 4881 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_200G) 4882 bufp += sprintf(bufp, "200G/"); 4883 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_400G) 4884 bufp += sprintf(bufp, "400G/"); 4885 if (bufp != buf) 4886 --bufp; 4887 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type)); 4888 4889 netdev_info(dev, "%s: Chelsio %s (%s) %s\n", 4890 dev->name, adap->params.vpd.id, adap->name, buf); 4891 } 4892 4893 /* 4894 * Free the following resources: 4895 * - memory used for tables 4896 * - MSI/MSI-X 4897 * - net devices 4898 * - resources FW is holding for us 4899 */ 4900 static void free_some_resources(struct adapter *adapter) 4901 { 4902 unsigned int i; 4903 4904 kvfree(adapter->smt); 4905 kvfree(adapter->l2t); 4906 t4_cleanup_sched(adapter); 4907 kvfree(adapter->tids.tid_tab); 4908 cxgb4_cleanup_tc_flower(adapter); 4909 cxgb4_cleanup_tc_u32(adapter); 4910 kfree(adapter->sge.egr_map); 4911 kfree(adapter->sge.ingr_map); 4912 kfree(adapter->sge.starving_fl); 4913 kfree(adapter->sge.txq_maperr); 4914 #ifdef CONFIG_DEBUG_FS 4915 kfree(adapter->sge.blocked_fl); 4916 #endif 4917 disable_msi(adapter); 4918 4919 for_each_port(adapter, i) 4920 if (adapter->port[i]) { 4921 struct port_info *pi = adap2pinfo(adapter, i); 4922 4923 if (pi->viid != 0) 4924 t4_free_vi(adapter, adapter->mbox, adapter->pf, 4925 0, pi->viid); 4926 kfree(adap2pinfo(adapter, i)->rss); 4927 free_netdev(adapter->port[i]); 4928 } 4929 if (adapter->flags & FW_OK) 4930 t4_fw_bye(adapter, adapter->pf); 4931 } 4932 4933 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN) 4934 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ 4935 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) 4936 #define SEGMENT_SIZE 128 4937 4938 static int get_chip_type(struct pci_dev *pdev, u32 pl_rev) 4939 { 4940 u16 device_id; 4941 4942 /* Retrieve adapter's device ID */ 4943 pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id); 4944 4945 switch (device_id >> 12) { 4946 case CHELSIO_T4: 4947 return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev); 4948 case CHELSIO_T5: 4949 return CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); 4950 case CHELSIO_T6: 4951 return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); 4952 default: 4953 dev_err(&pdev->dev, "Device %d is not supported\n", 4954 device_id); 4955 } 4956 return -EINVAL; 4957 } 4958 4959 #ifdef CONFIG_PCI_IOV 4960 static void cxgb4_mgmt_setup(struct net_device *dev) 4961 { 4962 dev->type = ARPHRD_NONE; 4963 dev->mtu = 0; 4964 dev->hard_header_len = 0; 4965 dev->addr_len = 0; 4966 dev->tx_queue_len = 0; 4967 dev->flags |= IFF_NOARP; 4968 dev->priv_flags |= IFF_NO_QUEUE; 4969 4970 /* Initialize the device structure. */ 4971 dev->netdev_ops = &cxgb4_mgmt_netdev_ops; 4972 dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops; 4973 dev->needs_free_netdev = true; 4974 } 4975 4976 static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) 4977 { 4978 struct adapter *adap = pci_get_drvdata(pdev); 4979 int err = 0; 4980 int current_vfs = pci_num_vf(pdev); 4981 u32 pcie_fw; 4982 4983 pcie_fw = readl(adap->regs + PCIE_FW_A); 4984 /* Check if cxgb4 is the MASTER and fw is initialized */ 4985 if (!(pcie_fw & PCIE_FW_INIT_F) || 4986 !(pcie_fw & PCIE_FW_MASTER_VLD_F) || 4987 PCIE_FW_MASTER_G(pcie_fw) != CXGB4_UNIFIED_PF) { 4988 dev_warn(&pdev->dev, 4989 "cxgb4 driver needs to be MASTER to support SRIOV\n"); 4990 return -EOPNOTSUPP; 4991 } 4992 4993 /* If any of the VF's is already assigned to Guest OS, then 4994 * SRIOV for the same cannot be modified 4995 */ 4996 if (current_vfs && pci_vfs_assigned(pdev)) { 4997 dev_err(&pdev->dev, 4998 "Cannot modify SR-IOV while VFs are assigned\n"); 4999 return current_vfs; 5000 } 5001 /* Note that the upper-level code ensures that we're never called with 5002 * a non-zero "num_vfs" when we already have VFs instantiated. But 5003 * it never hurts to code defensively. 5004 */ 5005 if (num_vfs != 0 && current_vfs != 0) 5006 return -EBUSY; 5007 5008 /* Nothing to do for no change. */ 5009 if (num_vfs == current_vfs) 5010 return num_vfs; 5011 5012 /* Disable SRIOV when zero is passed. */ 5013 if (!num_vfs) { 5014 pci_disable_sriov(pdev); 5015 /* free VF Management Interface */ 5016 unregister_netdev(adap->port[0]); 5017 free_netdev(adap->port[0]); 5018 adap->port[0] = NULL; 5019 5020 /* free VF resources */ 5021 adap->num_vfs = 0; 5022 kfree(adap->vfinfo); 5023 adap->vfinfo = NULL; 5024 return 0; 5025 } 5026 5027 if (!current_vfs) { 5028 struct fw_pfvf_cmd port_cmd, port_rpl; 5029 struct net_device *netdev; 5030 unsigned int pmask, port; 5031 struct pci_dev *pbridge; 5032 struct port_info *pi; 5033 char name[IFNAMSIZ]; 5034 u32 devcap2; 5035 u16 flags; 5036 int pos; 5037 5038 /* If we want to instantiate Virtual Functions, then our 5039 * parent bridge's PCI-E needs to support Alternative Routing 5040 * ID (ARI) because our VFs will show up at function offset 8 5041 * and above. 5042 */ 5043 pbridge = pdev->bus->self; 5044 pos = pci_find_capability(pbridge, PCI_CAP_ID_EXP); 5045 pci_read_config_word(pbridge, pos + PCI_EXP_FLAGS, &flags); 5046 pci_read_config_dword(pbridge, pos + PCI_EXP_DEVCAP2, &devcap2); 5047 5048 if ((flags & PCI_EXP_FLAGS_VERS) < 2 || 5049 !(devcap2 & PCI_EXP_DEVCAP2_ARI)) { 5050 /* Our parent bridge does not support ARI so issue a 5051 * warning and skip instantiating the VFs. They 5052 * won't be reachable. 5053 */ 5054 dev_warn(&pdev->dev, "Parent bridge %02x:%02x.%x doesn't support ARI; can't instantiate Virtual Functions\n", 5055 pbridge->bus->number, PCI_SLOT(pbridge->devfn), 5056 PCI_FUNC(pbridge->devfn)); 5057 return -ENOTSUPP; 5058 } 5059 memset(&port_cmd, 0, sizeof(port_cmd)); 5060 port_cmd.op_to_vfn = cpu_to_be32(FW_CMD_OP_V(FW_PFVF_CMD) | 5061 FW_CMD_REQUEST_F | 5062 FW_CMD_READ_F | 5063 FW_PFVF_CMD_PFN_V(adap->pf) | 5064 FW_PFVF_CMD_VFN_V(0)); 5065 port_cmd.retval_len16 = cpu_to_be32(FW_LEN16(port_cmd)); 5066 err = t4_wr_mbox(adap, adap->mbox, &port_cmd, sizeof(port_cmd), 5067 &port_rpl); 5068 if (err) 5069 return err; 5070 pmask = FW_PFVF_CMD_PMASK_G(be32_to_cpu(port_rpl.type_to_neq)); 5071 port = ffs(pmask) - 1; 5072 /* Allocate VF Management Interface. */ 5073 snprintf(name, IFNAMSIZ, "mgmtpf%d,%d", adap->adap_idx, 5074 adap->pf); 5075 netdev = alloc_netdev(sizeof(struct port_info), 5076 name, NET_NAME_UNKNOWN, cxgb4_mgmt_setup); 5077 if (!netdev) 5078 return -ENOMEM; 5079 5080 pi = netdev_priv(netdev); 5081 pi->adapter = adap; 5082 pi->lport = port; 5083 pi->tx_chan = port; 5084 SET_NETDEV_DEV(netdev, &pdev->dev); 5085 5086 adap->port[0] = netdev; 5087 pi->port_id = 0; 5088 5089 err = register_netdev(adap->port[0]); 5090 if (err) { 5091 pr_info("Unable to register VF mgmt netdev %s\n", name); 5092 free_netdev(adap->port[0]); 5093 adap->port[0] = NULL; 5094 return err; 5095 } 5096 /* Allocate and set up VF Information. */ 5097 adap->vfinfo = kcalloc(pci_sriov_get_totalvfs(pdev), 5098 sizeof(struct vf_info), GFP_KERNEL); 5099 if (!adap->vfinfo) { 5100 unregister_netdev(adap->port[0]); 5101 free_netdev(adap->port[0]); 5102 adap->port[0] = NULL; 5103 return -ENOMEM; 5104 } 5105 cxgb4_mgmt_fill_vf_station_mac_addr(adap); 5106 } 5107 /* Instantiate the requested number of VFs. */ 5108 err = pci_enable_sriov(pdev, num_vfs); 5109 if (err) { 5110 pr_info("Unable to instantiate %d VFs\n", num_vfs); 5111 if (!current_vfs) { 5112 unregister_netdev(adap->port[0]); 5113 free_netdev(adap->port[0]); 5114 adap->port[0] = NULL; 5115 kfree(adap->vfinfo); 5116 adap->vfinfo = NULL; 5117 } 5118 return err; 5119 } 5120 5121 adap->num_vfs = num_vfs; 5122 return num_vfs; 5123 } 5124 #endif /* CONFIG_PCI_IOV */ 5125 5126 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 5127 { 5128 int func, i, err, s_qpp, qpp, num_seg; 5129 struct port_info *pi; 5130 bool highdma = false; 5131 struct adapter *adapter = NULL; 5132 struct net_device *netdev; 5133 void __iomem *regs; 5134 u32 whoami, pl_rev; 5135 enum chip_type chip; 5136 static int adap_idx = 1; 5137 5138 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION); 5139 5140 err = pci_request_regions(pdev, KBUILD_MODNAME); 5141 if (err) { 5142 /* Just info, some other driver may have claimed the device. */ 5143 dev_info(&pdev->dev, "cannot obtain PCI resources\n"); 5144 return err; 5145 } 5146 5147 err = pci_enable_device(pdev); 5148 if (err) { 5149 dev_err(&pdev->dev, "cannot enable PCI device\n"); 5150 goto out_release_regions; 5151 } 5152 5153 regs = pci_ioremap_bar(pdev, 0); 5154 if (!regs) { 5155 dev_err(&pdev->dev, "cannot map device registers\n"); 5156 err = -ENOMEM; 5157 goto out_disable_device; 5158 } 5159 5160 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 5161 if (!adapter) { 5162 err = -ENOMEM; 5163 goto out_unmap_bar0; 5164 } 5165 5166 adapter->regs = regs; 5167 err = t4_wait_dev_ready(regs); 5168 if (err < 0) 5169 goto out_free_adapter; 5170 5171 /* We control everything through one PF */ 5172 whoami = readl(regs + PL_WHOAMI_A); 5173 pl_rev = REV_G(readl(regs + PL_REV_A)); 5174 chip = get_chip_type(pdev, pl_rev); 5175 func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ? 5176 SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); 5177 5178 adapter->pdev = pdev; 5179 adapter->pdev_dev = &pdev->dev; 5180 adapter->name = pci_name(pdev); 5181 adapter->mbox = func; 5182 adapter->pf = func; 5183 adapter->msg_enable = DFLT_MSG_ENABLE; 5184 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + 5185 (sizeof(struct mbox_cmd) * 5186 T4_OS_LOG_MBOX_CMDS), 5187 GFP_KERNEL); 5188 if (!adapter->mbox_log) { 5189 err = -ENOMEM; 5190 goto out_free_adapter; 5191 } 5192 spin_lock_init(&adapter->mbox_lock); 5193 INIT_LIST_HEAD(&adapter->mlist.list); 5194 pci_set_drvdata(pdev, adapter); 5195 5196 if (func != ent->driver_data) { 5197 pci_disable_device(pdev); 5198 pci_save_state(pdev); /* to restore SR-IOV later */ 5199 return 0; 5200 } 5201 5202 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 5203 highdma = true; 5204 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 5205 if (err) { 5206 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for " 5207 "coherent allocations\n"); 5208 goto out_free_adapter; 5209 } 5210 } else { 5211 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 5212 if (err) { 5213 dev_err(&pdev->dev, "no usable DMA configuration\n"); 5214 goto out_free_adapter; 5215 } 5216 } 5217 5218 pci_enable_pcie_error_reporting(pdev); 5219 pci_set_master(pdev); 5220 pci_save_state(pdev); 5221 adap_idx++; 5222 adapter->workq = create_singlethread_workqueue("cxgb4"); 5223 if (!adapter->workq) { 5224 err = -ENOMEM; 5225 goto out_free_adapter; 5226 } 5227 5228 adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS; 5229 5230 /* PCI device has been enabled */ 5231 adapter->flags |= DEV_ENABLED; 5232 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); 5233 5234 /* If possible, we use PCIe Relaxed Ordering Attribute to deliver 5235 * Ingress Packet Data to Free List Buffers in order to allow for 5236 * chipset performance optimizations between the Root Complex and 5237 * Memory Controllers. (Messages to the associated Ingress Queue 5238 * notifying new Packet Placement in the Free Lists Buffers will be 5239 * send without the Relaxed Ordering Attribute thus guaranteeing that 5240 * all preceding PCIe Transaction Layer Packets will be processed 5241 * first.) But some Root Complexes have various issues with Upstream 5242 * Transaction Layer Packets with the Relaxed Ordering Attribute set. 5243 * The PCIe devices which under the Root Complexes will be cleared the 5244 * Relaxed Ordering bit in the configuration space, So we check our 5245 * PCIe configuration space to see if it's flagged with advice against 5246 * using Relaxed Ordering. 5247 */ 5248 if (!pcie_relaxed_ordering_enabled(pdev)) 5249 adapter->flags |= ROOT_NO_RELAXED_ORDERING; 5250 5251 spin_lock_init(&adapter->stats_lock); 5252 spin_lock_init(&adapter->tid_release_lock); 5253 spin_lock_init(&adapter->win0_lock); 5254 5255 INIT_WORK(&adapter->tid_release_task, process_tid_release_list); 5256 INIT_WORK(&adapter->db_full_task, process_db_full); 5257 INIT_WORK(&adapter->db_drop_task, process_db_drop); 5258 5259 err = t4_prep_adapter(adapter); 5260 if (err) 5261 goto out_free_adapter; 5262 5263 5264 if (!is_t4(adapter->params.chip)) { 5265 s_qpp = (QUEUESPERPAGEPF0_S + 5266 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * 5267 adapter->pf); 5268 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter, 5269 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp); 5270 num_seg = PAGE_SIZE / SEGMENT_SIZE; 5271 5272 /* Each segment size is 128B. Write coalescing is enabled only 5273 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the 5274 * queue is less no of segments that can be accommodated in 5275 * a page size. 5276 */ 5277 if (qpp > num_seg) { 5278 dev_err(&pdev->dev, 5279 "Incorrect number of egress queues per page\n"); 5280 err = -EINVAL; 5281 goto out_free_adapter; 5282 } 5283 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), 5284 pci_resource_len(pdev, 2)); 5285 if (!adapter->bar2) { 5286 dev_err(&pdev->dev, "cannot map device bar2 region\n"); 5287 err = -ENOMEM; 5288 goto out_free_adapter; 5289 } 5290 } 5291 5292 setup_memwin(adapter); 5293 err = adap_init0(adapter); 5294 #ifdef CONFIG_DEBUG_FS 5295 bitmap_zero(adapter->sge.blocked_fl, adapter->sge.egr_sz); 5296 #endif 5297 setup_memwin_rdma(adapter); 5298 if (err) 5299 goto out_unmap_bar; 5300 5301 /* configure SGE_STAT_CFG_A to read WC stats */ 5302 if (!is_t4(adapter->params.chip)) 5303 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) | 5304 (is_t5(adapter->params.chip) ? STATMODE_V(0) : 5305 T6_STATMODE_V(0))); 5306 5307 for_each_port(adapter, i) { 5308 netdev = alloc_etherdev_mq(sizeof(struct port_info), 5309 MAX_ETH_QSETS); 5310 if (!netdev) { 5311 err = -ENOMEM; 5312 goto out_free_dev; 5313 } 5314 5315 SET_NETDEV_DEV(netdev, &pdev->dev); 5316 5317 adapter->port[i] = netdev; 5318 pi = netdev_priv(netdev); 5319 pi->adapter = adapter; 5320 pi->xact_addr_filt = -1; 5321 pi->port_id = i; 5322 netdev->irq = pdev->irq; 5323 5324 netdev->hw_features = NETIF_F_SG | TSO_FLAGS | 5325 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 5326 NETIF_F_RXCSUM | NETIF_F_RXHASH | 5327 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 5328 NETIF_F_HW_TC; 5329 5330 if (CHELSIO_CHIP_VERSION(chip) > CHELSIO_T5) 5331 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL; 5332 5333 if (highdma) 5334 netdev->hw_features |= NETIF_F_HIGHDMA; 5335 netdev->features |= netdev->hw_features; 5336 netdev->vlan_features = netdev->features & VLAN_FEAT; 5337 5338 netdev->priv_flags |= IFF_UNICAST_FLT; 5339 5340 /* MTU range: 81 - 9600 */ 5341 netdev->min_mtu = 81; /* accommodate SACK */ 5342 netdev->max_mtu = MAX_MTU; 5343 5344 netdev->netdev_ops = &cxgb4_netdev_ops; 5345 #ifdef CONFIG_CHELSIO_T4_DCB 5346 netdev->dcbnl_ops = &cxgb4_dcb_ops; 5347 cxgb4_dcb_state_init(netdev); 5348 #endif 5349 cxgb4_set_ethtool_ops(netdev); 5350 } 5351 5352 cxgb4_init_ethtool_dump(adapter); 5353 5354 pci_set_drvdata(pdev, adapter); 5355 5356 if (adapter->flags & FW_OK) { 5357 err = t4_port_init(adapter, func, func, 0); 5358 if (err) 5359 goto out_free_dev; 5360 } else if (adapter->params.nports == 1) { 5361 /* If we don't have a connection to the firmware -- possibly 5362 * because of an error -- grab the raw VPD parameters so we 5363 * can set the proper MAC Address on the debug network 5364 * interface that we've created. 5365 */ 5366 u8 hw_addr[ETH_ALEN]; 5367 u8 *na = adapter->params.vpd.na; 5368 5369 err = t4_get_raw_vpd_params(adapter, &adapter->params.vpd); 5370 if (!err) { 5371 for (i = 0; i < ETH_ALEN; i++) 5372 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 5373 hex2val(na[2 * i + 1])); 5374 t4_set_hw_addr(adapter, 0, hw_addr); 5375 } 5376 } 5377 5378 /* Configure queues and allocate tables now, they can be needed as 5379 * soon as the first register_netdev completes. 5380 */ 5381 cfg_queues(adapter); 5382 5383 adapter->smt = t4_init_smt(); 5384 if (!adapter->smt) { 5385 /* We tolerate a lack of SMT, giving up some functionality */ 5386 dev_warn(&pdev->dev, "could not allocate SMT, continuing\n"); 5387 } 5388 5389 adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end); 5390 if (!adapter->l2t) { 5391 /* We tolerate a lack of L2T, giving up some functionality */ 5392 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n"); 5393 adapter->params.offload = 0; 5394 } 5395 5396 #if IS_ENABLED(CONFIG_IPV6) 5397 if ((CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) && 5398 (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) { 5399 /* CLIP functionality is not present in hardware, 5400 * hence disable all offload features 5401 */ 5402 dev_warn(&pdev->dev, 5403 "CLIP not enabled in hardware, continuing\n"); 5404 adapter->params.offload = 0; 5405 } else { 5406 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start, 5407 adapter->clipt_end); 5408 if (!adapter->clipt) { 5409 /* We tolerate a lack of clip_table, giving up 5410 * some functionality 5411 */ 5412 dev_warn(&pdev->dev, 5413 "could not allocate Clip table, continuing\n"); 5414 adapter->params.offload = 0; 5415 } 5416 } 5417 #endif 5418 5419 for_each_port(adapter, i) { 5420 pi = adap2pinfo(adapter, i); 5421 pi->sched_tbl = t4_init_sched(adapter->params.nsched_cls); 5422 if (!pi->sched_tbl) 5423 dev_warn(&pdev->dev, 5424 "could not activate scheduling on port %d\n", 5425 i); 5426 } 5427 5428 if (tid_init(&adapter->tids) < 0) { 5429 dev_warn(&pdev->dev, "could not allocate TID table, " 5430 "continuing\n"); 5431 adapter->params.offload = 0; 5432 } else { 5433 adapter->tc_u32 = cxgb4_init_tc_u32(adapter); 5434 if (!adapter->tc_u32) 5435 dev_warn(&pdev->dev, 5436 "could not offload tc u32, continuing\n"); 5437 5438 if (cxgb4_init_tc_flower(adapter)) 5439 dev_warn(&pdev->dev, 5440 "could not offload tc flower, continuing\n"); 5441 } 5442 5443 if (is_offload(adapter) || is_hashfilter(adapter)) { 5444 if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) { 5445 u32 hash_base, hash_reg; 5446 5447 if (chip <= CHELSIO_T5) { 5448 hash_reg = LE_DB_TID_HASHBASE_A; 5449 hash_base = t4_read_reg(adapter, hash_reg); 5450 adapter->tids.hash_base = hash_base / 4; 5451 } else { 5452 hash_reg = T6_LE_DB_HASH_TID_BASE_A; 5453 hash_base = t4_read_reg(adapter, hash_reg); 5454 adapter->tids.hash_base = hash_base; 5455 } 5456 } 5457 } 5458 5459 /* See what interrupts we'll be using */ 5460 if (msi > 1 && enable_msix(adapter) == 0) 5461 adapter->flags |= USING_MSIX; 5462 else if (msi > 0 && pci_enable_msi(pdev) == 0) { 5463 adapter->flags |= USING_MSI; 5464 if (msi > 1) 5465 free_msix_info(adapter); 5466 } 5467 5468 /* check for PCI Express bandwidth capabiltites */ 5469 cxgb4_check_pcie_caps(adapter); 5470 5471 err = init_rss(adapter); 5472 if (err) 5473 goto out_free_dev; 5474 5475 /* 5476 * The card is now ready to go. If any errors occur during device 5477 * registration we do not fail the whole card but rather proceed only 5478 * with the ports we manage to register successfully. However we must 5479 * register at least one net device. 5480 */ 5481 for_each_port(adapter, i) { 5482 pi = adap2pinfo(adapter, i); 5483 adapter->port[i]->dev_port = pi->lport; 5484 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets); 5485 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets); 5486 5487 netif_carrier_off(adapter->port[i]); 5488 5489 err = register_netdev(adapter->port[i]); 5490 if (err) 5491 break; 5492 adapter->chan_map[pi->tx_chan] = i; 5493 print_port_info(adapter->port[i]); 5494 } 5495 if (i == 0) { 5496 dev_err(&pdev->dev, "could not register any net devices\n"); 5497 goto out_free_dev; 5498 } 5499 if (err) { 5500 dev_warn(&pdev->dev, "only %d net devices registered\n", i); 5501 err = 0; 5502 } 5503 5504 if (cxgb4_debugfs_root) { 5505 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev), 5506 cxgb4_debugfs_root); 5507 setup_debugfs(adapter); 5508 } 5509 5510 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */ 5511 pdev->needs_freset = 1; 5512 5513 if (is_uld(adapter)) { 5514 mutex_lock(&uld_mutex); 5515 list_add_tail(&adapter->list_node, &adapter_list); 5516 mutex_unlock(&uld_mutex); 5517 } 5518 5519 if (!is_t4(adapter->params.chip)) 5520 cxgb4_ptp_init(adapter); 5521 5522 print_adapter_info(adapter); 5523 setup_fw_sge_queues(adapter); 5524 return 0; 5525 5526 out_free_dev: 5527 free_some_resources(adapter); 5528 if (adapter->flags & USING_MSIX) 5529 free_msix_info(adapter); 5530 if (adapter->num_uld || adapter->num_ofld_uld) 5531 t4_uld_mem_free(adapter); 5532 out_unmap_bar: 5533 if (!is_t4(adapter->params.chip)) 5534 iounmap(adapter->bar2); 5535 out_free_adapter: 5536 if (adapter->workq) 5537 destroy_workqueue(adapter->workq); 5538 5539 kfree(adapter->mbox_log); 5540 kfree(adapter); 5541 out_unmap_bar0: 5542 iounmap(regs); 5543 out_disable_device: 5544 pci_disable_pcie_error_reporting(pdev); 5545 pci_disable_device(pdev); 5546 out_release_regions: 5547 pci_release_regions(pdev); 5548 return err; 5549 } 5550 5551 static void remove_one(struct pci_dev *pdev) 5552 { 5553 struct adapter *adapter = pci_get_drvdata(pdev); 5554 5555 if (!adapter) { 5556 pci_release_regions(pdev); 5557 return; 5558 } 5559 5560 adapter->flags |= SHUTTING_DOWN; 5561 5562 if (adapter->pf == 4) { 5563 int i; 5564 5565 /* Tear down per-adapter Work Queue first since it can contain 5566 * references to our adapter data structure. 5567 */ 5568 destroy_workqueue(adapter->workq); 5569 5570 if (is_uld(adapter)) { 5571 detach_ulds(adapter); 5572 t4_uld_clean_up(adapter); 5573 } 5574 5575 disable_interrupts(adapter); 5576 5577 for_each_port(adapter, i) 5578 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5579 unregister_netdev(adapter->port[i]); 5580 5581 debugfs_remove_recursive(adapter->debugfs_root); 5582 5583 if (!is_t4(adapter->params.chip)) 5584 cxgb4_ptp_stop(adapter); 5585 5586 /* If we allocated filters, free up state associated with any 5587 * valid filters ... 5588 */ 5589 clear_all_filters(adapter); 5590 5591 if (adapter->flags & FULL_INIT_DONE) 5592 cxgb_down(adapter); 5593 5594 if (adapter->flags & USING_MSIX) 5595 free_msix_info(adapter); 5596 if (adapter->num_uld || adapter->num_ofld_uld) 5597 t4_uld_mem_free(adapter); 5598 free_some_resources(adapter); 5599 #if IS_ENABLED(CONFIG_IPV6) 5600 t4_cleanup_clip_tbl(adapter); 5601 #endif 5602 iounmap(adapter->regs); 5603 if (!is_t4(adapter->params.chip)) 5604 iounmap(adapter->bar2); 5605 pci_disable_pcie_error_reporting(pdev); 5606 if ((adapter->flags & DEV_ENABLED)) { 5607 pci_disable_device(pdev); 5608 adapter->flags &= ~DEV_ENABLED; 5609 } 5610 pci_release_regions(pdev); 5611 kfree(adapter->mbox_log); 5612 synchronize_rcu(); 5613 kfree(adapter); 5614 } 5615 #ifdef CONFIG_PCI_IOV 5616 else { 5617 cxgb4_iov_configure(adapter->pdev, 0); 5618 } 5619 #endif 5620 } 5621 5622 /* "Shutdown" quiesces the device, stopping Ingress Packet and Interrupt 5623 * delivery. This is essentially a stripped down version of the PCI remove() 5624 * function where we do the minimal amount of work necessary to shutdown any 5625 * further activity. 5626 */ 5627 static void shutdown_one(struct pci_dev *pdev) 5628 { 5629 struct adapter *adapter = pci_get_drvdata(pdev); 5630 5631 /* As with remove_one() above (see extended comment), we only want do 5632 * do cleanup on PCI Devices which went all the way through init_one() 5633 * ... 5634 */ 5635 if (!adapter) { 5636 pci_release_regions(pdev); 5637 return; 5638 } 5639 5640 adapter->flags |= SHUTTING_DOWN; 5641 5642 if (adapter->pf == 4) { 5643 int i; 5644 5645 for_each_port(adapter, i) 5646 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5647 cxgb_close(adapter->port[i]); 5648 5649 if (is_uld(adapter)) { 5650 detach_ulds(adapter); 5651 t4_uld_clean_up(adapter); 5652 } 5653 5654 disable_interrupts(adapter); 5655 disable_msi(adapter); 5656 5657 t4_sge_stop(adapter); 5658 if (adapter->flags & FW_OK) 5659 t4_fw_bye(adapter, adapter->mbox); 5660 } 5661 } 5662 5663 static struct pci_driver cxgb4_driver = { 5664 .name = KBUILD_MODNAME, 5665 .id_table = cxgb4_pci_tbl, 5666 .probe = init_one, 5667 .remove = remove_one, 5668 .shutdown = shutdown_one, 5669 #ifdef CONFIG_PCI_IOV 5670 .sriov_configure = cxgb4_iov_configure, 5671 #endif 5672 .err_handler = &cxgb4_eeh, 5673 }; 5674 5675 static int __init cxgb4_init_module(void) 5676 { 5677 int ret; 5678 5679 /* Debugfs support is optional, just warn if this fails */ 5680 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); 5681 if (!cxgb4_debugfs_root) 5682 pr_warn("could not create debugfs entry, continuing\n"); 5683 5684 ret = pci_register_driver(&cxgb4_driver); 5685 if (ret < 0) 5686 debugfs_remove(cxgb4_debugfs_root); 5687 5688 #if IS_ENABLED(CONFIG_IPV6) 5689 if (!inet6addr_registered) { 5690 register_inet6addr_notifier(&cxgb4_inet6addr_notifier); 5691 inet6addr_registered = true; 5692 } 5693 #endif 5694 5695 return ret; 5696 } 5697 5698 static void __exit cxgb4_cleanup_module(void) 5699 { 5700 #if IS_ENABLED(CONFIG_IPV6) 5701 if (inet6addr_registered) { 5702 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier); 5703 inet6addr_registered = false; 5704 } 5705 #endif 5706 pci_unregister_driver(&cxgb4_driver); 5707 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ 5708 } 5709 5710 module_init(cxgb4_init_module); 5711 module_exit(cxgb4_cleanup_module); 5712