1 /* 2 * This file is part of the Chelsio T4 Ethernet driver for Linux. 3 * 4 * Copyright (c) 2003-2016 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/bitmap.h> 38 #include <linux/crc32.h> 39 #include <linux/ctype.h> 40 #include <linux/debugfs.h> 41 #include <linux/err.h> 42 #include <linux/etherdevice.h> 43 #include <linux/firmware.h> 44 #include <linux/if.h> 45 #include <linux/if_vlan.h> 46 #include <linux/init.h> 47 #include <linux/log2.h> 48 #include <linux/mdio.h> 49 #include <linux/module.h> 50 #include <linux/moduleparam.h> 51 #include <linux/mutex.h> 52 #include <linux/netdevice.h> 53 #include <linux/pci.h> 54 #include <linux/aer.h> 55 #include <linux/rtnetlink.h> 56 #include <linux/sched.h> 57 #include <linux/seq_file.h> 58 #include <linux/sockios.h> 59 #include <linux/vmalloc.h> 60 #include <linux/workqueue.h> 61 #include <net/neighbour.h> 62 #include <net/netevent.h> 63 #include <net/addrconf.h> 64 #include <net/bonding.h> 65 #include <net/addrconf.h> 66 #include <linux/uaccess.h> 67 #include <linux/crash_dump.h> 68 69 #include "cxgb4.h" 70 #include "cxgb4_filter.h" 71 #include "t4_regs.h" 72 #include "t4_values.h" 73 #include "t4_msg.h" 74 #include "t4fw_api.h" 75 #include "t4fw_version.h" 76 #include "cxgb4_dcb.h" 77 #include "cxgb4_debugfs.h" 78 #include "clip_tbl.h" 79 #include "l2t.h" 80 #include "sched.h" 81 #include "cxgb4_tc_u32.h" 82 #include "cxgb4_tc_flower.h" 83 #include "cxgb4_ptp.h" 84 85 char cxgb4_driver_name[] = KBUILD_MODNAME; 86 87 #ifdef DRV_VERSION 88 #undef DRV_VERSION 89 #endif 90 #define DRV_VERSION "2.0.0-ko" 91 const char cxgb4_driver_version[] = DRV_VERSION; 92 #define DRV_DESC "Chelsio T4/T5/T6 Network Driver" 93 94 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ 95 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ 96 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) 97 98 /* Macros needed to support the PCI Device ID Table ... 99 */ 100 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ 101 static const struct pci_device_id cxgb4_pci_tbl[] = { 102 #define CH_PCI_DEVICE_ID_FUNCTION 0x4 103 104 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is 105 * called for both. 106 */ 107 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0 108 109 #define CH_PCI_ID_TABLE_ENTRY(devid) \ 110 {PCI_VDEVICE(CHELSIO, (devid)), 4} 111 112 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ 113 { 0, } \ 114 } 115 116 #include "t4_pci_id_tbl.h" 117 118 #define FW4_FNAME "cxgb4/t4fw.bin" 119 #define FW5_FNAME "cxgb4/t5fw.bin" 120 #define FW6_FNAME "cxgb4/t6fw.bin" 121 #define FW4_CFNAME "cxgb4/t4-config.txt" 122 #define FW5_CFNAME "cxgb4/t5-config.txt" 123 #define FW6_CFNAME "cxgb4/t6-config.txt" 124 #define PHY_AQ1202_FIRMWARE "cxgb4/aq1202_fw.cld" 125 #define PHY_BCM84834_FIRMWARE "cxgb4/bcm8483.bin" 126 #define PHY_AQ1202_DEVICEID 0x4409 127 #define PHY_BCM84834_DEVICEID 0x4486 128 129 MODULE_DESCRIPTION(DRV_DESC); 130 MODULE_AUTHOR("Chelsio Communications"); 131 MODULE_LICENSE("Dual BSD/GPL"); 132 MODULE_VERSION(DRV_VERSION); 133 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl); 134 MODULE_FIRMWARE(FW4_FNAME); 135 MODULE_FIRMWARE(FW5_FNAME); 136 MODULE_FIRMWARE(FW6_FNAME); 137 138 /* 139 * The driver uses the best interrupt scheme available on a platform in the 140 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which 141 * of these schemes the driver may consider as follows: 142 * 143 * msi = 2: choose from among all three options 144 * msi = 1: only consider MSI and INTx interrupts 145 * msi = 0: force INTx interrupts 146 */ 147 static int msi = 2; 148 149 module_param(msi, int, 0644); 150 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)"); 151 152 /* 153 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers 154 * offset by 2 bytes in order to have the IP headers line up on 4-byte 155 * boundaries. This is a requirement for many architectures which will throw 156 * a machine check fault if an attempt is made to access one of the 4-byte IP 157 * header fields on a non-4-byte boundary. And it's a major performance issue 158 * even on some architectures which allow it like some implementations of the 159 * x86 ISA. However, some architectures don't mind this and for some very 160 * edge-case performance sensitive applications (like forwarding large volumes 161 * of small packets), setting this DMA offset to 0 will decrease the number of 162 * PCI-E Bus transfers enough to measurably affect performance. 163 */ 164 static int rx_dma_offset = 2; 165 166 /* TX Queue select used to determine what algorithm to use for selecting TX 167 * queue. Select between the kernel provided function (select_queue=0) or user 168 * cxgb_select_queue function (select_queue=1) 169 * 170 * Default: select_queue=0 171 */ 172 static int select_queue; 173 module_param(select_queue, int, 0644); 174 MODULE_PARM_DESC(select_queue, 175 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method."); 176 177 static struct dentry *cxgb4_debugfs_root; 178 179 LIST_HEAD(adapter_list); 180 DEFINE_MUTEX(uld_mutex); 181 182 static void link_report(struct net_device *dev) 183 { 184 if (!netif_carrier_ok(dev)) 185 netdev_info(dev, "link down\n"); 186 else { 187 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" }; 188 189 const char *s; 190 const struct port_info *p = netdev_priv(dev); 191 192 switch (p->link_cfg.speed) { 193 case 100: 194 s = "100Mbps"; 195 break; 196 case 1000: 197 s = "1Gbps"; 198 break; 199 case 10000: 200 s = "10Gbps"; 201 break; 202 case 25000: 203 s = "25Gbps"; 204 break; 205 case 40000: 206 s = "40Gbps"; 207 break; 208 case 100000: 209 s = "100Gbps"; 210 break; 211 default: 212 pr_info("%s: unsupported speed: %d\n", 213 dev->name, p->link_cfg.speed); 214 return; 215 } 216 217 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, 218 fc[p->link_cfg.fc]); 219 } 220 } 221 222 #ifdef CONFIG_CHELSIO_T4_DCB 223 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */ 224 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable) 225 { 226 struct port_info *pi = netdev_priv(dev); 227 struct adapter *adap = pi->adapter; 228 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; 229 int i; 230 231 /* We use a simple mapping of Port TX Queue Index to DCB 232 * Priority when we're enabling DCB. 233 */ 234 for (i = 0; i < pi->nqsets; i++, txq++) { 235 u32 name, value; 236 int err; 237 238 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 239 FW_PARAMS_PARAM_X_V( 240 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) | 241 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); 242 value = enable ? i : 0xffffffff; 243 244 /* Since we can be called while atomic (from "interrupt 245 * level") we need to issue the Set Parameters Commannd 246 * without sleeping (timeout < 0). 247 */ 248 err = t4_set_params_timeout(adap, adap->mbox, adap->pf, 0, 1, 249 &name, &value, 250 -FW_CMD_MAX_TIMEOUT); 251 252 if (err) 253 dev_err(adap->pdev_dev, 254 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n", 255 enable ? "set" : "unset", pi->port_id, i, -err); 256 else 257 txq->dcb_prio = value; 258 } 259 } 260 261 static int cxgb4_dcb_enabled(const struct net_device *dev) 262 { 263 struct port_info *pi = netdev_priv(dev); 264 265 if (!pi->dcb.enabled) 266 return 0; 267 268 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) || 269 (pi->dcb.state == CXGB4_DCB_STATE_HOST)); 270 } 271 #endif /* CONFIG_CHELSIO_T4_DCB */ 272 273 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat) 274 { 275 struct net_device *dev = adapter->port[port_id]; 276 277 /* Skip changes from disabled ports. */ 278 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) { 279 if (link_stat) 280 netif_carrier_on(dev); 281 else { 282 #ifdef CONFIG_CHELSIO_T4_DCB 283 if (cxgb4_dcb_enabled(dev)) { 284 cxgb4_dcb_reset(dev); 285 dcb_tx_queue_prio_enable(dev, false); 286 } 287 #endif /* CONFIG_CHELSIO_T4_DCB */ 288 netif_carrier_off(dev); 289 } 290 291 link_report(dev); 292 } 293 } 294 295 void t4_os_portmod_changed(const struct adapter *adap, int port_id) 296 { 297 static const char *mod_str[] = { 298 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" 299 }; 300 301 const struct net_device *dev = adap->port[port_id]; 302 const struct port_info *pi = netdev_priv(dev); 303 304 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 305 netdev_info(dev, "port module unplugged\n"); 306 else if (pi->mod_type < ARRAY_SIZE(mod_str)) 307 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]); 308 else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED) 309 netdev_info(dev, "%s: unsupported port module inserted\n", 310 dev->name); 311 else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN) 312 netdev_info(dev, "%s: unknown port module inserted\n", 313 dev->name); 314 else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR) 315 netdev_info(dev, "%s: transceiver module error\n", dev->name); 316 else 317 netdev_info(dev, "%s: unknown module type %d inserted\n", 318 dev->name, pi->mod_type); 319 } 320 321 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ 322 module_param(dbfifo_int_thresh, int, 0644); 323 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); 324 325 /* 326 * usecs to sleep while draining the dbfifo 327 */ 328 static int dbfifo_drain_delay = 1000; 329 module_param(dbfifo_drain_delay, int, 0644); 330 MODULE_PARM_DESC(dbfifo_drain_delay, 331 "usecs to sleep while draining the dbfifo"); 332 333 static inline int cxgb4_set_addr_hash(struct port_info *pi) 334 { 335 struct adapter *adap = pi->adapter; 336 u64 vec = 0; 337 bool ucast = false; 338 struct hash_mac_addr *entry; 339 340 /* Calculate the hash vector for the updated list and program it */ 341 list_for_each_entry(entry, &adap->mac_hlist, list) { 342 ucast |= is_unicast_ether_addr(entry->addr); 343 vec |= (1ULL << hash_mac_addr(entry->addr)); 344 } 345 return t4_set_addr_hash(adap, adap->mbox, pi->viid, ucast, 346 vec, false); 347 } 348 349 static int cxgb4_mac_sync(struct net_device *netdev, const u8 *mac_addr) 350 { 351 struct port_info *pi = netdev_priv(netdev); 352 struct adapter *adap = pi->adapter; 353 int ret; 354 u64 mhash = 0; 355 u64 uhash = 0; 356 bool free = false; 357 bool ucast = is_unicast_ether_addr(mac_addr); 358 const u8 *maclist[1] = {mac_addr}; 359 struct hash_mac_addr *new_entry; 360 361 ret = t4_alloc_mac_filt(adap, adap->mbox, pi->viid, free, 1, maclist, 362 NULL, ucast ? &uhash : &mhash, false); 363 if (ret < 0) 364 goto out; 365 /* if hash != 0, then add the addr to hash addr list 366 * so on the end we will calculate the hash for the 367 * list and program it 368 */ 369 if (uhash || mhash) { 370 new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC); 371 if (!new_entry) 372 return -ENOMEM; 373 ether_addr_copy(new_entry->addr, mac_addr); 374 list_add_tail(&new_entry->list, &adap->mac_hlist); 375 ret = cxgb4_set_addr_hash(pi); 376 } 377 out: 378 return ret < 0 ? ret : 0; 379 } 380 381 static int cxgb4_mac_unsync(struct net_device *netdev, const u8 *mac_addr) 382 { 383 struct port_info *pi = netdev_priv(netdev); 384 struct adapter *adap = pi->adapter; 385 int ret; 386 const u8 *maclist[1] = {mac_addr}; 387 struct hash_mac_addr *entry, *tmp; 388 389 /* If the MAC address to be removed is in the hash addr 390 * list, delete it from the list and update hash vector 391 */ 392 list_for_each_entry_safe(entry, tmp, &adap->mac_hlist, list) { 393 if (ether_addr_equal(entry->addr, mac_addr)) { 394 list_del(&entry->list); 395 kfree(entry); 396 return cxgb4_set_addr_hash(pi); 397 } 398 } 399 400 ret = t4_free_mac_filt(adap, adap->mbox, pi->viid, 1, maclist, false); 401 return ret < 0 ? -EINVAL : 0; 402 } 403 404 /* 405 * Set Rx properties of a port, such as promiscruity, address filters, and MTU. 406 * If @mtu is -1 it is left unchanged. 407 */ 408 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) 409 { 410 struct port_info *pi = netdev_priv(dev); 411 struct adapter *adapter = pi->adapter; 412 413 __dev_uc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 414 __dev_mc_sync(dev, cxgb4_mac_sync, cxgb4_mac_unsync); 415 416 return t4_set_rxmode(adapter, adapter->mbox, pi->viid, mtu, 417 (dev->flags & IFF_PROMISC) ? 1 : 0, 418 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1, 419 sleep_ok); 420 } 421 422 /** 423 * link_start - enable a port 424 * @dev: the port to enable 425 * 426 * Performs the MAC and PHY actions needed to enable a port. 427 */ 428 static int link_start(struct net_device *dev) 429 { 430 int ret; 431 struct port_info *pi = netdev_priv(dev); 432 unsigned int mb = pi->adapter->pf; 433 434 /* 435 * We do not set address filters and promiscuity here, the stack does 436 * that step explicitly. 437 */ 438 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1, 439 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); 440 if (ret == 0) { 441 ret = t4_change_mac(pi->adapter, mb, pi->viid, 442 pi->xact_addr_filt, dev->dev_addr, true, 443 true); 444 if (ret >= 0) { 445 pi->xact_addr_filt = ret; 446 ret = 0; 447 } 448 } 449 if (ret == 0) 450 ret = t4_link_l1cfg(pi->adapter, mb, pi->tx_chan, 451 &pi->link_cfg); 452 if (ret == 0) { 453 local_bh_disable(); 454 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true, 455 true, CXGB4_DCB_ENABLED); 456 local_bh_enable(); 457 } 458 459 return ret; 460 } 461 462 #ifdef CONFIG_CHELSIO_T4_DCB 463 /* Handle a Data Center Bridging update message from the firmware. */ 464 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd) 465 { 466 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid)); 467 struct net_device *dev = adap->port[adap->chan_map[port]]; 468 int old_dcb_enabled = cxgb4_dcb_enabled(dev); 469 int new_dcb_enabled; 470 471 cxgb4_dcb_handle_fw_update(adap, pcmd); 472 new_dcb_enabled = cxgb4_dcb_enabled(dev); 473 474 /* If the DCB has become enabled or disabled on the port then we're 475 * going to need to set up/tear down DCB Priority parameters for the 476 * TX Queues associated with the port. 477 */ 478 if (new_dcb_enabled != old_dcb_enabled) 479 dcb_tx_queue_prio_enable(dev, new_dcb_enabled); 480 } 481 #endif /* CONFIG_CHELSIO_T4_DCB */ 482 483 /* Response queue handler for the FW event queue. 484 */ 485 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, 486 const struct pkt_gl *gl) 487 { 488 u8 opcode = ((const struct rss_header *)rsp)->opcode; 489 490 rsp++; /* skip RSS header */ 491 492 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. 493 */ 494 if (unlikely(opcode == CPL_FW4_MSG && 495 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) { 496 rsp++; 497 opcode = ((const struct rss_header *)rsp)->opcode; 498 rsp++; 499 if (opcode != CPL_SGE_EGR_UPDATE) { 500 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" 501 , opcode); 502 goto out; 503 } 504 } 505 506 if (likely(opcode == CPL_SGE_EGR_UPDATE)) { 507 const struct cpl_sge_egr_update *p = (void *)rsp; 508 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid)); 509 struct sge_txq *txq; 510 511 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start]; 512 txq->restarts++; 513 if (txq->q_type == CXGB4_TXQ_ETH) { 514 struct sge_eth_txq *eq; 515 516 eq = container_of(txq, struct sge_eth_txq, q); 517 netif_tx_wake_queue(eq->txq); 518 } else { 519 struct sge_uld_txq *oq; 520 521 oq = container_of(txq, struct sge_uld_txq, q); 522 tasklet_schedule(&oq->qresume_tsk); 523 } 524 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { 525 const struct cpl_fw6_msg *p = (void *)rsp; 526 527 #ifdef CONFIG_CHELSIO_T4_DCB 528 const struct fw_port_cmd *pcmd = (const void *)p->data; 529 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid)); 530 unsigned int action = 531 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16)); 532 533 if (cmd == FW_PORT_CMD && 534 (action == FW_PORT_ACTION_GET_PORT_INFO || 535 action == FW_PORT_ACTION_GET_PORT_INFO32)) { 536 int port = FW_PORT_CMD_PORTID_G( 537 be32_to_cpu(pcmd->op_to_portid)); 538 struct net_device *dev; 539 int dcbxdis, state_input; 540 541 dev = q->adap->port[q->adap->chan_map[port]]; 542 dcbxdis = (action == FW_PORT_ACTION_GET_PORT_INFO 543 ? !!(pcmd->u.info.dcbxdis_pkd & 544 FW_PORT_CMD_DCBXDIS_F) 545 : !!(pcmd->u.info32.lstatus32_to_cbllen32 & 546 FW_PORT_CMD_DCBXDIS32_F)); 547 state_input = (dcbxdis 548 ? CXGB4_DCB_INPUT_FW_DISABLED 549 : CXGB4_DCB_INPUT_FW_ENABLED); 550 551 cxgb4_dcb_state_fsm(dev, state_input); 552 } 553 554 if (cmd == FW_PORT_CMD && 555 action == FW_PORT_ACTION_L2_DCB_CFG) 556 dcb_rpl(q->adap, pcmd); 557 else 558 #endif 559 if (p->type == 0) 560 t4_handle_fw_rpl(q->adap, p->data); 561 } else if (opcode == CPL_L2T_WRITE_RPL) { 562 const struct cpl_l2t_write_rpl *p = (void *)rsp; 563 564 do_l2t_write_rpl(q->adap, p); 565 } else if (opcode == CPL_SET_TCB_RPL) { 566 const struct cpl_set_tcb_rpl *p = (void *)rsp; 567 568 filter_rpl(q->adap, p); 569 } else 570 dev_err(q->adap->pdev_dev, 571 "unexpected CPL %#x on FW event queue\n", opcode); 572 out: 573 return 0; 574 } 575 576 static void disable_msi(struct adapter *adapter) 577 { 578 if (adapter->flags & USING_MSIX) { 579 pci_disable_msix(adapter->pdev); 580 adapter->flags &= ~USING_MSIX; 581 } else if (adapter->flags & USING_MSI) { 582 pci_disable_msi(adapter->pdev); 583 adapter->flags &= ~USING_MSI; 584 } 585 } 586 587 /* 588 * Interrupt handler for non-data events used with MSI-X. 589 */ 590 static irqreturn_t t4_nondata_intr(int irq, void *cookie) 591 { 592 struct adapter *adap = cookie; 593 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A)); 594 595 if (v & PFSW_F) { 596 adap->swintr = 1; 597 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v); 598 } 599 if (adap->flags & MASTER_PF) 600 t4_slow_intr_handler(adap); 601 return IRQ_HANDLED; 602 } 603 604 /* 605 * Name the MSI-X interrupts. 606 */ 607 static void name_msix_vecs(struct adapter *adap) 608 { 609 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc); 610 611 /* non-data interrupts */ 612 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name); 613 614 /* FW events */ 615 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", 616 adap->port[0]->name); 617 618 /* Ethernet queues */ 619 for_each_port(adap, j) { 620 struct net_device *d = adap->port[j]; 621 const struct port_info *pi = netdev_priv(d); 622 623 for (i = 0; i < pi->nqsets; i++, msi_idx++) 624 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d", 625 d->name, i); 626 } 627 } 628 629 static int request_msix_queue_irqs(struct adapter *adap) 630 { 631 struct sge *s = &adap->sge; 632 int err, ethqidx; 633 int msi_index = 2; 634 635 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0, 636 adap->msix_info[1].desc, &s->fw_evtq); 637 if (err) 638 return err; 639 640 for_each_ethrxq(s, ethqidx) { 641 err = request_irq(adap->msix_info[msi_index].vec, 642 t4_sge_intr_msix, 0, 643 adap->msix_info[msi_index].desc, 644 &s->ethrxq[ethqidx].rspq); 645 if (err) 646 goto unwind; 647 msi_index++; 648 } 649 return 0; 650 651 unwind: 652 while (--ethqidx >= 0) 653 free_irq(adap->msix_info[--msi_index].vec, 654 &s->ethrxq[ethqidx].rspq); 655 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 656 return err; 657 } 658 659 static void free_msix_queue_irqs(struct adapter *adap) 660 { 661 int i, msi_index = 2; 662 struct sge *s = &adap->sge; 663 664 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 665 for_each_ethrxq(s, i) 666 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq); 667 } 668 669 /** 670 * cxgb4_write_rss - write the RSS table for a given port 671 * @pi: the port 672 * @queues: array of queue indices for RSS 673 * 674 * Sets up the portion of the HW RSS table for the port's VI to distribute 675 * packets to the Rx queues in @queues. 676 * Should never be called before setting up sge eth rx queues 677 */ 678 int cxgb4_write_rss(const struct port_info *pi, const u16 *queues) 679 { 680 u16 *rss; 681 int i, err; 682 struct adapter *adapter = pi->adapter; 683 const struct sge_eth_rxq *rxq; 684 685 rxq = &adapter->sge.ethrxq[pi->first_qset]; 686 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL); 687 if (!rss) 688 return -ENOMEM; 689 690 /* map the queue indices to queue ids */ 691 for (i = 0; i < pi->rss_size; i++, queues++) 692 rss[i] = rxq[*queues].rspq.abs_id; 693 694 err = t4_config_rss_range(adapter, adapter->pf, pi->viid, 0, 695 pi->rss_size, rss, pi->rss_size); 696 /* If Tunnel All Lookup isn't specified in the global RSS 697 * Configuration, then we need to specify a default Ingress 698 * Queue for any ingress packets which aren't hashed. We'll 699 * use our first ingress queue ... 700 */ 701 if (!err) 702 err = t4_config_vi_rss(adapter, adapter->mbox, pi->viid, 703 FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F | 704 FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F | 705 FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F | 706 FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F | 707 FW_RSS_VI_CONFIG_CMD_UDPEN_F, 708 rss[0]); 709 kfree(rss); 710 return err; 711 } 712 713 /** 714 * setup_rss - configure RSS 715 * @adap: the adapter 716 * 717 * Sets up RSS for each port. 718 */ 719 static int setup_rss(struct adapter *adap) 720 { 721 int i, j, err; 722 723 for_each_port(adap, i) { 724 const struct port_info *pi = adap2pinfo(adap, i); 725 726 /* Fill default values with equal distribution */ 727 for (j = 0; j < pi->rss_size; j++) 728 pi->rss[j] = j % pi->nqsets; 729 730 err = cxgb4_write_rss(pi, pi->rss); 731 if (err) 732 return err; 733 } 734 return 0; 735 } 736 737 /* 738 * Return the channel of the ingress queue with the given qid. 739 */ 740 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid) 741 { 742 qid -= p->ingr_start; 743 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan; 744 } 745 746 /* 747 * Wait until all NAPI handlers are descheduled. 748 */ 749 static void quiesce_rx(struct adapter *adap) 750 { 751 int i; 752 753 for (i = 0; i < adap->sge.ingr_sz; i++) { 754 struct sge_rspq *q = adap->sge.ingr_map[i]; 755 756 if (q && q->handler) 757 napi_disable(&q->napi); 758 } 759 } 760 761 /* Disable interrupt and napi handler */ 762 static void disable_interrupts(struct adapter *adap) 763 { 764 if (adap->flags & FULL_INIT_DONE) { 765 t4_intr_disable(adap); 766 if (adap->flags & USING_MSIX) { 767 free_msix_queue_irqs(adap); 768 free_irq(adap->msix_info[0].vec, adap); 769 } else { 770 free_irq(adap->pdev->irq, adap); 771 } 772 quiesce_rx(adap); 773 } 774 } 775 776 /* 777 * Enable NAPI scheduling and interrupt generation for all Rx queues. 778 */ 779 static void enable_rx(struct adapter *adap) 780 { 781 int i; 782 783 for (i = 0; i < adap->sge.ingr_sz; i++) { 784 struct sge_rspq *q = adap->sge.ingr_map[i]; 785 786 if (!q) 787 continue; 788 if (q->handler) 789 napi_enable(&q->napi); 790 791 /* 0-increment GTS to start the timer and enable interrupts */ 792 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), 793 SEINTARM_V(q->intr_params) | 794 INGRESSQID_V(q->cntxt_id)); 795 } 796 } 797 798 799 static int setup_fw_sge_queues(struct adapter *adap) 800 { 801 struct sge *s = &adap->sge; 802 int err = 0; 803 804 bitmap_zero(s->starving_fl, s->egr_sz); 805 bitmap_zero(s->txq_maperr, s->egr_sz); 806 807 if (adap->flags & USING_MSIX) 808 adap->msi_idx = 1; /* vector 0 is for non-queue interrupts */ 809 else { 810 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0, 811 NULL, NULL, NULL, -1); 812 if (err) 813 return err; 814 adap->msi_idx = -((int)s->intrq.abs_id + 1); 815 } 816 817 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0], 818 adap->msi_idx, NULL, fwevtq_handler, NULL, -1); 819 if (err) 820 t4_free_sge_resources(adap); 821 return err; 822 } 823 824 /** 825 * setup_sge_queues - configure SGE Tx/Rx/response queues 826 * @adap: the adapter 827 * 828 * Determines how many sets of SGE queues to use and initializes them. 829 * We support multiple queue sets per port if we have MSI-X, otherwise 830 * just one queue set per port. 831 */ 832 static int setup_sge_queues(struct adapter *adap) 833 { 834 int err, i, j; 835 struct sge *s = &adap->sge; 836 struct sge_uld_rxq_info *rxq_info = NULL; 837 unsigned int cmplqid = 0; 838 839 if (is_uld(adap)) 840 rxq_info = s->uld_rxq_info[CXGB4_ULD_RDMA]; 841 842 for_each_port(adap, i) { 843 struct net_device *dev = adap->port[i]; 844 struct port_info *pi = netdev_priv(dev); 845 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset]; 846 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset]; 847 848 for (j = 0; j < pi->nqsets; j++, q++) { 849 if (adap->msi_idx > 0) 850 adap->msi_idx++; 851 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, 852 adap->msi_idx, &q->fl, 853 t4_ethrx_handler, 854 NULL, 855 t4_get_tp_ch_map(adap, 856 pi->tx_chan)); 857 if (err) 858 goto freeout; 859 q->rspq.idx = j; 860 memset(&q->stats, 0, sizeof(q->stats)); 861 } 862 for (j = 0; j < pi->nqsets; j++, t++) { 863 err = t4_sge_alloc_eth_txq(adap, t, dev, 864 netdev_get_tx_queue(dev, j), 865 s->fw_evtq.cntxt_id); 866 if (err) 867 goto freeout; 868 } 869 } 870 871 for_each_port(adap, i) { 872 /* Note that cmplqid below is 0 if we don't 873 * have RDMA queues, and that's the right value. 874 */ 875 if (rxq_info) 876 cmplqid = rxq_info->uldrxq[i].rspq.cntxt_id; 877 878 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i], 879 s->fw_evtq.cntxt_id, cmplqid); 880 if (err) 881 goto freeout; 882 } 883 884 if (!is_t4(adap->params.chip)) { 885 err = t4_sge_alloc_eth_txq(adap, &s->ptptxq, adap->port[0], 886 netdev_get_tx_queue(adap->port[0], 0) 887 , s->fw_evtq.cntxt_id); 888 if (err) 889 goto freeout; 890 } 891 892 t4_write_reg(adap, is_t4(adap->params.chip) ? 893 MPS_TRC_RSS_CONTROL_A : 894 MPS_T5_TRC_RSS_CONTROL_A, 895 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) | 896 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id)); 897 return 0; 898 freeout: 899 t4_free_sge_resources(adap); 900 return err; 901 } 902 903 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb, 904 void *accel_priv, select_queue_fallback_t fallback) 905 { 906 int txq; 907 908 #ifdef CONFIG_CHELSIO_T4_DCB 909 /* If a Data Center Bridging has been successfully negotiated on this 910 * link then we'll use the skb's priority to map it to a TX Queue. 911 * The skb's priority is determined via the VLAN Tag Priority Code 912 * Point field. 913 */ 914 if (cxgb4_dcb_enabled(dev) && !is_kdump_kernel()) { 915 u16 vlan_tci; 916 int err; 917 918 err = vlan_get_tag(skb, &vlan_tci); 919 if (unlikely(err)) { 920 if (net_ratelimit()) 921 netdev_warn(dev, 922 "TX Packet without VLAN Tag on DCB Link\n"); 923 txq = 0; 924 } else { 925 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 926 #ifdef CONFIG_CHELSIO_T4_FCOE 927 if (skb->protocol == htons(ETH_P_FCOE)) 928 txq = skb->priority & 0x7; 929 #endif /* CONFIG_CHELSIO_T4_FCOE */ 930 } 931 return txq; 932 } 933 #endif /* CONFIG_CHELSIO_T4_DCB */ 934 935 if (select_queue) { 936 txq = (skb_rx_queue_recorded(skb) 937 ? skb_get_rx_queue(skb) 938 : smp_processor_id()); 939 940 while (unlikely(txq >= dev->real_num_tx_queues)) 941 txq -= dev->real_num_tx_queues; 942 943 return txq; 944 } 945 946 return fallback(dev, skb) % dev->real_num_tx_queues; 947 } 948 949 static int closest_timer(const struct sge *s, int time) 950 { 951 int i, delta, match = 0, min_delta = INT_MAX; 952 953 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 954 delta = time - s->timer_val[i]; 955 if (delta < 0) 956 delta = -delta; 957 if (delta < min_delta) { 958 min_delta = delta; 959 match = i; 960 } 961 } 962 return match; 963 } 964 965 static int closest_thres(const struct sge *s, int thres) 966 { 967 int i, delta, match = 0, min_delta = INT_MAX; 968 969 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 970 delta = thres - s->counter_val[i]; 971 if (delta < 0) 972 delta = -delta; 973 if (delta < min_delta) { 974 min_delta = delta; 975 match = i; 976 } 977 } 978 return match; 979 } 980 981 /** 982 * cxgb4_set_rspq_intr_params - set a queue's interrupt holdoff parameters 983 * @q: the Rx queue 984 * @us: the hold-off time in us, or 0 to disable timer 985 * @cnt: the hold-off packet count, or 0 to disable counter 986 * 987 * Sets an Rx queue's interrupt hold-off time and packet count. At least 988 * one of the two needs to be enabled for the queue to generate interrupts. 989 */ 990 int cxgb4_set_rspq_intr_params(struct sge_rspq *q, 991 unsigned int us, unsigned int cnt) 992 { 993 struct adapter *adap = q->adap; 994 995 if ((us | cnt) == 0) 996 cnt = 1; 997 998 if (cnt) { 999 int err; 1000 u32 v, new_idx; 1001 1002 new_idx = closest_thres(&adap->sge, cnt); 1003 if (q->desc && q->pktcnt_idx != new_idx) { 1004 /* the queue has already been created, update it */ 1005 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 1006 FW_PARAMS_PARAM_X_V( 1007 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | 1008 FW_PARAMS_PARAM_YZ_V(q->cntxt_id); 1009 err = t4_set_params(adap, adap->mbox, adap->pf, 0, 1, 1010 &v, &new_idx); 1011 if (err) 1012 return err; 1013 } 1014 q->pktcnt_idx = new_idx; 1015 } 1016 1017 us = us == 0 ? 6 : closest_timer(&adap->sge, us); 1018 q->intr_params = QINTR_TIMER_IDX_V(us) | QINTR_CNT_EN_V(cnt > 0); 1019 return 0; 1020 } 1021 1022 static int cxgb_set_features(struct net_device *dev, netdev_features_t features) 1023 { 1024 const struct port_info *pi = netdev_priv(dev); 1025 netdev_features_t changed = dev->features ^ features; 1026 int err; 1027 1028 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX)) 1029 return 0; 1030 1031 err = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, -1, 1032 -1, -1, -1, 1033 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true); 1034 if (unlikely(err)) 1035 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX; 1036 return err; 1037 } 1038 1039 static int setup_debugfs(struct adapter *adap) 1040 { 1041 if (IS_ERR_OR_NULL(adap->debugfs_root)) 1042 return -1; 1043 1044 #ifdef CONFIG_DEBUG_FS 1045 t4_setup_debugfs(adap); 1046 #endif 1047 return 0; 1048 } 1049 1050 /* 1051 * upper-layer driver support 1052 */ 1053 1054 /* 1055 * Allocate an active-open TID and set it to the supplied value. 1056 */ 1057 int cxgb4_alloc_atid(struct tid_info *t, void *data) 1058 { 1059 int atid = -1; 1060 1061 spin_lock_bh(&t->atid_lock); 1062 if (t->afree) { 1063 union aopen_entry *p = t->afree; 1064 1065 atid = (p - t->atid_tab) + t->atid_base; 1066 t->afree = p->next; 1067 p->data = data; 1068 t->atids_in_use++; 1069 } 1070 spin_unlock_bh(&t->atid_lock); 1071 return atid; 1072 } 1073 EXPORT_SYMBOL(cxgb4_alloc_atid); 1074 1075 /* 1076 * Release an active-open TID. 1077 */ 1078 void cxgb4_free_atid(struct tid_info *t, unsigned int atid) 1079 { 1080 union aopen_entry *p = &t->atid_tab[atid - t->atid_base]; 1081 1082 spin_lock_bh(&t->atid_lock); 1083 p->next = t->afree; 1084 t->afree = p; 1085 t->atids_in_use--; 1086 spin_unlock_bh(&t->atid_lock); 1087 } 1088 EXPORT_SYMBOL(cxgb4_free_atid); 1089 1090 /* 1091 * Allocate a server TID and set it to the supplied value. 1092 */ 1093 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data) 1094 { 1095 int stid; 1096 1097 spin_lock_bh(&t->stid_lock); 1098 if (family == PF_INET) { 1099 stid = find_first_zero_bit(t->stid_bmap, t->nstids); 1100 if (stid < t->nstids) 1101 __set_bit(stid, t->stid_bmap); 1102 else 1103 stid = -1; 1104 } else { 1105 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 1); 1106 if (stid < 0) 1107 stid = -1; 1108 } 1109 if (stid >= 0) { 1110 t->stid_tab[stid].data = data; 1111 stid += t->stid_base; 1112 /* IPv6 requires max of 520 bits or 16 cells in TCAM 1113 * This is equivalent to 4 TIDs. With CLIP enabled it 1114 * needs 2 TIDs. 1115 */ 1116 if (family == PF_INET6) { 1117 t->stids_in_use += 2; 1118 t->v6_stids_in_use += 2; 1119 } else { 1120 t->stids_in_use++; 1121 } 1122 } 1123 spin_unlock_bh(&t->stid_lock); 1124 return stid; 1125 } 1126 EXPORT_SYMBOL(cxgb4_alloc_stid); 1127 1128 /* Allocate a server filter TID and set it to the supplied value. 1129 */ 1130 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data) 1131 { 1132 int stid; 1133 1134 spin_lock_bh(&t->stid_lock); 1135 if (family == PF_INET) { 1136 stid = find_next_zero_bit(t->stid_bmap, 1137 t->nstids + t->nsftids, t->nstids); 1138 if (stid < (t->nstids + t->nsftids)) 1139 __set_bit(stid, t->stid_bmap); 1140 else 1141 stid = -1; 1142 } else { 1143 stid = -1; 1144 } 1145 if (stid >= 0) { 1146 t->stid_tab[stid].data = data; 1147 stid -= t->nstids; 1148 stid += t->sftid_base; 1149 t->sftids_in_use++; 1150 } 1151 spin_unlock_bh(&t->stid_lock); 1152 return stid; 1153 } 1154 EXPORT_SYMBOL(cxgb4_alloc_sftid); 1155 1156 /* Release a server TID. 1157 */ 1158 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family) 1159 { 1160 /* Is it a server filter TID? */ 1161 if (t->nsftids && (stid >= t->sftid_base)) { 1162 stid -= t->sftid_base; 1163 stid += t->nstids; 1164 } else { 1165 stid -= t->stid_base; 1166 } 1167 1168 spin_lock_bh(&t->stid_lock); 1169 if (family == PF_INET) 1170 __clear_bit(stid, t->stid_bmap); 1171 else 1172 bitmap_release_region(t->stid_bmap, stid, 1); 1173 t->stid_tab[stid].data = NULL; 1174 if (stid < t->nstids) { 1175 if (family == PF_INET6) { 1176 t->stids_in_use -= 2; 1177 t->v6_stids_in_use -= 2; 1178 } else { 1179 t->stids_in_use--; 1180 } 1181 } else { 1182 t->sftids_in_use--; 1183 } 1184 1185 spin_unlock_bh(&t->stid_lock); 1186 } 1187 EXPORT_SYMBOL(cxgb4_free_stid); 1188 1189 /* 1190 * Populate a TID_RELEASE WR. Caller must properly size the skb. 1191 */ 1192 static void mk_tid_release(struct sk_buff *skb, unsigned int chan, 1193 unsigned int tid) 1194 { 1195 struct cpl_tid_release *req; 1196 1197 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); 1198 req = __skb_put(skb, sizeof(*req)); 1199 INIT_TP_WR(req, tid); 1200 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 1201 } 1202 1203 /* 1204 * Queue a TID release request and if necessary schedule a work queue to 1205 * process it. 1206 */ 1207 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, 1208 unsigned int tid) 1209 { 1210 void **p = &t->tid_tab[tid]; 1211 struct adapter *adap = container_of(t, struct adapter, tids); 1212 1213 spin_lock_bh(&adap->tid_release_lock); 1214 *p = adap->tid_release_head; 1215 /* Low 2 bits encode the Tx channel number */ 1216 adap->tid_release_head = (void **)((uintptr_t)p | chan); 1217 if (!adap->tid_release_task_busy) { 1218 adap->tid_release_task_busy = true; 1219 queue_work(adap->workq, &adap->tid_release_task); 1220 } 1221 spin_unlock_bh(&adap->tid_release_lock); 1222 } 1223 1224 /* 1225 * Process the list of pending TID release requests. 1226 */ 1227 static void process_tid_release_list(struct work_struct *work) 1228 { 1229 struct sk_buff *skb; 1230 struct adapter *adap; 1231 1232 adap = container_of(work, struct adapter, tid_release_task); 1233 1234 spin_lock_bh(&adap->tid_release_lock); 1235 while (adap->tid_release_head) { 1236 void **p = adap->tid_release_head; 1237 unsigned int chan = (uintptr_t)p & 3; 1238 p = (void *)p - chan; 1239 1240 adap->tid_release_head = *p; 1241 *p = NULL; 1242 spin_unlock_bh(&adap->tid_release_lock); 1243 1244 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release), 1245 GFP_KERNEL))) 1246 schedule_timeout_uninterruptible(1); 1247 1248 mk_tid_release(skb, chan, p - adap->tids.tid_tab); 1249 t4_ofld_send(adap, skb); 1250 spin_lock_bh(&adap->tid_release_lock); 1251 } 1252 adap->tid_release_task_busy = false; 1253 spin_unlock_bh(&adap->tid_release_lock); 1254 } 1255 1256 /* 1257 * Release a TID and inform HW. If we are unable to allocate the release 1258 * message we defer to a work queue. 1259 */ 1260 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid, 1261 unsigned short family) 1262 { 1263 struct sk_buff *skb; 1264 struct adapter *adap = container_of(t, struct adapter, tids); 1265 1266 WARN_ON(tid >= t->ntids); 1267 1268 if (t->tid_tab[tid]) { 1269 t->tid_tab[tid] = NULL; 1270 atomic_dec(&t->conns_in_use); 1271 if (t->hash_base && (tid >= t->hash_base)) { 1272 if (family == AF_INET6) 1273 atomic_sub(2, &t->hash_tids_in_use); 1274 else 1275 atomic_dec(&t->hash_tids_in_use); 1276 } else { 1277 if (family == AF_INET6) 1278 atomic_sub(2, &t->tids_in_use); 1279 else 1280 atomic_dec(&t->tids_in_use); 1281 } 1282 } 1283 1284 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); 1285 if (likely(skb)) { 1286 mk_tid_release(skb, chan, tid); 1287 t4_ofld_send(adap, skb); 1288 } else 1289 cxgb4_queue_tid_release(t, chan, tid); 1290 } 1291 EXPORT_SYMBOL(cxgb4_remove_tid); 1292 1293 /* 1294 * Allocate and initialize the TID tables. Returns 0 on success. 1295 */ 1296 static int tid_init(struct tid_info *t) 1297 { 1298 struct adapter *adap = container_of(t, struct adapter, tids); 1299 unsigned int max_ftids = t->nftids + t->nsftids; 1300 unsigned int natids = t->natids; 1301 unsigned int stid_bmap_size; 1302 unsigned int ftid_bmap_size; 1303 size_t size; 1304 1305 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids); 1306 ftid_bmap_size = BITS_TO_LONGS(t->nftids); 1307 size = t->ntids * sizeof(*t->tid_tab) + 1308 natids * sizeof(*t->atid_tab) + 1309 t->nstids * sizeof(*t->stid_tab) + 1310 t->nsftids * sizeof(*t->stid_tab) + 1311 stid_bmap_size * sizeof(long) + 1312 max_ftids * sizeof(*t->ftid_tab) + 1313 ftid_bmap_size * sizeof(long); 1314 1315 t->tid_tab = kvzalloc(size, GFP_KERNEL); 1316 if (!t->tid_tab) 1317 return -ENOMEM; 1318 1319 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 1320 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids]; 1321 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids]; 1322 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size]; 1323 t->ftid_bmap = (unsigned long *)&t->ftid_tab[max_ftids]; 1324 spin_lock_init(&t->stid_lock); 1325 spin_lock_init(&t->atid_lock); 1326 spin_lock_init(&t->ftid_lock); 1327 1328 t->stids_in_use = 0; 1329 t->v6_stids_in_use = 0; 1330 t->sftids_in_use = 0; 1331 t->afree = NULL; 1332 t->atids_in_use = 0; 1333 atomic_set(&t->tids_in_use, 0); 1334 atomic_set(&t->conns_in_use, 0); 1335 atomic_set(&t->hash_tids_in_use, 0); 1336 1337 /* Setup the free list for atid_tab and clear the stid bitmap. */ 1338 if (natids) { 1339 while (--natids) 1340 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 1341 t->afree = t->atid_tab; 1342 } 1343 1344 if (is_offload(adap)) { 1345 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids); 1346 /* Reserve stid 0 for T4/T5 adapters */ 1347 if (!t->stid_base && 1348 CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 1349 __set_bit(0, t->stid_bmap); 1350 } 1351 1352 bitmap_zero(t->ftid_bmap, t->nftids); 1353 return 0; 1354 } 1355 1356 /** 1357 * cxgb4_create_server - create an IP server 1358 * @dev: the device 1359 * @stid: the server TID 1360 * @sip: local IP address to bind server to 1361 * @sport: the server's TCP port 1362 * @queue: queue to direct messages from this server to 1363 * 1364 * Create an IP server for the given port and address. 1365 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1366 */ 1367 int cxgb4_create_server(const struct net_device *dev, unsigned int stid, 1368 __be32 sip, __be16 sport, __be16 vlan, 1369 unsigned int queue) 1370 { 1371 unsigned int chan; 1372 struct sk_buff *skb; 1373 struct adapter *adap; 1374 struct cpl_pass_open_req *req; 1375 int ret; 1376 1377 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1378 if (!skb) 1379 return -ENOMEM; 1380 1381 adap = netdev2adap(dev); 1382 req = __skb_put(skb, sizeof(*req)); 1383 INIT_TP_WR(req, 0); 1384 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid)); 1385 req->local_port = sport; 1386 req->peer_port = htons(0); 1387 req->local_ip = sip; 1388 req->peer_ip = htonl(0); 1389 chan = rxq_to_chan(&adap->sge, queue); 1390 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1391 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1392 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1393 ret = t4_mgmt_tx(adap, skb); 1394 return net_xmit_eval(ret); 1395 } 1396 EXPORT_SYMBOL(cxgb4_create_server); 1397 1398 /* cxgb4_create_server6 - create an IPv6 server 1399 * @dev: the device 1400 * @stid: the server TID 1401 * @sip: local IPv6 address to bind server to 1402 * @sport: the server's TCP port 1403 * @queue: queue to direct messages from this server to 1404 * 1405 * Create an IPv6 server for the given port and address. 1406 * Returns <0 on error and one of the %NET_XMIT_* values on success. 1407 */ 1408 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, 1409 const struct in6_addr *sip, __be16 sport, 1410 unsigned int queue) 1411 { 1412 unsigned int chan; 1413 struct sk_buff *skb; 1414 struct adapter *adap; 1415 struct cpl_pass_open_req6 *req; 1416 int ret; 1417 1418 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1419 if (!skb) 1420 return -ENOMEM; 1421 1422 adap = netdev2adap(dev); 1423 req = __skb_put(skb, sizeof(*req)); 1424 INIT_TP_WR(req, 0); 1425 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid)); 1426 req->local_port = sport; 1427 req->peer_port = htons(0); 1428 req->local_ip_hi = *(__be64 *)(sip->s6_addr); 1429 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8); 1430 req->peer_ip_hi = cpu_to_be64(0); 1431 req->peer_ip_lo = cpu_to_be64(0); 1432 chan = rxq_to_chan(&adap->sge, queue); 1433 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 1434 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 1435 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 1436 ret = t4_mgmt_tx(adap, skb); 1437 return net_xmit_eval(ret); 1438 } 1439 EXPORT_SYMBOL(cxgb4_create_server6); 1440 1441 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, 1442 unsigned int queue, bool ipv6) 1443 { 1444 struct sk_buff *skb; 1445 struct adapter *adap; 1446 struct cpl_close_listsvr_req *req; 1447 int ret; 1448 1449 adap = netdev2adap(dev); 1450 1451 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 1452 if (!skb) 1453 return -ENOMEM; 1454 1455 req = __skb_put(skb, sizeof(*req)); 1456 INIT_TP_WR(req, 0); 1457 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid)); 1458 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) : 1459 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue)); 1460 ret = t4_mgmt_tx(adap, skb); 1461 return net_xmit_eval(ret); 1462 } 1463 EXPORT_SYMBOL(cxgb4_remove_server); 1464 1465 /** 1466 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU 1467 * @mtus: the HW MTU table 1468 * @mtu: the target MTU 1469 * @idx: index of selected entry in the MTU table 1470 * 1471 * Returns the index and the value in the HW MTU table that is closest to 1472 * but does not exceed @mtu, unless @mtu is smaller than any value in the 1473 * table, in which case that smallest available value is selected. 1474 */ 1475 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, 1476 unsigned int *idx) 1477 { 1478 unsigned int i = 0; 1479 1480 while (i < NMTUS - 1 && mtus[i + 1] <= mtu) 1481 ++i; 1482 if (idx) 1483 *idx = i; 1484 return mtus[i]; 1485 } 1486 EXPORT_SYMBOL(cxgb4_best_mtu); 1487 1488 /** 1489 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned 1490 * @mtus: the HW MTU table 1491 * @header_size: Header Size 1492 * @data_size_max: maximum Data Segment Size 1493 * @data_size_align: desired Data Segment Size Alignment (2^N) 1494 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL) 1495 * 1496 * Similar to cxgb4_best_mtu() but instead of searching the Hardware 1497 * MTU Table based solely on a Maximum MTU parameter, we break that 1498 * parameter up into a Header Size and Maximum Data Segment Size, and 1499 * provide a desired Data Segment Size Alignment. If we find an MTU in 1500 * the Hardware MTU Table which will result in a Data Segment Size with 1501 * the requested alignment _and_ that MTU isn't "too far" from the 1502 * closest MTU, then we'll return that rather than the closest MTU. 1503 */ 1504 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, 1505 unsigned short header_size, 1506 unsigned short data_size_max, 1507 unsigned short data_size_align, 1508 unsigned int *mtu_idxp) 1509 { 1510 unsigned short max_mtu = header_size + data_size_max; 1511 unsigned short data_size_align_mask = data_size_align - 1; 1512 int mtu_idx, aligned_mtu_idx; 1513 1514 /* Scan the MTU Table till we find an MTU which is larger than our 1515 * Maximum MTU or we reach the end of the table. Along the way, 1516 * record the last MTU found, if any, which will result in a Data 1517 * Segment Length matching the requested alignment. 1518 */ 1519 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) { 1520 unsigned short data_size = mtus[mtu_idx] - header_size; 1521 1522 /* If this MTU minus the Header Size would result in a 1523 * Data Segment Size of the desired alignment, remember it. 1524 */ 1525 if ((data_size & data_size_align_mask) == 0) 1526 aligned_mtu_idx = mtu_idx; 1527 1528 /* If we're not at the end of the Hardware MTU Table and the 1529 * next element is larger than our Maximum MTU, drop out of 1530 * the loop. 1531 */ 1532 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu) 1533 break; 1534 } 1535 1536 /* If we fell out of the loop because we ran to the end of the table, 1537 * then we just have to use the last [largest] entry. 1538 */ 1539 if (mtu_idx == NMTUS) 1540 mtu_idx--; 1541 1542 /* If we found an MTU which resulted in the requested Data Segment 1543 * Length alignment and that's "not far" from the largest MTU which is 1544 * less than or equal to the maximum MTU, then use that. 1545 */ 1546 if (aligned_mtu_idx >= 0 && 1547 mtu_idx - aligned_mtu_idx <= 1) 1548 mtu_idx = aligned_mtu_idx; 1549 1550 /* If the caller has passed in an MTU Index pointer, pass the 1551 * MTU Index back. Return the MTU value. 1552 */ 1553 if (mtu_idxp) 1554 *mtu_idxp = mtu_idx; 1555 return mtus[mtu_idx]; 1556 } 1557 EXPORT_SYMBOL(cxgb4_best_aligned_mtu); 1558 1559 /** 1560 * cxgb4_tp_smt_idx - Get the Source Mac Table index for this VI 1561 * @chip: chip type 1562 * @viid: VI id of the given port 1563 * 1564 * Return the SMT index for this VI. 1565 */ 1566 unsigned int cxgb4_tp_smt_idx(enum chip_type chip, unsigned int viid) 1567 { 1568 /* In T4/T5, SMT contains 256 SMAC entries organized in 1569 * 128 rows of 2 entries each. 1570 * In T6, SMT contains 256 SMAC entries in 256 rows. 1571 * TODO: The below code needs to be updated when we add support 1572 * for 256 VFs. 1573 */ 1574 if (CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5) 1575 return ((viid & 0x7f) << 1); 1576 else 1577 return (viid & 0x7f); 1578 } 1579 EXPORT_SYMBOL(cxgb4_tp_smt_idx); 1580 1581 /** 1582 * cxgb4_port_chan - get the HW channel of a port 1583 * @dev: the net device for the port 1584 * 1585 * Return the HW Tx channel of the given port. 1586 */ 1587 unsigned int cxgb4_port_chan(const struct net_device *dev) 1588 { 1589 return netdev2pinfo(dev)->tx_chan; 1590 } 1591 EXPORT_SYMBOL(cxgb4_port_chan); 1592 1593 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo) 1594 { 1595 struct adapter *adap = netdev2adap(dev); 1596 u32 v1, v2, lp_count, hp_count; 1597 1598 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 1599 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 1600 if (is_t4(adap->params.chip)) { 1601 lp_count = LP_COUNT_G(v1); 1602 hp_count = HP_COUNT_G(v1); 1603 } else { 1604 lp_count = LP_COUNT_T5_G(v1); 1605 hp_count = HP_COUNT_T5_G(v2); 1606 } 1607 return lpfifo ? lp_count : hp_count; 1608 } 1609 EXPORT_SYMBOL(cxgb4_dbfifo_count); 1610 1611 /** 1612 * cxgb4_port_viid - get the VI id of a port 1613 * @dev: the net device for the port 1614 * 1615 * Return the VI id of the given port. 1616 */ 1617 unsigned int cxgb4_port_viid(const struct net_device *dev) 1618 { 1619 return netdev2pinfo(dev)->viid; 1620 } 1621 EXPORT_SYMBOL(cxgb4_port_viid); 1622 1623 /** 1624 * cxgb4_port_idx - get the index of a port 1625 * @dev: the net device for the port 1626 * 1627 * Return the index of the given port. 1628 */ 1629 unsigned int cxgb4_port_idx(const struct net_device *dev) 1630 { 1631 return netdev2pinfo(dev)->port_id; 1632 } 1633 EXPORT_SYMBOL(cxgb4_port_idx); 1634 1635 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, 1636 struct tp_tcp_stats *v6) 1637 { 1638 struct adapter *adap = pci_get_drvdata(pdev); 1639 1640 spin_lock(&adap->stats_lock); 1641 t4_tp_get_tcp_stats(adap, v4, v6); 1642 spin_unlock(&adap->stats_lock); 1643 } 1644 EXPORT_SYMBOL(cxgb4_get_tcp_stats); 1645 1646 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, 1647 const unsigned int *pgsz_order) 1648 { 1649 struct adapter *adap = netdev2adap(dev); 1650 1651 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask); 1652 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) | 1653 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) | 1654 HPZ3_V(pgsz_order[3])); 1655 } 1656 EXPORT_SYMBOL(cxgb4_iscsi_init); 1657 1658 int cxgb4_flush_eq_cache(struct net_device *dev) 1659 { 1660 struct adapter *adap = netdev2adap(dev); 1661 1662 return t4_sge_ctxt_flush(adap, adap->mbox); 1663 } 1664 EXPORT_SYMBOL(cxgb4_flush_eq_cache); 1665 1666 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) 1667 { 1668 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8; 1669 __be64 indices; 1670 int ret; 1671 1672 spin_lock(&adap->win0_lock); 1673 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr, 1674 sizeof(indices), (__be32 *)&indices, 1675 T4_MEMORY_READ); 1676 spin_unlock(&adap->win0_lock); 1677 if (!ret) { 1678 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff; 1679 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff; 1680 } 1681 return ret; 1682 } 1683 1684 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, 1685 u16 size) 1686 { 1687 struct adapter *adap = netdev2adap(dev); 1688 u16 hw_pidx, hw_cidx; 1689 int ret; 1690 1691 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); 1692 if (ret) 1693 goto out; 1694 1695 if (pidx != hw_pidx) { 1696 u16 delta; 1697 u32 val; 1698 1699 if (pidx >= hw_pidx) 1700 delta = pidx - hw_pidx; 1701 else 1702 delta = size - hw_pidx + pidx; 1703 1704 if (is_t4(adap->params.chip)) 1705 val = PIDX_V(delta); 1706 else 1707 val = PIDX_T5_V(delta); 1708 wmb(); 1709 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 1710 QID_V(qid) | val); 1711 } 1712 out: 1713 return ret; 1714 } 1715 EXPORT_SYMBOL(cxgb4_sync_txq_pidx); 1716 1717 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte) 1718 { 1719 struct adapter *adap; 1720 u32 offset, memtype, memaddr; 1721 u32 edc0_size, edc1_size, mc0_size, mc1_size, size; 1722 u32 edc0_end, edc1_end, mc0_end, mc1_end; 1723 int ret; 1724 1725 adap = netdev2adap(dev); 1726 1727 offset = ((stag >> 8) * 32) + adap->vres.stag.start; 1728 1729 /* Figure out where the offset lands in the Memory Type/Address scheme. 1730 * This code assumes that the memory is laid out starting at offset 0 1731 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0 1732 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have 1733 * MC0, and some have both MC0 and MC1. 1734 */ 1735 size = t4_read_reg(adap, MA_EDRAM0_BAR_A); 1736 edc0_size = EDRAM0_SIZE_G(size) << 20; 1737 size = t4_read_reg(adap, MA_EDRAM1_BAR_A); 1738 edc1_size = EDRAM1_SIZE_G(size) << 20; 1739 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); 1740 mc0_size = EXT_MEM0_SIZE_G(size) << 20; 1741 1742 edc0_end = edc0_size; 1743 edc1_end = edc0_end + edc1_size; 1744 mc0_end = edc1_end + mc0_size; 1745 1746 if (offset < edc0_end) { 1747 memtype = MEM_EDC0; 1748 memaddr = offset; 1749 } else if (offset < edc1_end) { 1750 memtype = MEM_EDC1; 1751 memaddr = offset - edc0_end; 1752 } else { 1753 if (offset < mc0_end) { 1754 memtype = MEM_MC0; 1755 memaddr = offset - edc1_end; 1756 } else if (is_t5(adap->params.chip)) { 1757 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); 1758 mc1_size = EXT_MEM1_SIZE_G(size) << 20; 1759 mc1_end = mc0_end + mc1_size; 1760 if (offset < mc1_end) { 1761 memtype = MEM_MC1; 1762 memaddr = offset - mc0_end; 1763 } else { 1764 /* offset beyond the end of any memory */ 1765 goto err; 1766 } 1767 } else { 1768 /* T4/T6 only has a single memory channel */ 1769 goto err; 1770 } 1771 } 1772 1773 spin_lock(&adap->win0_lock); 1774 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ); 1775 spin_unlock(&adap->win0_lock); 1776 return ret; 1777 1778 err: 1779 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n", 1780 stag, offset); 1781 return -EINVAL; 1782 } 1783 EXPORT_SYMBOL(cxgb4_read_tpte); 1784 1785 u64 cxgb4_read_sge_timestamp(struct net_device *dev) 1786 { 1787 u32 hi, lo; 1788 struct adapter *adap; 1789 1790 adap = netdev2adap(dev); 1791 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A); 1792 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A)); 1793 1794 return ((u64)hi << 32) | (u64)lo; 1795 } 1796 EXPORT_SYMBOL(cxgb4_read_sge_timestamp); 1797 1798 int cxgb4_bar2_sge_qregs(struct net_device *dev, 1799 unsigned int qid, 1800 enum cxgb4_bar2_qtype qtype, 1801 int user, 1802 u64 *pbar2_qoffset, 1803 unsigned int *pbar2_qid) 1804 { 1805 return t4_bar2_sge_qregs(netdev2adap(dev), 1806 qid, 1807 (qtype == CXGB4_BAR2_QTYPE_EGRESS 1808 ? T4_BAR2_QTYPE_EGRESS 1809 : T4_BAR2_QTYPE_INGRESS), 1810 user, 1811 pbar2_qoffset, 1812 pbar2_qid); 1813 } 1814 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs); 1815 1816 static struct pci_driver cxgb4_driver; 1817 1818 static void check_neigh_update(struct neighbour *neigh) 1819 { 1820 const struct device *parent; 1821 const struct net_device *netdev = neigh->dev; 1822 1823 if (is_vlan_dev(netdev)) 1824 netdev = vlan_dev_real_dev(netdev); 1825 parent = netdev->dev.parent; 1826 if (parent && parent->driver == &cxgb4_driver.driver) 1827 t4_l2t_update(dev_get_drvdata(parent), neigh); 1828 } 1829 1830 static int netevent_cb(struct notifier_block *nb, unsigned long event, 1831 void *data) 1832 { 1833 switch (event) { 1834 case NETEVENT_NEIGH_UPDATE: 1835 check_neigh_update(data); 1836 break; 1837 case NETEVENT_REDIRECT: 1838 default: 1839 break; 1840 } 1841 return 0; 1842 } 1843 1844 static bool netevent_registered; 1845 static struct notifier_block cxgb4_netevent_nb = { 1846 .notifier_call = netevent_cb 1847 }; 1848 1849 static void drain_db_fifo(struct adapter *adap, int usecs) 1850 { 1851 u32 v1, v2, lp_count, hp_count; 1852 1853 do { 1854 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 1855 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 1856 if (is_t4(adap->params.chip)) { 1857 lp_count = LP_COUNT_G(v1); 1858 hp_count = HP_COUNT_G(v1); 1859 } else { 1860 lp_count = LP_COUNT_T5_G(v1); 1861 hp_count = HP_COUNT_T5_G(v2); 1862 } 1863 1864 if (lp_count == 0 && hp_count == 0) 1865 break; 1866 set_current_state(TASK_UNINTERRUPTIBLE); 1867 schedule_timeout(usecs_to_jiffies(usecs)); 1868 } while (1); 1869 } 1870 1871 static void disable_txq_db(struct sge_txq *q) 1872 { 1873 unsigned long flags; 1874 1875 spin_lock_irqsave(&q->db_lock, flags); 1876 q->db_disabled = 1; 1877 spin_unlock_irqrestore(&q->db_lock, flags); 1878 } 1879 1880 static void enable_txq_db(struct adapter *adap, struct sge_txq *q) 1881 { 1882 spin_lock_irq(&q->db_lock); 1883 if (q->db_pidx_inc) { 1884 /* Make sure that all writes to the TX descriptors 1885 * are committed before we tell HW about them. 1886 */ 1887 wmb(); 1888 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 1889 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc)); 1890 q->db_pidx_inc = 0; 1891 } 1892 q->db_disabled = 0; 1893 spin_unlock_irq(&q->db_lock); 1894 } 1895 1896 static void disable_dbs(struct adapter *adap) 1897 { 1898 int i; 1899 1900 for_each_ethrxq(&adap->sge, i) 1901 disable_txq_db(&adap->sge.ethtxq[i].q); 1902 if (is_offload(adap)) { 1903 struct sge_uld_txq_info *txq_info = 1904 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 1905 1906 if (txq_info) { 1907 for_each_ofldtxq(&adap->sge, i) { 1908 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 1909 1910 disable_txq_db(&txq->q); 1911 } 1912 } 1913 } 1914 for_each_port(adap, i) 1915 disable_txq_db(&adap->sge.ctrlq[i].q); 1916 } 1917 1918 static void enable_dbs(struct adapter *adap) 1919 { 1920 int i; 1921 1922 for_each_ethrxq(&adap->sge, i) 1923 enable_txq_db(adap, &adap->sge.ethtxq[i].q); 1924 if (is_offload(adap)) { 1925 struct sge_uld_txq_info *txq_info = 1926 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 1927 1928 if (txq_info) { 1929 for_each_ofldtxq(&adap->sge, i) { 1930 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 1931 1932 enable_txq_db(adap, &txq->q); 1933 } 1934 } 1935 } 1936 for_each_port(adap, i) 1937 enable_txq_db(adap, &adap->sge.ctrlq[i].q); 1938 } 1939 1940 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) 1941 { 1942 enum cxgb4_uld type = CXGB4_ULD_RDMA; 1943 1944 if (adap->uld && adap->uld[type].handle) 1945 adap->uld[type].control(adap->uld[type].handle, cmd); 1946 } 1947 1948 static void process_db_full(struct work_struct *work) 1949 { 1950 struct adapter *adap; 1951 1952 adap = container_of(work, struct adapter, db_full_task); 1953 1954 drain_db_fifo(adap, dbfifo_drain_delay); 1955 enable_dbs(adap); 1956 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 1957 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 1958 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 1959 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 1960 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F); 1961 else 1962 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 1963 DBFIFO_LP_INT_F, DBFIFO_LP_INT_F); 1964 } 1965 1966 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) 1967 { 1968 u16 hw_pidx, hw_cidx; 1969 int ret; 1970 1971 spin_lock_irq(&q->db_lock); 1972 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); 1973 if (ret) 1974 goto out; 1975 if (q->db_pidx != hw_pidx) { 1976 u16 delta; 1977 u32 val; 1978 1979 if (q->db_pidx >= hw_pidx) 1980 delta = q->db_pidx - hw_pidx; 1981 else 1982 delta = q->size - hw_pidx + q->db_pidx; 1983 1984 if (is_t4(adap->params.chip)) 1985 val = PIDX_V(delta); 1986 else 1987 val = PIDX_T5_V(delta); 1988 wmb(); 1989 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 1990 QID_V(q->cntxt_id) | val); 1991 } 1992 out: 1993 q->db_disabled = 0; 1994 q->db_pidx_inc = 0; 1995 spin_unlock_irq(&q->db_lock); 1996 if (ret) 1997 CH_WARN(adap, "DB drop recovery failed.\n"); 1998 } 1999 2000 static void recover_all_queues(struct adapter *adap) 2001 { 2002 int i; 2003 2004 for_each_ethrxq(&adap->sge, i) 2005 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); 2006 if (is_offload(adap)) { 2007 struct sge_uld_txq_info *txq_info = 2008 adap->sge.uld_txq_info[CXGB4_TX_OFLD]; 2009 if (txq_info) { 2010 for_each_ofldtxq(&adap->sge, i) { 2011 struct sge_uld_txq *txq = &txq_info->uldtxq[i]; 2012 2013 sync_txq_pidx(adap, &txq->q); 2014 } 2015 } 2016 } 2017 for_each_port(adap, i) 2018 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); 2019 } 2020 2021 static void process_db_drop(struct work_struct *work) 2022 { 2023 struct adapter *adap; 2024 2025 adap = container_of(work, struct adapter, db_drop_task); 2026 2027 if (is_t4(adap->params.chip)) { 2028 drain_db_fifo(adap, dbfifo_drain_delay); 2029 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); 2030 drain_db_fifo(adap, dbfifo_drain_delay); 2031 recover_all_queues(adap); 2032 drain_db_fifo(adap, dbfifo_drain_delay); 2033 enable_dbs(adap); 2034 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 2035 } else if (is_t5(adap->params.chip)) { 2036 u32 dropped_db = t4_read_reg(adap, 0x010ac); 2037 u16 qid = (dropped_db >> 15) & 0x1ffff; 2038 u16 pidx_inc = dropped_db & 0x1fff; 2039 u64 bar2_qoffset; 2040 unsigned int bar2_qid; 2041 int ret; 2042 2043 ret = t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS, 2044 0, &bar2_qoffset, &bar2_qid); 2045 if (ret) 2046 dev_err(adap->pdev_dev, "doorbell drop recovery: " 2047 "qid=%d, pidx_inc=%d\n", qid, pidx_inc); 2048 else 2049 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid), 2050 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL); 2051 2052 /* Re-enable BAR2 WC */ 2053 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15); 2054 } 2055 2056 if (CHELSIO_CHIP_VERSION(adap->params.chip) <= CHELSIO_T5) 2057 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0); 2058 } 2059 2060 void t4_db_full(struct adapter *adap) 2061 { 2062 if (is_t4(adap->params.chip)) { 2063 disable_dbs(adap); 2064 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2065 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 2066 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0); 2067 queue_work(adap->workq, &adap->db_full_task); 2068 } 2069 } 2070 2071 void t4_db_dropped(struct adapter *adap) 2072 { 2073 if (is_t4(adap->params.chip)) { 2074 disable_dbs(adap); 2075 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 2076 } 2077 queue_work(adap->workq, &adap->db_drop_task); 2078 } 2079 2080 void t4_register_netevent_notifier(void) 2081 { 2082 if (!netevent_registered) { 2083 register_netevent_notifier(&cxgb4_netevent_nb); 2084 netevent_registered = true; 2085 } 2086 } 2087 2088 static void detach_ulds(struct adapter *adap) 2089 { 2090 unsigned int i; 2091 2092 mutex_lock(&uld_mutex); 2093 list_del(&adap->list_node); 2094 2095 for (i = 0; i < CXGB4_ULD_MAX; i++) 2096 if (adap->uld && adap->uld[i].handle) 2097 adap->uld[i].state_change(adap->uld[i].handle, 2098 CXGB4_STATE_DETACH); 2099 2100 if (netevent_registered && list_empty(&adapter_list)) { 2101 unregister_netevent_notifier(&cxgb4_netevent_nb); 2102 netevent_registered = false; 2103 } 2104 mutex_unlock(&uld_mutex); 2105 } 2106 2107 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state) 2108 { 2109 unsigned int i; 2110 2111 mutex_lock(&uld_mutex); 2112 for (i = 0; i < CXGB4_ULD_MAX; i++) 2113 if (adap->uld && adap->uld[i].handle) 2114 adap->uld[i].state_change(adap->uld[i].handle, 2115 new_state); 2116 mutex_unlock(&uld_mutex); 2117 } 2118 2119 #if IS_ENABLED(CONFIG_IPV6) 2120 static int cxgb4_inet6addr_handler(struct notifier_block *this, 2121 unsigned long event, void *data) 2122 { 2123 struct inet6_ifaddr *ifa = data; 2124 struct net_device *event_dev = ifa->idev->dev; 2125 const struct device *parent = NULL; 2126 #if IS_ENABLED(CONFIG_BONDING) 2127 struct adapter *adap; 2128 #endif 2129 if (is_vlan_dev(event_dev)) 2130 event_dev = vlan_dev_real_dev(event_dev); 2131 #if IS_ENABLED(CONFIG_BONDING) 2132 if (event_dev->flags & IFF_MASTER) { 2133 list_for_each_entry(adap, &adapter_list, list_node) { 2134 switch (event) { 2135 case NETDEV_UP: 2136 cxgb4_clip_get(adap->port[0], 2137 (const u32 *)ifa, 1); 2138 break; 2139 case NETDEV_DOWN: 2140 cxgb4_clip_release(adap->port[0], 2141 (const u32 *)ifa, 1); 2142 break; 2143 default: 2144 break; 2145 } 2146 } 2147 return NOTIFY_OK; 2148 } 2149 #endif 2150 2151 if (event_dev) 2152 parent = event_dev->dev.parent; 2153 2154 if (parent && parent->driver == &cxgb4_driver.driver) { 2155 switch (event) { 2156 case NETDEV_UP: 2157 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1); 2158 break; 2159 case NETDEV_DOWN: 2160 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1); 2161 break; 2162 default: 2163 break; 2164 } 2165 } 2166 return NOTIFY_OK; 2167 } 2168 2169 static bool inet6addr_registered; 2170 static struct notifier_block cxgb4_inet6addr_notifier = { 2171 .notifier_call = cxgb4_inet6addr_handler 2172 }; 2173 2174 static void update_clip(const struct adapter *adap) 2175 { 2176 int i; 2177 struct net_device *dev; 2178 int ret; 2179 2180 rcu_read_lock(); 2181 2182 for (i = 0; i < MAX_NPORTS; i++) { 2183 dev = adap->port[i]; 2184 ret = 0; 2185 2186 if (dev) 2187 ret = cxgb4_update_root_dev_clip(dev); 2188 2189 if (ret < 0) 2190 break; 2191 } 2192 rcu_read_unlock(); 2193 } 2194 #endif /* IS_ENABLED(CONFIG_IPV6) */ 2195 2196 /** 2197 * cxgb_up - enable the adapter 2198 * @adap: adapter being enabled 2199 * 2200 * Called when the first port is enabled, this function performs the 2201 * actions necessary to make an adapter operational, such as completing 2202 * the initialization of HW modules, and enabling interrupts. 2203 * 2204 * Must be called with the rtnl lock held. 2205 */ 2206 static int cxgb_up(struct adapter *adap) 2207 { 2208 int err; 2209 2210 mutex_lock(&uld_mutex); 2211 err = setup_sge_queues(adap); 2212 if (err) 2213 goto rel_lock; 2214 err = setup_rss(adap); 2215 if (err) 2216 goto freeq; 2217 2218 if (adap->flags & USING_MSIX) { 2219 name_msix_vecs(adap); 2220 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0, 2221 adap->msix_info[0].desc, adap); 2222 if (err) 2223 goto irq_err; 2224 err = request_msix_queue_irqs(adap); 2225 if (err) { 2226 free_irq(adap->msix_info[0].vec, adap); 2227 goto irq_err; 2228 } 2229 } else { 2230 err = request_irq(adap->pdev->irq, t4_intr_handler(adap), 2231 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED, 2232 adap->port[0]->name, adap); 2233 if (err) 2234 goto irq_err; 2235 } 2236 2237 enable_rx(adap); 2238 t4_sge_start(adap); 2239 t4_intr_enable(adap); 2240 adap->flags |= FULL_INIT_DONE; 2241 mutex_unlock(&uld_mutex); 2242 2243 notify_ulds(adap, CXGB4_STATE_UP); 2244 #if IS_ENABLED(CONFIG_IPV6) 2245 update_clip(adap); 2246 #endif 2247 /* Initialize hash mac addr list*/ 2248 INIT_LIST_HEAD(&adap->mac_hlist); 2249 return err; 2250 2251 irq_err: 2252 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err); 2253 freeq: 2254 t4_free_sge_resources(adap); 2255 rel_lock: 2256 mutex_unlock(&uld_mutex); 2257 return err; 2258 } 2259 2260 static void cxgb_down(struct adapter *adapter) 2261 { 2262 cancel_work_sync(&adapter->tid_release_task); 2263 cancel_work_sync(&adapter->db_full_task); 2264 cancel_work_sync(&adapter->db_drop_task); 2265 adapter->tid_release_task_busy = false; 2266 adapter->tid_release_head = NULL; 2267 2268 t4_sge_stop(adapter); 2269 t4_free_sge_resources(adapter); 2270 adapter->flags &= ~FULL_INIT_DONE; 2271 } 2272 2273 /* 2274 * net_device operations 2275 */ 2276 static int cxgb_open(struct net_device *dev) 2277 { 2278 int err; 2279 struct port_info *pi = netdev_priv(dev); 2280 struct adapter *adapter = pi->adapter; 2281 2282 netif_carrier_off(dev); 2283 2284 if (!(adapter->flags & FULL_INIT_DONE)) { 2285 err = cxgb_up(adapter); 2286 if (err < 0) 2287 return err; 2288 } 2289 2290 /* It's possible that the basic port information could have 2291 * changed since we first read it. 2292 */ 2293 err = t4_update_port_info(pi); 2294 if (err < 0) 2295 return err; 2296 2297 err = link_start(dev); 2298 if (!err) 2299 netif_tx_start_all_queues(dev); 2300 return err; 2301 } 2302 2303 static int cxgb_close(struct net_device *dev) 2304 { 2305 struct port_info *pi = netdev_priv(dev); 2306 struct adapter *adapter = pi->adapter; 2307 int ret; 2308 2309 netif_tx_stop_all_queues(dev); 2310 netif_carrier_off(dev); 2311 ret = t4_enable_vi(adapter, adapter->pf, pi->viid, false, false); 2312 #ifdef CONFIG_CHELSIO_T4_DCB 2313 cxgb4_dcb_reset(dev); 2314 dcb_tx_queue_prio_enable(dev, false); 2315 #endif 2316 return ret; 2317 } 2318 2319 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, 2320 __be32 sip, __be16 sport, __be16 vlan, 2321 unsigned int queue, unsigned char port, unsigned char mask) 2322 { 2323 int ret; 2324 struct filter_entry *f; 2325 struct adapter *adap; 2326 int i; 2327 u8 *val; 2328 2329 adap = netdev2adap(dev); 2330 2331 /* Adjust stid to correct filter index */ 2332 stid -= adap->tids.sftid_base; 2333 stid += adap->tids.nftids; 2334 2335 /* Check to make sure the filter requested is writable ... 2336 */ 2337 f = &adap->tids.ftid_tab[stid]; 2338 ret = writable_filter(f); 2339 if (ret) 2340 return ret; 2341 2342 /* Clear out any old resources being used by the filter before 2343 * we start constructing the new filter. 2344 */ 2345 if (f->valid) 2346 clear_filter(adap, f); 2347 2348 /* Clear out filter specifications */ 2349 memset(&f->fs, 0, sizeof(struct ch_filter_specification)); 2350 f->fs.val.lport = cpu_to_be16(sport); 2351 f->fs.mask.lport = ~0; 2352 val = (u8 *)&sip; 2353 if ((val[0] | val[1] | val[2] | val[3]) != 0) { 2354 for (i = 0; i < 4; i++) { 2355 f->fs.val.lip[i] = val[i]; 2356 f->fs.mask.lip[i] = ~0; 2357 } 2358 if (adap->params.tp.vlan_pri_map & PORT_F) { 2359 f->fs.val.iport = port; 2360 f->fs.mask.iport = mask; 2361 } 2362 } 2363 2364 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) { 2365 f->fs.val.proto = IPPROTO_TCP; 2366 f->fs.mask.proto = ~0; 2367 } 2368 2369 f->fs.dirsteer = 1; 2370 f->fs.iq = queue; 2371 /* Mark filter as locked */ 2372 f->locked = 1; 2373 f->fs.rpttid = 1; 2374 2375 /* Save the actual tid. We need this to get the corresponding 2376 * filter entry structure in filter_rpl. 2377 */ 2378 f->tid = stid + adap->tids.ftid_base; 2379 ret = set_filter_wr(adap, stid); 2380 if (ret) { 2381 clear_filter(adap, f); 2382 return ret; 2383 } 2384 2385 return 0; 2386 } 2387 EXPORT_SYMBOL(cxgb4_create_server_filter); 2388 2389 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, 2390 unsigned int queue, bool ipv6) 2391 { 2392 struct filter_entry *f; 2393 struct adapter *adap; 2394 2395 adap = netdev2adap(dev); 2396 2397 /* Adjust stid to correct filter index */ 2398 stid -= adap->tids.sftid_base; 2399 stid += adap->tids.nftids; 2400 2401 f = &adap->tids.ftid_tab[stid]; 2402 /* Unlock the filter */ 2403 f->locked = 0; 2404 2405 return delete_filter(adap, stid); 2406 } 2407 EXPORT_SYMBOL(cxgb4_remove_server_filter); 2408 2409 static void cxgb_get_stats(struct net_device *dev, 2410 struct rtnl_link_stats64 *ns) 2411 { 2412 struct port_stats stats; 2413 struct port_info *p = netdev_priv(dev); 2414 struct adapter *adapter = p->adapter; 2415 2416 /* Block retrieving statistics during EEH error 2417 * recovery. Otherwise, the recovery might fail 2418 * and the PCI device will be removed permanently 2419 */ 2420 spin_lock(&adapter->stats_lock); 2421 if (!netif_device_present(dev)) { 2422 spin_unlock(&adapter->stats_lock); 2423 return; 2424 } 2425 t4_get_port_stats_offset(adapter, p->tx_chan, &stats, 2426 &p->stats_base); 2427 spin_unlock(&adapter->stats_lock); 2428 2429 ns->tx_bytes = stats.tx_octets; 2430 ns->tx_packets = stats.tx_frames; 2431 ns->rx_bytes = stats.rx_octets; 2432 ns->rx_packets = stats.rx_frames; 2433 ns->multicast = stats.rx_mcast_frames; 2434 2435 /* detailed rx_errors */ 2436 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long + 2437 stats.rx_runt; 2438 ns->rx_over_errors = 0; 2439 ns->rx_crc_errors = stats.rx_fcs_err; 2440 ns->rx_frame_errors = stats.rx_symbol_err; 2441 ns->rx_dropped = stats.rx_ovflow0 + stats.rx_ovflow1 + 2442 stats.rx_ovflow2 + stats.rx_ovflow3 + 2443 stats.rx_trunc0 + stats.rx_trunc1 + 2444 stats.rx_trunc2 + stats.rx_trunc3; 2445 ns->rx_missed_errors = 0; 2446 2447 /* detailed tx_errors */ 2448 ns->tx_aborted_errors = 0; 2449 ns->tx_carrier_errors = 0; 2450 ns->tx_fifo_errors = 0; 2451 ns->tx_heartbeat_errors = 0; 2452 ns->tx_window_errors = 0; 2453 2454 ns->tx_errors = stats.tx_error_frames; 2455 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err + 2456 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors; 2457 } 2458 2459 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) 2460 { 2461 unsigned int mbox; 2462 int ret = 0, prtad, devad; 2463 struct port_info *pi = netdev_priv(dev); 2464 struct adapter *adapter = pi->adapter; 2465 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data; 2466 2467 switch (cmd) { 2468 case SIOCGMIIPHY: 2469 if (pi->mdio_addr < 0) 2470 return -EOPNOTSUPP; 2471 data->phy_id = pi->mdio_addr; 2472 break; 2473 case SIOCGMIIREG: 2474 case SIOCSMIIREG: 2475 if (mdio_phy_id_is_c45(data->phy_id)) { 2476 prtad = mdio_phy_id_prtad(data->phy_id); 2477 devad = mdio_phy_id_devad(data->phy_id); 2478 } else if (data->phy_id < 32) { 2479 prtad = data->phy_id; 2480 devad = 0; 2481 data->reg_num &= 0x1f; 2482 } else 2483 return -EINVAL; 2484 2485 mbox = pi->adapter->pf; 2486 if (cmd == SIOCGMIIREG) 2487 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad, 2488 data->reg_num, &data->val_out); 2489 else 2490 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad, 2491 data->reg_num, data->val_in); 2492 break; 2493 case SIOCGHWTSTAMP: 2494 return copy_to_user(req->ifr_data, &pi->tstamp_config, 2495 sizeof(pi->tstamp_config)) ? 2496 -EFAULT : 0; 2497 case SIOCSHWTSTAMP: 2498 if (copy_from_user(&pi->tstamp_config, req->ifr_data, 2499 sizeof(pi->tstamp_config))) 2500 return -EFAULT; 2501 2502 if (!is_t4(adapter->params.chip)) { 2503 switch (pi->tstamp_config.tx_type) { 2504 case HWTSTAMP_TX_OFF: 2505 case HWTSTAMP_TX_ON: 2506 break; 2507 default: 2508 return -ERANGE; 2509 } 2510 2511 switch (pi->tstamp_config.rx_filter) { 2512 case HWTSTAMP_FILTER_NONE: 2513 pi->rxtstamp = false; 2514 break; 2515 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 2516 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 2517 cxgb4_ptprx_timestamping(pi, pi->port_id, 2518 PTP_TS_L4); 2519 break; 2520 case HWTSTAMP_FILTER_PTP_V2_EVENT: 2521 cxgb4_ptprx_timestamping(pi, pi->port_id, 2522 PTP_TS_L2_L4); 2523 break; 2524 case HWTSTAMP_FILTER_ALL: 2525 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 2526 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 2527 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 2528 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 2529 pi->rxtstamp = true; 2530 break; 2531 default: 2532 pi->tstamp_config.rx_filter = 2533 HWTSTAMP_FILTER_NONE; 2534 return -ERANGE; 2535 } 2536 2537 if ((pi->tstamp_config.tx_type == HWTSTAMP_TX_OFF) && 2538 (pi->tstamp_config.rx_filter == 2539 HWTSTAMP_FILTER_NONE)) { 2540 if (cxgb4_ptp_txtype(adapter, pi->port_id) >= 0) 2541 pi->ptp_enable = false; 2542 } 2543 2544 if (pi->tstamp_config.rx_filter != 2545 HWTSTAMP_FILTER_NONE) { 2546 if (cxgb4_ptp_redirect_rx_packet(adapter, 2547 pi) >= 0) 2548 pi->ptp_enable = true; 2549 } 2550 } else { 2551 /* For T4 Adapters */ 2552 switch (pi->tstamp_config.rx_filter) { 2553 case HWTSTAMP_FILTER_NONE: 2554 pi->rxtstamp = false; 2555 break; 2556 case HWTSTAMP_FILTER_ALL: 2557 pi->rxtstamp = true; 2558 break; 2559 default: 2560 pi->tstamp_config.rx_filter = 2561 HWTSTAMP_FILTER_NONE; 2562 return -ERANGE; 2563 } 2564 } 2565 return copy_to_user(req->ifr_data, &pi->tstamp_config, 2566 sizeof(pi->tstamp_config)) ? 2567 -EFAULT : 0; 2568 default: 2569 return -EOPNOTSUPP; 2570 } 2571 return ret; 2572 } 2573 2574 static void cxgb_set_rxmode(struct net_device *dev) 2575 { 2576 /* unfortunately we can't return errors to the stack */ 2577 set_rxmode(dev, -1, false); 2578 } 2579 2580 static int cxgb_change_mtu(struct net_device *dev, int new_mtu) 2581 { 2582 int ret; 2583 struct port_info *pi = netdev_priv(dev); 2584 2585 ret = t4_set_rxmode(pi->adapter, pi->adapter->pf, pi->viid, new_mtu, -1, 2586 -1, -1, -1, true); 2587 if (!ret) 2588 dev->mtu = new_mtu; 2589 return ret; 2590 } 2591 2592 #ifdef CONFIG_PCI_IOV 2593 static int dummy_open(struct net_device *dev) 2594 { 2595 /* Turn carrier off since we don't have to transmit anything on this 2596 * interface. 2597 */ 2598 netif_carrier_off(dev); 2599 return 0; 2600 } 2601 2602 /* Fill MAC address that will be assigned by the FW */ 2603 static void fill_vf_station_mac_addr(struct adapter *adap) 2604 { 2605 unsigned int i; 2606 u8 hw_addr[ETH_ALEN], macaddr[ETH_ALEN]; 2607 int err; 2608 u8 *na; 2609 u16 a, b; 2610 2611 err = t4_get_raw_vpd_params(adap, &adap->params.vpd); 2612 if (!err) { 2613 na = adap->params.vpd.na; 2614 for (i = 0; i < ETH_ALEN; i++) 2615 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 2616 hex2val(na[2 * i + 1])); 2617 a = (hw_addr[0] << 8) | hw_addr[1]; 2618 b = (hw_addr[1] << 8) | hw_addr[2]; 2619 a ^= b; 2620 a |= 0x0200; /* locally assigned Ethernet MAC address */ 2621 a &= ~0x0100; /* not a multicast Ethernet MAC address */ 2622 macaddr[0] = a >> 8; 2623 macaddr[1] = a & 0xff; 2624 2625 for (i = 2; i < 5; i++) 2626 macaddr[i] = hw_addr[i + 1]; 2627 2628 for (i = 0; i < adap->num_vfs; i++) { 2629 macaddr[5] = adap->pf * 16 + i; 2630 ether_addr_copy(adap->vfinfo[i].vf_mac_addr, macaddr); 2631 } 2632 } 2633 } 2634 2635 static int cxgb_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 2636 { 2637 struct port_info *pi = netdev_priv(dev); 2638 struct adapter *adap = pi->adapter; 2639 int ret; 2640 2641 /* verify MAC addr is valid */ 2642 if (!is_valid_ether_addr(mac)) { 2643 dev_err(pi->adapter->pdev_dev, 2644 "Invalid Ethernet address %pM for VF %d\n", 2645 mac, vf); 2646 return -EINVAL; 2647 } 2648 2649 dev_info(pi->adapter->pdev_dev, 2650 "Setting MAC %pM on VF %d\n", mac, vf); 2651 ret = t4_set_vf_mac_acl(adap, vf + 1, 1, mac); 2652 if (!ret) 2653 ether_addr_copy(adap->vfinfo[vf].vf_mac_addr, mac); 2654 return ret; 2655 } 2656 2657 static int cxgb_get_vf_config(struct net_device *dev, 2658 int vf, struct ifla_vf_info *ivi) 2659 { 2660 struct port_info *pi = netdev_priv(dev); 2661 struct adapter *adap = pi->adapter; 2662 2663 if (vf >= adap->num_vfs) 2664 return -EINVAL; 2665 ivi->vf = vf; 2666 ivi->max_tx_rate = adap->vfinfo[vf].tx_rate; 2667 ivi->min_tx_rate = 0; 2668 ether_addr_copy(ivi->mac, adap->vfinfo[vf].vf_mac_addr); 2669 return 0; 2670 } 2671 2672 static int cxgb_get_phys_port_id(struct net_device *dev, 2673 struct netdev_phys_item_id *ppid) 2674 { 2675 struct port_info *pi = netdev_priv(dev); 2676 unsigned int phy_port_id; 2677 2678 phy_port_id = pi->adapter->adap_idx * 10 + pi->port_id; 2679 ppid->id_len = sizeof(phy_port_id); 2680 memcpy(ppid->id, &phy_port_id, ppid->id_len); 2681 return 0; 2682 } 2683 2684 static int cxgb_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 2685 int max_tx_rate) 2686 { 2687 struct port_info *pi = netdev_priv(dev); 2688 struct adapter *adap = pi->adapter; 2689 unsigned int link_ok, speed, mtu; 2690 u32 fw_pfvf, fw_class; 2691 int class_id = vf; 2692 int ret; 2693 u16 pktsize; 2694 2695 if (vf >= adap->num_vfs) 2696 return -EINVAL; 2697 2698 if (min_tx_rate) { 2699 dev_err(adap->pdev_dev, 2700 "Min tx rate (%d) (> 0) for VF %d is Invalid.\n", 2701 min_tx_rate, vf); 2702 return -EINVAL; 2703 } 2704 2705 ret = t4_get_link_params(pi, &link_ok, &speed, &mtu); 2706 if (ret != FW_SUCCESS) { 2707 dev_err(adap->pdev_dev, 2708 "Failed to get link information for VF %d\n", vf); 2709 return -EINVAL; 2710 } 2711 2712 if (!link_ok) { 2713 dev_err(adap->pdev_dev, "Link down for VF %d\n", vf); 2714 return -EINVAL; 2715 } 2716 2717 if (max_tx_rate > speed) { 2718 dev_err(adap->pdev_dev, 2719 "Max tx rate %d for VF %d can't be > link-speed %u", 2720 max_tx_rate, vf, speed); 2721 return -EINVAL; 2722 } 2723 2724 pktsize = mtu; 2725 /* subtract ethhdr size and 4 bytes crc since, f/w appends it */ 2726 pktsize = pktsize - sizeof(struct ethhdr) - 4; 2727 /* subtract ipv4 hdr size, tcp hdr size to get typical IPv4 MSS size */ 2728 pktsize = pktsize - sizeof(struct iphdr) - sizeof(struct tcphdr); 2729 /* configure Traffic Class for rate-limiting */ 2730 ret = t4_sched_params(adap, SCHED_CLASS_TYPE_PACKET, 2731 SCHED_CLASS_LEVEL_CL_RL, 2732 SCHED_CLASS_MODE_CLASS, 2733 SCHED_CLASS_RATEUNIT_BITS, 2734 SCHED_CLASS_RATEMODE_ABS, 2735 pi->tx_chan, class_id, 0, 2736 max_tx_rate * 1000, 0, pktsize); 2737 if (ret) { 2738 dev_err(adap->pdev_dev, "Err %d for Traffic Class config\n", 2739 ret); 2740 return -EINVAL; 2741 } 2742 dev_info(adap->pdev_dev, 2743 "Class %d with MSS %u configured with rate %u\n", 2744 class_id, pktsize, max_tx_rate); 2745 2746 /* bind VF to configured Traffic Class */ 2747 fw_pfvf = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | 2748 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_SCHEDCLASS_ETH)); 2749 fw_class = class_id; 2750 ret = t4_set_params(adap, adap->mbox, adap->pf, vf + 1, 1, &fw_pfvf, 2751 &fw_class); 2752 if (ret) { 2753 dev_err(adap->pdev_dev, 2754 "Err %d in binding VF %d to Traffic Class %d\n", 2755 ret, vf, class_id); 2756 return -EINVAL; 2757 } 2758 dev_info(adap->pdev_dev, "PF %d VF %d is bound to Class %d\n", 2759 adap->pf, vf, class_id); 2760 adap->vfinfo[vf].tx_rate = max_tx_rate; 2761 return 0; 2762 } 2763 2764 #endif 2765 2766 static int cxgb_set_mac_addr(struct net_device *dev, void *p) 2767 { 2768 int ret; 2769 struct sockaddr *addr = p; 2770 struct port_info *pi = netdev_priv(dev); 2771 2772 if (!is_valid_ether_addr(addr->sa_data)) 2773 return -EADDRNOTAVAIL; 2774 2775 ret = t4_change_mac(pi->adapter, pi->adapter->pf, pi->viid, 2776 pi->xact_addr_filt, addr->sa_data, true, true); 2777 if (ret < 0) 2778 return ret; 2779 2780 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 2781 pi->xact_addr_filt = ret; 2782 return 0; 2783 } 2784 2785 #ifdef CONFIG_NET_POLL_CONTROLLER 2786 static void cxgb_netpoll(struct net_device *dev) 2787 { 2788 struct port_info *pi = netdev_priv(dev); 2789 struct adapter *adap = pi->adapter; 2790 2791 if (adap->flags & USING_MSIX) { 2792 int i; 2793 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset]; 2794 2795 for (i = pi->nqsets; i; i--, rx++) 2796 t4_sge_intr_msix(0, &rx->rspq); 2797 } else 2798 t4_intr_handler(adap)(0, adap); 2799 } 2800 #endif 2801 2802 static int cxgb_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 2803 { 2804 struct port_info *pi = netdev_priv(dev); 2805 struct adapter *adap = pi->adapter; 2806 struct sched_class *e; 2807 struct ch_sched_params p; 2808 struct ch_sched_queue qe; 2809 u32 req_rate; 2810 int err = 0; 2811 2812 if (!can_sched(dev)) 2813 return -ENOTSUPP; 2814 2815 if (index < 0 || index > pi->nqsets - 1) 2816 return -EINVAL; 2817 2818 if (!(adap->flags & FULL_INIT_DONE)) { 2819 dev_err(adap->pdev_dev, 2820 "Failed to rate limit on queue %d. Link Down?\n", 2821 index); 2822 return -EINVAL; 2823 } 2824 2825 /* Convert from Mbps to Kbps */ 2826 req_rate = rate << 10; 2827 2828 /* Max rate is 10 Gbps */ 2829 if (req_rate >= SCHED_MAX_RATE_KBPS) { 2830 dev_err(adap->pdev_dev, 2831 "Invalid rate %u Mbps, Max rate is %u Gbps\n", 2832 rate, SCHED_MAX_RATE_KBPS); 2833 return -ERANGE; 2834 } 2835 2836 /* First unbind the queue from any existing class */ 2837 memset(&qe, 0, sizeof(qe)); 2838 qe.queue = index; 2839 qe.class = SCHED_CLS_NONE; 2840 2841 err = cxgb4_sched_class_unbind(dev, (void *)(&qe), SCHED_QUEUE); 2842 if (err) { 2843 dev_err(adap->pdev_dev, 2844 "Unbinding Queue %d on port %d fail. Err: %d\n", 2845 index, pi->port_id, err); 2846 return err; 2847 } 2848 2849 /* Queue already unbound */ 2850 if (!req_rate) 2851 return 0; 2852 2853 /* Fetch any available unused or matching scheduling class */ 2854 memset(&p, 0, sizeof(p)); 2855 p.type = SCHED_CLASS_TYPE_PACKET; 2856 p.u.params.level = SCHED_CLASS_LEVEL_CL_RL; 2857 p.u.params.mode = SCHED_CLASS_MODE_CLASS; 2858 p.u.params.rateunit = SCHED_CLASS_RATEUNIT_BITS; 2859 p.u.params.ratemode = SCHED_CLASS_RATEMODE_ABS; 2860 p.u.params.channel = pi->tx_chan; 2861 p.u.params.class = SCHED_CLS_NONE; 2862 p.u.params.minrate = 0; 2863 p.u.params.maxrate = req_rate; 2864 p.u.params.weight = 0; 2865 p.u.params.pktsize = dev->mtu; 2866 2867 e = cxgb4_sched_class_alloc(dev, &p); 2868 if (!e) 2869 return -ENOMEM; 2870 2871 /* Bind the queue to a scheduling class */ 2872 memset(&qe, 0, sizeof(qe)); 2873 qe.queue = index; 2874 qe.class = e->idx; 2875 2876 err = cxgb4_sched_class_bind(dev, (void *)(&qe), SCHED_QUEUE); 2877 if (err) 2878 dev_err(adap->pdev_dev, 2879 "Queue rate limiting failed. Err: %d\n", err); 2880 return err; 2881 } 2882 2883 static int cxgb_setup_tc_flower(struct net_device *dev, 2884 struct tc_cls_flower_offload *cls_flower) 2885 { 2886 if (!is_classid_clsact_ingress(cls_flower->common.classid) || 2887 cls_flower->common.chain_index) 2888 return -EOPNOTSUPP; 2889 2890 switch (cls_flower->command) { 2891 case TC_CLSFLOWER_REPLACE: 2892 return cxgb4_tc_flower_replace(dev, cls_flower); 2893 case TC_CLSFLOWER_DESTROY: 2894 return cxgb4_tc_flower_destroy(dev, cls_flower); 2895 case TC_CLSFLOWER_STATS: 2896 return cxgb4_tc_flower_stats(dev, cls_flower); 2897 default: 2898 return -EOPNOTSUPP; 2899 } 2900 } 2901 2902 static int cxgb_setup_tc_cls_u32(struct net_device *dev, 2903 struct tc_cls_u32_offload *cls_u32) 2904 { 2905 if (!is_classid_clsact_ingress(cls_u32->common.classid) || 2906 cls_u32->common.chain_index) 2907 return -EOPNOTSUPP; 2908 2909 switch (cls_u32->command) { 2910 case TC_CLSU32_NEW_KNODE: 2911 case TC_CLSU32_REPLACE_KNODE: 2912 return cxgb4_config_knode(dev, cls_u32); 2913 case TC_CLSU32_DELETE_KNODE: 2914 return cxgb4_delete_knode(dev, cls_u32); 2915 default: 2916 return -EOPNOTSUPP; 2917 } 2918 } 2919 2920 static int cxgb_setup_tc(struct net_device *dev, enum tc_setup_type type, 2921 void *type_data) 2922 { 2923 struct port_info *pi = netdev2pinfo(dev); 2924 struct adapter *adap = netdev2adap(dev); 2925 2926 if (!(adap->flags & FULL_INIT_DONE)) { 2927 dev_err(adap->pdev_dev, 2928 "Failed to setup tc on port %d. Link Down?\n", 2929 pi->port_id); 2930 return -EINVAL; 2931 } 2932 2933 switch (type) { 2934 case TC_SETUP_CLSU32: 2935 return cxgb_setup_tc_cls_u32(dev, type_data); 2936 case TC_SETUP_CLSFLOWER: 2937 return cxgb_setup_tc_flower(dev, type_data); 2938 default: 2939 return -EOPNOTSUPP; 2940 } 2941 } 2942 2943 static netdev_features_t cxgb_fix_features(struct net_device *dev, 2944 netdev_features_t features) 2945 { 2946 /* Disable GRO, if RX_CSUM is disabled */ 2947 if (!(features & NETIF_F_RXCSUM)) 2948 features &= ~NETIF_F_GRO; 2949 2950 return features; 2951 } 2952 2953 static const struct net_device_ops cxgb4_netdev_ops = { 2954 .ndo_open = cxgb_open, 2955 .ndo_stop = cxgb_close, 2956 .ndo_start_xmit = t4_eth_xmit, 2957 .ndo_select_queue = cxgb_select_queue, 2958 .ndo_get_stats64 = cxgb_get_stats, 2959 .ndo_set_rx_mode = cxgb_set_rxmode, 2960 .ndo_set_mac_address = cxgb_set_mac_addr, 2961 .ndo_set_features = cxgb_set_features, 2962 .ndo_validate_addr = eth_validate_addr, 2963 .ndo_do_ioctl = cxgb_ioctl, 2964 .ndo_change_mtu = cxgb_change_mtu, 2965 #ifdef CONFIG_NET_POLL_CONTROLLER 2966 .ndo_poll_controller = cxgb_netpoll, 2967 #endif 2968 #ifdef CONFIG_CHELSIO_T4_FCOE 2969 .ndo_fcoe_enable = cxgb_fcoe_enable, 2970 .ndo_fcoe_disable = cxgb_fcoe_disable, 2971 #endif /* CONFIG_CHELSIO_T4_FCOE */ 2972 .ndo_set_tx_maxrate = cxgb_set_tx_maxrate, 2973 .ndo_setup_tc = cxgb_setup_tc, 2974 .ndo_fix_features = cxgb_fix_features, 2975 }; 2976 2977 #ifdef CONFIG_PCI_IOV 2978 static const struct net_device_ops cxgb4_mgmt_netdev_ops = { 2979 .ndo_open = dummy_open, 2980 .ndo_set_vf_mac = cxgb_set_vf_mac, 2981 .ndo_get_vf_config = cxgb_get_vf_config, 2982 .ndo_set_vf_rate = cxgb_set_vf_rate, 2983 .ndo_get_phys_port_id = cxgb_get_phys_port_id, 2984 }; 2985 #endif 2986 2987 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 2988 { 2989 struct adapter *adapter = netdev2adap(dev); 2990 2991 strlcpy(info->driver, cxgb4_driver_name, sizeof(info->driver)); 2992 strlcpy(info->version, cxgb4_driver_version, 2993 sizeof(info->version)); 2994 strlcpy(info->bus_info, pci_name(adapter->pdev), 2995 sizeof(info->bus_info)); 2996 } 2997 2998 static const struct ethtool_ops cxgb4_mgmt_ethtool_ops = { 2999 .get_drvinfo = get_drvinfo, 3000 }; 3001 3002 void t4_fatal_err(struct adapter *adap) 3003 { 3004 int port; 3005 3006 if (pci_channel_offline(adap->pdev)) 3007 return; 3008 3009 /* Disable the SGE since ULDs are going to free resources that 3010 * could be exposed to the adapter. RDMA MWs for example... 3011 */ 3012 t4_shutdown_adapter(adap); 3013 for_each_port(adap, port) { 3014 struct net_device *dev = adap->port[port]; 3015 3016 /* If we get here in very early initialization the network 3017 * devices may not have been set up yet. 3018 */ 3019 if (!dev) 3020 continue; 3021 3022 netif_tx_stop_all_queues(dev); 3023 netif_carrier_off(dev); 3024 } 3025 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n"); 3026 } 3027 3028 static void setup_memwin(struct adapter *adap) 3029 { 3030 u32 nic_win_base = t4_get_util_window(adap); 3031 3032 t4_setup_memwin(adap, nic_win_base, MEMWIN_NIC); 3033 } 3034 3035 static void setup_memwin_rdma(struct adapter *adap) 3036 { 3037 if (adap->vres.ocq.size) { 3038 u32 start; 3039 unsigned int sz_kb; 3040 3041 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2); 3042 start &= PCI_BASE_ADDRESS_MEM_MASK; 3043 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres); 3044 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10; 3045 t4_write_reg(adap, 3046 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3), 3047 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb))); 3048 t4_write_reg(adap, 3049 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3), 3050 adap->vres.ocq.start); 3051 t4_read_reg(adap, 3052 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3)); 3053 } 3054 } 3055 3056 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c) 3057 { 3058 u32 v; 3059 int ret; 3060 3061 /* get device capabilities */ 3062 memset(c, 0, sizeof(*c)); 3063 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3064 FW_CMD_REQUEST_F | FW_CMD_READ_F); 3065 c->cfvalid_to_len16 = htonl(FW_LEN16(*c)); 3066 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), c); 3067 if (ret < 0) 3068 return ret; 3069 3070 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3071 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 3072 ret = t4_wr_mbox(adap, adap->mbox, c, sizeof(*c), NULL); 3073 if (ret < 0) 3074 return ret; 3075 3076 ret = t4_config_glbl_rss(adap, adap->pf, 3077 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 3078 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | 3079 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F); 3080 if (ret < 0) 3081 return ret; 3082 3083 ret = t4_cfg_pfvf(adap, adap->mbox, adap->pf, 0, adap->sge.egr_sz, 64, 3084 MAX_INGQ, 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, 3085 FW_CMD_CAP_PF); 3086 if (ret < 0) 3087 return ret; 3088 3089 t4_sge_init(adap); 3090 3091 /* tweak some settings */ 3092 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849); 3093 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12)); 3094 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A); 3095 v = t4_read_reg(adap, TP_PIO_DATA_A); 3096 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F); 3097 3098 /* first 4 Tx modulation queues point to consecutive Tx channels */ 3099 adap->params.tp.tx_modq_map = 0xE4; 3100 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A, 3101 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map)); 3102 3103 /* associate each Tx modulation queue with consecutive Tx channels */ 3104 v = 0x84218421; 3105 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3106 &v, 1, TP_TX_SCHED_HDR_A); 3107 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3108 &v, 1, TP_TX_SCHED_FIFO_A); 3109 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 3110 &v, 1, TP_TX_SCHED_PCMD_A); 3111 3112 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */ 3113 if (is_offload(adap)) { 3114 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A, 3115 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3116 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3117 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3118 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 3119 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A, 3120 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3121 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3122 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 3123 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 3124 } 3125 3126 /* get basic stuff going */ 3127 return t4_early_init(adap, adap->pf); 3128 } 3129 3130 /* 3131 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. 3132 */ 3133 #define MAX_ATIDS 8192U 3134 3135 /* 3136 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 3137 * 3138 * If the firmware we're dealing with has Configuration File support, then 3139 * we use that to perform all configuration 3140 */ 3141 3142 /* 3143 * Tweak configuration based on module parameters, etc. Most of these have 3144 * defaults assigned to them by Firmware Configuration Files (if we're using 3145 * them) but need to be explicitly set if we're using hard-coded 3146 * initialization. But even in the case of using Firmware Configuration 3147 * Files, we'd like to expose the ability to change these via module 3148 * parameters so these are essentially common tweaks/settings for 3149 * Configuration Files and hard-coded initialization ... 3150 */ 3151 static int adap_init0_tweaks(struct adapter *adapter) 3152 { 3153 /* 3154 * Fix up various Host-Dependent Parameters like Page Size, Cache 3155 * Line Size, etc. The firmware default is for a 4KB Page Size and 3156 * 64B Cache Line Size ... 3157 */ 3158 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES); 3159 3160 /* 3161 * Process module parameters which affect early initialization. 3162 */ 3163 if (rx_dma_offset != 2 && rx_dma_offset != 0) { 3164 dev_err(&adapter->pdev->dev, 3165 "Ignoring illegal rx_dma_offset=%d, using 2\n", 3166 rx_dma_offset); 3167 rx_dma_offset = 2; 3168 } 3169 t4_set_reg_field(adapter, SGE_CONTROL_A, 3170 PKTSHIFT_V(PKTSHIFT_M), 3171 PKTSHIFT_V(rx_dma_offset)); 3172 3173 /* 3174 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux 3175 * adds the pseudo header itself. 3176 */ 3177 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A, 3178 CSUM_HAS_PSEUDO_HDR_F, 0); 3179 3180 return 0; 3181 } 3182 3183 /* 10Gb/s-BT PHY Support. chip-external 10Gb/s-BT PHYs are complex chips 3184 * unto themselves and they contain their own firmware to perform their 3185 * tasks ... 3186 */ 3187 static int phy_aq1202_version(const u8 *phy_fw_data, 3188 size_t phy_fw_size) 3189 { 3190 int offset; 3191 3192 /* At offset 0x8 you're looking for the primary image's 3193 * starting offset which is 3 Bytes wide 3194 * 3195 * At offset 0xa of the primary image, you look for the offset 3196 * of the DRAM segment which is 3 Bytes wide. 3197 * 3198 * The FW version is at offset 0x27e of the DRAM and is 2 Bytes 3199 * wide 3200 */ 3201 #define be16(__p) (((__p)[0] << 8) | (__p)[1]) 3202 #define le16(__p) ((__p)[0] | ((__p)[1] << 8)) 3203 #define le24(__p) (le16(__p) | ((__p)[2] << 16)) 3204 3205 offset = le24(phy_fw_data + 0x8) << 12; 3206 offset = le24(phy_fw_data + offset + 0xa); 3207 return be16(phy_fw_data + offset + 0x27e); 3208 3209 #undef be16 3210 #undef le16 3211 #undef le24 3212 } 3213 3214 static struct info_10gbt_phy_fw { 3215 unsigned int phy_fw_id; /* PCI Device ID */ 3216 char *phy_fw_file; /* /lib/firmware/ PHY Firmware file */ 3217 int (*phy_fw_version)(const u8 *phy_fw_data, size_t phy_fw_size); 3218 int phy_flash; /* Has FLASH for PHY Firmware */ 3219 } phy_info_array[] = { 3220 { 3221 PHY_AQ1202_DEVICEID, 3222 PHY_AQ1202_FIRMWARE, 3223 phy_aq1202_version, 3224 1, 3225 }, 3226 { 3227 PHY_BCM84834_DEVICEID, 3228 PHY_BCM84834_FIRMWARE, 3229 NULL, 3230 0, 3231 }, 3232 { 0, NULL, NULL }, 3233 }; 3234 3235 static struct info_10gbt_phy_fw *find_phy_info(int devid) 3236 { 3237 int i; 3238 3239 for (i = 0; i < ARRAY_SIZE(phy_info_array); i++) { 3240 if (phy_info_array[i].phy_fw_id == devid) 3241 return &phy_info_array[i]; 3242 } 3243 return NULL; 3244 } 3245 3246 /* Handle updating of chip-external 10Gb/s-BT PHY firmware. This needs to 3247 * happen after the FW_RESET_CMD but before the FW_INITIALIZE_CMD. On error 3248 * we return a negative error number. If we transfer new firmware we return 1 3249 * (from t4_load_phy_fw()). If we don't do anything we return 0. 3250 */ 3251 static int adap_init0_phy(struct adapter *adap) 3252 { 3253 const struct firmware *phyf; 3254 int ret; 3255 struct info_10gbt_phy_fw *phy_info; 3256 3257 /* Use the device ID to determine which PHY file to flash. 3258 */ 3259 phy_info = find_phy_info(adap->pdev->device); 3260 if (!phy_info) { 3261 dev_warn(adap->pdev_dev, 3262 "No PHY Firmware file found for this PHY\n"); 3263 return -EOPNOTSUPP; 3264 } 3265 3266 /* If we have a T4 PHY firmware file under /lib/firmware/cxgb4/, then 3267 * use that. The adapter firmware provides us with a memory buffer 3268 * where we can load a PHY firmware file from the host if we want to 3269 * override the PHY firmware File in flash. 3270 */ 3271 ret = request_firmware_direct(&phyf, phy_info->phy_fw_file, 3272 adap->pdev_dev); 3273 if (ret < 0) { 3274 /* For adapters without FLASH attached to PHY for their 3275 * firmware, it's obviously a fatal error if we can't get the 3276 * firmware to the adapter. For adapters with PHY firmware 3277 * FLASH storage, it's worth a warning if we can't find the 3278 * PHY Firmware but we'll neuter the error ... 3279 */ 3280 dev_err(adap->pdev_dev, "unable to find PHY Firmware image " 3281 "/lib/firmware/%s, error %d\n", 3282 phy_info->phy_fw_file, -ret); 3283 if (phy_info->phy_flash) { 3284 int cur_phy_fw_ver = 0; 3285 3286 t4_phy_fw_ver(adap, &cur_phy_fw_ver); 3287 dev_warn(adap->pdev_dev, "continuing with, on-adapter " 3288 "FLASH copy, version %#x\n", cur_phy_fw_ver); 3289 ret = 0; 3290 } 3291 3292 return ret; 3293 } 3294 3295 /* Load PHY Firmware onto adapter. 3296 */ 3297 ret = t4_load_phy_fw(adap, MEMWIN_NIC, &adap->win0_lock, 3298 phy_info->phy_fw_version, 3299 (u8 *)phyf->data, phyf->size); 3300 if (ret < 0) 3301 dev_err(adap->pdev_dev, "PHY Firmware transfer error %d\n", 3302 -ret); 3303 else if (ret > 0) { 3304 int new_phy_fw_ver = 0; 3305 3306 if (phy_info->phy_fw_version) 3307 new_phy_fw_ver = phy_info->phy_fw_version(phyf->data, 3308 phyf->size); 3309 dev_info(adap->pdev_dev, "Successfully transferred PHY " 3310 "Firmware /lib/firmware/%s, version %#x\n", 3311 phy_info->phy_fw_file, new_phy_fw_ver); 3312 } 3313 3314 release_firmware(phyf); 3315 3316 return ret; 3317 } 3318 3319 /* 3320 * Attempt to initialize the adapter via a Firmware Configuration File. 3321 */ 3322 static int adap_init0_config(struct adapter *adapter, int reset) 3323 { 3324 struct fw_caps_config_cmd caps_cmd; 3325 const struct firmware *cf; 3326 unsigned long mtype = 0, maddr = 0; 3327 u32 finiver, finicsum, cfcsum; 3328 int ret; 3329 int config_issued = 0; 3330 char *fw_config_file, fw_config_file_path[256]; 3331 char *config_name = NULL; 3332 3333 /* 3334 * Reset device if necessary. 3335 */ 3336 if (reset) { 3337 ret = t4_fw_reset(adapter, adapter->mbox, 3338 PIORSTMODE_F | PIORST_F); 3339 if (ret < 0) 3340 goto bye; 3341 } 3342 3343 /* If this is a 10Gb/s-BT adapter make sure the chip-external 3344 * 10Gb/s-BT PHYs have up-to-date firmware. Note that this step needs 3345 * to be performed after any global adapter RESET above since some 3346 * PHYs only have local RAM copies of the PHY firmware. 3347 */ 3348 if (is_10gbt_device(adapter->pdev->device)) { 3349 ret = adap_init0_phy(adapter); 3350 if (ret < 0) 3351 goto bye; 3352 } 3353 /* 3354 * If we have a T4 configuration file under /lib/firmware/cxgb4/, 3355 * then use that. Otherwise, use the configuration file stored 3356 * in the adapter flash ... 3357 */ 3358 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { 3359 case CHELSIO_T4: 3360 fw_config_file = FW4_CFNAME; 3361 break; 3362 case CHELSIO_T5: 3363 fw_config_file = FW5_CFNAME; 3364 break; 3365 case CHELSIO_T6: 3366 fw_config_file = FW6_CFNAME; 3367 break; 3368 default: 3369 dev_err(adapter->pdev_dev, "Device %d is not supported\n", 3370 adapter->pdev->device); 3371 ret = -EINVAL; 3372 goto bye; 3373 } 3374 3375 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev); 3376 if (ret < 0) { 3377 config_name = "On FLASH"; 3378 mtype = FW_MEMTYPE_CF_FLASH; 3379 maddr = t4_flash_cfg_addr(adapter); 3380 } else { 3381 u32 params[7], val[7]; 3382 3383 sprintf(fw_config_file_path, 3384 "/lib/firmware/%s", fw_config_file); 3385 config_name = fw_config_file_path; 3386 3387 if (cf->size >= FLASH_CFG_MAX_SIZE) 3388 ret = -ENOMEM; 3389 else { 3390 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3391 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 3392 ret = t4_query_params(adapter, adapter->mbox, 3393 adapter->pf, 0, 1, params, val); 3394 if (ret == 0) { 3395 /* 3396 * For t4_memory_rw() below addresses and 3397 * sizes have to be in terms of multiples of 4 3398 * bytes. So, if the Configuration File isn't 3399 * a multiple of 4 bytes in length we'll have 3400 * to write that out separately since we can't 3401 * guarantee that the bytes following the 3402 * residual byte in the buffer returned by 3403 * request_firmware() are zeroed out ... 3404 */ 3405 size_t resid = cf->size & 0x3; 3406 size_t size = cf->size & ~0x3; 3407 __be32 *data = (__be32 *)cf->data; 3408 3409 mtype = FW_PARAMS_PARAM_Y_G(val[0]); 3410 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16; 3411 3412 spin_lock(&adapter->win0_lock); 3413 ret = t4_memory_rw(adapter, 0, mtype, maddr, 3414 size, data, T4_MEMORY_WRITE); 3415 if (ret == 0 && resid != 0) { 3416 union { 3417 __be32 word; 3418 char buf[4]; 3419 } last; 3420 int i; 3421 3422 last.word = data[size >> 2]; 3423 for (i = resid; i < 4; i++) 3424 last.buf[i] = 0; 3425 ret = t4_memory_rw(adapter, 0, mtype, 3426 maddr + size, 3427 4, &last.word, 3428 T4_MEMORY_WRITE); 3429 } 3430 spin_unlock(&adapter->win0_lock); 3431 } 3432 } 3433 3434 release_firmware(cf); 3435 if (ret) 3436 goto bye; 3437 } 3438 3439 /* 3440 * Issue a Capability Configuration command to the firmware to get it 3441 * to parse the Configuration File. We don't use t4_fw_config_file() 3442 * because we want the ability to modify various features after we've 3443 * processed the configuration file ... 3444 */ 3445 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3446 caps_cmd.op_to_write = 3447 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3448 FW_CMD_REQUEST_F | 3449 FW_CMD_READ_F); 3450 caps_cmd.cfvalid_to_len16 = 3451 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F | 3452 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) | 3453 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) | 3454 FW_LEN16(caps_cmd)); 3455 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 3456 &caps_cmd); 3457 3458 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware 3459 * Configuration File in FLASH), our last gasp effort is to use the 3460 * Firmware Configuration File which is embedded in the firmware. A 3461 * very few early versions of the firmware didn't have one embedded 3462 * but we can ignore those. 3463 */ 3464 if (ret == -ENOENT) { 3465 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3466 caps_cmd.op_to_write = 3467 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3468 FW_CMD_REQUEST_F | 3469 FW_CMD_READ_F); 3470 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3471 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, 3472 sizeof(caps_cmd), &caps_cmd); 3473 config_name = "Firmware Default"; 3474 } 3475 3476 config_issued = 1; 3477 if (ret < 0) 3478 goto bye; 3479 3480 finiver = ntohl(caps_cmd.finiver); 3481 finicsum = ntohl(caps_cmd.finicsum); 3482 cfcsum = ntohl(caps_cmd.cfcsum); 3483 if (finicsum != cfcsum) 3484 dev_warn(adapter->pdev_dev, "Configuration File checksum "\ 3485 "mismatch: [fini] csum=%#x, computed csum=%#x\n", 3486 finicsum, cfcsum); 3487 3488 /* 3489 * And now tell the firmware to use the configuration we just loaded. 3490 */ 3491 caps_cmd.op_to_write = 3492 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3493 FW_CMD_REQUEST_F | 3494 FW_CMD_WRITE_F); 3495 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3496 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 3497 NULL); 3498 if (ret < 0) 3499 goto bye; 3500 3501 /* 3502 * Tweak configuration based on system architecture, module 3503 * parameters, etc. 3504 */ 3505 ret = adap_init0_tweaks(adapter); 3506 if (ret < 0) 3507 goto bye; 3508 3509 /* 3510 * And finally tell the firmware to initialize itself using the 3511 * parameters from the Configuration File. 3512 */ 3513 ret = t4_fw_initialize(adapter, adapter->mbox); 3514 if (ret < 0) 3515 goto bye; 3516 3517 /* Emit Firmware Configuration File information and return 3518 * successfully. 3519 */ 3520 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\ 3521 "Configuration File \"%s\", version %#x, computed checksum %#x\n", 3522 config_name, finiver, cfcsum); 3523 return 0; 3524 3525 /* 3526 * Something bad happened. Return the error ... (If the "error" 3527 * is that there's no Configuration File on the adapter we don't 3528 * want to issue a warning since this is fairly common.) 3529 */ 3530 bye: 3531 if (config_issued && ret != -ENOENT) 3532 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n", 3533 config_name, -ret); 3534 return ret; 3535 } 3536 3537 static struct fw_info fw_info_array[] = { 3538 { 3539 .chip = CHELSIO_T4, 3540 .fs_name = FW4_CFNAME, 3541 .fw_mod_name = FW4_FNAME, 3542 .fw_hdr = { 3543 .chip = FW_HDR_CHIP_T4, 3544 .fw_ver = __cpu_to_be32(FW_VERSION(T4)), 3545 .intfver_nic = FW_INTFVER(T4, NIC), 3546 .intfver_vnic = FW_INTFVER(T4, VNIC), 3547 .intfver_ri = FW_INTFVER(T4, RI), 3548 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 3549 .intfver_fcoe = FW_INTFVER(T4, FCOE), 3550 }, 3551 }, { 3552 .chip = CHELSIO_T5, 3553 .fs_name = FW5_CFNAME, 3554 .fw_mod_name = FW5_FNAME, 3555 .fw_hdr = { 3556 .chip = FW_HDR_CHIP_T5, 3557 .fw_ver = __cpu_to_be32(FW_VERSION(T5)), 3558 .intfver_nic = FW_INTFVER(T5, NIC), 3559 .intfver_vnic = FW_INTFVER(T5, VNIC), 3560 .intfver_ri = FW_INTFVER(T5, RI), 3561 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 3562 .intfver_fcoe = FW_INTFVER(T5, FCOE), 3563 }, 3564 }, { 3565 .chip = CHELSIO_T6, 3566 .fs_name = FW6_CFNAME, 3567 .fw_mod_name = FW6_FNAME, 3568 .fw_hdr = { 3569 .chip = FW_HDR_CHIP_T6, 3570 .fw_ver = __cpu_to_be32(FW_VERSION(T6)), 3571 .intfver_nic = FW_INTFVER(T6, NIC), 3572 .intfver_vnic = FW_INTFVER(T6, VNIC), 3573 .intfver_ofld = FW_INTFVER(T6, OFLD), 3574 .intfver_ri = FW_INTFVER(T6, RI), 3575 .intfver_iscsipdu = FW_INTFVER(T6, ISCSIPDU), 3576 .intfver_iscsi = FW_INTFVER(T6, ISCSI), 3577 .intfver_fcoepdu = FW_INTFVER(T6, FCOEPDU), 3578 .intfver_fcoe = FW_INTFVER(T6, FCOE), 3579 }, 3580 } 3581 3582 }; 3583 3584 static struct fw_info *find_fw_info(int chip) 3585 { 3586 int i; 3587 3588 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) { 3589 if (fw_info_array[i].chip == chip) 3590 return &fw_info_array[i]; 3591 } 3592 return NULL; 3593 } 3594 3595 /* 3596 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 3597 */ 3598 static int adap_init0(struct adapter *adap) 3599 { 3600 int ret; 3601 u32 v, port_vec; 3602 enum dev_state state; 3603 u32 params[7], val[7]; 3604 struct fw_caps_config_cmd caps_cmd; 3605 int reset = 1; 3606 3607 /* Grab Firmware Device Log parameters as early as possible so we have 3608 * access to it for debugging, etc. 3609 */ 3610 ret = t4_init_devlog_params(adap); 3611 if (ret < 0) 3612 return ret; 3613 3614 /* Contact FW, advertising Master capability */ 3615 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, 3616 is_kdump_kernel() ? MASTER_MUST : MASTER_MAY, &state); 3617 if (ret < 0) { 3618 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n", 3619 ret); 3620 return ret; 3621 } 3622 if (ret == adap->mbox) 3623 adap->flags |= MASTER_PF; 3624 3625 /* 3626 * If we're the Master PF Driver and the device is uninitialized, 3627 * then let's consider upgrading the firmware ... (We always want 3628 * to check the firmware version number in order to A. get it for 3629 * later reporting and B. to warn if the currently loaded firmware 3630 * is excessively mismatched relative to the driver.) 3631 */ 3632 3633 t4_get_version_info(adap); 3634 ret = t4_check_fw_version(adap); 3635 /* If firmware is too old (not supported by driver) force an update. */ 3636 if (ret) 3637 state = DEV_STATE_UNINIT; 3638 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) { 3639 struct fw_info *fw_info; 3640 struct fw_hdr *card_fw; 3641 const struct firmware *fw; 3642 const u8 *fw_data = NULL; 3643 unsigned int fw_size = 0; 3644 3645 /* This is the firmware whose headers the driver was compiled 3646 * against 3647 */ 3648 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip)); 3649 if (fw_info == NULL) { 3650 dev_err(adap->pdev_dev, 3651 "unable to get firmware info for chip %d.\n", 3652 CHELSIO_CHIP_VERSION(adap->params.chip)); 3653 return -EINVAL; 3654 } 3655 3656 /* allocate memory to read the header of the firmware on the 3657 * card 3658 */ 3659 card_fw = kvzalloc(sizeof(*card_fw), GFP_KERNEL); 3660 3661 /* Get FW from from /lib/firmware/ */ 3662 ret = request_firmware(&fw, fw_info->fw_mod_name, 3663 adap->pdev_dev); 3664 if (ret < 0) { 3665 dev_err(adap->pdev_dev, 3666 "unable to load firmware image %s, error %d\n", 3667 fw_info->fw_mod_name, ret); 3668 } else { 3669 fw_data = fw->data; 3670 fw_size = fw->size; 3671 } 3672 3673 /* upgrade FW logic */ 3674 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw, 3675 state, &reset); 3676 3677 /* Cleaning up */ 3678 release_firmware(fw); 3679 kvfree(card_fw); 3680 3681 if (ret < 0) 3682 goto bye; 3683 } 3684 3685 /* 3686 * Grab VPD parameters. This should be done after we establish a 3687 * connection to the firmware since some of the VPD parameters 3688 * (notably the Core Clock frequency) are retrieved via requests to 3689 * the firmware. On the other hand, we need these fairly early on 3690 * so we do this right after getting ahold of the firmware. 3691 */ 3692 ret = t4_get_vpd_params(adap, &adap->params.vpd); 3693 if (ret < 0) 3694 goto bye; 3695 3696 /* 3697 * Find out what ports are available to us. Note that we need to do 3698 * this before calling adap_init0_no_config() since it needs nports 3699 * and portvec ... 3700 */ 3701 v = 3702 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3703 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 3704 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, &v, &port_vec); 3705 if (ret < 0) 3706 goto bye; 3707 3708 adap->params.nports = hweight32(port_vec); 3709 adap->params.portvec = port_vec; 3710 3711 /* If the firmware is initialized already, emit a simply note to that 3712 * effect. Otherwise, it's time to try initializing the adapter. 3713 */ 3714 if (state == DEV_STATE_INIT) { 3715 dev_info(adap->pdev_dev, "Coming up as %s: "\ 3716 "Adapter already initialized\n", 3717 adap->flags & MASTER_PF ? "MASTER" : "SLAVE"); 3718 } else { 3719 dev_info(adap->pdev_dev, "Coming up as MASTER: "\ 3720 "Initializing adapter\n"); 3721 3722 /* Find out whether we're dealing with a version of the 3723 * firmware which has configuration file support. 3724 */ 3725 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 3726 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 3727 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 1, 3728 params, val); 3729 3730 /* If the firmware doesn't support Configuration Files, 3731 * return an error. 3732 */ 3733 if (ret < 0) { 3734 dev_err(adap->pdev_dev, "firmware doesn't support " 3735 "Firmware Configuration Files\n"); 3736 goto bye; 3737 } 3738 3739 /* The firmware provides us with a memory buffer where we can 3740 * load a Configuration File from the host if we want to 3741 * override the Configuration File in flash. 3742 */ 3743 ret = adap_init0_config(adap, reset); 3744 if (ret == -ENOENT) { 3745 dev_err(adap->pdev_dev, "no Configuration File " 3746 "present on adapter.\n"); 3747 goto bye; 3748 } 3749 if (ret < 0) { 3750 dev_err(adap->pdev_dev, "could not initialize " 3751 "adapter, error %d\n", -ret); 3752 goto bye; 3753 } 3754 } 3755 3756 /* Give the SGE code a chance to pull in anything that it needs ... 3757 * Note that this must be called after we retrieve our VPD parameters 3758 * in order to know how to convert core ticks to seconds, etc. 3759 */ 3760 ret = t4_sge_init(adap); 3761 if (ret < 0) 3762 goto bye; 3763 3764 if (is_bypass_device(adap->pdev->device)) 3765 adap->params.bypass = 1; 3766 3767 /* 3768 * Grab some of our basic fundamental operating parameters. 3769 */ 3770 #define FW_PARAM_DEV(param) \ 3771 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \ 3772 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param)) 3773 3774 #define FW_PARAM_PFVF(param) \ 3775 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \ 3776 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \ 3777 FW_PARAMS_PARAM_Y_V(0) | \ 3778 FW_PARAMS_PARAM_Z_V(0) 3779 3780 params[0] = FW_PARAM_PFVF(EQ_START); 3781 params[1] = FW_PARAM_PFVF(L2T_START); 3782 params[2] = FW_PARAM_PFVF(L2T_END); 3783 params[3] = FW_PARAM_PFVF(FILTER_START); 3784 params[4] = FW_PARAM_PFVF(FILTER_END); 3785 params[5] = FW_PARAM_PFVF(IQFLINT_START); 3786 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, val); 3787 if (ret < 0) 3788 goto bye; 3789 adap->sge.egr_start = val[0]; 3790 adap->l2t_start = val[1]; 3791 adap->l2t_end = val[2]; 3792 adap->tids.ftid_base = val[3]; 3793 adap->tids.nftids = val[4] - val[3] + 1; 3794 adap->sge.ingr_start = val[5]; 3795 3796 /* qids (ingress/egress) returned from firmware can be anywhere 3797 * in the range from EQ(IQFLINT)_START to EQ(IQFLINT)_END. 3798 * Hence driver needs to allocate memory for this range to 3799 * store the queue info. Get the highest IQFLINT/EQ index returned 3800 * in FW_EQ_*_CMD.alloc command. 3801 */ 3802 params[0] = FW_PARAM_PFVF(EQ_END); 3803 params[1] = FW_PARAM_PFVF(IQFLINT_END); 3804 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3805 if (ret < 0) 3806 goto bye; 3807 adap->sge.egr_sz = val[0] - adap->sge.egr_start + 1; 3808 adap->sge.ingr_sz = val[1] - adap->sge.ingr_start + 1; 3809 3810 adap->sge.egr_map = kcalloc(adap->sge.egr_sz, 3811 sizeof(*adap->sge.egr_map), GFP_KERNEL); 3812 if (!adap->sge.egr_map) { 3813 ret = -ENOMEM; 3814 goto bye; 3815 } 3816 3817 adap->sge.ingr_map = kcalloc(adap->sge.ingr_sz, 3818 sizeof(*adap->sge.ingr_map), GFP_KERNEL); 3819 if (!adap->sge.ingr_map) { 3820 ret = -ENOMEM; 3821 goto bye; 3822 } 3823 3824 /* Allocate the memory for the vaious egress queue bitmaps 3825 * ie starving_fl, txq_maperr and blocked_fl. 3826 */ 3827 adap->sge.starving_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3828 sizeof(long), GFP_KERNEL); 3829 if (!adap->sge.starving_fl) { 3830 ret = -ENOMEM; 3831 goto bye; 3832 } 3833 3834 adap->sge.txq_maperr = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3835 sizeof(long), GFP_KERNEL); 3836 if (!adap->sge.txq_maperr) { 3837 ret = -ENOMEM; 3838 goto bye; 3839 } 3840 3841 #ifdef CONFIG_DEBUG_FS 3842 adap->sge.blocked_fl = kcalloc(BITS_TO_LONGS(adap->sge.egr_sz), 3843 sizeof(long), GFP_KERNEL); 3844 if (!adap->sge.blocked_fl) { 3845 ret = -ENOMEM; 3846 goto bye; 3847 } 3848 #endif 3849 3850 params[0] = FW_PARAM_PFVF(CLIP_START); 3851 params[1] = FW_PARAM_PFVF(CLIP_END); 3852 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3853 if (ret < 0) 3854 goto bye; 3855 adap->clipt_start = val[0]; 3856 adap->clipt_end = val[1]; 3857 3858 /* We don't yet have a PARAMs calls to retrieve the number of Traffic 3859 * Classes supported by the hardware/firmware so we hard code it here 3860 * for now. 3861 */ 3862 adap->params.nsched_cls = is_t4(adap->params.chip) ? 15 : 16; 3863 3864 /* query params related to active filter region */ 3865 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START); 3866 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END); 3867 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, val); 3868 /* If Active filter size is set we enable establishing 3869 * offload connection through firmware work request 3870 */ 3871 if ((val[0] != val[1]) && (ret >= 0)) { 3872 adap->flags |= FW_OFLD_CONN; 3873 adap->tids.aftid_base = val[0]; 3874 adap->tids.aftid_end = val[1]; 3875 } 3876 3877 /* If we're running on newer firmware, let it know that we're 3878 * prepared to deal with encapsulated CPL messages. Older 3879 * firmware won't understand this and we'll just get 3880 * unencapsulated messages ... 3881 */ 3882 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 3883 val[0] = 1; 3884 (void)t4_set_params(adap, adap->mbox, adap->pf, 0, 1, params, val); 3885 3886 /* 3887 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL 3888 * capability. Earlier versions of the firmware didn't have the 3889 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no 3890 * permission to use ULPTX MEMWRITE DSGL. 3891 */ 3892 if (is_t4(adap->params.chip)) { 3893 adap->params.ulptx_memwrite_dsgl = false; 3894 } else { 3895 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); 3896 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 3897 1, params, val); 3898 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0); 3899 } 3900 3901 /* See if FW supports FW_RI_FR_NSMR_TPTE_WR work request */ 3902 params[0] = FW_PARAM_DEV(RI_FR_NSMR_TPTE_WR); 3903 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 3904 1, params, val); 3905 adap->params.fr_nsmr_tpte_wr_support = (ret == 0 && val[0] != 0); 3906 3907 /* 3908 * Get device capabilities so we can determine what resources we need 3909 * to manage. 3910 */ 3911 memset(&caps_cmd, 0, sizeof(caps_cmd)); 3912 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 3913 FW_CMD_REQUEST_F | FW_CMD_READ_F); 3914 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 3915 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd), 3916 &caps_cmd); 3917 if (ret < 0) 3918 goto bye; 3919 3920 if (caps_cmd.ofldcaps) { 3921 /* query offload-related parameters */ 3922 params[0] = FW_PARAM_DEV(NTID); 3923 params[1] = FW_PARAM_PFVF(SERVER_START); 3924 params[2] = FW_PARAM_PFVF(SERVER_END); 3925 params[3] = FW_PARAM_PFVF(TDDP_START); 3926 params[4] = FW_PARAM_PFVF(TDDP_END); 3927 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 3928 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 3929 params, val); 3930 if (ret < 0) 3931 goto bye; 3932 adap->tids.ntids = val[0]; 3933 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); 3934 adap->tids.stid_base = val[1]; 3935 adap->tids.nstids = val[2] - val[1] + 1; 3936 /* 3937 * Setup server filter region. Divide the available filter 3938 * region into two parts. Regular filters get 1/3rd and server 3939 * filters get 2/3rd part. This is only enabled if workarond 3940 * path is enabled. 3941 * 1. For regular filters. 3942 * 2. Server filter: This are special filters which are used 3943 * to redirect SYN packets to offload queue. 3944 */ 3945 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) { 3946 adap->tids.sftid_base = adap->tids.ftid_base + 3947 DIV_ROUND_UP(adap->tids.nftids, 3); 3948 adap->tids.nsftids = adap->tids.nftids - 3949 DIV_ROUND_UP(adap->tids.nftids, 3); 3950 adap->tids.nftids = adap->tids.sftid_base - 3951 adap->tids.ftid_base; 3952 } 3953 adap->vres.ddp.start = val[3]; 3954 adap->vres.ddp.size = val[4] - val[3] + 1; 3955 adap->params.ofldq_wr_cred = val[5]; 3956 3957 adap->params.offload = 1; 3958 adap->num_ofld_uld += 1; 3959 } 3960 if (caps_cmd.rdmacaps) { 3961 params[0] = FW_PARAM_PFVF(STAG_START); 3962 params[1] = FW_PARAM_PFVF(STAG_END); 3963 params[2] = FW_PARAM_PFVF(RQ_START); 3964 params[3] = FW_PARAM_PFVF(RQ_END); 3965 params[4] = FW_PARAM_PFVF(PBL_START); 3966 params[5] = FW_PARAM_PFVF(PBL_END); 3967 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, 3968 params, val); 3969 if (ret < 0) 3970 goto bye; 3971 adap->vres.stag.start = val[0]; 3972 adap->vres.stag.size = val[1] - val[0] + 1; 3973 adap->vres.rq.start = val[2]; 3974 adap->vres.rq.size = val[3] - val[2] + 1; 3975 adap->vres.pbl.start = val[4]; 3976 adap->vres.pbl.size = val[5] - val[4] + 1; 3977 3978 params[0] = FW_PARAM_PFVF(SQRQ_START); 3979 params[1] = FW_PARAM_PFVF(SQRQ_END); 3980 params[2] = FW_PARAM_PFVF(CQ_START); 3981 params[3] = FW_PARAM_PFVF(CQ_END); 3982 params[4] = FW_PARAM_PFVF(OCQ_START); 3983 params[5] = FW_PARAM_PFVF(OCQ_END); 3984 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 6, params, 3985 val); 3986 if (ret < 0) 3987 goto bye; 3988 adap->vres.qp.start = val[0]; 3989 adap->vres.qp.size = val[1] - val[0] + 1; 3990 adap->vres.cq.start = val[2]; 3991 adap->vres.cq.size = val[3] - val[2] + 1; 3992 adap->vres.ocq.start = val[4]; 3993 adap->vres.ocq.size = val[5] - val[4] + 1; 3994 3995 params[0] = FW_PARAM_DEV(MAXORDIRD_QP); 3996 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER); 3997 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, params, 3998 val); 3999 if (ret < 0) { 4000 adap->params.max_ordird_qp = 8; 4001 adap->params.max_ird_adapter = 32 * adap->tids.ntids; 4002 ret = 0; 4003 } else { 4004 adap->params.max_ordird_qp = val[0]; 4005 adap->params.max_ird_adapter = val[1]; 4006 } 4007 dev_info(adap->pdev_dev, 4008 "max_ordird_qp %d max_ird_adapter %d\n", 4009 adap->params.max_ordird_qp, 4010 adap->params.max_ird_adapter); 4011 adap->num_ofld_uld += 2; 4012 } 4013 if (caps_cmd.iscsicaps) { 4014 params[0] = FW_PARAM_PFVF(ISCSI_START); 4015 params[1] = FW_PARAM_PFVF(ISCSI_END); 4016 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 4017 params, val); 4018 if (ret < 0) 4019 goto bye; 4020 adap->vres.iscsi.start = val[0]; 4021 adap->vres.iscsi.size = val[1] - val[0] + 1; 4022 /* LIO target and cxgb4i initiaitor */ 4023 adap->num_ofld_uld += 2; 4024 } 4025 if (caps_cmd.cryptocaps) { 4026 /* Should query params here...TODO */ 4027 params[0] = FW_PARAM_PFVF(NCRYPTO_LOOKASIDE); 4028 ret = t4_query_params(adap, adap->mbox, adap->pf, 0, 2, 4029 params, val); 4030 if (ret < 0) { 4031 if (ret != -EINVAL) 4032 goto bye; 4033 } else { 4034 adap->vres.ncrypto_fc = val[0]; 4035 } 4036 adap->params.crypto |= ULP_CRYPTO_LOOKASIDE; 4037 adap->num_uld += 1; 4038 } 4039 #undef FW_PARAM_PFVF 4040 #undef FW_PARAM_DEV 4041 4042 /* The MTU/MSS Table is initialized by now, so load their values. If 4043 * we're initializing the adapter, then we'll make any modifications 4044 * we want to the MTU/MSS Table and also initialize the congestion 4045 * parameters. 4046 */ 4047 t4_read_mtu_tbl(adap, adap->params.mtus, NULL); 4048 if (state != DEV_STATE_INIT) { 4049 int i; 4050 4051 /* The default MTU Table contains values 1492 and 1500. 4052 * However, for TCP, it's better to have two values which are 4053 * a multiple of 8 +/- 4 bytes apart near this popular MTU. 4054 * This allows us to have a TCP Data Payload which is a 4055 * multiple of 8 regardless of what combination of TCP Options 4056 * are in use (always a multiple of 4 bytes) which is 4057 * important for performance reasons. For instance, if no 4058 * options are in use, then we have a 20-byte IP header and a 4059 * 20-byte TCP header. In this case, a 1500-byte MSS would 4060 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes 4061 * which is not a multiple of 8. So using an MSS of 1488 in 4062 * this case results in a TCP Data Payload of 1448 bytes which 4063 * is a multiple of 8. On the other hand, if 12-byte TCP Time 4064 * Stamps have been negotiated, then an MTU of 1500 bytes 4065 * results in a TCP Data Payload of 1448 bytes which, as 4066 * above, is a multiple of 8 bytes ... 4067 */ 4068 for (i = 0; i < NMTUS; i++) 4069 if (adap->params.mtus[i] == 1492) { 4070 adap->params.mtus[i] = 1488; 4071 break; 4072 } 4073 4074 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 4075 adap->params.b_wnd); 4076 } 4077 t4_init_sge_params(adap); 4078 adap->flags |= FW_OK; 4079 t4_init_tp_params(adap); 4080 return 0; 4081 4082 /* 4083 * Something bad happened. If a command timed out or failed with EIO 4084 * FW does not operate within its spec or something catastrophic 4085 * happened to HW/FW, stop issuing commands. 4086 */ 4087 bye: 4088 kfree(adap->sge.egr_map); 4089 kfree(adap->sge.ingr_map); 4090 kfree(adap->sge.starving_fl); 4091 kfree(adap->sge.txq_maperr); 4092 #ifdef CONFIG_DEBUG_FS 4093 kfree(adap->sge.blocked_fl); 4094 #endif 4095 if (ret != -ETIMEDOUT && ret != -EIO) 4096 t4_fw_bye(adap, adap->mbox); 4097 return ret; 4098 } 4099 4100 /* EEH callbacks */ 4101 4102 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev, 4103 pci_channel_state_t state) 4104 { 4105 int i; 4106 struct adapter *adap = pci_get_drvdata(pdev); 4107 4108 if (!adap) 4109 goto out; 4110 4111 rtnl_lock(); 4112 adap->flags &= ~FW_OK; 4113 notify_ulds(adap, CXGB4_STATE_START_RECOVERY); 4114 spin_lock(&adap->stats_lock); 4115 for_each_port(adap, i) { 4116 struct net_device *dev = adap->port[i]; 4117 if (dev) { 4118 netif_device_detach(dev); 4119 netif_carrier_off(dev); 4120 } 4121 } 4122 spin_unlock(&adap->stats_lock); 4123 disable_interrupts(adap); 4124 if (adap->flags & FULL_INIT_DONE) 4125 cxgb_down(adap); 4126 rtnl_unlock(); 4127 if ((adap->flags & DEV_ENABLED)) { 4128 pci_disable_device(pdev); 4129 adap->flags &= ~DEV_ENABLED; 4130 } 4131 out: return state == pci_channel_io_perm_failure ? 4132 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 4133 } 4134 4135 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) 4136 { 4137 int i, ret; 4138 struct fw_caps_config_cmd c; 4139 struct adapter *adap = pci_get_drvdata(pdev); 4140 4141 if (!adap) { 4142 pci_restore_state(pdev); 4143 pci_save_state(pdev); 4144 return PCI_ERS_RESULT_RECOVERED; 4145 } 4146 4147 if (!(adap->flags & DEV_ENABLED)) { 4148 if (pci_enable_device(pdev)) { 4149 dev_err(&pdev->dev, "Cannot reenable PCI " 4150 "device after reset\n"); 4151 return PCI_ERS_RESULT_DISCONNECT; 4152 } 4153 adap->flags |= DEV_ENABLED; 4154 } 4155 4156 pci_set_master(pdev); 4157 pci_restore_state(pdev); 4158 pci_save_state(pdev); 4159 pci_cleanup_aer_uncorrect_error_status(pdev); 4160 4161 if (t4_wait_dev_ready(adap->regs) < 0) 4162 return PCI_ERS_RESULT_DISCONNECT; 4163 if (t4_fw_hello(adap, adap->mbox, adap->pf, MASTER_MUST, NULL) < 0) 4164 return PCI_ERS_RESULT_DISCONNECT; 4165 adap->flags |= FW_OK; 4166 if (adap_init1(adap, &c)) 4167 return PCI_ERS_RESULT_DISCONNECT; 4168 4169 for_each_port(adap, i) { 4170 struct port_info *p = adap2pinfo(adap, i); 4171 4172 ret = t4_alloc_vi(adap, adap->mbox, p->tx_chan, adap->pf, 0, 1, 4173 NULL, NULL); 4174 if (ret < 0) 4175 return PCI_ERS_RESULT_DISCONNECT; 4176 p->viid = ret; 4177 p->xact_addr_filt = -1; 4178 } 4179 4180 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 4181 adap->params.b_wnd); 4182 setup_memwin(adap); 4183 if (cxgb_up(adap)) 4184 return PCI_ERS_RESULT_DISCONNECT; 4185 return PCI_ERS_RESULT_RECOVERED; 4186 } 4187 4188 static void eeh_resume(struct pci_dev *pdev) 4189 { 4190 int i; 4191 struct adapter *adap = pci_get_drvdata(pdev); 4192 4193 if (!adap) 4194 return; 4195 4196 rtnl_lock(); 4197 for_each_port(adap, i) { 4198 struct net_device *dev = adap->port[i]; 4199 if (dev) { 4200 if (netif_running(dev)) { 4201 link_start(dev); 4202 cxgb_set_rxmode(dev); 4203 } 4204 netif_device_attach(dev); 4205 } 4206 } 4207 rtnl_unlock(); 4208 } 4209 4210 static const struct pci_error_handlers cxgb4_eeh = { 4211 .error_detected = eeh_err_detected, 4212 .slot_reset = eeh_slot_reset, 4213 .resume = eeh_resume, 4214 }; 4215 4216 /* Return true if the Link Configuration supports "High Speeds" (those greater 4217 * than 1Gb/s). 4218 */ 4219 static inline bool is_x_10g_port(const struct link_config *lc) 4220 { 4221 unsigned int speeds, high_speeds; 4222 4223 speeds = FW_PORT_CAP32_SPEED_V(FW_PORT_CAP32_SPEED_G(lc->pcaps)); 4224 high_speeds = speeds & 4225 ~(FW_PORT_CAP32_SPEED_100M | FW_PORT_CAP32_SPEED_1G); 4226 4227 return high_speeds != 0; 4228 } 4229 4230 /* 4231 * Perform default configuration of DMA queues depending on the number and type 4232 * of ports we found and the number of available CPUs. Most settings can be 4233 * modified by the admin prior to actual use. 4234 */ 4235 static void cfg_queues(struct adapter *adap) 4236 { 4237 struct sge *s = &adap->sge; 4238 int i = 0, n10g = 0, qidx = 0; 4239 #ifndef CONFIG_CHELSIO_T4_DCB 4240 int q10g = 0; 4241 #endif 4242 4243 /* Reduce memory usage in kdump environment, disable all offload. 4244 */ 4245 if (is_kdump_kernel() || (is_uld(adap) && t4_uld_mem_alloc(adap))) { 4246 adap->params.offload = 0; 4247 adap->params.crypto = 0; 4248 } 4249 4250 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg); 4251 #ifdef CONFIG_CHELSIO_T4_DCB 4252 /* For Data Center Bridging support we need to be able to support up 4253 * to 8 Traffic Priorities; each of which will be assigned to its 4254 * own TX Queue in order to prevent Head-Of-Line Blocking. 4255 */ 4256 if (adap->params.nports * 8 > MAX_ETH_QSETS) { 4257 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n", 4258 MAX_ETH_QSETS, adap->params.nports * 8); 4259 BUG_ON(1); 4260 } 4261 4262 for_each_port(adap, i) { 4263 struct port_info *pi = adap2pinfo(adap, i); 4264 4265 pi->first_qset = qidx; 4266 pi->nqsets = is_kdump_kernel() ? 1 : 8; 4267 qidx += pi->nqsets; 4268 } 4269 #else /* !CONFIG_CHELSIO_T4_DCB */ 4270 /* 4271 * We default to 1 queue per non-10G port and up to # of cores queues 4272 * per 10G port. 4273 */ 4274 if (n10g) 4275 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g; 4276 if (q10g > netif_get_num_default_rss_queues()) 4277 q10g = netif_get_num_default_rss_queues(); 4278 4279 if (is_kdump_kernel()) 4280 q10g = 1; 4281 4282 for_each_port(adap, i) { 4283 struct port_info *pi = adap2pinfo(adap, i); 4284 4285 pi->first_qset = qidx; 4286 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1; 4287 qidx += pi->nqsets; 4288 } 4289 #endif /* !CONFIG_CHELSIO_T4_DCB */ 4290 4291 s->ethqsets = qidx; 4292 s->max_ethqsets = qidx; /* MSI-X may lower it later */ 4293 4294 if (is_uld(adap)) { 4295 /* 4296 * For offload we use 1 queue/channel if all ports are up to 1G, 4297 * otherwise we divide all available queues amongst the channels 4298 * capped by the number of available cores. 4299 */ 4300 if (n10g) { 4301 i = min_t(int, MAX_OFLD_QSETS, num_online_cpus()); 4302 s->ofldqsets = roundup(i, adap->params.nports); 4303 } else { 4304 s->ofldqsets = adap->params.nports; 4305 } 4306 } 4307 4308 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { 4309 struct sge_eth_rxq *r = &s->ethrxq[i]; 4310 4311 init_rspq(adap, &r->rspq, 5, 10, 1024, 64); 4312 r->fl.size = 72; 4313 } 4314 4315 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) 4316 s->ethtxq[i].q.size = 1024; 4317 4318 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) 4319 s->ctrlq[i].q.size = 512; 4320 4321 if (!is_t4(adap->params.chip)) 4322 s->ptptxq.q.size = 8; 4323 4324 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64); 4325 init_rspq(adap, &s->intrq, 0, 1, 512, 64); 4326 } 4327 4328 /* 4329 * Reduce the number of Ethernet queues across all ports to at most n. 4330 * n provides at least one queue per port. 4331 */ 4332 static void reduce_ethqs(struct adapter *adap, int n) 4333 { 4334 int i; 4335 struct port_info *pi; 4336 4337 while (n < adap->sge.ethqsets) 4338 for_each_port(adap, i) { 4339 pi = adap2pinfo(adap, i); 4340 if (pi->nqsets > 1) { 4341 pi->nqsets--; 4342 adap->sge.ethqsets--; 4343 if (adap->sge.ethqsets <= n) 4344 break; 4345 } 4346 } 4347 4348 n = 0; 4349 for_each_port(adap, i) { 4350 pi = adap2pinfo(adap, i); 4351 pi->first_qset = n; 4352 n += pi->nqsets; 4353 } 4354 } 4355 4356 static int get_msix_info(struct adapter *adap) 4357 { 4358 struct uld_msix_info *msix_info; 4359 unsigned int max_ingq = 0; 4360 4361 if (is_offload(adap)) 4362 max_ingq += MAX_OFLD_QSETS * adap->num_ofld_uld; 4363 if (is_pci_uld(adap)) 4364 max_ingq += MAX_OFLD_QSETS * adap->num_uld; 4365 4366 if (!max_ingq) 4367 goto out; 4368 4369 msix_info = kcalloc(max_ingq, sizeof(*msix_info), GFP_KERNEL); 4370 if (!msix_info) 4371 return -ENOMEM; 4372 4373 adap->msix_bmap_ulds.msix_bmap = kcalloc(BITS_TO_LONGS(max_ingq), 4374 sizeof(long), GFP_KERNEL); 4375 if (!adap->msix_bmap_ulds.msix_bmap) { 4376 kfree(msix_info); 4377 return -ENOMEM; 4378 } 4379 spin_lock_init(&adap->msix_bmap_ulds.lock); 4380 adap->msix_info_ulds = msix_info; 4381 out: 4382 return 0; 4383 } 4384 4385 static void free_msix_info(struct adapter *adap) 4386 { 4387 if (!(adap->num_uld && adap->num_ofld_uld)) 4388 return; 4389 4390 kfree(adap->msix_info_ulds); 4391 kfree(adap->msix_bmap_ulds.msix_bmap); 4392 } 4393 4394 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */ 4395 #define EXTRA_VECS 2 4396 4397 static int enable_msix(struct adapter *adap) 4398 { 4399 int ofld_need = 0, uld_need = 0; 4400 int i, j, want, need, allocated; 4401 struct sge *s = &adap->sge; 4402 unsigned int nchan = adap->params.nports; 4403 struct msix_entry *entries; 4404 int max_ingq = MAX_INGQ; 4405 4406 if (is_pci_uld(adap)) 4407 max_ingq += (MAX_OFLD_QSETS * adap->num_uld); 4408 if (is_offload(adap)) 4409 max_ingq += (MAX_OFLD_QSETS * adap->num_ofld_uld); 4410 entries = kmalloc(sizeof(*entries) * (max_ingq + 1), 4411 GFP_KERNEL); 4412 if (!entries) 4413 return -ENOMEM; 4414 4415 /* map for msix */ 4416 if (get_msix_info(adap)) { 4417 adap->params.offload = 0; 4418 adap->params.crypto = 0; 4419 } 4420 4421 for (i = 0; i < max_ingq + 1; ++i) 4422 entries[i].entry = i; 4423 4424 want = s->max_ethqsets + EXTRA_VECS; 4425 if (is_offload(adap)) { 4426 want += adap->num_ofld_uld * s->ofldqsets; 4427 ofld_need = adap->num_ofld_uld * nchan; 4428 } 4429 if (is_pci_uld(adap)) { 4430 want += adap->num_uld * s->ofldqsets; 4431 uld_need = adap->num_uld * nchan; 4432 } 4433 #ifdef CONFIG_CHELSIO_T4_DCB 4434 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for 4435 * each port. 4436 */ 4437 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need + uld_need; 4438 #else 4439 need = adap->params.nports + EXTRA_VECS + ofld_need + uld_need; 4440 #endif 4441 allocated = pci_enable_msix_range(adap->pdev, entries, need, want); 4442 if (allocated < 0) { 4443 dev_info(adap->pdev_dev, "not enough MSI-X vectors left," 4444 " not using MSI-X\n"); 4445 kfree(entries); 4446 return allocated; 4447 } 4448 4449 /* Distribute available vectors to the various queue groups. 4450 * Every group gets its minimum requirement and NIC gets top 4451 * priority for leftovers. 4452 */ 4453 i = allocated - EXTRA_VECS - ofld_need - uld_need; 4454 if (i < s->max_ethqsets) { 4455 s->max_ethqsets = i; 4456 if (i < s->ethqsets) 4457 reduce_ethqs(adap, i); 4458 } 4459 if (is_uld(adap)) { 4460 if (allocated < want) 4461 s->nqs_per_uld = nchan; 4462 else 4463 s->nqs_per_uld = s->ofldqsets; 4464 } 4465 4466 for (i = 0; i < (s->max_ethqsets + EXTRA_VECS); ++i) 4467 adap->msix_info[i].vec = entries[i].vector; 4468 if (is_uld(adap)) { 4469 for (j = 0 ; i < allocated; ++i, j++) { 4470 adap->msix_info_ulds[j].vec = entries[i].vector; 4471 adap->msix_info_ulds[j].idx = i; 4472 } 4473 adap->msix_bmap_ulds.mapsize = j; 4474 } 4475 dev_info(adap->pdev_dev, "%d MSI-X vectors allocated, " 4476 "nic %d per uld %d\n", 4477 allocated, s->max_ethqsets, s->nqs_per_uld); 4478 4479 kfree(entries); 4480 return 0; 4481 } 4482 4483 #undef EXTRA_VECS 4484 4485 static int init_rss(struct adapter *adap) 4486 { 4487 unsigned int i; 4488 int err; 4489 4490 err = t4_init_rss_mode(adap, adap->mbox); 4491 if (err) 4492 return err; 4493 4494 for_each_port(adap, i) { 4495 struct port_info *pi = adap2pinfo(adap, i); 4496 4497 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); 4498 if (!pi->rss) 4499 return -ENOMEM; 4500 } 4501 return 0; 4502 } 4503 4504 static int cxgb4_get_pcie_dev_link_caps(struct adapter *adap, 4505 enum pci_bus_speed *speed, 4506 enum pcie_link_width *width) 4507 { 4508 u32 lnkcap1, lnkcap2; 4509 int err1, err2; 4510 4511 #define PCIE_MLW_CAP_SHIFT 4 /* start of MLW mask in link capabilities */ 4512 4513 *speed = PCI_SPEED_UNKNOWN; 4514 *width = PCIE_LNK_WIDTH_UNKNOWN; 4515 4516 err1 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP, 4517 &lnkcap1); 4518 err2 = pcie_capability_read_dword(adap->pdev, PCI_EXP_LNKCAP2, 4519 &lnkcap2); 4520 if (!err2 && lnkcap2) { /* PCIe r3.0-compliant */ 4521 if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_8_0GB) 4522 *speed = PCIE_SPEED_8_0GT; 4523 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_5_0GB) 4524 *speed = PCIE_SPEED_5_0GT; 4525 else if (lnkcap2 & PCI_EXP_LNKCAP2_SLS_2_5GB) 4526 *speed = PCIE_SPEED_2_5GT; 4527 } 4528 if (!err1) { 4529 *width = (lnkcap1 & PCI_EXP_LNKCAP_MLW) >> PCIE_MLW_CAP_SHIFT; 4530 if (!lnkcap2) { /* pre-r3.0 */ 4531 if (lnkcap1 & PCI_EXP_LNKCAP_SLS_5_0GB) 4532 *speed = PCIE_SPEED_5_0GT; 4533 else if (lnkcap1 & PCI_EXP_LNKCAP_SLS_2_5GB) 4534 *speed = PCIE_SPEED_2_5GT; 4535 } 4536 } 4537 4538 if (*speed == PCI_SPEED_UNKNOWN || *width == PCIE_LNK_WIDTH_UNKNOWN) 4539 return err1 ? err1 : err2 ? err2 : -EINVAL; 4540 return 0; 4541 } 4542 4543 static void cxgb4_check_pcie_caps(struct adapter *adap) 4544 { 4545 enum pcie_link_width width, width_cap; 4546 enum pci_bus_speed speed, speed_cap; 4547 4548 #define PCIE_SPEED_STR(speed) \ 4549 (speed == PCIE_SPEED_8_0GT ? "8.0GT/s" : \ 4550 speed == PCIE_SPEED_5_0GT ? "5.0GT/s" : \ 4551 speed == PCIE_SPEED_2_5GT ? "2.5GT/s" : \ 4552 "Unknown") 4553 4554 if (cxgb4_get_pcie_dev_link_caps(adap, &speed_cap, &width_cap)) { 4555 dev_warn(adap->pdev_dev, 4556 "Unable to determine PCIe device BW capabilities\n"); 4557 return; 4558 } 4559 4560 if (pcie_get_minimum_link(adap->pdev, &speed, &width) || 4561 speed == PCI_SPEED_UNKNOWN || width == PCIE_LNK_WIDTH_UNKNOWN) { 4562 dev_warn(adap->pdev_dev, 4563 "Unable to determine PCI Express bandwidth.\n"); 4564 return; 4565 } 4566 4567 dev_info(adap->pdev_dev, "PCIe link speed is %s, device supports %s\n", 4568 PCIE_SPEED_STR(speed), PCIE_SPEED_STR(speed_cap)); 4569 dev_info(adap->pdev_dev, "PCIe link width is x%d, device supports x%d\n", 4570 width, width_cap); 4571 if (speed < speed_cap || width < width_cap) 4572 dev_info(adap->pdev_dev, 4573 "A slot with more lanes and/or higher speed is " 4574 "suggested for optimal performance.\n"); 4575 } 4576 4577 /* Dump basic information about the adapter */ 4578 static void print_adapter_info(struct adapter *adapter) 4579 { 4580 /* Hardware/Firmware/etc. Version/Revision IDs */ 4581 t4_dump_version_info(adapter); 4582 4583 /* Software/Hardware configuration */ 4584 dev_info(adapter->pdev_dev, "Configuration: %sNIC %s, %s capable\n", 4585 is_offload(adapter) ? "R" : "", 4586 ((adapter->flags & USING_MSIX) ? "MSI-X" : 4587 (adapter->flags & USING_MSI) ? "MSI" : ""), 4588 is_offload(adapter) ? "Offload" : "non-Offload"); 4589 } 4590 4591 static void print_port_info(const struct net_device *dev) 4592 { 4593 char buf[80]; 4594 char *bufp = buf; 4595 const char *spd = ""; 4596 const struct port_info *pi = netdev_priv(dev); 4597 const struct adapter *adap = pi->adapter; 4598 4599 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB) 4600 spd = " 2.5 GT/s"; 4601 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB) 4602 spd = " 5 GT/s"; 4603 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB) 4604 spd = " 8 GT/s"; 4605 4606 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100M) 4607 bufp += sprintf(bufp, "100M/"); 4608 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_1G) 4609 bufp += sprintf(bufp, "1G/"); 4610 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_10G) 4611 bufp += sprintf(bufp, "10G/"); 4612 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_25G) 4613 bufp += sprintf(bufp, "25G/"); 4614 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_40G) 4615 bufp += sprintf(bufp, "40G/"); 4616 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_50G) 4617 bufp += sprintf(bufp, "50G/"); 4618 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_100G) 4619 bufp += sprintf(bufp, "100G/"); 4620 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_200G) 4621 bufp += sprintf(bufp, "200G/"); 4622 if (pi->link_cfg.pcaps & FW_PORT_CAP32_SPEED_400G) 4623 bufp += sprintf(bufp, "400G/"); 4624 if (bufp != buf) 4625 --bufp; 4626 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type)); 4627 4628 netdev_info(dev, "%s: Chelsio %s (%s) %s\n", 4629 dev->name, adap->params.vpd.id, adap->name, buf); 4630 } 4631 4632 /* 4633 * Free the following resources: 4634 * - memory used for tables 4635 * - MSI/MSI-X 4636 * - net devices 4637 * - resources FW is holding for us 4638 */ 4639 static void free_some_resources(struct adapter *adapter) 4640 { 4641 unsigned int i; 4642 4643 kvfree(adapter->l2t); 4644 t4_cleanup_sched(adapter); 4645 kvfree(adapter->tids.tid_tab); 4646 cxgb4_cleanup_tc_flower(adapter); 4647 cxgb4_cleanup_tc_u32(adapter); 4648 kfree(adapter->sge.egr_map); 4649 kfree(adapter->sge.ingr_map); 4650 kfree(adapter->sge.starving_fl); 4651 kfree(adapter->sge.txq_maperr); 4652 #ifdef CONFIG_DEBUG_FS 4653 kfree(adapter->sge.blocked_fl); 4654 #endif 4655 disable_msi(adapter); 4656 4657 for_each_port(adapter, i) 4658 if (adapter->port[i]) { 4659 struct port_info *pi = adap2pinfo(adapter, i); 4660 4661 if (pi->viid != 0) 4662 t4_free_vi(adapter, adapter->mbox, adapter->pf, 4663 0, pi->viid); 4664 kfree(adap2pinfo(adapter, i)->rss); 4665 free_netdev(adapter->port[i]); 4666 } 4667 if (adapter->flags & FW_OK) 4668 t4_fw_bye(adapter, adapter->pf); 4669 } 4670 4671 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN) 4672 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ 4673 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) 4674 #define SEGMENT_SIZE 128 4675 4676 static int get_chip_type(struct pci_dev *pdev, u32 pl_rev) 4677 { 4678 u16 device_id; 4679 4680 /* Retrieve adapter's device ID */ 4681 pci_read_config_word(pdev, PCI_DEVICE_ID, &device_id); 4682 4683 switch (device_id >> 12) { 4684 case CHELSIO_T4: 4685 return CHELSIO_CHIP_CODE(CHELSIO_T4, pl_rev); 4686 case CHELSIO_T5: 4687 return CHELSIO_CHIP_CODE(CHELSIO_T5, pl_rev); 4688 case CHELSIO_T6: 4689 return CHELSIO_CHIP_CODE(CHELSIO_T6, pl_rev); 4690 default: 4691 dev_err(&pdev->dev, "Device %d is not supported\n", 4692 device_id); 4693 } 4694 return -EINVAL; 4695 } 4696 4697 #ifdef CONFIG_PCI_IOV 4698 static void dummy_setup(struct net_device *dev) 4699 { 4700 dev->type = ARPHRD_NONE; 4701 dev->mtu = 0; 4702 dev->hard_header_len = 0; 4703 dev->addr_len = 0; 4704 dev->tx_queue_len = 0; 4705 dev->flags |= IFF_NOARP; 4706 dev->priv_flags |= IFF_NO_QUEUE; 4707 4708 /* Initialize the device structure. */ 4709 dev->netdev_ops = &cxgb4_mgmt_netdev_ops; 4710 dev->ethtool_ops = &cxgb4_mgmt_ethtool_ops; 4711 dev->needs_free_netdev = true; 4712 } 4713 4714 static int config_mgmt_dev(struct pci_dev *pdev) 4715 { 4716 struct adapter *adap = pci_get_drvdata(pdev); 4717 struct net_device *netdev; 4718 struct port_info *pi; 4719 char name[IFNAMSIZ]; 4720 int err; 4721 4722 snprintf(name, IFNAMSIZ, "mgmtpf%d%d", adap->adap_idx, adap->pf); 4723 netdev = alloc_netdev(sizeof(struct port_info), name, NET_NAME_UNKNOWN, 4724 dummy_setup); 4725 if (!netdev) 4726 return -ENOMEM; 4727 4728 pi = netdev_priv(netdev); 4729 pi->adapter = adap; 4730 pi->tx_chan = adap->pf % adap->params.nports; 4731 SET_NETDEV_DEV(netdev, &pdev->dev); 4732 4733 adap->port[0] = netdev; 4734 pi->port_id = 0; 4735 4736 err = register_netdev(adap->port[0]); 4737 if (err) { 4738 pr_info("Unable to register VF mgmt netdev %s\n", name); 4739 free_netdev(adap->port[0]); 4740 adap->port[0] = NULL; 4741 return err; 4742 } 4743 return 0; 4744 } 4745 4746 static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) 4747 { 4748 struct adapter *adap = pci_get_drvdata(pdev); 4749 int err = 0; 4750 int current_vfs = pci_num_vf(pdev); 4751 u32 pcie_fw; 4752 4753 pcie_fw = readl(adap->regs + PCIE_FW_A); 4754 /* Check if cxgb4 is the MASTER and fw is initialized */ 4755 if (!(pcie_fw & PCIE_FW_INIT_F) || 4756 !(pcie_fw & PCIE_FW_MASTER_VLD_F) || 4757 PCIE_FW_MASTER_G(pcie_fw) != 4) { 4758 dev_warn(&pdev->dev, 4759 "cxgb4 driver needs to be MASTER to support SRIOV\n"); 4760 return -EOPNOTSUPP; 4761 } 4762 4763 /* If any of the VF's is already assigned to Guest OS, then 4764 * SRIOV for the same cannot be modified 4765 */ 4766 if (current_vfs && pci_vfs_assigned(pdev)) { 4767 dev_err(&pdev->dev, 4768 "Cannot modify SR-IOV while VFs are assigned\n"); 4769 num_vfs = current_vfs; 4770 return num_vfs; 4771 } 4772 4773 /* Disable SRIOV when zero is passed. 4774 * One needs to disable SRIOV before modifying it, else 4775 * stack throws the below warning: 4776 * " 'n' VFs already enabled. Disable before enabling 'm' VFs." 4777 */ 4778 if (!num_vfs) { 4779 pci_disable_sriov(pdev); 4780 if (adap->port[0]) { 4781 unregister_netdev(adap->port[0]); 4782 adap->port[0] = NULL; 4783 } 4784 /* free VF resources */ 4785 kfree(adap->vfinfo); 4786 adap->vfinfo = NULL; 4787 adap->num_vfs = 0; 4788 return num_vfs; 4789 } 4790 4791 if (num_vfs != current_vfs) { 4792 err = pci_enable_sriov(pdev, num_vfs); 4793 if (err) 4794 return err; 4795 4796 adap->num_vfs = num_vfs; 4797 err = config_mgmt_dev(pdev); 4798 if (err) 4799 return err; 4800 } 4801 4802 adap->vfinfo = kcalloc(adap->num_vfs, 4803 sizeof(struct vf_info), GFP_KERNEL); 4804 if (adap->vfinfo) 4805 fill_vf_station_mac_addr(adap); 4806 return num_vfs; 4807 } 4808 #endif 4809 4810 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 4811 { 4812 int func, i, err, s_qpp, qpp, num_seg; 4813 struct port_info *pi; 4814 bool highdma = false; 4815 struct adapter *adapter = NULL; 4816 struct net_device *netdev; 4817 void __iomem *regs; 4818 u32 whoami, pl_rev; 4819 enum chip_type chip; 4820 static int adap_idx = 1; 4821 #ifdef CONFIG_PCI_IOV 4822 u32 v, port_vec; 4823 #endif 4824 4825 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION); 4826 4827 err = pci_request_regions(pdev, KBUILD_MODNAME); 4828 if (err) { 4829 /* Just info, some other driver may have claimed the device. */ 4830 dev_info(&pdev->dev, "cannot obtain PCI resources\n"); 4831 return err; 4832 } 4833 4834 err = pci_enable_device(pdev); 4835 if (err) { 4836 dev_err(&pdev->dev, "cannot enable PCI device\n"); 4837 goto out_release_regions; 4838 } 4839 4840 regs = pci_ioremap_bar(pdev, 0); 4841 if (!regs) { 4842 dev_err(&pdev->dev, "cannot map device registers\n"); 4843 err = -ENOMEM; 4844 goto out_disable_device; 4845 } 4846 4847 err = t4_wait_dev_ready(regs); 4848 if (err < 0) 4849 goto out_unmap_bar0; 4850 4851 /* We control everything through one PF */ 4852 whoami = readl(regs + PL_WHOAMI_A); 4853 pl_rev = REV_G(readl(regs + PL_REV_A)); 4854 chip = get_chip_type(pdev, pl_rev); 4855 func = CHELSIO_CHIP_VERSION(chip) <= CHELSIO_T5 ? 4856 SOURCEPF_G(whoami) : T6_SOURCEPF_G(whoami); 4857 if (func != ent->driver_data) { 4858 #ifndef CONFIG_PCI_IOV 4859 iounmap(regs); 4860 #endif 4861 pci_disable_device(pdev); 4862 pci_save_state(pdev); /* to restore SR-IOV later */ 4863 goto sriov; 4864 } 4865 4866 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 4867 highdma = true; 4868 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 4869 if (err) { 4870 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for " 4871 "coherent allocations\n"); 4872 goto out_unmap_bar0; 4873 } 4874 } else { 4875 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 4876 if (err) { 4877 dev_err(&pdev->dev, "no usable DMA configuration\n"); 4878 goto out_unmap_bar0; 4879 } 4880 } 4881 4882 pci_enable_pcie_error_reporting(pdev); 4883 pci_set_master(pdev); 4884 pci_save_state(pdev); 4885 4886 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 4887 if (!adapter) { 4888 err = -ENOMEM; 4889 goto out_unmap_bar0; 4890 } 4891 adap_idx++; 4892 4893 adapter->workq = create_singlethread_workqueue("cxgb4"); 4894 if (!adapter->workq) { 4895 err = -ENOMEM; 4896 goto out_free_adapter; 4897 } 4898 4899 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + 4900 (sizeof(struct mbox_cmd) * 4901 T4_OS_LOG_MBOX_CMDS), 4902 GFP_KERNEL); 4903 if (!adapter->mbox_log) { 4904 err = -ENOMEM; 4905 goto out_free_adapter; 4906 } 4907 adapter->mbox_log->size = T4_OS_LOG_MBOX_CMDS; 4908 4909 /* PCI device has been enabled */ 4910 adapter->flags |= DEV_ENABLED; 4911 4912 adapter->regs = regs; 4913 adapter->pdev = pdev; 4914 adapter->pdev_dev = &pdev->dev; 4915 adapter->name = pci_name(pdev); 4916 adapter->mbox = func; 4917 adapter->pf = func; 4918 adapter->msg_enable = DFLT_MSG_ENABLE; 4919 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); 4920 4921 /* If possible, we use PCIe Relaxed Ordering Attribute to deliver 4922 * Ingress Packet Data to Free List Buffers in order to allow for 4923 * chipset performance optimizations between the Root Complex and 4924 * Memory Controllers. (Messages to the associated Ingress Queue 4925 * notifying new Packet Placement in the Free Lists Buffers will be 4926 * send without the Relaxed Ordering Attribute thus guaranteeing that 4927 * all preceding PCIe Transaction Layer Packets will be processed 4928 * first.) But some Root Complexes have various issues with Upstream 4929 * Transaction Layer Packets with the Relaxed Ordering Attribute set. 4930 * The PCIe devices which under the Root Complexes will be cleared the 4931 * Relaxed Ordering bit in the configuration space, So we check our 4932 * PCIe configuration space to see if it's flagged with advice against 4933 * using Relaxed Ordering. 4934 */ 4935 if (!pcie_relaxed_ordering_enabled(pdev)) 4936 adapter->flags |= ROOT_NO_RELAXED_ORDERING; 4937 4938 spin_lock_init(&adapter->stats_lock); 4939 spin_lock_init(&adapter->tid_release_lock); 4940 spin_lock_init(&adapter->win0_lock); 4941 spin_lock_init(&adapter->mbox_lock); 4942 4943 INIT_LIST_HEAD(&adapter->mlist.list); 4944 4945 INIT_WORK(&adapter->tid_release_task, process_tid_release_list); 4946 INIT_WORK(&adapter->db_full_task, process_db_full); 4947 INIT_WORK(&adapter->db_drop_task, process_db_drop); 4948 4949 err = t4_prep_adapter(adapter); 4950 if (err) 4951 goto out_free_adapter; 4952 4953 4954 if (!is_t4(adapter->params.chip)) { 4955 s_qpp = (QUEUESPERPAGEPF0_S + 4956 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * 4957 adapter->pf); 4958 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter, 4959 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp); 4960 num_seg = PAGE_SIZE / SEGMENT_SIZE; 4961 4962 /* Each segment size is 128B. Write coalescing is enabled only 4963 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the 4964 * queue is less no of segments that can be accommodated in 4965 * a page size. 4966 */ 4967 if (qpp > num_seg) { 4968 dev_err(&pdev->dev, 4969 "Incorrect number of egress queues per page\n"); 4970 err = -EINVAL; 4971 goto out_free_adapter; 4972 } 4973 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), 4974 pci_resource_len(pdev, 2)); 4975 if (!adapter->bar2) { 4976 dev_err(&pdev->dev, "cannot map device bar2 region\n"); 4977 err = -ENOMEM; 4978 goto out_free_adapter; 4979 } 4980 } 4981 4982 setup_memwin(adapter); 4983 err = adap_init0(adapter); 4984 #ifdef CONFIG_DEBUG_FS 4985 bitmap_zero(adapter->sge.blocked_fl, adapter->sge.egr_sz); 4986 #endif 4987 setup_memwin_rdma(adapter); 4988 if (err) 4989 goto out_unmap_bar; 4990 4991 /* configure SGE_STAT_CFG_A to read WC stats */ 4992 if (!is_t4(adapter->params.chip)) 4993 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7) | 4994 (is_t5(adapter->params.chip) ? STATMODE_V(0) : 4995 T6_STATMODE_V(0))); 4996 4997 for_each_port(adapter, i) { 4998 netdev = alloc_etherdev_mq(sizeof(struct port_info), 4999 MAX_ETH_QSETS); 5000 if (!netdev) { 5001 err = -ENOMEM; 5002 goto out_free_dev; 5003 } 5004 5005 SET_NETDEV_DEV(netdev, &pdev->dev); 5006 5007 adapter->port[i] = netdev; 5008 pi = netdev_priv(netdev); 5009 pi->adapter = adapter; 5010 pi->xact_addr_filt = -1; 5011 pi->port_id = i; 5012 netdev->irq = pdev->irq; 5013 5014 netdev->hw_features = NETIF_F_SG | TSO_FLAGS | 5015 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 5016 NETIF_F_RXCSUM | NETIF_F_RXHASH | 5017 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 5018 NETIF_F_HW_TC; 5019 if (highdma) 5020 netdev->hw_features |= NETIF_F_HIGHDMA; 5021 netdev->features |= netdev->hw_features; 5022 netdev->vlan_features = netdev->features & VLAN_FEAT; 5023 5024 netdev->priv_flags |= IFF_UNICAST_FLT; 5025 5026 /* MTU range: 81 - 9600 */ 5027 netdev->min_mtu = 81; /* accommodate SACK */ 5028 netdev->max_mtu = MAX_MTU; 5029 5030 netdev->netdev_ops = &cxgb4_netdev_ops; 5031 #ifdef CONFIG_CHELSIO_T4_DCB 5032 netdev->dcbnl_ops = &cxgb4_dcb_ops; 5033 cxgb4_dcb_state_init(netdev); 5034 #endif 5035 cxgb4_set_ethtool_ops(netdev); 5036 } 5037 5038 pci_set_drvdata(pdev, adapter); 5039 5040 if (adapter->flags & FW_OK) { 5041 err = t4_port_init(adapter, func, func, 0); 5042 if (err) 5043 goto out_free_dev; 5044 } else if (adapter->params.nports == 1) { 5045 /* If we don't have a connection to the firmware -- possibly 5046 * because of an error -- grab the raw VPD parameters so we 5047 * can set the proper MAC Address on the debug network 5048 * interface that we've created. 5049 */ 5050 u8 hw_addr[ETH_ALEN]; 5051 u8 *na = adapter->params.vpd.na; 5052 5053 err = t4_get_raw_vpd_params(adapter, &adapter->params.vpd); 5054 if (!err) { 5055 for (i = 0; i < ETH_ALEN; i++) 5056 hw_addr[i] = (hex2val(na[2 * i + 0]) * 16 + 5057 hex2val(na[2 * i + 1])); 5058 t4_set_hw_addr(adapter, 0, hw_addr); 5059 } 5060 } 5061 5062 /* Configure queues and allocate tables now, they can be needed as 5063 * soon as the first register_netdev completes. 5064 */ 5065 cfg_queues(adapter); 5066 5067 adapter->l2t = t4_init_l2t(adapter->l2t_start, adapter->l2t_end); 5068 if (!adapter->l2t) { 5069 /* We tolerate a lack of L2T, giving up some functionality */ 5070 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n"); 5071 adapter->params.offload = 0; 5072 } 5073 5074 #if IS_ENABLED(CONFIG_IPV6) 5075 if ((CHELSIO_CHIP_VERSION(adapter->params.chip) <= CHELSIO_T5) && 5076 (!(t4_read_reg(adapter, LE_DB_CONFIG_A) & ASLIPCOMPEN_F))) { 5077 /* CLIP functionality is not present in hardware, 5078 * hence disable all offload features 5079 */ 5080 dev_warn(&pdev->dev, 5081 "CLIP not enabled in hardware, continuing\n"); 5082 adapter->params.offload = 0; 5083 } else { 5084 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start, 5085 adapter->clipt_end); 5086 if (!adapter->clipt) { 5087 /* We tolerate a lack of clip_table, giving up 5088 * some functionality 5089 */ 5090 dev_warn(&pdev->dev, 5091 "could not allocate Clip table, continuing\n"); 5092 adapter->params.offload = 0; 5093 } 5094 } 5095 #endif 5096 5097 for_each_port(adapter, i) { 5098 pi = adap2pinfo(adapter, i); 5099 pi->sched_tbl = t4_init_sched(adapter->params.nsched_cls); 5100 if (!pi->sched_tbl) 5101 dev_warn(&pdev->dev, 5102 "could not activate scheduling on port %d\n", 5103 i); 5104 } 5105 5106 if (tid_init(&adapter->tids) < 0) { 5107 dev_warn(&pdev->dev, "could not allocate TID table, " 5108 "continuing\n"); 5109 adapter->params.offload = 0; 5110 } else { 5111 adapter->tc_u32 = cxgb4_init_tc_u32(adapter); 5112 if (!adapter->tc_u32) 5113 dev_warn(&pdev->dev, 5114 "could not offload tc u32, continuing\n"); 5115 5116 cxgb4_init_tc_flower(adapter); 5117 } 5118 5119 if (is_offload(adapter)) { 5120 if (t4_read_reg(adapter, LE_DB_CONFIG_A) & HASHEN_F) { 5121 u32 hash_base, hash_reg; 5122 5123 if (chip <= CHELSIO_T5) { 5124 hash_reg = LE_DB_TID_HASHBASE_A; 5125 hash_base = t4_read_reg(adapter, hash_reg); 5126 adapter->tids.hash_base = hash_base / 4; 5127 } else { 5128 hash_reg = T6_LE_DB_HASH_TID_BASE_A; 5129 hash_base = t4_read_reg(adapter, hash_reg); 5130 adapter->tids.hash_base = hash_base; 5131 } 5132 } 5133 } 5134 5135 /* See what interrupts we'll be using */ 5136 if (msi > 1 && enable_msix(adapter) == 0) 5137 adapter->flags |= USING_MSIX; 5138 else if (msi > 0 && pci_enable_msi(pdev) == 0) { 5139 adapter->flags |= USING_MSI; 5140 if (msi > 1) 5141 free_msix_info(adapter); 5142 } 5143 5144 /* check for PCI Express bandwidth capabiltites */ 5145 cxgb4_check_pcie_caps(adapter); 5146 5147 err = init_rss(adapter); 5148 if (err) 5149 goto out_free_dev; 5150 5151 /* 5152 * The card is now ready to go. If any errors occur during device 5153 * registration we do not fail the whole card but rather proceed only 5154 * with the ports we manage to register successfully. However we must 5155 * register at least one net device. 5156 */ 5157 for_each_port(adapter, i) { 5158 pi = adap2pinfo(adapter, i); 5159 adapter->port[i]->dev_port = pi->lport; 5160 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets); 5161 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets); 5162 5163 netif_carrier_off(adapter->port[i]); 5164 5165 err = register_netdev(adapter->port[i]); 5166 if (err) 5167 break; 5168 adapter->chan_map[pi->tx_chan] = i; 5169 print_port_info(adapter->port[i]); 5170 } 5171 if (i == 0) { 5172 dev_err(&pdev->dev, "could not register any net devices\n"); 5173 goto out_free_dev; 5174 } 5175 if (err) { 5176 dev_warn(&pdev->dev, "only %d net devices registered\n", i); 5177 err = 0; 5178 } 5179 5180 if (cxgb4_debugfs_root) { 5181 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev), 5182 cxgb4_debugfs_root); 5183 setup_debugfs(adapter); 5184 } 5185 5186 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */ 5187 pdev->needs_freset = 1; 5188 5189 if (is_uld(adapter)) { 5190 mutex_lock(&uld_mutex); 5191 list_add_tail(&adapter->list_node, &adapter_list); 5192 mutex_unlock(&uld_mutex); 5193 } 5194 5195 if (!is_t4(adapter->params.chip)) 5196 cxgb4_ptp_init(adapter); 5197 5198 print_adapter_info(adapter); 5199 setup_fw_sge_queues(adapter); 5200 return 0; 5201 5202 sriov: 5203 #ifdef CONFIG_PCI_IOV 5204 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 5205 if (!adapter) { 5206 err = -ENOMEM; 5207 goto free_pci_region; 5208 } 5209 5210 adapter->pdev = pdev; 5211 adapter->pdev_dev = &pdev->dev; 5212 adapter->name = pci_name(pdev); 5213 adapter->mbox = func; 5214 adapter->pf = func; 5215 adapter->regs = regs; 5216 adapter->adap_idx = adap_idx; 5217 adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) + 5218 (sizeof(struct mbox_cmd) * 5219 T4_OS_LOG_MBOX_CMDS), 5220 GFP_KERNEL); 5221 if (!adapter->mbox_log) { 5222 err = -ENOMEM; 5223 goto free_adapter; 5224 } 5225 spin_lock_init(&adapter->mbox_lock); 5226 INIT_LIST_HEAD(&adapter->mlist.list); 5227 5228 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 5229 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 5230 err = t4_query_params(adapter, adapter->mbox, adapter->pf, 0, 1, 5231 &v, &port_vec); 5232 if (err < 0) { 5233 dev_err(adapter->pdev_dev, "Could not fetch port params\n"); 5234 goto free_mbox_log; 5235 } 5236 5237 adapter->params.nports = hweight32(port_vec); 5238 pci_set_drvdata(pdev, adapter); 5239 return 0; 5240 5241 free_mbox_log: 5242 kfree(adapter->mbox_log); 5243 free_adapter: 5244 kfree(adapter); 5245 free_pci_region: 5246 iounmap(regs); 5247 pci_disable_sriov(pdev); 5248 pci_release_regions(pdev); 5249 return err; 5250 #else 5251 return 0; 5252 #endif 5253 5254 out_free_dev: 5255 free_some_resources(adapter); 5256 if (adapter->flags & USING_MSIX) 5257 free_msix_info(adapter); 5258 if (adapter->num_uld || adapter->num_ofld_uld) 5259 t4_uld_mem_free(adapter); 5260 out_unmap_bar: 5261 if (!is_t4(adapter->params.chip)) 5262 iounmap(adapter->bar2); 5263 out_free_adapter: 5264 if (adapter->workq) 5265 destroy_workqueue(adapter->workq); 5266 5267 kfree(adapter->mbox_log); 5268 kfree(adapter); 5269 out_unmap_bar0: 5270 iounmap(regs); 5271 out_disable_device: 5272 pci_disable_pcie_error_reporting(pdev); 5273 pci_disable_device(pdev); 5274 out_release_regions: 5275 pci_release_regions(pdev); 5276 return err; 5277 } 5278 5279 static void remove_one(struct pci_dev *pdev) 5280 { 5281 struct adapter *adapter = pci_get_drvdata(pdev); 5282 5283 if (!adapter) { 5284 pci_release_regions(pdev); 5285 return; 5286 } 5287 5288 adapter->flags |= SHUTTING_DOWN; 5289 5290 if (adapter->pf == 4) { 5291 int i; 5292 5293 /* Tear down per-adapter Work Queue first since it can contain 5294 * references to our adapter data structure. 5295 */ 5296 destroy_workqueue(adapter->workq); 5297 5298 if (is_uld(adapter)) { 5299 detach_ulds(adapter); 5300 t4_uld_clean_up(adapter); 5301 } 5302 5303 disable_interrupts(adapter); 5304 5305 for_each_port(adapter, i) 5306 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5307 unregister_netdev(adapter->port[i]); 5308 5309 debugfs_remove_recursive(adapter->debugfs_root); 5310 5311 if (!is_t4(adapter->params.chip)) 5312 cxgb4_ptp_stop(adapter); 5313 5314 /* If we allocated filters, free up state associated with any 5315 * valid filters ... 5316 */ 5317 clear_all_filters(adapter); 5318 5319 if (adapter->flags & FULL_INIT_DONE) 5320 cxgb_down(adapter); 5321 5322 if (adapter->flags & USING_MSIX) 5323 free_msix_info(adapter); 5324 if (adapter->num_uld || adapter->num_ofld_uld) 5325 t4_uld_mem_free(adapter); 5326 free_some_resources(adapter); 5327 #if IS_ENABLED(CONFIG_IPV6) 5328 t4_cleanup_clip_tbl(adapter); 5329 #endif 5330 iounmap(adapter->regs); 5331 if (!is_t4(adapter->params.chip)) 5332 iounmap(adapter->bar2); 5333 pci_disable_pcie_error_reporting(pdev); 5334 if ((adapter->flags & DEV_ENABLED)) { 5335 pci_disable_device(pdev); 5336 adapter->flags &= ~DEV_ENABLED; 5337 } 5338 pci_release_regions(pdev); 5339 kfree(adapter->mbox_log); 5340 synchronize_rcu(); 5341 kfree(adapter); 5342 } 5343 #ifdef CONFIG_PCI_IOV 5344 else { 5345 if (adapter->port[0]) 5346 unregister_netdev(adapter->port[0]); 5347 iounmap(adapter->regs); 5348 kfree(adapter->vfinfo); 5349 kfree(adapter->mbox_log); 5350 kfree(adapter); 5351 pci_disable_sriov(pdev); 5352 pci_release_regions(pdev); 5353 } 5354 #endif 5355 } 5356 5357 /* "Shutdown" quiesces the device, stopping Ingress Packet and Interrupt 5358 * delivery. This is essentially a stripped down version of the PCI remove() 5359 * function where we do the minimal amount of work necessary to shutdown any 5360 * further activity. 5361 */ 5362 static void shutdown_one(struct pci_dev *pdev) 5363 { 5364 struct adapter *adapter = pci_get_drvdata(pdev); 5365 5366 /* As with remove_one() above (see extended comment), we only want do 5367 * do cleanup on PCI Devices which went all the way through init_one() 5368 * ... 5369 */ 5370 if (!adapter) { 5371 pci_release_regions(pdev); 5372 return; 5373 } 5374 5375 adapter->flags |= SHUTTING_DOWN; 5376 5377 if (adapter->pf == 4) { 5378 int i; 5379 5380 for_each_port(adapter, i) 5381 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 5382 cxgb_close(adapter->port[i]); 5383 5384 if (is_uld(adapter)) { 5385 detach_ulds(adapter); 5386 t4_uld_clean_up(adapter); 5387 } 5388 5389 disable_interrupts(adapter); 5390 disable_msi(adapter); 5391 5392 t4_sge_stop(adapter); 5393 if (adapter->flags & FW_OK) 5394 t4_fw_bye(adapter, adapter->mbox); 5395 } 5396 #ifdef CONFIG_PCI_IOV 5397 else { 5398 if (adapter->port[0]) 5399 unregister_netdev(adapter->port[0]); 5400 iounmap(adapter->regs); 5401 kfree(adapter->vfinfo); 5402 kfree(adapter->mbox_log); 5403 kfree(adapter); 5404 pci_disable_sriov(pdev); 5405 pci_release_regions(pdev); 5406 } 5407 #endif 5408 } 5409 5410 static struct pci_driver cxgb4_driver = { 5411 .name = KBUILD_MODNAME, 5412 .id_table = cxgb4_pci_tbl, 5413 .probe = init_one, 5414 .remove = remove_one, 5415 .shutdown = shutdown_one, 5416 #ifdef CONFIG_PCI_IOV 5417 .sriov_configure = cxgb4_iov_configure, 5418 #endif 5419 .err_handler = &cxgb4_eeh, 5420 }; 5421 5422 static int __init cxgb4_init_module(void) 5423 { 5424 int ret; 5425 5426 /* Debugfs support is optional, just warn if this fails */ 5427 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); 5428 if (!cxgb4_debugfs_root) 5429 pr_warn("could not create debugfs entry, continuing\n"); 5430 5431 ret = pci_register_driver(&cxgb4_driver); 5432 if (ret < 0) 5433 debugfs_remove(cxgb4_debugfs_root); 5434 5435 #if IS_ENABLED(CONFIG_IPV6) 5436 if (!inet6addr_registered) { 5437 register_inet6addr_notifier(&cxgb4_inet6addr_notifier); 5438 inet6addr_registered = true; 5439 } 5440 #endif 5441 5442 return ret; 5443 } 5444 5445 static void __exit cxgb4_cleanup_module(void) 5446 { 5447 #if IS_ENABLED(CONFIG_IPV6) 5448 if (inet6addr_registered) { 5449 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier); 5450 inet6addr_registered = false; 5451 } 5452 #endif 5453 pci_unregister_driver(&cxgb4_driver); 5454 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ 5455 } 5456 5457 module_init(cxgb4_init_module); 5458 module_exit(cxgb4_cleanup_module); 5459