1 /* 2 * This file is part of the Chelsio T4 Ethernet driver for Linux. 3 * 4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/bitmap.h> 38 #include <linux/crc32.h> 39 #include <linux/ctype.h> 40 #include <linux/debugfs.h> 41 #include <linux/err.h> 42 #include <linux/etherdevice.h> 43 #include <linux/firmware.h> 44 #include <linux/if.h> 45 #include <linux/if_vlan.h> 46 #include <linux/init.h> 47 #include <linux/log2.h> 48 #include <linux/mdio.h> 49 #include <linux/module.h> 50 #include <linux/moduleparam.h> 51 #include <linux/mutex.h> 52 #include <linux/netdevice.h> 53 #include <linux/pci.h> 54 #include <linux/aer.h> 55 #include <linux/rtnetlink.h> 56 #include <linux/sched.h> 57 #include <linux/seq_file.h> 58 #include <linux/sockios.h> 59 #include <linux/vmalloc.h> 60 #include <linux/workqueue.h> 61 #include <net/neighbour.h> 62 #include <net/netevent.h> 63 #include <net/addrconf.h> 64 #include <net/bonding.h> 65 #include <net/addrconf.h> 66 #include <asm/uaccess.h> 67 68 #include "cxgb4.h" 69 #include "t4_regs.h" 70 #include "t4_values.h" 71 #include "t4_msg.h" 72 #include "t4fw_api.h" 73 #include "t4fw_version.h" 74 #include "cxgb4_dcb.h" 75 #include "cxgb4_debugfs.h" 76 #include "clip_tbl.h" 77 #include "l2t.h" 78 79 #ifdef DRV_VERSION 80 #undef DRV_VERSION 81 #endif 82 #define DRV_VERSION "2.0.0-ko" 83 #define DRV_DESC "Chelsio T4/T5 Network Driver" 84 85 enum { 86 MAX_TXQ_ENTRIES = 16384, 87 MAX_CTRL_TXQ_ENTRIES = 1024, 88 MAX_RSPQ_ENTRIES = 16384, 89 MAX_RX_BUFFERS = 16384, 90 MIN_TXQ_ENTRIES = 32, 91 MIN_CTRL_TXQ_ENTRIES = 32, 92 MIN_RSPQ_ENTRIES = 128, 93 MIN_FL_ENTRIES = 16 94 }; 95 96 /* Host shadow copy of ingress filter entry. This is in host native format 97 * and doesn't match the ordering or bit order, etc. of the hardware of the 98 * firmware command. The use of bit-field structure elements is purely to 99 * remind ourselves of the field size limitations and save memory in the case 100 * where the filter table is large. 101 */ 102 struct filter_entry { 103 /* Administrative fields for filter. 104 */ 105 u32 valid:1; /* filter allocated and valid */ 106 u32 locked:1; /* filter is administratively locked */ 107 108 u32 pending:1; /* filter action is pending firmware reply */ 109 u32 smtidx:8; /* Source MAC Table index for smac */ 110 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 111 112 /* The filter itself. Most of this is a straight copy of information 113 * provided by the extended ioctl(). Some fields are translated to 114 * internal forms -- for instance the Ingress Queue ID passed in from 115 * the ioctl() is translated into the Absolute Ingress Queue ID. 116 */ 117 struct ch_filter_specification fs; 118 }; 119 120 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ 121 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ 122 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) 123 124 /* Macros needed to support the PCI Device ID Table ... 125 */ 126 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ 127 static struct pci_device_id cxgb4_pci_tbl[] = { 128 #define CH_PCI_DEVICE_ID_FUNCTION 0x4 129 130 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is 131 * called for both. 132 */ 133 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0 134 135 #define CH_PCI_ID_TABLE_ENTRY(devid) \ 136 {PCI_VDEVICE(CHELSIO, (devid)), 4} 137 138 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ 139 { 0, } \ 140 } 141 142 #include "t4_pci_id_tbl.h" 143 144 #define FW4_FNAME "cxgb4/t4fw.bin" 145 #define FW5_FNAME "cxgb4/t5fw.bin" 146 #define FW4_CFNAME "cxgb4/t4-config.txt" 147 #define FW5_CFNAME "cxgb4/t5-config.txt" 148 149 MODULE_DESCRIPTION(DRV_DESC); 150 MODULE_AUTHOR("Chelsio Communications"); 151 MODULE_LICENSE("Dual BSD/GPL"); 152 MODULE_VERSION(DRV_VERSION); 153 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl); 154 MODULE_FIRMWARE(FW4_FNAME); 155 MODULE_FIRMWARE(FW5_FNAME); 156 157 /* 158 * Normally we're willing to become the firmware's Master PF but will be happy 159 * if another PF has already become the Master and initialized the adapter. 160 * Setting "force_init" will cause this driver to forcibly establish itself as 161 * the Master PF and initialize the adapter. 162 */ 163 static uint force_init; 164 165 module_param(force_init, uint, 0644); 166 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter"); 167 168 /* 169 * Normally if the firmware we connect to has Configuration File support, we 170 * use that and only fall back to the old Driver-based initialization if the 171 * Configuration File fails for some reason. If force_old_init is set, then 172 * we'll always use the old Driver-based initialization sequence. 173 */ 174 static uint force_old_init; 175 176 module_param(force_old_init, uint, 0644); 177 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence, deprecated" 178 " parameter"); 179 180 static int dflt_msg_enable = DFLT_MSG_ENABLE; 181 182 module_param(dflt_msg_enable, int, 0644); 183 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap"); 184 185 /* 186 * The driver uses the best interrupt scheme available on a platform in the 187 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which 188 * of these schemes the driver may consider as follows: 189 * 190 * msi = 2: choose from among all three options 191 * msi = 1: only consider MSI and INTx interrupts 192 * msi = 0: force INTx interrupts 193 */ 194 static int msi = 2; 195 196 module_param(msi, int, 0644); 197 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)"); 198 199 /* 200 * Queue interrupt hold-off timer values. Queues default to the first of these 201 * upon creation. 202 */ 203 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 }; 204 205 module_param_array(intr_holdoff, uint, NULL, 0644); 206 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers " 207 "0..4 in microseconds, deprecated parameter"); 208 209 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 }; 210 211 module_param_array(intr_cnt, uint, NULL, 0644); 212 MODULE_PARM_DESC(intr_cnt, 213 "thresholds 1..3 for queue interrupt packet counters, " 214 "deprecated parameter"); 215 216 /* 217 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers 218 * offset by 2 bytes in order to have the IP headers line up on 4-byte 219 * boundaries. This is a requirement for many architectures which will throw 220 * a machine check fault if an attempt is made to access one of the 4-byte IP 221 * header fields on a non-4-byte boundary. And it's a major performance issue 222 * even on some architectures which allow it like some implementations of the 223 * x86 ISA. However, some architectures don't mind this and for some very 224 * edge-case performance sensitive applications (like forwarding large volumes 225 * of small packets), setting this DMA offset to 0 will decrease the number of 226 * PCI-E Bus transfers enough to measurably affect performance. 227 */ 228 static int rx_dma_offset = 2; 229 230 static bool vf_acls; 231 232 #ifdef CONFIG_PCI_IOV 233 module_param(vf_acls, bool, 0644); 234 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement, " 235 "deprecated parameter"); 236 237 /* Configure the number of PCI-E Virtual Function which are to be instantiated 238 * on SR-IOV Capable Physical Functions. 239 */ 240 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV]; 241 242 module_param_array(num_vf, uint, NULL, 0644); 243 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3"); 244 #endif 245 246 /* TX Queue select used to determine what algorithm to use for selecting TX 247 * queue. Select between the kernel provided function (select_queue=0) or user 248 * cxgb_select_queue function (select_queue=1) 249 * 250 * Default: select_queue=0 251 */ 252 static int select_queue; 253 module_param(select_queue, int, 0644); 254 MODULE_PARM_DESC(select_queue, 255 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method."); 256 257 static unsigned int tp_vlan_pri_map = HW_TPL_FR_MT_PR_IV_P_FC; 258 259 module_param(tp_vlan_pri_map, uint, 0644); 260 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration, " 261 "deprecated parameter"); 262 263 static struct dentry *cxgb4_debugfs_root; 264 265 static LIST_HEAD(adapter_list); 266 static DEFINE_MUTEX(uld_mutex); 267 /* Adapter list to be accessed from atomic context */ 268 static LIST_HEAD(adap_rcu_list); 269 static DEFINE_SPINLOCK(adap_rcu_lock); 270 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX]; 271 static const char *uld_str[] = { "RDMA", "iSCSI" }; 272 273 static void link_report(struct net_device *dev) 274 { 275 if (!netif_carrier_ok(dev)) 276 netdev_info(dev, "link down\n"); 277 else { 278 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" }; 279 280 const char *s = "10Mbps"; 281 const struct port_info *p = netdev_priv(dev); 282 283 switch (p->link_cfg.speed) { 284 case 10000: 285 s = "10Gbps"; 286 break; 287 case 1000: 288 s = "1000Mbps"; 289 break; 290 case 100: 291 s = "100Mbps"; 292 break; 293 case 40000: 294 s = "40Gbps"; 295 break; 296 } 297 298 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, 299 fc[p->link_cfg.fc]); 300 } 301 } 302 303 #ifdef CONFIG_CHELSIO_T4_DCB 304 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */ 305 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable) 306 { 307 struct port_info *pi = netdev_priv(dev); 308 struct adapter *adap = pi->adapter; 309 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; 310 int i; 311 312 /* We use a simple mapping of Port TX Queue Index to DCB 313 * Priority when we're enabling DCB. 314 */ 315 for (i = 0; i < pi->nqsets; i++, txq++) { 316 u32 name, value; 317 int err; 318 319 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 320 FW_PARAMS_PARAM_X_V( 321 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) | 322 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); 323 value = enable ? i : 0xffffffff; 324 325 /* Since we can be called while atomic (from "interrupt 326 * level") we need to issue the Set Parameters Commannd 327 * without sleeping (timeout < 0). 328 */ 329 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1, 330 &name, &value); 331 332 if (err) 333 dev_err(adap->pdev_dev, 334 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n", 335 enable ? "set" : "unset", pi->port_id, i, -err); 336 else 337 txq->dcb_prio = value; 338 } 339 } 340 #endif /* CONFIG_CHELSIO_T4_DCB */ 341 342 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat) 343 { 344 struct net_device *dev = adapter->port[port_id]; 345 346 /* Skip changes from disabled ports. */ 347 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) { 348 if (link_stat) 349 netif_carrier_on(dev); 350 else { 351 #ifdef CONFIG_CHELSIO_T4_DCB 352 cxgb4_dcb_state_init(dev); 353 dcb_tx_queue_prio_enable(dev, false); 354 #endif /* CONFIG_CHELSIO_T4_DCB */ 355 netif_carrier_off(dev); 356 } 357 358 link_report(dev); 359 } 360 } 361 362 void t4_os_portmod_changed(const struct adapter *adap, int port_id) 363 { 364 static const char *mod_str[] = { 365 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" 366 }; 367 368 const struct net_device *dev = adap->port[port_id]; 369 const struct port_info *pi = netdev_priv(dev); 370 371 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 372 netdev_info(dev, "port module unplugged\n"); 373 else if (pi->mod_type < ARRAY_SIZE(mod_str)) 374 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]); 375 } 376 377 /* 378 * Configure the exact and hash address filters to handle a port's multicast 379 * and secondary unicast MAC addresses. 380 */ 381 static int set_addr_filters(const struct net_device *dev, bool sleep) 382 { 383 u64 mhash = 0; 384 u64 uhash = 0; 385 bool free = true; 386 u16 filt_idx[7]; 387 const u8 *addr[7]; 388 int ret, naddr = 0; 389 const struct netdev_hw_addr *ha; 390 int uc_cnt = netdev_uc_count(dev); 391 int mc_cnt = netdev_mc_count(dev); 392 const struct port_info *pi = netdev_priv(dev); 393 unsigned int mb = pi->adapter->fn; 394 395 /* first do the secondary unicast addresses */ 396 netdev_for_each_uc_addr(ha, dev) { 397 addr[naddr++] = ha->addr; 398 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) { 399 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free, 400 naddr, addr, filt_idx, &uhash, sleep); 401 if (ret < 0) 402 return ret; 403 404 free = false; 405 naddr = 0; 406 } 407 } 408 409 /* next set up the multicast addresses */ 410 netdev_for_each_mc_addr(ha, dev) { 411 addr[naddr++] = ha->addr; 412 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) { 413 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free, 414 naddr, addr, filt_idx, &mhash, sleep); 415 if (ret < 0) 416 return ret; 417 418 free = false; 419 naddr = 0; 420 } 421 } 422 423 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0, 424 uhash | mhash, sleep); 425 } 426 427 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ 428 module_param(dbfifo_int_thresh, int, 0644); 429 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); 430 431 /* 432 * usecs to sleep while draining the dbfifo 433 */ 434 static int dbfifo_drain_delay = 1000; 435 module_param(dbfifo_drain_delay, int, 0644); 436 MODULE_PARM_DESC(dbfifo_drain_delay, 437 "usecs to sleep while draining the dbfifo"); 438 439 /* 440 * Set Rx properties of a port, such as promiscruity, address filters, and MTU. 441 * If @mtu is -1 it is left unchanged. 442 */ 443 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) 444 { 445 int ret; 446 struct port_info *pi = netdev_priv(dev); 447 448 ret = set_addr_filters(dev, sleep_ok); 449 if (ret == 0) 450 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu, 451 (dev->flags & IFF_PROMISC) ? 1 : 0, 452 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1, 453 sleep_ok); 454 return ret; 455 } 456 457 /** 458 * link_start - enable a port 459 * @dev: the port to enable 460 * 461 * Performs the MAC and PHY actions needed to enable a port. 462 */ 463 static int link_start(struct net_device *dev) 464 { 465 int ret; 466 struct port_info *pi = netdev_priv(dev); 467 unsigned int mb = pi->adapter->fn; 468 469 /* 470 * We do not set address filters and promiscuity here, the stack does 471 * that step explicitly. 472 */ 473 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1, 474 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); 475 if (ret == 0) { 476 ret = t4_change_mac(pi->adapter, mb, pi->viid, 477 pi->xact_addr_filt, dev->dev_addr, true, 478 true); 479 if (ret >= 0) { 480 pi->xact_addr_filt = ret; 481 ret = 0; 482 } 483 } 484 if (ret == 0) 485 ret = t4_link_start(pi->adapter, mb, pi->tx_chan, 486 &pi->link_cfg); 487 if (ret == 0) { 488 local_bh_disable(); 489 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true, 490 true, CXGB4_DCB_ENABLED); 491 local_bh_enable(); 492 } 493 494 return ret; 495 } 496 497 int cxgb4_dcb_enabled(const struct net_device *dev) 498 { 499 #ifdef CONFIG_CHELSIO_T4_DCB 500 struct port_info *pi = netdev_priv(dev); 501 502 if (!pi->dcb.enabled) 503 return 0; 504 505 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) || 506 (pi->dcb.state == CXGB4_DCB_STATE_HOST)); 507 #else 508 return 0; 509 #endif 510 } 511 EXPORT_SYMBOL(cxgb4_dcb_enabled); 512 513 #ifdef CONFIG_CHELSIO_T4_DCB 514 /* Handle a Data Center Bridging update message from the firmware. */ 515 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd) 516 { 517 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid)); 518 struct net_device *dev = adap->port[port]; 519 int old_dcb_enabled = cxgb4_dcb_enabled(dev); 520 int new_dcb_enabled; 521 522 cxgb4_dcb_handle_fw_update(adap, pcmd); 523 new_dcb_enabled = cxgb4_dcb_enabled(dev); 524 525 /* If the DCB has become enabled or disabled on the port then we're 526 * going to need to set up/tear down DCB Priority parameters for the 527 * TX Queues associated with the port. 528 */ 529 if (new_dcb_enabled != old_dcb_enabled) 530 dcb_tx_queue_prio_enable(dev, new_dcb_enabled); 531 } 532 #endif /* CONFIG_CHELSIO_T4_DCB */ 533 534 /* Clear a filter and release any of its resources that we own. This also 535 * clears the filter's "pending" status. 536 */ 537 static void clear_filter(struct adapter *adap, struct filter_entry *f) 538 { 539 /* If the new or old filter have loopback rewriteing rules then we'll 540 * need to free any existing Layer Two Table (L2T) entries of the old 541 * filter rule. The firmware will handle freeing up any Source MAC 542 * Table (SMT) entries used for rewriting Source MAC Addresses in 543 * loopback rules. 544 */ 545 if (f->l2t) 546 cxgb4_l2t_release(f->l2t); 547 548 /* The zeroing of the filter rule below clears the filter valid, 549 * pending, locked flags, l2t pointer, etc. so it's all we need for 550 * this operation. 551 */ 552 memset(f, 0, sizeof(*f)); 553 } 554 555 /* Handle a filter write/deletion reply. 556 */ 557 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl) 558 { 559 unsigned int idx = GET_TID(rpl); 560 unsigned int nidx = idx - adap->tids.ftid_base; 561 unsigned int ret; 562 struct filter_entry *f; 563 564 if (idx >= adap->tids.ftid_base && nidx < 565 (adap->tids.nftids + adap->tids.nsftids)) { 566 idx = nidx; 567 ret = TCB_COOKIE_G(rpl->cookie); 568 f = &adap->tids.ftid_tab[idx]; 569 570 if (ret == FW_FILTER_WR_FLT_DELETED) { 571 /* Clear the filter when we get confirmation from the 572 * hardware that the filter has been deleted. 573 */ 574 clear_filter(adap, f); 575 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) { 576 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n", 577 idx); 578 clear_filter(adap, f); 579 } else if (ret == FW_FILTER_WR_FLT_ADDED) { 580 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff; 581 f->pending = 0; /* asynchronous setup completed */ 582 f->valid = 1; 583 } else { 584 /* Something went wrong. Issue a warning about the 585 * problem and clear everything out. 586 */ 587 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n", 588 idx, ret); 589 clear_filter(adap, f); 590 } 591 } 592 } 593 594 /* Response queue handler for the FW event queue. 595 */ 596 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, 597 const struct pkt_gl *gl) 598 { 599 u8 opcode = ((const struct rss_header *)rsp)->opcode; 600 601 rsp++; /* skip RSS header */ 602 603 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. 604 */ 605 if (unlikely(opcode == CPL_FW4_MSG && 606 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) { 607 rsp++; 608 opcode = ((const struct rss_header *)rsp)->opcode; 609 rsp++; 610 if (opcode != CPL_SGE_EGR_UPDATE) { 611 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" 612 , opcode); 613 goto out; 614 } 615 } 616 617 if (likely(opcode == CPL_SGE_EGR_UPDATE)) { 618 const struct cpl_sge_egr_update *p = (void *)rsp; 619 unsigned int qid = EGR_QID_G(ntohl(p->opcode_qid)); 620 struct sge_txq *txq; 621 622 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start]; 623 txq->restarts++; 624 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) { 625 struct sge_eth_txq *eq; 626 627 eq = container_of(txq, struct sge_eth_txq, q); 628 netif_tx_wake_queue(eq->txq); 629 } else { 630 struct sge_ofld_txq *oq; 631 632 oq = container_of(txq, struct sge_ofld_txq, q); 633 tasklet_schedule(&oq->qresume_tsk); 634 } 635 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { 636 const struct cpl_fw6_msg *p = (void *)rsp; 637 638 #ifdef CONFIG_CHELSIO_T4_DCB 639 const struct fw_port_cmd *pcmd = (const void *)p->data; 640 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid)); 641 unsigned int action = 642 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16)); 643 644 if (cmd == FW_PORT_CMD && 645 action == FW_PORT_ACTION_GET_PORT_INFO) { 646 int port = FW_PORT_CMD_PORTID_G( 647 be32_to_cpu(pcmd->op_to_portid)); 648 struct net_device *dev = q->adap->port[port]; 649 int state_input = ((pcmd->u.info.dcbxdis_pkd & 650 FW_PORT_CMD_DCBXDIS_F) 651 ? CXGB4_DCB_INPUT_FW_DISABLED 652 : CXGB4_DCB_INPUT_FW_ENABLED); 653 654 cxgb4_dcb_state_fsm(dev, state_input); 655 } 656 657 if (cmd == FW_PORT_CMD && 658 action == FW_PORT_ACTION_L2_DCB_CFG) 659 dcb_rpl(q->adap, pcmd); 660 else 661 #endif 662 if (p->type == 0) 663 t4_handle_fw_rpl(q->adap, p->data); 664 } else if (opcode == CPL_L2T_WRITE_RPL) { 665 const struct cpl_l2t_write_rpl *p = (void *)rsp; 666 667 do_l2t_write_rpl(q->adap, p); 668 } else if (opcode == CPL_SET_TCB_RPL) { 669 const struct cpl_set_tcb_rpl *p = (void *)rsp; 670 671 filter_rpl(q->adap, p); 672 } else 673 dev_err(q->adap->pdev_dev, 674 "unexpected CPL %#x on FW event queue\n", opcode); 675 out: 676 return 0; 677 } 678 679 /** 680 * uldrx_handler - response queue handler for ULD queues 681 * @q: the response queue that received the packet 682 * @rsp: the response queue descriptor holding the offload message 683 * @gl: the gather list of packet fragments 684 * 685 * Deliver an ingress offload packet to a ULD. All processing is done by 686 * the ULD, we just maintain statistics. 687 */ 688 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp, 689 const struct pkt_gl *gl) 690 { 691 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq); 692 693 /* FW can send CPLs encapsulated in a CPL_FW4_MSG. 694 */ 695 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG && 696 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL) 697 rsp += 2; 698 699 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) { 700 rxq->stats.nomem++; 701 return -1; 702 } 703 if (gl == NULL) 704 rxq->stats.imm++; 705 else if (gl == CXGB4_MSG_AN) 706 rxq->stats.an++; 707 else 708 rxq->stats.pkts++; 709 return 0; 710 } 711 712 static void disable_msi(struct adapter *adapter) 713 { 714 if (adapter->flags & USING_MSIX) { 715 pci_disable_msix(adapter->pdev); 716 adapter->flags &= ~USING_MSIX; 717 } else if (adapter->flags & USING_MSI) { 718 pci_disable_msi(adapter->pdev); 719 adapter->flags &= ~USING_MSI; 720 } 721 } 722 723 /* 724 * Interrupt handler for non-data events used with MSI-X. 725 */ 726 static irqreturn_t t4_nondata_intr(int irq, void *cookie) 727 { 728 struct adapter *adap = cookie; 729 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A)); 730 731 if (v & PFSW_F) { 732 adap->swintr = 1; 733 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE_A), v); 734 } 735 t4_slow_intr_handler(adap); 736 return IRQ_HANDLED; 737 } 738 739 /* 740 * Name the MSI-X interrupts. 741 */ 742 static void name_msix_vecs(struct adapter *adap) 743 { 744 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc); 745 746 /* non-data interrupts */ 747 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name); 748 749 /* FW events */ 750 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", 751 adap->port[0]->name); 752 753 /* Ethernet queues */ 754 for_each_port(adap, j) { 755 struct net_device *d = adap->port[j]; 756 const struct port_info *pi = netdev_priv(d); 757 758 for (i = 0; i < pi->nqsets; i++, msi_idx++) 759 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d", 760 d->name, i); 761 } 762 763 /* offload queues */ 764 for_each_ofldrxq(&adap->sge, i) 765 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d", 766 adap->port[0]->name, i); 767 768 for_each_rdmarxq(&adap->sge, i) 769 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d", 770 adap->port[0]->name, i); 771 772 for_each_rdmaciq(&adap->sge, i) 773 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d", 774 adap->port[0]->name, i); 775 } 776 777 static int request_msix_queue_irqs(struct adapter *adap) 778 { 779 struct sge *s = &adap->sge; 780 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0; 781 int msi_index = 2; 782 783 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0, 784 adap->msix_info[1].desc, &s->fw_evtq); 785 if (err) 786 return err; 787 788 for_each_ethrxq(s, ethqidx) { 789 err = request_irq(adap->msix_info[msi_index].vec, 790 t4_sge_intr_msix, 0, 791 adap->msix_info[msi_index].desc, 792 &s->ethrxq[ethqidx].rspq); 793 if (err) 794 goto unwind; 795 msi_index++; 796 } 797 for_each_ofldrxq(s, ofldqidx) { 798 err = request_irq(adap->msix_info[msi_index].vec, 799 t4_sge_intr_msix, 0, 800 adap->msix_info[msi_index].desc, 801 &s->ofldrxq[ofldqidx].rspq); 802 if (err) 803 goto unwind; 804 msi_index++; 805 } 806 for_each_rdmarxq(s, rdmaqidx) { 807 err = request_irq(adap->msix_info[msi_index].vec, 808 t4_sge_intr_msix, 0, 809 adap->msix_info[msi_index].desc, 810 &s->rdmarxq[rdmaqidx].rspq); 811 if (err) 812 goto unwind; 813 msi_index++; 814 } 815 for_each_rdmaciq(s, rdmaciqqidx) { 816 err = request_irq(adap->msix_info[msi_index].vec, 817 t4_sge_intr_msix, 0, 818 adap->msix_info[msi_index].desc, 819 &s->rdmaciq[rdmaciqqidx].rspq); 820 if (err) 821 goto unwind; 822 msi_index++; 823 } 824 return 0; 825 826 unwind: 827 while (--rdmaciqqidx >= 0) 828 free_irq(adap->msix_info[--msi_index].vec, 829 &s->rdmaciq[rdmaciqqidx].rspq); 830 while (--rdmaqidx >= 0) 831 free_irq(adap->msix_info[--msi_index].vec, 832 &s->rdmarxq[rdmaqidx].rspq); 833 while (--ofldqidx >= 0) 834 free_irq(adap->msix_info[--msi_index].vec, 835 &s->ofldrxq[ofldqidx].rspq); 836 while (--ethqidx >= 0) 837 free_irq(adap->msix_info[--msi_index].vec, 838 &s->ethrxq[ethqidx].rspq); 839 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 840 return err; 841 } 842 843 static void free_msix_queue_irqs(struct adapter *adap) 844 { 845 int i, msi_index = 2; 846 struct sge *s = &adap->sge; 847 848 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 849 for_each_ethrxq(s, i) 850 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq); 851 for_each_ofldrxq(s, i) 852 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq); 853 for_each_rdmarxq(s, i) 854 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq); 855 for_each_rdmaciq(s, i) 856 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq); 857 } 858 859 /** 860 * write_rss - write the RSS table for a given port 861 * @pi: the port 862 * @queues: array of queue indices for RSS 863 * 864 * Sets up the portion of the HW RSS table for the port's VI to distribute 865 * packets to the Rx queues in @queues. 866 */ 867 static int write_rss(const struct port_info *pi, const u16 *queues) 868 { 869 u16 *rss; 870 int i, err; 871 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset]; 872 873 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL); 874 if (!rss) 875 return -ENOMEM; 876 877 /* map the queue indices to queue ids */ 878 for (i = 0; i < pi->rss_size; i++, queues++) 879 rss[i] = q[*queues].rspq.abs_id; 880 881 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0, 882 pi->rss_size, rss, pi->rss_size); 883 kfree(rss); 884 return err; 885 } 886 887 /** 888 * setup_rss - configure RSS 889 * @adap: the adapter 890 * 891 * Sets up RSS for each port. 892 */ 893 static int setup_rss(struct adapter *adap) 894 { 895 int i, err; 896 897 for_each_port(adap, i) { 898 const struct port_info *pi = adap2pinfo(adap, i); 899 900 err = write_rss(pi, pi->rss); 901 if (err) 902 return err; 903 } 904 return 0; 905 } 906 907 /* 908 * Return the channel of the ingress queue with the given qid. 909 */ 910 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid) 911 { 912 qid -= p->ingr_start; 913 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan; 914 } 915 916 /* 917 * Wait until all NAPI handlers are descheduled. 918 */ 919 static void quiesce_rx(struct adapter *adap) 920 { 921 int i; 922 923 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) { 924 struct sge_rspq *q = adap->sge.ingr_map[i]; 925 926 if (q && q->handler) { 927 napi_disable(&q->napi); 928 local_bh_disable(); 929 while (!cxgb_poll_lock_napi(q)) 930 mdelay(1); 931 local_bh_enable(); 932 } 933 934 } 935 } 936 937 /* 938 * Enable NAPI scheduling and interrupt generation for all Rx queues. 939 */ 940 static void enable_rx(struct adapter *adap) 941 { 942 int i; 943 944 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) { 945 struct sge_rspq *q = adap->sge.ingr_map[i]; 946 947 if (!q) 948 continue; 949 if (q->handler) { 950 cxgb_busy_poll_init_lock(q); 951 napi_enable(&q->napi); 952 } 953 /* 0-increment GTS to start the timer and enable interrupts */ 954 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS_A), 955 SEINTARM_V(q->intr_params) | 956 INGRESSQID_V(q->cntxt_id)); 957 } 958 } 959 960 static int alloc_ofld_rxqs(struct adapter *adap, struct sge_ofld_rxq *q, 961 unsigned int nq, unsigned int per_chan, int msi_idx, 962 u16 *ids) 963 { 964 int i, err; 965 966 for (i = 0; i < nq; i++, q++) { 967 if (msi_idx > 0) 968 msi_idx++; 969 err = t4_sge_alloc_rxq(adap, &q->rspq, false, 970 adap->port[i / per_chan], 971 msi_idx, q->fl.size ? &q->fl : NULL, 972 uldrx_handler); 973 if (err) 974 return err; 975 memset(&q->stats, 0, sizeof(q->stats)); 976 if (ids) 977 ids[i] = q->rspq.abs_id; 978 } 979 return 0; 980 } 981 982 /** 983 * setup_sge_queues - configure SGE Tx/Rx/response queues 984 * @adap: the adapter 985 * 986 * Determines how many sets of SGE queues to use and initializes them. 987 * We support multiple queue sets per port if we have MSI-X, otherwise 988 * just one queue set per port. 989 */ 990 static int setup_sge_queues(struct adapter *adap) 991 { 992 int err, msi_idx, i, j; 993 struct sge *s = &adap->sge; 994 995 bitmap_zero(s->starving_fl, MAX_EGRQ); 996 bitmap_zero(s->txq_maperr, MAX_EGRQ); 997 998 if (adap->flags & USING_MSIX) 999 msi_idx = 1; /* vector 0 is for non-queue interrupts */ 1000 else { 1001 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0, 1002 NULL, NULL); 1003 if (err) 1004 return err; 1005 msi_idx = -((int)s->intrq.abs_id + 1); 1006 } 1007 1008 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0], 1009 msi_idx, NULL, fwevtq_handler); 1010 if (err) { 1011 freeout: t4_free_sge_resources(adap); 1012 return err; 1013 } 1014 1015 for_each_port(adap, i) { 1016 struct net_device *dev = adap->port[i]; 1017 struct port_info *pi = netdev_priv(dev); 1018 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset]; 1019 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset]; 1020 1021 for (j = 0; j < pi->nqsets; j++, q++) { 1022 if (msi_idx > 0) 1023 msi_idx++; 1024 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, 1025 msi_idx, &q->fl, 1026 t4_ethrx_handler); 1027 if (err) 1028 goto freeout; 1029 q->rspq.idx = j; 1030 memset(&q->stats, 0, sizeof(q->stats)); 1031 } 1032 for (j = 0; j < pi->nqsets; j++, t++) { 1033 err = t4_sge_alloc_eth_txq(adap, t, dev, 1034 netdev_get_tx_queue(dev, j), 1035 s->fw_evtq.cntxt_id); 1036 if (err) 1037 goto freeout; 1038 } 1039 } 1040 1041 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */ 1042 for_each_ofldrxq(s, i) { 1043 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], 1044 adap->port[i / j], 1045 s->fw_evtq.cntxt_id); 1046 if (err) 1047 goto freeout; 1048 } 1049 1050 #define ALLOC_OFLD_RXQS(firstq, nq, per_chan, ids) do { \ 1051 err = alloc_ofld_rxqs(adap, firstq, nq, per_chan, msi_idx, ids); \ 1052 if (err) \ 1053 goto freeout; \ 1054 if (msi_idx > 0) \ 1055 msi_idx += nq; \ 1056 } while (0) 1057 1058 ALLOC_OFLD_RXQS(s->ofldrxq, s->ofldqsets, j, s->ofld_rxq); 1059 ALLOC_OFLD_RXQS(s->rdmarxq, s->rdmaqs, 1, s->rdma_rxq); 1060 j = s->rdmaciqs / adap->params.nports; /* rdmaq queues per channel */ 1061 ALLOC_OFLD_RXQS(s->rdmaciq, s->rdmaciqs, j, s->rdma_ciq); 1062 1063 #undef ALLOC_OFLD_RXQS 1064 1065 for_each_port(adap, i) { 1066 /* 1067 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't 1068 * have RDMA queues, and that's the right value. 1069 */ 1070 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i], 1071 s->fw_evtq.cntxt_id, 1072 s->rdmarxq[i].rspq.cntxt_id); 1073 if (err) 1074 goto freeout; 1075 } 1076 1077 t4_write_reg(adap, is_t4(adap->params.chip) ? 1078 MPS_TRC_RSS_CONTROL_A : 1079 MPS_T5_TRC_RSS_CONTROL_A, 1080 RSSCONTROL_V(netdev2pinfo(adap->port[0])->tx_chan) | 1081 QUEUENUMBER_V(s->ethrxq[0].rspq.abs_id)); 1082 return 0; 1083 } 1084 1085 /* 1086 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc. 1087 * The allocated memory is cleared. 1088 */ 1089 void *t4_alloc_mem(size_t size) 1090 { 1091 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 1092 1093 if (!p) 1094 p = vzalloc(size); 1095 return p; 1096 } 1097 1098 /* 1099 * Free memory allocated through alloc_mem(). 1100 */ 1101 void t4_free_mem(void *addr) 1102 { 1103 if (is_vmalloc_addr(addr)) 1104 vfree(addr); 1105 else 1106 kfree(addr); 1107 } 1108 1109 /* Send a Work Request to write the filter at a specified index. We construct 1110 * a Firmware Filter Work Request to have the work done and put the indicated 1111 * filter into "pending" mode which will prevent any further actions against 1112 * it till we get a reply from the firmware on the completion status of the 1113 * request. 1114 */ 1115 static int set_filter_wr(struct adapter *adapter, int fidx) 1116 { 1117 struct filter_entry *f = &adapter->tids.ftid_tab[fidx]; 1118 struct sk_buff *skb; 1119 struct fw_filter_wr *fwr; 1120 unsigned int ftid; 1121 1122 /* If the new filter requires loopback Destination MAC and/or VLAN 1123 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for 1124 * the filter. 1125 */ 1126 if (f->fs.newdmac || f->fs.newvlan) { 1127 /* allocate L2T entry for new filter */ 1128 f->l2t = t4_l2t_alloc_switching(adapter->l2t); 1129 if (f->l2t == NULL) 1130 return -EAGAIN; 1131 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan, 1132 f->fs.eport, f->fs.dmac)) { 1133 cxgb4_l2t_release(f->l2t); 1134 f->l2t = NULL; 1135 return -ENOMEM; 1136 } 1137 } 1138 1139 ftid = adapter->tids.ftid_base + fidx; 1140 1141 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL); 1142 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr)); 1143 memset(fwr, 0, sizeof(*fwr)); 1144 1145 /* It would be nice to put most of the following in t4_hw.c but most 1146 * of the work is translating the cxgbtool ch_filter_specification 1147 * into the Work Request and the definition of that structure is 1148 * currently in cxgbtool.h which isn't appropriate to pull into the 1149 * common code. We may eventually try to come up with a more neutral 1150 * filter specification structure but for now it's easiest to simply 1151 * put this fairly direct code in line ... 1152 */ 1153 fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR)); 1154 fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16)); 1155 fwr->tid_to_iq = 1156 htonl(FW_FILTER_WR_TID_V(ftid) | 1157 FW_FILTER_WR_RQTYPE_V(f->fs.type) | 1158 FW_FILTER_WR_NOREPLY_V(0) | 1159 FW_FILTER_WR_IQ_V(f->fs.iq)); 1160 fwr->del_filter_to_l2tix = 1161 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) | 1162 FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) | 1163 FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) | 1164 FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) | 1165 FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) | 1166 FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) | 1167 FW_FILTER_WR_DMAC_V(f->fs.newdmac) | 1168 FW_FILTER_WR_SMAC_V(f->fs.newsmac) | 1169 FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT || 1170 f->fs.newvlan == VLAN_REWRITE) | 1171 FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE || 1172 f->fs.newvlan == VLAN_REWRITE) | 1173 FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) | 1174 FW_FILTER_WR_TXCHAN_V(f->fs.eport) | 1175 FW_FILTER_WR_PRIO_V(f->fs.prio) | 1176 FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0)); 1177 fwr->ethtype = htons(f->fs.val.ethtype); 1178 fwr->ethtypem = htons(f->fs.mask.ethtype); 1179 fwr->frag_to_ovlan_vldm = 1180 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) | 1181 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) | 1182 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) | 1183 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) | 1184 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) | 1185 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld)); 1186 fwr->smac_sel = 0; 1187 fwr->rx_chan_rx_rpl_iq = 1188 htons(FW_FILTER_WR_RX_CHAN_V(0) | 1189 FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id)); 1190 fwr->maci_to_matchtypem = 1191 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) | 1192 FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) | 1193 FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) | 1194 FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) | 1195 FW_FILTER_WR_PORT_V(f->fs.val.iport) | 1196 FW_FILTER_WR_PORTM_V(f->fs.mask.iport) | 1197 FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) | 1198 FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype)); 1199 fwr->ptcl = f->fs.val.proto; 1200 fwr->ptclm = f->fs.mask.proto; 1201 fwr->ttyp = f->fs.val.tos; 1202 fwr->ttypm = f->fs.mask.tos; 1203 fwr->ivlan = htons(f->fs.val.ivlan); 1204 fwr->ivlanm = htons(f->fs.mask.ivlan); 1205 fwr->ovlan = htons(f->fs.val.ovlan); 1206 fwr->ovlanm = htons(f->fs.mask.ovlan); 1207 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip)); 1208 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm)); 1209 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip)); 1210 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm)); 1211 fwr->lp = htons(f->fs.val.lport); 1212 fwr->lpm = htons(f->fs.mask.lport); 1213 fwr->fp = htons(f->fs.val.fport); 1214 fwr->fpm = htons(f->fs.mask.fport); 1215 if (f->fs.newsmac) 1216 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma)); 1217 1218 /* Mark the filter as "pending" and ship off the Filter Work Request. 1219 * When we get the Work Request Reply we'll clear the pending status. 1220 */ 1221 f->pending = 1; 1222 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3); 1223 t4_ofld_send(adapter, skb); 1224 return 0; 1225 } 1226 1227 /* Delete the filter at a specified index. 1228 */ 1229 static int del_filter_wr(struct adapter *adapter, int fidx) 1230 { 1231 struct filter_entry *f = &adapter->tids.ftid_tab[fidx]; 1232 struct sk_buff *skb; 1233 struct fw_filter_wr *fwr; 1234 unsigned int len, ftid; 1235 1236 len = sizeof(*fwr); 1237 ftid = adapter->tids.ftid_base + fidx; 1238 1239 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); 1240 fwr = (struct fw_filter_wr *)__skb_put(skb, len); 1241 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id); 1242 1243 /* Mark the filter as "pending" and ship off the Filter Work Request. 1244 * When we get the Work Request Reply we'll clear the pending status. 1245 */ 1246 f->pending = 1; 1247 t4_mgmt_tx(adapter, skb); 1248 return 0; 1249 } 1250 1251 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb, 1252 void *accel_priv, select_queue_fallback_t fallback) 1253 { 1254 int txq; 1255 1256 #ifdef CONFIG_CHELSIO_T4_DCB 1257 /* If a Data Center Bridging has been successfully negotiated on this 1258 * link then we'll use the skb's priority to map it to a TX Queue. 1259 * The skb's priority is determined via the VLAN Tag Priority Code 1260 * Point field. 1261 */ 1262 if (cxgb4_dcb_enabled(dev)) { 1263 u16 vlan_tci; 1264 int err; 1265 1266 err = vlan_get_tag(skb, &vlan_tci); 1267 if (unlikely(err)) { 1268 if (net_ratelimit()) 1269 netdev_warn(dev, 1270 "TX Packet without VLAN Tag on DCB Link\n"); 1271 txq = 0; 1272 } else { 1273 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1274 } 1275 return txq; 1276 } 1277 #endif /* CONFIG_CHELSIO_T4_DCB */ 1278 1279 if (select_queue) { 1280 txq = (skb_rx_queue_recorded(skb) 1281 ? skb_get_rx_queue(skb) 1282 : smp_processor_id()); 1283 1284 while (unlikely(txq >= dev->real_num_tx_queues)) 1285 txq -= dev->real_num_tx_queues; 1286 1287 return txq; 1288 } 1289 1290 return fallback(dev, skb) % dev->real_num_tx_queues; 1291 } 1292 1293 static inline int is_offload(const struct adapter *adap) 1294 { 1295 return adap->params.offload; 1296 } 1297 1298 /* 1299 * Implementation of ethtool operations. 1300 */ 1301 1302 static u32 get_msglevel(struct net_device *dev) 1303 { 1304 return netdev2adap(dev)->msg_enable; 1305 } 1306 1307 static void set_msglevel(struct net_device *dev, u32 val) 1308 { 1309 netdev2adap(dev)->msg_enable = val; 1310 } 1311 1312 static char stats_strings[][ETH_GSTRING_LEN] = { 1313 "TxOctetsOK ", 1314 "TxFramesOK ", 1315 "TxBroadcastFrames ", 1316 "TxMulticastFrames ", 1317 "TxUnicastFrames ", 1318 "TxErrorFrames ", 1319 1320 "TxFrames64 ", 1321 "TxFrames65To127 ", 1322 "TxFrames128To255 ", 1323 "TxFrames256To511 ", 1324 "TxFrames512To1023 ", 1325 "TxFrames1024To1518 ", 1326 "TxFrames1519ToMax ", 1327 1328 "TxFramesDropped ", 1329 "TxPauseFrames ", 1330 "TxPPP0Frames ", 1331 "TxPPP1Frames ", 1332 "TxPPP2Frames ", 1333 "TxPPP3Frames ", 1334 "TxPPP4Frames ", 1335 "TxPPP5Frames ", 1336 "TxPPP6Frames ", 1337 "TxPPP7Frames ", 1338 1339 "RxOctetsOK ", 1340 "RxFramesOK ", 1341 "RxBroadcastFrames ", 1342 "RxMulticastFrames ", 1343 "RxUnicastFrames ", 1344 1345 "RxFramesTooLong ", 1346 "RxJabberErrors ", 1347 "RxFCSErrors ", 1348 "RxLengthErrors ", 1349 "RxSymbolErrors ", 1350 "RxRuntFrames ", 1351 1352 "RxFrames64 ", 1353 "RxFrames65To127 ", 1354 "RxFrames128To255 ", 1355 "RxFrames256To511 ", 1356 "RxFrames512To1023 ", 1357 "RxFrames1024To1518 ", 1358 "RxFrames1519ToMax ", 1359 1360 "RxPauseFrames ", 1361 "RxPPP0Frames ", 1362 "RxPPP1Frames ", 1363 "RxPPP2Frames ", 1364 "RxPPP3Frames ", 1365 "RxPPP4Frames ", 1366 "RxPPP5Frames ", 1367 "RxPPP6Frames ", 1368 "RxPPP7Frames ", 1369 1370 "RxBG0FramesDropped ", 1371 "RxBG1FramesDropped ", 1372 "RxBG2FramesDropped ", 1373 "RxBG3FramesDropped ", 1374 "RxBG0FramesTrunc ", 1375 "RxBG1FramesTrunc ", 1376 "RxBG2FramesTrunc ", 1377 "RxBG3FramesTrunc ", 1378 1379 "TSO ", 1380 "TxCsumOffload ", 1381 "RxCsumGood ", 1382 "VLANextractions ", 1383 "VLANinsertions ", 1384 "GROpackets ", 1385 "GROmerged ", 1386 "WriteCoalSuccess ", 1387 "WriteCoalFail ", 1388 }; 1389 1390 static int get_sset_count(struct net_device *dev, int sset) 1391 { 1392 switch (sset) { 1393 case ETH_SS_STATS: 1394 return ARRAY_SIZE(stats_strings); 1395 default: 1396 return -EOPNOTSUPP; 1397 } 1398 } 1399 1400 #define T4_REGMAP_SIZE (160 * 1024) 1401 #define T5_REGMAP_SIZE (332 * 1024) 1402 1403 static int get_regs_len(struct net_device *dev) 1404 { 1405 struct adapter *adap = netdev2adap(dev); 1406 if (is_t4(adap->params.chip)) 1407 return T4_REGMAP_SIZE; 1408 else 1409 return T5_REGMAP_SIZE; 1410 } 1411 1412 static int get_eeprom_len(struct net_device *dev) 1413 { 1414 return EEPROMSIZE; 1415 } 1416 1417 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 1418 { 1419 struct adapter *adapter = netdev2adap(dev); 1420 u32 exprom_vers; 1421 1422 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 1423 strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 1424 strlcpy(info->bus_info, pci_name(adapter->pdev), 1425 sizeof(info->bus_info)); 1426 1427 if (adapter->params.fw_vers) 1428 snprintf(info->fw_version, sizeof(info->fw_version), 1429 "%u.%u.%u.%u, TP %u.%u.%u.%u", 1430 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers), 1431 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers), 1432 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers), 1433 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers), 1434 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers), 1435 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers), 1436 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers), 1437 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers)); 1438 1439 if (!t4_get_exprom_version(adapter, &exprom_vers)) 1440 snprintf(info->erom_version, sizeof(info->erom_version), 1441 "%u.%u.%u.%u", 1442 FW_HDR_FW_VER_MAJOR_G(exprom_vers), 1443 FW_HDR_FW_VER_MINOR_G(exprom_vers), 1444 FW_HDR_FW_VER_MICRO_G(exprom_vers), 1445 FW_HDR_FW_VER_BUILD_G(exprom_vers)); 1446 } 1447 1448 static void get_strings(struct net_device *dev, u32 stringset, u8 *data) 1449 { 1450 if (stringset == ETH_SS_STATS) 1451 memcpy(data, stats_strings, sizeof(stats_strings)); 1452 } 1453 1454 /* 1455 * port stats maintained per queue of the port. They should be in the same 1456 * order as in stats_strings above. 1457 */ 1458 struct queue_port_stats { 1459 u64 tso; 1460 u64 tx_csum; 1461 u64 rx_csum; 1462 u64 vlan_ex; 1463 u64 vlan_ins; 1464 u64 gro_pkts; 1465 u64 gro_merged; 1466 }; 1467 1468 static void collect_sge_port_stats(const struct adapter *adap, 1469 const struct port_info *p, struct queue_port_stats *s) 1470 { 1471 int i; 1472 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset]; 1473 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset]; 1474 1475 memset(s, 0, sizeof(*s)); 1476 for (i = 0; i < p->nqsets; i++, rx++, tx++) { 1477 s->tso += tx->tso; 1478 s->tx_csum += tx->tx_cso; 1479 s->rx_csum += rx->stats.rx_cso; 1480 s->vlan_ex += rx->stats.vlan_ex; 1481 s->vlan_ins += tx->vlan_ins; 1482 s->gro_pkts += rx->stats.lro_pkts; 1483 s->gro_merged += rx->stats.lro_merged; 1484 } 1485 } 1486 1487 static void get_stats(struct net_device *dev, struct ethtool_stats *stats, 1488 u64 *data) 1489 { 1490 struct port_info *pi = netdev_priv(dev); 1491 struct adapter *adapter = pi->adapter; 1492 u32 val1, val2; 1493 1494 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data); 1495 1496 data += sizeof(struct port_stats) / sizeof(u64); 1497 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data); 1498 data += sizeof(struct queue_port_stats) / sizeof(u64); 1499 if (!is_t4(adapter->params.chip)) { 1500 t4_write_reg(adapter, SGE_STAT_CFG_A, STATSOURCE_T5_V(7)); 1501 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL_A); 1502 val2 = t4_read_reg(adapter, SGE_STAT_MATCH_A); 1503 *data = val1 - val2; 1504 data++; 1505 *data = val2; 1506 data++; 1507 } else { 1508 memset(data, 0, 2 * sizeof(u64)); 1509 *data += 2; 1510 } 1511 } 1512 1513 /* 1514 * Return a version number to identify the type of adapter. The scheme is: 1515 * - bits 0..9: chip version 1516 * - bits 10..15: chip revision 1517 * - bits 16..23: register dump version 1518 */ 1519 static inline unsigned int mk_adap_vers(const struct adapter *ap) 1520 { 1521 return CHELSIO_CHIP_VERSION(ap->params.chip) | 1522 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16); 1523 } 1524 1525 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start, 1526 unsigned int end) 1527 { 1528 u32 *p = buf + start; 1529 1530 for ( ; start <= end; start += sizeof(u32)) 1531 *p++ = t4_read_reg(ap, start); 1532 } 1533 1534 static void get_regs(struct net_device *dev, struct ethtool_regs *regs, 1535 void *buf) 1536 { 1537 static const unsigned int t4_reg_ranges[] = { 1538 0x1008, 0x1108, 1539 0x1180, 0x11b4, 1540 0x11fc, 0x123c, 1541 0x1300, 0x173c, 1542 0x1800, 0x18fc, 1543 0x3000, 0x30d8, 1544 0x30e0, 0x5924, 1545 0x5960, 0x59d4, 1546 0x5a00, 0x5af8, 1547 0x6000, 0x6098, 1548 0x6100, 0x6150, 1549 0x6200, 0x6208, 1550 0x6240, 0x6248, 1551 0x6280, 0x6338, 1552 0x6370, 0x638c, 1553 0x6400, 0x643c, 1554 0x6500, 0x6524, 1555 0x6a00, 0x6a38, 1556 0x6a60, 0x6a78, 1557 0x6b00, 0x6b84, 1558 0x6bf0, 0x6c84, 1559 0x6cf0, 0x6d84, 1560 0x6df0, 0x6e84, 1561 0x6ef0, 0x6f84, 1562 0x6ff0, 0x7084, 1563 0x70f0, 0x7184, 1564 0x71f0, 0x7284, 1565 0x72f0, 0x7384, 1566 0x73f0, 0x7450, 1567 0x7500, 0x7530, 1568 0x7600, 0x761c, 1569 0x7680, 0x76cc, 1570 0x7700, 0x7798, 1571 0x77c0, 0x77fc, 1572 0x7900, 0x79fc, 1573 0x7b00, 0x7c38, 1574 0x7d00, 0x7efc, 1575 0x8dc0, 0x8e1c, 1576 0x8e30, 0x8e78, 1577 0x8ea0, 0x8f6c, 1578 0x8fc0, 0x9074, 1579 0x90fc, 0x90fc, 1580 0x9400, 0x9458, 1581 0x9600, 0x96bc, 1582 0x9800, 0x9808, 1583 0x9820, 0x983c, 1584 0x9850, 0x9864, 1585 0x9c00, 0x9c6c, 1586 0x9c80, 0x9cec, 1587 0x9d00, 0x9d6c, 1588 0x9d80, 0x9dec, 1589 0x9e00, 0x9e6c, 1590 0x9e80, 0x9eec, 1591 0x9f00, 0x9f6c, 1592 0x9f80, 0x9fec, 1593 0xd004, 0xd03c, 1594 0xdfc0, 0xdfe0, 1595 0xe000, 0xea7c, 1596 0xf000, 0x11110, 1597 0x11118, 0x11190, 1598 0x19040, 0x1906c, 1599 0x19078, 0x19080, 1600 0x1908c, 0x19124, 1601 0x19150, 0x191b0, 1602 0x191d0, 0x191e8, 1603 0x19238, 0x1924c, 1604 0x193f8, 0x19474, 1605 0x19490, 0x194f8, 1606 0x19800, 0x19f30, 1607 0x1a000, 0x1a06c, 1608 0x1a0b0, 0x1a120, 1609 0x1a128, 0x1a138, 1610 0x1a190, 0x1a1c4, 1611 0x1a1fc, 0x1a1fc, 1612 0x1e040, 0x1e04c, 1613 0x1e284, 0x1e28c, 1614 0x1e2c0, 0x1e2c0, 1615 0x1e2e0, 0x1e2e0, 1616 0x1e300, 0x1e384, 1617 0x1e3c0, 0x1e3c8, 1618 0x1e440, 0x1e44c, 1619 0x1e684, 0x1e68c, 1620 0x1e6c0, 0x1e6c0, 1621 0x1e6e0, 0x1e6e0, 1622 0x1e700, 0x1e784, 1623 0x1e7c0, 0x1e7c8, 1624 0x1e840, 0x1e84c, 1625 0x1ea84, 0x1ea8c, 1626 0x1eac0, 0x1eac0, 1627 0x1eae0, 0x1eae0, 1628 0x1eb00, 0x1eb84, 1629 0x1ebc0, 0x1ebc8, 1630 0x1ec40, 0x1ec4c, 1631 0x1ee84, 0x1ee8c, 1632 0x1eec0, 0x1eec0, 1633 0x1eee0, 0x1eee0, 1634 0x1ef00, 0x1ef84, 1635 0x1efc0, 0x1efc8, 1636 0x1f040, 0x1f04c, 1637 0x1f284, 0x1f28c, 1638 0x1f2c0, 0x1f2c0, 1639 0x1f2e0, 0x1f2e0, 1640 0x1f300, 0x1f384, 1641 0x1f3c0, 0x1f3c8, 1642 0x1f440, 0x1f44c, 1643 0x1f684, 0x1f68c, 1644 0x1f6c0, 0x1f6c0, 1645 0x1f6e0, 0x1f6e0, 1646 0x1f700, 0x1f784, 1647 0x1f7c0, 0x1f7c8, 1648 0x1f840, 0x1f84c, 1649 0x1fa84, 0x1fa8c, 1650 0x1fac0, 0x1fac0, 1651 0x1fae0, 0x1fae0, 1652 0x1fb00, 0x1fb84, 1653 0x1fbc0, 0x1fbc8, 1654 0x1fc40, 0x1fc4c, 1655 0x1fe84, 0x1fe8c, 1656 0x1fec0, 0x1fec0, 1657 0x1fee0, 0x1fee0, 1658 0x1ff00, 0x1ff84, 1659 0x1ffc0, 0x1ffc8, 1660 0x20000, 0x2002c, 1661 0x20100, 0x2013c, 1662 0x20190, 0x201c8, 1663 0x20200, 0x20318, 1664 0x20400, 0x20528, 1665 0x20540, 0x20614, 1666 0x21000, 0x21040, 1667 0x2104c, 0x21060, 1668 0x210c0, 0x210ec, 1669 0x21200, 0x21268, 1670 0x21270, 0x21284, 1671 0x212fc, 0x21388, 1672 0x21400, 0x21404, 1673 0x21500, 0x21518, 1674 0x2152c, 0x2153c, 1675 0x21550, 0x21554, 1676 0x21600, 0x21600, 1677 0x21608, 0x21628, 1678 0x21630, 0x2163c, 1679 0x21700, 0x2171c, 1680 0x21780, 0x2178c, 1681 0x21800, 0x21c38, 1682 0x21c80, 0x21d7c, 1683 0x21e00, 0x21e04, 1684 0x22000, 0x2202c, 1685 0x22100, 0x2213c, 1686 0x22190, 0x221c8, 1687 0x22200, 0x22318, 1688 0x22400, 0x22528, 1689 0x22540, 0x22614, 1690 0x23000, 0x23040, 1691 0x2304c, 0x23060, 1692 0x230c0, 0x230ec, 1693 0x23200, 0x23268, 1694 0x23270, 0x23284, 1695 0x232fc, 0x23388, 1696 0x23400, 0x23404, 1697 0x23500, 0x23518, 1698 0x2352c, 0x2353c, 1699 0x23550, 0x23554, 1700 0x23600, 0x23600, 1701 0x23608, 0x23628, 1702 0x23630, 0x2363c, 1703 0x23700, 0x2371c, 1704 0x23780, 0x2378c, 1705 0x23800, 0x23c38, 1706 0x23c80, 0x23d7c, 1707 0x23e00, 0x23e04, 1708 0x24000, 0x2402c, 1709 0x24100, 0x2413c, 1710 0x24190, 0x241c8, 1711 0x24200, 0x24318, 1712 0x24400, 0x24528, 1713 0x24540, 0x24614, 1714 0x25000, 0x25040, 1715 0x2504c, 0x25060, 1716 0x250c0, 0x250ec, 1717 0x25200, 0x25268, 1718 0x25270, 0x25284, 1719 0x252fc, 0x25388, 1720 0x25400, 0x25404, 1721 0x25500, 0x25518, 1722 0x2552c, 0x2553c, 1723 0x25550, 0x25554, 1724 0x25600, 0x25600, 1725 0x25608, 0x25628, 1726 0x25630, 0x2563c, 1727 0x25700, 0x2571c, 1728 0x25780, 0x2578c, 1729 0x25800, 0x25c38, 1730 0x25c80, 0x25d7c, 1731 0x25e00, 0x25e04, 1732 0x26000, 0x2602c, 1733 0x26100, 0x2613c, 1734 0x26190, 0x261c8, 1735 0x26200, 0x26318, 1736 0x26400, 0x26528, 1737 0x26540, 0x26614, 1738 0x27000, 0x27040, 1739 0x2704c, 0x27060, 1740 0x270c0, 0x270ec, 1741 0x27200, 0x27268, 1742 0x27270, 0x27284, 1743 0x272fc, 0x27388, 1744 0x27400, 0x27404, 1745 0x27500, 0x27518, 1746 0x2752c, 0x2753c, 1747 0x27550, 0x27554, 1748 0x27600, 0x27600, 1749 0x27608, 0x27628, 1750 0x27630, 0x2763c, 1751 0x27700, 0x2771c, 1752 0x27780, 0x2778c, 1753 0x27800, 0x27c38, 1754 0x27c80, 0x27d7c, 1755 0x27e00, 0x27e04 1756 }; 1757 1758 static const unsigned int t5_reg_ranges[] = { 1759 0x1008, 0x1148, 1760 0x1180, 0x11b4, 1761 0x11fc, 0x123c, 1762 0x1280, 0x173c, 1763 0x1800, 0x18fc, 1764 0x3000, 0x3028, 1765 0x3060, 0x30d8, 1766 0x30e0, 0x30fc, 1767 0x3140, 0x357c, 1768 0x35a8, 0x35cc, 1769 0x35ec, 0x35ec, 1770 0x3600, 0x5624, 1771 0x56cc, 0x575c, 1772 0x580c, 0x5814, 1773 0x5890, 0x58bc, 1774 0x5940, 0x59dc, 1775 0x59fc, 0x5a18, 1776 0x5a60, 0x5a9c, 1777 0x5b9c, 0x5bfc, 1778 0x6000, 0x6040, 1779 0x6058, 0x614c, 1780 0x7700, 0x7798, 1781 0x77c0, 0x78fc, 1782 0x7b00, 0x7c54, 1783 0x7d00, 0x7efc, 1784 0x8dc0, 0x8de0, 1785 0x8df8, 0x8e84, 1786 0x8ea0, 0x8f84, 1787 0x8fc0, 0x90f8, 1788 0x9400, 0x9470, 1789 0x9600, 0x96f4, 1790 0x9800, 0x9808, 1791 0x9820, 0x983c, 1792 0x9850, 0x9864, 1793 0x9c00, 0x9c6c, 1794 0x9c80, 0x9cec, 1795 0x9d00, 0x9d6c, 1796 0x9d80, 0x9dec, 1797 0x9e00, 0x9e6c, 1798 0x9e80, 0x9eec, 1799 0x9f00, 0x9f6c, 1800 0x9f80, 0xa020, 1801 0xd004, 0xd03c, 1802 0xdfc0, 0xdfe0, 1803 0xe000, 0x11088, 1804 0x1109c, 0x11110, 1805 0x11118, 0x1117c, 1806 0x11190, 0x11204, 1807 0x19040, 0x1906c, 1808 0x19078, 0x19080, 1809 0x1908c, 0x19124, 1810 0x19150, 0x191b0, 1811 0x191d0, 0x191e8, 1812 0x19238, 0x19290, 1813 0x193f8, 0x19474, 1814 0x19490, 0x194cc, 1815 0x194f0, 0x194f8, 1816 0x19c00, 0x19c60, 1817 0x19c94, 0x19e10, 1818 0x19e50, 0x19f34, 1819 0x19f40, 0x19f50, 1820 0x19f90, 0x19fe4, 1821 0x1a000, 0x1a06c, 1822 0x1a0b0, 0x1a120, 1823 0x1a128, 0x1a138, 1824 0x1a190, 0x1a1c4, 1825 0x1a1fc, 0x1a1fc, 1826 0x1e008, 0x1e00c, 1827 0x1e040, 0x1e04c, 1828 0x1e284, 0x1e290, 1829 0x1e2c0, 0x1e2c0, 1830 0x1e2e0, 0x1e2e0, 1831 0x1e300, 0x1e384, 1832 0x1e3c0, 0x1e3c8, 1833 0x1e408, 0x1e40c, 1834 0x1e440, 0x1e44c, 1835 0x1e684, 0x1e690, 1836 0x1e6c0, 0x1e6c0, 1837 0x1e6e0, 0x1e6e0, 1838 0x1e700, 0x1e784, 1839 0x1e7c0, 0x1e7c8, 1840 0x1e808, 0x1e80c, 1841 0x1e840, 0x1e84c, 1842 0x1ea84, 0x1ea90, 1843 0x1eac0, 0x1eac0, 1844 0x1eae0, 0x1eae0, 1845 0x1eb00, 0x1eb84, 1846 0x1ebc0, 0x1ebc8, 1847 0x1ec08, 0x1ec0c, 1848 0x1ec40, 0x1ec4c, 1849 0x1ee84, 0x1ee90, 1850 0x1eec0, 0x1eec0, 1851 0x1eee0, 0x1eee0, 1852 0x1ef00, 0x1ef84, 1853 0x1efc0, 0x1efc8, 1854 0x1f008, 0x1f00c, 1855 0x1f040, 0x1f04c, 1856 0x1f284, 0x1f290, 1857 0x1f2c0, 0x1f2c0, 1858 0x1f2e0, 0x1f2e0, 1859 0x1f300, 0x1f384, 1860 0x1f3c0, 0x1f3c8, 1861 0x1f408, 0x1f40c, 1862 0x1f440, 0x1f44c, 1863 0x1f684, 0x1f690, 1864 0x1f6c0, 0x1f6c0, 1865 0x1f6e0, 0x1f6e0, 1866 0x1f700, 0x1f784, 1867 0x1f7c0, 0x1f7c8, 1868 0x1f808, 0x1f80c, 1869 0x1f840, 0x1f84c, 1870 0x1fa84, 0x1fa90, 1871 0x1fac0, 0x1fac0, 1872 0x1fae0, 0x1fae0, 1873 0x1fb00, 0x1fb84, 1874 0x1fbc0, 0x1fbc8, 1875 0x1fc08, 0x1fc0c, 1876 0x1fc40, 0x1fc4c, 1877 0x1fe84, 0x1fe90, 1878 0x1fec0, 0x1fec0, 1879 0x1fee0, 0x1fee0, 1880 0x1ff00, 0x1ff84, 1881 0x1ffc0, 0x1ffc8, 1882 0x30000, 0x30030, 1883 0x30100, 0x30144, 1884 0x30190, 0x301d0, 1885 0x30200, 0x30318, 1886 0x30400, 0x3052c, 1887 0x30540, 0x3061c, 1888 0x30800, 0x30834, 1889 0x308c0, 0x30908, 1890 0x30910, 0x309ac, 1891 0x30a00, 0x30a04, 1892 0x30a0c, 0x30a2c, 1893 0x30a44, 0x30a50, 1894 0x30a74, 0x30c24, 1895 0x30d08, 0x30d14, 1896 0x30d1c, 0x30d20, 1897 0x30d3c, 0x30d50, 1898 0x31200, 0x3120c, 1899 0x31220, 0x31220, 1900 0x31240, 0x31240, 1901 0x31600, 0x31600, 1902 0x31608, 0x3160c, 1903 0x31a00, 0x31a1c, 1904 0x31e04, 0x31e20, 1905 0x31e38, 0x31e3c, 1906 0x31e80, 0x31e80, 1907 0x31e88, 0x31ea8, 1908 0x31eb0, 0x31eb4, 1909 0x31ec8, 0x31ed4, 1910 0x31fb8, 0x32004, 1911 0x32208, 0x3223c, 1912 0x32600, 0x32630, 1913 0x32a00, 0x32abc, 1914 0x32b00, 0x32b70, 1915 0x33000, 0x33048, 1916 0x33060, 0x3309c, 1917 0x330f0, 0x33148, 1918 0x33160, 0x3319c, 1919 0x331f0, 0x332e4, 1920 0x332f8, 0x333e4, 1921 0x333f8, 0x33448, 1922 0x33460, 0x3349c, 1923 0x334f0, 0x33548, 1924 0x33560, 0x3359c, 1925 0x335f0, 0x336e4, 1926 0x336f8, 0x337e4, 1927 0x337f8, 0x337fc, 1928 0x33814, 0x33814, 1929 0x3382c, 0x3382c, 1930 0x33880, 0x3388c, 1931 0x338e8, 0x338ec, 1932 0x33900, 0x33948, 1933 0x33960, 0x3399c, 1934 0x339f0, 0x33ae4, 1935 0x33af8, 0x33b10, 1936 0x33b28, 0x33b28, 1937 0x33b3c, 0x33b50, 1938 0x33bf0, 0x33c10, 1939 0x33c28, 0x33c28, 1940 0x33c3c, 0x33c50, 1941 0x33cf0, 0x33cfc, 1942 0x34000, 0x34030, 1943 0x34100, 0x34144, 1944 0x34190, 0x341d0, 1945 0x34200, 0x34318, 1946 0x34400, 0x3452c, 1947 0x34540, 0x3461c, 1948 0x34800, 0x34834, 1949 0x348c0, 0x34908, 1950 0x34910, 0x349ac, 1951 0x34a00, 0x34a04, 1952 0x34a0c, 0x34a2c, 1953 0x34a44, 0x34a50, 1954 0x34a74, 0x34c24, 1955 0x34d08, 0x34d14, 1956 0x34d1c, 0x34d20, 1957 0x34d3c, 0x34d50, 1958 0x35200, 0x3520c, 1959 0x35220, 0x35220, 1960 0x35240, 0x35240, 1961 0x35600, 0x35600, 1962 0x35608, 0x3560c, 1963 0x35a00, 0x35a1c, 1964 0x35e04, 0x35e20, 1965 0x35e38, 0x35e3c, 1966 0x35e80, 0x35e80, 1967 0x35e88, 0x35ea8, 1968 0x35eb0, 0x35eb4, 1969 0x35ec8, 0x35ed4, 1970 0x35fb8, 0x36004, 1971 0x36208, 0x3623c, 1972 0x36600, 0x36630, 1973 0x36a00, 0x36abc, 1974 0x36b00, 0x36b70, 1975 0x37000, 0x37048, 1976 0x37060, 0x3709c, 1977 0x370f0, 0x37148, 1978 0x37160, 0x3719c, 1979 0x371f0, 0x372e4, 1980 0x372f8, 0x373e4, 1981 0x373f8, 0x37448, 1982 0x37460, 0x3749c, 1983 0x374f0, 0x37548, 1984 0x37560, 0x3759c, 1985 0x375f0, 0x376e4, 1986 0x376f8, 0x377e4, 1987 0x377f8, 0x377fc, 1988 0x37814, 0x37814, 1989 0x3782c, 0x3782c, 1990 0x37880, 0x3788c, 1991 0x378e8, 0x378ec, 1992 0x37900, 0x37948, 1993 0x37960, 0x3799c, 1994 0x379f0, 0x37ae4, 1995 0x37af8, 0x37b10, 1996 0x37b28, 0x37b28, 1997 0x37b3c, 0x37b50, 1998 0x37bf0, 0x37c10, 1999 0x37c28, 0x37c28, 2000 0x37c3c, 0x37c50, 2001 0x37cf0, 0x37cfc, 2002 0x38000, 0x38030, 2003 0x38100, 0x38144, 2004 0x38190, 0x381d0, 2005 0x38200, 0x38318, 2006 0x38400, 0x3852c, 2007 0x38540, 0x3861c, 2008 0x38800, 0x38834, 2009 0x388c0, 0x38908, 2010 0x38910, 0x389ac, 2011 0x38a00, 0x38a04, 2012 0x38a0c, 0x38a2c, 2013 0x38a44, 0x38a50, 2014 0x38a74, 0x38c24, 2015 0x38d08, 0x38d14, 2016 0x38d1c, 0x38d20, 2017 0x38d3c, 0x38d50, 2018 0x39200, 0x3920c, 2019 0x39220, 0x39220, 2020 0x39240, 0x39240, 2021 0x39600, 0x39600, 2022 0x39608, 0x3960c, 2023 0x39a00, 0x39a1c, 2024 0x39e04, 0x39e20, 2025 0x39e38, 0x39e3c, 2026 0x39e80, 0x39e80, 2027 0x39e88, 0x39ea8, 2028 0x39eb0, 0x39eb4, 2029 0x39ec8, 0x39ed4, 2030 0x39fb8, 0x3a004, 2031 0x3a208, 0x3a23c, 2032 0x3a600, 0x3a630, 2033 0x3aa00, 0x3aabc, 2034 0x3ab00, 0x3ab70, 2035 0x3b000, 0x3b048, 2036 0x3b060, 0x3b09c, 2037 0x3b0f0, 0x3b148, 2038 0x3b160, 0x3b19c, 2039 0x3b1f0, 0x3b2e4, 2040 0x3b2f8, 0x3b3e4, 2041 0x3b3f8, 0x3b448, 2042 0x3b460, 0x3b49c, 2043 0x3b4f0, 0x3b548, 2044 0x3b560, 0x3b59c, 2045 0x3b5f0, 0x3b6e4, 2046 0x3b6f8, 0x3b7e4, 2047 0x3b7f8, 0x3b7fc, 2048 0x3b814, 0x3b814, 2049 0x3b82c, 0x3b82c, 2050 0x3b880, 0x3b88c, 2051 0x3b8e8, 0x3b8ec, 2052 0x3b900, 0x3b948, 2053 0x3b960, 0x3b99c, 2054 0x3b9f0, 0x3bae4, 2055 0x3baf8, 0x3bb10, 2056 0x3bb28, 0x3bb28, 2057 0x3bb3c, 0x3bb50, 2058 0x3bbf0, 0x3bc10, 2059 0x3bc28, 0x3bc28, 2060 0x3bc3c, 0x3bc50, 2061 0x3bcf0, 0x3bcfc, 2062 0x3c000, 0x3c030, 2063 0x3c100, 0x3c144, 2064 0x3c190, 0x3c1d0, 2065 0x3c200, 0x3c318, 2066 0x3c400, 0x3c52c, 2067 0x3c540, 0x3c61c, 2068 0x3c800, 0x3c834, 2069 0x3c8c0, 0x3c908, 2070 0x3c910, 0x3c9ac, 2071 0x3ca00, 0x3ca04, 2072 0x3ca0c, 0x3ca2c, 2073 0x3ca44, 0x3ca50, 2074 0x3ca74, 0x3cc24, 2075 0x3cd08, 0x3cd14, 2076 0x3cd1c, 0x3cd20, 2077 0x3cd3c, 0x3cd50, 2078 0x3d200, 0x3d20c, 2079 0x3d220, 0x3d220, 2080 0x3d240, 0x3d240, 2081 0x3d600, 0x3d600, 2082 0x3d608, 0x3d60c, 2083 0x3da00, 0x3da1c, 2084 0x3de04, 0x3de20, 2085 0x3de38, 0x3de3c, 2086 0x3de80, 0x3de80, 2087 0x3de88, 0x3dea8, 2088 0x3deb0, 0x3deb4, 2089 0x3dec8, 0x3ded4, 2090 0x3dfb8, 0x3e004, 2091 0x3e208, 0x3e23c, 2092 0x3e600, 0x3e630, 2093 0x3ea00, 0x3eabc, 2094 0x3eb00, 0x3eb70, 2095 0x3f000, 0x3f048, 2096 0x3f060, 0x3f09c, 2097 0x3f0f0, 0x3f148, 2098 0x3f160, 0x3f19c, 2099 0x3f1f0, 0x3f2e4, 2100 0x3f2f8, 0x3f3e4, 2101 0x3f3f8, 0x3f448, 2102 0x3f460, 0x3f49c, 2103 0x3f4f0, 0x3f548, 2104 0x3f560, 0x3f59c, 2105 0x3f5f0, 0x3f6e4, 2106 0x3f6f8, 0x3f7e4, 2107 0x3f7f8, 0x3f7fc, 2108 0x3f814, 0x3f814, 2109 0x3f82c, 0x3f82c, 2110 0x3f880, 0x3f88c, 2111 0x3f8e8, 0x3f8ec, 2112 0x3f900, 0x3f948, 2113 0x3f960, 0x3f99c, 2114 0x3f9f0, 0x3fae4, 2115 0x3faf8, 0x3fb10, 2116 0x3fb28, 0x3fb28, 2117 0x3fb3c, 0x3fb50, 2118 0x3fbf0, 0x3fc10, 2119 0x3fc28, 0x3fc28, 2120 0x3fc3c, 0x3fc50, 2121 0x3fcf0, 0x3fcfc, 2122 0x40000, 0x4000c, 2123 0x40040, 0x40068, 2124 0x40080, 0x40144, 2125 0x40180, 0x4018c, 2126 0x40200, 0x40298, 2127 0x402ac, 0x4033c, 2128 0x403f8, 0x403fc, 2129 0x41304, 0x413c4, 2130 0x41400, 0x4141c, 2131 0x41480, 0x414d0, 2132 0x44000, 0x44078, 2133 0x440c0, 0x44278, 2134 0x442c0, 0x44478, 2135 0x444c0, 0x44678, 2136 0x446c0, 0x44878, 2137 0x448c0, 0x449fc, 2138 0x45000, 0x45068, 2139 0x45080, 0x45084, 2140 0x450a0, 0x450b0, 2141 0x45200, 0x45268, 2142 0x45280, 0x45284, 2143 0x452a0, 0x452b0, 2144 0x460c0, 0x460e4, 2145 0x47000, 0x4708c, 2146 0x47200, 0x47250, 2147 0x47400, 0x47420, 2148 0x47600, 0x47618, 2149 0x47800, 0x47814, 2150 0x48000, 0x4800c, 2151 0x48040, 0x48068, 2152 0x48080, 0x48144, 2153 0x48180, 0x4818c, 2154 0x48200, 0x48298, 2155 0x482ac, 0x4833c, 2156 0x483f8, 0x483fc, 2157 0x49304, 0x493c4, 2158 0x49400, 0x4941c, 2159 0x49480, 0x494d0, 2160 0x4c000, 0x4c078, 2161 0x4c0c0, 0x4c278, 2162 0x4c2c0, 0x4c478, 2163 0x4c4c0, 0x4c678, 2164 0x4c6c0, 0x4c878, 2165 0x4c8c0, 0x4c9fc, 2166 0x4d000, 0x4d068, 2167 0x4d080, 0x4d084, 2168 0x4d0a0, 0x4d0b0, 2169 0x4d200, 0x4d268, 2170 0x4d280, 0x4d284, 2171 0x4d2a0, 0x4d2b0, 2172 0x4e0c0, 0x4e0e4, 2173 0x4f000, 0x4f08c, 2174 0x4f200, 0x4f250, 2175 0x4f400, 0x4f420, 2176 0x4f600, 0x4f618, 2177 0x4f800, 0x4f814, 2178 0x50000, 0x500cc, 2179 0x50400, 0x50400, 2180 0x50800, 0x508cc, 2181 0x50c00, 0x50c00, 2182 0x51000, 0x5101c, 2183 0x51300, 0x51308, 2184 }; 2185 2186 int i; 2187 struct adapter *ap = netdev2adap(dev); 2188 static const unsigned int *reg_ranges; 2189 int arr_size = 0, buf_size = 0; 2190 2191 if (is_t4(ap->params.chip)) { 2192 reg_ranges = &t4_reg_ranges[0]; 2193 arr_size = ARRAY_SIZE(t4_reg_ranges); 2194 buf_size = T4_REGMAP_SIZE; 2195 } else { 2196 reg_ranges = &t5_reg_ranges[0]; 2197 arr_size = ARRAY_SIZE(t5_reg_ranges); 2198 buf_size = T5_REGMAP_SIZE; 2199 } 2200 2201 regs->version = mk_adap_vers(ap); 2202 2203 memset(buf, 0, buf_size); 2204 for (i = 0; i < arr_size; i += 2) 2205 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]); 2206 } 2207 2208 static int restart_autoneg(struct net_device *dev) 2209 { 2210 struct port_info *p = netdev_priv(dev); 2211 2212 if (!netif_running(dev)) 2213 return -EAGAIN; 2214 if (p->link_cfg.autoneg != AUTONEG_ENABLE) 2215 return -EINVAL; 2216 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan); 2217 return 0; 2218 } 2219 2220 static int identify_port(struct net_device *dev, 2221 enum ethtool_phys_id_state state) 2222 { 2223 unsigned int val; 2224 struct adapter *adap = netdev2adap(dev); 2225 2226 if (state == ETHTOOL_ID_ACTIVE) 2227 val = 0xffff; 2228 else if (state == ETHTOOL_ID_INACTIVE) 2229 val = 0; 2230 else 2231 return -EINVAL; 2232 2233 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val); 2234 } 2235 2236 static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps) 2237 { 2238 unsigned int v = 0; 2239 2240 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI || 2241 type == FW_PORT_TYPE_BT_XAUI) { 2242 v |= SUPPORTED_TP; 2243 if (caps & FW_PORT_CAP_SPEED_100M) 2244 v |= SUPPORTED_100baseT_Full; 2245 if (caps & FW_PORT_CAP_SPEED_1G) 2246 v |= SUPPORTED_1000baseT_Full; 2247 if (caps & FW_PORT_CAP_SPEED_10G) 2248 v |= SUPPORTED_10000baseT_Full; 2249 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) { 2250 v |= SUPPORTED_Backplane; 2251 if (caps & FW_PORT_CAP_SPEED_1G) 2252 v |= SUPPORTED_1000baseKX_Full; 2253 if (caps & FW_PORT_CAP_SPEED_10G) 2254 v |= SUPPORTED_10000baseKX4_Full; 2255 } else if (type == FW_PORT_TYPE_KR) 2256 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full; 2257 else if (type == FW_PORT_TYPE_BP_AP) 2258 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC | 2259 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full; 2260 else if (type == FW_PORT_TYPE_BP4_AP) 2261 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC | 2262 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full | 2263 SUPPORTED_10000baseKX4_Full; 2264 else if (type == FW_PORT_TYPE_FIBER_XFI || 2265 type == FW_PORT_TYPE_FIBER_XAUI || 2266 type == FW_PORT_TYPE_SFP || 2267 type == FW_PORT_TYPE_QSFP_10G || 2268 type == FW_PORT_TYPE_QSA) { 2269 v |= SUPPORTED_FIBRE; 2270 if (caps & FW_PORT_CAP_SPEED_1G) 2271 v |= SUPPORTED_1000baseT_Full; 2272 if (caps & FW_PORT_CAP_SPEED_10G) 2273 v |= SUPPORTED_10000baseT_Full; 2274 } else if (type == FW_PORT_TYPE_BP40_BA || 2275 type == FW_PORT_TYPE_QSFP) { 2276 v |= SUPPORTED_40000baseSR4_Full; 2277 v |= SUPPORTED_FIBRE; 2278 } 2279 2280 if (caps & FW_PORT_CAP_ANEG) 2281 v |= SUPPORTED_Autoneg; 2282 return v; 2283 } 2284 2285 static unsigned int to_fw_linkcaps(unsigned int caps) 2286 { 2287 unsigned int v = 0; 2288 2289 if (caps & ADVERTISED_100baseT_Full) 2290 v |= FW_PORT_CAP_SPEED_100M; 2291 if (caps & ADVERTISED_1000baseT_Full) 2292 v |= FW_PORT_CAP_SPEED_1G; 2293 if (caps & ADVERTISED_10000baseT_Full) 2294 v |= FW_PORT_CAP_SPEED_10G; 2295 if (caps & ADVERTISED_40000baseSR4_Full) 2296 v |= FW_PORT_CAP_SPEED_40G; 2297 return v; 2298 } 2299 2300 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd) 2301 { 2302 const struct port_info *p = netdev_priv(dev); 2303 2304 if (p->port_type == FW_PORT_TYPE_BT_SGMII || 2305 p->port_type == FW_PORT_TYPE_BT_XFI || 2306 p->port_type == FW_PORT_TYPE_BT_XAUI) 2307 cmd->port = PORT_TP; 2308 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI || 2309 p->port_type == FW_PORT_TYPE_FIBER_XAUI) 2310 cmd->port = PORT_FIBRE; 2311 else if (p->port_type == FW_PORT_TYPE_SFP || 2312 p->port_type == FW_PORT_TYPE_QSFP_10G || 2313 p->port_type == FW_PORT_TYPE_QSA || 2314 p->port_type == FW_PORT_TYPE_QSFP) { 2315 if (p->mod_type == FW_PORT_MOD_TYPE_LR || 2316 p->mod_type == FW_PORT_MOD_TYPE_SR || 2317 p->mod_type == FW_PORT_MOD_TYPE_ER || 2318 p->mod_type == FW_PORT_MOD_TYPE_LRM) 2319 cmd->port = PORT_FIBRE; 2320 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE || 2321 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE) 2322 cmd->port = PORT_DA; 2323 else 2324 cmd->port = PORT_OTHER; 2325 } else 2326 cmd->port = PORT_OTHER; 2327 2328 if (p->mdio_addr >= 0) { 2329 cmd->phy_address = p->mdio_addr; 2330 cmd->transceiver = XCVR_EXTERNAL; 2331 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ? 2332 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45; 2333 } else { 2334 cmd->phy_address = 0; /* not really, but no better option */ 2335 cmd->transceiver = XCVR_INTERNAL; 2336 cmd->mdio_support = 0; 2337 } 2338 2339 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported); 2340 cmd->advertising = from_fw_linkcaps(p->port_type, 2341 p->link_cfg.advertising); 2342 ethtool_cmd_speed_set(cmd, 2343 netif_carrier_ok(dev) ? p->link_cfg.speed : 0); 2344 cmd->duplex = DUPLEX_FULL; 2345 cmd->autoneg = p->link_cfg.autoneg; 2346 cmd->maxtxpkt = 0; 2347 cmd->maxrxpkt = 0; 2348 return 0; 2349 } 2350 2351 static unsigned int speed_to_caps(int speed) 2352 { 2353 if (speed == 100) 2354 return FW_PORT_CAP_SPEED_100M; 2355 if (speed == 1000) 2356 return FW_PORT_CAP_SPEED_1G; 2357 if (speed == 10000) 2358 return FW_PORT_CAP_SPEED_10G; 2359 if (speed == 40000) 2360 return FW_PORT_CAP_SPEED_40G; 2361 return 0; 2362 } 2363 2364 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd) 2365 { 2366 unsigned int cap; 2367 struct port_info *p = netdev_priv(dev); 2368 struct link_config *lc = &p->link_cfg; 2369 u32 speed = ethtool_cmd_speed(cmd); 2370 2371 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */ 2372 return -EINVAL; 2373 2374 if (!(lc->supported & FW_PORT_CAP_ANEG)) { 2375 /* 2376 * PHY offers a single speed. See if that's what's 2377 * being requested. 2378 */ 2379 if (cmd->autoneg == AUTONEG_DISABLE && 2380 (lc->supported & speed_to_caps(speed))) 2381 return 0; 2382 return -EINVAL; 2383 } 2384 2385 if (cmd->autoneg == AUTONEG_DISABLE) { 2386 cap = speed_to_caps(speed); 2387 2388 if (!(lc->supported & cap) || 2389 (speed == 1000) || 2390 (speed == 10000) || 2391 (speed == 40000)) 2392 return -EINVAL; 2393 lc->requested_speed = cap; 2394 lc->advertising = 0; 2395 } else { 2396 cap = to_fw_linkcaps(cmd->advertising); 2397 if (!(lc->supported & cap)) 2398 return -EINVAL; 2399 lc->requested_speed = 0; 2400 lc->advertising = cap | FW_PORT_CAP_ANEG; 2401 } 2402 lc->autoneg = cmd->autoneg; 2403 2404 if (netif_running(dev)) 2405 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan, 2406 lc); 2407 return 0; 2408 } 2409 2410 static void get_pauseparam(struct net_device *dev, 2411 struct ethtool_pauseparam *epause) 2412 { 2413 struct port_info *p = netdev_priv(dev); 2414 2415 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0; 2416 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0; 2417 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0; 2418 } 2419 2420 static int set_pauseparam(struct net_device *dev, 2421 struct ethtool_pauseparam *epause) 2422 { 2423 struct port_info *p = netdev_priv(dev); 2424 struct link_config *lc = &p->link_cfg; 2425 2426 if (epause->autoneg == AUTONEG_DISABLE) 2427 lc->requested_fc = 0; 2428 else if (lc->supported & FW_PORT_CAP_ANEG) 2429 lc->requested_fc = PAUSE_AUTONEG; 2430 else 2431 return -EINVAL; 2432 2433 if (epause->rx_pause) 2434 lc->requested_fc |= PAUSE_RX; 2435 if (epause->tx_pause) 2436 lc->requested_fc |= PAUSE_TX; 2437 if (netif_running(dev)) 2438 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan, 2439 lc); 2440 return 0; 2441 } 2442 2443 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) 2444 { 2445 const struct port_info *pi = netdev_priv(dev); 2446 const struct sge *s = &pi->adapter->sge; 2447 2448 e->rx_max_pending = MAX_RX_BUFFERS; 2449 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES; 2450 e->rx_jumbo_max_pending = 0; 2451 e->tx_max_pending = MAX_TXQ_ENTRIES; 2452 2453 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8; 2454 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size; 2455 e->rx_jumbo_pending = 0; 2456 e->tx_pending = s->ethtxq[pi->first_qset].q.size; 2457 } 2458 2459 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e) 2460 { 2461 int i; 2462 const struct port_info *pi = netdev_priv(dev); 2463 struct adapter *adapter = pi->adapter; 2464 struct sge *s = &adapter->sge; 2465 2466 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending || 2467 e->tx_pending > MAX_TXQ_ENTRIES || 2468 e->rx_mini_pending > MAX_RSPQ_ENTRIES || 2469 e->rx_mini_pending < MIN_RSPQ_ENTRIES || 2470 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES) 2471 return -EINVAL; 2472 2473 if (adapter->flags & FULL_INIT_DONE) 2474 return -EBUSY; 2475 2476 for (i = 0; i < pi->nqsets; ++i) { 2477 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending; 2478 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8; 2479 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending; 2480 } 2481 return 0; 2482 } 2483 2484 static int closest_timer(const struct sge *s, int time) 2485 { 2486 int i, delta, match = 0, min_delta = INT_MAX; 2487 2488 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 2489 delta = time - s->timer_val[i]; 2490 if (delta < 0) 2491 delta = -delta; 2492 if (delta < min_delta) { 2493 min_delta = delta; 2494 match = i; 2495 } 2496 } 2497 return match; 2498 } 2499 2500 static int closest_thres(const struct sge *s, int thres) 2501 { 2502 int i, delta, match = 0, min_delta = INT_MAX; 2503 2504 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 2505 delta = thres - s->counter_val[i]; 2506 if (delta < 0) 2507 delta = -delta; 2508 if (delta < min_delta) { 2509 min_delta = delta; 2510 match = i; 2511 } 2512 } 2513 return match; 2514 } 2515 2516 /* 2517 * Return a queue's interrupt hold-off time in us. 0 means no timer. 2518 */ 2519 unsigned int qtimer_val(const struct adapter *adap, 2520 const struct sge_rspq *q) 2521 { 2522 unsigned int idx = q->intr_params >> 1; 2523 2524 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0; 2525 } 2526 2527 /** 2528 * set_rspq_intr_params - set a queue's interrupt holdoff parameters 2529 * @q: the Rx queue 2530 * @us: the hold-off time in us, or 0 to disable timer 2531 * @cnt: the hold-off packet count, or 0 to disable counter 2532 * 2533 * Sets an Rx queue's interrupt hold-off time and packet count. At least 2534 * one of the two needs to be enabled for the queue to generate interrupts. 2535 */ 2536 static int set_rspq_intr_params(struct sge_rspq *q, 2537 unsigned int us, unsigned int cnt) 2538 { 2539 struct adapter *adap = q->adap; 2540 2541 if ((us | cnt) == 0) 2542 cnt = 1; 2543 2544 if (cnt) { 2545 int err; 2546 u32 v, new_idx; 2547 2548 new_idx = closest_thres(&adap->sge, cnt); 2549 if (q->desc && q->pktcnt_idx != new_idx) { 2550 /* the queue has already been created, update it */ 2551 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 2552 FW_PARAMS_PARAM_X_V( 2553 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | 2554 FW_PARAMS_PARAM_YZ_V(q->cntxt_id); 2555 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v, 2556 &new_idx); 2557 if (err) 2558 return err; 2559 } 2560 q->pktcnt_idx = new_idx; 2561 } 2562 2563 us = us == 0 ? 6 : closest_timer(&adap->sge, us); 2564 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0); 2565 return 0; 2566 } 2567 2568 /** 2569 * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete! 2570 * @dev: the network device 2571 * @us: the hold-off time in us, or 0 to disable timer 2572 * @cnt: the hold-off packet count, or 0 to disable counter 2573 * 2574 * Set the RX interrupt hold-off parameters for a network device. 2575 */ 2576 static int set_rx_intr_params(struct net_device *dev, 2577 unsigned int us, unsigned int cnt) 2578 { 2579 int i, err; 2580 struct port_info *pi = netdev_priv(dev); 2581 struct adapter *adap = pi->adapter; 2582 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; 2583 2584 for (i = 0; i < pi->nqsets; i++, q++) { 2585 err = set_rspq_intr_params(&q->rspq, us, cnt); 2586 if (err) 2587 return err; 2588 } 2589 return 0; 2590 } 2591 2592 static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx) 2593 { 2594 int i; 2595 struct port_info *pi = netdev_priv(dev); 2596 struct adapter *adap = pi->adapter; 2597 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; 2598 2599 for (i = 0; i < pi->nqsets; i++, q++) 2600 q->rspq.adaptive_rx = adaptive_rx; 2601 2602 return 0; 2603 } 2604 2605 static int get_adaptive_rx_setting(struct net_device *dev) 2606 { 2607 struct port_info *pi = netdev_priv(dev); 2608 struct adapter *adap = pi->adapter; 2609 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; 2610 2611 return q->rspq.adaptive_rx; 2612 } 2613 2614 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c) 2615 { 2616 set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce); 2617 return set_rx_intr_params(dev, c->rx_coalesce_usecs, 2618 c->rx_max_coalesced_frames); 2619 } 2620 2621 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c) 2622 { 2623 const struct port_info *pi = netdev_priv(dev); 2624 const struct adapter *adap = pi->adapter; 2625 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq; 2626 2627 c->rx_coalesce_usecs = qtimer_val(adap, rq); 2628 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ? 2629 adap->sge.counter_val[rq->pktcnt_idx] : 0; 2630 c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev); 2631 return 0; 2632 } 2633 2634 /** 2635 * eeprom_ptov - translate a physical EEPROM address to virtual 2636 * @phys_addr: the physical EEPROM address 2637 * @fn: the PCI function number 2638 * @sz: size of function-specific area 2639 * 2640 * Translate a physical EEPROM address to virtual. The first 1K is 2641 * accessed through virtual addresses starting at 31K, the rest is 2642 * accessed through virtual addresses starting at 0. 2643 * 2644 * The mapping is as follows: 2645 * [0..1K) -> [31K..32K) 2646 * [1K..1K+A) -> [31K-A..31K) 2647 * [1K+A..ES) -> [0..ES-A-1K) 2648 * 2649 * where A = @fn * @sz, and ES = EEPROM size. 2650 */ 2651 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz) 2652 { 2653 fn *= sz; 2654 if (phys_addr < 1024) 2655 return phys_addr + (31 << 10); 2656 if (phys_addr < 1024 + fn) 2657 return 31744 - fn + phys_addr - 1024; 2658 if (phys_addr < EEPROMSIZE) 2659 return phys_addr - 1024 - fn; 2660 return -EINVAL; 2661 } 2662 2663 /* 2664 * The next two routines implement eeprom read/write from physical addresses. 2665 */ 2666 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v) 2667 { 2668 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE); 2669 2670 if (vaddr >= 0) 2671 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v); 2672 return vaddr < 0 ? vaddr : 0; 2673 } 2674 2675 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v) 2676 { 2677 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE); 2678 2679 if (vaddr >= 0) 2680 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v); 2681 return vaddr < 0 ? vaddr : 0; 2682 } 2683 2684 #define EEPROM_MAGIC 0x38E2F10C 2685 2686 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e, 2687 u8 *data) 2688 { 2689 int i, err = 0; 2690 struct adapter *adapter = netdev2adap(dev); 2691 2692 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL); 2693 if (!buf) 2694 return -ENOMEM; 2695 2696 e->magic = EEPROM_MAGIC; 2697 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4) 2698 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]); 2699 2700 if (!err) 2701 memcpy(data, buf + e->offset, e->len); 2702 kfree(buf); 2703 return err; 2704 } 2705 2706 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, 2707 u8 *data) 2708 { 2709 u8 *buf; 2710 int err = 0; 2711 u32 aligned_offset, aligned_len, *p; 2712 struct adapter *adapter = netdev2adap(dev); 2713 2714 if (eeprom->magic != EEPROM_MAGIC) 2715 return -EINVAL; 2716 2717 aligned_offset = eeprom->offset & ~3; 2718 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3; 2719 2720 if (adapter->fn > 0) { 2721 u32 start = 1024 + adapter->fn * EEPROMPFSIZE; 2722 2723 if (aligned_offset < start || 2724 aligned_offset + aligned_len > start + EEPROMPFSIZE) 2725 return -EPERM; 2726 } 2727 2728 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) { 2729 /* 2730 * RMW possibly needed for first or last words. 2731 */ 2732 buf = kmalloc(aligned_len, GFP_KERNEL); 2733 if (!buf) 2734 return -ENOMEM; 2735 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf); 2736 if (!err && aligned_len > 4) 2737 err = eeprom_rd_phys(adapter, 2738 aligned_offset + aligned_len - 4, 2739 (u32 *)&buf[aligned_len - 4]); 2740 if (err) 2741 goto out; 2742 memcpy(buf + (eeprom->offset & 3), data, eeprom->len); 2743 } else 2744 buf = data; 2745 2746 err = t4_seeprom_wp(adapter, false); 2747 if (err) 2748 goto out; 2749 2750 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2751 err = eeprom_wr_phys(adapter, aligned_offset, *p); 2752 aligned_offset += 4; 2753 } 2754 2755 if (!err) 2756 err = t4_seeprom_wp(adapter, true); 2757 out: 2758 if (buf != data) 2759 kfree(buf); 2760 return err; 2761 } 2762 2763 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef) 2764 { 2765 int ret; 2766 const struct firmware *fw; 2767 struct adapter *adap = netdev2adap(netdev); 2768 unsigned int mbox = PCIE_FW_MASTER_M + 1; 2769 2770 ef->data[sizeof(ef->data) - 1] = '\0'; 2771 ret = request_firmware(&fw, ef->data, adap->pdev_dev); 2772 if (ret < 0) 2773 return ret; 2774 2775 /* If the adapter has been fully initialized then we'll go ahead and 2776 * try to get the firmware's cooperation in upgrading to the new 2777 * firmware image otherwise we'll try to do the entire job from the 2778 * host ... and we always "force" the operation in this path. 2779 */ 2780 if (adap->flags & FULL_INIT_DONE) 2781 mbox = adap->mbox; 2782 2783 ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1); 2784 release_firmware(fw); 2785 if (!ret) 2786 dev_info(adap->pdev_dev, "loaded firmware %s," 2787 " reload cxgb4 driver\n", ef->data); 2788 return ret; 2789 } 2790 2791 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC) 2792 #define BCAST_CRC 0xa0ccc1a6 2793 2794 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) 2795 { 2796 wol->supported = WAKE_BCAST | WAKE_MAGIC; 2797 wol->wolopts = netdev2adap(dev)->wol; 2798 memset(&wol->sopass, 0, sizeof(wol->sopass)); 2799 } 2800 2801 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) 2802 { 2803 int err = 0; 2804 struct port_info *pi = netdev_priv(dev); 2805 2806 if (wol->wolopts & ~WOL_SUPPORTED) 2807 return -EINVAL; 2808 t4_wol_magic_enable(pi->adapter, pi->tx_chan, 2809 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL); 2810 if (wol->wolopts & WAKE_BCAST) { 2811 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL, 2812 ~0ULL, 0, false); 2813 if (!err) 2814 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1, 2815 ~6ULL, ~0ULL, BCAST_CRC, true); 2816 } else 2817 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false); 2818 return err; 2819 } 2820 2821 static int cxgb_set_features(struct net_device *dev, netdev_features_t features) 2822 { 2823 const struct port_info *pi = netdev_priv(dev); 2824 netdev_features_t changed = dev->features ^ features; 2825 int err; 2826 2827 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX)) 2828 return 0; 2829 2830 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1, 2831 -1, -1, -1, 2832 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true); 2833 if (unlikely(err)) 2834 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX; 2835 return err; 2836 } 2837 2838 static u32 get_rss_table_size(struct net_device *dev) 2839 { 2840 const struct port_info *pi = netdev_priv(dev); 2841 2842 return pi->rss_size; 2843 } 2844 2845 static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc) 2846 { 2847 const struct port_info *pi = netdev_priv(dev); 2848 unsigned int n = pi->rss_size; 2849 2850 if (hfunc) 2851 *hfunc = ETH_RSS_HASH_TOP; 2852 if (!p) 2853 return 0; 2854 while (n--) 2855 p[n] = pi->rss[n]; 2856 return 0; 2857 } 2858 2859 static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key, 2860 const u8 hfunc) 2861 { 2862 unsigned int i; 2863 struct port_info *pi = netdev_priv(dev); 2864 2865 /* We require at least one supported parameter to be changed and no 2866 * change in any of the unsupported parameters 2867 */ 2868 if (key || 2869 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) 2870 return -EOPNOTSUPP; 2871 if (!p) 2872 return 0; 2873 2874 for (i = 0; i < pi->rss_size; i++) 2875 pi->rss[i] = p[i]; 2876 if (pi->adapter->flags & FULL_INIT_DONE) 2877 return write_rss(pi, pi->rss); 2878 return 0; 2879 } 2880 2881 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, 2882 u32 *rules) 2883 { 2884 const struct port_info *pi = netdev_priv(dev); 2885 2886 switch (info->cmd) { 2887 case ETHTOOL_GRXFH: { 2888 unsigned int v = pi->rss_mode; 2889 2890 info->data = 0; 2891 switch (info->flow_type) { 2892 case TCP_V4_FLOW: 2893 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) 2894 info->data = RXH_IP_SRC | RXH_IP_DST | 2895 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2896 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) 2897 info->data = RXH_IP_SRC | RXH_IP_DST; 2898 break; 2899 case UDP_V4_FLOW: 2900 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) && 2901 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F)) 2902 info->data = RXH_IP_SRC | RXH_IP_DST | 2903 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2904 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) 2905 info->data = RXH_IP_SRC | RXH_IP_DST; 2906 break; 2907 case SCTP_V4_FLOW: 2908 case AH_ESP_V4_FLOW: 2909 case IPV4_FLOW: 2910 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) 2911 info->data = RXH_IP_SRC | RXH_IP_DST; 2912 break; 2913 case TCP_V6_FLOW: 2914 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) 2915 info->data = RXH_IP_SRC | RXH_IP_DST | 2916 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2917 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) 2918 info->data = RXH_IP_SRC | RXH_IP_DST; 2919 break; 2920 case UDP_V6_FLOW: 2921 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) && 2922 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F)) 2923 info->data = RXH_IP_SRC | RXH_IP_DST | 2924 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2925 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) 2926 info->data = RXH_IP_SRC | RXH_IP_DST; 2927 break; 2928 case SCTP_V6_FLOW: 2929 case AH_ESP_V6_FLOW: 2930 case IPV6_FLOW: 2931 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) 2932 info->data = RXH_IP_SRC | RXH_IP_DST; 2933 break; 2934 } 2935 return 0; 2936 } 2937 case ETHTOOL_GRXRINGS: 2938 info->data = pi->nqsets; 2939 return 0; 2940 } 2941 return -EOPNOTSUPP; 2942 } 2943 2944 static const struct ethtool_ops cxgb_ethtool_ops = { 2945 .get_settings = get_settings, 2946 .set_settings = set_settings, 2947 .get_drvinfo = get_drvinfo, 2948 .get_msglevel = get_msglevel, 2949 .set_msglevel = set_msglevel, 2950 .get_ringparam = get_sge_param, 2951 .set_ringparam = set_sge_param, 2952 .get_coalesce = get_coalesce, 2953 .set_coalesce = set_coalesce, 2954 .get_eeprom_len = get_eeprom_len, 2955 .get_eeprom = get_eeprom, 2956 .set_eeprom = set_eeprom, 2957 .get_pauseparam = get_pauseparam, 2958 .set_pauseparam = set_pauseparam, 2959 .get_link = ethtool_op_get_link, 2960 .get_strings = get_strings, 2961 .set_phys_id = identify_port, 2962 .nway_reset = restart_autoneg, 2963 .get_sset_count = get_sset_count, 2964 .get_ethtool_stats = get_stats, 2965 .get_regs_len = get_regs_len, 2966 .get_regs = get_regs, 2967 .get_wol = get_wol, 2968 .set_wol = set_wol, 2969 .get_rxnfc = get_rxnfc, 2970 .get_rxfh_indir_size = get_rss_table_size, 2971 .get_rxfh = get_rss_table, 2972 .set_rxfh = set_rss_table, 2973 .flash_device = set_flash, 2974 }; 2975 2976 static int setup_debugfs(struct adapter *adap) 2977 { 2978 if (IS_ERR_OR_NULL(adap->debugfs_root)) 2979 return -1; 2980 2981 #ifdef CONFIG_DEBUG_FS 2982 t4_setup_debugfs(adap); 2983 #endif 2984 return 0; 2985 } 2986 2987 /* 2988 * upper-layer driver support 2989 */ 2990 2991 /* 2992 * Allocate an active-open TID and set it to the supplied value. 2993 */ 2994 int cxgb4_alloc_atid(struct tid_info *t, void *data) 2995 { 2996 int atid = -1; 2997 2998 spin_lock_bh(&t->atid_lock); 2999 if (t->afree) { 3000 union aopen_entry *p = t->afree; 3001 3002 atid = (p - t->atid_tab) + t->atid_base; 3003 t->afree = p->next; 3004 p->data = data; 3005 t->atids_in_use++; 3006 } 3007 spin_unlock_bh(&t->atid_lock); 3008 return atid; 3009 } 3010 EXPORT_SYMBOL(cxgb4_alloc_atid); 3011 3012 /* 3013 * Release an active-open TID. 3014 */ 3015 void cxgb4_free_atid(struct tid_info *t, unsigned int atid) 3016 { 3017 union aopen_entry *p = &t->atid_tab[atid - t->atid_base]; 3018 3019 spin_lock_bh(&t->atid_lock); 3020 p->next = t->afree; 3021 t->afree = p; 3022 t->atids_in_use--; 3023 spin_unlock_bh(&t->atid_lock); 3024 } 3025 EXPORT_SYMBOL(cxgb4_free_atid); 3026 3027 /* 3028 * Allocate a server TID and set it to the supplied value. 3029 */ 3030 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data) 3031 { 3032 int stid; 3033 3034 spin_lock_bh(&t->stid_lock); 3035 if (family == PF_INET) { 3036 stid = find_first_zero_bit(t->stid_bmap, t->nstids); 3037 if (stid < t->nstids) 3038 __set_bit(stid, t->stid_bmap); 3039 else 3040 stid = -1; 3041 } else { 3042 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2); 3043 if (stid < 0) 3044 stid = -1; 3045 } 3046 if (stid >= 0) { 3047 t->stid_tab[stid].data = data; 3048 stid += t->stid_base; 3049 /* IPv6 requires max of 520 bits or 16 cells in TCAM 3050 * This is equivalent to 4 TIDs. With CLIP enabled it 3051 * needs 2 TIDs. 3052 */ 3053 if (family == PF_INET) 3054 t->stids_in_use++; 3055 else 3056 t->stids_in_use += 4; 3057 } 3058 spin_unlock_bh(&t->stid_lock); 3059 return stid; 3060 } 3061 EXPORT_SYMBOL(cxgb4_alloc_stid); 3062 3063 /* Allocate a server filter TID and set it to the supplied value. 3064 */ 3065 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data) 3066 { 3067 int stid; 3068 3069 spin_lock_bh(&t->stid_lock); 3070 if (family == PF_INET) { 3071 stid = find_next_zero_bit(t->stid_bmap, 3072 t->nstids + t->nsftids, t->nstids); 3073 if (stid < (t->nstids + t->nsftids)) 3074 __set_bit(stid, t->stid_bmap); 3075 else 3076 stid = -1; 3077 } else { 3078 stid = -1; 3079 } 3080 if (stid >= 0) { 3081 t->stid_tab[stid].data = data; 3082 stid -= t->nstids; 3083 stid += t->sftid_base; 3084 t->stids_in_use++; 3085 } 3086 spin_unlock_bh(&t->stid_lock); 3087 return stid; 3088 } 3089 EXPORT_SYMBOL(cxgb4_alloc_sftid); 3090 3091 /* Release a server TID. 3092 */ 3093 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family) 3094 { 3095 /* Is it a server filter TID? */ 3096 if (t->nsftids && (stid >= t->sftid_base)) { 3097 stid -= t->sftid_base; 3098 stid += t->nstids; 3099 } else { 3100 stid -= t->stid_base; 3101 } 3102 3103 spin_lock_bh(&t->stid_lock); 3104 if (family == PF_INET) 3105 __clear_bit(stid, t->stid_bmap); 3106 else 3107 bitmap_release_region(t->stid_bmap, stid, 2); 3108 t->stid_tab[stid].data = NULL; 3109 if (family == PF_INET) 3110 t->stids_in_use--; 3111 else 3112 t->stids_in_use -= 4; 3113 spin_unlock_bh(&t->stid_lock); 3114 } 3115 EXPORT_SYMBOL(cxgb4_free_stid); 3116 3117 /* 3118 * Populate a TID_RELEASE WR. Caller must properly size the skb. 3119 */ 3120 static void mk_tid_release(struct sk_buff *skb, unsigned int chan, 3121 unsigned int tid) 3122 { 3123 struct cpl_tid_release *req; 3124 3125 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); 3126 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req)); 3127 INIT_TP_WR(req, tid); 3128 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 3129 } 3130 3131 /* 3132 * Queue a TID release request and if necessary schedule a work queue to 3133 * process it. 3134 */ 3135 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, 3136 unsigned int tid) 3137 { 3138 void **p = &t->tid_tab[tid]; 3139 struct adapter *adap = container_of(t, struct adapter, tids); 3140 3141 spin_lock_bh(&adap->tid_release_lock); 3142 *p = adap->tid_release_head; 3143 /* Low 2 bits encode the Tx channel number */ 3144 adap->tid_release_head = (void **)((uintptr_t)p | chan); 3145 if (!adap->tid_release_task_busy) { 3146 adap->tid_release_task_busy = true; 3147 queue_work(adap->workq, &adap->tid_release_task); 3148 } 3149 spin_unlock_bh(&adap->tid_release_lock); 3150 } 3151 3152 /* 3153 * Process the list of pending TID release requests. 3154 */ 3155 static void process_tid_release_list(struct work_struct *work) 3156 { 3157 struct sk_buff *skb; 3158 struct adapter *adap; 3159 3160 adap = container_of(work, struct adapter, tid_release_task); 3161 3162 spin_lock_bh(&adap->tid_release_lock); 3163 while (adap->tid_release_head) { 3164 void **p = adap->tid_release_head; 3165 unsigned int chan = (uintptr_t)p & 3; 3166 p = (void *)p - chan; 3167 3168 adap->tid_release_head = *p; 3169 *p = NULL; 3170 spin_unlock_bh(&adap->tid_release_lock); 3171 3172 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release), 3173 GFP_KERNEL))) 3174 schedule_timeout_uninterruptible(1); 3175 3176 mk_tid_release(skb, chan, p - adap->tids.tid_tab); 3177 t4_ofld_send(adap, skb); 3178 spin_lock_bh(&adap->tid_release_lock); 3179 } 3180 adap->tid_release_task_busy = false; 3181 spin_unlock_bh(&adap->tid_release_lock); 3182 } 3183 3184 /* 3185 * Release a TID and inform HW. If we are unable to allocate the release 3186 * message we defer to a work queue. 3187 */ 3188 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid) 3189 { 3190 void *old; 3191 struct sk_buff *skb; 3192 struct adapter *adap = container_of(t, struct adapter, tids); 3193 3194 old = t->tid_tab[tid]; 3195 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); 3196 if (likely(skb)) { 3197 t->tid_tab[tid] = NULL; 3198 mk_tid_release(skb, chan, tid); 3199 t4_ofld_send(adap, skb); 3200 } else 3201 cxgb4_queue_tid_release(t, chan, tid); 3202 if (old) 3203 atomic_dec(&t->tids_in_use); 3204 } 3205 EXPORT_SYMBOL(cxgb4_remove_tid); 3206 3207 /* 3208 * Allocate and initialize the TID tables. Returns 0 on success. 3209 */ 3210 static int tid_init(struct tid_info *t) 3211 { 3212 size_t size; 3213 unsigned int stid_bmap_size; 3214 unsigned int natids = t->natids; 3215 struct adapter *adap = container_of(t, struct adapter, tids); 3216 3217 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids); 3218 size = t->ntids * sizeof(*t->tid_tab) + 3219 natids * sizeof(*t->atid_tab) + 3220 t->nstids * sizeof(*t->stid_tab) + 3221 t->nsftids * sizeof(*t->stid_tab) + 3222 stid_bmap_size * sizeof(long) + 3223 t->nftids * sizeof(*t->ftid_tab) + 3224 t->nsftids * sizeof(*t->ftid_tab); 3225 3226 t->tid_tab = t4_alloc_mem(size); 3227 if (!t->tid_tab) 3228 return -ENOMEM; 3229 3230 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 3231 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids]; 3232 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids]; 3233 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size]; 3234 spin_lock_init(&t->stid_lock); 3235 spin_lock_init(&t->atid_lock); 3236 3237 t->stids_in_use = 0; 3238 t->afree = NULL; 3239 t->atids_in_use = 0; 3240 atomic_set(&t->tids_in_use, 0); 3241 3242 /* Setup the free list for atid_tab and clear the stid bitmap. */ 3243 if (natids) { 3244 while (--natids) 3245 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 3246 t->afree = t->atid_tab; 3247 } 3248 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids); 3249 /* Reserve stid 0 for T4/T5 adapters */ 3250 if (!t->stid_base && 3251 (is_t4(adap->params.chip) || is_t5(adap->params.chip))) 3252 __set_bit(0, t->stid_bmap); 3253 3254 return 0; 3255 } 3256 3257 /** 3258 * cxgb4_create_server - create an IP server 3259 * @dev: the device 3260 * @stid: the server TID 3261 * @sip: local IP address to bind server to 3262 * @sport: the server's TCP port 3263 * @queue: queue to direct messages from this server to 3264 * 3265 * Create an IP server for the given port and address. 3266 * Returns <0 on error and one of the %NET_XMIT_* values on success. 3267 */ 3268 int cxgb4_create_server(const struct net_device *dev, unsigned int stid, 3269 __be32 sip, __be16 sport, __be16 vlan, 3270 unsigned int queue) 3271 { 3272 unsigned int chan; 3273 struct sk_buff *skb; 3274 struct adapter *adap; 3275 struct cpl_pass_open_req *req; 3276 int ret; 3277 3278 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 3279 if (!skb) 3280 return -ENOMEM; 3281 3282 adap = netdev2adap(dev); 3283 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req)); 3284 INIT_TP_WR(req, 0); 3285 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid)); 3286 req->local_port = sport; 3287 req->peer_port = htons(0); 3288 req->local_ip = sip; 3289 req->peer_ip = htonl(0); 3290 chan = rxq_to_chan(&adap->sge, queue); 3291 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 3292 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 3293 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 3294 ret = t4_mgmt_tx(adap, skb); 3295 return net_xmit_eval(ret); 3296 } 3297 EXPORT_SYMBOL(cxgb4_create_server); 3298 3299 /* cxgb4_create_server6 - create an IPv6 server 3300 * @dev: the device 3301 * @stid: the server TID 3302 * @sip: local IPv6 address to bind server to 3303 * @sport: the server's TCP port 3304 * @queue: queue to direct messages from this server to 3305 * 3306 * Create an IPv6 server for the given port and address. 3307 * Returns <0 on error and one of the %NET_XMIT_* values on success. 3308 */ 3309 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, 3310 const struct in6_addr *sip, __be16 sport, 3311 unsigned int queue) 3312 { 3313 unsigned int chan; 3314 struct sk_buff *skb; 3315 struct adapter *adap; 3316 struct cpl_pass_open_req6 *req; 3317 int ret; 3318 3319 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 3320 if (!skb) 3321 return -ENOMEM; 3322 3323 adap = netdev2adap(dev); 3324 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req)); 3325 INIT_TP_WR(req, 0); 3326 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid)); 3327 req->local_port = sport; 3328 req->peer_port = htons(0); 3329 req->local_ip_hi = *(__be64 *)(sip->s6_addr); 3330 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8); 3331 req->peer_ip_hi = cpu_to_be64(0); 3332 req->peer_ip_lo = cpu_to_be64(0); 3333 chan = rxq_to_chan(&adap->sge, queue); 3334 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 3335 req->opt1 = cpu_to_be64(CONN_POLICY_V(CPL_CONN_POLICY_ASK) | 3336 SYN_RSS_ENABLE_F | SYN_RSS_QUEUE_V(queue)); 3337 ret = t4_mgmt_tx(adap, skb); 3338 return net_xmit_eval(ret); 3339 } 3340 EXPORT_SYMBOL(cxgb4_create_server6); 3341 3342 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, 3343 unsigned int queue, bool ipv6) 3344 { 3345 struct sk_buff *skb; 3346 struct adapter *adap; 3347 struct cpl_close_listsvr_req *req; 3348 int ret; 3349 3350 adap = netdev2adap(dev); 3351 3352 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 3353 if (!skb) 3354 return -ENOMEM; 3355 3356 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req)); 3357 INIT_TP_WR(req, 0); 3358 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid)); 3359 req->reply_ctrl = htons(NO_REPLY_V(0) | (ipv6 ? LISTSVR_IPV6_V(1) : 3360 LISTSVR_IPV6_V(0)) | QUEUENO_V(queue)); 3361 ret = t4_mgmt_tx(adap, skb); 3362 return net_xmit_eval(ret); 3363 } 3364 EXPORT_SYMBOL(cxgb4_remove_server); 3365 3366 /** 3367 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU 3368 * @mtus: the HW MTU table 3369 * @mtu: the target MTU 3370 * @idx: index of selected entry in the MTU table 3371 * 3372 * Returns the index and the value in the HW MTU table that is closest to 3373 * but does not exceed @mtu, unless @mtu is smaller than any value in the 3374 * table, in which case that smallest available value is selected. 3375 */ 3376 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, 3377 unsigned int *idx) 3378 { 3379 unsigned int i = 0; 3380 3381 while (i < NMTUS - 1 && mtus[i + 1] <= mtu) 3382 ++i; 3383 if (idx) 3384 *idx = i; 3385 return mtus[i]; 3386 } 3387 EXPORT_SYMBOL(cxgb4_best_mtu); 3388 3389 /** 3390 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned 3391 * @mtus: the HW MTU table 3392 * @header_size: Header Size 3393 * @data_size_max: maximum Data Segment Size 3394 * @data_size_align: desired Data Segment Size Alignment (2^N) 3395 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL) 3396 * 3397 * Similar to cxgb4_best_mtu() but instead of searching the Hardware 3398 * MTU Table based solely on a Maximum MTU parameter, we break that 3399 * parameter up into a Header Size and Maximum Data Segment Size, and 3400 * provide a desired Data Segment Size Alignment. If we find an MTU in 3401 * the Hardware MTU Table which will result in a Data Segment Size with 3402 * the requested alignment _and_ that MTU isn't "too far" from the 3403 * closest MTU, then we'll return that rather than the closest MTU. 3404 */ 3405 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, 3406 unsigned short header_size, 3407 unsigned short data_size_max, 3408 unsigned short data_size_align, 3409 unsigned int *mtu_idxp) 3410 { 3411 unsigned short max_mtu = header_size + data_size_max; 3412 unsigned short data_size_align_mask = data_size_align - 1; 3413 int mtu_idx, aligned_mtu_idx; 3414 3415 /* Scan the MTU Table till we find an MTU which is larger than our 3416 * Maximum MTU or we reach the end of the table. Along the way, 3417 * record the last MTU found, if any, which will result in a Data 3418 * Segment Length matching the requested alignment. 3419 */ 3420 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) { 3421 unsigned short data_size = mtus[mtu_idx] - header_size; 3422 3423 /* If this MTU minus the Header Size would result in a 3424 * Data Segment Size of the desired alignment, remember it. 3425 */ 3426 if ((data_size & data_size_align_mask) == 0) 3427 aligned_mtu_idx = mtu_idx; 3428 3429 /* If we're not at the end of the Hardware MTU Table and the 3430 * next element is larger than our Maximum MTU, drop out of 3431 * the loop. 3432 */ 3433 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu) 3434 break; 3435 } 3436 3437 /* If we fell out of the loop because we ran to the end of the table, 3438 * then we just have to use the last [largest] entry. 3439 */ 3440 if (mtu_idx == NMTUS) 3441 mtu_idx--; 3442 3443 /* If we found an MTU which resulted in the requested Data Segment 3444 * Length alignment and that's "not far" from the largest MTU which is 3445 * less than or equal to the maximum MTU, then use that. 3446 */ 3447 if (aligned_mtu_idx >= 0 && 3448 mtu_idx - aligned_mtu_idx <= 1) 3449 mtu_idx = aligned_mtu_idx; 3450 3451 /* If the caller has passed in an MTU Index pointer, pass the 3452 * MTU Index back. Return the MTU value. 3453 */ 3454 if (mtu_idxp) 3455 *mtu_idxp = mtu_idx; 3456 return mtus[mtu_idx]; 3457 } 3458 EXPORT_SYMBOL(cxgb4_best_aligned_mtu); 3459 3460 /** 3461 * cxgb4_port_chan - get the HW channel of a port 3462 * @dev: the net device for the port 3463 * 3464 * Return the HW Tx channel of the given port. 3465 */ 3466 unsigned int cxgb4_port_chan(const struct net_device *dev) 3467 { 3468 return netdev2pinfo(dev)->tx_chan; 3469 } 3470 EXPORT_SYMBOL(cxgb4_port_chan); 3471 3472 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo) 3473 { 3474 struct adapter *adap = netdev2adap(dev); 3475 u32 v1, v2, lp_count, hp_count; 3476 3477 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 3478 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 3479 if (is_t4(adap->params.chip)) { 3480 lp_count = LP_COUNT_G(v1); 3481 hp_count = HP_COUNT_G(v1); 3482 } else { 3483 lp_count = LP_COUNT_T5_G(v1); 3484 hp_count = HP_COUNT_T5_G(v2); 3485 } 3486 return lpfifo ? lp_count : hp_count; 3487 } 3488 EXPORT_SYMBOL(cxgb4_dbfifo_count); 3489 3490 /** 3491 * cxgb4_port_viid - get the VI id of a port 3492 * @dev: the net device for the port 3493 * 3494 * Return the VI id of the given port. 3495 */ 3496 unsigned int cxgb4_port_viid(const struct net_device *dev) 3497 { 3498 return netdev2pinfo(dev)->viid; 3499 } 3500 EXPORT_SYMBOL(cxgb4_port_viid); 3501 3502 /** 3503 * cxgb4_port_idx - get the index of a port 3504 * @dev: the net device for the port 3505 * 3506 * Return the index of the given port. 3507 */ 3508 unsigned int cxgb4_port_idx(const struct net_device *dev) 3509 { 3510 return netdev2pinfo(dev)->port_id; 3511 } 3512 EXPORT_SYMBOL(cxgb4_port_idx); 3513 3514 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, 3515 struct tp_tcp_stats *v6) 3516 { 3517 struct adapter *adap = pci_get_drvdata(pdev); 3518 3519 spin_lock(&adap->stats_lock); 3520 t4_tp_get_tcp_stats(adap, v4, v6); 3521 spin_unlock(&adap->stats_lock); 3522 } 3523 EXPORT_SYMBOL(cxgb4_get_tcp_stats); 3524 3525 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, 3526 const unsigned int *pgsz_order) 3527 { 3528 struct adapter *adap = netdev2adap(dev); 3529 3530 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK_A, tag_mask); 3531 t4_write_reg(adap, ULP_RX_ISCSI_PSZ_A, HPZ0_V(pgsz_order[0]) | 3532 HPZ1_V(pgsz_order[1]) | HPZ2_V(pgsz_order[2]) | 3533 HPZ3_V(pgsz_order[3])); 3534 } 3535 EXPORT_SYMBOL(cxgb4_iscsi_init); 3536 3537 int cxgb4_flush_eq_cache(struct net_device *dev) 3538 { 3539 struct adapter *adap = netdev2adap(dev); 3540 int ret; 3541 3542 ret = t4_fwaddrspace_write(adap, adap->mbox, 3543 0xe1000000 + SGE_CTXT_CMD_A, 0x20000000); 3544 return ret; 3545 } 3546 EXPORT_SYMBOL(cxgb4_flush_eq_cache); 3547 3548 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) 3549 { 3550 u32 addr = t4_read_reg(adap, SGE_DBQ_CTXT_BADDR_A) + 24 * qid + 8; 3551 __be64 indices; 3552 int ret; 3553 3554 spin_lock(&adap->win0_lock); 3555 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr, 3556 sizeof(indices), (__be32 *)&indices, 3557 T4_MEMORY_READ); 3558 spin_unlock(&adap->win0_lock); 3559 if (!ret) { 3560 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff; 3561 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff; 3562 } 3563 return ret; 3564 } 3565 3566 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, 3567 u16 size) 3568 { 3569 struct adapter *adap = netdev2adap(dev); 3570 u16 hw_pidx, hw_cidx; 3571 int ret; 3572 3573 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); 3574 if (ret) 3575 goto out; 3576 3577 if (pidx != hw_pidx) { 3578 u16 delta; 3579 u32 val; 3580 3581 if (pidx >= hw_pidx) 3582 delta = pidx - hw_pidx; 3583 else 3584 delta = size - hw_pidx + pidx; 3585 3586 if (is_t4(adap->params.chip)) 3587 val = PIDX_V(delta); 3588 else 3589 val = PIDX_T5_V(delta); 3590 wmb(); 3591 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 3592 QID_V(qid) | val); 3593 } 3594 out: 3595 return ret; 3596 } 3597 EXPORT_SYMBOL(cxgb4_sync_txq_pidx); 3598 3599 void cxgb4_disable_db_coalescing(struct net_device *dev) 3600 { 3601 struct adapter *adap; 3602 3603 adap = netdev2adap(dev); 3604 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 3605 NOCOALESCE_F); 3606 } 3607 EXPORT_SYMBOL(cxgb4_disable_db_coalescing); 3608 3609 void cxgb4_enable_db_coalescing(struct net_device *dev) 3610 { 3611 struct adapter *adap; 3612 3613 adap = netdev2adap(dev); 3614 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, NOCOALESCE_F, 0); 3615 } 3616 EXPORT_SYMBOL(cxgb4_enable_db_coalescing); 3617 3618 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte) 3619 { 3620 struct adapter *adap; 3621 u32 offset, memtype, memaddr; 3622 u32 edc0_size, edc1_size, mc0_size, mc1_size, size; 3623 u32 edc0_end, edc1_end, mc0_end, mc1_end; 3624 int ret; 3625 3626 adap = netdev2adap(dev); 3627 3628 offset = ((stag >> 8) * 32) + adap->vres.stag.start; 3629 3630 /* Figure out where the offset lands in the Memory Type/Address scheme. 3631 * This code assumes that the memory is laid out starting at offset 0 3632 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0 3633 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have 3634 * MC0, and some have both MC0 and MC1. 3635 */ 3636 size = t4_read_reg(adap, MA_EDRAM0_BAR_A); 3637 edc0_size = EDRAM0_SIZE_G(size) << 20; 3638 size = t4_read_reg(adap, MA_EDRAM1_BAR_A); 3639 edc1_size = EDRAM1_SIZE_G(size) << 20; 3640 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); 3641 mc0_size = EXT_MEM0_SIZE_G(size) << 20; 3642 3643 edc0_end = edc0_size; 3644 edc1_end = edc0_end + edc1_size; 3645 mc0_end = edc1_end + mc0_size; 3646 3647 if (offset < edc0_end) { 3648 memtype = MEM_EDC0; 3649 memaddr = offset; 3650 } else if (offset < edc1_end) { 3651 memtype = MEM_EDC1; 3652 memaddr = offset - edc0_end; 3653 } else { 3654 if (offset < mc0_end) { 3655 memtype = MEM_MC0; 3656 memaddr = offset - edc1_end; 3657 } else if (is_t4(adap->params.chip)) { 3658 /* T4 only has a single memory channel */ 3659 goto err; 3660 } else { 3661 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); 3662 mc1_size = EXT_MEM1_SIZE_G(size) << 20; 3663 mc1_end = mc0_end + mc1_size; 3664 if (offset < mc1_end) { 3665 memtype = MEM_MC1; 3666 memaddr = offset - mc0_end; 3667 } else { 3668 /* offset beyond the end of any memory */ 3669 goto err; 3670 } 3671 } 3672 } 3673 3674 spin_lock(&adap->win0_lock); 3675 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ); 3676 spin_unlock(&adap->win0_lock); 3677 return ret; 3678 3679 err: 3680 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n", 3681 stag, offset); 3682 return -EINVAL; 3683 } 3684 EXPORT_SYMBOL(cxgb4_read_tpte); 3685 3686 u64 cxgb4_read_sge_timestamp(struct net_device *dev) 3687 { 3688 u32 hi, lo; 3689 struct adapter *adap; 3690 3691 adap = netdev2adap(dev); 3692 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO_A); 3693 hi = TSVAL_G(t4_read_reg(adap, SGE_TIMESTAMP_HI_A)); 3694 3695 return ((u64)hi << 32) | (u64)lo; 3696 } 3697 EXPORT_SYMBOL(cxgb4_read_sge_timestamp); 3698 3699 int cxgb4_bar2_sge_qregs(struct net_device *dev, 3700 unsigned int qid, 3701 enum cxgb4_bar2_qtype qtype, 3702 u64 *pbar2_qoffset, 3703 unsigned int *pbar2_qid) 3704 { 3705 return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev), 3706 qid, 3707 (qtype == CXGB4_BAR2_QTYPE_EGRESS 3708 ? T4_BAR2_QTYPE_EGRESS 3709 : T4_BAR2_QTYPE_INGRESS), 3710 pbar2_qoffset, 3711 pbar2_qid); 3712 } 3713 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs); 3714 3715 static struct pci_driver cxgb4_driver; 3716 3717 static void check_neigh_update(struct neighbour *neigh) 3718 { 3719 const struct device *parent; 3720 const struct net_device *netdev = neigh->dev; 3721 3722 if (netdev->priv_flags & IFF_802_1Q_VLAN) 3723 netdev = vlan_dev_real_dev(netdev); 3724 parent = netdev->dev.parent; 3725 if (parent && parent->driver == &cxgb4_driver.driver) 3726 t4_l2t_update(dev_get_drvdata(parent), neigh); 3727 } 3728 3729 static int netevent_cb(struct notifier_block *nb, unsigned long event, 3730 void *data) 3731 { 3732 switch (event) { 3733 case NETEVENT_NEIGH_UPDATE: 3734 check_neigh_update(data); 3735 break; 3736 case NETEVENT_REDIRECT: 3737 default: 3738 break; 3739 } 3740 return 0; 3741 } 3742 3743 static bool netevent_registered; 3744 static struct notifier_block cxgb4_netevent_nb = { 3745 .notifier_call = netevent_cb 3746 }; 3747 3748 static void drain_db_fifo(struct adapter *adap, int usecs) 3749 { 3750 u32 v1, v2, lp_count, hp_count; 3751 3752 do { 3753 v1 = t4_read_reg(adap, SGE_DBFIFO_STATUS_A); 3754 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2_A); 3755 if (is_t4(adap->params.chip)) { 3756 lp_count = LP_COUNT_G(v1); 3757 hp_count = HP_COUNT_G(v1); 3758 } else { 3759 lp_count = LP_COUNT_T5_G(v1); 3760 hp_count = HP_COUNT_T5_G(v2); 3761 } 3762 3763 if (lp_count == 0 && hp_count == 0) 3764 break; 3765 set_current_state(TASK_UNINTERRUPTIBLE); 3766 schedule_timeout(usecs_to_jiffies(usecs)); 3767 } while (1); 3768 } 3769 3770 static void disable_txq_db(struct sge_txq *q) 3771 { 3772 unsigned long flags; 3773 3774 spin_lock_irqsave(&q->db_lock, flags); 3775 q->db_disabled = 1; 3776 spin_unlock_irqrestore(&q->db_lock, flags); 3777 } 3778 3779 static void enable_txq_db(struct adapter *adap, struct sge_txq *q) 3780 { 3781 spin_lock_irq(&q->db_lock); 3782 if (q->db_pidx_inc) { 3783 /* Make sure that all writes to the TX descriptors 3784 * are committed before we tell HW about them. 3785 */ 3786 wmb(); 3787 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 3788 QID_V(q->cntxt_id) | PIDX_V(q->db_pidx_inc)); 3789 q->db_pidx_inc = 0; 3790 } 3791 q->db_disabled = 0; 3792 spin_unlock_irq(&q->db_lock); 3793 } 3794 3795 static void disable_dbs(struct adapter *adap) 3796 { 3797 int i; 3798 3799 for_each_ethrxq(&adap->sge, i) 3800 disable_txq_db(&adap->sge.ethtxq[i].q); 3801 for_each_ofldrxq(&adap->sge, i) 3802 disable_txq_db(&adap->sge.ofldtxq[i].q); 3803 for_each_port(adap, i) 3804 disable_txq_db(&adap->sge.ctrlq[i].q); 3805 } 3806 3807 static void enable_dbs(struct adapter *adap) 3808 { 3809 int i; 3810 3811 for_each_ethrxq(&adap->sge, i) 3812 enable_txq_db(adap, &adap->sge.ethtxq[i].q); 3813 for_each_ofldrxq(&adap->sge, i) 3814 enable_txq_db(adap, &adap->sge.ofldtxq[i].q); 3815 for_each_port(adap, i) 3816 enable_txq_db(adap, &adap->sge.ctrlq[i].q); 3817 } 3818 3819 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) 3820 { 3821 if (adap->uld_handle[CXGB4_ULD_RDMA]) 3822 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA], 3823 cmd); 3824 } 3825 3826 static void process_db_full(struct work_struct *work) 3827 { 3828 struct adapter *adap; 3829 3830 adap = container_of(work, struct adapter, db_full_task); 3831 3832 drain_db_fifo(adap, dbfifo_drain_delay); 3833 enable_dbs(adap); 3834 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 3835 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 3836 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 3837 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F); 3838 } 3839 3840 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) 3841 { 3842 u16 hw_pidx, hw_cidx; 3843 int ret; 3844 3845 spin_lock_irq(&q->db_lock); 3846 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); 3847 if (ret) 3848 goto out; 3849 if (q->db_pidx != hw_pidx) { 3850 u16 delta; 3851 u32 val; 3852 3853 if (q->db_pidx >= hw_pidx) 3854 delta = q->db_pidx - hw_pidx; 3855 else 3856 delta = q->size - hw_pidx + q->db_pidx; 3857 3858 if (is_t4(adap->params.chip)) 3859 val = PIDX_V(delta); 3860 else 3861 val = PIDX_T5_V(delta); 3862 wmb(); 3863 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL_A), 3864 QID_V(q->cntxt_id) | val); 3865 } 3866 out: 3867 q->db_disabled = 0; 3868 q->db_pidx_inc = 0; 3869 spin_unlock_irq(&q->db_lock); 3870 if (ret) 3871 CH_WARN(adap, "DB drop recovery failed.\n"); 3872 } 3873 static void recover_all_queues(struct adapter *adap) 3874 { 3875 int i; 3876 3877 for_each_ethrxq(&adap->sge, i) 3878 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); 3879 for_each_ofldrxq(&adap->sge, i) 3880 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q); 3881 for_each_port(adap, i) 3882 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); 3883 } 3884 3885 static void process_db_drop(struct work_struct *work) 3886 { 3887 struct adapter *adap; 3888 3889 adap = container_of(work, struct adapter, db_drop_task); 3890 3891 if (is_t4(adap->params.chip)) { 3892 drain_db_fifo(adap, dbfifo_drain_delay); 3893 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); 3894 drain_db_fifo(adap, dbfifo_drain_delay); 3895 recover_all_queues(adap); 3896 drain_db_fifo(adap, dbfifo_drain_delay); 3897 enable_dbs(adap); 3898 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 3899 } else { 3900 u32 dropped_db = t4_read_reg(adap, 0x010ac); 3901 u16 qid = (dropped_db >> 15) & 0x1ffff; 3902 u16 pidx_inc = dropped_db & 0x1fff; 3903 u64 bar2_qoffset; 3904 unsigned int bar2_qid; 3905 int ret; 3906 3907 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS, 3908 &bar2_qoffset, &bar2_qid); 3909 if (ret) 3910 dev_err(adap->pdev_dev, "doorbell drop recovery: " 3911 "qid=%d, pidx_inc=%d\n", qid, pidx_inc); 3912 else 3913 writel(PIDX_T5_V(pidx_inc) | QID_V(bar2_qid), 3914 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL); 3915 3916 /* Re-enable BAR2 WC */ 3917 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15); 3918 } 3919 3920 t4_set_reg_field(adap, SGE_DOORBELL_CONTROL_A, DROPPED_DB_F, 0); 3921 } 3922 3923 void t4_db_full(struct adapter *adap) 3924 { 3925 if (is_t4(adap->params.chip)) { 3926 disable_dbs(adap); 3927 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 3928 t4_set_reg_field(adap, SGE_INT_ENABLE3_A, 3929 DBFIFO_HP_INT_F | DBFIFO_LP_INT_F, 0); 3930 queue_work(adap->workq, &adap->db_full_task); 3931 } 3932 } 3933 3934 void t4_db_dropped(struct adapter *adap) 3935 { 3936 if (is_t4(adap->params.chip)) { 3937 disable_dbs(adap); 3938 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 3939 } 3940 queue_work(adap->workq, &adap->db_drop_task); 3941 } 3942 3943 static void uld_attach(struct adapter *adap, unsigned int uld) 3944 { 3945 void *handle; 3946 struct cxgb4_lld_info lli; 3947 unsigned short i; 3948 3949 lli.pdev = adap->pdev; 3950 lli.pf = adap->fn; 3951 lli.l2t = adap->l2t; 3952 lli.tids = &adap->tids; 3953 lli.ports = adap->port; 3954 lli.vr = &adap->vres; 3955 lli.mtus = adap->params.mtus; 3956 if (uld == CXGB4_ULD_RDMA) { 3957 lli.rxq_ids = adap->sge.rdma_rxq; 3958 lli.ciq_ids = adap->sge.rdma_ciq; 3959 lli.nrxq = adap->sge.rdmaqs; 3960 lli.nciq = adap->sge.rdmaciqs; 3961 } else if (uld == CXGB4_ULD_ISCSI) { 3962 lli.rxq_ids = adap->sge.ofld_rxq; 3963 lli.nrxq = adap->sge.ofldqsets; 3964 } 3965 lli.ntxq = adap->sge.ofldqsets; 3966 lli.nchan = adap->params.nports; 3967 lli.nports = adap->params.nports; 3968 lli.wr_cred = adap->params.ofldq_wr_cred; 3969 lli.adapter_type = adap->params.chip; 3970 lli.iscsi_iolen = MAXRXDATA_G(t4_read_reg(adap, TP_PARA_REG2_A)); 3971 lli.cclk_ps = 1000000000 / adap->params.vpd.cclk; 3972 lli.udb_density = 1 << adap->params.sge.eq_qpp; 3973 lli.ucq_density = 1 << adap->params.sge.iq_qpp; 3974 lli.filt_mode = adap->params.tp.vlan_pri_map; 3975 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */ 3976 for (i = 0; i < NCHAN; i++) 3977 lli.tx_modq[i] = i; 3978 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS_A); 3979 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL_A); 3980 lli.fw_vers = adap->params.fw_vers; 3981 lli.dbfifo_int_thresh = dbfifo_int_thresh; 3982 lli.sge_ingpadboundary = adap->sge.fl_align; 3983 lli.sge_egrstatuspagesize = adap->sge.stat_len; 3984 lli.sge_pktshift = adap->sge.pktshift; 3985 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN; 3986 lli.max_ordird_qp = adap->params.max_ordird_qp; 3987 lli.max_ird_adapter = adap->params.max_ird_adapter; 3988 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl; 3989 3990 handle = ulds[uld].add(&lli); 3991 if (IS_ERR(handle)) { 3992 dev_warn(adap->pdev_dev, 3993 "could not attach to the %s driver, error %ld\n", 3994 uld_str[uld], PTR_ERR(handle)); 3995 return; 3996 } 3997 3998 adap->uld_handle[uld] = handle; 3999 4000 if (!netevent_registered) { 4001 register_netevent_notifier(&cxgb4_netevent_nb); 4002 netevent_registered = true; 4003 } 4004 4005 if (adap->flags & FULL_INIT_DONE) 4006 ulds[uld].state_change(handle, CXGB4_STATE_UP); 4007 } 4008 4009 static void attach_ulds(struct adapter *adap) 4010 { 4011 unsigned int i; 4012 4013 spin_lock(&adap_rcu_lock); 4014 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list); 4015 spin_unlock(&adap_rcu_lock); 4016 4017 mutex_lock(&uld_mutex); 4018 list_add_tail(&adap->list_node, &adapter_list); 4019 for (i = 0; i < CXGB4_ULD_MAX; i++) 4020 if (ulds[i].add) 4021 uld_attach(adap, i); 4022 mutex_unlock(&uld_mutex); 4023 } 4024 4025 static void detach_ulds(struct adapter *adap) 4026 { 4027 unsigned int i; 4028 4029 mutex_lock(&uld_mutex); 4030 list_del(&adap->list_node); 4031 for (i = 0; i < CXGB4_ULD_MAX; i++) 4032 if (adap->uld_handle[i]) { 4033 ulds[i].state_change(adap->uld_handle[i], 4034 CXGB4_STATE_DETACH); 4035 adap->uld_handle[i] = NULL; 4036 } 4037 if (netevent_registered && list_empty(&adapter_list)) { 4038 unregister_netevent_notifier(&cxgb4_netevent_nb); 4039 netevent_registered = false; 4040 } 4041 mutex_unlock(&uld_mutex); 4042 4043 spin_lock(&adap_rcu_lock); 4044 list_del_rcu(&adap->rcu_node); 4045 spin_unlock(&adap_rcu_lock); 4046 } 4047 4048 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state) 4049 { 4050 unsigned int i; 4051 4052 mutex_lock(&uld_mutex); 4053 for (i = 0; i < CXGB4_ULD_MAX; i++) 4054 if (adap->uld_handle[i]) 4055 ulds[i].state_change(adap->uld_handle[i], new_state); 4056 mutex_unlock(&uld_mutex); 4057 } 4058 4059 /** 4060 * cxgb4_register_uld - register an upper-layer driver 4061 * @type: the ULD type 4062 * @p: the ULD methods 4063 * 4064 * Registers an upper-layer driver with this driver and notifies the ULD 4065 * about any presently available devices that support its type. Returns 4066 * %-EBUSY if a ULD of the same type is already registered. 4067 */ 4068 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p) 4069 { 4070 int ret = 0; 4071 struct adapter *adap; 4072 4073 if (type >= CXGB4_ULD_MAX) 4074 return -EINVAL; 4075 mutex_lock(&uld_mutex); 4076 if (ulds[type].add) { 4077 ret = -EBUSY; 4078 goto out; 4079 } 4080 ulds[type] = *p; 4081 list_for_each_entry(adap, &adapter_list, list_node) 4082 uld_attach(adap, type); 4083 out: mutex_unlock(&uld_mutex); 4084 return ret; 4085 } 4086 EXPORT_SYMBOL(cxgb4_register_uld); 4087 4088 /** 4089 * cxgb4_unregister_uld - unregister an upper-layer driver 4090 * @type: the ULD type 4091 * 4092 * Unregisters an existing upper-layer driver. 4093 */ 4094 int cxgb4_unregister_uld(enum cxgb4_uld type) 4095 { 4096 struct adapter *adap; 4097 4098 if (type >= CXGB4_ULD_MAX) 4099 return -EINVAL; 4100 mutex_lock(&uld_mutex); 4101 list_for_each_entry(adap, &adapter_list, list_node) 4102 adap->uld_handle[type] = NULL; 4103 ulds[type].add = NULL; 4104 mutex_unlock(&uld_mutex); 4105 return 0; 4106 } 4107 EXPORT_SYMBOL(cxgb4_unregister_uld); 4108 4109 #if IS_ENABLED(CONFIG_IPV6) 4110 static int cxgb4_inet6addr_handler(struct notifier_block *this, 4111 unsigned long event, void *data) 4112 { 4113 struct inet6_ifaddr *ifa = data; 4114 struct net_device *event_dev = ifa->idev->dev; 4115 const struct device *parent = NULL; 4116 #if IS_ENABLED(CONFIG_BONDING) 4117 struct adapter *adap; 4118 #endif 4119 if (event_dev->priv_flags & IFF_802_1Q_VLAN) 4120 event_dev = vlan_dev_real_dev(event_dev); 4121 #if IS_ENABLED(CONFIG_BONDING) 4122 if (event_dev->flags & IFF_MASTER) { 4123 list_for_each_entry(adap, &adapter_list, list_node) { 4124 switch (event) { 4125 case NETDEV_UP: 4126 cxgb4_clip_get(adap->port[0], 4127 (const u32 *)ifa, 1); 4128 break; 4129 case NETDEV_DOWN: 4130 cxgb4_clip_release(adap->port[0], 4131 (const u32 *)ifa, 1); 4132 break; 4133 default: 4134 break; 4135 } 4136 } 4137 return NOTIFY_OK; 4138 } 4139 #endif 4140 4141 if (event_dev) 4142 parent = event_dev->dev.parent; 4143 4144 if (parent && parent->driver == &cxgb4_driver.driver) { 4145 switch (event) { 4146 case NETDEV_UP: 4147 cxgb4_clip_get(event_dev, (const u32 *)ifa, 1); 4148 break; 4149 case NETDEV_DOWN: 4150 cxgb4_clip_release(event_dev, (const u32 *)ifa, 1); 4151 break; 4152 default: 4153 break; 4154 } 4155 } 4156 return NOTIFY_OK; 4157 } 4158 4159 static bool inet6addr_registered; 4160 static struct notifier_block cxgb4_inet6addr_notifier = { 4161 .notifier_call = cxgb4_inet6addr_handler 4162 }; 4163 4164 static void update_clip(const struct adapter *adap) 4165 { 4166 int i; 4167 struct net_device *dev; 4168 int ret; 4169 4170 rcu_read_lock(); 4171 4172 for (i = 0; i < MAX_NPORTS; i++) { 4173 dev = adap->port[i]; 4174 ret = 0; 4175 4176 if (dev) 4177 ret = cxgb4_update_root_dev_clip(dev); 4178 4179 if (ret < 0) 4180 break; 4181 } 4182 rcu_read_unlock(); 4183 } 4184 #endif /* IS_ENABLED(CONFIG_IPV6) */ 4185 4186 /** 4187 * cxgb_up - enable the adapter 4188 * @adap: adapter being enabled 4189 * 4190 * Called when the first port is enabled, this function performs the 4191 * actions necessary to make an adapter operational, such as completing 4192 * the initialization of HW modules, and enabling interrupts. 4193 * 4194 * Must be called with the rtnl lock held. 4195 */ 4196 static int cxgb_up(struct adapter *adap) 4197 { 4198 int err; 4199 4200 err = setup_sge_queues(adap); 4201 if (err) 4202 goto out; 4203 err = setup_rss(adap); 4204 if (err) 4205 goto freeq; 4206 4207 if (adap->flags & USING_MSIX) { 4208 name_msix_vecs(adap); 4209 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0, 4210 adap->msix_info[0].desc, adap); 4211 if (err) 4212 goto irq_err; 4213 4214 err = request_msix_queue_irqs(adap); 4215 if (err) { 4216 free_irq(adap->msix_info[0].vec, adap); 4217 goto irq_err; 4218 } 4219 } else { 4220 err = request_irq(adap->pdev->irq, t4_intr_handler(adap), 4221 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED, 4222 adap->port[0]->name, adap); 4223 if (err) 4224 goto irq_err; 4225 } 4226 enable_rx(adap); 4227 t4_sge_start(adap); 4228 t4_intr_enable(adap); 4229 adap->flags |= FULL_INIT_DONE; 4230 notify_ulds(adap, CXGB4_STATE_UP); 4231 #if IS_ENABLED(CONFIG_IPV6) 4232 update_clip(adap); 4233 #endif 4234 out: 4235 return err; 4236 irq_err: 4237 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err); 4238 freeq: 4239 t4_free_sge_resources(adap); 4240 goto out; 4241 } 4242 4243 static void cxgb_down(struct adapter *adapter) 4244 { 4245 t4_intr_disable(adapter); 4246 cancel_work_sync(&adapter->tid_release_task); 4247 cancel_work_sync(&adapter->db_full_task); 4248 cancel_work_sync(&adapter->db_drop_task); 4249 adapter->tid_release_task_busy = false; 4250 adapter->tid_release_head = NULL; 4251 4252 if (adapter->flags & USING_MSIX) { 4253 free_msix_queue_irqs(adapter); 4254 free_irq(adapter->msix_info[0].vec, adapter); 4255 } else 4256 free_irq(adapter->pdev->irq, adapter); 4257 quiesce_rx(adapter); 4258 t4_sge_stop(adapter); 4259 t4_free_sge_resources(adapter); 4260 adapter->flags &= ~FULL_INIT_DONE; 4261 } 4262 4263 /* 4264 * net_device operations 4265 */ 4266 static int cxgb_open(struct net_device *dev) 4267 { 4268 int err; 4269 struct port_info *pi = netdev_priv(dev); 4270 struct adapter *adapter = pi->adapter; 4271 4272 netif_carrier_off(dev); 4273 4274 if (!(adapter->flags & FULL_INIT_DONE)) { 4275 err = cxgb_up(adapter); 4276 if (err < 0) 4277 return err; 4278 } 4279 4280 err = link_start(dev); 4281 if (!err) 4282 netif_tx_start_all_queues(dev); 4283 return err; 4284 } 4285 4286 static int cxgb_close(struct net_device *dev) 4287 { 4288 struct port_info *pi = netdev_priv(dev); 4289 struct adapter *adapter = pi->adapter; 4290 4291 netif_tx_stop_all_queues(dev); 4292 netif_carrier_off(dev); 4293 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false); 4294 } 4295 4296 /* Return an error number if the indicated filter isn't writable ... 4297 */ 4298 static int writable_filter(struct filter_entry *f) 4299 { 4300 if (f->locked) 4301 return -EPERM; 4302 if (f->pending) 4303 return -EBUSY; 4304 4305 return 0; 4306 } 4307 4308 /* Delete the filter at the specified index (if valid). The checks for all 4309 * the common problems with doing this like the filter being locked, currently 4310 * pending in another operation, etc. 4311 */ 4312 static int delete_filter(struct adapter *adapter, unsigned int fidx) 4313 { 4314 struct filter_entry *f; 4315 int ret; 4316 4317 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids) 4318 return -EINVAL; 4319 4320 f = &adapter->tids.ftid_tab[fidx]; 4321 ret = writable_filter(f); 4322 if (ret) 4323 return ret; 4324 if (f->valid) 4325 return del_filter_wr(adapter, fidx); 4326 4327 return 0; 4328 } 4329 4330 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, 4331 __be32 sip, __be16 sport, __be16 vlan, 4332 unsigned int queue, unsigned char port, unsigned char mask) 4333 { 4334 int ret; 4335 struct filter_entry *f; 4336 struct adapter *adap; 4337 int i; 4338 u8 *val; 4339 4340 adap = netdev2adap(dev); 4341 4342 /* Adjust stid to correct filter index */ 4343 stid -= adap->tids.sftid_base; 4344 stid += adap->tids.nftids; 4345 4346 /* Check to make sure the filter requested is writable ... 4347 */ 4348 f = &adap->tids.ftid_tab[stid]; 4349 ret = writable_filter(f); 4350 if (ret) 4351 return ret; 4352 4353 /* Clear out any old resources being used by the filter before 4354 * we start constructing the new filter. 4355 */ 4356 if (f->valid) 4357 clear_filter(adap, f); 4358 4359 /* Clear out filter specifications */ 4360 memset(&f->fs, 0, sizeof(struct ch_filter_specification)); 4361 f->fs.val.lport = cpu_to_be16(sport); 4362 f->fs.mask.lport = ~0; 4363 val = (u8 *)&sip; 4364 if ((val[0] | val[1] | val[2] | val[3]) != 0) { 4365 for (i = 0; i < 4; i++) { 4366 f->fs.val.lip[i] = val[i]; 4367 f->fs.mask.lip[i] = ~0; 4368 } 4369 if (adap->params.tp.vlan_pri_map & PORT_F) { 4370 f->fs.val.iport = port; 4371 f->fs.mask.iport = mask; 4372 } 4373 } 4374 4375 if (adap->params.tp.vlan_pri_map & PROTOCOL_F) { 4376 f->fs.val.proto = IPPROTO_TCP; 4377 f->fs.mask.proto = ~0; 4378 } 4379 4380 f->fs.dirsteer = 1; 4381 f->fs.iq = queue; 4382 /* Mark filter as locked */ 4383 f->locked = 1; 4384 f->fs.rpttid = 1; 4385 4386 ret = set_filter_wr(adap, stid); 4387 if (ret) { 4388 clear_filter(adap, f); 4389 return ret; 4390 } 4391 4392 return 0; 4393 } 4394 EXPORT_SYMBOL(cxgb4_create_server_filter); 4395 4396 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, 4397 unsigned int queue, bool ipv6) 4398 { 4399 int ret; 4400 struct filter_entry *f; 4401 struct adapter *adap; 4402 4403 adap = netdev2adap(dev); 4404 4405 /* Adjust stid to correct filter index */ 4406 stid -= adap->tids.sftid_base; 4407 stid += adap->tids.nftids; 4408 4409 f = &adap->tids.ftid_tab[stid]; 4410 /* Unlock the filter */ 4411 f->locked = 0; 4412 4413 ret = delete_filter(adap, stid); 4414 if (ret) 4415 return ret; 4416 4417 return 0; 4418 } 4419 EXPORT_SYMBOL(cxgb4_remove_server_filter); 4420 4421 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev, 4422 struct rtnl_link_stats64 *ns) 4423 { 4424 struct port_stats stats; 4425 struct port_info *p = netdev_priv(dev); 4426 struct adapter *adapter = p->adapter; 4427 4428 /* Block retrieving statistics during EEH error 4429 * recovery. Otherwise, the recovery might fail 4430 * and the PCI device will be removed permanently 4431 */ 4432 spin_lock(&adapter->stats_lock); 4433 if (!netif_device_present(dev)) { 4434 spin_unlock(&adapter->stats_lock); 4435 return ns; 4436 } 4437 t4_get_port_stats(adapter, p->tx_chan, &stats); 4438 spin_unlock(&adapter->stats_lock); 4439 4440 ns->tx_bytes = stats.tx_octets; 4441 ns->tx_packets = stats.tx_frames; 4442 ns->rx_bytes = stats.rx_octets; 4443 ns->rx_packets = stats.rx_frames; 4444 ns->multicast = stats.rx_mcast_frames; 4445 4446 /* detailed rx_errors */ 4447 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long + 4448 stats.rx_runt; 4449 ns->rx_over_errors = 0; 4450 ns->rx_crc_errors = stats.rx_fcs_err; 4451 ns->rx_frame_errors = stats.rx_symbol_err; 4452 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 + 4453 stats.rx_ovflow2 + stats.rx_ovflow3 + 4454 stats.rx_trunc0 + stats.rx_trunc1 + 4455 stats.rx_trunc2 + stats.rx_trunc3; 4456 ns->rx_missed_errors = 0; 4457 4458 /* detailed tx_errors */ 4459 ns->tx_aborted_errors = 0; 4460 ns->tx_carrier_errors = 0; 4461 ns->tx_fifo_errors = 0; 4462 ns->tx_heartbeat_errors = 0; 4463 ns->tx_window_errors = 0; 4464 4465 ns->tx_errors = stats.tx_error_frames; 4466 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err + 4467 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors; 4468 return ns; 4469 } 4470 4471 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) 4472 { 4473 unsigned int mbox; 4474 int ret = 0, prtad, devad; 4475 struct port_info *pi = netdev_priv(dev); 4476 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data; 4477 4478 switch (cmd) { 4479 case SIOCGMIIPHY: 4480 if (pi->mdio_addr < 0) 4481 return -EOPNOTSUPP; 4482 data->phy_id = pi->mdio_addr; 4483 break; 4484 case SIOCGMIIREG: 4485 case SIOCSMIIREG: 4486 if (mdio_phy_id_is_c45(data->phy_id)) { 4487 prtad = mdio_phy_id_prtad(data->phy_id); 4488 devad = mdio_phy_id_devad(data->phy_id); 4489 } else if (data->phy_id < 32) { 4490 prtad = data->phy_id; 4491 devad = 0; 4492 data->reg_num &= 0x1f; 4493 } else 4494 return -EINVAL; 4495 4496 mbox = pi->adapter->fn; 4497 if (cmd == SIOCGMIIREG) 4498 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad, 4499 data->reg_num, &data->val_out); 4500 else 4501 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad, 4502 data->reg_num, data->val_in); 4503 break; 4504 default: 4505 return -EOPNOTSUPP; 4506 } 4507 return ret; 4508 } 4509 4510 static void cxgb_set_rxmode(struct net_device *dev) 4511 { 4512 /* unfortunately we can't return errors to the stack */ 4513 set_rxmode(dev, -1, false); 4514 } 4515 4516 static int cxgb_change_mtu(struct net_device *dev, int new_mtu) 4517 { 4518 int ret; 4519 struct port_info *pi = netdev_priv(dev); 4520 4521 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */ 4522 return -EINVAL; 4523 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1, 4524 -1, -1, -1, true); 4525 if (!ret) 4526 dev->mtu = new_mtu; 4527 return ret; 4528 } 4529 4530 static int cxgb_set_mac_addr(struct net_device *dev, void *p) 4531 { 4532 int ret; 4533 struct sockaddr *addr = p; 4534 struct port_info *pi = netdev_priv(dev); 4535 4536 if (!is_valid_ether_addr(addr->sa_data)) 4537 return -EADDRNOTAVAIL; 4538 4539 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid, 4540 pi->xact_addr_filt, addr->sa_data, true, true); 4541 if (ret < 0) 4542 return ret; 4543 4544 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 4545 pi->xact_addr_filt = ret; 4546 return 0; 4547 } 4548 4549 #ifdef CONFIG_NET_POLL_CONTROLLER 4550 static void cxgb_netpoll(struct net_device *dev) 4551 { 4552 struct port_info *pi = netdev_priv(dev); 4553 struct adapter *adap = pi->adapter; 4554 4555 if (adap->flags & USING_MSIX) { 4556 int i; 4557 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset]; 4558 4559 for (i = pi->nqsets; i; i--, rx++) 4560 t4_sge_intr_msix(0, &rx->rspq); 4561 } else 4562 t4_intr_handler(adap)(0, adap); 4563 } 4564 #endif 4565 4566 static const struct net_device_ops cxgb4_netdev_ops = { 4567 .ndo_open = cxgb_open, 4568 .ndo_stop = cxgb_close, 4569 .ndo_start_xmit = t4_eth_xmit, 4570 .ndo_select_queue = cxgb_select_queue, 4571 .ndo_get_stats64 = cxgb_get_stats, 4572 .ndo_set_rx_mode = cxgb_set_rxmode, 4573 .ndo_set_mac_address = cxgb_set_mac_addr, 4574 .ndo_set_features = cxgb_set_features, 4575 .ndo_validate_addr = eth_validate_addr, 4576 .ndo_do_ioctl = cxgb_ioctl, 4577 .ndo_change_mtu = cxgb_change_mtu, 4578 #ifdef CONFIG_NET_POLL_CONTROLLER 4579 .ndo_poll_controller = cxgb_netpoll, 4580 #endif 4581 #ifdef CONFIG_NET_RX_BUSY_POLL 4582 .ndo_busy_poll = cxgb_busy_poll, 4583 #endif 4584 4585 }; 4586 4587 void t4_fatal_err(struct adapter *adap) 4588 { 4589 t4_set_reg_field(adap, SGE_CONTROL_A, GLOBALENABLE_F, 0); 4590 t4_intr_disable(adap); 4591 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n"); 4592 } 4593 4594 /* Return the specified PCI-E Configuration Space register from our Physical 4595 * Function. We try first via a Firmware LDST Command since we prefer to let 4596 * the firmware own all of these registers, but if that fails we go for it 4597 * directly ourselves. 4598 */ 4599 static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg) 4600 { 4601 struct fw_ldst_cmd ldst_cmd; 4602 u32 val; 4603 int ret; 4604 4605 /* Construct and send the Firmware LDST Command to retrieve the 4606 * specified PCI-E Configuration Space register. 4607 */ 4608 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 4609 ldst_cmd.op_to_addrspace = 4610 htonl(FW_CMD_OP_V(FW_LDST_CMD) | 4611 FW_CMD_REQUEST_F | 4612 FW_CMD_READ_F | 4613 FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE)); 4614 ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd)); 4615 ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1); 4616 ldst_cmd.u.pcie.ctrl_to_fn = 4617 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn)); 4618 ldst_cmd.u.pcie.r = reg; 4619 ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd), 4620 &ldst_cmd); 4621 4622 /* If the LDST Command suucceeded, exctract the returned register 4623 * value. Otherwise read it directly ourself. 4624 */ 4625 if (ret == 0) 4626 val = ntohl(ldst_cmd.u.pcie.data[0]); 4627 else 4628 t4_hw_pci_read_cfg4(adap, reg, &val); 4629 4630 return val; 4631 } 4632 4633 static void setup_memwin(struct adapter *adap) 4634 { 4635 u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture; 4636 4637 if (is_t4(adap->params.chip)) { 4638 u32 bar0; 4639 4640 /* Truncation intentional: we only read the bottom 32-bits of 4641 * the 64-bit BAR0/BAR1 ... We use the hardware backdoor 4642 * mechanism to read BAR0 instead of using 4643 * pci_resource_start() because we could be operating from 4644 * within a Virtual Machine which is trapping our accesses to 4645 * our Configuration Space and we need to set up the PCI-E 4646 * Memory Window decoders with the actual addresses which will 4647 * be coming across the PCI-E link. 4648 */ 4649 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0); 4650 bar0 &= PCI_BASE_ADDRESS_MEM_MASK; 4651 adap->t4_bar0 = bar0; 4652 4653 mem_win0_base = bar0 + MEMWIN0_BASE; 4654 mem_win1_base = bar0 + MEMWIN1_BASE; 4655 mem_win2_base = bar0 + MEMWIN2_BASE; 4656 mem_win2_aperture = MEMWIN2_APERTURE; 4657 } else { 4658 /* For T5, only relative offset inside the PCIe BAR is passed */ 4659 mem_win0_base = MEMWIN0_BASE; 4660 mem_win1_base = MEMWIN1_BASE; 4661 mem_win2_base = MEMWIN2_BASE_T5; 4662 mem_win2_aperture = MEMWIN2_APERTURE_T5; 4663 } 4664 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 0), 4665 mem_win0_base | BIR_V(0) | 4666 WINDOW_V(ilog2(MEMWIN0_APERTURE) - 10)); 4667 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 1), 4668 mem_win1_base | BIR_V(0) | 4669 WINDOW_V(ilog2(MEMWIN1_APERTURE) - 10)); 4670 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2), 4671 mem_win2_base | BIR_V(0) | 4672 WINDOW_V(ilog2(mem_win2_aperture) - 10)); 4673 t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 2)); 4674 } 4675 4676 static void setup_memwin_rdma(struct adapter *adap) 4677 { 4678 if (adap->vres.ocq.size) { 4679 u32 start; 4680 unsigned int sz_kb; 4681 4682 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2); 4683 start &= PCI_BASE_ADDRESS_MEM_MASK; 4684 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres); 4685 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10; 4686 t4_write_reg(adap, 4687 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN_A, 3), 4688 start | BIR_V(1) | WINDOW_V(ilog2(sz_kb))); 4689 t4_write_reg(adap, 4690 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3), 4691 adap->vres.ocq.start); 4692 t4_read_reg(adap, 4693 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET_A, 3)); 4694 } 4695 } 4696 4697 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c) 4698 { 4699 u32 v; 4700 int ret; 4701 4702 /* get device capabilities */ 4703 memset(c, 0, sizeof(*c)); 4704 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4705 FW_CMD_REQUEST_F | FW_CMD_READ_F); 4706 c->cfvalid_to_len16 = htonl(FW_LEN16(*c)); 4707 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c); 4708 if (ret < 0) 4709 return ret; 4710 4711 /* select capabilities we'll be using */ 4712 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) { 4713 if (!vf_acls) 4714 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM); 4715 else 4716 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM); 4717 } else if (vf_acls) { 4718 dev_err(adap->pdev_dev, "virtualization ACLs not supported"); 4719 return ret; 4720 } 4721 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4722 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 4723 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL); 4724 if (ret < 0) 4725 return ret; 4726 4727 ret = t4_config_glbl_rss(adap, adap->fn, 4728 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 4729 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | 4730 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F); 4731 if (ret < 0) 4732 return ret; 4733 4734 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ, 4735 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF); 4736 if (ret < 0) 4737 return ret; 4738 4739 t4_sge_init(adap); 4740 4741 /* tweak some settings */ 4742 t4_write_reg(adap, TP_SHIFT_CNT_A, 0x64f8849); 4743 t4_write_reg(adap, ULP_RX_TDDP_PSZ_A, HPZ0_V(PAGE_SHIFT - 12)); 4744 t4_write_reg(adap, TP_PIO_ADDR_A, TP_INGRESS_CONFIG_A); 4745 v = t4_read_reg(adap, TP_PIO_DATA_A); 4746 t4_write_reg(adap, TP_PIO_DATA_A, v & ~CSUM_HAS_PSEUDO_HDR_F); 4747 4748 /* first 4 Tx modulation queues point to consecutive Tx channels */ 4749 adap->params.tp.tx_modq_map = 0xE4; 4750 t4_write_reg(adap, TP_TX_MOD_QUEUE_REQ_MAP_A, 4751 TX_MOD_QUEUE_REQ_MAP_V(adap->params.tp.tx_modq_map)); 4752 4753 /* associate each Tx modulation queue with consecutive Tx channels */ 4754 v = 0x84218421; 4755 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 4756 &v, 1, TP_TX_SCHED_HDR_A); 4757 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 4758 &v, 1, TP_TX_SCHED_FIFO_A); 4759 t4_write_indirect(adap, TP_PIO_ADDR_A, TP_PIO_DATA_A, 4760 &v, 1, TP_TX_SCHED_PCMD_A); 4761 4762 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */ 4763 if (is_offload(adap)) { 4764 t4_write_reg(adap, TP_TX_MOD_QUEUE_WEIGHT0_A, 4765 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4766 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4767 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4768 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 4769 t4_write_reg(adap, TP_TX_MOD_CHANNEL_WEIGHT_A, 4770 TX_MODQ_WEIGHT0_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4771 TX_MODQ_WEIGHT1_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4772 TX_MODQ_WEIGHT2_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4773 TX_MODQ_WEIGHT3_V(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 4774 } 4775 4776 /* get basic stuff going */ 4777 return t4_early_init(adap, adap->fn); 4778 } 4779 4780 /* 4781 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. 4782 */ 4783 #define MAX_ATIDS 8192U 4784 4785 /* 4786 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 4787 * 4788 * If the firmware we're dealing with has Configuration File support, then 4789 * we use that to perform all configuration 4790 */ 4791 4792 /* 4793 * Tweak configuration based on module parameters, etc. Most of these have 4794 * defaults assigned to them by Firmware Configuration Files (if we're using 4795 * them) but need to be explicitly set if we're using hard-coded 4796 * initialization. But even in the case of using Firmware Configuration 4797 * Files, we'd like to expose the ability to change these via module 4798 * parameters so these are essentially common tweaks/settings for 4799 * Configuration Files and hard-coded initialization ... 4800 */ 4801 static int adap_init0_tweaks(struct adapter *adapter) 4802 { 4803 /* 4804 * Fix up various Host-Dependent Parameters like Page Size, Cache 4805 * Line Size, etc. The firmware default is for a 4KB Page Size and 4806 * 64B Cache Line Size ... 4807 */ 4808 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES); 4809 4810 /* 4811 * Process module parameters which affect early initialization. 4812 */ 4813 if (rx_dma_offset != 2 && rx_dma_offset != 0) { 4814 dev_err(&adapter->pdev->dev, 4815 "Ignoring illegal rx_dma_offset=%d, using 2\n", 4816 rx_dma_offset); 4817 rx_dma_offset = 2; 4818 } 4819 t4_set_reg_field(adapter, SGE_CONTROL_A, 4820 PKTSHIFT_V(PKTSHIFT_M), 4821 PKTSHIFT_V(rx_dma_offset)); 4822 4823 /* 4824 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux 4825 * adds the pseudo header itself. 4826 */ 4827 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG_A, 4828 CSUM_HAS_PSEUDO_HDR_F, 0); 4829 4830 return 0; 4831 } 4832 4833 /* 4834 * Attempt to initialize the adapter via a Firmware Configuration File. 4835 */ 4836 static int adap_init0_config(struct adapter *adapter, int reset) 4837 { 4838 struct fw_caps_config_cmd caps_cmd; 4839 const struct firmware *cf; 4840 unsigned long mtype = 0, maddr = 0; 4841 u32 finiver, finicsum, cfcsum; 4842 int ret; 4843 int config_issued = 0; 4844 char *fw_config_file, fw_config_file_path[256]; 4845 char *config_name = NULL; 4846 4847 /* 4848 * Reset device if necessary. 4849 */ 4850 if (reset) { 4851 ret = t4_fw_reset(adapter, adapter->mbox, 4852 PIORSTMODE_F | PIORST_F); 4853 if (ret < 0) 4854 goto bye; 4855 } 4856 4857 /* 4858 * If we have a T4 configuration file under /lib/firmware/cxgb4/, 4859 * then use that. Otherwise, use the configuration file stored 4860 * in the adapter flash ... 4861 */ 4862 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { 4863 case CHELSIO_T4: 4864 fw_config_file = FW4_CFNAME; 4865 break; 4866 case CHELSIO_T5: 4867 fw_config_file = FW5_CFNAME; 4868 break; 4869 default: 4870 dev_err(adapter->pdev_dev, "Device %d is not supported\n", 4871 adapter->pdev->device); 4872 ret = -EINVAL; 4873 goto bye; 4874 } 4875 4876 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev); 4877 if (ret < 0) { 4878 config_name = "On FLASH"; 4879 mtype = FW_MEMTYPE_CF_FLASH; 4880 maddr = t4_flash_cfg_addr(adapter); 4881 } else { 4882 u32 params[7], val[7]; 4883 4884 sprintf(fw_config_file_path, 4885 "/lib/firmware/%s", fw_config_file); 4886 config_name = fw_config_file_path; 4887 4888 if (cf->size >= FLASH_CFG_MAX_SIZE) 4889 ret = -ENOMEM; 4890 else { 4891 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 4892 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 4893 ret = t4_query_params(adapter, adapter->mbox, 4894 adapter->fn, 0, 1, params, val); 4895 if (ret == 0) { 4896 /* 4897 * For t4_memory_rw() below addresses and 4898 * sizes have to be in terms of multiples of 4 4899 * bytes. So, if the Configuration File isn't 4900 * a multiple of 4 bytes in length we'll have 4901 * to write that out separately since we can't 4902 * guarantee that the bytes following the 4903 * residual byte in the buffer returned by 4904 * request_firmware() are zeroed out ... 4905 */ 4906 size_t resid = cf->size & 0x3; 4907 size_t size = cf->size & ~0x3; 4908 __be32 *data = (__be32 *)cf->data; 4909 4910 mtype = FW_PARAMS_PARAM_Y_G(val[0]); 4911 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16; 4912 4913 spin_lock(&adapter->win0_lock); 4914 ret = t4_memory_rw(adapter, 0, mtype, maddr, 4915 size, data, T4_MEMORY_WRITE); 4916 if (ret == 0 && resid != 0) { 4917 union { 4918 __be32 word; 4919 char buf[4]; 4920 } last; 4921 int i; 4922 4923 last.word = data[size >> 2]; 4924 for (i = resid; i < 4; i++) 4925 last.buf[i] = 0; 4926 ret = t4_memory_rw(adapter, 0, mtype, 4927 maddr + size, 4928 4, &last.word, 4929 T4_MEMORY_WRITE); 4930 } 4931 spin_unlock(&adapter->win0_lock); 4932 } 4933 } 4934 4935 release_firmware(cf); 4936 if (ret) 4937 goto bye; 4938 } 4939 4940 /* 4941 * Issue a Capability Configuration command to the firmware to get it 4942 * to parse the Configuration File. We don't use t4_fw_config_file() 4943 * because we want the ability to modify various features after we've 4944 * processed the configuration file ... 4945 */ 4946 memset(&caps_cmd, 0, sizeof(caps_cmd)); 4947 caps_cmd.op_to_write = 4948 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4949 FW_CMD_REQUEST_F | 4950 FW_CMD_READ_F); 4951 caps_cmd.cfvalid_to_len16 = 4952 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F | 4953 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) | 4954 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) | 4955 FW_LEN16(caps_cmd)); 4956 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 4957 &caps_cmd); 4958 4959 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware 4960 * Configuration File in FLASH), our last gasp effort is to use the 4961 * Firmware Configuration File which is embedded in the firmware. A 4962 * very few early versions of the firmware didn't have one embedded 4963 * but we can ignore those. 4964 */ 4965 if (ret == -ENOENT) { 4966 memset(&caps_cmd, 0, sizeof(caps_cmd)); 4967 caps_cmd.op_to_write = 4968 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4969 FW_CMD_REQUEST_F | 4970 FW_CMD_READ_F); 4971 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 4972 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, 4973 sizeof(caps_cmd), &caps_cmd); 4974 config_name = "Firmware Default"; 4975 } 4976 4977 config_issued = 1; 4978 if (ret < 0) 4979 goto bye; 4980 4981 finiver = ntohl(caps_cmd.finiver); 4982 finicsum = ntohl(caps_cmd.finicsum); 4983 cfcsum = ntohl(caps_cmd.cfcsum); 4984 if (finicsum != cfcsum) 4985 dev_warn(adapter->pdev_dev, "Configuration File checksum "\ 4986 "mismatch: [fini] csum=%#x, computed csum=%#x\n", 4987 finicsum, cfcsum); 4988 4989 /* 4990 * And now tell the firmware to use the configuration we just loaded. 4991 */ 4992 caps_cmd.op_to_write = 4993 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4994 FW_CMD_REQUEST_F | 4995 FW_CMD_WRITE_F); 4996 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 4997 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 4998 NULL); 4999 if (ret < 0) 5000 goto bye; 5001 5002 /* 5003 * Tweak configuration based on system architecture, module 5004 * parameters, etc. 5005 */ 5006 ret = adap_init0_tweaks(adapter); 5007 if (ret < 0) 5008 goto bye; 5009 5010 /* 5011 * And finally tell the firmware to initialize itself using the 5012 * parameters from the Configuration File. 5013 */ 5014 ret = t4_fw_initialize(adapter, adapter->mbox); 5015 if (ret < 0) 5016 goto bye; 5017 5018 /* Emit Firmware Configuration File information and return 5019 * successfully. 5020 */ 5021 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\ 5022 "Configuration File \"%s\", version %#x, computed checksum %#x\n", 5023 config_name, finiver, cfcsum); 5024 return 0; 5025 5026 /* 5027 * Something bad happened. Return the error ... (If the "error" 5028 * is that there's no Configuration File on the adapter we don't 5029 * want to issue a warning since this is fairly common.) 5030 */ 5031 bye: 5032 if (config_issued && ret != -ENOENT) 5033 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n", 5034 config_name, -ret); 5035 return ret; 5036 } 5037 5038 static struct fw_info fw_info_array[] = { 5039 { 5040 .chip = CHELSIO_T4, 5041 .fs_name = FW4_CFNAME, 5042 .fw_mod_name = FW4_FNAME, 5043 .fw_hdr = { 5044 .chip = FW_HDR_CHIP_T4, 5045 .fw_ver = __cpu_to_be32(FW_VERSION(T4)), 5046 .intfver_nic = FW_INTFVER(T4, NIC), 5047 .intfver_vnic = FW_INTFVER(T4, VNIC), 5048 .intfver_ri = FW_INTFVER(T4, RI), 5049 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 5050 .intfver_fcoe = FW_INTFVER(T4, FCOE), 5051 }, 5052 }, { 5053 .chip = CHELSIO_T5, 5054 .fs_name = FW5_CFNAME, 5055 .fw_mod_name = FW5_FNAME, 5056 .fw_hdr = { 5057 .chip = FW_HDR_CHIP_T5, 5058 .fw_ver = __cpu_to_be32(FW_VERSION(T5)), 5059 .intfver_nic = FW_INTFVER(T5, NIC), 5060 .intfver_vnic = FW_INTFVER(T5, VNIC), 5061 .intfver_ri = FW_INTFVER(T5, RI), 5062 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 5063 .intfver_fcoe = FW_INTFVER(T5, FCOE), 5064 }, 5065 } 5066 }; 5067 5068 static struct fw_info *find_fw_info(int chip) 5069 { 5070 int i; 5071 5072 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) { 5073 if (fw_info_array[i].chip == chip) 5074 return &fw_info_array[i]; 5075 } 5076 return NULL; 5077 } 5078 5079 /* 5080 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 5081 */ 5082 static int adap_init0(struct adapter *adap) 5083 { 5084 int ret; 5085 u32 v, port_vec; 5086 enum dev_state state; 5087 u32 params[7], val[7]; 5088 struct fw_caps_config_cmd caps_cmd; 5089 struct fw_devlog_cmd devlog_cmd; 5090 u32 devlog_meminfo; 5091 int reset = 1; 5092 5093 /* Contact FW, advertising Master capability */ 5094 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state); 5095 if (ret < 0) { 5096 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n", 5097 ret); 5098 return ret; 5099 } 5100 if (ret == adap->mbox) 5101 adap->flags |= MASTER_PF; 5102 5103 /* 5104 * If we're the Master PF Driver and the device is uninitialized, 5105 * then let's consider upgrading the firmware ... (We always want 5106 * to check the firmware version number in order to A. get it for 5107 * later reporting and B. to warn if the currently loaded firmware 5108 * is excessively mismatched relative to the driver.) 5109 */ 5110 t4_get_fw_version(adap, &adap->params.fw_vers); 5111 t4_get_tp_version(adap, &adap->params.tp_vers); 5112 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) { 5113 struct fw_info *fw_info; 5114 struct fw_hdr *card_fw; 5115 const struct firmware *fw; 5116 const u8 *fw_data = NULL; 5117 unsigned int fw_size = 0; 5118 5119 /* This is the firmware whose headers the driver was compiled 5120 * against 5121 */ 5122 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip)); 5123 if (fw_info == NULL) { 5124 dev_err(adap->pdev_dev, 5125 "unable to get firmware info for chip %d.\n", 5126 CHELSIO_CHIP_VERSION(adap->params.chip)); 5127 return -EINVAL; 5128 } 5129 5130 /* allocate memory to read the header of the firmware on the 5131 * card 5132 */ 5133 card_fw = t4_alloc_mem(sizeof(*card_fw)); 5134 5135 /* Get FW from from /lib/firmware/ */ 5136 ret = request_firmware(&fw, fw_info->fw_mod_name, 5137 adap->pdev_dev); 5138 if (ret < 0) { 5139 dev_err(adap->pdev_dev, 5140 "unable to load firmware image %s, error %d\n", 5141 fw_info->fw_mod_name, ret); 5142 } else { 5143 fw_data = fw->data; 5144 fw_size = fw->size; 5145 } 5146 5147 /* upgrade FW logic */ 5148 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw, 5149 state, &reset); 5150 5151 /* Cleaning up */ 5152 release_firmware(fw); 5153 t4_free_mem(card_fw); 5154 5155 if (ret < 0) 5156 goto bye; 5157 } 5158 5159 /* 5160 * Grab VPD parameters. This should be done after we establish a 5161 * connection to the firmware since some of the VPD parameters 5162 * (notably the Core Clock frequency) are retrieved via requests to 5163 * the firmware. On the other hand, we need these fairly early on 5164 * so we do this right after getting ahold of the firmware. 5165 */ 5166 ret = get_vpd_params(adap, &adap->params.vpd); 5167 if (ret < 0) 5168 goto bye; 5169 5170 /* Read firmware device log parameters. We really need to find a way 5171 * to get these parameters initialized with some default values (which 5172 * are likely to be correct) for the case where we either don't 5173 * attache to the firmware or it's crashed when we probe the adapter. 5174 * That way we'll still be able to perform early firmware startup 5175 * debugging ... If the request to get the Firmware's Device Log 5176 * parameters fails, we'll live so we don't make that a fatal error. 5177 */ 5178 memset(&devlog_cmd, 0, sizeof(devlog_cmd)); 5179 devlog_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_DEVLOG_CMD) | 5180 FW_CMD_REQUEST_F | FW_CMD_READ_F); 5181 devlog_cmd.retval_len16 = htonl(FW_LEN16(devlog_cmd)); 5182 ret = t4_wr_mbox(adap, adap->mbox, &devlog_cmd, sizeof(devlog_cmd), 5183 &devlog_cmd); 5184 if (ret == 0) { 5185 devlog_meminfo = 5186 ntohl(devlog_cmd.memtype_devlog_memaddr16_devlog); 5187 adap->params.devlog.memtype = 5188 FW_DEVLOG_CMD_MEMTYPE_DEVLOG_G(devlog_meminfo); 5189 adap->params.devlog.start = 5190 FW_DEVLOG_CMD_MEMADDR16_DEVLOG_G(devlog_meminfo) << 4; 5191 adap->params.devlog.size = ntohl(devlog_cmd.memsize_devlog); 5192 } 5193 5194 /* 5195 * Find out what ports are available to us. Note that we need to do 5196 * this before calling adap_init0_no_config() since it needs nports 5197 * and portvec ... 5198 */ 5199 v = 5200 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 5201 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 5202 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec); 5203 if (ret < 0) 5204 goto bye; 5205 5206 adap->params.nports = hweight32(port_vec); 5207 adap->params.portvec = port_vec; 5208 5209 /* If the firmware is initialized already, emit a simply note to that 5210 * effect. Otherwise, it's time to try initializing the adapter. 5211 */ 5212 if (state == DEV_STATE_INIT) { 5213 dev_info(adap->pdev_dev, "Coming up as %s: "\ 5214 "Adapter already initialized\n", 5215 adap->flags & MASTER_PF ? "MASTER" : "SLAVE"); 5216 } else { 5217 dev_info(adap->pdev_dev, "Coming up as MASTER: "\ 5218 "Initializing adapter\n"); 5219 5220 /* Find out whether we're dealing with a version of the 5221 * firmware which has configuration file support. 5222 */ 5223 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 5224 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 5225 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, 5226 params, val); 5227 5228 /* If the firmware doesn't support Configuration Files, 5229 * return an error. 5230 */ 5231 if (ret < 0) { 5232 dev_err(adap->pdev_dev, "firmware doesn't support " 5233 "Firmware Configuration Files\n"); 5234 goto bye; 5235 } 5236 5237 /* The firmware provides us with a memory buffer where we can 5238 * load a Configuration File from the host if we want to 5239 * override the Configuration File in flash. 5240 */ 5241 ret = adap_init0_config(adap, reset); 5242 if (ret == -ENOENT) { 5243 dev_err(adap->pdev_dev, "no Configuration File " 5244 "present on adapter.\n"); 5245 goto bye; 5246 } 5247 if (ret < 0) { 5248 dev_err(adap->pdev_dev, "could not initialize " 5249 "adapter, error %d\n", -ret); 5250 goto bye; 5251 } 5252 } 5253 5254 /* Give the SGE code a chance to pull in anything that it needs ... 5255 * Note that this must be called after we retrieve our VPD parameters 5256 * in order to know how to convert core ticks to seconds, etc. 5257 */ 5258 ret = t4_sge_init(adap); 5259 if (ret < 0) 5260 goto bye; 5261 5262 if (is_bypass_device(adap->pdev->device)) 5263 adap->params.bypass = 1; 5264 5265 /* 5266 * Grab some of our basic fundamental operating parameters. 5267 */ 5268 #define FW_PARAM_DEV(param) \ 5269 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \ 5270 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param)) 5271 5272 #define FW_PARAM_PFVF(param) \ 5273 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \ 5274 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \ 5275 FW_PARAMS_PARAM_Y_V(0) | \ 5276 FW_PARAMS_PARAM_Z_V(0) 5277 5278 params[0] = FW_PARAM_PFVF(EQ_START); 5279 params[1] = FW_PARAM_PFVF(L2T_START); 5280 params[2] = FW_PARAM_PFVF(L2T_END); 5281 params[3] = FW_PARAM_PFVF(FILTER_START); 5282 params[4] = FW_PARAM_PFVF(FILTER_END); 5283 params[5] = FW_PARAM_PFVF(IQFLINT_START); 5284 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val); 5285 if (ret < 0) 5286 goto bye; 5287 adap->sge.egr_start = val[0]; 5288 adap->l2t_start = val[1]; 5289 adap->l2t_end = val[2]; 5290 adap->tids.ftid_base = val[3]; 5291 adap->tids.nftids = val[4] - val[3] + 1; 5292 adap->sge.ingr_start = val[5]; 5293 5294 params[0] = FW_PARAM_PFVF(CLIP_START); 5295 params[1] = FW_PARAM_PFVF(CLIP_END); 5296 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val); 5297 if (ret < 0) 5298 goto bye; 5299 adap->clipt_start = val[0]; 5300 adap->clipt_end = val[1]; 5301 5302 /* query params related to active filter region */ 5303 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START); 5304 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END); 5305 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val); 5306 /* If Active filter size is set we enable establishing 5307 * offload connection through firmware work request 5308 */ 5309 if ((val[0] != val[1]) && (ret >= 0)) { 5310 adap->flags |= FW_OFLD_CONN; 5311 adap->tids.aftid_base = val[0]; 5312 adap->tids.aftid_end = val[1]; 5313 } 5314 5315 /* If we're running on newer firmware, let it know that we're 5316 * prepared to deal with encapsulated CPL messages. Older 5317 * firmware won't understand this and we'll just get 5318 * unencapsulated messages ... 5319 */ 5320 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 5321 val[0] = 1; 5322 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val); 5323 5324 /* 5325 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL 5326 * capability. Earlier versions of the firmware didn't have the 5327 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no 5328 * permission to use ULPTX MEMWRITE DSGL. 5329 */ 5330 if (is_t4(adap->params.chip)) { 5331 adap->params.ulptx_memwrite_dsgl = false; 5332 } else { 5333 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); 5334 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 5335 1, params, val); 5336 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0); 5337 } 5338 5339 /* 5340 * Get device capabilities so we can determine what resources we need 5341 * to manage. 5342 */ 5343 memset(&caps_cmd, 0, sizeof(caps_cmd)); 5344 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 5345 FW_CMD_REQUEST_F | FW_CMD_READ_F); 5346 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 5347 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd), 5348 &caps_cmd); 5349 if (ret < 0) 5350 goto bye; 5351 5352 if (caps_cmd.ofldcaps) { 5353 /* query offload-related parameters */ 5354 params[0] = FW_PARAM_DEV(NTID); 5355 params[1] = FW_PARAM_PFVF(SERVER_START); 5356 params[2] = FW_PARAM_PFVF(SERVER_END); 5357 params[3] = FW_PARAM_PFVF(TDDP_START); 5358 params[4] = FW_PARAM_PFVF(TDDP_END); 5359 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 5360 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, 5361 params, val); 5362 if (ret < 0) 5363 goto bye; 5364 adap->tids.ntids = val[0]; 5365 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); 5366 adap->tids.stid_base = val[1]; 5367 adap->tids.nstids = val[2] - val[1] + 1; 5368 /* 5369 * Setup server filter region. Divide the available filter 5370 * region into two parts. Regular filters get 1/3rd and server 5371 * filters get 2/3rd part. This is only enabled if workarond 5372 * path is enabled. 5373 * 1. For regular filters. 5374 * 2. Server filter: This are special filters which are used 5375 * to redirect SYN packets to offload queue. 5376 */ 5377 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) { 5378 adap->tids.sftid_base = adap->tids.ftid_base + 5379 DIV_ROUND_UP(adap->tids.nftids, 3); 5380 adap->tids.nsftids = adap->tids.nftids - 5381 DIV_ROUND_UP(adap->tids.nftids, 3); 5382 adap->tids.nftids = adap->tids.sftid_base - 5383 adap->tids.ftid_base; 5384 } 5385 adap->vres.ddp.start = val[3]; 5386 adap->vres.ddp.size = val[4] - val[3] + 1; 5387 adap->params.ofldq_wr_cred = val[5]; 5388 5389 adap->params.offload = 1; 5390 } 5391 if (caps_cmd.rdmacaps) { 5392 params[0] = FW_PARAM_PFVF(STAG_START); 5393 params[1] = FW_PARAM_PFVF(STAG_END); 5394 params[2] = FW_PARAM_PFVF(RQ_START); 5395 params[3] = FW_PARAM_PFVF(RQ_END); 5396 params[4] = FW_PARAM_PFVF(PBL_START); 5397 params[5] = FW_PARAM_PFVF(PBL_END); 5398 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, 5399 params, val); 5400 if (ret < 0) 5401 goto bye; 5402 adap->vres.stag.start = val[0]; 5403 adap->vres.stag.size = val[1] - val[0] + 1; 5404 adap->vres.rq.start = val[2]; 5405 adap->vres.rq.size = val[3] - val[2] + 1; 5406 adap->vres.pbl.start = val[4]; 5407 adap->vres.pbl.size = val[5] - val[4] + 1; 5408 5409 params[0] = FW_PARAM_PFVF(SQRQ_START); 5410 params[1] = FW_PARAM_PFVF(SQRQ_END); 5411 params[2] = FW_PARAM_PFVF(CQ_START); 5412 params[3] = FW_PARAM_PFVF(CQ_END); 5413 params[4] = FW_PARAM_PFVF(OCQ_START); 5414 params[5] = FW_PARAM_PFVF(OCQ_END); 5415 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, 5416 val); 5417 if (ret < 0) 5418 goto bye; 5419 adap->vres.qp.start = val[0]; 5420 adap->vres.qp.size = val[1] - val[0] + 1; 5421 adap->vres.cq.start = val[2]; 5422 adap->vres.cq.size = val[3] - val[2] + 1; 5423 adap->vres.ocq.start = val[4]; 5424 adap->vres.ocq.size = val[5] - val[4] + 1; 5425 5426 params[0] = FW_PARAM_DEV(MAXORDIRD_QP); 5427 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER); 5428 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, 5429 val); 5430 if (ret < 0) { 5431 adap->params.max_ordird_qp = 8; 5432 adap->params.max_ird_adapter = 32 * adap->tids.ntids; 5433 ret = 0; 5434 } else { 5435 adap->params.max_ordird_qp = val[0]; 5436 adap->params.max_ird_adapter = val[1]; 5437 } 5438 dev_info(adap->pdev_dev, 5439 "max_ordird_qp %d max_ird_adapter %d\n", 5440 adap->params.max_ordird_qp, 5441 adap->params.max_ird_adapter); 5442 } 5443 if (caps_cmd.iscsicaps) { 5444 params[0] = FW_PARAM_PFVF(ISCSI_START); 5445 params[1] = FW_PARAM_PFVF(ISCSI_END); 5446 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, 5447 params, val); 5448 if (ret < 0) 5449 goto bye; 5450 adap->vres.iscsi.start = val[0]; 5451 adap->vres.iscsi.size = val[1] - val[0] + 1; 5452 } 5453 #undef FW_PARAM_PFVF 5454 #undef FW_PARAM_DEV 5455 5456 /* The MTU/MSS Table is initialized by now, so load their values. If 5457 * we're initializing the adapter, then we'll make any modifications 5458 * we want to the MTU/MSS Table and also initialize the congestion 5459 * parameters. 5460 */ 5461 t4_read_mtu_tbl(adap, adap->params.mtus, NULL); 5462 if (state != DEV_STATE_INIT) { 5463 int i; 5464 5465 /* The default MTU Table contains values 1492 and 1500. 5466 * However, for TCP, it's better to have two values which are 5467 * a multiple of 8 +/- 4 bytes apart near this popular MTU. 5468 * This allows us to have a TCP Data Payload which is a 5469 * multiple of 8 regardless of what combination of TCP Options 5470 * are in use (always a multiple of 4 bytes) which is 5471 * important for performance reasons. For instance, if no 5472 * options are in use, then we have a 20-byte IP header and a 5473 * 20-byte TCP header. In this case, a 1500-byte MSS would 5474 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes 5475 * which is not a multiple of 8. So using an MSS of 1488 in 5476 * this case results in a TCP Data Payload of 1448 bytes which 5477 * is a multiple of 8. On the other hand, if 12-byte TCP Time 5478 * Stamps have been negotiated, then an MTU of 1500 bytes 5479 * results in a TCP Data Payload of 1448 bytes which, as 5480 * above, is a multiple of 8 bytes ... 5481 */ 5482 for (i = 0; i < NMTUS; i++) 5483 if (adap->params.mtus[i] == 1492) { 5484 adap->params.mtus[i] = 1488; 5485 break; 5486 } 5487 5488 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 5489 adap->params.b_wnd); 5490 } 5491 t4_init_sge_params(adap); 5492 t4_init_tp_params(adap); 5493 adap->flags |= FW_OK; 5494 return 0; 5495 5496 /* 5497 * Something bad happened. If a command timed out or failed with EIO 5498 * FW does not operate within its spec or something catastrophic 5499 * happened to HW/FW, stop issuing commands. 5500 */ 5501 bye: 5502 if (ret != -ETIMEDOUT && ret != -EIO) 5503 t4_fw_bye(adap, adap->mbox); 5504 return ret; 5505 } 5506 5507 /* EEH callbacks */ 5508 5509 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev, 5510 pci_channel_state_t state) 5511 { 5512 int i; 5513 struct adapter *adap = pci_get_drvdata(pdev); 5514 5515 if (!adap) 5516 goto out; 5517 5518 rtnl_lock(); 5519 adap->flags &= ~FW_OK; 5520 notify_ulds(adap, CXGB4_STATE_START_RECOVERY); 5521 spin_lock(&adap->stats_lock); 5522 for_each_port(adap, i) { 5523 struct net_device *dev = adap->port[i]; 5524 5525 netif_device_detach(dev); 5526 netif_carrier_off(dev); 5527 } 5528 spin_unlock(&adap->stats_lock); 5529 if (adap->flags & FULL_INIT_DONE) 5530 cxgb_down(adap); 5531 rtnl_unlock(); 5532 if ((adap->flags & DEV_ENABLED)) { 5533 pci_disable_device(pdev); 5534 adap->flags &= ~DEV_ENABLED; 5535 } 5536 out: return state == pci_channel_io_perm_failure ? 5537 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 5538 } 5539 5540 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) 5541 { 5542 int i, ret; 5543 struct fw_caps_config_cmd c; 5544 struct adapter *adap = pci_get_drvdata(pdev); 5545 5546 if (!adap) { 5547 pci_restore_state(pdev); 5548 pci_save_state(pdev); 5549 return PCI_ERS_RESULT_RECOVERED; 5550 } 5551 5552 if (!(adap->flags & DEV_ENABLED)) { 5553 if (pci_enable_device(pdev)) { 5554 dev_err(&pdev->dev, "Cannot reenable PCI " 5555 "device after reset\n"); 5556 return PCI_ERS_RESULT_DISCONNECT; 5557 } 5558 adap->flags |= DEV_ENABLED; 5559 } 5560 5561 pci_set_master(pdev); 5562 pci_restore_state(pdev); 5563 pci_save_state(pdev); 5564 pci_cleanup_aer_uncorrect_error_status(pdev); 5565 5566 if (t4_wait_dev_ready(adap->regs) < 0) 5567 return PCI_ERS_RESULT_DISCONNECT; 5568 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0) 5569 return PCI_ERS_RESULT_DISCONNECT; 5570 adap->flags |= FW_OK; 5571 if (adap_init1(adap, &c)) 5572 return PCI_ERS_RESULT_DISCONNECT; 5573 5574 for_each_port(adap, i) { 5575 struct port_info *p = adap2pinfo(adap, i); 5576 5577 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1, 5578 NULL, NULL); 5579 if (ret < 0) 5580 return PCI_ERS_RESULT_DISCONNECT; 5581 p->viid = ret; 5582 p->xact_addr_filt = -1; 5583 } 5584 5585 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 5586 adap->params.b_wnd); 5587 setup_memwin(adap); 5588 if (cxgb_up(adap)) 5589 return PCI_ERS_RESULT_DISCONNECT; 5590 return PCI_ERS_RESULT_RECOVERED; 5591 } 5592 5593 static void eeh_resume(struct pci_dev *pdev) 5594 { 5595 int i; 5596 struct adapter *adap = pci_get_drvdata(pdev); 5597 5598 if (!adap) 5599 return; 5600 5601 rtnl_lock(); 5602 for_each_port(adap, i) { 5603 struct net_device *dev = adap->port[i]; 5604 5605 if (netif_running(dev)) { 5606 link_start(dev); 5607 cxgb_set_rxmode(dev); 5608 } 5609 netif_device_attach(dev); 5610 } 5611 rtnl_unlock(); 5612 } 5613 5614 static const struct pci_error_handlers cxgb4_eeh = { 5615 .error_detected = eeh_err_detected, 5616 .slot_reset = eeh_slot_reset, 5617 .resume = eeh_resume, 5618 }; 5619 5620 static inline bool is_x_10g_port(const struct link_config *lc) 5621 { 5622 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 || 5623 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0; 5624 } 5625 5626 static inline void init_rspq(struct adapter *adap, struct sge_rspq *q, 5627 unsigned int us, unsigned int cnt, 5628 unsigned int size, unsigned int iqe_size) 5629 { 5630 q->adap = adap; 5631 set_rspq_intr_params(q, us, cnt); 5632 q->iqe_len = iqe_size; 5633 q->size = size; 5634 } 5635 5636 /* 5637 * Perform default configuration of DMA queues depending on the number and type 5638 * of ports we found and the number of available CPUs. Most settings can be 5639 * modified by the admin prior to actual use. 5640 */ 5641 static void cfg_queues(struct adapter *adap) 5642 { 5643 struct sge *s = &adap->sge; 5644 int i, n10g = 0, qidx = 0; 5645 #ifndef CONFIG_CHELSIO_T4_DCB 5646 int q10g = 0; 5647 #endif 5648 int ciq_size; 5649 5650 for_each_port(adap, i) 5651 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg); 5652 #ifdef CONFIG_CHELSIO_T4_DCB 5653 /* For Data Center Bridging support we need to be able to support up 5654 * to 8 Traffic Priorities; each of which will be assigned to its 5655 * own TX Queue in order to prevent Head-Of-Line Blocking. 5656 */ 5657 if (adap->params.nports * 8 > MAX_ETH_QSETS) { 5658 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n", 5659 MAX_ETH_QSETS, adap->params.nports * 8); 5660 BUG_ON(1); 5661 } 5662 5663 for_each_port(adap, i) { 5664 struct port_info *pi = adap2pinfo(adap, i); 5665 5666 pi->first_qset = qidx; 5667 pi->nqsets = 8; 5668 qidx += pi->nqsets; 5669 } 5670 #else /* !CONFIG_CHELSIO_T4_DCB */ 5671 /* 5672 * We default to 1 queue per non-10G port and up to # of cores queues 5673 * per 10G port. 5674 */ 5675 if (n10g) 5676 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g; 5677 if (q10g > netif_get_num_default_rss_queues()) 5678 q10g = netif_get_num_default_rss_queues(); 5679 5680 for_each_port(adap, i) { 5681 struct port_info *pi = adap2pinfo(adap, i); 5682 5683 pi->first_qset = qidx; 5684 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1; 5685 qidx += pi->nqsets; 5686 } 5687 #endif /* !CONFIG_CHELSIO_T4_DCB */ 5688 5689 s->ethqsets = qidx; 5690 s->max_ethqsets = qidx; /* MSI-X may lower it later */ 5691 5692 if (is_offload(adap)) { 5693 /* 5694 * For offload we use 1 queue/channel if all ports are up to 1G, 5695 * otherwise we divide all available queues amongst the channels 5696 * capped by the number of available cores. 5697 */ 5698 if (n10g) { 5699 i = min_t(int, ARRAY_SIZE(s->ofldrxq), 5700 num_online_cpus()); 5701 s->ofldqsets = roundup(i, adap->params.nports); 5702 } else 5703 s->ofldqsets = adap->params.nports; 5704 /* For RDMA one Rx queue per channel suffices */ 5705 s->rdmaqs = adap->params.nports; 5706 /* Try and allow at least 1 CIQ per cpu rounding down 5707 * to the number of ports, with a minimum of 1 per port. 5708 * A 2 port card in a 6 cpu system: 6 CIQs, 3 / port. 5709 * A 4 port card in a 6 cpu system: 4 CIQs, 1 / port. 5710 * A 4 port card in a 2 cpu system: 4 CIQs, 1 / port. 5711 */ 5712 s->rdmaciqs = min_t(int, MAX_RDMA_CIQS, num_online_cpus()); 5713 s->rdmaciqs = (s->rdmaciqs / adap->params.nports) * 5714 adap->params.nports; 5715 s->rdmaciqs = max_t(int, s->rdmaciqs, adap->params.nports); 5716 } 5717 5718 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { 5719 struct sge_eth_rxq *r = &s->ethrxq[i]; 5720 5721 init_rspq(adap, &r->rspq, 5, 10, 1024, 64); 5722 r->fl.size = 72; 5723 } 5724 5725 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) 5726 s->ethtxq[i].q.size = 1024; 5727 5728 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) 5729 s->ctrlq[i].q.size = 512; 5730 5731 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++) 5732 s->ofldtxq[i].q.size = 1024; 5733 5734 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) { 5735 struct sge_ofld_rxq *r = &s->ofldrxq[i]; 5736 5737 init_rspq(adap, &r->rspq, 5, 1, 1024, 64); 5738 r->rspq.uld = CXGB4_ULD_ISCSI; 5739 r->fl.size = 72; 5740 } 5741 5742 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) { 5743 struct sge_ofld_rxq *r = &s->rdmarxq[i]; 5744 5745 init_rspq(adap, &r->rspq, 5, 1, 511, 64); 5746 r->rspq.uld = CXGB4_ULD_RDMA; 5747 r->fl.size = 72; 5748 } 5749 5750 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids; 5751 if (ciq_size > SGE_MAX_IQ_SIZE) { 5752 CH_WARN(adap, "CIQ size too small for available IQs\n"); 5753 ciq_size = SGE_MAX_IQ_SIZE; 5754 } 5755 5756 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) { 5757 struct sge_ofld_rxq *r = &s->rdmaciq[i]; 5758 5759 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64); 5760 r->rspq.uld = CXGB4_ULD_RDMA; 5761 } 5762 5763 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64); 5764 init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64); 5765 } 5766 5767 /* 5768 * Reduce the number of Ethernet queues across all ports to at most n. 5769 * n provides at least one queue per port. 5770 */ 5771 static void reduce_ethqs(struct adapter *adap, int n) 5772 { 5773 int i; 5774 struct port_info *pi; 5775 5776 while (n < adap->sge.ethqsets) 5777 for_each_port(adap, i) { 5778 pi = adap2pinfo(adap, i); 5779 if (pi->nqsets > 1) { 5780 pi->nqsets--; 5781 adap->sge.ethqsets--; 5782 if (adap->sge.ethqsets <= n) 5783 break; 5784 } 5785 } 5786 5787 n = 0; 5788 for_each_port(adap, i) { 5789 pi = adap2pinfo(adap, i); 5790 pi->first_qset = n; 5791 n += pi->nqsets; 5792 } 5793 } 5794 5795 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */ 5796 #define EXTRA_VECS 2 5797 5798 static int enable_msix(struct adapter *adap) 5799 { 5800 int ofld_need = 0; 5801 int i, want, need, allocated; 5802 struct sge *s = &adap->sge; 5803 unsigned int nchan = adap->params.nports; 5804 struct msix_entry *entries; 5805 5806 entries = kmalloc(sizeof(*entries) * (MAX_INGQ + 1), 5807 GFP_KERNEL); 5808 if (!entries) 5809 return -ENOMEM; 5810 5811 for (i = 0; i < MAX_INGQ + 1; ++i) 5812 entries[i].entry = i; 5813 5814 want = s->max_ethqsets + EXTRA_VECS; 5815 if (is_offload(adap)) { 5816 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets; 5817 /* need nchan for each possible ULD */ 5818 ofld_need = 3 * nchan; 5819 } 5820 #ifdef CONFIG_CHELSIO_T4_DCB 5821 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for 5822 * each port. 5823 */ 5824 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need; 5825 #else 5826 need = adap->params.nports + EXTRA_VECS + ofld_need; 5827 #endif 5828 allocated = pci_enable_msix_range(adap->pdev, entries, need, want); 5829 if (allocated < 0) { 5830 dev_info(adap->pdev_dev, "not enough MSI-X vectors left," 5831 " not using MSI-X\n"); 5832 kfree(entries); 5833 return allocated; 5834 } 5835 5836 /* Distribute available vectors to the various queue groups. 5837 * Every group gets its minimum requirement and NIC gets top 5838 * priority for leftovers. 5839 */ 5840 i = allocated - EXTRA_VECS - ofld_need; 5841 if (i < s->max_ethqsets) { 5842 s->max_ethqsets = i; 5843 if (i < s->ethqsets) 5844 reduce_ethqs(adap, i); 5845 } 5846 if (is_offload(adap)) { 5847 if (allocated < want) { 5848 s->rdmaqs = nchan; 5849 s->rdmaciqs = nchan; 5850 } 5851 5852 /* leftovers go to OFLD */ 5853 i = allocated - EXTRA_VECS - s->max_ethqsets - 5854 s->rdmaqs - s->rdmaciqs; 5855 s->ofldqsets = (i / nchan) * nchan; /* round down */ 5856 } 5857 for (i = 0; i < allocated; ++i) 5858 adap->msix_info[i].vec = entries[i].vector; 5859 5860 kfree(entries); 5861 return 0; 5862 } 5863 5864 #undef EXTRA_VECS 5865 5866 static int init_rss(struct adapter *adap) 5867 { 5868 unsigned int i, j; 5869 5870 for_each_port(adap, i) { 5871 struct port_info *pi = adap2pinfo(adap, i); 5872 5873 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); 5874 if (!pi->rss) 5875 return -ENOMEM; 5876 for (j = 0; j < pi->rss_size; j++) 5877 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets); 5878 } 5879 return 0; 5880 } 5881 5882 static void print_port_info(const struct net_device *dev) 5883 { 5884 char buf[80]; 5885 char *bufp = buf; 5886 const char *spd = ""; 5887 const struct port_info *pi = netdev_priv(dev); 5888 const struct adapter *adap = pi->adapter; 5889 5890 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB) 5891 spd = " 2.5 GT/s"; 5892 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB) 5893 spd = " 5 GT/s"; 5894 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB) 5895 spd = " 8 GT/s"; 5896 5897 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M) 5898 bufp += sprintf(bufp, "100/"); 5899 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G) 5900 bufp += sprintf(bufp, "1000/"); 5901 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) 5902 bufp += sprintf(bufp, "10G/"); 5903 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) 5904 bufp += sprintf(bufp, "40G/"); 5905 if (bufp != buf) 5906 --bufp; 5907 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type)); 5908 5909 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n", 5910 adap->params.vpd.id, 5911 CHELSIO_CHIP_RELEASE(adap->params.chip), buf, 5912 is_offload(adap) ? "R" : "", adap->params.pci.width, spd, 5913 (adap->flags & USING_MSIX) ? " MSI-X" : 5914 (adap->flags & USING_MSI) ? " MSI" : ""); 5915 netdev_info(dev, "S/N: %s, P/N: %s\n", 5916 adap->params.vpd.sn, adap->params.vpd.pn); 5917 } 5918 5919 static void enable_pcie_relaxed_ordering(struct pci_dev *dev) 5920 { 5921 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); 5922 } 5923 5924 /* 5925 * Free the following resources: 5926 * - memory used for tables 5927 * - MSI/MSI-X 5928 * - net devices 5929 * - resources FW is holding for us 5930 */ 5931 static void free_some_resources(struct adapter *adapter) 5932 { 5933 unsigned int i; 5934 5935 t4_free_mem(adapter->l2t); 5936 t4_free_mem(adapter->tids.tid_tab); 5937 disable_msi(adapter); 5938 5939 for_each_port(adapter, i) 5940 if (adapter->port[i]) { 5941 kfree(adap2pinfo(adapter, i)->rss); 5942 free_netdev(adapter->port[i]); 5943 } 5944 if (adapter->flags & FW_OK) 5945 t4_fw_bye(adapter, adapter->fn); 5946 } 5947 5948 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN) 5949 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ 5950 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) 5951 #define SEGMENT_SIZE 128 5952 5953 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 5954 { 5955 int func, i, err, s_qpp, qpp, num_seg; 5956 struct port_info *pi; 5957 bool highdma = false; 5958 struct adapter *adapter = NULL; 5959 void __iomem *regs; 5960 5961 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION); 5962 5963 err = pci_request_regions(pdev, KBUILD_MODNAME); 5964 if (err) { 5965 /* Just info, some other driver may have claimed the device. */ 5966 dev_info(&pdev->dev, "cannot obtain PCI resources\n"); 5967 return err; 5968 } 5969 5970 err = pci_enable_device(pdev); 5971 if (err) { 5972 dev_err(&pdev->dev, "cannot enable PCI device\n"); 5973 goto out_release_regions; 5974 } 5975 5976 regs = pci_ioremap_bar(pdev, 0); 5977 if (!regs) { 5978 dev_err(&pdev->dev, "cannot map device registers\n"); 5979 err = -ENOMEM; 5980 goto out_disable_device; 5981 } 5982 5983 err = t4_wait_dev_ready(regs); 5984 if (err < 0) 5985 goto out_unmap_bar0; 5986 5987 /* We control everything through one PF */ 5988 func = SOURCEPF_G(readl(regs + PL_WHOAMI_A)); 5989 if (func != ent->driver_data) { 5990 iounmap(regs); 5991 pci_disable_device(pdev); 5992 pci_save_state(pdev); /* to restore SR-IOV later */ 5993 goto sriov; 5994 } 5995 5996 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 5997 highdma = true; 5998 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 5999 if (err) { 6000 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for " 6001 "coherent allocations\n"); 6002 goto out_unmap_bar0; 6003 } 6004 } else { 6005 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 6006 if (err) { 6007 dev_err(&pdev->dev, "no usable DMA configuration\n"); 6008 goto out_unmap_bar0; 6009 } 6010 } 6011 6012 pci_enable_pcie_error_reporting(pdev); 6013 enable_pcie_relaxed_ordering(pdev); 6014 pci_set_master(pdev); 6015 pci_save_state(pdev); 6016 6017 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 6018 if (!adapter) { 6019 err = -ENOMEM; 6020 goto out_unmap_bar0; 6021 } 6022 6023 adapter->workq = create_singlethread_workqueue("cxgb4"); 6024 if (!adapter->workq) { 6025 err = -ENOMEM; 6026 goto out_free_adapter; 6027 } 6028 6029 /* PCI device has been enabled */ 6030 adapter->flags |= DEV_ENABLED; 6031 6032 adapter->regs = regs; 6033 adapter->pdev = pdev; 6034 adapter->pdev_dev = &pdev->dev; 6035 adapter->mbox = func; 6036 adapter->fn = func; 6037 adapter->msg_enable = dflt_msg_enable; 6038 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); 6039 6040 spin_lock_init(&adapter->stats_lock); 6041 spin_lock_init(&adapter->tid_release_lock); 6042 spin_lock_init(&adapter->win0_lock); 6043 6044 INIT_WORK(&adapter->tid_release_task, process_tid_release_list); 6045 INIT_WORK(&adapter->db_full_task, process_db_full); 6046 INIT_WORK(&adapter->db_drop_task, process_db_drop); 6047 6048 err = t4_prep_adapter(adapter); 6049 if (err) 6050 goto out_free_adapter; 6051 6052 6053 if (!is_t4(adapter->params.chip)) { 6054 s_qpp = (QUEUESPERPAGEPF0_S + 6055 (QUEUESPERPAGEPF1_S - QUEUESPERPAGEPF0_S) * 6056 adapter->fn); 6057 qpp = 1 << QUEUESPERPAGEPF0_G(t4_read_reg(adapter, 6058 SGE_EGRESS_QUEUES_PER_PAGE_PF_A) >> s_qpp); 6059 num_seg = PAGE_SIZE / SEGMENT_SIZE; 6060 6061 /* Each segment size is 128B. Write coalescing is enabled only 6062 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the 6063 * queue is less no of segments that can be accommodated in 6064 * a page size. 6065 */ 6066 if (qpp > num_seg) { 6067 dev_err(&pdev->dev, 6068 "Incorrect number of egress queues per page\n"); 6069 err = -EINVAL; 6070 goto out_free_adapter; 6071 } 6072 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), 6073 pci_resource_len(pdev, 2)); 6074 if (!adapter->bar2) { 6075 dev_err(&pdev->dev, "cannot map device bar2 region\n"); 6076 err = -ENOMEM; 6077 goto out_free_adapter; 6078 } 6079 } 6080 6081 setup_memwin(adapter); 6082 err = adap_init0(adapter); 6083 setup_memwin_rdma(adapter); 6084 if (err) 6085 goto out_unmap_bar; 6086 6087 for_each_port(adapter, i) { 6088 struct net_device *netdev; 6089 6090 netdev = alloc_etherdev_mq(sizeof(struct port_info), 6091 MAX_ETH_QSETS); 6092 if (!netdev) { 6093 err = -ENOMEM; 6094 goto out_free_dev; 6095 } 6096 6097 SET_NETDEV_DEV(netdev, &pdev->dev); 6098 6099 adapter->port[i] = netdev; 6100 pi = netdev_priv(netdev); 6101 pi->adapter = adapter; 6102 pi->xact_addr_filt = -1; 6103 pi->port_id = i; 6104 netdev->irq = pdev->irq; 6105 6106 netdev->hw_features = NETIF_F_SG | TSO_FLAGS | 6107 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 6108 NETIF_F_RXCSUM | NETIF_F_RXHASH | 6109 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 6110 if (highdma) 6111 netdev->hw_features |= NETIF_F_HIGHDMA; 6112 netdev->features |= netdev->hw_features; 6113 netdev->vlan_features = netdev->features & VLAN_FEAT; 6114 6115 netdev->priv_flags |= IFF_UNICAST_FLT; 6116 6117 netdev->netdev_ops = &cxgb4_netdev_ops; 6118 #ifdef CONFIG_CHELSIO_T4_DCB 6119 netdev->dcbnl_ops = &cxgb4_dcb_ops; 6120 cxgb4_dcb_state_init(netdev); 6121 #endif 6122 netdev->ethtool_ops = &cxgb_ethtool_ops; 6123 } 6124 6125 pci_set_drvdata(pdev, adapter); 6126 6127 if (adapter->flags & FW_OK) { 6128 err = t4_port_init(adapter, func, func, 0); 6129 if (err) 6130 goto out_free_dev; 6131 } 6132 6133 /* 6134 * Configure queues and allocate tables now, they can be needed as 6135 * soon as the first register_netdev completes. 6136 */ 6137 cfg_queues(adapter); 6138 6139 adapter->l2t = t4_init_l2t(); 6140 if (!adapter->l2t) { 6141 /* We tolerate a lack of L2T, giving up some functionality */ 6142 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n"); 6143 adapter->params.offload = 0; 6144 } 6145 6146 #if IS_ENABLED(CONFIG_IPV6) 6147 adapter->clipt = t4_init_clip_tbl(adapter->clipt_start, 6148 adapter->clipt_end); 6149 if (!adapter->clipt) { 6150 /* We tolerate a lack of clip_table, giving up 6151 * some functionality 6152 */ 6153 dev_warn(&pdev->dev, 6154 "could not allocate Clip table, continuing\n"); 6155 adapter->params.offload = 0; 6156 } 6157 #endif 6158 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) { 6159 dev_warn(&pdev->dev, "could not allocate TID table, " 6160 "continuing\n"); 6161 adapter->params.offload = 0; 6162 } 6163 6164 /* See what interrupts we'll be using */ 6165 if (msi > 1 && enable_msix(adapter) == 0) 6166 adapter->flags |= USING_MSIX; 6167 else if (msi > 0 && pci_enable_msi(pdev) == 0) 6168 adapter->flags |= USING_MSI; 6169 6170 err = init_rss(adapter); 6171 if (err) 6172 goto out_free_dev; 6173 6174 /* 6175 * The card is now ready to go. If any errors occur during device 6176 * registration we do not fail the whole card but rather proceed only 6177 * with the ports we manage to register successfully. However we must 6178 * register at least one net device. 6179 */ 6180 for_each_port(adapter, i) { 6181 pi = adap2pinfo(adapter, i); 6182 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets); 6183 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets); 6184 6185 err = register_netdev(adapter->port[i]); 6186 if (err) 6187 break; 6188 adapter->chan_map[pi->tx_chan] = i; 6189 print_port_info(adapter->port[i]); 6190 } 6191 if (i == 0) { 6192 dev_err(&pdev->dev, "could not register any net devices\n"); 6193 goto out_free_dev; 6194 } 6195 if (err) { 6196 dev_warn(&pdev->dev, "only %d net devices registered\n", i); 6197 err = 0; 6198 } 6199 6200 if (cxgb4_debugfs_root) { 6201 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev), 6202 cxgb4_debugfs_root); 6203 setup_debugfs(adapter); 6204 } 6205 6206 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */ 6207 pdev->needs_freset = 1; 6208 6209 if (is_offload(adapter)) 6210 attach_ulds(adapter); 6211 6212 sriov: 6213 #ifdef CONFIG_PCI_IOV 6214 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0) 6215 if (pci_enable_sriov(pdev, num_vf[func]) == 0) 6216 dev_info(&pdev->dev, 6217 "instantiated %u virtual functions\n", 6218 num_vf[func]); 6219 #endif 6220 return 0; 6221 6222 out_free_dev: 6223 free_some_resources(adapter); 6224 out_unmap_bar: 6225 if (!is_t4(adapter->params.chip)) 6226 iounmap(adapter->bar2); 6227 out_free_adapter: 6228 if (adapter->workq) 6229 destroy_workqueue(adapter->workq); 6230 6231 kfree(adapter); 6232 out_unmap_bar0: 6233 iounmap(regs); 6234 out_disable_device: 6235 pci_disable_pcie_error_reporting(pdev); 6236 pci_disable_device(pdev); 6237 out_release_regions: 6238 pci_release_regions(pdev); 6239 return err; 6240 } 6241 6242 static void remove_one(struct pci_dev *pdev) 6243 { 6244 struct adapter *adapter = pci_get_drvdata(pdev); 6245 6246 #ifdef CONFIG_PCI_IOV 6247 pci_disable_sriov(pdev); 6248 6249 #endif 6250 6251 if (adapter) { 6252 int i; 6253 6254 /* Tear down per-adapter Work Queue first since it can contain 6255 * references to our adapter data structure. 6256 */ 6257 destroy_workqueue(adapter->workq); 6258 6259 if (is_offload(adapter)) 6260 detach_ulds(adapter); 6261 6262 for_each_port(adapter, i) 6263 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 6264 unregister_netdev(adapter->port[i]); 6265 6266 debugfs_remove_recursive(adapter->debugfs_root); 6267 6268 /* If we allocated filters, free up state associated with any 6269 * valid filters ... 6270 */ 6271 if (adapter->tids.ftid_tab) { 6272 struct filter_entry *f = &adapter->tids.ftid_tab[0]; 6273 for (i = 0; i < (adapter->tids.nftids + 6274 adapter->tids.nsftids); i++, f++) 6275 if (f->valid) 6276 clear_filter(adapter, f); 6277 } 6278 6279 if (adapter->flags & FULL_INIT_DONE) 6280 cxgb_down(adapter); 6281 6282 free_some_resources(adapter); 6283 #if IS_ENABLED(CONFIG_IPV6) 6284 t4_cleanup_clip_tbl(adapter); 6285 #endif 6286 iounmap(adapter->regs); 6287 if (!is_t4(adapter->params.chip)) 6288 iounmap(adapter->bar2); 6289 pci_disable_pcie_error_reporting(pdev); 6290 if ((adapter->flags & DEV_ENABLED)) { 6291 pci_disable_device(pdev); 6292 adapter->flags &= ~DEV_ENABLED; 6293 } 6294 pci_release_regions(pdev); 6295 synchronize_rcu(); 6296 kfree(adapter); 6297 } else 6298 pci_release_regions(pdev); 6299 } 6300 6301 static struct pci_driver cxgb4_driver = { 6302 .name = KBUILD_MODNAME, 6303 .id_table = cxgb4_pci_tbl, 6304 .probe = init_one, 6305 .remove = remove_one, 6306 .shutdown = remove_one, 6307 .err_handler = &cxgb4_eeh, 6308 }; 6309 6310 static int __init cxgb4_init_module(void) 6311 { 6312 int ret; 6313 6314 /* Debugfs support is optional, just warn if this fails */ 6315 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); 6316 if (!cxgb4_debugfs_root) 6317 pr_warn("could not create debugfs entry, continuing\n"); 6318 6319 ret = pci_register_driver(&cxgb4_driver); 6320 if (ret < 0) 6321 debugfs_remove(cxgb4_debugfs_root); 6322 6323 #if IS_ENABLED(CONFIG_IPV6) 6324 if (!inet6addr_registered) { 6325 register_inet6addr_notifier(&cxgb4_inet6addr_notifier); 6326 inet6addr_registered = true; 6327 } 6328 #endif 6329 6330 return ret; 6331 } 6332 6333 static void __exit cxgb4_cleanup_module(void) 6334 { 6335 #if IS_ENABLED(CONFIG_IPV6) 6336 if (inet6addr_registered) { 6337 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier); 6338 inet6addr_registered = false; 6339 } 6340 #endif 6341 pci_unregister_driver(&cxgb4_driver); 6342 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ 6343 } 6344 6345 module_init(cxgb4_init_module); 6346 module_exit(cxgb4_cleanup_module); 6347