1 /* 2 * This file is part of the Chelsio T4 Ethernet driver for Linux. 3 * 4 * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 36 37 #include <linux/bitmap.h> 38 #include <linux/crc32.h> 39 #include <linux/ctype.h> 40 #include <linux/debugfs.h> 41 #include <linux/err.h> 42 #include <linux/etherdevice.h> 43 #include <linux/firmware.h> 44 #include <linux/if.h> 45 #include <linux/if_vlan.h> 46 #include <linux/init.h> 47 #include <linux/log2.h> 48 #include <linux/mdio.h> 49 #include <linux/module.h> 50 #include <linux/moduleparam.h> 51 #include <linux/mutex.h> 52 #include <linux/netdevice.h> 53 #include <linux/pci.h> 54 #include <linux/aer.h> 55 #include <linux/rtnetlink.h> 56 #include <linux/sched.h> 57 #include <linux/seq_file.h> 58 #include <linux/sockios.h> 59 #include <linux/vmalloc.h> 60 #include <linux/workqueue.h> 61 #include <net/neighbour.h> 62 #include <net/netevent.h> 63 #include <net/addrconf.h> 64 #include <net/bonding.h> 65 #include <asm/uaccess.h> 66 67 #include "cxgb4.h" 68 #include "t4_regs.h" 69 #include "t4_msg.h" 70 #include "t4fw_api.h" 71 #include "cxgb4_dcb.h" 72 #include "cxgb4_debugfs.h" 73 #include "l2t.h" 74 75 #ifdef DRV_VERSION 76 #undef DRV_VERSION 77 #endif 78 #define DRV_VERSION "2.0.0-ko" 79 #define DRV_DESC "Chelsio T4/T5 Network Driver" 80 81 /* 82 * Max interrupt hold-off timer value in us. Queues fall back to this value 83 * under extreme memory pressure so it's largish to give the system time to 84 * recover. 85 */ 86 #define MAX_SGE_TIMERVAL 200U 87 88 enum { 89 /* 90 * Physical Function provisioning constants. 91 */ 92 PFRES_NVI = 4, /* # of Virtual Interfaces */ 93 PFRES_NETHCTRL = 128, /* # of EQs used for ETH or CTRL Qs */ 94 PFRES_NIQFLINT = 128, /* # of ingress Qs/w Free List(s)/intr 95 */ 96 PFRES_NEQ = 256, /* # of egress queues */ 97 PFRES_NIQ = 0, /* # of ingress queues */ 98 PFRES_TC = 0, /* PCI-E traffic class */ 99 PFRES_NEXACTF = 128, /* # of exact MPS filters */ 100 101 PFRES_R_CAPS = FW_CMD_CAP_PF, 102 PFRES_WX_CAPS = FW_CMD_CAP_PF, 103 104 #ifdef CONFIG_PCI_IOV 105 /* 106 * Virtual Function provisioning constants. We need two extra Ingress 107 * Queues with Interrupt capability to serve as the VF's Firmware 108 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) -- 109 * neither will have Free Lists associated with them). For each 110 * Ethernet/Control Egress Queue and for each Free List, we need an 111 * Egress Context. 112 */ 113 VFRES_NPORTS = 1, /* # of "ports" per VF */ 114 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */ 115 116 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */ 117 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */ 118 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */ 119 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */ 120 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */ 121 VFRES_TC = 0, /* PCI-E traffic class */ 122 VFRES_NEXACTF = 16, /* # of exact MPS filters */ 123 124 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT, 125 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF, 126 #endif 127 }; 128 129 /* 130 * Provide a Port Access Rights Mask for the specified PF/VF. This is very 131 * static and likely not to be useful in the long run. We really need to 132 * implement some form of persistent configuration which the firmware 133 * controls. 134 */ 135 static unsigned int pfvfres_pmask(struct adapter *adapter, 136 unsigned int pf, unsigned int vf) 137 { 138 unsigned int portn, portvec; 139 140 /* 141 * Give PF's access to all of the ports. 142 */ 143 if (vf == 0) 144 return FW_PFVF_CMD_PMASK_M; 145 146 /* 147 * For VFs, we'll assign them access to the ports based purely on the 148 * PF. We assign active ports in order, wrapping around if there are 149 * fewer active ports than PFs: e.g. active port[pf % nports]. 150 * Unfortunately the adapter's port_info structs haven't been 151 * initialized yet so we have to compute this. 152 */ 153 if (adapter->params.nports == 0) 154 return 0; 155 156 portn = pf % adapter->params.nports; 157 portvec = adapter->params.portvec; 158 for (;;) { 159 /* 160 * Isolate the lowest set bit in the port vector. If we're at 161 * the port number that we want, return that as the pmask. 162 * otherwise mask that bit out of the port vector and 163 * decrement our port number ... 164 */ 165 unsigned int pmask = portvec ^ (portvec & (portvec-1)); 166 if (portn == 0) 167 return pmask; 168 portn--; 169 portvec &= ~pmask; 170 } 171 /*NOTREACHED*/ 172 } 173 174 enum { 175 MAX_TXQ_ENTRIES = 16384, 176 MAX_CTRL_TXQ_ENTRIES = 1024, 177 MAX_RSPQ_ENTRIES = 16384, 178 MAX_RX_BUFFERS = 16384, 179 MIN_TXQ_ENTRIES = 32, 180 MIN_CTRL_TXQ_ENTRIES = 32, 181 MIN_RSPQ_ENTRIES = 128, 182 MIN_FL_ENTRIES = 16 183 }; 184 185 /* Host shadow copy of ingress filter entry. This is in host native format 186 * and doesn't match the ordering or bit order, etc. of the hardware of the 187 * firmware command. The use of bit-field structure elements is purely to 188 * remind ourselves of the field size limitations and save memory in the case 189 * where the filter table is large. 190 */ 191 struct filter_entry { 192 /* Administrative fields for filter. 193 */ 194 u32 valid:1; /* filter allocated and valid */ 195 u32 locked:1; /* filter is administratively locked */ 196 197 u32 pending:1; /* filter action is pending firmware reply */ 198 u32 smtidx:8; /* Source MAC Table index for smac */ 199 struct l2t_entry *l2t; /* Layer Two Table entry for dmac */ 200 201 /* The filter itself. Most of this is a straight copy of information 202 * provided by the extended ioctl(). Some fields are translated to 203 * internal forms -- for instance the Ingress Queue ID passed in from 204 * the ioctl() is translated into the Absolute Ingress Queue ID. 205 */ 206 struct ch_filter_specification fs; 207 }; 208 209 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \ 210 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\ 211 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR) 212 213 /* Macros needed to support the PCI Device ID Table ... 214 */ 215 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ 216 static struct pci_device_id cxgb4_pci_tbl[] = { 217 #define CH_PCI_DEVICE_ID_FUNCTION 0x4 218 219 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is 220 * called for both. 221 */ 222 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0 223 224 #define CH_PCI_ID_TABLE_ENTRY(devid) \ 225 {PCI_VDEVICE(CHELSIO, (devid)), 4} 226 227 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ 228 { 0, } \ 229 } 230 231 #include "t4_pci_id_tbl.h" 232 233 #define FW4_FNAME "cxgb4/t4fw.bin" 234 #define FW5_FNAME "cxgb4/t5fw.bin" 235 #define FW4_CFNAME "cxgb4/t4-config.txt" 236 #define FW5_CFNAME "cxgb4/t5-config.txt" 237 238 MODULE_DESCRIPTION(DRV_DESC); 239 MODULE_AUTHOR("Chelsio Communications"); 240 MODULE_LICENSE("Dual BSD/GPL"); 241 MODULE_VERSION(DRV_VERSION); 242 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl); 243 MODULE_FIRMWARE(FW4_FNAME); 244 MODULE_FIRMWARE(FW5_FNAME); 245 246 /* 247 * Normally we're willing to become the firmware's Master PF but will be happy 248 * if another PF has already become the Master and initialized the adapter. 249 * Setting "force_init" will cause this driver to forcibly establish itself as 250 * the Master PF and initialize the adapter. 251 */ 252 static uint force_init; 253 254 module_param(force_init, uint, 0644); 255 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter"); 256 257 /* 258 * Normally if the firmware we connect to has Configuration File support, we 259 * use that and only fall back to the old Driver-based initialization if the 260 * Configuration File fails for some reason. If force_old_init is set, then 261 * we'll always use the old Driver-based initialization sequence. 262 */ 263 static uint force_old_init; 264 265 module_param(force_old_init, uint, 0644); 266 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence"); 267 268 static int dflt_msg_enable = DFLT_MSG_ENABLE; 269 270 module_param(dflt_msg_enable, int, 0644); 271 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap"); 272 273 /* 274 * The driver uses the best interrupt scheme available on a platform in the 275 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which 276 * of these schemes the driver may consider as follows: 277 * 278 * msi = 2: choose from among all three options 279 * msi = 1: only consider MSI and INTx interrupts 280 * msi = 0: force INTx interrupts 281 */ 282 static int msi = 2; 283 284 module_param(msi, int, 0644); 285 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)"); 286 287 /* 288 * Queue interrupt hold-off timer values. Queues default to the first of these 289 * upon creation. 290 */ 291 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 }; 292 293 module_param_array(intr_holdoff, uint, NULL, 0644); 294 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers " 295 "0..4 in microseconds"); 296 297 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 }; 298 299 module_param_array(intr_cnt, uint, NULL, 0644); 300 MODULE_PARM_DESC(intr_cnt, 301 "thresholds 1..3 for queue interrupt packet counters"); 302 303 /* 304 * Normally we tell the chip to deliver Ingress Packets into our DMA buffers 305 * offset by 2 bytes in order to have the IP headers line up on 4-byte 306 * boundaries. This is a requirement for many architectures which will throw 307 * a machine check fault if an attempt is made to access one of the 4-byte IP 308 * header fields on a non-4-byte boundary. And it's a major performance issue 309 * even on some architectures which allow it like some implementations of the 310 * x86 ISA. However, some architectures don't mind this and for some very 311 * edge-case performance sensitive applications (like forwarding large volumes 312 * of small packets), setting this DMA offset to 0 will decrease the number of 313 * PCI-E Bus transfers enough to measurably affect performance. 314 */ 315 static int rx_dma_offset = 2; 316 317 static bool vf_acls; 318 319 #ifdef CONFIG_PCI_IOV 320 module_param(vf_acls, bool, 0644); 321 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement"); 322 323 /* Configure the number of PCI-E Virtual Function which are to be instantiated 324 * on SR-IOV Capable Physical Functions. 325 */ 326 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV]; 327 328 module_param_array(num_vf, uint, NULL, 0644); 329 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3"); 330 #endif 331 332 /* TX Queue select used to determine what algorithm to use for selecting TX 333 * queue. Select between the kernel provided function (select_queue=0) or user 334 * cxgb_select_queue function (select_queue=1) 335 * 336 * Default: select_queue=0 337 */ 338 static int select_queue; 339 module_param(select_queue, int, 0644); 340 MODULE_PARM_DESC(select_queue, 341 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method."); 342 343 /* 344 * The filter TCAM has a fixed portion and a variable portion. The fixed 345 * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP 346 * ports. The variable portion is 36 bits which can include things like Exact 347 * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits), 348 * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would 349 * far exceed the 36-bit budget for this "compressed" header portion of the 350 * filter. Thus, we have a scarce resource which must be carefully managed. 351 * 352 * By default we set this up to mostly match the set of filter matching 353 * capabilities of T3 but with accommodations for some of T4's more 354 * interesting features: 355 * 356 * { IP Fragment (1), MPS Match Type (3), IP Protocol (8), 357 * [Inner] VLAN (17), Port (3), FCoE (1) } 358 */ 359 enum { 360 TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC, 361 TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT, 362 TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT, 363 }; 364 365 static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT; 366 367 module_param(tp_vlan_pri_map, uint, 0644); 368 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration"); 369 370 static struct dentry *cxgb4_debugfs_root; 371 372 static LIST_HEAD(adapter_list); 373 static DEFINE_MUTEX(uld_mutex); 374 /* Adapter list to be accessed from atomic context */ 375 static LIST_HEAD(adap_rcu_list); 376 static DEFINE_SPINLOCK(adap_rcu_lock); 377 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX]; 378 static const char *uld_str[] = { "RDMA", "iSCSI" }; 379 380 static void link_report(struct net_device *dev) 381 { 382 if (!netif_carrier_ok(dev)) 383 netdev_info(dev, "link down\n"); 384 else { 385 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" }; 386 387 const char *s = "10Mbps"; 388 const struct port_info *p = netdev_priv(dev); 389 390 switch (p->link_cfg.speed) { 391 case 10000: 392 s = "10Gbps"; 393 break; 394 case 1000: 395 s = "1000Mbps"; 396 break; 397 case 100: 398 s = "100Mbps"; 399 break; 400 case 40000: 401 s = "40Gbps"; 402 break; 403 } 404 405 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, 406 fc[p->link_cfg.fc]); 407 } 408 } 409 410 #ifdef CONFIG_CHELSIO_T4_DCB 411 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */ 412 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable) 413 { 414 struct port_info *pi = netdev_priv(dev); 415 struct adapter *adap = pi->adapter; 416 struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset]; 417 int i; 418 419 /* We use a simple mapping of Port TX Queue Index to DCB 420 * Priority when we're enabling DCB. 421 */ 422 for (i = 0; i < pi->nqsets; i++, txq++) { 423 u32 name, value; 424 int err; 425 426 name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 427 FW_PARAMS_PARAM_X_V( 428 FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) | 429 FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id)); 430 value = enable ? i : 0xffffffff; 431 432 /* Since we can be called while atomic (from "interrupt 433 * level") we need to issue the Set Parameters Commannd 434 * without sleeping (timeout < 0). 435 */ 436 err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1, 437 &name, &value); 438 439 if (err) 440 dev_err(adap->pdev_dev, 441 "Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n", 442 enable ? "set" : "unset", pi->port_id, i, -err); 443 else 444 txq->dcb_prio = value; 445 } 446 } 447 #endif /* CONFIG_CHELSIO_T4_DCB */ 448 449 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat) 450 { 451 struct net_device *dev = adapter->port[port_id]; 452 453 /* Skip changes from disabled ports. */ 454 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) { 455 if (link_stat) 456 netif_carrier_on(dev); 457 else { 458 #ifdef CONFIG_CHELSIO_T4_DCB 459 cxgb4_dcb_state_init(dev); 460 dcb_tx_queue_prio_enable(dev, false); 461 #endif /* CONFIG_CHELSIO_T4_DCB */ 462 netif_carrier_off(dev); 463 } 464 465 link_report(dev); 466 } 467 } 468 469 void t4_os_portmod_changed(const struct adapter *adap, int port_id) 470 { 471 static const char *mod_str[] = { 472 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM" 473 }; 474 475 const struct net_device *dev = adap->port[port_id]; 476 const struct port_info *pi = netdev_priv(dev); 477 478 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE) 479 netdev_info(dev, "port module unplugged\n"); 480 else if (pi->mod_type < ARRAY_SIZE(mod_str)) 481 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]); 482 } 483 484 /* 485 * Configure the exact and hash address filters to handle a port's multicast 486 * and secondary unicast MAC addresses. 487 */ 488 static int set_addr_filters(const struct net_device *dev, bool sleep) 489 { 490 u64 mhash = 0; 491 u64 uhash = 0; 492 bool free = true; 493 u16 filt_idx[7]; 494 const u8 *addr[7]; 495 int ret, naddr = 0; 496 const struct netdev_hw_addr *ha; 497 int uc_cnt = netdev_uc_count(dev); 498 int mc_cnt = netdev_mc_count(dev); 499 const struct port_info *pi = netdev_priv(dev); 500 unsigned int mb = pi->adapter->fn; 501 502 /* first do the secondary unicast addresses */ 503 netdev_for_each_uc_addr(ha, dev) { 504 addr[naddr++] = ha->addr; 505 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) { 506 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free, 507 naddr, addr, filt_idx, &uhash, sleep); 508 if (ret < 0) 509 return ret; 510 511 free = false; 512 naddr = 0; 513 } 514 } 515 516 /* next set up the multicast addresses */ 517 netdev_for_each_mc_addr(ha, dev) { 518 addr[naddr++] = ha->addr; 519 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) { 520 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free, 521 naddr, addr, filt_idx, &mhash, sleep); 522 if (ret < 0) 523 return ret; 524 525 free = false; 526 naddr = 0; 527 } 528 } 529 530 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0, 531 uhash | mhash, sleep); 532 } 533 534 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */ 535 module_param(dbfifo_int_thresh, int, 0644); 536 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold"); 537 538 /* 539 * usecs to sleep while draining the dbfifo 540 */ 541 static int dbfifo_drain_delay = 1000; 542 module_param(dbfifo_drain_delay, int, 0644); 543 MODULE_PARM_DESC(dbfifo_drain_delay, 544 "usecs to sleep while draining the dbfifo"); 545 546 /* 547 * Set Rx properties of a port, such as promiscruity, address filters, and MTU. 548 * If @mtu is -1 it is left unchanged. 549 */ 550 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok) 551 { 552 int ret; 553 struct port_info *pi = netdev_priv(dev); 554 555 ret = set_addr_filters(dev, sleep_ok); 556 if (ret == 0) 557 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu, 558 (dev->flags & IFF_PROMISC) ? 1 : 0, 559 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1, 560 sleep_ok); 561 return ret; 562 } 563 564 /** 565 * link_start - enable a port 566 * @dev: the port to enable 567 * 568 * Performs the MAC and PHY actions needed to enable a port. 569 */ 570 static int link_start(struct net_device *dev) 571 { 572 int ret; 573 struct port_info *pi = netdev_priv(dev); 574 unsigned int mb = pi->adapter->fn; 575 576 /* 577 * We do not set address filters and promiscuity here, the stack does 578 * that step explicitly. 579 */ 580 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1, 581 !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true); 582 if (ret == 0) { 583 ret = t4_change_mac(pi->adapter, mb, pi->viid, 584 pi->xact_addr_filt, dev->dev_addr, true, 585 true); 586 if (ret >= 0) { 587 pi->xact_addr_filt = ret; 588 ret = 0; 589 } 590 } 591 if (ret == 0) 592 ret = t4_link_start(pi->adapter, mb, pi->tx_chan, 593 &pi->link_cfg); 594 if (ret == 0) { 595 local_bh_disable(); 596 ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true, 597 true, CXGB4_DCB_ENABLED); 598 local_bh_enable(); 599 } 600 601 return ret; 602 } 603 604 int cxgb4_dcb_enabled(const struct net_device *dev) 605 { 606 #ifdef CONFIG_CHELSIO_T4_DCB 607 struct port_info *pi = netdev_priv(dev); 608 609 if (!pi->dcb.enabled) 610 return 0; 611 612 return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) || 613 (pi->dcb.state == CXGB4_DCB_STATE_HOST)); 614 #else 615 return 0; 616 #endif 617 } 618 EXPORT_SYMBOL(cxgb4_dcb_enabled); 619 620 #ifdef CONFIG_CHELSIO_T4_DCB 621 /* Handle a Data Center Bridging update message from the firmware. */ 622 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd) 623 { 624 int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid)); 625 struct net_device *dev = adap->port[port]; 626 int old_dcb_enabled = cxgb4_dcb_enabled(dev); 627 int new_dcb_enabled; 628 629 cxgb4_dcb_handle_fw_update(adap, pcmd); 630 new_dcb_enabled = cxgb4_dcb_enabled(dev); 631 632 /* If the DCB has become enabled or disabled on the port then we're 633 * going to need to set up/tear down DCB Priority parameters for the 634 * TX Queues associated with the port. 635 */ 636 if (new_dcb_enabled != old_dcb_enabled) 637 dcb_tx_queue_prio_enable(dev, new_dcb_enabled); 638 } 639 #endif /* CONFIG_CHELSIO_T4_DCB */ 640 641 /* Clear a filter and release any of its resources that we own. This also 642 * clears the filter's "pending" status. 643 */ 644 static void clear_filter(struct adapter *adap, struct filter_entry *f) 645 { 646 /* If the new or old filter have loopback rewriteing rules then we'll 647 * need to free any existing Layer Two Table (L2T) entries of the old 648 * filter rule. The firmware will handle freeing up any Source MAC 649 * Table (SMT) entries used for rewriting Source MAC Addresses in 650 * loopback rules. 651 */ 652 if (f->l2t) 653 cxgb4_l2t_release(f->l2t); 654 655 /* The zeroing of the filter rule below clears the filter valid, 656 * pending, locked flags, l2t pointer, etc. so it's all we need for 657 * this operation. 658 */ 659 memset(f, 0, sizeof(*f)); 660 } 661 662 /* Handle a filter write/deletion reply. 663 */ 664 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl) 665 { 666 unsigned int idx = GET_TID(rpl); 667 unsigned int nidx = idx - adap->tids.ftid_base; 668 unsigned int ret; 669 struct filter_entry *f; 670 671 if (idx >= adap->tids.ftid_base && nidx < 672 (adap->tids.nftids + adap->tids.nsftids)) { 673 idx = nidx; 674 ret = GET_TCB_COOKIE(rpl->cookie); 675 f = &adap->tids.ftid_tab[idx]; 676 677 if (ret == FW_FILTER_WR_FLT_DELETED) { 678 /* Clear the filter when we get confirmation from the 679 * hardware that the filter has been deleted. 680 */ 681 clear_filter(adap, f); 682 } else if (ret == FW_FILTER_WR_SMT_TBL_FULL) { 683 dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n", 684 idx); 685 clear_filter(adap, f); 686 } else if (ret == FW_FILTER_WR_FLT_ADDED) { 687 f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff; 688 f->pending = 0; /* asynchronous setup completed */ 689 f->valid = 1; 690 } else { 691 /* Something went wrong. Issue a warning about the 692 * problem and clear everything out. 693 */ 694 dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n", 695 idx, ret); 696 clear_filter(adap, f); 697 } 698 } 699 } 700 701 /* Response queue handler for the FW event queue. 702 */ 703 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp, 704 const struct pkt_gl *gl) 705 { 706 u8 opcode = ((const struct rss_header *)rsp)->opcode; 707 708 rsp++; /* skip RSS header */ 709 710 /* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG. 711 */ 712 if (unlikely(opcode == CPL_FW4_MSG && 713 ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) { 714 rsp++; 715 opcode = ((const struct rss_header *)rsp)->opcode; 716 rsp++; 717 if (opcode != CPL_SGE_EGR_UPDATE) { 718 dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n" 719 , opcode); 720 goto out; 721 } 722 } 723 724 if (likely(opcode == CPL_SGE_EGR_UPDATE)) { 725 const struct cpl_sge_egr_update *p = (void *)rsp; 726 unsigned int qid = EGR_QID(ntohl(p->opcode_qid)); 727 struct sge_txq *txq; 728 729 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start]; 730 txq->restarts++; 731 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) { 732 struct sge_eth_txq *eq; 733 734 eq = container_of(txq, struct sge_eth_txq, q); 735 netif_tx_wake_queue(eq->txq); 736 } else { 737 struct sge_ofld_txq *oq; 738 739 oq = container_of(txq, struct sge_ofld_txq, q); 740 tasklet_schedule(&oq->qresume_tsk); 741 } 742 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) { 743 const struct cpl_fw6_msg *p = (void *)rsp; 744 745 #ifdef CONFIG_CHELSIO_T4_DCB 746 const struct fw_port_cmd *pcmd = (const void *)p->data; 747 unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid)); 748 unsigned int action = 749 FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16)); 750 751 if (cmd == FW_PORT_CMD && 752 action == FW_PORT_ACTION_GET_PORT_INFO) { 753 int port = FW_PORT_CMD_PORTID_G( 754 be32_to_cpu(pcmd->op_to_portid)); 755 struct net_device *dev = q->adap->port[port]; 756 int state_input = ((pcmd->u.info.dcbxdis_pkd & 757 FW_PORT_CMD_DCBXDIS_F) 758 ? CXGB4_DCB_INPUT_FW_DISABLED 759 : CXGB4_DCB_INPUT_FW_ENABLED); 760 761 cxgb4_dcb_state_fsm(dev, state_input); 762 } 763 764 if (cmd == FW_PORT_CMD && 765 action == FW_PORT_ACTION_L2_DCB_CFG) 766 dcb_rpl(q->adap, pcmd); 767 else 768 #endif 769 if (p->type == 0) 770 t4_handle_fw_rpl(q->adap, p->data); 771 } else if (opcode == CPL_L2T_WRITE_RPL) { 772 const struct cpl_l2t_write_rpl *p = (void *)rsp; 773 774 do_l2t_write_rpl(q->adap, p); 775 } else if (opcode == CPL_SET_TCB_RPL) { 776 const struct cpl_set_tcb_rpl *p = (void *)rsp; 777 778 filter_rpl(q->adap, p); 779 } else 780 dev_err(q->adap->pdev_dev, 781 "unexpected CPL %#x on FW event queue\n", opcode); 782 out: 783 return 0; 784 } 785 786 /** 787 * uldrx_handler - response queue handler for ULD queues 788 * @q: the response queue that received the packet 789 * @rsp: the response queue descriptor holding the offload message 790 * @gl: the gather list of packet fragments 791 * 792 * Deliver an ingress offload packet to a ULD. All processing is done by 793 * the ULD, we just maintain statistics. 794 */ 795 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp, 796 const struct pkt_gl *gl) 797 { 798 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq); 799 800 /* FW can send CPLs encapsulated in a CPL_FW4_MSG. 801 */ 802 if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG && 803 ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL) 804 rsp += 2; 805 806 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) { 807 rxq->stats.nomem++; 808 return -1; 809 } 810 if (gl == NULL) 811 rxq->stats.imm++; 812 else if (gl == CXGB4_MSG_AN) 813 rxq->stats.an++; 814 else 815 rxq->stats.pkts++; 816 return 0; 817 } 818 819 static void disable_msi(struct adapter *adapter) 820 { 821 if (adapter->flags & USING_MSIX) { 822 pci_disable_msix(adapter->pdev); 823 adapter->flags &= ~USING_MSIX; 824 } else if (adapter->flags & USING_MSI) { 825 pci_disable_msi(adapter->pdev); 826 adapter->flags &= ~USING_MSI; 827 } 828 } 829 830 /* 831 * Interrupt handler for non-data events used with MSI-X. 832 */ 833 static irqreturn_t t4_nondata_intr(int irq, void *cookie) 834 { 835 struct adapter *adap = cookie; 836 837 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE)); 838 if (v & PFSW) { 839 adap->swintr = 1; 840 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v); 841 } 842 t4_slow_intr_handler(adap); 843 return IRQ_HANDLED; 844 } 845 846 /* 847 * Name the MSI-X interrupts. 848 */ 849 static void name_msix_vecs(struct adapter *adap) 850 { 851 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc); 852 853 /* non-data interrupts */ 854 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name); 855 856 /* FW events */ 857 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq", 858 adap->port[0]->name); 859 860 /* Ethernet queues */ 861 for_each_port(adap, j) { 862 struct net_device *d = adap->port[j]; 863 const struct port_info *pi = netdev_priv(d); 864 865 for (i = 0; i < pi->nqsets; i++, msi_idx++) 866 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d", 867 d->name, i); 868 } 869 870 /* offload queues */ 871 for_each_ofldrxq(&adap->sge, i) 872 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d", 873 adap->port[0]->name, i); 874 875 for_each_rdmarxq(&adap->sge, i) 876 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d", 877 adap->port[0]->name, i); 878 879 for_each_rdmaciq(&adap->sge, i) 880 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d", 881 adap->port[0]->name, i); 882 } 883 884 static int request_msix_queue_irqs(struct adapter *adap) 885 { 886 struct sge *s = &adap->sge; 887 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0; 888 int msi_index = 2; 889 890 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0, 891 adap->msix_info[1].desc, &s->fw_evtq); 892 if (err) 893 return err; 894 895 for_each_ethrxq(s, ethqidx) { 896 err = request_irq(adap->msix_info[msi_index].vec, 897 t4_sge_intr_msix, 0, 898 adap->msix_info[msi_index].desc, 899 &s->ethrxq[ethqidx].rspq); 900 if (err) 901 goto unwind; 902 msi_index++; 903 } 904 for_each_ofldrxq(s, ofldqidx) { 905 err = request_irq(adap->msix_info[msi_index].vec, 906 t4_sge_intr_msix, 0, 907 adap->msix_info[msi_index].desc, 908 &s->ofldrxq[ofldqidx].rspq); 909 if (err) 910 goto unwind; 911 msi_index++; 912 } 913 for_each_rdmarxq(s, rdmaqidx) { 914 err = request_irq(adap->msix_info[msi_index].vec, 915 t4_sge_intr_msix, 0, 916 adap->msix_info[msi_index].desc, 917 &s->rdmarxq[rdmaqidx].rspq); 918 if (err) 919 goto unwind; 920 msi_index++; 921 } 922 for_each_rdmaciq(s, rdmaciqqidx) { 923 err = request_irq(adap->msix_info[msi_index].vec, 924 t4_sge_intr_msix, 0, 925 adap->msix_info[msi_index].desc, 926 &s->rdmaciq[rdmaciqqidx].rspq); 927 if (err) 928 goto unwind; 929 msi_index++; 930 } 931 return 0; 932 933 unwind: 934 while (--rdmaciqqidx >= 0) 935 free_irq(adap->msix_info[--msi_index].vec, 936 &s->rdmaciq[rdmaciqqidx].rspq); 937 while (--rdmaqidx >= 0) 938 free_irq(adap->msix_info[--msi_index].vec, 939 &s->rdmarxq[rdmaqidx].rspq); 940 while (--ofldqidx >= 0) 941 free_irq(adap->msix_info[--msi_index].vec, 942 &s->ofldrxq[ofldqidx].rspq); 943 while (--ethqidx >= 0) 944 free_irq(adap->msix_info[--msi_index].vec, 945 &s->ethrxq[ethqidx].rspq); 946 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 947 return err; 948 } 949 950 static void free_msix_queue_irqs(struct adapter *adap) 951 { 952 int i, msi_index = 2; 953 struct sge *s = &adap->sge; 954 955 free_irq(adap->msix_info[1].vec, &s->fw_evtq); 956 for_each_ethrxq(s, i) 957 free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq); 958 for_each_ofldrxq(s, i) 959 free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq); 960 for_each_rdmarxq(s, i) 961 free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq); 962 for_each_rdmaciq(s, i) 963 free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq); 964 } 965 966 /** 967 * write_rss - write the RSS table for a given port 968 * @pi: the port 969 * @queues: array of queue indices for RSS 970 * 971 * Sets up the portion of the HW RSS table for the port's VI to distribute 972 * packets to the Rx queues in @queues. 973 */ 974 static int write_rss(const struct port_info *pi, const u16 *queues) 975 { 976 u16 *rss; 977 int i, err; 978 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset]; 979 980 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL); 981 if (!rss) 982 return -ENOMEM; 983 984 /* map the queue indices to queue ids */ 985 for (i = 0; i < pi->rss_size; i++, queues++) 986 rss[i] = q[*queues].rspq.abs_id; 987 988 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0, 989 pi->rss_size, rss, pi->rss_size); 990 kfree(rss); 991 return err; 992 } 993 994 /** 995 * setup_rss - configure RSS 996 * @adap: the adapter 997 * 998 * Sets up RSS for each port. 999 */ 1000 static int setup_rss(struct adapter *adap) 1001 { 1002 int i, err; 1003 1004 for_each_port(adap, i) { 1005 const struct port_info *pi = adap2pinfo(adap, i); 1006 1007 err = write_rss(pi, pi->rss); 1008 if (err) 1009 return err; 1010 } 1011 return 0; 1012 } 1013 1014 /* 1015 * Return the channel of the ingress queue with the given qid. 1016 */ 1017 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid) 1018 { 1019 qid -= p->ingr_start; 1020 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan; 1021 } 1022 1023 /* 1024 * Wait until all NAPI handlers are descheduled. 1025 */ 1026 static void quiesce_rx(struct adapter *adap) 1027 { 1028 int i; 1029 1030 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) { 1031 struct sge_rspq *q = adap->sge.ingr_map[i]; 1032 1033 if (q && q->handler) 1034 napi_disable(&q->napi); 1035 } 1036 } 1037 1038 /* 1039 * Enable NAPI scheduling and interrupt generation for all Rx queues. 1040 */ 1041 static void enable_rx(struct adapter *adap) 1042 { 1043 int i; 1044 1045 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) { 1046 struct sge_rspq *q = adap->sge.ingr_map[i]; 1047 1048 if (!q) 1049 continue; 1050 if (q->handler) 1051 napi_enable(&q->napi); 1052 /* 0-increment GTS to start the timer and enable interrupts */ 1053 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS), 1054 SEINTARM(q->intr_params) | 1055 INGRESSQID(q->cntxt_id)); 1056 } 1057 } 1058 1059 /** 1060 * setup_sge_queues - configure SGE Tx/Rx/response queues 1061 * @adap: the adapter 1062 * 1063 * Determines how many sets of SGE queues to use and initializes them. 1064 * We support multiple queue sets per port if we have MSI-X, otherwise 1065 * just one queue set per port. 1066 */ 1067 static int setup_sge_queues(struct adapter *adap) 1068 { 1069 int err, msi_idx, i, j; 1070 struct sge *s = &adap->sge; 1071 1072 bitmap_zero(s->starving_fl, MAX_EGRQ); 1073 bitmap_zero(s->txq_maperr, MAX_EGRQ); 1074 1075 if (adap->flags & USING_MSIX) 1076 msi_idx = 1; /* vector 0 is for non-queue interrupts */ 1077 else { 1078 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0, 1079 NULL, NULL); 1080 if (err) 1081 return err; 1082 msi_idx = -((int)s->intrq.abs_id + 1); 1083 } 1084 1085 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0], 1086 msi_idx, NULL, fwevtq_handler); 1087 if (err) { 1088 freeout: t4_free_sge_resources(adap); 1089 return err; 1090 } 1091 1092 for_each_port(adap, i) { 1093 struct net_device *dev = adap->port[i]; 1094 struct port_info *pi = netdev_priv(dev); 1095 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset]; 1096 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset]; 1097 1098 for (j = 0; j < pi->nqsets; j++, q++) { 1099 if (msi_idx > 0) 1100 msi_idx++; 1101 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, 1102 msi_idx, &q->fl, 1103 t4_ethrx_handler); 1104 if (err) 1105 goto freeout; 1106 q->rspq.idx = j; 1107 memset(&q->stats, 0, sizeof(q->stats)); 1108 } 1109 for (j = 0; j < pi->nqsets; j++, t++) { 1110 err = t4_sge_alloc_eth_txq(adap, t, dev, 1111 netdev_get_tx_queue(dev, j), 1112 s->fw_evtq.cntxt_id); 1113 if (err) 1114 goto freeout; 1115 } 1116 } 1117 1118 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */ 1119 for_each_ofldrxq(s, i) { 1120 struct sge_ofld_rxq *q = &s->ofldrxq[i]; 1121 struct net_device *dev = adap->port[i / j]; 1122 1123 if (msi_idx > 0) 1124 msi_idx++; 1125 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx, 1126 q->fl.size ? &q->fl : NULL, 1127 uldrx_handler); 1128 if (err) 1129 goto freeout; 1130 memset(&q->stats, 0, sizeof(q->stats)); 1131 s->ofld_rxq[i] = q->rspq.abs_id; 1132 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev, 1133 s->fw_evtq.cntxt_id); 1134 if (err) 1135 goto freeout; 1136 } 1137 1138 for_each_rdmarxq(s, i) { 1139 struct sge_ofld_rxq *q = &s->rdmarxq[i]; 1140 1141 if (msi_idx > 0) 1142 msi_idx++; 1143 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i], 1144 msi_idx, q->fl.size ? &q->fl : NULL, 1145 uldrx_handler); 1146 if (err) 1147 goto freeout; 1148 memset(&q->stats, 0, sizeof(q->stats)); 1149 s->rdma_rxq[i] = q->rspq.abs_id; 1150 } 1151 1152 for_each_rdmaciq(s, i) { 1153 struct sge_ofld_rxq *q = &s->rdmaciq[i]; 1154 1155 if (msi_idx > 0) 1156 msi_idx++; 1157 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i], 1158 msi_idx, q->fl.size ? &q->fl : NULL, 1159 uldrx_handler); 1160 if (err) 1161 goto freeout; 1162 memset(&q->stats, 0, sizeof(q->stats)); 1163 s->rdma_ciq[i] = q->rspq.abs_id; 1164 } 1165 1166 for_each_port(adap, i) { 1167 /* 1168 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't 1169 * have RDMA queues, and that's the right value. 1170 */ 1171 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i], 1172 s->fw_evtq.cntxt_id, 1173 s->rdmarxq[i].rspq.cntxt_id); 1174 if (err) 1175 goto freeout; 1176 } 1177 1178 t4_write_reg(adap, is_t4(adap->params.chip) ? 1179 MPS_TRC_RSS_CONTROL : 1180 MPS_T5_TRC_RSS_CONTROL, 1181 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) | 1182 QUEUENUMBER(s->ethrxq[0].rspq.abs_id)); 1183 return 0; 1184 } 1185 1186 /* 1187 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc. 1188 * The allocated memory is cleared. 1189 */ 1190 void *t4_alloc_mem(size_t size) 1191 { 1192 void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN); 1193 1194 if (!p) 1195 p = vzalloc(size); 1196 return p; 1197 } 1198 1199 /* 1200 * Free memory allocated through alloc_mem(). 1201 */ 1202 void t4_free_mem(void *addr) 1203 { 1204 if (is_vmalloc_addr(addr)) 1205 vfree(addr); 1206 else 1207 kfree(addr); 1208 } 1209 1210 /* Send a Work Request to write the filter at a specified index. We construct 1211 * a Firmware Filter Work Request to have the work done and put the indicated 1212 * filter into "pending" mode which will prevent any further actions against 1213 * it till we get a reply from the firmware on the completion status of the 1214 * request. 1215 */ 1216 static int set_filter_wr(struct adapter *adapter, int fidx) 1217 { 1218 struct filter_entry *f = &adapter->tids.ftid_tab[fidx]; 1219 struct sk_buff *skb; 1220 struct fw_filter_wr *fwr; 1221 unsigned int ftid; 1222 1223 /* If the new filter requires loopback Destination MAC and/or VLAN 1224 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for 1225 * the filter. 1226 */ 1227 if (f->fs.newdmac || f->fs.newvlan) { 1228 /* allocate L2T entry for new filter */ 1229 f->l2t = t4_l2t_alloc_switching(adapter->l2t); 1230 if (f->l2t == NULL) 1231 return -EAGAIN; 1232 if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan, 1233 f->fs.eport, f->fs.dmac)) { 1234 cxgb4_l2t_release(f->l2t); 1235 f->l2t = NULL; 1236 return -ENOMEM; 1237 } 1238 } 1239 1240 ftid = adapter->tids.ftid_base + fidx; 1241 1242 skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL); 1243 fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr)); 1244 memset(fwr, 0, sizeof(*fwr)); 1245 1246 /* It would be nice to put most of the following in t4_hw.c but most 1247 * of the work is translating the cxgbtool ch_filter_specification 1248 * into the Work Request and the definition of that structure is 1249 * currently in cxgbtool.h which isn't appropriate to pull into the 1250 * common code. We may eventually try to come up with a more neutral 1251 * filter specification structure but for now it's easiest to simply 1252 * put this fairly direct code in line ... 1253 */ 1254 fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR)); 1255 fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16)); 1256 fwr->tid_to_iq = 1257 htonl(FW_FILTER_WR_TID_V(ftid) | 1258 FW_FILTER_WR_RQTYPE_V(f->fs.type) | 1259 FW_FILTER_WR_NOREPLY_V(0) | 1260 FW_FILTER_WR_IQ_V(f->fs.iq)); 1261 fwr->del_filter_to_l2tix = 1262 htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) | 1263 FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) | 1264 FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) | 1265 FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) | 1266 FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) | 1267 FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) | 1268 FW_FILTER_WR_DMAC_V(f->fs.newdmac) | 1269 FW_FILTER_WR_SMAC_V(f->fs.newsmac) | 1270 FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT || 1271 f->fs.newvlan == VLAN_REWRITE) | 1272 FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE || 1273 f->fs.newvlan == VLAN_REWRITE) | 1274 FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) | 1275 FW_FILTER_WR_TXCHAN_V(f->fs.eport) | 1276 FW_FILTER_WR_PRIO_V(f->fs.prio) | 1277 FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0)); 1278 fwr->ethtype = htons(f->fs.val.ethtype); 1279 fwr->ethtypem = htons(f->fs.mask.ethtype); 1280 fwr->frag_to_ovlan_vldm = 1281 (FW_FILTER_WR_FRAG_V(f->fs.val.frag) | 1282 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) | 1283 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) | 1284 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) | 1285 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) | 1286 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld)); 1287 fwr->smac_sel = 0; 1288 fwr->rx_chan_rx_rpl_iq = 1289 htons(FW_FILTER_WR_RX_CHAN_V(0) | 1290 FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id)); 1291 fwr->maci_to_matchtypem = 1292 htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) | 1293 FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) | 1294 FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) | 1295 FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) | 1296 FW_FILTER_WR_PORT_V(f->fs.val.iport) | 1297 FW_FILTER_WR_PORTM_V(f->fs.mask.iport) | 1298 FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) | 1299 FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype)); 1300 fwr->ptcl = f->fs.val.proto; 1301 fwr->ptclm = f->fs.mask.proto; 1302 fwr->ttyp = f->fs.val.tos; 1303 fwr->ttypm = f->fs.mask.tos; 1304 fwr->ivlan = htons(f->fs.val.ivlan); 1305 fwr->ivlanm = htons(f->fs.mask.ivlan); 1306 fwr->ovlan = htons(f->fs.val.ovlan); 1307 fwr->ovlanm = htons(f->fs.mask.ovlan); 1308 memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip)); 1309 memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm)); 1310 memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip)); 1311 memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm)); 1312 fwr->lp = htons(f->fs.val.lport); 1313 fwr->lpm = htons(f->fs.mask.lport); 1314 fwr->fp = htons(f->fs.val.fport); 1315 fwr->fpm = htons(f->fs.mask.fport); 1316 if (f->fs.newsmac) 1317 memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma)); 1318 1319 /* Mark the filter as "pending" and ship off the Filter Work Request. 1320 * When we get the Work Request Reply we'll clear the pending status. 1321 */ 1322 f->pending = 1; 1323 set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3); 1324 t4_ofld_send(adapter, skb); 1325 return 0; 1326 } 1327 1328 /* Delete the filter at a specified index. 1329 */ 1330 static int del_filter_wr(struct adapter *adapter, int fidx) 1331 { 1332 struct filter_entry *f = &adapter->tids.ftid_tab[fidx]; 1333 struct sk_buff *skb; 1334 struct fw_filter_wr *fwr; 1335 unsigned int len, ftid; 1336 1337 len = sizeof(*fwr); 1338 ftid = adapter->tids.ftid_base + fidx; 1339 1340 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL); 1341 fwr = (struct fw_filter_wr *)__skb_put(skb, len); 1342 t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id); 1343 1344 /* Mark the filter as "pending" and ship off the Filter Work Request. 1345 * When we get the Work Request Reply we'll clear the pending status. 1346 */ 1347 f->pending = 1; 1348 t4_mgmt_tx(adapter, skb); 1349 return 0; 1350 } 1351 1352 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb, 1353 void *accel_priv, select_queue_fallback_t fallback) 1354 { 1355 int txq; 1356 1357 #ifdef CONFIG_CHELSIO_T4_DCB 1358 /* If a Data Center Bridging has been successfully negotiated on this 1359 * link then we'll use the skb's priority to map it to a TX Queue. 1360 * The skb's priority is determined via the VLAN Tag Priority Code 1361 * Point field. 1362 */ 1363 if (cxgb4_dcb_enabled(dev)) { 1364 u16 vlan_tci; 1365 int err; 1366 1367 err = vlan_get_tag(skb, &vlan_tci); 1368 if (unlikely(err)) { 1369 if (net_ratelimit()) 1370 netdev_warn(dev, 1371 "TX Packet without VLAN Tag on DCB Link\n"); 1372 txq = 0; 1373 } else { 1374 txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT; 1375 } 1376 return txq; 1377 } 1378 #endif /* CONFIG_CHELSIO_T4_DCB */ 1379 1380 if (select_queue) { 1381 txq = (skb_rx_queue_recorded(skb) 1382 ? skb_get_rx_queue(skb) 1383 : smp_processor_id()); 1384 1385 while (unlikely(txq >= dev->real_num_tx_queues)) 1386 txq -= dev->real_num_tx_queues; 1387 1388 return txq; 1389 } 1390 1391 return fallback(dev, skb) % dev->real_num_tx_queues; 1392 } 1393 1394 static inline int is_offload(const struct adapter *adap) 1395 { 1396 return adap->params.offload; 1397 } 1398 1399 /* 1400 * Implementation of ethtool operations. 1401 */ 1402 1403 static u32 get_msglevel(struct net_device *dev) 1404 { 1405 return netdev2adap(dev)->msg_enable; 1406 } 1407 1408 static void set_msglevel(struct net_device *dev, u32 val) 1409 { 1410 netdev2adap(dev)->msg_enable = val; 1411 } 1412 1413 static char stats_strings[][ETH_GSTRING_LEN] = { 1414 "TxOctetsOK ", 1415 "TxFramesOK ", 1416 "TxBroadcastFrames ", 1417 "TxMulticastFrames ", 1418 "TxUnicastFrames ", 1419 "TxErrorFrames ", 1420 1421 "TxFrames64 ", 1422 "TxFrames65To127 ", 1423 "TxFrames128To255 ", 1424 "TxFrames256To511 ", 1425 "TxFrames512To1023 ", 1426 "TxFrames1024To1518 ", 1427 "TxFrames1519ToMax ", 1428 1429 "TxFramesDropped ", 1430 "TxPauseFrames ", 1431 "TxPPP0Frames ", 1432 "TxPPP1Frames ", 1433 "TxPPP2Frames ", 1434 "TxPPP3Frames ", 1435 "TxPPP4Frames ", 1436 "TxPPP5Frames ", 1437 "TxPPP6Frames ", 1438 "TxPPP7Frames ", 1439 1440 "RxOctetsOK ", 1441 "RxFramesOK ", 1442 "RxBroadcastFrames ", 1443 "RxMulticastFrames ", 1444 "RxUnicastFrames ", 1445 1446 "RxFramesTooLong ", 1447 "RxJabberErrors ", 1448 "RxFCSErrors ", 1449 "RxLengthErrors ", 1450 "RxSymbolErrors ", 1451 "RxRuntFrames ", 1452 1453 "RxFrames64 ", 1454 "RxFrames65To127 ", 1455 "RxFrames128To255 ", 1456 "RxFrames256To511 ", 1457 "RxFrames512To1023 ", 1458 "RxFrames1024To1518 ", 1459 "RxFrames1519ToMax ", 1460 1461 "RxPauseFrames ", 1462 "RxPPP0Frames ", 1463 "RxPPP1Frames ", 1464 "RxPPP2Frames ", 1465 "RxPPP3Frames ", 1466 "RxPPP4Frames ", 1467 "RxPPP5Frames ", 1468 "RxPPP6Frames ", 1469 "RxPPP7Frames ", 1470 1471 "RxBG0FramesDropped ", 1472 "RxBG1FramesDropped ", 1473 "RxBG2FramesDropped ", 1474 "RxBG3FramesDropped ", 1475 "RxBG0FramesTrunc ", 1476 "RxBG1FramesTrunc ", 1477 "RxBG2FramesTrunc ", 1478 "RxBG3FramesTrunc ", 1479 1480 "TSO ", 1481 "TxCsumOffload ", 1482 "RxCsumGood ", 1483 "VLANextractions ", 1484 "VLANinsertions ", 1485 "GROpackets ", 1486 "GROmerged ", 1487 "WriteCoalSuccess ", 1488 "WriteCoalFail ", 1489 }; 1490 1491 static int get_sset_count(struct net_device *dev, int sset) 1492 { 1493 switch (sset) { 1494 case ETH_SS_STATS: 1495 return ARRAY_SIZE(stats_strings); 1496 default: 1497 return -EOPNOTSUPP; 1498 } 1499 } 1500 1501 #define T4_REGMAP_SIZE (160 * 1024) 1502 #define T5_REGMAP_SIZE (332 * 1024) 1503 1504 static int get_regs_len(struct net_device *dev) 1505 { 1506 struct adapter *adap = netdev2adap(dev); 1507 if (is_t4(adap->params.chip)) 1508 return T4_REGMAP_SIZE; 1509 else 1510 return T5_REGMAP_SIZE; 1511 } 1512 1513 static int get_eeprom_len(struct net_device *dev) 1514 { 1515 return EEPROMSIZE; 1516 } 1517 1518 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info) 1519 { 1520 struct adapter *adapter = netdev2adap(dev); 1521 1522 strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver)); 1523 strlcpy(info->version, DRV_VERSION, sizeof(info->version)); 1524 strlcpy(info->bus_info, pci_name(adapter->pdev), 1525 sizeof(info->bus_info)); 1526 1527 if (adapter->params.fw_vers) 1528 snprintf(info->fw_version, sizeof(info->fw_version), 1529 "%u.%u.%u.%u, TP %u.%u.%u.%u", 1530 FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers), 1531 FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers), 1532 FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers), 1533 FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers), 1534 FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers), 1535 FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers), 1536 FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers), 1537 FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers)); 1538 } 1539 1540 static void get_strings(struct net_device *dev, u32 stringset, u8 *data) 1541 { 1542 if (stringset == ETH_SS_STATS) 1543 memcpy(data, stats_strings, sizeof(stats_strings)); 1544 } 1545 1546 /* 1547 * port stats maintained per queue of the port. They should be in the same 1548 * order as in stats_strings above. 1549 */ 1550 struct queue_port_stats { 1551 u64 tso; 1552 u64 tx_csum; 1553 u64 rx_csum; 1554 u64 vlan_ex; 1555 u64 vlan_ins; 1556 u64 gro_pkts; 1557 u64 gro_merged; 1558 }; 1559 1560 static void collect_sge_port_stats(const struct adapter *adap, 1561 const struct port_info *p, struct queue_port_stats *s) 1562 { 1563 int i; 1564 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset]; 1565 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset]; 1566 1567 memset(s, 0, sizeof(*s)); 1568 for (i = 0; i < p->nqsets; i++, rx++, tx++) { 1569 s->tso += tx->tso; 1570 s->tx_csum += tx->tx_cso; 1571 s->rx_csum += rx->stats.rx_cso; 1572 s->vlan_ex += rx->stats.vlan_ex; 1573 s->vlan_ins += tx->vlan_ins; 1574 s->gro_pkts += rx->stats.lro_pkts; 1575 s->gro_merged += rx->stats.lro_merged; 1576 } 1577 } 1578 1579 static void get_stats(struct net_device *dev, struct ethtool_stats *stats, 1580 u64 *data) 1581 { 1582 struct port_info *pi = netdev_priv(dev); 1583 struct adapter *adapter = pi->adapter; 1584 u32 val1, val2; 1585 1586 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data); 1587 1588 data += sizeof(struct port_stats) / sizeof(u64); 1589 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data); 1590 data += sizeof(struct queue_port_stats) / sizeof(u64); 1591 if (!is_t4(adapter->params.chip)) { 1592 t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7)); 1593 val1 = t4_read_reg(adapter, SGE_STAT_TOTAL); 1594 val2 = t4_read_reg(adapter, SGE_STAT_MATCH); 1595 *data = val1 - val2; 1596 data++; 1597 *data = val2; 1598 data++; 1599 } else { 1600 memset(data, 0, 2 * sizeof(u64)); 1601 *data += 2; 1602 } 1603 } 1604 1605 /* 1606 * Return a version number to identify the type of adapter. The scheme is: 1607 * - bits 0..9: chip version 1608 * - bits 10..15: chip revision 1609 * - bits 16..23: register dump version 1610 */ 1611 static inline unsigned int mk_adap_vers(const struct adapter *ap) 1612 { 1613 return CHELSIO_CHIP_VERSION(ap->params.chip) | 1614 (CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16); 1615 } 1616 1617 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start, 1618 unsigned int end) 1619 { 1620 u32 *p = buf + start; 1621 1622 for ( ; start <= end; start += sizeof(u32)) 1623 *p++ = t4_read_reg(ap, start); 1624 } 1625 1626 static void get_regs(struct net_device *dev, struct ethtool_regs *regs, 1627 void *buf) 1628 { 1629 static const unsigned int t4_reg_ranges[] = { 1630 0x1008, 0x1108, 1631 0x1180, 0x11b4, 1632 0x11fc, 0x123c, 1633 0x1300, 0x173c, 1634 0x1800, 0x18fc, 1635 0x3000, 0x30d8, 1636 0x30e0, 0x5924, 1637 0x5960, 0x59d4, 1638 0x5a00, 0x5af8, 1639 0x6000, 0x6098, 1640 0x6100, 0x6150, 1641 0x6200, 0x6208, 1642 0x6240, 0x6248, 1643 0x6280, 0x6338, 1644 0x6370, 0x638c, 1645 0x6400, 0x643c, 1646 0x6500, 0x6524, 1647 0x6a00, 0x6a38, 1648 0x6a60, 0x6a78, 1649 0x6b00, 0x6b84, 1650 0x6bf0, 0x6c84, 1651 0x6cf0, 0x6d84, 1652 0x6df0, 0x6e84, 1653 0x6ef0, 0x6f84, 1654 0x6ff0, 0x7084, 1655 0x70f0, 0x7184, 1656 0x71f0, 0x7284, 1657 0x72f0, 0x7384, 1658 0x73f0, 0x7450, 1659 0x7500, 0x7530, 1660 0x7600, 0x761c, 1661 0x7680, 0x76cc, 1662 0x7700, 0x7798, 1663 0x77c0, 0x77fc, 1664 0x7900, 0x79fc, 1665 0x7b00, 0x7c38, 1666 0x7d00, 0x7efc, 1667 0x8dc0, 0x8e1c, 1668 0x8e30, 0x8e78, 1669 0x8ea0, 0x8f6c, 1670 0x8fc0, 0x9074, 1671 0x90fc, 0x90fc, 1672 0x9400, 0x9458, 1673 0x9600, 0x96bc, 1674 0x9800, 0x9808, 1675 0x9820, 0x983c, 1676 0x9850, 0x9864, 1677 0x9c00, 0x9c6c, 1678 0x9c80, 0x9cec, 1679 0x9d00, 0x9d6c, 1680 0x9d80, 0x9dec, 1681 0x9e00, 0x9e6c, 1682 0x9e80, 0x9eec, 1683 0x9f00, 0x9f6c, 1684 0x9f80, 0x9fec, 1685 0xd004, 0xd03c, 1686 0xdfc0, 0xdfe0, 1687 0xe000, 0xea7c, 1688 0xf000, 0x11110, 1689 0x11118, 0x11190, 1690 0x19040, 0x1906c, 1691 0x19078, 0x19080, 1692 0x1908c, 0x19124, 1693 0x19150, 0x191b0, 1694 0x191d0, 0x191e8, 1695 0x19238, 0x1924c, 1696 0x193f8, 0x19474, 1697 0x19490, 0x194f8, 1698 0x19800, 0x19f30, 1699 0x1a000, 0x1a06c, 1700 0x1a0b0, 0x1a120, 1701 0x1a128, 0x1a138, 1702 0x1a190, 0x1a1c4, 1703 0x1a1fc, 0x1a1fc, 1704 0x1e040, 0x1e04c, 1705 0x1e284, 0x1e28c, 1706 0x1e2c0, 0x1e2c0, 1707 0x1e2e0, 0x1e2e0, 1708 0x1e300, 0x1e384, 1709 0x1e3c0, 0x1e3c8, 1710 0x1e440, 0x1e44c, 1711 0x1e684, 0x1e68c, 1712 0x1e6c0, 0x1e6c0, 1713 0x1e6e0, 0x1e6e0, 1714 0x1e700, 0x1e784, 1715 0x1e7c0, 0x1e7c8, 1716 0x1e840, 0x1e84c, 1717 0x1ea84, 0x1ea8c, 1718 0x1eac0, 0x1eac0, 1719 0x1eae0, 0x1eae0, 1720 0x1eb00, 0x1eb84, 1721 0x1ebc0, 0x1ebc8, 1722 0x1ec40, 0x1ec4c, 1723 0x1ee84, 0x1ee8c, 1724 0x1eec0, 0x1eec0, 1725 0x1eee0, 0x1eee0, 1726 0x1ef00, 0x1ef84, 1727 0x1efc0, 0x1efc8, 1728 0x1f040, 0x1f04c, 1729 0x1f284, 0x1f28c, 1730 0x1f2c0, 0x1f2c0, 1731 0x1f2e0, 0x1f2e0, 1732 0x1f300, 0x1f384, 1733 0x1f3c0, 0x1f3c8, 1734 0x1f440, 0x1f44c, 1735 0x1f684, 0x1f68c, 1736 0x1f6c0, 0x1f6c0, 1737 0x1f6e0, 0x1f6e0, 1738 0x1f700, 0x1f784, 1739 0x1f7c0, 0x1f7c8, 1740 0x1f840, 0x1f84c, 1741 0x1fa84, 0x1fa8c, 1742 0x1fac0, 0x1fac0, 1743 0x1fae0, 0x1fae0, 1744 0x1fb00, 0x1fb84, 1745 0x1fbc0, 0x1fbc8, 1746 0x1fc40, 0x1fc4c, 1747 0x1fe84, 0x1fe8c, 1748 0x1fec0, 0x1fec0, 1749 0x1fee0, 0x1fee0, 1750 0x1ff00, 0x1ff84, 1751 0x1ffc0, 0x1ffc8, 1752 0x20000, 0x2002c, 1753 0x20100, 0x2013c, 1754 0x20190, 0x201c8, 1755 0x20200, 0x20318, 1756 0x20400, 0x20528, 1757 0x20540, 0x20614, 1758 0x21000, 0x21040, 1759 0x2104c, 0x21060, 1760 0x210c0, 0x210ec, 1761 0x21200, 0x21268, 1762 0x21270, 0x21284, 1763 0x212fc, 0x21388, 1764 0x21400, 0x21404, 1765 0x21500, 0x21518, 1766 0x2152c, 0x2153c, 1767 0x21550, 0x21554, 1768 0x21600, 0x21600, 1769 0x21608, 0x21628, 1770 0x21630, 0x2163c, 1771 0x21700, 0x2171c, 1772 0x21780, 0x2178c, 1773 0x21800, 0x21c38, 1774 0x21c80, 0x21d7c, 1775 0x21e00, 0x21e04, 1776 0x22000, 0x2202c, 1777 0x22100, 0x2213c, 1778 0x22190, 0x221c8, 1779 0x22200, 0x22318, 1780 0x22400, 0x22528, 1781 0x22540, 0x22614, 1782 0x23000, 0x23040, 1783 0x2304c, 0x23060, 1784 0x230c0, 0x230ec, 1785 0x23200, 0x23268, 1786 0x23270, 0x23284, 1787 0x232fc, 0x23388, 1788 0x23400, 0x23404, 1789 0x23500, 0x23518, 1790 0x2352c, 0x2353c, 1791 0x23550, 0x23554, 1792 0x23600, 0x23600, 1793 0x23608, 0x23628, 1794 0x23630, 0x2363c, 1795 0x23700, 0x2371c, 1796 0x23780, 0x2378c, 1797 0x23800, 0x23c38, 1798 0x23c80, 0x23d7c, 1799 0x23e00, 0x23e04, 1800 0x24000, 0x2402c, 1801 0x24100, 0x2413c, 1802 0x24190, 0x241c8, 1803 0x24200, 0x24318, 1804 0x24400, 0x24528, 1805 0x24540, 0x24614, 1806 0x25000, 0x25040, 1807 0x2504c, 0x25060, 1808 0x250c0, 0x250ec, 1809 0x25200, 0x25268, 1810 0x25270, 0x25284, 1811 0x252fc, 0x25388, 1812 0x25400, 0x25404, 1813 0x25500, 0x25518, 1814 0x2552c, 0x2553c, 1815 0x25550, 0x25554, 1816 0x25600, 0x25600, 1817 0x25608, 0x25628, 1818 0x25630, 0x2563c, 1819 0x25700, 0x2571c, 1820 0x25780, 0x2578c, 1821 0x25800, 0x25c38, 1822 0x25c80, 0x25d7c, 1823 0x25e00, 0x25e04, 1824 0x26000, 0x2602c, 1825 0x26100, 0x2613c, 1826 0x26190, 0x261c8, 1827 0x26200, 0x26318, 1828 0x26400, 0x26528, 1829 0x26540, 0x26614, 1830 0x27000, 0x27040, 1831 0x2704c, 0x27060, 1832 0x270c0, 0x270ec, 1833 0x27200, 0x27268, 1834 0x27270, 0x27284, 1835 0x272fc, 0x27388, 1836 0x27400, 0x27404, 1837 0x27500, 0x27518, 1838 0x2752c, 0x2753c, 1839 0x27550, 0x27554, 1840 0x27600, 0x27600, 1841 0x27608, 0x27628, 1842 0x27630, 0x2763c, 1843 0x27700, 0x2771c, 1844 0x27780, 0x2778c, 1845 0x27800, 0x27c38, 1846 0x27c80, 0x27d7c, 1847 0x27e00, 0x27e04 1848 }; 1849 1850 static const unsigned int t5_reg_ranges[] = { 1851 0x1008, 0x1148, 1852 0x1180, 0x11b4, 1853 0x11fc, 0x123c, 1854 0x1280, 0x173c, 1855 0x1800, 0x18fc, 1856 0x3000, 0x3028, 1857 0x3060, 0x30d8, 1858 0x30e0, 0x30fc, 1859 0x3140, 0x357c, 1860 0x35a8, 0x35cc, 1861 0x35ec, 0x35ec, 1862 0x3600, 0x5624, 1863 0x56cc, 0x575c, 1864 0x580c, 0x5814, 1865 0x5890, 0x58bc, 1866 0x5940, 0x59dc, 1867 0x59fc, 0x5a18, 1868 0x5a60, 0x5a9c, 1869 0x5b9c, 0x5bfc, 1870 0x6000, 0x6040, 1871 0x6058, 0x614c, 1872 0x7700, 0x7798, 1873 0x77c0, 0x78fc, 1874 0x7b00, 0x7c54, 1875 0x7d00, 0x7efc, 1876 0x8dc0, 0x8de0, 1877 0x8df8, 0x8e84, 1878 0x8ea0, 0x8f84, 1879 0x8fc0, 0x90f8, 1880 0x9400, 0x9470, 1881 0x9600, 0x96f4, 1882 0x9800, 0x9808, 1883 0x9820, 0x983c, 1884 0x9850, 0x9864, 1885 0x9c00, 0x9c6c, 1886 0x9c80, 0x9cec, 1887 0x9d00, 0x9d6c, 1888 0x9d80, 0x9dec, 1889 0x9e00, 0x9e6c, 1890 0x9e80, 0x9eec, 1891 0x9f00, 0x9f6c, 1892 0x9f80, 0xa020, 1893 0xd004, 0xd03c, 1894 0xdfc0, 0xdfe0, 1895 0xe000, 0x11088, 1896 0x1109c, 0x11110, 1897 0x11118, 0x1117c, 1898 0x11190, 0x11204, 1899 0x19040, 0x1906c, 1900 0x19078, 0x19080, 1901 0x1908c, 0x19124, 1902 0x19150, 0x191b0, 1903 0x191d0, 0x191e8, 1904 0x19238, 0x19290, 1905 0x193f8, 0x19474, 1906 0x19490, 0x194cc, 1907 0x194f0, 0x194f8, 1908 0x19c00, 0x19c60, 1909 0x19c94, 0x19e10, 1910 0x19e50, 0x19f34, 1911 0x19f40, 0x19f50, 1912 0x19f90, 0x19fe4, 1913 0x1a000, 0x1a06c, 1914 0x1a0b0, 0x1a120, 1915 0x1a128, 0x1a138, 1916 0x1a190, 0x1a1c4, 1917 0x1a1fc, 0x1a1fc, 1918 0x1e008, 0x1e00c, 1919 0x1e040, 0x1e04c, 1920 0x1e284, 0x1e290, 1921 0x1e2c0, 0x1e2c0, 1922 0x1e2e0, 0x1e2e0, 1923 0x1e300, 0x1e384, 1924 0x1e3c0, 0x1e3c8, 1925 0x1e408, 0x1e40c, 1926 0x1e440, 0x1e44c, 1927 0x1e684, 0x1e690, 1928 0x1e6c0, 0x1e6c0, 1929 0x1e6e0, 0x1e6e0, 1930 0x1e700, 0x1e784, 1931 0x1e7c0, 0x1e7c8, 1932 0x1e808, 0x1e80c, 1933 0x1e840, 0x1e84c, 1934 0x1ea84, 0x1ea90, 1935 0x1eac0, 0x1eac0, 1936 0x1eae0, 0x1eae0, 1937 0x1eb00, 0x1eb84, 1938 0x1ebc0, 0x1ebc8, 1939 0x1ec08, 0x1ec0c, 1940 0x1ec40, 0x1ec4c, 1941 0x1ee84, 0x1ee90, 1942 0x1eec0, 0x1eec0, 1943 0x1eee0, 0x1eee0, 1944 0x1ef00, 0x1ef84, 1945 0x1efc0, 0x1efc8, 1946 0x1f008, 0x1f00c, 1947 0x1f040, 0x1f04c, 1948 0x1f284, 0x1f290, 1949 0x1f2c0, 0x1f2c0, 1950 0x1f2e0, 0x1f2e0, 1951 0x1f300, 0x1f384, 1952 0x1f3c0, 0x1f3c8, 1953 0x1f408, 0x1f40c, 1954 0x1f440, 0x1f44c, 1955 0x1f684, 0x1f690, 1956 0x1f6c0, 0x1f6c0, 1957 0x1f6e0, 0x1f6e0, 1958 0x1f700, 0x1f784, 1959 0x1f7c0, 0x1f7c8, 1960 0x1f808, 0x1f80c, 1961 0x1f840, 0x1f84c, 1962 0x1fa84, 0x1fa90, 1963 0x1fac0, 0x1fac0, 1964 0x1fae0, 0x1fae0, 1965 0x1fb00, 0x1fb84, 1966 0x1fbc0, 0x1fbc8, 1967 0x1fc08, 0x1fc0c, 1968 0x1fc40, 0x1fc4c, 1969 0x1fe84, 0x1fe90, 1970 0x1fec0, 0x1fec0, 1971 0x1fee0, 0x1fee0, 1972 0x1ff00, 0x1ff84, 1973 0x1ffc0, 0x1ffc8, 1974 0x30000, 0x30030, 1975 0x30100, 0x30144, 1976 0x30190, 0x301d0, 1977 0x30200, 0x30318, 1978 0x30400, 0x3052c, 1979 0x30540, 0x3061c, 1980 0x30800, 0x30834, 1981 0x308c0, 0x30908, 1982 0x30910, 0x309ac, 1983 0x30a00, 0x30a04, 1984 0x30a0c, 0x30a2c, 1985 0x30a44, 0x30a50, 1986 0x30a74, 0x30c24, 1987 0x30d08, 0x30d14, 1988 0x30d1c, 0x30d20, 1989 0x30d3c, 0x30d50, 1990 0x31200, 0x3120c, 1991 0x31220, 0x31220, 1992 0x31240, 0x31240, 1993 0x31600, 0x31600, 1994 0x31608, 0x3160c, 1995 0x31a00, 0x31a1c, 1996 0x31e04, 0x31e20, 1997 0x31e38, 0x31e3c, 1998 0x31e80, 0x31e80, 1999 0x31e88, 0x31ea8, 2000 0x31eb0, 0x31eb4, 2001 0x31ec8, 0x31ed4, 2002 0x31fb8, 0x32004, 2003 0x32208, 0x3223c, 2004 0x32600, 0x32630, 2005 0x32a00, 0x32abc, 2006 0x32b00, 0x32b70, 2007 0x33000, 0x33048, 2008 0x33060, 0x3309c, 2009 0x330f0, 0x33148, 2010 0x33160, 0x3319c, 2011 0x331f0, 0x332e4, 2012 0x332f8, 0x333e4, 2013 0x333f8, 0x33448, 2014 0x33460, 0x3349c, 2015 0x334f0, 0x33548, 2016 0x33560, 0x3359c, 2017 0x335f0, 0x336e4, 2018 0x336f8, 0x337e4, 2019 0x337f8, 0x337fc, 2020 0x33814, 0x33814, 2021 0x3382c, 0x3382c, 2022 0x33880, 0x3388c, 2023 0x338e8, 0x338ec, 2024 0x33900, 0x33948, 2025 0x33960, 0x3399c, 2026 0x339f0, 0x33ae4, 2027 0x33af8, 0x33b10, 2028 0x33b28, 0x33b28, 2029 0x33b3c, 0x33b50, 2030 0x33bf0, 0x33c10, 2031 0x33c28, 0x33c28, 2032 0x33c3c, 0x33c50, 2033 0x33cf0, 0x33cfc, 2034 0x34000, 0x34030, 2035 0x34100, 0x34144, 2036 0x34190, 0x341d0, 2037 0x34200, 0x34318, 2038 0x34400, 0x3452c, 2039 0x34540, 0x3461c, 2040 0x34800, 0x34834, 2041 0x348c0, 0x34908, 2042 0x34910, 0x349ac, 2043 0x34a00, 0x34a04, 2044 0x34a0c, 0x34a2c, 2045 0x34a44, 0x34a50, 2046 0x34a74, 0x34c24, 2047 0x34d08, 0x34d14, 2048 0x34d1c, 0x34d20, 2049 0x34d3c, 0x34d50, 2050 0x35200, 0x3520c, 2051 0x35220, 0x35220, 2052 0x35240, 0x35240, 2053 0x35600, 0x35600, 2054 0x35608, 0x3560c, 2055 0x35a00, 0x35a1c, 2056 0x35e04, 0x35e20, 2057 0x35e38, 0x35e3c, 2058 0x35e80, 0x35e80, 2059 0x35e88, 0x35ea8, 2060 0x35eb0, 0x35eb4, 2061 0x35ec8, 0x35ed4, 2062 0x35fb8, 0x36004, 2063 0x36208, 0x3623c, 2064 0x36600, 0x36630, 2065 0x36a00, 0x36abc, 2066 0x36b00, 0x36b70, 2067 0x37000, 0x37048, 2068 0x37060, 0x3709c, 2069 0x370f0, 0x37148, 2070 0x37160, 0x3719c, 2071 0x371f0, 0x372e4, 2072 0x372f8, 0x373e4, 2073 0x373f8, 0x37448, 2074 0x37460, 0x3749c, 2075 0x374f0, 0x37548, 2076 0x37560, 0x3759c, 2077 0x375f0, 0x376e4, 2078 0x376f8, 0x377e4, 2079 0x377f8, 0x377fc, 2080 0x37814, 0x37814, 2081 0x3782c, 0x3782c, 2082 0x37880, 0x3788c, 2083 0x378e8, 0x378ec, 2084 0x37900, 0x37948, 2085 0x37960, 0x3799c, 2086 0x379f0, 0x37ae4, 2087 0x37af8, 0x37b10, 2088 0x37b28, 0x37b28, 2089 0x37b3c, 0x37b50, 2090 0x37bf0, 0x37c10, 2091 0x37c28, 0x37c28, 2092 0x37c3c, 0x37c50, 2093 0x37cf0, 0x37cfc, 2094 0x38000, 0x38030, 2095 0x38100, 0x38144, 2096 0x38190, 0x381d0, 2097 0x38200, 0x38318, 2098 0x38400, 0x3852c, 2099 0x38540, 0x3861c, 2100 0x38800, 0x38834, 2101 0x388c0, 0x38908, 2102 0x38910, 0x389ac, 2103 0x38a00, 0x38a04, 2104 0x38a0c, 0x38a2c, 2105 0x38a44, 0x38a50, 2106 0x38a74, 0x38c24, 2107 0x38d08, 0x38d14, 2108 0x38d1c, 0x38d20, 2109 0x38d3c, 0x38d50, 2110 0x39200, 0x3920c, 2111 0x39220, 0x39220, 2112 0x39240, 0x39240, 2113 0x39600, 0x39600, 2114 0x39608, 0x3960c, 2115 0x39a00, 0x39a1c, 2116 0x39e04, 0x39e20, 2117 0x39e38, 0x39e3c, 2118 0x39e80, 0x39e80, 2119 0x39e88, 0x39ea8, 2120 0x39eb0, 0x39eb4, 2121 0x39ec8, 0x39ed4, 2122 0x39fb8, 0x3a004, 2123 0x3a208, 0x3a23c, 2124 0x3a600, 0x3a630, 2125 0x3aa00, 0x3aabc, 2126 0x3ab00, 0x3ab70, 2127 0x3b000, 0x3b048, 2128 0x3b060, 0x3b09c, 2129 0x3b0f0, 0x3b148, 2130 0x3b160, 0x3b19c, 2131 0x3b1f0, 0x3b2e4, 2132 0x3b2f8, 0x3b3e4, 2133 0x3b3f8, 0x3b448, 2134 0x3b460, 0x3b49c, 2135 0x3b4f0, 0x3b548, 2136 0x3b560, 0x3b59c, 2137 0x3b5f0, 0x3b6e4, 2138 0x3b6f8, 0x3b7e4, 2139 0x3b7f8, 0x3b7fc, 2140 0x3b814, 0x3b814, 2141 0x3b82c, 0x3b82c, 2142 0x3b880, 0x3b88c, 2143 0x3b8e8, 0x3b8ec, 2144 0x3b900, 0x3b948, 2145 0x3b960, 0x3b99c, 2146 0x3b9f0, 0x3bae4, 2147 0x3baf8, 0x3bb10, 2148 0x3bb28, 0x3bb28, 2149 0x3bb3c, 0x3bb50, 2150 0x3bbf0, 0x3bc10, 2151 0x3bc28, 0x3bc28, 2152 0x3bc3c, 0x3bc50, 2153 0x3bcf0, 0x3bcfc, 2154 0x3c000, 0x3c030, 2155 0x3c100, 0x3c144, 2156 0x3c190, 0x3c1d0, 2157 0x3c200, 0x3c318, 2158 0x3c400, 0x3c52c, 2159 0x3c540, 0x3c61c, 2160 0x3c800, 0x3c834, 2161 0x3c8c0, 0x3c908, 2162 0x3c910, 0x3c9ac, 2163 0x3ca00, 0x3ca04, 2164 0x3ca0c, 0x3ca2c, 2165 0x3ca44, 0x3ca50, 2166 0x3ca74, 0x3cc24, 2167 0x3cd08, 0x3cd14, 2168 0x3cd1c, 0x3cd20, 2169 0x3cd3c, 0x3cd50, 2170 0x3d200, 0x3d20c, 2171 0x3d220, 0x3d220, 2172 0x3d240, 0x3d240, 2173 0x3d600, 0x3d600, 2174 0x3d608, 0x3d60c, 2175 0x3da00, 0x3da1c, 2176 0x3de04, 0x3de20, 2177 0x3de38, 0x3de3c, 2178 0x3de80, 0x3de80, 2179 0x3de88, 0x3dea8, 2180 0x3deb0, 0x3deb4, 2181 0x3dec8, 0x3ded4, 2182 0x3dfb8, 0x3e004, 2183 0x3e208, 0x3e23c, 2184 0x3e600, 0x3e630, 2185 0x3ea00, 0x3eabc, 2186 0x3eb00, 0x3eb70, 2187 0x3f000, 0x3f048, 2188 0x3f060, 0x3f09c, 2189 0x3f0f0, 0x3f148, 2190 0x3f160, 0x3f19c, 2191 0x3f1f0, 0x3f2e4, 2192 0x3f2f8, 0x3f3e4, 2193 0x3f3f8, 0x3f448, 2194 0x3f460, 0x3f49c, 2195 0x3f4f0, 0x3f548, 2196 0x3f560, 0x3f59c, 2197 0x3f5f0, 0x3f6e4, 2198 0x3f6f8, 0x3f7e4, 2199 0x3f7f8, 0x3f7fc, 2200 0x3f814, 0x3f814, 2201 0x3f82c, 0x3f82c, 2202 0x3f880, 0x3f88c, 2203 0x3f8e8, 0x3f8ec, 2204 0x3f900, 0x3f948, 2205 0x3f960, 0x3f99c, 2206 0x3f9f0, 0x3fae4, 2207 0x3faf8, 0x3fb10, 2208 0x3fb28, 0x3fb28, 2209 0x3fb3c, 0x3fb50, 2210 0x3fbf0, 0x3fc10, 2211 0x3fc28, 0x3fc28, 2212 0x3fc3c, 0x3fc50, 2213 0x3fcf0, 0x3fcfc, 2214 0x40000, 0x4000c, 2215 0x40040, 0x40068, 2216 0x40080, 0x40144, 2217 0x40180, 0x4018c, 2218 0x40200, 0x40298, 2219 0x402ac, 0x4033c, 2220 0x403f8, 0x403fc, 2221 0x41304, 0x413c4, 2222 0x41400, 0x4141c, 2223 0x41480, 0x414d0, 2224 0x44000, 0x44078, 2225 0x440c0, 0x44278, 2226 0x442c0, 0x44478, 2227 0x444c0, 0x44678, 2228 0x446c0, 0x44878, 2229 0x448c0, 0x449fc, 2230 0x45000, 0x45068, 2231 0x45080, 0x45084, 2232 0x450a0, 0x450b0, 2233 0x45200, 0x45268, 2234 0x45280, 0x45284, 2235 0x452a0, 0x452b0, 2236 0x460c0, 0x460e4, 2237 0x47000, 0x4708c, 2238 0x47200, 0x47250, 2239 0x47400, 0x47420, 2240 0x47600, 0x47618, 2241 0x47800, 0x47814, 2242 0x48000, 0x4800c, 2243 0x48040, 0x48068, 2244 0x48080, 0x48144, 2245 0x48180, 0x4818c, 2246 0x48200, 0x48298, 2247 0x482ac, 0x4833c, 2248 0x483f8, 0x483fc, 2249 0x49304, 0x493c4, 2250 0x49400, 0x4941c, 2251 0x49480, 0x494d0, 2252 0x4c000, 0x4c078, 2253 0x4c0c0, 0x4c278, 2254 0x4c2c0, 0x4c478, 2255 0x4c4c0, 0x4c678, 2256 0x4c6c0, 0x4c878, 2257 0x4c8c0, 0x4c9fc, 2258 0x4d000, 0x4d068, 2259 0x4d080, 0x4d084, 2260 0x4d0a0, 0x4d0b0, 2261 0x4d200, 0x4d268, 2262 0x4d280, 0x4d284, 2263 0x4d2a0, 0x4d2b0, 2264 0x4e0c0, 0x4e0e4, 2265 0x4f000, 0x4f08c, 2266 0x4f200, 0x4f250, 2267 0x4f400, 0x4f420, 2268 0x4f600, 0x4f618, 2269 0x4f800, 0x4f814, 2270 0x50000, 0x500cc, 2271 0x50400, 0x50400, 2272 0x50800, 0x508cc, 2273 0x50c00, 0x50c00, 2274 0x51000, 0x5101c, 2275 0x51300, 0x51308, 2276 }; 2277 2278 int i; 2279 struct adapter *ap = netdev2adap(dev); 2280 static const unsigned int *reg_ranges; 2281 int arr_size = 0, buf_size = 0; 2282 2283 if (is_t4(ap->params.chip)) { 2284 reg_ranges = &t4_reg_ranges[0]; 2285 arr_size = ARRAY_SIZE(t4_reg_ranges); 2286 buf_size = T4_REGMAP_SIZE; 2287 } else { 2288 reg_ranges = &t5_reg_ranges[0]; 2289 arr_size = ARRAY_SIZE(t5_reg_ranges); 2290 buf_size = T5_REGMAP_SIZE; 2291 } 2292 2293 regs->version = mk_adap_vers(ap); 2294 2295 memset(buf, 0, buf_size); 2296 for (i = 0; i < arr_size; i += 2) 2297 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]); 2298 } 2299 2300 static int restart_autoneg(struct net_device *dev) 2301 { 2302 struct port_info *p = netdev_priv(dev); 2303 2304 if (!netif_running(dev)) 2305 return -EAGAIN; 2306 if (p->link_cfg.autoneg != AUTONEG_ENABLE) 2307 return -EINVAL; 2308 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan); 2309 return 0; 2310 } 2311 2312 static int identify_port(struct net_device *dev, 2313 enum ethtool_phys_id_state state) 2314 { 2315 unsigned int val; 2316 struct adapter *adap = netdev2adap(dev); 2317 2318 if (state == ETHTOOL_ID_ACTIVE) 2319 val = 0xffff; 2320 else if (state == ETHTOOL_ID_INACTIVE) 2321 val = 0; 2322 else 2323 return -EINVAL; 2324 2325 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val); 2326 } 2327 2328 static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps) 2329 { 2330 unsigned int v = 0; 2331 2332 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI || 2333 type == FW_PORT_TYPE_BT_XAUI) { 2334 v |= SUPPORTED_TP; 2335 if (caps & FW_PORT_CAP_SPEED_100M) 2336 v |= SUPPORTED_100baseT_Full; 2337 if (caps & FW_PORT_CAP_SPEED_1G) 2338 v |= SUPPORTED_1000baseT_Full; 2339 if (caps & FW_PORT_CAP_SPEED_10G) 2340 v |= SUPPORTED_10000baseT_Full; 2341 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) { 2342 v |= SUPPORTED_Backplane; 2343 if (caps & FW_PORT_CAP_SPEED_1G) 2344 v |= SUPPORTED_1000baseKX_Full; 2345 if (caps & FW_PORT_CAP_SPEED_10G) 2346 v |= SUPPORTED_10000baseKX4_Full; 2347 } else if (type == FW_PORT_TYPE_KR) 2348 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full; 2349 else if (type == FW_PORT_TYPE_BP_AP) 2350 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC | 2351 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full; 2352 else if (type == FW_PORT_TYPE_BP4_AP) 2353 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC | 2354 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full | 2355 SUPPORTED_10000baseKX4_Full; 2356 else if (type == FW_PORT_TYPE_FIBER_XFI || 2357 type == FW_PORT_TYPE_FIBER_XAUI || 2358 type == FW_PORT_TYPE_SFP || 2359 type == FW_PORT_TYPE_QSFP_10G || 2360 type == FW_PORT_TYPE_QSA) { 2361 v |= SUPPORTED_FIBRE; 2362 if (caps & FW_PORT_CAP_SPEED_1G) 2363 v |= SUPPORTED_1000baseT_Full; 2364 if (caps & FW_PORT_CAP_SPEED_10G) 2365 v |= SUPPORTED_10000baseT_Full; 2366 } else if (type == FW_PORT_TYPE_BP40_BA || 2367 type == FW_PORT_TYPE_QSFP) { 2368 v |= SUPPORTED_40000baseSR4_Full; 2369 v |= SUPPORTED_FIBRE; 2370 } 2371 2372 if (caps & FW_PORT_CAP_ANEG) 2373 v |= SUPPORTED_Autoneg; 2374 return v; 2375 } 2376 2377 static unsigned int to_fw_linkcaps(unsigned int caps) 2378 { 2379 unsigned int v = 0; 2380 2381 if (caps & ADVERTISED_100baseT_Full) 2382 v |= FW_PORT_CAP_SPEED_100M; 2383 if (caps & ADVERTISED_1000baseT_Full) 2384 v |= FW_PORT_CAP_SPEED_1G; 2385 if (caps & ADVERTISED_10000baseT_Full) 2386 v |= FW_PORT_CAP_SPEED_10G; 2387 if (caps & ADVERTISED_40000baseSR4_Full) 2388 v |= FW_PORT_CAP_SPEED_40G; 2389 return v; 2390 } 2391 2392 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd) 2393 { 2394 const struct port_info *p = netdev_priv(dev); 2395 2396 if (p->port_type == FW_PORT_TYPE_BT_SGMII || 2397 p->port_type == FW_PORT_TYPE_BT_XFI || 2398 p->port_type == FW_PORT_TYPE_BT_XAUI) 2399 cmd->port = PORT_TP; 2400 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI || 2401 p->port_type == FW_PORT_TYPE_FIBER_XAUI) 2402 cmd->port = PORT_FIBRE; 2403 else if (p->port_type == FW_PORT_TYPE_SFP || 2404 p->port_type == FW_PORT_TYPE_QSFP_10G || 2405 p->port_type == FW_PORT_TYPE_QSA || 2406 p->port_type == FW_PORT_TYPE_QSFP) { 2407 if (p->mod_type == FW_PORT_MOD_TYPE_LR || 2408 p->mod_type == FW_PORT_MOD_TYPE_SR || 2409 p->mod_type == FW_PORT_MOD_TYPE_ER || 2410 p->mod_type == FW_PORT_MOD_TYPE_LRM) 2411 cmd->port = PORT_FIBRE; 2412 else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE || 2413 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE) 2414 cmd->port = PORT_DA; 2415 else 2416 cmd->port = PORT_OTHER; 2417 } else 2418 cmd->port = PORT_OTHER; 2419 2420 if (p->mdio_addr >= 0) { 2421 cmd->phy_address = p->mdio_addr; 2422 cmd->transceiver = XCVR_EXTERNAL; 2423 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ? 2424 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45; 2425 } else { 2426 cmd->phy_address = 0; /* not really, but no better option */ 2427 cmd->transceiver = XCVR_INTERNAL; 2428 cmd->mdio_support = 0; 2429 } 2430 2431 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported); 2432 cmd->advertising = from_fw_linkcaps(p->port_type, 2433 p->link_cfg.advertising); 2434 ethtool_cmd_speed_set(cmd, 2435 netif_carrier_ok(dev) ? p->link_cfg.speed : 0); 2436 cmd->duplex = DUPLEX_FULL; 2437 cmd->autoneg = p->link_cfg.autoneg; 2438 cmd->maxtxpkt = 0; 2439 cmd->maxrxpkt = 0; 2440 return 0; 2441 } 2442 2443 static unsigned int speed_to_caps(int speed) 2444 { 2445 if (speed == 100) 2446 return FW_PORT_CAP_SPEED_100M; 2447 if (speed == 1000) 2448 return FW_PORT_CAP_SPEED_1G; 2449 if (speed == 10000) 2450 return FW_PORT_CAP_SPEED_10G; 2451 if (speed == 40000) 2452 return FW_PORT_CAP_SPEED_40G; 2453 return 0; 2454 } 2455 2456 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd) 2457 { 2458 unsigned int cap; 2459 struct port_info *p = netdev_priv(dev); 2460 struct link_config *lc = &p->link_cfg; 2461 u32 speed = ethtool_cmd_speed(cmd); 2462 2463 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */ 2464 return -EINVAL; 2465 2466 if (!(lc->supported & FW_PORT_CAP_ANEG)) { 2467 /* 2468 * PHY offers a single speed. See if that's what's 2469 * being requested. 2470 */ 2471 if (cmd->autoneg == AUTONEG_DISABLE && 2472 (lc->supported & speed_to_caps(speed))) 2473 return 0; 2474 return -EINVAL; 2475 } 2476 2477 if (cmd->autoneg == AUTONEG_DISABLE) { 2478 cap = speed_to_caps(speed); 2479 2480 if (!(lc->supported & cap) || 2481 (speed == 1000) || 2482 (speed == 10000) || 2483 (speed == 40000)) 2484 return -EINVAL; 2485 lc->requested_speed = cap; 2486 lc->advertising = 0; 2487 } else { 2488 cap = to_fw_linkcaps(cmd->advertising); 2489 if (!(lc->supported & cap)) 2490 return -EINVAL; 2491 lc->requested_speed = 0; 2492 lc->advertising = cap | FW_PORT_CAP_ANEG; 2493 } 2494 lc->autoneg = cmd->autoneg; 2495 2496 if (netif_running(dev)) 2497 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan, 2498 lc); 2499 return 0; 2500 } 2501 2502 static void get_pauseparam(struct net_device *dev, 2503 struct ethtool_pauseparam *epause) 2504 { 2505 struct port_info *p = netdev_priv(dev); 2506 2507 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0; 2508 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0; 2509 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0; 2510 } 2511 2512 static int set_pauseparam(struct net_device *dev, 2513 struct ethtool_pauseparam *epause) 2514 { 2515 struct port_info *p = netdev_priv(dev); 2516 struct link_config *lc = &p->link_cfg; 2517 2518 if (epause->autoneg == AUTONEG_DISABLE) 2519 lc->requested_fc = 0; 2520 else if (lc->supported & FW_PORT_CAP_ANEG) 2521 lc->requested_fc = PAUSE_AUTONEG; 2522 else 2523 return -EINVAL; 2524 2525 if (epause->rx_pause) 2526 lc->requested_fc |= PAUSE_RX; 2527 if (epause->tx_pause) 2528 lc->requested_fc |= PAUSE_TX; 2529 if (netif_running(dev)) 2530 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan, 2531 lc); 2532 return 0; 2533 } 2534 2535 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e) 2536 { 2537 const struct port_info *pi = netdev_priv(dev); 2538 const struct sge *s = &pi->adapter->sge; 2539 2540 e->rx_max_pending = MAX_RX_BUFFERS; 2541 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES; 2542 e->rx_jumbo_max_pending = 0; 2543 e->tx_max_pending = MAX_TXQ_ENTRIES; 2544 2545 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8; 2546 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size; 2547 e->rx_jumbo_pending = 0; 2548 e->tx_pending = s->ethtxq[pi->first_qset].q.size; 2549 } 2550 2551 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e) 2552 { 2553 int i; 2554 const struct port_info *pi = netdev_priv(dev); 2555 struct adapter *adapter = pi->adapter; 2556 struct sge *s = &adapter->sge; 2557 2558 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending || 2559 e->tx_pending > MAX_TXQ_ENTRIES || 2560 e->rx_mini_pending > MAX_RSPQ_ENTRIES || 2561 e->rx_mini_pending < MIN_RSPQ_ENTRIES || 2562 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES) 2563 return -EINVAL; 2564 2565 if (adapter->flags & FULL_INIT_DONE) 2566 return -EBUSY; 2567 2568 for (i = 0; i < pi->nqsets; ++i) { 2569 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending; 2570 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8; 2571 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending; 2572 } 2573 return 0; 2574 } 2575 2576 static int closest_timer(const struct sge *s, int time) 2577 { 2578 int i, delta, match = 0, min_delta = INT_MAX; 2579 2580 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) { 2581 delta = time - s->timer_val[i]; 2582 if (delta < 0) 2583 delta = -delta; 2584 if (delta < min_delta) { 2585 min_delta = delta; 2586 match = i; 2587 } 2588 } 2589 return match; 2590 } 2591 2592 static int closest_thres(const struct sge *s, int thres) 2593 { 2594 int i, delta, match = 0, min_delta = INT_MAX; 2595 2596 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) { 2597 delta = thres - s->counter_val[i]; 2598 if (delta < 0) 2599 delta = -delta; 2600 if (delta < min_delta) { 2601 min_delta = delta; 2602 match = i; 2603 } 2604 } 2605 return match; 2606 } 2607 2608 /* 2609 * Return a queue's interrupt hold-off time in us. 0 means no timer. 2610 */ 2611 static unsigned int qtimer_val(const struct adapter *adap, 2612 const struct sge_rspq *q) 2613 { 2614 unsigned int idx = q->intr_params >> 1; 2615 2616 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0; 2617 } 2618 2619 /** 2620 * set_rspq_intr_params - set a queue's interrupt holdoff parameters 2621 * @q: the Rx queue 2622 * @us: the hold-off time in us, or 0 to disable timer 2623 * @cnt: the hold-off packet count, or 0 to disable counter 2624 * 2625 * Sets an Rx queue's interrupt hold-off time and packet count. At least 2626 * one of the two needs to be enabled for the queue to generate interrupts. 2627 */ 2628 static int set_rspq_intr_params(struct sge_rspq *q, 2629 unsigned int us, unsigned int cnt) 2630 { 2631 struct adapter *adap = q->adap; 2632 2633 if ((us | cnt) == 0) 2634 cnt = 1; 2635 2636 if (cnt) { 2637 int err; 2638 u32 v, new_idx; 2639 2640 new_idx = closest_thres(&adap->sge, cnt); 2641 if (q->desc && q->pktcnt_idx != new_idx) { 2642 /* the queue has already been created, update it */ 2643 v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) | 2644 FW_PARAMS_PARAM_X_V( 2645 FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) | 2646 FW_PARAMS_PARAM_YZ_V(q->cntxt_id); 2647 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v, 2648 &new_idx); 2649 if (err) 2650 return err; 2651 } 2652 q->pktcnt_idx = new_idx; 2653 } 2654 2655 us = us == 0 ? 6 : closest_timer(&adap->sge, us); 2656 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0); 2657 return 0; 2658 } 2659 2660 /** 2661 * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete! 2662 * @dev: the network device 2663 * @us: the hold-off time in us, or 0 to disable timer 2664 * @cnt: the hold-off packet count, or 0 to disable counter 2665 * 2666 * Set the RX interrupt hold-off parameters for a network device. 2667 */ 2668 static int set_rx_intr_params(struct net_device *dev, 2669 unsigned int us, unsigned int cnt) 2670 { 2671 int i, err; 2672 struct port_info *pi = netdev_priv(dev); 2673 struct adapter *adap = pi->adapter; 2674 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; 2675 2676 for (i = 0; i < pi->nqsets; i++, q++) { 2677 err = set_rspq_intr_params(&q->rspq, us, cnt); 2678 if (err) 2679 return err; 2680 } 2681 return 0; 2682 } 2683 2684 static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx) 2685 { 2686 int i; 2687 struct port_info *pi = netdev_priv(dev); 2688 struct adapter *adap = pi->adapter; 2689 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; 2690 2691 for (i = 0; i < pi->nqsets; i++, q++) 2692 q->rspq.adaptive_rx = adaptive_rx; 2693 2694 return 0; 2695 } 2696 2697 static int get_adaptive_rx_setting(struct net_device *dev) 2698 { 2699 struct port_info *pi = netdev_priv(dev); 2700 struct adapter *adap = pi->adapter; 2701 struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset]; 2702 2703 return q->rspq.adaptive_rx; 2704 } 2705 2706 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c) 2707 { 2708 set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce); 2709 return set_rx_intr_params(dev, c->rx_coalesce_usecs, 2710 c->rx_max_coalesced_frames); 2711 } 2712 2713 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c) 2714 { 2715 const struct port_info *pi = netdev_priv(dev); 2716 const struct adapter *adap = pi->adapter; 2717 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq; 2718 2719 c->rx_coalesce_usecs = qtimer_val(adap, rq); 2720 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ? 2721 adap->sge.counter_val[rq->pktcnt_idx] : 0; 2722 c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev); 2723 return 0; 2724 } 2725 2726 /** 2727 * eeprom_ptov - translate a physical EEPROM address to virtual 2728 * @phys_addr: the physical EEPROM address 2729 * @fn: the PCI function number 2730 * @sz: size of function-specific area 2731 * 2732 * Translate a physical EEPROM address to virtual. The first 1K is 2733 * accessed through virtual addresses starting at 31K, the rest is 2734 * accessed through virtual addresses starting at 0. 2735 * 2736 * The mapping is as follows: 2737 * [0..1K) -> [31K..32K) 2738 * [1K..1K+A) -> [31K-A..31K) 2739 * [1K+A..ES) -> [0..ES-A-1K) 2740 * 2741 * where A = @fn * @sz, and ES = EEPROM size. 2742 */ 2743 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz) 2744 { 2745 fn *= sz; 2746 if (phys_addr < 1024) 2747 return phys_addr + (31 << 10); 2748 if (phys_addr < 1024 + fn) 2749 return 31744 - fn + phys_addr - 1024; 2750 if (phys_addr < EEPROMSIZE) 2751 return phys_addr - 1024 - fn; 2752 return -EINVAL; 2753 } 2754 2755 /* 2756 * The next two routines implement eeprom read/write from physical addresses. 2757 */ 2758 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v) 2759 { 2760 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE); 2761 2762 if (vaddr >= 0) 2763 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v); 2764 return vaddr < 0 ? vaddr : 0; 2765 } 2766 2767 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v) 2768 { 2769 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE); 2770 2771 if (vaddr >= 0) 2772 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v); 2773 return vaddr < 0 ? vaddr : 0; 2774 } 2775 2776 #define EEPROM_MAGIC 0x38E2F10C 2777 2778 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e, 2779 u8 *data) 2780 { 2781 int i, err = 0; 2782 struct adapter *adapter = netdev2adap(dev); 2783 2784 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL); 2785 if (!buf) 2786 return -ENOMEM; 2787 2788 e->magic = EEPROM_MAGIC; 2789 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4) 2790 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]); 2791 2792 if (!err) 2793 memcpy(data, buf + e->offset, e->len); 2794 kfree(buf); 2795 return err; 2796 } 2797 2798 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom, 2799 u8 *data) 2800 { 2801 u8 *buf; 2802 int err = 0; 2803 u32 aligned_offset, aligned_len, *p; 2804 struct adapter *adapter = netdev2adap(dev); 2805 2806 if (eeprom->magic != EEPROM_MAGIC) 2807 return -EINVAL; 2808 2809 aligned_offset = eeprom->offset & ~3; 2810 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3; 2811 2812 if (adapter->fn > 0) { 2813 u32 start = 1024 + adapter->fn * EEPROMPFSIZE; 2814 2815 if (aligned_offset < start || 2816 aligned_offset + aligned_len > start + EEPROMPFSIZE) 2817 return -EPERM; 2818 } 2819 2820 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) { 2821 /* 2822 * RMW possibly needed for first or last words. 2823 */ 2824 buf = kmalloc(aligned_len, GFP_KERNEL); 2825 if (!buf) 2826 return -ENOMEM; 2827 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf); 2828 if (!err && aligned_len > 4) 2829 err = eeprom_rd_phys(adapter, 2830 aligned_offset + aligned_len - 4, 2831 (u32 *)&buf[aligned_len - 4]); 2832 if (err) 2833 goto out; 2834 memcpy(buf + (eeprom->offset & 3), data, eeprom->len); 2835 } else 2836 buf = data; 2837 2838 err = t4_seeprom_wp(adapter, false); 2839 if (err) 2840 goto out; 2841 2842 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) { 2843 err = eeprom_wr_phys(adapter, aligned_offset, *p); 2844 aligned_offset += 4; 2845 } 2846 2847 if (!err) 2848 err = t4_seeprom_wp(adapter, true); 2849 out: 2850 if (buf != data) 2851 kfree(buf); 2852 return err; 2853 } 2854 2855 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef) 2856 { 2857 int ret; 2858 const struct firmware *fw; 2859 struct adapter *adap = netdev2adap(netdev); 2860 unsigned int mbox = PCIE_FW_MASTER_M + 1; 2861 2862 ef->data[sizeof(ef->data) - 1] = '\0'; 2863 ret = request_firmware(&fw, ef->data, adap->pdev_dev); 2864 if (ret < 0) 2865 return ret; 2866 2867 /* If the adapter has been fully initialized then we'll go ahead and 2868 * try to get the firmware's cooperation in upgrading to the new 2869 * firmware image otherwise we'll try to do the entire job from the 2870 * host ... and we always "force" the operation in this path. 2871 */ 2872 if (adap->flags & FULL_INIT_DONE) 2873 mbox = adap->mbox; 2874 2875 ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1); 2876 release_firmware(fw); 2877 if (!ret) 2878 dev_info(adap->pdev_dev, "loaded firmware %s," 2879 " reload cxgb4 driver\n", ef->data); 2880 return ret; 2881 } 2882 2883 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC) 2884 #define BCAST_CRC 0xa0ccc1a6 2885 2886 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol) 2887 { 2888 wol->supported = WAKE_BCAST | WAKE_MAGIC; 2889 wol->wolopts = netdev2adap(dev)->wol; 2890 memset(&wol->sopass, 0, sizeof(wol->sopass)); 2891 } 2892 2893 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol) 2894 { 2895 int err = 0; 2896 struct port_info *pi = netdev_priv(dev); 2897 2898 if (wol->wolopts & ~WOL_SUPPORTED) 2899 return -EINVAL; 2900 t4_wol_magic_enable(pi->adapter, pi->tx_chan, 2901 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL); 2902 if (wol->wolopts & WAKE_BCAST) { 2903 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL, 2904 ~0ULL, 0, false); 2905 if (!err) 2906 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1, 2907 ~6ULL, ~0ULL, BCAST_CRC, true); 2908 } else 2909 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false); 2910 return err; 2911 } 2912 2913 static int cxgb_set_features(struct net_device *dev, netdev_features_t features) 2914 { 2915 const struct port_info *pi = netdev_priv(dev); 2916 netdev_features_t changed = dev->features ^ features; 2917 int err; 2918 2919 if (!(changed & NETIF_F_HW_VLAN_CTAG_RX)) 2920 return 0; 2921 2922 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1, 2923 -1, -1, -1, 2924 !!(features & NETIF_F_HW_VLAN_CTAG_RX), true); 2925 if (unlikely(err)) 2926 dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX; 2927 return err; 2928 } 2929 2930 static u32 get_rss_table_size(struct net_device *dev) 2931 { 2932 const struct port_info *pi = netdev_priv(dev); 2933 2934 return pi->rss_size; 2935 } 2936 2937 static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc) 2938 { 2939 const struct port_info *pi = netdev_priv(dev); 2940 unsigned int n = pi->rss_size; 2941 2942 if (hfunc) 2943 *hfunc = ETH_RSS_HASH_TOP; 2944 if (!p) 2945 return 0; 2946 while (n--) 2947 p[n] = pi->rss[n]; 2948 return 0; 2949 } 2950 2951 static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key, 2952 const u8 hfunc) 2953 { 2954 unsigned int i; 2955 struct port_info *pi = netdev_priv(dev); 2956 2957 /* We require at least one supported parameter to be changed and no 2958 * change in any of the unsupported parameters 2959 */ 2960 if (key || 2961 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) 2962 return -EOPNOTSUPP; 2963 if (!p) 2964 return 0; 2965 2966 for (i = 0; i < pi->rss_size; i++) 2967 pi->rss[i] = p[i]; 2968 if (pi->adapter->flags & FULL_INIT_DONE) 2969 return write_rss(pi, pi->rss); 2970 return 0; 2971 } 2972 2973 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info, 2974 u32 *rules) 2975 { 2976 const struct port_info *pi = netdev_priv(dev); 2977 2978 switch (info->cmd) { 2979 case ETHTOOL_GRXFH: { 2980 unsigned int v = pi->rss_mode; 2981 2982 info->data = 0; 2983 switch (info->flow_type) { 2984 case TCP_V4_FLOW: 2985 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) 2986 info->data = RXH_IP_SRC | RXH_IP_DST | 2987 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2988 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) 2989 info->data = RXH_IP_SRC | RXH_IP_DST; 2990 break; 2991 case UDP_V4_FLOW: 2992 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) && 2993 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F)) 2994 info->data = RXH_IP_SRC | RXH_IP_DST | 2995 RXH_L4_B_0_1 | RXH_L4_B_2_3; 2996 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) 2997 info->data = RXH_IP_SRC | RXH_IP_DST; 2998 break; 2999 case SCTP_V4_FLOW: 3000 case AH_ESP_V4_FLOW: 3001 case IPV4_FLOW: 3002 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F) 3003 info->data = RXH_IP_SRC | RXH_IP_DST; 3004 break; 3005 case TCP_V6_FLOW: 3006 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) 3007 info->data = RXH_IP_SRC | RXH_IP_DST | 3008 RXH_L4_B_0_1 | RXH_L4_B_2_3; 3009 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) 3010 info->data = RXH_IP_SRC | RXH_IP_DST; 3011 break; 3012 case UDP_V6_FLOW: 3013 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) && 3014 (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F)) 3015 info->data = RXH_IP_SRC | RXH_IP_DST | 3016 RXH_L4_B_0_1 | RXH_L4_B_2_3; 3017 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) 3018 info->data = RXH_IP_SRC | RXH_IP_DST; 3019 break; 3020 case SCTP_V6_FLOW: 3021 case AH_ESP_V6_FLOW: 3022 case IPV6_FLOW: 3023 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F) 3024 info->data = RXH_IP_SRC | RXH_IP_DST; 3025 break; 3026 } 3027 return 0; 3028 } 3029 case ETHTOOL_GRXRINGS: 3030 info->data = pi->nqsets; 3031 return 0; 3032 } 3033 return -EOPNOTSUPP; 3034 } 3035 3036 static const struct ethtool_ops cxgb_ethtool_ops = { 3037 .get_settings = get_settings, 3038 .set_settings = set_settings, 3039 .get_drvinfo = get_drvinfo, 3040 .get_msglevel = get_msglevel, 3041 .set_msglevel = set_msglevel, 3042 .get_ringparam = get_sge_param, 3043 .set_ringparam = set_sge_param, 3044 .get_coalesce = get_coalesce, 3045 .set_coalesce = set_coalesce, 3046 .get_eeprom_len = get_eeprom_len, 3047 .get_eeprom = get_eeprom, 3048 .set_eeprom = set_eeprom, 3049 .get_pauseparam = get_pauseparam, 3050 .set_pauseparam = set_pauseparam, 3051 .get_link = ethtool_op_get_link, 3052 .get_strings = get_strings, 3053 .set_phys_id = identify_port, 3054 .nway_reset = restart_autoneg, 3055 .get_sset_count = get_sset_count, 3056 .get_ethtool_stats = get_stats, 3057 .get_regs_len = get_regs_len, 3058 .get_regs = get_regs, 3059 .get_wol = get_wol, 3060 .set_wol = set_wol, 3061 .get_rxnfc = get_rxnfc, 3062 .get_rxfh_indir_size = get_rss_table_size, 3063 .get_rxfh = get_rss_table, 3064 .set_rxfh = set_rss_table, 3065 .flash_device = set_flash, 3066 }; 3067 3068 static int setup_debugfs(struct adapter *adap) 3069 { 3070 if (IS_ERR_OR_NULL(adap->debugfs_root)) 3071 return -1; 3072 3073 #ifdef CONFIG_DEBUG_FS 3074 t4_setup_debugfs(adap); 3075 #endif 3076 return 0; 3077 } 3078 3079 /* 3080 * upper-layer driver support 3081 */ 3082 3083 /* 3084 * Allocate an active-open TID and set it to the supplied value. 3085 */ 3086 int cxgb4_alloc_atid(struct tid_info *t, void *data) 3087 { 3088 int atid = -1; 3089 3090 spin_lock_bh(&t->atid_lock); 3091 if (t->afree) { 3092 union aopen_entry *p = t->afree; 3093 3094 atid = (p - t->atid_tab) + t->atid_base; 3095 t->afree = p->next; 3096 p->data = data; 3097 t->atids_in_use++; 3098 } 3099 spin_unlock_bh(&t->atid_lock); 3100 return atid; 3101 } 3102 EXPORT_SYMBOL(cxgb4_alloc_atid); 3103 3104 /* 3105 * Release an active-open TID. 3106 */ 3107 void cxgb4_free_atid(struct tid_info *t, unsigned int atid) 3108 { 3109 union aopen_entry *p = &t->atid_tab[atid - t->atid_base]; 3110 3111 spin_lock_bh(&t->atid_lock); 3112 p->next = t->afree; 3113 t->afree = p; 3114 t->atids_in_use--; 3115 spin_unlock_bh(&t->atid_lock); 3116 } 3117 EXPORT_SYMBOL(cxgb4_free_atid); 3118 3119 /* 3120 * Allocate a server TID and set it to the supplied value. 3121 */ 3122 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data) 3123 { 3124 int stid; 3125 3126 spin_lock_bh(&t->stid_lock); 3127 if (family == PF_INET) { 3128 stid = find_first_zero_bit(t->stid_bmap, t->nstids); 3129 if (stid < t->nstids) 3130 __set_bit(stid, t->stid_bmap); 3131 else 3132 stid = -1; 3133 } else { 3134 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2); 3135 if (stid < 0) 3136 stid = -1; 3137 } 3138 if (stid >= 0) { 3139 t->stid_tab[stid].data = data; 3140 stid += t->stid_base; 3141 /* IPv6 requires max of 520 bits or 16 cells in TCAM 3142 * This is equivalent to 4 TIDs. With CLIP enabled it 3143 * needs 2 TIDs. 3144 */ 3145 if (family == PF_INET) 3146 t->stids_in_use++; 3147 else 3148 t->stids_in_use += 4; 3149 } 3150 spin_unlock_bh(&t->stid_lock); 3151 return stid; 3152 } 3153 EXPORT_SYMBOL(cxgb4_alloc_stid); 3154 3155 /* Allocate a server filter TID and set it to the supplied value. 3156 */ 3157 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data) 3158 { 3159 int stid; 3160 3161 spin_lock_bh(&t->stid_lock); 3162 if (family == PF_INET) { 3163 stid = find_next_zero_bit(t->stid_bmap, 3164 t->nstids + t->nsftids, t->nstids); 3165 if (stid < (t->nstids + t->nsftids)) 3166 __set_bit(stid, t->stid_bmap); 3167 else 3168 stid = -1; 3169 } else { 3170 stid = -1; 3171 } 3172 if (stid >= 0) { 3173 t->stid_tab[stid].data = data; 3174 stid -= t->nstids; 3175 stid += t->sftid_base; 3176 t->stids_in_use++; 3177 } 3178 spin_unlock_bh(&t->stid_lock); 3179 return stid; 3180 } 3181 EXPORT_SYMBOL(cxgb4_alloc_sftid); 3182 3183 /* Release a server TID. 3184 */ 3185 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family) 3186 { 3187 /* Is it a server filter TID? */ 3188 if (t->nsftids && (stid >= t->sftid_base)) { 3189 stid -= t->sftid_base; 3190 stid += t->nstids; 3191 } else { 3192 stid -= t->stid_base; 3193 } 3194 3195 spin_lock_bh(&t->stid_lock); 3196 if (family == PF_INET) 3197 __clear_bit(stid, t->stid_bmap); 3198 else 3199 bitmap_release_region(t->stid_bmap, stid, 2); 3200 t->stid_tab[stid].data = NULL; 3201 if (family == PF_INET) 3202 t->stids_in_use--; 3203 else 3204 t->stids_in_use -= 4; 3205 spin_unlock_bh(&t->stid_lock); 3206 } 3207 EXPORT_SYMBOL(cxgb4_free_stid); 3208 3209 /* 3210 * Populate a TID_RELEASE WR. Caller must properly size the skb. 3211 */ 3212 static void mk_tid_release(struct sk_buff *skb, unsigned int chan, 3213 unsigned int tid) 3214 { 3215 struct cpl_tid_release *req; 3216 3217 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan); 3218 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req)); 3219 INIT_TP_WR(req, tid); 3220 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 3221 } 3222 3223 /* 3224 * Queue a TID release request and if necessary schedule a work queue to 3225 * process it. 3226 */ 3227 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan, 3228 unsigned int tid) 3229 { 3230 void **p = &t->tid_tab[tid]; 3231 struct adapter *adap = container_of(t, struct adapter, tids); 3232 3233 spin_lock_bh(&adap->tid_release_lock); 3234 *p = adap->tid_release_head; 3235 /* Low 2 bits encode the Tx channel number */ 3236 adap->tid_release_head = (void **)((uintptr_t)p | chan); 3237 if (!adap->tid_release_task_busy) { 3238 adap->tid_release_task_busy = true; 3239 queue_work(adap->workq, &adap->tid_release_task); 3240 } 3241 spin_unlock_bh(&adap->tid_release_lock); 3242 } 3243 3244 /* 3245 * Process the list of pending TID release requests. 3246 */ 3247 static void process_tid_release_list(struct work_struct *work) 3248 { 3249 struct sk_buff *skb; 3250 struct adapter *adap; 3251 3252 adap = container_of(work, struct adapter, tid_release_task); 3253 3254 spin_lock_bh(&adap->tid_release_lock); 3255 while (adap->tid_release_head) { 3256 void **p = adap->tid_release_head; 3257 unsigned int chan = (uintptr_t)p & 3; 3258 p = (void *)p - chan; 3259 3260 adap->tid_release_head = *p; 3261 *p = NULL; 3262 spin_unlock_bh(&adap->tid_release_lock); 3263 3264 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release), 3265 GFP_KERNEL))) 3266 schedule_timeout_uninterruptible(1); 3267 3268 mk_tid_release(skb, chan, p - adap->tids.tid_tab); 3269 t4_ofld_send(adap, skb); 3270 spin_lock_bh(&adap->tid_release_lock); 3271 } 3272 adap->tid_release_task_busy = false; 3273 spin_unlock_bh(&adap->tid_release_lock); 3274 } 3275 3276 /* 3277 * Release a TID and inform HW. If we are unable to allocate the release 3278 * message we defer to a work queue. 3279 */ 3280 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid) 3281 { 3282 void *old; 3283 struct sk_buff *skb; 3284 struct adapter *adap = container_of(t, struct adapter, tids); 3285 3286 old = t->tid_tab[tid]; 3287 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); 3288 if (likely(skb)) { 3289 t->tid_tab[tid] = NULL; 3290 mk_tid_release(skb, chan, tid); 3291 t4_ofld_send(adap, skb); 3292 } else 3293 cxgb4_queue_tid_release(t, chan, tid); 3294 if (old) 3295 atomic_dec(&t->tids_in_use); 3296 } 3297 EXPORT_SYMBOL(cxgb4_remove_tid); 3298 3299 /* 3300 * Allocate and initialize the TID tables. Returns 0 on success. 3301 */ 3302 static int tid_init(struct tid_info *t) 3303 { 3304 size_t size; 3305 unsigned int stid_bmap_size; 3306 unsigned int natids = t->natids; 3307 struct adapter *adap = container_of(t, struct adapter, tids); 3308 3309 stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids); 3310 size = t->ntids * sizeof(*t->tid_tab) + 3311 natids * sizeof(*t->atid_tab) + 3312 t->nstids * sizeof(*t->stid_tab) + 3313 t->nsftids * sizeof(*t->stid_tab) + 3314 stid_bmap_size * sizeof(long) + 3315 t->nftids * sizeof(*t->ftid_tab) + 3316 t->nsftids * sizeof(*t->ftid_tab); 3317 3318 t->tid_tab = t4_alloc_mem(size); 3319 if (!t->tid_tab) 3320 return -ENOMEM; 3321 3322 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids]; 3323 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids]; 3324 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids]; 3325 t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size]; 3326 spin_lock_init(&t->stid_lock); 3327 spin_lock_init(&t->atid_lock); 3328 3329 t->stids_in_use = 0; 3330 t->afree = NULL; 3331 t->atids_in_use = 0; 3332 atomic_set(&t->tids_in_use, 0); 3333 3334 /* Setup the free list for atid_tab and clear the stid bitmap. */ 3335 if (natids) { 3336 while (--natids) 3337 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 3338 t->afree = t->atid_tab; 3339 } 3340 bitmap_zero(t->stid_bmap, t->nstids + t->nsftids); 3341 /* Reserve stid 0 for T4/T5 adapters */ 3342 if (!t->stid_base && 3343 (is_t4(adap->params.chip) || is_t5(adap->params.chip))) 3344 __set_bit(0, t->stid_bmap); 3345 3346 return 0; 3347 } 3348 3349 int cxgb4_clip_get(const struct net_device *dev, 3350 const struct in6_addr *lip) 3351 { 3352 struct adapter *adap; 3353 struct fw_clip_cmd c; 3354 3355 adap = netdev2adap(dev); 3356 memset(&c, 0, sizeof(c)); 3357 c.op_to_write = htonl(FW_CMD_OP_V(FW_CLIP_CMD) | 3358 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 3359 c.alloc_to_len16 = htonl(FW_CLIP_CMD_ALLOC_F | FW_LEN16(c)); 3360 c.ip_hi = *(__be64 *)(lip->s6_addr); 3361 c.ip_lo = *(__be64 *)(lip->s6_addr + 8); 3362 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false); 3363 } 3364 EXPORT_SYMBOL(cxgb4_clip_get); 3365 3366 int cxgb4_clip_release(const struct net_device *dev, 3367 const struct in6_addr *lip) 3368 { 3369 struct adapter *adap; 3370 struct fw_clip_cmd c; 3371 3372 adap = netdev2adap(dev); 3373 memset(&c, 0, sizeof(c)); 3374 c.op_to_write = htonl(FW_CMD_OP_V(FW_CLIP_CMD) | 3375 FW_CMD_REQUEST_F | FW_CMD_READ_F); 3376 c.alloc_to_len16 = htonl(FW_CLIP_CMD_FREE_F | FW_LEN16(c)); 3377 c.ip_hi = *(__be64 *)(lip->s6_addr); 3378 c.ip_lo = *(__be64 *)(lip->s6_addr + 8); 3379 return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false); 3380 } 3381 EXPORT_SYMBOL(cxgb4_clip_release); 3382 3383 /** 3384 * cxgb4_create_server - create an IP server 3385 * @dev: the device 3386 * @stid: the server TID 3387 * @sip: local IP address to bind server to 3388 * @sport: the server's TCP port 3389 * @queue: queue to direct messages from this server to 3390 * 3391 * Create an IP server for the given port and address. 3392 * Returns <0 on error and one of the %NET_XMIT_* values on success. 3393 */ 3394 int cxgb4_create_server(const struct net_device *dev, unsigned int stid, 3395 __be32 sip, __be16 sport, __be16 vlan, 3396 unsigned int queue) 3397 { 3398 unsigned int chan; 3399 struct sk_buff *skb; 3400 struct adapter *adap; 3401 struct cpl_pass_open_req *req; 3402 int ret; 3403 3404 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 3405 if (!skb) 3406 return -ENOMEM; 3407 3408 adap = netdev2adap(dev); 3409 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req)); 3410 INIT_TP_WR(req, 0); 3411 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid)); 3412 req->local_port = sport; 3413 req->peer_port = htons(0); 3414 req->local_ip = sip; 3415 req->peer_ip = htonl(0); 3416 chan = rxq_to_chan(&adap->sge, queue); 3417 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 3418 req->opt1 = cpu_to_be64(CONN_POLICY_ASK | 3419 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue)); 3420 ret = t4_mgmt_tx(adap, skb); 3421 return net_xmit_eval(ret); 3422 } 3423 EXPORT_SYMBOL(cxgb4_create_server); 3424 3425 /* cxgb4_create_server6 - create an IPv6 server 3426 * @dev: the device 3427 * @stid: the server TID 3428 * @sip: local IPv6 address to bind server to 3429 * @sport: the server's TCP port 3430 * @queue: queue to direct messages from this server to 3431 * 3432 * Create an IPv6 server for the given port and address. 3433 * Returns <0 on error and one of the %NET_XMIT_* values on success. 3434 */ 3435 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid, 3436 const struct in6_addr *sip, __be16 sport, 3437 unsigned int queue) 3438 { 3439 unsigned int chan; 3440 struct sk_buff *skb; 3441 struct adapter *adap; 3442 struct cpl_pass_open_req6 *req; 3443 int ret; 3444 3445 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 3446 if (!skb) 3447 return -ENOMEM; 3448 3449 adap = netdev2adap(dev); 3450 req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req)); 3451 INIT_TP_WR(req, 0); 3452 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid)); 3453 req->local_port = sport; 3454 req->peer_port = htons(0); 3455 req->local_ip_hi = *(__be64 *)(sip->s6_addr); 3456 req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8); 3457 req->peer_ip_hi = cpu_to_be64(0); 3458 req->peer_ip_lo = cpu_to_be64(0); 3459 chan = rxq_to_chan(&adap->sge, queue); 3460 req->opt0 = cpu_to_be64(TX_CHAN_V(chan)); 3461 req->opt1 = cpu_to_be64(CONN_POLICY_ASK | 3462 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue)); 3463 ret = t4_mgmt_tx(adap, skb); 3464 return net_xmit_eval(ret); 3465 } 3466 EXPORT_SYMBOL(cxgb4_create_server6); 3467 3468 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid, 3469 unsigned int queue, bool ipv6) 3470 { 3471 struct sk_buff *skb; 3472 struct adapter *adap; 3473 struct cpl_close_listsvr_req *req; 3474 int ret; 3475 3476 adap = netdev2adap(dev); 3477 3478 skb = alloc_skb(sizeof(*req), GFP_KERNEL); 3479 if (!skb) 3480 return -ENOMEM; 3481 3482 req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req)); 3483 INIT_TP_WR(req, 0); 3484 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid)); 3485 req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) : 3486 LISTSVR_IPV6(0)) | QUEUENO(queue)); 3487 ret = t4_mgmt_tx(adap, skb); 3488 return net_xmit_eval(ret); 3489 } 3490 EXPORT_SYMBOL(cxgb4_remove_server); 3491 3492 /** 3493 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU 3494 * @mtus: the HW MTU table 3495 * @mtu: the target MTU 3496 * @idx: index of selected entry in the MTU table 3497 * 3498 * Returns the index and the value in the HW MTU table that is closest to 3499 * but does not exceed @mtu, unless @mtu is smaller than any value in the 3500 * table, in which case that smallest available value is selected. 3501 */ 3502 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu, 3503 unsigned int *idx) 3504 { 3505 unsigned int i = 0; 3506 3507 while (i < NMTUS - 1 && mtus[i + 1] <= mtu) 3508 ++i; 3509 if (idx) 3510 *idx = i; 3511 return mtus[i]; 3512 } 3513 EXPORT_SYMBOL(cxgb4_best_mtu); 3514 3515 /** 3516 * cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned 3517 * @mtus: the HW MTU table 3518 * @header_size: Header Size 3519 * @data_size_max: maximum Data Segment Size 3520 * @data_size_align: desired Data Segment Size Alignment (2^N) 3521 * @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL) 3522 * 3523 * Similar to cxgb4_best_mtu() but instead of searching the Hardware 3524 * MTU Table based solely on a Maximum MTU parameter, we break that 3525 * parameter up into a Header Size and Maximum Data Segment Size, and 3526 * provide a desired Data Segment Size Alignment. If we find an MTU in 3527 * the Hardware MTU Table which will result in a Data Segment Size with 3528 * the requested alignment _and_ that MTU isn't "too far" from the 3529 * closest MTU, then we'll return that rather than the closest MTU. 3530 */ 3531 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus, 3532 unsigned short header_size, 3533 unsigned short data_size_max, 3534 unsigned short data_size_align, 3535 unsigned int *mtu_idxp) 3536 { 3537 unsigned short max_mtu = header_size + data_size_max; 3538 unsigned short data_size_align_mask = data_size_align - 1; 3539 int mtu_idx, aligned_mtu_idx; 3540 3541 /* Scan the MTU Table till we find an MTU which is larger than our 3542 * Maximum MTU or we reach the end of the table. Along the way, 3543 * record the last MTU found, if any, which will result in a Data 3544 * Segment Length matching the requested alignment. 3545 */ 3546 for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) { 3547 unsigned short data_size = mtus[mtu_idx] - header_size; 3548 3549 /* If this MTU minus the Header Size would result in a 3550 * Data Segment Size of the desired alignment, remember it. 3551 */ 3552 if ((data_size & data_size_align_mask) == 0) 3553 aligned_mtu_idx = mtu_idx; 3554 3555 /* If we're not at the end of the Hardware MTU Table and the 3556 * next element is larger than our Maximum MTU, drop out of 3557 * the loop. 3558 */ 3559 if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu) 3560 break; 3561 } 3562 3563 /* If we fell out of the loop because we ran to the end of the table, 3564 * then we just have to use the last [largest] entry. 3565 */ 3566 if (mtu_idx == NMTUS) 3567 mtu_idx--; 3568 3569 /* If we found an MTU which resulted in the requested Data Segment 3570 * Length alignment and that's "not far" from the largest MTU which is 3571 * less than or equal to the maximum MTU, then use that. 3572 */ 3573 if (aligned_mtu_idx >= 0 && 3574 mtu_idx - aligned_mtu_idx <= 1) 3575 mtu_idx = aligned_mtu_idx; 3576 3577 /* If the caller has passed in an MTU Index pointer, pass the 3578 * MTU Index back. Return the MTU value. 3579 */ 3580 if (mtu_idxp) 3581 *mtu_idxp = mtu_idx; 3582 return mtus[mtu_idx]; 3583 } 3584 EXPORT_SYMBOL(cxgb4_best_aligned_mtu); 3585 3586 /** 3587 * cxgb4_port_chan - get the HW channel of a port 3588 * @dev: the net device for the port 3589 * 3590 * Return the HW Tx channel of the given port. 3591 */ 3592 unsigned int cxgb4_port_chan(const struct net_device *dev) 3593 { 3594 return netdev2pinfo(dev)->tx_chan; 3595 } 3596 EXPORT_SYMBOL(cxgb4_port_chan); 3597 3598 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo) 3599 { 3600 struct adapter *adap = netdev2adap(dev); 3601 u32 v1, v2, lp_count, hp_count; 3602 3603 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS); 3604 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2); 3605 if (is_t4(adap->params.chip)) { 3606 lp_count = G_LP_COUNT(v1); 3607 hp_count = G_HP_COUNT(v1); 3608 } else { 3609 lp_count = G_LP_COUNT_T5(v1); 3610 hp_count = G_HP_COUNT_T5(v2); 3611 } 3612 return lpfifo ? lp_count : hp_count; 3613 } 3614 EXPORT_SYMBOL(cxgb4_dbfifo_count); 3615 3616 /** 3617 * cxgb4_port_viid - get the VI id of a port 3618 * @dev: the net device for the port 3619 * 3620 * Return the VI id of the given port. 3621 */ 3622 unsigned int cxgb4_port_viid(const struct net_device *dev) 3623 { 3624 return netdev2pinfo(dev)->viid; 3625 } 3626 EXPORT_SYMBOL(cxgb4_port_viid); 3627 3628 /** 3629 * cxgb4_port_idx - get the index of a port 3630 * @dev: the net device for the port 3631 * 3632 * Return the index of the given port. 3633 */ 3634 unsigned int cxgb4_port_idx(const struct net_device *dev) 3635 { 3636 return netdev2pinfo(dev)->port_id; 3637 } 3638 EXPORT_SYMBOL(cxgb4_port_idx); 3639 3640 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4, 3641 struct tp_tcp_stats *v6) 3642 { 3643 struct adapter *adap = pci_get_drvdata(pdev); 3644 3645 spin_lock(&adap->stats_lock); 3646 t4_tp_get_tcp_stats(adap, v4, v6); 3647 spin_unlock(&adap->stats_lock); 3648 } 3649 EXPORT_SYMBOL(cxgb4_get_tcp_stats); 3650 3651 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask, 3652 const unsigned int *pgsz_order) 3653 { 3654 struct adapter *adap = netdev2adap(dev); 3655 3656 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask); 3657 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) | 3658 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) | 3659 HPZ3(pgsz_order[3])); 3660 } 3661 EXPORT_SYMBOL(cxgb4_iscsi_init); 3662 3663 int cxgb4_flush_eq_cache(struct net_device *dev) 3664 { 3665 struct adapter *adap = netdev2adap(dev); 3666 int ret; 3667 3668 ret = t4_fwaddrspace_write(adap, adap->mbox, 3669 0xe1000000 + A_SGE_CTXT_CMD, 0x20000000); 3670 return ret; 3671 } 3672 EXPORT_SYMBOL(cxgb4_flush_eq_cache); 3673 3674 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx) 3675 { 3676 u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8; 3677 __be64 indices; 3678 int ret; 3679 3680 spin_lock(&adap->win0_lock); 3681 ret = t4_memory_rw(adap, 0, MEM_EDC0, addr, 3682 sizeof(indices), (__be32 *)&indices, 3683 T4_MEMORY_READ); 3684 spin_unlock(&adap->win0_lock); 3685 if (!ret) { 3686 *cidx = (be64_to_cpu(indices) >> 25) & 0xffff; 3687 *pidx = (be64_to_cpu(indices) >> 9) & 0xffff; 3688 } 3689 return ret; 3690 } 3691 3692 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx, 3693 u16 size) 3694 { 3695 struct adapter *adap = netdev2adap(dev); 3696 u16 hw_pidx, hw_cidx; 3697 int ret; 3698 3699 ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx); 3700 if (ret) 3701 goto out; 3702 3703 if (pidx != hw_pidx) { 3704 u16 delta; 3705 3706 if (pidx >= hw_pidx) 3707 delta = pidx - hw_pidx; 3708 else 3709 delta = size - hw_pidx + pidx; 3710 wmb(); 3711 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL), 3712 QID(qid) | PIDX(delta)); 3713 } 3714 out: 3715 return ret; 3716 } 3717 EXPORT_SYMBOL(cxgb4_sync_txq_pidx); 3718 3719 void cxgb4_disable_db_coalescing(struct net_device *dev) 3720 { 3721 struct adapter *adap; 3722 3723 adap = netdev2adap(dev); 3724 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 3725 F_NOCOALESCE); 3726 } 3727 EXPORT_SYMBOL(cxgb4_disable_db_coalescing); 3728 3729 void cxgb4_enable_db_coalescing(struct net_device *dev) 3730 { 3731 struct adapter *adap; 3732 3733 adap = netdev2adap(dev); 3734 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0); 3735 } 3736 EXPORT_SYMBOL(cxgb4_enable_db_coalescing); 3737 3738 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte) 3739 { 3740 struct adapter *adap; 3741 u32 offset, memtype, memaddr; 3742 u32 edc0_size, edc1_size, mc0_size, mc1_size, size; 3743 u32 edc0_end, edc1_end, mc0_end, mc1_end; 3744 int ret; 3745 3746 adap = netdev2adap(dev); 3747 3748 offset = ((stag >> 8) * 32) + adap->vres.stag.start; 3749 3750 /* Figure out where the offset lands in the Memory Type/Address scheme. 3751 * This code assumes that the memory is laid out starting at offset 0 3752 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0 3753 * and EDC1. Some cards will have neither MC0 nor MC1, most cards have 3754 * MC0, and some have both MC0 and MC1. 3755 */ 3756 size = t4_read_reg(adap, MA_EDRAM0_BAR_A); 3757 edc0_size = EDRAM0_SIZE_G(size) << 20; 3758 size = t4_read_reg(adap, MA_EDRAM1_BAR_A); 3759 edc1_size = EDRAM1_SIZE_G(size) << 20; 3760 size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A); 3761 mc0_size = EXT_MEM0_SIZE_G(size) << 20; 3762 3763 edc0_end = edc0_size; 3764 edc1_end = edc0_end + edc1_size; 3765 mc0_end = edc1_end + mc0_size; 3766 3767 if (offset < edc0_end) { 3768 memtype = MEM_EDC0; 3769 memaddr = offset; 3770 } else if (offset < edc1_end) { 3771 memtype = MEM_EDC1; 3772 memaddr = offset - edc0_end; 3773 } else { 3774 if (offset < mc0_end) { 3775 memtype = MEM_MC0; 3776 memaddr = offset - edc1_end; 3777 } else if (is_t4(adap->params.chip)) { 3778 /* T4 only has a single memory channel */ 3779 goto err; 3780 } else { 3781 size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A); 3782 mc1_size = EXT_MEM1_SIZE_G(size) << 20; 3783 mc1_end = mc0_end + mc1_size; 3784 if (offset < mc1_end) { 3785 memtype = MEM_MC1; 3786 memaddr = offset - mc0_end; 3787 } else { 3788 /* offset beyond the end of any memory */ 3789 goto err; 3790 } 3791 } 3792 } 3793 3794 spin_lock(&adap->win0_lock); 3795 ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ); 3796 spin_unlock(&adap->win0_lock); 3797 return ret; 3798 3799 err: 3800 dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n", 3801 stag, offset); 3802 return -EINVAL; 3803 } 3804 EXPORT_SYMBOL(cxgb4_read_tpte); 3805 3806 u64 cxgb4_read_sge_timestamp(struct net_device *dev) 3807 { 3808 u32 hi, lo; 3809 struct adapter *adap; 3810 3811 adap = netdev2adap(dev); 3812 lo = t4_read_reg(adap, SGE_TIMESTAMP_LO); 3813 hi = GET_TSVAL(t4_read_reg(adap, SGE_TIMESTAMP_HI)); 3814 3815 return ((u64)hi << 32) | (u64)lo; 3816 } 3817 EXPORT_SYMBOL(cxgb4_read_sge_timestamp); 3818 3819 int cxgb4_bar2_sge_qregs(struct net_device *dev, 3820 unsigned int qid, 3821 enum cxgb4_bar2_qtype qtype, 3822 u64 *pbar2_qoffset, 3823 unsigned int *pbar2_qid) 3824 { 3825 return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev), 3826 qid, 3827 (qtype == CXGB4_BAR2_QTYPE_EGRESS 3828 ? T4_BAR2_QTYPE_EGRESS 3829 : T4_BAR2_QTYPE_INGRESS), 3830 pbar2_qoffset, 3831 pbar2_qid); 3832 } 3833 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs); 3834 3835 static struct pci_driver cxgb4_driver; 3836 3837 static void check_neigh_update(struct neighbour *neigh) 3838 { 3839 const struct device *parent; 3840 const struct net_device *netdev = neigh->dev; 3841 3842 if (netdev->priv_flags & IFF_802_1Q_VLAN) 3843 netdev = vlan_dev_real_dev(netdev); 3844 parent = netdev->dev.parent; 3845 if (parent && parent->driver == &cxgb4_driver.driver) 3846 t4_l2t_update(dev_get_drvdata(parent), neigh); 3847 } 3848 3849 static int netevent_cb(struct notifier_block *nb, unsigned long event, 3850 void *data) 3851 { 3852 switch (event) { 3853 case NETEVENT_NEIGH_UPDATE: 3854 check_neigh_update(data); 3855 break; 3856 case NETEVENT_REDIRECT: 3857 default: 3858 break; 3859 } 3860 return 0; 3861 } 3862 3863 static bool netevent_registered; 3864 static struct notifier_block cxgb4_netevent_nb = { 3865 .notifier_call = netevent_cb 3866 }; 3867 3868 static void drain_db_fifo(struct adapter *adap, int usecs) 3869 { 3870 u32 v1, v2, lp_count, hp_count; 3871 3872 do { 3873 v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS); 3874 v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2); 3875 if (is_t4(adap->params.chip)) { 3876 lp_count = G_LP_COUNT(v1); 3877 hp_count = G_HP_COUNT(v1); 3878 } else { 3879 lp_count = G_LP_COUNT_T5(v1); 3880 hp_count = G_HP_COUNT_T5(v2); 3881 } 3882 3883 if (lp_count == 0 && hp_count == 0) 3884 break; 3885 set_current_state(TASK_UNINTERRUPTIBLE); 3886 schedule_timeout(usecs_to_jiffies(usecs)); 3887 } while (1); 3888 } 3889 3890 static void disable_txq_db(struct sge_txq *q) 3891 { 3892 unsigned long flags; 3893 3894 spin_lock_irqsave(&q->db_lock, flags); 3895 q->db_disabled = 1; 3896 spin_unlock_irqrestore(&q->db_lock, flags); 3897 } 3898 3899 static void enable_txq_db(struct adapter *adap, struct sge_txq *q) 3900 { 3901 spin_lock_irq(&q->db_lock); 3902 if (q->db_pidx_inc) { 3903 /* Make sure that all writes to the TX descriptors 3904 * are committed before we tell HW about them. 3905 */ 3906 wmb(); 3907 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL), 3908 QID(q->cntxt_id) | PIDX(q->db_pidx_inc)); 3909 q->db_pidx_inc = 0; 3910 } 3911 q->db_disabled = 0; 3912 spin_unlock_irq(&q->db_lock); 3913 } 3914 3915 static void disable_dbs(struct adapter *adap) 3916 { 3917 int i; 3918 3919 for_each_ethrxq(&adap->sge, i) 3920 disable_txq_db(&adap->sge.ethtxq[i].q); 3921 for_each_ofldrxq(&adap->sge, i) 3922 disable_txq_db(&adap->sge.ofldtxq[i].q); 3923 for_each_port(adap, i) 3924 disable_txq_db(&adap->sge.ctrlq[i].q); 3925 } 3926 3927 static void enable_dbs(struct adapter *adap) 3928 { 3929 int i; 3930 3931 for_each_ethrxq(&adap->sge, i) 3932 enable_txq_db(adap, &adap->sge.ethtxq[i].q); 3933 for_each_ofldrxq(&adap->sge, i) 3934 enable_txq_db(adap, &adap->sge.ofldtxq[i].q); 3935 for_each_port(adap, i) 3936 enable_txq_db(adap, &adap->sge.ctrlq[i].q); 3937 } 3938 3939 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd) 3940 { 3941 if (adap->uld_handle[CXGB4_ULD_RDMA]) 3942 ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA], 3943 cmd); 3944 } 3945 3946 static void process_db_full(struct work_struct *work) 3947 { 3948 struct adapter *adap; 3949 3950 adap = container_of(work, struct adapter, db_full_task); 3951 3952 drain_db_fifo(adap, dbfifo_drain_delay); 3953 enable_dbs(adap); 3954 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 3955 t4_set_reg_field(adap, SGE_INT_ENABLE3, 3956 DBFIFO_HP_INT | DBFIFO_LP_INT, 3957 DBFIFO_HP_INT | DBFIFO_LP_INT); 3958 } 3959 3960 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q) 3961 { 3962 u16 hw_pidx, hw_cidx; 3963 int ret; 3964 3965 spin_lock_irq(&q->db_lock); 3966 ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx); 3967 if (ret) 3968 goto out; 3969 if (q->db_pidx != hw_pidx) { 3970 u16 delta; 3971 3972 if (q->db_pidx >= hw_pidx) 3973 delta = q->db_pidx - hw_pidx; 3974 else 3975 delta = q->size - hw_pidx + q->db_pidx; 3976 wmb(); 3977 t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL), 3978 QID(q->cntxt_id) | PIDX(delta)); 3979 } 3980 out: 3981 q->db_disabled = 0; 3982 q->db_pidx_inc = 0; 3983 spin_unlock_irq(&q->db_lock); 3984 if (ret) 3985 CH_WARN(adap, "DB drop recovery failed.\n"); 3986 } 3987 static void recover_all_queues(struct adapter *adap) 3988 { 3989 int i; 3990 3991 for_each_ethrxq(&adap->sge, i) 3992 sync_txq_pidx(adap, &adap->sge.ethtxq[i].q); 3993 for_each_ofldrxq(&adap->sge, i) 3994 sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q); 3995 for_each_port(adap, i) 3996 sync_txq_pidx(adap, &adap->sge.ctrlq[i].q); 3997 } 3998 3999 static void process_db_drop(struct work_struct *work) 4000 { 4001 struct adapter *adap; 4002 4003 adap = container_of(work, struct adapter, db_drop_task); 4004 4005 if (is_t4(adap->params.chip)) { 4006 drain_db_fifo(adap, dbfifo_drain_delay); 4007 notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP); 4008 drain_db_fifo(adap, dbfifo_drain_delay); 4009 recover_all_queues(adap); 4010 drain_db_fifo(adap, dbfifo_drain_delay); 4011 enable_dbs(adap); 4012 notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY); 4013 } else { 4014 u32 dropped_db = t4_read_reg(adap, 0x010ac); 4015 u16 qid = (dropped_db >> 15) & 0x1ffff; 4016 u16 pidx_inc = dropped_db & 0x1fff; 4017 u64 bar2_qoffset; 4018 unsigned int bar2_qid; 4019 int ret; 4020 4021 ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS, 4022 &bar2_qoffset, &bar2_qid); 4023 if (ret) 4024 dev_err(adap->pdev_dev, "doorbell drop recovery: " 4025 "qid=%d, pidx_inc=%d\n", qid, pidx_inc); 4026 else 4027 writel(PIDX_T5(pidx_inc) | QID(bar2_qid), 4028 adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL); 4029 4030 /* Re-enable BAR2 WC */ 4031 t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15); 4032 } 4033 4034 t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0); 4035 } 4036 4037 void t4_db_full(struct adapter *adap) 4038 { 4039 if (is_t4(adap->params.chip)) { 4040 disable_dbs(adap); 4041 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 4042 t4_set_reg_field(adap, SGE_INT_ENABLE3, 4043 DBFIFO_HP_INT | DBFIFO_LP_INT, 0); 4044 queue_work(adap->workq, &adap->db_full_task); 4045 } 4046 } 4047 4048 void t4_db_dropped(struct adapter *adap) 4049 { 4050 if (is_t4(adap->params.chip)) { 4051 disable_dbs(adap); 4052 notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL); 4053 } 4054 queue_work(adap->workq, &adap->db_drop_task); 4055 } 4056 4057 static void uld_attach(struct adapter *adap, unsigned int uld) 4058 { 4059 void *handle; 4060 struct cxgb4_lld_info lli; 4061 unsigned short i; 4062 4063 lli.pdev = adap->pdev; 4064 lli.pf = adap->fn; 4065 lli.l2t = adap->l2t; 4066 lli.tids = &adap->tids; 4067 lli.ports = adap->port; 4068 lli.vr = &adap->vres; 4069 lli.mtus = adap->params.mtus; 4070 if (uld == CXGB4_ULD_RDMA) { 4071 lli.rxq_ids = adap->sge.rdma_rxq; 4072 lli.ciq_ids = adap->sge.rdma_ciq; 4073 lli.nrxq = adap->sge.rdmaqs; 4074 lli.nciq = adap->sge.rdmaciqs; 4075 } else if (uld == CXGB4_ULD_ISCSI) { 4076 lli.rxq_ids = adap->sge.ofld_rxq; 4077 lli.nrxq = adap->sge.ofldqsets; 4078 } 4079 lli.ntxq = adap->sge.ofldqsets; 4080 lli.nchan = adap->params.nports; 4081 lli.nports = adap->params.nports; 4082 lli.wr_cred = adap->params.ofldq_wr_cred; 4083 lli.adapter_type = adap->params.chip; 4084 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2)); 4085 lli.cclk_ps = 1000000000 / adap->params.vpd.cclk; 4086 lli.udb_density = 1 << adap->params.sge.eq_qpp; 4087 lli.ucq_density = 1 << adap->params.sge.iq_qpp; 4088 lli.filt_mode = adap->params.tp.vlan_pri_map; 4089 /* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */ 4090 for (i = 0; i < NCHAN; i++) 4091 lli.tx_modq[i] = i; 4092 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS); 4093 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL); 4094 lli.fw_vers = adap->params.fw_vers; 4095 lli.dbfifo_int_thresh = dbfifo_int_thresh; 4096 lli.sge_ingpadboundary = adap->sge.fl_align; 4097 lli.sge_egrstatuspagesize = adap->sge.stat_len; 4098 lli.sge_pktshift = adap->sge.pktshift; 4099 lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN; 4100 lli.max_ordird_qp = adap->params.max_ordird_qp; 4101 lli.max_ird_adapter = adap->params.max_ird_adapter; 4102 lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl; 4103 4104 handle = ulds[uld].add(&lli); 4105 if (IS_ERR(handle)) { 4106 dev_warn(adap->pdev_dev, 4107 "could not attach to the %s driver, error %ld\n", 4108 uld_str[uld], PTR_ERR(handle)); 4109 return; 4110 } 4111 4112 adap->uld_handle[uld] = handle; 4113 4114 if (!netevent_registered) { 4115 register_netevent_notifier(&cxgb4_netevent_nb); 4116 netevent_registered = true; 4117 } 4118 4119 if (adap->flags & FULL_INIT_DONE) 4120 ulds[uld].state_change(handle, CXGB4_STATE_UP); 4121 } 4122 4123 static void attach_ulds(struct adapter *adap) 4124 { 4125 unsigned int i; 4126 4127 spin_lock(&adap_rcu_lock); 4128 list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list); 4129 spin_unlock(&adap_rcu_lock); 4130 4131 mutex_lock(&uld_mutex); 4132 list_add_tail(&adap->list_node, &adapter_list); 4133 for (i = 0; i < CXGB4_ULD_MAX; i++) 4134 if (ulds[i].add) 4135 uld_attach(adap, i); 4136 mutex_unlock(&uld_mutex); 4137 } 4138 4139 static void detach_ulds(struct adapter *adap) 4140 { 4141 unsigned int i; 4142 4143 mutex_lock(&uld_mutex); 4144 list_del(&adap->list_node); 4145 for (i = 0; i < CXGB4_ULD_MAX; i++) 4146 if (adap->uld_handle[i]) { 4147 ulds[i].state_change(adap->uld_handle[i], 4148 CXGB4_STATE_DETACH); 4149 adap->uld_handle[i] = NULL; 4150 } 4151 if (netevent_registered && list_empty(&adapter_list)) { 4152 unregister_netevent_notifier(&cxgb4_netevent_nb); 4153 netevent_registered = false; 4154 } 4155 mutex_unlock(&uld_mutex); 4156 4157 spin_lock(&adap_rcu_lock); 4158 list_del_rcu(&adap->rcu_node); 4159 spin_unlock(&adap_rcu_lock); 4160 } 4161 4162 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state) 4163 { 4164 unsigned int i; 4165 4166 mutex_lock(&uld_mutex); 4167 for (i = 0; i < CXGB4_ULD_MAX; i++) 4168 if (adap->uld_handle[i]) 4169 ulds[i].state_change(adap->uld_handle[i], new_state); 4170 mutex_unlock(&uld_mutex); 4171 } 4172 4173 /** 4174 * cxgb4_register_uld - register an upper-layer driver 4175 * @type: the ULD type 4176 * @p: the ULD methods 4177 * 4178 * Registers an upper-layer driver with this driver and notifies the ULD 4179 * about any presently available devices that support its type. Returns 4180 * %-EBUSY if a ULD of the same type is already registered. 4181 */ 4182 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p) 4183 { 4184 int ret = 0; 4185 struct adapter *adap; 4186 4187 if (type >= CXGB4_ULD_MAX) 4188 return -EINVAL; 4189 mutex_lock(&uld_mutex); 4190 if (ulds[type].add) { 4191 ret = -EBUSY; 4192 goto out; 4193 } 4194 ulds[type] = *p; 4195 list_for_each_entry(adap, &adapter_list, list_node) 4196 uld_attach(adap, type); 4197 out: mutex_unlock(&uld_mutex); 4198 return ret; 4199 } 4200 EXPORT_SYMBOL(cxgb4_register_uld); 4201 4202 /** 4203 * cxgb4_unregister_uld - unregister an upper-layer driver 4204 * @type: the ULD type 4205 * 4206 * Unregisters an existing upper-layer driver. 4207 */ 4208 int cxgb4_unregister_uld(enum cxgb4_uld type) 4209 { 4210 struct adapter *adap; 4211 4212 if (type >= CXGB4_ULD_MAX) 4213 return -EINVAL; 4214 mutex_lock(&uld_mutex); 4215 list_for_each_entry(adap, &adapter_list, list_node) 4216 adap->uld_handle[type] = NULL; 4217 ulds[type].add = NULL; 4218 mutex_unlock(&uld_mutex); 4219 return 0; 4220 } 4221 EXPORT_SYMBOL(cxgb4_unregister_uld); 4222 4223 /* Check if netdev on which event is occured belongs to us or not. Return 4224 * success (true) if it belongs otherwise failure (false). 4225 * Called with rcu_read_lock() held. 4226 */ 4227 #if IS_ENABLED(CONFIG_IPV6) 4228 static bool cxgb4_netdev(const struct net_device *netdev) 4229 { 4230 struct adapter *adap; 4231 int i; 4232 4233 list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node) 4234 for (i = 0; i < MAX_NPORTS; i++) 4235 if (adap->port[i] == netdev) 4236 return true; 4237 return false; 4238 } 4239 4240 static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa, 4241 unsigned long event) 4242 { 4243 int ret = NOTIFY_DONE; 4244 4245 rcu_read_lock(); 4246 if (cxgb4_netdev(event_dev)) { 4247 switch (event) { 4248 case NETDEV_UP: 4249 ret = cxgb4_clip_get(event_dev, &ifa->addr); 4250 if (ret < 0) { 4251 rcu_read_unlock(); 4252 return ret; 4253 } 4254 ret = NOTIFY_OK; 4255 break; 4256 case NETDEV_DOWN: 4257 cxgb4_clip_release(event_dev, &ifa->addr); 4258 ret = NOTIFY_OK; 4259 break; 4260 default: 4261 break; 4262 } 4263 } 4264 rcu_read_unlock(); 4265 return ret; 4266 } 4267 4268 static int cxgb4_inet6addr_handler(struct notifier_block *this, 4269 unsigned long event, void *data) 4270 { 4271 struct inet6_ifaddr *ifa = data; 4272 struct net_device *event_dev; 4273 int ret = NOTIFY_DONE; 4274 struct bonding *bond = netdev_priv(ifa->idev->dev); 4275 struct list_head *iter; 4276 struct slave *slave; 4277 struct pci_dev *first_pdev = NULL; 4278 4279 if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) { 4280 event_dev = vlan_dev_real_dev(ifa->idev->dev); 4281 ret = clip_add(event_dev, ifa, event); 4282 } else if (ifa->idev->dev->flags & IFF_MASTER) { 4283 /* It is possible that two different adapters are bonded in one 4284 * bond. We need to find such different adapters and add clip 4285 * in all of them only once. 4286 */ 4287 bond_for_each_slave(bond, slave, iter) { 4288 if (!first_pdev) { 4289 ret = clip_add(slave->dev, ifa, event); 4290 /* If clip_add is success then only initialize 4291 * first_pdev since it means it is our device 4292 */ 4293 if (ret == NOTIFY_OK) 4294 first_pdev = to_pci_dev( 4295 slave->dev->dev.parent); 4296 } else if (first_pdev != 4297 to_pci_dev(slave->dev->dev.parent)) 4298 ret = clip_add(slave->dev, ifa, event); 4299 } 4300 } else 4301 ret = clip_add(ifa->idev->dev, ifa, event); 4302 4303 return ret; 4304 } 4305 4306 static struct notifier_block cxgb4_inet6addr_notifier = { 4307 .notifier_call = cxgb4_inet6addr_handler 4308 }; 4309 4310 /* Retrieves IPv6 addresses from a root device (bond, vlan) associated with 4311 * a physical device. 4312 * The physical device reference is needed to send the actul CLIP command. 4313 */ 4314 static int update_dev_clip(struct net_device *root_dev, struct net_device *dev) 4315 { 4316 struct inet6_dev *idev = NULL; 4317 struct inet6_ifaddr *ifa; 4318 int ret = 0; 4319 4320 idev = __in6_dev_get(root_dev); 4321 if (!idev) 4322 return ret; 4323 4324 read_lock_bh(&idev->lock); 4325 list_for_each_entry(ifa, &idev->addr_list, if_list) { 4326 ret = cxgb4_clip_get(dev, &ifa->addr); 4327 if (ret < 0) 4328 break; 4329 } 4330 read_unlock_bh(&idev->lock); 4331 4332 return ret; 4333 } 4334 4335 static int update_root_dev_clip(struct net_device *dev) 4336 { 4337 struct net_device *root_dev = NULL; 4338 int i, ret = 0; 4339 4340 /* First populate the real net device's IPv6 addresses */ 4341 ret = update_dev_clip(dev, dev); 4342 if (ret) 4343 return ret; 4344 4345 /* Parse all bond and vlan devices layered on top of the physical dev */ 4346 root_dev = netdev_master_upper_dev_get_rcu(dev); 4347 if (root_dev) { 4348 ret = update_dev_clip(root_dev, dev); 4349 if (ret) 4350 return ret; 4351 } 4352 4353 for (i = 0; i < VLAN_N_VID; i++) { 4354 root_dev = __vlan_find_dev_deep_rcu(dev, htons(ETH_P_8021Q), i); 4355 if (!root_dev) 4356 continue; 4357 4358 ret = update_dev_clip(root_dev, dev); 4359 if (ret) 4360 break; 4361 } 4362 return ret; 4363 } 4364 4365 static void update_clip(const struct adapter *adap) 4366 { 4367 int i; 4368 struct net_device *dev; 4369 int ret; 4370 4371 rcu_read_lock(); 4372 4373 for (i = 0; i < MAX_NPORTS; i++) { 4374 dev = adap->port[i]; 4375 ret = 0; 4376 4377 if (dev) 4378 ret = update_root_dev_clip(dev); 4379 4380 if (ret < 0) 4381 break; 4382 } 4383 rcu_read_unlock(); 4384 } 4385 #endif /* IS_ENABLED(CONFIG_IPV6) */ 4386 4387 /** 4388 * cxgb_up - enable the adapter 4389 * @adap: adapter being enabled 4390 * 4391 * Called when the first port is enabled, this function performs the 4392 * actions necessary to make an adapter operational, such as completing 4393 * the initialization of HW modules, and enabling interrupts. 4394 * 4395 * Must be called with the rtnl lock held. 4396 */ 4397 static int cxgb_up(struct adapter *adap) 4398 { 4399 int err; 4400 4401 err = setup_sge_queues(adap); 4402 if (err) 4403 goto out; 4404 err = setup_rss(adap); 4405 if (err) 4406 goto freeq; 4407 4408 if (adap->flags & USING_MSIX) { 4409 name_msix_vecs(adap); 4410 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0, 4411 adap->msix_info[0].desc, adap); 4412 if (err) 4413 goto irq_err; 4414 4415 err = request_msix_queue_irqs(adap); 4416 if (err) { 4417 free_irq(adap->msix_info[0].vec, adap); 4418 goto irq_err; 4419 } 4420 } else { 4421 err = request_irq(adap->pdev->irq, t4_intr_handler(adap), 4422 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED, 4423 adap->port[0]->name, adap); 4424 if (err) 4425 goto irq_err; 4426 } 4427 enable_rx(adap); 4428 t4_sge_start(adap); 4429 t4_intr_enable(adap); 4430 adap->flags |= FULL_INIT_DONE; 4431 notify_ulds(adap, CXGB4_STATE_UP); 4432 #if IS_ENABLED(CONFIG_IPV6) 4433 update_clip(adap); 4434 #endif 4435 out: 4436 return err; 4437 irq_err: 4438 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err); 4439 freeq: 4440 t4_free_sge_resources(adap); 4441 goto out; 4442 } 4443 4444 static void cxgb_down(struct adapter *adapter) 4445 { 4446 t4_intr_disable(adapter); 4447 cancel_work_sync(&adapter->tid_release_task); 4448 cancel_work_sync(&adapter->db_full_task); 4449 cancel_work_sync(&adapter->db_drop_task); 4450 adapter->tid_release_task_busy = false; 4451 adapter->tid_release_head = NULL; 4452 4453 if (adapter->flags & USING_MSIX) { 4454 free_msix_queue_irqs(adapter); 4455 free_irq(adapter->msix_info[0].vec, adapter); 4456 } else 4457 free_irq(adapter->pdev->irq, adapter); 4458 quiesce_rx(adapter); 4459 t4_sge_stop(adapter); 4460 t4_free_sge_resources(adapter); 4461 adapter->flags &= ~FULL_INIT_DONE; 4462 } 4463 4464 /* 4465 * net_device operations 4466 */ 4467 static int cxgb_open(struct net_device *dev) 4468 { 4469 int err; 4470 struct port_info *pi = netdev_priv(dev); 4471 struct adapter *adapter = pi->adapter; 4472 4473 netif_carrier_off(dev); 4474 4475 if (!(adapter->flags & FULL_INIT_DONE)) { 4476 err = cxgb_up(adapter); 4477 if (err < 0) 4478 return err; 4479 } 4480 4481 err = link_start(dev); 4482 if (!err) 4483 netif_tx_start_all_queues(dev); 4484 return err; 4485 } 4486 4487 static int cxgb_close(struct net_device *dev) 4488 { 4489 struct port_info *pi = netdev_priv(dev); 4490 struct adapter *adapter = pi->adapter; 4491 4492 netif_tx_stop_all_queues(dev); 4493 netif_carrier_off(dev); 4494 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false); 4495 } 4496 4497 /* Return an error number if the indicated filter isn't writable ... 4498 */ 4499 static int writable_filter(struct filter_entry *f) 4500 { 4501 if (f->locked) 4502 return -EPERM; 4503 if (f->pending) 4504 return -EBUSY; 4505 4506 return 0; 4507 } 4508 4509 /* Delete the filter at the specified index (if valid). The checks for all 4510 * the common problems with doing this like the filter being locked, currently 4511 * pending in another operation, etc. 4512 */ 4513 static int delete_filter(struct adapter *adapter, unsigned int fidx) 4514 { 4515 struct filter_entry *f; 4516 int ret; 4517 4518 if (fidx >= adapter->tids.nftids + adapter->tids.nsftids) 4519 return -EINVAL; 4520 4521 f = &adapter->tids.ftid_tab[fidx]; 4522 ret = writable_filter(f); 4523 if (ret) 4524 return ret; 4525 if (f->valid) 4526 return del_filter_wr(adapter, fidx); 4527 4528 return 0; 4529 } 4530 4531 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid, 4532 __be32 sip, __be16 sport, __be16 vlan, 4533 unsigned int queue, unsigned char port, unsigned char mask) 4534 { 4535 int ret; 4536 struct filter_entry *f; 4537 struct adapter *adap; 4538 int i; 4539 u8 *val; 4540 4541 adap = netdev2adap(dev); 4542 4543 /* Adjust stid to correct filter index */ 4544 stid -= adap->tids.sftid_base; 4545 stid += adap->tids.nftids; 4546 4547 /* Check to make sure the filter requested is writable ... 4548 */ 4549 f = &adap->tids.ftid_tab[stid]; 4550 ret = writable_filter(f); 4551 if (ret) 4552 return ret; 4553 4554 /* Clear out any old resources being used by the filter before 4555 * we start constructing the new filter. 4556 */ 4557 if (f->valid) 4558 clear_filter(adap, f); 4559 4560 /* Clear out filter specifications */ 4561 memset(&f->fs, 0, sizeof(struct ch_filter_specification)); 4562 f->fs.val.lport = cpu_to_be16(sport); 4563 f->fs.mask.lport = ~0; 4564 val = (u8 *)&sip; 4565 if ((val[0] | val[1] | val[2] | val[3]) != 0) { 4566 for (i = 0; i < 4; i++) { 4567 f->fs.val.lip[i] = val[i]; 4568 f->fs.mask.lip[i] = ~0; 4569 } 4570 if (adap->params.tp.vlan_pri_map & F_PORT) { 4571 f->fs.val.iport = port; 4572 f->fs.mask.iport = mask; 4573 } 4574 } 4575 4576 if (adap->params.tp.vlan_pri_map & F_PROTOCOL) { 4577 f->fs.val.proto = IPPROTO_TCP; 4578 f->fs.mask.proto = ~0; 4579 } 4580 4581 f->fs.dirsteer = 1; 4582 f->fs.iq = queue; 4583 /* Mark filter as locked */ 4584 f->locked = 1; 4585 f->fs.rpttid = 1; 4586 4587 ret = set_filter_wr(adap, stid); 4588 if (ret) { 4589 clear_filter(adap, f); 4590 return ret; 4591 } 4592 4593 return 0; 4594 } 4595 EXPORT_SYMBOL(cxgb4_create_server_filter); 4596 4597 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid, 4598 unsigned int queue, bool ipv6) 4599 { 4600 int ret; 4601 struct filter_entry *f; 4602 struct adapter *adap; 4603 4604 adap = netdev2adap(dev); 4605 4606 /* Adjust stid to correct filter index */ 4607 stid -= adap->tids.sftid_base; 4608 stid += adap->tids.nftids; 4609 4610 f = &adap->tids.ftid_tab[stid]; 4611 /* Unlock the filter */ 4612 f->locked = 0; 4613 4614 ret = delete_filter(adap, stid); 4615 if (ret) 4616 return ret; 4617 4618 return 0; 4619 } 4620 EXPORT_SYMBOL(cxgb4_remove_server_filter); 4621 4622 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev, 4623 struct rtnl_link_stats64 *ns) 4624 { 4625 struct port_stats stats; 4626 struct port_info *p = netdev_priv(dev); 4627 struct adapter *adapter = p->adapter; 4628 4629 /* Block retrieving statistics during EEH error 4630 * recovery. Otherwise, the recovery might fail 4631 * and the PCI device will be removed permanently 4632 */ 4633 spin_lock(&adapter->stats_lock); 4634 if (!netif_device_present(dev)) { 4635 spin_unlock(&adapter->stats_lock); 4636 return ns; 4637 } 4638 t4_get_port_stats(adapter, p->tx_chan, &stats); 4639 spin_unlock(&adapter->stats_lock); 4640 4641 ns->tx_bytes = stats.tx_octets; 4642 ns->tx_packets = stats.tx_frames; 4643 ns->rx_bytes = stats.rx_octets; 4644 ns->rx_packets = stats.rx_frames; 4645 ns->multicast = stats.rx_mcast_frames; 4646 4647 /* detailed rx_errors */ 4648 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long + 4649 stats.rx_runt; 4650 ns->rx_over_errors = 0; 4651 ns->rx_crc_errors = stats.rx_fcs_err; 4652 ns->rx_frame_errors = stats.rx_symbol_err; 4653 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 + 4654 stats.rx_ovflow2 + stats.rx_ovflow3 + 4655 stats.rx_trunc0 + stats.rx_trunc1 + 4656 stats.rx_trunc2 + stats.rx_trunc3; 4657 ns->rx_missed_errors = 0; 4658 4659 /* detailed tx_errors */ 4660 ns->tx_aborted_errors = 0; 4661 ns->tx_carrier_errors = 0; 4662 ns->tx_fifo_errors = 0; 4663 ns->tx_heartbeat_errors = 0; 4664 ns->tx_window_errors = 0; 4665 4666 ns->tx_errors = stats.tx_error_frames; 4667 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err + 4668 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors; 4669 return ns; 4670 } 4671 4672 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd) 4673 { 4674 unsigned int mbox; 4675 int ret = 0, prtad, devad; 4676 struct port_info *pi = netdev_priv(dev); 4677 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data; 4678 4679 switch (cmd) { 4680 case SIOCGMIIPHY: 4681 if (pi->mdio_addr < 0) 4682 return -EOPNOTSUPP; 4683 data->phy_id = pi->mdio_addr; 4684 break; 4685 case SIOCGMIIREG: 4686 case SIOCSMIIREG: 4687 if (mdio_phy_id_is_c45(data->phy_id)) { 4688 prtad = mdio_phy_id_prtad(data->phy_id); 4689 devad = mdio_phy_id_devad(data->phy_id); 4690 } else if (data->phy_id < 32) { 4691 prtad = data->phy_id; 4692 devad = 0; 4693 data->reg_num &= 0x1f; 4694 } else 4695 return -EINVAL; 4696 4697 mbox = pi->adapter->fn; 4698 if (cmd == SIOCGMIIREG) 4699 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad, 4700 data->reg_num, &data->val_out); 4701 else 4702 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad, 4703 data->reg_num, data->val_in); 4704 break; 4705 default: 4706 return -EOPNOTSUPP; 4707 } 4708 return ret; 4709 } 4710 4711 static void cxgb_set_rxmode(struct net_device *dev) 4712 { 4713 /* unfortunately we can't return errors to the stack */ 4714 set_rxmode(dev, -1, false); 4715 } 4716 4717 static int cxgb_change_mtu(struct net_device *dev, int new_mtu) 4718 { 4719 int ret; 4720 struct port_info *pi = netdev_priv(dev); 4721 4722 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */ 4723 return -EINVAL; 4724 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1, 4725 -1, -1, -1, true); 4726 if (!ret) 4727 dev->mtu = new_mtu; 4728 return ret; 4729 } 4730 4731 static int cxgb_set_mac_addr(struct net_device *dev, void *p) 4732 { 4733 int ret; 4734 struct sockaddr *addr = p; 4735 struct port_info *pi = netdev_priv(dev); 4736 4737 if (!is_valid_ether_addr(addr->sa_data)) 4738 return -EADDRNOTAVAIL; 4739 4740 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid, 4741 pi->xact_addr_filt, addr->sa_data, true, true); 4742 if (ret < 0) 4743 return ret; 4744 4745 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len); 4746 pi->xact_addr_filt = ret; 4747 return 0; 4748 } 4749 4750 #ifdef CONFIG_NET_POLL_CONTROLLER 4751 static void cxgb_netpoll(struct net_device *dev) 4752 { 4753 struct port_info *pi = netdev_priv(dev); 4754 struct adapter *adap = pi->adapter; 4755 4756 if (adap->flags & USING_MSIX) { 4757 int i; 4758 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset]; 4759 4760 for (i = pi->nqsets; i; i--, rx++) 4761 t4_sge_intr_msix(0, &rx->rspq); 4762 } else 4763 t4_intr_handler(adap)(0, adap); 4764 } 4765 #endif 4766 4767 static const struct net_device_ops cxgb4_netdev_ops = { 4768 .ndo_open = cxgb_open, 4769 .ndo_stop = cxgb_close, 4770 .ndo_start_xmit = t4_eth_xmit, 4771 .ndo_select_queue = cxgb_select_queue, 4772 .ndo_get_stats64 = cxgb_get_stats, 4773 .ndo_set_rx_mode = cxgb_set_rxmode, 4774 .ndo_set_mac_address = cxgb_set_mac_addr, 4775 .ndo_set_features = cxgb_set_features, 4776 .ndo_validate_addr = eth_validate_addr, 4777 .ndo_do_ioctl = cxgb_ioctl, 4778 .ndo_change_mtu = cxgb_change_mtu, 4779 #ifdef CONFIG_NET_POLL_CONTROLLER 4780 .ndo_poll_controller = cxgb_netpoll, 4781 #endif 4782 }; 4783 4784 void t4_fatal_err(struct adapter *adap) 4785 { 4786 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0); 4787 t4_intr_disable(adap); 4788 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n"); 4789 } 4790 4791 /* Return the specified PCI-E Configuration Space register from our Physical 4792 * Function. We try first via a Firmware LDST Command since we prefer to let 4793 * the firmware own all of these registers, but if that fails we go for it 4794 * directly ourselves. 4795 */ 4796 static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg) 4797 { 4798 struct fw_ldst_cmd ldst_cmd; 4799 u32 val; 4800 int ret; 4801 4802 /* Construct and send the Firmware LDST Command to retrieve the 4803 * specified PCI-E Configuration Space register. 4804 */ 4805 memset(&ldst_cmd, 0, sizeof(ldst_cmd)); 4806 ldst_cmd.op_to_addrspace = 4807 htonl(FW_CMD_OP_V(FW_LDST_CMD) | 4808 FW_CMD_REQUEST_F | 4809 FW_CMD_READ_F | 4810 FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE)); 4811 ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd)); 4812 ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1); 4813 ldst_cmd.u.pcie.ctrl_to_fn = 4814 (FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn)); 4815 ldst_cmd.u.pcie.r = reg; 4816 ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd), 4817 &ldst_cmd); 4818 4819 /* If the LDST Command suucceeded, exctract the returned register 4820 * value. Otherwise read it directly ourself. 4821 */ 4822 if (ret == 0) 4823 val = ntohl(ldst_cmd.u.pcie.data[0]); 4824 else 4825 t4_hw_pci_read_cfg4(adap, reg, &val); 4826 4827 return val; 4828 } 4829 4830 static void setup_memwin(struct adapter *adap) 4831 { 4832 u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture; 4833 4834 if (is_t4(adap->params.chip)) { 4835 u32 bar0; 4836 4837 /* Truncation intentional: we only read the bottom 32-bits of 4838 * the 64-bit BAR0/BAR1 ... We use the hardware backdoor 4839 * mechanism to read BAR0 instead of using 4840 * pci_resource_start() because we could be operating from 4841 * within a Virtual Machine which is trapping our accesses to 4842 * our Configuration Space and we need to set up the PCI-E 4843 * Memory Window decoders with the actual addresses which will 4844 * be coming across the PCI-E link. 4845 */ 4846 bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0); 4847 bar0 &= PCI_BASE_ADDRESS_MEM_MASK; 4848 adap->t4_bar0 = bar0; 4849 4850 mem_win0_base = bar0 + MEMWIN0_BASE; 4851 mem_win1_base = bar0 + MEMWIN1_BASE; 4852 mem_win2_base = bar0 + MEMWIN2_BASE; 4853 mem_win2_aperture = MEMWIN2_APERTURE; 4854 } else { 4855 /* For T5, only relative offset inside the PCIe BAR is passed */ 4856 mem_win0_base = MEMWIN0_BASE; 4857 mem_win1_base = MEMWIN1_BASE; 4858 mem_win2_base = MEMWIN2_BASE_T5; 4859 mem_win2_aperture = MEMWIN2_APERTURE_T5; 4860 } 4861 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0), 4862 mem_win0_base | BIR(0) | 4863 WINDOW(ilog2(MEMWIN0_APERTURE) - 10)); 4864 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1), 4865 mem_win1_base | BIR(0) | 4866 WINDOW(ilog2(MEMWIN1_APERTURE) - 10)); 4867 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2), 4868 mem_win2_base | BIR(0) | 4869 WINDOW(ilog2(mem_win2_aperture) - 10)); 4870 t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2)); 4871 } 4872 4873 static void setup_memwin_rdma(struct adapter *adap) 4874 { 4875 if (adap->vres.ocq.size) { 4876 u32 start; 4877 unsigned int sz_kb; 4878 4879 start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2); 4880 start &= PCI_BASE_ADDRESS_MEM_MASK; 4881 start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres); 4882 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10; 4883 t4_write_reg(adap, 4884 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3), 4885 start | BIR(1) | WINDOW(ilog2(sz_kb))); 4886 t4_write_reg(adap, 4887 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3), 4888 adap->vres.ocq.start); 4889 t4_read_reg(adap, 4890 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3)); 4891 } 4892 } 4893 4894 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c) 4895 { 4896 u32 v; 4897 int ret; 4898 4899 /* get device capabilities */ 4900 memset(c, 0, sizeof(*c)); 4901 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4902 FW_CMD_REQUEST_F | FW_CMD_READ_F); 4903 c->cfvalid_to_len16 = htonl(FW_LEN16(*c)); 4904 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c); 4905 if (ret < 0) 4906 return ret; 4907 4908 /* select capabilities we'll be using */ 4909 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) { 4910 if (!vf_acls) 4911 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM); 4912 else 4913 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM); 4914 } else if (vf_acls) { 4915 dev_err(adap->pdev_dev, "virtualization ACLs not supported"); 4916 return ret; 4917 } 4918 c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 4919 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 4920 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL); 4921 if (ret < 0) 4922 return ret; 4923 4924 ret = t4_config_glbl_rss(adap, adap->fn, 4925 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 4926 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | 4927 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F); 4928 if (ret < 0) 4929 return ret; 4930 4931 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ, 4932 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF); 4933 if (ret < 0) 4934 return ret; 4935 4936 t4_sge_init(adap); 4937 4938 /* tweak some settings */ 4939 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849); 4940 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12)); 4941 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG); 4942 v = t4_read_reg(adap, TP_PIO_DATA); 4943 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR); 4944 4945 /* first 4 Tx modulation queues point to consecutive Tx channels */ 4946 adap->params.tp.tx_modq_map = 0xE4; 4947 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP, 4948 V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map)); 4949 4950 /* associate each Tx modulation queue with consecutive Tx channels */ 4951 v = 0x84218421; 4952 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA, 4953 &v, 1, A_TP_TX_SCHED_HDR); 4954 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA, 4955 &v, 1, A_TP_TX_SCHED_FIFO); 4956 t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA, 4957 &v, 1, A_TP_TX_SCHED_PCMD); 4958 4959 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */ 4960 if (is_offload(adap)) { 4961 t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0, 4962 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4963 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4964 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4965 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 4966 t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT, 4967 V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4968 V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4969 V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) | 4970 V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT)); 4971 } 4972 4973 /* get basic stuff going */ 4974 return t4_early_init(adap, adap->fn); 4975 } 4976 4977 /* 4978 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower. 4979 */ 4980 #define MAX_ATIDS 8192U 4981 4982 /* 4983 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 4984 * 4985 * If the firmware we're dealing with has Configuration File support, then 4986 * we use that to perform all configuration 4987 */ 4988 4989 /* 4990 * Tweak configuration based on module parameters, etc. Most of these have 4991 * defaults assigned to them by Firmware Configuration Files (if we're using 4992 * them) but need to be explicitly set if we're using hard-coded 4993 * initialization. But even in the case of using Firmware Configuration 4994 * Files, we'd like to expose the ability to change these via module 4995 * parameters so these are essentially common tweaks/settings for 4996 * Configuration Files and hard-coded initialization ... 4997 */ 4998 static int adap_init0_tweaks(struct adapter *adapter) 4999 { 5000 /* 5001 * Fix up various Host-Dependent Parameters like Page Size, Cache 5002 * Line Size, etc. The firmware default is for a 4KB Page Size and 5003 * 64B Cache Line Size ... 5004 */ 5005 t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES); 5006 5007 /* 5008 * Process module parameters which affect early initialization. 5009 */ 5010 if (rx_dma_offset != 2 && rx_dma_offset != 0) { 5011 dev_err(&adapter->pdev->dev, 5012 "Ignoring illegal rx_dma_offset=%d, using 2\n", 5013 rx_dma_offset); 5014 rx_dma_offset = 2; 5015 } 5016 t4_set_reg_field(adapter, SGE_CONTROL, 5017 PKTSHIFT_MASK, 5018 PKTSHIFT(rx_dma_offset)); 5019 5020 /* 5021 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux 5022 * adds the pseudo header itself. 5023 */ 5024 t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG, 5025 CSUM_HAS_PSEUDO_HDR, 0); 5026 5027 return 0; 5028 } 5029 5030 /* 5031 * Attempt to initialize the adapter via a Firmware Configuration File. 5032 */ 5033 static int adap_init0_config(struct adapter *adapter, int reset) 5034 { 5035 struct fw_caps_config_cmd caps_cmd; 5036 const struct firmware *cf; 5037 unsigned long mtype = 0, maddr = 0; 5038 u32 finiver, finicsum, cfcsum; 5039 int ret; 5040 int config_issued = 0; 5041 char *fw_config_file, fw_config_file_path[256]; 5042 char *config_name = NULL; 5043 5044 /* 5045 * Reset device if necessary. 5046 */ 5047 if (reset) { 5048 ret = t4_fw_reset(adapter, adapter->mbox, 5049 PIORSTMODE | PIORST); 5050 if (ret < 0) 5051 goto bye; 5052 } 5053 5054 /* 5055 * If we have a T4 configuration file under /lib/firmware/cxgb4/, 5056 * then use that. Otherwise, use the configuration file stored 5057 * in the adapter flash ... 5058 */ 5059 switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) { 5060 case CHELSIO_T4: 5061 fw_config_file = FW4_CFNAME; 5062 break; 5063 case CHELSIO_T5: 5064 fw_config_file = FW5_CFNAME; 5065 break; 5066 default: 5067 dev_err(adapter->pdev_dev, "Device %d is not supported\n", 5068 adapter->pdev->device); 5069 ret = -EINVAL; 5070 goto bye; 5071 } 5072 5073 ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev); 5074 if (ret < 0) { 5075 config_name = "On FLASH"; 5076 mtype = FW_MEMTYPE_CF_FLASH; 5077 maddr = t4_flash_cfg_addr(adapter); 5078 } else { 5079 u32 params[7], val[7]; 5080 5081 sprintf(fw_config_file_path, 5082 "/lib/firmware/%s", fw_config_file); 5083 config_name = fw_config_file_path; 5084 5085 if (cf->size >= FLASH_CFG_MAX_SIZE) 5086 ret = -ENOMEM; 5087 else { 5088 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 5089 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF)); 5090 ret = t4_query_params(adapter, adapter->mbox, 5091 adapter->fn, 0, 1, params, val); 5092 if (ret == 0) { 5093 /* 5094 * For t4_memory_rw() below addresses and 5095 * sizes have to be in terms of multiples of 4 5096 * bytes. So, if the Configuration File isn't 5097 * a multiple of 4 bytes in length we'll have 5098 * to write that out separately since we can't 5099 * guarantee that the bytes following the 5100 * residual byte in the buffer returned by 5101 * request_firmware() are zeroed out ... 5102 */ 5103 size_t resid = cf->size & 0x3; 5104 size_t size = cf->size & ~0x3; 5105 __be32 *data = (__be32 *)cf->data; 5106 5107 mtype = FW_PARAMS_PARAM_Y_G(val[0]); 5108 maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16; 5109 5110 spin_lock(&adapter->win0_lock); 5111 ret = t4_memory_rw(adapter, 0, mtype, maddr, 5112 size, data, T4_MEMORY_WRITE); 5113 if (ret == 0 && resid != 0) { 5114 union { 5115 __be32 word; 5116 char buf[4]; 5117 } last; 5118 int i; 5119 5120 last.word = data[size >> 2]; 5121 for (i = resid; i < 4; i++) 5122 last.buf[i] = 0; 5123 ret = t4_memory_rw(adapter, 0, mtype, 5124 maddr + size, 5125 4, &last.word, 5126 T4_MEMORY_WRITE); 5127 } 5128 spin_unlock(&adapter->win0_lock); 5129 } 5130 } 5131 5132 release_firmware(cf); 5133 if (ret) 5134 goto bye; 5135 } 5136 5137 /* 5138 * Issue a Capability Configuration command to the firmware to get it 5139 * to parse the Configuration File. We don't use t4_fw_config_file() 5140 * because we want the ability to modify various features after we've 5141 * processed the configuration file ... 5142 */ 5143 memset(&caps_cmd, 0, sizeof(caps_cmd)); 5144 caps_cmd.op_to_write = 5145 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 5146 FW_CMD_REQUEST_F | 5147 FW_CMD_READ_F); 5148 caps_cmd.cfvalid_to_len16 = 5149 htonl(FW_CAPS_CONFIG_CMD_CFVALID_F | 5150 FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) | 5151 FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) | 5152 FW_LEN16(caps_cmd)); 5153 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 5154 &caps_cmd); 5155 5156 /* If the CAPS_CONFIG failed with an ENOENT (for a Firmware 5157 * Configuration File in FLASH), our last gasp effort is to use the 5158 * Firmware Configuration File which is embedded in the firmware. A 5159 * very few early versions of the firmware didn't have one embedded 5160 * but we can ignore those. 5161 */ 5162 if (ret == -ENOENT) { 5163 memset(&caps_cmd, 0, sizeof(caps_cmd)); 5164 caps_cmd.op_to_write = 5165 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 5166 FW_CMD_REQUEST_F | 5167 FW_CMD_READ_F); 5168 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 5169 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, 5170 sizeof(caps_cmd), &caps_cmd); 5171 config_name = "Firmware Default"; 5172 } 5173 5174 config_issued = 1; 5175 if (ret < 0) 5176 goto bye; 5177 5178 finiver = ntohl(caps_cmd.finiver); 5179 finicsum = ntohl(caps_cmd.finicsum); 5180 cfcsum = ntohl(caps_cmd.cfcsum); 5181 if (finicsum != cfcsum) 5182 dev_warn(adapter->pdev_dev, "Configuration File checksum "\ 5183 "mismatch: [fini] csum=%#x, computed csum=%#x\n", 5184 finicsum, cfcsum); 5185 5186 /* 5187 * And now tell the firmware to use the configuration we just loaded. 5188 */ 5189 caps_cmd.op_to_write = 5190 htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 5191 FW_CMD_REQUEST_F | 5192 FW_CMD_WRITE_F); 5193 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 5194 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 5195 NULL); 5196 if (ret < 0) 5197 goto bye; 5198 5199 /* 5200 * Tweak configuration based on system architecture, module 5201 * parameters, etc. 5202 */ 5203 ret = adap_init0_tweaks(adapter); 5204 if (ret < 0) 5205 goto bye; 5206 5207 /* 5208 * And finally tell the firmware to initialize itself using the 5209 * parameters from the Configuration File. 5210 */ 5211 ret = t4_fw_initialize(adapter, adapter->mbox); 5212 if (ret < 0) 5213 goto bye; 5214 5215 /* 5216 * Return successfully and note that we're operating with parameters 5217 * not supplied by the driver, rather than from hard-wired 5218 * initialization constants burried in the driver. 5219 */ 5220 adapter->flags |= USING_SOFT_PARAMS; 5221 dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\ 5222 "Configuration File \"%s\", version %#x, computed checksum %#x\n", 5223 config_name, finiver, cfcsum); 5224 return 0; 5225 5226 /* 5227 * Something bad happened. Return the error ... (If the "error" 5228 * is that there's no Configuration File on the adapter we don't 5229 * want to issue a warning since this is fairly common.) 5230 */ 5231 bye: 5232 if (config_issued && ret != -ENOENT) 5233 dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n", 5234 config_name, -ret); 5235 return ret; 5236 } 5237 5238 /* 5239 * Attempt to initialize the adapter via hard-coded, driver supplied 5240 * parameters ... 5241 */ 5242 static int adap_init0_no_config(struct adapter *adapter, int reset) 5243 { 5244 struct sge *s = &adapter->sge; 5245 struct fw_caps_config_cmd caps_cmd; 5246 u32 v; 5247 int i, ret; 5248 5249 /* 5250 * Reset device if necessary 5251 */ 5252 if (reset) { 5253 ret = t4_fw_reset(adapter, adapter->mbox, 5254 PIORSTMODE | PIORST); 5255 if (ret < 0) 5256 goto bye; 5257 } 5258 5259 /* 5260 * Get device capabilities and select which we'll be using. 5261 */ 5262 memset(&caps_cmd, 0, sizeof(caps_cmd)); 5263 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 5264 FW_CMD_REQUEST_F | FW_CMD_READ_F); 5265 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 5266 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 5267 &caps_cmd); 5268 if (ret < 0) 5269 goto bye; 5270 5271 if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) { 5272 if (!vf_acls) 5273 caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM); 5274 else 5275 caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM); 5276 } else if (vf_acls) { 5277 dev_err(adapter->pdev_dev, "virtualization ACLs not supported"); 5278 goto bye; 5279 } 5280 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 5281 FW_CMD_REQUEST_F | FW_CMD_WRITE_F); 5282 ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd), 5283 NULL); 5284 if (ret < 0) 5285 goto bye; 5286 5287 /* 5288 * Tweak configuration based on system architecture, module 5289 * parameters, etc. 5290 */ 5291 ret = adap_init0_tweaks(adapter); 5292 if (ret < 0) 5293 goto bye; 5294 5295 /* 5296 * Select RSS Global Mode we want to use. We use "Basic Virtual" 5297 * mode which maps each Virtual Interface to its own section of 5298 * the RSS Table and we turn on all map and hash enables ... 5299 */ 5300 adapter->flags |= RSS_TNLALLLOOKUP; 5301 ret = t4_config_glbl_rss(adapter, adapter->mbox, 5302 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL, 5303 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F | 5304 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_F | 5305 ((adapter->flags & RSS_TNLALLLOOKUP) ? 5306 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F : 0)); 5307 if (ret < 0) 5308 goto bye; 5309 5310 /* 5311 * Set up our own fundamental resource provisioning ... 5312 */ 5313 ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0, 5314 PFRES_NEQ, PFRES_NETHCTRL, 5315 PFRES_NIQFLINT, PFRES_NIQ, 5316 PFRES_TC, PFRES_NVI, 5317 FW_PFVF_CMD_CMASK_M, 5318 pfvfres_pmask(adapter, adapter->fn, 0), 5319 PFRES_NEXACTF, 5320 PFRES_R_CAPS, PFRES_WX_CAPS); 5321 if (ret < 0) 5322 goto bye; 5323 5324 /* 5325 * Perform low level SGE initialization. We need to do this before we 5326 * send the firmware the INITIALIZE command because that will cause 5327 * any other PF Drivers which are waiting for the Master 5328 * Initialization to proceed forward. 5329 */ 5330 for (i = 0; i < SGE_NTIMERS - 1; i++) 5331 s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL); 5332 s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL; 5333 s->counter_val[0] = 1; 5334 for (i = 1; i < SGE_NCOUNTERS; i++) 5335 s->counter_val[i] = min(intr_cnt[i - 1], 5336 THRESHOLD_0_GET(THRESHOLD_0_MASK)); 5337 t4_sge_init(adapter); 5338 5339 #ifdef CONFIG_PCI_IOV 5340 /* 5341 * Provision resource limits for Virtual Functions. We currently 5342 * grant them all the same static resource limits except for the Port 5343 * Access Rights Mask which we're assigning based on the PF. All of 5344 * the static provisioning stuff for both the PF and VF really needs 5345 * to be managed in a persistent manner for each device which the 5346 * firmware controls. 5347 */ 5348 { 5349 int pf, vf; 5350 5351 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) { 5352 if (num_vf[pf] <= 0) 5353 continue; 5354 5355 /* VF numbering starts at 1! */ 5356 for (vf = 1; vf <= num_vf[pf]; vf++) { 5357 ret = t4_cfg_pfvf(adapter, adapter->mbox, 5358 pf, vf, 5359 VFRES_NEQ, VFRES_NETHCTRL, 5360 VFRES_NIQFLINT, VFRES_NIQ, 5361 VFRES_TC, VFRES_NVI, 5362 FW_PFVF_CMD_CMASK_M, 5363 pfvfres_pmask( 5364 adapter, pf, vf), 5365 VFRES_NEXACTF, 5366 VFRES_R_CAPS, VFRES_WX_CAPS); 5367 if (ret < 0) 5368 dev_warn(adapter->pdev_dev, 5369 "failed to "\ 5370 "provision pf/vf=%d/%d; " 5371 "err=%d\n", pf, vf, ret); 5372 } 5373 } 5374 } 5375 #endif 5376 5377 /* 5378 * Set up the default filter mode. Later we'll want to implement this 5379 * via a firmware command, etc. ... This needs to be done before the 5380 * firmare initialization command ... If the selected set of fields 5381 * isn't equal to the default value, we'll need to make sure that the 5382 * field selections will fit in the 36-bit budget. 5383 */ 5384 if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) { 5385 int j, bits = 0; 5386 5387 for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++) 5388 switch (tp_vlan_pri_map & (1 << j)) { 5389 case 0: 5390 /* compressed filter field not enabled */ 5391 break; 5392 case FCOE_MASK: 5393 bits += 1; 5394 break; 5395 case PORT_MASK: 5396 bits += 3; 5397 break; 5398 case VNIC_ID_MASK: 5399 bits += 17; 5400 break; 5401 case VLAN_MASK: 5402 bits += 17; 5403 break; 5404 case TOS_MASK: 5405 bits += 8; 5406 break; 5407 case PROTOCOL_MASK: 5408 bits += 8; 5409 break; 5410 case ETHERTYPE_MASK: 5411 bits += 16; 5412 break; 5413 case MACMATCH_MASK: 5414 bits += 9; 5415 break; 5416 case MPSHITTYPE_MASK: 5417 bits += 3; 5418 break; 5419 case FRAGMENTATION_MASK: 5420 bits += 1; 5421 break; 5422 } 5423 5424 if (bits > 36) { 5425 dev_err(adapter->pdev_dev, 5426 "tp_vlan_pri_map=%#x needs %d bits > 36;"\ 5427 " using %#x\n", tp_vlan_pri_map, bits, 5428 TP_VLAN_PRI_MAP_DEFAULT); 5429 tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT; 5430 } 5431 } 5432 v = tp_vlan_pri_map; 5433 t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA, 5434 &v, 1, TP_VLAN_PRI_MAP); 5435 5436 /* 5437 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order 5438 * to support any of the compressed filter fields above. Newer 5439 * versions of the firmware do this automatically but it doesn't hurt 5440 * to set it here. Meanwhile, we do _not_ need to set Lookup Every 5441 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets 5442 * since the firmware automatically turns this on and off when we have 5443 * a non-zero number of filters active (since it does have a 5444 * performance impact). 5445 */ 5446 if (tp_vlan_pri_map) 5447 t4_set_reg_field(adapter, TP_GLOBAL_CONFIG, 5448 FIVETUPLELOOKUP_MASK, 5449 FIVETUPLELOOKUP_MASK); 5450 5451 /* 5452 * Tweak some settings. 5453 */ 5454 t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) | 5455 RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) | 5456 PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) | 5457 KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9)); 5458 5459 /* 5460 * Get basic stuff going by issuing the Firmware Initialize command. 5461 * Note that this _must_ be after all PFVF commands ... 5462 */ 5463 ret = t4_fw_initialize(adapter, adapter->mbox); 5464 if (ret < 0) 5465 goto bye; 5466 5467 /* 5468 * Return successfully! 5469 */ 5470 dev_info(adapter->pdev_dev, "Successfully configured using built-in "\ 5471 "driver parameters\n"); 5472 return 0; 5473 5474 /* 5475 * Something bad happened. Return the error ... 5476 */ 5477 bye: 5478 return ret; 5479 } 5480 5481 static struct fw_info fw_info_array[] = { 5482 { 5483 .chip = CHELSIO_T4, 5484 .fs_name = FW4_CFNAME, 5485 .fw_mod_name = FW4_FNAME, 5486 .fw_hdr = { 5487 .chip = FW_HDR_CHIP_T4, 5488 .fw_ver = __cpu_to_be32(FW_VERSION(T4)), 5489 .intfver_nic = FW_INTFVER(T4, NIC), 5490 .intfver_vnic = FW_INTFVER(T4, VNIC), 5491 .intfver_ri = FW_INTFVER(T4, RI), 5492 .intfver_iscsi = FW_INTFVER(T4, ISCSI), 5493 .intfver_fcoe = FW_INTFVER(T4, FCOE), 5494 }, 5495 }, { 5496 .chip = CHELSIO_T5, 5497 .fs_name = FW5_CFNAME, 5498 .fw_mod_name = FW5_FNAME, 5499 .fw_hdr = { 5500 .chip = FW_HDR_CHIP_T5, 5501 .fw_ver = __cpu_to_be32(FW_VERSION(T5)), 5502 .intfver_nic = FW_INTFVER(T5, NIC), 5503 .intfver_vnic = FW_INTFVER(T5, VNIC), 5504 .intfver_ri = FW_INTFVER(T5, RI), 5505 .intfver_iscsi = FW_INTFVER(T5, ISCSI), 5506 .intfver_fcoe = FW_INTFVER(T5, FCOE), 5507 }, 5508 } 5509 }; 5510 5511 static struct fw_info *find_fw_info(int chip) 5512 { 5513 int i; 5514 5515 for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) { 5516 if (fw_info_array[i].chip == chip) 5517 return &fw_info_array[i]; 5518 } 5519 return NULL; 5520 } 5521 5522 /* 5523 * Phase 0 of initialization: contact FW, obtain config, perform basic init. 5524 */ 5525 static int adap_init0(struct adapter *adap) 5526 { 5527 int ret; 5528 u32 v, port_vec; 5529 enum dev_state state; 5530 u32 params[7], val[7]; 5531 struct fw_caps_config_cmd caps_cmd; 5532 int reset = 1; 5533 5534 /* Contact FW, advertising Master capability */ 5535 ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state); 5536 if (ret < 0) { 5537 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n", 5538 ret); 5539 return ret; 5540 } 5541 if (ret == adap->mbox) 5542 adap->flags |= MASTER_PF; 5543 5544 /* 5545 * If we're the Master PF Driver and the device is uninitialized, 5546 * then let's consider upgrading the firmware ... (We always want 5547 * to check the firmware version number in order to A. get it for 5548 * later reporting and B. to warn if the currently loaded firmware 5549 * is excessively mismatched relative to the driver.) 5550 */ 5551 t4_get_fw_version(adap, &adap->params.fw_vers); 5552 t4_get_tp_version(adap, &adap->params.tp_vers); 5553 if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) { 5554 struct fw_info *fw_info; 5555 struct fw_hdr *card_fw; 5556 const struct firmware *fw; 5557 const u8 *fw_data = NULL; 5558 unsigned int fw_size = 0; 5559 5560 /* This is the firmware whose headers the driver was compiled 5561 * against 5562 */ 5563 fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip)); 5564 if (fw_info == NULL) { 5565 dev_err(adap->pdev_dev, 5566 "unable to get firmware info for chip %d.\n", 5567 CHELSIO_CHIP_VERSION(adap->params.chip)); 5568 return -EINVAL; 5569 } 5570 5571 /* allocate memory to read the header of the firmware on the 5572 * card 5573 */ 5574 card_fw = t4_alloc_mem(sizeof(*card_fw)); 5575 5576 /* Get FW from from /lib/firmware/ */ 5577 ret = request_firmware(&fw, fw_info->fw_mod_name, 5578 adap->pdev_dev); 5579 if (ret < 0) { 5580 dev_err(adap->pdev_dev, 5581 "unable to load firmware image %s, error %d\n", 5582 fw_info->fw_mod_name, ret); 5583 } else { 5584 fw_data = fw->data; 5585 fw_size = fw->size; 5586 } 5587 5588 /* upgrade FW logic */ 5589 ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw, 5590 state, &reset); 5591 5592 /* Cleaning up */ 5593 if (fw != NULL) 5594 release_firmware(fw); 5595 t4_free_mem(card_fw); 5596 5597 if (ret < 0) 5598 goto bye; 5599 } 5600 5601 /* 5602 * Grab VPD parameters. This should be done after we establish a 5603 * connection to the firmware since some of the VPD parameters 5604 * (notably the Core Clock frequency) are retrieved via requests to 5605 * the firmware. On the other hand, we need these fairly early on 5606 * so we do this right after getting ahold of the firmware. 5607 */ 5608 ret = get_vpd_params(adap, &adap->params.vpd); 5609 if (ret < 0) 5610 goto bye; 5611 5612 /* 5613 * Find out what ports are available to us. Note that we need to do 5614 * this before calling adap_init0_no_config() since it needs nports 5615 * and portvec ... 5616 */ 5617 v = 5618 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 5619 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC); 5620 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec); 5621 if (ret < 0) 5622 goto bye; 5623 5624 adap->params.nports = hweight32(port_vec); 5625 adap->params.portvec = port_vec; 5626 5627 /* 5628 * If the firmware is initialized already (and we're not forcing a 5629 * master initialization), note that we're living with existing 5630 * adapter parameters. Otherwise, it's time to try initializing the 5631 * adapter ... 5632 */ 5633 if (state == DEV_STATE_INIT) { 5634 dev_info(adap->pdev_dev, "Coming up as %s: "\ 5635 "Adapter already initialized\n", 5636 adap->flags & MASTER_PF ? "MASTER" : "SLAVE"); 5637 adap->flags |= USING_SOFT_PARAMS; 5638 } else { 5639 dev_info(adap->pdev_dev, "Coming up as MASTER: "\ 5640 "Initializing adapter\n"); 5641 /* 5642 * If the firmware doesn't support Configuration 5643 * Files warn user and exit, 5644 */ 5645 if (ret < 0) 5646 dev_warn(adap->pdev_dev, "Firmware doesn't support " 5647 "configuration file.\n"); 5648 if (force_old_init) 5649 ret = adap_init0_no_config(adap, reset); 5650 else { 5651 /* 5652 * Find out whether we're dealing with a version of 5653 * the firmware which has configuration file support. 5654 */ 5655 params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | 5656 FW_PARAMS_PARAM_X_V( 5657 FW_PARAMS_PARAM_DEV_CF)); 5658 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, 5659 params, val); 5660 5661 /* 5662 * If the firmware doesn't support Configuration 5663 * Files, use the old Driver-based, hard-wired 5664 * initialization. Otherwise, try using the 5665 * Configuration File support and fall back to the 5666 * Driver-based initialization if there's no 5667 * Configuration File found. 5668 */ 5669 if (ret < 0) 5670 ret = adap_init0_no_config(adap, reset); 5671 else { 5672 /* 5673 * The firmware provides us with a memory 5674 * buffer where we can load a Configuration 5675 * File from the host if we want to override 5676 * the Configuration File in flash. 5677 */ 5678 5679 ret = adap_init0_config(adap, reset); 5680 if (ret == -ENOENT) { 5681 dev_info(adap->pdev_dev, 5682 "No Configuration File present " 5683 "on adapter. Using hard-wired " 5684 "configuration parameters.\n"); 5685 ret = adap_init0_no_config(adap, reset); 5686 } 5687 } 5688 } 5689 if (ret < 0) { 5690 dev_err(adap->pdev_dev, 5691 "could not initialize adapter, error %d\n", 5692 -ret); 5693 goto bye; 5694 } 5695 } 5696 5697 /* 5698 * If we're living with non-hard-coded parameters (either from a 5699 * Firmware Configuration File or values programmed by a different PF 5700 * Driver), give the SGE code a chance to pull in anything that it 5701 * needs ... Note that this must be called after we retrieve our VPD 5702 * parameters in order to know how to convert core ticks to seconds. 5703 */ 5704 if (adap->flags & USING_SOFT_PARAMS) { 5705 ret = t4_sge_init(adap); 5706 if (ret < 0) 5707 goto bye; 5708 } 5709 5710 if (is_bypass_device(adap->pdev->device)) 5711 adap->params.bypass = 1; 5712 5713 /* 5714 * Grab some of our basic fundamental operating parameters. 5715 */ 5716 #define FW_PARAM_DEV(param) \ 5717 (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \ 5718 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param)) 5719 5720 #define FW_PARAM_PFVF(param) \ 5721 FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \ 5722 FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)| \ 5723 FW_PARAMS_PARAM_Y_V(0) | \ 5724 FW_PARAMS_PARAM_Z_V(0) 5725 5726 params[0] = FW_PARAM_PFVF(EQ_START); 5727 params[1] = FW_PARAM_PFVF(L2T_START); 5728 params[2] = FW_PARAM_PFVF(L2T_END); 5729 params[3] = FW_PARAM_PFVF(FILTER_START); 5730 params[4] = FW_PARAM_PFVF(FILTER_END); 5731 params[5] = FW_PARAM_PFVF(IQFLINT_START); 5732 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val); 5733 if (ret < 0) 5734 goto bye; 5735 adap->sge.egr_start = val[0]; 5736 adap->l2t_start = val[1]; 5737 adap->l2t_end = val[2]; 5738 adap->tids.ftid_base = val[3]; 5739 adap->tids.nftids = val[4] - val[3] + 1; 5740 adap->sge.ingr_start = val[5]; 5741 5742 /* query params related to active filter region */ 5743 params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START); 5744 params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END); 5745 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val); 5746 /* If Active filter size is set we enable establishing 5747 * offload connection through firmware work request 5748 */ 5749 if ((val[0] != val[1]) && (ret >= 0)) { 5750 adap->flags |= FW_OFLD_CONN; 5751 adap->tids.aftid_base = val[0]; 5752 adap->tids.aftid_end = val[1]; 5753 } 5754 5755 /* If we're running on newer firmware, let it know that we're 5756 * prepared to deal with encapsulated CPL messages. Older 5757 * firmware won't understand this and we'll just get 5758 * unencapsulated messages ... 5759 */ 5760 params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP); 5761 val[0] = 1; 5762 (void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val); 5763 5764 /* 5765 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL 5766 * capability. Earlier versions of the firmware didn't have the 5767 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no 5768 * permission to use ULPTX MEMWRITE DSGL. 5769 */ 5770 if (is_t4(adap->params.chip)) { 5771 adap->params.ulptx_memwrite_dsgl = false; 5772 } else { 5773 params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL); 5774 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 5775 1, params, val); 5776 adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0); 5777 } 5778 5779 /* 5780 * Get device capabilities so we can determine what resources we need 5781 * to manage. 5782 */ 5783 memset(&caps_cmd, 0, sizeof(caps_cmd)); 5784 caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) | 5785 FW_CMD_REQUEST_F | FW_CMD_READ_F); 5786 caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd)); 5787 ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd), 5788 &caps_cmd); 5789 if (ret < 0) 5790 goto bye; 5791 5792 if (caps_cmd.ofldcaps) { 5793 /* query offload-related parameters */ 5794 params[0] = FW_PARAM_DEV(NTID); 5795 params[1] = FW_PARAM_PFVF(SERVER_START); 5796 params[2] = FW_PARAM_PFVF(SERVER_END); 5797 params[3] = FW_PARAM_PFVF(TDDP_START); 5798 params[4] = FW_PARAM_PFVF(TDDP_END); 5799 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ); 5800 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, 5801 params, val); 5802 if (ret < 0) 5803 goto bye; 5804 adap->tids.ntids = val[0]; 5805 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS); 5806 adap->tids.stid_base = val[1]; 5807 adap->tids.nstids = val[2] - val[1] + 1; 5808 /* 5809 * Setup server filter region. Divide the availble filter 5810 * region into two parts. Regular filters get 1/3rd and server 5811 * filters get 2/3rd part. This is only enabled if workarond 5812 * path is enabled. 5813 * 1. For regular filters. 5814 * 2. Server filter: This are special filters which are used 5815 * to redirect SYN packets to offload queue. 5816 */ 5817 if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) { 5818 adap->tids.sftid_base = adap->tids.ftid_base + 5819 DIV_ROUND_UP(adap->tids.nftids, 3); 5820 adap->tids.nsftids = adap->tids.nftids - 5821 DIV_ROUND_UP(adap->tids.nftids, 3); 5822 adap->tids.nftids = adap->tids.sftid_base - 5823 adap->tids.ftid_base; 5824 } 5825 adap->vres.ddp.start = val[3]; 5826 adap->vres.ddp.size = val[4] - val[3] + 1; 5827 adap->params.ofldq_wr_cred = val[5]; 5828 5829 adap->params.offload = 1; 5830 } 5831 if (caps_cmd.rdmacaps) { 5832 params[0] = FW_PARAM_PFVF(STAG_START); 5833 params[1] = FW_PARAM_PFVF(STAG_END); 5834 params[2] = FW_PARAM_PFVF(RQ_START); 5835 params[3] = FW_PARAM_PFVF(RQ_END); 5836 params[4] = FW_PARAM_PFVF(PBL_START); 5837 params[5] = FW_PARAM_PFVF(PBL_END); 5838 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, 5839 params, val); 5840 if (ret < 0) 5841 goto bye; 5842 adap->vres.stag.start = val[0]; 5843 adap->vres.stag.size = val[1] - val[0] + 1; 5844 adap->vres.rq.start = val[2]; 5845 adap->vres.rq.size = val[3] - val[2] + 1; 5846 adap->vres.pbl.start = val[4]; 5847 adap->vres.pbl.size = val[5] - val[4] + 1; 5848 5849 params[0] = FW_PARAM_PFVF(SQRQ_START); 5850 params[1] = FW_PARAM_PFVF(SQRQ_END); 5851 params[2] = FW_PARAM_PFVF(CQ_START); 5852 params[3] = FW_PARAM_PFVF(CQ_END); 5853 params[4] = FW_PARAM_PFVF(OCQ_START); 5854 params[5] = FW_PARAM_PFVF(OCQ_END); 5855 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, 5856 val); 5857 if (ret < 0) 5858 goto bye; 5859 adap->vres.qp.start = val[0]; 5860 adap->vres.qp.size = val[1] - val[0] + 1; 5861 adap->vres.cq.start = val[2]; 5862 adap->vres.cq.size = val[3] - val[2] + 1; 5863 adap->vres.ocq.start = val[4]; 5864 adap->vres.ocq.size = val[5] - val[4] + 1; 5865 5866 params[0] = FW_PARAM_DEV(MAXORDIRD_QP); 5867 params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER); 5868 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, 5869 val); 5870 if (ret < 0) { 5871 adap->params.max_ordird_qp = 8; 5872 adap->params.max_ird_adapter = 32 * adap->tids.ntids; 5873 ret = 0; 5874 } else { 5875 adap->params.max_ordird_qp = val[0]; 5876 adap->params.max_ird_adapter = val[1]; 5877 } 5878 dev_info(adap->pdev_dev, 5879 "max_ordird_qp %d max_ird_adapter %d\n", 5880 adap->params.max_ordird_qp, 5881 adap->params.max_ird_adapter); 5882 } 5883 if (caps_cmd.iscsicaps) { 5884 params[0] = FW_PARAM_PFVF(ISCSI_START); 5885 params[1] = FW_PARAM_PFVF(ISCSI_END); 5886 ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, 5887 params, val); 5888 if (ret < 0) 5889 goto bye; 5890 adap->vres.iscsi.start = val[0]; 5891 adap->vres.iscsi.size = val[1] - val[0] + 1; 5892 } 5893 #undef FW_PARAM_PFVF 5894 #undef FW_PARAM_DEV 5895 5896 /* The MTU/MSS Table is initialized by now, so load their values. If 5897 * we're initializing the adapter, then we'll make any modifications 5898 * we want to the MTU/MSS Table and also initialize the congestion 5899 * parameters. 5900 */ 5901 t4_read_mtu_tbl(adap, adap->params.mtus, NULL); 5902 if (state != DEV_STATE_INIT) { 5903 int i; 5904 5905 /* The default MTU Table contains values 1492 and 1500. 5906 * However, for TCP, it's better to have two values which are 5907 * a multiple of 8 +/- 4 bytes apart near this popular MTU. 5908 * This allows us to have a TCP Data Payload which is a 5909 * multiple of 8 regardless of what combination of TCP Options 5910 * are in use (always a multiple of 4 bytes) which is 5911 * important for performance reasons. For instance, if no 5912 * options are in use, then we have a 20-byte IP header and a 5913 * 20-byte TCP header. In this case, a 1500-byte MSS would 5914 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes 5915 * which is not a multiple of 8. So using an MSS of 1488 in 5916 * this case results in a TCP Data Payload of 1448 bytes which 5917 * is a multiple of 8. On the other hand, if 12-byte TCP Time 5918 * Stamps have been negotiated, then an MTU of 1500 bytes 5919 * results in a TCP Data Payload of 1448 bytes which, as 5920 * above, is a multiple of 8 bytes ... 5921 */ 5922 for (i = 0; i < NMTUS; i++) 5923 if (adap->params.mtus[i] == 1492) { 5924 adap->params.mtus[i] = 1488; 5925 break; 5926 } 5927 5928 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 5929 adap->params.b_wnd); 5930 } 5931 t4_init_sge_params(adap); 5932 t4_init_tp_params(adap); 5933 adap->flags |= FW_OK; 5934 return 0; 5935 5936 /* 5937 * Something bad happened. If a command timed out or failed with EIO 5938 * FW does not operate within its spec or something catastrophic 5939 * happened to HW/FW, stop issuing commands. 5940 */ 5941 bye: 5942 if (ret != -ETIMEDOUT && ret != -EIO) 5943 t4_fw_bye(adap, adap->mbox); 5944 return ret; 5945 } 5946 5947 /* EEH callbacks */ 5948 5949 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev, 5950 pci_channel_state_t state) 5951 { 5952 int i; 5953 struct adapter *adap = pci_get_drvdata(pdev); 5954 5955 if (!adap) 5956 goto out; 5957 5958 rtnl_lock(); 5959 adap->flags &= ~FW_OK; 5960 notify_ulds(adap, CXGB4_STATE_START_RECOVERY); 5961 spin_lock(&adap->stats_lock); 5962 for_each_port(adap, i) { 5963 struct net_device *dev = adap->port[i]; 5964 5965 netif_device_detach(dev); 5966 netif_carrier_off(dev); 5967 } 5968 spin_unlock(&adap->stats_lock); 5969 if (adap->flags & FULL_INIT_DONE) 5970 cxgb_down(adap); 5971 rtnl_unlock(); 5972 if ((adap->flags & DEV_ENABLED)) { 5973 pci_disable_device(pdev); 5974 adap->flags &= ~DEV_ENABLED; 5975 } 5976 out: return state == pci_channel_io_perm_failure ? 5977 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET; 5978 } 5979 5980 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev) 5981 { 5982 int i, ret; 5983 struct fw_caps_config_cmd c; 5984 struct adapter *adap = pci_get_drvdata(pdev); 5985 5986 if (!adap) { 5987 pci_restore_state(pdev); 5988 pci_save_state(pdev); 5989 return PCI_ERS_RESULT_RECOVERED; 5990 } 5991 5992 if (!(adap->flags & DEV_ENABLED)) { 5993 if (pci_enable_device(pdev)) { 5994 dev_err(&pdev->dev, "Cannot reenable PCI " 5995 "device after reset\n"); 5996 return PCI_ERS_RESULT_DISCONNECT; 5997 } 5998 adap->flags |= DEV_ENABLED; 5999 } 6000 6001 pci_set_master(pdev); 6002 pci_restore_state(pdev); 6003 pci_save_state(pdev); 6004 pci_cleanup_aer_uncorrect_error_status(pdev); 6005 6006 if (t4_wait_dev_ready(adap->regs) < 0) 6007 return PCI_ERS_RESULT_DISCONNECT; 6008 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0) 6009 return PCI_ERS_RESULT_DISCONNECT; 6010 adap->flags |= FW_OK; 6011 if (adap_init1(adap, &c)) 6012 return PCI_ERS_RESULT_DISCONNECT; 6013 6014 for_each_port(adap, i) { 6015 struct port_info *p = adap2pinfo(adap, i); 6016 6017 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1, 6018 NULL, NULL); 6019 if (ret < 0) 6020 return PCI_ERS_RESULT_DISCONNECT; 6021 p->viid = ret; 6022 p->xact_addr_filt = -1; 6023 } 6024 6025 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd, 6026 adap->params.b_wnd); 6027 setup_memwin(adap); 6028 if (cxgb_up(adap)) 6029 return PCI_ERS_RESULT_DISCONNECT; 6030 return PCI_ERS_RESULT_RECOVERED; 6031 } 6032 6033 static void eeh_resume(struct pci_dev *pdev) 6034 { 6035 int i; 6036 struct adapter *adap = pci_get_drvdata(pdev); 6037 6038 if (!adap) 6039 return; 6040 6041 rtnl_lock(); 6042 for_each_port(adap, i) { 6043 struct net_device *dev = adap->port[i]; 6044 6045 if (netif_running(dev)) { 6046 link_start(dev); 6047 cxgb_set_rxmode(dev); 6048 } 6049 netif_device_attach(dev); 6050 } 6051 rtnl_unlock(); 6052 } 6053 6054 static const struct pci_error_handlers cxgb4_eeh = { 6055 .error_detected = eeh_err_detected, 6056 .slot_reset = eeh_slot_reset, 6057 .resume = eeh_resume, 6058 }; 6059 6060 static inline bool is_x_10g_port(const struct link_config *lc) 6061 { 6062 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 || 6063 (lc->supported & FW_PORT_CAP_SPEED_40G) != 0; 6064 } 6065 6066 static inline void init_rspq(struct adapter *adap, struct sge_rspq *q, 6067 unsigned int us, unsigned int cnt, 6068 unsigned int size, unsigned int iqe_size) 6069 { 6070 q->adap = adap; 6071 set_rspq_intr_params(q, us, cnt); 6072 q->iqe_len = iqe_size; 6073 q->size = size; 6074 } 6075 6076 /* 6077 * Perform default configuration of DMA queues depending on the number and type 6078 * of ports we found and the number of available CPUs. Most settings can be 6079 * modified by the admin prior to actual use. 6080 */ 6081 static void cfg_queues(struct adapter *adap) 6082 { 6083 struct sge *s = &adap->sge; 6084 int i, n10g = 0, qidx = 0; 6085 #ifndef CONFIG_CHELSIO_T4_DCB 6086 int q10g = 0; 6087 #endif 6088 int ciq_size; 6089 6090 for_each_port(adap, i) 6091 n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg); 6092 #ifdef CONFIG_CHELSIO_T4_DCB 6093 /* For Data Center Bridging support we need to be able to support up 6094 * to 8 Traffic Priorities; each of which will be assigned to its 6095 * own TX Queue in order to prevent Head-Of-Line Blocking. 6096 */ 6097 if (adap->params.nports * 8 > MAX_ETH_QSETS) { 6098 dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n", 6099 MAX_ETH_QSETS, adap->params.nports * 8); 6100 BUG_ON(1); 6101 } 6102 6103 for_each_port(adap, i) { 6104 struct port_info *pi = adap2pinfo(adap, i); 6105 6106 pi->first_qset = qidx; 6107 pi->nqsets = 8; 6108 qidx += pi->nqsets; 6109 } 6110 #else /* !CONFIG_CHELSIO_T4_DCB */ 6111 /* 6112 * We default to 1 queue per non-10G port and up to # of cores queues 6113 * per 10G port. 6114 */ 6115 if (n10g) 6116 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g; 6117 if (q10g > netif_get_num_default_rss_queues()) 6118 q10g = netif_get_num_default_rss_queues(); 6119 6120 for_each_port(adap, i) { 6121 struct port_info *pi = adap2pinfo(adap, i); 6122 6123 pi->first_qset = qidx; 6124 pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1; 6125 qidx += pi->nqsets; 6126 } 6127 #endif /* !CONFIG_CHELSIO_T4_DCB */ 6128 6129 s->ethqsets = qidx; 6130 s->max_ethqsets = qidx; /* MSI-X may lower it later */ 6131 6132 if (is_offload(adap)) { 6133 /* 6134 * For offload we use 1 queue/channel if all ports are up to 1G, 6135 * otherwise we divide all available queues amongst the channels 6136 * capped by the number of available cores. 6137 */ 6138 if (n10g) { 6139 i = min_t(int, ARRAY_SIZE(s->ofldrxq), 6140 num_online_cpus()); 6141 s->ofldqsets = roundup(i, adap->params.nports); 6142 } else 6143 s->ofldqsets = adap->params.nports; 6144 /* For RDMA one Rx queue per channel suffices */ 6145 s->rdmaqs = adap->params.nports; 6146 s->rdmaciqs = adap->params.nports; 6147 } 6148 6149 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) { 6150 struct sge_eth_rxq *r = &s->ethrxq[i]; 6151 6152 init_rspq(adap, &r->rspq, 5, 10, 1024, 64); 6153 r->fl.size = 72; 6154 } 6155 6156 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++) 6157 s->ethtxq[i].q.size = 1024; 6158 6159 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++) 6160 s->ctrlq[i].q.size = 512; 6161 6162 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++) 6163 s->ofldtxq[i].q.size = 1024; 6164 6165 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) { 6166 struct sge_ofld_rxq *r = &s->ofldrxq[i]; 6167 6168 init_rspq(adap, &r->rspq, 5, 1, 1024, 64); 6169 r->rspq.uld = CXGB4_ULD_ISCSI; 6170 r->fl.size = 72; 6171 } 6172 6173 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) { 6174 struct sge_ofld_rxq *r = &s->rdmarxq[i]; 6175 6176 init_rspq(adap, &r->rspq, 5, 1, 511, 64); 6177 r->rspq.uld = CXGB4_ULD_RDMA; 6178 r->fl.size = 72; 6179 } 6180 6181 ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids; 6182 if (ciq_size > SGE_MAX_IQ_SIZE) { 6183 CH_WARN(adap, "CIQ size too small for available IQs\n"); 6184 ciq_size = SGE_MAX_IQ_SIZE; 6185 } 6186 6187 for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) { 6188 struct sge_ofld_rxq *r = &s->rdmaciq[i]; 6189 6190 init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64); 6191 r->rspq.uld = CXGB4_ULD_RDMA; 6192 } 6193 6194 init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64); 6195 init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64); 6196 } 6197 6198 /* 6199 * Reduce the number of Ethernet queues across all ports to at most n. 6200 * n provides at least one queue per port. 6201 */ 6202 static void reduce_ethqs(struct adapter *adap, int n) 6203 { 6204 int i; 6205 struct port_info *pi; 6206 6207 while (n < adap->sge.ethqsets) 6208 for_each_port(adap, i) { 6209 pi = adap2pinfo(adap, i); 6210 if (pi->nqsets > 1) { 6211 pi->nqsets--; 6212 adap->sge.ethqsets--; 6213 if (adap->sge.ethqsets <= n) 6214 break; 6215 } 6216 } 6217 6218 n = 0; 6219 for_each_port(adap, i) { 6220 pi = adap2pinfo(adap, i); 6221 pi->first_qset = n; 6222 n += pi->nqsets; 6223 } 6224 } 6225 6226 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */ 6227 #define EXTRA_VECS 2 6228 6229 static int enable_msix(struct adapter *adap) 6230 { 6231 int ofld_need = 0; 6232 int i, want, need; 6233 struct sge *s = &adap->sge; 6234 unsigned int nchan = adap->params.nports; 6235 struct msix_entry entries[MAX_INGQ + 1]; 6236 6237 for (i = 0; i < ARRAY_SIZE(entries); ++i) 6238 entries[i].entry = i; 6239 6240 want = s->max_ethqsets + EXTRA_VECS; 6241 if (is_offload(adap)) { 6242 want += s->rdmaqs + s->rdmaciqs + s->ofldqsets; 6243 /* need nchan for each possible ULD */ 6244 ofld_need = 3 * nchan; 6245 } 6246 #ifdef CONFIG_CHELSIO_T4_DCB 6247 /* For Data Center Bridging we need 8 Ethernet TX Priority Queues for 6248 * each port. 6249 */ 6250 need = 8 * adap->params.nports + EXTRA_VECS + ofld_need; 6251 #else 6252 need = adap->params.nports + EXTRA_VECS + ofld_need; 6253 #endif 6254 want = pci_enable_msix_range(adap->pdev, entries, need, want); 6255 if (want < 0) 6256 return want; 6257 6258 /* 6259 * Distribute available vectors to the various queue groups. 6260 * Every group gets its minimum requirement and NIC gets top 6261 * priority for leftovers. 6262 */ 6263 i = want - EXTRA_VECS - ofld_need; 6264 if (i < s->max_ethqsets) { 6265 s->max_ethqsets = i; 6266 if (i < s->ethqsets) 6267 reduce_ethqs(adap, i); 6268 } 6269 if (is_offload(adap)) { 6270 i = want - EXTRA_VECS - s->max_ethqsets; 6271 i -= ofld_need - nchan; 6272 s->ofldqsets = (i / nchan) * nchan; /* round down */ 6273 } 6274 for (i = 0; i < want; ++i) 6275 adap->msix_info[i].vec = entries[i].vector; 6276 6277 return 0; 6278 } 6279 6280 #undef EXTRA_VECS 6281 6282 static int init_rss(struct adapter *adap) 6283 { 6284 unsigned int i, j; 6285 6286 for_each_port(adap, i) { 6287 struct port_info *pi = adap2pinfo(adap, i); 6288 6289 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL); 6290 if (!pi->rss) 6291 return -ENOMEM; 6292 for (j = 0; j < pi->rss_size; j++) 6293 pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets); 6294 } 6295 return 0; 6296 } 6297 6298 static void print_port_info(const struct net_device *dev) 6299 { 6300 char buf[80]; 6301 char *bufp = buf; 6302 const char *spd = ""; 6303 const struct port_info *pi = netdev_priv(dev); 6304 const struct adapter *adap = pi->adapter; 6305 6306 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB) 6307 spd = " 2.5 GT/s"; 6308 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB) 6309 spd = " 5 GT/s"; 6310 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB) 6311 spd = " 8 GT/s"; 6312 6313 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M) 6314 bufp += sprintf(bufp, "100/"); 6315 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G) 6316 bufp += sprintf(bufp, "1000/"); 6317 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G) 6318 bufp += sprintf(bufp, "10G/"); 6319 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G) 6320 bufp += sprintf(bufp, "40G/"); 6321 if (bufp != buf) 6322 --bufp; 6323 sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type)); 6324 6325 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n", 6326 adap->params.vpd.id, 6327 CHELSIO_CHIP_RELEASE(adap->params.chip), buf, 6328 is_offload(adap) ? "R" : "", adap->params.pci.width, spd, 6329 (adap->flags & USING_MSIX) ? " MSI-X" : 6330 (adap->flags & USING_MSI) ? " MSI" : ""); 6331 netdev_info(dev, "S/N: %s, P/N: %s\n", 6332 adap->params.vpd.sn, adap->params.vpd.pn); 6333 } 6334 6335 static void enable_pcie_relaxed_ordering(struct pci_dev *dev) 6336 { 6337 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN); 6338 } 6339 6340 /* 6341 * Free the following resources: 6342 * - memory used for tables 6343 * - MSI/MSI-X 6344 * - net devices 6345 * - resources FW is holding for us 6346 */ 6347 static void free_some_resources(struct adapter *adapter) 6348 { 6349 unsigned int i; 6350 6351 t4_free_mem(adapter->l2t); 6352 t4_free_mem(adapter->tids.tid_tab); 6353 disable_msi(adapter); 6354 6355 for_each_port(adapter, i) 6356 if (adapter->port[i]) { 6357 kfree(adap2pinfo(adapter, i)->rss); 6358 free_netdev(adapter->port[i]); 6359 } 6360 if (adapter->flags & FW_OK) 6361 t4_fw_bye(adapter, adapter->fn); 6362 } 6363 6364 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN) 6365 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \ 6366 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA) 6367 #define SEGMENT_SIZE 128 6368 6369 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) 6370 { 6371 int func, i, err, s_qpp, qpp, num_seg; 6372 struct port_info *pi; 6373 bool highdma = false; 6374 struct adapter *adapter = NULL; 6375 void __iomem *regs; 6376 6377 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION); 6378 6379 err = pci_request_regions(pdev, KBUILD_MODNAME); 6380 if (err) { 6381 /* Just info, some other driver may have claimed the device. */ 6382 dev_info(&pdev->dev, "cannot obtain PCI resources\n"); 6383 return err; 6384 } 6385 6386 err = pci_enable_device(pdev); 6387 if (err) { 6388 dev_err(&pdev->dev, "cannot enable PCI device\n"); 6389 goto out_release_regions; 6390 } 6391 6392 regs = pci_ioremap_bar(pdev, 0); 6393 if (!regs) { 6394 dev_err(&pdev->dev, "cannot map device registers\n"); 6395 err = -ENOMEM; 6396 goto out_disable_device; 6397 } 6398 6399 err = t4_wait_dev_ready(regs); 6400 if (err < 0) 6401 goto out_unmap_bar0; 6402 6403 /* We control everything through one PF */ 6404 func = SOURCEPF_GET(readl(regs + PL_WHOAMI)); 6405 if (func != ent->driver_data) { 6406 iounmap(regs); 6407 pci_disable_device(pdev); 6408 pci_save_state(pdev); /* to restore SR-IOV later */ 6409 goto sriov; 6410 } 6411 6412 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) { 6413 highdma = true; 6414 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64)); 6415 if (err) { 6416 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for " 6417 "coherent allocations\n"); 6418 goto out_unmap_bar0; 6419 } 6420 } else { 6421 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32)); 6422 if (err) { 6423 dev_err(&pdev->dev, "no usable DMA configuration\n"); 6424 goto out_unmap_bar0; 6425 } 6426 } 6427 6428 pci_enable_pcie_error_reporting(pdev); 6429 enable_pcie_relaxed_ordering(pdev); 6430 pci_set_master(pdev); 6431 pci_save_state(pdev); 6432 6433 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 6434 if (!adapter) { 6435 err = -ENOMEM; 6436 goto out_unmap_bar0; 6437 } 6438 6439 adapter->workq = create_singlethread_workqueue("cxgb4"); 6440 if (!adapter->workq) { 6441 err = -ENOMEM; 6442 goto out_free_adapter; 6443 } 6444 6445 /* PCI device has been enabled */ 6446 adapter->flags |= DEV_ENABLED; 6447 6448 adapter->regs = regs; 6449 adapter->pdev = pdev; 6450 adapter->pdev_dev = &pdev->dev; 6451 adapter->mbox = func; 6452 adapter->fn = func; 6453 adapter->msg_enable = dflt_msg_enable; 6454 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map)); 6455 6456 spin_lock_init(&adapter->stats_lock); 6457 spin_lock_init(&adapter->tid_release_lock); 6458 spin_lock_init(&adapter->win0_lock); 6459 6460 INIT_WORK(&adapter->tid_release_task, process_tid_release_list); 6461 INIT_WORK(&adapter->db_full_task, process_db_full); 6462 INIT_WORK(&adapter->db_drop_task, process_db_drop); 6463 6464 err = t4_prep_adapter(adapter); 6465 if (err) 6466 goto out_free_adapter; 6467 6468 6469 if (!is_t4(adapter->params.chip)) { 6470 s_qpp = QUEUESPERPAGEPF1 * adapter->fn; 6471 qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter, 6472 SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp); 6473 num_seg = PAGE_SIZE / SEGMENT_SIZE; 6474 6475 /* Each segment size is 128B. Write coalescing is enabled only 6476 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the 6477 * queue is less no of segments that can be accommodated in 6478 * a page size. 6479 */ 6480 if (qpp > num_seg) { 6481 dev_err(&pdev->dev, 6482 "Incorrect number of egress queues per page\n"); 6483 err = -EINVAL; 6484 goto out_free_adapter; 6485 } 6486 adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2), 6487 pci_resource_len(pdev, 2)); 6488 if (!adapter->bar2) { 6489 dev_err(&pdev->dev, "cannot map device bar2 region\n"); 6490 err = -ENOMEM; 6491 goto out_free_adapter; 6492 } 6493 } 6494 6495 setup_memwin(adapter); 6496 err = adap_init0(adapter); 6497 setup_memwin_rdma(adapter); 6498 if (err) 6499 goto out_unmap_bar; 6500 6501 for_each_port(adapter, i) { 6502 struct net_device *netdev; 6503 6504 netdev = alloc_etherdev_mq(sizeof(struct port_info), 6505 MAX_ETH_QSETS); 6506 if (!netdev) { 6507 err = -ENOMEM; 6508 goto out_free_dev; 6509 } 6510 6511 SET_NETDEV_DEV(netdev, &pdev->dev); 6512 6513 adapter->port[i] = netdev; 6514 pi = netdev_priv(netdev); 6515 pi->adapter = adapter; 6516 pi->xact_addr_filt = -1; 6517 pi->port_id = i; 6518 netdev->irq = pdev->irq; 6519 6520 netdev->hw_features = NETIF_F_SG | TSO_FLAGS | 6521 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 6522 NETIF_F_RXCSUM | NETIF_F_RXHASH | 6523 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 6524 if (highdma) 6525 netdev->hw_features |= NETIF_F_HIGHDMA; 6526 netdev->features |= netdev->hw_features; 6527 netdev->vlan_features = netdev->features & VLAN_FEAT; 6528 6529 netdev->priv_flags |= IFF_UNICAST_FLT; 6530 6531 netdev->netdev_ops = &cxgb4_netdev_ops; 6532 #ifdef CONFIG_CHELSIO_T4_DCB 6533 netdev->dcbnl_ops = &cxgb4_dcb_ops; 6534 cxgb4_dcb_state_init(netdev); 6535 #endif 6536 netdev->ethtool_ops = &cxgb_ethtool_ops; 6537 } 6538 6539 pci_set_drvdata(pdev, adapter); 6540 6541 if (adapter->flags & FW_OK) { 6542 err = t4_port_init(adapter, func, func, 0); 6543 if (err) 6544 goto out_free_dev; 6545 } 6546 6547 /* 6548 * Configure queues and allocate tables now, they can be needed as 6549 * soon as the first register_netdev completes. 6550 */ 6551 cfg_queues(adapter); 6552 6553 adapter->l2t = t4_init_l2t(); 6554 if (!adapter->l2t) { 6555 /* We tolerate a lack of L2T, giving up some functionality */ 6556 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n"); 6557 adapter->params.offload = 0; 6558 } 6559 6560 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) { 6561 dev_warn(&pdev->dev, "could not allocate TID table, " 6562 "continuing\n"); 6563 adapter->params.offload = 0; 6564 } 6565 6566 /* See what interrupts we'll be using */ 6567 if (msi > 1 && enable_msix(adapter) == 0) 6568 adapter->flags |= USING_MSIX; 6569 else if (msi > 0 && pci_enable_msi(pdev) == 0) 6570 adapter->flags |= USING_MSI; 6571 6572 err = init_rss(adapter); 6573 if (err) 6574 goto out_free_dev; 6575 6576 /* 6577 * The card is now ready to go. If any errors occur during device 6578 * registration we do not fail the whole card but rather proceed only 6579 * with the ports we manage to register successfully. However we must 6580 * register at least one net device. 6581 */ 6582 for_each_port(adapter, i) { 6583 pi = adap2pinfo(adapter, i); 6584 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets); 6585 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets); 6586 6587 err = register_netdev(adapter->port[i]); 6588 if (err) 6589 break; 6590 adapter->chan_map[pi->tx_chan] = i; 6591 print_port_info(adapter->port[i]); 6592 } 6593 if (i == 0) { 6594 dev_err(&pdev->dev, "could not register any net devices\n"); 6595 goto out_free_dev; 6596 } 6597 if (err) { 6598 dev_warn(&pdev->dev, "only %d net devices registered\n", i); 6599 err = 0; 6600 } 6601 6602 if (cxgb4_debugfs_root) { 6603 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev), 6604 cxgb4_debugfs_root); 6605 setup_debugfs(adapter); 6606 } 6607 6608 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */ 6609 pdev->needs_freset = 1; 6610 6611 if (is_offload(adapter)) 6612 attach_ulds(adapter); 6613 6614 sriov: 6615 #ifdef CONFIG_PCI_IOV 6616 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0) 6617 if (pci_enable_sriov(pdev, num_vf[func]) == 0) 6618 dev_info(&pdev->dev, 6619 "instantiated %u virtual functions\n", 6620 num_vf[func]); 6621 #endif 6622 return 0; 6623 6624 out_free_dev: 6625 free_some_resources(adapter); 6626 out_unmap_bar: 6627 if (!is_t4(adapter->params.chip)) 6628 iounmap(adapter->bar2); 6629 out_free_adapter: 6630 if (adapter->workq) 6631 destroy_workqueue(adapter->workq); 6632 6633 kfree(adapter); 6634 out_unmap_bar0: 6635 iounmap(regs); 6636 out_disable_device: 6637 pci_disable_pcie_error_reporting(pdev); 6638 pci_disable_device(pdev); 6639 out_release_regions: 6640 pci_release_regions(pdev); 6641 return err; 6642 } 6643 6644 static void remove_one(struct pci_dev *pdev) 6645 { 6646 struct adapter *adapter = pci_get_drvdata(pdev); 6647 6648 #ifdef CONFIG_PCI_IOV 6649 pci_disable_sriov(pdev); 6650 6651 #endif 6652 6653 if (adapter) { 6654 int i; 6655 6656 /* Tear down per-adapter Work Queue first since it can contain 6657 * references to our adapter data structure. 6658 */ 6659 destroy_workqueue(adapter->workq); 6660 6661 if (is_offload(adapter)) 6662 detach_ulds(adapter); 6663 6664 for_each_port(adapter, i) 6665 if (adapter->port[i]->reg_state == NETREG_REGISTERED) 6666 unregister_netdev(adapter->port[i]); 6667 6668 debugfs_remove_recursive(adapter->debugfs_root); 6669 6670 /* If we allocated filters, free up state associated with any 6671 * valid filters ... 6672 */ 6673 if (adapter->tids.ftid_tab) { 6674 struct filter_entry *f = &adapter->tids.ftid_tab[0]; 6675 for (i = 0; i < (adapter->tids.nftids + 6676 adapter->tids.nsftids); i++, f++) 6677 if (f->valid) 6678 clear_filter(adapter, f); 6679 } 6680 6681 if (adapter->flags & FULL_INIT_DONE) 6682 cxgb_down(adapter); 6683 6684 free_some_resources(adapter); 6685 iounmap(adapter->regs); 6686 if (!is_t4(adapter->params.chip)) 6687 iounmap(adapter->bar2); 6688 pci_disable_pcie_error_reporting(pdev); 6689 if ((adapter->flags & DEV_ENABLED)) { 6690 pci_disable_device(pdev); 6691 adapter->flags &= ~DEV_ENABLED; 6692 } 6693 pci_release_regions(pdev); 6694 synchronize_rcu(); 6695 kfree(adapter); 6696 } else 6697 pci_release_regions(pdev); 6698 } 6699 6700 static struct pci_driver cxgb4_driver = { 6701 .name = KBUILD_MODNAME, 6702 .id_table = cxgb4_pci_tbl, 6703 .probe = init_one, 6704 .remove = remove_one, 6705 .shutdown = remove_one, 6706 .err_handler = &cxgb4_eeh, 6707 }; 6708 6709 static int __init cxgb4_init_module(void) 6710 { 6711 int ret; 6712 6713 /* Debugfs support is optional, just warn if this fails */ 6714 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL); 6715 if (!cxgb4_debugfs_root) 6716 pr_warn("could not create debugfs entry, continuing\n"); 6717 6718 ret = pci_register_driver(&cxgb4_driver); 6719 if (ret < 0) 6720 debugfs_remove(cxgb4_debugfs_root); 6721 6722 #if IS_ENABLED(CONFIG_IPV6) 6723 register_inet6addr_notifier(&cxgb4_inet6addr_notifier); 6724 #endif 6725 6726 return ret; 6727 } 6728 6729 static void __exit cxgb4_cleanup_module(void) 6730 { 6731 #if IS_ENABLED(CONFIG_IPV6) 6732 unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier); 6733 #endif 6734 pci_unregister_driver(&cxgb4_driver); 6735 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */ 6736 } 6737 6738 module_init(cxgb4_init_module); 6739 module_exit(cxgb4_cleanup_module); 6740