1 /*
2  * This file is part of the Chelsio T4 Ethernet driver for Linux.
3  *
4  * Copyright (c) 2003-2014 Chelsio Communications, Inc. All rights reserved.
5  *
6  * This software is available to you under a choice of one of two
7  * licenses.  You may choose to be licensed under the terms of the GNU
8  * General Public License (GPL) Version 2, available from the file
9  * COPYING in the main directory of this source tree, or the
10  * OpenIB.org BSD license below:
11  *
12  *     Redistribution and use in source and binary forms, with or
13  *     without modification, are permitted provided that the following
14  *     conditions are met:
15  *
16  *      - Redistributions of source code must retain the above
17  *        copyright notice, this list of conditions and the following
18  *        disclaimer.
19  *
20  *      - Redistributions in binary form must reproduce the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer in the documentation and/or other materials
23  *        provided with the distribution.
24  *
25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32  * SOFTWARE.
33  */
34 
35 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36 
37 #include <linux/bitmap.h>
38 #include <linux/crc32.h>
39 #include <linux/ctype.h>
40 #include <linux/debugfs.h>
41 #include <linux/err.h>
42 #include <linux/etherdevice.h>
43 #include <linux/firmware.h>
44 #include <linux/if.h>
45 #include <linux/if_vlan.h>
46 #include <linux/init.h>
47 #include <linux/log2.h>
48 #include <linux/mdio.h>
49 #include <linux/module.h>
50 #include <linux/moduleparam.h>
51 #include <linux/mutex.h>
52 #include <linux/netdevice.h>
53 #include <linux/pci.h>
54 #include <linux/aer.h>
55 #include <linux/rtnetlink.h>
56 #include <linux/sched.h>
57 #include <linux/seq_file.h>
58 #include <linux/sockios.h>
59 #include <linux/vmalloc.h>
60 #include <linux/workqueue.h>
61 #include <net/neighbour.h>
62 #include <net/netevent.h>
63 #include <net/addrconf.h>
64 #include <net/bonding.h>
65 #include <asm/uaccess.h>
66 
67 #include "cxgb4.h"
68 #include "t4_regs.h"
69 #include "t4_msg.h"
70 #include "t4fw_api.h"
71 #include "cxgb4_dcb.h"
72 #include "cxgb4_debugfs.h"
73 #include "l2t.h"
74 
75 #ifdef DRV_VERSION
76 #undef DRV_VERSION
77 #endif
78 #define DRV_VERSION "2.0.0-ko"
79 #define DRV_DESC "Chelsio T4/T5 Network Driver"
80 
81 /*
82  * Max interrupt hold-off timer value in us.  Queues fall back to this value
83  * under extreme memory pressure so it's largish to give the system time to
84  * recover.
85  */
86 #define MAX_SGE_TIMERVAL 200U
87 
88 enum {
89 	/*
90 	 * Physical Function provisioning constants.
91 	 */
92 	PFRES_NVI = 4,			/* # of Virtual Interfaces */
93 	PFRES_NETHCTRL = 128,		/* # of EQs used for ETH or CTRL Qs */
94 	PFRES_NIQFLINT = 128,		/* # of ingress Qs/w Free List(s)/intr
95 					 */
96 	PFRES_NEQ = 256,		/* # of egress queues */
97 	PFRES_NIQ = 0,			/* # of ingress queues */
98 	PFRES_TC = 0,			/* PCI-E traffic class */
99 	PFRES_NEXACTF = 128,		/* # of exact MPS filters */
100 
101 	PFRES_R_CAPS = FW_CMD_CAP_PF,
102 	PFRES_WX_CAPS = FW_CMD_CAP_PF,
103 
104 #ifdef CONFIG_PCI_IOV
105 	/*
106 	 * Virtual Function provisioning constants.  We need two extra Ingress
107 	 * Queues with Interrupt capability to serve as the VF's Firmware
108 	 * Event Queue and Forwarded Interrupt Queue (when using MSI mode) --
109 	 * neither will have Free Lists associated with them).  For each
110 	 * Ethernet/Control Egress Queue and for each Free List, we need an
111 	 * Egress Context.
112 	 */
113 	VFRES_NPORTS = 1,		/* # of "ports" per VF */
114 	VFRES_NQSETS = 2,		/* # of "Queue Sets" per VF */
115 
116 	VFRES_NVI = VFRES_NPORTS,	/* # of Virtual Interfaces */
117 	VFRES_NETHCTRL = VFRES_NQSETS,	/* # of EQs used for ETH or CTRL Qs */
118 	VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
119 	VFRES_NEQ = VFRES_NQSETS*2,	/* # of egress queues */
120 	VFRES_NIQ = 0,			/* # of non-fl/int ingress queues */
121 	VFRES_TC = 0,			/* PCI-E traffic class */
122 	VFRES_NEXACTF = 16,		/* # of exact MPS filters */
123 
124 	VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
125 	VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
126 #endif
127 };
128 
129 /*
130  * Provide a Port Access Rights Mask for the specified PF/VF.  This is very
131  * static and likely not to be useful in the long run.  We really need to
132  * implement some form of persistent configuration which the firmware
133  * controls.
134  */
135 static unsigned int pfvfres_pmask(struct adapter *adapter,
136 				  unsigned int pf, unsigned int vf)
137 {
138 	unsigned int portn, portvec;
139 
140 	/*
141 	 * Give PF's access to all of the ports.
142 	 */
143 	if (vf == 0)
144 		return FW_PFVF_CMD_PMASK_M;
145 
146 	/*
147 	 * For VFs, we'll assign them access to the ports based purely on the
148 	 * PF.  We assign active ports in order, wrapping around if there are
149 	 * fewer active ports than PFs: e.g. active port[pf % nports].
150 	 * Unfortunately the adapter's port_info structs haven't been
151 	 * initialized yet so we have to compute this.
152 	 */
153 	if (adapter->params.nports == 0)
154 		return 0;
155 
156 	portn = pf % adapter->params.nports;
157 	portvec = adapter->params.portvec;
158 	for (;;) {
159 		/*
160 		 * Isolate the lowest set bit in the port vector.  If we're at
161 		 * the port number that we want, return that as the pmask.
162 		 * otherwise mask that bit out of the port vector and
163 		 * decrement our port number ...
164 		 */
165 		unsigned int pmask = portvec ^ (portvec & (portvec-1));
166 		if (portn == 0)
167 			return pmask;
168 		portn--;
169 		portvec &= ~pmask;
170 	}
171 	/*NOTREACHED*/
172 }
173 
174 enum {
175 	MAX_TXQ_ENTRIES      = 16384,
176 	MAX_CTRL_TXQ_ENTRIES = 1024,
177 	MAX_RSPQ_ENTRIES     = 16384,
178 	MAX_RX_BUFFERS       = 16384,
179 	MIN_TXQ_ENTRIES      = 32,
180 	MIN_CTRL_TXQ_ENTRIES = 32,
181 	MIN_RSPQ_ENTRIES     = 128,
182 	MIN_FL_ENTRIES       = 16
183 };
184 
185 /* Host shadow copy of ingress filter entry.  This is in host native format
186  * and doesn't match the ordering or bit order, etc. of the hardware of the
187  * firmware command.  The use of bit-field structure elements is purely to
188  * remind ourselves of the field size limitations and save memory in the case
189  * where the filter table is large.
190  */
191 struct filter_entry {
192 	/* Administrative fields for filter.
193 	 */
194 	u32 valid:1;            /* filter allocated and valid */
195 	u32 locked:1;           /* filter is administratively locked */
196 
197 	u32 pending:1;          /* filter action is pending firmware reply */
198 	u32 smtidx:8;           /* Source MAC Table index for smac */
199 	struct l2t_entry *l2t;  /* Layer Two Table entry for dmac */
200 
201 	/* The filter itself.  Most of this is a straight copy of information
202 	 * provided by the extended ioctl().  Some fields are translated to
203 	 * internal forms -- for instance the Ingress Queue ID passed in from
204 	 * the ioctl() is translated into the Absolute Ingress Queue ID.
205 	 */
206 	struct ch_filter_specification fs;
207 };
208 
209 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
210 			 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
211 			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
212 
213 /* Macros needed to support the PCI Device ID Table ...
214  */
215 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
216 	static struct pci_device_id cxgb4_pci_tbl[] = {
217 #define CH_PCI_DEVICE_ID_FUNCTION 0x4
218 
219 /* Include PCI Device IDs for both PF4 and PF0-3 so our PCI probe() routine is
220  * called for both.
221  */
222 #define CH_PCI_DEVICE_ID_FUNCTION2 0x0
223 
224 #define CH_PCI_ID_TABLE_ENTRY(devid) \
225 		{PCI_VDEVICE(CHELSIO, (devid)), 4}
226 
227 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
228 		{ 0, } \
229 	}
230 
231 #include "t4_pci_id_tbl.h"
232 
233 #define FW4_FNAME "cxgb4/t4fw.bin"
234 #define FW5_FNAME "cxgb4/t5fw.bin"
235 #define FW4_CFNAME "cxgb4/t4-config.txt"
236 #define FW5_CFNAME "cxgb4/t5-config.txt"
237 
238 MODULE_DESCRIPTION(DRV_DESC);
239 MODULE_AUTHOR("Chelsio Communications");
240 MODULE_LICENSE("Dual BSD/GPL");
241 MODULE_VERSION(DRV_VERSION);
242 MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
243 MODULE_FIRMWARE(FW4_FNAME);
244 MODULE_FIRMWARE(FW5_FNAME);
245 
246 /*
247  * Normally we're willing to become the firmware's Master PF but will be happy
248  * if another PF has already become the Master and initialized the adapter.
249  * Setting "force_init" will cause this driver to forcibly establish itself as
250  * the Master PF and initialize the adapter.
251  */
252 static uint force_init;
253 
254 module_param(force_init, uint, 0644);
255 MODULE_PARM_DESC(force_init, "Forcibly become Master PF and initialize adapter");
256 
257 /*
258  * Normally if the firmware we connect to has Configuration File support, we
259  * use that and only fall back to the old Driver-based initialization if the
260  * Configuration File fails for some reason.  If force_old_init is set, then
261  * we'll always use the old Driver-based initialization sequence.
262  */
263 static uint force_old_init;
264 
265 module_param(force_old_init, uint, 0644);
266 MODULE_PARM_DESC(force_old_init, "Force old initialization sequence");
267 
268 static int dflt_msg_enable = DFLT_MSG_ENABLE;
269 
270 module_param(dflt_msg_enable, int, 0644);
271 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
272 
273 /*
274  * The driver uses the best interrupt scheme available on a platform in the
275  * order MSI-X, MSI, legacy INTx interrupts.  This parameter determines which
276  * of these schemes the driver may consider as follows:
277  *
278  * msi = 2: choose from among all three options
279  * msi = 1: only consider MSI and INTx interrupts
280  * msi = 0: force INTx interrupts
281  */
282 static int msi = 2;
283 
284 module_param(msi, int, 0644);
285 MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
286 
287 /*
288  * Queue interrupt hold-off timer values.  Queues default to the first of these
289  * upon creation.
290  */
291 static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
292 
293 module_param_array(intr_holdoff, uint, NULL, 0644);
294 MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
295 		 "0..4 in microseconds");
296 
297 static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
298 
299 module_param_array(intr_cnt, uint, NULL, 0644);
300 MODULE_PARM_DESC(intr_cnt,
301 		 "thresholds 1..3 for queue interrupt packet counters");
302 
303 /*
304  * Normally we tell the chip to deliver Ingress Packets into our DMA buffers
305  * offset by 2 bytes in order to have the IP headers line up on 4-byte
306  * boundaries.  This is a requirement for many architectures which will throw
307  * a machine check fault if an attempt is made to access one of the 4-byte IP
308  * header fields on a non-4-byte boundary.  And it's a major performance issue
309  * even on some architectures which allow it like some implementations of the
310  * x86 ISA.  However, some architectures don't mind this and for some very
311  * edge-case performance sensitive applications (like forwarding large volumes
312  * of small packets), setting this DMA offset to 0 will decrease the number of
313  * PCI-E Bus transfers enough to measurably affect performance.
314  */
315 static int rx_dma_offset = 2;
316 
317 static bool vf_acls;
318 
319 #ifdef CONFIG_PCI_IOV
320 module_param(vf_acls, bool, 0644);
321 MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
322 
323 /* Configure the number of PCI-E Virtual Function which are to be instantiated
324  * on SR-IOV Capable Physical Functions.
325  */
326 static unsigned int num_vf[NUM_OF_PF_WITH_SRIOV];
327 
328 module_param_array(num_vf, uint, NULL, 0644);
329 MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
330 #endif
331 
332 /* TX Queue select used to determine what algorithm to use for selecting TX
333  * queue. Select between the kernel provided function (select_queue=0) or user
334  * cxgb_select_queue function (select_queue=1)
335  *
336  * Default: select_queue=0
337  */
338 static int select_queue;
339 module_param(select_queue, int, 0644);
340 MODULE_PARM_DESC(select_queue,
341 		 "Select between kernel provided method of selecting or driver method of selecting TX queue. Default is kernel method.");
342 
343 /*
344  * The filter TCAM has a fixed portion and a variable portion.  The fixed
345  * portion can match on source/destination IP IPv4/IPv6 addresses and TCP/UDP
346  * ports.  The variable portion is 36 bits which can include things like Exact
347  * Match MAC Index (9 bits), Ether Type (16 bits), IP Protocol (8 bits),
348  * [Inner] VLAN Tag (17 bits), etc. which, if all were somehow selected, would
349  * far exceed the 36-bit budget for this "compressed" header portion of the
350  * filter.  Thus, we have a scarce resource which must be carefully managed.
351  *
352  * By default we set this up to mostly match the set of filter matching
353  * capabilities of T3 but with accommodations for some of T4's more
354  * interesting features:
355  *
356  *   { IP Fragment (1), MPS Match Type (3), IP Protocol (8),
357  *     [Inner] VLAN (17), Port (3), FCoE (1) }
358  */
359 enum {
360 	TP_VLAN_PRI_MAP_DEFAULT = HW_TPL_FR_MT_PR_IV_P_FC,
361 	TP_VLAN_PRI_MAP_FIRST = FCOE_SHIFT,
362 	TP_VLAN_PRI_MAP_LAST = FRAGMENTATION_SHIFT,
363 };
364 
365 static unsigned int tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
366 
367 module_param(tp_vlan_pri_map, uint, 0644);
368 MODULE_PARM_DESC(tp_vlan_pri_map, "global compressed filter configuration");
369 
370 static struct dentry *cxgb4_debugfs_root;
371 
372 static LIST_HEAD(adapter_list);
373 static DEFINE_MUTEX(uld_mutex);
374 /* Adapter list to be accessed from atomic context */
375 static LIST_HEAD(adap_rcu_list);
376 static DEFINE_SPINLOCK(adap_rcu_lock);
377 static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
378 static const char *uld_str[] = { "RDMA", "iSCSI" };
379 
380 static void link_report(struct net_device *dev)
381 {
382 	if (!netif_carrier_ok(dev))
383 		netdev_info(dev, "link down\n");
384 	else {
385 		static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
386 
387 		const char *s = "10Mbps";
388 		const struct port_info *p = netdev_priv(dev);
389 
390 		switch (p->link_cfg.speed) {
391 		case 10000:
392 			s = "10Gbps";
393 			break;
394 		case 1000:
395 			s = "1000Mbps";
396 			break;
397 		case 100:
398 			s = "100Mbps";
399 			break;
400 		case 40000:
401 			s = "40Gbps";
402 			break;
403 		}
404 
405 		netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
406 			    fc[p->link_cfg.fc]);
407 	}
408 }
409 
410 #ifdef CONFIG_CHELSIO_T4_DCB
411 /* Set up/tear down Data Center Bridging Priority mapping for a net device. */
412 static void dcb_tx_queue_prio_enable(struct net_device *dev, int enable)
413 {
414 	struct port_info *pi = netdev_priv(dev);
415 	struct adapter *adap = pi->adapter;
416 	struct sge_eth_txq *txq = &adap->sge.ethtxq[pi->first_qset];
417 	int i;
418 
419 	/* We use a simple mapping of Port TX Queue Index to DCB
420 	 * Priority when we're enabling DCB.
421 	 */
422 	for (i = 0; i < pi->nqsets; i++, txq++) {
423 		u32 name, value;
424 		int err;
425 
426 		name = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
427 			FW_PARAMS_PARAM_X_V(
428 				FW_PARAMS_PARAM_DMAQ_EQ_DCBPRIO_ETH) |
429 			FW_PARAMS_PARAM_YZ_V(txq->q.cntxt_id));
430 		value = enable ? i : 0xffffffff;
431 
432 		/* Since we can be called while atomic (from "interrupt
433 		 * level") we need to issue the Set Parameters Commannd
434 		 * without sleeping (timeout < 0).
435 		 */
436 		err = t4_set_params_nosleep(adap, adap->mbox, adap->fn, 0, 1,
437 					    &name, &value);
438 
439 		if (err)
440 			dev_err(adap->pdev_dev,
441 				"Can't %s DCB Priority on port %d, TX Queue %d: err=%d\n",
442 				enable ? "set" : "unset", pi->port_id, i, -err);
443 		else
444 			txq->dcb_prio = value;
445 	}
446 }
447 #endif /* CONFIG_CHELSIO_T4_DCB */
448 
449 void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
450 {
451 	struct net_device *dev = adapter->port[port_id];
452 
453 	/* Skip changes from disabled ports. */
454 	if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
455 		if (link_stat)
456 			netif_carrier_on(dev);
457 		else {
458 #ifdef CONFIG_CHELSIO_T4_DCB
459 			cxgb4_dcb_state_init(dev);
460 			dcb_tx_queue_prio_enable(dev, false);
461 #endif /* CONFIG_CHELSIO_T4_DCB */
462 			netif_carrier_off(dev);
463 		}
464 
465 		link_report(dev);
466 	}
467 }
468 
469 void t4_os_portmod_changed(const struct adapter *adap, int port_id)
470 {
471 	static const char *mod_str[] = {
472 		NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
473 	};
474 
475 	const struct net_device *dev = adap->port[port_id];
476 	const struct port_info *pi = netdev_priv(dev);
477 
478 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
479 		netdev_info(dev, "port module unplugged\n");
480 	else if (pi->mod_type < ARRAY_SIZE(mod_str))
481 		netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
482 }
483 
484 /*
485  * Configure the exact and hash address filters to handle a port's multicast
486  * and secondary unicast MAC addresses.
487  */
488 static int set_addr_filters(const struct net_device *dev, bool sleep)
489 {
490 	u64 mhash = 0;
491 	u64 uhash = 0;
492 	bool free = true;
493 	u16 filt_idx[7];
494 	const u8 *addr[7];
495 	int ret, naddr = 0;
496 	const struct netdev_hw_addr *ha;
497 	int uc_cnt = netdev_uc_count(dev);
498 	int mc_cnt = netdev_mc_count(dev);
499 	const struct port_info *pi = netdev_priv(dev);
500 	unsigned int mb = pi->adapter->fn;
501 
502 	/* first do the secondary unicast addresses */
503 	netdev_for_each_uc_addr(ha, dev) {
504 		addr[naddr++] = ha->addr;
505 		if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
506 			ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
507 					naddr, addr, filt_idx, &uhash, sleep);
508 			if (ret < 0)
509 				return ret;
510 
511 			free = false;
512 			naddr = 0;
513 		}
514 	}
515 
516 	/* next set up the multicast addresses */
517 	netdev_for_each_mc_addr(ha, dev) {
518 		addr[naddr++] = ha->addr;
519 		if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
520 			ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
521 					naddr, addr, filt_idx, &mhash, sleep);
522 			if (ret < 0)
523 				return ret;
524 
525 			free = false;
526 			naddr = 0;
527 		}
528 	}
529 
530 	return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
531 				uhash | mhash, sleep);
532 }
533 
534 int dbfifo_int_thresh = 10; /* 10 == 640 entry threshold */
535 module_param(dbfifo_int_thresh, int, 0644);
536 MODULE_PARM_DESC(dbfifo_int_thresh, "doorbell fifo interrupt threshold");
537 
538 /*
539  * usecs to sleep while draining the dbfifo
540  */
541 static int dbfifo_drain_delay = 1000;
542 module_param(dbfifo_drain_delay, int, 0644);
543 MODULE_PARM_DESC(dbfifo_drain_delay,
544 		 "usecs to sleep while draining the dbfifo");
545 
546 /*
547  * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
548  * If @mtu is -1 it is left unchanged.
549  */
550 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
551 {
552 	int ret;
553 	struct port_info *pi = netdev_priv(dev);
554 
555 	ret = set_addr_filters(dev, sleep_ok);
556 	if (ret == 0)
557 		ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
558 				    (dev->flags & IFF_PROMISC) ? 1 : 0,
559 				    (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
560 				    sleep_ok);
561 	return ret;
562 }
563 
564 /**
565  *	link_start - enable a port
566  *	@dev: the port to enable
567  *
568  *	Performs the MAC and PHY actions needed to enable a port.
569  */
570 static int link_start(struct net_device *dev)
571 {
572 	int ret;
573 	struct port_info *pi = netdev_priv(dev);
574 	unsigned int mb = pi->adapter->fn;
575 
576 	/*
577 	 * We do not set address filters and promiscuity here, the stack does
578 	 * that step explicitly.
579 	 */
580 	ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
581 			    !!(dev->features & NETIF_F_HW_VLAN_CTAG_RX), true);
582 	if (ret == 0) {
583 		ret = t4_change_mac(pi->adapter, mb, pi->viid,
584 				    pi->xact_addr_filt, dev->dev_addr, true,
585 				    true);
586 		if (ret >= 0) {
587 			pi->xact_addr_filt = ret;
588 			ret = 0;
589 		}
590 	}
591 	if (ret == 0)
592 		ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
593 				    &pi->link_cfg);
594 	if (ret == 0) {
595 		local_bh_disable();
596 		ret = t4_enable_vi_params(pi->adapter, mb, pi->viid, true,
597 					  true, CXGB4_DCB_ENABLED);
598 		local_bh_enable();
599 	}
600 
601 	return ret;
602 }
603 
604 int cxgb4_dcb_enabled(const struct net_device *dev)
605 {
606 #ifdef CONFIG_CHELSIO_T4_DCB
607 	struct port_info *pi = netdev_priv(dev);
608 
609 	if (!pi->dcb.enabled)
610 		return 0;
611 
612 	return ((pi->dcb.state == CXGB4_DCB_STATE_FW_ALLSYNCED) ||
613 		(pi->dcb.state == CXGB4_DCB_STATE_HOST));
614 #else
615 	return 0;
616 #endif
617 }
618 EXPORT_SYMBOL(cxgb4_dcb_enabled);
619 
620 #ifdef CONFIG_CHELSIO_T4_DCB
621 /* Handle a Data Center Bridging update message from the firmware. */
622 static void dcb_rpl(struct adapter *adap, const struct fw_port_cmd *pcmd)
623 {
624 	int port = FW_PORT_CMD_PORTID_G(ntohl(pcmd->op_to_portid));
625 	struct net_device *dev = adap->port[port];
626 	int old_dcb_enabled = cxgb4_dcb_enabled(dev);
627 	int new_dcb_enabled;
628 
629 	cxgb4_dcb_handle_fw_update(adap, pcmd);
630 	new_dcb_enabled = cxgb4_dcb_enabled(dev);
631 
632 	/* If the DCB has become enabled or disabled on the port then we're
633 	 * going to need to set up/tear down DCB Priority parameters for the
634 	 * TX Queues associated with the port.
635 	 */
636 	if (new_dcb_enabled != old_dcb_enabled)
637 		dcb_tx_queue_prio_enable(dev, new_dcb_enabled);
638 }
639 #endif /* CONFIG_CHELSIO_T4_DCB */
640 
641 /* Clear a filter and release any of its resources that we own.  This also
642  * clears the filter's "pending" status.
643  */
644 static void clear_filter(struct adapter *adap, struct filter_entry *f)
645 {
646 	/* If the new or old filter have loopback rewriteing rules then we'll
647 	 * need to free any existing Layer Two Table (L2T) entries of the old
648 	 * filter rule.  The firmware will handle freeing up any Source MAC
649 	 * Table (SMT) entries used for rewriting Source MAC Addresses in
650 	 * loopback rules.
651 	 */
652 	if (f->l2t)
653 		cxgb4_l2t_release(f->l2t);
654 
655 	/* The zeroing of the filter rule below clears the filter valid,
656 	 * pending, locked flags, l2t pointer, etc. so it's all we need for
657 	 * this operation.
658 	 */
659 	memset(f, 0, sizeof(*f));
660 }
661 
662 /* Handle a filter write/deletion reply.
663  */
664 static void filter_rpl(struct adapter *adap, const struct cpl_set_tcb_rpl *rpl)
665 {
666 	unsigned int idx = GET_TID(rpl);
667 	unsigned int nidx = idx - adap->tids.ftid_base;
668 	unsigned int ret;
669 	struct filter_entry *f;
670 
671 	if (idx >= adap->tids.ftid_base && nidx <
672 	   (adap->tids.nftids + adap->tids.nsftids)) {
673 		idx = nidx;
674 		ret = GET_TCB_COOKIE(rpl->cookie);
675 		f = &adap->tids.ftid_tab[idx];
676 
677 		if (ret == FW_FILTER_WR_FLT_DELETED) {
678 			/* Clear the filter when we get confirmation from the
679 			 * hardware that the filter has been deleted.
680 			 */
681 			clear_filter(adap, f);
682 		} else if (ret == FW_FILTER_WR_SMT_TBL_FULL) {
683 			dev_err(adap->pdev_dev, "filter %u setup failed due to full SMT\n",
684 				idx);
685 			clear_filter(adap, f);
686 		} else if (ret == FW_FILTER_WR_FLT_ADDED) {
687 			f->smtidx = (be64_to_cpu(rpl->oldval) >> 24) & 0xff;
688 			f->pending = 0;  /* asynchronous setup completed */
689 			f->valid = 1;
690 		} else {
691 			/* Something went wrong.  Issue a warning about the
692 			 * problem and clear everything out.
693 			 */
694 			dev_err(adap->pdev_dev, "filter %u setup failed with error %u\n",
695 				idx, ret);
696 			clear_filter(adap, f);
697 		}
698 	}
699 }
700 
701 /* Response queue handler for the FW event queue.
702  */
703 static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
704 			  const struct pkt_gl *gl)
705 {
706 	u8 opcode = ((const struct rss_header *)rsp)->opcode;
707 
708 	rsp++;                                          /* skip RSS header */
709 
710 	/* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
711 	 */
712 	if (unlikely(opcode == CPL_FW4_MSG &&
713 	   ((const struct cpl_fw4_msg *)rsp)->type == FW_TYPE_RSSCPL)) {
714 		rsp++;
715 		opcode = ((const struct rss_header *)rsp)->opcode;
716 		rsp++;
717 		if (opcode != CPL_SGE_EGR_UPDATE) {
718 			dev_err(q->adap->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
719 				, opcode);
720 			goto out;
721 		}
722 	}
723 
724 	if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
725 		const struct cpl_sge_egr_update *p = (void *)rsp;
726 		unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
727 		struct sge_txq *txq;
728 
729 		txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
730 		txq->restarts++;
731 		if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
732 			struct sge_eth_txq *eq;
733 
734 			eq = container_of(txq, struct sge_eth_txq, q);
735 			netif_tx_wake_queue(eq->txq);
736 		} else {
737 			struct sge_ofld_txq *oq;
738 
739 			oq = container_of(txq, struct sge_ofld_txq, q);
740 			tasklet_schedule(&oq->qresume_tsk);
741 		}
742 	} else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
743 		const struct cpl_fw6_msg *p = (void *)rsp;
744 
745 #ifdef CONFIG_CHELSIO_T4_DCB
746 		const struct fw_port_cmd *pcmd = (const void *)p->data;
747 		unsigned int cmd = FW_CMD_OP_G(ntohl(pcmd->op_to_portid));
748 		unsigned int action =
749 			FW_PORT_CMD_ACTION_G(ntohl(pcmd->action_to_len16));
750 
751 		if (cmd == FW_PORT_CMD &&
752 		    action == FW_PORT_ACTION_GET_PORT_INFO) {
753 			int port = FW_PORT_CMD_PORTID_G(
754 					be32_to_cpu(pcmd->op_to_portid));
755 			struct net_device *dev = q->adap->port[port];
756 			int state_input = ((pcmd->u.info.dcbxdis_pkd &
757 					    FW_PORT_CMD_DCBXDIS_F)
758 					   ? CXGB4_DCB_INPUT_FW_DISABLED
759 					   : CXGB4_DCB_INPUT_FW_ENABLED);
760 
761 			cxgb4_dcb_state_fsm(dev, state_input);
762 		}
763 
764 		if (cmd == FW_PORT_CMD &&
765 		    action == FW_PORT_ACTION_L2_DCB_CFG)
766 			dcb_rpl(q->adap, pcmd);
767 		else
768 #endif
769 			if (p->type == 0)
770 				t4_handle_fw_rpl(q->adap, p->data);
771 	} else if (opcode == CPL_L2T_WRITE_RPL) {
772 		const struct cpl_l2t_write_rpl *p = (void *)rsp;
773 
774 		do_l2t_write_rpl(q->adap, p);
775 	} else if (opcode == CPL_SET_TCB_RPL) {
776 		const struct cpl_set_tcb_rpl *p = (void *)rsp;
777 
778 		filter_rpl(q->adap, p);
779 	} else
780 		dev_err(q->adap->pdev_dev,
781 			"unexpected CPL %#x on FW event queue\n", opcode);
782 out:
783 	return 0;
784 }
785 
786 /**
787  *	uldrx_handler - response queue handler for ULD queues
788  *	@q: the response queue that received the packet
789  *	@rsp: the response queue descriptor holding the offload message
790  *	@gl: the gather list of packet fragments
791  *
792  *	Deliver an ingress offload packet to a ULD.  All processing is done by
793  *	the ULD, we just maintain statistics.
794  */
795 static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
796 			 const struct pkt_gl *gl)
797 {
798 	struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
799 
800 	/* FW can send CPLs encapsulated in a CPL_FW4_MSG.
801 	 */
802 	if (((const struct rss_header *)rsp)->opcode == CPL_FW4_MSG &&
803 	    ((const struct cpl_fw4_msg *)(rsp + 1))->type == FW_TYPE_RSSCPL)
804 		rsp += 2;
805 
806 	if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
807 		rxq->stats.nomem++;
808 		return -1;
809 	}
810 	if (gl == NULL)
811 		rxq->stats.imm++;
812 	else if (gl == CXGB4_MSG_AN)
813 		rxq->stats.an++;
814 	else
815 		rxq->stats.pkts++;
816 	return 0;
817 }
818 
819 static void disable_msi(struct adapter *adapter)
820 {
821 	if (adapter->flags & USING_MSIX) {
822 		pci_disable_msix(adapter->pdev);
823 		adapter->flags &= ~USING_MSIX;
824 	} else if (adapter->flags & USING_MSI) {
825 		pci_disable_msi(adapter->pdev);
826 		adapter->flags &= ~USING_MSI;
827 	}
828 }
829 
830 /*
831  * Interrupt handler for non-data events used with MSI-X.
832  */
833 static irqreturn_t t4_nondata_intr(int irq, void *cookie)
834 {
835 	struct adapter *adap = cookie;
836 
837 	u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
838 	if (v & PFSW) {
839 		adap->swintr = 1;
840 		t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
841 	}
842 	t4_slow_intr_handler(adap);
843 	return IRQ_HANDLED;
844 }
845 
846 /*
847  * Name the MSI-X interrupts.
848  */
849 static void name_msix_vecs(struct adapter *adap)
850 {
851 	int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
852 
853 	/* non-data interrupts */
854 	snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
855 
856 	/* FW events */
857 	snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
858 		 adap->port[0]->name);
859 
860 	/* Ethernet queues */
861 	for_each_port(adap, j) {
862 		struct net_device *d = adap->port[j];
863 		const struct port_info *pi = netdev_priv(d);
864 
865 		for (i = 0; i < pi->nqsets; i++, msi_idx++)
866 			snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
867 				 d->name, i);
868 	}
869 
870 	/* offload queues */
871 	for_each_ofldrxq(&adap->sge, i)
872 		snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
873 			 adap->port[0]->name, i);
874 
875 	for_each_rdmarxq(&adap->sge, i)
876 		snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
877 			 adap->port[0]->name, i);
878 
879 	for_each_rdmaciq(&adap->sge, i)
880 		snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma-ciq%d",
881 			 adap->port[0]->name, i);
882 }
883 
884 static int request_msix_queue_irqs(struct adapter *adap)
885 {
886 	struct sge *s = &adap->sge;
887 	int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, rdmaciqqidx = 0;
888 	int msi_index = 2;
889 
890 	err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
891 			  adap->msix_info[1].desc, &s->fw_evtq);
892 	if (err)
893 		return err;
894 
895 	for_each_ethrxq(s, ethqidx) {
896 		err = request_irq(adap->msix_info[msi_index].vec,
897 				  t4_sge_intr_msix, 0,
898 				  adap->msix_info[msi_index].desc,
899 				  &s->ethrxq[ethqidx].rspq);
900 		if (err)
901 			goto unwind;
902 		msi_index++;
903 	}
904 	for_each_ofldrxq(s, ofldqidx) {
905 		err = request_irq(adap->msix_info[msi_index].vec,
906 				  t4_sge_intr_msix, 0,
907 				  adap->msix_info[msi_index].desc,
908 				  &s->ofldrxq[ofldqidx].rspq);
909 		if (err)
910 			goto unwind;
911 		msi_index++;
912 	}
913 	for_each_rdmarxq(s, rdmaqidx) {
914 		err = request_irq(adap->msix_info[msi_index].vec,
915 				  t4_sge_intr_msix, 0,
916 				  adap->msix_info[msi_index].desc,
917 				  &s->rdmarxq[rdmaqidx].rspq);
918 		if (err)
919 			goto unwind;
920 		msi_index++;
921 	}
922 	for_each_rdmaciq(s, rdmaciqqidx) {
923 		err = request_irq(adap->msix_info[msi_index].vec,
924 				  t4_sge_intr_msix, 0,
925 				  adap->msix_info[msi_index].desc,
926 				  &s->rdmaciq[rdmaciqqidx].rspq);
927 		if (err)
928 			goto unwind;
929 		msi_index++;
930 	}
931 	return 0;
932 
933 unwind:
934 	while (--rdmaciqqidx >= 0)
935 		free_irq(adap->msix_info[--msi_index].vec,
936 			 &s->rdmaciq[rdmaciqqidx].rspq);
937 	while (--rdmaqidx >= 0)
938 		free_irq(adap->msix_info[--msi_index].vec,
939 			 &s->rdmarxq[rdmaqidx].rspq);
940 	while (--ofldqidx >= 0)
941 		free_irq(adap->msix_info[--msi_index].vec,
942 			 &s->ofldrxq[ofldqidx].rspq);
943 	while (--ethqidx >= 0)
944 		free_irq(adap->msix_info[--msi_index].vec,
945 			 &s->ethrxq[ethqidx].rspq);
946 	free_irq(adap->msix_info[1].vec, &s->fw_evtq);
947 	return err;
948 }
949 
950 static void free_msix_queue_irqs(struct adapter *adap)
951 {
952 	int i, msi_index = 2;
953 	struct sge *s = &adap->sge;
954 
955 	free_irq(adap->msix_info[1].vec, &s->fw_evtq);
956 	for_each_ethrxq(s, i)
957 		free_irq(adap->msix_info[msi_index++].vec, &s->ethrxq[i].rspq);
958 	for_each_ofldrxq(s, i)
959 		free_irq(adap->msix_info[msi_index++].vec, &s->ofldrxq[i].rspq);
960 	for_each_rdmarxq(s, i)
961 		free_irq(adap->msix_info[msi_index++].vec, &s->rdmarxq[i].rspq);
962 	for_each_rdmaciq(s, i)
963 		free_irq(adap->msix_info[msi_index++].vec, &s->rdmaciq[i].rspq);
964 }
965 
966 /**
967  *	write_rss - write the RSS table for a given port
968  *	@pi: the port
969  *	@queues: array of queue indices for RSS
970  *
971  *	Sets up the portion of the HW RSS table for the port's VI to distribute
972  *	packets to the Rx queues in @queues.
973  */
974 static int write_rss(const struct port_info *pi, const u16 *queues)
975 {
976 	u16 *rss;
977 	int i, err;
978 	const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
979 
980 	rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
981 	if (!rss)
982 		return -ENOMEM;
983 
984 	/* map the queue indices to queue ids */
985 	for (i = 0; i < pi->rss_size; i++, queues++)
986 		rss[i] = q[*queues].rspq.abs_id;
987 
988 	err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
989 				  pi->rss_size, rss, pi->rss_size);
990 	kfree(rss);
991 	return err;
992 }
993 
994 /**
995  *	setup_rss - configure RSS
996  *	@adap: the adapter
997  *
998  *	Sets up RSS for each port.
999  */
1000 static int setup_rss(struct adapter *adap)
1001 {
1002 	int i, err;
1003 
1004 	for_each_port(adap, i) {
1005 		const struct port_info *pi = adap2pinfo(adap, i);
1006 
1007 		err = write_rss(pi, pi->rss);
1008 		if (err)
1009 			return err;
1010 	}
1011 	return 0;
1012 }
1013 
1014 /*
1015  * Return the channel of the ingress queue with the given qid.
1016  */
1017 static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
1018 {
1019 	qid -= p->ingr_start;
1020 	return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
1021 }
1022 
1023 /*
1024  * Wait until all NAPI handlers are descheduled.
1025  */
1026 static void quiesce_rx(struct adapter *adap)
1027 {
1028 	int i;
1029 
1030 	for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
1031 		struct sge_rspq *q = adap->sge.ingr_map[i];
1032 
1033 		if (q && q->handler)
1034 			napi_disable(&q->napi);
1035 	}
1036 }
1037 
1038 /*
1039  * Enable NAPI scheduling and interrupt generation for all Rx queues.
1040  */
1041 static void enable_rx(struct adapter *adap)
1042 {
1043 	int i;
1044 
1045 	for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
1046 		struct sge_rspq *q = adap->sge.ingr_map[i];
1047 
1048 		if (!q)
1049 			continue;
1050 		if (q->handler)
1051 			napi_enable(&q->napi);
1052 		/* 0-increment GTS to start the timer and enable interrupts */
1053 		t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
1054 			     SEINTARM(q->intr_params) |
1055 			     INGRESSQID(q->cntxt_id));
1056 	}
1057 }
1058 
1059 /**
1060  *	setup_sge_queues - configure SGE Tx/Rx/response queues
1061  *	@adap: the adapter
1062  *
1063  *	Determines how many sets of SGE queues to use and initializes them.
1064  *	We support multiple queue sets per port if we have MSI-X, otherwise
1065  *	just one queue set per port.
1066  */
1067 static int setup_sge_queues(struct adapter *adap)
1068 {
1069 	int err, msi_idx, i, j;
1070 	struct sge *s = &adap->sge;
1071 
1072 	bitmap_zero(s->starving_fl, MAX_EGRQ);
1073 	bitmap_zero(s->txq_maperr, MAX_EGRQ);
1074 
1075 	if (adap->flags & USING_MSIX)
1076 		msi_idx = 1;         /* vector 0 is for non-queue interrupts */
1077 	else {
1078 		err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
1079 				       NULL, NULL);
1080 		if (err)
1081 			return err;
1082 		msi_idx = -((int)s->intrq.abs_id + 1);
1083 	}
1084 
1085 	err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
1086 			       msi_idx, NULL, fwevtq_handler);
1087 	if (err) {
1088 freeout:	t4_free_sge_resources(adap);
1089 		return err;
1090 	}
1091 
1092 	for_each_port(adap, i) {
1093 		struct net_device *dev = adap->port[i];
1094 		struct port_info *pi = netdev_priv(dev);
1095 		struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
1096 		struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
1097 
1098 		for (j = 0; j < pi->nqsets; j++, q++) {
1099 			if (msi_idx > 0)
1100 				msi_idx++;
1101 			err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
1102 					       msi_idx, &q->fl,
1103 					       t4_ethrx_handler);
1104 			if (err)
1105 				goto freeout;
1106 			q->rspq.idx = j;
1107 			memset(&q->stats, 0, sizeof(q->stats));
1108 		}
1109 		for (j = 0; j < pi->nqsets; j++, t++) {
1110 			err = t4_sge_alloc_eth_txq(adap, t, dev,
1111 					netdev_get_tx_queue(dev, j),
1112 					s->fw_evtq.cntxt_id);
1113 			if (err)
1114 				goto freeout;
1115 		}
1116 	}
1117 
1118 	j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
1119 	for_each_ofldrxq(s, i) {
1120 		struct sge_ofld_rxq *q = &s->ofldrxq[i];
1121 		struct net_device *dev = adap->port[i / j];
1122 
1123 		if (msi_idx > 0)
1124 			msi_idx++;
1125 		err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
1126 				       q->fl.size ? &q->fl : NULL,
1127 				       uldrx_handler);
1128 		if (err)
1129 			goto freeout;
1130 		memset(&q->stats, 0, sizeof(q->stats));
1131 		s->ofld_rxq[i] = q->rspq.abs_id;
1132 		err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
1133 					    s->fw_evtq.cntxt_id);
1134 		if (err)
1135 			goto freeout;
1136 	}
1137 
1138 	for_each_rdmarxq(s, i) {
1139 		struct sge_ofld_rxq *q = &s->rdmarxq[i];
1140 
1141 		if (msi_idx > 0)
1142 			msi_idx++;
1143 		err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1144 				       msi_idx, q->fl.size ? &q->fl : NULL,
1145 				       uldrx_handler);
1146 		if (err)
1147 			goto freeout;
1148 		memset(&q->stats, 0, sizeof(q->stats));
1149 		s->rdma_rxq[i] = q->rspq.abs_id;
1150 	}
1151 
1152 	for_each_rdmaciq(s, i) {
1153 		struct sge_ofld_rxq *q = &s->rdmaciq[i];
1154 
1155 		if (msi_idx > 0)
1156 			msi_idx++;
1157 		err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
1158 				       msi_idx, q->fl.size ? &q->fl : NULL,
1159 				       uldrx_handler);
1160 		if (err)
1161 			goto freeout;
1162 		memset(&q->stats, 0, sizeof(q->stats));
1163 		s->rdma_ciq[i] = q->rspq.abs_id;
1164 	}
1165 
1166 	for_each_port(adap, i) {
1167 		/*
1168 		 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
1169 		 * have RDMA queues, and that's the right value.
1170 		 */
1171 		err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
1172 					    s->fw_evtq.cntxt_id,
1173 					    s->rdmarxq[i].rspq.cntxt_id);
1174 		if (err)
1175 			goto freeout;
1176 	}
1177 
1178 	t4_write_reg(adap, is_t4(adap->params.chip) ?
1179 				MPS_TRC_RSS_CONTROL :
1180 				MPS_T5_TRC_RSS_CONTROL,
1181 		     RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
1182 		     QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
1183 	return 0;
1184 }
1185 
1186 /*
1187  * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
1188  * The allocated memory is cleared.
1189  */
1190 void *t4_alloc_mem(size_t size)
1191 {
1192 	void *p = kzalloc(size, GFP_KERNEL | __GFP_NOWARN);
1193 
1194 	if (!p)
1195 		p = vzalloc(size);
1196 	return p;
1197 }
1198 
1199 /*
1200  * Free memory allocated through alloc_mem().
1201  */
1202 void t4_free_mem(void *addr)
1203 {
1204 	if (is_vmalloc_addr(addr))
1205 		vfree(addr);
1206 	else
1207 		kfree(addr);
1208 }
1209 
1210 /* Send a Work Request to write the filter at a specified index.  We construct
1211  * a Firmware Filter Work Request to have the work done and put the indicated
1212  * filter into "pending" mode which will prevent any further actions against
1213  * it till we get a reply from the firmware on the completion status of the
1214  * request.
1215  */
1216 static int set_filter_wr(struct adapter *adapter, int fidx)
1217 {
1218 	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1219 	struct sk_buff *skb;
1220 	struct fw_filter_wr *fwr;
1221 	unsigned int ftid;
1222 
1223 	/* If the new filter requires loopback Destination MAC and/or VLAN
1224 	 * rewriting then we need to allocate a Layer 2 Table (L2T) entry for
1225 	 * the filter.
1226 	 */
1227 	if (f->fs.newdmac || f->fs.newvlan) {
1228 		/* allocate L2T entry for new filter */
1229 		f->l2t = t4_l2t_alloc_switching(adapter->l2t);
1230 		if (f->l2t == NULL)
1231 			return -EAGAIN;
1232 		if (t4_l2t_set_switching(adapter, f->l2t, f->fs.vlan,
1233 					f->fs.eport, f->fs.dmac)) {
1234 			cxgb4_l2t_release(f->l2t);
1235 			f->l2t = NULL;
1236 			return -ENOMEM;
1237 		}
1238 	}
1239 
1240 	ftid = adapter->tids.ftid_base + fidx;
1241 
1242 	skb = alloc_skb(sizeof(*fwr), GFP_KERNEL | __GFP_NOFAIL);
1243 	fwr = (struct fw_filter_wr *)__skb_put(skb, sizeof(*fwr));
1244 	memset(fwr, 0, sizeof(*fwr));
1245 
1246 	/* It would be nice to put most of the following in t4_hw.c but most
1247 	 * of the work is translating the cxgbtool ch_filter_specification
1248 	 * into the Work Request and the definition of that structure is
1249 	 * currently in cxgbtool.h which isn't appropriate to pull into the
1250 	 * common code.  We may eventually try to come up with a more neutral
1251 	 * filter specification structure but for now it's easiest to simply
1252 	 * put this fairly direct code in line ...
1253 	 */
1254 	fwr->op_pkd = htonl(FW_WR_OP_V(FW_FILTER_WR));
1255 	fwr->len16_pkd = htonl(FW_WR_LEN16_V(sizeof(*fwr)/16));
1256 	fwr->tid_to_iq =
1257 		htonl(FW_FILTER_WR_TID_V(ftid) |
1258 		      FW_FILTER_WR_RQTYPE_V(f->fs.type) |
1259 		      FW_FILTER_WR_NOREPLY_V(0) |
1260 		      FW_FILTER_WR_IQ_V(f->fs.iq));
1261 	fwr->del_filter_to_l2tix =
1262 		htonl(FW_FILTER_WR_RPTTID_V(f->fs.rpttid) |
1263 		      FW_FILTER_WR_DROP_V(f->fs.action == FILTER_DROP) |
1264 		      FW_FILTER_WR_DIRSTEER_V(f->fs.dirsteer) |
1265 		      FW_FILTER_WR_MASKHASH_V(f->fs.maskhash) |
1266 		      FW_FILTER_WR_DIRSTEERHASH_V(f->fs.dirsteerhash) |
1267 		      FW_FILTER_WR_LPBK_V(f->fs.action == FILTER_SWITCH) |
1268 		      FW_FILTER_WR_DMAC_V(f->fs.newdmac) |
1269 		      FW_FILTER_WR_SMAC_V(f->fs.newsmac) |
1270 		      FW_FILTER_WR_INSVLAN_V(f->fs.newvlan == VLAN_INSERT ||
1271 					     f->fs.newvlan == VLAN_REWRITE) |
1272 		      FW_FILTER_WR_RMVLAN_V(f->fs.newvlan == VLAN_REMOVE ||
1273 					    f->fs.newvlan == VLAN_REWRITE) |
1274 		      FW_FILTER_WR_HITCNTS_V(f->fs.hitcnts) |
1275 		      FW_FILTER_WR_TXCHAN_V(f->fs.eport) |
1276 		      FW_FILTER_WR_PRIO_V(f->fs.prio) |
1277 		      FW_FILTER_WR_L2TIX_V(f->l2t ? f->l2t->idx : 0));
1278 	fwr->ethtype = htons(f->fs.val.ethtype);
1279 	fwr->ethtypem = htons(f->fs.mask.ethtype);
1280 	fwr->frag_to_ovlan_vldm =
1281 		(FW_FILTER_WR_FRAG_V(f->fs.val.frag) |
1282 		 FW_FILTER_WR_FRAGM_V(f->fs.mask.frag) |
1283 		 FW_FILTER_WR_IVLAN_VLD_V(f->fs.val.ivlan_vld) |
1284 		 FW_FILTER_WR_OVLAN_VLD_V(f->fs.val.ovlan_vld) |
1285 		 FW_FILTER_WR_IVLAN_VLDM_V(f->fs.mask.ivlan_vld) |
1286 		 FW_FILTER_WR_OVLAN_VLDM_V(f->fs.mask.ovlan_vld));
1287 	fwr->smac_sel = 0;
1288 	fwr->rx_chan_rx_rpl_iq =
1289 		htons(FW_FILTER_WR_RX_CHAN_V(0) |
1290 		      FW_FILTER_WR_RX_RPL_IQ_V(adapter->sge.fw_evtq.abs_id));
1291 	fwr->maci_to_matchtypem =
1292 		htonl(FW_FILTER_WR_MACI_V(f->fs.val.macidx) |
1293 		      FW_FILTER_WR_MACIM_V(f->fs.mask.macidx) |
1294 		      FW_FILTER_WR_FCOE_V(f->fs.val.fcoe) |
1295 		      FW_FILTER_WR_FCOEM_V(f->fs.mask.fcoe) |
1296 		      FW_FILTER_WR_PORT_V(f->fs.val.iport) |
1297 		      FW_FILTER_WR_PORTM_V(f->fs.mask.iport) |
1298 		      FW_FILTER_WR_MATCHTYPE_V(f->fs.val.matchtype) |
1299 		      FW_FILTER_WR_MATCHTYPEM_V(f->fs.mask.matchtype));
1300 	fwr->ptcl = f->fs.val.proto;
1301 	fwr->ptclm = f->fs.mask.proto;
1302 	fwr->ttyp = f->fs.val.tos;
1303 	fwr->ttypm = f->fs.mask.tos;
1304 	fwr->ivlan = htons(f->fs.val.ivlan);
1305 	fwr->ivlanm = htons(f->fs.mask.ivlan);
1306 	fwr->ovlan = htons(f->fs.val.ovlan);
1307 	fwr->ovlanm = htons(f->fs.mask.ovlan);
1308 	memcpy(fwr->lip, f->fs.val.lip, sizeof(fwr->lip));
1309 	memcpy(fwr->lipm, f->fs.mask.lip, sizeof(fwr->lipm));
1310 	memcpy(fwr->fip, f->fs.val.fip, sizeof(fwr->fip));
1311 	memcpy(fwr->fipm, f->fs.mask.fip, sizeof(fwr->fipm));
1312 	fwr->lp = htons(f->fs.val.lport);
1313 	fwr->lpm = htons(f->fs.mask.lport);
1314 	fwr->fp = htons(f->fs.val.fport);
1315 	fwr->fpm = htons(f->fs.mask.fport);
1316 	if (f->fs.newsmac)
1317 		memcpy(fwr->sma, f->fs.smac, sizeof(fwr->sma));
1318 
1319 	/* Mark the filter as "pending" and ship off the Filter Work Request.
1320 	 * When we get the Work Request Reply we'll clear the pending status.
1321 	 */
1322 	f->pending = 1;
1323 	set_wr_txq(skb, CPL_PRIORITY_CONTROL, f->fs.val.iport & 0x3);
1324 	t4_ofld_send(adapter, skb);
1325 	return 0;
1326 }
1327 
1328 /* Delete the filter at a specified index.
1329  */
1330 static int del_filter_wr(struct adapter *adapter, int fidx)
1331 {
1332 	struct filter_entry *f = &adapter->tids.ftid_tab[fidx];
1333 	struct sk_buff *skb;
1334 	struct fw_filter_wr *fwr;
1335 	unsigned int len, ftid;
1336 
1337 	len = sizeof(*fwr);
1338 	ftid = adapter->tids.ftid_base + fidx;
1339 
1340 	skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
1341 	fwr = (struct fw_filter_wr *)__skb_put(skb, len);
1342 	t4_mk_filtdelwr(ftid, fwr, adapter->sge.fw_evtq.abs_id);
1343 
1344 	/* Mark the filter as "pending" and ship off the Filter Work Request.
1345 	 * When we get the Work Request Reply we'll clear the pending status.
1346 	 */
1347 	f->pending = 1;
1348 	t4_mgmt_tx(adapter, skb);
1349 	return 0;
1350 }
1351 
1352 static u16 cxgb_select_queue(struct net_device *dev, struct sk_buff *skb,
1353 			     void *accel_priv, select_queue_fallback_t fallback)
1354 {
1355 	int txq;
1356 
1357 #ifdef CONFIG_CHELSIO_T4_DCB
1358 	/* If a Data Center Bridging has been successfully negotiated on this
1359 	 * link then we'll use the skb's priority to map it to a TX Queue.
1360 	 * The skb's priority is determined via the VLAN Tag Priority Code
1361 	 * Point field.
1362 	 */
1363 	if (cxgb4_dcb_enabled(dev)) {
1364 		u16 vlan_tci;
1365 		int err;
1366 
1367 		err = vlan_get_tag(skb, &vlan_tci);
1368 		if (unlikely(err)) {
1369 			if (net_ratelimit())
1370 				netdev_warn(dev,
1371 					    "TX Packet without VLAN Tag on DCB Link\n");
1372 			txq = 0;
1373 		} else {
1374 			txq = (vlan_tci & VLAN_PRIO_MASK) >> VLAN_PRIO_SHIFT;
1375 		}
1376 		return txq;
1377 	}
1378 #endif /* CONFIG_CHELSIO_T4_DCB */
1379 
1380 	if (select_queue) {
1381 		txq = (skb_rx_queue_recorded(skb)
1382 			? skb_get_rx_queue(skb)
1383 			: smp_processor_id());
1384 
1385 		while (unlikely(txq >= dev->real_num_tx_queues))
1386 			txq -= dev->real_num_tx_queues;
1387 
1388 		return txq;
1389 	}
1390 
1391 	return fallback(dev, skb) % dev->real_num_tx_queues;
1392 }
1393 
1394 static inline int is_offload(const struct adapter *adap)
1395 {
1396 	return adap->params.offload;
1397 }
1398 
1399 /*
1400  * Implementation of ethtool operations.
1401  */
1402 
1403 static u32 get_msglevel(struct net_device *dev)
1404 {
1405 	return netdev2adap(dev)->msg_enable;
1406 }
1407 
1408 static void set_msglevel(struct net_device *dev, u32 val)
1409 {
1410 	netdev2adap(dev)->msg_enable = val;
1411 }
1412 
1413 static char stats_strings[][ETH_GSTRING_LEN] = {
1414 	"TxOctetsOK         ",
1415 	"TxFramesOK         ",
1416 	"TxBroadcastFrames  ",
1417 	"TxMulticastFrames  ",
1418 	"TxUnicastFrames    ",
1419 	"TxErrorFrames      ",
1420 
1421 	"TxFrames64         ",
1422 	"TxFrames65To127    ",
1423 	"TxFrames128To255   ",
1424 	"TxFrames256To511   ",
1425 	"TxFrames512To1023  ",
1426 	"TxFrames1024To1518 ",
1427 	"TxFrames1519ToMax  ",
1428 
1429 	"TxFramesDropped    ",
1430 	"TxPauseFrames      ",
1431 	"TxPPP0Frames       ",
1432 	"TxPPP1Frames       ",
1433 	"TxPPP2Frames       ",
1434 	"TxPPP3Frames       ",
1435 	"TxPPP4Frames       ",
1436 	"TxPPP5Frames       ",
1437 	"TxPPP6Frames       ",
1438 	"TxPPP7Frames       ",
1439 
1440 	"RxOctetsOK         ",
1441 	"RxFramesOK         ",
1442 	"RxBroadcastFrames  ",
1443 	"RxMulticastFrames  ",
1444 	"RxUnicastFrames    ",
1445 
1446 	"RxFramesTooLong    ",
1447 	"RxJabberErrors     ",
1448 	"RxFCSErrors        ",
1449 	"RxLengthErrors     ",
1450 	"RxSymbolErrors     ",
1451 	"RxRuntFrames       ",
1452 
1453 	"RxFrames64         ",
1454 	"RxFrames65To127    ",
1455 	"RxFrames128To255   ",
1456 	"RxFrames256To511   ",
1457 	"RxFrames512To1023  ",
1458 	"RxFrames1024To1518 ",
1459 	"RxFrames1519ToMax  ",
1460 
1461 	"RxPauseFrames      ",
1462 	"RxPPP0Frames       ",
1463 	"RxPPP1Frames       ",
1464 	"RxPPP2Frames       ",
1465 	"RxPPP3Frames       ",
1466 	"RxPPP4Frames       ",
1467 	"RxPPP5Frames       ",
1468 	"RxPPP6Frames       ",
1469 	"RxPPP7Frames       ",
1470 
1471 	"RxBG0FramesDropped ",
1472 	"RxBG1FramesDropped ",
1473 	"RxBG2FramesDropped ",
1474 	"RxBG3FramesDropped ",
1475 	"RxBG0FramesTrunc   ",
1476 	"RxBG1FramesTrunc   ",
1477 	"RxBG2FramesTrunc   ",
1478 	"RxBG3FramesTrunc   ",
1479 
1480 	"TSO                ",
1481 	"TxCsumOffload      ",
1482 	"RxCsumGood         ",
1483 	"VLANextractions    ",
1484 	"VLANinsertions     ",
1485 	"GROpackets         ",
1486 	"GROmerged          ",
1487 	"WriteCoalSuccess   ",
1488 	"WriteCoalFail      ",
1489 };
1490 
1491 static int get_sset_count(struct net_device *dev, int sset)
1492 {
1493 	switch (sset) {
1494 	case ETH_SS_STATS:
1495 		return ARRAY_SIZE(stats_strings);
1496 	default:
1497 		return -EOPNOTSUPP;
1498 	}
1499 }
1500 
1501 #define T4_REGMAP_SIZE (160 * 1024)
1502 #define T5_REGMAP_SIZE (332 * 1024)
1503 
1504 static int get_regs_len(struct net_device *dev)
1505 {
1506 	struct adapter *adap = netdev2adap(dev);
1507 	if (is_t4(adap->params.chip))
1508 		return T4_REGMAP_SIZE;
1509 	else
1510 		return T5_REGMAP_SIZE;
1511 }
1512 
1513 static int get_eeprom_len(struct net_device *dev)
1514 {
1515 	return EEPROMSIZE;
1516 }
1517 
1518 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1519 {
1520 	struct adapter *adapter = netdev2adap(dev);
1521 
1522 	strlcpy(info->driver, KBUILD_MODNAME, sizeof(info->driver));
1523 	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1524 	strlcpy(info->bus_info, pci_name(adapter->pdev),
1525 		sizeof(info->bus_info));
1526 
1527 	if (adapter->params.fw_vers)
1528 		snprintf(info->fw_version, sizeof(info->fw_version),
1529 			"%u.%u.%u.%u, TP %u.%u.%u.%u",
1530 			FW_HDR_FW_VER_MAJOR_G(adapter->params.fw_vers),
1531 			FW_HDR_FW_VER_MINOR_G(adapter->params.fw_vers),
1532 			FW_HDR_FW_VER_MICRO_G(adapter->params.fw_vers),
1533 			FW_HDR_FW_VER_BUILD_G(adapter->params.fw_vers),
1534 			FW_HDR_FW_VER_MAJOR_G(adapter->params.tp_vers),
1535 			FW_HDR_FW_VER_MINOR_G(adapter->params.tp_vers),
1536 			FW_HDR_FW_VER_MICRO_G(adapter->params.tp_vers),
1537 			FW_HDR_FW_VER_BUILD_G(adapter->params.tp_vers));
1538 }
1539 
1540 static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1541 {
1542 	if (stringset == ETH_SS_STATS)
1543 		memcpy(data, stats_strings, sizeof(stats_strings));
1544 }
1545 
1546 /*
1547  * port stats maintained per queue of the port.  They should be in the same
1548  * order as in stats_strings above.
1549  */
1550 struct queue_port_stats {
1551 	u64 tso;
1552 	u64 tx_csum;
1553 	u64 rx_csum;
1554 	u64 vlan_ex;
1555 	u64 vlan_ins;
1556 	u64 gro_pkts;
1557 	u64 gro_merged;
1558 };
1559 
1560 static void collect_sge_port_stats(const struct adapter *adap,
1561 		const struct port_info *p, struct queue_port_stats *s)
1562 {
1563 	int i;
1564 	const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1565 	const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1566 
1567 	memset(s, 0, sizeof(*s));
1568 	for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1569 		s->tso += tx->tso;
1570 		s->tx_csum += tx->tx_cso;
1571 		s->rx_csum += rx->stats.rx_cso;
1572 		s->vlan_ex += rx->stats.vlan_ex;
1573 		s->vlan_ins += tx->vlan_ins;
1574 		s->gro_pkts += rx->stats.lro_pkts;
1575 		s->gro_merged += rx->stats.lro_merged;
1576 	}
1577 }
1578 
1579 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1580 		      u64 *data)
1581 {
1582 	struct port_info *pi = netdev_priv(dev);
1583 	struct adapter *adapter = pi->adapter;
1584 	u32 val1, val2;
1585 
1586 	t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1587 
1588 	data += sizeof(struct port_stats) / sizeof(u64);
1589 	collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1590 	data += sizeof(struct queue_port_stats) / sizeof(u64);
1591 	if (!is_t4(adapter->params.chip)) {
1592 		t4_write_reg(adapter, SGE_STAT_CFG, STATSOURCE_T5(7));
1593 		val1 = t4_read_reg(adapter, SGE_STAT_TOTAL);
1594 		val2 = t4_read_reg(adapter, SGE_STAT_MATCH);
1595 		*data = val1 - val2;
1596 		data++;
1597 		*data = val2;
1598 		data++;
1599 	} else {
1600 		memset(data, 0, 2 * sizeof(u64));
1601 		*data += 2;
1602 	}
1603 }
1604 
1605 /*
1606  * Return a version number to identify the type of adapter.  The scheme is:
1607  * - bits 0..9: chip version
1608  * - bits 10..15: chip revision
1609  * - bits 16..23: register dump version
1610  */
1611 static inline unsigned int mk_adap_vers(const struct adapter *ap)
1612 {
1613 	return CHELSIO_CHIP_VERSION(ap->params.chip) |
1614 		(CHELSIO_CHIP_RELEASE(ap->params.chip) << 10) | (1 << 16);
1615 }
1616 
1617 static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1618 			   unsigned int end)
1619 {
1620 	u32 *p = buf + start;
1621 
1622 	for ( ; start <= end; start += sizeof(u32))
1623 		*p++ = t4_read_reg(ap, start);
1624 }
1625 
1626 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1627 		     void *buf)
1628 {
1629 	static const unsigned int t4_reg_ranges[] = {
1630 		0x1008, 0x1108,
1631 		0x1180, 0x11b4,
1632 		0x11fc, 0x123c,
1633 		0x1300, 0x173c,
1634 		0x1800, 0x18fc,
1635 		0x3000, 0x30d8,
1636 		0x30e0, 0x5924,
1637 		0x5960, 0x59d4,
1638 		0x5a00, 0x5af8,
1639 		0x6000, 0x6098,
1640 		0x6100, 0x6150,
1641 		0x6200, 0x6208,
1642 		0x6240, 0x6248,
1643 		0x6280, 0x6338,
1644 		0x6370, 0x638c,
1645 		0x6400, 0x643c,
1646 		0x6500, 0x6524,
1647 		0x6a00, 0x6a38,
1648 		0x6a60, 0x6a78,
1649 		0x6b00, 0x6b84,
1650 		0x6bf0, 0x6c84,
1651 		0x6cf0, 0x6d84,
1652 		0x6df0, 0x6e84,
1653 		0x6ef0, 0x6f84,
1654 		0x6ff0, 0x7084,
1655 		0x70f0, 0x7184,
1656 		0x71f0, 0x7284,
1657 		0x72f0, 0x7384,
1658 		0x73f0, 0x7450,
1659 		0x7500, 0x7530,
1660 		0x7600, 0x761c,
1661 		0x7680, 0x76cc,
1662 		0x7700, 0x7798,
1663 		0x77c0, 0x77fc,
1664 		0x7900, 0x79fc,
1665 		0x7b00, 0x7c38,
1666 		0x7d00, 0x7efc,
1667 		0x8dc0, 0x8e1c,
1668 		0x8e30, 0x8e78,
1669 		0x8ea0, 0x8f6c,
1670 		0x8fc0, 0x9074,
1671 		0x90fc, 0x90fc,
1672 		0x9400, 0x9458,
1673 		0x9600, 0x96bc,
1674 		0x9800, 0x9808,
1675 		0x9820, 0x983c,
1676 		0x9850, 0x9864,
1677 		0x9c00, 0x9c6c,
1678 		0x9c80, 0x9cec,
1679 		0x9d00, 0x9d6c,
1680 		0x9d80, 0x9dec,
1681 		0x9e00, 0x9e6c,
1682 		0x9e80, 0x9eec,
1683 		0x9f00, 0x9f6c,
1684 		0x9f80, 0x9fec,
1685 		0xd004, 0xd03c,
1686 		0xdfc0, 0xdfe0,
1687 		0xe000, 0xea7c,
1688 		0xf000, 0x11110,
1689 		0x11118, 0x11190,
1690 		0x19040, 0x1906c,
1691 		0x19078, 0x19080,
1692 		0x1908c, 0x19124,
1693 		0x19150, 0x191b0,
1694 		0x191d0, 0x191e8,
1695 		0x19238, 0x1924c,
1696 		0x193f8, 0x19474,
1697 		0x19490, 0x194f8,
1698 		0x19800, 0x19f30,
1699 		0x1a000, 0x1a06c,
1700 		0x1a0b0, 0x1a120,
1701 		0x1a128, 0x1a138,
1702 		0x1a190, 0x1a1c4,
1703 		0x1a1fc, 0x1a1fc,
1704 		0x1e040, 0x1e04c,
1705 		0x1e284, 0x1e28c,
1706 		0x1e2c0, 0x1e2c0,
1707 		0x1e2e0, 0x1e2e0,
1708 		0x1e300, 0x1e384,
1709 		0x1e3c0, 0x1e3c8,
1710 		0x1e440, 0x1e44c,
1711 		0x1e684, 0x1e68c,
1712 		0x1e6c0, 0x1e6c0,
1713 		0x1e6e0, 0x1e6e0,
1714 		0x1e700, 0x1e784,
1715 		0x1e7c0, 0x1e7c8,
1716 		0x1e840, 0x1e84c,
1717 		0x1ea84, 0x1ea8c,
1718 		0x1eac0, 0x1eac0,
1719 		0x1eae0, 0x1eae0,
1720 		0x1eb00, 0x1eb84,
1721 		0x1ebc0, 0x1ebc8,
1722 		0x1ec40, 0x1ec4c,
1723 		0x1ee84, 0x1ee8c,
1724 		0x1eec0, 0x1eec0,
1725 		0x1eee0, 0x1eee0,
1726 		0x1ef00, 0x1ef84,
1727 		0x1efc0, 0x1efc8,
1728 		0x1f040, 0x1f04c,
1729 		0x1f284, 0x1f28c,
1730 		0x1f2c0, 0x1f2c0,
1731 		0x1f2e0, 0x1f2e0,
1732 		0x1f300, 0x1f384,
1733 		0x1f3c0, 0x1f3c8,
1734 		0x1f440, 0x1f44c,
1735 		0x1f684, 0x1f68c,
1736 		0x1f6c0, 0x1f6c0,
1737 		0x1f6e0, 0x1f6e0,
1738 		0x1f700, 0x1f784,
1739 		0x1f7c0, 0x1f7c8,
1740 		0x1f840, 0x1f84c,
1741 		0x1fa84, 0x1fa8c,
1742 		0x1fac0, 0x1fac0,
1743 		0x1fae0, 0x1fae0,
1744 		0x1fb00, 0x1fb84,
1745 		0x1fbc0, 0x1fbc8,
1746 		0x1fc40, 0x1fc4c,
1747 		0x1fe84, 0x1fe8c,
1748 		0x1fec0, 0x1fec0,
1749 		0x1fee0, 0x1fee0,
1750 		0x1ff00, 0x1ff84,
1751 		0x1ffc0, 0x1ffc8,
1752 		0x20000, 0x2002c,
1753 		0x20100, 0x2013c,
1754 		0x20190, 0x201c8,
1755 		0x20200, 0x20318,
1756 		0x20400, 0x20528,
1757 		0x20540, 0x20614,
1758 		0x21000, 0x21040,
1759 		0x2104c, 0x21060,
1760 		0x210c0, 0x210ec,
1761 		0x21200, 0x21268,
1762 		0x21270, 0x21284,
1763 		0x212fc, 0x21388,
1764 		0x21400, 0x21404,
1765 		0x21500, 0x21518,
1766 		0x2152c, 0x2153c,
1767 		0x21550, 0x21554,
1768 		0x21600, 0x21600,
1769 		0x21608, 0x21628,
1770 		0x21630, 0x2163c,
1771 		0x21700, 0x2171c,
1772 		0x21780, 0x2178c,
1773 		0x21800, 0x21c38,
1774 		0x21c80, 0x21d7c,
1775 		0x21e00, 0x21e04,
1776 		0x22000, 0x2202c,
1777 		0x22100, 0x2213c,
1778 		0x22190, 0x221c8,
1779 		0x22200, 0x22318,
1780 		0x22400, 0x22528,
1781 		0x22540, 0x22614,
1782 		0x23000, 0x23040,
1783 		0x2304c, 0x23060,
1784 		0x230c0, 0x230ec,
1785 		0x23200, 0x23268,
1786 		0x23270, 0x23284,
1787 		0x232fc, 0x23388,
1788 		0x23400, 0x23404,
1789 		0x23500, 0x23518,
1790 		0x2352c, 0x2353c,
1791 		0x23550, 0x23554,
1792 		0x23600, 0x23600,
1793 		0x23608, 0x23628,
1794 		0x23630, 0x2363c,
1795 		0x23700, 0x2371c,
1796 		0x23780, 0x2378c,
1797 		0x23800, 0x23c38,
1798 		0x23c80, 0x23d7c,
1799 		0x23e00, 0x23e04,
1800 		0x24000, 0x2402c,
1801 		0x24100, 0x2413c,
1802 		0x24190, 0x241c8,
1803 		0x24200, 0x24318,
1804 		0x24400, 0x24528,
1805 		0x24540, 0x24614,
1806 		0x25000, 0x25040,
1807 		0x2504c, 0x25060,
1808 		0x250c0, 0x250ec,
1809 		0x25200, 0x25268,
1810 		0x25270, 0x25284,
1811 		0x252fc, 0x25388,
1812 		0x25400, 0x25404,
1813 		0x25500, 0x25518,
1814 		0x2552c, 0x2553c,
1815 		0x25550, 0x25554,
1816 		0x25600, 0x25600,
1817 		0x25608, 0x25628,
1818 		0x25630, 0x2563c,
1819 		0x25700, 0x2571c,
1820 		0x25780, 0x2578c,
1821 		0x25800, 0x25c38,
1822 		0x25c80, 0x25d7c,
1823 		0x25e00, 0x25e04,
1824 		0x26000, 0x2602c,
1825 		0x26100, 0x2613c,
1826 		0x26190, 0x261c8,
1827 		0x26200, 0x26318,
1828 		0x26400, 0x26528,
1829 		0x26540, 0x26614,
1830 		0x27000, 0x27040,
1831 		0x2704c, 0x27060,
1832 		0x270c0, 0x270ec,
1833 		0x27200, 0x27268,
1834 		0x27270, 0x27284,
1835 		0x272fc, 0x27388,
1836 		0x27400, 0x27404,
1837 		0x27500, 0x27518,
1838 		0x2752c, 0x2753c,
1839 		0x27550, 0x27554,
1840 		0x27600, 0x27600,
1841 		0x27608, 0x27628,
1842 		0x27630, 0x2763c,
1843 		0x27700, 0x2771c,
1844 		0x27780, 0x2778c,
1845 		0x27800, 0x27c38,
1846 		0x27c80, 0x27d7c,
1847 		0x27e00, 0x27e04
1848 	};
1849 
1850 	static const unsigned int t5_reg_ranges[] = {
1851 		0x1008, 0x1148,
1852 		0x1180, 0x11b4,
1853 		0x11fc, 0x123c,
1854 		0x1280, 0x173c,
1855 		0x1800, 0x18fc,
1856 		0x3000, 0x3028,
1857 		0x3060, 0x30d8,
1858 		0x30e0, 0x30fc,
1859 		0x3140, 0x357c,
1860 		0x35a8, 0x35cc,
1861 		0x35ec, 0x35ec,
1862 		0x3600, 0x5624,
1863 		0x56cc, 0x575c,
1864 		0x580c, 0x5814,
1865 		0x5890, 0x58bc,
1866 		0x5940, 0x59dc,
1867 		0x59fc, 0x5a18,
1868 		0x5a60, 0x5a9c,
1869 		0x5b9c, 0x5bfc,
1870 		0x6000, 0x6040,
1871 		0x6058, 0x614c,
1872 		0x7700, 0x7798,
1873 		0x77c0, 0x78fc,
1874 		0x7b00, 0x7c54,
1875 		0x7d00, 0x7efc,
1876 		0x8dc0, 0x8de0,
1877 		0x8df8, 0x8e84,
1878 		0x8ea0, 0x8f84,
1879 		0x8fc0, 0x90f8,
1880 		0x9400, 0x9470,
1881 		0x9600, 0x96f4,
1882 		0x9800, 0x9808,
1883 		0x9820, 0x983c,
1884 		0x9850, 0x9864,
1885 		0x9c00, 0x9c6c,
1886 		0x9c80, 0x9cec,
1887 		0x9d00, 0x9d6c,
1888 		0x9d80, 0x9dec,
1889 		0x9e00, 0x9e6c,
1890 		0x9e80, 0x9eec,
1891 		0x9f00, 0x9f6c,
1892 		0x9f80, 0xa020,
1893 		0xd004, 0xd03c,
1894 		0xdfc0, 0xdfe0,
1895 		0xe000, 0x11088,
1896 		0x1109c, 0x11110,
1897 		0x11118, 0x1117c,
1898 		0x11190, 0x11204,
1899 		0x19040, 0x1906c,
1900 		0x19078, 0x19080,
1901 		0x1908c, 0x19124,
1902 		0x19150, 0x191b0,
1903 		0x191d0, 0x191e8,
1904 		0x19238, 0x19290,
1905 		0x193f8, 0x19474,
1906 		0x19490, 0x194cc,
1907 		0x194f0, 0x194f8,
1908 		0x19c00, 0x19c60,
1909 		0x19c94, 0x19e10,
1910 		0x19e50, 0x19f34,
1911 		0x19f40, 0x19f50,
1912 		0x19f90, 0x19fe4,
1913 		0x1a000, 0x1a06c,
1914 		0x1a0b0, 0x1a120,
1915 		0x1a128, 0x1a138,
1916 		0x1a190, 0x1a1c4,
1917 		0x1a1fc, 0x1a1fc,
1918 		0x1e008, 0x1e00c,
1919 		0x1e040, 0x1e04c,
1920 		0x1e284, 0x1e290,
1921 		0x1e2c0, 0x1e2c0,
1922 		0x1e2e0, 0x1e2e0,
1923 		0x1e300, 0x1e384,
1924 		0x1e3c0, 0x1e3c8,
1925 		0x1e408, 0x1e40c,
1926 		0x1e440, 0x1e44c,
1927 		0x1e684, 0x1e690,
1928 		0x1e6c0, 0x1e6c0,
1929 		0x1e6e0, 0x1e6e0,
1930 		0x1e700, 0x1e784,
1931 		0x1e7c0, 0x1e7c8,
1932 		0x1e808, 0x1e80c,
1933 		0x1e840, 0x1e84c,
1934 		0x1ea84, 0x1ea90,
1935 		0x1eac0, 0x1eac0,
1936 		0x1eae0, 0x1eae0,
1937 		0x1eb00, 0x1eb84,
1938 		0x1ebc0, 0x1ebc8,
1939 		0x1ec08, 0x1ec0c,
1940 		0x1ec40, 0x1ec4c,
1941 		0x1ee84, 0x1ee90,
1942 		0x1eec0, 0x1eec0,
1943 		0x1eee0, 0x1eee0,
1944 		0x1ef00, 0x1ef84,
1945 		0x1efc0, 0x1efc8,
1946 		0x1f008, 0x1f00c,
1947 		0x1f040, 0x1f04c,
1948 		0x1f284, 0x1f290,
1949 		0x1f2c0, 0x1f2c0,
1950 		0x1f2e0, 0x1f2e0,
1951 		0x1f300, 0x1f384,
1952 		0x1f3c0, 0x1f3c8,
1953 		0x1f408, 0x1f40c,
1954 		0x1f440, 0x1f44c,
1955 		0x1f684, 0x1f690,
1956 		0x1f6c0, 0x1f6c0,
1957 		0x1f6e0, 0x1f6e0,
1958 		0x1f700, 0x1f784,
1959 		0x1f7c0, 0x1f7c8,
1960 		0x1f808, 0x1f80c,
1961 		0x1f840, 0x1f84c,
1962 		0x1fa84, 0x1fa90,
1963 		0x1fac0, 0x1fac0,
1964 		0x1fae0, 0x1fae0,
1965 		0x1fb00, 0x1fb84,
1966 		0x1fbc0, 0x1fbc8,
1967 		0x1fc08, 0x1fc0c,
1968 		0x1fc40, 0x1fc4c,
1969 		0x1fe84, 0x1fe90,
1970 		0x1fec0, 0x1fec0,
1971 		0x1fee0, 0x1fee0,
1972 		0x1ff00, 0x1ff84,
1973 		0x1ffc0, 0x1ffc8,
1974 		0x30000, 0x30030,
1975 		0x30100, 0x30144,
1976 		0x30190, 0x301d0,
1977 		0x30200, 0x30318,
1978 		0x30400, 0x3052c,
1979 		0x30540, 0x3061c,
1980 		0x30800, 0x30834,
1981 		0x308c0, 0x30908,
1982 		0x30910, 0x309ac,
1983 		0x30a00, 0x30a04,
1984 		0x30a0c, 0x30a2c,
1985 		0x30a44, 0x30a50,
1986 		0x30a74, 0x30c24,
1987 		0x30d08, 0x30d14,
1988 		0x30d1c, 0x30d20,
1989 		0x30d3c, 0x30d50,
1990 		0x31200, 0x3120c,
1991 		0x31220, 0x31220,
1992 		0x31240, 0x31240,
1993 		0x31600, 0x31600,
1994 		0x31608, 0x3160c,
1995 		0x31a00, 0x31a1c,
1996 		0x31e04, 0x31e20,
1997 		0x31e38, 0x31e3c,
1998 		0x31e80, 0x31e80,
1999 		0x31e88, 0x31ea8,
2000 		0x31eb0, 0x31eb4,
2001 		0x31ec8, 0x31ed4,
2002 		0x31fb8, 0x32004,
2003 		0x32208, 0x3223c,
2004 		0x32600, 0x32630,
2005 		0x32a00, 0x32abc,
2006 		0x32b00, 0x32b70,
2007 		0x33000, 0x33048,
2008 		0x33060, 0x3309c,
2009 		0x330f0, 0x33148,
2010 		0x33160, 0x3319c,
2011 		0x331f0, 0x332e4,
2012 		0x332f8, 0x333e4,
2013 		0x333f8, 0x33448,
2014 		0x33460, 0x3349c,
2015 		0x334f0, 0x33548,
2016 		0x33560, 0x3359c,
2017 		0x335f0, 0x336e4,
2018 		0x336f8, 0x337e4,
2019 		0x337f8, 0x337fc,
2020 		0x33814, 0x33814,
2021 		0x3382c, 0x3382c,
2022 		0x33880, 0x3388c,
2023 		0x338e8, 0x338ec,
2024 		0x33900, 0x33948,
2025 		0x33960, 0x3399c,
2026 		0x339f0, 0x33ae4,
2027 		0x33af8, 0x33b10,
2028 		0x33b28, 0x33b28,
2029 		0x33b3c, 0x33b50,
2030 		0x33bf0, 0x33c10,
2031 		0x33c28, 0x33c28,
2032 		0x33c3c, 0x33c50,
2033 		0x33cf0, 0x33cfc,
2034 		0x34000, 0x34030,
2035 		0x34100, 0x34144,
2036 		0x34190, 0x341d0,
2037 		0x34200, 0x34318,
2038 		0x34400, 0x3452c,
2039 		0x34540, 0x3461c,
2040 		0x34800, 0x34834,
2041 		0x348c0, 0x34908,
2042 		0x34910, 0x349ac,
2043 		0x34a00, 0x34a04,
2044 		0x34a0c, 0x34a2c,
2045 		0x34a44, 0x34a50,
2046 		0x34a74, 0x34c24,
2047 		0x34d08, 0x34d14,
2048 		0x34d1c, 0x34d20,
2049 		0x34d3c, 0x34d50,
2050 		0x35200, 0x3520c,
2051 		0x35220, 0x35220,
2052 		0x35240, 0x35240,
2053 		0x35600, 0x35600,
2054 		0x35608, 0x3560c,
2055 		0x35a00, 0x35a1c,
2056 		0x35e04, 0x35e20,
2057 		0x35e38, 0x35e3c,
2058 		0x35e80, 0x35e80,
2059 		0x35e88, 0x35ea8,
2060 		0x35eb0, 0x35eb4,
2061 		0x35ec8, 0x35ed4,
2062 		0x35fb8, 0x36004,
2063 		0x36208, 0x3623c,
2064 		0x36600, 0x36630,
2065 		0x36a00, 0x36abc,
2066 		0x36b00, 0x36b70,
2067 		0x37000, 0x37048,
2068 		0x37060, 0x3709c,
2069 		0x370f0, 0x37148,
2070 		0x37160, 0x3719c,
2071 		0x371f0, 0x372e4,
2072 		0x372f8, 0x373e4,
2073 		0x373f8, 0x37448,
2074 		0x37460, 0x3749c,
2075 		0x374f0, 0x37548,
2076 		0x37560, 0x3759c,
2077 		0x375f0, 0x376e4,
2078 		0x376f8, 0x377e4,
2079 		0x377f8, 0x377fc,
2080 		0x37814, 0x37814,
2081 		0x3782c, 0x3782c,
2082 		0x37880, 0x3788c,
2083 		0x378e8, 0x378ec,
2084 		0x37900, 0x37948,
2085 		0x37960, 0x3799c,
2086 		0x379f0, 0x37ae4,
2087 		0x37af8, 0x37b10,
2088 		0x37b28, 0x37b28,
2089 		0x37b3c, 0x37b50,
2090 		0x37bf0, 0x37c10,
2091 		0x37c28, 0x37c28,
2092 		0x37c3c, 0x37c50,
2093 		0x37cf0, 0x37cfc,
2094 		0x38000, 0x38030,
2095 		0x38100, 0x38144,
2096 		0x38190, 0x381d0,
2097 		0x38200, 0x38318,
2098 		0x38400, 0x3852c,
2099 		0x38540, 0x3861c,
2100 		0x38800, 0x38834,
2101 		0x388c0, 0x38908,
2102 		0x38910, 0x389ac,
2103 		0x38a00, 0x38a04,
2104 		0x38a0c, 0x38a2c,
2105 		0x38a44, 0x38a50,
2106 		0x38a74, 0x38c24,
2107 		0x38d08, 0x38d14,
2108 		0x38d1c, 0x38d20,
2109 		0x38d3c, 0x38d50,
2110 		0x39200, 0x3920c,
2111 		0x39220, 0x39220,
2112 		0x39240, 0x39240,
2113 		0x39600, 0x39600,
2114 		0x39608, 0x3960c,
2115 		0x39a00, 0x39a1c,
2116 		0x39e04, 0x39e20,
2117 		0x39e38, 0x39e3c,
2118 		0x39e80, 0x39e80,
2119 		0x39e88, 0x39ea8,
2120 		0x39eb0, 0x39eb4,
2121 		0x39ec8, 0x39ed4,
2122 		0x39fb8, 0x3a004,
2123 		0x3a208, 0x3a23c,
2124 		0x3a600, 0x3a630,
2125 		0x3aa00, 0x3aabc,
2126 		0x3ab00, 0x3ab70,
2127 		0x3b000, 0x3b048,
2128 		0x3b060, 0x3b09c,
2129 		0x3b0f0, 0x3b148,
2130 		0x3b160, 0x3b19c,
2131 		0x3b1f0, 0x3b2e4,
2132 		0x3b2f8, 0x3b3e4,
2133 		0x3b3f8, 0x3b448,
2134 		0x3b460, 0x3b49c,
2135 		0x3b4f0, 0x3b548,
2136 		0x3b560, 0x3b59c,
2137 		0x3b5f0, 0x3b6e4,
2138 		0x3b6f8, 0x3b7e4,
2139 		0x3b7f8, 0x3b7fc,
2140 		0x3b814, 0x3b814,
2141 		0x3b82c, 0x3b82c,
2142 		0x3b880, 0x3b88c,
2143 		0x3b8e8, 0x3b8ec,
2144 		0x3b900, 0x3b948,
2145 		0x3b960, 0x3b99c,
2146 		0x3b9f0, 0x3bae4,
2147 		0x3baf8, 0x3bb10,
2148 		0x3bb28, 0x3bb28,
2149 		0x3bb3c, 0x3bb50,
2150 		0x3bbf0, 0x3bc10,
2151 		0x3bc28, 0x3bc28,
2152 		0x3bc3c, 0x3bc50,
2153 		0x3bcf0, 0x3bcfc,
2154 		0x3c000, 0x3c030,
2155 		0x3c100, 0x3c144,
2156 		0x3c190, 0x3c1d0,
2157 		0x3c200, 0x3c318,
2158 		0x3c400, 0x3c52c,
2159 		0x3c540, 0x3c61c,
2160 		0x3c800, 0x3c834,
2161 		0x3c8c0, 0x3c908,
2162 		0x3c910, 0x3c9ac,
2163 		0x3ca00, 0x3ca04,
2164 		0x3ca0c, 0x3ca2c,
2165 		0x3ca44, 0x3ca50,
2166 		0x3ca74, 0x3cc24,
2167 		0x3cd08, 0x3cd14,
2168 		0x3cd1c, 0x3cd20,
2169 		0x3cd3c, 0x3cd50,
2170 		0x3d200, 0x3d20c,
2171 		0x3d220, 0x3d220,
2172 		0x3d240, 0x3d240,
2173 		0x3d600, 0x3d600,
2174 		0x3d608, 0x3d60c,
2175 		0x3da00, 0x3da1c,
2176 		0x3de04, 0x3de20,
2177 		0x3de38, 0x3de3c,
2178 		0x3de80, 0x3de80,
2179 		0x3de88, 0x3dea8,
2180 		0x3deb0, 0x3deb4,
2181 		0x3dec8, 0x3ded4,
2182 		0x3dfb8, 0x3e004,
2183 		0x3e208, 0x3e23c,
2184 		0x3e600, 0x3e630,
2185 		0x3ea00, 0x3eabc,
2186 		0x3eb00, 0x3eb70,
2187 		0x3f000, 0x3f048,
2188 		0x3f060, 0x3f09c,
2189 		0x3f0f0, 0x3f148,
2190 		0x3f160, 0x3f19c,
2191 		0x3f1f0, 0x3f2e4,
2192 		0x3f2f8, 0x3f3e4,
2193 		0x3f3f8, 0x3f448,
2194 		0x3f460, 0x3f49c,
2195 		0x3f4f0, 0x3f548,
2196 		0x3f560, 0x3f59c,
2197 		0x3f5f0, 0x3f6e4,
2198 		0x3f6f8, 0x3f7e4,
2199 		0x3f7f8, 0x3f7fc,
2200 		0x3f814, 0x3f814,
2201 		0x3f82c, 0x3f82c,
2202 		0x3f880, 0x3f88c,
2203 		0x3f8e8, 0x3f8ec,
2204 		0x3f900, 0x3f948,
2205 		0x3f960, 0x3f99c,
2206 		0x3f9f0, 0x3fae4,
2207 		0x3faf8, 0x3fb10,
2208 		0x3fb28, 0x3fb28,
2209 		0x3fb3c, 0x3fb50,
2210 		0x3fbf0, 0x3fc10,
2211 		0x3fc28, 0x3fc28,
2212 		0x3fc3c, 0x3fc50,
2213 		0x3fcf0, 0x3fcfc,
2214 		0x40000, 0x4000c,
2215 		0x40040, 0x40068,
2216 		0x40080, 0x40144,
2217 		0x40180, 0x4018c,
2218 		0x40200, 0x40298,
2219 		0x402ac, 0x4033c,
2220 		0x403f8, 0x403fc,
2221 		0x41304, 0x413c4,
2222 		0x41400, 0x4141c,
2223 		0x41480, 0x414d0,
2224 		0x44000, 0x44078,
2225 		0x440c0, 0x44278,
2226 		0x442c0, 0x44478,
2227 		0x444c0, 0x44678,
2228 		0x446c0, 0x44878,
2229 		0x448c0, 0x449fc,
2230 		0x45000, 0x45068,
2231 		0x45080, 0x45084,
2232 		0x450a0, 0x450b0,
2233 		0x45200, 0x45268,
2234 		0x45280, 0x45284,
2235 		0x452a0, 0x452b0,
2236 		0x460c0, 0x460e4,
2237 		0x47000, 0x4708c,
2238 		0x47200, 0x47250,
2239 		0x47400, 0x47420,
2240 		0x47600, 0x47618,
2241 		0x47800, 0x47814,
2242 		0x48000, 0x4800c,
2243 		0x48040, 0x48068,
2244 		0x48080, 0x48144,
2245 		0x48180, 0x4818c,
2246 		0x48200, 0x48298,
2247 		0x482ac, 0x4833c,
2248 		0x483f8, 0x483fc,
2249 		0x49304, 0x493c4,
2250 		0x49400, 0x4941c,
2251 		0x49480, 0x494d0,
2252 		0x4c000, 0x4c078,
2253 		0x4c0c0, 0x4c278,
2254 		0x4c2c0, 0x4c478,
2255 		0x4c4c0, 0x4c678,
2256 		0x4c6c0, 0x4c878,
2257 		0x4c8c0, 0x4c9fc,
2258 		0x4d000, 0x4d068,
2259 		0x4d080, 0x4d084,
2260 		0x4d0a0, 0x4d0b0,
2261 		0x4d200, 0x4d268,
2262 		0x4d280, 0x4d284,
2263 		0x4d2a0, 0x4d2b0,
2264 		0x4e0c0, 0x4e0e4,
2265 		0x4f000, 0x4f08c,
2266 		0x4f200, 0x4f250,
2267 		0x4f400, 0x4f420,
2268 		0x4f600, 0x4f618,
2269 		0x4f800, 0x4f814,
2270 		0x50000, 0x500cc,
2271 		0x50400, 0x50400,
2272 		0x50800, 0x508cc,
2273 		0x50c00, 0x50c00,
2274 		0x51000, 0x5101c,
2275 		0x51300, 0x51308,
2276 	};
2277 
2278 	int i;
2279 	struct adapter *ap = netdev2adap(dev);
2280 	static const unsigned int *reg_ranges;
2281 	int arr_size = 0, buf_size = 0;
2282 
2283 	if (is_t4(ap->params.chip)) {
2284 		reg_ranges = &t4_reg_ranges[0];
2285 		arr_size = ARRAY_SIZE(t4_reg_ranges);
2286 		buf_size = T4_REGMAP_SIZE;
2287 	} else {
2288 		reg_ranges = &t5_reg_ranges[0];
2289 		arr_size = ARRAY_SIZE(t5_reg_ranges);
2290 		buf_size = T5_REGMAP_SIZE;
2291 	}
2292 
2293 	regs->version = mk_adap_vers(ap);
2294 
2295 	memset(buf, 0, buf_size);
2296 	for (i = 0; i < arr_size; i += 2)
2297 		reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
2298 }
2299 
2300 static int restart_autoneg(struct net_device *dev)
2301 {
2302 	struct port_info *p = netdev_priv(dev);
2303 
2304 	if (!netif_running(dev))
2305 		return -EAGAIN;
2306 	if (p->link_cfg.autoneg != AUTONEG_ENABLE)
2307 		return -EINVAL;
2308 	t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
2309 	return 0;
2310 }
2311 
2312 static int identify_port(struct net_device *dev,
2313 			 enum ethtool_phys_id_state state)
2314 {
2315 	unsigned int val;
2316 	struct adapter *adap = netdev2adap(dev);
2317 
2318 	if (state == ETHTOOL_ID_ACTIVE)
2319 		val = 0xffff;
2320 	else if (state == ETHTOOL_ID_INACTIVE)
2321 		val = 0;
2322 	else
2323 		return -EINVAL;
2324 
2325 	return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
2326 }
2327 
2328 static unsigned int from_fw_linkcaps(enum fw_port_type type, unsigned int caps)
2329 {
2330 	unsigned int v = 0;
2331 
2332 	if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
2333 	    type == FW_PORT_TYPE_BT_XAUI) {
2334 		v |= SUPPORTED_TP;
2335 		if (caps & FW_PORT_CAP_SPEED_100M)
2336 			v |= SUPPORTED_100baseT_Full;
2337 		if (caps & FW_PORT_CAP_SPEED_1G)
2338 			v |= SUPPORTED_1000baseT_Full;
2339 		if (caps & FW_PORT_CAP_SPEED_10G)
2340 			v |= SUPPORTED_10000baseT_Full;
2341 	} else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
2342 		v |= SUPPORTED_Backplane;
2343 		if (caps & FW_PORT_CAP_SPEED_1G)
2344 			v |= SUPPORTED_1000baseKX_Full;
2345 		if (caps & FW_PORT_CAP_SPEED_10G)
2346 			v |= SUPPORTED_10000baseKX4_Full;
2347 	} else if (type == FW_PORT_TYPE_KR)
2348 		v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
2349 	else if (type == FW_PORT_TYPE_BP_AP)
2350 		v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2351 		     SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
2352 	else if (type == FW_PORT_TYPE_BP4_AP)
2353 		v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
2354 		     SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
2355 		     SUPPORTED_10000baseKX4_Full;
2356 	else if (type == FW_PORT_TYPE_FIBER_XFI ||
2357 		 type == FW_PORT_TYPE_FIBER_XAUI ||
2358 		 type == FW_PORT_TYPE_SFP ||
2359 		 type == FW_PORT_TYPE_QSFP_10G ||
2360 		 type == FW_PORT_TYPE_QSA) {
2361 		v |= SUPPORTED_FIBRE;
2362 		if (caps & FW_PORT_CAP_SPEED_1G)
2363 			v |= SUPPORTED_1000baseT_Full;
2364 		if (caps & FW_PORT_CAP_SPEED_10G)
2365 			v |= SUPPORTED_10000baseT_Full;
2366 	} else if (type == FW_PORT_TYPE_BP40_BA ||
2367 		   type == FW_PORT_TYPE_QSFP) {
2368 		v |= SUPPORTED_40000baseSR4_Full;
2369 		v |= SUPPORTED_FIBRE;
2370 	}
2371 
2372 	if (caps & FW_PORT_CAP_ANEG)
2373 		v |= SUPPORTED_Autoneg;
2374 	return v;
2375 }
2376 
2377 static unsigned int to_fw_linkcaps(unsigned int caps)
2378 {
2379 	unsigned int v = 0;
2380 
2381 	if (caps & ADVERTISED_100baseT_Full)
2382 		v |= FW_PORT_CAP_SPEED_100M;
2383 	if (caps & ADVERTISED_1000baseT_Full)
2384 		v |= FW_PORT_CAP_SPEED_1G;
2385 	if (caps & ADVERTISED_10000baseT_Full)
2386 		v |= FW_PORT_CAP_SPEED_10G;
2387 	if (caps & ADVERTISED_40000baseSR4_Full)
2388 		v |= FW_PORT_CAP_SPEED_40G;
2389 	return v;
2390 }
2391 
2392 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2393 {
2394 	const struct port_info *p = netdev_priv(dev);
2395 
2396 	if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
2397 	    p->port_type == FW_PORT_TYPE_BT_XFI ||
2398 	    p->port_type == FW_PORT_TYPE_BT_XAUI)
2399 		cmd->port = PORT_TP;
2400 	else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
2401 		 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
2402 		cmd->port = PORT_FIBRE;
2403 	else if (p->port_type == FW_PORT_TYPE_SFP ||
2404 		 p->port_type == FW_PORT_TYPE_QSFP_10G ||
2405 		 p->port_type == FW_PORT_TYPE_QSA ||
2406 		 p->port_type == FW_PORT_TYPE_QSFP) {
2407 		if (p->mod_type == FW_PORT_MOD_TYPE_LR ||
2408 		    p->mod_type == FW_PORT_MOD_TYPE_SR ||
2409 		    p->mod_type == FW_PORT_MOD_TYPE_ER ||
2410 		    p->mod_type == FW_PORT_MOD_TYPE_LRM)
2411 			cmd->port = PORT_FIBRE;
2412 		else if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
2413 			 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
2414 			cmd->port = PORT_DA;
2415 		else
2416 			cmd->port = PORT_OTHER;
2417 	} else
2418 		cmd->port = PORT_OTHER;
2419 
2420 	if (p->mdio_addr >= 0) {
2421 		cmd->phy_address = p->mdio_addr;
2422 		cmd->transceiver = XCVR_EXTERNAL;
2423 		cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
2424 			MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
2425 	} else {
2426 		cmd->phy_address = 0;  /* not really, but no better option */
2427 		cmd->transceiver = XCVR_INTERNAL;
2428 		cmd->mdio_support = 0;
2429 	}
2430 
2431 	cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
2432 	cmd->advertising = from_fw_linkcaps(p->port_type,
2433 					    p->link_cfg.advertising);
2434 	ethtool_cmd_speed_set(cmd,
2435 			      netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
2436 	cmd->duplex = DUPLEX_FULL;
2437 	cmd->autoneg = p->link_cfg.autoneg;
2438 	cmd->maxtxpkt = 0;
2439 	cmd->maxrxpkt = 0;
2440 	return 0;
2441 }
2442 
2443 static unsigned int speed_to_caps(int speed)
2444 {
2445 	if (speed == 100)
2446 		return FW_PORT_CAP_SPEED_100M;
2447 	if (speed == 1000)
2448 		return FW_PORT_CAP_SPEED_1G;
2449 	if (speed == 10000)
2450 		return FW_PORT_CAP_SPEED_10G;
2451 	if (speed == 40000)
2452 		return FW_PORT_CAP_SPEED_40G;
2453 	return 0;
2454 }
2455 
2456 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
2457 {
2458 	unsigned int cap;
2459 	struct port_info *p = netdev_priv(dev);
2460 	struct link_config *lc = &p->link_cfg;
2461 	u32 speed = ethtool_cmd_speed(cmd);
2462 
2463 	if (cmd->duplex != DUPLEX_FULL)     /* only full-duplex supported */
2464 		return -EINVAL;
2465 
2466 	if (!(lc->supported & FW_PORT_CAP_ANEG)) {
2467 		/*
2468 		 * PHY offers a single speed.  See if that's what's
2469 		 * being requested.
2470 		 */
2471 		if (cmd->autoneg == AUTONEG_DISABLE &&
2472 		    (lc->supported & speed_to_caps(speed)))
2473 			return 0;
2474 		return -EINVAL;
2475 	}
2476 
2477 	if (cmd->autoneg == AUTONEG_DISABLE) {
2478 		cap = speed_to_caps(speed);
2479 
2480 		if (!(lc->supported & cap) ||
2481 		    (speed == 1000) ||
2482 		    (speed == 10000) ||
2483 		    (speed == 40000))
2484 			return -EINVAL;
2485 		lc->requested_speed = cap;
2486 		lc->advertising = 0;
2487 	} else {
2488 		cap = to_fw_linkcaps(cmd->advertising);
2489 		if (!(lc->supported & cap))
2490 			return -EINVAL;
2491 		lc->requested_speed = 0;
2492 		lc->advertising = cap | FW_PORT_CAP_ANEG;
2493 	}
2494 	lc->autoneg = cmd->autoneg;
2495 
2496 	if (netif_running(dev))
2497 		return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2498 				     lc);
2499 	return 0;
2500 }
2501 
2502 static void get_pauseparam(struct net_device *dev,
2503 			   struct ethtool_pauseparam *epause)
2504 {
2505 	struct port_info *p = netdev_priv(dev);
2506 
2507 	epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
2508 	epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
2509 	epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
2510 }
2511 
2512 static int set_pauseparam(struct net_device *dev,
2513 			  struct ethtool_pauseparam *epause)
2514 {
2515 	struct port_info *p = netdev_priv(dev);
2516 	struct link_config *lc = &p->link_cfg;
2517 
2518 	if (epause->autoneg == AUTONEG_DISABLE)
2519 		lc->requested_fc = 0;
2520 	else if (lc->supported & FW_PORT_CAP_ANEG)
2521 		lc->requested_fc = PAUSE_AUTONEG;
2522 	else
2523 		return -EINVAL;
2524 
2525 	if (epause->rx_pause)
2526 		lc->requested_fc |= PAUSE_RX;
2527 	if (epause->tx_pause)
2528 		lc->requested_fc |= PAUSE_TX;
2529 	if (netif_running(dev))
2530 		return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
2531 				     lc);
2532 	return 0;
2533 }
2534 
2535 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2536 {
2537 	const struct port_info *pi = netdev_priv(dev);
2538 	const struct sge *s = &pi->adapter->sge;
2539 
2540 	e->rx_max_pending = MAX_RX_BUFFERS;
2541 	e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
2542 	e->rx_jumbo_max_pending = 0;
2543 	e->tx_max_pending = MAX_TXQ_ENTRIES;
2544 
2545 	e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
2546 	e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
2547 	e->rx_jumbo_pending = 0;
2548 	e->tx_pending = s->ethtxq[pi->first_qset].q.size;
2549 }
2550 
2551 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
2552 {
2553 	int i;
2554 	const struct port_info *pi = netdev_priv(dev);
2555 	struct adapter *adapter = pi->adapter;
2556 	struct sge *s = &adapter->sge;
2557 
2558 	if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
2559 	    e->tx_pending > MAX_TXQ_ENTRIES ||
2560 	    e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
2561 	    e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
2562 	    e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
2563 		return -EINVAL;
2564 
2565 	if (adapter->flags & FULL_INIT_DONE)
2566 		return -EBUSY;
2567 
2568 	for (i = 0; i < pi->nqsets; ++i) {
2569 		s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
2570 		s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
2571 		s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
2572 	}
2573 	return 0;
2574 }
2575 
2576 static int closest_timer(const struct sge *s, int time)
2577 {
2578 	int i, delta, match = 0, min_delta = INT_MAX;
2579 
2580 	for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
2581 		delta = time - s->timer_val[i];
2582 		if (delta < 0)
2583 			delta = -delta;
2584 		if (delta < min_delta) {
2585 			min_delta = delta;
2586 			match = i;
2587 		}
2588 	}
2589 	return match;
2590 }
2591 
2592 static int closest_thres(const struct sge *s, int thres)
2593 {
2594 	int i, delta, match = 0, min_delta = INT_MAX;
2595 
2596 	for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
2597 		delta = thres - s->counter_val[i];
2598 		if (delta < 0)
2599 			delta = -delta;
2600 		if (delta < min_delta) {
2601 			min_delta = delta;
2602 			match = i;
2603 		}
2604 	}
2605 	return match;
2606 }
2607 
2608 /*
2609  * Return a queue's interrupt hold-off time in us.  0 means no timer.
2610  */
2611 static unsigned int qtimer_val(const struct adapter *adap,
2612 			       const struct sge_rspq *q)
2613 {
2614 	unsigned int idx = q->intr_params >> 1;
2615 
2616 	return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
2617 }
2618 
2619 /**
2620  *	set_rspq_intr_params - set a queue's interrupt holdoff parameters
2621  *	@q: the Rx queue
2622  *	@us: the hold-off time in us, or 0 to disable timer
2623  *	@cnt: the hold-off packet count, or 0 to disable counter
2624  *
2625  *	Sets an Rx queue's interrupt hold-off time and packet count.  At least
2626  *	one of the two needs to be enabled for the queue to generate interrupts.
2627  */
2628 static int set_rspq_intr_params(struct sge_rspq *q,
2629 				unsigned int us, unsigned int cnt)
2630 {
2631 	struct adapter *adap = q->adap;
2632 
2633 	if ((us | cnt) == 0)
2634 		cnt = 1;
2635 
2636 	if (cnt) {
2637 		int err;
2638 		u32 v, new_idx;
2639 
2640 		new_idx = closest_thres(&adap->sge, cnt);
2641 		if (q->desc && q->pktcnt_idx != new_idx) {
2642 			/* the queue has already been created, update it */
2643 			v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
2644 			    FW_PARAMS_PARAM_X_V(
2645 					FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
2646 			    FW_PARAMS_PARAM_YZ_V(q->cntxt_id);
2647 			err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
2648 					    &new_idx);
2649 			if (err)
2650 				return err;
2651 		}
2652 		q->pktcnt_idx = new_idx;
2653 	}
2654 
2655 	us = us == 0 ? 6 : closest_timer(&adap->sge, us);
2656 	q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
2657 	return 0;
2658 }
2659 
2660 /**
2661  * set_rx_intr_params - set a net devices's RX interrupt holdoff paramete!
2662  * @dev: the network device
2663  * @us: the hold-off time in us, or 0 to disable timer
2664  * @cnt: the hold-off packet count, or 0 to disable counter
2665  *
2666  * Set the RX interrupt hold-off parameters for a network device.
2667  */
2668 static int set_rx_intr_params(struct net_device *dev,
2669 			      unsigned int us, unsigned int cnt)
2670 {
2671 	int i, err;
2672 	struct port_info *pi = netdev_priv(dev);
2673 	struct adapter *adap = pi->adapter;
2674 	struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2675 
2676 	for (i = 0; i < pi->nqsets; i++, q++) {
2677 		err = set_rspq_intr_params(&q->rspq, us, cnt);
2678 		if (err)
2679 			return err;
2680 	}
2681 	return 0;
2682 }
2683 
2684 static int set_adaptive_rx_setting(struct net_device *dev, int adaptive_rx)
2685 {
2686 	int i;
2687 	struct port_info *pi = netdev_priv(dev);
2688 	struct adapter *adap = pi->adapter;
2689 	struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2690 
2691 	for (i = 0; i < pi->nqsets; i++, q++)
2692 		q->rspq.adaptive_rx = adaptive_rx;
2693 
2694 	return 0;
2695 }
2696 
2697 static int get_adaptive_rx_setting(struct net_device *dev)
2698 {
2699 	struct port_info *pi = netdev_priv(dev);
2700 	struct adapter *adap = pi->adapter;
2701 	struct sge_eth_rxq *q = &adap->sge.ethrxq[pi->first_qset];
2702 
2703 	return q->rspq.adaptive_rx;
2704 }
2705 
2706 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2707 {
2708 	set_adaptive_rx_setting(dev, c->use_adaptive_rx_coalesce);
2709 	return set_rx_intr_params(dev, c->rx_coalesce_usecs,
2710 				  c->rx_max_coalesced_frames);
2711 }
2712 
2713 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2714 {
2715 	const struct port_info *pi = netdev_priv(dev);
2716 	const struct adapter *adap = pi->adapter;
2717 	const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
2718 
2719 	c->rx_coalesce_usecs = qtimer_val(adap, rq);
2720 	c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
2721 		adap->sge.counter_val[rq->pktcnt_idx] : 0;
2722 	c->use_adaptive_rx_coalesce = get_adaptive_rx_setting(dev);
2723 	return 0;
2724 }
2725 
2726 /**
2727  *	eeprom_ptov - translate a physical EEPROM address to virtual
2728  *	@phys_addr: the physical EEPROM address
2729  *	@fn: the PCI function number
2730  *	@sz: size of function-specific area
2731  *
2732  *	Translate a physical EEPROM address to virtual.  The first 1K is
2733  *	accessed through virtual addresses starting at 31K, the rest is
2734  *	accessed through virtual addresses starting at 0.
2735  *
2736  *	The mapping is as follows:
2737  *	[0..1K) -> [31K..32K)
2738  *	[1K..1K+A) -> [31K-A..31K)
2739  *	[1K+A..ES) -> [0..ES-A-1K)
2740  *
2741  *	where A = @fn * @sz, and ES = EEPROM size.
2742  */
2743 static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
2744 {
2745 	fn *= sz;
2746 	if (phys_addr < 1024)
2747 		return phys_addr + (31 << 10);
2748 	if (phys_addr < 1024 + fn)
2749 		return 31744 - fn + phys_addr - 1024;
2750 	if (phys_addr < EEPROMSIZE)
2751 		return phys_addr - 1024 - fn;
2752 	return -EINVAL;
2753 }
2754 
2755 /*
2756  * The next two routines implement eeprom read/write from physical addresses.
2757  */
2758 static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
2759 {
2760 	int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2761 
2762 	if (vaddr >= 0)
2763 		vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
2764 	return vaddr < 0 ? vaddr : 0;
2765 }
2766 
2767 static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
2768 {
2769 	int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
2770 
2771 	if (vaddr >= 0)
2772 		vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
2773 	return vaddr < 0 ? vaddr : 0;
2774 }
2775 
2776 #define EEPROM_MAGIC 0x38E2F10C
2777 
2778 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2779 		      u8 *data)
2780 {
2781 	int i, err = 0;
2782 	struct adapter *adapter = netdev2adap(dev);
2783 
2784 	u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2785 	if (!buf)
2786 		return -ENOMEM;
2787 
2788 	e->magic = EEPROM_MAGIC;
2789 	for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2790 		err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
2791 
2792 	if (!err)
2793 		memcpy(data, buf + e->offset, e->len);
2794 	kfree(buf);
2795 	return err;
2796 }
2797 
2798 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2799 		      u8 *data)
2800 {
2801 	u8 *buf;
2802 	int err = 0;
2803 	u32 aligned_offset, aligned_len, *p;
2804 	struct adapter *adapter = netdev2adap(dev);
2805 
2806 	if (eeprom->magic != EEPROM_MAGIC)
2807 		return -EINVAL;
2808 
2809 	aligned_offset = eeprom->offset & ~3;
2810 	aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2811 
2812 	if (adapter->fn > 0) {
2813 		u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
2814 
2815 		if (aligned_offset < start ||
2816 		    aligned_offset + aligned_len > start + EEPROMPFSIZE)
2817 			return -EPERM;
2818 	}
2819 
2820 	if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2821 		/*
2822 		 * RMW possibly needed for first or last words.
2823 		 */
2824 		buf = kmalloc(aligned_len, GFP_KERNEL);
2825 		if (!buf)
2826 			return -ENOMEM;
2827 		err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
2828 		if (!err && aligned_len > 4)
2829 			err = eeprom_rd_phys(adapter,
2830 					     aligned_offset + aligned_len - 4,
2831 					     (u32 *)&buf[aligned_len - 4]);
2832 		if (err)
2833 			goto out;
2834 		memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2835 	} else
2836 		buf = data;
2837 
2838 	err = t4_seeprom_wp(adapter, false);
2839 	if (err)
2840 		goto out;
2841 
2842 	for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
2843 		err = eeprom_wr_phys(adapter, aligned_offset, *p);
2844 		aligned_offset += 4;
2845 	}
2846 
2847 	if (!err)
2848 		err = t4_seeprom_wp(adapter, true);
2849 out:
2850 	if (buf != data)
2851 		kfree(buf);
2852 	return err;
2853 }
2854 
2855 static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
2856 {
2857 	int ret;
2858 	const struct firmware *fw;
2859 	struct adapter *adap = netdev2adap(netdev);
2860 	unsigned int mbox = PCIE_FW_MASTER_M + 1;
2861 
2862 	ef->data[sizeof(ef->data) - 1] = '\0';
2863 	ret = request_firmware(&fw, ef->data, adap->pdev_dev);
2864 	if (ret < 0)
2865 		return ret;
2866 
2867 	/* If the adapter has been fully initialized then we'll go ahead and
2868 	 * try to get the firmware's cooperation in upgrading to the new
2869 	 * firmware image otherwise we'll try to do the entire job from the
2870 	 * host ... and we always "force" the operation in this path.
2871 	 */
2872 	if (adap->flags & FULL_INIT_DONE)
2873 		mbox = adap->mbox;
2874 
2875 	ret = t4_fw_upgrade(adap, mbox, fw->data, fw->size, 1);
2876 	release_firmware(fw);
2877 	if (!ret)
2878 		dev_info(adap->pdev_dev, "loaded firmware %s,"
2879 			 " reload cxgb4 driver\n", ef->data);
2880 	return ret;
2881 }
2882 
2883 #define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
2884 #define BCAST_CRC 0xa0ccc1a6
2885 
2886 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2887 {
2888 	wol->supported = WAKE_BCAST | WAKE_MAGIC;
2889 	wol->wolopts = netdev2adap(dev)->wol;
2890 	memset(&wol->sopass, 0, sizeof(wol->sopass));
2891 }
2892 
2893 static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2894 {
2895 	int err = 0;
2896 	struct port_info *pi = netdev_priv(dev);
2897 
2898 	if (wol->wolopts & ~WOL_SUPPORTED)
2899 		return -EINVAL;
2900 	t4_wol_magic_enable(pi->adapter, pi->tx_chan,
2901 			    (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
2902 	if (wol->wolopts & WAKE_BCAST) {
2903 		err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
2904 					~0ULL, 0, false);
2905 		if (!err)
2906 			err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
2907 						~6ULL, ~0ULL, BCAST_CRC, true);
2908 	} else
2909 		t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
2910 	return err;
2911 }
2912 
2913 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2914 {
2915 	const struct port_info *pi = netdev_priv(dev);
2916 	netdev_features_t changed = dev->features ^ features;
2917 	int err;
2918 
2919 	if (!(changed & NETIF_F_HW_VLAN_CTAG_RX))
2920 		return 0;
2921 
2922 	err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
2923 			    -1, -1, -1,
2924 			    !!(features & NETIF_F_HW_VLAN_CTAG_RX), true);
2925 	if (unlikely(err))
2926 		dev->features = features ^ NETIF_F_HW_VLAN_CTAG_RX;
2927 	return err;
2928 }
2929 
2930 static u32 get_rss_table_size(struct net_device *dev)
2931 {
2932 	const struct port_info *pi = netdev_priv(dev);
2933 
2934 	return pi->rss_size;
2935 }
2936 
2937 static int get_rss_table(struct net_device *dev, u32 *p, u8 *key, u8 *hfunc)
2938 {
2939 	const struct port_info *pi = netdev_priv(dev);
2940 	unsigned int n = pi->rss_size;
2941 
2942 	if (hfunc)
2943 		*hfunc = ETH_RSS_HASH_TOP;
2944 	if (!p)
2945 		return 0;
2946 	while (n--)
2947 		p[n] = pi->rss[n];
2948 	return 0;
2949 }
2950 
2951 static int set_rss_table(struct net_device *dev, const u32 *p, const u8 *key,
2952 			 const u8 hfunc)
2953 {
2954 	unsigned int i;
2955 	struct port_info *pi = netdev_priv(dev);
2956 
2957 	/* We require at least one supported parameter to be changed and no
2958 	 * change in any of the unsupported parameters
2959 	 */
2960 	if (key ||
2961 	    (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP))
2962 		return -EOPNOTSUPP;
2963 	if (!p)
2964 		return 0;
2965 
2966 	for (i = 0; i < pi->rss_size; i++)
2967 		pi->rss[i] = p[i];
2968 	if (pi->adapter->flags & FULL_INIT_DONE)
2969 		return write_rss(pi, pi->rss);
2970 	return 0;
2971 }
2972 
2973 static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
2974 		     u32 *rules)
2975 {
2976 	const struct port_info *pi = netdev_priv(dev);
2977 
2978 	switch (info->cmd) {
2979 	case ETHTOOL_GRXFH: {
2980 		unsigned int v = pi->rss_mode;
2981 
2982 		info->data = 0;
2983 		switch (info->flow_type) {
2984 		case TCP_V4_FLOW:
2985 			if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F)
2986 				info->data = RXH_IP_SRC | RXH_IP_DST |
2987 					     RXH_L4_B_0_1 | RXH_L4_B_2_3;
2988 			else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2989 				info->data = RXH_IP_SRC | RXH_IP_DST;
2990 			break;
2991 		case UDP_V4_FLOW:
2992 			if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN_F) &&
2993 			    (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
2994 				info->data = RXH_IP_SRC | RXH_IP_DST |
2995 					     RXH_L4_B_0_1 | RXH_L4_B_2_3;
2996 			else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
2997 				info->data = RXH_IP_SRC | RXH_IP_DST;
2998 			break;
2999 		case SCTP_V4_FLOW:
3000 		case AH_ESP_V4_FLOW:
3001 		case IPV4_FLOW:
3002 			if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN_F)
3003 				info->data = RXH_IP_SRC | RXH_IP_DST;
3004 			break;
3005 		case TCP_V6_FLOW:
3006 			if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F)
3007 				info->data = RXH_IP_SRC | RXH_IP_DST |
3008 					     RXH_L4_B_0_1 | RXH_L4_B_2_3;
3009 			else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
3010 				info->data = RXH_IP_SRC | RXH_IP_DST;
3011 			break;
3012 		case UDP_V6_FLOW:
3013 			if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN_F) &&
3014 			    (v & FW_RSS_VI_CONFIG_CMD_UDPEN_F))
3015 				info->data = RXH_IP_SRC | RXH_IP_DST |
3016 					     RXH_L4_B_0_1 | RXH_L4_B_2_3;
3017 			else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
3018 				info->data = RXH_IP_SRC | RXH_IP_DST;
3019 			break;
3020 		case SCTP_V6_FLOW:
3021 		case AH_ESP_V6_FLOW:
3022 		case IPV6_FLOW:
3023 			if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN_F)
3024 				info->data = RXH_IP_SRC | RXH_IP_DST;
3025 			break;
3026 		}
3027 		return 0;
3028 	}
3029 	case ETHTOOL_GRXRINGS:
3030 		info->data = pi->nqsets;
3031 		return 0;
3032 	}
3033 	return -EOPNOTSUPP;
3034 }
3035 
3036 static const struct ethtool_ops cxgb_ethtool_ops = {
3037 	.get_settings      = get_settings,
3038 	.set_settings      = set_settings,
3039 	.get_drvinfo       = get_drvinfo,
3040 	.get_msglevel      = get_msglevel,
3041 	.set_msglevel      = set_msglevel,
3042 	.get_ringparam     = get_sge_param,
3043 	.set_ringparam     = set_sge_param,
3044 	.get_coalesce      = get_coalesce,
3045 	.set_coalesce      = set_coalesce,
3046 	.get_eeprom_len    = get_eeprom_len,
3047 	.get_eeprom        = get_eeprom,
3048 	.set_eeprom        = set_eeprom,
3049 	.get_pauseparam    = get_pauseparam,
3050 	.set_pauseparam    = set_pauseparam,
3051 	.get_link          = ethtool_op_get_link,
3052 	.get_strings       = get_strings,
3053 	.set_phys_id       = identify_port,
3054 	.nway_reset        = restart_autoneg,
3055 	.get_sset_count    = get_sset_count,
3056 	.get_ethtool_stats = get_stats,
3057 	.get_regs_len      = get_regs_len,
3058 	.get_regs          = get_regs,
3059 	.get_wol           = get_wol,
3060 	.set_wol           = set_wol,
3061 	.get_rxnfc         = get_rxnfc,
3062 	.get_rxfh_indir_size = get_rss_table_size,
3063 	.get_rxfh	   = get_rss_table,
3064 	.set_rxfh	   = set_rss_table,
3065 	.flash_device      = set_flash,
3066 };
3067 
3068 static int setup_debugfs(struct adapter *adap)
3069 {
3070 	if (IS_ERR_OR_NULL(adap->debugfs_root))
3071 		return -1;
3072 
3073 #ifdef CONFIG_DEBUG_FS
3074 	t4_setup_debugfs(adap);
3075 #endif
3076 	return 0;
3077 }
3078 
3079 /*
3080  * upper-layer driver support
3081  */
3082 
3083 /*
3084  * Allocate an active-open TID and set it to the supplied value.
3085  */
3086 int cxgb4_alloc_atid(struct tid_info *t, void *data)
3087 {
3088 	int atid = -1;
3089 
3090 	spin_lock_bh(&t->atid_lock);
3091 	if (t->afree) {
3092 		union aopen_entry *p = t->afree;
3093 
3094 		atid = (p - t->atid_tab) + t->atid_base;
3095 		t->afree = p->next;
3096 		p->data = data;
3097 		t->atids_in_use++;
3098 	}
3099 	spin_unlock_bh(&t->atid_lock);
3100 	return atid;
3101 }
3102 EXPORT_SYMBOL(cxgb4_alloc_atid);
3103 
3104 /*
3105  * Release an active-open TID.
3106  */
3107 void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
3108 {
3109 	union aopen_entry *p = &t->atid_tab[atid - t->atid_base];
3110 
3111 	spin_lock_bh(&t->atid_lock);
3112 	p->next = t->afree;
3113 	t->afree = p;
3114 	t->atids_in_use--;
3115 	spin_unlock_bh(&t->atid_lock);
3116 }
3117 EXPORT_SYMBOL(cxgb4_free_atid);
3118 
3119 /*
3120  * Allocate a server TID and set it to the supplied value.
3121  */
3122 int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
3123 {
3124 	int stid;
3125 
3126 	spin_lock_bh(&t->stid_lock);
3127 	if (family == PF_INET) {
3128 		stid = find_first_zero_bit(t->stid_bmap, t->nstids);
3129 		if (stid < t->nstids)
3130 			__set_bit(stid, t->stid_bmap);
3131 		else
3132 			stid = -1;
3133 	} else {
3134 		stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
3135 		if (stid < 0)
3136 			stid = -1;
3137 	}
3138 	if (stid >= 0) {
3139 		t->stid_tab[stid].data = data;
3140 		stid += t->stid_base;
3141 		/* IPv6 requires max of 520 bits or 16 cells in TCAM
3142 		 * This is equivalent to 4 TIDs. With CLIP enabled it
3143 		 * needs 2 TIDs.
3144 		 */
3145 		if (family == PF_INET)
3146 			t->stids_in_use++;
3147 		else
3148 			t->stids_in_use += 4;
3149 	}
3150 	spin_unlock_bh(&t->stid_lock);
3151 	return stid;
3152 }
3153 EXPORT_SYMBOL(cxgb4_alloc_stid);
3154 
3155 /* Allocate a server filter TID and set it to the supplied value.
3156  */
3157 int cxgb4_alloc_sftid(struct tid_info *t, int family, void *data)
3158 {
3159 	int stid;
3160 
3161 	spin_lock_bh(&t->stid_lock);
3162 	if (family == PF_INET) {
3163 		stid = find_next_zero_bit(t->stid_bmap,
3164 				t->nstids + t->nsftids, t->nstids);
3165 		if (stid < (t->nstids + t->nsftids))
3166 			__set_bit(stid, t->stid_bmap);
3167 		else
3168 			stid = -1;
3169 	} else {
3170 		stid = -1;
3171 	}
3172 	if (stid >= 0) {
3173 		t->stid_tab[stid].data = data;
3174 		stid -= t->nstids;
3175 		stid += t->sftid_base;
3176 		t->stids_in_use++;
3177 	}
3178 	spin_unlock_bh(&t->stid_lock);
3179 	return stid;
3180 }
3181 EXPORT_SYMBOL(cxgb4_alloc_sftid);
3182 
3183 /* Release a server TID.
3184  */
3185 void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
3186 {
3187 	/* Is it a server filter TID? */
3188 	if (t->nsftids && (stid >= t->sftid_base)) {
3189 		stid -= t->sftid_base;
3190 		stid += t->nstids;
3191 	} else {
3192 		stid -= t->stid_base;
3193 	}
3194 
3195 	spin_lock_bh(&t->stid_lock);
3196 	if (family == PF_INET)
3197 		__clear_bit(stid, t->stid_bmap);
3198 	else
3199 		bitmap_release_region(t->stid_bmap, stid, 2);
3200 	t->stid_tab[stid].data = NULL;
3201 	if (family == PF_INET)
3202 		t->stids_in_use--;
3203 	else
3204 		t->stids_in_use -= 4;
3205 	spin_unlock_bh(&t->stid_lock);
3206 }
3207 EXPORT_SYMBOL(cxgb4_free_stid);
3208 
3209 /*
3210  * Populate a TID_RELEASE WR.  Caller must properly size the skb.
3211  */
3212 static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
3213 			   unsigned int tid)
3214 {
3215 	struct cpl_tid_release *req;
3216 
3217 	set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
3218 	req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
3219 	INIT_TP_WR(req, tid);
3220 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
3221 }
3222 
3223 /*
3224  * Queue a TID release request and if necessary schedule a work queue to
3225  * process it.
3226  */
3227 static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
3228 				    unsigned int tid)
3229 {
3230 	void **p = &t->tid_tab[tid];
3231 	struct adapter *adap = container_of(t, struct adapter, tids);
3232 
3233 	spin_lock_bh(&adap->tid_release_lock);
3234 	*p = adap->tid_release_head;
3235 	/* Low 2 bits encode the Tx channel number */
3236 	adap->tid_release_head = (void **)((uintptr_t)p | chan);
3237 	if (!adap->tid_release_task_busy) {
3238 		adap->tid_release_task_busy = true;
3239 		queue_work(adap->workq, &adap->tid_release_task);
3240 	}
3241 	spin_unlock_bh(&adap->tid_release_lock);
3242 }
3243 
3244 /*
3245  * Process the list of pending TID release requests.
3246  */
3247 static void process_tid_release_list(struct work_struct *work)
3248 {
3249 	struct sk_buff *skb;
3250 	struct adapter *adap;
3251 
3252 	adap = container_of(work, struct adapter, tid_release_task);
3253 
3254 	spin_lock_bh(&adap->tid_release_lock);
3255 	while (adap->tid_release_head) {
3256 		void **p = adap->tid_release_head;
3257 		unsigned int chan = (uintptr_t)p & 3;
3258 		p = (void *)p - chan;
3259 
3260 		adap->tid_release_head = *p;
3261 		*p = NULL;
3262 		spin_unlock_bh(&adap->tid_release_lock);
3263 
3264 		while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
3265 					 GFP_KERNEL)))
3266 			schedule_timeout_uninterruptible(1);
3267 
3268 		mk_tid_release(skb, chan, p - adap->tids.tid_tab);
3269 		t4_ofld_send(adap, skb);
3270 		spin_lock_bh(&adap->tid_release_lock);
3271 	}
3272 	adap->tid_release_task_busy = false;
3273 	spin_unlock_bh(&adap->tid_release_lock);
3274 }
3275 
3276 /*
3277  * Release a TID and inform HW.  If we are unable to allocate the release
3278  * message we defer to a work queue.
3279  */
3280 void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
3281 {
3282 	void *old;
3283 	struct sk_buff *skb;
3284 	struct adapter *adap = container_of(t, struct adapter, tids);
3285 
3286 	old = t->tid_tab[tid];
3287 	skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
3288 	if (likely(skb)) {
3289 		t->tid_tab[tid] = NULL;
3290 		mk_tid_release(skb, chan, tid);
3291 		t4_ofld_send(adap, skb);
3292 	} else
3293 		cxgb4_queue_tid_release(t, chan, tid);
3294 	if (old)
3295 		atomic_dec(&t->tids_in_use);
3296 }
3297 EXPORT_SYMBOL(cxgb4_remove_tid);
3298 
3299 /*
3300  * Allocate and initialize the TID tables.  Returns 0 on success.
3301  */
3302 static int tid_init(struct tid_info *t)
3303 {
3304 	size_t size;
3305 	unsigned int stid_bmap_size;
3306 	unsigned int natids = t->natids;
3307 	struct adapter *adap = container_of(t, struct adapter, tids);
3308 
3309 	stid_bmap_size = BITS_TO_LONGS(t->nstids + t->nsftids);
3310 	size = t->ntids * sizeof(*t->tid_tab) +
3311 	       natids * sizeof(*t->atid_tab) +
3312 	       t->nstids * sizeof(*t->stid_tab) +
3313 	       t->nsftids * sizeof(*t->stid_tab) +
3314 	       stid_bmap_size * sizeof(long) +
3315 	       t->nftids * sizeof(*t->ftid_tab) +
3316 	       t->nsftids * sizeof(*t->ftid_tab);
3317 
3318 	t->tid_tab = t4_alloc_mem(size);
3319 	if (!t->tid_tab)
3320 		return -ENOMEM;
3321 
3322 	t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
3323 	t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
3324 	t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids + t->nsftids];
3325 	t->ftid_tab = (struct filter_entry *)&t->stid_bmap[stid_bmap_size];
3326 	spin_lock_init(&t->stid_lock);
3327 	spin_lock_init(&t->atid_lock);
3328 
3329 	t->stids_in_use = 0;
3330 	t->afree = NULL;
3331 	t->atids_in_use = 0;
3332 	atomic_set(&t->tids_in_use, 0);
3333 
3334 	/* Setup the free list for atid_tab and clear the stid bitmap. */
3335 	if (natids) {
3336 		while (--natids)
3337 			t->atid_tab[natids - 1].next = &t->atid_tab[natids];
3338 		t->afree = t->atid_tab;
3339 	}
3340 	bitmap_zero(t->stid_bmap, t->nstids + t->nsftids);
3341 	/* Reserve stid 0 for T4/T5 adapters */
3342 	if (!t->stid_base &&
3343 	    (is_t4(adap->params.chip) || is_t5(adap->params.chip)))
3344 		__set_bit(0, t->stid_bmap);
3345 
3346 	return 0;
3347 }
3348 
3349 int cxgb4_clip_get(const struct net_device *dev,
3350 		   const struct in6_addr *lip)
3351 {
3352 	struct adapter *adap;
3353 	struct fw_clip_cmd c;
3354 
3355 	adap = netdev2adap(dev);
3356 	memset(&c, 0, sizeof(c));
3357 	c.op_to_write = htonl(FW_CMD_OP_V(FW_CLIP_CMD) |
3358 			FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
3359 	c.alloc_to_len16 = htonl(FW_CLIP_CMD_ALLOC_F | FW_LEN16(c));
3360 	c.ip_hi = *(__be64 *)(lip->s6_addr);
3361 	c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3362 	return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3363 }
3364 EXPORT_SYMBOL(cxgb4_clip_get);
3365 
3366 int cxgb4_clip_release(const struct net_device *dev,
3367 		       const struct in6_addr *lip)
3368 {
3369 	struct adapter *adap;
3370 	struct fw_clip_cmd c;
3371 
3372 	adap = netdev2adap(dev);
3373 	memset(&c, 0, sizeof(c));
3374 	c.op_to_write = htonl(FW_CMD_OP_V(FW_CLIP_CMD) |
3375 			FW_CMD_REQUEST_F | FW_CMD_READ_F);
3376 	c.alloc_to_len16 = htonl(FW_CLIP_CMD_FREE_F | FW_LEN16(c));
3377 	c.ip_hi = *(__be64 *)(lip->s6_addr);
3378 	c.ip_lo = *(__be64 *)(lip->s6_addr + 8);
3379 	return t4_wr_mbox_meat(adap, adap->mbox, &c, sizeof(c), &c, false);
3380 }
3381 EXPORT_SYMBOL(cxgb4_clip_release);
3382 
3383 /**
3384  *	cxgb4_create_server - create an IP server
3385  *	@dev: the device
3386  *	@stid: the server TID
3387  *	@sip: local IP address to bind server to
3388  *	@sport: the server's TCP port
3389  *	@queue: queue to direct messages from this server to
3390  *
3391  *	Create an IP server for the given port and address.
3392  *	Returns <0 on error and one of the %NET_XMIT_* values on success.
3393  */
3394 int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
3395 			__be32 sip, __be16 sport, __be16 vlan,
3396 			unsigned int queue)
3397 {
3398 	unsigned int chan;
3399 	struct sk_buff *skb;
3400 	struct adapter *adap;
3401 	struct cpl_pass_open_req *req;
3402 	int ret;
3403 
3404 	skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3405 	if (!skb)
3406 		return -ENOMEM;
3407 
3408 	adap = netdev2adap(dev);
3409 	req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
3410 	INIT_TP_WR(req, 0);
3411 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
3412 	req->local_port = sport;
3413 	req->peer_port = htons(0);
3414 	req->local_ip = sip;
3415 	req->peer_ip = htonl(0);
3416 	chan = rxq_to_chan(&adap->sge, queue);
3417 	req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
3418 	req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3419 				SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3420 	ret = t4_mgmt_tx(adap, skb);
3421 	return net_xmit_eval(ret);
3422 }
3423 EXPORT_SYMBOL(cxgb4_create_server);
3424 
3425 /*	cxgb4_create_server6 - create an IPv6 server
3426  *	@dev: the device
3427  *	@stid: the server TID
3428  *	@sip: local IPv6 address to bind server to
3429  *	@sport: the server's TCP port
3430  *	@queue: queue to direct messages from this server to
3431  *
3432  *	Create an IPv6 server for the given port and address.
3433  *	Returns <0 on error and one of the %NET_XMIT_* values on success.
3434  */
3435 int cxgb4_create_server6(const struct net_device *dev, unsigned int stid,
3436 			 const struct in6_addr *sip, __be16 sport,
3437 			 unsigned int queue)
3438 {
3439 	unsigned int chan;
3440 	struct sk_buff *skb;
3441 	struct adapter *adap;
3442 	struct cpl_pass_open_req6 *req;
3443 	int ret;
3444 
3445 	skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3446 	if (!skb)
3447 		return -ENOMEM;
3448 
3449 	adap = netdev2adap(dev);
3450 	req = (struct cpl_pass_open_req6 *)__skb_put(skb, sizeof(*req));
3451 	INIT_TP_WR(req, 0);
3452 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ6, stid));
3453 	req->local_port = sport;
3454 	req->peer_port = htons(0);
3455 	req->local_ip_hi = *(__be64 *)(sip->s6_addr);
3456 	req->local_ip_lo = *(__be64 *)(sip->s6_addr + 8);
3457 	req->peer_ip_hi = cpu_to_be64(0);
3458 	req->peer_ip_lo = cpu_to_be64(0);
3459 	chan = rxq_to_chan(&adap->sge, queue);
3460 	req->opt0 = cpu_to_be64(TX_CHAN_V(chan));
3461 	req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
3462 				SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
3463 	ret = t4_mgmt_tx(adap, skb);
3464 	return net_xmit_eval(ret);
3465 }
3466 EXPORT_SYMBOL(cxgb4_create_server6);
3467 
3468 int cxgb4_remove_server(const struct net_device *dev, unsigned int stid,
3469 			unsigned int queue, bool ipv6)
3470 {
3471 	struct sk_buff *skb;
3472 	struct adapter *adap;
3473 	struct cpl_close_listsvr_req *req;
3474 	int ret;
3475 
3476 	adap = netdev2adap(dev);
3477 
3478 	skb = alloc_skb(sizeof(*req), GFP_KERNEL);
3479 	if (!skb)
3480 		return -ENOMEM;
3481 
3482 	req = (struct cpl_close_listsvr_req *)__skb_put(skb, sizeof(*req));
3483 	INIT_TP_WR(req, 0);
3484 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_LISTSRV_REQ, stid));
3485 	req->reply_ctrl = htons(NO_REPLY(0) | (ipv6 ? LISTSVR_IPV6(1) :
3486 				LISTSVR_IPV6(0)) | QUEUENO(queue));
3487 	ret = t4_mgmt_tx(adap, skb);
3488 	return net_xmit_eval(ret);
3489 }
3490 EXPORT_SYMBOL(cxgb4_remove_server);
3491 
3492 /**
3493  *	cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
3494  *	@mtus: the HW MTU table
3495  *	@mtu: the target MTU
3496  *	@idx: index of selected entry in the MTU table
3497  *
3498  *	Returns the index and the value in the HW MTU table that is closest to
3499  *	but does not exceed @mtu, unless @mtu is smaller than any value in the
3500  *	table, in which case that smallest available value is selected.
3501  */
3502 unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
3503 			    unsigned int *idx)
3504 {
3505 	unsigned int i = 0;
3506 
3507 	while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
3508 		++i;
3509 	if (idx)
3510 		*idx = i;
3511 	return mtus[i];
3512 }
3513 EXPORT_SYMBOL(cxgb4_best_mtu);
3514 
3515 /**
3516  *     cxgb4_best_aligned_mtu - find best MTU, [hopefully] data size aligned
3517  *     @mtus: the HW MTU table
3518  *     @header_size: Header Size
3519  *     @data_size_max: maximum Data Segment Size
3520  *     @data_size_align: desired Data Segment Size Alignment (2^N)
3521  *     @mtu_idxp: HW MTU Table Index return value pointer (possibly NULL)
3522  *
3523  *     Similar to cxgb4_best_mtu() but instead of searching the Hardware
3524  *     MTU Table based solely on a Maximum MTU parameter, we break that
3525  *     parameter up into a Header Size and Maximum Data Segment Size, and
3526  *     provide a desired Data Segment Size Alignment.  If we find an MTU in
3527  *     the Hardware MTU Table which will result in a Data Segment Size with
3528  *     the requested alignment _and_ that MTU isn't "too far" from the
3529  *     closest MTU, then we'll return that rather than the closest MTU.
3530  */
3531 unsigned int cxgb4_best_aligned_mtu(const unsigned short *mtus,
3532 				    unsigned short header_size,
3533 				    unsigned short data_size_max,
3534 				    unsigned short data_size_align,
3535 				    unsigned int *mtu_idxp)
3536 {
3537 	unsigned short max_mtu = header_size + data_size_max;
3538 	unsigned short data_size_align_mask = data_size_align - 1;
3539 	int mtu_idx, aligned_mtu_idx;
3540 
3541 	/* Scan the MTU Table till we find an MTU which is larger than our
3542 	 * Maximum MTU or we reach the end of the table.  Along the way,
3543 	 * record the last MTU found, if any, which will result in a Data
3544 	 * Segment Length matching the requested alignment.
3545 	 */
3546 	for (mtu_idx = 0, aligned_mtu_idx = -1; mtu_idx < NMTUS; mtu_idx++) {
3547 		unsigned short data_size = mtus[mtu_idx] - header_size;
3548 
3549 		/* If this MTU minus the Header Size would result in a
3550 		 * Data Segment Size of the desired alignment, remember it.
3551 		 */
3552 		if ((data_size & data_size_align_mask) == 0)
3553 			aligned_mtu_idx = mtu_idx;
3554 
3555 		/* If we're not at the end of the Hardware MTU Table and the
3556 		 * next element is larger than our Maximum MTU, drop out of
3557 		 * the loop.
3558 		 */
3559 		if (mtu_idx+1 < NMTUS && mtus[mtu_idx+1] > max_mtu)
3560 			break;
3561 	}
3562 
3563 	/* If we fell out of the loop because we ran to the end of the table,
3564 	 * then we just have to use the last [largest] entry.
3565 	 */
3566 	if (mtu_idx == NMTUS)
3567 		mtu_idx--;
3568 
3569 	/* If we found an MTU which resulted in the requested Data Segment
3570 	 * Length alignment and that's "not far" from the largest MTU which is
3571 	 * less than or equal to the maximum MTU, then use that.
3572 	 */
3573 	if (aligned_mtu_idx >= 0 &&
3574 	    mtu_idx - aligned_mtu_idx <= 1)
3575 		mtu_idx = aligned_mtu_idx;
3576 
3577 	/* If the caller has passed in an MTU Index pointer, pass the
3578 	 * MTU Index back.  Return the MTU value.
3579 	 */
3580 	if (mtu_idxp)
3581 		*mtu_idxp = mtu_idx;
3582 	return mtus[mtu_idx];
3583 }
3584 EXPORT_SYMBOL(cxgb4_best_aligned_mtu);
3585 
3586 /**
3587  *	cxgb4_port_chan - get the HW channel of a port
3588  *	@dev: the net device for the port
3589  *
3590  *	Return the HW Tx channel of the given port.
3591  */
3592 unsigned int cxgb4_port_chan(const struct net_device *dev)
3593 {
3594 	return netdev2pinfo(dev)->tx_chan;
3595 }
3596 EXPORT_SYMBOL(cxgb4_port_chan);
3597 
3598 unsigned int cxgb4_dbfifo_count(const struct net_device *dev, int lpfifo)
3599 {
3600 	struct adapter *adap = netdev2adap(dev);
3601 	u32 v1, v2, lp_count, hp_count;
3602 
3603 	v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3604 	v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
3605 	if (is_t4(adap->params.chip)) {
3606 		lp_count = G_LP_COUNT(v1);
3607 		hp_count = G_HP_COUNT(v1);
3608 	} else {
3609 		lp_count = G_LP_COUNT_T5(v1);
3610 		hp_count = G_HP_COUNT_T5(v2);
3611 	}
3612 	return lpfifo ? lp_count : hp_count;
3613 }
3614 EXPORT_SYMBOL(cxgb4_dbfifo_count);
3615 
3616 /**
3617  *	cxgb4_port_viid - get the VI id of a port
3618  *	@dev: the net device for the port
3619  *
3620  *	Return the VI id of the given port.
3621  */
3622 unsigned int cxgb4_port_viid(const struct net_device *dev)
3623 {
3624 	return netdev2pinfo(dev)->viid;
3625 }
3626 EXPORT_SYMBOL(cxgb4_port_viid);
3627 
3628 /**
3629  *	cxgb4_port_idx - get the index of a port
3630  *	@dev: the net device for the port
3631  *
3632  *	Return the index of the given port.
3633  */
3634 unsigned int cxgb4_port_idx(const struct net_device *dev)
3635 {
3636 	return netdev2pinfo(dev)->port_id;
3637 }
3638 EXPORT_SYMBOL(cxgb4_port_idx);
3639 
3640 void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
3641 			 struct tp_tcp_stats *v6)
3642 {
3643 	struct adapter *adap = pci_get_drvdata(pdev);
3644 
3645 	spin_lock(&adap->stats_lock);
3646 	t4_tp_get_tcp_stats(adap, v4, v6);
3647 	spin_unlock(&adap->stats_lock);
3648 }
3649 EXPORT_SYMBOL(cxgb4_get_tcp_stats);
3650 
3651 void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
3652 		      const unsigned int *pgsz_order)
3653 {
3654 	struct adapter *adap = netdev2adap(dev);
3655 
3656 	t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
3657 	t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
3658 		     HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
3659 		     HPZ3(pgsz_order[3]));
3660 }
3661 EXPORT_SYMBOL(cxgb4_iscsi_init);
3662 
3663 int cxgb4_flush_eq_cache(struct net_device *dev)
3664 {
3665 	struct adapter *adap = netdev2adap(dev);
3666 	int ret;
3667 
3668 	ret = t4_fwaddrspace_write(adap, adap->mbox,
3669 				   0xe1000000 + A_SGE_CTXT_CMD, 0x20000000);
3670 	return ret;
3671 }
3672 EXPORT_SYMBOL(cxgb4_flush_eq_cache);
3673 
3674 static int read_eq_indices(struct adapter *adap, u16 qid, u16 *pidx, u16 *cidx)
3675 {
3676 	u32 addr = t4_read_reg(adap, A_SGE_DBQ_CTXT_BADDR) + 24 * qid + 8;
3677 	__be64 indices;
3678 	int ret;
3679 
3680 	spin_lock(&adap->win0_lock);
3681 	ret = t4_memory_rw(adap, 0, MEM_EDC0, addr,
3682 			   sizeof(indices), (__be32 *)&indices,
3683 			   T4_MEMORY_READ);
3684 	spin_unlock(&adap->win0_lock);
3685 	if (!ret) {
3686 		*cidx = (be64_to_cpu(indices) >> 25) & 0xffff;
3687 		*pidx = (be64_to_cpu(indices) >> 9) & 0xffff;
3688 	}
3689 	return ret;
3690 }
3691 
3692 int cxgb4_sync_txq_pidx(struct net_device *dev, u16 qid, u16 pidx,
3693 			u16 size)
3694 {
3695 	struct adapter *adap = netdev2adap(dev);
3696 	u16 hw_pidx, hw_cidx;
3697 	int ret;
3698 
3699 	ret = read_eq_indices(adap, qid, &hw_pidx, &hw_cidx);
3700 	if (ret)
3701 		goto out;
3702 
3703 	if (pidx != hw_pidx) {
3704 		u16 delta;
3705 
3706 		if (pidx >= hw_pidx)
3707 			delta = pidx - hw_pidx;
3708 		else
3709 			delta = size - hw_pidx + pidx;
3710 		wmb();
3711 		t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3712 			     QID(qid) | PIDX(delta));
3713 	}
3714 out:
3715 	return ret;
3716 }
3717 EXPORT_SYMBOL(cxgb4_sync_txq_pidx);
3718 
3719 void cxgb4_disable_db_coalescing(struct net_device *dev)
3720 {
3721 	struct adapter *adap;
3722 
3723 	adap = netdev2adap(dev);
3724 	t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE,
3725 			 F_NOCOALESCE);
3726 }
3727 EXPORT_SYMBOL(cxgb4_disable_db_coalescing);
3728 
3729 void cxgb4_enable_db_coalescing(struct net_device *dev)
3730 {
3731 	struct adapter *adap;
3732 
3733 	adap = netdev2adap(dev);
3734 	t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_NOCOALESCE, 0);
3735 }
3736 EXPORT_SYMBOL(cxgb4_enable_db_coalescing);
3737 
3738 int cxgb4_read_tpte(struct net_device *dev, u32 stag, __be32 *tpte)
3739 {
3740 	struct adapter *adap;
3741 	u32 offset, memtype, memaddr;
3742 	u32 edc0_size, edc1_size, mc0_size, mc1_size, size;
3743 	u32 edc0_end, edc1_end, mc0_end, mc1_end;
3744 	int ret;
3745 
3746 	adap = netdev2adap(dev);
3747 
3748 	offset = ((stag >> 8) * 32) + adap->vres.stag.start;
3749 
3750 	/* Figure out where the offset lands in the Memory Type/Address scheme.
3751 	 * This code assumes that the memory is laid out starting at offset 0
3752 	 * with no breaks as: EDC0, EDC1, MC0, MC1. All cards have both EDC0
3753 	 * and EDC1.  Some cards will have neither MC0 nor MC1, most cards have
3754 	 * MC0, and some have both MC0 and MC1.
3755 	 */
3756 	size = t4_read_reg(adap, MA_EDRAM0_BAR_A);
3757 	edc0_size = EDRAM0_SIZE_G(size) << 20;
3758 	size = t4_read_reg(adap, MA_EDRAM1_BAR_A);
3759 	edc1_size = EDRAM1_SIZE_G(size) << 20;
3760 	size = t4_read_reg(adap, MA_EXT_MEMORY0_BAR_A);
3761 	mc0_size = EXT_MEM0_SIZE_G(size) << 20;
3762 
3763 	edc0_end = edc0_size;
3764 	edc1_end = edc0_end + edc1_size;
3765 	mc0_end = edc1_end + mc0_size;
3766 
3767 	if (offset < edc0_end) {
3768 		memtype = MEM_EDC0;
3769 		memaddr = offset;
3770 	} else if (offset < edc1_end) {
3771 		memtype = MEM_EDC1;
3772 		memaddr = offset - edc0_end;
3773 	} else {
3774 		if (offset < mc0_end) {
3775 			memtype = MEM_MC0;
3776 			memaddr = offset - edc1_end;
3777 		} else if (is_t4(adap->params.chip)) {
3778 			/* T4 only has a single memory channel */
3779 			goto err;
3780 		} else {
3781 			size = t4_read_reg(adap, MA_EXT_MEMORY1_BAR_A);
3782 			mc1_size = EXT_MEM1_SIZE_G(size) << 20;
3783 			mc1_end = mc0_end + mc1_size;
3784 			if (offset < mc1_end) {
3785 				memtype = MEM_MC1;
3786 				memaddr = offset - mc0_end;
3787 			} else {
3788 				/* offset beyond the end of any memory */
3789 				goto err;
3790 			}
3791 		}
3792 	}
3793 
3794 	spin_lock(&adap->win0_lock);
3795 	ret = t4_memory_rw(adap, 0, memtype, memaddr, 32, tpte, T4_MEMORY_READ);
3796 	spin_unlock(&adap->win0_lock);
3797 	return ret;
3798 
3799 err:
3800 	dev_err(adap->pdev_dev, "stag %#x, offset %#x out of range\n",
3801 		stag, offset);
3802 	return -EINVAL;
3803 }
3804 EXPORT_SYMBOL(cxgb4_read_tpte);
3805 
3806 u64 cxgb4_read_sge_timestamp(struct net_device *dev)
3807 {
3808 	u32 hi, lo;
3809 	struct adapter *adap;
3810 
3811 	adap = netdev2adap(dev);
3812 	lo = t4_read_reg(adap, SGE_TIMESTAMP_LO);
3813 	hi = GET_TSVAL(t4_read_reg(adap, SGE_TIMESTAMP_HI));
3814 
3815 	return ((u64)hi << 32) | (u64)lo;
3816 }
3817 EXPORT_SYMBOL(cxgb4_read_sge_timestamp);
3818 
3819 int cxgb4_bar2_sge_qregs(struct net_device *dev,
3820 			 unsigned int qid,
3821 			 enum cxgb4_bar2_qtype qtype,
3822 			 u64 *pbar2_qoffset,
3823 			 unsigned int *pbar2_qid)
3824 {
3825 	return cxgb4_t4_bar2_sge_qregs(netdev2adap(dev),
3826 				 qid,
3827 				 (qtype == CXGB4_BAR2_QTYPE_EGRESS
3828 				  ? T4_BAR2_QTYPE_EGRESS
3829 				  : T4_BAR2_QTYPE_INGRESS),
3830 				 pbar2_qoffset,
3831 				 pbar2_qid);
3832 }
3833 EXPORT_SYMBOL(cxgb4_bar2_sge_qregs);
3834 
3835 static struct pci_driver cxgb4_driver;
3836 
3837 static void check_neigh_update(struct neighbour *neigh)
3838 {
3839 	const struct device *parent;
3840 	const struct net_device *netdev = neigh->dev;
3841 
3842 	if (netdev->priv_flags & IFF_802_1Q_VLAN)
3843 		netdev = vlan_dev_real_dev(netdev);
3844 	parent = netdev->dev.parent;
3845 	if (parent && parent->driver == &cxgb4_driver.driver)
3846 		t4_l2t_update(dev_get_drvdata(parent), neigh);
3847 }
3848 
3849 static int netevent_cb(struct notifier_block *nb, unsigned long event,
3850 		       void *data)
3851 {
3852 	switch (event) {
3853 	case NETEVENT_NEIGH_UPDATE:
3854 		check_neigh_update(data);
3855 		break;
3856 	case NETEVENT_REDIRECT:
3857 	default:
3858 		break;
3859 	}
3860 	return 0;
3861 }
3862 
3863 static bool netevent_registered;
3864 static struct notifier_block cxgb4_netevent_nb = {
3865 	.notifier_call = netevent_cb
3866 };
3867 
3868 static void drain_db_fifo(struct adapter *adap, int usecs)
3869 {
3870 	u32 v1, v2, lp_count, hp_count;
3871 
3872 	do {
3873 		v1 = t4_read_reg(adap, A_SGE_DBFIFO_STATUS);
3874 		v2 = t4_read_reg(adap, SGE_DBFIFO_STATUS2);
3875 		if (is_t4(adap->params.chip)) {
3876 			lp_count = G_LP_COUNT(v1);
3877 			hp_count = G_HP_COUNT(v1);
3878 		} else {
3879 			lp_count = G_LP_COUNT_T5(v1);
3880 			hp_count = G_HP_COUNT_T5(v2);
3881 		}
3882 
3883 		if (lp_count == 0 && hp_count == 0)
3884 			break;
3885 		set_current_state(TASK_UNINTERRUPTIBLE);
3886 		schedule_timeout(usecs_to_jiffies(usecs));
3887 	} while (1);
3888 }
3889 
3890 static void disable_txq_db(struct sge_txq *q)
3891 {
3892 	unsigned long flags;
3893 
3894 	spin_lock_irqsave(&q->db_lock, flags);
3895 	q->db_disabled = 1;
3896 	spin_unlock_irqrestore(&q->db_lock, flags);
3897 }
3898 
3899 static void enable_txq_db(struct adapter *adap, struct sge_txq *q)
3900 {
3901 	spin_lock_irq(&q->db_lock);
3902 	if (q->db_pidx_inc) {
3903 		/* Make sure that all writes to the TX descriptors
3904 		 * are committed before we tell HW about them.
3905 		 */
3906 		wmb();
3907 		t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3908 			     QID(q->cntxt_id) | PIDX(q->db_pidx_inc));
3909 		q->db_pidx_inc = 0;
3910 	}
3911 	q->db_disabled = 0;
3912 	spin_unlock_irq(&q->db_lock);
3913 }
3914 
3915 static void disable_dbs(struct adapter *adap)
3916 {
3917 	int i;
3918 
3919 	for_each_ethrxq(&adap->sge, i)
3920 		disable_txq_db(&adap->sge.ethtxq[i].q);
3921 	for_each_ofldrxq(&adap->sge, i)
3922 		disable_txq_db(&adap->sge.ofldtxq[i].q);
3923 	for_each_port(adap, i)
3924 		disable_txq_db(&adap->sge.ctrlq[i].q);
3925 }
3926 
3927 static void enable_dbs(struct adapter *adap)
3928 {
3929 	int i;
3930 
3931 	for_each_ethrxq(&adap->sge, i)
3932 		enable_txq_db(adap, &adap->sge.ethtxq[i].q);
3933 	for_each_ofldrxq(&adap->sge, i)
3934 		enable_txq_db(adap, &adap->sge.ofldtxq[i].q);
3935 	for_each_port(adap, i)
3936 		enable_txq_db(adap, &adap->sge.ctrlq[i].q);
3937 }
3938 
3939 static void notify_rdma_uld(struct adapter *adap, enum cxgb4_control cmd)
3940 {
3941 	if (adap->uld_handle[CXGB4_ULD_RDMA])
3942 		ulds[CXGB4_ULD_RDMA].control(adap->uld_handle[CXGB4_ULD_RDMA],
3943 				cmd);
3944 }
3945 
3946 static void process_db_full(struct work_struct *work)
3947 {
3948 	struct adapter *adap;
3949 
3950 	adap = container_of(work, struct adapter, db_full_task);
3951 
3952 	drain_db_fifo(adap, dbfifo_drain_delay);
3953 	enable_dbs(adap);
3954 	notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
3955 	t4_set_reg_field(adap, SGE_INT_ENABLE3,
3956 			 DBFIFO_HP_INT | DBFIFO_LP_INT,
3957 			 DBFIFO_HP_INT | DBFIFO_LP_INT);
3958 }
3959 
3960 static void sync_txq_pidx(struct adapter *adap, struct sge_txq *q)
3961 {
3962 	u16 hw_pidx, hw_cidx;
3963 	int ret;
3964 
3965 	spin_lock_irq(&q->db_lock);
3966 	ret = read_eq_indices(adap, (u16)q->cntxt_id, &hw_pidx, &hw_cidx);
3967 	if (ret)
3968 		goto out;
3969 	if (q->db_pidx != hw_pidx) {
3970 		u16 delta;
3971 
3972 		if (q->db_pidx >= hw_pidx)
3973 			delta = q->db_pidx - hw_pidx;
3974 		else
3975 			delta = q->size - hw_pidx + q->db_pidx;
3976 		wmb();
3977 		t4_write_reg(adap, MYPF_REG(SGE_PF_KDOORBELL),
3978 			     QID(q->cntxt_id) | PIDX(delta));
3979 	}
3980 out:
3981 	q->db_disabled = 0;
3982 	q->db_pidx_inc = 0;
3983 	spin_unlock_irq(&q->db_lock);
3984 	if (ret)
3985 		CH_WARN(adap, "DB drop recovery failed.\n");
3986 }
3987 static void recover_all_queues(struct adapter *adap)
3988 {
3989 	int i;
3990 
3991 	for_each_ethrxq(&adap->sge, i)
3992 		sync_txq_pidx(adap, &adap->sge.ethtxq[i].q);
3993 	for_each_ofldrxq(&adap->sge, i)
3994 		sync_txq_pidx(adap, &adap->sge.ofldtxq[i].q);
3995 	for_each_port(adap, i)
3996 		sync_txq_pidx(adap, &adap->sge.ctrlq[i].q);
3997 }
3998 
3999 static void process_db_drop(struct work_struct *work)
4000 {
4001 	struct adapter *adap;
4002 
4003 	adap = container_of(work, struct adapter, db_drop_task);
4004 
4005 	if (is_t4(adap->params.chip)) {
4006 		drain_db_fifo(adap, dbfifo_drain_delay);
4007 		notify_rdma_uld(adap, CXGB4_CONTROL_DB_DROP);
4008 		drain_db_fifo(adap, dbfifo_drain_delay);
4009 		recover_all_queues(adap);
4010 		drain_db_fifo(adap, dbfifo_drain_delay);
4011 		enable_dbs(adap);
4012 		notify_rdma_uld(adap, CXGB4_CONTROL_DB_EMPTY);
4013 	} else {
4014 		u32 dropped_db = t4_read_reg(adap, 0x010ac);
4015 		u16 qid = (dropped_db >> 15) & 0x1ffff;
4016 		u16 pidx_inc = dropped_db & 0x1fff;
4017 		u64 bar2_qoffset;
4018 		unsigned int bar2_qid;
4019 		int ret;
4020 
4021 		ret = cxgb4_t4_bar2_sge_qregs(adap, qid, T4_BAR2_QTYPE_EGRESS,
4022 					&bar2_qoffset, &bar2_qid);
4023 		if (ret)
4024 			dev_err(adap->pdev_dev, "doorbell drop recovery: "
4025 				"qid=%d, pidx_inc=%d\n", qid, pidx_inc);
4026 		else
4027 			writel(PIDX_T5(pidx_inc) | QID(bar2_qid),
4028 			       adap->bar2 + bar2_qoffset + SGE_UDB_KDOORBELL);
4029 
4030 		/* Re-enable BAR2 WC */
4031 		t4_set_reg_field(adap, 0x10b0, 1<<15, 1<<15);
4032 	}
4033 
4034 	t4_set_reg_field(adap, A_SGE_DOORBELL_CONTROL, F_DROPPED_DB, 0);
4035 }
4036 
4037 void t4_db_full(struct adapter *adap)
4038 {
4039 	if (is_t4(adap->params.chip)) {
4040 		disable_dbs(adap);
4041 		notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
4042 		t4_set_reg_field(adap, SGE_INT_ENABLE3,
4043 				 DBFIFO_HP_INT | DBFIFO_LP_INT, 0);
4044 		queue_work(adap->workq, &adap->db_full_task);
4045 	}
4046 }
4047 
4048 void t4_db_dropped(struct adapter *adap)
4049 {
4050 	if (is_t4(adap->params.chip)) {
4051 		disable_dbs(adap);
4052 		notify_rdma_uld(adap, CXGB4_CONTROL_DB_FULL);
4053 	}
4054 	queue_work(adap->workq, &adap->db_drop_task);
4055 }
4056 
4057 static void uld_attach(struct adapter *adap, unsigned int uld)
4058 {
4059 	void *handle;
4060 	struct cxgb4_lld_info lli;
4061 	unsigned short i;
4062 
4063 	lli.pdev = adap->pdev;
4064 	lli.pf = adap->fn;
4065 	lli.l2t = adap->l2t;
4066 	lli.tids = &adap->tids;
4067 	lli.ports = adap->port;
4068 	lli.vr = &adap->vres;
4069 	lli.mtus = adap->params.mtus;
4070 	if (uld == CXGB4_ULD_RDMA) {
4071 		lli.rxq_ids = adap->sge.rdma_rxq;
4072 		lli.ciq_ids = adap->sge.rdma_ciq;
4073 		lli.nrxq = adap->sge.rdmaqs;
4074 		lli.nciq = adap->sge.rdmaciqs;
4075 	} else if (uld == CXGB4_ULD_ISCSI) {
4076 		lli.rxq_ids = adap->sge.ofld_rxq;
4077 		lli.nrxq = adap->sge.ofldqsets;
4078 	}
4079 	lli.ntxq = adap->sge.ofldqsets;
4080 	lli.nchan = adap->params.nports;
4081 	lli.nports = adap->params.nports;
4082 	lli.wr_cred = adap->params.ofldq_wr_cred;
4083 	lli.adapter_type = adap->params.chip;
4084 	lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
4085 	lli.cclk_ps = 1000000000 / adap->params.vpd.cclk;
4086 	lli.udb_density = 1 << adap->params.sge.eq_qpp;
4087 	lli.ucq_density = 1 << adap->params.sge.iq_qpp;
4088 	lli.filt_mode = adap->params.tp.vlan_pri_map;
4089 	/* MODQ_REQ_MAP sets queues 0-3 to chan 0-3 */
4090 	for (i = 0; i < NCHAN; i++)
4091 		lli.tx_modq[i] = i;
4092 	lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
4093 	lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
4094 	lli.fw_vers = adap->params.fw_vers;
4095 	lli.dbfifo_int_thresh = dbfifo_int_thresh;
4096 	lli.sge_ingpadboundary = adap->sge.fl_align;
4097 	lli.sge_egrstatuspagesize = adap->sge.stat_len;
4098 	lli.sge_pktshift = adap->sge.pktshift;
4099 	lli.enable_fw_ofld_conn = adap->flags & FW_OFLD_CONN;
4100 	lli.max_ordird_qp = adap->params.max_ordird_qp;
4101 	lli.max_ird_adapter = adap->params.max_ird_adapter;
4102 	lli.ulptx_memwrite_dsgl = adap->params.ulptx_memwrite_dsgl;
4103 
4104 	handle = ulds[uld].add(&lli);
4105 	if (IS_ERR(handle)) {
4106 		dev_warn(adap->pdev_dev,
4107 			 "could not attach to the %s driver, error %ld\n",
4108 			 uld_str[uld], PTR_ERR(handle));
4109 		return;
4110 	}
4111 
4112 	adap->uld_handle[uld] = handle;
4113 
4114 	if (!netevent_registered) {
4115 		register_netevent_notifier(&cxgb4_netevent_nb);
4116 		netevent_registered = true;
4117 	}
4118 
4119 	if (adap->flags & FULL_INIT_DONE)
4120 		ulds[uld].state_change(handle, CXGB4_STATE_UP);
4121 }
4122 
4123 static void attach_ulds(struct adapter *adap)
4124 {
4125 	unsigned int i;
4126 
4127 	spin_lock(&adap_rcu_lock);
4128 	list_add_tail_rcu(&adap->rcu_node, &adap_rcu_list);
4129 	spin_unlock(&adap_rcu_lock);
4130 
4131 	mutex_lock(&uld_mutex);
4132 	list_add_tail(&adap->list_node, &adapter_list);
4133 	for (i = 0; i < CXGB4_ULD_MAX; i++)
4134 		if (ulds[i].add)
4135 			uld_attach(adap, i);
4136 	mutex_unlock(&uld_mutex);
4137 }
4138 
4139 static void detach_ulds(struct adapter *adap)
4140 {
4141 	unsigned int i;
4142 
4143 	mutex_lock(&uld_mutex);
4144 	list_del(&adap->list_node);
4145 	for (i = 0; i < CXGB4_ULD_MAX; i++)
4146 		if (adap->uld_handle[i]) {
4147 			ulds[i].state_change(adap->uld_handle[i],
4148 					     CXGB4_STATE_DETACH);
4149 			adap->uld_handle[i] = NULL;
4150 		}
4151 	if (netevent_registered && list_empty(&adapter_list)) {
4152 		unregister_netevent_notifier(&cxgb4_netevent_nb);
4153 		netevent_registered = false;
4154 	}
4155 	mutex_unlock(&uld_mutex);
4156 
4157 	spin_lock(&adap_rcu_lock);
4158 	list_del_rcu(&adap->rcu_node);
4159 	spin_unlock(&adap_rcu_lock);
4160 }
4161 
4162 static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
4163 {
4164 	unsigned int i;
4165 
4166 	mutex_lock(&uld_mutex);
4167 	for (i = 0; i < CXGB4_ULD_MAX; i++)
4168 		if (adap->uld_handle[i])
4169 			ulds[i].state_change(adap->uld_handle[i], new_state);
4170 	mutex_unlock(&uld_mutex);
4171 }
4172 
4173 /**
4174  *	cxgb4_register_uld - register an upper-layer driver
4175  *	@type: the ULD type
4176  *	@p: the ULD methods
4177  *
4178  *	Registers an upper-layer driver with this driver and notifies the ULD
4179  *	about any presently available devices that support its type.  Returns
4180  *	%-EBUSY if a ULD of the same type is already registered.
4181  */
4182 int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
4183 {
4184 	int ret = 0;
4185 	struct adapter *adap;
4186 
4187 	if (type >= CXGB4_ULD_MAX)
4188 		return -EINVAL;
4189 	mutex_lock(&uld_mutex);
4190 	if (ulds[type].add) {
4191 		ret = -EBUSY;
4192 		goto out;
4193 	}
4194 	ulds[type] = *p;
4195 	list_for_each_entry(adap, &adapter_list, list_node)
4196 		uld_attach(adap, type);
4197 out:	mutex_unlock(&uld_mutex);
4198 	return ret;
4199 }
4200 EXPORT_SYMBOL(cxgb4_register_uld);
4201 
4202 /**
4203  *	cxgb4_unregister_uld - unregister an upper-layer driver
4204  *	@type: the ULD type
4205  *
4206  *	Unregisters an existing upper-layer driver.
4207  */
4208 int cxgb4_unregister_uld(enum cxgb4_uld type)
4209 {
4210 	struct adapter *adap;
4211 
4212 	if (type >= CXGB4_ULD_MAX)
4213 		return -EINVAL;
4214 	mutex_lock(&uld_mutex);
4215 	list_for_each_entry(adap, &adapter_list, list_node)
4216 		adap->uld_handle[type] = NULL;
4217 	ulds[type].add = NULL;
4218 	mutex_unlock(&uld_mutex);
4219 	return 0;
4220 }
4221 EXPORT_SYMBOL(cxgb4_unregister_uld);
4222 
4223 /* Check if netdev on which event is occured belongs to us or not. Return
4224  * success (true) if it belongs otherwise failure (false).
4225  * Called with rcu_read_lock() held.
4226  */
4227 #if IS_ENABLED(CONFIG_IPV6)
4228 static bool cxgb4_netdev(const struct net_device *netdev)
4229 {
4230 	struct adapter *adap;
4231 	int i;
4232 
4233 	list_for_each_entry_rcu(adap, &adap_rcu_list, rcu_node)
4234 		for (i = 0; i < MAX_NPORTS; i++)
4235 			if (adap->port[i] == netdev)
4236 				return true;
4237 	return false;
4238 }
4239 
4240 static int clip_add(struct net_device *event_dev, struct inet6_ifaddr *ifa,
4241 		    unsigned long event)
4242 {
4243 	int ret = NOTIFY_DONE;
4244 
4245 	rcu_read_lock();
4246 	if (cxgb4_netdev(event_dev)) {
4247 		switch (event) {
4248 		case NETDEV_UP:
4249 			ret = cxgb4_clip_get(event_dev, &ifa->addr);
4250 			if (ret < 0) {
4251 				rcu_read_unlock();
4252 				return ret;
4253 			}
4254 			ret = NOTIFY_OK;
4255 			break;
4256 		case NETDEV_DOWN:
4257 			cxgb4_clip_release(event_dev, &ifa->addr);
4258 			ret = NOTIFY_OK;
4259 			break;
4260 		default:
4261 			break;
4262 		}
4263 	}
4264 	rcu_read_unlock();
4265 	return ret;
4266 }
4267 
4268 static int cxgb4_inet6addr_handler(struct notifier_block *this,
4269 		unsigned long event, void *data)
4270 {
4271 	struct inet6_ifaddr *ifa = data;
4272 	struct net_device *event_dev;
4273 	int ret = NOTIFY_DONE;
4274 	struct bonding *bond = netdev_priv(ifa->idev->dev);
4275 	struct list_head *iter;
4276 	struct slave *slave;
4277 	struct pci_dev *first_pdev = NULL;
4278 
4279 	if (ifa->idev->dev->priv_flags & IFF_802_1Q_VLAN) {
4280 		event_dev = vlan_dev_real_dev(ifa->idev->dev);
4281 		ret = clip_add(event_dev, ifa, event);
4282 	} else if (ifa->idev->dev->flags & IFF_MASTER) {
4283 		/* It is possible that two different adapters are bonded in one
4284 		 * bond. We need to find such different adapters and add clip
4285 		 * in all of them only once.
4286 		 */
4287 		bond_for_each_slave(bond, slave, iter) {
4288 			if (!first_pdev) {
4289 				ret = clip_add(slave->dev, ifa, event);
4290 				/* If clip_add is success then only initialize
4291 				 * first_pdev since it means it is our device
4292 				 */
4293 				if (ret == NOTIFY_OK)
4294 					first_pdev = to_pci_dev(
4295 							slave->dev->dev.parent);
4296 			} else if (first_pdev !=
4297 				   to_pci_dev(slave->dev->dev.parent))
4298 					ret = clip_add(slave->dev, ifa, event);
4299 		}
4300 	} else
4301 		ret = clip_add(ifa->idev->dev, ifa, event);
4302 
4303 	return ret;
4304 }
4305 
4306 static struct notifier_block cxgb4_inet6addr_notifier = {
4307 	.notifier_call = cxgb4_inet6addr_handler
4308 };
4309 
4310 /* Retrieves IPv6 addresses from a root device (bond, vlan) associated with
4311  * a physical device.
4312  * The physical device reference is needed to send the actul CLIP command.
4313  */
4314 static int update_dev_clip(struct net_device *root_dev, struct net_device *dev)
4315 {
4316 	struct inet6_dev *idev = NULL;
4317 	struct inet6_ifaddr *ifa;
4318 	int ret = 0;
4319 
4320 	idev = __in6_dev_get(root_dev);
4321 	if (!idev)
4322 		return ret;
4323 
4324 	read_lock_bh(&idev->lock);
4325 	list_for_each_entry(ifa, &idev->addr_list, if_list) {
4326 		ret = cxgb4_clip_get(dev, &ifa->addr);
4327 		if (ret < 0)
4328 			break;
4329 	}
4330 	read_unlock_bh(&idev->lock);
4331 
4332 	return ret;
4333 }
4334 
4335 static int update_root_dev_clip(struct net_device *dev)
4336 {
4337 	struct net_device *root_dev = NULL;
4338 	int i, ret = 0;
4339 
4340 	/* First populate the real net device's IPv6 addresses */
4341 	ret = update_dev_clip(dev, dev);
4342 	if (ret)
4343 		return ret;
4344 
4345 	/* Parse all bond and vlan devices layered on top of the physical dev */
4346 	root_dev = netdev_master_upper_dev_get_rcu(dev);
4347 	if (root_dev) {
4348 		ret = update_dev_clip(root_dev, dev);
4349 		if (ret)
4350 			return ret;
4351 	}
4352 
4353 	for (i = 0; i < VLAN_N_VID; i++) {
4354 		root_dev = __vlan_find_dev_deep_rcu(dev, htons(ETH_P_8021Q), i);
4355 		if (!root_dev)
4356 			continue;
4357 
4358 		ret = update_dev_clip(root_dev, dev);
4359 		if (ret)
4360 			break;
4361 	}
4362 	return ret;
4363 }
4364 
4365 static void update_clip(const struct adapter *adap)
4366 {
4367 	int i;
4368 	struct net_device *dev;
4369 	int ret;
4370 
4371 	rcu_read_lock();
4372 
4373 	for (i = 0; i < MAX_NPORTS; i++) {
4374 		dev = adap->port[i];
4375 		ret = 0;
4376 
4377 		if (dev)
4378 			ret = update_root_dev_clip(dev);
4379 
4380 		if (ret < 0)
4381 			break;
4382 	}
4383 	rcu_read_unlock();
4384 }
4385 #endif /* IS_ENABLED(CONFIG_IPV6) */
4386 
4387 /**
4388  *	cxgb_up - enable the adapter
4389  *	@adap: adapter being enabled
4390  *
4391  *	Called when the first port is enabled, this function performs the
4392  *	actions necessary to make an adapter operational, such as completing
4393  *	the initialization of HW modules, and enabling interrupts.
4394  *
4395  *	Must be called with the rtnl lock held.
4396  */
4397 static int cxgb_up(struct adapter *adap)
4398 {
4399 	int err;
4400 
4401 	err = setup_sge_queues(adap);
4402 	if (err)
4403 		goto out;
4404 	err = setup_rss(adap);
4405 	if (err)
4406 		goto freeq;
4407 
4408 	if (adap->flags & USING_MSIX) {
4409 		name_msix_vecs(adap);
4410 		err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
4411 				  adap->msix_info[0].desc, adap);
4412 		if (err)
4413 			goto irq_err;
4414 
4415 		err = request_msix_queue_irqs(adap);
4416 		if (err) {
4417 			free_irq(adap->msix_info[0].vec, adap);
4418 			goto irq_err;
4419 		}
4420 	} else {
4421 		err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
4422 				  (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
4423 				  adap->port[0]->name, adap);
4424 		if (err)
4425 			goto irq_err;
4426 	}
4427 	enable_rx(adap);
4428 	t4_sge_start(adap);
4429 	t4_intr_enable(adap);
4430 	adap->flags |= FULL_INIT_DONE;
4431 	notify_ulds(adap, CXGB4_STATE_UP);
4432 #if IS_ENABLED(CONFIG_IPV6)
4433 	update_clip(adap);
4434 #endif
4435  out:
4436 	return err;
4437  irq_err:
4438 	dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
4439  freeq:
4440 	t4_free_sge_resources(adap);
4441 	goto out;
4442 }
4443 
4444 static void cxgb_down(struct adapter *adapter)
4445 {
4446 	t4_intr_disable(adapter);
4447 	cancel_work_sync(&adapter->tid_release_task);
4448 	cancel_work_sync(&adapter->db_full_task);
4449 	cancel_work_sync(&adapter->db_drop_task);
4450 	adapter->tid_release_task_busy = false;
4451 	adapter->tid_release_head = NULL;
4452 
4453 	if (adapter->flags & USING_MSIX) {
4454 		free_msix_queue_irqs(adapter);
4455 		free_irq(adapter->msix_info[0].vec, adapter);
4456 	} else
4457 		free_irq(adapter->pdev->irq, adapter);
4458 	quiesce_rx(adapter);
4459 	t4_sge_stop(adapter);
4460 	t4_free_sge_resources(adapter);
4461 	adapter->flags &= ~FULL_INIT_DONE;
4462 }
4463 
4464 /*
4465  * net_device operations
4466  */
4467 static int cxgb_open(struct net_device *dev)
4468 {
4469 	int err;
4470 	struct port_info *pi = netdev_priv(dev);
4471 	struct adapter *adapter = pi->adapter;
4472 
4473 	netif_carrier_off(dev);
4474 
4475 	if (!(adapter->flags & FULL_INIT_DONE)) {
4476 		err = cxgb_up(adapter);
4477 		if (err < 0)
4478 			return err;
4479 	}
4480 
4481 	err = link_start(dev);
4482 	if (!err)
4483 		netif_tx_start_all_queues(dev);
4484 	return err;
4485 }
4486 
4487 static int cxgb_close(struct net_device *dev)
4488 {
4489 	struct port_info *pi = netdev_priv(dev);
4490 	struct adapter *adapter = pi->adapter;
4491 
4492 	netif_tx_stop_all_queues(dev);
4493 	netif_carrier_off(dev);
4494 	return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
4495 }
4496 
4497 /* Return an error number if the indicated filter isn't writable ...
4498  */
4499 static int writable_filter(struct filter_entry *f)
4500 {
4501 	if (f->locked)
4502 		return -EPERM;
4503 	if (f->pending)
4504 		return -EBUSY;
4505 
4506 	return 0;
4507 }
4508 
4509 /* Delete the filter at the specified index (if valid).  The checks for all
4510  * the common problems with doing this like the filter being locked, currently
4511  * pending in another operation, etc.
4512  */
4513 static int delete_filter(struct adapter *adapter, unsigned int fidx)
4514 {
4515 	struct filter_entry *f;
4516 	int ret;
4517 
4518 	if (fidx >= adapter->tids.nftids + adapter->tids.nsftids)
4519 		return -EINVAL;
4520 
4521 	f = &adapter->tids.ftid_tab[fidx];
4522 	ret = writable_filter(f);
4523 	if (ret)
4524 		return ret;
4525 	if (f->valid)
4526 		return del_filter_wr(adapter, fidx);
4527 
4528 	return 0;
4529 }
4530 
4531 int cxgb4_create_server_filter(const struct net_device *dev, unsigned int stid,
4532 		__be32 sip, __be16 sport, __be16 vlan,
4533 		unsigned int queue, unsigned char port, unsigned char mask)
4534 {
4535 	int ret;
4536 	struct filter_entry *f;
4537 	struct adapter *adap;
4538 	int i;
4539 	u8 *val;
4540 
4541 	adap = netdev2adap(dev);
4542 
4543 	/* Adjust stid to correct filter index */
4544 	stid -= adap->tids.sftid_base;
4545 	stid += adap->tids.nftids;
4546 
4547 	/* Check to make sure the filter requested is writable ...
4548 	 */
4549 	f = &adap->tids.ftid_tab[stid];
4550 	ret = writable_filter(f);
4551 	if (ret)
4552 		return ret;
4553 
4554 	/* Clear out any old resources being used by the filter before
4555 	 * we start constructing the new filter.
4556 	 */
4557 	if (f->valid)
4558 		clear_filter(adap, f);
4559 
4560 	/* Clear out filter specifications */
4561 	memset(&f->fs, 0, sizeof(struct ch_filter_specification));
4562 	f->fs.val.lport = cpu_to_be16(sport);
4563 	f->fs.mask.lport  = ~0;
4564 	val = (u8 *)&sip;
4565 	if ((val[0] | val[1] | val[2] | val[3]) != 0) {
4566 		for (i = 0; i < 4; i++) {
4567 			f->fs.val.lip[i] = val[i];
4568 			f->fs.mask.lip[i] = ~0;
4569 		}
4570 		if (adap->params.tp.vlan_pri_map & F_PORT) {
4571 			f->fs.val.iport = port;
4572 			f->fs.mask.iport = mask;
4573 		}
4574 	}
4575 
4576 	if (adap->params.tp.vlan_pri_map & F_PROTOCOL) {
4577 		f->fs.val.proto = IPPROTO_TCP;
4578 		f->fs.mask.proto = ~0;
4579 	}
4580 
4581 	f->fs.dirsteer = 1;
4582 	f->fs.iq = queue;
4583 	/* Mark filter as locked */
4584 	f->locked = 1;
4585 	f->fs.rpttid = 1;
4586 
4587 	ret = set_filter_wr(adap, stid);
4588 	if (ret) {
4589 		clear_filter(adap, f);
4590 		return ret;
4591 	}
4592 
4593 	return 0;
4594 }
4595 EXPORT_SYMBOL(cxgb4_create_server_filter);
4596 
4597 int cxgb4_remove_server_filter(const struct net_device *dev, unsigned int stid,
4598 		unsigned int queue, bool ipv6)
4599 {
4600 	int ret;
4601 	struct filter_entry *f;
4602 	struct adapter *adap;
4603 
4604 	adap = netdev2adap(dev);
4605 
4606 	/* Adjust stid to correct filter index */
4607 	stid -= adap->tids.sftid_base;
4608 	stid += adap->tids.nftids;
4609 
4610 	f = &adap->tids.ftid_tab[stid];
4611 	/* Unlock the filter */
4612 	f->locked = 0;
4613 
4614 	ret = delete_filter(adap, stid);
4615 	if (ret)
4616 		return ret;
4617 
4618 	return 0;
4619 }
4620 EXPORT_SYMBOL(cxgb4_remove_server_filter);
4621 
4622 static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
4623 						struct rtnl_link_stats64 *ns)
4624 {
4625 	struct port_stats stats;
4626 	struct port_info *p = netdev_priv(dev);
4627 	struct adapter *adapter = p->adapter;
4628 
4629 	/* Block retrieving statistics during EEH error
4630 	 * recovery. Otherwise, the recovery might fail
4631 	 * and the PCI device will be removed permanently
4632 	 */
4633 	spin_lock(&adapter->stats_lock);
4634 	if (!netif_device_present(dev)) {
4635 		spin_unlock(&adapter->stats_lock);
4636 		return ns;
4637 	}
4638 	t4_get_port_stats(adapter, p->tx_chan, &stats);
4639 	spin_unlock(&adapter->stats_lock);
4640 
4641 	ns->tx_bytes   = stats.tx_octets;
4642 	ns->tx_packets = stats.tx_frames;
4643 	ns->rx_bytes   = stats.rx_octets;
4644 	ns->rx_packets = stats.rx_frames;
4645 	ns->multicast  = stats.rx_mcast_frames;
4646 
4647 	/* detailed rx_errors */
4648 	ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
4649 			       stats.rx_runt;
4650 	ns->rx_over_errors   = 0;
4651 	ns->rx_crc_errors    = stats.rx_fcs_err;
4652 	ns->rx_frame_errors  = stats.rx_symbol_err;
4653 	ns->rx_fifo_errors   = stats.rx_ovflow0 + stats.rx_ovflow1 +
4654 			       stats.rx_ovflow2 + stats.rx_ovflow3 +
4655 			       stats.rx_trunc0 + stats.rx_trunc1 +
4656 			       stats.rx_trunc2 + stats.rx_trunc3;
4657 	ns->rx_missed_errors = 0;
4658 
4659 	/* detailed tx_errors */
4660 	ns->tx_aborted_errors   = 0;
4661 	ns->tx_carrier_errors   = 0;
4662 	ns->tx_fifo_errors      = 0;
4663 	ns->tx_heartbeat_errors = 0;
4664 	ns->tx_window_errors    = 0;
4665 
4666 	ns->tx_errors = stats.tx_error_frames;
4667 	ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
4668 		ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
4669 	return ns;
4670 }
4671 
4672 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
4673 {
4674 	unsigned int mbox;
4675 	int ret = 0, prtad, devad;
4676 	struct port_info *pi = netdev_priv(dev);
4677 	struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
4678 
4679 	switch (cmd) {
4680 	case SIOCGMIIPHY:
4681 		if (pi->mdio_addr < 0)
4682 			return -EOPNOTSUPP;
4683 		data->phy_id = pi->mdio_addr;
4684 		break;
4685 	case SIOCGMIIREG:
4686 	case SIOCSMIIREG:
4687 		if (mdio_phy_id_is_c45(data->phy_id)) {
4688 			prtad = mdio_phy_id_prtad(data->phy_id);
4689 			devad = mdio_phy_id_devad(data->phy_id);
4690 		} else if (data->phy_id < 32) {
4691 			prtad = data->phy_id;
4692 			devad = 0;
4693 			data->reg_num &= 0x1f;
4694 		} else
4695 			return -EINVAL;
4696 
4697 		mbox = pi->adapter->fn;
4698 		if (cmd == SIOCGMIIREG)
4699 			ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
4700 					 data->reg_num, &data->val_out);
4701 		else
4702 			ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
4703 					 data->reg_num, data->val_in);
4704 		break;
4705 	default:
4706 		return -EOPNOTSUPP;
4707 	}
4708 	return ret;
4709 }
4710 
4711 static void cxgb_set_rxmode(struct net_device *dev)
4712 {
4713 	/* unfortunately we can't return errors to the stack */
4714 	set_rxmode(dev, -1, false);
4715 }
4716 
4717 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
4718 {
4719 	int ret;
4720 	struct port_info *pi = netdev_priv(dev);
4721 
4722 	if (new_mtu < 81 || new_mtu > MAX_MTU)         /* accommodate SACK */
4723 		return -EINVAL;
4724 	ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
4725 			    -1, -1, -1, true);
4726 	if (!ret)
4727 		dev->mtu = new_mtu;
4728 	return ret;
4729 }
4730 
4731 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
4732 {
4733 	int ret;
4734 	struct sockaddr *addr = p;
4735 	struct port_info *pi = netdev_priv(dev);
4736 
4737 	if (!is_valid_ether_addr(addr->sa_data))
4738 		return -EADDRNOTAVAIL;
4739 
4740 	ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
4741 			    pi->xact_addr_filt, addr->sa_data, true, true);
4742 	if (ret < 0)
4743 		return ret;
4744 
4745 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
4746 	pi->xact_addr_filt = ret;
4747 	return 0;
4748 }
4749 
4750 #ifdef CONFIG_NET_POLL_CONTROLLER
4751 static void cxgb_netpoll(struct net_device *dev)
4752 {
4753 	struct port_info *pi = netdev_priv(dev);
4754 	struct adapter *adap = pi->adapter;
4755 
4756 	if (adap->flags & USING_MSIX) {
4757 		int i;
4758 		struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
4759 
4760 		for (i = pi->nqsets; i; i--, rx++)
4761 			t4_sge_intr_msix(0, &rx->rspq);
4762 	} else
4763 		t4_intr_handler(adap)(0, adap);
4764 }
4765 #endif
4766 
4767 static const struct net_device_ops cxgb4_netdev_ops = {
4768 	.ndo_open             = cxgb_open,
4769 	.ndo_stop             = cxgb_close,
4770 	.ndo_start_xmit       = t4_eth_xmit,
4771 	.ndo_select_queue     =	cxgb_select_queue,
4772 	.ndo_get_stats64      = cxgb_get_stats,
4773 	.ndo_set_rx_mode      = cxgb_set_rxmode,
4774 	.ndo_set_mac_address  = cxgb_set_mac_addr,
4775 	.ndo_set_features     = cxgb_set_features,
4776 	.ndo_validate_addr    = eth_validate_addr,
4777 	.ndo_do_ioctl         = cxgb_ioctl,
4778 	.ndo_change_mtu       = cxgb_change_mtu,
4779 #ifdef CONFIG_NET_POLL_CONTROLLER
4780 	.ndo_poll_controller  = cxgb_netpoll,
4781 #endif
4782 };
4783 
4784 void t4_fatal_err(struct adapter *adap)
4785 {
4786 	t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
4787 	t4_intr_disable(adap);
4788 	dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
4789 }
4790 
4791 /* Return the specified PCI-E Configuration Space register from our Physical
4792  * Function.  We try first via a Firmware LDST Command since we prefer to let
4793  * the firmware own all of these registers, but if that fails we go for it
4794  * directly ourselves.
4795  */
4796 static u32 t4_read_pcie_cfg4(struct adapter *adap, int reg)
4797 {
4798 	struct fw_ldst_cmd ldst_cmd;
4799 	u32 val;
4800 	int ret;
4801 
4802 	/* Construct and send the Firmware LDST Command to retrieve the
4803 	 * specified PCI-E Configuration Space register.
4804 	 */
4805 	memset(&ldst_cmd, 0, sizeof(ldst_cmd));
4806 	ldst_cmd.op_to_addrspace =
4807 		htonl(FW_CMD_OP_V(FW_LDST_CMD) |
4808 		      FW_CMD_REQUEST_F |
4809 		      FW_CMD_READ_F |
4810 		      FW_LDST_CMD_ADDRSPACE_V(FW_LDST_ADDRSPC_FUNC_PCIE));
4811 	ldst_cmd.cycles_to_len16 = htonl(FW_LEN16(ldst_cmd));
4812 	ldst_cmd.u.pcie.select_naccess = FW_LDST_CMD_NACCESS_V(1);
4813 	ldst_cmd.u.pcie.ctrl_to_fn =
4814 		(FW_LDST_CMD_LC_F | FW_LDST_CMD_FN_V(adap->fn));
4815 	ldst_cmd.u.pcie.r = reg;
4816 	ret = t4_wr_mbox(adap, adap->mbox, &ldst_cmd, sizeof(ldst_cmd),
4817 			 &ldst_cmd);
4818 
4819 	/* If the LDST Command suucceeded, exctract the returned register
4820 	 * value.  Otherwise read it directly ourself.
4821 	 */
4822 	if (ret == 0)
4823 		val = ntohl(ldst_cmd.u.pcie.data[0]);
4824 	else
4825 		t4_hw_pci_read_cfg4(adap, reg, &val);
4826 
4827 	return val;
4828 }
4829 
4830 static void setup_memwin(struct adapter *adap)
4831 {
4832 	u32 mem_win0_base, mem_win1_base, mem_win2_base, mem_win2_aperture;
4833 
4834 	if (is_t4(adap->params.chip)) {
4835 		u32 bar0;
4836 
4837 		/* Truncation intentional: we only read the bottom 32-bits of
4838 		 * the 64-bit BAR0/BAR1 ...  We use the hardware backdoor
4839 		 * mechanism to read BAR0 instead of using
4840 		 * pci_resource_start() because we could be operating from
4841 		 * within a Virtual Machine which is trapping our accesses to
4842 		 * our Configuration Space and we need to set up the PCI-E
4843 		 * Memory Window decoders with the actual addresses which will
4844 		 * be coming across the PCI-E link.
4845 		 */
4846 		bar0 = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_0);
4847 		bar0 &= PCI_BASE_ADDRESS_MEM_MASK;
4848 		adap->t4_bar0 = bar0;
4849 
4850 		mem_win0_base = bar0 + MEMWIN0_BASE;
4851 		mem_win1_base = bar0 + MEMWIN1_BASE;
4852 		mem_win2_base = bar0 + MEMWIN2_BASE;
4853 		mem_win2_aperture = MEMWIN2_APERTURE;
4854 	} else {
4855 		/* For T5, only relative offset inside the PCIe BAR is passed */
4856 		mem_win0_base = MEMWIN0_BASE;
4857 		mem_win1_base = MEMWIN1_BASE;
4858 		mem_win2_base = MEMWIN2_BASE_T5;
4859 		mem_win2_aperture = MEMWIN2_APERTURE_T5;
4860 	}
4861 	t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
4862 		     mem_win0_base | BIR(0) |
4863 		     WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
4864 	t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
4865 		     mem_win1_base | BIR(0) |
4866 		     WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
4867 	t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
4868 		     mem_win2_base | BIR(0) |
4869 		     WINDOW(ilog2(mem_win2_aperture) - 10));
4870 	t4_read_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2));
4871 }
4872 
4873 static void setup_memwin_rdma(struct adapter *adap)
4874 {
4875 	if (adap->vres.ocq.size) {
4876 		u32 start;
4877 		unsigned int sz_kb;
4878 
4879 		start = t4_read_pcie_cfg4(adap, PCI_BASE_ADDRESS_2);
4880 		start &= PCI_BASE_ADDRESS_MEM_MASK;
4881 		start += OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
4882 		sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
4883 		t4_write_reg(adap,
4884 			     PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
4885 			     start | BIR(1) | WINDOW(ilog2(sz_kb)));
4886 		t4_write_reg(adap,
4887 			     PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
4888 			     adap->vres.ocq.start);
4889 		t4_read_reg(adap,
4890 			    PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
4891 	}
4892 }
4893 
4894 static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
4895 {
4896 	u32 v;
4897 	int ret;
4898 
4899 	/* get device capabilities */
4900 	memset(c, 0, sizeof(*c));
4901 	c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4902 			       FW_CMD_REQUEST_F | FW_CMD_READ_F);
4903 	c->cfvalid_to_len16 = htonl(FW_LEN16(*c));
4904 	ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
4905 	if (ret < 0)
4906 		return ret;
4907 
4908 	/* select capabilities we'll be using */
4909 	if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
4910 		if (!vf_acls)
4911 			c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
4912 		else
4913 			c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
4914 	} else if (vf_acls) {
4915 		dev_err(adap->pdev_dev, "virtualization ACLs not supported");
4916 		return ret;
4917 	}
4918 	c->op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
4919 			       FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
4920 	ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
4921 	if (ret < 0)
4922 		return ret;
4923 
4924 	ret = t4_config_glbl_rss(adap, adap->fn,
4925 				 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
4926 				 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
4927 				 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F);
4928 	if (ret < 0)
4929 		return ret;
4930 
4931 	ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
4932 			  0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
4933 	if (ret < 0)
4934 		return ret;
4935 
4936 	t4_sge_init(adap);
4937 
4938 	/* tweak some settings */
4939 	t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
4940 	t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
4941 	t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
4942 	v = t4_read_reg(adap, TP_PIO_DATA);
4943 	t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
4944 
4945 	/* first 4 Tx modulation queues point to consecutive Tx channels */
4946 	adap->params.tp.tx_modq_map = 0xE4;
4947 	t4_write_reg(adap, A_TP_TX_MOD_QUEUE_REQ_MAP,
4948 		     V_TX_MOD_QUEUE_REQ_MAP(adap->params.tp.tx_modq_map));
4949 
4950 	/* associate each Tx modulation queue with consecutive Tx channels */
4951 	v = 0x84218421;
4952 	t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4953 			  &v, 1, A_TP_TX_SCHED_HDR);
4954 	t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4955 			  &v, 1, A_TP_TX_SCHED_FIFO);
4956 	t4_write_indirect(adap, TP_PIO_ADDR, TP_PIO_DATA,
4957 			  &v, 1, A_TP_TX_SCHED_PCMD);
4958 
4959 #define T4_TX_MODQ_10G_WEIGHT_DEFAULT 16 /* in KB units */
4960 	if (is_offload(adap)) {
4961 		t4_write_reg(adap, A_TP_TX_MOD_QUEUE_WEIGHT0,
4962 			     V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4963 			     V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4964 			     V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4965 			     V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4966 		t4_write_reg(adap, A_TP_TX_MOD_CHANNEL_WEIGHT,
4967 			     V_TX_MODQ_WEIGHT0(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4968 			     V_TX_MODQ_WEIGHT1(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4969 			     V_TX_MODQ_WEIGHT2(T4_TX_MODQ_10G_WEIGHT_DEFAULT) |
4970 			     V_TX_MODQ_WEIGHT3(T4_TX_MODQ_10G_WEIGHT_DEFAULT));
4971 	}
4972 
4973 	/* get basic stuff going */
4974 	return t4_early_init(adap, adap->fn);
4975 }
4976 
4977 /*
4978  * Max # of ATIDs.  The absolute HW max is 16K but we keep it lower.
4979  */
4980 #define MAX_ATIDS 8192U
4981 
4982 /*
4983  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
4984  *
4985  * If the firmware we're dealing with has Configuration File support, then
4986  * we use that to perform all configuration
4987  */
4988 
4989 /*
4990  * Tweak configuration based on module parameters, etc.  Most of these have
4991  * defaults assigned to them by Firmware Configuration Files (if we're using
4992  * them) but need to be explicitly set if we're using hard-coded
4993  * initialization.  But even in the case of using Firmware Configuration
4994  * Files, we'd like to expose the ability to change these via module
4995  * parameters so these are essentially common tweaks/settings for
4996  * Configuration Files and hard-coded initialization ...
4997  */
4998 static int adap_init0_tweaks(struct adapter *adapter)
4999 {
5000 	/*
5001 	 * Fix up various Host-Dependent Parameters like Page Size, Cache
5002 	 * Line Size, etc.  The firmware default is for a 4KB Page Size and
5003 	 * 64B Cache Line Size ...
5004 	 */
5005 	t4_fixup_host_params(adapter, PAGE_SIZE, L1_CACHE_BYTES);
5006 
5007 	/*
5008 	 * Process module parameters which affect early initialization.
5009 	 */
5010 	if (rx_dma_offset != 2 && rx_dma_offset != 0) {
5011 		dev_err(&adapter->pdev->dev,
5012 			"Ignoring illegal rx_dma_offset=%d, using 2\n",
5013 			rx_dma_offset);
5014 		rx_dma_offset = 2;
5015 	}
5016 	t4_set_reg_field(adapter, SGE_CONTROL,
5017 			 PKTSHIFT_MASK,
5018 			 PKTSHIFT(rx_dma_offset));
5019 
5020 	/*
5021 	 * Don't include the "IP Pseudo Header" in CPL_RX_PKT checksums: Linux
5022 	 * adds the pseudo header itself.
5023 	 */
5024 	t4_tp_wr_bits_indirect(adapter, TP_INGRESS_CONFIG,
5025 			       CSUM_HAS_PSEUDO_HDR, 0);
5026 
5027 	return 0;
5028 }
5029 
5030 /*
5031  * Attempt to initialize the adapter via a Firmware Configuration File.
5032  */
5033 static int adap_init0_config(struct adapter *adapter, int reset)
5034 {
5035 	struct fw_caps_config_cmd caps_cmd;
5036 	const struct firmware *cf;
5037 	unsigned long mtype = 0, maddr = 0;
5038 	u32 finiver, finicsum, cfcsum;
5039 	int ret;
5040 	int config_issued = 0;
5041 	char *fw_config_file, fw_config_file_path[256];
5042 	char *config_name = NULL;
5043 
5044 	/*
5045 	 * Reset device if necessary.
5046 	 */
5047 	if (reset) {
5048 		ret = t4_fw_reset(adapter, adapter->mbox,
5049 				  PIORSTMODE | PIORST);
5050 		if (ret < 0)
5051 			goto bye;
5052 	}
5053 
5054 	/*
5055 	 * If we have a T4 configuration file under /lib/firmware/cxgb4/,
5056 	 * then use that.  Otherwise, use the configuration file stored
5057 	 * in the adapter flash ...
5058 	 */
5059 	switch (CHELSIO_CHIP_VERSION(adapter->params.chip)) {
5060 	case CHELSIO_T4:
5061 		fw_config_file = FW4_CFNAME;
5062 		break;
5063 	case CHELSIO_T5:
5064 		fw_config_file = FW5_CFNAME;
5065 		break;
5066 	default:
5067 		dev_err(adapter->pdev_dev, "Device %d is not supported\n",
5068 		       adapter->pdev->device);
5069 		ret = -EINVAL;
5070 		goto bye;
5071 	}
5072 
5073 	ret = request_firmware(&cf, fw_config_file, adapter->pdev_dev);
5074 	if (ret < 0) {
5075 		config_name = "On FLASH";
5076 		mtype = FW_MEMTYPE_CF_FLASH;
5077 		maddr = t4_flash_cfg_addr(adapter);
5078 	} else {
5079 		u32 params[7], val[7];
5080 
5081 		sprintf(fw_config_file_path,
5082 			"/lib/firmware/%s", fw_config_file);
5083 		config_name = fw_config_file_path;
5084 
5085 		if (cf->size >= FLASH_CFG_MAX_SIZE)
5086 			ret = -ENOMEM;
5087 		else {
5088 			params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5089 			     FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_CF));
5090 			ret = t4_query_params(adapter, adapter->mbox,
5091 					      adapter->fn, 0, 1, params, val);
5092 			if (ret == 0) {
5093 				/*
5094 				 * For t4_memory_rw() below addresses and
5095 				 * sizes have to be in terms of multiples of 4
5096 				 * bytes.  So, if the Configuration File isn't
5097 				 * a multiple of 4 bytes in length we'll have
5098 				 * to write that out separately since we can't
5099 				 * guarantee that the bytes following the
5100 				 * residual byte in the buffer returned by
5101 				 * request_firmware() are zeroed out ...
5102 				 */
5103 				size_t resid = cf->size & 0x3;
5104 				size_t size = cf->size & ~0x3;
5105 				__be32 *data = (__be32 *)cf->data;
5106 
5107 				mtype = FW_PARAMS_PARAM_Y_G(val[0]);
5108 				maddr = FW_PARAMS_PARAM_Z_G(val[0]) << 16;
5109 
5110 				spin_lock(&adapter->win0_lock);
5111 				ret = t4_memory_rw(adapter, 0, mtype, maddr,
5112 						   size, data, T4_MEMORY_WRITE);
5113 				if (ret == 0 && resid != 0) {
5114 					union {
5115 						__be32 word;
5116 						char buf[4];
5117 					} last;
5118 					int i;
5119 
5120 					last.word = data[size >> 2];
5121 					for (i = resid; i < 4; i++)
5122 						last.buf[i] = 0;
5123 					ret = t4_memory_rw(adapter, 0, mtype,
5124 							   maddr + size,
5125 							   4, &last.word,
5126 							   T4_MEMORY_WRITE);
5127 				}
5128 				spin_unlock(&adapter->win0_lock);
5129 			}
5130 		}
5131 
5132 		release_firmware(cf);
5133 		if (ret)
5134 			goto bye;
5135 	}
5136 
5137 	/*
5138 	 * Issue a Capability Configuration command to the firmware to get it
5139 	 * to parse the Configuration File.  We don't use t4_fw_config_file()
5140 	 * because we want the ability to modify various features after we've
5141 	 * processed the configuration file ...
5142 	 */
5143 	memset(&caps_cmd, 0, sizeof(caps_cmd));
5144 	caps_cmd.op_to_write =
5145 		htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5146 		      FW_CMD_REQUEST_F |
5147 		      FW_CMD_READ_F);
5148 	caps_cmd.cfvalid_to_len16 =
5149 		htonl(FW_CAPS_CONFIG_CMD_CFVALID_F |
5150 		      FW_CAPS_CONFIG_CMD_MEMTYPE_CF_V(mtype) |
5151 		      FW_CAPS_CONFIG_CMD_MEMADDR64K_CF_V(maddr >> 16) |
5152 		      FW_LEN16(caps_cmd));
5153 	ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5154 			 &caps_cmd);
5155 
5156 	/* If the CAPS_CONFIG failed with an ENOENT (for a Firmware
5157 	 * Configuration File in FLASH), our last gasp effort is to use the
5158 	 * Firmware Configuration File which is embedded in the firmware.  A
5159 	 * very few early versions of the firmware didn't have one embedded
5160 	 * but we can ignore those.
5161 	 */
5162 	if (ret == -ENOENT) {
5163 		memset(&caps_cmd, 0, sizeof(caps_cmd));
5164 		caps_cmd.op_to_write =
5165 			htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5166 					FW_CMD_REQUEST_F |
5167 					FW_CMD_READ_F);
5168 		caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5169 		ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd,
5170 				sizeof(caps_cmd), &caps_cmd);
5171 		config_name = "Firmware Default";
5172 	}
5173 
5174 	config_issued = 1;
5175 	if (ret < 0)
5176 		goto bye;
5177 
5178 	finiver = ntohl(caps_cmd.finiver);
5179 	finicsum = ntohl(caps_cmd.finicsum);
5180 	cfcsum = ntohl(caps_cmd.cfcsum);
5181 	if (finicsum != cfcsum)
5182 		dev_warn(adapter->pdev_dev, "Configuration File checksum "\
5183 			 "mismatch: [fini] csum=%#x, computed csum=%#x\n",
5184 			 finicsum, cfcsum);
5185 
5186 	/*
5187 	 * And now tell the firmware to use the configuration we just loaded.
5188 	 */
5189 	caps_cmd.op_to_write =
5190 		htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5191 		      FW_CMD_REQUEST_F |
5192 		      FW_CMD_WRITE_F);
5193 	caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5194 	ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5195 			 NULL);
5196 	if (ret < 0)
5197 		goto bye;
5198 
5199 	/*
5200 	 * Tweak configuration based on system architecture, module
5201 	 * parameters, etc.
5202 	 */
5203 	ret = adap_init0_tweaks(adapter);
5204 	if (ret < 0)
5205 		goto bye;
5206 
5207 	/*
5208 	 * And finally tell the firmware to initialize itself using the
5209 	 * parameters from the Configuration File.
5210 	 */
5211 	ret = t4_fw_initialize(adapter, adapter->mbox);
5212 	if (ret < 0)
5213 		goto bye;
5214 
5215 	/*
5216 	 * Return successfully and note that we're operating with parameters
5217 	 * not supplied by the driver, rather than from hard-wired
5218 	 * initialization constants burried in the driver.
5219 	 */
5220 	adapter->flags |= USING_SOFT_PARAMS;
5221 	dev_info(adapter->pdev_dev, "Successfully configured using Firmware "\
5222 		 "Configuration File \"%s\", version %#x, computed checksum %#x\n",
5223 		 config_name, finiver, cfcsum);
5224 	return 0;
5225 
5226 	/*
5227 	 * Something bad happened.  Return the error ...  (If the "error"
5228 	 * is that there's no Configuration File on the adapter we don't
5229 	 * want to issue a warning since this is fairly common.)
5230 	 */
5231 bye:
5232 	if (config_issued && ret != -ENOENT)
5233 		dev_warn(adapter->pdev_dev, "\"%s\" configuration file error %d\n",
5234 			 config_name, -ret);
5235 	return ret;
5236 }
5237 
5238 /*
5239  * Attempt to initialize the adapter via hard-coded, driver supplied
5240  * parameters ...
5241  */
5242 static int adap_init0_no_config(struct adapter *adapter, int reset)
5243 {
5244 	struct sge *s = &adapter->sge;
5245 	struct fw_caps_config_cmd caps_cmd;
5246 	u32 v;
5247 	int i, ret;
5248 
5249 	/*
5250 	 * Reset device if necessary
5251 	 */
5252 	if (reset) {
5253 		ret = t4_fw_reset(adapter, adapter->mbox,
5254 				  PIORSTMODE | PIORST);
5255 		if (ret < 0)
5256 			goto bye;
5257 	}
5258 
5259 	/*
5260 	 * Get device capabilities and select which we'll be using.
5261 	 */
5262 	memset(&caps_cmd, 0, sizeof(caps_cmd));
5263 	caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5264 				     FW_CMD_REQUEST_F | FW_CMD_READ_F);
5265 	caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5266 	ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5267 			 &caps_cmd);
5268 	if (ret < 0)
5269 		goto bye;
5270 
5271 	if (caps_cmd.niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
5272 		if (!vf_acls)
5273 			caps_cmd.niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
5274 		else
5275 			caps_cmd.niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
5276 	} else if (vf_acls) {
5277 		dev_err(adapter->pdev_dev, "virtualization ACLs not supported");
5278 		goto bye;
5279 	}
5280 	caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5281 			      FW_CMD_REQUEST_F | FW_CMD_WRITE_F);
5282 	ret = t4_wr_mbox(adapter, adapter->mbox, &caps_cmd, sizeof(caps_cmd),
5283 			 NULL);
5284 	if (ret < 0)
5285 		goto bye;
5286 
5287 	/*
5288 	 * Tweak configuration based on system architecture, module
5289 	 * parameters, etc.
5290 	 */
5291 	ret = adap_init0_tweaks(adapter);
5292 	if (ret < 0)
5293 		goto bye;
5294 
5295 	/*
5296 	 * Select RSS Global Mode we want to use.  We use "Basic Virtual"
5297 	 * mode which maps each Virtual Interface to its own section of
5298 	 * the RSS Table and we turn on all map and hash enables ...
5299 	 */
5300 	adapter->flags |= RSS_TNLALLLOOKUP;
5301 	ret = t4_config_glbl_rss(adapter, adapter->mbox,
5302 				 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
5303 				 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN_F |
5304 				 FW_RSS_GLB_CONFIG_CMD_HASHTOEPLITZ_F |
5305 				 ((adapter->flags & RSS_TNLALLLOOKUP) ?
5306 					FW_RSS_GLB_CONFIG_CMD_TNLALLLKP_F : 0));
5307 	if (ret < 0)
5308 		goto bye;
5309 
5310 	/*
5311 	 * Set up our own fundamental resource provisioning ...
5312 	 */
5313 	ret = t4_cfg_pfvf(adapter, adapter->mbox, adapter->fn, 0,
5314 			  PFRES_NEQ, PFRES_NETHCTRL,
5315 			  PFRES_NIQFLINT, PFRES_NIQ,
5316 			  PFRES_TC, PFRES_NVI,
5317 			  FW_PFVF_CMD_CMASK_M,
5318 			  pfvfres_pmask(adapter, adapter->fn, 0),
5319 			  PFRES_NEXACTF,
5320 			  PFRES_R_CAPS, PFRES_WX_CAPS);
5321 	if (ret < 0)
5322 		goto bye;
5323 
5324 	/*
5325 	 * Perform low level SGE initialization.  We need to do this before we
5326 	 * send the firmware the INITIALIZE command because that will cause
5327 	 * any other PF Drivers which are waiting for the Master
5328 	 * Initialization to proceed forward.
5329 	 */
5330 	for (i = 0; i < SGE_NTIMERS - 1; i++)
5331 		s->timer_val[i] = min(intr_holdoff[i], MAX_SGE_TIMERVAL);
5332 	s->timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
5333 	s->counter_val[0] = 1;
5334 	for (i = 1; i < SGE_NCOUNTERS; i++)
5335 		s->counter_val[i] = min(intr_cnt[i - 1],
5336 					THRESHOLD_0_GET(THRESHOLD_0_MASK));
5337 	t4_sge_init(adapter);
5338 
5339 #ifdef CONFIG_PCI_IOV
5340 	/*
5341 	 * Provision resource limits for Virtual Functions.  We currently
5342 	 * grant them all the same static resource limits except for the Port
5343 	 * Access Rights Mask which we're assigning based on the PF.  All of
5344 	 * the static provisioning stuff for both the PF and VF really needs
5345 	 * to be managed in a persistent manner for each device which the
5346 	 * firmware controls.
5347 	 */
5348 	{
5349 		int pf, vf;
5350 
5351 		for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
5352 			if (num_vf[pf] <= 0)
5353 				continue;
5354 
5355 			/* VF numbering starts at 1! */
5356 			for (vf = 1; vf <= num_vf[pf]; vf++) {
5357 				ret = t4_cfg_pfvf(adapter, adapter->mbox,
5358 						  pf, vf,
5359 						  VFRES_NEQ, VFRES_NETHCTRL,
5360 						  VFRES_NIQFLINT, VFRES_NIQ,
5361 						  VFRES_TC, VFRES_NVI,
5362 						  FW_PFVF_CMD_CMASK_M,
5363 						  pfvfres_pmask(
5364 						  adapter, pf, vf),
5365 						  VFRES_NEXACTF,
5366 						  VFRES_R_CAPS, VFRES_WX_CAPS);
5367 				if (ret < 0)
5368 					dev_warn(adapter->pdev_dev,
5369 						 "failed to "\
5370 						 "provision pf/vf=%d/%d; "
5371 						 "err=%d\n", pf, vf, ret);
5372 			}
5373 		}
5374 	}
5375 #endif
5376 
5377 	/*
5378 	 * Set up the default filter mode.  Later we'll want to implement this
5379 	 * via a firmware command, etc. ...  This needs to be done before the
5380 	 * firmare initialization command ...  If the selected set of fields
5381 	 * isn't equal to the default value, we'll need to make sure that the
5382 	 * field selections will fit in the 36-bit budget.
5383 	 */
5384 	if (tp_vlan_pri_map != TP_VLAN_PRI_MAP_DEFAULT) {
5385 		int j, bits = 0;
5386 
5387 		for (j = TP_VLAN_PRI_MAP_FIRST; j <= TP_VLAN_PRI_MAP_LAST; j++)
5388 			switch (tp_vlan_pri_map & (1 << j)) {
5389 			case 0:
5390 				/* compressed filter field not enabled */
5391 				break;
5392 			case FCOE_MASK:
5393 				bits +=  1;
5394 				break;
5395 			case PORT_MASK:
5396 				bits +=  3;
5397 				break;
5398 			case VNIC_ID_MASK:
5399 				bits += 17;
5400 				break;
5401 			case VLAN_MASK:
5402 				bits += 17;
5403 				break;
5404 			case TOS_MASK:
5405 				bits +=  8;
5406 				break;
5407 			case PROTOCOL_MASK:
5408 				bits +=  8;
5409 				break;
5410 			case ETHERTYPE_MASK:
5411 				bits += 16;
5412 				break;
5413 			case MACMATCH_MASK:
5414 				bits +=  9;
5415 				break;
5416 			case MPSHITTYPE_MASK:
5417 				bits +=  3;
5418 				break;
5419 			case FRAGMENTATION_MASK:
5420 				bits +=  1;
5421 				break;
5422 			}
5423 
5424 		if (bits > 36) {
5425 			dev_err(adapter->pdev_dev,
5426 				"tp_vlan_pri_map=%#x needs %d bits > 36;"\
5427 				" using %#x\n", tp_vlan_pri_map, bits,
5428 				TP_VLAN_PRI_MAP_DEFAULT);
5429 			tp_vlan_pri_map = TP_VLAN_PRI_MAP_DEFAULT;
5430 		}
5431 	}
5432 	v = tp_vlan_pri_map;
5433 	t4_write_indirect(adapter, TP_PIO_ADDR, TP_PIO_DATA,
5434 			  &v, 1, TP_VLAN_PRI_MAP);
5435 
5436 	/*
5437 	 * We need Five Tuple Lookup mode to be set in TP_GLOBAL_CONFIG order
5438 	 * to support any of the compressed filter fields above.  Newer
5439 	 * versions of the firmware do this automatically but it doesn't hurt
5440 	 * to set it here.  Meanwhile, we do _not_ need to set Lookup Every
5441 	 * Packet in TP_INGRESS_CONFIG to support matching non-TCP packets
5442 	 * since the firmware automatically turns this on and off when we have
5443 	 * a non-zero number of filters active (since it does have a
5444 	 * performance impact).
5445 	 */
5446 	if (tp_vlan_pri_map)
5447 		t4_set_reg_field(adapter, TP_GLOBAL_CONFIG,
5448 				 FIVETUPLELOOKUP_MASK,
5449 				 FIVETUPLELOOKUP_MASK);
5450 
5451 	/*
5452 	 * Tweak some settings.
5453 	 */
5454 	t4_write_reg(adapter, TP_SHIFT_CNT, SYNSHIFTMAX(6) |
5455 		     RXTSHIFTMAXR1(4) | RXTSHIFTMAXR2(15) |
5456 		     PERSHIFTBACKOFFMAX(8) | PERSHIFTMAX(8) |
5457 		     KEEPALIVEMAXR1(4) | KEEPALIVEMAXR2(9));
5458 
5459 	/*
5460 	 * Get basic stuff going by issuing the Firmware Initialize command.
5461 	 * Note that this _must_ be after all PFVF commands ...
5462 	 */
5463 	ret = t4_fw_initialize(adapter, adapter->mbox);
5464 	if (ret < 0)
5465 		goto bye;
5466 
5467 	/*
5468 	 * Return successfully!
5469 	 */
5470 	dev_info(adapter->pdev_dev, "Successfully configured using built-in "\
5471 		 "driver parameters\n");
5472 	return 0;
5473 
5474 	/*
5475 	 * Something bad happened.  Return the error ...
5476 	 */
5477 bye:
5478 	return ret;
5479 }
5480 
5481 static struct fw_info fw_info_array[] = {
5482 	{
5483 		.chip = CHELSIO_T4,
5484 		.fs_name = FW4_CFNAME,
5485 		.fw_mod_name = FW4_FNAME,
5486 		.fw_hdr = {
5487 			.chip = FW_HDR_CHIP_T4,
5488 			.fw_ver = __cpu_to_be32(FW_VERSION(T4)),
5489 			.intfver_nic = FW_INTFVER(T4, NIC),
5490 			.intfver_vnic = FW_INTFVER(T4, VNIC),
5491 			.intfver_ri = FW_INTFVER(T4, RI),
5492 			.intfver_iscsi = FW_INTFVER(T4, ISCSI),
5493 			.intfver_fcoe = FW_INTFVER(T4, FCOE),
5494 		},
5495 	}, {
5496 		.chip = CHELSIO_T5,
5497 		.fs_name = FW5_CFNAME,
5498 		.fw_mod_name = FW5_FNAME,
5499 		.fw_hdr = {
5500 			.chip = FW_HDR_CHIP_T5,
5501 			.fw_ver = __cpu_to_be32(FW_VERSION(T5)),
5502 			.intfver_nic = FW_INTFVER(T5, NIC),
5503 			.intfver_vnic = FW_INTFVER(T5, VNIC),
5504 			.intfver_ri = FW_INTFVER(T5, RI),
5505 			.intfver_iscsi = FW_INTFVER(T5, ISCSI),
5506 			.intfver_fcoe = FW_INTFVER(T5, FCOE),
5507 		},
5508 	}
5509 };
5510 
5511 static struct fw_info *find_fw_info(int chip)
5512 {
5513 	int i;
5514 
5515 	for (i = 0; i < ARRAY_SIZE(fw_info_array); i++) {
5516 		if (fw_info_array[i].chip == chip)
5517 			return &fw_info_array[i];
5518 	}
5519 	return NULL;
5520 }
5521 
5522 /*
5523  * Phase 0 of initialization: contact FW, obtain config, perform basic init.
5524  */
5525 static int adap_init0(struct adapter *adap)
5526 {
5527 	int ret;
5528 	u32 v, port_vec;
5529 	enum dev_state state;
5530 	u32 params[7], val[7];
5531 	struct fw_caps_config_cmd caps_cmd;
5532 	int reset = 1;
5533 
5534 	/* Contact FW, advertising Master capability */
5535 	ret = t4_fw_hello(adap, adap->mbox, adap->mbox, MASTER_MAY, &state);
5536 	if (ret < 0) {
5537 		dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
5538 			ret);
5539 		return ret;
5540 	}
5541 	if (ret == adap->mbox)
5542 		adap->flags |= MASTER_PF;
5543 
5544 	/*
5545 	 * If we're the Master PF Driver and the device is uninitialized,
5546 	 * then let's consider upgrading the firmware ...  (We always want
5547 	 * to check the firmware version number in order to A. get it for
5548 	 * later reporting and B. to warn if the currently loaded firmware
5549 	 * is excessively mismatched relative to the driver.)
5550 	 */
5551 	t4_get_fw_version(adap, &adap->params.fw_vers);
5552 	t4_get_tp_version(adap, &adap->params.tp_vers);
5553 	if ((adap->flags & MASTER_PF) && state != DEV_STATE_INIT) {
5554 		struct fw_info *fw_info;
5555 		struct fw_hdr *card_fw;
5556 		const struct firmware *fw;
5557 		const u8 *fw_data = NULL;
5558 		unsigned int fw_size = 0;
5559 
5560 		/* This is the firmware whose headers the driver was compiled
5561 		 * against
5562 		 */
5563 		fw_info = find_fw_info(CHELSIO_CHIP_VERSION(adap->params.chip));
5564 		if (fw_info == NULL) {
5565 			dev_err(adap->pdev_dev,
5566 				"unable to get firmware info for chip %d.\n",
5567 				CHELSIO_CHIP_VERSION(adap->params.chip));
5568 			return -EINVAL;
5569 		}
5570 
5571 		/* allocate memory to read the header of the firmware on the
5572 		 * card
5573 		 */
5574 		card_fw = t4_alloc_mem(sizeof(*card_fw));
5575 
5576 		/* Get FW from from /lib/firmware/ */
5577 		ret = request_firmware(&fw, fw_info->fw_mod_name,
5578 				       adap->pdev_dev);
5579 		if (ret < 0) {
5580 			dev_err(adap->pdev_dev,
5581 				"unable to load firmware image %s, error %d\n",
5582 				fw_info->fw_mod_name, ret);
5583 		} else {
5584 			fw_data = fw->data;
5585 			fw_size = fw->size;
5586 		}
5587 
5588 		/* upgrade FW logic */
5589 		ret = t4_prep_fw(adap, fw_info, fw_data, fw_size, card_fw,
5590 				 state, &reset);
5591 
5592 		/* Cleaning up */
5593 		if (fw != NULL)
5594 			release_firmware(fw);
5595 		t4_free_mem(card_fw);
5596 
5597 		if (ret < 0)
5598 			goto bye;
5599 	}
5600 
5601 	/*
5602 	 * Grab VPD parameters.  This should be done after we establish a
5603 	 * connection to the firmware since some of the VPD parameters
5604 	 * (notably the Core Clock frequency) are retrieved via requests to
5605 	 * the firmware.  On the other hand, we need these fairly early on
5606 	 * so we do this right after getting ahold of the firmware.
5607 	 */
5608 	ret = get_vpd_params(adap, &adap->params.vpd);
5609 	if (ret < 0)
5610 		goto bye;
5611 
5612 	/*
5613 	 * Find out what ports are available to us.  Note that we need to do
5614 	 * this before calling adap_init0_no_config() since it needs nports
5615 	 * and portvec ...
5616 	 */
5617 	v =
5618 	    FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5619 	    FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_PORTVEC);
5620 	ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1, &v, &port_vec);
5621 	if (ret < 0)
5622 		goto bye;
5623 
5624 	adap->params.nports = hweight32(port_vec);
5625 	adap->params.portvec = port_vec;
5626 
5627 	/*
5628 	 * If the firmware is initialized already (and we're not forcing a
5629 	 * master initialization), note that we're living with existing
5630 	 * adapter parameters.  Otherwise, it's time to try initializing the
5631 	 * adapter ...
5632 	 */
5633 	if (state == DEV_STATE_INIT) {
5634 		dev_info(adap->pdev_dev, "Coming up as %s: "\
5635 			 "Adapter already initialized\n",
5636 			 adap->flags & MASTER_PF ? "MASTER" : "SLAVE");
5637 		adap->flags |= USING_SOFT_PARAMS;
5638 	} else {
5639 		dev_info(adap->pdev_dev, "Coming up as MASTER: "\
5640 			 "Initializing adapter\n");
5641 		/*
5642 		 * If the firmware doesn't support Configuration
5643 		 * Files warn user and exit,
5644 		 */
5645 		if (ret < 0)
5646 			dev_warn(adap->pdev_dev, "Firmware doesn't support "
5647 				 "configuration file.\n");
5648 		if (force_old_init)
5649 			ret = adap_init0_no_config(adap, reset);
5650 		else {
5651 			/*
5652 			 * Find out whether we're dealing with a version of
5653 			 * the firmware which has configuration file support.
5654 			 */
5655 			params[0] = (FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) |
5656 				     FW_PARAMS_PARAM_X_V(
5657 					     FW_PARAMS_PARAM_DEV_CF));
5658 			ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 1,
5659 					      params, val);
5660 
5661 			/*
5662 			 * If the firmware doesn't support Configuration
5663 			 * Files, use the old Driver-based, hard-wired
5664 			 * initialization.  Otherwise, try using the
5665 			 * Configuration File support and fall back to the
5666 			 * Driver-based initialization if there's no
5667 			 * Configuration File found.
5668 			 */
5669 			if (ret < 0)
5670 				ret = adap_init0_no_config(adap, reset);
5671 			else {
5672 				/*
5673 				 * The firmware provides us with a memory
5674 				 * buffer where we can load a Configuration
5675 				 * File from the host if we want to override
5676 				 * the Configuration File in flash.
5677 				 */
5678 
5679 				ret = adap_init0_config(adap, reset);
5680 				if (ret == -ENOENT) {
5681 					dev_info(adap->pdev_dev,
5682 					    "No Configuration File present "
5683 					    "on adapter. Using hard-wired "
5684 					    "configuration parameters.\n");
5685 					ret = adap_init0_no_config(adap, reset);
5686 				}
5687 			}
5688 		}
5689 		if (ret < 0) {
5690 			dev_err(adap->pdev_dev,
5691 				"could not initialize adapter, error %d\n",
5692 				-ret);
5693 			goto bye;
5694 		}
5695 	}
5696 
5697 	/*
5698 	 * If we're living with non-hard-coded parameters (either from a
5699 	 * Firmware Configuration File or values programmed by a different PF
5700 	 * Driver), give the SGE code a chance to pull in anything that it
5701 	 * needs ...  Note that this must be called after we retrieve our VPD
5702 	 * parameters in order to know how to convert core ticks to seconds.
5703 	 */
5704 	if (adap->flags & USING_SOFT_PARAMS) {
5705 		ret = t4_sge_init(adap);
5706 		if (ret < 0)
5707 			goto bye;
5708 	}
5709 
5710 	if (is_bypass_device(adap->pdev->device))
5711 		adap->params.bypass = 1;
5712 
5713 	/*
5714 	 * Grab some of our basic fundamental operating parameters.
5715 	 */
5716 #define FW_PARAM_DEV(param) \
5717 	(FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DEV) | \
5718 	FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_DEV_##param))
5719 
5720 #define FW_PARAM_PFVF(param) \
5721 	FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) | \
5722 	FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_##param)|  \
5723 	FW_PARAMS_PARAM_Y_V(0) | \
5724 	FW_PARAMS_PARAM_Z_V(0)
5725 
5726 	params[0] = FW_PARAM_PFVF(EQ_START);
5727 	params[1] = FW_PARAM_PFVF(L2T_START);
5728 	params[2] = FW_PARAM_PFVF(L2T_END);
5729 	params[3] = FW_PARAM_PFVF(FILTER_START);
5730 	params[4] = FW_PARAM_PFVF(FILTER_END);
5731 	params[5] = FW_PARAM_PFVF(IQFLINT_START);
5732 	ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params, val);
5733 	if (ret < 0)
5734 		goto bye;
5735 	adap->sge.egr_start = val[0];
5736 	adap->l2t_start = val[1];
5737 	adap->l2t_end = val[2];
5738 	adap->tids.ftid_base = val[3];
5739 	adap->tids.nftids = val[4] - val[3] + 1;
5740 	adap->sge.ingr_start = val[5];
5741 
5742 	/* query params related to active filter region */
5743 	params[0] = FW_PARAM_PFVF(ACTIVE_FILTER_START);
5744 	params[1] = FW_PARAM_PFVF(ACTIVE_FILTER_END);
5745 	ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params, val);
5746 	/* If Active filter size is set we enable establishing
5747 	 * offload connection through firmware work request
5748 	 */
5749 	if ((val[0] != val[1]) && (ret >= 0)) {
5750 		adap->flags |= FW_OFLD_CONN;
5751 		adap->tids.aftid_base = val[0];
5752 		adap->tids.aftid_end = val[1];
5753 	}
5754 
5755 	/* If we're running on newer firmware, let it know that we're
5756 	 * prepared to deal with encapsulated CPL messages.  Older
5757 	 * firmware won't understand this and we'll just get
5758 	 * unencapsulated messages ...
5759 	 */
5760 	params[0] = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
5761 	val[0] = 1;
5762 	(void) t4_set_params(adap, adap->mbox, adap->fn, 0, 1, params, val);
5763 
5764 	/*
5765 	 * Find out whether we're allowed to use the T5+ ULPTX MEMWRITE DSGL
5766 	 * capability.  Earlier versions of the firmware didn't have the
5767 	 * ULPTX_MEMWRITE_DSGL so we'll interpret a query failure as no
5768 	 * permission to use ULPTX MEMWRITE DSGL.
5769 	 */
5770 	if (is_t4(adap->params.chip)) {
5771 		adap->params.ulptx_memwrite_dsgl = false;
5772 	} else {
5773 		params[0] = FW_PARAM_DEV(ULPTX_MEMWRITE_DSGL);
5774 		ret = t4_query_params(adap, adap->mbox, adap->fn, 0,
5775 				      1, params, val);
5776 		adap->params.ulptx_memwrite_dsgl = (ret == 0 && val[0] != 0);
5777 	}
5778 
5779 	/*
5780 	 * Get device capabilities so we can determine what resources we need
5781 	 * to manage.
5782 	 */
5783 	memset(&caps_cmd, 0, sizeof(caps_cmd));
5784 	caps_cmd.op_to_write = htonl(FW_CMD_OP_V(FW_CAPS_CONFIG_CMD) |
5785 				     FW_CMD_REQUEST_F | FW_CMD_READ_F);
5786 	caps_cmd.cfvalid_to_len16 = htonl(FW_LEN16(caps_cmd));
5787 	ret = t4_wr_mbox(adap, adap->mbox, &caps_cmd, sizeof(caps_cmd),
5788 			 &caps_cmd);
5789 	if (ret < 0)
5790 		goto bye;
5791 
5792 	if (caps_cmd.ofldcaps) {
5793 		/* query offload-related parameters */
5794 		params[0] = FW_PARAM_DEV(NTID);
5795 		params[1] = FW_PARAM_PFVF(SERVER_START);
5796 		params[2] = FW_PARAM_PFVF(SERVER_END);
5797 		params[3] = FW_PARAM_PFVF(TDDP_START);
5798 		params[4] = FW_PARAM_PFVF(TDDP_END);
5799 		params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
5800 		ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5801 				      params, val);
5802 		if (ret < 0)
5803 			goto bye;
5804 		adap->tids.ntids = val[0];
5805 		adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
5806 		adap->tids.stid_base = val[1];
5807 		adap->tids.nstids = val[2] - val[1] + 1;
5808 		/*
5809 		 * Setup server filter region. Divide the availble filter
5810 		 * region into two parts. Regular filters get 1/3rd and server
5811 		 * filters get 2/3rd part. This is only enabled if workarond
5812 		 * path is enabled.
5813 		 * 1. For regular filters.
5814 		 * 2. Server filter: This are special filters which are used
5815 		 * to redirect SYN packets to offload queue.
5816 		 */
5817 		if (adap->flags & FW_OFLD_CONN && !is_bypass(adap)) {
5818 			adap->tids.sftid_base = adap->tids.ftid_base +
5819 					DIV_ROUND_UP(adap->tids.nftids, 3);
5820 			adap->tids.nsftids = adap->tids.nftids -
5821 					 DIV_ROUND_UP(adap->tids.nftids, 3);
5822 			adap->tids.nftids = adap->tids.sftid_base -
5823 						adap->tids.ftid_base;
5824 		}
5825 		adap->vres.ddp.start = val[3];
5826 		adap->vres.ddp.size = val[4] - val[3] + 1;
5827 		adap->params.ofldq_wr_cred = val[5];
5828 
5829 		adap->params.offload = 1;
5830 	}
5831 	if (caps_cmd.rdmacaps) {
5832 		params[0] = FW_PARAM_PFVF(STAG_START);
5833 		params[1] = FW_PARAM_PFVF(STAG_END);
5834 		params[2] = FW_PARAM_PFVF(RQ_START);
5835 		params[3] = FW_PARAM_PFVF(RQ_END);
5836 		params[4] = FW_PARAM_PFVF(PBL_START);
5837 		params[5] = FW_PARAM_PFVF(PBL_END);
5838 		ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6,
5839 				      params, val);
5840 		if (ret < 0)
5841 			goto bye;
5842 		adap->vres.stag.start = val[0];
5843 		adap->vres.stag.size = val[1] - val[0] + 1;
5844 		adap->vres.rq.start = val[2];
5845 		adap->vres.rq.size = val[3] - val[2] + 1;
5846 		adap->vres.pbl.start = val[4];
5847 		adap->vres.pbl.size = val[5] - val[4] + 1;
5848 
5849 		params[0] = FW_PARAM_PFVF(SQRQ_START);
5850 		params[1] = FW_PARAM_PFVF(SQRQ_END);
5851 		params[2] = FW_PARAM_PFVF(CQ_START);
5852 		params[3] = FW_PARAM_PFVF(CQ_END);
5853 		params[4] = FW_PARAM_PFVF(OCQ_START);
5854 		params[5] = FW_PARAM_PFVF(OCQ_END);
5855 		ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 6, params,
5856 				      val);
5857 		if (ret < 0)
5858 			goto bye;
5859 		adap->vres.qp.start = val[0];
5860 		adap->vres.qp.size = val[1] - val[0] + 1;
5861 		adap->vres.cq.start = val[2];
5862 		adap->vres.cq.size = val[3] - val[2] + 1;
5863 		adap->vres.ocq.start = val[4];
5864 		adap->vres.ocq.size = val[5] - val[4] + 1;
5865 
5866 		params[0] = FW_PARAM_DEV(MAXORDIRD_QP);
5867 		params[1] = FW_PARAM_DEV(MAXIRD_ADAPTER);
5868 		ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2, params,
5869 				      val);
5870 		if (ret < 0) {
5871 			adap->params.max_ordird_qp = 8;
5872 			adap->params.max_ird_adapter = 32 * adap->tids.ntids;
5873 			ret = 0;
5874 		} else {
5875 			adap->params.max_ordird_qp = val[0];
5876 			adap->params.max_ird_adapter = val[1];
5877 		}
5878 		dev_info(adap->pdev_dev,
5879 			 "max_ordird_qp %d max_ird_adapter %d\n",
5880 			 adap->params.max_ordird_qp,
5881 			 adap->params.max_ird_adapter);
5882 	}
5883 	if (caps_cmd.iscsicaps) {
5884 		params[0] = FW_PARAM_PFVF(ISCSI_START);
5885 		params[1] = FW_PARAM_PFVF(ISCSI_END);
5886 		ret = t4_query_params(adap, adap->mbox, adap->fn, 0, 2,
5887 				      params, val);
5888 		if (ret < 0)
5889 			goto bye;
5890 		adap->vres.iscsi.start = val[0];
5891 		adap->vres.iscsi.size = val[1] - val[0] + 1;
5892 	}
5893 #undef FW_PARAM_PFVF
5894 #undef FW_PARAM_DEV
5895 
5896 	/* The MTU/MSS Table is initialized by now, so load their values.  If
5897 	 * we're initializing the adapter, then we'll make any modifications
5898 	 * we want to the MTU/MSS Table and also initialize the congestion
5899 	 * parameters.
5900 	 */
5901 	t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
5902 	if (state != DEV_STATE_INIT) {
5903 		int i;
5904 
5905 		/* The default MTU Table contains values 1492 and 1500.
5906 		 * However, for TCP, it's better to have two values which are
5907 		 * a multiple of 8 +/- 4 bytes apart near this popular MTU.
5908 		 * This allows us to have a TCP Data Payload which is a
5909 		 * multiple of 8 regardless of what combination of TCP Options
5910 		 * are in use (always a multiple of 4 bytes) which is
5911 		 * important for performance reasons.  For instance, if no
5912 		 * options are in use, then we have a 20-byte IP header and a
5913 		 * 20-byte TCP header.  In this case, a 1500-byte MSS would
5914 		 * result in a TCP Data Payload of 1500 - 40 == 1460 bytes
5915 		 * which is not a multiple of 8.  So using an MSS of 1488 in
5916 		 * this case results in a TCP Data Payload of 1448 bytes which
5917 		 * is a multiple of 8.  On the other hand, if 12-byte TCP Time
5918 		 * Stamps have been negotiated, then an MTU of 1500 bytes
5919 		 * results in a TCP Data Payload of 1448 bytes which, as
5920 		 * above, is a multiple of 8 bytes ...
5921 		 */
5922 		for (i = 0; i < NMTUS; i++)
5923 			if (adap->params.mtus[i] == 1492) {
5924 				adap->params.mtus[i] = 1488;
5925 				break;
5926 			}
5927 
5928 		t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
5929 			     adap->params.b_wnd);
5930 	}
5931 	t4_init_sge_params(adap);
5932 	t4_init_tp_params(adap);
5933 	adap->flags |= FW_OK;
5934 	return 0;
5935 
5936 	/*
5937 	 * Something bad happened.  If a command timed out or failed with EIO
5938 	 * FW does not operate within its spec or something catastrophic
5939 	 * happened to HW/FW, stop issuing commands.
5940 	 */
5941 bye:
5942 	if (ret != -ETIMEDOUT && ret != -EIO)
5943 		t4_fw_bye(adap, adap->mbox);
5944 	return ret;
5945 }
5946 
5947 /* EEH callbacks */
5948 
5949 static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
5950 					 pci_channel_state_t state)
5951 {
5952 	int i;
5953 	struct adapter *adap = pci_get_drvdata(pdev);
5954 
5955 	if (!adap)
5956 		goto out;
5957 
5958 	rtnl_lock();
5959 	adap->flags &= ~FW_OK;
5960 	notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
5961 	spin_lock(&adap->stats_lock);
5962 	for_each_port(adap, i) {
5963 		struct net_device *dev = adap->port[i];
5964 
5965 		netif_device_detach(dev);
5966 		netif_carrier_off(dev);
5967 	}
5968 	spin_unlock(&adap->stats_lock);
5969 	if (adap->flags & FULL_INIT_DONE)
5970 		cxgb_down(adap);
5971 	rtnl_unlock();
5972 	if ((adap->flags & DEV_ENABLED)) {
5973 		pci_disable_device(pdev);
5974 		adap->flags &= ~DEV_ENABLED;
5975 	}
5976 out:	return state == pci_channel_io_perm_failure ?
5977 		PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
5978 }
5979 
5980 static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
5981 {
5982 	int i, ret;
5983 	struct fw_caps_config_cmd c;
5984 	struct adapter *adap = pci_get_drvdata(pdev);
5985 
5986 	if (!adap) {
5987 		pci_restore_state(pdev);
5988 		pci_save_state(pdev);
5989 		return PCI_ERS_RESULT_RECOVERED;
5990 	}
5991 
5992 	if (!(adap->flags & DEV_ENABLED)) {
5993 		if (pci_enable_device(pdev)) {
5994 			dev_err(&pdev->dev, "Cannot reenable PCI "
5995 					    "device after reset\n");
5996 			return PCI_ERS_RESULT_DISCONNECT;
5997 		}
5998 		adap->flags |= DEV_ENABLED;
5999 	}
6000 
6001 	pci_set_master(pdev);
6002 	pci_restore_state(pdev);
6003 	pci_save_state(pdev);
6004 	pci_cleanup_aer_uncorrect_error_status(pdev);
6005 
6006 	if (t4_wait_dev_ready(adap->regs) < 0)
6007 		return PCI_ERS_RESULT_DISCONNECT;
6008 	if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL) < 0)
6009 		return PCI_ERS_RESULT_DISCONNECT;
6010 	adap->flags |= FW_OK;
6011 	if (adap_init1(adap, &c))
6012 		return PCI_ERS_RESULT_DISCONNECT;
6013 
6014 	for_each_port(adap, i) {
6015 		struct port_info *p = adap2pinfo(adap, i);
6016 
6017 		ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
6018 				  NULL, NULL);
6019 		if (ret < 0)
6020 			return PCI_ERS_RESULT_DISCONNECT;
6021 		p->viid = ret;
6022 		p->xact_addr_filt = -1;
6023 	}
6024 
6025 	t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
6026 		     adap->params.b_wnd);
6027 	setup_memwin(adap);
6028 	if (cxgb_up(adap))
6029 		return PCI_ERS_RESULT_DISCONNECT;
6030 	return PCI_ERS_RESULT_RECOVERED;
6031 }
6032 
6033 static void eeh_resume(struct pci_dev *pdev)
6034 {
6035 	int i;
6036 	struct adapter *adap = pci_get_drvdata(pdev);
6037 
6038 	if (!adap)
6039 		return;
6040 
6041 	rtnl_lock();
6042 	for_each_port(adap, i) {
6043 		struct net_device *dev = adap->port[i];
6044 
6045 		if (netif_running(dev)) {
6046 			link_start(dev);
6047 			cxgb_set_rxmode(dev);
6048 		}
6049 		netif_device_attach(dev);
6050 	}
6051 	rtnl_unlock();
6052 }
6053 
6054 static const struct pci_error_handlers cxgb4_eeh = {
6055 	.error_detected = eeh_err_detected,
6056 	.slot_reset     = eeh_slot_reset,
6057 	.resume         = eeh_resume,
6058 };
6059 
6060 static inline bool is_x_10g_port(const struct link_config *lc)
6061 {
6062 	return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0 ||
6063 	       (lc->supported & FW_PORT_CAP_SPEED_40G) != 0;
6064 }
6065 
6066 static inline void init_rspq(struct adapter *adap, struct sge_rspq *q,
6067 			     unsigned int us, unsigned int cnt,
6068 			     unsigned int size, unsigned int iqe_size)
6069 {
6070 	q->adap = adap;
6071 	set_rspq_intr_params(q, us, cnt);
6072 	q->iqe_len = iqe_size;
6073 	q->size = size;
6074 }
6075 
6076 /*
6077  * Perform default configuration of DMA queues depending on the number and type
6078  * of ports we found and the number of available CPUs.  Most settings can be
6079  * modified by the admin prior to actual use.
6080  */
6081 static void cfg_queues(struct adapter *adap)
6082 {
6083 	struct sge *s = &adap->sge;
6084 	int i, n10g = 0, qidx = 0;
6085 #ifndef CONFIG_CHELSIO_T4_DCB
6086 	int q10g = 0;
6087 #endif
6088 	int ciq_size;
6089 
6090 	for_each_port(adap, i)
6091 		n10g += is_x_10g_port(&adap2pinfo(adap, i)->link_cfg);
6092 #ifdef CONFIG_CHELSIO_T4_DCB
6093 	/* For Data Center Bridging support we need to be able to support up
6094 	 * to 8 Traffic Priorities; each of which will be assigned to its
6095 	 * own TX Queue in order to prevent Head-Of-Line Blocking.
6096 	 */
6097 	if (adap->params.nports * 8 > MAX_ETH_QSETS) {
6098 		dev_err(adap->pdev_dev, "MAX_ETH_QSETS=%d < %d!\n",
6099 			MAX_ETH_QSETS, adap->params.nports * 8);
6100 		BUG_ON(1);
6101 	}
6102 
6103 	for_each_port(adap, i) {
6104 		struct port_info *pi = adap2pinfo(adap, i);
6105 
6106 		pi->first_qset = qidx;
6107 		pi->nqsets = 8;
6108 		qidx += pi->nqsets;
6109 	}
6110 #else /* !CONFIG_CHELSIO_T4_DCB */
6111 	/*
6112 	 * We default to 1 queue per non-10G port and up to # of cores queues
6113 	 * per 10G port.
6114 	 */
6115 	if (n10g)
6116 		q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
6117 	if (q10g > netif_get_num_default_rss_queues())
6118 		q10g = netif_get_num_default_rss_queues();
6119 
6120 	for_each_port(adap, i) {
6121 		struct port_info *pi = adap2pinfo(adap, i);
6122 
6123 		pi->first_qset = qidx;
6124 		pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
6125 		qidx += pi->nqsets;
6126 	}
6127 #endif /* !CONFIG_CHELSIO_T4_DCB */
6128 
6129 	s->ethqsets = qidx;
6130 	s->max_ethqsets = qidx;   /* MSI-X may lower it later */
6131 
6132 	if (is_offload(adap)) {
6133 		/*
6134 		 * For offload we use 1 queue/channel if all ports are up to 1G,
6135 		 * otherwise we divide all available queues amongst the channels
6136 		 * capped by the number of available cores.
6137 		 */
6138 		if (n10g) {
6139 			i = min_t(int, ARRAY_SIZE(s->ofldrxq),
6140 				  num_online_cpus());
6141 			s->ofldqsets = roundup(i, adap->params.nports);
6142 		} else
6143 			s->ofldqsets = adap->params.nports;
6144 		/* For RDMA one Rx queue per channel suffices */
6145 		s->rdmaqs = adap->params.nports;
6146 		s->rdmaciqs = adap->params.nports;
6147 	}
6148 
6149 	for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
6150 		struct sge_eth_rxq *r = &s->ethrxq[i];
6151 
6152 		init_rspq(adap, &r->rspq, 5, 10, 1024, 64);
6153 		r->fl.size = 72;
6154 	}
6155 
6156 	for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
6157 		s->ethtxq[i].q.size = 1024;
6158 
6159 	for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
6160 		s->ctrlq[i].q.size = 512;
6161 
6162 	for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
6163 		s->ofldtxq[i].q.size = 1024;
6164 
6165 	for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
6166 		struct sge_ofld_rxq *r = &s->ofldrxq[i];
6167 
6168 		init_rspq(adap, &r->rspq, 5, 1, 1024, 64);
6169 		r->rspq.uld = CXGB4_ULD_ISCSI;
6170 		r->fl.size = 72;
6171 	}
6172 
6173 	for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
6174 		struct sge_ofld_rxq *r = &s->rdmarxq[i];
6175 
6176 		init_rspq(adap, &r->rspq, 5, 1, 511, 64);
6177 		r->rspq.uld = CXGB4_ULD_RDMA;
6178 		r->fl.size = 72;
6179 	}
6180 
6181 	ciq_size = 64 + adap->vres.cq.size + adap->tids.nftids;
6182 	if (ciq_size > SGE_MAX_IQ_SIZE) {
6183 		CH_WARN(adap, "CIQ size too small for available IQs\n");
6184 		ciq_size = SGE_MAX_IQ_SIZE;
6185 	}
6186 
6187 	for (i = 0; i < ARRAY_SIZE(s->rdmaciq); i++) {
6188 		struct sge_ofld_rxq *r = &s->rdmaciq[i];
6189 
6190 		init_rspq(adap, &r->rspq, 5, 1, ciq_size, 64);
6191 		r->rspq.uld = CXGB4_ULD_RDMA;
6192 	}
6193 
6194 	init_rspq(adap, &s->fw_evtq, 0, 1, 1024, 64);
6195 	init_rspq(adap, &s->intrq, 0, 1, 2 * MAX_INGQ, 64);
6196 }
6197 
6198 /*
6199  * Reduce the number of Ethernet queues across all ports to at most n.
6200  * n provides at least one queue per port.
6201  */
6202 static void reduce_ethqs(struct adapter *adap, int n)
6203 {
6204 	int i;
6205 	struct port_info *pi;
6206 
6207 	while (n < adap->sge.ethqsets)
6208 		for_each_port(adap, i) {
6209 			pi = adap2pinfo(adap, i);
6210 			if (pi->nqsets > 1) {
6211 				pi->nqsets--;
6212 				adap->sge.ethqsets--;
6213 				if (adap->sge.ethqsets <= n)
6214 					break;
6215 			}
6216 		}
6217 
6218 	n = 0;
6219 	for_each_port(adap, i) {
6220 		pi = adap2pinfo(adap, i);
6221 		pi->first_qset = n;
6222 		n += pi->nqsets;
6223 	}
6224 }
6225 
6226 /* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
6227 #define EXTRA_VECS 2
6228 
6229 static int enable_msix(struct adapter *adap)
6230 {
6231 	int ofld_need = 0;
6232 	int i, want, need;
6233 	struct sge *s = &adap->sge;
6234 	unsigned int nchan = adap->params.nports;
6235 	struct msix_entry entries[MAX_INGQ + 1];
6236 
6237 	for (i = 0; i < ARRAY_SIZE(entries); ++i)
6238 		entries[i].entry = i;
6239 
6240 	want = s->max_ethqsets + EXTRA_VECS;
6241 	if (is_offload(adap)) {
6242 		want += s->rdmaqs + s->rdmaciqs + s->ofldqsets;
6243 		/* need nchan for each possible ULD */
6244 		ofld_need = 3 * nchan;
6245 	}
6246 #ifdef CONFIG_CHELSIO_T4_DCB
6247 	/* For Data Center Bridging we need 8 Ethernet TX Priority Queues for
6248 	 * each port.
6249 	 */
6250 	need = 8 * adap->params.nports + EXTRA_VECS + ofld_need;
6251 #else
6252 	need = adap->params.nports + EXTRA_VECS + ofld_need;
6253 #endif
6254 	want = pci_enable_msix_range(adap->pdev, entries, need, want);
6255 	if (want < 0)
6256 		return want;
6257 
6258 	/*
6259 	 * Distribute available vectors to the various queue groups.
6260 	 * Every group gets its minimum requirement and NIC gets top
6261 	 * priority for leftovers.
6262 	 */
6263 	i = want - EXTRA_VECS - ofld_need;
6264 	if (i < s->max_ethqsets) {
6265 		s->max_ethqsets = i;
6266 		if (i < s->ethqsets)
6267 			reduce_ethqs(adap, i);
6268 	}
6269 	if (is_offload(adap)) {
6270 		i = want - EXTRA_VECS - s->max_ethqsets;
6271 		i -= ofld_need - nchan;
6272 		s->ofldqsets = (i / nchan) * nchan;  /* round down */
6273 	}
6274 	for (i = 0; i < want; ++i)
6275 		adap->msix_info[i].vec = entries[i].vector;
6276 
6277 	return 0;
6278 }
6279 
6280 #undef EXTRA_VECS
6281 
6282 static int init_rss(struct adapter *adap)
6283 {
6284 	unsigned int i, j;
6285 
6286 	for_each_port(adap, i) {
6287 		struct port_info *pi = adap2pinfo(adap, i);
6288 
6289 		pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
6290 		if (!pi->rss)
6291 			return -ENOMEM;
6292 		for (j = 0; j < pi->rss_size; j++)
6293 			pi->rss[j] = ethtool_rxfh_indir_default(j, pi->nqsets);
6294 	}
6295 	return 0;
6296 }
6297 
6298 static void print_port_info(const struct net_device *dev)
6299 {
6300 	char buf[80];
6301 	char *bufp = buf;
6302 	const char *spd = "";
6303 	const struct port_info *pi = netdev_priv(dev);
6304 	const struct adapter *adap = pi->adapter;
6305 
6306 	if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
6307 		spd = " 2.5 GT/s";
6308 	else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
6309 		spd = " 5 GT/s";
6310 	else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_8_0GB)
6311 		spd = " 8 GT/s";
6312 
6313 	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
6314 		bufp += sprintf(bufp, "100/");
6315 	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
6316 		bufp += sprintf(bufp, "1000/");
6317 	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
6318 		bufp += sprintf(bufp, "10G/");
6319 	if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_40G)
6320 		bufp += sprintf(bufp, "40G/");
6321 	if (bufp != buf)
6322 		--bufp;
6323 	sprintf(bufp, "BASE-%s", t4_get_port_type_description(pi->port_type));
6324 
6325 	netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
6326 		    adap->params.vpd.id,
6327 		    CHELSIO_CHIP_RELEASE(adap->params.chip), buf,
6328 		    is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
6329 		    (adap->flags & USING_MSIX) ? " MSI-X" :
6330 		    (adap->flags & USING_MSI) ? " MSI" : "");
6331 	netdev_info(dev, "S/N: %s, P/N: %s\n",
6332 		    adap->params.vpd.sn, adap->params.vpd.pn);
6333 }
6334 
6335 static void enable_pcie_relaxed_ordering(struct pci_dev *dev)
6336 {
6337 	pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
6338 }
6339 
6340 /*
6341  * Free the following resources:
6342  * - memory used for tables
6343  * - MSI/MSI-X
6344  * - net devices
6345  * - resources FW is holding for us
6346  */
6347 static void free_some_resources(struct adapter *adapter)
6348 {
6349 	unsigned int i;
6350 
6351 	t4_free_mem(adapter->l2t);
6352 	t4_free_mem(adapter->tids.tid_tab);
6353 	disable_msi(adapter);
6354 
6355 	for_each_port(adapter, i)
6356 		if (adapter->port[i]) {
6357 			kfree(adap2pinfo(adapter, i)->rss);
6358 			free_netdev(adapter->port[i]);
6359 		}
6360 	if (adapter->flags & FW_OK)
6361 		t4_fw_bye(adapter, adapter->fn);
6362 }
6363 
6364 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
6365 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
6366 		   NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
6367 #define SEGMENT_SIZE 128
6368 
6369 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
6370 {
6371 	int func, i, err, s_qpp, qpp, num_seg;
6372 	struct port_info *pi;
6373 	bool highdma = false;
6374 	struct adapter *adapter = NULL;
6375 	void __iomem *regs;
6376 
6377 	printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
6378 
6379 	err = pci_request_regions(pdev, KBUILD_MODNAME);
6380 	if (err) {
6381 		/* Just info, some other driver may have claimed the device. */
6382 		dev_info(&pdev->dev, "cannot obtain PCI resources\n");
6383 		return err;
6384 	}
6385 
6386 	err = pci_enable_device(pdev);
6387 	if (err) {
6388 		dev_err(&pdev->dev, "cannot enable PCI device\n");
6389 		goto out_release_regions;
6390 	}
6391 
6392 	regs = pci_ioremap_bar(pdev, 0);
6393 	if (!regs) {
6394 		dev_err(&pdev->dev, "cannot map device registers\n");
6395 		err = -ENOMEM;
6396 		goto out_disable_device;
6397 	}
6398 
6399 	err = t4_wait_dev_ready(regs);
6400 	if (err < 0)
6401 		goto out_unmap_bar0;
6402 
6403 	/* We control everything through one PF */
6404 	func = SOURCEPF_GET(readl(regs + PL_WHOAMI));
6405 	if (func != ent->driver_data) {
6406 		iounmap(regs);
6407 		pci_disable_device(pdev);
6408 		pci_save_state(pdev);        /* to restore SR-IOV later */
6409 		goto sriov;
6410 	}
6411 
6412 	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
6413 		highdma = true;
6414 		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
6415 		if (err) {
6416 			dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
6417 				"coherent allocations\n");
6418 			goto out_unmap_bar0;
6419 		}
6420 	} else {
6421 		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
6422 		if (err) {
6423 			dev_err(&pdev->dev, "no usable DMA configuration\n");
6424 			goto out_unmap_bar0;
6425 		}
6426 	}
6427 
6428 	pci_enable_pcie_error_reporting(pdev);
6429 	enable_pcie_relaxed_ordering(pdev);
6430 	pci_set_master(pdev);
6431 	pci_save_state(pdev);
6432 
6433 	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
6434 	if (!adapter) {
6435 		err = -ENOMEM;
6436 		goto out_unmap_bar0;
6437 	}
6438 
6439 	adapter->workq = create_singlethread_workqueue("cxgb4");
6440 	if (!adapter->workq) {
6441 		err = -ENOMEM;
6442 		goto out_free_adapter;
6443 	}
6444 
6445 	/* PCI device has been enabled */
6446 	adapter->flags |= DEV_ENABLED;
6447 
6448 	adapter->regs = regs;
6449 	adapter->pdev = pdev;
6450 	adapter->pdev_dev = &pdev->dev;
6451 	adapter->mbox = func;
6452 	adapter->fn = func;
6453 	adapter->msg_enable = dflt_msg_enable;
6454 	memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
6455 
6456 	spin_lock_init(&adapter->stats_lock);
6457 	spin_lock_init(&adapter->tid_release_lock);
6458 	spin_lock_init(&adapter->win0_lock);
6459 
6460 	INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
6461 	INIT_WORK(&adapter->db_full_task, process_db_full);
6462 	INIT_WORK(&adapter->db_drop_task, process_db_drop);
6463 
6464 	err = t4_prep_adapter(adapter);
6465 	if (err)
6466 		goto out_free_adapter;
6467 
6468 
6469 	if (!is_t4(adapter->params.chip)) {
6470 		s_qpp = QUEUESPERPAGEPF1 * adapter->fn;
6471 		qpp = 1 << QUEUESPERPAGEPF0_GET(t4_read_reg(adapter,
6472 		      SGE_EGRESS_QUEUES_PER_PAGE_PF) >> s_qpp);
6473 		num_seg = PAGE_SIZE / SEGMENT_SIZE;
6474 
6475 		/* Each segment size is 128B. Write coalescing is enabled only
6476 		 * when SGE_EGRESS_QUEUES_PER_PAGE_PF reg value for the
6477 		 * queue is less no of segments that can be accommodated in
6478 		 * a page size.
6479 		 */
6480 		if (qpp > num_seg) {
6481 			dev_err(&pdev->dev,
6482 				"Incorrect number of egress queues per page\n");
6483 			err = -EINVAL;
6484 			goto out_free_adapter;
6485 		}
6486 		adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
6487 		pci_resource_len(pdev, 2));
6488 		if (!adapter->bar2) {
6489 			dev_err(&pdev->dev, "cannot map device bar2 region\n");
6490 			err = -ENOMEM;
6491 			goto out_free_adapter;
6492 		}
6493 	}
6494 
6495 	setup_memwin(adapter);
6496 	err = adap_init0(adapter);
6497 	setup_memwin_rdma(adapter);
6498 	if (err)
6499 		goto out_unmap_bar;
6500 
6501 	for_each_port(adapter, i) {
6502 		struct net_device *netdev;
6503 
6504 		netdev = alloc_etherdev_mq(sizeof(struct port_info),
6505 					   MAX_ETH_QSETS);
6506 		if (!netdev) {
6507 			err = -ENOMEM;
6508 			goto out_free_dev;
6509 		}
6510 
6511 		SET_NETDEV_DEV(netdev, &pdev->dev);
6512 
6513 		adapter->port[i] = netdev;
6514 		pi = netdev_priv(netdev);
6515 		pi->adapter = adapter;
6516 		pi->xact_addr_filt = -1;
6517 		pi->port_id = i;
6518 		netdev->irq = pdev->irq;
6519 
6520 		netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
6521 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
6522 			NETIF_F_RXCSUM | NETIF_F_RXHASH |
6523 			NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
6524 		if (highdma)
6525 			netdev->hw_features |= NETIF_F_HIGHDMA;
6526 		netdev->features |= netdev->hw_features;
6527 		netdev->vlan_features = netdev->features & VLAN_FEAT;
6528 
6529 		netdev->priv_flags |= IFF_UNICAST_FLT;
6530 
6531 		netdev->netdev_ops = &cxgb4_netdev_ops;
6532 #ifdef CONFIG_CHELSIO_T4_DCB
6533 		netdev->dcbnl_ops = &cxgb4_dcb_ops;
6534 		cxgb4_dcb_state_init(netdev);
6535 #endif
6536 		netdev->ethtool_ops = &cxgb_ethtool_ops;
6537 	}
6538 
6539 	pci_set_drvdata(pdev, adapter);
6540 
6541 	if (adapter->flags & FW_OK) {
6542 		err = t4_port_init(adapter, func, func, 0);
6543 		if (err)
6544 			goto out_free_dev;
6545 	}
6546 
6547 	/*
6548 	 * Configure queues and allocate tables now, they can be needed as
6549 	 * soon as the first register_netdev completes.
6550 	 */
6551 	cfg_queues(adapter);
6552 
6553 	adapter->l2t = t4_init_l2t();
6554 	if (!adapter->l2t) {
6555 		/* We tolerate a lack of L2T, giving up some functionality */
6556 		dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
6557 		adapter->params.offload = 0;
6558 	}
6559 
6560 	if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
6561 		dev_warn(&pdev->dev, "could not allocate TID table, "
6562 			 "continuing\n");
6563 		adapter->params.offload = 0;
6564 	}
6565 
6566 	/* See what interrupts we'll be using */
6567 	if (msi > 1 && enable_msix(adapter) == 0)
6568 		adapter->flags |= USING_MSIX;
6569 	else if (msi > 0 && pci_enable_msi(pdev) == 0)
6570 		adapter->flags |= USING_MSI;
6571 
6572 	err = init_rss(adapter);
6573 	if (err)
6574 		goto out_free_dev;
6575 
6576 	/*
6577 	 * The card is now ready to go.  If any errors occur during device
6578 	 * registration we do not fail the whole card but rather proceed only
6579 	 * with the ports we manage to register successfully.  However we must
6580 	 * register at least one net device.
6581 	 */
6582 	for_each_port(adapter, i) {
6583 		pi = adap2pinfo(adapter, i);
6584 		netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
6585 		netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
6586 
6587 		err = register_netdev(adapter->port[i]);
6588 		if (err)
6589 			break;
6590 		adapter->chan_map[pi->tx_chan] = i;
6591 		print_port_info(adapter->port[i]);
6592 	}
6593 	if (i == 0) {
6594 		dev_err(&pdev->dev, "could not register any net devices\n");
6595 		goto out_free_dev;
6596 	}
6597 	if (err) {
6598 		dev_warn(&pdev->dev, "only %d net devices registered\n", i);
6599 		err = 0;
6600 	}
6601 
6602 	if (cxgb4_debugfs_root) {
6603 		adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
6604 							   cxgb4_debugfs_root);
6605 		setup_debugfs(adapter);
6606 	}
6607 
6608 	/* PCIe EEH recovery on powerpc platforms needs fundamental reset */
6609 	pdev->needs_freset = 1;
6610 
6611 	if (is_offload(adapter))
6612 		attach_ulds(adapter);
6613 
6614 sriov:
6615 #ifdef CONFIG_PCI_IOV
6616 	if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
6617 		if (pci_enable_sriov(pdev, num_vf[func]) == 0)
6618 			dev_info(&pdev->dev,
6619 				 "instantiated %u virtual functions\n",
6620 				 num_vf[func]);
6621 #endif
6622 	return 0;
6623 
6624  out_free_dev:
6625 	free_some_resources(adapter);
6626  out_unmap_bar:
6627 	if (!is_t4(adapter->params.chip))
6628 		iounmap(adapter->bar2);
6629  out_free_adapter:
6630 	if (adapter->workq)
6631 		destroy_workqueue(adapter->workq);
6632 
6633 	kfree(adapter);
6634  out_unmap_bar0:
6635 	iounmap(regs);
6636  out_disable_device:
6637 	pci_disable_pcie_error_reporting(pdev);
6638 	pci_disable_device(pdev);
6639  out_release_regions:
6640 	pci_release_regions(pdev);
6641 	return err;
6642 }
6643 
6644 static void remove_one(struct pci_dev *pdev)
6645 {
6646 	struct adapter *adapter = pci_get_drvdata(pdev);
6647 
6648 #ifdef CONFIG_PCI_IOV
6649 	pci_disable_sriov(pdev);
6650 
6651 #endif
6652 
6653 	if (adapter) {
6654 		int i;
6655 
6656 		/* Tear down per-adapter Work Queue first since it can contain
6657 		 * references to our adapter data structure.
6658 		 */
6659 		destroy_workqueue(adapter->workq);
6660 
6661 		if (is_offload(adapter))
6662 			detach_ulds(adapter);
6663 
6664 		for_each_port(adapter, i)
6665 			if (adapter->port[i]->reg_state == NETREG_REGISTERED)
6666 				unregister_netdev(adapter->port[i]);
6667 
6668 		debugfs_remove_recursive(adapter->debugfs_root);
6669 
6670 		/* If we allocated filters, free up state associated with any
6671 		 * valid filters ...
6672 		 */
6673 		if (adapter->tids.ftid_tab) {
6674 			struct filter_entry *f = &adapter->tids.ftid_tab[0];
6675 			for (i = 0; i < (adapter->tids.nftids +
6676 					adapter->tids.nsftids); i++, f++)
6677 				if (f->valid)
6678 					clear_filter(adapter, f);
6679 		}
6680 
6681 		if (adapter->flags & FULL_INIT_DONE)
6682 			cxgb_down(adapter);
6683 
6684 		free_some_resources(adapter);
6685 		iounmap(adapter->regs);
6686 		if (!is_t4(adapter->params.chip))
6687 			iounmap(adapter->bar2);
6688 		pci_disable_pcie_error_reporting(pdev);
6689 		if ((adapter->flags & DEV_ENABLED)) {
6690 			pci_disable_device(pdev);
6691 			adapter->flags &= ~DEV_ENABLED;
6692 		}
6693 		pci_release_regions(pdev);
6694 		synchronize_rcu();
6695 		kfree(adapter);
6696 	} else
6697 		pci_release_regions(pdev);
6698 }
6699 
6700 static struct pci_driver cxgb4_driver = {
6701 	.name     = KBUILD_MODNAME,
6702 	.id_table = cxgb4_pci_tbl,
6703 	.probe    = init_one,
6704 	.remove   = remove_one,
6705 	.shutdown = remove_one,
6706 	.err_handler = &cxgb4_eeh,
6707 };
6708 
6709 static int __init cxgb4_init_module(void)
6710 {
6711 	int ret;
6712 
6713 	/* Debugfs support is optional, just warn if this fails */
6714 	cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
6715 	if (!cxgb4_debugfs_root)
6716 		pr_warn("could not create debugfs entry, continuing\n");
6717 
6718 	ret = pci_register_driver(&cxgb4_driver);
6719 	if (ret < 0)
6720 		debugfs_remove(cxgb4_debugfs_root);
6721 
6722 #if IS_ENABLED(CONFIG_IPV6)
6723 	register_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6724 #endif
6725 
6726 	return ret;
6727 }
6728 
6729 static void __exit cxgb4_cleanup_module(void)
6730 {
6731 #if IS_ENABLED(CONFIG_IPV6)
6732 	unregister_inet6addr_notifier(&cxgb4_inet6addr_notifier);
6733 #endif
6734 	pci_unregister_driver(&cxgb4_driver);
6735 	debugfs_remove(cxgb4_debugfs_root);  /* NULL ok */
6736 }
6737 
6738 module_init(cxgb4_init_module);
6739 module_exit(cxgb4_cleanup_module);
6740