1 /*
2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34 
35 #include <linux/module.h>
36 #include <linux/moduleparam.h>
37 #include <linux/init.h>
38 #include <linux/pci.h>
39 #include <linux/dma-mapping.h>
40 #include <linux/netdevice.h>
41 #include <linux/etherdevice.h>
42 #include <linux/if_vlan.h>
43 #include <linux/mdio.h>
44 #include <linux/sockios.h>
45 #include <linux/workqueue.h>
46 #include <linux/proc_fs.h>
47 #include <linux/rtnetlink.h>
48 #include <linux/firmware.h>
49 #include <linux/log2.h>
50 #include <linux/stringify.h>
51 #include <linux/sched.h>
52 #include <linux/slab.h>
53 #include <asm/uaccess.h>
54 
55 #include "common.h"
56 #include "cxgb3_ioctl.h"
57 #include "regs.h"
58 #include "cxgb3_offload.h"
59 #include "version.h"
60 
61 #include "cxgb3_ctl_defs.h"
62 #include "t3_cpl.h"
63 #include "firmware_exports.h"
64 
65 enum {
66 	MAX_TXQ_ENTRIES = 16384,
67 	MAX_CTRL_TXQ_ENTRIES = 1024,
68 	MAX_RSPQ_ENTRIES = 16384,
69 	MAX_RX_BUFFERS = 16384,
70 	MAX_RX_JUMBO_BUFFERS = 16384,
71 	MIN_TXQ_ENTRIES = 4,
72 	MIN_CTRL_TXQ_ENTRIES = 4,
73 	MIN_RSPQ_ENTRIES = 32,
74 	MIN_FL_ENTRIES = 32
75 };
76 
77 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
78 
79 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
80 			 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
81 			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
82 
83 #define EEPROM_MAGIC 0x38E2F10C
84 
85 #define CH_DEVICE(devid, idx) \
86 	{ PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
87 
88 static DEFINE_PCI_DEVICE_TABLE(cxgb3_pci_tbl) = {
89 	CH_DEVICE(0x20, 0),	/* PE9000 */
90 	CH_DEVICE(0x21, 1),	/* T302E */
91 	CH_DEVICE(0x22, 2),	/* T310E */
92 	CH_DEVICE(0x23, 3),	/* T320X */
93 	CH_DEVICE(0x24, 1),	/* T302X */
94 	CH_DEVICE(0x25, 3),	/* T320E */
95 	CH_DEVICE(0x26, 2),	/* T310X */
96 	CH_DEVICE(0x30, 2),	/* T3B10 */
97 	CH_DEVICE(0x31, 3),	/* T3B20 */
98 	CH_DEVICE(0x32, 1),	/* T3B02 */
99 	CH_DEVICE(0x35, 6),	/* T3C20-derived T3C10 */
100 	CH_DEVICE(0x36, 3),	/* S320E-CR */
101 	CH_DEVICE(0x37, 7),	/* N320E-G2 */
102 	{0,}
103 };
104 
105 MODULE_DESCRIPTION(DRV_DESC);
106 MODULE_AUTHOR("Chelsio Communications");
107 MODULE_LICENSE("Dual BSD/GPL");
108 MODULE_VERSION(DRV_VERSION);
109 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
110 
111 static int dflt_msg_enable = DFLT_MSG_ENABLE;
112 
113 module_param(dflt_msg_enable, int, 0644);
114 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
115 
116 /*
117  * The driver uses the best interrupt scheme available on a platform in the
118  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
119  * of these schemes the driver may consider as follows:
120  *
121  * msi = 2: choose from among all three options
122  * msi = 1: only consider MSI and pin interrupts
123  * msi = 0: force pin interrupts
124  */
125 static int msi = 2;
126 
127 module_param(msi, int, 0644);
128 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
129 
130 /*
131  * The driver enables offload as a default.
132  * To disable it, use ofld_disable = 1.
133  */
134 
135 static int ofld_disable = 0;
136 
137 module_param(ofld_disable, int, 0644);
138 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
139 
140 /*
141  * We have work elements that we need to cancel when an interface is taken
142  * down.  Normally the work elements would be executed by keventd but that
143  * can deadlock because of linkwatch.  If our close method takes the rtnl
144  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
145  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
146  * for our work to complete.  Get our own work queue to solve this.
147  */
148 struct workqueue_struct *cxgb3_wq;
149 
150 /**
151  *	link_report - show link status and link speed/duplex
152  *	@p: the port whose settings are to be reported
153  *
154  *	Shows the link status, speed, and duplex of a port.
155  */
156 static void link_report(struct net_device *dev)
157 {
158 	if (!netif_carrier_ok(dev))
159 		netdev_info(dev, "link down\n");
160 	else {
161 		const char *s = "10Mbps";
162 		const struct port_info *p = netdev_priv(dev);
163 
164 		switch (p->link_config.speed) {
165 		case SPEED_10000:
166 			s = "10Gbps";
167 			break;
168 		case SPEED_1000:
169 			s = "1000Mbps";
170 			break;
171 		case SPEED_100:
172 			s = "100Mbps";
173 			break;
174 		}
175 
176 		netdev_info(dev, "link up, %s, %s-duplex\n",
177 			    s, p->link_config.duplex == DUPLEX_FULL
178 			    ? "full" : "half");
179 	}
180 }
181 
182 static void enable_tx_fifo_drain(struct adapter *adapter,
183 				 struct port_info *pi)
184 {
185 	t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
186 			 F_ENDROPPKT);
187 	t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
188 	t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
189 	t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
190 }
191 
192 static void disable_tx_fifo_drain(struct adapter *adapter,
193 				  struct port_info *pi)
194 {
195 	t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
196 			 F_ENDROPPKT, 0);
197 }
198 
199 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
200 {
201 	struct net_device *dev = adap->port[port_id];
202 	struct port_info *pi = netdev_priv(dev);
203 
204 	if (state == netif_carrier_ok(dev))
205 		return;
206 
207 	if (state) {
208 		struct cmac *mac = &pi->mac;
209 
210 		netif_carrier_on(dev);
211 
212 		disable_tx_fifo_drain(adap, pi);
213 
214 		/* Clear local faults */
215 		t3_xgm_intr_disable(adap, pi->port_id);
216 		t3_read_reg(adap, A_XGM_INT_STATUS +
217 				    pi->mac.offset);
218 		t3_write_reg(adap,
219 			     A_XGM_INT_CAUSE + pi->mac.offset,
220 			     F_XGM_INT);
221 
222 		t3_set_reg_field(adap,
223 				 A_XGM_INT_ENABLE +
224 				 pi->mac.offset,
225 				 F_XGM_INT, F_XGM_INT);
226 		t3_xgm_intr_enable(adap, pi->port_id);
227 
228 		t3_mac_enable(mac, MAC_DIRECTION_TX);
229 	} else {
230 		netif_carrier_off(dev);
231 
232 		/* Flush TX FIFO */
233 		enable_tx_fifo_drain(adap, pi);
234 	}
235 	link_report(dev);
236 }
237 
238 /**
239  *	t3_os_link_changed - handle link status changes
240  *	@adapter: the adapter associated with the link change
241  *	@port_id: the port index whose limk status has changed
242  *	@link_stat: the new status of the link
243  *	@speed: the new speed setting
244  *	@duplex: the new duplex setting
245  *	@pause: the new flow-control setting
246  *
247  *	This is the OS-dependent handler for link status changes.  The OS
248  *	neutral handler takes care of most of the processing for these events,
249  *	then calls this handler for any OS-specific processing.
250  */
251 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
252 			int speed, int duplex, int pause)
253 {
254 	struct net_device *dev = adapter->port[port_id];
255 	struct port_info *pi = netdev_priv(dev);
256 	struct cmac *mac = &pi->mac;
257 
258 	/* Skip changes from disabled ports. */
259 	if (!netif_running(dev))
260 		return;
261 
262 	if (link_stat != netif_carrier_ok(dev)) {
263 		if (link_stat) {
264 			disable_tx_fifo_drain(adapter, pi);
265 
266 			t3_mac_enable(mac, MAC_DIRECTION_RX);
267 
268 			/* Clear local faults */
269 			t3_xgm_intr_disable(adapter, pi->port_id);
270 			t3_read_reg(adapter, A_XGM_INT_STATUS +
271 				    pi->mac.offset);
272 			t3_write_reg(adapter,
273 				     A_XGM_INT_CAUSE + pi->mac.offset,
274 				     F_XGM_INT);
275 
276 			t3_set_reg_field(adapter,
277 					 A_XGM_INT_ENABLE + pi->mac.offset,
278 					 F_XGM_INT, F_XGM_INT);
279 			t3_xgm_intr_enable(adapter, pi->port_id);
280 
281 			netif_carrier_on(dev);
282 		} else {
283 			netif_carrier_off(dev);
284 
285 			t3_xgm_intr_disable(adapter, pi->port_id);
286 			t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
287 			t3_set_reg_field(adapter,
288 					 A_XGM_INT_ENABLE + pi->mac.offset,
289 					 F_XGM_INT, 0);
290 
291 			if (is_10G(adapter))
292 				pi->phy.ops->power_down(&pi->phy, 1);
293 
294 			t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
295 			t3_mac_disable(mac, MAC_DIRECTION_RX);
296 			t3_link_start(&pi->phy, mac, &pi->link_config);
297 
298 			/* Flush TX FIFO */
299 			enable_tx_fifo_drain(adapter, pi);
300 		}
301 
302 		link_report(dev);
303 	}
304 }
305 
306 /**
307  *	t3_os_phymod_changed - handle PHY module changes
308  *	@phy: the PHY reporting the module change
309  *	@mod_type: new module type
310  *
311  *	This is the OS-dependent handler for PHY module changes.  It is
312  *	invoked when a PHY module is removed or inserted for any OS-specific
313  *	processing.
314  */
315 void t3_os_phymod_changed(struct adapter *adap, int port_id)
316 {
317 	static const char *mod_str[] = {
318 		NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
319 	};
320 
321 	const struct net_device *dev = adap->port[port_id];
322 	const struct port_info *pi = netdev_priv(dev);
323 
324 	if (pi->phy.modtype == phy_modtype_none)
325 		netdev_info(dev, "PHY module unplugged\n");
326 	else
327 		netdev_info(dev, "%s PHY module inserted\n",
328 			    mod_str[pi->phy.modtype]);
329 }
330 
331 static void cxgb_set_rxmode(struct net_device *dev)
332 {
333 	struct port_info *pi = netdev_priv(dev);
334 
335 	t3_mac_set_rx_mode(&pi->mac, dev);
336 }
337 
338 /**
339  *	link_start - enable a port
340  *	@dev: the device to enable
341  *
342  *	Performs the MAC and PHY actions needed to enable a port.
343  */
344 static void link_start(struct net_device *dev)
345 {
346 	struct port_info *pi = netdev_priv(dev);
347 	struct cmac *mac = &pi->mac;
348 
349 	t3_mac_reset(mac);
350 	t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
351 	t3_mac_set_mtu(mac, dev->mtu);
352 	t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
353 	t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
354 	t3_mac_set_rx_mode(mac, dev);
355 	t3_link_start(&pi->phy, mac, &pi->link_config);
356 	t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
357 }
358 
359 static inline void cxgb_disable_msi(struct adapter *adapter)
360 {
361 	if (adapter->flags & USING_MSIX) {
362 		pci_disable_msix(adapter->pdev);
363 		adapter->flags &= ~USING_MSIX;
364 	} else if (adapter->flags & USING_MSI) {
365 		pci_disable_msi(adapter->pdev);
366 		adapter->flags &= ~USING_MSI;
367 	}
368 }
369 
370 /*
371  * Interrupt handler for asynchronous events used with MSI-X.
372  */
373 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
374 {
375 	t3_slow_intr_handler(cookie);
376 	return IRQ_HANDLED;
377 }
378 
379 /*
380  * Name the MSI-X interrupts.
381  */
382 static void name_msix_vecs(struct adapter *adap)
383 {
384 	int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
385 
386 	snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
387 	adap->msix_info[0].desc[n] = 0;
388 
389 	for_each_port(adap, j) {
390 		struct net_device *d = adap->port[j];
391 		const struct port_info *pi = netdev_priv(d);
392 
393 		for (i = 0; i < pi->nqsets; i++, msi_idx++) {
394 			snprintf(adap->msix_info[msi_idx].desc, n,
395 				 "%s-%d", d->name, pi->first_qset + i);
396 			adap->msix_info[msi_idx].desc[n] = 0;
397 		}
398 	}
399 }
400 
401 static int request_msix_data_irqs(struct adapter *adap)
402 {
403 	int i, j, err, qidx = 0;
404 
405 	for_each_port(adap, i) {
406 		int nqsets = adap2pinfo(adap, i)->nqsets;
407 
408 		for (j = 0; j < nqsets; ++j) {
409 			err = request_irq(adap->msix_info[qidx + 1].vec,
410 					  t3_intr_handler(adap,
411 							  adap->sge.qs[qidx].
412 							  rspq.polling), 0,
413 					  adap->msix_info[qidx + 1].desc,
414 					  &adap->sge.qs[qidx]);
415 			if (err) {
416 				while (--qidx >= 0)
417 					free_irq(adap->msix_info[qidx + 1].vec,
418 						 &adap->sge.qs[qidx]);
419 				return err;
420 			}
421 			qidx++;
422 		}
423 	}
424 	return 0;
425 }
426 
427 static void free_irq_resources(struct adapter *adapter)
428 {
429 	if (adapter->flags & USING_MSIX) {
430 		int i, n = 0;
431 
432 		free_irq(adapter->msix_info[0].vec, adapter);
433 		for_each_port(adapter, i)
434 			n += adap2pinfo(adapter, i)->nqsets;
435 
436 		for (i = 0; i < n; ++i)
437 			free_irq(adapter->msix_info[i + 1].vec,
438 				 &adapter->sge.qs[i]);
439 	} else
440 		free_irq(adapter->pdev->irq, adapter);
441 }
442 
443 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
444 			      unsigned long n)
445 {
446 	int attempts = 10;
447 
448 	while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
449 		if (!--attempts)
450 			return -ETIMEDOUT;
451 		msleep(10);
452 	}
453 	return 0;
454 }
455 
456 static int init_tp_parity(struct adapter *adap)
457 {
458 	int i;
459 	struct sk_buff *skb;
460 	struct cpl_set_tcb_field *greq;
461 	unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
462 
463 	t3_tp_set_offload_mode(adap, 1);
464 
465 	for (i = 0; i < 16; i++) {
466 		struct cpl_smt_write_req *req;
467 
468 		skb = alloc_skb(sizeof(*req), GFP_KERNEL);
469 		if (!skb)
470 			skb = adap->nofail_skb;
471 		if (!skb)
472 			goto alloc_skb_fail;
473 
474 		req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
475 		memset(req, 0, sizeof(*req));
476 		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
477 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
478 		req->mtu_idx = NMTUS - 1;
479 		req->iff = i;
480 		t3_mgmt_tx(adap, skb);
481 		if (skb == adap->nofail_skb) {
482 			await_mgmt_replies(adap, cnt, i + 1);
483 			adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
484 			if (!adap->nofail_skb)
485 				goto alloc_skb_fail;
486 		}
487 	}
488 
489 	for (i = 0; i < 2048; i++) {
490 		struct cpl_l2t_write_req *req;
491 
492 		skb = alloc_skb(sizeof(*req), GFP_KERNEL);
493 		if (!skb)
494 			skb = adap->nofail_skb;
495 		if (!skb)
496 			goto alloc_skb_fail;
497 
498 		req = (struct cpl_l2t_write_req *)__skb_put(skb, sizeof(*req));
499 		memset(req, 0, sizeof(*req));
500 		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
501 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
502 		req->params = htonl(V_L2T_W_IDX(i));
503 		t3_mgmt_tx(adap, skb);
504 		if (skb == adap->nofail_skb) {
505 			await_mgmt_replies(adap, cnt, 16 + i + 1);
506 			adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
507 			if (!adap->nofail_skb)
508 				goto alloc_skb_fail;
509 		}
510 	}
511 
512 	for (i = 0; i < 2048; i++) {
513 		struct cpl_rte_write_req *req;
514 
515 		skb = alloc_skb(sizeof(*req), GFP_KERNEL);
516 		if (!skb)
517 			skb = adap->nofail_skb;
518 		if (!skb)
519 			goto alloc_skb_fail;
520 
521 		req = (struct cpl_rte_write_req *)__skb_put(skb, sizeof(*req));
522 		memset(req, 0, sizeof(*req));
523 		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
524 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
525 		req->l2t_idx = htonl(V_L2T_W_IDX(i));
526 		t3_mgmt_tx(adap, skb);
527 		if (skb == adap->nofail_skb) {
528 			await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
529 			adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
530 			if (!adap->nofail_skb)
531 				goto alloc_skb_fail;
532 		}
533 	}
534 
535 	skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
536 	if (!skb)
537 		skb = adap->nofail_skb;
538 	if (!skb)
539 		goto alloc_skb_fail;
540 
541 	greq = (struct cpl_set_tcb_field *)__skb_put(skb, sizeof(*greq));
542 	memset(greq, 0, sizeof(*greq));
543 	greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
544 	OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
545 	greq->mask = cpu_to_be64(1);
546 	t3_mgmt_tx(adap, skb);
547 
548 	i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
549 	if (skb == adap->nofail_skb) {
550 		i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
551 		adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
552 	}
553 
554 	t3_tp_set_offload_mode(adap, 0);
555 	return i;
556 
557 alloc_skb_fail:
558 	t3_tp_set_offload_mode(adap, 0);
559 	return -ENOMEM;
560 }
561 
562 /**
563  *	setup_rss - configure RSS
564  *	@adap: the adapter
565  *
566  *	Sets up RSS to distribute packets to multiple receive queues.  We
567  *	configure the RSS CPU lookup table to distribute to the number of HW
568  *	receive queues, and the response queue lookup table to narrow that
569  *	down to the response queues actually configured for each port.
570  *	We always configure the RSS mapping for two ports since the mapping
571  *	table has plenty of entries.
572  */
573 static void setup_rss(struct adapter *adap)
574 {
575 	int i;
576 	unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
577 	unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
578 	u8 cpus[SGE_QSETS + 1];
579 	u16 rspq_map[RSS_TABLE_SIZE];
580 
581 	for (i = 0; i < SGE_QSETS; ++i)
582 		cpus[i] = i;
583 	cpus[SGE_QSETS] = 0xff;	/* terminator */
584 
585 	for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
586 		rspq_map[i] = i % nq0;
587 		rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
588 	}
589 
590 	t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
591 		      F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
592 		      V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
593 }
594 
595 static void ring_dbs(struct adapter *adap)
596 {
597 	int i, j;
598 
599 	for (i = 0; i < SGE_QSETS; i++) {
600 		struct sge_qset *qs = &adap->sge.qs[i];
601 
602 		if (qs->adap)
603 			for (j = 0; j < SGE_TXQ_PER_SET; j++)
604 				t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
605 	}
606 }
607 
608 static void init_napi(struct adapter *adap)
609 {
610 	int i;
611 
612 	for (i = 0; i < SGE_QSETS; i++) {
613 		struct sge_qset *qs = &adap->sge.qs[i];
614 
615 		if (qs->adap)
616 			netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
617 				       64);
618 	}
619 
620 	/*
621 	 * netif_napi_add() can be called only once per napi_struct because it
622 	 * adds each new napi_struct to a list.  Be careful not to call it a
623 	 * second time, e.g., during EEH recovery, by making a note of it.
624 	 */
625 	adap->flags |= NAPI_INIT;
626 }
627 
628 /*
629  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
630  * both netdevices representing interfaces and the dummy ones for the extra
631  * queues.
632  */
633 static void quiesce_rx(struct adapter *adap)
634 {
635 	int i;
636 
637 	for (i = 0; i < SGE_QSETS; i++)
638 		if (adap->sge.qs[i].adap)
639 			napi_disable(&adap->sge.qs[i].napi);
640 }
641 
642 static void enable_all_napi(struct adapter *adap)
643 {
644 	int i;
645 	for (i = 0; i < SGE_QSETS; i++)
646 		if (adap->sge.qs[i].adap)
647 			napi_enable(&adap->sge.qs[i].napi);
648 }
649 
650 /**
651  *	setup_sge_qsets - configure SGE Tx/Rx/response queues
652  *	@adap: the adapter
653  *
654  *	Determines how many sets of SGE queues to use and initializes them.
655  *	We support multiple queue sets per port if we have MSI-X, otherwise
656  *	just one queue set per port.
657  */
658 static int setup_sge_qsets(struct adapter *adap)
659 {
660 	int i, j, err, irq_idx = 0, qset_idx = 0;
661 	unsigned int ntxq = SGE_TXQ_PER_SET;
662 
663 	if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
664 		irq_idx = -1;
665 
666 	for_each_port(adap, i) {
667 		struct net_device *dev = adap->port[i];
668 		struct port_info *pi = netdev_priv(dev);
669 
670 		pi->qs = &adap->sge.qs[pi->first_qset];
671 		for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
672 			err = t3_sge_alloc_qset(adap, qset_idx, 1,
673 				(adap->flags & USING_MSIX) ? qset_idx + 1 :
674 							     irq_idx,
675 				&adap->params.sge.qset[qset_idx], ntxq, dev,
676 				netdev_get_tx_queue(dev, j));
677 			if (err) {
678 				t3_free_sge_resources(adap);
679 				return err;
680 			}
681 		}
682 	}
683 
684 	return 0;
685 }
686 
687 static ssize_t attr_show(struct device *d, char *buf,
688 			 ssize_t(*format) (struct net_device *, char *))
689 {
690 	ssize_t len;
691 
692 	/* Synchronize with ioctls that may shut down the device */
693 	rtnl_lock();
694 	len = (*format) (to_net_dev(d), buf);
695 	rtnl_unlock();
696 	return len;
697 }
698 
699 static ssize_t attr_store(struct device *d,
700 			  const char *buf, size_t len,
701 			  ssize_t(*set) (struct net_device *, unsigned int),
702 			  unsigned int min_val, unsigned int max_val)
703 {
704 	char *endp;
705 	ssize_t ret;
706 	unsigned int val;
707 
708 	if (!capable(CAP_NET_ADMIN))
709 		return -EPERM;
710 
711 	val = simple_strtoul(buf, &endp, 0);
712 	if (endp == buf || val < min_val || val > max_val)
713 		return -EINVAL;
714 
715 	rtnl_lock();
716 	ret = (*set) (to_net_dev(d), val);
717 	if (!ret)
718 		ret = len;
719 	rtnl_unlock();
720 	return ret;
721 }
722 
723 #define CXGB3_SHOW(name, val_expr) \
724 static ssize_t format_##name(struct net_device *dev, char *buf) \
725 { \
726 	struct port_info *pi = netdev_priv(dev); \
727 	struct adapter *adap = pi->adapter; \
728 	return sprintf(buf, "%u\n", val_expr); \
729 } \
730 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
731 			   char *buf) \
732 { \
733 	return attr_show(d, buf, format_##name); \
734 }
735 
736 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
737 {
738 	struct port_info *pi = netdev_priv(dev);
739 	struct adapter *adap = pi->adapter;
740 	int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
741 
742 	if (adap->flags & FULL_INIT_DONE)
743 		return -EBUSY;
744 	if (val && adap->params.rev == 0)
745 		return -EINVAL;
746 	if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
747 	    min_tids)
748 		return -EINVAL;
749 	adap->params.mc5.nfilters = val;
750 	return 0;
751 }
752 
753 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
754 			      const char *buf, size_t len)
755 {
756 	return attr_store(d, buf, len, set_nfilters, 0, ~0);
757 }
758 
759 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
760 {
761 	struct port_info *pi = netdev_priv(dev);
762 	struct adapter *adap = pi->adapter;
763 
764 	if (adap->flags & FULL_INIT_DONE)
765 		return -EBUSY;
766 	if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
767 	    MC5_MIN_TIDS)
768 		return -EINVAL;
769 	adap->params.mc5.nservers = val;
770 	return 0;
771 }
772 
773 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
774 			      const char *buf, size_t len)
775 {
776 	return attr_store(d, buf, len, set_nservers, 0, ~0);
777 }
778 
779 #define CXGB3_ATTR_R(name, val_expr) \
780 CXGB3_SHOW(name, val_expr) \
781 static DEVICE_ATTR(name, S_IRUGO, show_##name, NULL)
782 
783 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
784 CXGB3_SHOW(name, val_expr) \
785 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_method)
786 
787 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
788 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
789 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
790 
791 static struct attribute *cxgb3_attrs[] = {
792 	&dev_attr_cam_size.attr,
793 	&dev_attr_nfilters.attr,
794 	&dev_attr_nservers.attr,
795 	NULL
796 };
797 
798 static struct attribute_group cxgb3_attr_group = {.attrs = cxgb3_attrs };
799 
800 static ssize_t tm_attr_show(struct device *d,
801 			    char *buf, int sched)
802 {
803 	struct port_info *pi = netdev_priv(to_net_dev(d));
804 	struct adapter *adap = pi->adapter;
805 	unsigned int v, addr, bpt, cpt;
806 	ssize_t len;
807 
808 	addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
809 	rtnl_lock();
810 	t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
811 	v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
812 	if (sched & 1)
813 		v >>= 16;
814 	bpt = (v >> 8) & 0xff;
815 	cpt = v & 0xff;
816 	if (!cpt)
817 		len = sprintf(buf, "disabled\n");
818 	else {
819 		v = (adap->params.vpd.cclk * 1000) / cpt;
820 		len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
821 	}
822 	rtnl_unlock();
823 	return len;
824 }
825 
826 static ssize_t tm_attr_store(struct device *d,
827 			     const char *buf, size_t len, int sched)
828 {
829 	struct port_info *pi = netdev_priv(to_net_dev(d));
830 	struct adapter *adap = pi->adapter;
831 	unsigned int val;
832 	char *endp;
833 	ssize_t ret;
834 
835 	if (!capable(CAP_NET_ADMIN))
836 		return -EPERM;
837 
838 	val = simple_strtoul(buf, &endp, 0);
839 	if (endp == buf || val > 10000000)
840 		return -EINVAL;
841 
842 	rtnl_lock();
843 	ret = t3_config_sched(adap, val, sched);
844 	if (!ret)
845 		ret = len;
846 	rtnl_unlock();
847 	return ret;
848 }
849 
850 #define TM_ATTR(name, sched) \
851 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
852 			   char *buf) \
853 { \
854 	return tm_attr_show(d, buf, sched); \
855 } \
856 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
857 			    const char *buf, size_t len) \
858 { \
859 	return tm_attr_store(d, buf, len, sched); \
860 } \
861 static DEVICE_ATTR(name, S_IRUGO | S_IWUSR, show_##name, store_##name)
862 
863 TM_ATTR(sched0, 0);
864 TM_ATTR(sched1, 1);
865 TM_ATTR(sched2, 2);
866 TM_ATTR(sched3, 3);
867 TM_ATTR(sched4, 4);
868 TM_ATTR(sched5, 5);
869 TM_ATTR(sched6, 6);
870 TM_ATTR(sched7, 7);
871 
872 static struct attribute *offload_attrs[] = {
873 	&dev_attr_sched0.attr,
874 	&dev_attr_sched1.attr,
875 	&dev_attr_sched2.attr,
876 	&dev_attr_sched3.attr,
877 	&dev_attr_sched4.attr,
878 	&dev_attr_sched5.attr,
879 	&dev_attr_sched6.attr,
880 	&dev_attr_sched7.attr,
881 	NULL
882 };
883 
884 static struct attribute_group offload_attr_group = {.attrs = offload_attrs };
885 
886 /*
887  * Sends an sk_buff to an offload queue driver
888  * after dealing with any active network taps.
889  */
890 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
891 {
892 	int ret;
893 
894 	local_bh_disable();
895 	ret = t3_offload_tx(tdev, skb);
896 	local_bh_enable();
897 	return ret;
898 }
899 
900 static int write_smt_entry(struct adapter *adapter, int idx)
901 {
902 	struct cpl_smt_write_req *req;
903 	struct port_info *pi = netdev_priv(adapter->port[idx]);
904 	struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
905 
906 	if (!skb)
907 		return -ENOMEM;
908 
909 	req = (struct cpl_smt_write_req *)__skb_put(skb, sizeof(*req));
910 	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
911 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
912 	req->mtu_idx = NMTUS - 1;	/* should be 0 but there's a T3 bug */
913 	req->iff = idx;
914 	memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
915 	memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
916 	skb->priority = 1;
917 	offload_tx(&adapter->tdev, skb);
918 	return 0;
919 }
920 
921 static int init_smt(struct adapter *adapter)
922 {
923 	int i;
924 
925 	for_each_port(adapter, i)
926 	    write_smt_entry(adapter, i);
927 	return 0;
928 }
929 
930 static void init_port_mtus(struct adapter *adapter)
931 {
932 	unsigned int mtus = adapter->port[0]->mtu;
933 
934 	if (adapter->port[1])
935 		mtus |= adapter->port[1]->mtu << 16;
936 	t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
937 }
938 
939 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
940 			      int hi, int port)
941 {
942 	struct sk_buff *skb;
943 	struct mngt_pktsched_wr *req;
944 	int ret;
945 
946 	skb = alloc_skb(sizeof(*req), GFP_KERNEL);
947 	if (!skb)
948 		skb = adap->nofail_skb;
949 	if (!skb)
950 		return -ENOMEM;
951 
952 	req = (struct mngt_pktsched_wr *)skb_put(skb, sizeof(*req));
953 	req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
954 	req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
955 	req->sched = sched;
956 	req->idx = qidx;
957 	req->min = lo;
958 	req->max = hi;
959 	req->binding = port;
960 	ret = t3_mgmt_tx(adap, skb);
961 	if (skb == adap->nofail_skb) {
962 		adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
963 					     GFP_KERNEL);
964 		if (!adap->nofail_skb)
965 			ret = -ENOMEM;
966 	}
967 
968 	return ret;
969 }
970 
971 static int bind_qsets(struct adapter *adap)
972 {
973 	int i, j, err = 0;
974 
975 	for_each_port(adap, i) {
976 		const struct port_info *pi = adap2pinfo(adap, i);
977 
978 		for (j = 0; j < pi->nqsets; ++j) {
979 			int ret = send_pktsched_cmd(adap, 1,
980 						    pi->first_qset + j, -1,
981 						    -1, i);
982 			if (ret)
983 				err = ret;
984 		}
985 	}
986 
987 	return err;
988 }
989 
990 #define FW_VERSION __stringify(FW_VERSION_MAJOR) "."			\
991 	__stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO)
992 #define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin"
993 #define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "."		\
994 	__stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO)
995 #define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin"
996 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
997 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
998 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
999 MODULE_FIRMWARE(FW_FNAME);
1000 MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin");
1001 MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin");
1002 MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME);
1003 MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME);
1004 MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME);
1005 
1006 static inline const char *get_edc_fw_name(int edc_idx)
1007 {
1008 	const char *fw_name = NULL;
1009 
1010 	switch (edc_idx) {
1011 	case EDC_OPT_AEL2005:
1012 		fw_name = AEL2005_OPT_EDC_NAME;
1013 		break;
1014 	case EDC_TWX_AEL2005:
1015 		fw_name = AEL2005_TWX_EDC_NAME;
1016 		break;
1017 	case EDC_TWX_AEL2020:
1018 		fw_name = AEL2020_TWX_EDC_NAME;
1019 		break;
1020 	}
1021 	return fw_name;
1022 }
1023 
1024 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1025 {
1026 	struct adapter *adapter = phy->adapter;
1027 	const struct firmware *fw;
1028 	char buf[64];
1029 	u32 csum;
1030 	const __be32 *p;
1031 	u16 *cache = phy->phy_cache;
1032 	int i, ret;
1033 
1034 	snprintf(buf, sizeof(buf), get_edc_fw_name(edc_idx));
1035 
1036 	ret = request_firmware(&fw, buf, &adapter->pdev->dev);
1037 	if (ret < 0) {
1038 		dev_err(&adapter->pdev->dev,
1039 			"could not upgrade firmware: unable to load %s\n",
1040 			buf);
1041 		return ret;
1042 	}
1043 
1044 	/* check size, take checksum in account */
1045 	if (fw->size > size + 4) {
1046 		CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1047 		       (unsigned int)fw->size, size + 4);
1048 		ret = -EINVAL;
1049 	}
1050 
1051 	/* compute checksum */
1052 	p = (const __be32 *)fw->data;
1053 	for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1054 		csum += ntohl(p[i]);
1055 
1056 	if (csum != 0xffffffff) {
1057 		CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1058 		       csum);
1059 		ret = -EINVAL;
1060 	}
1061 
1062 	for (i = 0; i < size / 4 ; i++) {
1063 		*cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1064 		*cache++ = be32_to_cpu(p[i]) & 0xffff;
1065 	}
1066 
1067 	release_firmware(fw);
1068 
1069 	return ret;
1070 }
1071 
1072 static int upgrade_fw(struct adapter *adap)
1073 {
1074 	int ret;
1075 	const struct firmware *fw;
1076 	struct device *dev = &adap->pdev->dev;
1077 
1078 	ret = request_firmware(&fw, FW_FNAME, dev);
1079 	if (ret < 0) {
1080 		dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1081 			FW_FNAME);
1082 		return ret;
1083 	}
1084 	ret = t3_load_fw(adap, fw->data, fw->size);
1085 	release_firmware(fw);
1086 
1087 	if (ret == 0)
1088 		dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1089 			 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1090 	else
1091 		dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1092 			FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1093 
1094 	return ret;
1095 }
1096 
1097 static inline char t3rev2char(struct adapter *adapter)
1098 {
1099 	char rev = 0;
1100 
1101 	switch(adapter->params.rev) {
1102 	case T3_REV_B:
1103 	case T3_REV_B2:
1104 		rev = 'b';
1105 		break;
1106 	case T3_REV_C:
1107 		rev = 'c';
1108 		break;
1109 	}
1110 	return rev;
1111 }
1112 
1113 static int update_tpsram(struct adapter *adap)
1114 {
1115 	const struct firmware *tpsram;
1116 	char buf[64];
1117 	struct device *dev = &adap->pdev->dev;
1118 	int ret;
1119 	char rev;
1120 
1121 	rev = t3rev2char(adap);
1122 	if (!rev)
1123 		return 0;
1124 
1125 	snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
1126 
1127 	ret = request_firmware(&tpsram, buf, dev);
1128 	if (ret < 0) {
1129 		dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1130 			buf);
1131 		return ret;
1132 	}
1133 
1134 	ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1135 	if (ret)
1136 		goto release_tpsram;
1137 
1138 	ret = t3_set_proto_sram(adap, tpsram->data);
1139 	if (ret == 0)
1140 		dev_info(dev,
1141 			 "successful update of protocol engine "
1142 			 "to %d.%d.%d\n",
1143 			 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1144 	else
1145 		dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1146 			TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1147 	if (ret)
1148 		dev_err(dev, "loading protocol SRAM failed\n");
1149 
1150 release_tpsram:
1151 	release_firmware(tpsram);
1152 
1153 	return ret;
1154 }
1155 
1156 /**
1157  * t3_synchronize_rx - wait for current Rx processing on a port to complete
1158  * @adap: the adapter
1159  * @p: the port
1160  *
1161  * Ensures that current Rx processing on any of the queues associated with
1162  * the given port completes before returning.  We do this by acquiring and
1163  * releasing the locks of the response queues associated with the port.
1164  */
1165 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
1166 {
1167 	int i;
1168 
1169 	for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1170 		struct sge_rspq *q = &adap->sge.qs[i].rspq;
1171 
1172 		spin_lock_irq(&q->lock);
1173 		spin_unlock_irq(&q->lock);
1174 	}
1175 }
1176 
1177 static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features)
1178 {
1179 	struct port_info *pi = netdev_priv(dev);
1180 	struct adapter *adapter = pi->adapter;
1181 
1182 	if (adapter->params.rev > 0) {
1183 		t3_set_vlan_accel(adapter, 1 << pi->port_id,
1184 				  features & NETIF_F_HW_VLAN_CTAG_RX);
1185 	} else {
1186 		/* single control for all ports */
1187 		unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_CTAG_RX;
1188 
1189 		for_each_port(adapter, i)
1190 			have_vlans |=
1191 				adapter->port[i]->features &
1192 				NETIF_F_HW_VLAN_CTAG_RX;
1193 
1194 		t3_set_vlan_accel(adapter, 1, have_vlans);
1195 	}
1196 	t3_synchronize_rx(adapter, pi);
1197 }
1198 
1199 /**
1200  *	cxgb_up - enable the adapter
1201  *	@adapter: adapter being enabled
1202  *
1203  *	Called when the first port is enabled, this function performs the
1204  *	actions necessary to make an adapter operational, such as completing
1205  *	the initialization of HW modules, and enabling interrupts.
1206  *
1207  *	Must be called with the rtnl lock held.
1208  */
1209 static int cxgb_up(struct adapter *adap)
1210 {
1211 	int i, err;
1212 
1213 	if (!(adap->flags & FULL_INIT_DONE)) {
1214 		err = t3_check_fw_version(adap);
1215 		if (err == -EINVAL) {
1216 			err = upgrade_fw(adap);
1217 			CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1218 				FW_VERSION_MAJOR, FW_VERSION_MINOR,
1219 				FW_VERSION_MICRO, err ? "failed" : "succeeded");
1220 		}
1221 
1222 		err = t3_check_tpsram_version(adap);
1223 		if (err == -EINVAL) {
1224 			err = update_tpsram(adap);
1225 			CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1226 				TP_VERSION_MAJOR, TP_VERSION_MINOR,
1227 				TP_VERSION_MICRO, err ? "failed" : "succeeded");
1228 		}
1229 
1230 		/*
1231 		 * Clear interrupts now to catch errors if t3_init_hw fails.
1232 		 * We clear them again later as initialization may trigger
1233 		 * conditions that can interrupt.
1234 		 */
1235 		t3_intr_clear(adap);
1236 
1237 		err = t3_init_hw(adap, 0);
1238 		if (err)
1239 			goto out;
1240 
1241 		t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1242 		t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1243 
1244 		err = setup_sge_qsets(adap);
1245 		if (err)
1246 			goto out;
1247 
1248 		for_each_port(adap, i)
1249 			cxgb_vlan_mode(adap->port[i], adap->port[i]->features);
1250 
1251 		setup_rss(adap);
1252 		if (!(adap->flags & NAPI_INIT))
1253 			init_napi(adap);
1254 
1255 		t3_start_sge_timers(adap);
1256 		adap->flags |= FULL_INIT_DONE;
1257 	}
1258 
1259 	t3_intr_clear(adap);
1260 
1261 	if (adap->flags & USING_MSIX) {
1262 		name_msix_vecs(adap);
1263 		err = request_irq(adap->msix_info[0].vec,
1264 				  t3_async_intr_handler, 0,
1265 				  adap->msix_info[0].desc, adap);
1266 		if (err)
1267 			goto irq_err;
1268 
1269 		err = request_msix_data_irqs(adap);
1270 		if (err) {
1271 			free_irq(adap->msix_info[0].vec, adap);
1272 			goto irq_err;
1273 		}
1274 	} else if ((err = request_irq(adap->pdev->irq,
1275 				      t3_intr_handler(adap,
1276 						      adap->sge.qs[0].rspq.
1277 						      polling),
1278 				      (adap->flags & USING_MSI) ?
1279 				       0 : IRQF_SHARED,
1280 				      adap->name, adap)))
1281 		goto irq_err;
1282 
1283 	enable_all_napi(adap);
1284 	t3_sge_start(adap);
1285 	t3_intr_enable(adap);
1286 
1287 	if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1288 	    is_offload(adap) && init_tp_parity(adap) == 0)
1289 		adap->flags |= TP_PARITY_INIT;
1290 
1291 	if (adap->flags & TP_PARITY_INIT) {
1292 		t3_write_reg(adap, A_TP_INT_CAUSE,
1293 			     F_CMCACHEPERR | F_ARPLUTPERR);
1294 		t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1295 	}
1296 
1297 	if (!(adap->flags & QUEUES_BOUND)) {
1298 		int ret = bind_qsets(adap);
1299 
1300 		if (ret < 0) {
1301 			CH_ERR(adap, "failed to bind qsets, err %d\n", ret);
1302 			t3_intr_disable(adap);
1303 			free_irq_resources(adap);
1304 			err = ret;
1305 			goto out;
1306 		}
1307 		adap->flags |= QUEUES_BOUND;
1308 	}
1309 
1310 out:
1311 	return err;
1312 irq_err:
1313 	CH_ERR(adap, "request_irq failed, err %d\n", err);
1314 	goto out;
1315 }
1316 
1317 /*
1318  * Release resources when all the ports and offloading have been stopped.
1319  */
1320 static void cxgb_down(struct adapter *adapter, int on_wq)
1321 {
1322 	t3_sge_stop(adapter);
1323 	spin_lock_irq(&adapter->work_lock);	/* sync with PHY intr task */
1324 	t3_intr_disable(adapter);
1325 	spin_unlock_irq(&adapter->work_lock);
1326 
1327 	free_irq_resources(adapter);
1328 	quiesce_rx(adapter);
1329 	t3_sge_stop(adapter);
1330 	if (!on_wq)
1331 		flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */
1332 }
1333 
1334 static void schedule_chk_task(struct adapter *adap)
1335 {
1336 	unsigned int timeo;
1337 
1338 	timeo = adap->params.linkpoll_period ?
1339 	    (HZ * adap->params.linkpoll_period) / 10 :
1340 	    adap->params.stats_update_period * HZ;
1341 	if (timeo)
1342 		queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1343 }
1344 
1345 static int offload_open(struct net_device *dev)
1346 {
1347 	struct port_info *pi = netdev_priv(dev);
1348 	struct adapter *adapter = pi->adapter;
1349 	struct t3cdev *tdev = dev2t3cdev(dev);
1350 	int adap_up = adapter->open_device_map & PORT_MASK;
1351 	int err;
1352 
1353 	if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1354 		return 0;
1355 
1356 	if (!adap_up && (err = cxgb_up(adapter)) < 0)
1357 		goto out;
1358 
1359 	t3_tp_set_offload_mode(adapter, 1);
1360 	tdev->lldev = adapter->port[0];
1361 	err = cxgb3_offload_activate(adapter);
1362 	if (err)
1363 		goto out;
1364 
1365 	init_port_mtus(adapter);
1366 	t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1367 		     adapter->params.b_wnd,
1368 		     adapter->params.rev == 0 ?
1369 		     adapter->port[0]->mtu : 0xffff);
1370 	init_smt(adapter);
1371 
1372 	if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1373 		dev_dbg(&dev->dev, "cannot create sysfs group\n");
1374 
1375 	/* Call back all registered clients */
1376 	cxgb3_add_clients(tdev);
1377 
1378 out:
1379 	/* restore them in case the offload module has changed them */
1380 	if (err) {
1381 		t3_tp_set_offload_mode(adapter, 0);
1382 		clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1383 		cxgb3_set_dummy_ops(tdev);
1384 	}
1385 	return err;
1386 }
1387 
1388 static int offload_close(struct t3cdev *tdev)
1389 {
1390 	struct adapter *adapter = tdev2adap(tdev);
1391 	struct t3c_data *td = T3C_DATA(tdev);
1392 
1393 	if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1394 		return 0;
1395 
1396 	/* Call back all registered clients */
1397 	cxgb3_remove_clients(tdev);
1398 
1399 	sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1400 
1401 	/* Flush work scheduled while releasing TIDs */
1402 	flush_work(&td->tid_release_task);
1403 
1404 	tdev->lldev = NULL;
1405 	cxgb3_set_dummy_ops(tdev);
1406 	t3_tp_set_offload_mode(adapter, 0);
1407 	clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1408 
1409 	if (!adapter->open_device_map)
1410 		cxgb_down(adapter, 0);
1411 
1412 	cxgb3_offload_deactivate(adapter);
1413 	return 0;
1414 }
1415 
1416 static int cxgb_open(struct net_device *dev)
1417 {
1418 	struct port_info *pi = netdev_priv(dev);
1419 	struct adapter *adapter = pi->adapter;
1420 	int other_ports = adapter->open_device_map & PORT_MASK;
1421 	int err;
1422 
1423 	if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1424 		return err;
1425 
1426 	set_bit(pi->port_id, &adapter->open_device_map);
1427 	if (is_offload(adapter) && !ofld_disable) {
1428 		err = offload_open(dev);
1429 		if (err)
1430 			pr_warn("Could not initialize offload capabilities\n");
1431 	}
1432 
1433 	netif_set_real_num_tx_queues(dev, pi->nqsets);
1434 	err = netif_set_real_num_rx_queues(dev, pi->nqsets);
1435 	if (err)
1436 		return err;
1437 	link_start(dev);
1438 	t3_port_intr_enable(adapter, pi->port_id);
1439 	netif_tx_start_all_queues(dev);
1440 	if (!other_ports)
1441 		schedule_chk_task(adapter);
1442 
1443 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1444 	return 0;
1445 }
1446 
1447 static int __cxgb_close(struct net_device *dev, int on_wq)
1448 {
1449 	struct port_info *pi = netdev_priv(dev);
1450 	struct adapter *adapter = pi->adapter;
1451 
1452 
1453 	if (!adapter->open_device_map)
1454 		return 0;
1455 
1456 	/* Stop link fault interrupts */
1457 	t3_xgm_intr_disable(adapter, pi->port_id);
1458 	t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1459 
1460 	t3_port_intr_disable(adapter, pi->port_id);
1461 	netif_tx_stop_all_queues(dev);
1462 	pi->phy.ops->power_down(&pi->phy, 1);
1463 	netif_carrier_off(dev);
1464 	t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1465 
1466 	spin_lock_irq(&adapter->work_lock);	/* sync with update task */
1467 	clear_bit(pi->port_id, &adapter->open_device_map);
1468 	spin_unlock_irq(&adapter->work_lock);
1469 
1470 	if (!(adapter->open_device_map & PORT_MASK))
1471 		cancel_delayed_work_sync(&adapter->adap_check_task);
1472 
1473 	if (!adapter->open_device_map)
1474 		cxgb_down(adapter, on_wq);
1475 
1476 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1477 	return 0;
1478 }
1479 
1480 static int cxgb_close(struct net_device *dev)
1481 {
1482 	return __cxgb_close(dev, 0);
1483 }
1484 
1485 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1486 {
1487 	struct port_info *pi = netdev_priv(dev);
1488 	struct adapter *adapter = pi->adapter;
1489 	struct net_device_stats *ns = &pi->netstats;
1490 	const struct mac_stats *pstats;
1491 
1492 	spin_lock(&adapter->stats_lock);
1493 	pstats = t3_mac_update_stats(&pi->mac);
1494 	spin_unlock(&adapter->stats_lock);
1495 
1496 	ns->tx_bytes = pstats->tx_octets;
1497 	ns->tx_packets = pstats->tx_frames;
1498 	ns->rx_bytes = pstats->rx_octets;
1499 	ns->rx_packets = pstats->rx_frames;
1500 	ns->multicast = pstats->rx_mcast_frames;
1501 
1502 	ns->tx_errors = pstats->tx_underrun;
1503 	ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1504 	    pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1505 	    pstats->rx_fifo_ovfl;
1506 
1507 	/* detailed rx_errors */
1508 	ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1509 	ns->rx_over_errors = 0;
1510 	ns->rx_crc_errors = pstats->rx_fcs_errs;
1511 	ns->rx_frame_errors = pstats->rx_symbol_errs;
1512 	ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1513 	ns->rx_missed_errors = pstats->rx_cong_drops;
1514 
1515 	/* detailed tx_errors */
1516 	ns->tx_aborted_errors = 0;
1517 	ns->tx_carrier_errors = 0;
1518 	ns->tx_fifo_errors = pstats->tx_underrun;
1519 	ns->tx_heartbeat_errors = 0;
1520 	ns->tx_window_errors = 0;
1521 	return ns;
1522 }
1523 
1524 static u32 get_msglevel(struct net_device *dev)
1525 {
1526 	struct port_info *pi = netdev_priv(dev);
1527 	struct adapter *adapter = pi->adapter;
1528 
1529 	return adapter->msg_enable;
1530 }
1531 
1532 static void set_msglevel(struct net_device *dev, u32 val)
1533 {
1534 	struct port_info *pi = netdev_priv(dev);
1535 	struct adapter *adapter = pi->adapter;
1536 
1537 	adapter->msg_enable = val;
1538 }
1539 
1540 static char stats_strings[][ETH_GSTRING_LEN] = {
1541 	"TxOctetsOK         ",
1542 	"TxFramesOK         ",
1543 	"TxMulticastFramesOK",
1544 	"TxBroadcastFramesOK",
1545 	"TxPauseFrames      ",
1546 	"TxUnderrun         ",
1547 	"TxExtUnderrun      ",
1548 
1549 	"TxFrames64         ",
1550 	"TxFrames65To127    ",
1551 	"TxFrames128To255   ",
1552 	"TxFrames256To511   ",
1553 	"TxFrames512To1023  ",
1554 	"TxFrames1024To1518 ",
1555 	"TxFrames1519ToMax  ",
1556 
1557 	"RxOctetsOK         ",
1558 	"RxFramesOK         ",
1559 	"RxMulticastFramesOK",
1560 	"RxBroadcastFramesOK",
1561 	"RxPauseFrames      ",
1562 	"RxFCSErrors        ",
1563 	"RxSymbolErrors     ",
1564 	"RxShortErrors      ",
1565 	"RxJabberErrors     ",
1566 	"RxLengthErrors     ",
1567 	"RxFIFOoverflow     ",
1568 
1569 	"RxFrames64         ",
1570 	"RxFrames65To127    ",
1571 	"RxFrames128To255   ",
1572 	"RxFrames256To511   ",
1573 	"RxFrames512To1023  ",
1574 	"RxFrames1024To1518 ",
1575 	"RxFrames1519ToMax  ",
1576 
1577 	"PhyFIFOErrors      ",
1578 	"TSO                ",
1579 	"VLANextractions    ",
1580 	"VLANinsertions     ",
1581 	"TxCsumOffload      ",
1582 	"RxCsumGood         ",
1583 	"LroAggregated      ",
1584 	"LroFlushed         ",
1585 	"LroNoDesc          ",
1586 	"RxDrops            ",
1587 
1588 	"CheckTXEnToggled   ",
1589 	"CheckResets        ",
1590 
1591 	"LinkFaults         ",
1592 };
1593 
1594 static int get_sset_count(struct net_device *dev, int sset)
1595 {
1596 	switch (sset) {
1597 	case ETH_SS_STATS:
1598 		return ARRAY_SIZE(stats_strings);
1599 	default:
1600 		return -EOPNOTSUPP;
1601 	}
1602 }
1603 
1604 #define T3_REGMAP_SIZE (3 * 1024)
1605 
1606 static int get_regs_len(struct net_device *dev)
1607 {
1608 	return T3_REGMAP_SIZE;
1609 }
1610 
1611 static int get_eeprom_len(struct net_device *dev)
1612 {
1613 	return EEPROMSIZE;
1614 }
1615 
1616 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1617 {
1618 	struct port_info *pi = netdev_priv(dev);
1619 	struct adapter *adapter = pi->adapter;
1620 	u32 fw_vers = 0;
1621 	u32 tp_vers = 0;
1622 
1623 	spin_lock(&adapter->stats_lock);
1624 	t3_get_fw_version(adapter, &fw_vers);
1625 	t3_get_tp_version(adapter, &tp_vers);
1626 	spin_unlock(&adapter->stats_lock);
1627 
1628 	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
1629 	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1630 	strlcpy(info->bus_info, pci_name(adapter->pdev),
1631 		sizeof(info->bus_info));
1632 	if (fw_vers)
1633 		snprintf(info->fw_version, sizeof(info->fw_version),
1634 			 "%s %u.%u.%u TP %u.%u.%u",
1635 			 G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1636 			 G_FW_VERSION_MAJOR(fw_vers),
1637 			 G_FW_VERSION_MINOR(fw_vers),
1638 			 G_FW_VERSION_MICRO(fw_vers),
1639 			 G_TP_VERSION_MAJOR(tp_vers),
1640 			 G_TP_VERSION_MINOR(tp_vers),
1641 			 G_TP_VERSION_MICRO(tp_vers));
1642 }
1643 
1644 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1645 {
1646 	if (stringset == ETH_SS_STATS)
1647 		memcpy(data, stats_strings, sizeof(stats_strings));
1648 }
1649 
1650 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1651 					    struct port_info *p, int idx)
1652 {
1653 	int i;
1654 	unsigned long tot = 0;
1655 
1656 	for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1657 		tot += adapter->sge.qs[i].port_stats[idx];
1658 	return tot;
1659 }
1660 
1661 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1662 		      u64 *data)
1663 {
1664 	struct port_info *pi = netdev_priv(dev);
1665 	struct adapter *adapter = pi->adapter;
1666 	const struct mac_stats *s;
1667 
1668 	spin_lock(&adapter->stats_lock);
1669 	s = t3_mac_update_stats(&pi->mac);
1670 	spin_unlock(&adapter->stats_lock);
1671 
1672 	*data++ = s->tx_octets;
1673 	*data++ = s->tx_frames;
1674 	*data++ = s->tx_mcast_frames;
1675 	*data++ = s->tx_bcast_frames;
1676 	*data++ = s->tx_pause;
1677 	*data++ = s->tx_underrun;
1678 	*data++ = s->tx_fifo_urun;
1679 
1680 	*data++ = s->tx_frames_64;
1681 	*data++ = s->tx_frames_65_127;
1682 	*data++ = s->tx_frames_128_255;
1683 	*data++ = s->tx_frames_256_511;
1684 	*data++ = s->tx_frames_512_1023;
1685 	*data++ = s->tx_frames_1024_1518;
1686 	*data++ = s->tx_frames_1519_max;
1687 
1688 	*data++ = s->rx_octets;
1689 	*data++ = s->rx_frames;
1690 	*data++ = s->rx_mcast_frames;
1691 	*data++ = s->rx_bcast_frames;
1692 	*data++ = s->rx_pause;
1693 	*data++ = s->rx_fcs_errs;
1694 	*data++ = s->rx_symbol_errs;
1695 	*data++ = s->rx_short;
1696 	*data++ = s->rx_jabber;
1697 	*data++ = s->rx_too_long;
1698 	*data++ = s->rx_fifo_ovfl;
1699 
1700 	*data++ = s->rx_frames_64;
1701 	*data++ = s->rx_frames_65_127;
1702 	*data++ = s->rx_frames_128_255;
1703 	*data++ = s->rx_frames_256_511;
1704 	*data++ = s->rx_frames_512_1023;
1705 	*data++ = s->rx_frames_1024_1518;
1706 	*data++ = s->rx_frames_1519_max;
1707 
1708 	*data++ = pi->phy.fifo_errors;
1709 
1710 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1711 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1712 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1713 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1714 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1715 	*data++ = 0;
1716 	*data++ = 0;
1717 	*data++ = 0;
1718 	*data++ = s->rx_cong_drops;
1719 
1720 	*data++ = s->num_toggled;
1721 	*data++ = s->num_resets;
1722 
1723 	*data++ = s->link_faults;
1724 }
1725 
1726 static inline void reg_block_dump(struct adapter *ap, void *buf,
1727 				  unsigned int start, unsigned int end)
1728 {
1729 	u32 *p = buf + start;
1730 
1731 	for (; start <= end; start += sizeof(u32))
1732 		*p++ = t3_read_reg(ap, start);
1733 }
1734 
1735 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1736 		     void *buf)
1737 {
1738 	struct port_info *pi = netdev_priv(dev);
1739 	struct adapter *ap = pi->adapter;
1740 
1741 	/*
1742 	 * Version scheme:
1743 	 * bits 0..9: chip version
1744 	 * bits 10..15: chip revision
1745 	 * bit 31: set for PCIe cards
1746 	 */
1747 	regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1748 
1749 	/*
1750 	 * We skip the MAC statistics registers because they are clear-on-read.
1751 	 * Also reading multi-register stats would need to synchronize with the
1752 	 * periodic mac stats accumulation.  Hard to justify the complexity.
1753 	 */
1754 	memset(buf, 0, T3_REGMAP_SIZE);
1755 	reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1756 	reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1757 	reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1758 	reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1759 	reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1760 	reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1761 		       XGM_REG(A_XGM_SERDES_STAT3, 1));
1762 	reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1763 		       XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1764 }
1765 
1766 static int restart_autoneg(struct net_device *dev)
1767 {
1768 	struct port_info *p = netdev_priv(dev);
1769 
1770 	if (!netif_running(dev))
1771 		return -EAGAIN;
1772 	if (p->link_config.autoneg != AUTONEG_ENABLE)
1773 		return -EINVAL;
1774 	p->phy.ops->autoneg_restart(&p->phy);
1775 	return 0;
1776 }
1777 
1778 static int set_phys_id(struct net_device *dev,
1779 		       enum ethtool_phys_id_state state)
1780 {
1781 	struct port_info *pi = netdev_priv(dev);
1782 	struct adapter *adapter = pi->adapter;
1783 
1784 	switch (state) {
1785 	case ETHTOOL_ID_ACTIVE:
1786 		return 1;	/* cycle on/off once per second */
1787 
1788 	case ETHTOOL_ID_OFF:
1789 		t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
1790 		break;
1791 
1792 	case ETHTOOL_ID_ON:
1793 	case ETHTOOL_ID_INACTIVE:
1794 		t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1795 			 F_GPIO0_OUT_VAL);
1796 	}
1797 
1798 	return 0;
1799 }
1800 
1801 static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1802 {
1803 	struct port_info *p = netdev_priv(dev);
1804 
1805 	cmd->supported = p->link_config.supported;
1806 	cmd->advertising = p->link_config.advertising;
1807 
1808 	if (netif_carrier_ok(dev)) {
1809 		ethtool_cmd_speed_set(cmd, p->link_config.speed);
1810 		cmd->duplex = p->link_config.duplex;
1811 	} else {
1812 		ethtool_cmd_speed_set(cmd, SPEED_UNKNOWN);
1813 		cmd->duplex = DUPLEX_UNKNOWN;
1814 	}
1815 
1816 	cmd->port = (cmd->supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1817 	cmd->phy_address = p->phy.mdio.prtad;
1818 	cmd->transceiver = XCVR_EXTERNAL;
1819 	cmd->autoneg = p->link_config.autoneg;
1820 	cmd->maxtxpkt = 0;
1821 	cmd->maxrxpkt = 0;
1822 	return 0;
1823 }
1824 
1825 static int speed_duplex_to_caps(int speed, int duplex)
1826 {
1827 	int cap = 0;
1828 
1829 	switch (speed) {
1830 	case SPEED_10:
1831 		if (duplex == DUPLEX_FULL)
1832 			cap = SUPPORTED_10baseT_Full;
1833 		else
1834 			cap = SUPPORTED_10baseT_Half;
1835 		break;
1836 	case SPEED_100:
1837 		if (duplex == DUPLEX_FULL)
1838 			cap = SUPPORTED_100baseT_Full;
1839 		else
1840 			cap = SUPPORTED_100baseT_Half;
1841 		break;
1842 	case SPEED_1000:
1843 		if (duplex == DUPLEX_FULL)
1844 			cap = SUPPORTED_1000baseT_Full;
1845 		else
1846 			cap = SUPPORTED_1000baseT_Half;
1847 		break;
1848 	case SPEED_10000:
1849 		if (duplex == DUPLEX_FULL)
1850 			cap = SUPPORTED_10000baseT_Full;
1851 	}
1852 	return cap;
1853 }
1854 
1855 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1856 		      ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1857 		      ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1858 		      ADVERTISED_10000baseT_Full)
1859 
1860 static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1861 {
1862 	struct port_info *p = netdev_priv(dev);
1863 	struct link_config *lc = &p->link_config;
1864 
1865 	if (!(lc->supported & SUPPORTED_Autoneg)) {
1866 		/*
1867 		 * PHY offers a single speed/duplex.  See if that's what's
1868 		 * being requested.
1869 		 */
1870 		if (cmd->autoneg == AUTONEG_DISABLE) {
1871 			u32 speed = ethtool_cmd_speed(cmd);
1872 			int cap = speed_duplex_to_caps(speed, cmd->duplex);
1873 			if (lc->supported & cap)
1874 				return 0;
1875 		}
1876 		return -EINVAL;
1877 	}
1878 
1879 	if (cmd->autoneg == AUTONEG_DISABLE) {
1880 		u32 speed = ethtool_cmd_speed(cmd);
1881 		int cap = speed_duplex_to_caps(speed, cmd->duplex);
1882 
1883 		if (!(lc->supported & cap) || (speed == SPEED_1000))
1884 			return -EINVAL;
1885 		lc->requested_speed = speed;
1886 		lc->requested_duplex = cmd->duplex;
1887 		lc->advertising = 0;
1888 	} else {
1889 		cmd->advertising &= ADVERTISED_MASK;
1890 		cmd->advertising &= lc->supported;
1891 		if (!cmd->advertising)
1892 			return -EINVAL;
1893 		lc->requested_speed = SPEED_INVALID;
1894 		lc->requested_duplex = DUPLEX_INVALID;
1895 		lc->advertising = cmd->advertising | ADVERTISED_Autoneg;
1896 	}
1897 	lc->autoneg = cmd->autoneg;
1898 	if (netif_running(dev))
1899 		t3_link_start(&p->phy, &p->mac, lc);
1900 	return 0;
1901 }
1902 
1903 static void get_pauseparam(struct net_device *dev,
1904 			   struct ethtool_pauseparam *epause)
1905 {
1906 	struct port_info *p = netdev_priv(dev);
1907 
1908 	epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1909 	epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1910 	epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1911 }
1912 
1913 static int set_pauseparam(struct net_device *dev,
1914 			  struct ethtool_pauseparam *epause)
1915 {
1916 	struct port_info *p = netdev_priv(dev);
1917 	struct link_config *lc = &p->link_config;
1918 
1919 	if (epause->autoneg == AUTONEG_DISABLE)
1920 		lc->requested_fc = 0;
1921 	else if (lc->supported & SUPPORTED_Autoneg)
1922 		lc->requested_fc = PAUSE_AUTONEG;
1923 	else
1924 		return -EINVAL;
1925 
1926 	if (epause->rx_pause)
1927 		lc->requested_fc |= PAUSE_RX;
1928 	if (epause->tx_pause)
1929 		lc->requested_fc |= PAUSE_TX;
1930 	if (lc->autoneg == AUTONEG_ENABLE) {
1931 		if (netif_running(dev))
1932 			t3_link_start(&p->phy, &p->mac, lc);
1933 	} else {
1934 		lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1935 		if (netif_running(dev))
1936 			t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1937 	}
1938 	return 0;
1939 }
1940 
1941 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1942 {
1943 	struct port_info *pi = netdev_priv(dev);
1944 	struct adapter *adapter = pi->adapter;
1945 	const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1946 
1947 	e->rx_max_pending = MAX_RX_BUFFERS;
1948 	e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1949 	e->tx_max_pending = MAX_TXQ_ENTRIES;
1950 
1951 	e->rx_pending = q->fl_size;
1952 	e->rx_mini_pending = q->rspq_size;
1953 	e->rx_jumbo_pending = q->jumbo_size;
1954 	e->tx_pending = q->txq_size[0];
1955 }
1956 
1957 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1958 {
1959 	struct port_info *pi = netdev_priv(dev);
1960 	struct adapter *adapter = pi->adapter;
1961 	struct qset_params *q;
1962 	int i;
1963 
1964 	if (e->rx_pending > MAX_RX_BUFFERS ||
1965 	    e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1966 	    e->tx_pending > MAX_TXQ_ENTRIES ||
1967 	    e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1968 	    e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1969 	    e->rx_pending < MIN_FL_ENTRIES ||
1970 	    e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1971 	    e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1972 		return -EINVAL;
1973 
1974 	if (adapter->flags & FULL_INIT_DONE)
1975 		return -EBUSY;
1976 
1977 	q = &adapter->params.sge.qset[pi->first_qset];
1978 	for (i = 0; i < pi->nqsets; ++i, ++q) {
1979 		q->rspq_size = e->rx_mini_pending;
1980 		q->fl_size = e->rx_pending;
1981 		q->jumbo_size = e->rx_jumbo_pending;
1982 		q->txq_size[0] = e->tx_pending;
1983 		q->txq_size[1] = e->tx_pending;
1984 		q->txq_size[2] = e->tx_pending;
1985 	}
1986 	return 0;
1987 }
1988 
1989 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1990 {
1991 	struct port_info *pi = netdev_priv(dev);
1992 	struct adapter *adapter = pi->adapter;
1993 	struct qset_params *qsp;
1994 	struct sge_qset *qs;
1995 	int i;
1996 
1997 	if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
1998 		return -EINVAL;
1999 
2000 	for (i = 0; i < pi->nqsets; i++) {
2001 		qsp = &adapter->params.sge.qset[i];
2002 		qs = &adapter->sge.qs[i];
2003 		qsp->coalesce_usecs = c->rx_coalesce_usecs;
2004 		t3_update_qset_coalesce(qs, qsp);
2005 	}
2006 
2007 	return 0;
2008 }
2009 
2010 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2011 {
2012 	struct port_info *pi = netdev_priv(dev);
2013 	struct adapter *adapter = pi->adapter;
2014 	struct qset_params *q = adapter->params.sge.qset;
2015 
2016 	c->rx_coalesce_usecs = q->coalesce_usecs;
2017 	return 0;
2018 }
2019 
2020 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2021 		      u8 * data)
2022 {
2023 	struct port_info *pi = netdev_priv(dev);
2024 	struct adapter *adapter = pi->adapter;
2025 	int i, err = 0;
2026 
2027 	u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2028 	if (!buf)
2029 		return -ENOMEM;
2030 
2031 	e->magic = EEPROM_MAGIC;
2032 	for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2033 		err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
2034 
2035 	if (!err)
2036 		memcpy(data, buf + e->offset, e->len);
2037 	kfree(buf);
2038 	return err;
2039 }
2040 
2041 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2042 		      u8 * data)
2043 {
2044 	struct port_info *pi = netdev_priv(dev);
2045 	struct adapter *adapter = pi->adapter;
2046 	u32 aligned_offset, aligned_len;
2047 	__le32 *p;
2048 	u8 *buf;
2049 	int err;
2050 
2051 	if (eeprom->magic != EEPROM_MAGIC)
2052 		return -EINVAL;
2053 
2054 	aligned_offset = eeprom->offset & ~3;
2055 	aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2056 
2057 	if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2058 		buf = kmalloc(aligned_len, GFP_KERNEL);
2059 		if (!buf)
2060 			return -ENOMEM;
2061 		err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2062 		if (!err && aligned_len > 4)
2063 			err = t3_seeprom_read(adapter,
2064 					      aligned_offset + aligned_len - 4,
2065 					      (__le32 *) & buf[aligned_len - 4]);
2066 		if (err)
2067 			goto out;
2068 		memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2069 	} else
2070 		buf = data;
2071 
2072 	err = t3_seeprom_wp(adapter, 0);
2073 	if (err)
2074 		goto out;
2075 
2076 	for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2077 		err = t3_seeprom_write(adapter, aligned_offset, *p);
2078 		aligned_offset += 4;
2079 	}
2080 
2081 	if (!err)
2082 		err = t3_seeprom_wp(adapter, 1);
2083 out:
2084 	if (buf != data)
2085 		kfree(buf);
2086 	return err;
2087 }
2088 
2089 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2090 {
2091 	wol->supported = 0;
2092 	wol->wolopts = 0;
2093 	memset(&wol->sopass, 0, sizeof(wol->sopass));
2094 }
2095 
2096 static const struct ethtool_ops cxgb_ethtool_ops = {
2097 	.get_settings = get_settings,
2098 	.set_settings = set_settings,
2099 	.get_drvinfo = get_drvinfo,
2100 	.get_msglevel = get_msglevel,
2101 	.set_msglevel = set_msglevel,
2102 	.get_ringparam = get_sge_param,
2103 	.set_ringparam = set_sge_param,
2104 	.get_coalesce = get_coalesce,
2105 	.set_coalesce = set_coalesce,
2106 	.get_eeprom_len = get_eeprom_len,
2107 	.get_eeprom = get_eeprom,
2108 	.set_eeprom = set_eeprom,
2109 	.get_pauseparam = get_pauseparam,
2110 	.set_pauseparam = set_pauseparam,
2111 	.get_link = ethtool_op_get_link,
2112 	.get_strings = get_strings,
2113 	.set_phys_id = set_phys_id,
2114 	.nway_reset = restart_autoneg,
2115 	.get_sset_count = get_sset_count,
2116 	.get_ethtool_stats = get_stats,
2117 	.get_regs_len = get_regs_len,
2118 	.get_regs = get_regs,
2119 	.get_wol = get_wol,
2120 };
2121 
2122 static int in_range(int val, int lo, int hi)
2123 {
2124 	return val < 0 || (val <= hi && val >= lo);
2125 }
2126 
2127 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2128 {
2129 	struct port_info *pi = netdev_priv(dev);
2130 	struct adapter *adapter = pi->adapter;
2131 	u32 cmd;
2132 	int ret;
2133 
2134 	if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2135 		return -EFAULT;
2136 
2137 	switch (cmd) {
2138 	case CHELSIO_SET_QSET_PARAMS:{
2139 		int i;
2140 		struct qset_params *q;
2141 		struct ch_qset_params t;
2142 		int q1 = pi->first_qset;
2143 		int nqsets = pi->nqsets;
2144 
2145 		if (!capable(CAP_NET_ADMIN))
2146 			return -EPERM;
2147 		if (copy_from_user(&t, useraddr, sizeof(t)))
2148 			return -EFAULT;
2149 		if (t.qset_idx >= SGE_QSETS)
2150 			return -EINVAL;
2151 		if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2152 		    !in_range(t.cong_thres, 0, 255) ||
2153 		    !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2154 			      MAX_TXQ_ENTRIES) ||
2155 		    !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2156 			      MAX_TXQ_ENTRIES) ||
2157 		    !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2158 			      MAX_CTRL_TXQ_ENTRIES) ||
2159 		    !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2160 			      MAX_RX_BUFFERS) ||
2161 		    !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2162 			      MAX_RX_JUMBO_BUFFERS) ||
2163 		    !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2164 			      MAX_RSPQ_ENTRIES))
2165 			return -EINVAL;
2166 
2167 		if ((adapter->flags & FULL_INIT_DONE) &&
2168 			(t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2169 			t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2170 			t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2171 			t.polling >= 0 || t.cong_thres >= 0))
2172 			return -EBUSY;
2173 
2174 		/* Allow setting of any available qset when offload enabled */
2175 		if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2176 			q1 = 0;
2177 			for_each_port(adapter, i) {
2178 				pi = adap2pinfo(adapter, i);
2179 				nqsets += pi->first_qset + pi->nqsets;
2180 			}
2181 		}
2182 
2183 		if (t.qset_idx < q1)
2184 			return -EINVAL;
2185 		if (t.qset_idx > q1 + nqsets - 1)
2186 			return -EINVAL;
2187 
2188 		q = &adapter->params.sge.qset[t.qset_idx];
2189 
2190 		if (t.rspq_size >= 0)
2191 			q->rspq_size = t.rspq_size;
2192 		if (t.fl_size[0] >= 0)
2193 			q->fl_size = t.fl_size[0];
2194 		if (t.fl_size[1] >= 0)
2195 			q->jumbo_size = t.fl_size[1];
2196 		if (t.txq_size[0] >= 0)
2197 			q->txq_size[0] = t.txq_size[0];
2198 		if (t.txq_size[1] >= 0)
2199 			q->txq_size[1] = t.txq_size[1];
2200 		if (t.txq_size[2] >= 0)
2201 			q->txq_size[2] = t.txq_size[2];
2202 		if (t.cong_thres >= 0)
2203 			q->cong_thres = t.cong_thres;
2204 		if (t.intr_lat >= 0) {
2205 			struct sge_qset *qs =
2206 				&adapter->sge.qs[t.qset_idx];
2207 
2208 			q->coalesce_usecs = t.intr_lat;
2209 			t3_update_qset_coalesce(qs, q);
2210 		}
2211 		if (t.polling >= 0) {
2212 			if (adapter->flags & USING_MSIX)
2213 				q->polling = t.polling;
2214 			else {
2215 				/* No polling with INTx for T3A */
2216 				if (adapter->params.rev == 0 &&
2217 					!(adapter->flags & USING_MSI))
2218 					t.polling = 0;
2219 
2220 				for (i = 0; i < SGE_QSETS; i++) {
2221 					q = &adapter->params.sge.
2222 						qset[i];
2223 					q->polling = t.polling;
2224 				}
2225 			}
2226 		}
2227 
2228 		if (t.lro >= 0) {
2229 			if (t.lro)
2230 				dev->wanted_features |= NETIF_F_GRO;
2231 			else
2232 				dev->wanted_features &= ~NETIF_F_GRO;
2233 			netdev_update_features(dev);
2234 		}
2235 
2236 		break;
2237 	}
2238 	case CHELSIO_GET_QSET_PARAMS:{
2239 		struct qset_params *q;
2240 		struct ch_qset_params t;
2241 		int q1 = pi->first_qset;
2242 		int nqsets = pi->nqsets;
2243 		int i;
2244 
2245 		if (copy_from_user(&t, useraddr, sizeof(t)))
2246 			return -EFAULT;
2247 
2248 		/* Display qsets for all ports when offload enabled */
2249 		if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2250 			q1 = 0;
2251 			for_each_port(adapter, i) {
2252 				pi = adap2pinfo(adapter, i);
2253 				nqsets = pi->first_qset + pi->nqsets;
2254 			}
2255 		}
2256 
2257 		if (t.qset_idx >= nqsets)
2258 			return -EINVAL;
2259 
2260 		q = &adapter->params.sge.qset[q1 + t.qset_idx];
2261 		t.rspq_size = q->rspq_size;
2262 		t.txq_size[0] = q->txq_size[0];
2263 		t.txq_size[1] = q->txq_size[1];
2264 		t.txq_size[2] = q->txq_size[2];
2265 		t.fl_size[0] = q->fl_size;
2266 		t.fl_size[1] = q->jumbo_size;
2267 		t.polling = q->polling;
2268 		t.lro = !!(dev->features & NETIF_F_GRO);
2269 		t.intr_lat = q->coalesce_usecs;
2270 		t.cong_thres = q->cong_thres;
2271 		t.qnum = q1;
2272 
2273 		if (adapter->flags & USING_MSIX)
2274 			t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2275 		else
2276 			t.vector = adapter->pdev->irq;
2277 
2278 		if (copy_to_user(useraddr, &t, sizeof(t)))
2279 			return -EFAULT;
2280 		break;
2281 	}
2282 	case CHELSIO_SET_QSET_NUM:{
2283 		struct ch_reg edata;
2284 		unsigned int i, first_qset = 0, other_qsets = 0;
2285 
2286 		if (!capable(CAP_NET_ADMIN))
2287 			return -EPERM;
2288 		if (adapter->flags & FULL_INIT_DONE)
2289 			return -EBUSY;
2290 		if (copy_from_user(&edata, useraddr, sizeof(edata)))
2291 			return -EFAULT;
2292 		if (edata.val < 1 ||
2293 			(edata.val > 1 && !(adapter->flags & USING_MSIX)))
2294 			return -EINVAL;
2295 
2296 		for_each_port(adapter, i)
2297 			if (adapter->port[i] && adapter->port[i] != dev)
2298 				other_qsets += adap2pinfo(adapter, i)->nqsets;
2299 
2300 		if (edata.val + other_qsets > SGE_QSETS)
2301 			return -EINVAL;
2302 
2303 		pi->nqsets = edata.val;
2304 
2305 		for_each_port(adapter, i)
2306 			if (adapter->port[i]) {
2307 				pi = adap2pinfo(adapter, i);
2308 				pi->first_qset = first_qset;
2309 				first_qset += pi->nqsets;
2310 			}
2311 		break;
2312 	}
2313 	case CHELSIO_GET_QSET_NUM:{
2314 		struct ch_reg edata;
2315 
2316 		memset(&edata, 0, sizeof(struct ch_reg));
2317 
2318 		edata.cmd = CHELSIO_GET_QSET_NUM;
2319 		edata.val = pi->nqsets;
2320 		if (copy_to_user(useraddr, &edata, sizeof(edata)))
2321 			return -EFAULT;
2322 		break;
2323 	}
2324 	case CHELSIO_LOAD_FW:{
2325 		u8 *fw_data;
2326 		struct ch_mem_range t;
2327 
2328 		if (!capable(CAP_SYS_RAWIO))
2329 			return -EPERM;
2330 		if (copy_from_user(&t, useraddr, sizeof(t)))
2331 			return -EFAULT;
2332 		/* Check t.len sanity ? */
2333 		fw_data = memdup_user(useraddr + sizeof(t), t.len);
2334 		if (IS_ERR(fw_data))
2335 			return PTR_ERR(fw_data);
2336 
2337 		ret = t3_load_fw(adapter, fw_data, t.len);
2338 		kfree(fw_data);
2339 		if (ret)
2340 			return ret;
2341 		break;
2342 	}
2343 	case CHELSIO_SETMTUTAB:{
2344 		struct ch_mtus m;
2345 		int i;
2346 
2347 		if (!is_offload(adapter))
2348 			return -EOPNOTSUPP;
2349 		if (!capable(CAP_NET_ADMIN))
2350 			return -EPERM;
2351 		if (offload_running(adapter))
2352 			return -EBUSY;
2353 		if (copy_from_user(&m, useraddr, sizeof(m)))
2354 			return -EFAULT;
2355 		if (m.nmtus != NMTUS)
2356 			return -EINVAL;
2357 		if (m.mtus[0] < 81)	/* accommodate SACK */
2358 			return -EINVAL;
2359 
2360 		/* MTUs must be in ascending order */
2361 		for (i = 1; i < NMTUS; ++i)
2362 			if (m.mtus[i] < m.mtus[i - 1])
2363 				return -EINVAL;
2364 
2365 		memcpy(adapter->params.mtus, m.mtus,
2366 			sizeof(adapter->params.mtus));
2367 		break;
2368 	}
2369 	case CHELSIO_GET_PM:{
2370 		struct tp_params *p = &adapter->params.tp;
2371 		struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2372 
2373 		if (!is_offload(adapter))
2374 			return -EOPNOTSUPP;
2375 		m.tx_pg_sz = p->tx_pg_size;
2376 		m.tx_num_pg = p->tx_num_pgs;
2377 		m.rx_pg_sz = p->rx_pg_size;
2378 		m.rx_num_pg = p->rx_num_pgs;
2379 		m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2380 		if (copy_to_user(useraddr, &m, sizeof(m)))
2381 			return -EFAULT;
2382 		break;
2383 	}
2384 	case CHELSIO_SET_PM:{
2385 		struct ch_pm m;
2386 		struct tp_params *p = &adapter->params.tp;
2387 
2388 		if (!is_offload(adapter))
2389 			return -EOPNOTSUPP;
2390 		if (!capable(CAP_NET_ADMIN))
2391 			return -EPERM;
2392 		if (adapter->flags & FULL_INIT_DONE)
2393 			return -EBUSY;
2394 		if (copy_from_user(&m, useraddr, sizeof(m)))
2395 			return -EFAULT;
2396 		if (!is_power_of_2(m.rx_pg_sz) ||
2397 			!is_power_of_2(m.tx_pg_sz))
2398 			return -EINVAL;	/* not power of 2 */
2399 		if (!(m.rx_pg_sz & 0x14000))
2400 			return -EINVAL;	/* not 16KB or 64KB */
2401 		if (!(m.tx_pg_sz & 0x1554000))
2402 			return -EINVAL;
2403 		if (m.tx_num_pg == -1)
2404 			m.tx_num_pg = p->tx_num_pgs;
2405 		if (m.rx_num_pg == -1)
2406 			m.rx_num_pg = p->rx_num_pgs;
2407 		if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2408 			return -EINVAL;
2409 		if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2410 			m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2411 			return -EINVAL;
2412 		p->rx_pg_size = m.rx_pg_sz;
2413 		p->tx_pg_size = m.tx_pg_sz;
2414 		p->rx_num_pgs = m.rx_num_pg;
2415 		p->tx_num_pgs = m.tx_num_pg;
2416 		break;
2417 	}
2418 	case CHELSIO_GET_MEM:{
2419 		struct ch_mem_range t;
2420 		struct mc7 *mem;
2421 		u64 buf[32];
2422 
2423 		if (!is_offload(adapter))
2424 			return -EOPNOTSUPP;
2425 		if (!(adapter->flags & FULL_INIT_DONE))
2426 			return -EIO;	/* need the memory controllers */
2427 		if (copy_from_user(&t, useraddr, sizeof(t)))
2428 			return -EFAULT;
2429 		if ((t.addr & 7) || (t.len & 7))
2430 			return -EINVAL;
2431 		if (t.mem_id == MEM_CM)
2432 			mem = &adapter->cm;
2433 		else if (t.mem_id == MEM_PMRX)
2434 			mem = &adapter->pmrx;
2435 		else if (t.mem_id == MEM_PMTX)
2436 			mem = &adapter->pmtx;
2437 		else
2438 			return -EINVAL;
2439 
2440 		/*
2441 		 * Version scheme:
2442 		 * bits 0..9: chip version
2443 		 * bits 10..15: chip revision
2444 		 */
2445 		t.version = 3 | (adapter->params.rev << 10);
2446 		if (copy_to_user(useraddr, &t, sizeof(t)))
2447 			return -EFAULT;
2448 
2449 		/*
2450 		 * Read 256 bytes at a time as len can be large and we don't
2451 		 * want to use huge intermediate buffers.
2452 		 */
2453 		useraddr += sizeof(t);	/* advance to start of buffer */
2454 		while (t.len) {
2455 			unsigned int chunk =
2456 				min_t(unsigned int, t.len, sizeof(buf));
2457 
2458 			ret =
2459 				t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2460 						buf);
2461 			if (ret)
2462 				return ret;
2463 			if (copy_to_user(useraddr, buf, chunk))
2464 				return -EFAULT;
2465 			useraddr += chunk;
2466 			t.addr += chunk;
2467 			t.len -= chunk;
2468 		}
2469 		break;
2470 	}
2471 	case CHELSIO_SET_TRACE_FILTER:{
2472 		struct ch_trace t;
2473 		const struct trace_params *tp;
2474 
2475 		if (!capable(CAP_NET_ADMIN))
2476 			return -EPERM;
2477 		if (!offload_running(adapter))
2478 			return -EAGAIN;
2479 		if (copy_from_user(&t, useraddr, sizeof(t)))
2480 			return -EFAULT;
2481 
2482 		tp = (const struct trace_params *)&t.sip;
2483 		if (t.config_tx)
2484 			t3_config_trace_filter(adapter, tp, 0,
2485 						t.invert_match,
2486 						t.trace_tx);
2487 		if (t.config_rx)
2488 			t3_config_trace_filter(adapter, tp, 1,
2489 						t.invert_match,
2490 						t.trace_rx);
2491 		break;
2492 	}
2493 	default:
2494 		return -EOPNOTSUPP;
2495 	}
2496 	return 0;
2497 }
2498 
2499 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2500 {
2501 	struct mii_ioctl_data *data = if_mii(req);
2502 	struct port_info *pi = netdev_priv(dev);
2503 	struct adapter *adapter = pi->adapter;
2504 
2505 	switch (cmd) {
2506 	case SIOCGMIIREG:
2507 	case SIOCSMIIREG:
2508 		/* Convert phy_id from older PRTAD/DEVAD format */
2509 		if (is_10G(adapter) &&
2510 		    !mdio_phy_id_is_c45(data->phy_id) &&
2511 		    (data->phy_id & 0x1f00) &&
2512 		    !(data->phy_id & 0xe0e0))
2513 			data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2514 						       data->phy_id & 0x1f);
2515 		/* FALLTHRU */
2516 	case SIOCGMIIPHY:
2517 		return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2518 	case SIOCCHIOCTL:
2519 		return cxgb_extension_ioctl(dev, req->ifr_data);
2520 	default:
2521 		return -EOPNOTSUPP;
2522 	}
2523 }
2524 
2525 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2526 {
2527 	struct port_info *pi = netdev_priv(dev);
2528 	struct adapter *adapter = pi->adapter;
2529 	int ret;
2530 
2531 	if (new_mtu < 81)	/* accommodate SACK */
2532 		return -EINVAL;
2533 	if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2534 		return ret;
2535 	dev->mtu = new_mtu;
2536 	init_port_mtus(adapter);
2537 	if (adapter->params.rev == 0 && offload_running(adapter))
2538 		t3_load_mtus(adapter, adapter->params.mtus,
2539 			     adapter->params.a_wnd, adapter->params.b_wnd,
2540 			     adapter->port[0]->mtu);
2541 	return 0;
2542 }
2543 
2544 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2545 {
2546 	struct port_info *pi = netdev_priv(dev);
2547 	struct adapter *adapter = pi->adapter;
2548 	struct sockaddr *addr = p;
2549 
2550 	if (!is_valid_ether_addr(addr->sa_data))
2551 		return -EADDRNOTAVAIL;
2552 
2553 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2554 	t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
2555 	if (offload_running(adapter))
2556 		write_smt_entry(adapter, pi->port_id);
2557 	return 0;
2558 }
2559 
2560 static netdev_features_t cxgb_fix_features(struct net_device *dev,
2561 	netdev_features_t features)
2562 {
2563 	/*
2564 	 * Since there is no support for separate rx/tx vlan accel
2565 	 * enable/disable make sure tx flag is always in same state as rx.
2566 	 */
2567 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
2568 		features |= NETIF_F_HW_VLAN_CTAG_TX;
2569 	else
2570 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2571 
2572 	return features;
2573 }
2574 
2575 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2576 {
2577 	netdev_features_t changed = dev->features ^ features;
2578 
2579 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2580 		cxgb_vlan_mode(dev, features);
2581 
2582 	return 0;
2583 }
2584 
2585 #ifdef CONFIG_NET_POLL_CONTROLLER
2586 static void cxgb_netpoll(struct net_device *dev)
2587 {
2588 	struct port_info *pi = netdev_priv(dev);
2589 	struct adapter *adapter = pi->adapter;
2590 	int qidx;
2591 
2592 	for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2593 		struct sge_qset *qs = &adapter->sge.qs[qidx];
2594 		void *source;
2595 
2596 		if (adapter->flags & USING_MSIX)
2597 			source = qs;
2598 		else
2599 			source = adapter;
2600 
2601 		t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2602 	}
2603 }
2604 #endif
2605 
2606 /*
2607  * Periodic accumulation of MAC statistics.
2608  */
2609 static void mac_stats_update(struct adapter *adapter)
2610 {
2611 	int i;
2612 
2613 	for_each_port(adapter, i) {
2614 		struct net_device *dev = adapter->port[i];
2615 		struct port_info *p = netdev_priv(dev);
2616 
2617 		if (netif_running(dev)) {
2618 			spin_lock(&adapter->stats_lock);
2619 			t3_mac_update_stats(&p->mac);
2620 			spin_unlock(&adapter->stats_lock);
2621 		}
2622 	}
2623 }
2624 
2625 static void check_link_status(struct adapter *adapter)
2626 {
2627 	int i;
2628 
2629 	for_each_port(adapter, i) {
2630 		struct net_device *dev = adapter->port[i];
2631 		struct port_info *p = netdev_priv(dev);
2632 		int link_fault;
2633 
2634 		spin_lock_irq(&adapter->work_lock);
2635 		link_fault = p->link_fault;
2636 		spin_unlock_irq(&adapter->work_lock);
2637 
2638 		if (link_fault) {
2639 			t3_link_fault(adapter, i);
2640 			continue;
2641 		}
2642 
2643 		if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2644 			t3_xgm_intr_disable(adapter, i);
2645 			t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2646 
2647 			t3_link_changed(adapter, i);
2648 			t3_xgm_intr_enable(adapter, i);
2649 		}
2650 	}
2651 }
2652 
2653 static void check_t3b2_mac(struct adapter *adapter)
2654 {
2655 	int i;
2656 
2657 	if (!rtnl_trylock())	/* synchronize with ifdown */
2658 		return;
2659 
2660 	for_each_port(adapter, i) {
2661 		struct net_device *dev = adapter->port[i];
2662 		struct port_info *p = netdev_priv(dev);
2663 		int status;
2664 
2665 		if (!netif_running(dev))
2666 			continue;
2667 
2668 		status = 0;
2669 		if (netif_running(dev) && netif_carrier_ok(dev))
2670 			status = t3b2_mac_watchdog_task(&p->mac);
2671 		if (status == 1)
2672 			p->mac.stats.num_toggled++;
2673 		else if (status == 2) {
2674 			struct cmac *mac = &p->mac;
2675 
2676 			t3_mac_set_mtu(mac, dev->mtu);
2677 			t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
2678 			cxgb_set_rxmode(dev);
2679 			t3_link_start(&p->phy, mac, &p->link_config);
2680 			t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2681 			t3_port_intr_enable(adapter, p->port_id);
2682 			p->mac.stats.num_resets++;
2683 		}
2684 	}
2685 	rtnl_unlock();
2686 }
2687 
2688 
2689 static void t3_adap_check_task(struct work_struct *work)
2690 {
2691 	struct adapter *adapter = container_of(work, struct adapter,
2692 					       adap_check_task.work);
2693 	const struct adapter_params *p = &adapter->params;
2694 	int port;
2695 	unsigned int v, status, reset;
2696 
2697 	adapter->check_task_cnt++;
2698 
2699 	check_link_status(adapter);
2700 
2701 	/* Accumulate MAC stats if needed */
2702 	if (!p->linkpoll_period ||
2703 	    (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2704 	    p->stats_update_period) {
2705 		mac_stats_update(adapter);
2706 		adapter->check_task_cnt = 0;
2707 	}
2708 
2709 	if (p->rev == T3_REV_B2)
2710 		check_t3b2_mac(adapter);
2711 
2712 	/*
2713 	 * Scan the XGMAC's to check for various conditions which we want to
2714 	 * monitor in a periodic polling manner rather than via an interrupt
2715 	 * condition.  This is used for conditions which would otherwise flood
2716 	 * the system with interrupts and we only really need to know that the
2717 	 * conditions are "happening" ...  For each condition we count the
2718 	 * detection of the condition and reset it for the next polling loop.
2719 	 */
2720 	for_each_port(adapter, port) {
2721 		struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2722 		u32 cause;
2723 
2724 		cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2725 		reset = 0;
2726 		if (cause & F_RXFIFO_OVERFLOW) {
2727 			mac->stats.rx_fifo_ovfl++;
2728 			reset |= F_RXFIFO_OVERFLOW;
2729 		}
2730 
2731 		t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2732 	}
2733 
2734 	/*
2735 	 * We do the same as above for FL_EMPTY interrupts.
2736 	 */
2737 	status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2738 	reset = 0;
2739 
2740 	if (status & F_FLEMPTY) {
2741 		struct sge_qset *qs = &adapter->sge.qs[0];
2742 		int i = 0;
2743 
2744 		reset |= F_FLEMPTY;
2745 
2746 		v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2747 		    0xffff;
2748 
2749 		while (v) {
2750 			qs->fl[i].empty += (v & 1);
2751 			if (i)
2752 				qs++;
2753 			i ^= 1;
2754 			v >>= 1;
2755 		}
2756 	}
2757 
2758 	t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2759 
2760 	/* Schedule the next check update if any port is active. */
2761 	spin_lock_irq(&adapter->work_lock);
2762 	if (adapter->open_device_map & PORT_MASK)
2763 		schedule_chk_task(adapter);
2764 	spin_unlock_irq(&adapter->work_lock);
2765 }
2766 
2767 static void db_full_task(struct work_struct *work)
2768 {
2769 	struct adapter *adapter = container_of(work, struct adapter,
2770 					       db_full_task);
2771 
2772 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2773 }
2774 
2775 static void db_empty_task(struct work_struct *work)
2776 {
2777 	struct adapter *adapter = container_of(work, struct adapter,
2778 					       db_empty_task);
2779 
2780 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2781 }
2782 
2783 static void db_drop_task(struct work_struct *work)
2784 {
2785 	struct adapter *adapter = container_of(work, struct adapter,
2786 					       db_drop_task);
2787 	unsigned long delay = 1000;
2788 	unsigned short r;
2789 
2790 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2791 
2792 	/*
2793 	 * Sleep a while before ringing the driver qset dbs.
2794 	 * The delay is between 1000-2023 usecs.
2795 	 */
2796 	get_random_bytes(&r, 2);
2797 	delay += r & 1023;
2798 	set_current_state(TASK_UNINTERRUPTIBLE);
2799 	schedule_timeout(usecs_to_jiffies(delay));
2800 	ring_dbs(adapter);
2801 }
2802 
2803 /*
2804  * Processes external (PHY) interrupts in process context.
2805  */
2806 static void ext_intr_task(struct work_struct *work)
2807 {
2808 	struct adapter *adapter = container_of(work, struct adapter,
2809 					       ext_intr_handler_task);
2810 	int i;
2811 
2812 	/* Disable link fault interrupts */
2813 	for_each_port(adapter, i) {
2814 		struct net_device *dev = adapter->port[i];
2815 		struct port_info *p = netdev_priv(dev);
2816 
2817 		t3_xgm_intr_disable(adapter, i);
2818 		t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2819 	}
2820 
2821 	/* Re-enable link fault interrupts */
2822 	t3_phy_intr_handler(adapter);
2823 
2824 	for_each_port(adapter, i)
2825 		t3_xgm_intr_enable(adapter, i);
2826 
2827 	/* Now reenable external interrupts */
2828 	spin_lock_irq(&adapter->work_lock);
2829 	if (adapter->slow_intr_mask) {
2830 		adapter->slow_intr_mask |= F_T3DBG;
2831 		t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2832 		t3_write_reg(adapter, A_PL_INT_ENABLE0,
2833 			     adapter->slow_intr_mask);
2834 	}
2835 	spin_unlock_irq(&adapter->work_lock);
2836 }
2837 
2838 /*
2839  * Interrupt-context handler for external (PHY) interrupts.
2840  */
2841 void t3_os_ext_intr_handler(struct adapter *adapter)
2842 {
2843 	/*
2844 	 * Schedule a task to handle external interrupts as they may be slow
2845 	 * and we use a mutex to protect MDIO registers.  We disable PHY
2846 	 * interrupts in the meantime and let the task reenable them when
2847 	 * it's done.
2848 	 */
2849 	spin_lock(&adapter->work_lock);
2850 	if (adapter->slow_intr_mask) {
2851 		adapter->slow_intr_mask &= ~F_T3DBG;
2852 		t3_write_reg(adapter, A_PL_INT_ENABLE0,
2853 			     adapter->slow_intr_mask);
2854 		queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2855 	}
2856 	spin_unlock(&adapter->work_lock);
2857 }
2858 
2859 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2860 {
2861 	struct net_device *netdev = adapter->port[port_id];
2862 	struct port_info *pi = netdev_priv(netdev);
2863 
2864 	spin_lock(&adapter->work_lock);
2865 	pi->link_fault = 1;
2866 	spin_unlock(&adapter->work_lock);
2867 }
2868 
2869 static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq)
2870 {
2871 	int i, ret = 0;
2872 
2873 	if (is_offload(adapter) &&
2874 	    test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2875 		cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2876 		offload_close(&adapter->tdev);
2877 	}
2878 
2879 	/* Stop all ports */
2880 	for_each_port(adapter, i) {
2881 		struct net_device *netdev = adapter->port[i];
2882 
2883 		if (netif_running(netdev))
2884 			__cxgb_close(netdev, on_wq);
2885 	}
2886 
2887 	/* Stop SGE timers */
2888 	t3_stop_sge_timers(adapter);
2889 
2890 	adapter->flags &= ~FULL_INIT_DONE;
2891 
2892 	if (reset)
2893 		ret = t3_reset_adapter(adapter);
2894 
2895 	pci_disable_device(adapter->pdev);
2896 
2897 	return ret;
2898 }
2899 
2900 static int t3_reenable_adapter(struct adapter *adapter)
2901 {
2902 	if (pci_enable_device(adapter->pdev)) {
2903 		dev_err(&adapter->pdev->dev,
2904 			"Cannot re-enable PCI device after reset.\n");
2905 		goto err;
2906 	}
2907 	pci_set_master(adapter->pdev);
2908 	pci_restore_state(adapter->pdev);
2909 	pci_save_state(adapter->pdev);
2910 
2911 	/* Free sge resources */
2912 	t3_free_sge_resources(adapter);
2913 
2914 	if (t3_replay_prep_adapter(adapter))
2915 		goto err;
2916 
2917 	return 0;
2918 err:
2919 	return -1;
2920 }
2921 
2922 static void t3_resume_ports(struct adapter *adapter)
2923 {
2924 	int i;
2925 
2926 	/* Restart the ports */
2927 	for_each_port(adapter, i) {
2928 		struct net_device *netdev = adapter->port[i];
2929 
2930 		if (netif_running(netdev)) {
2931 			if (cxgb_open(netdev)) {
2932 				dev_err(&adapter->pdev->dev,
2933 					"can't bring device back up"
2934 					" after reset\n");
2935 				continue;
2936 			}
2937 		}
2938 	}
2939 
2940 	if (is_offload(adapter) && !ofld_disable)
2941 		cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2942 }
2943 
2944 /*
2945  * processes a fatal error.
2946  * Bring the ports down, reset the chip, bring the ports back up.
2947  */
2948 static void fatal_error_task(struct work_struct *work)
2949 {
2950 	struct adapter *adapter = container_of(work, struct adapter,
2951 					       fatal_error_handler_task);
2952 	int err = 0;
2953 
2954 	rtnl_lock();
2955 	err = t3_adapter_error(adapter, 1, 1);
2956 	if (!err)
2957 		err = t3_reenable_adapter(adapter);
2958 	if (!err)
2959 		t3_resume_ports(adapter);
2960 
2961 	CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2962 	rtnl_unlock();
2963 }
2964 
2965 void t3_fatal_err(struct adapter *adapter)
2966 {
2967 	unsigned int fw_status[4];
2968 
2969 	if (adapter->flags & FULL_INIT_DONE) {
2970 		t3_sge_stop(adapter);
2971 		t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
2972 		t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
2973 		t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
2974 		t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
2975 
2976 		spin_lock(&adapter->work_lock);
2977 		t3_intr_disable(adapter);
2978 		queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
2979 		spin_unlock(&adapter->work_lock);
2980 	}
2981 	CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
2982 	if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
2983 		CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
2984 			 fw_status[0], fw_status[1],
2985 			 fw_status[2], fw_status[3]);
2986 }
2987 
2988 /**
2989  * t3_io_error_detected - called when PCI error is detected
2990  * @pdev: Pointer to PCI device
2991  * @state: The current pci connection state
2992  *
2993  * This function is called after a PCI bus error affecting
2994  * this device has been detected.
2995  */
2996 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
2997 					     pci_channel_state_t state)
2998 {
2999 	struct adapter *adapter = pci_get_drvdata(pdev);
3000 
3001 	if (state == pci_channel_io_perm_failure)
3002 		return PCI_ERS_RESULT_DISCONNECT;
3003 
3004 	t3_adapter_error(adapter, 0, 0);
3005 
3006 	/* Request a slot reset. */
3007 	return PCI_ERS_RESULT_NEED_RESET;
3008 }
3009 
3010 /**
3011  * t3_io_slot_reset - called after the pci bus has been reset.
3012  * @pdev: Pointer to PCI device
3013  *
3014  * Restart the card from scratch, as if from a cold-boot.
3015  */
3016 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3017 {
3018 	struct adapter *adapter = pci_get_drvdata(pdev);
3019 
3020 	if (!t3_reenable_adapter(adapter))
3021 		return PCI_ERS_RESULT_RECOVERED;
3022 
3023 	return PCI_ERS_RESULT_DISCONNECT;
3024 }
3025 
3026 /**
3027  * t3_io_resume - called when traffic can start flowing again.
3028  * @pdev: Pointer to PCI device
3029  *
3030  * This callback is called when the error recovery driver tells us that
3031  * its OK to resume normal operation.
3032  */
3033 static void t3_io_resume(struct pci_dev *pdev)
3034 {
3035 	struct adapter *adapter = pci_get_drvdata(pdev);
3036 
3037 	CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3038 		 t3_read_reg(adapter, A_PCIE_PEX_ERR));
3039 
3040 	rtnl_lock();
3041 	t3_resume_ports(adapter);
3042 	rtnl_unlock();
3043 }
3044 
3045 static const struct pci_error_handlers t3_err_handler = {
3046 	.error_detected = t3_io_error_detected,
3047 	.slot_reset = t3_io_slot_reset,
3048 	.resume = t3_io_resume,
3049 };
3050 
3051 /*
3052  * Set the number of qsets based on the number of CPUs and the number of ports,
3053  * not to exceed the number of available qsets, assuming there are enough qsets
3054  * per port in HW.
3055  */
3056 static void set_nqsets(struct adapter *adap)
3057 {
3058 	int i, j = 0;
3059 	int num_cpus = netif_get_num_default_rss_queues();
3060 	int hwports = adap->params.nports;
3061 	int nqsets = adap->msix_nvectors - 1;
3062 
3063 	if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3064 		if (hwports == 2 &&
3065 		    (hwports * nqsets > SGE_QSETS ||
3066 		     num_cpus >= nqsets / hwports))
3067 			nqsets /= hwports;
3068 		if (nqsets > num_cpus)
3069 			nqsets = num_cpus;
3070 		if (nqsets < 1 || hwports == 4)
3071 			nqsets = 1;
3072 	} else
3073 		nqsets = 1;
3074 
3075 	for_each_port(adap, i) {
3076 		struct port_info *pi = adap2pinfo(adap, i);
3077 
3078 		pi->first_qset = j;
3079 		pi->nqsets = nqsets;
3080 		j = pi->first_qset + nqsets;
3081 
3082 		dev_info(&adap->pdev->dev,
3083 			 "Port %d using %d queue sets.\n", i, nqsets);
3084 	}
3085 }
3086 
3087 static int cxgb_enable_msix(struct adapter *adap)
3088 {
3089 	struct msix_entry entries[SGE_QSETS + 1];
3090 	int vectors;
3091 	int i;
3092 
3093 	vectors = ARRAY_SIZE(entries);
3094 	for (i = 0; i < vectors; ++i)
3095 		entries[i].entry = i;
3096 
3097 	vectors = pci_enable_msix_range(adap->pdev, entries,
3098 					adap->params.nports + 1, vectors);
3099 	if (vectors < 0)
3100 		return vectors;
3101 
3102 	for (i = 0; i < vectors; ++i)
3103 		adap->msix_info[i].vec = entries[i].vector;
3104 	adap->msix_nvectors = vectors;
3105 
3106 	return 0;
3107 }
3108 
3109 static void print_port_info(struct adapter *adap, const struct adapter_info *ai)
3110 {
3111 	static const char *pci_variant[] = {
3112 		"PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3113 	};
3114 
3115 	int i;
3116 	char buf[80];
3117 
3118 	if (is_pcie(adap))
3119 		snprintf(buf, sizeof(buf), "%s x%d",
3120 			 pci_variant[adap->params.pci.variant],
3121 			 adap->params.pci.width);
3122 	else
3123 		snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3124 			 pci_variant[adap->params.pci.variant],
3125 			 adap->params.pci.speed, adap->params.pci.width);
3126 
3127 	for_each_port(adap, i) {
3128 		struct net_device *dev = adap->port[i];
3129 		const struct port_info *pi = netdev_priv(dev);
3130 
3131 		if (!test_bit(i, &adap->registered_device_map))
3132 			continue;
3133 		netdev_info(dev, "%s %s %sNIC (rev %d) %s%s\n",
3134 			    ai->desc, pi->phy.desc,
3135 			    is_offload(adap) ? "R" : "", adap->params.rev, buf,
3136 			    (adap->flags & USING_MSIX) ? " MSI-X" :
3137 			    (adap->flags & USING_MSI) ? " MSI" : "");
3138 		if (adap->name == dev->name && adap->params.vpd.mclk)
3139 			pr_info("%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3140 			       adap->name, t3_mc7_size(&adap->cm) >> 20,
3141 			       t3_mc7_size(&adap->pmtx) >> 20,
3142 			       t3_mc7_size(&adap->pmrx) >> 20,
3143 			       adap->params.vpd.sn);
3144 	}
3145 }
3146 
3147 static const struct net_device_ops cxgb_netdev_ops = {
3148 	.ndo_open		= cxgb_open,
3149 	.ndo_stop		= cxgb_close,
3150 	.ndo_start_xmit		= t3_eth_xmit,
3151 	.ndo_get_stats		= cxgb_get_stats,
3152 	.ndo_validate_addr	= eth_validate_addr,
3153 	.ndo_set_rx_mode	= cxgb_set_rxmode,
3154 	.ndo_do_ioctl		= cxgb_ioctl,
3155 	.ndo_change_mtu		= cxgb_change_mtu,
3156 	.ndo_set_mac_address	= cxgb_set_mac_addr,
3157 	.ndo_fix_features	= cxgb_fix_features,
3158 	.ndo_set_features	= cxgb_set_features,
3159 #ifdef CONFIG_NET_POLL_CONTROLLER
3160 	.ndo_poll_controller	= cxgb_netpoll,
3161 #endif
3162 };
3163 
3164 static void cxgb3_init_iscsi_mac(struct net_device *dev)
3165 {
3166 	struct port_info *pi = netdev_priv(dev);
3167 
3168 	memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3169 	pi->iscsic.mac_addr[3] |= 0x80;
3170 }
3171 
3172 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3173 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3174 			NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3175 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
3176 {
3177 	int i, err, pci_using_dac = 0;
3178 	resource_size_t mmio_start, mmio_len;
3179 	const struct adapter_info *ai;
3180 	struct adapter *adapter = NULL;
3181 	struct port_info *pi;
3182 
3183 	pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
3184 
3185 	if (!cxgb3_wq) {
3186 		cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3187 		if (!cxgb3_wq) {
3188 			pr_err("cannot initialize work queue\n");
3189 			return -ENOMEM;
3190 		}
3191 	}
3192 
3193 	err = pci_enable_device(pdev);
3194 	if (err) {
3195 		dev_err(&pdev->dev, "cannot enable PCI device\n");
3196 		goto out;
3197 	}
3198 
3199 	err = pci_request_regions(pdev, DRV_NAME);
3200 	if (err) {
3201 		/* Just info, some other driver may have claimed the device. */
3202 		dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3203 		goto out_disable_device;
3204 	}
3205 
3206 	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3207 		pci_using_dac = 1;
3208 		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3209 		if (err) {
3210 			dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3211 			       "coherent allocations\n");
3212 			goto out_release_regions;
3213 		}
3214 	} else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3215 		dev_err(&pdev->dev, "no usable DMA configuration\n");
3216 		goto out_release_regions;
3217 	}
3218 
3219 	pci_set_master(pdev);
3220 	pci_save_state(pdev);
3221 
3222 	mmio_start = pci_resource_start(pdev, 0);
3223 	mmio_len = pci_resource_len(pdev, 0);
3224 	ai = t3_get_adapter_info(ent->driver_data);
3225 
3226 	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3227 	if (!adapter) {
3228 		err = -ENOMEM;
3229 		goto out_release_regions;
3230 	}
3231 
3232 	adapter->nofail_skb =
3233 		alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3234 	if (!adapter->nofail_skb) {
3235 		dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3236 		err = -ENOMEM;
3237 		goto out_free_adapter;
3238 	}
3239 
3240 	adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3241 	if (!adapter->regs) {
3242 		dev_err(&pdev->dev, "cannot map device registers\n");
3243 		err = -ENOMEM;
3244 		goto out_free_adapter;
3245 	}
3246 
3247 	adapter->pdev = pdev;
3248 	adapter->name = pci_name(pdev);
3249 	adapter->msg_enable = dflt_msg_enable;
3250 	adapter->mmio_len = mmio_len;
3251 
3252 	mutex_init(&adapter->mdio_lock);
3253 	spin_lock_init(&adapter->work_lock);
3254 	spin_lock_init(&adapter->stats_lock);
3255 
3256 	INIT_LIST_HEAD(&adapter->adapter_list);
3257 	INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3258 	INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3259 
3260 	INIT_WORK(&adapter->db_full_task, db_full_task);
3261 	INIT_WORK(&adapter->db_empty_task, db_empty_task);
3262 	INIT_WORK(&adapter->db_drop_task, db_drop_task);
3263 
3264 	INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3265 
3266 	for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3267 		struct net_device *netdev;
3268 
3269 		netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3270 		if (!netdev) {
3271 			err = -ENOMEM;
3272 			goto out_free_dev;
3273 		}
3274 
3275 		SET_NETDEV_DEV(netdev, &pdev->dev);
3276 
3277 		adapter->port[i] = netdev;
3278 		pi = netdev_priv(netdev);
3279 		pi->adapter = adapter;
3280 		pi->port_id = i;
3281 		netif_carrier_off(netdev);
3282 		netdev->irq = pdev->irq;
3283 		netdev->mem_start = mmio_start;
3284 		netdev->mem_end = mmio_start + mmio_len - 1;
3285 		netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
3286 			NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX;
3287 		netdev->features |= netdev->hw_features |
3288 				    NETIF_F_HW_VLAN_CTAG_TX;
3289 		netdev->vlan_features |= netdev->features & VLAN_FEAT;
3290 		if (pci_using_dac)
3291 			netdev->features |= NETIF_F_HIGHDMA;
3292 
3293 		netdev->netdev_ops = &cxgb_netdev_ops;
3294 		netdev->ethtool_ops = &cxgb_ethtool_ops;
3295 	}
3296 
3297 	pci_set_drvdata(pdev, adapter);
3298 	if (t3_prep_adapter(adapter, ai, 1) < 0) {
3299 		err = -ENODEV;
3300 		goto out_free_dev;
3301 	}
3302 
3303 	/*
3304 	 * The card is now ready to go.  If any errors occur during device
3305 	 * registration we do not fail the whole card but rather proceed only
3306 	 * with the ports we manage to register successfully.  However we must
3307 	 * register at least one net device.
3308 	 */
3309 	for_each_port(adapter, i) {
3310 		err = register_netdev(adapter->port[i]);
3311 		if (err)
3312 			dev_warn(&pdev->dev,
3313 				 "cannot register net device %s, skipping\n",
3314 				 adapter->port[i]->name);
3315 		else {
3316 			/*
3317 			 * Change the name we use for messages to the name of
3318 			 * the first successfully registered interface.
3319 			 */
3320 			if (!adapter->registered_device_map)
3321 				adapter->name = adapter->port[i]->name;
3322 
3323 			__set_bit(i, &adapter->registered_device_map);
3324 		}
3325 	}
3326 	if (!adapter->registered_device_map) {
3327 		dev_err(&pdev->dev, "could not register any net devices\n");
3328 		goto out_free_dev;
3329 	}
3330 
3331 	for_each_port(adapter, i)
3332 		cxgb3_init_iscsi_mac(adapter->port[i]);
3333 
3334 	/* Driver's ready. Reflect it on LEDs */
3335 	t3_led_ready(adapter);
3336 
3337 	if (is_offload(adapter)) {
3338 		__set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3339 		cxgb3_adapter_ofld(adapter);
3340 	}
3341 
3342 	/* See what interrupts we'll be using */
3343 	if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3344 		adapter->flags |= USING_MSIX;
3345 	else if (msi > 0 && pci_enable_msi(pdev) == 0)
3346 		adapter->flags |= USING_MSI;
3347 
3348 	set_nqsets(adapter);
3349 
3350 	err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3351 				 &cxgb3_attr_group);
3352 
3353 	print_port_info(adapter, ai);
3354 	return 0;
3355 
3356 out_free_dev:
3357 	iounmap(adapter->regs);
3358 	for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3359 		if (adapter->port[i])
3360 			free_netdev(adapter->port[i]);
3361 
3362 out_free_adapter:
3363 	kfree(adapter);
3364 
3365 out_release_regions:
3366 	pci_release_regions(pdev);
3367 out_disable_device:
3368 	pci_disable_device(pdev);
3369 out:
3370 	return err;
3371 }
3372 
3373 static void remove_one(struct pci_dev *pdev)
3374 {
3375 	struct adapter *adapter = pci_get_drvdata(pdev);
3376 
3377 	if (adapter) {
3378 		int i;
3379 
3380 		t3_sge_stop(adapter);
3381 		sysfs_remove_group(&adapter->port[0]->dev.kobj,
3382 				   &cxgb3_attr_group);
3383 
3384 		if (is_offload(adapter)) {
3385 			cxgb3_adapter_unofld(adapter);
3386 			if (test_bit(OFFLOAD_DEVMAP_BIT,
3387 				     &adapter->open_device_map))
3388 				offload_close(&adapter->tdev);
3389 		}
3390 
3391 		for_each_port(adapter, i)
3392 		    if (test_bit(i, &adapter->registered_device_map))
3393 			unregister_netdev(adapter->port[i]);
3394 
3395 		t3_stop_sge_timers(adapter);
3396 		t3_free_sge_resources(adapter);
3397 		cxgb_disable_msi(adapter);
3398 
3399 		for_each_port(adapter, i)
3400 			if (adapter->port[i])
3401 				free_netdev(adapter->port[i]);
3402 
3403 		iounmap(adapter->regs);
3404 		if (adapter->nofail_skb)
3405 			kfree_skb(adapter->nofail_skb);
3406 		kfree(adapter);
3407 		pci_release_regions(pdev);
3408 		pci_disable_device(pdev);
3409 	}
3410 }
3411 
3412 static struct pci_driver driver = {
3413 	.name = DRV_NAME,
3414 	.id_table = cxgb3_pci_tbl,
3415 	.probe = init_one,
3416 	.remove = remove_one,
3417 	.err_handler = &t3_err_handler,
3418 };
3419 
3420 static int __init cxgb3_init_module(void)
3421 {
3422 	int ret;
3423 
3424 	cxgb3_offload_init();
3425 
3426 	ret = pci_register_driver(&driver);
3427 	return ret;
3428 }
3429 
3430 static void __exit cxgb3_cleanup_module(void)
3431 {
3432 	pci_unregister_driver(&driver);
3433 	if (cxgb3_wq)
3434 		destroy_workqueue(cxgb3_wq);
3435 }
3436 
3437 module_init(cxgb3_init_module);
3438 module_exit(cxgb3_cleanup_module);
3439