1 /*
2  * Copyright (c) 2003-2008 Chelsio, Inc. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
34 
35 #include <linux/module.h>
36 #include <linux/init.h>
37 #include <linux/pci.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/netdevice.h>
40 #include <linux/etherdevice.h>
41 #include <linux/if_vlan.h>
42 #include <linux/mdio.h>
43 #include <linux/sockios.h>
44 #include <linux/workqueue.h>
45 #include <linux/proc_fs.h>
46 #include <linux/rtnetlink.h>
47 #include <linux/firmware.h>
48 #include <linux/log2.h>
49 #include <linux/stringify.h>
50 #include <linux/sched.h>
51 #include <linux/slab.h>
52 #include <linux/uaccess.h>
53 #include <linux/nospec.h>
54 
55 #include "common.h"
56 #include "cxgb3_ioctl.h"
57 #include "regs.h"
58 #include "cxgb3_offload.h"
59 #include "version.h"
60 
61 #include "cxgb3_ctl_defs.h"
62 #include "t3_cpl.h"
63 #include "firmware_exports.h"
64 
65 enum {
66 	MAX_TXQ_ENTRIES = 16384,
67 	MAX_CTRL_TXQ_ENTRIES = 1024,
68 	MAX_RSPQ_ENTRIES = 16384,
69 	MAX_RX_BUFFERS = 16384,
70 	MAX_RX_JUMBO_BUFFERS = 16384,
71 	MIN_TXQ_ENTRIES = 4,
72 	MIN_CTRL_TXQ_ENTRIES = 4,
73 	MIN_RSPQ_ENTRIES = 32,
74 	MIN_FL_ENTRIES = 32
75 };
76 
77 #define PORT_MASK ((1 << MAX_NPORTS) - 1)
78 
79 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
80 			 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
81 			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
82 
83 #define EEPROM_MAGIC 0x38E2F10C
84 
85 #define CH_DEVICE(devid, idx) \
86 	{ PCI_VENDOR_ID_CHELSIO, devid, PCI_ANY_ID, PCI_ANY_ID, 0, 0, idx }
87 
88 static const struct pci_device_id cxgb3_pci_tbl[] = {
89 	CH_DEVICE(0x20, 0),	/* PE9000 */
90 	CH_DEVICE(0x21, 1),	/* T302E */
91 	CH_DEVICE(0x22, 2),	/* T310E */
92 	CH_DEVICE(0x23, 3),	/* T320X */
93 	CH_DEVICE(0x24, 1),	/* T302X */
94 	CH_DEVICE(0x25, 3),	/* T320E */
95 	CH_DEVICE(0x26, 2),	/* T310X */
96 	CH_DEVICE(0x30, 2),	/* T3B10 */
97 	CH_DEVICE(0x31, 3),	/* T3B20 */
98 	CH_DEVICE(0x32, 1),	/* T3B02 */
99 	CH_DEVICE(0x35, 6),	/* T3C20-derived T3C10 */
100 	CH_DEVICE(0x36, 3),	/* S320E-CR */
101 	CH_DEVICE(0x37, 7),	/* N320E-G2 */
102 	{0,}
103 };
104 
105 MODULE_DESCRIPTION(DRV_DESC);
106 MODULE_AUTHOR("Chelsio Communications");
107 MODULE_LICENSE("Dual BSD/GPL");
108 MODULE_VERSION(DRV_VERSION);
109 MODULE_DEVICE_TABLE(pci, cxgb3_pci_tbl);
110 
111 static int dflt_msg_enable = DFLT_MSG_ENABLE;
112 
113 module_param(dflt_msg_enable, int, 0644);
114 MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T3 default message enable bitmap");
115 
116 /*
117  * The driver uses the best interrupt scheme available on a platform in the
118  * order MSI-X, MSI, legacy pin interrupts.  This parameter determines which
119  * of these schemes the driver may consider as follows:
120  *
121  * msi = 2: choose from among all three options
122  * msi = 1: only consider MSI and pin interrupts
123  * msi = 0: force pin interrupts
124  */
125 static int msi = 2;
126 
127 module_param(msi, int, 0644);
128 MODULE_PARM_DESC(msi, "whether to use MSI or MSI-X");
129 
130 /*
131  * The driver enables offload as a default.
132  * To disable it, use ofld_disable = 1.
133  */
134 
135 static int ofld_disable = 0;
136 
137 module_param(ofld_disable, int, 0644);
138 MODULE_PARM_DESC(ofld_disable, "whether to enable offload at init time or not");
139 
140 /*
141  * We have work elements that we need to cancel when an interface is taken
142  * down.  Normally the work elements would be executed by keventd but that
143  * can deadlock because of linkwatch.  If our close method takes the rtnl
144  * lock and linkwatch is ahead of our work elements in keventd, linkwatch
145  * will block keventd as it needs the rtnl lock, and we'll deadlock waiting
146  * for our work to complete.  Get our own work queue to solve this.
147  */
148 struct workqueue_struct *cxgb3_wq;
149 
150 /**
151  *	link_report - show link status and link speed/duplex
152  *	@p: the port whose settings are to be reported
153  *
154  *	Shows the link status, speed, and duplex of a port.
155  */
156 static void link_report(struct net_device *dev)
157 {
158 	if (!netif_carrier_ok(dev))
159 		netdev_info(dev, "link down\n");
160 	else {
161 		const char *s = "10Mbps";
162 		const struct port_info *p = netdev_priv(dev);
163 
164 		switch (p->link_config.speed) {
165 		case SPEED_10000:
166 			s = "10Gbps";
167 			break;
168 		case SPEED_1000:
169 			s = "1000Mbps";
170 			break;
171 		case SPEED_100:
172 			s = "100Mbps";
173 			break;
174 		}
175 
176 		netdev_info(dev, "link up, %s, %s-duplex\n",
177 			    s, p->link_config.duplex == DUPLEX_FULL
178 			    ? "full" : "half");
179 	}
180 }
181 
182 static void enable_tx_fifo_drain(struct adapter *adapter,
183 				 struct port_info *pi)
184 {
185 	t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset, 0,
186 			 F_ENDROPPKT);
187 	t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, 0);
188 	t3_write_reg(adapter, A_XGM_TX_CTRL + pi->mac.offset, F_TXEN);
189 	t3_write_reg(adapter, A_XGM_RX_CTRL + pi->mac.offset, F_RXEN);
190 }
191 
192 static void disable_tx_fifo_drain(struct adapter *adapter,
193 				  struct port_info *pi)
194 {
195 	t3_set_reg_field(adapter, A_XGM_TXFIFO_CFG + pi->mac.offset,
196 			 F_ENDROPPKT, 0);
197 }
198 
199 void t3_os_link_fault(struct adapter *adap, int port_id, int state)
200 {
201 	struct net_device *dev = adap->port[port_id];
202 	struct port_info *pi = netdev_priv(dev);
203 
204 	if (state == netif_carrier_ok(dev))
205 		return;
206 
207 	if (state) {
208 		struct cmac *mac = &pi->mac;
209 
210 		netif_carrier_on(dev);
211 
212 		disable_tx_fifo_drain(adap, pi);
213 
214 		/* Clear local faults */
215 		t3_xgm_intr_disable(adap, pi->port_id);
216 		t3_read_reg(adap, A_XGM_INT_STATUS +
217 				    pi->mac.offset);
218 		t3_write_reg(adap,
219 			     A_XGM_INT_CAUSE + pi->mac.offset,
220 			     F_XGM_INT);
221 
222 		t3_set_reg_field(adap,
223 				 A_XGM_INT_ENABLE +
224 				 pi->mac.offset,
225 				 F_XGM_INT, F_XGM_INT);
226 		t3_xgm_intr_enable(adap, pi->port_id);
227 
228 		t3_mac_enable(mac, MAC_DIRECTION_TX);
229 	} else {
230 		netif_carrier_off(dev);
231 
232 		/* Flush TX FIFO */
233 		enable_tx_fifo_drain(adap, pi);
234 	}
235 	link_report(dev);
236 }
237 
238 /**
239  *	t3_os_link_changed - handle link status changes
240  *	@adapter: the adapter associated with the link change
241  *	@port_id: the port index whose limk status has changed
242  *	@link_stat: the new status of the link
243  *	@speed: the new speed setting
244  *	@duplex: the new duplex setting
245  *	@pause: the new flow-control setting
246  *
247  *	This is the OS-dependent handler for link status changes.  The OS
248  *	neutral handler takes care of most of the processing for these events,
249  *	then calls this handler for any OS-specific processing.
250  */
251 void t3_os_link_changed(struct adapter *adapter, int port_id, int link_stat,
252 			int speed, int duplex, int pause)
253 {
254 	struct net_device *dev = adapter->port[port_id];
255 	struct port_info *pi = netdev_priv(dev);
256 	struct cmac *mac = &pi->mac;
257 
258 	/* Skip changes from disabled ports. */
259 	if (!netif_running(dev))
260 		return;
261 
262 	if (link_stat != netif_carrier_ok(dev)) {
263 		if (link_stat) {
264 			disable_tx_fifo_drain(adapter, pi);
265 
266 			t3_mac_enable(mac, MAC_DIRECTION_RX);
267 
268 			/* Clear local faults */
269 			t3_xgm_intr_disable(adapter, pi->port_id);
270 			t3_read_reg(adapter, A_XGM_INT_STATUS +
271 				    pi->mac.offset);
272 			t3_write_reg(adapter,
273 				     A_XGM_INT_CAUSE + pi->mac.offset,
274 				     F_XGM_INT);
275 
276 			t3_set_reg_field(adapter,
277 					 A_XGM_INT_ENABLE + pi->mac.offset,
278 					 F_XGM_INT, F_XGM_INT);
279 			t3_xgm_intr_enable(adapter, pi->port_id);
280 
281 			netif_carrier_on(dev);
282 		} else {
283 			netif_carrier_off(dev);
284 
285 			t3_xgm_intr_disable(adapter, pi->port_id);
286 			t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
287 			t3_set_reg_field(adapter,
288 					 A_XGM_INT_ENABLE + pi->mac.offset,
289 					 F_XGM_INT, 0);
290 
291 			if (is_10G(adapter))
292 				pi->phy.ops->power_down(&pi->phy, 1);
293 
294 			t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
295 			t3_mac_disable(mac, MAC_DIRECTION_RX);
296 			t3_link_start(&pi->phy, mac, &pi->link_config);
297 
298 			/* Flush TX FIFO */
299 			enable_tx_fifo_drain(adapter, pi);
300 		}
301 
302 		link_report(dev);
303 	}
304 }
305 
306 /**
307  *	t3_os_phymod_changed - handle PHY module changes
308  *	@phy: the PHY reporting the module change
309  *	@mod_type: new module type
310  *
311  *	This is the OS-dependent handler for PHY module changes.  It is
312  *	invoked when a PHY module is removed or inserted for any OS-specific
313  *	processing.
314  */
315 void t3_os_phymod_changed(struct adapter *adap, int port_id)
316 {
317 	static const char *mod_str[] = {
318 		NULL, "SR", "LR", "LRM", "TWINAX", "TWINAX", "unknown"
319 	};
320 
321 	const struct net_device *dev = adap->port[port_id];
322 	const struct port_info *pi = netdev_priv(dev);
323 
324 	if (pi->phy.modtype == phy_modtype_none)
325 		netdev_info(dev, "PHY module unplugged\n");
326 	else
327 		netdev_info(dev, "%s PHY module inserted\n",
328 			    mod_str[pi->phy.modtype]);
329 }
330 
331 static void cxgb_set_rxmode(struct net_device *dev)
332 {
333 	struct port_info *pi = netdev_priv(dev);
334 
335 	t3_mac_set_rx_mode(&pi->mac, dev);
336 }
337 
338 /**
339  *	link_start - enable a port
340  *	@dev: the device to enable
341  *
342  *	Performs the MAC and PHY actions needed to enable a port.
343  */
344 static void link_start(struct net_device *dev)
345 {
346 	struct port_info *pi = netdev_priv(dev);
347 	struct cmac *mac = &pi->mac;
348 
349 	t3_mac_reset(mac);
350 	t3_mac_set_num_ucast(mac, MAX_MAC_IDX);
351 	t3_mac_set_mtu(mac, dev->mtu);
352 	t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
353 	t3_mac_set_address(mac, SAN_MAC_IDX, pi->iscsic.mac_addr);
354 	t3_mac_set_rx_mode(mac, dev);
355 	t3_link_start(&pi->phy, mac, &pi->link_config);
356 	t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
357 }
358 
359 static inline void cxgb_disable_msi(struct adapter *adapter)
360 {
361 	if (adapter->flags & USING_MSIX) {
362 		pci_disable_msix(adapter->pdev);
363 		adapter->flags &= ~USING_MSIX;
364 	} else if (adapter->flags & USING_MSI) {
365 		pci_disable_msi(adapter->pdev);
366 		adapter->flags &= ~USING_MSI;
367 	}
368 }
369 
370 /*
371  * Interrupt handler for asynchronous events used with MSI-X.
372  */
373 static irqreturn_t t3_async_intr_handler(int irq, void *cookie)
374 {
375 	t3_slow_intr_handler(cookie);
376 	return IRQ_HANDLED;
377 }
378 
379 /*
380  * Name the MSI-X interrupts.
381  */
382 static void name_msix_vecs(struct adapter *adap)
383 {
384 	int i, j, msi_idx = 1, n = sizeof(adap->msix_info[0].desc) - 1;
385 
386 	snprintf(adap->msix_info[0].desc, n, "%s", adap->name);
387 	adap->msix_info[0].desc[n] = 0;
388 
389 	for_each_port(adap, j) {
390 		struct net_device *d = adap->port[j];
391 		const struct port_info *pi = netdev_priv(d);
392 
393 		for (i = 0; i < pi->nqsets; i++, msi_idx++) {
394 			snprintf(adap->msix_info[msi_idx].desc, n,
395 				 "%s-%d", d->name, pi->first_qset + i);
396 			adap->msix_info[msi_idx].desc[n] = 0;
397 		}
398 	}
399 }
400 
401 static int request_msix_data_irqs(struct adapter *adap)
402 {
403 	int i, j, err, qidx = 0;
404 
405 	for_each_port(adap, i) {
406 		int nqsets = adap2pinfo(adap, i)->nqsets;
407 
408 		for (j = 0; j < nqsets; ++j) {
409 			err = request_irq(adap->msix_info[qidx + 1].vec,
410 					  t3_intr_handler(adap,
411 							  adap->sge.qs[qidx].
412 							  rspq.polling), 0,
413 					  adap->msix_info[qidx + 1].desc,
414 					  &adap->sge.qs[qidx]);
415 			if (err) {
416 				while (--qidx >= 0)
417 					free_irq(adap->msix_info[qidx + 1].vec,
418 						 &adap->sge.qs[qidx]);
419 				return err;
420 			}
421 			qidx++;
422 		}
423 	}
424 	return 0;
425 }
426 
427 static void free_irq_resources(struct adapter *adapter)
428 {
429 	if (adapter->flags & USING_MSIX) {
430 		int i, n = 0;
431 
432 		free_irq(adapter->msix_info[0].vec, adapter);
433 		for_each_port(adapter, i)
434 			n += adap2pinfo(adapter, i)->nqsets;
435 
436 		for (i = 0; i < n; ++i)
437 			free_irq(adapter->msix_info[i + 1].vec,
438 				 &adapter->sge.qs[i]);
439 	} else
440 		free_irq(adapter->pdev->irq, adapter);
441 }
442 
443 static int await_mgmt_replies(struct adapter *adap, unsigned long init_cnt,
444 			      unsigned long n)
445 {
446 	int attempts = 10;
447 
448 	while (adap->sge.qs[0].rspq.offload_pkts < init_cnt + n) {
449 		if (!--attempts)
450 			return -ETIMEDOUT;
451 		msleep(10);
452 	}
453 	return 0;
454 }
455 
456 static int init_tp_parity(struct adapter *adap)
457 {
458 	int i;
459 	struct sk_buff *skb;
460 	struct cpl_set_tcb_field *greq;
461 	unsigned long cnt = adap->sge.qs[0].rspq.offload_pkts;
462 
463 	t3_tp_set_offload_mode(adap, 1);
464 
465 	for (i = 0; i < 16; i++) {
466 		struct cpl_smt_write_req *req;
467 
468 		skb = alloc_skb(sizeof(*req), GFP_KERNEL);
469 		if (!skb)
470 			skb = adap->nofail_skb;
471 		if (!skb)
472 			goto alloc_skb_fail;
473 
474 		req = __skb_put_zero(skb, sizeof(*req));
475 		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
476 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, i));
477 		req->mtu_idx = NMTUS - 1;
478 		req->iff = i;
479 		t3_mgmt_tx(adap, skb);
480 		if (skb == adap->nofail_skb) {
481 			await_mgmt_replies(adap, cnt, i + 1);
482 			adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
483 			if (!adap->nofail_skb)
484 				goto alloc_skb_fail;
485 		}
486 	}
487 
488 	for (i = 0; i < 2048; i++) {
489 		struct cpl_l2t_write_req *req;
490 
491 		skb = alloc_skb(sizeof(*req), GFP_KERNEL);
492 		if (!skb)
493 			skb = adap->nofail_skb;
494 		if (!skb)
495 			goto alloc_skb_fail;
496 
497 		req = __skb_put_zero(skb, sizeof(*req));
498 		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
499 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_L2T_WRITE_REQ, i));
500 		req->params = htonl(V_L2T_W_IDX(i));
501 		t3_mgmt_tx(adap, skb);
502 		if (skb == adap->nofail_skb) {
503 			await_mgmt_replies(adap, cnt, 16 + i + 1);
504 			adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
505 			if (!adap->nofail_skb)
506 				goto alloc_skb_fail;
507 		}
508 	}
509 
510 	for (i = 0; i < 2048; i++) {
511 		struct cpl_rte_write_req *req;
512 
513 		skb = alloc_skb(sizeof(*req), GFP_KERNEL);
514 		if (!skb)
515 			skb = adap->nofail_skb;
516 		if (!skb)
517 			goto alloc_skb_fail;
518 
519 		req = __skb_put_zero(skb, sizeof(*req));
520 		req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
521 		OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_RTE_WRITE_REQ, i));
522 		req->l2t_idx = htonl(V_L2T_W_IDX(i));
523 		t3_mgmt_tx(adap, skb);
524 		if (skb == adap->nofail_skb) {
525 			await_mgmt_replies(adap, cnt, 16 + 2048 + i + 1);
526 			adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
527 			if (!adap->nofail_skb)
528 				goto alloc_skb_fail;
529 		}
530 	}
531 
532 	skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
533 	if (!skb)
534 		skb = adap->nofail_skb;
535 	if (!skb)
536 		goto alloc_skb_fail;
537 
538 	greq = __skb_put_zero(skb, sizeof(*greq));
539 	greq->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
540 	OPCODE_TID(greq) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, 0));
541 	greq->mask = cpu_to_be64(1);
542 	t3_mgmt_tx(adap, skb);
543 
544 	i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
545 	if (skb == adap->nofail_skb) {
546 		i = await_mgmt_replies(adap, cnt, 16 + 2048 + 2048 + 1);
547 		adap->nofail_skb = alloc_skb(sizeof(*greq), GFP_KERNEL);
548 	}
549 
550 	t3_tp_set_offload_mode(adap, 0);
551 	return i;
552 
553 alloc_skb_fail:
554 	t3_tp_set_offload_mode(adap, 0);
555 	return -ENOMEM;
556 }
557 
558 /**
559  *	setup_rss - configure RSS
560  *	@adap: the adapter
561  *
562  *	Sets up RSS to distribute packets to multiple receive queues.  We
563  *	configure the RSS CPU lookup table to distribute to the number of HW
564  *	receive queues, and the response queue lookup table to narrow that
565  *	down to the response queues actually configured for each port.
566  *	We always configure the RSS mapping for two ports since the mapping
567  *	table has plenty of entries.
568  */
569 static void setup_rss(struct adapter *adap)
570 {
571 	int i;
572 	unsigned int nq0 = adap2pinfo(adap, 0)->nqsets;
573 	unsigned int nq1 = adap->port[1] ? adap2pinfo(adap, 1)->nqsets : 1;
574 	u8 cpus[SGE_QSETS + 1];
575 	u16 rspq_map[RSS_TABLE_SIZE + 1];
576 
577 	for (i = 0; i < SGE_QSETS; ++i)
578 		cpus[i] = i;
579 	cpus[SGE_QSETS] = 0xff;	/* terminator */
580 
581 	for (i = 0; i < RSS_TABLE_SIZE / 2; ++i) {
582 		rspq_map[i] = i % nq0;
583 		rspq_map[i + RSS_TABLE_SIZE / 2] = (i % nq1) + nq0;
584 	}
585 	rspq_map[RSS_TABLE_SIZE] = 0xffff; /* terminator */
586 
587 	t3_config_rss(adap, F_RQFEEDBACKENABLE | F_TNLLKPEN | F_TNLMAPEN |
588 		      F_TNLPRTEN | F_TNL2TUPEN | F_TNL4TUPEN |
589 		      V_RRCPLCPUSIZE(6) | F_HASHTOEPLITZ, cpus, rspq_map);
590 }
591 
592 static void ring_dbs(struct adapter *adap)
593 {
594 	int i, j;
595 
596 	for (i = 0; i < SGE_QSETS; i++) {
597 		struct sge_qset *qs = &adap->sge.qs[i];
598 
599 		if (qs->adap)
600 			for (j = 0; j < SGE_TXQ_PER_SET; j++)
601 				t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX | V_EGRCNTX(qs->txq[j].cntxt_id));
602 	}
603 }
604 
605 static void init_napi(struct adapter *adap)
606 {
607 	int i;
608 
609 	for (i = 0; i < SGE_QSETS; i++) {
610 		struct sge_qset *qs = &adap->sge.qs[i];
611 
612 		if (qs->adap)
613 			netif_napi_add(qs->netdev, &qs->napi, qs->napi.poll,
614 				       64);
615 	}
616 
617 	/*
618 	 * netif_napi_add() can be called only once per napi_struct because it
619 	 * adds each new napi_struct to a list.  Be careful not to call it a
620 	 * second time, e.g., during EEH recovery, by making a note of it.
621 	 */
622 	adap->flags |= NAPI_INIT;
623 }
624 
625 /*
626  * Wait until all NAPI handlers are descheduled.  This includes the handlers of
627  * both netdevices representing interfaces and the dummy ones for the extra
628  * queues.
629  */
630 static void quiesce_rx(struct adapter *adap)
631 {
632 	int i;
633 
634 	for (i = 0; i < SGE_QSETS; i++)
635 		if (adap->sge.qs[i].adap)
636 			napi_disable(&adap->sge.qs[i].napi);
637 }
638 
639 static void enable_all_napi(struct adapter *adap)
640 {
641 	int i;
642 	for (i = 0; i < SGE_QSETS; i++)
643 		if (adap->sge.qs[i].adap)
644 			napi_enable(&adap->sge.qs[i].napi);
645 }
646 
647 /**
648  *	setup_sge_qsets - configure SGE Tx/Rx/response queues
649  *	@adap: the adapter
650  *
651  *	Determines how many sets of SGE queues to use and initializes them.
652  *	We support multiple queue sets per port if we have MSI-X, otherwise
653  *	just one queue set per port.
654  */
655 static int setup_sge_qsets(struct adapter *adap)
656 {
657 	int i, j, err, irq_idx = 0, qset_idx = 0;
658 	unsigned int ntxq = SGE_TXQ_PER_SET;
659 
660 	if (adap->params.rev > 0 && !(adap->flags & USING_MSI))
661 		irq_idx = -1;
662 
663 	for_each_port(adap, i) {
664 		struct net_device *dev = adap->port[i];
665 		struct port_info *pi = netdev_priv(dev);
666 
667 		pi->qs = &adap->sge.qs[pi->first_qset];
668 		for (j = 0; j < pi->nqsets; ++j, ++qset_idx) {
669 			err = t3_sge_alloc_qset(adap, qset_idx, 1,
670 				(adap->flags & USING_MSIX) ? qset_idx + 1 :
671 							     irq_idx,
672 				&adap->params.sge.qset[qset_idx], ntxq, dev,
673 				netdev_get_tx_queue(dev, j));
674 			if (err) {
675 				t3_free_sge_resources(adap);
676 				return err;
677 			}
678 		}
679 	}
680 
681 	return 0;
682 }
683 
684 static ssize_t attr_show(struct device *d, char *buf,
685 			 ssize_t(*format) (struct net_device *, char *))
686 {
687 	ssize_t len;
688 
689 	/* Synchronize with ioctls that may shut down the device */
690 	rtnl_lock();
691 	len = (*format) (to_net_dev(d), buf);
692 	rtnl_unlock();
693 	return len;
694 }
695 
696 static ssize_t attr_store(struct device *d,
697 			  const char *buf, size_t len,
698 			  ssize_t(*set) (struct net_device *, unsigned int),
699 			  unsigned int min_val, unsigned int max_val)
700 {
701 	ssize_t ret;
702 	unsigned int val;
703 
704 	if (!capable(CAP_NET_ADMIN))
705 		return -EPERM;
706 
707 	ret = kstrtouint(buf, 0, &val);
708 	if (ret)
709 		return ret;
710 	if (val < min_val || val > max_val)
711 		return -EINVAL;
712 
713 	rtnl_lock();
714 	ret = (*set) (to_net_dev(d), val);
715 	if (!ret)
716 		ret = len;
717 	rtnl_unlock();
718 	return ret;
719 }
720 
721 #define CXGB3_SHOW(name, val_expr) \
722 static ssize_t format_##name(struct net_device *dev, char *buf) \
723 { \
724 	struct port_info *pi = netdev_priv(dev); \
725 	struct adapter *adap = pi->adapter; \
726 	return sprintf(buf, "%u\n", val_expr); \
727 } \
728 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
729 			   char *buf) \
730 { \
731 	return attr_show(d, buf, format_##name); \
732 }
733 
734 static ssize_t set_nfilters(struct net_device *dev, unsigned int val)
735 {
736 	struct port_info *pi = netdev_priv(dev);
737 	struct adapter *adap = pi->adapter;
738 	int min_tids = is_offload(adap) ? MC5_MIN_TIDS : 0;
739 
740 	if (adap->flags & FULL_INIT_DONE)
741 		return -EBUSY;
742 	if (val && adap->params.rev == 0)
743 		return -EINVAL;
744 	if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nservers -
745 	    min_tids)
746 		return -EINVAL;
747 	adap->params.mc5.nfilters = val;
748 	return 0;
749 }
750 
751 static ssize_t store_nfilters(struct device *d, struct device_attribute *attr,
752 			      const char *buf, size_t len)
753 {
754 	return attr_store(d, buf, len, set_nfilters, 0, ~0);
755 }
756 
757 static ssize_t set_nservers(struct net_device *dev, unsigned int val)
758 {
759 	struct port_info *pi = netdev_priv(dev);
760 	struct adapter *adap = pi->adapter;
761 
762 	if (adap->flags & FULL_INIT_DONE)
763 		return -EBUSY;
764 	if (val > t3_mc5_size(&adap->mc5) - adap->params.mc5.nfilters -
765 	    MC5_MIN_TIDS)
766 		return -EINVAL;
767 	adap->params.mc5.nservers = val;
768 	return 0;
769 }
770 
771 static ssize_t store_nservers(struct device *d, struct device_attribute *attr,
772 			      const char *buf, size_t len)
773 {
774 	return attr_store(d, buf, len, set_nservers, 0, ~0);
775 }
776 
777 #define CXGB3_ATTR_R(name, val_expr) \
778 CXGB3_SHOW(name, val_expr) \
779 static DEVICE_ATTR(name, 0444, show_##name, NULL)
780 
781 #define CXGB3_ATTR_RW(name, val_expr, store_method) \
782 CXGB3_SHOW(name, val_expr) \
783 static DEVICE_ATTR(name, 0644, show_##name, store_method)
784 
785 CXGB3_ATTR_R(cam_size, t3_mc5_size(&adap->mc5));
786 CXGB3_ATTR_RW(nfilters, adap->params.mc5.nfilters, store_nfilters);
787 CXGB3_ATTR_RW(nservers, adap->params.mc5.nservers, store_nservers);
788 
789 static struct attribute *cxgb3_attrs[] = {
790 	&dev_attr_cam_size.attr,
791 	&dev_attr_nfilters.attr,
792 	&dev_attr_nservers.attr,
793 	NULL
794 };
795 
796 static const struct attribute_group cxgb3_attr_group = {
797 	.attrs = cxgb3_attrs,
798 };
799 
800 static ssize_t tm_attr_show(struct device *d,
801 			    char *buf, int sched)
802 {
803 	struct port_info *pi = netdev_priv(to_net_dev(d));
804 	struct adapter *adap = pi->adapter;
805 	unsigned int v, addr, bpt, cpt;
806 	ssize_t len;
807 
808 	addr = A_TP_TX_MOD_Q1_Q0_RATE_LIMIT - sched / 2;
809 	rtnl_lock();
810 	t3_write_reg(adap, A_TP_TM_PIO_ADDR, addr);
811 	v = t3_read_reg(adap, A_TP_TM_PIO_DATA);
812 	if (sched & 1)
813 		v >>= 16;
814 	bpt = (v >> 8) & 0xff;
815 	cpt = v & 0xff;
816 	if (!cpt)
817 		len = sprintf(buf, "disabled\n");
818 	else {
819 		v = (adap->params.vpd.cclk * 1000) / cpt;
820 		len = sprintf(buf, "%u Kbps\n", (v * bpt) / 125);
821 	}
822 	rtnl_unlock();
823 	return len;
824 }
825 
826 static ssize_t tm_attr_store(struct device *d,
827 			     const char *buf, size_t len, int sched)
828 {
829 	struct port_info *pi = netdev_priv(to_net_dev(d));
830 	struct adapter *adap = pi->adapter;
831 	unsigned int val;
832 	ssize_t ret;
833 
834 	if (!capable(CAP_NET_ADMIN))
835 		return -EPERM;
836 
837 	ret = kstrtouint(buf, 0, &val);
838 	if (ret)
839 		return ret;
840 	if (val > 10000000)
841 		return -EINVAL;
842 
843 	rtnl_lock();
844 	ret = t3_config_sched(adap, val, sched);
845 	if (!ret)
846 		ret = len;
847 	rtnl_unlock();
848 	return ret;
849 }
850 
851 #define TM_ATTR(name, sched) \
852 static ssize_t show_##name(struct device *d, struct device_attribute *attr, \
853 			   char *buf) \
854 { \
855 	return tm_attr_show(d, buf, sched); \
856 } \
857 static ssize_t store_##name(struct device *d, struct device_attribute *attr, \
858 			    const char *buf, size_t len) \
859 { \
860 	return tm_attr_store(d, buf, len, sched); \
861 } \
862 static DEVICE_ATTR(name, 0644, show_##name, store_##name)
863 
864 TM_ATTR(sched0, 0);
865 TM_ATTR(sched1, 1);
866 TM_ATTR(sched2, 2);
867 TM_ATTR(sched3, 3);
868 TM_ATTR(sched4, 4);
869 TM_ATTR(sched5, 5);
870 TM_ATTR(sched6, 6);
871 TM_ATTR(sched7, 7);
872 
873 static struct attribute *offload_attrs[] = {
874 	&dev_attr_sched0.attr,
875 	&dev_attr_sched1.attr,
876 	&dev_attr_sched2.attr,
877 	&dev_attr_sched3.attr,
878 	&dev_attr_sched4.attr,
879 	&dev_attr_sched5.attr,
880 	&dev_attr_sched6.attr,
881 	&dev_attr_sched7.attr,
882 	NULL
883 };
884 
885 static const struct attribute_group offload_attr_group = {
886 	.attrs = offload_attrs,
887 };
888 
889 /*
890  * Sends an sk_buff to an offload queue driver
891  * after dealing with any active network taps.
892  */
893 static inline int offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
894 {
895 	int ret;
896 
897 	local_bh_disable();
898 	ret = t3_offload_tx(tdev, skb);
899 	local_bh_enable();
900 	return ret;
901 }
902 
903 static int write_smt_entry(struct adapter *adapter, int idx)
904 {
905 	struct cpl_smt_write_req *req;
906 	struct port_info *pi = netdev_priv(adapter->port[idx]);
907 	struct sk_buff *skb = alloc_skb(sizeof(*req), GFP_KERNEL);
908 
909 	if (!skb)
910 		return -ENOMEM;
911 
912 	req = __skb_put(skb, sizeof(*req));
913 	req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD));
914 	OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SMT_WRITE_REQ, idx));
915 	req->mtu_idx = NMTUS - 1;	/* should be 0 but there's a T3 bug */
916 	req->iff = idx;
917 	memcpy(req->src_mac0, adapter->port[idx]->dev_addr, ETH_ALEN);
918 	memcpy(req->src_mac1, pi->iscsic.mac_addr, ETH_ALEN);
919 	skb->priority = 1;
920 	offload_tx(&adapter->tdev, skb);
921 	return 0;
922 }
923 
924 static int init_smt(struct adapter *adapter)
925 {
926 	int i;
927 
928 	for_each_port(adapter, i)
929 	    write_smt_entry(adapter, i);
930 	return 0;
931 }
932 
933 static void init_port_mtus(struct adapter *adapter)
934 {
935 	unsigned int mtus = adapter->port[0]->mtu;
936 
937 	if (adapter->port[1])
938 		mtus |= adapter->port[1]->mtu << 16;
939 	t3_write_reg(adapter, A_TP_MTU_PORT_TABLE, mtus);
940 }
941 
942 static int send_pktsched_cmd(struct adapter *adap, int sched, int qidx, int lo,
943 			      int hi, int port)
944 {
945 	struct sk_buff *skb;
946 	struct mngt_pktsched_wr *req;
947 	int ret;
948 
949 	skb = alloc_skb(sizeof(*req), GFP_KERNEL);
950 	if (!skb)
951 		skb = adap->nofail_skb;
952 	if (!skb)
953 		return -ENOMEM;
954 
955 	req = skb_put(skb, sizeof(*req));
956 	req->wr_hi = htonl(V_WR_OP(FW_WROPCODE_MNGT));
957 	req->mngt_opcode = FW_MNGTOPCODE_PKTSCHED_SET;
958 	req->sched = sched;
959 	req->idx = qidx;
960 	req->min = lo;
961 	req->max = hi;
962 	req->binding = port;
963 	ret = t3_mgmt_tx(adap, skb);
964 	if (skb == adap->nofail_skb) {
965 		adap->nofail_skb = alloc_skb(sizeof(struct cpl_set_tcb_field),
966 					     GFP_KERNEL);
967 		if (!adap->nofail_skb)
968 			ret = -ENOMEM;
969 	}
970 
971 	return ret;
972 }
973 
974 static int bind_qsets(struct adapter *adap)
975 {
976 	int i, j, err = 0;
977 
978 	for_each_port(adap, i) {
979 		const struct port_info *pi = adap2pinfo(adap, i);
980 
981 		for (j = 0; j < pi->nqsets; ++j) {
982 			int ret = send_pktsched_cmd(adap, 1,
983 						    pi->first_qset + j, -1,
984 						    -1, i);
985 			if (ret)
986 				err = ret;
987 		}
988 	}
989 
990 	return err;
991 }
992 
993 #define FW_VERSION __stringify(FW_VERSION_MAJOR) "."			\
994 	__stringify(FW_VERSION_MINOR) "." __stringify(FW_VERSION_MICRO)
995 #define FW_FNAME "cxgb3/t3fw-" FW_VERSION ".bin"
996 #define TPSRAM_VERSION __stringify(TP_VERSION_MAJOR) "."		\
997 	__stringify(TP_VERSION_MINOR) "." __stringify(TP_VERSION_MICRO)
998 #define TPSRAM_NAME "cxgb3/t3%c_psram-" TPSRAM_VERSION ".bin"
999 #define AEL2005_OPT_EDC_NAME "cxgb3/ael2005_opt_edc.bin"
1000 #define AEL2005_TWX_EDC_NAME "cxgb3/ael2005_twx_edc.bin"
1001 #define AEL2020_TWX_EDC_NAME "cxgb3/ael2020_twx_edc.bin"
1002 MODULE_FIRMWARE(FW_FNAME);
1003 MODULE_FIRMWARE("cxgb3/t3b_psram-" TPSRAM_VERSION ".bin");
1004 MODULE_FIRMWARE("cxgb3/t3c_psram-" TPSRAM_VERSION ".bin");
1005 MODULE_FIRMWARE(AEL2005_OPT_EDC_NAME);
1006 MODULE_FIRMWARE(AEL2005_TWX_EDC_NAME);
1007 MODULE_FIRMWARE(AEL2020_TWX_EDC_NAME);
1008 
1009 static inline const char *get_edc_fw_name(int edc_idx)
1010 {
1011 	const char *fw_name = NULL;
1012 
1013 	switch (edc_idx) {
1014 	case EDC_OPT_AEL2005:
1015 		fw_name = AEL2005_OPT_EDC_NAME;
1016 		break;
1017 	case EDC_TWX_AEL2005:
1018 		fw_name = AEL2005_TWX_EDC_NAME;
1019 		break;
1020 	case EDC_TWX_AEL2020:
1021 		fw_name = AEL2020_TWX_EDC_NAME;
1022 		break;
1023 	}
1024 	return fw_name;
1025 }
1026 
1027 int t3_get_edc_fw(struct cphy *phy, int edc_idx, int size)
1028 {
1029 	struct adapter *adapter = phy->adapter;
1030 	const struct firmware *fw;
1031 	const char *fw_name;
1032 	u32 csum;
1033 	const __be32 *p;
1034 	u16 *cache = phy->phy_cache;
1035 	int i, ret = -EINVAL;
1036 
1037 	fw_name = get_edc_fw_name(edc_idx);
1038 	if (fw_name)
1039 		ret = request_firmware(&fw, fw_name, &adapter->pdev->dev);
1040 	if (ret < 0) {
1041 		dev_err(&adapter->pdev->dev,
1042 			"could not upgrade firmware: unable to load %s\n",
1043 			fw_name);
1044 		return ret;
1045 	}
1046 
1047 	/* check size, take checksum in account */
1048 	if (fw->size > size + 4) {
1049 		CH_ERR(adapter, "firmware image too large %u, expected %d\n",
1050 		       (unsigned int)fw->size, size + 4);
1051 		ret = -EINVAL;
1052 	}
1053 
1054 	/* compute checksum */
1055 	p = (const __be32 *)fw->data;
1056 	for (csum = 0, i = 0; i < fw->size / sizeof(csum); i++)
1057 		csum += ntohl(p[i]);
1058 
1059 	if (csum != 0xffffffff) {
1060 		CH_ERR(adapter, "corrupted firmware image, checksum %u\n",
1061 		       csum);
1062 		ret = -EINVAL;
1063 	}
1064 
1065 	for (i = 0; i < size / 4 ; i++) {
1066 		*cache++ = (be32_to_cpu(p[i]) & 0xffff0000) >> 16;
1067 		*cache++ = be32_to_cpu(p[i]) & 0xffff;
1068 	}
1069 
1070 	release_firmware(fw);
1071 
1072 	return ret;
1073 }
1074 
1075 static int upgrade_fw(struct adapter *adap)
1076 {
1077 	int ret;
1078 	const struct firmware *fw;
1079 	struct device *dev = &adap->pdev->dev;
1080 
1081 	ret = request_firmware(&fw, FW_FNAME, dev);
1082 	if (ret < 0) {
1083 		dev_err(dev, "could not upgrade firmware: unable to load %s\n",
1084 			FW_FNAME);
1085 		return ret;
1086 	}
1087 	ret = t3_load_fw(adap, fw->data, fw->size);
1088 	release_firmware(fw);
1089 
1090 	if (ret == 0)
1091 		dev_info(dev, "successful upgrade to firmware %d.%d.%d\n",
1092 			 FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1093 	else
1094 		dev_err(dev, "failed to upgrade to firmware %d.%d.%d\n",
1095 			FW_VERSION_MAJOR, FW_VERSION_MINOR, FW_VERSION_MICRO);
1096 
1097 	return ret;
1098 }
1099 
1100 static inline char t3rev2char(struct adapter *adapter)
1101 {
1102 	char rev = 0;
1103 
1104 	switch(adapter->params.rev) {
1105 	case T3_REV_B:
1106 	case T3_REV_B2:
1107 		rev = 'b';
1108 		break;
1109 	case T3_REV_C:
1110 		rev = 'c';
1111 		break;
1112 	}
1113 	return rev;
1114 }
1115 
1116 static int update_tpsram(struct adapter *adap)
1117 {
1118 	const struct firmware *tpsram;
1119 	char buf[64];
1120 	struct device *dev = &adap->pdev->dev;
1121 	int ret;
1122 	char rev;
1123 
1124 	rev = t3rev2char(adap);
1125 	if (!rev)
1126 		return 0;
1127 
1128 	snprintf(buf, sizeof(buf), TPSRAM_NAME, rev);
1129 
1130 	ret = request_firmware(&tpsram, buf, dev);
1131 	if (ret < 0) {
1132 		dev_err(dev, "could not load TP SRAM: unable to load %s\n",
1133 			buf);
1134 		return ret;
1135 	}
1136 
1137 	ret = t3_check_tpsram(adap, tpsram->data, tpsram->size);
1138 	if (ret)
1139 		goto release_tpsram;
1140 
1141 	ret = t3_set_proto_sram(adap, tpsram->data);
1142 	if (ret == 0)
1143 		dev_info(dev,
1144 			 "successful update of protocol engine "
1145 			 "to %d.%d.%d\n",
1146 			 TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1147 	else
1148 		dev_err(dev, "failed to update of protocol engine %d.%d.%d\n",
1149 			TP_VERSION_MAJOR, TP_VERSION_MINOR, TP_VERSION_MICRO);
1150 	if (ret)
1151 		dev_err(dev, "loading protocol SRAM failed\n");
1152 
1153 release_tpsram:
1154 	release_firmware(tpsram);
1155 
1156 	return ret;
1157 }
1158 
1159 /**
1160  * t3_synchronize_rx - wait for current Rx processing on a port to complete
1161  * @adap: the adapter
1162  * @p: the port
1163  *
1164  * Ensures that current Rx processing on any of the queues associated with
1165  * the given port completes before returning.  We do this by acquiring and
1166  * releasing the locks of the response queues associated with the port.
1167  */
1168 static void t3_synchronize_rx(struct adapter *adap, const struct port_info *p)
1169 {
1170 	int i;
1171 
1172 	for (i = p->first_qset; i < p->first_qset + p->nqsets; i++) {
1173 		struct sge_rspq *q = &adap->sge.qs[i].rspq;
1174 
1175 		spin_lock_irq(&q->lock);
1176 		spin_unlock_irq(&q->lock);
1177 	}
1178 }
1179 
1180 static void cxgb_vlan_mode(struct net_device *dev, netdev_features_t features)
1181 {
1182 	struct port_info *pi = netdev_priv(dev);
1183 	struct adapter *adapter = pi->adapter;
1184 
1185 	if (adapter->params.rev > 0) {
1186 		t3_set_vlan_accel(adapter, 1 << pi->port_id,
1187 				  features & NETIF_F_HW_VLAN_CTAG_RX);
1188 	} else {
1189 		/* single control for all ports */
1190 		unsigned int i, have_vlans = features & NETIF_F_HW_VLAN_CTAG_RX;
1191 
1192 		for_each_port(adapter, i)
1193 			have_vlans |=
1194 				adapter->port[i]->features &
1195 				NETIF_F_HW_VLAN_CTAG_RX;
1196 
1197 		t3_set_vlan_accel(adapter, 1, have_vlans);
1198 	}
1199 	t3_synchronize_rx(adapter, pi);
1200 }
1201 
1202 /**
1203  *	cxgb_up - enable the adapter
1204  *	@adapter: adapter being enabled
1205  *
1206  *	Called when the first port is enabled, this function performs the
1207  *	actions necessary to make an adapter operational, such as completing
1208  *	the initialization of HW modules, and enabling interrupts.
1209  *
1210  *	Must be called with the rtnl lock held.
1211  */
1212 static int cxgb_up(struct adapter *adap)
1213 {
1214 	int i, err;
1215 
1216 	if (!(adap->flags & FULL_INIT_DONE)) {
1217 		err = t3_check_fw_version(adap);
1218 		if (err == -EINVAL) {
1219 			err = upgrade_fw(adap);
1220 			CH_WARN(adap, "FW upgrade to %d.%d.%d %s\n",
1221 				FW_VERSION_MAJOR, FW_VERSION_MINOR,
1222 				FW_VERSION_MICRO, err ? "failed" : "succeeded");
1223 		}
1224 
1225 		err = t3_check_tpsram_version(adap);
1226 		if (err == -EINVAL) {
1227 			err = update_tpsram(adap);
1228 			CH_WARN(adap, "TP upgrade to %d.%d.%d %s\n",
1229 				TP_VERSION_MAJOR, TP_VERSION_MINOR,
1230 				TP_VERSION_MICRO, err ? "failed" : "succeeded");
1231 		}
1232 
1233 		/*
1234 		 * Clear interrupts now to catch errors if t3_init_hw fails.
1235 		 * We clear them again later as initialization may trigger
1236 		 * conditions that can interrupt.
1237 		 */
1238 		t3_intr_clear(adap);
1239 
1240 		err = t3_init_hw(adap, 0);
1241 		if (err)
1242 			goto out;
1243 
1244 		t3_set_reg_field(adap, A_TP_PARA_REG5, 0, F_RXDDPOFFINIT);
1245 		t3_write_reg(adap, A_ULPRX_TDDP_PSZ, V_HPZ0(PAGE_SHIFT - 12));
1246 
1247 		err = setup_sge_qsets(adap);
1248 		if (err)
1249 			goto out;
1250 
1251 		for_each_port(adap, i)
1252 			cxgb_vlan_mode(adap->port[i], adap->port[i]->features);
1253 
1254 		setup_rss(adap);
1255 		if (!(adap->flags & NAPI_INIT))
1256 			init_napi(adap);
1257 
1258 		t3_start_sge_timers(adap);
1259 		adap->flags |= FULL_INIT_DONE;
1260 	}
1261 
1262 	t3_intr_clear(adap);
1263 
1264 	if (adap->flags & USING_MSIX) {
1265 		name_msix_vecs(adap);
1266 		err = request_irq(adap->msix_info[0].vec,
1267 				  t3_async_intr_handler, 0,
1268 				  adap->msix_info[0].desc, adap);
1269 		if (err)
1270 			goto irq_err;
1271 
1272 		err = request_msix_data_irqs(adap);
1273 		if (err) {
1274 			free_irq(adap->msix_info[0].vec, adap);
1275 			goto irq_err;
1276 		}
1277 	} else if ((err = request_irq(adap->pdev->irq,
1278 				      t3_intr_handler(adap,
1279 						      adap->sge.qs[0].rspq.
1280 						      polling),
1281 				      (adap->flags & USING_MSI) ?
1282 				       0 : IRQF_SHARED,
1283 				      adap->name, adap)))
1284 		goto irq_err;
1285 
1286 	enable_all_napi(adap);
1287 	t3_sge_start(adap);
1288 	t3_intr_enable(adap);
1289 
1290 	if (adap->params.rev >= T3_REV_C && !(adap->flags & TP_PARITY_INIT) &&
1291 	    is_offload(adap) && init_tp_parity(adap) == 0)
1292 		adap->flags |= TP_PARITY_INIT;
1293 
1294 	if (adap->flags & TP_PARITY_INIT) {
1295 		t3_write_reg(adap, A_TP_INT_CAUSE,
1296 			     F_CMCACHEPERR | F_ARPLUTPERR);
1297 		t3_write_reg(adap, A_TP_INT_ENABLE, 0x7fbfffff);
1298 	}
1299 
1300 	if (!(adap->flags & QUEUES_BOUND)) {
1301 		int ret = bind_qsets(adap);
1302 
1303 		if (ret < 0) {
1304 			CH_ERR(adap, "failed to bind qsets, err %d\n", ret);
1305 			t3_intr_disable(adap);
1306 			free_irq_resources(adap);
1307 			err = ret;
1308 			goto out;
1309 		}
1310 		adap->flags |= QUEUES_BOUND;
1311 	}
1312 
1313 out:
1314 	return err;
1315 irq_err:
1316 	CH_ERR(adap, "request_irq failed, err %d\n", err);
1317 	goto out;
1318 }
1319 
1320 /*
1321  * Release resources when all the ports and offloading have been stopped.
1322  */
1323 static void cxgb_down(struct adapter *adapter, int on_wq)
1324 {
1325 	t3_sge_stop(adapter);
1326 	spin_lock_irq(&adapter->work_lock);	/* sync with PHY intr task */
1327 	t3_intr_disable(adapter);
1328 	spin_unlock_irq(&adapter->work_lock);
1329 
1330 	free_irq_resources(adapter);
1331 	quiesce_rx(adapter);
1332 	t3_sge_stop(adapter);
1333 	if (!on_wq)
1334 		flush_workqueue(cxgb3_wq);/* wait for external IRQ handler */
1335 }
1336 
1337 static void schedule_chk_task(struct adapter *adap)
1338 {
1339 	unsigned int timeo;
1340 
1341 	timeo = adap->params.linkpoll_period ?
1342 	    (HZ * adap->params.linkpoll_period) / 10 :
1343 	    adap->params.stats_update_period * HZ;
1344 	if (timeo)
1345 		queue_delayed_work(cxgb3_wq, &adap->adap_check_task, timeo);
1346 }
1347 
1348 static int offload_open(struct net_device *dev)
1349 {
1350 	struct port_info *pi = netdev_priv(dev);
1351 	struct adapter *adapter = pi->adapter;
1352 	struct t3cdev *tdev = dev2t3cdev(dev);
1353 	int adap_up = adapter->open_device_map & PORT_MASK;
1354 	int err;
1355 
1356 	if (test_and_set_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1357 		return 0;
1358 
1359 	if (!adap_up && (err = cxgb_up(adapter)) < 0)
1360 		goto out;
1361 
1362 	t3_tp_set_offload_mode(adapter, 1);
1363 	tdev->lldev = adapter->port[0];
1364 	err = cxgb3_offload_activate(adapter);
1365 	if (err)
1366 		goto out;
1367 
1368 	init_port_mtus(adapter);
1369 	t3_load_mtus(adapter, adapter->params.mtus, adapter->params.a_wnd,
1370 		     adapter->params.b_wnd,
1371 		     adapter->params.rev == 0 ?
1372 		     adapter->port[0]->mtu : 0xffff);
1373 	init_smt(adapter);
1374 
1375 	if (sysfs_create_group(&tdev->lldev->dev.kobj, &offload_attr_group))
1376 		dev_dbg(&dev->dev, "cannot create sysfs group\n");
1377 
1378 	/* Call back all registered clients */
1379 	cxgb3_add_clients(tdev);
1380 
1381 out:
1382 	/* restore them in case the offload module has changed them */
1383 	if (err) {
1384 		t3_tp_set_offload_mode(adapter, 0);
1385 		clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1386 		cxgb3_set_dummy_ops(tdev);
1387 	}
1388 	return err;
1389 }
1390 
1391 static int offload_close(struct t3cdev *tdev)
1392 {
1393 	struct adapter *adapter = tdev2adap(tdev);
1394 	struct t3c_data *td = T3C_DATA(tdev);
1395 
1396 	if (!test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map))
1397 		return 0;
1398 
1399 	/* Call back all registered clients */
1400 	cxgb3_remove_clients(tdev);
1401 
1402 	sysfs_remove_group(&tdev->lldev->dev.kobj, &offload_attr_group);
1403 
1404 	/* Flush work scheduled while releasing TIDs */
1405 	flush_work(&td->tid_release_task);
1406 
1407 	tdev->lldev = NULL;
1408 	cxgb3_set_dummy_ops(tdev);
1409 	t3_tp_set_offload_mode(adapter, 0);
1410 	clear_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map);
1411 
1412 	if (!adapter->open_device_map)
1413 		cxgb_down(adapter, 0);
1414 
1415 	cxgb3_offload_deactivate(adapter);
1416 	return 0;
1417 }
1418 
1419 static int cxgb_open(struct net_device *dev)
1420 {
1421 	struct port_info *pi = netdev_priv(dev);
1422 	struct adapter *adapter = pi->adapter;
1423 	int other_ports = adapter->open_device_map & PORT_MASK;
1424 	int err;
1425 
1426 	if (!adapter->open_device_map && (err = cxgb_up(adapter)) < 0)
1427 		return err;
1428 
1429 	set_bit(pi->port_id, &adapter->open_device_map);
1430 	if (is_offload(adapter) && !ofld_disable) {
1431 		err = offload_open(dev);
1432 		if (err)
1433 			pr_warn("Could not initialize offload capabilities\n");
1434 	}
1435 
1436 	netif_set_real_num_tx_queues(dev, pi->nqsets);
1437 	err = netif_set_real_num_rx_queues(dev, pi->nqsets);
1438 	if (err)
1439 		return err;
1440 	link_start(dev);
1441 	t3_port_intr_enable(adapter, pi->port_id);
1442 	netif_tx_start_all_queues(dev);
1443 	if (!other_ports)
1444 		schedule_chk_task(adapter);
1445 
1446 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_UP, pi->port_id);
1447 	return 0;
1448 }
1449 
1450 static int __cxgb_close(struct net_device *dev, int on_wq)
1451 {
1452 	struct port_info *pi = netdev_priv(dev);
1453 	struct adapter *adapter = pi->adapter;
1454 
1455 
1456 	if (!adapter->open_device_map)
1457 		return 0;
1458 
1459 	/* Stop link fault interrupts */
1460 	t3_xgm_intr_disable(adapter, pi->port_id);
1461 	t3_read_reg(adapter, A_XGM_INT_STATUS + pi->mac.offset);
1462 
1463 	t3_port_intr_disable(adapter, pi->port_id);
1464 	netif_tx_stop_all_queues(dev);
1465 	pi->phy.ops->power_down(&pi->phy, 1);
1466 	netif_carrier_off(dev);
1467 	t3_mac_disable(&pi->mac, MAC_DIRECTION_TX | MAC_DIRECTION_RX);
1468 
1469 	spin_lock_irq(&adapter->work_lock);	/* sync with update task */
1470 	clear_bit(pi->port_id, &adapter->open_device_map);
1471 	spin_unlock_irq(&adapter->work_lock);
1472 
1473 	if (!(adapter->open_device_map & PORT_MASK))
1474 		cancel_delayed_work_sync(&adapter->adap_check_task);
1475 
1476 	if (!adapter->open_device_map)
1477 		cxgb_down(adapter, on_wq);
1478 
1479 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_PORT_DOWN, pi->port_id);
1480 	return 0;
1481 }
1482 
1483 static int cxgb_close(struct net_device *dev)
1484 {
1485 	return __cxgb_close(dev, 0);
1486 }
1487 
1488 static struct net_device_stats *cxgb_get_stats(struct net_device *dev)
1489 {
1490 	struct port_info *pi = netdev_priv(dev);
1491 	struct adapter *adapter = pi->adapter;
1492 	struct net_device_stats *ns = &dev->stats;
1493 	const struct mac_stats *pstats;
1494 
1495 	spin_lock(&adapter->stats_lock);
1496 	pstats = t3_mac_update_stats(&pi->mac);
1497 	spin_unlock(&adapter->stats_lock);
1498 
1499 	ns->tx_bytes = pstats->tx_octets;
1500 	ns->tx_packets = pstats->tx_frames;
1501 	ns->rx_bytes = pstats->rx_octets;
1502 	ns->rx_packets = pstats->rx_frames;
1503 	ns->multicast = pstats->rx_mcast_frames;
1504 
1505 	ns->tx_errors = pstats->tx_underrun;
1506 	ns->rx_errors = pstats->rx_symbol_errs + pstats->rx_fcs_errs +
1507 	    pstats->rx_too_long + pstats->rx_jabber + pstats->rx_short +
1508 	    pstats->rx_fifo_ovfl;
1509 
1510 	/* detailed rx_errors */
1511 	ns->rx_length_errors = pstats->rx_jabber + pstats->rx_too_long;
1512 	ns->rx_over_errors = 0;
1513 	ns->rx_crc_errors = pstats->rx_fcs_errs;
1514 	ns->rx_frame_errors = pstats->rx_symbol_errs;
1515 	ns->rx_fifo_errors = pstats->rx_fifo_ovfl;
1516 	ns->rx_missed_errors = pstats->rx_cong_drops;
1517 
1518 	/* detailed tx_errors */
1519 	ns->tx_aborted_errors = 0;
1520 	ns->tx_carrier_errors = 0;
1521 	ns->tx_fifo_errors = pstats->tx_underrun;
1522 	ns->tx_heartbeat_errors = 0;
1523 	ns->tx_window_errors = 0;
1524 	return ns;
1525 }
1526 
1527 static u32 get_msglevel(struct net_device *dev)
1528 {
1529 	struct port_info *pi = netdev_priv(dev);
1530 	struct adapter *adapter = pi->adapter;
1531 
1532 	return adapter->msg_enable;
1533 }
1534 
1535 static void set_msglevel(struct net_device *dev, u32 val)
1536 {
1537 	struct port_info *pi = netdev_priv(dev);
1538 	struct adapter *adapter = pi->adapter;
1539 
1540 	adapter->msg_enable = val;
1541 }
1542 
1543 static const char stats_strings[][ETH_GSTRING_LEN] = {
1544 	"TxOctetsOK         ",
1545 	"TxFramesOK         ",
1546 	"TxMulticastFramesOK",
1547 	"TxBroadcastFramesOK",
1548 	"TxPauseFrames      ",
1549 	"TxUnderrun         ",
1550 	"TxExtUnderrun      ",
1551 
1552 	"TxFrames64         ",
1553 	"TxFrames65To127    ",
1554 	"TxFrames128To255   ",
1555 	"TxFrames256To511   ",
1556 	"TxFrames512To1023  ",
1557 	"TxFrames1024To1518 ",
1558 	"TxFrames1519ToMax  ",
1559 
1560 	"RxOctetsOK         ",
1561 	"RxFramesOK         ",
1562 	"RxMulticastFramesOK",
1563 	"RxBroadcastFramesOK",
1564 	"RxPauseFrames      ",
1565 	"RxFCSErrors        ",
1566 	"RxSymbolErrors     ",
1567 	"RxShortErrors      ",
1568 	"RxJabberErrors     ",
1569 	"RxLengthErrors     ",
1570 	"RxFIFOoverflow     ",
1571 
1572 	"RxFrames64         ",
1573 	"RxFrames65To127    ",
1574 	"RxFrames128To255   ",
1575 	"RxFrames256To511   ",
1576 	"RxFrames512To1023  ",
1577 	"RxFrames1024To1518 ",
1578 	"RxFrames1519ToMax  ",
1579 
1580 	"PhyFIFOErrors      ",
1581 	"TSO                ",
1582 	"VLANextractions    ",
1583 	"VLANinsertions     ",
1584 	"TxCsumOffload      ",
1585 	"RxCsumGood         ",
1586 	"LroAggregated      ",
1587 	"LroFlushed         ",
1588 	"LroNoDesc          ",
1589 	"RxDrops            ",
1590 
1591 	"CheckTXEnToggled   ",
1592 	"CheckResets        ",
1593 
1594 	"LinkFaults         ",
1595 };
1596 
1597 static int get_sset_count(struct net_device *dev, int sset)
1598 {
1599 	switch (sset) {
1600 	case ETH_SS_STATS:
1601 		return ARRAY_SIZE(stats_strings);
1602 	default:
1603 		return -EOPNOTSUPP;
1604 	}
1605 }
1606 
1607 #define T3_REGMAP_SIZE (3 * 1024)
1608 
1609 static int get_regs_len(struct net_device *dev)
1610 {
1611 	return T3_REGMAP_SIZE;
1612 }
1613 
1614 static int get_eeprom_len(struct net_device *dev)
1615 {
1616 	return EEPROMSIZE;
1617 }
1618 
1619 static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1620 {
1621 	struct port_info *pi = netdev_priv(dev);
1622 	struct adapter *adapter = pi->adapter;
1623 	u32 fw_vers = 0;
1624 	u32 tp_vers = 0;
1625 
1626 	spin_lock(&adapter->stats_lock);
1627 	t3_get_fw_version(adapter, &fw_vers);
1628 	t3_get_tp_version(adapter, &tp_vers);
1629 	spin_unlock(&adapter->stats_lock);
1630 
1631 	strlcpy(info->driver, DRV_NAME, sizeof(info->driver));
1632 	strlcpy(info->version, DRV_VERSION, sizeof(info->version));
1633 	strlcpy(info->bus_info, pci_name(adapter->pdev),
1634 		sizeof(info->bus_info));
1635 	if (fw_vers)
1636 		snprintf(info->fw_version, sizeof(info->fw_version),
1637 			 "%s %u.%u.%u TP %u.%u.%u",
1638 			 G_FW_VERSION_TYPE(fw_vers) ? "T" : "N",
1639 			 G_FW_VERSION_MAJOR(fw_vers),
1640 			 G_FW_VERSION_MINOR(fw_vers),
1641 			 G_FW_VERSION_MICRO(fw_vers),
1642 			 G_TP_VERSION_MAJOR(tp_vers),
1643 			 G_TP_VERSION_MINOR(tp_vers),
1644 			 G_TP_VERSION_MICRO(tp_vers));
1645 }
1646 
1647 static void get_strings(struct net_device *dev, u32 stringset, u8 * data)
1648 {
1649 	if (stringset == ETH_SS_STATS)
1650 		memcpy(data, stats_strings, sizeof(stats_strings));
1651 }
1652 
1653 static unsigned long collect_sge_port_stats(struct adapter *adapter,
1654 					    struct port_info *p, int idx)
1655 {
1656 	int i;
1657 	unsigned long tot = 0;
1658 
1659 	for (i = p->first_qset; i < p->first_qset + p->nqsets; ++i)
1660 		tot += adapter->sge.qs[i].port_stats[idx];
1661 	return tot;
1662 }
1663 
1664 static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1665 		      u64 *data)
1666 {
1667 	struct port_info *pi = netdev_priv(dev);
1668 	struct adapter *adapter = pi->adapter;
1669 	const struct mac_stats *s;
1670 
1671 	spin_lock(&adapter->stats_lock);
1672 	s = t3_mac_update_stats(&pi->mac);
1673 	spin_unlock(&adapter->stats_lock);
1674 
1675 	*data++ = s->tx_octets;
1676 	*data++ = s->tx_frames;
1677 	*data++ = s->tx_mcast_frames;
1678 	*data++ = s->tx_bcast_frames;
1679 	*data++ = s->tx_pause;
1680 	*data++ = s->tx_underrun;
1681 	*data++ = s->tx_fifo_urun;
1682 
1683 	*data++ = s->tx_frames_64;
1684 	*data++ = s->tx_frames_65_127;
1685 	*data++ = s->tx_frames_128_255;
1686 	*data++ = s->tx_frames_256_511;
1687 	*data++ = s->tx_frames_512_1023;
1688 	*data++ = s->tx_frames_1024_1518;
1689 	*data++ = s->tx_frames_1519_max;
1690 
1691 	*data++ = s->rx_octets;
1692 	*data++ = s->rx_frames;
1693 	*data++ = s->rx_mcast_frames;
1694 	*data++ = s->rx_bcast_frames;
1695 	*data++ = s->rx_pause;
1696 	*data++ = s->rx_fcs_errs;
1697 	*data++ = s->rx_symbol_errs;
1698 	*data++ = s->rx_short;
1699 	*data++ = s->rx_jabber;
1700 	*data++ = s->rx_too_long;
1701 	*data++ = s->rx_fifo_ovfl;
1702 
1703 	*data++ = s->rx_frames_64;
1704 	*data++ = s->rx_frames_65_127;
1705 	*data++ = s->rx_frames_128_255;
1706 	*data++ = s->rx_frames_256_511;
1707 	*data++ = s->rx_frames_512_1023;
1708 	*data++ = s->rx_frames_1024_1518;
1709 	*data++ = s->rx_frames_1519_max;
1710 
1711 	*data++ = pi->phy.fifo_errors;
1712 
1713 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TSO);
1714 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANEX);
1715 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_VLANINS);
1716 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_TX_CSUM);
1717 	*data++ = collect_sge_port_stats(adapter, pi, SGE_PSTAT_RX_CSUM_GOOD);
1718 	*data++ = 0;
1719 	*data++ = 0;
1720 	*data++ = 0;
1721 	*data++ = s->rx_cong_drops;
1722 
1723 	*data++ = s->num_toggled;
1724 	*data++ = s->num_resets;
1725 
1726 	*data++ = s->link_faults;
1727 }
1728 
1729 static inline void reg_block_dump(struct adapter *ap, void *buf,
1730 				  unsigned int start, unsigned int end)
1731 {
1732 	u32 *p = buf + start;
1733 
1734 	for (; start <= end; start += sizeof(u32))
1735 		*p++ = t3_read_reg(ap, start);
1736 }
1737 
1738 static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1739 		     void *buf)
1740 {
1741 	struct port_info *pi = netdev_priv(dev);
1742 	struct adapter *ap = pi->adapter;
1743 
1744 	/*
1745 	 * Version scheme:
1746 	 * bits 0..9: chip version
1747 	 * bits 10..15: chip revision
1748 	 * bit 31: set for PCIe cards
1749 	 */
1750 	regs->version = 3 | (ap->params.rev << 10) | (is_pcie(ap) << 31);
1751 
1752 	/*
1753 	 * We skip the MAC statistics registers because they are clear-on-read.
1754 	 * Also reading multi-register stats would need to synchronize with the
1755 	 * periodic mac stats accumulation.  Hard to justify the complexity.
1756 	 */
1757 	memset(buf, 0, T3_REGMAP_SIZE);
1758 	reg_block_dump(ap, buf, 0, A_SG_RSPQ_CREDIT_RETURN);
1759 	reg_block_dump(ap, buf, A_SG_HI_DRB_HI_THRSH, A_ULPRX_PBL_ULIMIT);
1760 	reg_block_dump(ap, buf, A_ULPTX_CONFIG, A_MPS_INT_CAUSE);
1761 	reg_block_dump(ap, buf, A_CPL_SWITCH_CNTRL, A_CPL_MAP_TBL_DATA);
1762 	reg_block_dump(ap, buf, A_SMB_GLOBAL_TIME_CFG, A_XGM_SERDES_STAT3);
1763 	reg_block_dump(ap, buf, A_XGM_SERDES_STATUS0,
1764 		       XGM_REG(A_XGM_SERDES_STAT3, 1));
1765 	reg_block_dump(ap, buf, XGM_REG(A_XGM_SERDES_STATUS0, 1),
1766 		       XGM_REG(A_XGM_RX_SPI4_SOP_EOP_CNT, 1));
1767 }
1768 
1769 static int restart_autoneg(struct net_device *dev)
1770 {
1771 	struct port_info *p = netdev_priv(dev);
1772 
1773 	if (!netif_running(dev))
1774 		return -EAGAIN;
1775 	if (p->link_config.autoneg != AUTONEG_ENABLE)
1776 		return -EINVAL;
1777 	p->phy.ops->autoneg_restart(&p->phy);
1778 	return 0;
1779 }
1780 
1781 static int set_phys_id(struct net_device *dev,
1782 		       enum ethtool_phys_id_state state)
1783 {
1784 	struct port_info *pi = netdev_priv(dev);
1785 	struct adapter *adapter = pi->adapter;
1786 
1787 	switch (state) {
1788 	case ETHTOOL_ID_ACTIVE:
1789 		return 1;	/* cycle on/off once per second */
1790 
1791 	case ETHTOOL_ID_OFF:
1792 		t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
1793 		break;
1794 
1795 	case ETHTOOL_ID_ON:
1796 	case ETHTOOL_ID_INACTIVE:
1797 		t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL,
1798 			 F_GPIO0_OUT_VAL);
1799 	}
1800 
1801 	return 0;
1802 }
1803 
1804 static int get_link_ksettings(struct net_device *dev,
1805 			      struct ethtool_link_ksettings *cmd)
1806 {
1807 	struct port_info *p = netdev_priv(dev);
1808 	u32 supported;
1809 
1810 	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported,
1811 						p->link_config.supported);
1812 	ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising,
1813 						p->link_config.advertising);
1814 
1815 	if (netif_carrier_ok(dev)) {
1816 		cmd->base.speed = p->link_config.speed;
1817 		cmd->base.duplex = p->link_config.duplex;
1818 	} else {
1819 		cmd->base.speed = SPEED_UNKNOWN;
1820 		cmd->base.duplex = DUPLEX_UNKNOWN;
1821 	}
1822 
1823 	ethtool_convert_link_mode_to_legacy_u32(&supported,
1824 						cmd->link_modes.supported);
1825 
1826 	cmd->base.port = (supported & SUPPORTED_TP) ? PORT_TP : PORT_FIBRE;
1827 	cmd->base.phy_address = p->phy.mdio.prtad;
1828 	cmd->base.autoneg = p->link_config.autoneg;
1829 	return 0;
1830 }
1831 
1832 static int speed_duplex_to_caps(int speed, int duplex)
1833 {
1834 	int cap = 0;
1835 
1836 	switch (speed) {
1837 	case SPEED_10:
1838 		if (duplex == DUPLEX_FULL)
1839 			cap = SUPPORTED_10baseT_Full;
1840 		else
1841 			cap = SUPPORTED_10baseT_Half;
1842 		break;
1843 	case SPEED_100:
1844 		if (duplex == DUPLEX_FULL)
1845 			cap = SUPPORTED_100baseT_Full;
1846 		else
1847 			cap = SUPPORTED_100baseT_Half;
1848 		break;
1849 	case SPEED_1000:
1850 		if (duplex == DUPLEX_FULL)
1851 			cap = SUPPORTED_1000baseT_Full;
1852 		else
1853 			cap = SUPPORTED_1000baseT_Half;
1854 		break;
1855 	case SPEED_10000:
1856 		if (duplex == DUPLEX_FULL)
1857 			cap = SUPPORTED_10000baseT_Full;
1858 	}
1859 	return cap;
1860 }
1861 
1862 #define ADVERTISED_MASK (ADVERTISED_10baseT_Half | ADVERTISED_10baseT_Full | \
1863 		      ADVERTISED_100baseT_Half | ADVERTISED_100baseT_Full | \
1864 		      ADVERTISED_1000baseT_Half | ADVERTISED_1000baseT_Full | \
1865 		      ADVERTISED_10000baseT_Full)
1866 
1867 static int set_link_ksettings(struct net_device *dev,
1868 			      const struct ethtool_link_ksettings *cmd)
1869 {
1870 	struct port_info *p = netdev_priv(dev);
1871 	struct link_config *lc = &p->link_config;
1872 	u32 advertising;
1873 
1874 	ethtool_convert_link_mode_to_legacy_u32(&advertising,
1875 						cmd->link_modes.advertising);
1876 
1877 	if (!(lc->supported & SUPPORTED_Autoneg)) {
1878 		/*
1879 		 * PHY offers a single speed/duplex.  See if that's what's
1880 		 * being requested.
1881 		 */
1882 		if (cmd->base.autoneg == AUTONEG_DISABLE) {
1883 			u32 speed = cmd->base.speed;
1884 			int cap = speed_duplex_to_caps(speed, cmd->base.duplex);
1885 			if (lc->supported & cap)
1886 				return 0;
1887 		}
1888 		return -EINVAL;
1889 	}
1890 
1891 	if (cmd->base.autoneg == AUTONEG_DISABLE) {
1892 		u32 speed = cmd->base.speed;
1893 		int cap = speed_duplex_to_caps(speed, cmd->base.duplex);
1894 
1895 		if (!(lc->supported & cap) || (speed == SPEED_1000))
1896 			return -EINVAL;
1897 		lc->requested_speed = speed;
1898 		lc->requested_duplex = cmd->base.duplex;
1899 		lc->advertising = 0;
1900 	} else {
1901 		advertising &= ADVERTISED_MASK;
1902 		advertising &= lc->supported;
1903 		if (!advertising)
1904 			return -EINVAL;
1905 		lc->requested_speed = SPEED_INVALID;
1906 		lc->requested_duplex = DUPLEX_INVALID;
1907 		lc->advertising = advertising | ADVERTISED_Autoneg;
1908 	}
1909 	lc->autoneg = cmd->base.autoneg;
1910 	if (netif_running(dev))
1911 		t3_link_start(&p->phy, &p->mac, lc);
1912 	return 0;
1913 }
1914 
1915 static void get_pauseparam(struct net_device *dev,
1916 			   struct ethtool_pauseparam *epause)
1917 {
1918 	struct port_info *p = netdev_priv(dev);
1919 
1920 	epause->autoneg = (p->link_config.requested_fc & PAUSE_AUTONEG) != 0;
1921 	epause->rx_pause = (p->link_config.fc & PAUSE_RX) != 0;
1922 	epause->tx_pause = (p->link_config.fc & PAUSE_TX) != 0;
1923 }
1924 
1925 static int set_pauseparam(struct net_device *dev,
1926 			  struct ethtool_pauseparam *epause)
1927 {
1928 	struct port_info *p = netdev_priv(dev);
1929 	struct link_config *lc = &p->link_config;
1930 
1931 	if (epause->autoneg == AUTONEG_DISABLE)
1932 		lc->requested_fc = 0;
1933 	else if (lc->supported & SUPPORTED_Autoneg)
1934 		lc->requested_fc = PAUSE_AUTONEG;
1935 	else
1936 		return -EINVAL;
1937 
1938 	if (epause->rx_pause)
1939 		lc->requested_fc |= PAUSE_RX;
1940 	if (epause->tx_pause)
1941 		lc->requested_fc |= PAUSE_TX;
1942 	if (lc->autoneg == AUTONEG_ENABLE) {
1943 		if (netif_running(dev))
1944 			t3_link_start(&p->phy, &p->mac, lc);
1945 	} else {
1946 		lc->fc = lc->requested_fc & (PAUSE_RX | PAUSE_TX);
1947 		if (netif_running(dev))
1948 			t3_mac_set_speed_duplex_fc(&p->mac, -1, -1, lc->fc);
1949 	}
1950 	return 0;
1951 }
1952 
1953 static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1954 {
1955 	struct port_info *pi = netdev_priv(dev);
1956 	struct adapter *adapter = pi->adapter;
1957 	const struct qset_params *q = &adapter->params.sge.qset[pi->first_qset];
1958 
1959 	e->rx_max_pending = MAX_RX_BUFFERS;
1960 	e->rx_jumbo_max_pending = MAX_RX_JUMBO_BUFFERS;
1961 	e->tx_max_pending = MAX_TXQ_ENTRIES;
1962 
1963 	e->rx_pending = q->fl_size;
1964 	e->rx_mini_pending = q->rspq_size;
1965 	e->rx_jumbo_pending = q->jumbo_size;
1966 	e->tx_pending = q->txq_size[0];
1967 }
1968 
1969 static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1970 {
1971 	struct port_info *pi = netdev_priv(dev);
1972 	struct adapter *adapter = pi->adapter;
1973 	struct qset_params *q;
1974 	int i;
1975 
1976 	if (e->rx_pending > MAX_RX_BUFFERS ||
1977 	    e->rx_jumbo_pending > MAX_RX_JUMBO_BUFFERS ||
1978 	    e->tx_pending > MAX_TXQ_ENTRIES ||
1979 	    e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1980 	    e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1981 	    e->rx_pending < MIN_FL_ENTRIES ||
1982 	    e->rx_jumbo_pending < MIN_FL_ENTRIES ||
1983 	    e->tx_pending < adapter->params.nports * MIN_TXQ_ENTRIES)
1984 		return -EINVAL;
1985 
1986 	if (adapter->flags & FULL_INIT_DONE)
1987 		return -EBUSY;
1988 
1989 	q = &adapter->params.sge.qset[pi->first_qset];
1990 	for (i = 0; i < pi->nqsets; ++i, ++q) {
1991 		q->rspq_size = e->rx_mini_pending;
1992 		q->fl_size = e->rx_pending;
1993 		q->jumbo_size = e->rx_jumbo_pending;
1994 		q->txq_size[0] = e->tx_pending;
1995 		q->txq_size[1] = e->tx_pending;
1996 		q->txq_size[2] = e->tx_pending;
1997 	}
1998 	return 0;
1999 }
2000 
2001 static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2002 {
2003 	struct port_info *pi = netdev_priv(dev);
2004 	struct adapter *adapter = pi->adapter;
2005 	struct qset_params *qsp;
2006 	struct sge_qset *qs;
2007 	int i;
2008 
2009 	if (c->rx_coalesce_usecs * 10 > M_NEWTIMER)
2010 		return -EINVAL;
2011 
2012 	for (i = 0; i < pi->nqsets; i++) {
2013 		qsp = &adapter->params.sge.qset[i];
2014 		qs = &adapter->sge.qs[i];
2015 		qsp->coalesce_usecs = c->rx_coalesce_usecs;
2016 		t3_update_qset_coalesce(qs, qsp);
2017 	}
2018 
2019 	return 0;
2020 }
2021 
2022 static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
2023 {
2024 	struct port_info *pi = netdev_priv(dev);
2025 	struct adapter *adapter = pi->adapter;
2026 	struct qset_params *q = adapter->params.sge.qset;
2027 
2028 	c->rx_coalesce_usecs = q->coalesce_usecs;
2029 	return 0;
2030 }
2031 
2032 static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
2033 		      u8 * data)
2034 {
2035 	struct port_info *pi = netdev_priv(dev);
2036 	struct adapter *adapter = pi->adapter;
2037 	int i, err = 0;
2038 
2039 	u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
2040 	if (!buf)
2041 		return -ENOMEM;
2042 
2043 	e->magic = EEPROM_MAGIC;
2044 	for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
2045 		err = t3_seeprom_read(adapter, i, (__le32 *) & buf[i]);
2046 
2047 	if (!err)
2048 		memcpy(data, buf + e->offset, e->len);
2049 	kfree(buf);
2050 	return err;
2051 }
2052 
2053 static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
2054 		      u8 * data)
2055 {
2056 	struct port_info *pi = netdev_priv(dev);
2057 	struct adapter *adapter = pi->adapter;
2058 	u32 aligned_offset, aligned_len;
2059 	__le32 *p;
2060 	u8 *buf;
2061 	int err;
2062 
2063 	if (eeprom->magic != EEPROM_MAGIC)
2064 		return -EINVAL;
2065 
2066 	aligned_offset = eeprom->offset & ~3;
2067 	aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
2068 
2069 	if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
2070 		buf = kmalloc(aligned_len, GFP_KERNEL);
2071 		if (!buf)
2072 			return -ENOMEM;
2073 		err = t3_seeprom_read(adapter, aligned_offset, (__le32 *) buf);
2074 		if (!err && aligned_len > 4)
2075 			err = t3_seeprom_read(adapter,
2076 					      aligned_offset + aligned_len - 4,
2077 					      (__le32 *) & buf[aligned_len - 4]);
2078 		if (err)
2079 			goto out;
2080 		memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
2081 	} else
2082 		buf = data;
2083 
2084 	err = t3_seeprom_wp(adapter, 0);
2085 	if (err)
2086 		goto out;
2087 
2088 	for (p = (__le32 *) buf; !err && aligned_len; aligned_len -= 4, p++) {
2089 		err = t3_seeprom_write(adapter, aligned_offset, *p);
2090 		aligned_offset += 4;
2091 	}
2092 
2093 	if (!err)
2094 		err = t3_seeprom_wp(adapter, 1);
2095 out:
2096 	if (buf != data)
2097 		kfree(buf);
2098 	return err;
2099 }
2100 
2101 static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
2102 {
2103 	wol->supported = 0;
2104 	wol->wolopts = 0;
2105 	memset(&wol->sopass, 0, sizeof(wol->sopass));
2106 }
2107 
2108 static const struct ethtool_ops cxgb_ethtool_ops = {
2109 	.get_drvinfo = get_drvinfo,
2110 	.get_msglevel = get_msglevel,
2111 	.set_msglevel = set_msglevel,
2112 	.get_ringparam = get_sge_param,
2113 	.set_ringparam = set_sge_param,
2114 	.get_coalesce = get_coalesce,
2115 	.set_coalesce = set_coalesce,
2116 	.get_eeprom_len = get_eeprom_len,
2117 	.get_eeprom = get_eeprom,
2118 	.set_eeprom = set_eeprom,
2119 	.get_pauseparam = get_pauseparam,
2120 	.set_pauseparam = set_pauseparam,
2121 	.get_link = ethtool_op_get_link,
2122 	.get_strings = get_strings,
2123 	.set_phys_id = set_phys_id,
2124 	.nway_reset = restart_autoneg,
2125 	.get_sset_count = get_sset_count,
2126 	.get_ethtool_stats = get_stats,
2127 	.get_regs_len = get_regs_len,
2128 	.get_regs = get_regs,
2129 	.get_wol = get_wol,
2130 	.get_link_ksettings = get_link_ksettings,
2131 	.set_link_ksettings = set_link_ksettings,
2132 };
2133 
2134 static int in_range(int val, int lo, int hi)
2135 {
2136 	return val < 0 || (val <= hi && val >= lo);
2137 }
2138 
2139 static int cxgb_extension_ioctl(struct net_device *dev, void __user *useraddr)
2140 {
2141 	struct port_info *pi = netdev_priv(dev);
2142 	struct adapter *adapter = pi->adapter;
2143 	u32 cmd;
2144 	int ret;
2145 
2146 	if (copy_from_user(&cmd, useraddr, sizeof(cmd)))
2147 		return -EFAULT;
2148 
2149 	switch (cmd) {
2150 	case CHELSIO_SET_QSET_PARAMS:{
2151 		int i;
2152 		struct qset_params *q;
2153 		struct ch_qset_params t;
2154 		int q1 = pi->first_qset;
2155 		int nqsets = pi->nqsets;
2156 
2157 		if (!capable(CAP_NET_ADMIN))
2158 			return -EPERM;
2159 		if (copy_from_user(&t, useraddr, sizeof(t)))
2160 			return -EFAULT;
2161 		if (t.cmd != CHELSIO_SET_QSET_PARAMS)
2162 			return -EINVAL;
2163 		if (t.qset_idx >= SGE_QSETS)
2164 			return -EINVAL;
2165 		if (!in_range(t.intr_lat, 0, M_NEWTIMER) ||
2166 		    !in_range(t.cong_thres, 0, 255) ||
2167 		    !in_range(t.txq_size[0], MIN_TXQ_ENTRIES,
2168 			      MAX_TXQ_ENTRIES) ||
2169 		    !in_range(t.txq_size[1], MIN_TXQ_ENTRIES,
2170 			      MAX_TXQ_ENTRIES) ||
2171 		    !in_range(t.txq_size[2], MIN_CTRL_TXQ_ENTRIES,
2172 			      MAX_CTRL_TXQ_ENTRIES) ||
2173 		    !in_range(t.fl_size[0], MIN_FL_ENTRIES,
2174 			      MAX_RX_BUFFERS) ||
2175 		    !in_range(t.fl_size[1], MIN_FL_ENTRIES,
2176 			      MAX_RX_JUMBO_BUFFERS) ||
2177 		    !in_range(t.rspq_size, MIN_RSPQ_ENTRIES,
2178 			      MAX_RSPQ_ENTRIES))
2179 			return -EINVAL;
2180 
2181 		if ((adapter->flags & FULL_INIT_DONE) &&
2182 			(t.rspq_size >= 0 || t.fl_size[0] >= 0 ||
2183 			t.fl_size[1] >= 0 || t.txq_size[0] >= 0 ||
2184 			t.txq_size[1] >= 0 || t.txq_size[2] >= 0 ||
2185 			t.polling >= 0 || t.cong_thres >= 0))
2186 			return -EBUSY;
2187 
2188 		/* Allow setting of any available qset when offload enabled */
2189 		if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2190 			q1 = 0;
2191 			for_each_port(adapter, i) {
2192 				pi = adap2pinfo(adapter, i);
2193 				nqsets += pi->first_qset + pi->nqsets;
2194 			}
2195 		}
2196 
2197 		if (t.qset_idx < q1)
2198 			return -EINVAL;
2199 		if (t.qset_idx > q1 + nqsets - 1)
2200 			return -EINVAL;
2201 
2202 		q = &adapter->params.sge.qset[t.qset_idx];
2203 
2204 		if (t.rspq_size >= 0)
2205 			q->rspq_size = t.rspq_size;
2206 		if (t.fl_size[0] >= 0)
2207 			q->fl_size = t.fl_size[0];
2208 		if (t.fl_size[1] >= 0)
2209 			q->jumbo_size = t.fl_size[1];
2210 		if (t.txq_size[0] >= 0)
2211 			q->txq_size[0] = t.txq_size[0];
2212 		if (t.txq_size[1] >= 0)
2213 			q->txq_size[1] = t.txq_size[1];
2214 		if (t.txq_size[2] >= 0)
2215 			q->txq_size[2] = t.txq_size[2];
2216 		if (t.cong_thres >= 0)
2217 			q->cong_thres = t.cong_thres;
2218 		if (t.intr_lat >= 0) {
2219 			struct sge_qset *qs =
2220 				&adapter->sge.qs[t.qset_idx];
2221 
2222 			q->coalesce_usecs = t.intr_lat;
2223 			t3_update_qset_coalesce(qs, q);
2224 		}
2225 		if (t.polling >= 0) {
2226 			if (adapter->flags & USING_MSIX)
2227 				q->polling = t.polling;
2228 			else {
2229 				/* No polling with INTx for T3A */
2230 				if (adapter->params.rev == 0 &&
2231 					!(adapter->flags & USING_MSI))
2232 					t.polling = 0;
2233 
2234 				for (i = 0; i < SGE_QSETS; i++) {
2235 					q = &adapter->params.sge.
2236 						qset[i];
2237 					q->polling = t.polling;
2238 				}
2239 			}
2240 		}
2241 
2242 		if (t.lro >= 0) {
2243 			if (t.lro)
2244 				dev->wanted_features |= NETIF_F_GRO;
2245 			else
2246 				dev->wanted_features &= ~NETIF_F_GRO;
2247 			netdev_update_features(dev);
2248 		}
2249 
2250 		break;
2251 	}
2252 	case CHELSIO_GET_QSET_PARAMS:{
2253 		struct qset_params *q;
2254 		struct ch_qset_params t;
2255 		int q1 = pi->first_qset;
2256 		int nqsets = pi->nqsets;
2257 		int i;
2258 
2259 		if (copy_from_user(&t, useraddr, sizeof(t)))
2260 			return -EFAULT;
2261 
2262 		if (t.cmd != CHELSIO_GET_QSET_PARAMS)
2263 			return -EINVAL;
2264 
2265 		/* Display qsets for all ports when offload enabled */
2266 		if (test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2267 			q1 = 0;
2268 			for_each_port(adapter, i) {
2269 				pi = adap2pinfo(adapter, i);
2270 				nqsets = pi->first_qset + pi->nqsets;
2271 			}
2272 		}
2273 
2274 		if (t.qset_idx >= nqsets)
2275 			return -EINVAL;
2276 		t.qset_idx = array_index_nospec(t.qset_idx, nqsets);
2277 
2278 		q = &adapter->params.sge.qset[q1 + t.qset_idx];
2279 		t.rspq_size = q->rspq_size;
2280 		t.txq_size[0] = q->txq_size[0];
2281 		t.txq_size[1] = q->txq_size[1];
2282 		t.txq_size[2] = q->txq_size[2];
2283 		t.fl_size[0] = q->fl_size;
2284 		t.fl_size[1] = q->jumbo_size;
2285 		t.polling = q->polling;
2286 		t.lro = !!(dev->features & NETIF_F_GRO);
2287 		t.intr_lat = q->coalesce_usecs;
2288 		t.cong_thres = q->cong_thres;
2289 		t.qnum = q1;
2290 
2291 		if (adapter->flags & USING_MSIX)
2292 			t.vector = adapter->msix_info[q1 + t.qset_idx + 1].vec;
2293 		else
2294 			t.vector = adapter->pdev->irq;
2295 
2296 		if (copy_to_user(useraddr, &t, sizeof(t)))
2297 			return -EFAULT;
2298 		break;
2299 	}
2300 	case CHELSIO_SET_QSET_NUM:{
2301 		struct ch_reg edata;
2302 		unsigned int i, first_qset = 0, other_qsets = 0;
2303 
2304 		if (!capable(CAP_NET_ADMIN))
2305 			return -EPERM;
2306 		if (adapter->flags & FULL_INIT_DONE)
2307 			return -EBUSY;
2308 		if (copy_from_user(&edata, useraddr, sizeof(edata)))
2309 			return -EFAULT;
2310 		if (edata.cmd != CHELSIO_SET_QSET_NUM)
2311 			return -EINVAL;
2312 		if (edata.val < 1 ||
2313 			(edata.val > 1 && !(adapter->flags & USING_MSIX)))
2314 			return -EINVAL;
2315 
2316 		for_each_port(adapter, i)
2317 			if (adapter->port[i] && adapter->port[i] != dev)
2318 				other_qsets += adap2pinfo(adapter, i)->nqsets;
2319 
2320 		if (edata.val + other_qsets > SGE_QSETS)
2321 			return -EINVAL;
2322 
2323 		pi->nqsets = edata.val;
2324 
2325 		for_each_port(adapter, i)
2326 			if (adapter->port[i]) {
2327 				pi = adap2pinfo(adapter, i);
2328 				pi->first_qset = first_qset;
2329 				first_qset += pi->nqsets;
2330 			}
2331 		break;
2332 	}
2333 	case CHELSIO_GET_QSET_NUM:{
2334 		struct ch_reg edata;
2335 
2336 		memset(&edata, 0, sizeof(struct ch_reg));
2337 
2338 		edata.cmd = CHELSIO_GET_QSET_NUM;
2339 		edata.val = pi->nqsets;
2340 		if (copy_to_user(useraddr, &edata, sizeof(edata)))
2341 			return -EFAULT;
2342 		break;
2343 	}
2344 	case CHELSIO_LOAD_FW:{
2345 		u8 *fw_data;
2346 		struct ch_mem_range t;
2347 
2348 		if (!capable(CAP_SYS_RAWIO))
2349 			return -EPERM;
2350 		if (copy_from_user(&t, useraddr, sizeof(t)))
2351 			return -EFAULT;
2352 		if (t.cmd != CHELSIO_LOAD_FW)
2353 			return -EINVAL;
2354 		/* Check t.len sanity ? */
2355 		fw_data = memdup_user(useraddr + sizeof(t), t.len);
2356 		if (IS_ERR(fw_data))
2357 			return PTR_ERR(fw_data);
2358 
2359 		ret = t3_load_fw(adapter, fw_data, t.len);
2360 		kfree(fw_data);
2361 		if (ret)
2362 			return ret;
2363 		break;
2364 	}
2365 	case CHELSIO_SETMTUTAB:{
2366 		struct ch_mtus m;
2367 		int i;
2368 
2369 		if (!is_offload(adapter))
2370 			return -EOPNOTSUPP;
2371 		if (!capable(CAP_NET_ADMIN))
2372 			return -EPERM;
2373 		if (offload_running(adapter))
2374 			return -EBUSY;
2375 		if (copy_from_user(&m, useraddr, sizeof(m)))
2376 			return -EFAULT;
2377 		if (m.cmd != CHELSIO_SETMTUTAB)
2378 			return -EINVAL;
2379 		if (m.nmtus != NMTUS)
2380 			return -EINVAL;
2381 		if (m.mtus[0] < 81)	/* accommodate SACK */
2382 			return -EINVAL;
2383 
2384 		/* MTUs must be in ascending order */
2385 		for (i = 1; i < NMTUS; ++i)
2386 			if (m.mtus[i] < m.mtus[i - 1])
2387 				return -EINVAL;
2388 
2389 		memcpy(adapter->params.mtus, m.mtus,
2390 			sizeof(adapter->params.mtus));
2391 		break;
2392 	}
2393 	case CHELSIO_GET_PM:{
2394 		struct tp_params *p = &adapter->params.tp;
2395 		struct ch_pm m = {.cmd = CHELSIO_GET_PM };
2396 
2397 		if (!is_offload(adapter))
2398 			return -EOPNOTSUPP;
2399 		m.tx_pg_sz = p->tx_pg_size;
2400 		m.tx_num_pg = p->tx_num_pgs;
2401 		m.rx_pg_sz = p->rx_pg_size;
2402 		m.rx_num_pg = p->rx_num_pgs;
2403 		m.pm_total = p->pmtx_size + p->chan_rx_size * p->nchan;
2404 		if (copy_to_user(useraddr, &m, sizeof(m)))
2405 			return -EFAULT;
2406 		break;
2407 	}
2408 	case CHELSIO_SET_PM:{
2409 		struct ch_pm m;
2410 		struct tp_params *p = &adapter->params.tp;
2411 
2412 		if (!is_offload(adapter))
2413 			return -EOPNOTSUPP;
2414 		if (!capable(CAP_NET_ADMIN))
2415 			return -EPERM;
2416 		if (adapter->flags & FULL_INIT_DONE)
2417 			return -EBUSY;
2418 		if (copy_from_user(&m, useraddr, sizeof(m)))
2419 			return -EFAULT;
2420 		if (m.cmd != CHELSIO_SET_PM)
2421 			return -EINVAL;
2422 		if (!is_power_of_2(m.rx_pg_sz) ||
2423 			!is_power_of_2(m.tx_pg_sz))
2424 			return -EINVAL;	/* not power of 2 */
2425 		if (!(m.rx_pg_sz & 0x14000))
2426 			return -EINVAL;	/* not 16KB or 64KB */
2427 		if (!(m.tx_pg_sz & 0x1554000))
2428 			return -EINVAL;
2429 		if (m.tx_num_pg == -1)
2430 			m.tx_num_pg = p->tx_num_pgs;
2431 		if (m.rx_num_pg == -1)
2432 			m.rx_num_pg = p->rx_num_pgs;
2433 		if (m.tx_num_pg % 24 || m.rx_num_pg % 24)
2434 			return -EINVAL;
2435 		if (m.rx_num_pg * m.rx_pg_sz > p->chan_rx_size ||
2436 			m.tx_num_pg * m.tx_pg_sz > p->chan_tx_size)
2437 			return -EINVAL;
2438 		p->rx_pg_size = m.rx_pg_sz;
2439 		p->tx_pg_size = m.tx_pg_sz;
2440 		p->rx_num_pgs = m.rx_num_pg;
2441 		p->tx_num_pgs = m.tx_num_pg;
2442 		break;
2443 	}
2444 	case CHELSIO_GET_MEM:{
2445 		struct ch_mem_range t;
2446 		struct mc7 *mem;
2447 		u64 buf[32];
2448 
2449 		if (!is_offload(adapter))
2450 			return -EOPNOTSUPP;
2451 		if (!(adapter->flags & FULL_INIT_DONE))
2452 			return -EIO;	/* need the memory controllers */
2453 		if (copy_from_user(&t, useraddr, sizeof(t)))
2454 			return -EFAULT;
2455 		if (t.cmd != CHELSIO_GET_MEM)
2456 			return -EINVAL;
2457 		if ((t.addr & 7) || (t.len & 7))
2458 			return -EINVAL;
2459 		if (t.mem_id == MEM_CM)
2460 			mem = &adapter->cm;
2461 		else if (t.mem_id == MEM_PMRX)
2462 			mem = &adapter->pmrx;
2463 		else if (t.mem_id == MEM_PMTX)
2464 			mem = &adapter->pmtx;
2465 		else
2466 			return -EINVAL;
2467 
2468 		/*
2469 		 * Version scheme:
2470 		 * bits 0..9: chip version
2471 		 * bits 10..15: chip revision
2472 		 */
2473 		t.version = 3 | (adapter->params.rev << 10);
2474 		if (copy_to_user(useraddr, &t, sizeof(t)))
2475 			return -EFAULT;
2476 
2477 		/*
2478 		 * Read 256 bytes at a time as len can be large and we don't
2479 		 * want to use huge intermediate buffers.
2480 		 */
2481 		useraddr += sizeof(t);	/* advance to start of buffer */
2482 		while (t.len) {
2483 			unsigned int chunk =
2484 				min_t(unsigned int, t.len, sizeof(buf));
2485 
2486 			ret =
2487 				t3_mc7_bd_read(mem, t.addr / 8, chunk / 8,
2488 						buf);
2489 			if (ret)
2490 				return ret;
2491 			if (copy_to_user(useraddr, buf, chunk))
2492 				return -EFAULT;
2493 			useraddr += chunk;
2494 			t.addr += chunk;
2495 			t.len -= chunk;
2496 		}
2497 		break;
2498 	}
2499 	case CHELSIO_SET_TRACE_FILTER:{
2500 		struct ch_trace t;
2501 		const struct trace_params *tp;
2502 
2503 		if (!capable(CAP_NET_ADMIN))
2504 			return -EPERM;
2505 		if (!offload_running(adapter))
2506 			return -EAGAIN;
2507 		if (copy_from_user(&t, useraddr, sizeof(t)))
2508 			return -EFAULT;
2509 		if (t.cmd != CHELSIO_SET_TRACE_FILTER)
2510 			return -EINVAL;
2511 
2512 		tp = (const struct trace_params *)&t.sip;
2513 		if (t.config_tx)
2514 			t3_config_trace_filter(adapter, tp, 0,
2515 						t.invert_match,
2516 						t.trace_tx);
2517 		if (t.config_rx)
2518 			t3_config_trace_filter(adapter, tp, 1,
2519 						t.invert_match,
2520 						t.trace_rx);
2521 		break;
2522 	}
2523 	default:
2524 		return -EOPNOTSUPP;
2525 	}
2526 	return 0;
2527 }
2528 
2529 static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2530 {
2531 	struct mii_ioctl_data *data = if_mii(req);
2532 	struct port_info *pi = netdev_priv(dev);
2533 	struct adapter *adapter = pi->adapter;
2534 
2535 	switch (cmd) {
2536 	case SIOCGMIIREG:
2537 	case SIOCSMIIREG:
2538 		/* Convert phy_id from older PRTAD/DEVAD format */
2539 		if (is_10G(adapter) &&
2540 		    !mdio_phy_id_is_c45(data->phy_id) &&
2541 		    (data->phy_id & 0x1f00) &&
2542 		    !(data->phy_id & 0xe0e0))
2543 			data->phy_id = mdio_phy_id_c45(data->phy_id >> 8,
2544 						       data->phy_id & 0x1f);
2545 		/* FALLTHRU */
2546 	case SIOCGMIIPHY:
2547 		return mdio_mii_ioctl(&pi->phy.mdio, data, cmd);
2548 	case SIOCCHIOCTL:
2549 		return cxgb_extension_ioctl(dev, req->ifr_data);
2550 	default:
2551 		return -EOPNOTSUPP;
2552 	}
2553 }
2554 
2555 static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2556 {
2557 	struct port_info *pi = netdev_priv(dev);
2558 	struct adapter *adapter = pi->adapter;
2559 	int ret;
2560 
2561 	if ((ret = t3_mac_set_mtu(&pi->mac, new_mtu)))
2562 		return ret;
2563 	dev->mtu = new_mtu;
2564 	init_port_mtus(adapter);
2565 	if (adapter->params.rev == 0 && offload_running(adapter))
2566 		t3_load_mtus(adapter, adapter->params.mtus,
2567 			     adapter->params.a_wnd, adapter->params.b_wnd,
2568 			     adapter->port[0]->mtu);
2569 	return 0;
2570 }
2571 
2572 static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2573 {
2574 	struct port_info *pi = netdev_priv(dev);
2575 	struct adapter *adapter = pi->adapter;
2576 	struct sockaddr *addr = p;
2577 
2578 	if (!is_valid_ether_addr(addr->sa_data))
2579 		return -EADDRNOTAVAIL;
2580 
2581 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2582 	t3_mac_set_address(&pi->mac, LAN_MAC_IDX, dev->dev_addr);
2583 	if (offload_running(adapter))
2584 		write_smt_entry(adapter, pi->port_id);
2585 	return 0;
2586 }
2587 
2588 static netdev_features_t cxgb_fix_features(struct net_device *dev,
2589 	netdev_features_t features)
2590 {
2591 	/*
2592 	 * Since there is no support for separate rx/tx vlan accel
2593 	 * enable/disable make sure tx flag is always in same state as rx.
2594 	 */
2595 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
2596 		features |= NETIF_F_HW_VLAN_CTAG_TX;
2597 	else
2598 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2599 
2600 	return features;
2601 }
2602 
2603 static int cxgb_set_features(struct net_device *dev, netdev_features_t features)
2604 {
2605 	netdev_features_t changed = dev->features ^ features;
2606 
2607 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2608 		cxgb_vlan_mode(dev, features);
2609 
2610 	return 0;
2611 }
2612 
2613 #ifdef CONFIG_NET_POLL_CONTROLLER
2614 static void cxgb_netpoll(struct net_device *dev)
2615 {
2616 	struct port_info *pi = netdev_priv(dev);
2617 	struct adapter *adapter = pi->adapter;
2618 	int qidx;
2619 
2620 	for (qidx = pi->first_qset; qidx < pi->first_qset + pi->nqsets; qidx++) {
2621 		struct sge_qset *qs = &adapter->sge.qs[qidx];
2622 		void *source;
2623 
2624 		if (adapter->flags & USING_MSIX)
2625 			source = qs;
2626 		else
2627 			source = adapter;
2628 
2629 		t3_intr_handler(adapter, qs->rspq.polling) (0, source);
2630 	}
2631 }
2632 #endif
2633 
2634 /*
2635  * Periodic accumulation of MAC statistics.
2636  */
2637 static void mac_stats_update(struct adapter *adapter)
2638 {
2639 	int i;
2640 
2641 	for_each_port(adapter, i) {
2642 		struct net_device *dev = adapter->port[i];
2643 		struct port_info *p = netdev_priv(dev);
2644 
2645 		if (netif_running(dev)) {
2646 			spin_lock(&adapter->stats_lock);
2647 			t3_mac_update_stats(&p->mac);
2648 			spin_unlock(&adapter->stats_lock);
2649 		}
2650 	}
2651 }
2652 
2653 static void check_link_status(struct adapter *adapter)
2654 {
2655 	int i;
2656 
2657 	for_each_port(adapter, i) {
2658 		struct net_device *dev = adapter->port[i];
2659 		struct port_info *p = netdev_priv(dev);
2660 		int link_fault;
2661 
2662 		spin_lock_irq(&adapter->work_lock);
2663 		link_fault = p->link_fault;
2664 		spin_unlock_irq(&adapter->work_lock);
2665 
2666 		if (link_fault) {
2667 			t3_link_fault(adapter, i);
2668 			continue;
2669 		}
2670 
2671 		if (!(p->phy.caps & SUPPORTED_IRQ) && netif_running(dev)) {
2672 			t3_xgm_intr_disable(adapter, i);
2673 			t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2674 
2675 			t3_link_changed(adapter, i);
2676 			t3_xgm_intr_enable(adapter, i);
2677 		}
2678 	}
2679 }
2680 
2681 static void check_t3b2_mac(struct adapter *adapter)
2682 {
2683 	int i;
2684 
2685 	if (!rtnl_trylock())	/* synchronize with ifdown */
2686 		return;
2687 
2688 	for_each_port(adapter, i) {
2689 		struct net_device *dev = adapter->port[i];
2690 		struct port_info *p = netdev_priv(dev);
2691 		int status;
2692 
2693 		if (!netif_running(dev))
2694 			continue;
2695 
2696 		status = 0;
2697 		if (netif_running(dev) && netif_carrier_ok(dev))
2698 			status = t3b2_mac_watchdog_task(&p->mac);
2699 		if (status == 1)
2700 			p->mac.stats.num_toggled++;
2701 		else if (status == 2) {
2702 			struct cmac *mac = &p->mac;
2703 
2704 			t3_mac_set_mtu(mac, dev->mtu);
2705 			t3_mac_set_address(mac, LAN_MAC_IDX, dev->dev_addr);
2706 			cxgb_set_rxmode(dev);
2707 			t3_link_start(&p->phy, mac, &p->link_config);
2708 			t3_mac_enable(mac, MAC_DIRECTION_RX | MAC_DIRECTION_TX);
2709 			t3_port_intr_enable(adapter, p->port_id);
2710 			p->mac.stats.num_resets++;
2711 		}
2712 	}
2713 	rtnl_unlock();
2714 }
2715 
2716 
2717 static void t3_adap_check_task(struct work_struct *work)
2718 {
2719 	struct adapter *adapter = container_of(work, struct adapter,
2720 					       adap_check_task.work);
2721 	const struct adapter_params *p = &adapter->params;
2722 	int port;
2723 	unsigned int v, status, reset;
2724 
2725 	adapter->check_task_cnt++;
2726 
2727 	check_link_status(adapter);
2728 
2729 	/* Accumulate MAC stats if needed */
2730 	if (!p->linkpoll_period ||
2731 	    (adapter->check_task_cnt * p->linkpoll_period) / 10 >=
2732 	    p->stats_update_period) {
2733 		mac_stats_update(adapter);
2734 		adapter->check_task_cnt = 0;
2735 	}
2736 
2737 	if (p->rev == T3_REV_B2)
2738 		check_t3b2_mac(adapter);
2739 
2740 	/*
2741 	 * Scan the XGMAC's to check for various conditions which we want to
2742 	 * monitor in a periodic polling manner rather than via an interrupt
2743 	 * condition.  This is used for conditions which would otherwise flood
2744 	 * the system with interrupts and we only really need to know that the
2745 	 * conditions are "happening" ...  For each condition we count the
2746 	 * detection of the condition and reset it for the next polling loop.
2747 	 */
2748 	for_each_port(adapter, port) {
2749 		struct cmac *mac =  &adap2pinfo(adapter, port)->mac;
2750 		u32 cause;
2751 
2752 		cause = t3_read_reg(adapter, A_XGM_INT_CAUSE + mac->offset);
2753 		reset = 0;
2754 		if (cause & F_RXFIFO_OVERFLOW) {
2755 			mac->stats.rx_fifo_ovfl++;
2756 			reset |= F_RXFIFO_OVERFLOW;
2757 		}
2758 
2759 		t3_write_reg(adapter, A_XGM_INT_CAUSE + mac->offset, reset);
2760 	}
2761 
2762 	/*
2763 	 * We do the same as above for FL_EMPTY interrupts.
2764 	 */
2765 	status = t3_read_reg(adapter, A_SG_INT_CAUSE);
2766 	reset = 0;
2767 
2768 	if (status & F_FLEMPTY) {
2769 		struct sge_qset *qs = &adapter->sge.qs[0];
2770 		int i = 0;
2771 
2772 		reset |= F_FLEMPTY;
2773 
2774 		v = (t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS) >> S_FL0EMPTY) &
2775 		    0xffff;
2776 
2777 		while (v) {
2778 			qs->fl[i].empty += (v & 1);
2779 			if (i)
2780 				qs++;
2781 			i ^= 1;
2782 			v >>= 1;
2783 		}
2784 	}
2785 
2786 	t3_write_reg(adapter, A_SG_INT_CAUSE, reset);
2787 
2788 	/* Schedule the next check update if any port is active. */
2789 	spin_lock_irq(&adapter->work_lock);
2790 	if (adapter->open_device_map & PORT_MASK)
2791 		schedule_chk_task(adapter);
2792 	spin_unlock_irq(&adapter->work_lock);
2793 }
2794 
2795 static void db_full_task(struct work_struct *work)
2796 {
2797 	struct adapter *adapter = container_of(work, struct adapter,
2798 					       db_full_task);
2799 
2800 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_FULL, 0);
2801 }
2802 
2803 static void db_empty_task(struct work_struct *work)
2804 {
2805 	struct adapter *adapter = container_of(work, struct adapter,
2806 					       db_empty_task);
2807 
2808 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_EMPTY, 0);
2809 }
2810 
2811 static void db_drop_task(struct work_struct *work)
2812 {
2813 	struct adapter *adapter = container_of(work, struct adapter,
2814 					       db_drop_task);
2815 	unsigned long delay = 1000;
2816 	unsigned short r;
2817 
2818 	cxgb3_event_notify(&adapter->tdev, OFFLOAD_DB_DROP, 0);
2819 
2820 	/*
2821 	 * Sleep a while before ringing the driver qset dbs.
2822 	 * The delay is between 1000-2023 usecs.
2823 	 */
2824 	get_random_bytes(&r, 2);
2825 	delay += r & 1023;
2826 	set_current_state(TASK_UNINTERRUPTIBLE);
2827 	schedule_timeout(usecs_to_jiffies(delay));
2828 	ring_dbs(adapter);
2829 }
2830 
2831 /*
2832  * Processes external (PHY) interrupts in process context.
2833  */
2834 static void ext_intr_task(struct work_struct *work)
2835 {
2836 	struct adapter *adapter = container_of(work, struct adapter,
2837 					       ext_intr_handler_task);
2838 	int i;
2839 
2840 	/* Disable link fault interrupts */
2841 	for_each_port(adapter, i) {
2842 		struct net_device *dev = adapter->port[i];
2843 		struct port_info *p = netdev_priv(dev);
2844 
2845 		t3_xgm_intr_disable(adapter, i);
2846 		t3_read_reg(adapter, A_XGM_INT_STATUS + p->mac.offset);
2847 	}
2848 
2849 	/* Re-enable link fault interrupts */
2850 	t3_phy_intr_handler(adapter);
2851 
2852 	for_each_port(adapter, i)
2853 		t3_xgm_intr_enable(adapter, i);
2854 
2855 	/* Now reenable external interrupts */
2856 	spin_lock_irq(&adapter->work_lock);
2857 	if (adapter->slow_intr_mask) {
2858 		adapter->slow_intr_mask |= F_T3DBG;
2859 		t3_write_reg(adapter, A_PL_INT_CAUSE0, F_T3DBG);
2860 		t3_write_reg(adapter, A_PL_INT_ENABLE0,
2861 			     adapter->slow_intr_mask);
2862 	}
2863 	spin_unlock_irq(&adapter->work_lock);
2864 }
2865 
2866 /*
2867  * Interrupt-context handler for external (PHY) interrupts.
2868  */
2869 void t3_os_ext_intr_handler(struct adapter *adapter)
2870 {
2871 	/*
2872 	 * Schedule a task to handle external interrupts as they may be slow
2873 	 * and we use a mutex to protect MDIO registers.  We disable PHY
2874 	 * interrupts in the meantime and let the task reenable them when
2875 	 * it's done.
2876 	 */
2877 	spin_lock(&adapter->work_lock);
2878 	if (adapter->slow_intr_mask) {
2879 		adapter->slow_intr_mask &= ~F_T3DBG;
2880 		t3_write_reg(adapter, A_PL_INT_ENABLE0,
2881 			     adapter->slow_intr_mask);
2882 		queue_work(cxgb3_wq, &adapter->ext_intr_handler_task);
2883 	}
2884 	spin_unlock(&adapter->work_lock);
2885 }
2886 
2887 void t3_os_link_fault_handler(struct adapter *adapter, int port_id)
2888 {
2889 	struct net_device *netdev = adapter->port[port_id];
2890 	struct port_info *pi = netdev_priv(netdev);
2891 
2892 	spin_lock(&adapter->work_lock);
2893 	pi->link_fault = 1;
2894 	spin_unlock(&adapter->work_lock);
2895 }
2896 
2897 static int t3_adapter_error(struct adapter *adapter, int reset, int on_wq)
2898 {
2899 	int i, ret = 0;
2900 
2901 	if (is_offload(adapter) &&
2902 	    test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map)) {
2903 		cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_DOWN, 0);
2904 		offload_close(&adapter->tdev);
2905 	}
2906 
2907 	/* Stop all ports */
2908 	for_each_port(adapter, i) {
2909 		struct net_device *netdev = adapter->port[i];
2910 
2911 		if (netif_running(netdev))
2912 			__cxgb_close(netdev, on_wq);
2913 	}
2914 
2915 	/* Stop SGE timers */
2916 	t3_stop_sge_timers(adapter);
2917 
2918 	adapter->flags &= ~FULL_INIT_DONE;
2919 
2920 	if (reset)
2921 		ret = t3_reset_adapter(adapter);
2922 
2923 	pci_disable_device(adapter->pdev);
2924 
2925 	return ret;
2926 }
2927 
2928 static int t3_reenable_adapter(struct adapter *adapter)
2929 {
2930 	if (pci_enable_device(adapter->pdev)) {
2931 		dev_err(&adapter->pdev->dev,
2932 			"Cannot re-enable PCI device after reset.\n");
2933 		goto err;
2934 	}
2935 	pci_set_master(adapter->pdev);
2936 	pci_restore_state(adapter->pdev);
2937 	pci_save_state(adapter->pdev);
2938 
2939 	/* Free sge resources */
2940 	t3_free_sge_resources(adapter);
2941 
2942 	if (t3_replay_prep_adapter(adapter))
2943 		goto err;
2944 
2945 	return 0;
2946 err:
2947 	return -1;
2948 }
2949 
2950 static void t3_resume_ports(struct adapter *adapter)
2951 {
2952 	int i;
2953 
2954 	/* Restart the ports */
2955 	for_each_port(adapter, i) {
2956 		struct net_device *netdev = adapter->port[i];
2957 
2958 		if (netif_running(netdev)) {
2959 			if (cxgb_open(netdev)) {
2960 				dev_err(&adapter->pdev->dev,
2961 					"can't bring device back up"
2962 					" after reset\n");
2963 				continue;
2964 			}
2965 		}
2966 	}
2967 
2968 	if (is_offload(adapter) && !ofld_disable)
2969 		cxgb3_event_notify(&adapter->tdev, OFFLOAD_STATUS_UP, 0);
2970 }
2971 
2972 /*
2973  * processes a fatal error.
2974  * Bring the ports down, reset the chip, bring the ports back up.
2975  */
2976 static void fatal_error_task(struct work_struct *work)
2977 {
2978 	struct adapter *adapter = container_of(work, struct adapter,
2979 					       fatal_error_handler_task);
2980 	int err = 0;
2981 
2982 	rtnl_lock();
2983 	err = t3_adapter_error(adapter, 1, 1);
2984 	if (!err)
2985 		err = t3_reenable_adapter(adapter);
2986 	if (!err)
2987 		t3_resume_ports(adapter);
2988 
2989 	CH_ALERT(adapter, "adapter reset %s\n", err ? "failed" : "succeeded");
2990 	rtnl_unlock();
2991 }
2992 
2993 void t3_fatal_err(struct adapter *adapter)
2994 {
2995 	unsigned int fw_status[4];
2996 
2997 	if (adapter->flags & FULL_INIT_DONE) {
2998 		t3_sge_stop(adapter);
2999 		t3_write_reg(adapter, A_XGM_TX_CTRL, 0);
3000 		t3_write_reg(adapter, A_XGM_RX_CTRL, 0);
3001 		t3_write_reg(adapter, XGM_REG(A_XGM_TX_CTRL, 1), 0);
3002 		t3_write_reg(adapter, XGM_REG(A_XGM_RX_CTRL, 1), 0);
3003 
3004 		spin_lock(&adapter->work_lock);
3005 		t3_intr_disable(adapter);
3006 		queue_work(cxgb3_wq, &adapter->fatal_error_handler_task);
3007 		spin_unlock(&adapter->work_lock);
3008 	}
3009 	CH_ALERT(adapter, "encountered fatal error, operation suspended\n");
3010 	if (!t3_cim_ctl_blk_read(adapter, 0xa0, 4, fw_status))
3011 		CH_ALERT(adapter, "FW status: 0x%x, 0x%x, 0x%x, 0x%x\n",
3012 			 fw_status[0], fw_status[1],
3013 			 fw_status[2], fw_status[3]);
3014 }
3015 
3016 /**
3017  * t3_io_error_detected - called when PCI error is detected
3018  * @pdev: Pointer to PCI device
3019  * @state: The current pci connection state
3020  *
3021  * This function is called after a PCI bus error affecting
3022  * this device has been detected.
3023  */
3024 static pci_ers_result_t t3_io_error_detected(struct pci_dev *pdev,
3025 					     pci_channel_state_t state)
3026 {
3027 	struct adapter *adapter = pci_get_drvdata(pdev);
3028 
3029 	if (state == pci_channel_io_perm_failure)
3030 		return PCI_ERS_RESULT_DISCONNECT;
3031 
3032 	t3_adapter_error(adapter, 0, 0);
3033 
3034 	/* Request a slot reset. */
3035 	return PCI_ERS_RESULT_NEED_RESET;
3036 }
3037 
3038 /**
3039  * t3_io_slot_reset - called after the pci bus has been reset.
3040  * @pdev: Pointer to PCI device
3041  *
3042  * Restart the card from scratch, as if from a cold-boot.
3043  */
3044 static pci_ers_result_t t3_io_slot_reset(struct pci_dev *pdev)
3045 {
3046 	struct adapter *adapter = pci_get_drvdata(pdev);
3047 
3048 	if (!t3_reenable_adapter(adapter))
3049 		return PCI_ERS_RESULT_RECOVERED;
3050 
3051 	return PCI_ERS_RESULT_DISCONNECT;
3052 }
3053 
3054 /**
3055  * t3_io_resume - called when traffic can start flowing again.
3056  * @pdev: Pointer to PCI device
3057  *
3058  * This callback is called when the error recovery driver tells us that
3059  * its OK to resume normal operation.
3060  */
3061 static void t3_io_resume(struct pci_dev *pdev)
3062 {
3063 	struct adapter *adapter = pci_get_drvdata(pdev);
3064 
3065 	CH_ALERT(adapter, "adapter recovering, PEX ERR 0x%x\n",
3066 		 t3_read_reg(adapter, A_PCIE_PEX_ERR));
3067 
3068 	rtnl_lock();
3069 	t3_resume_ports(adapter);
3070 	rtnl_unlock();
3071 }
3072 
3073 static const struct pci_error_handlers t3_err_handler = {
3074 	.error_detected = t3_io_error_detected,
3075 	.slot_reset = t3_io_slot_reset,
3076 	.resume = t3_io_resume,
3077 };
3078 
3079 /*
3080  * Set the number of qsets based on the number of CPUs and the number of ports,
3081  * not to exceed the number of available qsets, assuming there are enough qsets
3082  * per port in HW.
3083  */
3084 static void set_nqsets(struct adapter *adap)
3085 {
3086 	int i, j = 0;
3087 	int num_cpus = netif_get_num_default_rss_queues();
3088 	int hwports = adap->params.nports;
3089 	int nqsets = adap->msix_nvectors - 1;
3090 
3091 	if (adap->params.rev > 0 && adap->flags & USING_MSIX) {
3092 		if (hwports == 2 &&
3093 		    (hwports * nqsets > SGE_QSETS ||
3094 		     num_cpus >= nqsets / hwports))
3095 			nqsets /= hwports;
3096 		if (nqsets > num_cpus)
3097 			nqsets = num_cpus;
3098 		if (nqsets < 1 || hwports == 4)
3099 			nqsets = 1;
3100 	} else
3101 		nqsets = 1;
3102 
3103 	for_each_port(adap, i) {
3104 		struct port_info *pi = adap2pinfo(adap, i);
3105 
3106 		pi->first_qset = j;
3107 		pi->nqsets = nqsets;
3108 		j = pi->first_qset + nqsets;
3109 
3110 		dev_info(&adap->pdev->dev,
3111 			 "Port %d using %d queue sets.\n", i, nqsets);
3112 	}
3113 }
3114 
3115 static int cxgb_enable_msix(struct adapter *adap)
3116 {
3117 	struct msix_entry entries[SGE_QSETS + 1];
3118 	int vectors;
3119 	int i;
3120 
3121 	vectors = ARRAY_SIZE(entries);
3122 	for (i = 0; i < vectors; ++i)
3123 		entries[i].entry = i;
3124 
3125 	vectors = pci_enable_msix_range(adap->pdev, entries,
3126 					adap->params.nports + 1, vectors);
3127 	if (vectors < 0)
3128 		return vectors;
3129 
3130 	for (i = 0; i < vectors; ++i)
3131 		adap->msix_info[i].vec = entries[i].vector;
3132 	adap->msix_nvectors = vectors;
3133 
3134 	return 0;
3135 }
3136 
3137 static void print_port_info(struct adapter *adap, const struct adapter_info *ai)
3138 {
3139 	static const char *pci_variant[] = {
3140 		"PCI", "PCI-X", "PCI-X ECC", "PCI-X 266", "PCI Express"
3141 	};
3142 
3143 	int i;
3144 	char buf[80];
3145 
3146 	if (is_pcie(adap))
3147 		snprintf(buf, sizeof(buf), "%s x%d",
3148 			 pci_variant[adap->params.pci.variant],
3149 			 adap->params.pci.width);
3150 	else
3151 		snprintf(buf, sizeof(buf), "%s %dMHz/%d-bit",
3152 			 pci_variant[adap->params.pci.variant],
3153 			 adap->params.pci.speed, adap->params.pci.width);
3154 
3155 	for_each_port(adap, i) {
3156 		struct net_device *dev = adap->port[i];
3157 		const struct port_info *pi = netdev_priv(dev);
3158 
3159 		if (!test_bit(i, &adap->registered_device_map))
3160 			continue;
3161 		netdev_info(dev, "%s %s %sNIC (rev %d) %s%s\n",
3162 			    ai->desc, pi->phy.desc,
3163 			    is_offload(adap) ? "R" : "", adap->params.rev, buf,
3164 			    (adap->flags & USING_MSIX) ? " MSI-X" :
3165 			    (adap->flags & USING_MSI) ? " MSI" : "");
3166 		if (adap->name == dev->name && adap->params.vpd.mclk)
3167 			pr_info("%s: %uMB CM, %uMB PMTX, %uMB PMRX, S/N: %s\n",
3168 			       adap->name, t3_mc7_size(&adap->cm) >> 20,
3169 			       t3_mc7_size(&adap->pmtx) >> 20,
3170 			       t3_mc7_size(&adap->pmrx) >> 20,
3171 			       adap->params.vpd.sn);
3172 	}
3173 }
3174 
3175 static const struct net_device_ops cxgb_netdev_ops = {
3176 	.ndo_open		= cxgb_open,
3177 	.ndo_stop		= cxgb_close,
3178 	.ndo_start_xmit		= t3_eth_xmit,
3179 	.ndo_get_stats		= cxgb_get_stats,
3180 	.ndo_validate_addr	= eth_validate_addr,
3181 	.ndo_set_rx_mode	= cxgb_set_rxmode,
3182 	.ndo_do_ioctl		= cxgb_ioctl,
3183 	.ndo_change_mtu		= cxgb_change_mtu,
3184 	.ndo_set_mac_address	= cxgb_set_mac_addr,
3185 	.ndo_fix_features	= cxgb_fix_features,
3186 	.ndo_set_features	= cxgb_set_features,
3187 #ifdef CONFIG_NET_POLL_CONTROLLER
3188 	.ndo_poll_controller	= cxgb_netpoll,
3189 #endif
3190 };
3191 
3192 static void cxgb3_init_iscsi_mac(struct net_device *dev)
3193 {
3194 	struct port_info *pi = netdev_priv(dev);
3195 
3196 	memcpy(pi->iscsic.mac_addr, dev->dev_addr, ETH_ALEN);
3197 	pi->iscsic.mac_addr[3] |= 0x80;
3198 }
3199 
3200 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3201 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3202 			NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3203 static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent)
3204 {
3205 	int i, err, pci_using_dac = 0;
3206 	resource_size_t mmio_start, mmio_len;
3207 	const struct adapter_info *ai;
3208 	struct adapter *adapter = NULL;
3209 	struct port_info *pi;
3210 
3211 	pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
3212 
3213 	if (!cxgb3_wq) {
3214 		cxgb3_wq = create_singlethread_workqueue(DRV_NAME);
3215 		if (!cxgb3_wq) {
3216 			pr_err("cannot initialize work queue\n");
3217 			return -ENOMEM;
3218 		}
3219 	}
3220 
3221 	err = pci_enable_device(pdev);
3222 	if (err) {
3223 		dev_err(&pdev->dev, "cannot enable PCI device\n");
3224 		goto out;
3225 	}
3226 
3227 	err = pci_request_regions(pdev, DRV_NAME);
3228 	if (err) {
3229 		/* Just info, some other driver may have claimed the device. */
3230 		dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3231 		goto out_disable_device;
3232 	}
3233 
3234 	if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3235 		pci_using_dac = 1;
3236 		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3237 		if (err) {
3238 			dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3239 			       "coherent allocations\n");
3240 			goto out_release_regions;
3241 		}
3242 	} else if ((err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32))) != 0) {
3243 		dev_err(&pdev->dev, "no usable DMA configuration\n");
3244 		goto out_release_regions;
3245 	}
3246 
3247 	pci_set_master(pdev);
3248 	pci_save_state(pdev);
3249 
3250 	mmio_start = pci_resource_start(pdev, 0);
3251 	mmio_len = pci_resource_len(pdev, 0);
3252 	ai = t3_get_adapter_info(ent->driver_data);
3253 
3254 	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3255 	if (!adapter) {
3256 		err = -ENOMEM;
3257 		goto out_release_regions;
3258 	}
3259 
3260 	adapter->nofail_skb =
3261 		alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_KERNEL);
3262 	if (!adapter->nofail_skb) {
3263 		dev_err(&pdev->dev, "cannot allocate nofail buffer\n");
3264 		err = -ENOMEM;
3265 		goto out_free_adapter;
3266 	}
3267 
3268 	adapter->regs = ioremap_nocache(mmio_start, mmio_len);
3269 	if (!adapter->regs) {
3270 		dev_err(&pdev->dev, "cannot map device registers\n");
3271 		err = -ENOMEM;
3272 		goto out_free_adapter_nofail;
3273 	}
3274 
3275 	adapter->pdev = pdev;
3276 	adapter->name = pci_name(pdev);
3277 	adapter->msg_enable = dflt_msg_enable;
3278 	adapter->mmio_len = mmio_len;
3279 
3280 	mutex_init(&adapter->mdio_lock);
3281 	spin_lock_init(&adapter->work_lock);
3282 	spin_lock_init(&adapter->stats_lock);
3283 
3284 	INIT_LIST_HEAD(&adapter->adapter_list);
3285 	INIT_WORK(&adapter->ext_intr_handler_task, ext_intr_task);
3286 	INIT_WORK(&adapter->fatal_error_handler_task, fatal_error_task);
3287 
3288 	INIT_WORK(&adapter->db_full_task, db_full_task);
3289 	INIT_WORK(&adapter->db_empty_task, db_empty_task);
3290 	INIT_WORK(&adapter->db_drop_task, db_drop_task);
3291 
3292 	INIT_DELAYED_WORK(&adapter->adap_check_task, t3_adap_check_task);
3293 
3294 	for (i = 0; i < ai->nports0 + ai->nports1; ++i) {
3295 		struct net_device *netdev;
3296 
3297 		netdev = alloc_etherdev_mq(sizeof(struct port_info), SGE_QSETS);
3298 		if (!netdev) {
3299 			err = -ENOMEM;
3300 			goto out_free_dev;
3301 		}
3302 
3303 		SET_NETDEV_DEV(netdev, &pdev->dev);
3304 
3305 		adapter->port[i] = netdev;
3306 		pi = netdev_priv(netdev);
3307 		pi->adapter = adapter;
3308 		pi->port_id = i;
3309 		netif_carrier_off(netdev);
3310 		netdev->irq = pdev->irq;
3311 		netdev->mem_start = mmio_start;
3312 		netdev->mem_end = mmio_start + mmio_len - 1;
3313 		netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM |
3314 			NETIF_F_TSO | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX;
3315 		netdev->features |= netdev->hw_features |
3316 				    NETIF_F_HW_VLAN_CTAG_TX;
3317 		netdev->vlan_features |= netdev->features & VLAN_FEAT;
3318 		if (pci_using_dac)
3319 			netdev->features |= NETIF_F_HIGHDMA;
3320 
3321 		netdev->netdev_ops = &cxgb_netdev_ops;
3322 		netdev->ethtool_ops = &cxgb_ethtool_ops;
3323 		netdev->min_mtu = 81;
3324 		netdev->max_mtu = ETH_MAX_MTU;
3325 		netdev->dev_port = pi->port_id;
3326 	}
3327 
3328 	pci_set_drvdata(pdev, adapter);
3329 	if (t3_prep_adapter(adapter, ai, 1) < 0) {
3330 		err = -ENODEV;
3331 		goto out_free_dev;
3332 	}
3333 
3334 	/*
3335 	 * The card is now ready to go.  If any errors occur during device
3336 	 * registration we do not fail the whole card but rather proceed only
3337 	 * with the ports we manage to register successfully.  However we must
3338 	 * register at least one net device.
3339 	 */
3340 	for_each_port(adapter, i) {
3341 		err = register_netdev(adapter->port[i]);
3342 		if (err)
3343 			dev_warn(&pdev->dev,
3344 				 "cannot register net device %s, skipping\n",
3345 				 adapter->port[i]->name);
3346 		else {
3347 			/*
3348 			 * Change the name we use for messages to the name of
3349 			 * the first successfully registered interface.
3350 			 */
3351 			if (!adapter->registered_device_map)
3352 				adapter->name = adapter->port[i]->name;
3353 
3354 			__set_bit(i, &adapter->registered_device_map);
3355 		}
3356 	}
3357 	if (!adapter->registered_device_map) {
3358 		dev_err(&pdev->dev, "could not register any net devices\n");
3359 		goto out_free_dev;
3360 	}
3361 
3362 	for_each_port(adapter, i)
3363 		cxgb3_init_iscsi_mac(adapter->port[i]);
3364 
3365 	/* Driver's ready. Reflect it on LEDs */
3366 	t3_led_ready(adapter);
3367 
3368 	if (is_offload(adapter)) {
3369 		__set_bit(OFFLOAD_DEVMAP_BIT, &adapter->registered_device_map);
3370 		cxgb3_adapter_ofld(adapter);
3371 	}
3372 
3373 	/* See what interrupts we'll be using */
3374 	if (msi > 1 && cxgb_enable_msix(adapter) == 0)
3375 		adapter->flags |= USING_MSIX;
3376 	else if (msi > 0 && pci_enable_msi(pdev) == 0)
3377 		adapter->flags |= USING_MSI;
3378 
3379 	set_nqsets(adapter);
3380 
3381 	err = sysfs_create_group(&adapter->port[0]->dev.kobj,
3382 				 &cxgb3_attr_group);
3383 	if (err) {
3384 		dev_err(&pdev->dev, "cannot create sysfs group\n");
3385 		goto out_close_led;
3386 	}
3387 
3388 	print_port_info(adapter, ai);
3389 	return 0;
3390 
3391 out_close_led:
3392 	t3_set_reg_field(adapter, A_T3DBG_GPIO_EN, F_GPIO0_OUT_VAL, 0);
3393 
3394 out_free_dev:
3395 	iounmap(adapter->regs);
3396 	for (i = ai->nports0 + ai->nports1 - 1; i >= 0; --i)
3397 		if (adapter->port[i])
3398 			free_netdev(adapter->port[i]);
3399 
3400 out_free_adapter_nofail:
3401 	kfree_skb(adapter->nofail_skb);
3402 
3403 out_free_adapter:
3404 	kfree(adapter);
3405 
3406 out_release_regions:
3407 	pci_release_regions(pdev);
3408 out_disable_device:
3409 	pci_disable_device(pdev);
3410 out:
3411 	return err;
3412 }
3413 
3414 static void remove_one(struct pci_dev *pdev)
3415 {
3416 	struct adapter *adapter = pci_get_drvdata(pdev);
3417 
3418 	if (adapter) {
3419 		int i;
3420 
3421 		t3_sge_stop(adapter);
3422 		sysfs_remove_group(&adapter->port[0]->dev.kobj,
3423 				   &cxgb3_attr_group);
3424 
3425 		if (is_offload(adapter)) {
3426 			cxgb3_adapter_unofld(adapter);
3427 			if (test_bit(OFFLOAD_DEVMAP_BIT,
3428 				     &adapter->open_device_map))
3429 				offload_close(&adapter->tdev);
3430 		}
3431 
3432 		for_each_port(adapter, i)
3433 		    if (test_bit(i, &adapter->registered_device_map))
3434 			unregister_netdev(adapter->port[i]);
3435 
3436 		t3_stop_sge_timers(adapter);
3437 		t3_free_sge_resources(adapter);
3438 		cxgb_disable_msi(adapter);
3439 
3440 		for_each_port(adapter, i)
3441 			if (adapter->port[i])
3442 				free_netdev(adapter->port[i]);
3443 
3444 		iounmap(adapter->regs);
3445 		kfree_skb(adapter->nofail_skb);
3446 		kfree(adapter);
3447 		pci_release_regions(pdev);
3448 		pci_disable_device(pdev);
3449 	}
3450 }
3451 
3452 static struct pci_driver driver = {
3453 	.name = DRV_NAME,
3454 	.id_table = cxgb3_pci_tbl,
3455 	.probe = init_one,
3456 	.remove = remove_one,
3457 	.err_handler = &t3_err_handler,
3458 };
3459 
3460 static int __init cxgb3_init_module(void)
3461 {
3462 	int ret;
3463 
3464 	cxgb3_offload_init();
3465 
3466 	ret = pci_register_driver(&driver);
3467 	return ret;
3468 }
3469 
3470 static void __exit cxgb3_cleanup_module(void)
3471 {
3472 	pci_unregister_driver(&driver);
3473 	if (cxgb3_wq)
3474 		destroy_workqueue(cxgb3_wq);
3475 }
3476 
3477 module_init(cxgb3_init_module);
3478 module_exit(cxgb3_cleanup_module);
3479