1 /*
2  * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet
3  * driver for Linux.
4  *
5  * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 
36 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
37 
38 #include <linux/module.h>
39 #include <linux/moduleparam.h>
40 #include <linux/init.h>
41 #include <linux/pci.h>
42 #include <linux/dma-mapping.h>
43 #include <linux/netdevice.h>
44 #include <linux/etherdevice.h>
45 #include <linux/debugfs.h>
46 #include <linux/ethtool.h>
47 #include <linux/mdio.h>
48 
49 #include "t4vf_common.h"
50 #include "t4vf_defs.h"
51 
52 #include "../cxgb4/t4_regs.h"
53 #include "../cxgb4/t4_msg.h"
54 
55 /*
56  * Generic information about the driver.
57  */
58 #define DRV_DESC "Chelsio T4/T5/T6 Virtual Function (VF) Network Driver"
59 
60 /*
61  * Module Parameters.
62  * ==================
63  */
64 
65 /*
66  * Default ethtool "message level" for adapters.
67  */
68 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
69 			 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
70 			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
71 
72 /*
73  * The driver uses the best interrupt scheme available on a platform in the
74  * order MSI-X then MSI.  This parameter determines which of these schemes the
75  * driver may consider as follows:
76  *
77  *     msi = 2: choose from among MSI-X and MSI
78  *     msi = 1: only consider MSI interrupts
79  *
80  * Note that unlike the Physical Function driver, this Virtual Function driver
81  * does _not_ support legacy INTx interrupts (this limitation is mandated by
82  * the PCI-E SR-IOV standard).
83  */
84 #define MSI_MSIX	2
85 #define MSI_MSI		1
86 #define MSI_DEFAULT	MSI_MSIX
87 
88 static int msi = MSI_DEFAULT;
89 
90 module_param(msi, int, 0644);
91 MODULE_PARM_DESC(msi, "whether to use MSI-X or MSI");
92 
93 /*
94  * Fundamental constants.
95  * ======================
96  */
97 
98 enum {
99 	MAX_TXQ_ENTRIES		= 16384,
100 	MAX_RSPQ_ENTRIES	= 16384,
101 	MAX_RX_BUFFERS		= 16384,
102 
103 	MIN_TXQ_ENTRIES		= 32,
104 	MIN_RSPQ_ENTRIES	= 128,
105 	MIN_FL_ENTRIES		= 16,
106 
107 	/*
108 	 * For purposes of manipulating the Free List size we need to
109 	 * recognize that Free Lists are actually Egress Queues (the host
110 	 * produces free buffers which the hardware consumes), Egress Queues
111 	 * indices are all in units of Egress Context Units bytes, and free
112 	 * list entries are 64-bit PCI DMA addresses.  And since the state of
113 	 * the Producer Index == the Consumer Index implies an EMPTY list, we
114 	 * always have at least one Egress Unit's worth of Free List entries
115 	 * unused.  See sge.c for more details ...
116 	 */
117 	EQ_UNIT = SGE_EQ_IDXSIZE,
118 	FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
119 	MIN_FL_RESID = FL_PER_EQ_UNIT,
120 };
121 
122 /*
123  * Global driver state.
124  * ====================
125  */
126 
127 static struct dentry *cxgb4vf_debugfs_root;
128 
129 /*
130  * OS "Callback" functions.
131  * ========================
132  */
133 
134 /*
135  * The link status has changed on the indicated "port" (Virtual Interface).
136  */
137 void t4vf_os_link_changed(struct adapter *adapter, int pidx, int link_ok)
138 {
139 	struct net_device *dev = adapter->port[pidx];
140 
141 	/*
142 	 * If the port is disabled or the current recorded "link up"
143 	 * status matches the new status, just return.
144 	 */
145 	if (!netif_running(dev) || link_ok == netif_carrier_ok(dev))
146 		return;
147 
148 	/*
149 	 * Tell the OS that the link status has changed and print a short
150 	 * informative message on the console about the event.
151 	 */
152 	if (link_ok) {
153 		const char *s;
154 		const char *fc;
155 		const struct port_info *pi = netdev_priv(dev);
156 
157 		netif_carrier_on(dev);
158 
159 		switch (pi->link_cfg.speed) {
160 		case 100:
161 			s = "100Mbps";
162 			break;
163 		case 1000:
164 			s = "1Gbps";
165 			break;
166 		case 10000:
167 			s = "10Gbps";
168 			break;
169 		case 25000:
170 			s = "25Gbps";
171 			break;
172 		case 40000:
173 			s = "40Gbps";
174 			break;
175 		case 100000:
176 			s = "100Gbps";
177 			break;
178 
179 		default:
180 			s = "unknown";
181 			break;
182 		}
183 
184 		switch ((int)pi->link_cfg.fc) {
185 		case PAUSE_RX:
186 			fc = "RX";
187 			break;
188 
189 		case PAUSE_TX:
190 			fc = "TX";
191 			break;
192 
193 		case PAUSE_RX | PAUSE_TX:
194 			fc = "RX/TX";
195 			break;
196 
197 		default:
198 			fc = "no";
199 			break;
200 		}
201 
202 		netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, fc);
203 	} else {
204 		netif_carrier_off(dev);
205 		netdev_info(dev, "link down\n");
206 	}
207 }
208 
209 /*
210  * THe port module type has changed on the indicated "port" (Virtual
211  * Interface).
212  */
213 void t4vf_os_portmod_changed(struct adapter *adapter, int pidx)
214 {
215 	static const char * const mod_str[] = {
216 		NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
217 	};
218 	const struct net_device *dev = adapter->port[pidx];
219 	const struct port_info *pi = netdev_priv(dev);
220 
221 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
222 		dev_info(adapter->pdev_dev, "%s: port module unplugged\n",
223 			 dev->name);
224 	else if (pi->mod_type < ARRAY_SIZE(mod_str))
225 		dev_info(adapter->pdev_dev, "%s: %s port module inserted\n",
226 			 dev->name, mod_str[pi->mod_type]);
227 	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
228 		dev_info(adapter->pdev_dev, "%s: unsupported optical port "
229 			 "module inserted\n", dev->name);
230 	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
231 		dev_info(adapter->pdev_dev, "%s: unknown port module inserted,"
232 			 "forcing TWINAX\n", dev->name);
233 	else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR)
234 		dev_info(adapter->pdev_dev, "%s: transceiver module error\n",
235 			 dev->name);
236 	else
237 		dev_info(adapter->pdev_dev, "%s: unknown module type %d "
238 			 "inserted\n", dev->name, pi->mod_type);
239 }
240 
241 static int cxgb4vf_set_addr_hash(struct port_info *pi)
242 {
243 	struct adapter *adapter = pi->adapter;
244 	u64 vec = 0;
245 	bool ucast = false;
246 	struct hash_mac_addr *entry;
247 
248 	/* Calculate the hash vector for the updated list and program it */
249 	list_for_each_entry(entry, &adapter->mac_hlist, list) {
250 		ucast |= is_unicast_ether_addr(entry->addr);
251 		vec |= (1ULL << hash_mac_addr(entry->addr));
252 	}
253 	return t4vf_set_addr_hash(adapter, pi->viid, ucast, vec, false);
254 }
255 
256 /**
257  *	cxgb4vf_change_mac - Update match filter for a MAC address.
258  *	@pi: the port_info
259  *	@viid: the VI id
260  *	@tcam_idx: TCAM index of existing filter for old value of MAC address,
261  *		   or -1
262  *	@addr: the new MAC address value
263  *	@persistent: whether a new MAC allocation should be persistent
264  *
265  *	Modifies an MPS filter and sets it to the new MAC address if
266  *	@tcam_idx >= 0, or adds the MAC address to a new filter if
267  *	@tcam_idx < 0. In the latter case the address is added persistently
268  *	if @persist is %true.
269  *	Addresses are programmed to hash region, if tcam runs out of entries.
270  *
271  */
272 static int cxgb4vf_change_mac(struct port_info *pi, unsigned int viid,
273 			      int *tcam_idx, const u8 *addr, bool persistent)
274 {
275 	struct hash_mac_addr *new_entry, *entry;
276 	struct adapter *adapter = pi->adapter;
277 	int ret;
278 
279 	ret = t4vf_change_mac(adapter, viid, *tcam_idx, addr, persistent);
280 	/* We ran out of TCAM entries. try programming hash region. */
281 	if (ret == -ENOMEM) {
282 		/* If the MAC address to be updated is in the hash addr
283 		 * list, update it from the list
284 		 */
285 		list_for_each_entry(entry, &adapter->mac_hlist, list) {
286 			if (entry->iface_mac) {
287 				ether_addr_copy(entry->addr, addr);
288 				goto set_hash;
289 			}
290 		}
291 		new_entry = kzalloc(sizeof(*new_entry), GFP_KERNEL);
292 		if (!new_entry)
293 			return -ENOMEM;
294 		ether_addr_copy(new_entry->addr, addr);
295 		new_entry->iface_mac = true;
296 		list_add_tail(&new_entry->list, &adapter->mac_hlist);
297 set_hash:
298 		ret = cxgb4vf_set_addr_hash(pi);
299 	} else if (ret >= 0) {
300 		*tcam_idx = ret;
301 		ret = 0;
302 	}
303 
304 	return ret;
305 }
306 
307 /*
308  * Net device operations.
309  * ======================
310  */
311 
312 
313 
314 
315 /*
316  * Perform the MAC and PHY actions needed to enable a "port" (Virtual
317  * Interface).
318  */
319 static int link_start(struct net_device *dev)
320 {
321 	int ret;
322 	struct port_info *pi = netdev_priv(dev);
323 
324 	/*
325 	 * We do not set address filters and promiscuity here, the stack does
326 	 * that step explicitly. Enable vlan accel.
327 	 */
328 	ret = t4vf_set_rxmode(pi->adapter, pi->viid, dev->mtu, -1, -1, -1, 1,
329 			      true);
330 	if (ret == 0)
331 		ret = cxgb4vf_change_mac(pi, pi->viid,
332 					 &pi->xact_addr_filt,
333 					 dev->dev_addr, true);
334 
335 	/*
336 	 * We don't need to actually "start the link" itself since the
337 	 * firmware will do that for us when the first Virtual Interface
338 	 * is enabled on a port.
339 	 */
340 	if (ret == 0)
341 		ret = t4vf_enable_pi(pi->adapter, pi, true, true);
342 
343 	return ret;
344 }
345 
346 /*
347  * Name the MSI-X interrupts.
348  */
349 static void name_msix_vecs(struct adapter *adapter)
350 {
351 	int namelen = sizeof(adapter->msix_info[0].desc) - 1;
352 	int pidx;
353 
354 	/*
355 	 * Firmware events.
356 	 */
357 	snprintf(adapter->msix_info[MSIX_FW].desc, namelen,
358 		 "%s-FWeventq", adapter->name);
359 	adapter->msix_info[MSIX_FW].desc[namelen] = 0;
360 
361 	/*
362 	 * Ethernet queues.
363 	 */
364 	for_each_port(adapter, pidx) {
365 		struct net_device *dev = adapter->port[pidx];
366 		const struct port_info *pi = netdev_priv(dev);
367 		int qs, msi;
368 
369 		for (qs = 0, msi = MSIX_IQFLINT; qs < pi->nqsets; qs++, msi++) {
370 			snprintf(adapter->msix_info[msi].desc, namelen,
371 				 "%s-%d", dev->name, qs);
372 			adapter->msix_info[msi].desc[namelen] = 0;
373 		}
374 	}
375 }
376 
377 /*
378  * Request all of our MSI-X resources.
379  */
380 static int request_msix_queue_irqs(struct adapter *adapter)
381 {
382 	struct sge *s = &adapter->sge;
383 	int rxq, msi, err;
384 
385 	/*
386 	 * Firmware events.
387 	 */
388 	err = request_irq(adapter->msix_info[MSIX_FW].vec, t4vf_sge_intr_msix,
389 			  0, adapter->msix_info[MSIX_FW].desc, &s->fw_evtq);
390 	if (err)
391 		return err;
392 
393 	/*
394 	 * Ethernet queues.
395 	 */
396 	msi = MSIX_IQFLINT;
397 	for_each_ethrxq(s, rxq) {
398 		err = request_irq(adapter->msix_info[msi].vec,
399 				  t4vf_sge_intr_msix, 0,
400 				  adapter->msix_info[msi].desc,
401 				  &s->ethrxq[rxq].rspq);
402 		if (err)
403 			goto err_free_irqs;
404 		msi++;
405 	}
406 	return 0;
407 
408 err_free_irqs:
409 	while (--rxq >= 0)
410 		free_irq(adapter->msix_info[--msi].vec, &s->ethrxq[rxq].rspq);
411 	free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq);
412 	return err;
413 }
414 
415 /*
416  * Free our MSI-X resources.
417  */
418 static void free_msix_queue_irqs(struct adapter *adapter)
419 {
420 	struct sge *s = &adapter->sge;
421 	int rxq, msi;
422 
423 	free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq);
424 	msi = MSIX_IQFLINT;
425 	for_each_ethrxq(s, rxq)
426 		free_irq(adapter->msix_info[msi++].vec,
427 			 &s->ethrxq[rxq].rspq);
428 }
429 
430 /*
431  * Turn on NAPI and start up interrupts on a response queue.
432  */
433 static void qenable(struct sge_rspq *rspq)
434 {
435 	napi_enable(&rspq->napi);
436 
437 	/*
438 	 * 0-increment the Going To Sleep register to start the timer and
439 	 * enable interrupts.
440 	 */
441 	t4_write_reg(rspq->adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
442 		     CIDXINC_V(0) |
443 		     SEINTARM_V(rspq->intr_params) |
444 		     INGRESSQID_V(rspq->cntxt_id));
445 }
446 
447 /*
448  * Enable NAPI scheduling and interrupt generation for all Receive Queues.
449  */
450 static void enable_rx(struct adapter *adapter)
451 {
452 	int rxq;
453 	struct sge *s = &adapter->sge;
454 
455 	for_each_ethrxq(s, rxq)
456 		qenable(&s->ethrxq[rxq].rspq);
457 	qenable(&s->fw_evtq);
458 
459 	/*
460 	 * The interrupt queue doesn't use NAPI so we do the 0-increment of
461 	 * its Going To Sleep register here to get it started.
462 	 */
463 	if (adapter->flags & CXGB4VF_USING_MSI)
464 		t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
465 			     CIDXINC_V(0) |
466 			     SEINTARM_V(s->intrq.intr_params) |
467 			     INGRESSQID_V(s->intrq.cntxt_id));
468 
469 }
470 
471 /*
472  * Wait until all NAPI handlers are descheduled.
473  */
474 static void quiesce_rx(struct adapter *adapter)
475 {
476 	struct sge *s = &adapter->sge;
477 	int rxq;
478 
479 	for_each_ethrxq(s, rxq)
480 		napi_disable(&s->ethrxq[rxq].rspq.napi);
481 	napi_disable(&s->fw_evtq.napi);
482 }
483 
484 /*
485  * Response queue handler for the firmware event queue.
486  */
487 static int fwevtq_handler(struct sge_rspq *rspq, const __be64 *rsp,
488 			  const struct pkt_gl *gl)
489 {
490 	/*
491 	 * Extract response opcode and get pointer to CPL message body.
492 	 */
493 	struct adapter *adapter = rspq->adapter;
494 	u8 opcode = ((const struct rss_header *)rsp)->opcode;
495 	void *cpl = (void *)(rsp + 1);
496 
497 	switch (opcode) {
498 	case CPL_FW6_MSG: {
499 		/*
500 		 * We've received an asynchronous message from the firmware.
501 		 */
502 		const struct cpl_fw6_msg *fw_msg = cpl;
503 		if (fw_msg->type == FW6_TYPE_CMD_RPL)
504 			t4vf_handle_fw_rpl(adapter, fw_msg->data);
505 		break;
506 	}
507 
508 	case CPL_FW4_MSG: {
509 		/* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
510 		 */
511 		const struct cpl_sge_egr_update *p = (void *)(rsp + 3);
512 		opcode = CPL_OPCODE_G(ntohl(p->opcode_qid));
513 		if (opcode != CPL_SGE_EGR_UPDATE) {
514 			dev_err(adapter->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
515 				, opcode);
516 			break;
517 		}
518 		cpl = (void *)p;
519 	}
520 		fallthrough;
521 
522 	case CPL_SGE_EGR_UPDATE: {
523 		/*
524 		 * We've received an Egress Queue Status Update message.  We
525 		 * get these, if the SGE is configured to send these when the
526 		 * firmware passes certain points in processing our TX
527 		 * Ethernet Queue or if we make an explicit request for one.
528 		 * We use these updates to determine when we may need to
529 		 * restart a TX Ethernet Queue which was stopped for lack of
530 		 * free TX Queue Descriptors ...
531 		 */
532 		const struct cpl_sge_egr_update *p = cpl;
533 		unsigned int qid = EGR_QID_G(be32_to_cpu(p->opcode_qid));
534 		struct sge *s = &adapter->sge;
535 		struct sge_txq *tq;
536 		struct sge_eth_txq *txq;
537 		unsigned int eq_idx;
538 
539 		/*
540 		 * Perform sanity checking on the Queue ID to make sure it
541 		 * really refers to one of our TX Ethernet Egress Queues which
542 		 * is active and matches the queue's ID.  None of these error
543 		 * conditions should ever happen so we may want to either make
544 		 * them fatal and/or conditionalized under DEBUG.
545 		 */
546 		eq_idx = EQ_IDX(s, qid);
547 		if (unlikely(eq_idx >= MAX_EGRQ)) {
548 			dev_err(adapter->pdev_dev,
549 				"Egress Update QID %d out of range\n", qid);
550 			break;
551 		}
552 		tq = s->egr_map[eq_idx];
553 		if (unlikely(tq == NULL)) {
554 			dev_err(adapter->pdev_dev,
555 				"Egress Update QID %d TXQ=NULL\n", qid);
556 			break;
557 		}
558 		txq = container_of(tq, struct sge_eth_txq, q);
559 		if (unlikely(tq->abs_id != qid)) {
560 			dev_err(adapter->pdev_dev,
561 				"Egress Update QID %d refers to TXQ %d\n",
562 				qid, tq->abs_id);
563 			break;
564 		}
565 
566 		/*
567 		 * Restart a stopped TX Queue which has less than half of its
568 		 * TX ring in use ...
569 		 */
570 		txq->q.restarts++;
571 		netif_tx_wake_queue(txq->txq);
572 		break;
573 	}
574 
575 	default:
576 		dev_err(adapter->pdev_dev,
577 			"unexpected CPL %#x on FW event queue\n", opcode);
578 	}
579 
580 	return 0;
581 }
582 
583 /*
584  * Allocate SGE TX/RX response queues.  Determine how many sets of SGE queues
585  * to use and initializes them.  We support multiple "Queue Sets" per port if
586  * we have MSI-X, otherwise just one queue set per port.
587  */
588 static int setup_sge_queues(struct adapter *adapter)
589 {
590 	struct sge *s = &adapter->sge;
591 	int err, pidx, msix;
592 
593 	/*
594 	 * Clear "Queue Set" Free List Starving and TX Queue Mapping Error
595 	 * state.
596 	 */
597 	bitmap_zero(s->starving_fl, MAX_EGRQ);
598 
599 	/*
600 	 * If we're using MSI interrupt mode we need to set up a "forwarded
601 	 * interrupt" queue which we'll set up with our MSI vector.  The rest
602 	 * of the ingress queues will be set up to forward their interrupts to
603 	 * this queue ...  This must be first since t4vf_sge_alloc_rxq() uses
604 	 * the intrq's queue ID as the interrupt forwarding queue for the
605 	 * subsequent calls ...
606 	 */
607 	if (adapter->flags & CXGB4VF_USING_MSI) {
608 		err = t4vf_sge_alloc_rxq(adapter, &s->intrq, false,
609 					 adapter->port[0], 0, NULL, NULL);
610 		if (err)
611 			goto err_free_queues;
612 	}
613 
614 	/*
615 	 * Allocate our ingress queue for asynchronous firmware messages.
616 	 */
617 	err = t4vf_sge_alloc_rxq(adapter, &s->fw_evtq, true, adapter->port[0],
618 				 MSIX_FW, NULL, fwevtq_handler);
619 	if (err)
620 		goto err_free_queues;
621 
622 	/*
623 	 * Allocate each "port"'s initial Queue Sets.  These can be changed
624 	 * later on ... up to the point where any interface on the adapter is
625 	 * brought up at which point lots of things get nailed down
626 	 * permanently ...
627 	 */
628 	msix = MSIX_IQFLINT;
629 	for_each_port(adapter, pidx) {
630 		struct net_device *dev = adapter->port[pidx];
631 		struct port_info *pi = netdev_priv(dev);
632 		struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset];
633 		struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset];
634 		int qs;
635 
636 		for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
637 			err = t4vf_sge_alloc_rxq(adapter, &rxq->rspq, false,
638 						 dev, msix++,
639 						 &rxq->fl, t4vf_ethrx_handler);
640 			if (err)
641 				goto err_free_queues;
642 
643 			err = t4vf_sge_alloc_eth_txq(adapter, txq, dev,
644 					     netdev_get_tx_queue(dev, qs),
645 					     s->fw_evtq.cntxt_id);
646 			if (err)
647 				goto err_free_queues;
648 
649 			rxq->rspq.idx = qs;
650 			memset(&rxq->stats, 0, sizeof(rxq->stats));
651 		}
652 	}
653 
654 	/*
655 	 * Create the reverse mappings for the queues.
656 	 */
657 	s->egr_base = s->ethtxq[0].q.abs_id - s->ethtxq[0].q.cntxt_id;
658 	s->ingr_base = s->ethrxq[0].rspq.abs_id - s->ethrxq[0].rspq.cntxt_id;
659 	IQ_MAP(s, s->fw_evtq.abs_id) = &s->fw_evtq;
660 	for_each_port(adapter, pidx) {
661 		struct net_device *dev = adapter->port[pidx];
662 		struct port_info *pi = netdev_priv(dev);
663 		struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset];
664 		struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset];
665 		int qs;
666 
667 		for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
668 			IQ_MAP(s, rxq->rspq.abs_id) = &rxq->rspq;
669 			EQ_MAP(s, txq->q.abs_id) = &txq->q;
670 
671 			/*
672 			 * The FW_IQ_CMD doesn't return the Absolute Queue IDs
673 			 * for Free Lists but since all of the Egress Queues
674 			 * (including Free Lists) have Relative Queue IDs
675 			 * which are computed as Absolute - Base Queue ID, we
676 			 * can synthesize the Absolute Queue IDs for the Free
677 			 * Lists.  This is useful for debugging purposes when
678 			 * we want to dump Queue Contexts via the PF Driver.
679 			 */
680 			rxq->fl.abs_id = rxq->fl.cntxt_id + s->egr_base;
681 			EQ_MAP(s, rxq->fl.abs_id) = &rxq->fl;
682 		}
683 	}
684 	return 0;
685 
686 err_free_queues:
687 	t4vf_free_sge_resources(adapter);
688 	return err;
689 }
690 
691 /*
692  * Set up Receive Side Scaling (RSS) to distribute packets to multiple receive
693  * queues.  We configure the RSS CPU lookup table to distribute to the number
694  * of HW receive queues, and the response queue lookup table to narrow that
695  * down to the response queues actually configured for each "port" (Virtual
696  * Interface).  We always configure the RSS mapping for all ports since the
697  * mapping table has plenty of entries.
698  */
699 static int setup_rss(struct adapter *adapter)
700 {
701 	int pidx;
702 
703 	for_each_port(adapter, pidx) {
704 		struct port_info *pi = adap2pinfo(adapter, pidx);
705 		struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset];
706 		u16 rss[MAX_PORT_QSETS];
707 		int qs, err;
708 
709 		for (qs = 0; qs < pi->nqsets; qs++)
710 			rss[qs] = rxq[qs].rspq.abs_id;
711 
712 		err = t4vf_config_rss_range(adapter, pi->viid,
713 					    0, pi->rss_size, rss, pi->nqsets);
714 		if (err)
715 			return err;
716 
717 		/*
718 		 * Perform Global RSS Mode-specific initialization.
719 		 */
720 		switch (adapter->params.rss.mode) {
721 		case FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL:
722 			/*
723 			 * If Tunnel All Lookup isn't specified in the global
724 			 * RSS Configuration, then we need to specify a
725 			 * default Ingress Queue for any ingress packets which
726 			 * aren't hashed.  We'll use our first ingress queue
727 			 * ...
728 			 */
729 			if (!adapter->params.rss.u.basicvirtual.tnlalllookup) {
730 				union rss_vi_config config;
731 				err = t4vf_read_rss_vi_config(adapter,
732 							      pi->viid,
733 							      &config);
734 				if (err)
735 					return err;
736 				config.basicvirtual.defaultq =
737 					rxq[0].rspq.abs_id;
738 				err = t4vf_write_rss_vi_config(adapter,
739 							       pi->viid,
740 							       &config);
741 				if (err)
742 					return err;
743 			}
744 			break;
745 		}
746 	}
747 
748 	return 0;
749 }
750 
751 /*
752  * Bring the adapter up.  Called whenever we go from no "ports" open to having
753  * one open.  This function performs the actions necessary to make an adapter
754  * operational, such as completing the initialization of HW modules, and
755  * enabling interrupts.  Must be called with the rtnl lock held.  (Note that
756  * this is called "cxgb_up" in the PF Driver.)
757  */
758 static int adapter_up(struct adapter *adapter)
759 {
760 	int err;
761 
762 	/*
763 	 * If this is the first time we've been called, perform basic
764 	 * adapter setup.  Once we've done this, many of our adapter
765 	 * parameters can no longer be changed ...
766 	 */
767 	if ((adapter->flags & CXGB4VF_FULL_INIT_DONE) == 0) {
768 		err = setup_sge_queues(adapter);
769 		if (err)
770 			return err;
771 		err = setup_rss(adapter);
772 		if (err) {
773 			t4vf_free_sge_resources(adapter);
774 			return err;
775 		}
776 
777 		if (adapter->flags & CXGB4VF_USING_MSIX)
778 			name_msix_vecs(adapter);
779 
780 		adapter->flags |= CXGB4VF_FULL_INIT_DONE;
781 	}
782 
783 	/*
784 	 * Acquire our interrupt resources.  We only support MSI-X and MSI.
785 	 */
786 	BUG_ON((adapter->flags &
787 	       (CXGB4VF_USING_MSIX | CXGB4VF_USING_MSI)) == 0);
788 	if (adapter->flags & CXGB4VF_USING_MSIX)
789 		err = request_msix_queue_irqs(adapter);
790 	else
791 		err = request_irq(adapter->pdev->irq,
792 				  t4vf_intr_handler(adapter), 0,
793 				  adapter->name, adapter);
794 	if (err) {
795 		dev_err(adapter->pdev_dev, "request_irq failed, err %d\n",
796 			err);
797 		return err;
798 	}
799 
800 	/*
801 	 * Enable NAPI ingress processing and return success.
802 	 */
803 	enable_rx(adapter);
804 	t4vf_sge_start(adapter);
805 
806 	return 0;
807 }
808 
809 /*
810  * Bring the adapter down.  Called whenever the last "port" (Virtual
811  * Interface) closed.  (Note that this routine is called "cxgb_down" in the PF
812  * Driver.)
813  */
814 static void adapter_down(struct adapter *adapter)
815 {
816 	/*
817 	 * Free interrupt resources.
818 	 */
819 	if (adapter->flags & CXGB4VF_USING_MSIX)
820 		free_msix_queue_irqs(adapter);
821 	else
822 		free_irq(adapter->pdev->irq, adapter);
823 
824 	/*
825 	 * Wait for NAPI handlers to finish.
826 	 */
827 	quiesce_rx(adapter);
828 }
829 
830 /*
831  * Start up a net device.
832  */
833 static int cxgb4vf_open(struct net_device *dev)
834 {
835 	int err;
836 	struct port_info *pi = netdev_priv(dev);
837 	struct adapter *adapter = pi->adapter;
838 
839 	/*
840 	 * If we don't have a connection to the firmware there's nothing we
841 	 * can do.
842 	 */
843 	if (!(adapter->flags & CXGB4VF_FW_OK))
844 		return -ENXIO;
845 
846 	/*
847 	 * If this is the first interface that we're opening on the "adapter",
848 	 * bring the "adapter" up now.
849 	 */
850 	if (adapter->open_device_map == 0) {
851 		err = adapter_up(adapter);
852 		if (err)
853 			return err;
854 	}
855 
856 	/* It's possible that the basic port information could have
857 	 * changed since we first read it.
858 	 */
859 	err = t4vf_update_port_info(pi);
860 	if (err < 0)
861 		return err;
862 
863 	/*
864 	 * Note that this interface is up and start everything up ...
865 	 */
866 	err = link_start(dev);
867 	if (err)
868 		goto err_unwind;
869 
870 	pi->vlan_id = t4vf_get_vf_vlan_acl(adapter);
871 
872 	netif_tx_start_all_queues(dev);
873 	set_bit(pi->port_id, &adapter->open_device_map);
874 	return 0;
875 
876 err_unwind:
877 	if (adapter->open_device_map == 0)
878 		adapter_down(adapter);
879 	return err;
880 }
881 
882 /*
883  * Shut down a net device.  This routine is called "cxgb_close" in the PF
884  * Driver ...
885  */
886 static int cxgb4vf_stop(struct net_device *dev)
887 {
888 	struct port_info *pi = netdev_priv(dev);
889 	struct adapter *adapter = pi->adapter;
890 
891 	netif_tx_stop_all_queues(dev);
892 	netif_carrier_off(dev);
893 	t4vf_enable_pi(adapter, pi, false, false);
894 
895 	clear_bit(pi->port_id, &adapter->open_device_map);
896 	if (adapter->open_device_map == 0)
897 		adapter_down(adapter);
898 	return 0;
899 }
900 
901 /*
902  * Translate our basic statistics into the standard "ifconfig" statistics.
903  */
904 static struct net_device_stats *cxgb4vf_get_stats(struct net_device *dev)
905 {
906 	struct t4vf_port_stats stats;
907 	struct port_info *pi = netdev2pinfo(dev);
908 	struct adapter *adapter = pi->adapter;
909 	struct net_device_stats *ns = &dev->stats;
910 	int err;
911 
912 	spin_lock(&adapter->stats_lock);
913 	err = t4vf_get_port_stats(adapter, pi->pidx, &stats);
914 	spin_unlock(&adapter->stats_lock);
915 
916 	memset(ns, 0, sizeof(*ns));
917 	if (err)
918 		return ns;
919 
920 	ns->tx_bytes = (stats.tx_bcast_bytes + stats.tx_mcast_bytes +
921 			stats.tx_ucast_bytes + stats.tx_offload_bytes);
922 	ns->tx_packets = (stats.tx_bcast_frames + stats.tx_mcast_frames +
923 			  stats.tx_ucast_frames + stats.tx_offload_frames);
924 	ns->rx_bytes = (stats.rx_bcast_bytes + stats.rx_mcast_bytes +
925 			stats.rx_ucast_bytes);
926 	ns->rx_packets = (stats.rx_bcast_frames + stats.rx_mcast_frames +
927 			  stats.rx_ucast_frames);
928 	ns->multicast = stats.rx_mcast_frames;
929 	ns->tx_errors = stats.tx_drop_frames;
930 	ns->rx_errors = stats.rx_err_frames;
931 
932 	return ns;
933 }
934 
935 static int cxgb4vf_mac_sync(struct net_device *netdev, const u8 *mac_addr)
936 {
937 	struct port_info *pi = netdev_priv(netdev);
938 	struct adapter *adapter = pi->adapter;
939 	int ret;
940 	u64 mhash = 0;
941 	u64 uhash = 0;
942 	bool free = false;
943 	bool ucast = is_unicast_ether_addr(mac_addr);
944 	const u8 *maclist[1] = {mac_addr};
945 	struct hash_mac_addr *new_entry;
946 
947 	ret = t4vf_alloc_mac_filt(adapter, pi->viid, free, 1, maclist,
948 				  NULL, ucast ? &uhash : &mhash, false);
949 	if (ret < 0)
950 		goto out;
951 	/* if hash != 0, then add the addr to hash addr list
952 	 * so on the end we will calculate the hash for the
953 	 * list and program it
954 	 */
955 	if (uhash || mhash) {
956 		new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC);
957 		if (!new_entry)
958 			return -ENOMEM;
959 		ether_addr_copy(new_entry->addr, mac_addr);
960 		list_add_tail(&new_entry->list, &adapter->mac_hlist);
961 		ret = cxgb4vf_set_addr_hash(pi);
962 	}
963 out:
964 	return ret < 0 ? ret : 0;
965 }
966 
967 static int cxgb4vf_mac_unsync(struct net_device *netdev, const u8 *mac_addr)
968 {
969 	struct port_info *pi = netdev_priv(netdev);
970 	struct adapter *adapter = pi->adapter;
971 	int ret;
972 	const u8 *maclist[1] = {mac_addr};
973 	struct hash_mac_addr *entry, *tmp;
974 
975 	/* If the MAC address to be removed is in the hash addr
976 	 * list, delete it from the list and update hash vector
977 	 */
978 	list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist, list) {
979 		if (ether_addr_equal(entry->addr, mac_addr)) {
980 			list_del(&entry->list);
981 			kfree(entry);
982 			return cxgb4vf_set_addr_hash(pi);
983 		}
984 	}
985 
986 	ret = t4vf_free_mac_filt(adapter, pi->viid, 1, maclist, false);
987 	return ret < 0 ? -EINVAL : 0;
988 }
989 
990 /*
991  * Set RX properties of a port, such as promiscruity, address filters, and MTU.
992  * If @mtu is -1 it is left unchanged.
993  */
994 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
995 {
996 	struct port_info *pi = netdev_priv(dev);
997 
998 	__dev_uc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync);
999 	__dev_mc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync);
1000 	return t4vf_set_rxmode(pi->adapter, pi->viid, -1,
1001 			       (dev->flags & IFF_PROMISC) != 0,
1002 			       (dev->flags & IFF_ALLMULTI) != 0,
1003 			       1, -1, sleep_ok);
1004 }
1005 
1006 /*
1007  * Set the current receive modes on the device.
1008  */
1009 static void cxgb4vf_set_rxmode(struct net_device *dev)
1010 {
1011 	/* unfortunately we can't return errors to the stack */
1012 	set_rxmode(dev, -1, false);
1013 }
1014 
1015 /*
1016  * Find the entry in the interrupt holdoff timer value array which comes
1017  * closest to the specified interrupt holdoff value.
1018  */
1019 static int closest_timer(const struct sge *s, int us)
1020 {
1021 	int i, timer_idx = 0, min_delta = INT_MAX;
1022 
1023 	for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1024 		int delta = us - s->timer_val[i];
1025 		if (delta < 0)
1026 			delta = -delta;
1027 		if (delta < min_delta) {
1028 			min_delta = delta;
1029 			timer_idx = i;
1030 		}
1031 	}
1032 	return timer_idx;
1033 }
1034 
1035 static int closest_thres(const struct sge *s, int thres)
1036 {
1037 	int i, delta, pktcnt_idx = 0, min_delta = INT_MAX;
1038 
1039 	for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1040 		delta = thres - s->counter_val[i];
1041 		if (delta < 0)
1042 			delta = -delta;
1043 		if (delta < min_delta) {
1044 			min_delta = delta;
1045 			pktcnt_idx = i;
1046 		}
1047 	}
1048 	return pktcnt_idx;
1049 }
1050 
1051 /*
1052  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1053  */
1054 static unsigned int qtimer_val(const struct adapter *adapter,
1055 			       const struct sge_rspq *rspq)
1056 {
1057 	unsigned int timer_idx = QINTR_TIMER_IDX_G(rspq->intr_params);
1058 
1059 	return timer_idx < SGE_NTIMERS
1060 		? adapter->sge.timer_val[timer_idx]
1061 		: 0;
1062 }
1063 
1064 /**
1065  *	set_rxq_intr_params - set a queue's interrupt holdoff parameters
1066  *	@adapter: the adapter
1067  *	@rspq: the RX response queue
1068  *	@us: the hold-off time in us, or 0 to disable timer
1069  *	@cnt: the hold-off packet count, or 0 to disable counter
1070  *
1071  *	Sets an RX response queue's interrupt hold-off time and packet count.
1072  *	At least one of the two needs to be enabled for the queue to generate
1073  *	interrupts.
1074  */
1075 static int set_rxq_intr_params(struct adapter *adapter, struct sge_rspq *rspq,
1076 			       unsigned int us, unsigned int cnt)
1077 {
1078 	unsigned int timer_idx;
1079 
1080 	/*
1081 	 * If both the interrupt holdoff timer and count are specified as
1082 	 * zero, default to a holdoff count of 1 ...
1083 	 */
1084 	if ((us | cnt) == 0)
1085 		cnt = 1;
1086 
1087 	/*
1088 	 * If an interrupt holdoff count has been specified, then find the
1089 	 * closest configured holdoff count and use that.  If the response
1090 	 * queue has already been created, then update its queue context
1091 	 * parameters ...
1092 	 */
1093 	if (cnt) {
1094 		int err;
1095 		u32 v, pktcnt_idx;
1096 
1097 		pktcnt_idx = closest_thres(&adapter->sge, cnt);
1098 		if (rspq->desc && rspq->pktcnt_idx != pktcnt_idx) {
1099 			v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
1100 			    FW_PARAMS_PARAM_X_V(
1101 					FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1102 			    FW_PARAMS_PARAM_YZ_V(rspq->cntxt_id);
1103 			err = t4vf_set_params(adapter, 1, &v, &pktcnt_idx);
1104 			if (err)
1105 				return err;
1106 		}
1107 		rspq->pktcnt_idx = pktcnt_idx;
1108 	}
1109 
1110 	/*
1111 	 * Compute the closest holdoff timer index from the supplied holdoff
1112 	 * timer value.
1113 	 */
1114 	timer_idx = (us == 0
1115 		     ? SGE_TIMER_RSTRT_CNTR
1116 		     : closest_timer(&adapter->sge, us));
1117 
1118 	/*
1119 	 * Update the response queue's interrupt coalescing parameters and
1120 	 * return success.
1121 	 */
1122 	rspq->intr_params = (QINTR_TIMER_IDX_V(timer_idx) |
1123 			     QINTR_CNT_EN_V(cnt > 0));
1124 	return 0;
1125 }
1126 
1127 /*
1128  * Return a version number to identify the type of adapter.  The scheme is:
1129  * - bits 0..9: chip version
1130  * - bits 10..15: chip revision
1131  */
1132 static inline unsigned int mk_adap_vers(const struct adapter *adapter)
1133 {
1134 	/*
1135 	 * Chip version 4, revision 0x3f (cxgb4vf).
1136 	 */
1137 	return CHELSIO_CHIP_VERSION(adapter->params.chip) | (0x3f << 10);
1138 }
1139 
1140 /*
1141  * Execute the specified ioctl command.
1142  */
1143 static int cxgb4vf_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
1144 {
1145 	int ret = 0;
1146 
1147 	switch (cmd) {
1148 	    /*
1149 	     * The VF Driver doesn't have access to any of the other
1150 	     * common Ethernet device ioctl()'s (like reading/writing
1151 	     * PHY registers, etc.
1152 	     */
1153 
1154 	default:
1155 		ret = -EOPNOTSUPP;
1156 		break;
1157 	}
1158 	return ret;
1159 }
1160 
1161 /*
1162  * Change the device's MTU.
1163  */
1164 static int cxgb4vf_change_mtu(struct net_device *dev, int new_mtu)
1165 {
1166 	int ret;
1167 	struct port_info *pi = netdev_priv(dev);
1168 
1169 	ret = t4vf_set_rxmode(pi->adapter, pi->viid, new_mtu,
1170 			      -1, -1, -1, -1, true);
1171 	if (!ret)
1172 		dev->mtu = new_mtu;
1173 	return ret;
1174 }
1175 
1176 static netdev_features_t cxgb4vf_fix_features(struct net_device *dev,
1177 	netdev_features_t features)
1178 {
1179 	/*
1180 	 * Since there is no support for separate rx/tx vlan accel
1181 	 * enable/disable make sure tx flag is always in same state as rx.
1182 	 */
1183 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
1184 		features |= NETIF_F_HW_VLAN_CTAG_TX;
1185 	else
1186 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
1187 
1188 	return features;
1189 }
1190 
1191 static int cxgb4vf_set_features(struct net_device *dev,
1192 	netdev_features_t features)
1193 {
1194 	struct port_info *pi = netdev_priv(dev);
1195 	netdev_features_t changed = dev->features ^ features;
1196 
1197 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1198 		t4vf_set_rxmode(pi->adapter, pi->viid, -1, -1, -1, -1,
1199 				features & NETIF_F_HW_VLAN_CTAG_TX, 0);
1200 
1201 	return 0;
1202 }
1203 
1204 /*
1205  * Change the devices MAC address.
1206  */
1207 static int cxgb4vf_set_mac_addr(struct net_device *dev, void *_addr)
1208 {
1209 	int ret;
1210 	struct sockaddr *addr = _addr;
1211 	struct port_info *pi = netdev_priv(dev);
1212 
1213 	if (!is_valid_ether_addr(addr->sa_data))
1214 		return -EADDRNOTAVAIL;
1215 
1216 	ret = cxgb4vf_change_mac(pi, pi->viid, &pi->xact_addr_filt,
1217 				 addr->sa_data, true);
1218 	if (ret < 0)
1219 		return ret;
1220 
1221 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1222 	return 0;
1223 }
1224 
1225 #ifdef CONFIG_NET_POLL_CONTROLLER
1226 /*
1227  * Poll all of our receive queues.  This is called outside of normal interrupt
1228  * context.
1229  */
1230 static void cxgb4vf_poll_controller(struct net_device *dev)
1231 {
1232 	struct port_info *pi = netdev_priv(dev);
1233 	struct adapter *adapter = pi->adapter;
1234 
1235 	if (adapter->flags & CXGB4VF_USING_MSIX) {
1236 		struct sge_eth_rxq *rxq;
1237 		int nqsets;
1238 
1239 		rxq = &adapter->sge.ethrxq[pi->first_qset];
1240 		for (nqsets = pi->nqsets; nqsets; nqsets--) {
1241 			t4vf_sge_intr_msix(0, &rxq->rspq);
1242 			rxq++;
1243 		}
1244 	} else
1245 		t4vf_intr_handler(adapter)(0, adapter);
1246 }
1247 #endif
1248 
1249 /*
1250  * Ethtool operations.
1251  * ===================
1252  *
1253  * Note that we don't support any ethtool operations which change the physical
1254  * state of the port to which we're linked.
1255  */
1256 
1257 /**
1258  *	from_fw_port_mod_type - translate Firmware Port/Module type to Ethtool
1259  *	@port_type: Firmware Port Type
1260  *	@mod_type: Firmware Module Type
1261  *
1262  *	Translate Firmware Port/Module type to Ethtool Port Type.
1263  */
1264 static int from_fw_port_mod_type(enum fw_port_type port_type,
1265 				 enum fw_port_module_type mod_type)
1266 {
1267 	if (port_type == FW_PORT_TYPE_BT_SGMII ||
1268 	    port_type == FW_PORT_TYPE_BT_XFI ||
1269 	    port_type == FW_PORT_TYPE_BT_XAUI) {
1270 		return PORT_TP;
1271 	} else if (port_type == FW_PORT_TYPE_FIBER_XFI ||
1272 		   port_type == FW_PORT_TYPE_FIBER_XAUI) {
1273 		return PORT_FIBRE;
1274 	} else if (port_type == FW_PORT_TYPE_SFP ||
1275 		   port_type == FW_PORT_TYPE_QSFP_10G ||
1276 		   port_type == FW_PORT_TYPE_QSA ||
1277 		   port_type == FW_PORT_TYPE_QSFP ||
1278 		   port_type == FW_PORT_TYPE_CR4_QSFP ||
1279 		   port_type == FW_PORT_TYPE_CR_QSFP ||
1280 		   port_type == FW_PORT_TYPE_CR2_QSFP ||
1281 		   port_type == FW_PORT_TYPE_SFP28) {
1282 		if (mod_type == FW_PORT_MOD_TYPE_LR ||
1283 		    mod_type == FW_PORT_MOD_TYPE_SR ||
1284 		    mod_type == FW_PORT_MOD_TYPE_ER ||
1285 		    mod_type == FW_PORT_MOD_TYPE_LRM)
1286 			return PORT_FIBRE;
1287 		else if (mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1288 			 mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1289 			return PORT_DA;
1290 		else
1291 			return PORT_OTHER;
1292 	} else if (port_type == FW_PORT_TYPE_KR4_100G ||
1293 		   port_type == FW_PORT_TYPE_KR_SFP28 ||
1294 		   port_type == FW_PORT_TYPE_KR_XLAUI) {
1295 		return PORT_NONE;
1296 	}
1297 
1298 	return PORT_OTHER;
1299 }
1300 
1301 /**
1302  *	fw_caps_to_lmm - translate Firmware to ethtool Link Mode Mask
1303  *	@port_type: Firmware Port Type
1304  *	@fw_caps: Firmware Port Capabilities
1305  *	@link_mode_mask: ethtool Link Mode Mask
1306  *
1307  *	Translate a Firmware Port Capabilities specification to an ethtool
1308  *	Link Mode Mask.
1309  */
1310 static void fw_caps_to_lmm(enum fw_port_type port_type,
1311 			   unsigned int fw_caps,
1312 			   unsigned long *link_mode_mask)
1313 {
1314 	#define SET_LMM(__lmm_name) \
1315 		__set_bit(ETHTOOL_LINK_MODE_ ## __lmm_name ## _BIT, \
1316 			  link_mode_mask)
1317 
1318 	#define FW_CAPS_TO_LMM(__fw_name, __lmm_name) \
1319 		do { \
1320 			if (fw_caps & FW_PORT_CAP32_ ## __fw_name) \
1321 				SET_LMM(__lmm_name); \
1322 		} while (0)
1323 
1324 	switch (port_type) {
1325 	case FW_PORT_TYPE_BT_SGMII:
1326 	case FW_PORT_TYPE_BT_XFI:
1327 	case FW_PORT_TYPE_BT_XAUI:
1328 		SET_LMM(TP);
1329 		FW_CAPS_TO_LMM(SPEED_100M, 100baseT_Full);
1330 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1331 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full);
1332 		break;
1333 
1334 	case FW_PORT_TYPE_KX4:
1335 	case FW_PORT_TYPE_KX:
1336 		SET_LMM(Backplane);
1337 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
1338 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKX4_Full);
1339 		break;
1340 
1341 	case FW_PORT_TYPE_KR:
1342 		SET_LMM(Backplane);
1343 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1344 		break;
1345 
1346 	case FW_PORT_TYPE_BP_AP:
1347 		SET_LMM(Backplane);
1348 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
1349 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseR_FEC);
1350 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1351 		break;
1352 
1353 	case FW_PORT_TYPE_BP4_AP:
1354 		SET_LMM(Backplane);
1355 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
1356 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseR_FEC);
1357 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1358 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKX4_Full);
1359 		break;
1360 
1361 	case FW_PORT_TYPE_FIBER_XFI:
1362 	case FW_PORT_TYPE_FIBER_XAUI:
1363 	case FW_PORT_TYPE_SFP:
1364 	case FW_PORT_TYPE_QSFP_10G:
1365 	case FW_PORT_TYPE_QSA:
1366 		SET_LMM(FIBRE);
1367 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1368 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full);
1369 		break;
1370 
1371 	case FW_PORT_TYPE_BP40_BA:
1372 	case FW_PORT_TYPE_QSFP:
1373 		SET_LMM(FIBRE);
1374 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1375 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full);
1376 		FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full);
1377 		break;
1378 
1379 	case FW_PORT_TYPE_CR_QSFP:
1380 	case FW_PORT_TYPE_SFP28:
1381 		SET_LMM(FIBRE);
1382 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1383 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full);
1384 		FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full);
1385 		break;
1386 
1387 	case FW_PORT_TYPE_KR_SFP28:
1388 		SET_LMM(Backplane);
1389 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1390 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1391 		FW_CAPS_TO_LMM(SPEED_25G, 25000baseKR_Full);
1392 		break;
1393 
1394 	case FW_PORT_TYPE_KR_XLAUI:
1395 		SET_LMM(Backplane);
1396 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
1397 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1398 		FW_CAPS_TO_LMM(SPEED_40G, 40000baseKR4_Full);
1399 		break;
1400 
1401 	case FW_PORT_TYPE_CR2_QSFP:
1402 		SET_LMM(FIBRE);
1403 		FW_CAPS_TO_LMM(SPEED_50G, 50000baseSR2_Full);
1404 		break;
1405 
1406 	case FW_PORT_TYPE_KR4_100G:
1407 	case FW_PORT_TYPE_CR4_QSFP:
1408 		SET_LMM(FIBRE);
1409 		FW_CAPS_TO_LMM(SPEED_1G,  1000baseT_Full);
1410 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1411 		FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full);
1412 		FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full);
1413 		FW_CAPS_TO_LMM(SPEED_50G, 50000baseCR2_Full);
1414 		FW_CAPS_TO_LMM(SPEED_100G, 100000baseCR4_Full);
1415 		break;
1416 
1417 	default:
1418 		break;
1419 	}
1420 
1421 	if (fw_caps & FW_PORT_CAP32_FEC_V(FW_PORT_CAP32_FEC_M)) {
1422 		FW_CAPS_TO_LMM(FEC_RS, FEC_RS);
1423 		FW_CAPS_TO_LMM(FEC_BASER_RS, FEC_BASER);
1424 	} else {
1425 		SET_LMM(FEC_NONE);
1426 	}
1427 
1428 	FW_CAPS_TO_LMM(ANEG, Autoneg);
1429 	FW_CAPS_TO_LMM(802_3_PAUSE, Pause);
1430 	FW_CAPS_TO_LMM(802_3_ASM_DIR, Asym_Pause);
1431 
1432 	#undef FW_CAPS_TO_LMM
1433 	#undef SET_LMM
1434 }
1435 
1436 static int cxgb4vf_get_link_ksettings(struct net_device *dev,
1437 				  struct ethtool_link_ksettings *link_ksettings)
1438 {
1439 	struct port_info *pi = netdev_priv(dev);
1440 	struct ethtool_link_settings *base = &link_ksettings->base;
1441 
1442 	/* For the nonce, the Firmware doesn't send up Port State changes
1443 	 * when the Virtual Interface attached to the Port is down.  So
1444 	 * if it's down, let's grab any changes.
1445 	 */
1446 	if (!netif_running(dev))
1447 		(void)t4vf_update_port_info(pi);
1448 
1449 	ethtool_link_ksettings_zero_link_mode(link_ksettings, supported);
1450 	ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising);
1451 	ethtool_link_ksettings_zero_link_mode(link_ksettings, lp_advertising);
1452 
1453 	base->port = from_fw_port_mod_type(pi->port_type, pi->mod_type);
1454 
1455 	if (pi->mdio_addr >= 0) {
1456 		base->phy_address = pi->mdio_addr;
1457 		base->mdio_support = (pi->port_type == FW_PORT_TYPE_BT_SGMII
1458 				      ? ETH_MDIO_SUPPORTS_C22
1459 				      : ETH_MDIO_SUPPORTS_C45);
1460 	} else {
1461 		base->phy_address = 255;
1462 		base->mdio_support = 0;
1463 	}
1464 
1465 	fw_caps_to_lmm(pi->port_type, pi->link_cfg.pcaps,
1466 		       link_ksettings->link_modes.supported);
1467 	fw_caps_to_lmm(pi->port_type, pi->link_cfg.acaps,
1468 		       link_ksettings->link_modes.advertising);
1469 	fw_caps_to_lmm(pi->port_type, pi->link_cfg.lpacaps,
1470 		       link_ksettings->link_modes.lp_advertising);
1471 
1472 	if (netif_carrier_ok(dev)) {
1473 		base->speed = pi->link_cfg.speed;
1474 		base->duplex = DUPLEX_FULL;
1475 	} else {
1476 		base->speed = SPEED_UNKNOWN;
1477 		base->duplex = DUPLEX_UNKNOWN;
1478 	}
1479 
1480 	base->autoneg = pi->link_cfg.autoneg;
1481 	if (pi->link_cfg.pcaps & FW_PORT_CAP32_ANEG)
1482 		ethtool_link_ksettings_add_link_mode(link_ksettings,
1483 						     supported, Autoneg);
1484 	if (pi->link_cfg.autoneg)
1485 		ethtool_link_ksettings_add_link_mode(link_ksettings,
1486 						     advertising, Autoneg);
1487 
1488 	return 0;
1489 }
1490 
1491 /* Translate the Firmware FEC value into the ethtool value. */
1492 static inline unsigned int fwcap_to_eth_fec(unsigned int fw_fec)
1493 {
1494 	unsigned int eth_fec = 0;
1495 
1496 	if (fw_fec & FW_PORT_CAP32_FEC_RS)
1497 		eth_fec |= ETHTOOL_FEC_RS;
1498 	if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS)
1499 		eth_fec |= ETHTOOL_FEC_BASER;
1500 
1501 	/* if nothing is set, then FEC is off */
1502 	if (!eth_fec)
1503 		eth_fec = ETHTOOL_FEC_OFF;
1504 
1505 	return eth_fec;
1506 }
1507 
1508 /* Translate Common Code FEC value into ethtool value. */
1509 static inline unsigned int cc_to_eth_fec(unsigned int cc_fec)
1510 {
1511 	unsigned int eth_fec = 0;
1512 
1513 	if (cc_fec & FEC_AUTO)
1514 		eth_fec |= ETHTOOL_FEC_AUTO;
1515 	if (cc_fec & FEC_RS)
1516 		eth_fec |= ETHTOOL_FEC_RS;
1517 	if (cc_fec & FEC_BASER_RS)
1518 		eth_fec |= ETHTOOL_FEC_BASER;
1519 
1520 	/* if nothing is set, then FEC is off */
1521 	if (!eth_fec)
1522 		eth_fec = ETHTOOL_FEC_OFF;
1523 
1524 	return eth_fec;
1525 }
1526 
1527 static int cxgb4vf_get_fecparam(struct net_device *dev,
1528 				struct ethtool_fecparam *fec)
1529 {
1530 	const struct port_info *pi = netdev_priv(dev);
1531 	const struct link_config *lc = &pi->link_cfg;
1532 
1533 	/* Translate the Firmware FEC Support into the ethtool value.  We
1534 	 * always support IEEE 802.3 "automatic" selection of Link FEC type if
1535 	 * any FEC is supported.
1536 	 */
1537 	fec->fec = fwcap_to_eth_fec(lc->pcaps);
1538 	if (fec->fec != ETHTOOL_FEC_OFF)
1539 		fec->fec |= ETHTOOL_FEC_AUTO;
1540 
1541 	/* Translate the current internal FEC parameters into the
1542 	 * ethtool values.
1543 	 */
1544 	fec->active_fec = cc_to_eth_fec(lc->fec);
1545 	return 0;
1546 }
1547 
1548 /*
1549  * Return our driver information.
1550  */
1551 static void cxgb4vf_get_drvinfo(struct net_device *dev,
1552 				struct ethtool_drvinfo *drvinfo)
1553 {
1554 	struct adapter *adapter = netdev2adap(dev);
1555 
1556 	strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
1557 	strlcpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)),
1558 		sizeof(drvinfo->bus_info));
1559 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
1560 		 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1561 		 FW_HDR_FW_VER_MAJOR_G(adapter->params.dev.fwrev),
1562 		 FW_HDR_FW_VER_MINOR_G(adapter->params.dev.fwrev),
1563 		 FW_HDR_FW_VER_MICRO_G(adapter->params.dev.fwrev),
1564 		 FW_HDR_FW_VER_BUILD_G(adapter->params.dev.fwrev),
1565 		 FW_HDR_FW_VER_MAJOR_G(adapter->params.dev.tprev),
1566 		 FW_HDR_FW_VER_MINOR_G(adapter->params.dev.tprev),
1567 		 FW_HDR_FW_VER_MICRO_G(adapter->params.dev.tprev),
1568 		 FW_HDR_FW_VER_BUILD_G(adapter->params.dev.tprev));
1569 }
1570 
1571 /*
1572  * Return current adapter message level.
1573  */
1574 static u32 cxgb4vf_get_msglevel(struct net_device *dev)
1575 {
1576 	return netdev2adap(dev)->msg_enable;
1577 }
1578 
1579 /*
1580  * Set current adapter message level.
1581  */
1582 static void cxgb4vf_set_msglevel(struct net_device *dev, u32 msglevel)
1583 {
1584 	netdev2adap(dev)->msg_enable = msglevel;
1585 }
1586 
1587 /*
1588  * Return the device's current Queue Set ring size parameters along with the
1589  * allowed maximum values.  Since ethtool doesn't understand the concept of
1590  * multi-queue devices, we just return the current values associated with the
1591  * first Queue Set.
1592  */
1593 static void cxgb4vf_get_ringparam(struct net_device *dev,
1594 				  struct ethtool_ringparam *rp)
1595 {
1596 	const struct port_info *pi = netdev_priv(dev);
1597 	const struct sge *s = &pi->adapter->sge;
1598 
1599 	rp->rx_max_pending = MAX_RX_BUFFERS;
1600 	rp->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1601 	rp->rx_jumbo_max_pending = 0;
1602 	rp->tx_max_pending = MAX_TXQ_ENTRIES;
1603 
1604 	rp->rx_pending = s->ethrxq[pi->first_qset].fl.size - MIN_FL_RESID;
1605 	rp->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1606 	rp->rx_jumbo_pending = 0;
1607 	rp->tx_pending = s->ethtxq[pi->first_qset].q.size;
1608 }
1609 
1610 /*
1611  * Set the Queue Set ring size parameters for the device.  Again, since
1612  * ethtool doesn't allow for the concept of multiple queues per device, we'll
1613  * apply these new values across all of the Queue Sets associated with the
1614  * device -- after vetting them of course!
1615  */
1616 static int cxgb4vf_set_ringparam(struct net_device *dev,
1617 				 struct ethtool_ringparam *rp)
1618 {
1619 	const struct port_info *pi = netdev_priv(dev);
1620 	struct adapter *adapter = pi->adapter;
1621 	struct sge *s = &adapter->sge;
1622 	int qs;
1623 
1624 	if (rp->rx_pending > MAX_RX_BUFFERS ||
1625 	    rp->rx_jumbo_pending ||
1626 	    rp->tx_pending > MAX_TXQ_ENTRIES ||
1627 	    rp->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1628 	    rp->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1629 	    rp->rx_pending < MIN_FL_ENTRIES ||
1630 	    rp->tx_pending < MIN_TXQ_ENTRIES)
1631 		return -EINVAL;
1632 
1633 	if (adapter->flags & CXGB4VF_FULL_INIT_DONE)
1634 		return -EBUSY;
1635 
1636 	for (qs = pi->first_qset; qs < pi->first_qset + pi->nqsets; qs++) {
1637 		s->ethrxq[qs].fl.size = rp->rx_pending + MIN_FL_RESID;
1638 		s->ethrxq[qs].rspq.size = rp->rx_mini_pending;
1639 		s->ethtxq[qs].q.size = rp->tx_pending;
1640 	}
1641 	return 0;
1642 }
1643 
1644 /*
1645  * Return the interrupt holdoff timer and count for the first Queue Set on the
1646  * device.  Our extension ioctl() (the cxgbtool interface) allows the
1647  * interrupt holdoff timer to be read on all of the device's Queue Sets.
1648  */
1649 static int cxgb4vf_get_coalesce(struct net_device *dev,
1650 				struct ethtool_coalesce *coalesce,
1651 				struct kernel_ethtool_coalesce *kernel_coal,
1652 				struct netlink_ext_ack *extack)
1653 {
1654 	const struct port_info *pi = netdev_priv(dev);
1655 	const struct adapter *adapter = pi->adapter;
1656 	const struct sge_rspq *rspq = &adapter->sge.ethrxq[pi->first_qset].rspq;
1657 
1658 	coalesce->rx_coalesce_usecs = qtimer_val(adapter, rspq);
1659 	coalesce->rx_max_coalesced_frames =
1660 		((rspq->intr_params & QINTR_CNT_EN_F)
1661 		 ? adapter->sge.counter_val[rspq->pktcnt_idx]
1662 		 : 0);
1663 	return 0;
1664 }
1665 
1666 /*
1667  * Set the RX interrupt holdoff timer and count for the first Queue Set on the
1668  * interface.  Our extension ioctl() (the cxgbtool interface) allows us to set
1669  * the interrupt holdoff timer on any of the device's Queue Sets.
1670  */
1671 static int cxgb4vf_set_coalesce(struct net_device *dev,
1672 				struct ethtool_coalesce *coalesce,
1673 				struct kernel_ethtool_coalesce *kernel_coal,
1674 				struct netlink_ext_ack *extack)
1675 {
1676 	const struct port_info *pi = netdev_priv(dev);
1677 	struct adapter *adapter = pi->adapter;
1678 
1679 	return set_rxq_intr_params(adapter,
1680 				   &adapter->sge.ethrxq[pi->first_qset].rspq,
1681 				   coalesce->rx_coalesce_usecs,
1682 				   coalesce->rx_max_coalesced_frames);
1683 }
1684 
1685 /*
1686  * Report current port link pause parameter settings.
1687  */
1688 static void cxgb4vf_get_pauseparam(struct net_device *dev,
1689 				   struct ethtool_pauseparam *pauseparam)
1690 {
1691 	struct port_info *pi = netdev_priv(dev);
1692 
1693 	pauseparam->autoneg = (pi->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1694 	pauseparam->rx_pause = (pi->link_cfg.advertised_fc & PAUSE_RX) != 0;
1695 	pauseparam->tx_pause = (pi->link_cfg.advertised_fc & PAUSE_TX) != 0;
1696 }
1697 
1698 /*
1699  * Identify the port by blinking the port's LED.
1700  */
1701 static int cxgb4vf_phys_id(struct net_device *dev,
1702 			   enum ethtool_phys_id_state state)
1703 {
1704 	unsigned int val;
1705 	struct port_info *pi = netdev_priv(dev);
1706 
1707 	if (state == ETHTOOL_ID_ACTIVE)
1708 		val = 0xffff;
1709 	else if (state == ETHTOOL_ID_INACTIVE)
1710 		val = 0;
1711 	else
1712 		return -EINVAL;
1713 
1714 	return t4vf_identify_port(pi->adapter, pi->viid, val);
1715 }
1716 
1717 /*
1718  * Port stats maintained per queue of the port.
1719  */
1720 struct queue_port_stats {
1721 	u64 tso;
1722 	u64 tx_csum;
1723 	u64 rx_csum;
1724 	u64 vlan_ex;
1725 	u64 vlan_ins;
1726 	u64 lro_pkts;
1727 	u64 lro_merged;
1728 };
1729 
1730 /*
1731  * Strings for the ETH_SS_STATS statistics set ("ethtool -S").  Note that
1732  * these need to match the order of statistics returned by
1733  * t4vf_get_port_stats().
1734  */
1735 static const char stats_strings[][ETH_GSTRING_LEN] = {
1736 	/*
1737 	 * These must match the layout of the t4vf_port_stats structure.
1738 	 */
1739 	"TxBroadcastBytes  ",
1740 	"TxBroadcastFrames ",
1741 	"TxMulticastBytes  ",
1742 	"TxMulticastFrames ",
1743 	"TxUnicastBytes    ",
1744 	"TxUnicastFrames   ",
1745 	"TxDroppedFrames   ",
1746 	"TxOffloadBytes    ",
1747 	"TxOffloadFrames   ",
1748 	"RxBroadcastBytes  ",
1749 	"RxBroadcastFrames ",
1750 	"RxMulticastBytes  ",
1751 	"RxMulticastFrames ",
1752 	"RxUnicastBytes    ",
1753 	"RxUnicastFrames   ",
1754 	"RxErrorFrames     ",
1755 
1756 	/*
1757 	 * These are accumulated per-queue statistics and must match the
1758 	 * order of the fields in the queue_port_stats structure.
1759 	 */
1760 	"TSO               ",
1761 	"TxCsumOffload     ",
1762 	"RxCsumGood        ",
1763 	"VLANextractions   ",
1764 	"VLANinsertions    ",
1765 	"GROPackets        ",
1766 	"GROMerged         ",
1767 };
1768 
1769 /*
1770  * Return the number of statistics in the specified statistics set.
1771  */
1772 static int cxgb4vf_get_sset_count(struct net_device *dev, int sset)
1773 {
1774 	switch (sset) {
1775 	case ETH_SS_STATS:
1776 		return ARRAY_SIZE(stats_strings);
1777 	default:
1778 		return -EOPNOTSUPP;
1779 	}
1780 	/*NOTREACHED*/
1781 }
1782 
1783 /*
1784  * Return the strings for the specified statistics set.
1785  */
1786 static void cxgb4vf_get_strings(struct net_device *dev,
1787 				u32 sset,
1788 				u8 *data)
1789 {
1790 	switch (sset) {
1791 	case ETH_SS_STATS:
1792 		memcpy(data, stats_strings, sizeof(stats_strings));
1793 		break;
1794 	}
1795 }
1796 
1797 /*
1798  * Small utility routine to accumulate queue statistics across the queues of
1799  * a "port".
1800  */
1801 static void collect_sge_port_stats(const struct adapter *adapter,
1802 				   const struct port_info *pi,
1803 				   struct queue_port_stats *stats)
1804 {
1805 	const struct sge_eth_txq *txq = &adapter->sge.ethtxq[pi->first_qset];
1806 	const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset];
1807 	int qs;
1808 
1809 	memset(stats, 0, sizeof(*stats));
1810 	for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
1811 		stats->tso += txq->tso;
1812 		stats->tx_csum += txq->tx_cso;
1813 		stats->rx_csum += rxq->stats.rx_cso;
1814 		stats->vlan_ex += rxq->stats.vlan_ex;
1815 		stats->vlan_ins += txq->vlan_ins;
1816 		stats->lro_pkts += rxq->stats.lro_pkts;
1817 		stats->lro_merged += rxq->stats.lro_merged;
1818 	}
1819 }
1820 
1821 /*
1822  * Return the ETH_SS_STATS statistics set.
1823  */
1824 static void cxgb4vf_get_ethtool_stats(struct net_device *dev,
1825 				      struct ethtool_stats *stats,
1826 				      u64 *data)
1827 {
1828 	struct port_info *pi = netdev2pinfo(dev);
1829 	struct adapter *adapter = pi->adapter;
1830 	int err = t4vf_get_port_stats(adapter, pi->pidx,
1831 				      (struct t4vf_port_stats *)data);
1832 	if (err)
1833 		memset(data, 0, sizeof(struct t4vf_port_stats));
1834 
1835 	data += sizeof(struct t4vf_port_stats) / sizeof(u64);
1836 	collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1837 }
1838 
1839 /*
1840  * Return the size of our register map.
1841  */
1842 static int cxgb4vf_get_regs_len(struct net_device *dev)
1843 {
1844 	return T4VF_REGMAP_SIZE;
1845 }
1846 
1847 /*
1848  * Dump a block of registers, start to end inclusive, into a buffer.
1849  */
1850 static void reg_block_dump(struct adapter *adapter, void *regbuf,
1851 			   unsigned int start, unsigned int end)
1852 {
1853 	u32 *bp = regbuf + start - T4VF_REGMAP_START;
1854 
1855 	for ( ; start <= end; start += sizeof(u32)) {
1856 		/*
1857 		 * Avoid reading the Mailbox Control register since that
1858 		 * can trigger a Mailbox Ownership Arbitration cycle and
1859 		 * interfere with communication with the firmware.
1860 		 */
1861 		if (start == T4VF_CIM_BASE_ADDR + CIM_VF_EXT_MAILBOX_CTRL)
1862 			*bp++ = 0xffff;
1863 		else
1864 			*bp++ = t4_read_reg(adapter, start);
1865 	}
1866 }
1867 
1868 /*
1869  * Copy our entire register map into the provided buffer.
1870  */
1871 static void cxgb4vf_get_regs(struct net_device *dev,
1872 			     struct ethtool_regs *regs,
1873 			     void *regbuf)
1874 {
1875 	struct adapter *adapter = netdev2adap(dev);
1876 
1877 	regs->version = mk_adap_vers(adapter);
1878 
1879 	/*
1880 	 * Fill in register buffer with our register map.
1881 	 */
1882 	memset(regbuf, 0, T4VF_REGMAP_SIZE);
1883 
1884 	reg_block_dump(adapter, regbuf,
1885 		       T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_FIRST,
1886 		       T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_LAST);
1887 	reg_block_dump(adapter, regbuf,
1888 		       T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_FIRST,
1889 		       T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_LAST);
1890 
1891 	/* T5 adds new registers in the PL Register map.
1892 	 */
1893 	reg_block_dump(adapter, regbuf,
1894 		       T4VF_PL_BASE_ADDR + T4VF_MOD_MAP_PL_FIRST,
1895 		       T4VF_PL_BASE_ADDR + (is_t4(adapter->params.chip)
1896 		       ? PL_VF_WHOAMI_A : PL_VF_REVISION_A));
1897 	reg_block_dump(adapter, regbuf,
1898 		       T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_FIRST,
1899 		       T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_LAST);
1900 
1901 	reg_block_dump(adapter, regbuf,
1902 		       T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_FIRST,
1903 		       T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_LAST);
1904 }
1905 
1906 /*
1907  * Report current Wake On LAN settings.
1908  */
1909 static void cxgb4vf_get_wol(struct net_device *dev,
1910 			    struct ethtool_wolinfo *wol)
1911 {
1912 	wol->supported = 0;
1913 	wol->wolopts = 0;
1914 	memset(&wol->sopass, 0, sizeof(wol->sopass));
1915 }
1916 
1917 /*
1918  * TCP Segmentation Offload flags which we support.
1919  */
1920 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1921 #define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
1922 		   NETIF_F_GRO | NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
1923 
1924 static const struct ethtool_ops cxgb4vf_ethtool_ops = {
1925 	.supported_coalesce_params = ETHTOOL_COALESCE_RX_USECS |
1926 				     ETHTOOL_COALESCE_RX_MAX_FRAMES,
1927 	.get_link_ksettings	= cxgb4vf_get_link_ksettings,
1928 	.get_fecparam		= cxgb4vf_get_fecparam,
1929 	.get_drvinfo		= cxgb4vf_get_drvinfo,
1930 	.get_msglevel		= cxgb4vf_get_msglevel,
1931 	.set_msglevel		= cxgb4vf_set_msglevel,
1932 	.get_ringparam		= cxgb4vf_get_ringparam,
1933 	.set_ringparam		= cxgb4vf_set_ringparam,
1934 	.get_coalesce		= cxgb4vf_get_coalesce,
1935 	.set_coalesce		= cxgb4vf_set_coalesce,
1936 	.get_pauseparam		= cxgb4vf_get_pauseparam,
1937 	.get_link		= ethtool_op_get_link,
1938 	.get_strings		= cxgb4vf_get_strings,
1939 	.set_phys_id		= cxgb4vf_phys_id,
1940 	.get_sset_count		= cxgb4vf_get_sset_count,
1941 	.get_ethtool_stats	= cxgb4vf_get_ethtool_stats,
1942 	.get_regs_len		= cxgb4vf_get_regs_len,
1943 	.get_regs		= cxgb4vf_get_regs,
1944 	.get_wol		= cxgb4vf_get_wol,
1945 };
1946 
1947 /*
1948  * /sys/kernel/debug/cxgb4vf support code and data.
1949  * ================================================
1950  */
1951 
1952 /*
1953  * Show Firmware Mailbox Command/Reply Log
1954  *
1955  * Note that we don't do any locking when dumping the Firmware Mailbox Log so
1956  * it's possible that we can catch things during a log update and therefore
1957  * see partially corrupted log entries.  But i9t's probably Good Enough(tm).
1958  * If we ever decide that we want to make sure that we're dumping a coherent
1959  * log, we'd need to perform locking in the mailbox logging and in
1960  * mboxlog_open() where we'd need to grab the entire mailbox log in one go
1961  * like we do for the Firmware Device Log.  But as stated above, meh ...
1962  */
1963 static int mboxlog_show(struct seq_file *seq, void *v)
1964 {
1965 	struct adapter *adapter = seq->private;
1966 	struct mbox_cmd_log *log = adapter->mbox_log;
1967 	struct mbox_cmd *entry;
1968 	int entry_idx, i;
1969 
1970 	if (v == SEQ_START_TOKEN) {
1971 		seq_printf(seq,
1972 			   "%10s  %15s  %5s  %5s  %s\n",
1973 			   "Seq#", "Tstamp", "Atime", "Etime",
1974 			   "Command/Reply");
1975 		return 0;
1976 	}
1977 
1978 	entry_idx = log->cursor + ((uintptr_t)v - 2);
1979 	if (entry_idx >= log->size)
1980 		entry_idx -= log->size;
1981 	entry = mbox_cmd_log_entry(log, entry_idx);
1982 
1983 	/* skip over unused entries */
1984 	if (entry->timestamp == 0)
1985 		return 0;
1986 
1987 	seq_printf(seq, "%10u  %15llu  %5d  %5d",
1988 		   entry->seqno, entry->timestamp,
1989 		   entry->access, entry->execute);
1990 	for (i = 0; i < MBOX_LEN / 8; i++) {
1991 		u64 flit = entry->cmd[i];
1992 		u32 hi = (u32)(flit >> 32);
1993 		u32 lo = (u32)flit;
1994 
1995 		seq_printf(seq, "  %08x %08x", hi, lo);
1996 	}
1997 	seq_puts(seq, "\n");
1998 	return 0;
1999 }
2000 
2001 static inline void *mboxlog_get_idx(struct seq_file *seq, loff_t pos)
2002 {
2003 	struct adapter *adapter = seq->private;
2004 	struct mbox_cmd_log *log = adapter->mbox_log;
2005 
2006 	return ((pos <= log->size) ? (void *)(uintptr_t)(pos + 1) : NULL);
2007 }
2008 
2009 static void *mboxlog_start(struct seq_file *seq, loff_t *pos)
2010 {
2011 	return *pos ? mboxlog_get_idx(seq, *pos) : SEQ_START_TOKEN;
2012 }
2013 
2014 static void *mboxlog_next(struct seq_file *seq, void *v, loff_t *pos)
2015 {
2016 	++*pos;
2017 	return mboxlog_get_idx(seq, *pos);
2018 }
2019 
2020 static void mboxlog_stop(struct seq_file *seq, void *v)
2021 {
2022 }
2023 
2024 static const struct seq_operations mboxlog_sops = {
2025 	.start = mboxlog_start,
2026 	.next  = mboxlog_next,
2027 	.stop  = mboxlog_stop,
2028 	.show  = mboxlog_show
2029 };
2030 
2031 DEFINE_SEQ_ATTRIBUTE(mboxlog);
2032 /*
2033  * Show SGE Queue Set information.  We display QPL Queues Sets per line.
2034  */
2035 #define QPL	4
2036 
2037 static int sge_qinfo_show(struct seq_file *seq, void *v)
2038 {
2039 	struct adapter *adapter = seq->private;
2040 	int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL);
2041 	int qs, r = (uintptr_t)v - 1;
2042 
2043 	if (r)
2044 		seq_putc(seq, '\n');
2045 
2046 	#define S3(fmt_spec, s, v) \
2047 		do {\
2048 			seq_printf(seq, "%-12s", s); \
2049 			for (qs = 0; qs < n; ++qs) \
2050 				seq_printf(seq, " %16" fmt_spec, v); \
2051 			seq_putc(seq, '\n'); \
2052 		} while (0)
2053 	#define S(s, v)		S3("s", s, v)
2054 	#define T(s, v)		S3("u", s, txq[qs].v)
2055 	#define R(s, v)		S3("u", s, rxq[qs].v)
2056 
2057 	if (r < eth_entries) {
2058 		const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL];
2059 		const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL];
2060 		int n = min(QPL, adapter->sge.ethqsets - QPL * r);
2061 
2062 		S("QType:", "Ethernet");
2063 		S("Interface:",
2064 		  (rxq[qs].rspq.netdev
2065 		   ? rxq[qs].rspq.netdev->name
2066 		   : "N/A"));
2067 		S3("d", "Port:",
2068 		   (rxq[qs].rspq.netdev
2069 		    ? ((struct port_info *)
2070 		       netdev_priv(rxq[qs].rspq.netdev))->port_id
2071 		    : -1));
2072 		T("TxQ ID:", q.abs_id);
2073 		T("TxQ size:", q.size);
2074 		T("TxQ inuse:", q.in_use);
2075 		T("TxQ PIdx:", q.pidx);
2076 		T("TxQ CIdx:", q.cidx);
2077 		R("RspQ ID:", rspq.abs_id);
2078 		R("RspQ size:", rspq.size);
2079 		R("RspQE size:", rspq.iqe_len);
2080 		S3("u", "Intr delay:", qtimer_val(adapter, &rxq[qs].rspq));
2081 		S3("u", "Intr pktcnt:",
2082 		   adapter->sge.counter_val[rxq[qs].rspq.pktcnt_idx]);
2083 		R("RspQ CIdx:", rspq.cidx);
2084 		R("RspQ Gen:", rspq.gen);
2085 		R("FL ID:", fl.abs_id);
2086 		R("FL size:", fl.size - MIN_FL_RESID);
2087 		R("FL avail:", fl.avail);
2088 		R("FL PIdx:", fl.pidx);
2089 		R("FL CIdx:", fl.cidx);
2090 		return 0;
2091 	}
2092 
2093 	r -= eth_entries;
2094 	if (r == 0) {
2095 		const struct sge_rspq *evtq = &adapter->sge.fw_evtq;
2096 
2097 		seq_printf(seq, "%-12s %16s\n", "QType:", "FW event queue");
2098 		seq_printf(seq, "%-12s %16u\n", "RspQ ID:", evtq->abs_id);
2099 		seq_printf(seq, "%-12s %16u\n", "Intr delay:",
2100 			   qtimer_val(adapter, evtq));
2101 		seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:",
2102 			   adapter->sge.counter_val[evtq->pktcnt_idx]);
2103 		seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", evtq->cidx);
2104 		seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", evtq->gen);
2105 	} else if (r == 1) {
2106 		const struct sge_rspq *intrq = &adapter->sge.intrq;
2107 
2108 		seq_printf(seq, "%-12s %16s\n", "QType:", "Interrupt Queue");
2109 		seq_printf(seq, "%-12s %16u\n", "RspQ ID:", intrq->abs_id);
2110 		seq_printf(seq, "%-12s %16u\n", "Intr delay:",
2111 			   qtimer_val(adapter, intrq));
2112 		seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:",
2113 			   adapter->sge.counter_val[intrq->pktcnt_idx]);
2114 		seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", intrq->cidx);
2115 		seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", intrq->gen);
2116 	}
2117 
2118 	#undef R
2119 	#undef T
2120 	#undef S
2121 	#undef S3
2122 
2123 	return 0;
2124 }
2125 
2126 /*
2127  * Return the number of "entries" in our "file".  We group the multi-Queue
2128  * sections with QPL Queue Sets per "entry".  The sections of the output are:
2129  *
2130  *     Ethernet RX/TX Queue Sets
2131  *     Firmware Event Queue
2132  *     Forwarded Interrupt Queue (if in MSI mode)
2133  */
2134 static int sge_queue_entries(const struct adapter *adapter)
2135 {
2136 	return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 +
2137 		((adapter->flags & CXGB4VF_USING_MSI) != 0);
2138 }
2139 
2140 static void *sge_queue_start(struct seq_file *seq, loff_t *pos)
2141 {
2142 	int entries = sge_queue_entries(seq->private);
2143 
2144 	return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
2145 }
2146 
2147 static void sge_queue_stop(struct seq_file *seq, void *v)
2148 {
2149 }
2150 
2151 static void *sge_queue_next(struct seq_file *seq, void *v, loff_t *pos)
2152 {
2153 	int entries = sge_queue_entries(seq->private);
2154 
2155 	++*pos;
2156 	return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
2157 }
2158 
2159 static const struct seq_operations sge_qinfo_sops = {
2160 	.start = sge_queue_start,
2161 	.next  = sge_queue_next,
2162 	.stop  = sge_queue_stop,
2163 	.show  = sge_qinfo_show
2164 };
2165 
2166 DEFINE_SEQ_ATTRIBUTE(sge_qinfo);
2167 
2168 /*
2169  * Show SGE Queue Set statistics.  We display QPL Queues Sets per line.
2170  */
2171 #define QPL	4
2172 
2173 static int sge_qstats_show(struct seq_file *seq, void *v)
2174 {
2175 	struct adapter *adapter = seq->private;
2176 	int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL);
2177 	int qs, r = (uintptr_t)v - 1;
2178 
2179 	if (r)
2180 		seq_putc(seq, '\n');
2181 
2182 	#define S3(fmt, s, v) \
2183 		do { \
2184 			seq_printf(seq, "%-16s", s); \
2185 			for (qs = 0; qs < n; ++qs) \
2186 				seq_printf(seq, " %8" fmt, v); \
2187 			seq_putc(seq, '\n'); \
2188 		} while (0)
2189 	#define S(s, v)		S3("s", s, v)
2190 
2191 	#define T3(fmt, s, v)	S3(fmt, s, txq[qs].v)
2192 	#define T(s, v)		T3("lu", s, v)
2193 
2194 	#define R3(fmt, s, v)	S3(fmt, s, rxq[qs].v)
2195 	#define R(s, v)		R3("lu", s, v)
2196 
2197 	if (r < eth_entries) {
2198 		const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL];
2199 		const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL];
2200 		int n = min(QPL, adapter->sge.ethqsets - QPL * r);
2201 
2202 		S("QType:", "Ethernet");
2203 		S("Interface:",
2204 		  (rxq[qs].rspq.netdev
2205 		   ? rxq[qs].rspq.netdev->name
2206 		   : "N/A"));
2207 		R3("u", "RspQNullInts:", rspq.unhandled_irqs);
2208 		R("RxPackets:", stats.pkts);
2209 		R("RxCSO:", stats.rx_cso);
2210 		R("VLANxtract:", stats.vlan_ex);
2211 		R("LROmerged:", stats.lro_merged);
2212 		R("LROpackets:", stats.lro_pkts);
2213 		R("RxDrops:", stats.rx_drops);
2214 		T("TSO:", tso);
2215 		T("TxCSO:", tx_cso);
2216 		T("VLANins:", vlan_ins);
2217 		T("TxQFull:", q.stops);
2218 		T("TxQRestarts:", q.restarts);
2219 		T("TxMapErr:", mapping_err);
2220 		R("FLAllocErr:", fl.alloc_failed);
2221 		R("FLLrgAlcErr:", fl.large_alloc_failed);
2222 		R("FLStarving:", fl.starving);
2223 		return 0;
2224 	}
2225 
2226 	r -= eth_entries;
2227 	if (r == 0) {
2228 		const struct sge_rspq *evtq = &adapter->sge.fw_evtq;
2229 
2230 		seq_printf(seq, "%-8s %16s\n", "QType:", "FW event queue");
2231 		seq_printf(seq, "%-16s %8u\n", "RspQNullInts:",
2232 			   evtq->unhandled_irqs);
2233 		seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", evtq->cidx);
2234 		seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", evtq->gen);
2235 	} else if (r == 1) {
2236 		const struct sge_rspq *intrq = &adapter->sge.intrq;
2237 
2238 		seq_printf(seq, "%-8s %16s\n", "QType:", "Interrupt Queue");
2239 		seq_printf(seq, "%-16s %8u\n", "RspQNullInts:",
2240 			   intrq->unhandled_irqs);
2241 		seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", intrq->cidx);
2242 		seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", intrq->gen);
2243 	}
2244 
2245 	#undef R
2246 	#undef T
2247 	#undef S
2248 	#undef R3
2249 	#undef T3
2250 	#undef S3
2251 
2252 	return 0;
2253 }
2254 
2255 /*
2256  * Return the number of "entries" in our "file".  We group the multi-Queue
2257  * sections with QPL Queue Sets per "entry".  The sections of the output are:
2258  *
2259  *     Ethernet RX/TX Queue Sets
2260  *     Firmware Event Queue
2261  *     Forwarded Interrupt Queue (if in MSI mode)
2262  */
2263 static int sge_qstats_entries(const struct adapter *adapter)
2264 {
2265 	return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 +
2266 		((adapter->flags & CXGB4VF_USING_MSI) != 0);
2267 }
2268 
2269 static void *sge_qstats_start(struct seq_file *seq, loff_t *pos)
2270 {
2271 	int entries = sge_qstats_entries(seq->private);
2272 
2273 	return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
2274 }
2275 
2276 static void sge_qstats_stop(struct seq_file *seq, void *v)
2277 {
2278 }
2279 
2280 static void *sge_qstats_next(struct seq_file *seq, void *v, loff_t *pos)
2281 {
2282 	int entries = sge_qstats_entries(seq->private);
2283 
2284 	(*pos)++;
2285 	return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
2286 }
2287 
2288 static const struct seq_operations sge_qstats_sops = {
2289 	.start = sge_qstats_start,
2290 	.next  = sge_qstats_next,
2291 	.stop  = sge_qstats_stop,
2292 	.show  = sge_qstats_show
2293 };
2294 
2295 DEFINE_SEQ_ATTRIBUTE(sge_qstats);
2296 
2297 /*
2298  * Show PCI-E SR-IOV Virtual Function Resource Limits.
2299  */
2300 static int resources_show(struct seq_file *seq, void *v)
2301 {
2302 	struct adapter *adapter = seq->private;
2303 	struct vf_resources *vfres = &adapter->params.vfres;
2304 
2305 	#define S(desc, fmt, var) \
2306 		seq_printf(seq, "%-60s " fmt "\n", \
2307 			   desc " (" #var "):", vfres->var)
2308 
2309 	S("Virtual Interfaces", "%d", nvi);
2310 	S("Egress Queues", "%d", neq);
2311 	S("Ethernet Control", "%d", nethctrl);
2312 	S("Ingress Queues/w Free Lists/Interrupts", "%d", niqflint);
2313 	S("Ingress Queues", "%d", niq);
2314 	S("Traffic Class", "%d", tc);
2315 	S("Port Access Rights Mask", "%#x", pmask);
2316 	S("MAC Address Filters", "%d", nexactf);
2317 	S("Firmware Command Read Capabilities", "%#x", r_caps);
2318 	S("Firmware Command Write/Execute Capabilities", "%#x", wx_caps);
2319 
2320 	#undef S
2321 
2322 	return 0;
2323 }
2324 DEFINE_SHOW_ATTRIBUTE(resources);
2325 
2326 /*
2327  * Show Virtual Interfaces.
2328  */
2329 static int interfaces_show(struct seq_file *seq, void *v)
2330 {
2331 	if (v == SEQ_START_TOKEN) {
2332 		seq_puts(seq, "Interface  Port   VIID\n");
2333 	} else {
2334 		struct adapter *adapter = seq->private;
2335 		int pidx = (uintptr_t)v - 2;
2336 		struct net_device *dev = adapter->port[pidx];
2337 		struct port_info *pi = netdev_priv(dev);
2338 
2339 		seq_printf(seq, "%9s  %4d  %#5x\n",
2340 			   dev->name, pi->port_id, pi->viid);
2341 	}
2342 	return 0;
2343 }
2344 
2345 static inline void *interfaces_get_idx(struct adapter *adapter, loff_t pos)
2346 {
2347 	return pos <= adapter->params.nports
2348 		? (void *)(uintptr_t)(pos + 1)
2349 		: NULL;
2350 }
2351 
2352 static void *interfaces_start(struct seq_file *seq, loff_t *pos)
2353 {
2354 	return *pos
2355 		? interfaces_get_idx(seq->private, *pos)
2356 		: SEQ_START_TOKEN;
2357 }
2358 
2359 static void *interfaces_next(struct seq_file *seq, void *v, loff_t *pos)
2360 {
2361 	(*pos)++;
2362 	return interfaces_get_idx(seq->private, *pos);
2363 }
2364 
2365 static void interfaces_stop(struct seq_file *seq, void *v)
2366 {
2367 }
2368 
2369 static const struct seq_operations interfaces_sops = {
2370 	.start = interfaces_start,
2371 	.next  = interfaces_next,
2372 	.stop  = interfaces_stop,
2373 	.show  = interfaces_show
2374 };
2375 
2376 DEFINE_SEQ_ATTRIBUTE(interfaces);
2377 
2378 /*
2379  * /sys/kernel/debugfs/cxgb4vf/ files list.
2380  */
2381 struct cxgb4vf_debugfs_entry {
2382 	const char *name;		/* name of debugfs node */
2383 	umode_t mode;			/* file system mode */
2384 	const struct file_operations *fops;
2385 };
2386 
2387 static struct cxgb4vf_debugfs_entry debugfs_files[] = {
2388 	{ "mboxlog",    0444, &mboxlog_fops },
2389 	{ "sge_qinfo",  0444, &sge_qinfo_fops },
2390 	{ "sge_qstats", 0444, &sge_qstats_fops },
2391 	{ "resources",  0444, &resources_fops },
2392 	{ "interfaces", 0444, &interfaces_fops },
2393 };
2394 
2395 /*
2396  * Module and device initialization and cleanup code.
2397  * ==================================================
2398  */
2399 
2400 /*
2401  * Set up out /sys/kernel/debug/cxgb4vf sub-nodes.  We assume that the
2402  * directory (debugfs_root) has already been set up.
2403  */
2404 static int setup_debugfs(struct adapter *adapter)
2405 {
2406 	int i;
2407 
2408 	BUG_ON(IS_ERR_OR_NULL(adapter->debugfs_root));
2409 
2410 	/*
2411 	 * Debugfs support is best effort.
2412 	 */
2413 	for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
2414 		debugfs_create_file(debugfs_files[i].name,
2415 				    debugfs_files[i].mode,
2416 				    adapter->debugfs_root, adapter,
2417 				    debugfs_files[i].fops);
2418 
2419 	return 0;
2420 }
2421 
2422 /*
2423  * Tear down the /sys/kernel/debug/cxgb4vf sub-nodes created above.  We leave
2424  * it to our caller to tear down the directory (debugfs_root).
2425  */
2426 static void cleanup_debugfs(struct adapter *adapter)
2427 {
2428 	BUG_ON(IS_ERR_OR_NULL(adapter->debugfs_root));
2429 
2430 	/*
2431 	 * Unlike our sister routine cleanup_proc(), we don't need to remove
2432 	 * individual entries because a call will be made to
2433 	 * debugfs_remove_recursive().  We just need to clean up any ancillary
2434 	 * persistent state.
2435 	 */
2436 	/* nothing to do */
2437 }
2438 
2439 /* Figure out how many Ports and Queue Sets we can support.  This depends on
2440  * knowing our Virtual Function Resources and may be called a second time if
2441  * we fall back from MSI-X to MSI Interrupt Mode.
2442  */
2443 static void size_nports_qsets(struct adapter *adapter)
2444 {
2445 	struct vf_resources *vfres = &adapter->params.vfres;
2446 	unsigned int ethqsets, pmask_nports;
2447 
2448 	/* The number of "ports" which we support is equal to the number of
2449 	 * Virtual Interfaces with which we've been provisioned.
2450 	 */
2451 	adapter->params.nports = vfres->nvi;
2452 	if (adapter->params.nports > MAX_NPORTS) {
2453 		dev_warn(adapter->pdev_dev, "only using %d of %d maximum"
2454 			 " allowed virtual interfaces\n", MAX_NPORTS,
2455 			 adapter->params.nports);
2456 		adapter->params.nports = MAX_NPORTS;
2457 	}
2458 
2459 	/* We may have been provisioned with more VIs than the number of
2460 	 * ports we're allowed to access (our Port Access Rights Mask).
2461 	 * This is obviously a configuration conflict but we don't want to
2462 	 * crash the kernel or anything silly just because of that.
2463 	 */
2464 	pmask_nports = hweight32(adapter->params.vfres.pmask);
2465 	if (pmask_nports < adapter->params.nports) {
2466 		dev_warn(adapter->pdev_dev, "only using %d of %d provisioned"
2467 			 " virtual interfaces; limited by Port Access Rights"
2468 			 " mask %#x\n", pmask_nports, adapter->params.nports,
2469 			 adapter->params.vfres.pmask);
2470 		adapter->params.nports = pmask_nports;
2471 	}
2472 
2473 	/* We need to reserve an Ingress Queue for the Asynchronous Firmware
2474 	 * Event Queue.  And if we're using MSI Interrupts, we'll also need to
2475 	 * reserve an Ingress Queue for a Forwarded Interrupts.
2476 	 *
2477 	 * The rest of the FL/Intr-capable ingress queues will be matched up
2478 	 * one-for-one with Ethernet/Control egress queues in order to form
2479 	 * "Queue Sets" which will be aportioned between the "ports".  For
2480 	 * each Queue Set, we'll need the ability to allocate two Egress
2481 	 * Contexts -- one for the Ingress Queue Free List and one for the TX
2482 	 * Ethernet Queue.
2483 	 *
2484 	 * Note that even if we're currently configured to use MSI-X
2485 	 * Interrupts (module variable msi == MSI_MSIX) we may get downgraded
2486 	 * to MSI Interrupts if we can't get enough MSI-X Interrupts.  If that
2487 	 * happens we'll need to adjust things later.
2488 	 */
2489 	ethqsets = vfres->niqflint - 1 - (msi == MSI_MSI);
2490 	if (vfres->nethctrl != ethqsets)
2491 		ethqsets = min(vfres->nethctrl, ethqsets);
2492 	if (vfres->neq < ethqsets*2)
2493 		ethqsets = vfres->neq/2;
2494 	if (ethqsets > MAX_ETH_QSETS)
2495 		ethqsets = MAX_ETH_QSETS;
2496 	adapter->sge.max_ethqsets = ethqsets;
2497 
2498 	if (adapter->sge.max_ethqsets < adapter->params.nports) {
2499 		dev_warn(adapter->pdev_dev, "only using %d of %d available"
2500 			 " virtual interfaces (too few Queue Sets)\n",
2501 			 adapter->sge.max_ethqsets, adapter->params.nports);
2502 		adapter->params.nports = adapter->sge.max_ethqsets;
2503 	}
2504 }
2505 
2506 /*
2507  * Perform early "adapter" initialization.  This is where we discover what
2508  * adapter parameters we're going to be using and initialize basic adapter
2509  * hardware support.
2510  */
2511 static int adap_init0(struct adapter *adapter)
2512 {
2513 	struct sge_params *sge_params = &adapter->params.sge;
2514 	struct sge *s = &adapter->sge;
2515 	int err;
2516 	u32 param, val = 0;
2517 
2518 	/*
2519 	 * Some environments do not properly handle PCIE FLRs -- e.g. in Linux
2520 	 * 2.6.31 and later we can't call pci_reset_function() in order to
2521 	 * issue an FLR because of a self- deadlock on the device semaphore.
2522 	 * Meanwhile, the OS infrastructure doesn't issue FLRs in all the
2523 	 * cases where they're needed -- for instance, some versions of KVM
2524 	 * fail to reset "Assigned Devices" when the VM reboots.  Therefore we
2525 	 * use the firmware based reset in order to reset any per function
2526 	 * state.
2527 	 */
2528 	err = t4vf_fw_reset(adapter);
2529 	if (err < 0) {
2530 		dev_err(adapter->pdev_dev, "FW reset failed: err=%d\n", err);
2531 		return err;
2532 	}
2533 
2534 	/*
2535 	 * Grab basic operational parameters.  These will predominantly have
2536 	 * been set up by the Physical Function Driver or will be hard coded
2537 	 * into the adapter.  We just have to live with them ...  Note that
2538 	 * we _must_ get our VPD parameters before our SGE parameters because
2539 	 * we need to know the adapter's core clock from the VPD in order to
2540 	 * properly decode the SGE Timer Values.
2541 	 */
2542 	err = t4vf_get_dev_params(adapter);
2543 	if (err) {
2544 		dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2545 			" device parameters: err=%d\n", err);
2546 		return err;
2547 	}
2548 	err = t4vf_get_vpd_params(adapter);
2549 	if (err) {
2550 		dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2551 			" VPD parameters: err=%d\n", err);
2552 		return err;
2553 	}
2554 	err = t4vf_get_sge_params(adapter);
2555 	if (err) {
2556 		dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2557 			" SGE parameters: err=%d\n", err);
2558 		return err;
2559 	}
2560 	err = t4vf_get_rss_glb_config(adapter);
2561 	if (err) {
2562 		dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2563 			" RSS parameters: err=%d\n", err);
2564 		return err;
2565 	}
2566 	if (adapter->params.rss.mode !=
2567 	    FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
2568 		dev_err(adapter->pdev_dev, "unable to operate with global RSS"
2569 			" mode %d\n", adapter->params.rss.mode);
2570 		return -EINVAL;
2571 	}
2572 	err = t4vf_sge_init(adapter);
2573 	if (err) {
2574 		dev_err(adapter->pdev_dev, "unable to use adapter parameters:"
2575 			" err=%d\n", err);
2576 		return err;
2577 	}
2578 
2579 	/* If we're running on newer firmware, let it know that we're
2580 	 * prepared to deal with encapsulated CPL messages.  Older
2581 	 * firmware won't understand this and we'll just get
2582 	 * unencapsulated messages ...
2583 	 */
2584 	param = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
2585 		FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP);
2586 	val = 1;
2587 	(void) t4vf_set_params(adapter, 1, &param, &val);
2588 
2589 	/*
2590 	 * Retrieve our RX interrupt holdoff timer values and counter
2591 	 * threshold values from the SGE parameters.
2592 	 */
2593 	s->timer_val[0] = core_ticks_to_us(adapter,
2594 		TIMERVALUE0_G(sge_params->sge_timer_value_0_and_1));
2595 	s->timer_val[1] = core_ticks_to_us(adapter,
2596 		TIMERVALUE1_G(sge_params->sge_timer_value_0_and_1));
2597 	s->timer_val[2] = core_ticks_to_us(adapter,
2598 		TIMERVALUE0_G(sge_params->sge_timer_value_2_and_3));
2599 	s->timer_val[3] = core_ticks_to_us(adapter,
2600 		TIMERVALUE1_G(sge_params->sge_timer_value_2_and_3));
2601 	s->timer_val[4] = core_ticks_to_us(adapter,
2602 		TIMERVALUE0_G(sge_params->sge_timer_value_4_and_5));
2603 	s->timer_val[5] = core_ticks_to_us(adapter,
2604 		TIMERVALUE1_G(sge_params->sge_timer_value_4_and_5));
2605 
2606 	s->counter_val[0] = THRESHOLD_0_G(sge_params->sge_ingress_rx_threshold);
2607 	s->counter_val[1] = THRESHOLD_1_G(sge_params->sge_ingress_rx_threshold);
2608 	s->counter_val[2] = THRESHOLD_2_G(sge_params->sge_ingress_rx_threshold);
2609 	s->counter_val[3] = THRESHOLD_3_G(sge_params->sge_ingress_rx_threshold);
2610 
2611 	/*
2612 	 * Grab our Virtual Interface resource allocation, extract the
2613 	 * features that we're interested in and do a bit of sanity testing on
2614 	 * what we discover.
2615 	 */
2616 	err = t4vf_get_vfres(adapter);
2617 	if (err) {
2618 		dev_err(adapter->pdev_dev, "unable to get virtual interface"
2619 			" resources: err=%d\n", err);
2620 		return err;
2621 	}
2622 
2623 	/* Check for various parameter sanity issues */
2624 	if (adapter->params.vfres.pmask == 0) {
2625 		dev_err(adapter->pdev_dev, "no port access configured\n"
2626 			"usable!\n");
2627 		return -EINVAL;
2628 	}
2629 	if (adapter->params.vfres.nvi == 0) {
2630 		dev_err(adapter->pdev_dev, "no virtual interfaces configured/"
2631 			"usable!\n");
2632 		return -EINVAL;
2633 	}
2634 
2635 	/* Initialize nports and max_ethqsets now that we have our Virtual
2636 	 * Function Resources.
2637 	 */
2638 	size_nports_qsets(adapter);
2639 
2640 	adapter->flags |= CXGB4VF_FW_OK;
2641 	return 0;
2642 }
2643 
2644 static inline void init_rspq(struct sge_rspq *rspq, u8 timer_idx,
2645 			     u8 pkt_cnt_idx, unsigned int size,
2646 			     unsigned int iqe_size)
2647 {
2648 	rspq->intr_params = (QINTR_TIMER_IDX_V(timer_idx) |
2649 			     (pkt_cnt_idx < SGE_NCOUNTERS ?
2650 			      QINTR_CNT_EN_F : 0));
2651 	rspq->pktcnt_idx = (pkt_cnt_idx < SGE_NCOUNTERS
2652 			    ? pkt_cnt_idx
2653 			    : 0);
2654 	rspq->iqe_len = iqe_size;
2655 	rspq->size = size;
2656 }
2657 
2658 /*
2659  * Perform default configuration of DMA queues depending on the number and
2660  * type of ports we found and the number of available CPUs.  Most settings can
2661  * be modified by the admin via ethtool and cxgbtool prior to the adapter
2662  * being brought up for the first time.
2663  */
2664 static void cfg_queues(struct adapter *adapter)
2665 {
2666 	struct sge *s = &adapter->sge;
2667 	int q10g, n10g, qidx, pidx, qs;
2668 	size_t iqe_size;
2669 
2670 	/*
2671 	 * We should not be called till we know how many Queue Sets we can
2672 	 * support.  In particular, this means that we need to know what kind
2673 	 * of interrupts we'll be using ...
2674 	 */
2675 	BUG_ON((adapter->flags &
2676 	       (CXGB4VF_USING_MSIX | CXGB4VF_USING_MSI)) == 0);
2677 
2678 	/*
2679 	 * Count the number of 10GbE Virtual Interfaces that we have.
2680 	 */
2681 	n10g = 0;
2682 	for_each_port(adapter, pidx)
2683 		n10g += is_x_10g_port(&adap2pinfo(adapter, pidx)->link_cfg);
2684 
2685 	/*
2686 	 * We default to 1 queue per non-10G port and up to # of cores queues
2687 	 * per 10G port.
2688 	 */
2689 	if (n10g == 0)
2690 		q10g = 0;
2691 	else {
2692 		int n1g = (adapter->params.nports - n10g);
2693 		q10g = (adapter->sge.max_ethqsets - n1g) / n10g;
2694 		if (q10g > num_online_cpus())
2695 			q10g = num_online_cpus();
2696 	}
2697 
2698 	/*
2699 	 * Allocate the "Queue Sets" to the various Virtual Interfaces.
2700 	 * The layout will be established in setup_sge_queues() when the
2701 	 * adapter is brough up for the first time.
2702 	 */
2703 	qidx = 0;
2704 	for_each_port(adapter, pidx) {
2705 		struct port_info *pi = adap2pinfo(adapter, pidx);
2706 
2707 		pi->first_qset = qidx;
2708 		pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
2709 		qidx += pi->nqsets;
2710 	}
2711 	s->ethqsets = qidx;
2712 
2713 	/*
2714 	 * The Ingress Queue Entry Size for our various Response Queues needs
2715 	 * to be big enough to accommodate the largest message we can receive
2716 	 * from the chip/firmware; which is 64 bytes ...
2717 	 */
2718 	iqe_size = 64;
2719 
2720 	/*
2721 	 * Set up default Queue Set parameters ...  Start off with the
2722 	 * shortest interrupt holdoff timer.
2723 	 */
2724 	for (qs = 0; qs < s->max_ethqsets; qs++) {
2725 		struct sge_eth_rxq *rxq = &s->ethrxq[qs];
2726 		struct sge_eth_txq *txq = &s->ethtxq[qs];
2727 
2728 		init_rspq(&rxq->rspq, 0, 0, 1024, iqe_size);
2729 		rxq->fl.size = 72;
2730 		txq->q.size = 1024;
2731 	}
2732 
2733 	/*
2734 	 * The firmware event queue is used for link state changes and
2735 	 * notifications of TX DMA completions.
2736 	 */
2737 	init_rspq(&s->fw_evtq, SGE_TIMER_RSTRT_CNTR, 0, 512, iqe_size);
2738 
2739 	/*
2740 	 * The forwarded interrupt queue is used when we're in MSI interrupt
2741 	 * mode.  In this mode all interrupts associated with RX queues will
2742 	 * be forwarded to a single queue which we'll associate with our MSI
2743 	 * interrupt vector.  The messages dropped in the forwarded interrupt
2744 	 * queue will indicate which ingress queue needs servicing ...  This
2745 	 * queue needs to be large enough to accommodate all of the ingress
2746 	 * queues which are forwarding their interrupt (+1 to prevent the PIDX
2747 	 * from equalling the CIDX if every ingress queue has an outstanding
2748 	 * interrupt).  The queue doesn't need to be any larger because no
2749 	 * ingress queue will ever have more than one outstanding interrupt at
2750 	 * any time ...
2751 	 */
2752 	init_rspq(&s->intrq, SGE_TIMER_RSTRT_CNTR, 0, MSIX_ENTRIES + 1,
2753 		  iqe_size);
2754 }
2755 
2756 /*
2757  * Reduce the number of Ethernet queues across all ports to at most n.
2758  * n provides at least one queue per port.
2759  */
2760 static void reduce_ethqs(struct adapter *adapter, int n)
2761 {
2762 	int i;
2763 	struct port_info *pi;
2764 
2765 	/*
2766 	 * While we have too many active Ether Queue Sets, interate across the
2767 	 * "ports" and reduce their individual Queue Set allocations.
2768 	 */
2769 	BUG_ON(n < adapter->params.nports);
2770 	while (n < adapter->sge.ethqsets)
2771 		for_each_port(adapter, i) {
2772 			pi = adap2pinfo(adapter, i);
2773 			if (pi->nqsets > 1) {
2774 				pi->nqsets--;
2775 				adapter->sge.ethqsets--;
2776 				if (adapter->sge.ethqsets <= n)
2777 					break;
2778 			}
2779 		}
2780 
2781 	/*
2782 	 * Reassign the starting Queue Sets for each of the "ports" ...
2783 	 */
2784 	n = 0;
2785 	for_each_port(adapter, i) {
2786 		pi = adap2pinfo(adapter, i);
2787 		pi->first_qset = n;
2788 		n += pi->nqsets;
2789 	}
2790 }
2791 
2792 /*
2793  * We need to grab enough MSI-X vectors to cover our interrupt needs.  Ideally
2794  * we get a separate MSI-X vector for every "Queue Set" plus any extras we
2795  * need.  Minimally we need one for every Virtual Interface plus those needed
2796  * for our "extras".  Note that this process may lower the maximum number of
2797  * allowed Queue Sets ...
2798  */
2799 static int enable_msix(struct adapter *adapter)
2800 {
2801 	int i, want, need, nqsets;
2802 	struct msix_entry entries[MSIX_ENTRIES];
2803 	struct sge *s = &adapter->sge;
2804 
2805 	for (i = 0; i < MSIX_ENTRIES; ++i)
2806 		entries[i].entry = i;
2807 
2808 	/*
2809 	 * We _want_ enough MSI-X interrupts to cover all of our "Queue Sets"
2810 	 * plus those needed for our "extras" (for example, the firmware
2811 	 * message queue).  We _need_ at least one "Queue Set" per Virtual
2812 	 * Interface plus those needed for our "extras".  So now we get to see
2813 	 * if the song is right ...
2814 	 */
2815 	want = s->max_ethqsets + MSIX_EXTRAS;
2816 	need = adapter->params.nports + MSIX_EXTRAS;
2817 
2818 	want = pci_enable_msix_range(adapter->pdev, entries, need, want);
2819 	if (want < 0)
2820 		return want;
2821 
2822 	nqsets = want - MSIX_EXTRAS;
2823 	if (nqsets < s->max_ethqsets) {
2824 		dev_warn(adapter->pdev_dev, "only enough MSI-X vectors"
2825 			 " for %d Queue Sets\n", nqsets);
2826 		s->max_ethqsets = nqsets;
2827 		if (nqsets < s->ethqsets)
2828 			reduce_ethqs(adapter, nqsets);
2829 	}
2830 	for (i = 0; i < want; ++i)
2831 		adapter->msix_info[i].vec = entries[i].vector;
2832 
2833 	return 0;
2834 }
2835 
2836 static const struct net_device_ops cxgb4vf_netdev_ops	= {
2837 	.ndo_open		= cxgb4vf_open,
2838 	.ndo_stop		= cxgb4vf_stop,
2839 	.ndo_start_xmit		= t4vf_eth_xmit,
2840 	.ndo_get_stats		= cxgb4vf_get_stats,
2841 	.ndo_set_rx_mode	= cxgb4vf_set_rxmode,
2842 	.ndo_set_mac_address	= cxgb4vf_set_mac_addr,
2843 	.ndo_validate_addr	= eth_validate_addr,
2844 	.ndo_eth_ioctl		= cxgb4vf_do_ioctl,
2845 	.ndo_change_mtu		= cxgb4vf_change_mtu,
2846 	.ndo_fix_features	= cxgb4vf_fix_features,
2847 	.ndo_set_features	= cxgb4vf_set_features,
2848 #ifdef CONFIG_NET_POLL_CONTROLLER
2849 	.ndo_poll_controller	= cxgb4vf_poll_controller,
2850 #endif
2851 };
2852 
2853 /**
2854  *	cxgb4vf_get_port_mask - Get port mask for the VF based on mac
2855  *				address stored on the adapter
2856  *	@adapter: The adapter
2857  *
2858  *	Find the the port mask for the VF based on the index of mac
2859  *	address stored in the adapter. If no mac address is stored on
2860  *	the adapter for the VF, use the port mask received from the
2861  *	firmware.
2862  */
2863 static unsigned int cxgb4vf_get_port_mask(struct adapter *adapter)
2864 {
2865 	unsigned int naddr = 1, pidx = 0;
2866 	unsigned int pmask, rmask = 0;
2867 	u8 mac[ETH_ALEN];
2868 	int err;
2869 
2870 	pmask = adapter->params.vfres.pmask;
2871 	while (pmask) {
2872 		if (pmask & 1) {
2873 			err = t4vf_get_vf_mac_acl(adapter, pidx, &naddr, mac);
2874 			if (!err && !is_zero_ether_addr(mac))
2875 				rmask |= (1 << pidx);
2876 		}
2877 		pmask >>= 1;
2878 		pidx++;
2879 	}
2880 	if (!rmask)
2881 		rmask = adapter->params.vfres.pmask;
2882 
2883 	return rmask;
2884 }
2885 
2886 /*
2887  * "Probe" a device: initialize a device and construct all kernel and driver
2888  * state needed to manage the device.  This routine is called "init_one" in
2889  * the PF Driver ...
2890  */
2891 static int cxgb4vf_pci_probe(struct pci_dev *pdev,
2892 			     const struct pci_device_id *ent)
2893 {
2894 	struct adapter *adapter;
2895 	struct net_device *netdev;
2896 	struct port_info *pi;
2897 	unsigned int pmask;
2898 	int pci_using_dac;
2899 	int err, pidx;
2900 
2901 	/*
2902 	 * Initialize generic PCI device state.
2903 	 */
2904 	err = pci_enable_device(pdev);
2905 	if (err) {
2906 		dev_err(&pdev->dev, "cannot enable PCI device\n");
2907 		return err;
2908 	}
2909 
2910 	/*
2911 	 * Reserve PCI resources for the device.  If we can't get them some
2912 	 * other driver may have already claimed the device ...
2913 	 */
2914 	err = pci_request_regions(pdev, KBUILD_MODNAME);
2915 	if (err) {
2916 		dev_err(&pdev->dev, "cannot obtain PCI resources\n");
2917 		goto err_disable_device;
2918 	}
2919 
2920 	/*
2921 	 * Set up our DMA mask: try for 64-bit address masking first and
2922 	 * fall back to 32-bit if we can't get 64 bits ...
2923 	 */
2924 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2925 	if (err == 0) {
2926 		pci_using_dac = 1;
2927 	} else {
2928 		err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
2929 		if (err != 0) {
2930 			dev_err(&pdev->dev, "no usable DMA configuration\n");
2931 			goto err_release_regions;
2932 		}
2933 		pci_using_dac = 0;
2934 	}
2935 
2936 	/*
2937 	 * Enable bus mastering for the device ...
2938 	 */
2939 	pci_set_master(pdev);
2940 
2941 	/*
2942 	 * Allocate our adapter data structure and attach it to the device.
2943 	 */
2944 	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2945 	if (!adapter) {
2946 		err = -ENOMEM;
2947 		goto err_release_regions;
2948 	}
2949 	pci_set_drvdata(pdev, adapter);
2950 	adapter->pdev = pdev;
2951 	adapter->pdev_dev = &pdev->dev;
2952 
2953 	adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
2954 				    (sizeof(struct mbox_cmd) *
2955 				     T4VF_OS_LOG_MBOX_CMDS),
2956 				    GFP_KERNEL);
2957 	if (!adapter->mbox_log) {
2958 		err = -ENOMEM;
2959 		goto err_free_adapter;
2960 	}
2961 	adapter->mbox_log->size = T4VF_OS_LOG_MBOX_CMDS;
2962 
2963 	/*
2964 	 * Initialize SMP data synchronization resources.
2965 	 */
2966 	spin_lock_init(&adapter->stats_lock);
2967 	spin_lock_init(&adapter->mbox_lock);
2968 	INIT_LIST_HEAD(&adapter->mlist.list);
2969 
2970 	/*
2971 	 * Map our I/O registers in BAR0.
2972 	 */
2973 	adapter->regs = pci_ioremap_bar(pdev, 0);
2974 	if (!adapter->regs) {
2975 		dev_err(&pdev->dev, "cannot map device registers\n");
2976 		err = -ENOMEM;
2977 		goto err_free_adapter;
2978 	}
2979 
2980 	/* Wait for the device to become ready before proceeding ...
2981 	 */
2982 	err = t4vf_prep_adapter(adapter);
2983 	if (err) {
2984 		dev_err(adapter->pdev_dev, "device didn't become ready:"
2985 			" err=%d\n", err);
2986 		goto err_unmap_bar0;
2987 	}
2988 
2989 	/* For T5 and later we want to use the new BAR-based User Doorbells,
2990 	 * so we need to map BAR2 here ...
2991 	 */
2992 	if (!is_t4(adapter->params.chip)) {
2993 		adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
2994 					   pci_resource_len(pdev, 2));
2995 		if (!adapter->bar2) {
2996 			dev_err(adapter->pdev_dev, "cannot map BAR2 doorbells\n");
2997 			err = -ENOMEM;
2998 			goto err_unmap_bar0;
2999 		}
3000 	}
3001 	/*
3002 	 * Initialize adapter level features.
3003 	 */
3004 	adapter->name = pci_name(pdev);
3005 	adapter->msg_enable = DFLT_MSG_ENABLE;
3006 
3007 	/* If possible, we use PCIe Relaxed Ordering Attribute to deliver
3008 	 * Ingress Packet Data to Free List Buffers in order to allow for
3009 	 * chipset performance optimizations between the Root Complex and
3010 	 * Memory Controllers.  (Messages to the associated Ingress Queue
3011 	 * notifying new Packet Placement in the Free Lists Buffers will be
3012 	 * send without the Relaxed Ordering Attribute thus guaranteeing that
3013 	 * all preceding PCIe Transaction Layer Packets will be processed
3014 	 * first.)  But some Root Complexes have various issues with Upstream
3015 	 * Transaction Layer Packets with the Relaxed Ordering Attribute set.
3016 	 * The PCIe devices which under the Root Complexes will be cleared the
3017 	 * Relaxed Ordering bit in the configuration space, So we check our
3018 	 * PCIe configuration space to see if it's flagged with advice against
3019 	 * using Relaxed Ordering.
3020 	 */
3021 	if (!pcie_relaxed_ordering_enabled(pdev))
3022 		adapter->flags |= CXGB4VF_ROOT_NO_RELAXED_ORDERING;
3023 
3024 	err = adap_init0(adapter);
3025 	if (err)
3026 		dev_err(&pdev->dev,
3027 			"Adapter initialization failed, error %d. Continuing in debug mode\n",
3028 			err);
3029 
3030 	/* Initialize hash mac addr list */
3031 	INIT_LIST_HEAD(&adapter->mac_hlist);
3032 
3033 	/*
3034 	 * Allocate our "adapter ports" and stitch everything together.
3035 	 */
3036 	pmask = cxgb4vf_get_port_mask(adapter);
3037 	for_each_port(adapter, pidx) {
3038 		int port_id, viid;
3039 		u8 mac[ETH_ALEN];
3040 		unsigned int naddr = 1;
3041 
3042 		/*
3043 		 * We simplistically allocate our virtual interfaces
3044 		 * sequentially across the port numbers to which we have
3045 		 * access rights.  This should be configurable in some manner
3046 		 * ...
3047 		 */
3048 		if (pmask == 0)
3049 			break;
3050 		port_id = ffs(pmask) - 1;
3051 		pmask &= ~(1 << port_id);
3052 
3053 		/*
3054 		 * Allocate our network device and stitch things together.
3055 		 */
3056 		netdev = alloc_etherdev_mq(sizeof(struct port_info),
3057 					   MAX_PORT_QSETS);
3058 		if (netdev == NULL) {
3059 			err = -ENOMEM;
3060 			goto err_free_dev;
3061 		}
3062 		adapter->port[pidx] = netdev;
3063 		SET_NETDEV_DEV(netdev, &pdev->dev);
3064 		pi = netdev_priv(netdev);
3065 		pi->adapter = adapter;
3066 		pi->pidx = pidx;
3067 		pi->port_id = port_id;
3068 
3069 		/*
3070 		 * Initialize the starting state of our "port" and register
3071 		 * it.
3072 		 */
3073 		pi->xact_addr_filt = -1;
3074 		netdev->irq = pdev->irq;
3075 
3076 		netdev->hw_features = NETIF_F_SG | TSO_FLAGS | NETIF_F_GRO |
3077 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM |
3078 			NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
3079 		netdev->features = netdev->hw_features;
3080 		if (pci_using_dac)
3081 			netdev->features |= NETIF_F_HIGHDMA;
3082 		netdev->vlan_features = netdev->features & VLAN_FEAT;
3083 
3084 		netdev->priv_flags |= IFF_UNICAST_FLT;
3085 		netdev->min_mtu = 81;
3086 		netdev->max_mtu = ETH_MAX_MTU;
3087 
3088 		netdev->netdev_ops = &cxgb4vf_netdev_ops;
3089 		netdev->ethtool_ops = &cxgb4vf_ethtool_ops;
3090 		netdev->dev_port = pi->port_id;
3091 
3092 		/*
3093 		 * If we haven't been able to contact the firmware, there's
3094 		 * nothing else we can do for this "port" ...
3095 		 */
3096 		if (!(adapter->flags & CXGB4VF_FW_OK))
3097 			continue;
3098 
3099 		viid = t4vf_alloc_vi(adapter, port_id);
3100 		if (viid < 0) {
3101 			dev_err(&pdev->dev,
3102 				"cannot allocate VI for port %d: err=%d\n",
3103 				port_id, viid);
3104 			err = viid;
3105 			goto err_free_dev;
3106 		}
3107 		pi->viid = viid;
3108 
3109 		/*
3110 		 * Initialize the hardware/software state for the port.
3111 		 */
3112 		err = t4vf_port_init(adapter, pidx);
3113 		if (err) {
3114 			dev_err(&pdev->dev, "cannot initialize port %d\n",
3115 				pidx);
3116 			goto err_free_dev;
3117 		}
3118 
3119 		err = t4vf_get_vf_mac_acl(adapter, port_id, &naddr, mac);
3120 		if (err) {
3121 			dev_err(&pdev->dev,
3122 				"unable to determine MAC ACL address, "
3123 				"continuing anyway.. (status %d)\n", err);
3124 		} else if (naddr && adapter->params.vfres.nvi == 1) {
3125 			struct sockaddr addr;
3126 
3127 			ether_addr_copy(addr.sa_data, mac);
3128 			err = cxgb4vf_set_mac_addr(netdev, &addr);
3129 			if (err) {
3130 				dev_err(&pdev->dev,
3131 					"unable to set MAC address %pM\n",
3132 					mac);
3133 				goto err_free_dev;
3134 			}
3135 			dev_info(&pdev->dev,
3136 				 "Using assigned MAC ACL: %pM\n", mac);
3137 		}
3138 	}
3139 
3140 	/* See what interrupts we'll be using.  If we've been configured to
3141 	 * use MSI-X interrupts, try to enable them but fall back to using
3142 	 * MSI interrupts if we can't enable MSI-X interrupts.  If we can't
3143 	 * get MSI interrupts we bail with the error.
3144 	 */
3145 	if (msi == MSI_MSIX && enable_msix(adapter) == 0)
3146 		adapter->flags |= CXGB4VF_USING_MSIX;
3147 	else {
3148 		if (msi == MSI_MSIX) {
3149 			dev_info(adapter->pdev_dev,
3150 				 "Unable to use MSI-X Interrupts; falling "
3151 				 "back to MSI Interrupts\n");
3152 
3153 			/* We're going to need a Forwarded Interrupt Queue so
3154 			 * that may cut into how many Queue Sets we can
3155 			 * support.
3156 			 */
3157 			msi = MSI_MSI;
3158 			size_nports_qsets(adapter);
3159 		}
3160 		err = pci_enable_msi(pdev);
3161 		if (err) {
3162 			dev_err(&pdev->dev, "Unable to allocate MSI Interrupts;"
3163 				" err=%d\n", err);
3164 			goto err_free_dev;
3165 		}
3166 		adapter->flags |= CXGB4VF_USING_MSI;
3167 	}
3168 
3169 	/* Now that we know how many "ports" we have and what interrupt
3170 	 * mechanism we're going to use, we can configure our queue resources.
3171 	 */
3172 	cfg_queues(adapter);
3173 
3174 	/*
3175 	 * The "card" is now ready to go.  If any errors occur during device
3176 	 * registration we do not fail the whole "card" but rather proceed
3177 	 * only with the ports we manage to register successfully.  However we
3178 	 * must register at least one net device.
3179 	 */
3180 	for_each_port(adapter, pidx) {
3181 		struct port_info *pi = netdev_priv(adapter->port[pidx]);
3182 		netdev = adapter->port[pidx];
3183 		if (netdev == NULL)
3184 			continue;
3185 
3186 		netif_set_real_num_tx_queues(netdev, pi->nqsets);
3187 		netif_set_real_num_rx_queues(netdev, pi->nqsets);
3188 
3189 		err = register_netdev(netdev);
3190 		if (err) {
3191 			dev_warn(&pdev->dev, "cannot register net device %s,"
3192 				 " skipping\n", netdev->name);
3193 			continue;
3194 		}
3195 
3196 		netif_carrier_off(netdev);
3197 		set_bit(pidx, &adapter->registered_device_map);
3198 	}
3199 	if (adapter->registered_device_map == 0) {
3200 		dev_err(&pdev->dev, "could not register any net devices\n");
3201 		goto err_disable_interrupts;
3202 	}
3203 
3204 	/*
3205 	 * Set up our debugfs entries.
3206 	 */
3207 	if (!IS_ERR_OR_NULL(cxgb4vf_debugfs_root)) {
3208 		adapter->debugfs_root =
3209 			debugfs_create_dir(pci_name(pdev),
3210 					   cxgb4vf_debugfs_root);
3211 		setup_debugfs(adapter);
3212 	}
3213 
3214 	/*
3215 	 * Print a short notice on the existence and configuration of the new
3216 	 * VF network device ...
3217 	 */
3218 	for_each_port(adapter, pidx) {
3219 		dev_info(adapter->pdev_dev, "%s: Chelsio VF NIC PCIe %s\n",
3220 			 adapter->port[pidx]->name,
3221 			 (adapter->flags & CXGB4VF_USING_MSIX) ? "MSI-X" :
3222 			 (adapter->flags & CXGB4VF_USING_MSI)  ? "MSI" : "");
3223 	}
3224 
3225 	/*
3226 	 * Return success!
3227 	 */
3228 	return 0;
3229 
3230 	/*
3231 	 * Error recovery and exit code.  Unwind state that's been created
3232 	 * so far and return the error.
3233 	 */
3234 err_disable_interrupts:
3235 	if (adapter->flags & CXGB4VF_USING_MSIX) {
3236 		pci_disable_msix(adapter->pdev);
3237 		adapter->flags &= ~CXGB4VF_USING_MSIX;
3238 	} else if (adapter->flags & CXGB4VF_USING_MSI) {
3239 		pci_disable_msi(adapter->pdev);
3240 		adapter->flags &= ~CXGB4VF_USING_MSI;
3241 	}
3242 
3243 err_free_dev:
3244 	for_each_port(adapter, pidx) {
3245 		netdev = adapter->port[pidx];
3246 		if (netdev == NULL)
3247 			continue;
3248 		pi = netdev_priv(netdev);
3249 		if (pi->viid)
3250 			t4vf_free_vi(adapter, pi->viid);
3251 		if (test_bit(pidx, &adapter->registered_device_map))
3252 			unregister_netdev(netdev);
3253 		free_netdev(netdev);
3254 	}
3255 
3256 	if (!is_t4(adapter->params.chip))
3257 		iounmap(adapter->bar2);
3258 
3259 err_unmap_bar0:
3260 	iounmap(adapter->regs);
3261 
3262 err_free_adapter:
3263 	kfree(adapter->mbox_log);
3264 	kfree(adapter);
3265 
3266 err_release_regions:
3267 	pci_release_regions(pdev);
3268 	pci_clear_master(pdev);
3269 
3270 err_disable_device:
3271 	pci_disable_device(pdev);
3272 
3273 	return err;
3274 }
3275 
3276 /*
3277  * "Remove" a device: tear down all kernel and driver state created in the
3278  * "probe" routine and quiesce the device (disable interrupts, etc.).  (Note
3279  * that this is called "remove_one" in the PF Driver.)
3280  */
3281 static void cxgb4vf_pci_remove(struct pci_dev *pdev)
3282 {
3283 	struct adapter *adapter = pci_get_drvdata(pdev);
3284 	struct hash_mac_addr *entry, *tmp;
3285 
3286 	/*
3287 	 * Tear down driver state associated with device.
3288 	 */
3289 	if (adapter) {
3290 		int pidx;
3291 
3292 		/*
3293 		 * Stop all of our activity.  Unregister network port,
3294 		 * disable interrupts, etc.
3295 		 */
3296 		for_each_port(adapter, pidx)
3297 			if (test_bit(pidx, &adapter->registered_device_map))
3298 				unregister_netdev(adapter->port[pidx]);
3299 		t4vf_sge_stop(adapter);
3300 		if (adapter->flags & CXGB4VF_USING_MSIX) {
3301 			pci_disable_msix(adapter->pdev);
3302 			adapter->flags &= ~CXGB4VF_USING_MSIX;
3303 		} else if (adapter->flags & CXGB4VF_USING_MSI) {
3304 			pci_disable_msi(adapter->pdev);
3305 			adapter->flags &= ~CXGB4VF_USING_MSI;
3306 		}
3307 
3308 		/*
3309 		 * Tear down our debugfs entries.
3310 		 */
3311 		if (!IS_ERR_OR_NULL(adapter->debugfs_root)) {
3312 			cleanup_debugfs(adapter);
3313 			debugfs_remove_recursive(adapter->debugfs_root);
3314 		}
3315 
3316 		/*
3317 		 * Free all of the various resources which we've acquired ...
3318 		 */
3319 		t4vf_free_sge_resources(adapter);
3320 		for_each_port(adapter, pidx) {
3321 			struct net_device *netdev = adapter->port[pidx];
3322 			struct port_info *pi;
3323 
3324 			if (netdev == NULL)
3325 				continue;
3326 
3327 			pi = netdev_priv(netdev);
3328 			if (pi->viid)
3329 				t4vf_free_vi(adapter, pi->viid);
3330 			free_netdev(netdev);
3331 		}
3332 		iounmap(adapter->regs);
3333 		if (!is_t4(adapter->params.chip))
3334 			iounmap(adapter->bar2);
3335 		kfree(adapter->mbox_log);
3336 		list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist,
3337 					 list) {
3338 			list_del(&entry->list);
3339 			kfree(entry);
3340 		}
3341 		kfree(adapter);
3342 	}
3343 
3344 	/*
3345 	 * Disable the device and release its PCI resources.
3346 	 */
3347 	pci_disable_device(pdev);
3348 	pci_clear_master(pdev);
3349 	pci_release_regions(pdev);
3350 }
3351 
3352 /*
3353  * "Shutdown" quiesce the device, stopping Ingress Packet and Interrupt
3354  * delivery.
3355  */
3356 static void cxgb4vf_pci_shutdown(struct pci_dev *pdev)
3357 {
3358 	struct adapter *adapter;
3359 	int pidx;
3360 
3361 	adapter = pci_get_drvdata(pdev);
3362 	if (!adapter)
3363 		return;
3364 
3365 	/* Disable all Virtual Interfaces.  This will shut down the
3366 	 * delivery of all ingress packets into the chip for these
3367 	 * Virtual Interfaces.
3368 	 */
3369 	for_each_port(adapter, pidx)
3370 		if (test_bit(pidx, &adapter->registered_device_map))
3371 			unregister_netdev(adapter->port[pidx]);
3372 
3373 	/* Free up all Queues which will prevent further DMA and
3374 	 * Interrupts allowing various internal pathways to drain.
3375 	 */
3376 	t4vf_sge_stop(adapter);
3377 	if (adapter->flags & CXGB4VF_USING_MSIX) {
3378 		pci_disable_msix(adapter->pdev);
3379 		adapter->flags &= ~CXGB4VF_USING_MSIX;
3380 	} else if (adapter->flags & CXGB4VF_USING_MSI) {
3381 		pci_disable_msi(adapter->pdev);
3382 		adapter->flags &= ~CXGB4VF_USING_MSI;
3383 	}
3384 
3385 	/*
3386 	 * Free up all Queues which will prevent further DMA and
3387 	 * Interrupts allowing various internal pathways to drain.
3388 	 */
3389 	t4vf_free_sge_resources(adapter);
3390 	pci_set_drvdata(pdev, NULL);
3391 }
3392 
3393 /* Macros needed to support the PCI Device ID Table ...
3394  */
3395 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
3396 	static const struct pci_device_id cxgb4vf_pci_tbl[] = {
3397 #define CH_PCI_DEVICE_ID_FUNCTION	0x8
3398 
3399 #define CH_PCI_ID_TABLE_ENTRY(devid) \
3400 		{ PCI_VDEVICE(CHELSIO, (devid)), 0 }
3401 
3402 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END { 0, } }
3403 
3404 #include "../cxgb4/t4_pci_id_tbl.h"
3405 
3406 MODULE_DESCRIPTION(DRV_DESC);
3407 MODULE_AUTHOR("Chelsio Communications");
3408 MODULE_LICENSE("Dual BSD/GPL");
3409 MODULE_DEVICE_TABLE(pci, cxgb4vf_pci_tbl);
3410 
3411 static struct pci_driver cxgb4vf_driver = {
3412 	.name		= KBUILD_MODNAME,
3413 	.id_table	= cxgb4vf_pci_tbl,
3414 	.probe		= cxgb4vf_pci_probe,
3415 	.remove		= cxgb4vf_pci_remove,
3416 	.shutdown	= cxgb4vf_pci_shutdown,
3417 };
3418 
3419 /*
3420  * Initialize global driver state.
3421  */
3422 static int __init cxgb4vf_module_init(void)
3423 {
3424 	int ret;
3425 
3426 	/*
3427 	 * Vet our module parameters.
3428 	 */
3429 	if (msi != MSI_MSIX && msi != MSI_MSI) {
3430 		pr_warn("bad module parameter msi=%d; must be %d (MSI-X or MSI) or %d (MSI)\n",
3431 			msi, MSI_MSIX, MSI_MSI);
3432 		return -EINVAL;
3433 	}
3434 
3435 	/* Debugfs support is optional, debugfs will warn if this fails */
3436 	cxgb4vf_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3437 
3438 	ret = pci_register_driver(&cxgb4vf_driver);
3439 	if (ret < 0)
3440 		debugfs_remove(cxgb4vf_debugfs_root);
3441 	return ret;
3442 }
3443 
3444 /*
3445  * Tear down global driver state.
3446  */
3447 static void __exit cxgb4vf_module_exit(void)
3448 {
3449 	pci_unregister_driver(&cxgb4vf_driver);
3450 	debugfs_remove(cxgb4vf_debugfs_root);
3451 }
3452 
3453 module_init(cxgb4vf_module_init);
3454 module_exit(cxgb4vf_module_exit);
3455