1 /*
2  * Copyright 2008-2010 Cisco Systems, Inc.  All rights reserved.
3  * Copyright 2007 Nuova Systems, Inc.  All rights reserved.
4  *
5  * This program is free software; you may redistribute it and/or modify
6  * it under the terms of the GNU General Public License as published by
7  * the Free Software Foundation; version 2 of the License.
8  *
9  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
10  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
11  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
12  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
13  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
14  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
15  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
16  * SOFTWARE.
17  *
18  */
19 
20 #include <linux/module.h>
21 #include <linux/kernel.h>
22 #include <linux/string.h>
23 #include <linux/errno.h>
24 #include <linux/types.h>
25 #include <linux/init.h>
26 #include <linux/interrupt.h>
27 #include <linux/workqueue.h>
28 #include <linux/pci.h>
29 #include <linux/netdevice.h>
30 #include <linux/etherdevice.h>
31 #include <linux/if.h>
32 #include <linux/if_ether.h>
33 #include <linux/if_vlan.h>
34 #include <linux/in.h>
35 #include <linux/ip.h>
36 #include <linux/ipv6.h>
37 #include <linux/tcp.h>
38 #include <linux/rtnetlink.h>
39 #include <linux/prefetch.h>
40 #include <net/ip6_checksum.h>
41 #include <linux/ktime.h>
42 #include <linux/numa.h>
43 #ifdef CONFIG_RFS_ACCEL
44 #include <linux/cpu_rmap.h>
45 #endif
46 #include <linux/crash_dump.h>
47 #include <net/busy_poll.h>
48 #include <net/vxlan.h>
49 
50 #include "cq_enet_desc.h"
51 #include "vnic_dev.h"
52 #include "vnic_intr.h"
53 #include "vnic_stats.h"
54 #include "vnic_vic.h"
55 #include "enic_res.h"
56 #include "enic.h"
57 #include "enic_dev.h"
58 #include "enic_pp.h"
59 #include "enic_clsf.h"
60 
61 #define ENIC_NOTIFY_TIMER_PERIOD	(2 * HZ)
62 #define WQ_ENET_MAX_DESC_LEN		(1 << WQ_ENET_LEN_BITS)
63 #define MAX_TSO				(1 << 16)
64 #define ENIC_DESC_MAX_SPLITS		(MAX_TSO / WQ_ENET_MAX_DESC_LEN + 1)
65 
66 #define PCI_DEVICE_ID_CISCO_VIC_ENET         0x0043  /* ethernet vnic */
67 #define PCI_DEVICE_ID_CISCO_VIC_ENET_DYN     0x0044  /* enet dynamic vnic */
68 #define PCI_DEVICE_ID_CISCO_VIC_ENET_VF      0x0071  /* enet SRIOV VF */
69 
70 #define RX_COPYBREAK_DEFAULT		256
71 
72 /* Supported devices */
73 static const struct pci_device_id enic_id_table[] = {
74 	{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET) },
75 	{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_DYN) },
76 	{ PCI_VDEVICE(CISCO, PCI_DEVICE_ID_CISCO_VIC_ENET_VF) },
77 	{ 0, }	/* end of table */
78 };
79 
80 MODULE_DESCRIPTION(DRV_DESCRIPTION);
81 MODULE_AUTHOR("Scott Feldman <scofeldm@cisco.com>");
82 MODULE_LICENSE("GPL");
83 MODULE_VERSION(DRV_VERSION);
84 MODULE_DEVICE_TABLE(pci, enic_id_table);
85 
86 #define ENIC_LARGE_PKT_THRESHOLD		1000
87 #define ENIC_MAX_COALESCE_TIMERS		10
88 /*  Interrupt moderation table, which will be used to decide the
89  *  coalescing timer values
90  *  {rx_rate in Mbps, mapping percentage of the range}
91  */
92 static struct enic_intr_mod_table mod_table[ENIC_MAX_COALESCE_TIMERS + 1] = {
93 	{4000,  0},
94 	{4400, 10},
95 	{5060, 20},
96 	{5230, 30},
97 	{5540, 40},
98 	{5820, 50},
99 	{6120, 60},
100 	{6435, 70},
101 	{6745, 80},
102 	{7000, 90},
103 	{0xFFFFFFFF, 100}
104 };
105 
106 /* This table helps the driver to pick different ranges for rx coalescing
107  * timer depending on the link speed.
108  */
109 static struct enic_intr_mod_range mod_range[ENIC_MAX_LINK_SPEEDS] = {
110 	{0,  0}, /* 0  - 4  Gbps */
111 	{0,  3}, /* 4  - 10 Gbps */
112 	{3,  6}, /* 10 - 40 Gbps */
113 };
114 
115 static void enic_init_affinity_hint(struct enic *enic)
116 {
117 	int numa_node = dev_to_node(&enic->pdev->dev);
118 	int i;
119 
120 	for (i = 0; i < enic->intr_count; i++) {
121 		if (enic_is_err_intr(enic, i) || enic_is_notify_intr(enic, i) ||
122 		    (cpumask_available(enic->msix[i].affinity_mask) &&
123 		     !cpumask_empty(enic->msix[i].affinity_mask)))
124 			continue;
125 		if (zalloc_cpumask_var(&enic->msix[i].affinity_mask,
126 				       GFP_KERNEL))
127 			cpumask_set_cpu(cpumask_local_spread(i, numa_node),
128 					enic->msix[i].affinity_mask);
129 	}
130 }
131 
132 static void enic_free_affinity_hint(struct enic *enic)
133 {
134 	int i;
135 
136 	for (i = 0; i < enic->intr_count; i++) {
137 		if (enic_is_err_intr(enic, i) || enic_is_notify_intr(enic, i))
138 			continue;
139 		free_cpumask_var(enic->msix[i].affinity_mask);
140 	}
141 }
142 
143 static void enic_set_affinity_hint(struct enic *enic)
144 {
145 	int i;
146 	int err;
147 
148 	for (i = 0; i < enic->intr_count; i++) {
149 		if (enic_is_err_intr(enic, i)		||
150 		    enic_is_notify_intr(enic, i)	||
151 		    !cpumask_available(enic->msix[i].affinity_mask) ||
152 		    cpumask_empty(enic->msix[i].affinity_mask))
153 			continue;
154 		err = irq_set_affinity_hint(enic->msix_entry[i].vector,
155 					    enic->msix[i].affinity_mask);
156 		if (err)
157 			netdev_warn(enic->netdev, "irq_set_affinity_hint failed, err %d\n",
158 				    err);
159 	}
160 
161 	for (i = 0; i < enic->wq_count; i++) {
162 		int wq_intr = enic_msix_wq_intr(enic, i);
163 
164 		if (cpumask_available(enic->msix[wq_intr].affinity_mask) &&
165 		    !cpumask_empty(enic->msix[wq_intr].affinity_mask))
166 			netif_set_xps_queue(enic->netdev,
167 					    enic->msix[wq_intr].affinity_mask,
168 					    i);
169 	}
170 }
171 
172 static void enic_unset_affinity_hint(struct enic *enic)
173 {
174 	int i;
175 
176 	for (i = 0; i < enic->intr_count; i++)
177 		irq_set_affinity_hint(enic->msix_entry[i].vector, NULL);
178 }
179 
180 static void enic_udp_tunnel_add(struct net_device *netdev,
181 				struct udp_tunnel_info *ti)
182 {
183 	struct enic *enic = netdev_priv(netdev);
184 	__be16 port = ti->port;
185 	int err;
186 
187 	spin_lock_bh(&enic->devcmd_lock);
188 
189 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN) {
190 		netdev_info(netdev, "udp_tnl: only vxlan tunnel offload supported");
191 		goto error;
192 	}
193 
194 	switch (ti->sa_family) {
195 	case AF_INET6:
196 		if (!(enic->vxlan.flags & ENIC_VXLAN_OUTER_IPV6)) {
197 			netdev_info(netdev, "vxlan: only IPv4 offload supported");
198 			goto error;
199 		}
200 		/* Fall through */
201 	case AF_INET:
202 		break;
203 	default:
204 		goto error;
205 	}
206 
207 	if (enic->vxlan.vxlan_udp_port_number) {
208 		if (ntohs(port) == enic->vxlan.vxlan_udp_port_number)
209 			netdev_warn(netdev, "vxlan: udp port already offloaded");
210 		else
211 			netdev_info(netdev, "vxlan: offload supported for only one UDP port");
212 
213 		goto error;
214 	}
215 	if ((vnic_dev_get_res_count(enic->vdev, RES_TYPE_WQ) != 1) &&
216 	    !(enic->vxlan.flags & ENIC_VXLAN_MULTI_WQ)) {
217 		netdev_info(netdev, "vxlan: vxlan offload with multi wq not supported on this adapter");
218 		goto error;
219 	}
220 
221 	err = vnic_dev_overlay_offload_cfg(enic->vdev,
222 					   OVERLAY_CFG_VXLAN_PORT_UPDATE,
223 					   ntohs(port));
224 	if (err)
225 		goto error;
226 
227 	err = vnic_dev_overlay_offload_ctrl(enic->vdev, OVERLAY_FEATURE_VXLAN,
228 					    enic->vxlan.patch_level);
229 	if (err)
230 		goto error;
231 
232 	enic->vxlan.vxlan_udp_port_number = ntohs(port);
233 
234 	netdev_info(netdev, "vxlan fw-vers-%d: offload enabled for udp port: %d, sa_family: %d ",
235 		    (int)enic->vxlan.patch_level, ntohs(port), ti->sa_family);
236 
237 	goto unlock;
238 
239 error:
240 	netdev_info(netdev, "failed to offload udp port: %d, sa_family: %d, type: %d",
241 		    ntohs(port), ti->sa_family, ti->type);
242 unlock:
243 	spin_unlock_bh(&enic->devcmd_lock);
244 }
245 
246 static void enic_udp_tunnel_del(struct net_device *netdev,
247 				struct udp_tunnel_info *ti)
248 {
249 	struct enic *enic = netdev_priv(netdev);
250 	int err;
251 
252 	spin_lock_bh(&enic->devcmd_lock);
253 
254 	if ((ntohs(ti->port) != enic->vxlan.vxlan_udp_port_number) ||
255 	    ti->type != UDP_TUNNEL_TYPE_VXLAN) {
256 		netdev_info(netdev, "udp_tnl: port:%d, sa_family: %d, type: %d not offloaded",
257 			    ntohs(ti->port), ti->sa_family, ti->type);
258 		goto unlock;
259 	}
260 
261 	err = vnic_dev_overlay_offload_ctrl(enic->vdev, OVERLAY_FEATURE_VXLAN,
262 					    OVERLAY_OFFLOAD_DISABLE);
263 	if (err) {
264 		netdev_err(netdev, "vxlan: del offload udp port: %d failed",
265 			   ntohs(ti->port));
266 		goto unlock;
267 	}
268 
269 	enic->vxlan.vxlan_udp_port_number = 0;
270 
271 	netdev_info(netdev, "vxlan: del offload udp port %d, family %d\n",
272 		    ntohs(ti->port), ti->sa_family);
273 
274 unlock:
275 	spin_unlock_bh(&enic->devcmd_lock);
276 }
277 
278 static netdev_features_t enic_features_check(struct sk_buff *skb,
279 					     struct net_device *dev,
280 					     netdev_features_t features)
281 {
282 	const struct ethhdr *eth = (struct ethhdr *)skb_inner_mac_header(skb);
283 	struct enic *enic = netdev_priv(dev);
284 	struct udphdr *udph;
285 	u16 port = 0;
286 	u8 proto;
287 
288 	if (!skb->encapsulation)
289 		return features;
290 
291 	features = vxlan_features_check(skb, features);
292 
293 	switch (vlan_get_protocol(skb)) {
294 	case htons(ETH_P_IPV6):
295 		if (!(enic->vxlan.flags & ENIC_VXLAN_OUTER_IPV6))
296 			goto out;
297 		proto = ipv6_hdr(skb)->nexthdr;
298 		break;
299 	case htons(ETH_P_IP):
300 		proto = ip_hdr(skb)->protocol;
301 		break;
302 	default:
303 		goto out;
304 	}
305 
306 	switch (eth->h_proto) {
307 	case ntohs(ETH_P_IPV6):
308 		if (!(enic->vxlan.flags & ENIC_VXLAN_INNER_IPV6))
309 			goto out;
310 		/* Fall through */
311 	case ntohs(ETH_P_IP):
312 		break;
313 	default:
314 		goto out;
315 	}
316 
317 
318 	if (proto == IPPROTO_UDP) {
319 		udph = udp_hdr(skb);
320 		port = be16_to_cpu(udph->dest);
321 	}
322 
323 	/* HW supports offload of only one UDP port. Remove CSUM and GSO MASK
324 	 * for other UDP port tunnels
325 	 */
326 	if (port  != enic->vxlan.vxlan_udp_port_number)
327 		goto out;
328 
329 	return features;
330 
331 out:
332 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
333 }
334 
335 int enic_is_dynamic(struct enic *enic)
336 {
337 	return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_DYN;
338 }
339 
340 int enic_sriov_enabled(struct enic *enic)
341 {
342 	return (enic->priv_flags & ENIC_SRIOV_ENABLED) ? 1 : 0;
343 }
344 
345 static int enic_is_sriov_vf(struct enic *enic)
346 {
347 	return enic->pdev->device == PCI_DEVICE_ID_CISCO_VIC_ENET_VF;
348 }
349 
350 int enic_is_valid_vf(struct enic *enic, int vf)
351 {
352 #ifdef CONFIG_PCI_IOV
353 	return vf >= 0 && vf < enic->num_vfs;
354 #else
355 	return 0;
356 #endif
357 }
358 
359 static void enic_free_wq_buf(struct vnic_wq *wq, struct vnic_wq_buf *buf)
360 {
361 	struct enic *enic = vnic_dev_priv(wq->vdev);
362 
363 	if (buf->sop)
364 		pci_unmap_single(enic->pdev, buf->dma_addr,
365 			buf->len, PCI_DMA_TODEVICE);
366 	else
367 		pci_unmap_page(enic->pdev, buf->dma_addr,
368 			buf->len, PCI_DMA_TODEVICE);
369 
370 	if (buf->os_buf)
371 		dev_kfree_skb_any(buf->os_buf);
372 }
373 
374 static void enic_wq_free_buf(struct vnic_wq *wq,
375 	struct cq_desc *cq_desc, struct vnic_wq_buf *buf, void *opaque)
376 {
377 	enic_free_wq_buf(wq, buf);
378 }
379 
380 static int enic_wq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
381 	u8 type, u16 q_number, u16 completed_index, void *opaque)
382 {
383 	struct enic *enic = vnic_dev_priv(vdev);
384 
385 	spin_lock(&enic->wq_lock[q_number]);
386 
387 	vnic_wq_service(&enic->wq[q_number], cq_desc,
388 		completed_index, enic_wq_free_buf,
389 		opaque);
390 
391 	if (netif_tx_queue_stopped(netdev_get_tx_queue(enic->netdev, q_number)) &&
392 	    vnic_wq_desc_avail(&enic->wq[q_number]) >=
393 	    (MAX_SKB_FRAGS + ENIC_DESC_MAX_SPLITS))
394 		netif_wake_subqueue(enic->netdev, q_number);
395 
396 	spin_unlock(&enic->wq_lock[q_number]);
397 
398 	return 0;
399 }
400 
401 static bool enic_log_q_error(struct enic *enic)
402 {
403 	unsigned int i;
404 	u32 error_status;
405 	bool err = false;
406 
407 	for (i = 0; i < enic->wq_count; i++) {
408 		error_status = vnic_wq_error_status(&enic->wq[i]);
409 		err |= error_status;
410 		if (error_status)
411 			netdev_err(enic->netdev, "WQ[%d] error_status %d\n",
412 				i, error_status);
413 	}
414 
415 	for (i = 0; i < enic->rq_count; i++) {
416 		error_status = vnic_rq_error_status(&enic->rq[i]);
417 		err |= error_status;
418 		if (error_status)
419 			netdev_err(enic->netdev, "RQ[%d] error_status %d\n",
420 				i, error_status);
421 	}
422 
423 	return err;
424 }
425 
426 static void enic_msglvl_check(struct enic *enic)
427 {
428 	u32 msg_enable = vnic_dev_msg_lvl(enic->vdev);
429 
430 	if (msg_enable != enic->msg_enable) {
431 		netdev_info(enic->netdev, "msg lvl changed from 0x%x to 0x%x\n",
432 			enic->msg_enable, msg_enable);
433 		enic->msg_enable = msg_enable;
434 	}
435 }
436 
437 static void enic_mtu_check(struct enic *enic)
438 {
439 	u32 mtu = vnic_dev_mtu(enic->vdev);
440 	struct net_device *netdev = enic->netdev;
441 
442 	if (mtu && mtu != enic->port_mtu) {
443 		enic->port_mtu = mtu;
444 		if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic)) {
445 			mtu = max_t(int, ENIC_MIN_MTU,
446 				min_t(int, ENIC_MAX_MTU, mtu));
447 			if (mtu != netdev->mtu)
448 				schedule_work(&enic->change_mtu_work);
449 		} else {
450 			if (mtu < netdev->mtu)
451 				netdev_warn(netdev,
452 					"interface MTU (%d) set higher "
453 					"than switch port MTU (%d)\n",
454 					netdev->mtu, mtu);
455 		}
456 	}
457 }
458 
459 static void enic_link_check(struct enic *enic)
460 {
461 	int link_status = vnic_dev_link_status(enic->vdev);
462 	int carrier_ok = netif_carrier_ok(enic->netdev);
463 
464 	if (link_status && !carrier_ok) {
465 		netdev_info(enic->netdev, "Link UP\n");
466 		netif_carrier_on(enic->netdev);
467 	} else if (!link_status && carrier_ok) {
468 		netdev_info(enic->netdev, "Link DOWN\n");
469 		netif_carrier_off(enic->netdev);
470 	}
471 }
472 
473 static void enic_notify_check(struct enic *enic)
474 {
475 	enic_msglvl_check(enic);
476 	enic_mtu_check(enic);
477 	enic_link_check(enic);
478 }
479 
480 #define ENIC_TEST_INTR(pba, i) (pba & (1 << i))
481 
482 static irqreturn_t enic_isr_legacy(int irq, void *data)
483 {
484 	struct net_device *netdev = data;
485 	struct enic *enic = netdev_priv(netdev);
486 	unsigned int io_intr = enic_legacy_io_intr();
487 	unsigned int err_intr = enic_legacy_err_intr();
488 	unsigned int notify_intr = enic_legacy_notify_intr();
489 	u32 pba;
490 
491 	vnic_intr_mask(&enic->intr[io_intr]);
492 
493 	pba = vnic_intr_legacy_pba(enic->legacy_pba);
494 	if (!pba) {
495 		vnic_intr_unmask(&enic->intr[io_intr]);
496 		return IRQ_NONE;	/* not our interrupt */
497 	}
498 
499 	if (ENIC_TEST_INTR(pba, notify_intr)) {
500 		enic_notify_check(enic);
501 		vnic_intr_return_all_credits(&enic->intr[notify_intr]);
502 	}
503 
504 	if (ENIC_TEST_INTR(pba, err_intr)) {
505 		vnic_intr_return_all_credits(&enic->intr[err_intr]);
506 		enic_log_q_error(enic);
507 		/* schedule recovery from WQ/RQ error */
508 		schedule_work(&enic->reset);
509 		return IRQ_HANDLED;
510 	}
511 
512 	if (ENIC_TEST_INTR(pba, io_intr))
513 		napi_schedule_irqoff(&enic->napi[0]);
514 	else
515 		vnic_intr_unmask(&enic->intr[io_intr]);
516 
517 	return IRQ_HANDLED;
518 }
519 
520 static irqreturn_t enic_isr_msi(int irq, void *data)
521 {
522 	struct enic *enic = data;
523 
524 	/* With MSI, there is no sharing of interrupts, so this is
525 	 * our interrupt and there is no need to ack it.  The device
526 	 * is not providing per-vector masking, so the OS will not
527 	 * write to PCI config space to mask/unmask the interrupt.
528 	 * We're using mask_on_assertion for MSI, so the device
529 	 * automatically masks the interrupt when the interrupt is
530 	 * generated.  Later, when exiting polling, the interrupt
531 	 * will be unmasked (see enic_poll).
532 	 *
533 	 * Also, the device uses the same PCIe Traffic Class (TC)
534 	 * for Memory Write data and MSI, so there are no ordering
535 	 * issues; the MSI will always arrive at the Root Complex
536 	 * _after_ corresponding Memory Writes (i.e. descriptor
537 	 * writes).
538 	 */
539 
540 	napi_schedule_irqoff(&enic->napi[0]);
541 
542 	return IRQ_HANDLED;
543 }
544 
545 static irqreturn_t enic_isr_msix(int irq, void *data)
546 {
547 	struct napi_struct *napi = data;
548 
549 	napi_schedule_irqoff(napi);
550 
551 	return IRQ_HANDLED;
552 }
553 
554 static irqreturn_t enic_isr_msix_err(int irq, void *data)
555 {
556 	struct enic *enic = data;
557 	unsigned int intr = enic_msix_err_intr(enic);
558 
559 	vnic_intr_return_all_credits(&enic->intr[intr]);
560 
561 	if (enic_log_q_error(enic))
562 		/* schedule recovery from WQ/RQ error */
563 		schedule_work(&enic->reset);
564 
565 	return IRQ_HANDLED;
566 }
567 
568 static irqreturn_t enic_isr_msix_notify(int irq, void *data)
569 {
570 	struct enic *enic = data;
571 	unsigned int intr = enic_msix_notify_intr(enic);
572 
573 	enic_notify_check(enic);
574 	vnic_intr_return_all_credits(&enic->intr[intr]);
575 
576 	return IRQ_HANDLED;
577 }
578 
579 static int enic_queue_wq_skb_cont(struct enic *enic, struct vnic_wq *wq,
580 				  struct sk_buff *skb, unsigned int len_left,
581 				  int loopback)
582 {
583 	const skb_frag_t *frag;
584 	dma_addr_t dma_addr;
585 
586 	/* Queue additional data fragments */
587 	for (frag = skb_shinfo(skb)->frags; len_left; frag++) {
588 		len_left -= skb_frag_size(frag);
589 		dma_addr = skb_frag_dma_map(&enic->pdev->dev, frag, 0,
590 					    skb_frag_size(frag),
591 					    DMA_TO_DEVICE);
592 		if (unlikely(enic_dma_map_check(enic, dma_addr)))
593 			return -ENOMEM;
594 		enic_queue_wq_desc_cont(wq, skb, dma_addr, skb_frag_size(frag),
595 					(len_left == 0),	/* EOP? */
596 					loopback);
597 	}
598 
599 	return 0;
600 }
601 
602 static int enic_queue_wq_skb_vlan(struct enic *enic, struct vnic_wq *wq,
603 				  struct sk_buff *skb, int vlan_tag_insert,
604 				  unsigned int vlan_tag, int loopback)
605 {
606 	unsigned int head_len = skb_headlen(skb);
607 	unsigned int len_left = skb->len - head_len;
608 	int eop = (len_left == 0);
609 	dma_addr_t dma_addr;
610 	int err = 0;
611 
612 	dma_addr = pci_map_single(enic->pdev, skb->data, head_len,
613 				  PCI_DMA_TODEVICE);
614 	if (unlikely(enic_dma_map_check(enic, dma_addr)))
615 		return -ENOMEM;
616 
617 	/* Queue the main skb fragment. The fragments are no larger
618 	 * than max MTU(9000)+ETH_HDR_LEN(14) bytes, which is less
619 	 * than WQ_ENET_MAX_DESC_LEN length. So only one descriptor
620 	 * per fragment is queued.
621 	 */
622 	enic_queue_wq_desc(wq, skb, dma_addr, head_len,	vlan_tag_insert,
623 			   vlan_tag, eop, loopback);
624 
625 	if (!eop)
626 		err = enic_queue_wq_skb_cont(enic, wq, skb, len_left, loopback);
627 
628 	return err;
629 }
630 
631 static int enic_queue_wq_skb_csum_l4(struct enic *enic, struct vnic_wq *wq,
632 				     struct sk_buff *skb, int vlan_tag_insert,
633 				     unsigned int vlan_tag, int loopback)
634 {
635 	unsigned int head_len = skb_headlen(skb);
636 	unsigned int len_left = skb->len - head_len;
637 	unsigned int hdr_len = skb_checksum_start_offset(skb);
638 	unsigned int csum_offset = hdr_len + skb->csum_offset;
639 	int eop = (len_left == 0);
640 	dma_addr_t dma_addr;
641 	int err = 0;
642 
643 	dma_addr = pci_map_single(enic->pdev, skb->data, head_len,
644 				  PCI_DMA_TODEVICE);
645 	if (unlikely(enic_dma_map_check(enic, dma_addr)))
646 		return -ENOMEM;
647 
648 	/* Queue the main skb fragment. The fragments are no larger
649 	 * than max MTU(9000)+ETH_HDR_LEN(14) bytes, which is less
650 	 * than WQ_ENET_MAX_DESC_LEN length. So only one descriptor
651 	 * per fragment is queued.
652 	 */
653 	enic_queue_wq_desc_csum_l4(wq, skb, dma_addr, head_len,	csum_offset,
654 				   hdr_len, vlan_tag_insert, vlan_tag, eop,
655 				   loopback);
656 
657 	if (!eop)
658 		err = enic_queue_wq_skb_cont(enic, wq, skb, len_left, loopback);
659 
660 	return err;
661 }
662 
663 static void enic_preload_tcp_csum_encap(struct sk_buff *skb)
664 {
665 	const struct ethhdr *eth = (struct ethhdr *)skb_inner_mac_header(skb);
666 
667 	switch (eth->h_proto) {
668 	case ntohs(ETH_P_IP):
669 		inner_ip_hdr(skb)->check = 0;
670 		inner_tcp_hdr(skb)->check =
671 			~csum_tcpudp_magic(inner_ip_hdr(skb)->saddr,
672 					   inner_ip_hdr(skb)->daddr, 0,
673 					   IPPROTO_TCP, 0);
674 		break;
675 	case ntohs(ETH_P_IPV6):
676 		inner_tcp_hdr(skb)->check =
677 			~csum_ipv6_magic(&inner_ipv6_hdr(skb)->saddr,
678 					 &inner_ipv6_hdr(skb)->daddr, 0,
679 					 IPPROTO_TCP, 0);
680 		break;
681 	default:
682 		WARN_ONCE(1, "Non ipv4/ipv6 inner pkt for encap offload");
683 		break;
684 	}
685 }
686 
687 static void enic_preload_tcp_csum(struct sk_buff *skb)
688 {
689 	/* Preload TCP csum field with IP pseudo hdr calculated
690 	 * with IP length set to zero.  HW will later add in length
691 	 * to each TCP segment resulting from the TSO.
692 	 */
693 
694 	if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
695 		ip_hdr(skb)->check = 0;
696 		tcp_hdr(skb)->check = ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
697 			ip_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
698 	} else if (skb->protocol == cpu_to_be16(ETH_P_IPV6)) {
699 		tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
700 			&ipv6_hdr(skb)->daddr, 0, IPPROTO_TCP, 0);
701 	}
702 }
703 
704 static int enic_queue_wq_skb_tso(struct enic *enic, struct vnic_wq *wq,
705 				 struct sk_buff *skb, unsigned int mss,
706 				 int vlan_tag_insert, unsigned int vlan_tag,
707 				 int loopback)
708 {
709 	unsigned int frag_len_left = skb_headlen(skb);
710 	unsigned int len_left = skb->len - frag_len_left;
711 	int eop = (len_left == 0);
712 	unsigned int offset = 0;
713 	unsigned int hdr_len;
714 	dma_addr_t dma_addr;
715 	unsigned int len;
716 	skb_frag_t *frag;
717 
718 	if (skb->encapsulation) {
719 		hdr_len = skb_inner_transport_header(skb) - skb->data;
720 		hdr_len += inner_tcp_hdrlen(skb);
721 		enic_preload_tcp_csum_encap(skb);
722 	} else {
723 		hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
724 		enic_preload_tcp_csum(skb);
725 	}
726 
727 	/* Queue WQ_ENET_MAX_DESC_LEN length descriptors
728 	 * for the main skb fragment
729 	 */
730 	while (frag_len_left) {
731 		len = min(frag_len_left, (unsigned int)WQ_ENET_MAX_DESC_LEN);
732 		dma_addr = pci_map_single(enic->pdev, skb->data + offset, len,
733 					  PCI_DMA_TODEVICE);
734 		if (unlikely(enic_dma_map_check(enic, dma_addr)))
735 			return -ENOMEM;
736 		enic_queue_wq_desc_tso(wq, skb, dma_addr, len, mss, hdr_len,
737 				       vlan_tag_insert, vlan_tag,
738 				       eop && (len == frag_len_left), loopback);
739 		frag_len_left -= len;
740 		offset += len;
741 	}
742 
743 	if (eop)
744 		return 0;
745 
746 	/* Queue WQ_ENET_MAX_DESC_LEN length descriptors
747 	 * for additional data fragments
748 	 */
749 	for (frag = skb_shinfo(skb)->frags; len_left; frag++) {
750 		len_left -= skb_frag_size(frag);
751 		frag_len_left = skb_frag_size(frag);
752 		offset = 0;
753 
754 		while (frag_len_left) {
755 			len = min(frag_len_left,
756 				(unsigned int)WQ_ENET_MAX_DESC_LEN);
757 			dma_addr = skb_frag_dma_map(&enic->pdev->dev, frag,
758 						    offset, len,
759 						    DMA_TO_DEVICE);
760 			if (unlikely(enic_dma_map_check(enic, dma_addr)))
761 				return -ENOMEM;
762 			enic_queue_wq_desc_cont(wq, skb, dma_addr, len,
763 						(len_left == 0) &&
764 						 (len == frag_len_left),/*EOP*/
765 						loopback);
766 			frag_len_left -= len;
767 			offset += len;
768 		}
769 	}
770 
771 	return 0;
772 }
773 
774 static inline int enic_queue_wq_skb_encap(struct enic *enic, struct vnic_wq *wq,
775 					  struct sk_buff *skb,
776 					  int vlan_tag_insert,
777 					  unsigned int vlan_tag, int loopback)
778 {
779 	unsigned int head_len = skb_headlen(skb);
780 	unsigned int len_left = skb->len - head_len;
781 	/* Hardware will overwrite the checksum fields, calculating from
782 	 * scratch and ignoring the value placed by software.
783 	 * Offload mode = 00
784 	 * mss[2], mss[1], mss[0] bits are set
785 	 */
786 	unsigned int mss_or_csum = 7;
787 	int eop = (len_left == 0);
788 	dma_addr_t dma_addr;
789 	int err = 0;
790 
791 	dma_addr = pci_map_single(enic->pdev, skb->data, head_len,
792 				  PCI_DMA_TODEVICE);
793 	if (unlikely(enic_dma_map_check(enic, dma_addr)))
794 		return -ENOMEM;
795 
796 	enic_queue_wq_desc_ex(wq, skb, dma_addr, head_len, mss_or_csum, 0,
797 			      vlan_tag_insert, vlan_tag,
798 			      WQ_ENET_OFFLOAD_MODE_CSUM, eop, 1 /* SOP */, eop,
799 			      loopback);
800 	if (!eop)
801 		err = enic_queue_wq_skb_cont(enic, wq, skb, len_left, loopback);
802 
803 	return err;
804 }
805 
806 static inline void enic_queue_wq_skb(struct enic *enic,
807 	struct vnic_wq *wq, struct sk_buff *skb)
808 {
809 	unsigned int mss = skb_shinfo(skb)->gso_size;
810 	unsigned int vlan_tag = 0;
811 	int vlan_tag_insert = 0;
812 	int loopback = 0;
813 	int err;
814 
815 	if (skb_vlan_tag_present(skb)) {
816 		/* VLAN tag from trunking driver */
817 		vlan_tag_insert = 1;
818 		vlan_tag = skb_vlan_tag_get(skb);
819 	} else if (enic->loop_enable) {
820 		vlan_tag = enic->loop_tag;
821 		loopback = 1;
822 	}
823 
824 	if (mss)
825 		err = enic_queue_wq_skb_tso(enic, wq, skb, mss,
826 					    vlan_tag_insert, vlan_tag,
827 					    loopback);
828 	else if (skb->encapsulation)
829 		err = enic_queue_wq_skb_encap(enic, wq, skb, vlan_tag_insert,
830 					      vlan_tag, loopback);
831 	else if	(skb->ip_summed == CHECKSUM_PARTIAL)
832 		err = enic_queue_wq_skb_csum_l4(enic, wq, skb, vlan_tag_insert,
833 						vlan_tag, loopback);
834 	else
835 		err = enic_queue_wq_skb_vlan(enic, wq, skb, vlan_tag_insert,
836 					     vlan_tag, loopback);
837 	if (unlikely(err)) {
838 		struct vnic_wq_buf *buf;
839 
840 		buf = wq->to_use->prev;
841 		/* while not EOP of previous pkt && queue not empty.
842 		 * For all non EOP bufs, os_buf is NULL.
843 		 */
844 		while (!buf->os_buf && (buf->next != wq->to_clean)) {
845 			enic_free_wq_buf(wq, buf);
846 			wq->ring.desc_avail++;
847 			buf = buf->prev;
848 		}
849 		wq->to_use = buf->next;
850 		dev_kfree_skb(skb);
851 	}
852 }
853 
854 /* netif_tx_lock held, process context with BHs disabled, or BH */
855 static netdev_tx_t enic_hard_start_xmit(struct sk_buff *skb,
856 	struct net_device *netdev)
857 {
858 	struct enic *enic = netdev_priv(netdev);
859 	struct vnic_wq *wq;
860 	unsigned int txq_map;
861 	struct netdev_queue *txq;
862 
863 	if (skb->len <= 0) {
864 		dev_kfree_skb_any(skb);
865 		return NETDEV_TX_OK;
866 	}
867 
868 	txq_map = skb_get_queue_mapping(skb) % enic->wq_count;
869 	wq = &enic->wq[txq_map];
870 	txq = netdev_get_tx_queue(netdev, txq_map);
871 
872 	/* Non-TSO sends must fit within ENIC_NON_TSO_MAX_DESC descs,
873 	 * which is very likely.  In the off chance it's going to take
874 	 * more than * ENIC_NON_TSO_MAX_DESC, linearize the skb.
875 	 */
876 
877 	if (skb_shinfo(skb)->gso_size == 0 &&
878 	    skb_shinfo(skb)->nr_frags + 1 > ENIC_NON_TSO_MAX_DESC &&
879 	    skb_linearize(skb)) {
880 		dev_kfree_skb_any(skb);
881 		return NETDEV_TX_OK;
882 	}
883 
884 	spin_lock(&enic->wq_lock[txq_map]);
885 
886 	if (vnic_wq_desc_avail(wq) <
887 	    skb_shinfo(skb)->nr_frags + ENIC_DESC_MAX_SPLITS) {
888 		netif_tx_stop_queue(txq);
889 		/* This is a hard error, log it */
890 		netdev_err(netdev, "BUG! Tx ring full when queue awake!\n");
891 		spin_unlock(&enic->wq_lock[txq_map]);
892 		return NETDEV_TX_BUSY;
893 	}
894 
895 	enic_queue_wq_skb(enic, wq, skb);
896 
897 	if (vnic_wq_desc_avail(wq) < MAX_SKB_FRAGS + ENIC_DESC_MAX_SPLITS)
898 		netif_tx_stop_queue(txq);
899 	skb_tx_timestamp(skb);
900 	if (!netdev_xmit_more() || netif_xmit_stopped(txq))
901 		vnic_wq_doorbell(wq);
902 
903 	spin_unlock(&enic->wq_lock[txq_map]);
904 
905 	return NETDEV_TX_OK;
906 }
907 
908 /* dev_base_lock rwlock held, nominally process context */
909 static void enic_get_stats(struct net_device *netdev,
910 			   struct rtnl_link_stats64 *net_stats)
911 {
912 	struct enic *enic = netdev_priv(netdev);
913 	struct vnic_stats *stats;
914 	int err;
915 
916 	err = enic_dev_stats_dump(enic, &stats);
917 	/* return only when pci_zalloc_consistent fails in vnic_dev_stats_dump
918 	 * For other failures, like devcmd failure, we return previously
919 	 * recorded stats.
920 	 */
921 	if (err == -ENOMEM)
922 		return;
923 
924 	net_stats->tx_packets = stats->tx.tx_frames_ok;
925 	net_stats->tx_bytes = stats->tx.tx_bytes_ok;
926 	net_stats->tx_errors = stats->tx.tx_errors;
927 	net_stats->tx_dropped = stats->tx.tx_drops;
928 
929 	net_stats->rx_packets = stats->rx.rx_frames_ok;
930 	net_stats->rx_bytes = stats->rx.rx_bytes_ok;
931 	net_stats->rx_errors = stats->rx.rx_errors;
932 	net_stats->multicast = stats->rx.rx_multicast_frames_ok;
933 	net_stats->rx_over_errors = enic->rq_truncated_pkts;
934 	net_stats->rx_crc_errors = enic->rq_bad_fcs;
935 	net_stats->rx_dropped = stats->rx.rx_no_bufs + stats->rx.rx_drop;
936 }
937 
938 static int enic_mc_sync(struct net_device *netdev, const u8 *mc_addr)
939 {
940 	struct enic *enic = netdev_priv(netdev);
941 
942 	if (enic->mc_count == ENIC_MULTICAST_PERFECT_FILTERS) {
943 		unsigned int mc_count = netdev_mc_count(netdev);
944 
945 		netdev_warn(netdev, "Registering only %d out of %d multicast addresses\n",
946 			    ENIC_MULTICAST_PERFECT_FILTERS, mc_count);
947 
948 		return -ENOSPC;
949 	}
950 
951 	enic_dev_add_addr(enic, mc_addr);
952 	enic->mc_count++;
953 
954 	return 0;
955 }
956 
957 static int enic_mc_unsync(struct net_device *netdev, const u8 *mc_addr)
958 {
959 	struct enic *enic = netdev_priv(netdev);
960 
961 	enic_dev_del_addr(enic, mc_addr);
962 	enic->mc_count--;
963 
964 	return 0;
965 }
966 
967 static int enic_uc_sync(struct net_device *netdev, const u8 *uc_addr)
968 {
969 	struct enic *enic = netdev_priv(netdev);
970 
971 	if (enic->uc_count == ENIC_UNICAST_PERFECT_FILTERS) {
972 		unsigned int uc_count = netdev_uc_count(netdev);
973 
974 		netdev_warn(netdev, "Registering only %d out of %d unicast addresses\n",
975 			    ENIC_UNICAST_PERFECT_FILTERS, uc_count);
976 
977 		return -ENOSPC;
978 	}
979 
980 	enic_dev_add_addr(enic, uc_addr);
981 	enic->uc_count++;
982 
983 	return 0;
984 }
985 
986 static int enic_uc_unsync(struct net_device *netdev, const u8 *uc_addr)
987 {
988 	struct enic *enic = netdev_priv(netdev);
989 
990 	enic_dev_del_addr(enic, uc_addr);
991 	enic->uc_count--;
992 
993 	return 0;
994 }
995 
996 void enic_reset_addr_lists(struct enic *enic)
997 {
998 	struct net_device *netdev = enic->netdev;
999 
1000 	__dev_uc_unsync(netdev, NULL);
1001 	__dev_mc_unsync(netdev, NULL);
1002 
1003 	enic->mc_count = 0;
1004 	enic->uc_count = 0;
1005 	enic->flags = 0;
1006 }
1007 
1008 static int enic_set_mac_addr(struct net_device *netdev, char *addr)
1009 {
1010 	struct enic *enic = netdev_priv(netdev);
1011 
1012 	if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic)) {
1013 		if (!is_valid_ether_addr(addr) && !is_zero_ether_addr(addr))
1014 			return -EADDRNOTAVAIL;
1015 	} else {
1016 		if (!is_valid_ether_addr(addr))
1017 			return -EADDRNOTAVAIL;
1018 	}
1019 
1020 	memcpy(netdev->dev_addr, addr, netdev->addr_len);
1021 
1022 	return 0;
1023 }
1024 
1025 static int enic_set_mac_address_dynamic(struct net_device *netdev, void *p)
1026 {
1027 	struct enic *enic = netdev_priv(netdev);
1028 	struct sockaddr *saddr = p;
1029 	char *addr = saddr->sa_data;
1030 	int err;
1031 
1032 	if (netif_running(enic->netdev)) {
1033 		err = enic_dev_del_station_addr(enic);
1034 		if (err)
1035 			return err;
1036 	}
1037 
1038 	err = enic_set_mac_addr(netdev, addr);
1039 	if (err)
1040 		return err;
1041 
1042 	if (netif_running(enic->netdev)) {
1043 		err = enic_dev_add_station_addr(enic);
1044 		if (err)
1045 			return err;
1046 	}
1047 
1048 	return err;
1049 }
1050 
1051 static int enic_set_mac_address(struct net_device *netdev, void *p)
1052 {
1053 	struct sockaddr *saddr = p;
1054 	char *addr = saddr->sa_data;
1055 	struct enic *enic = netdev_priv(netdev);
1056 	int err;
1057 
1058 	err = enic_dev_del_station_addr(enic);
1059 	if (err)
1060 		return err;
1061 
1062 	err = enic_set_mac_addr(netdev, addr);
1063 	if (err)
1064 		return err;
1065 
1066 	return enic_dev_add_station_addr(enic);
1067 }
1068 
1069 /* netif_tx_lock held, BHs disabled */
1070 static void enic_set_rx_mode(struct net_device *netdev)
1071 {
1072 	struct enic *enic = netdev_priv(netdev);
1073 	int directed = 1;
1074 	int multicast = (netdev->flags & IFF_MULTICAST) ? 1 : 0;
1075 	int broadcast = (netdev->flags & IFF_BROADCAST) ? 1 : 0;
1076 	int promisc = (netdev->flags & IFF_PROMISC) ||
1077 		netdev_uc_count(netdev) > ENIC_UNICAST_PERFECT_FILTERS;
1078 	int allmulti = (netdev->flags & IFF_ALLMULTI) ||
1079 		netdev_mc_count(netdev) > ENIC_MULTICAST_PERFECT_FILTERS;
1080 	unsigned int flags = netdev->flags |
1081 		(allmulti ? IFF_ALLMULTI : 0) |
1082 		(promisc ? IFF_PROMISC : 0);
1083 
1084 	if (enic->flags != flags) {
1085 		enic->flags = flags;
1086 		enic_dev_packet_filter(enic, directed,
1087 			multicast, broadcast, promisc, allmulti);
1088 	}
1089 
1090 	if (!promisc) {
1091 		__dev_uc_sync(netdev, enic_uc_sync, enic_uc_unsync);
1092 		if (!allmulti)
1093 			__dev_mc_sync(netdev, enic_mc_sync, enic_mc_unsync);
1094 	}
1095 }
1096 
1097 /* netif_tx_lock held, BHs disabled */
1098 static void enic_tx_timeout(struct net_device *netdev)
1099 {
1100 	struct enic *enic = netdev_priv(netdev);
1101 	schedule_work(&enic->tx_hang_reset);
1102 }
1103 
1104 static int enic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
1105 {
1106 	struct enic *enic = netdev_priv(netdev);
1107 	struct enic_port_profile *pp;
1108 	int err;
1109 
1110 	ENIC_PP_BY_INDEX(enic, vf, pp, &err);
1111 	if (err)
1112 		return err;
1113 
1114 	if (is_valid_ether_addr(mac) || is_zero_ether_addr(mac)) {
1115 		if (vf == PORT_SELF_VF) {
1116 			memcpy(pp->vf_mac, mac, ETH_ALEN);
1117 			return 0;
1118 		} else {
1119 			/*
1120 			 * For sriov vf's set the mac in hw
1121 			 */
1122 			ENIC_DEVCMD_PROXY_BY_INDEX(vf, err, enic,
1123 				vnic_dev_set_mac_addr, mac);
1124 			return enic_dev_status_to_errno(err);
1125 		}
1126 	} else
1127 		return -EINVAL;
1128 }
1129 
1130 static int enic_set_vf_port(struct net_device *netdev, int vf,
1131 	struct nlattr *port[])
1132 {
1133 	struct enic *enic = netdev_priv(netdev);
1134 	struct enic_port_profile prev_pp;
1135 	struct enic_port_profile *pp;
1136 	int err = 0, restore_pp = 1;
1137 
1138 	ENIC_PP_BY_INDEX(enic, vf, pp, &err);
1139 	if (err)
1140 		return err;
1141 
1142 	if (!port[IFLA_PORT_REQUEST])
1143 		return -EOPNOTSUPP;
1144 
1145 	memcpy(&prev_pp, pp, sizeof(*enic->pp));
1146 	memset(pp, 0, sizeof(*enic->pp));
1147 
1148 	pp->set |= ENIC_SET_REQUEST;
1149 	pp->request = nla_get_u8(port[IFLA_PORT_REQUEST]);
1150 
1151 	if (port[IFLA_PORT_PROFILE]) {
1152 		pp->set |= ENIC_SET_NAME;
1153 		memcpy(pp->name, nla_data(port[IFLA_PORT_PROFILE]),
1154 			PORT_PROFILE_MAX);
1155 	}
1156 
1157 	if (port[IFLA_PORT_INSTANCE_UUID]) {
1158 		pp->set |= ENIC_SET_INSTANCE;
1159 		memcpy(pp->instance_uuid,
1160 			nla_data(port[IFLA_PORT_INSTANCE_UUID]), PORT_UUID_MAX);
1161 	}
1162 
1163 	if (port[IFLA_PORT_HOST_UUID]) {
1164 		pp->set |= ENIC_SET_HOST;
1165 		memcpy(pp->host_uuid,
1166 			nla_data(port[IFLA_PORT_HOST_UUID]), PORT_UUID_MAX);
1167 	}
1168 
1169 	if (vf == PORT_SELF_VF) {
1170 		/* Special case handling: mac came from IFLA_VF_MAC */
1171 		if (!is_zero_ether_addr(prev_pp.vf_mac))
1172 			memcpy(pp->mac_addr, prev_pp.vf_mac, ETH_ALEN);
1173 
1174 		if (is_zero_ether_addr(netdev->dev_addr))
1175 			eth_hw_addr_random(netdev);
1176 	} else {
1177 		/* SR-IOV VF: get mac from adapter */
1178 		ENIC_DEVCMD_PROXY_BY_INDEX(vf, err, enic,
1179 			vnic_dev_get_mac_addr, pp->mac_addr);
1180 		if (err) {
1181 			netdev_err(netdev, "Error getting mac for vf %d\n", vf);
1182 			memcpy(pp, &prev_pp, sizeof(*pp));
1183 			return enic_dev_status_to_errno(err);
1184 		}
1185 	}
1186 
1187 	err = enic_process_set_pp_request(enic, vf, &prev_pp, &restore_pp);
1188 	if (err) {
1189 		if (restore_pp) {
1190 			/* Things are still the way they were: Implicit
1191 			 * DISASSOCIATE failed
1192 			 */
1193 			memcpy(pp, &prev_pp, sizeof(*pp));
1194 		} else {
1195 			memset(pp, 0, sizeof(*pp));
1196 			if (vf == PORT_SELF_VF)
1197 				eth_zero_addr(netdev->dev_addr);
1198 		}
1199 	} else {
1200 		/* Set flag to indicate that the port assoc/disassoc
1201 		 * request has been sent out to fw
1202 		 */
1203 		pp->set |= ENIC_PORT_REQUEST_APPLIED;
1204 
1205 		/* If DISASSOCIATE, clean up all assigned/saved macaddresses */
1206 		if (pp->request == PORT_REQUEST_DISASSOCIATE) {
1207 			eth_zero_addr(pp->mac_addr);
1208 			if (vf == PORT_SELF_VF)
1209 				eth_zero_addr(netdev->dev_addr);
1210 		}
1211 	}
1212 
1213 	if (vf == PORT_SELF_VF)
1214 		eth_zero_addr(pp->vf_mac);
1215 
1216 	return err;
1217 }
1218 
1219 static int enic_get_vf_port(struct net_device *netdev, int vf,
1220 	struct sk_buff *skb)
1221 {
1222 	struct enic *enic = netdev_priv(netdev);
1223 	u16 response = PORT_PROFILE_RESPONSE_SUCCESS;
1224 	struct enic_port_profile *pp;
1225 	int err;
1226 
1227 	ENIC_PP_BY_INDEX(enic, vf, pp, &err);
1228 	if (err)
1229 		return err;
1230 
1231 	if (!(pp->set & ENIC_PORT_REQUEST_APPLIED))
1232 		return -ENODATA;
1233 
1234 	err = enic_process_get_pp_request(enic, vf, pp->request, &response);
1235 	if (err)
1236 		return err;
1237 
1238 	if (nla_put_u16(skb, IFLA_PORT_REQUEST, pp->request) ||
1239 	    nla_put_u16(skb, IFLA_PORT_RESPONSE, response) ||
1240 	    ((pp->set & ENIC_SET_NAME) &&
1241 	     nla_put(skb, IFLA_PORT_PROFILE, PORT_PROFILE_MAX, pp->name)) ||
1242 	    ((pp->set & ENIC_SET_INSTANCE) &&
1243 	     nla_put(skb, IFLA_PORT_INSTANCE_UUID, PORT_UUID_MAX,
1244 		     pp->instance_uuid)) ||
1245 	    ((pp->set & ENIC_SET_HOST) &&
1246 	     nla_put(skb, IFLA_PORT_HOST_UUID, PORT_UUID_MAX, pp->host_uuid)))
1247 		goto nla_put_failure;
1248 	return 0;
1249 
1250 nla_put_failure:
1251 	return -EMSGSIZE;
1252 }
1253 
1254 static void enic_free_rq_buf(struct vnic_rq *rq, struct vnic_rq_buf *buf)
1255 {
1256 	struct enic *enic = vnic_dev_priv(rq->vdev);
1257 
1258 	if (!buf->os_buf)
1259 		return;
1260 
1261 	pci_unmap_single(enic->pdev, buf->dma_addr,
1262 		buf->len, PCI_DMA_FROMDEVICE);
1263 	dev_kfree_skb_any(buf->os_buf);
1264 	buf->os_buf = NULL;
1265 }
1266 
1267 static int enic_rq_alloc_buf(struct vnic_rq *rq)
1268 {
1269 	struct enic *enic = vnic_dev_priv(rq->vdev);
1270 	struct net_device *netdev = enic->netdev;
1271 	struct sk_buff *skb;
1272 	unsigned int len = netdev->mtu + VLAN_ETH_HLEN;
1273 	unsigned int os_buf_index = 0;
1274 	dma_addr_t dma_addr;
1275 	struct vnic_rq_buf *buf = rq->to_use;
1276 
1277 	if (buf->os_buf) {
1278 		enic_queue_rq_desc(rq, buf->os_buf, os_buf_index, buf->dma_addr,
1279 				   buf->len);
1280 
1281 		return 0;
1282 	}
1283 	skb = netdev_alloc_skb_ip_align(netdev, len);
1284 	if (!skb)
1285 		return -ENOMEM;
1286 
1287 	dma_addr = pci_map_single(enic->pdev, skb->data, len,
1288 				  PCI_DMA_FROMDEVICE);
1289 	if (unlikely(enic_dma_map_check(enic, dma_addr))) {
1290 		dev_kfree_skb(skb);
1291 		return -ENOMEM;
1292 	}
1293 
1294 	enic_queue_rq_desc(rq, skb, os_buf_index,
1295 		dma_addr, len);
1296 
1297 	return 0;
1298 }
1299 
1300 static void enic_intr_update_pkt_size(struct vnic_rx_bytes_counter *pkt_size,
1301 				      u32 pkt_len)
1302 {
1303 	if (ENIC_LARGE_PKT_THRESHOLD <= pkt_len)
1304 		pkt_size->large_pkt_bytes_cnt += pkt_len;
1305 	else
1306 		pkt_size->small_pkt_bytes_cnt += pkt_len;
1307 }
1308 
1309 static bool enic_rxcopybreak(struct net_device *netdev, struct sk_buff **skb,
1310 			     struct vnic_rq_buf *buf, u16 len)
1311 {
1312 	struct enic *enic = netdev_priv(netdev);
1313 	struct sk_buff *new_skb;
1314 
1315 	if (len > enic->rx_copybreak)
1316 		return false;
1317 	new_skb = netdev_alloc_skb_ip_align(netdev, len);
1318 	if (!new_skb)
1319 		return false;
1320 	pci_dma_sync_single_for_cpu(enic->pdev, buf->dma_addr, len,
1321 				    DMA_FROM_DEVICE);
1322 	memcpy(new_skb->data, (*skb)->data, len);
1323 	*skb = new_skb;
1324 
1325 	return true;
1326 }
1327 
1328 static void enic_rq_indicate_buf(struct vnic_rq *rq,
1329 	struct cq_desc *cq_desc, struct vnic_rq_buf *buf,
1330 	int skipped, void *opaque)
1331 {
1332 	struct enic *enic = vnic_dev_priv(rq->vdev);
1333 	struct net_device *netdev = enic->netdev;
1334 	struct sk_buff *skb;
1335 	struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)];
1336 
1337 	u8 type, color, eop, sop, ingress_port, vlan_stripped;
1338 	u8 fcoe, fcoe_sof, fcoe_fc_crc_ok, fcoe_enc_error, fcoe_eof;
1339 	u8 tcp_udp_csum_ok, udp, tcp, ipv4_csum_ok;
1340 	u8 ipv6, ipv4, ipv4_fragment, fcs_ok, rss_type, csum_not_calc;
1341 	u8 packet_error;
1342 	u16 q_number, completed_index, bytes_written, vlan_tci, checksum;
1343 	u32 rss_hash;
1344 	bool outer_csum_ok = true, encap = false;
1345 
1346 	if (skipped)
1347 		return;
1348 
1349 	skb = buf->os_buf;
1350 
1351 	cq_enet_rq_desc_dec((struct cq_enet_rq_desc *)cq_desc,
1352 		&type, &color, &q_number, &completed_index,
1353 		&ingress_port, &fcoe, &eop, &sop, &rss_type,
1354 		&csum_not_calc, &rss_hash, &bytes_written,
1355 		&packet_error, &vlan_stripped, &vlan_tci, &checksum,
1356 		&fcoe_sof, &fcoe_fc_crc_ok, &fcoe_enc_error,
1357 		&fcoe_eof, &tcp_udp_csum_ok, &udp, &tcp,
1358 		&ipv4_csum_ok, &ipv6, &ipv4, &ipv4_fragment,
1359 		&fcs_ok);
1360 
1361 	if (packet_error) {
1362 
1363 		if (!fcs_ok) {
1364 			if (bytes_written > 0)
1365 				enic->rq_bad_fcs++;
1366 			else if (bytes_written == 0)
1367 				enic->rq_truncated_pkts++;
1368 		}
1369 
1370 		pci_unmap_single(enic->pdev, buf->dma_addr, buf->len,
1371 				 PCI_DMA_FROMDEVICE);
1372 		dev_kfree_skb_any(skb);
1373 		buf->os_buf = NULL;
1374 
1375 		return;
1376 	}
1377 
1378 	if (eop && bytes_written > 0) {
1379 
1380 		/* Good receive
1381 		 */
1382 
1383 		if (!enic_rxcopybreak(netdev, &skb, buf, bytes_written)) {
1384 			buf->os_buf = NULL;
1385 			pci_unmap_single(enic->pdev, buf->dma_addr, buf->len,
1386 					 PCI_DMA_FROMDEVICE);
1387 		}
1388 		prefetch(skb->data - NET_IP_ALIGN);
1389 
1390 		skb_put(skb, bytes_written);
1391 		skb->protocol = eth_type_trans(skb, netdev);
1392 		skb_record_rx_queue(skb, q_number);
1393 		if ((netdev->features & NETIF_F_RXHASH) && rss_hash &&
1394 		    (type == 3)) {
1395 			switch (rss_type) {
1396 			case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv4:
1397 			case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6:
1398 			case CQ_ENET_RQ_DESC_RSS_TYPE_TCP_IPv6_EX:
1399 				skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L4);
1400 				break;
1401 			case CQ_ENET_RQ_DESC_RSS_TYPE_IPv4:
1402 			case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6:
1403 			case CQ_ENET_RQ_DESC_RSS_TYPE_IPv6_EX:
1404 				skb_set_hash(skb, rss_hash, PKT_HASH_TYPE_L3);
1405 				break;
1406 			}
1407 		}
1408 		if (enic->vxlan.vxlan_udp_port_number) {
1409 			switch (enic->vxlan.patch_level) {
1410 			case 0:
1411 				if (fcoe) {
1412 					encap = true;
1413 					outer_csum_ok = fcoe_fc_crc_ok;
1414 				}
1415 				break;
1416 			case 2:
1417 				if ((type == 7) &&
1418 				    (rss_hash & BIT(0))) {
1419 					encap = true;
1420 					outer_csum_ok = (rss_hash & BIT(1)) &&
1421 							(rss_hash & BIT(2));
1422 				}
1423 				break;
1424 			}
1425 		}
1426 
1427 		/* Hardware does not provide whole packet checksum. It only
1428 		 * provides pseudo checksum. Since hw validates the packet
1429 		 * checksum but not provide us the checksum value. use
1430 		 * CHECSUM_UNNECESSARY.
1431 		 *
1432 		 * In case of encap pkt tcp_udp_csum_ok/tcp_udp_csum_ok is
1433 		 * inner csum_ok. outer_csum_ok is set by hw when outer udp
1434 		 * csum is correct or is zero.
1435 		 */
1436 		if ((netdev->features & NETIF_F_RXCSUM) && !csum_not_calc &&
1437 		    tcp_udp_csum_ok && outer_csum_ok &&
1438 		    (ipv4_csum_ok || ipv6)) {
1439 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1440 			skb->csum_level = encap;
1441 		}
1442 
1443 		if (vlan_stripped)
1444 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci);
1445 
1446 		skb_mark_napi_id(skb, &enic->napi[rq->index]);
1447 		if (!(netdev->features & NETIF_F_GRO))
1448 			netif_receive_skb(skb);
1449 		else
1450 			napi_gro_receive(&enic->napi[q_number], skb);
1451 		if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
1452 			enic_intr_update_pkt_size(&cq->pkt_size_counter,
1453 						  bytes_written);
1454 	} else {
1455 
1456 		/* Buffer overflow
1457 		 */
1458 
1459 		pci_unmap_single(enic->pdev, buf->dma_addr, buf->len,
1460 				 PCI_DMA_FROMDEVICE);
1461 		dev_kfree_skb_any(skb);
1462 		buf->os_buf = NULL;
1463 	}
1464 }
1465 
1466 static int enic_rq_service(struct vnic_dev *vdev, struct cq_desc *cq_desc,
1467 	u8 type, u16 q_number, u16 completed_index, void *opaque)
1468 {
1469 	struct enic *enic = vnic_dev_priv(vdev);
1470 
1471 	vnic_rq_service(&enic->rq[q_number], cq_desc,
1472 		completed_index, VNIC_RQ_RETURN_DESC,
1473 		enic_rq_indicate_buf, opaque);
1474 
1475 	return 0;
1476 }
1477 
1478 static void enic_set_int_moderation(struct enic *enic, struct vnic_rq *rq)
1479 {
1480 	unsigned int intr = enic_msix_rq_intr(enic, rq->index);
1481 	struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)];
1482 	u32 timer = cq->tobe_rx_coal_timeval;
1483 
1484 	if (cq->tobe_rx_coal_timeval != cq->cur_rx_coal_timeval) {
1485 		vnic_intr_coalescing_timer_set(&enic->intr[intr], timer);
1486 		cq->cur_rx_coal_timeval = cq->tobe_rx_coal_timeval;
1487 	}
1488 }
1489 
1490 static void enic_calc_int_moderation(struct enic *enic, struct vnic_rq *rq)
1491 {
1492 	struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting;
1493 	struct vnic_cq *cq = &enic->cq[enic_cq_rq(enic, rq->index)];
1494 	struct vnic_rx_bytes_counter *pkt_size_counter = &cq->pkt_size_counter;
1495 	int index;
1496 	u32 timer;
1497 	u32 range_start;
1498 	u32 traffic;
1499 	u64 delta;
1500 	ktime_t now = ktime_get();
1501 
1502 	delta = ktime_us_delta(now, cq->prev_ts);
1503 	if (delta < ENIC_AIC_TS_BREAK)
1504 		return;
1505 	cq->prev_ts = now;
1506 
1507 	traffic = pkt_size_counter->large_pkt_bytes_cnt +
1508 		  pkt_size_counter->small_pkt_bytes_cnt;
1509 	/* The table takes Mbps
1510 	 * traffic *= 8    => bits
1511 	 * traffic *= (10^6 / delta)    => bps
1512 	 * traffic /= 10^6     => Mbps
1513 	 *
1514 	 * Combining, traffic *= (8 / delta)
1515 	 */
1516 
1517 	traffic <<= 3;
1518 	traffic = delta > UINT_MAX ? 0 : traffic / (u32)delta;
1519 
1520 	for (index = 0; index < ENIC_MAX_COALESCE_TIMERS; index++)
1521 		if (traffic < mod_table[index].rx_rate)
1522 			break;
1523 	range_start = (pkt_size_counter->small_pkt_bytes_cnt >
1524 		       pkt_size_counter->large_pkt_bytes_cnt << 1) ?
1525 		      rx_coal->small_pkt_range_start :
1526 		      rx_coal->large_pkt_range_start;
1527 	timer = range_start + ((rx_coal->range_end - range_start) *
1528 			       mod_table[index].range_percent / 100);
1529 	/* Damping */
1530 	cq->tobe_rx_coal_timeval = (timer + cq->tobe_rx_coal_timeval) >> 1;
1531 
1532 	pkt_size_counter->large_pkt_bytes_cnt = 0;
1533 	pkt_size_counter->small_pkt_bytes_cnt = 0;
1534 }
1535 
1536 static int enic_poll(struct napi_struct *napi, int budget)
1537 {
1538 	struct net_device *netdev = napi->dev;
1539 	struct enic *enic = netdev_priv(netdev);
1540 	unsigned int cq_rq = enic_cq_rq(enic, 0);
1541 	unsigned int cq_wq = enic_cq_wq(enic, 0);
1542 	unsigned int intr = enic_legacy_io_intr();
1543 	unsigned int rq_work_to_do = budget;
1544 	unsigned int wq_work_to_do = ENIC_WQ_NAPI_BUDGET;
1545 	unsigned int  work_done, rq_work_done = 0, wq_work_done;
1546 	int err;
1547 
1548 	wq_work_done = vnic_cq_service(&enic->cq[cq_wq], wq_work_to_do,
1549 				       enic_wq_service, NULL);
1550 
1551 	if (budget > 0)
1552 		rq_work_done = vnic_cq_service(&enic->cq[cq_rq],
1553 			rq_work_to_do, enic_rq_service, NULL);
1554 
1555 	/* Accumulate intr event credits for this polling
1556 	 * cycle.  An intr event is the completion of a
1557 	 * a WQ or RQ packet.
1558 	 */
1559 
1560 	work_done = rq_work_done + wq_work_done;
1561 
1562 	if (work_done > 0)
1563 		vnic_intr_return_credits(&enic->intr[intr],
1564 			work_done,
1565 			0 /* don't unmask intr */,
1566 			0 /* don't reset intr timer */);
1567 
1568 	err = vnic_rq_fill(&enic->rq[0], enic_rq_alloc_buf);
1569 
1570 	/* Buffer allocation failed. Stay in polling
1571 	 * mode so we can try to fill the ring again.
1572 	 */
1573 
1574 	if (err)
1575 		rq_work_done = rq_work_to_do;
1576 	if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
1577 		/* Call the function which refreshes the intr coalescing timer
1578 		 * value based on the traffic.
1579 		 */
1580 		enic_calc_int_moderation(enic, &enic->rq[0]);
1581 
1582 	if ((rq_work_done < budget) && napi_complete_done(napi, rq_work_done)) {
1583 
1584 		/* Some work done, but not enough to stay in polling,
1585 		 * exit polling
1586 		 */
1587 
1588 		if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
1589 			enic_set_int_moderation(enic, &enic->rq[0]);
1590 		vnic_intr_unmask(&enic->intr[intr]);
1591 	}
1592 
1593 	return rq_work_done;
1594 }
1595 
1596 #ifdef CONFIG_RFS_ACCEL
1597 static void enic_free_rx_cpu_rmap(struct enic *enic)
1598 {
1599 	free_irq_cpu_rmap(enic->netdev->rx_cpu_rmap);
1600 	enic->netdev->rx_cpu_rmap = NULL;
1601 }
1602 
1603 static void enic_set_rx_cpu_rmap(struct enic *enic)
1604 {
1605 	int i, res;
1606 
1607 	if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX) {
1608 		enic->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(enic->rq_count);
1609 		if (unlikely(!enic->netdev->rx_cpu_rmap))
1610 			return;
1611 		for (i = 0; i < enic->rq_count; i++) {
1612 			res = irq_cpu_rmap_add(enic->netdev->rx_cpu_rmap,
1613 					       enic->msix_entry[i].vector);
1614 			if (unlikely(res)) {
1615 				enic_free_rx_cpu_rmap(enic);
1616 				return;
1617 			}
1618 		}
1619 	}
1620 }
1621 
1622 #else
1623 
1624 static void enic_free_rx_cpu_rmap(struct enic *enic)
1625 {
1626 }
1627 
1628 static void enic_set_rx_cpu_rmap(struct enic *enic)
1629 {
1630 }
1631 
1632 #endif /* CONFIG_RFS_ACCEL */
1633 
1634 static int enic_poll_msix_wq(struct napi_struct *napi, int budget)
1635 {
1636 	struct net_device *netdev = napi->dev;
1637 	struct enic *enic = netdev_priv(netdev);
1638 	unsigned int wq_index = (napi - &enic->napi[0]) - enic->rq_count;
1639 	struct vnic_wq *wq = &enic->wq[wq_index];
1640 	unsigned int cq;
1641 	unsigned int intr;
1642 	unsigned int wq_work_to_do = ENIC_WQ_NAPI_BUDGET;
1643 	unsigned int wq_work_done;
1644 	unsigned int wq_irq;
1645 
1646 	wq_irq = wq->index;
1647 	cq = enic_cq_wq(enic, wq_irq);
1648 	intr = enic_msix_wq_intr(enic, wq_irq);
1649 	wq_work_done = vnic_cq_service(&enic->cq[cq], wq_work_to_do,
1650 				       enic_wq_service, NULL);
1651 
1652 	vnic_intr_return_credits(&enic->intr[intr], wq_work_done,
1653 				 0 /* don't unmask intr */,
1654 				 1 /* reset intr timer */);
1655 	if (!wq_work_done) {
1656 		napi_complete(napi);
1657 		vnic_intr_unmask(&enic->intr[intr]);
1658 		return 0;
1659 	}
1660 
1661 	return budget;
1662 }
1663 
1664 static int enic_poll_msix_rq(struct napi_struct *napi, int budget)
1665 {
1666 	struct net_device *netdev = napi->dev;
1667 	struct enic *enic = netdev_priv(netdev);
1668 	unsigned int rq = (napi - &enic->napi[0]);
1669 	unsigned int cq = enic_cq_rq(enic, rq);
1670 	unsigned int intr = enic_msix_rq_intr(enic, rq);
1671 	unsigned int work_to_do = budget;
1672 	unsigned int work_done = 0;
1673 	int err;
1674 
1675 	/* Service RQ
1676 	 */
1677 
1678 	if (budget > 0)
1679 		work_done = vnic_cq_service(&enic->cq[cq],
1680 			work_to_do, enic_rq_service, NULL);
1681 
1682 	/* Return intr event credits for this polling
1683 	 * cycle.  An intr event is the completion of a
1684 	 * RQ packet.
1685 	 */
1686 
1687 	if (work_done > 0)
1688 		vnic_intr_return_credits(&enic->intr[intr],
1689 			work_done,
1690 			0 /* don't unmask intr */,
1691 			0 /* don't reset intr timer */);
1692 
1693 	err = vnic_rq_fill(&enic->rq[rq], enic_rq_alloc_buf);
1694 
1695 	/* Buffer allocation failed. Stay in polling mode
1696 	 * so we can try to fill the ring again.
1697 	 */
1698 
1699 	if (err)
1700 		work_done = work_to_do;
1701 	if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
1702 		/* Call the function which refreshes the intr coalescing timer
1703 		 * value based on the traffic.
1704 		 */
1705 		enic_calc_int_moderation(enic, &enic->rq[rq]);
1706 
1707 	if ((work_done < budget) && napi_complete_done(napi, work_done)) {
1708 
1709 		/* Some work done, but not enough to stay in polling,
1710 		 * exit polling
1711 		 */
1712 
1713 		if (enic->rx_coalesce_setting.use_adaptive_rx_coalesce)
1714 			enic_set_int_moderation(enic, &enic->rq[rq]);
1715 		vnic_intr_unmask(&enic->intr[intr]);
1716 	}
1717 
1718 	return work_done;
1719 }
1720 
1721 static void enic_notify_timer(struct timer_list *t)
1722 {
1723 	struct enic *enic = from_timer(enic, t, notify_timer);
1724 
1725 	enic_notify_check(enic);
1726 
1727 	mod_timer(&enic->notify_timer,
1728 		round_jiffies(jiffies + ENIC_NOTIFY_TIMER_PERIOD));
1729 }
1730 
1731 static void enic_free_intr(struct enic *enic)
1732 {
1733 	struct net_device *netdev = enic->netdev;
1734 	unsigned int i;
1735 
1736 	enic_free_rx_cpu_rmap(enic);
1737 	switch (vnic_dev_get_intr_mode(enic->vdev)) {
1738 	case VNIC_DEV_INTR_MODE_INTX:
1739 		free_irq(enic->pdev->irq, netdev);
1740 		break;
1741 	case VNIC_DEV_INTR_MODE_MSI:
1742 		free_irq(enic->pdev->irq, enic);
1743 		break;
1744 	case VNIC_DEV_INTR_MODE_MSIX:
1745 		for (i = 0; i < ARRAY_SIZE(enic->msix); i++)
1746 			if (enic->msix[i].requested)
1747 				free_irq(enic->msix_entry[i].vector,
1748 					enic->msix[i].devid);
1749 		break;
1750 	default:
1751 		break;
1752 	}
1753 }
1754 
1755 static int enic_request_intr(struct enic *enic)
1756 {
1757 	struct net_device *netdev = enic->netdev;
1758 	unsigned int i, intr;
1759 	int err = 0;
1760 
1761 	enic_set_rx_cpu_rmap(enic);
1762 	switch (vnic_dev_get_intr_mode(enic->vdev)) {
1763 
1764 	case VNIC_DEV_INTR_MODE_INTX:
1765 
1766 		err = request_irq(enic->pdev->irq, enic_isr_legacy,
1767 			IRQF_SHARED, netdev->name, netdev);
1768 		break;
1769 
1770 	case VNIC_DEV_INTR_MODE_MSI:
1771 
1772 		err = request_irq(enic->pdev->irq, enic_isr_msi,
1773 			0, netdev->name, enic);
1774 		break;
1775 
1776 	case VNIC_DEV_INTR_MODE_MSIX:
1777 
1778 		for (i = 0; i < enic->rq_count; i++) {
1779 			intr = enic_msix_rq_intr(enic, i);
1780 			snprintf(enic->msix[intr].devname,
1781 				sizeof(enic->msix[intr].devname),
1782 				"%s-rx-%u", netdev->name, i);
1783 			enic->msix[intr].isr = enic_isr_msix;
1784 			enic->msix[intr].devid = &enic->napi[i];
1785 		}
1786 
1787 		for (i = 0; i < enic->wq_count; i++) {
1788 			int wq = enic_cq_wq(enic, i);
1789 
1790 			intr = enic_msix_wq_intr(enic, i);
1791 			snprintf(enic->msix[intr].devname,
1792 				sizeof(enic->msix[intr].devname),
1793 				"%s-tx-%u", netdev->name, i);
1794 			enic->msix[intr].isr = enic_isr_msix;
1795 			enic->msix[intr].devid = &enic->napi[wq];
1796 		}
1797 
1798 		intr = enic_msix_err_intr(enic);
1799 		snprintf(enic->msix[intr].devname,
1800 			sizeof(enic->msix[intr].devname),
1801 			"%s-err", netdev->name);
1802 		enic->msix[intr].isr = enic_isr_msix_err;
1803 		enic->msix[intr].devid = enic;
1804 
1805 		intr = enic_msix_notify_intr(enic);
1806 		snprintf(enic->msix[intr].devname,
1807 			sizeof(enic->msix[intr].devname),
1808 			"%s-notify", netdev->name);
1809 		enic->msix[intr].isr = enic_isr_msix_notify;
1810 		enic->msix[intr].devid = enic;
1811 
1812 		for (i = 0; i < ARRAY_SIZE(enic->msix); i++)
1813 			enic->msix[i].requested = 0;
1814 
1815 		for (i = 0; i < enic->intr_count; i++) {
1816 			err = request_irq(enic->msix_entry[i].vector,
1817 				enic->msix[i].isr, 0,
1818 				enic->msix[i].devname,
1819 				enic->msix[i].devid);
1820 			if (err) {
1821 				enic_free_intr(enic);
1822 				break;
1823 			}
1824 			enic->msix[i].requested = 1;
1825 		}
1826 
1827 		break;
1828 
1829 	default:
1830 		break;
1831 	}
1832 
1833 	return err;
1834 }
1835 
1836 static void enic_synchronize_irqs(struct enic *enic)
1837 {
1838 	unsigned int i;
1839 
1840 	switch (vnic_dev_get_intr_mode(enic->vdev)) {
1841 	case VNIC_DEV_INTR_MODE_INTX:
1842 	case VNIC_DEV_INTR_MODE_MSI:
1843 		synchronize_irq(enic->pdev->irq);
1844 		break;
1845 	case VNIC_DEV_INTR_MODE_MSIX:
1846 		for (i = 0; i < enic->intr_count; i++)
1847 			synchronize_irq(enic->msix_entry[i].vector);
1848 		break;
1849 	default:
1850 		break;
1851 	}
1852 }
1853 
1854 static void enic_set_rx_coal_setting(struct enic *enic)
1855 {
1856 	unsigned int speed;
1857 	int index = -1;
1858 	struct enic_rx_coal *rx_coal = &enic->rx_coalesce_setting;
1859 
1860 	/* 1. Read the link speed from fw
1861 	 * 2. Pick the default range for the speed
1862 	 * 3. Update it in enic->rx_coalesce_setting
1863 	 */
1864 	speed = vnic_dev_port_speed(enic->vdev);
1865 	if (ENIC_LINK_SPEED_10G < speed)
1866 		index = ENIC_LINK_40G_INDEX;
1867 	else if (ENIC_LINK_SPEED_4G < speed)
1868 		index = ENIC_LINK_10G_INDEX;
1869 	else
1870 		index = ENIC_LINK_4G_INDEX;
1871 
1872 	rx_coal->small_pkt_range_start = mod_range[index].small_pkt_range_start;
1873 	rx_coal->large_pkt_range_start = mod_range[index].large_pkt_range_start;
1874 	rx_coal->range_end = ENIC_RX_COALESCE_RANGE_END;
1875 
1876 	/* Start with the value provided by UCSM */
1877 	for (index = 0; index < enic->rq_count; index++)
1878 		enic->cq[index].cur_rx_coal_timeval =
1879 				enic->config.intr_timer_usec;
1880 
1881 	rx_coal->use_adaptive_rx_coalesce = 1;
1882 }
1883 
1884 static int enic_dev_notify_set(struct enic *enic)
1885 {
1886 	int err;
1887 
1888 	spin_lock_bh(&enic->devcmd_lock);
1889 	switch (vnic_dev_get_intr_mode(enic->vdev)) {
1890 	case VNIC_DEV_INTR_MODE_INTX:
1891 		err = vnic_dev_notify_set(enic->vdev,
1892 			enic_legacy_notify_intr());
1893 		break;
1894 	case VNIC_DEV_INTR_MODE_MSIX:
1895 		err = vnic_dev_notify_set(enic->vdev,
1896 			enic_msix_notify_intr(enic));
1897 		break;
1898 	default:
1899 		err = vnic_dev_notify_set(enic->vdev, -1 /* no intr */);
1900 		break;
1901 	}
1902 	spin_unlock_bh(&enic->devcmd_lock);
1903 
1904 	return err;
1905 }
1906 
1907 static void enic_notify_timer_start(struct enic *enic)
1908 {
1909 	switch (vnic_dev_get_intr_mode(enic->vdev)) {
1910 	case VNIC_DEV_INTR_MODE_MSI:
1911 		mod_timer(&enic->notify_timer, jiffies);
1912 		break;
1913 	default:
1914 		/* Using intr for notification for INTx/MSI-X */
1915 		break;
1916 	}
1917 }
1918 
1919 /* rtnl lock is held, process context */
1920 static int enic_open(struct net_device *netdev)
1921 {
1922 	struct enic *enic = netdev_priv(netdev);
1923 	unsigned int i;
1924 	int err, ret;
1925 
1926 	err = enic_request_intr(enic);
1927 	if (err) {
1928 		netdev_err(netdev, "Unable to request irq.\n");
1929 		return err;
1930 	}
1931 	enic_init_affinity_hint(enic);
1932 	enic_set_affinity_hint(enic);
1933 
1934 	err = enic_dev_notify_set(enic);
1935 	if (err) {
1936 		netdev_err(netdev,
1937 			"Failed to alloc notify buffer, aborting.\n");
1938 		goto err_out_free_intr;
1939 	}
1940 
1941 	for (i = 0; i < enic->rq_count; i++) {
1942 		/* enable rq before updating rq desc */
1943 		vnic_rq_enable(&enic->rq[i]);
1944 		vnic_rq_fill(&enic->rq[i], enic_rq_alloc_buf);
1945 		/* Need at least one buffer on ring to get going */
1946 		if (vnic_rq_desc_used(&enic->rq[i]) == 0) {
1947 			netdev_err(netdev, "Unable to alloc receive buffers\n");
1948 			err = -ENOMEM;
1949 			goto err_out_free_rq;
1950 		}
1951 	}
1952 
1953 	for (i = 0; i < enic->wq_count; i++)
1954 		vnic_wq_enable(&enic->wq[i]);
1955 
1956 	if (!enic_is_dynamic(enic) && !enic_is_sriov_vf(enic))
1957 		enic_dev_add_station_addr(enic);
1958 
1959 	enic_set_rx_mode(netdev);
1960 
1961 	netif_tx_wake_all_queues(netdev);
1962 
1963 	for (i = 0; i < enic->rq_count; i++)
1964 		napi_enable(&enic->napi[i]);
1965 
1966 	if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX)
1967 		for (i = 0; i < enic->wq_count; i++)
1968 			napi_enable(&enic->napi[enic_cq_wq(enic, i)]);
1969 	enic_dev_enable(enic);
1970 
1971 	for (i = 0; i < enic->intr_count; i++)
1972 		vnic_intr_unmask(&enic->intr[i]);
1973 
1974 	enic_notify_timer_start(enic);
1975 	enic_rfs_timer_start(enic);
1976 
1977 	return 0;
1978 
1979 err_out_free_rq:
1980 	for (i = 0; i < enic->rq_count; i++) {
1981 		ret = vnic_rq_disable(&enic->rq[i]);
1982 		if (!ret)
1983 			vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
1984 	}
1985 	enic_dev_notify_unset(enic);
1986 err_out_free_intr:
1987 	enic_unset_affinity_hint(enic);
1988 	enic_free_intr(enic);
1989 
1990 	return err;
1991 }
1992 
1993 /* rtnl lock is held, process context */
1994 static int enic_stop(struct net_device *netdev)
1995 {
1996 	struct enic *enic = netdev_priv(netdev);
1997 	unsigned int i;
1998 	int err;
1999 
2000 	for (i = 0; i < enic->intr_count; i++) {
2001 		vnic_intr_mask(&enic->intr[i]);
2002 		(void)vnic_intr_masked(&enic->intr[i]); /* flush write */
2003 	}
2004 
2005 	enic_synchronize_irqs(enic);
2006 
2007 	del_timer_sync(&enic->notify_timer);
2008 	enic_rfs_flw_tbl_free(enic);
2009 
2010 	enic_dev_disable(enic);
2011 
2012 	for (i = 0; i < enic->rq_count; i++)
2013 		napi_disable(&enic->napi[i]);
2014 
2015 	netif_carrier_off(netdev);
2016 	netif_tx_disable(netdev);
2017 	if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX)
2018 		for (i = 0; i < enic->wq_count; i++)
2019 			napi_disable(&enic->napi[enic_cq_wq(enic, i)]);
2020 
2021 	if (!enic_is_dynamic(enic) && !enic_is_sriov_vf(enic))
2022 		enic_dev_del_station_addr(enic);
2023 
2024 	for (i = 0; i < enic->wq_count; i++) {
2025 		err = vnic_wq_disable(&enic->wq[i]);
2026 		if (err)
2027 			return err;
2028 	}
2029 	for (i = 0; i < enic->rq_count; i++) {
2030 		err = vnic_rq_disable(&enic->rq[i]);
2031 		if (err)
2032 			return err;
2033 	}
2034 
2035 	enic_dev_notify_unset(enic);
2036 	enic_unset_affinity_hint(enic);
2037 	enic_free_intr(enic);
2038 
2039 	for (i = 0; i < enic->wq_count; i++)
2040 		vnic_wq_clean(&enic->wq[i], enic_free_wq_buf);
2041 	for (i = 0; i < enic->rq_count; i++)
2042 		vnic_rq_clean(&enic->rq[i], enic_free_rq_buf);
2043 	for (i = 0; i < enic->cq_count; i++)
2044 		vnic_cq_clean(&enic->cq[i]);
2045 	for (i = 0; i < enic->intr_count; i++)
2046 		vnic_intr_clean(&enic->intr[i]);
2047 
2048 	return 0;
2049 }
2050 
2051 static int _enic_change_mtu(struct net_device *netdev, int new_mtu)
2052 {
2053 	bool running = netif_running(netdev);
2054 	int err = 0;
2055 
2056 	ASSERT_RTNL();
2057 	if (running) {
2058 		err = enic_stop(netdev);
2059 		if (err)
2060 			return err;
2061 	}
2062 
2063 	netdev->mtu = new_mtu;
2064 
2065 	if (running) {
2066 		err = enic_open(netdev);
2067 		if (err)
2068 			return err;
2069 	}
2070 
2071 	return 0;
2072 }
2073 
2074 static int enic_change_mtu(struct net_device *netdev, int new_mtu)
2075 {
2076 	struct enic *enic = netdev_priv(netdev);
2077 
2078 	if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic))
2079 		return -EOPNOTSUPP;
2080 
2081 	if (netdev->mtu > enic->port_mtu)
2082 		netdev_warn(netdev,
2083 			    "interface MTU (%d) set higher than port MTU (%d)\n",
2084 			    netdev->mtu, enic->port_mtu);
2085 
2086 	return _enic_change_mtu(netdev, new_mtu);
2087 }
2088 
2089 static void enic_change_mtu_work(struct work_struct *work)
2090 {
2091 	struct enic *enic = container_of(work, struct enic, change_mtu_work);
2092 	struct net_device *netdev = enic->netdev;
2093 	int new_mtu = vnic_dev_mtu(enic->vdev);
2094 
2095 	rtnl_lock();
2096 	(void)_enic_change_mtu(netdev, new_mtu);
2097 	rtnl_unlock();
2098 
2099 	netdev_info(netdev, "interface MTU set as %d\n", netdev->mtu);
2100 }
2101 
2102 #ifdef CONFIG_NET_POLL_CONTROLLER
2103 static void enic_poll_controller(struct net_device *netdev)
2104 {
2105 	struct enic *enic = netdev_priv(netdev);
2106 	struct vnic_dev *vdev = enic->vdev;
2107 	unsigned int i, intr;
2108 
2109 	switch (vnic_dev_get_intr_mode(vdev)) {
2110 	case VNIC_DEV_INTR_MODE_MSIX:
2111 		for (i = 0; i < enic->rq_count; i++) {
2112 			intr = enic_msix_rq_intr(enic, i);
2113 			enic_isr_msix(enic->msix_entry[intr].vector,
2114 				      &enic->napi[i]);
2115 		}
2116 
2117 		for (i = 0; i < enic->wq_count; i++) {
2118 			intr = enic_msix_wq_intr(enic, i);
2119 			enic_isr_msix(enic->msix_entry[intr].vector,
2120 				      &enic->napi[enic_cq_wq(enic, i)]);
2121 		}
2122 
2123 		break;
2124 	case VNIC_DEV_INTR_MODE_MSI:
2125 		enic_isr_msi(enic->pdev->irq, enic);
2126 		break;
2127 	case VNIC_DEV_INTR_MODE_INTX:
2128 		enic_isr_legacy(enic->pdev->irq, netdev);
2129 		break;
2130 	default:
2131 		break;
2132 	}
2133 }
2134 #endif
2135 
2136 static int enic_dev_wait(struct vnic_dev *vdev,
2137 	int (*start)(struct vnic_dev *, int),
2138 	int (*finished)(struct vnic_dev *, int *),
2139 	int arg)
2140 {
2141 	unsigned long time;
2142 	int done;
2143 	int err;
2144 
2145 	BUG_ON(in_interrupt());
2146 
2147 	err = start(vdev, arg);
2148 	if (err)
2149 		return err;
2150 
2151 	/* Wait for func to complete...2 seconds max
2152 	 */
2153 
2154 	time = jiffies + (HZ * 2);
2155 	do {
2156 
2157 		err = finished(vdev, &done);
2158 		if (err)
2159 			return err;
2160 
2161 		if (done)
2162 			return 0;
2163 
2164 		schedule_timeout_uninterruptible(HZ / 10);
2165 
2166 	} while (time_after(time, jiffies));
2167 
2168 	return -ETIMEDOUT;
2169 }
2170 
2171 static int enic_dev_open(struct enic *enic)
2172 {
2173 	int err;
2174 	u32 flags = CMD_OPENF_IG_DESCCACHE;
2175 
2176 	err = enic_dev_wait(enic->vdev, vnic_dev_open,
2177 		vnic_dev_open_done, flags);
2178 	if (err)
2179 		dev_err(enic_get_dev(enic), "vNIC device open failed, err %d\n",
2180 			err);
2181 
2182 	return err;
2183 }
2184 
2185 static int enic_dev_soft_reset(struct enic *enic)
2186 {
2187 	int err;
2188 
2189 	err = enic_dev_wait(enic->vdev, vnic_dev_soft_reset,
2190 			    vnic_dev_soft_reset_done, 0);
2191 	if (err)
2192 		netdev_err(enic->netdev, "vNIC soft reset failed, err %d\n",
2193 			   err);
2194 
2195 	return err;
2196 }
2197 
2198 static int enic_dev_hang_reset(struct enic *enic)
2199 {
2200 	int err;
2201 
2202 	err = enic_dev_wait(enic->vdev, vnic_dev_hang_reset,
2203 		vnic_dev_hang_reset_done, 0);
2204 	if (err)
2205 		netdev_err(enic->netdev, "vNIC hang reset failed, err %d\n",
2206 			err);
2207 
2208 	return err;
2209 }
2210 
2211 int __enic_set_rsskey(struct enic *enic)
2212 {
2213 	union vnic_rss_key *rss_key_buf_va;
2214 	dma_addr_t rss_key_buf_pa;
2215 	int i, kidx, bidx, err;
2216 
2217 	rss_key_buf_va = pci_zalloc_consistent(enic->pdev,
2218 					       sizeof(union vnic_rss_key),
2219 					       &rss_key_buf_pa);
2220 	if (!rss_key_buf_va)
2221 		return -ENOMEM;
2222 
2223 	for (i = 0; i < ENIC_RSS_LEN; i++) {
2224 		kidx = i / ENIC_RSS_BYTES_PER_KEY;
2225 		bidx = i % ENIC_RSS_BYTES_PER_KEY;
2226 		rss_key_buf_va->key[kidx].b[bidx] = enic->rss_key[i];
2227 	}
2228 	spin_lock_bh(&enic->devcmd_lock);
2229 	err = enic_set_rss_key(enic,
2230 		rss_key_buf_pa,
2231 		sizeof(union vnic_rss_key));
2232 	spin_unlock_bh(&enic->devcmd_lock);
2233 
2234 	pci_free_consistent(enic->pdev, sizeof(union vnic_rss_key),
2235 		rss_key_buf_va, rss_key_buf_pa);
2236 
2237 	return err;
2238 }
2239 
2240 static int enic_set_rsskey(struct enic *enic)
2241 {
2242 	netdev_rss_key_fill(enic->rss_key, ENIC_RSS_LEN);
2243 
2244 	return __enic_set_rsskey(enic);
2245 }
2246 
2247 static int enic_set_rsscpu(struct enic *enic, u8 rss_hash_bits)
2248 {
2249 	dma_addr_t rss_cpu_buf_pa;
2250 	union vnic_rss_cpu *rss_cpu_buf_va = NULL;
2251 	unsigned int i;
2252 	int err;
2253 
2254 	rss_cpu_buf_va = pci_alloc_consistent(enic->pdev,
2255 		sizeof(union vnic_rss_cpu), &rss_cpu_buf_pa);
2256 	if (!rss_cpu_buf_va)
2257 		return -ENOMEM;
2258 
2259 	for (i = 0; i < (1 << rss_hash_bits); i++)
2260 		(*rss_cpu_buf_va).cpu[i/4].b[i%4] = i % enic->rq_count;
2261 
2262 	spin_lock_bh(&enic->devcmd_lock);
2263 	err = enic_set_rss_cpu(enic,
2264 		rss_cpu_buf_pa,
2265 		sizeof(union vnic_rss_cpu));
2266 	spin_unlock_bh(&enic->devcmd_lock);
2267 
2268 	pci_free_consistent(enic->pdev, sizeof(union vnic_rss_cpu),
2269 		rss_cpu_buf_va, rss_cpu_buf_pa);
2270 
2271 	return err;
2272 }
2273 
2274 static int enic_set_niccfg(struct enic *enic, u8 rss_default_cpu,
2275 	u8 rss_hash_type, u8 rss_hash_bits, u8 rss_base_cpu, u8 rss_enable)
2276 {
2277 	const u8 tso_ipid_split_en = 0;
2278 	const u8 ig_vlan_strip_en = 1;
2279 	int err;
2280 
2281 	/* Enable VLAN tag stripping.
2282 	*/
2283 
2284 	spin_lock_bh(&enic->devcmd_lock);
2285 	err = enic_set_nic_cfg(enic,
2286 		rss_default_cpu, rss_hash_type,
2287 		rss_hash_bits, rss_base_cpu,
2288 		rss_enable, tso_ipid_split_en,
2289 		ig_vlan_strip_en);
2290 	spin_unlock_bh(&enic->devcmd_lock);
2291 
2292 	return err;
2293 }
2294 
2295 static int enic_set_rss_nic_cfg(struct enic *enic)
2296 {
2297 	struct device *dev = enic_get_dev(enic);
2298 	const u8 rss_default_cpu = 0;
2299 	const u8 rss_hash_bits = 7;
2300 	const u8 rss_base_cpu = 0;
2301 	u8 rss_hash_type;
2302 	int res;
2303 	u8 rss_enable = ENIC_SETTING(enic, RSS) && (enic->rq_count > 1);
2304 
2305 	spin_lock_bh(&enic->devcmd_lock);
2306 	res = vnic_dev_capable_rss_hash_type(enic->vdev, &rss_hash_type);
2307 	spin_unlock_bh(&enic->devcmd_lock);
2308 	if (res) {
2309 		/* defaults for old adapters
2310 		 */
2311 		rss_hash_type = NIC_CFG_RSS_HASH_TYPE_IPV4	|
2312 				NIC_CFG_RSS_HASH_TYPE_TCP_IPV4	|
2313 				NIC_CFG_RSS_HASH_TYPE_IPV6	|
2314 				NIC_CFG_RSS_HASH_TYPE_TCP_IPV6;
2315 	}
2316 
2317 	if (rss_enable) {
2318 		if (!enic_set_rsskey(enic)) {
2319 			if (enic_set_rsscpu(enic, rss_hash_bits)) {
2320 				rss_enable = 0;
2321 				dev_warn(dev, "RSS disabled, "
2322 					"Failed to set RSS cpu indirection table.");
2323 			}
2324 		} else {
2325 			rss_enable = 0;
2326 			dev_warn(dev, "RSS disabled, Failed to set RSS key.\n");
2327 		}
2328 	}
2329 
2330 	return enic_set_niccfg(enic, rss_default_cpu, rss_hash_type,
2331 		rss_hash_bits, rss_base_cpu, rss_enable);
2332 }
2333 
2334 static void enic_reset(struct work_struct *work)
2335 {
2336 	struct enic *enic = container_of(work, struct enic, reset);
2337 
2338 	if (!netif_running(enic->netdev))
2339 		return;
2340 
2341 	rtnl_lock();
2342 
2343 	spin_lock(&enic->enic_api_lock);
2344 	enic_stop(enic->netdev);
2345 	enic_dev_soft_reset(enic);
2346 	enic_reset_addr_lists(enic);
2347 	enic_init_vnic_resources(enic);
2348 	enic_set_rss_nic_cfg(enic);
2349 	enic_dev_set_ig_vlan_rewrite_mode(enic);
2350 	enic_open(enic->netdev);
2351 	spin_unlock(&enic->enic_api_lock);
2352 	call_netdevice_notifiers(NETDEV_REBOOT, enic->netdev);
2353 
2354 	rtnl_unlock();
2355 }
2356 
2357 static void enic_tx_hang_reset(struct work_struct *work)
2358 {
2359 	struct enic *enic = container_of(work, struct enic, tx_hang_reset);
2360 
2361 	rtnl_lock();
2362 
2363 	spin_lock(&enic->enic_api_lock);
2364 	enic_dev_hang_notify(enic);
2365 	enic_stop(enic->netdev);
2366 	enic_dev_hang_reset(enic);
2367 	enic_reset_addr_lists(enic);
2368 	enic_init_vnic_resources(enic);
2369 	enic_set_rss_nic_cfg(enic);
2370 	enic_dev_set_ig_vlan_rewrite_mode(enic);
2371 	enic_open(enic->netdev);
2372 	spin_unlock(&enic->enic_api_lock);
2373 	call_netdevice_notifiers(NETDEV_REBOOT, enic->netdev);
2374 
2375 	rtnl_unlock();
2376 }
2377 
2378 static int enic_set_intr_mode(struct enic *enic)
2379 {
2380 	unsigned int n = min_t(unsigned int, enic->rq_count, ENIC_RQ_MAX);
2381 	unsigned int m = min_t(unsigned int, enic->wq_count, ENIC_WQ_MAX);
2382 	unsigned int i;
2383 
2384 	/* Set interrupt mode (INTx, MSI, MSI-X) depending
2385 	 * on system capabilities.
2386 	 *
2387 	 * Try MSI-X first
2388 	 *
2389 	 * We need n RQs, m WQs, n+m CQs, and n+m+2 INTRs
2390 	 * (the second to last INTR is used for WQ/RQ errors)
2391 	 * (the last INTR is used for notifications)
2392 	 */
2393 
2394 	BUG_ON(ARRAY_SIZE(enic->msix_entry) < n + m + 2);
2395 	for (i = 0; i < n + m + 2; i++)
2396 		enic->msix_entry[i].entry = i;
2397 
2398 	/* Use multiple RQs if RSS is enabled
2399 	 */
2400 
2401 	if (ENIC_SETTING(enic, RSS) &&
2402 	    enic->config.intr_mode < 1 &&
2403 	    enic->rq_count >= n &&
2404 	    enic->wq_count >= m &&
2405 	    enic->cq_count >= n + m &&
2406 	    enic->intr_count >= n + m + 2) {
2407 
2408 		if (pci_enable_msix_range(enic->pdev, enic->msix_entry,
2409 					  n + m + 2, n + m + 2) > 0) {
2410 
2411 			enic->rq_count = n;
2412 			enic->wq_count = m;
2413 			enic->cq_count = n + m;
2414 			enic->intr_count = n + m + 2;
2415 
2416 			vnic_dev_set_intr_mode(enic->vdev,
2417 				VNIC_DEV_INTR_MODE_MSIX);
2418 
2419 			return 0;
2420 		}
2421 	}
2422 
2423 	if (enic->config.intr_mode < 1 &&
2424 	    enic->rq_count >= 1 &&
2425 	    enic->wq_count >= m &&
2426 	    enic->cq_count >= 1 + m &&
2427 	    enic->intr_count >= 1 + m + 2) {
2428 		if (pci_enable_msix_range(enic->pdev, enic->msix_entry,
2429 					  1 + m + 2, 1 + m + 2) > 0) {
2430 
2431 			enic->rq_count = 1;
2432 			enic->wq_count = m;
2433 			enic->cq_count = 1 + m;
2434 			enic->intr_count = 1 + m + 2;
2435 
2436 			vnic_dev_set_intr_mode(enic->vdev,
2437 				VNIC_DEV_INTR_MODE_MSIX);
2438 
2439 			return 0;
2440 		}
2441 	}
2442 
2443 	/* Next try MSI
2444 	 *
2445 	 * We need 1 RQ, 1 WQ, 2 CQs, and 1 INTR
2446 	 */
2447 
2448 	if (enic->config.intr_mode < 2 &&
2449 	    enic->rq_count >= 1 &&
2450 	    enic->wq_count >= 1 &&
2451 	    enic->cq_count >= 2 &&
2452 	    enic->intr_count >= 1 &&
2453 	    !pci_enable_msi(enic->pdev)) {
2454 
2455 		enic->rq_count = 1;
2456 		enic->wq_count = 1;
2457 		enic->cq_count = 2;
2458 		enic->intr_count = 1;
2459 
2460 		vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_MSI);
2461 
2462 		return 0;
2463 	}
2464 
2465 	/* Next try INTx
2466 	 *
2467 	 * We need 1 RQ, 1 WQ, 2 CQs, and 3 INTRs
2468 	 * (the first INTR is used for WQ/RQ)
2469 	 * (the second INTR is used for WQ/RQ errors)
2470 	 * (the last INTR is used for notifications)
2471 	 */
2472 
2473 	if (enic->config.intr_mode < 3 &&
2474 	    enic->rq_count >= 1 &&
2475 	    enic->wq_count >= 1 &&
2476 	    enic->cq_count >= 2 &&
2477 	    enic->intr_count >= 3) {
2478 
2479 		enic->rq_count = 1;
2480 		enic->wq_count = 1;
2481 		enic->cq_count = 2;
2482 		enic->intr_count = 3;
2483 
2484 		vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_INTX);
2485 
2486 		return 0;
2487 	}
2488 
2489 	vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_UNKNOWN);
2490 
2491 	return -EINVAL;
2492 }
2493 
2494 static void enic_clear_intr_mode(struct enic *enic)
2495 {
2496 	switch (vnic_dev_get_intr_mode(enic->vdev)) {
2497 	case VNIC_DEV_INTR_MODE_MSIX:
2498 		pci_disable_msix(enic->pdev);
2499 		break;
2500 	case VNIC_DEV_INTR_MODE_MSI:
2501 		pci_disable_msi(enic->pdev);
2502 		break;
2503 	default:
2504 		break;
2505 	}
2506 
2507 	vnic_dev_set_intr_mode(enic->vdev, VNIC_DEV_INTR_MODE_UNKNOWN);
2508 }
2509 
2510 static const struct net_device_ops enic_netdev_dynamic_ops = {
2511 	.ndo_open		= enic_open,
2512 	.ndo_stop		= enic_stop,
2513 	.ndo_start_xmit		= enic_hard_start_xmit,
2514 	.ndo_get_stats64	= enic_get_stats,
2515 	.ndo_validate_addr	= eth_validate_addr,
2516 	.ndo_set_rx_mode	= enic_set_rx_mode,
2517 	.ndo_set_mac_address	= enic_set_mac_address_dynamic,
2518 	.ndo_change_mtu		= enic_change_mtu,
2519 	.ndo_vlan_rx_add_vid	= enic_vlan_rx_add_vid,
2520 	.ndo_vlan_rx_kill_vid	= enic_vlan_rx_kill_vid,
2521 	.ndo_tx_timeout		= enic_tx_timeout,
2522 	.ndo_set_vf_port	= enic_set_vf_port,
2523 	.ndo_get_vf_port	= enic_get_vf_port,
2524 	.ndo_set_vf_mac		= enic_set_vf_mac,
2525 #ifdef CONFIG_NET_POLL_CONTROLLER
2526 	.ndo_poll_controller	= enic_poll_controller,
2527 #endif
2528 #ifdef CONFIG_RFS_ACCEL
2529 	.ndo_rx_flow_steer	= enic_rx_flow_steer,
2530 #endif
2531 	.ndo_udp_tunnel_add	= enic_udp_tunnel_add,
2532 	.ndo_udp_tunnel_del	= enic_udp_tunnel_del,
2533 	.ndo_features_check	= enic_features_check,
2534 };
2535 
2536 static const struct net_device_ops enic_netdev_ops = {
2537 	.ndo_open		= enic_open,
2538 	.ndo_stop		= enic_stop,
2539 	.ndo_start_xmit		= enic_hard_start_xmit,
2540 	.ndo_get_stats64	= enic_get_stats,
2541 	.ndo_validate_addr	= eth_validate_addr,
2542 	.ndo_set_mac_address	= enic_set_mac_address,
2543 	.ndo_set_rx_mode	= enic_set_rx_mode,
2544 	.ndo_change_mtu		= enic_change_mtu,
2545 	.ndo_vlan_rx_add_vid	= enic_vlan_rx_add_vid,
2546 	.ndo_vlan_rx_kill_vid	= enic_vlan_rx_kill_vid,
2547 	.ndo_tx_timeout		= enic_tx_timeout,
2548 	.ndo_set_vf_port	= enic_set_vf_port,
2549 	.ndo_get_vf_port	= enic_get_vf_port,
2550 	.ndo_set_vf_mac		= enic_set_vf_mac,
2551 #ifdef CONFIG_NET_POLL_CONTROLLER
2552 	.ndo_poll_controller	= enic_poll_controller,
2553 #endif
2554 #ifdef CONFIG_RFS_ACCEL
2555 	.ndo_rx_flow_steer	= enic_rx_flow_steer,
2556 #endif
2557 	.ndo_udp_tunnel_add	= enic_udp_tunnel_add,
2558 	.ndo_udp_tunnel_del	= enic_udp_tunnel_del,
2559 	.ndo_features_check	= enic_features_check,
2560 };
2561 
2562 static void enic_dev_deinit(struct enic *enic)
2563 {
2564 	unsigned int i;
2565 
2566 	for (i = 0; i < enic->rq_count; i++) {
2567 		napi_hash_del(&enic->napi[i]);
2568 		netif_napi_del(&enic->napi[i]);
2569 	}
2570 	if (vnic_dev_get_intr_mode(enic->vdev) == VNIC_DEV_INTR_MODE_MSIX)
2571 		for (i = 0; i < enic->wq_count; i++)
2572 			netif_napi_del(&enic->napi[enic_cq_wq(enic, i)]);
2573 
2574 	enic_free_vnic_resources(enic);
2575 	enic_clear_intr_mode(enic);
2576 	enic_free_affinity_hint(enic);
2577 }
2578 
2579 static void enic_kdump_kernel_config(struct enic *enic)
2580 {
2581 	if (is_kdump_kernel()) {
2582 		dev_info(enic_get_dev(enic), "Running from within kdump kernel. Using minimal resources\n");
2583 		enic->rq_count = 1;
2584 		enic->wq_count = 1;
2585 		enic->config.rq_desc_count = ENIC_MIN_RQ_DESCS;
2586 		enic->config.wq_desc_count = ENIC_MIN_WQ_DESCS;
2587 		enic->config.mtu = min_t(u16, 1500, enic->config.mtu);
2588 	}
2589 }
2590 
2591 static int enic_dev_init(struct enic *enic)
2592 {
2593 	struct device *dev = enic_get_dev(enic);
2594 	struct net_device *netdev = enic->netdev;
2595 	unsigned int i;
2596 	int err;
2597 
2598 	/* Get interrupt coalesce timer info */
2599 	err = enic_dev_intr_coal_timer_info(enic);
2600 	if (err) {
2601 		dev_warn(dev, "Using default conversion factor for "
2602 			"interrupt coalesce timer\n");
2603 		vnic_dev_intr_coal_timer_info_default(enic->vdev);
2604 	}
2605 
2606 	/* Get vNIC configuration
2607 	 */
2608 
2609 	err = enic_get_vnic_config(enic);
2610 	if (err) {
2611 		dev_err(dev, "Get vNIC configuration failed, aborting\n");
2612 		return err;
2613 	}
2614 
2615 	/* Get available resource counts
2616 	 */
2617 
2618 	enic_get_res_counts(enic);
2619 
2620 	/* modify resource count if we are in kdump_kernel
2621 	 */
2622 	enic_kdump_kernel_config(enic);
2623 
2624 	/* Set interrupt mode based on resource counts and system
2625 	 * capabilities
2626 	 */
2627 
2628 	err = enic_set_intr_mode(enic);
2629 	if (err) {
2630 		dev_err(dev, "Failed to set intr mode based on resource "
2631 			"counts and system capabilities, aborting\n");
2632 		return err;
2633 	}
2634 
2635 	/* Allocate and configure vNIC resources
2636 	 */
2637 
2638 	err = enic_alloc_vnic_resources(enic);
2639 	if (err) {
2640 		dev_err(dev, "Failed to alloc vNIC resources, aborting\n");
2641 		goto err_out_free_vnic_resources;
2642 	}
2643 
2644 	enic_init_vnic_resources(enic);
2645 
2646 	err = enic_set_rss_nic_cfg(enic);
2647 	if (err) {
2648 		dev_err(dev, "Failed to config nic, aborting\n");
2649 		goto err_out_free_vnic_resources;
2650 	}
2651 
2652 	switch (vnic_dev_get_intr_mode(enic->vdev)) {
2653 	default:
2654 		netif_napi_add(netdev, &enic->napi[0], enic_poll, 64);
2655 		break;
2656 	case VNIC_DEV_INTR_MODE_MSIX:
2657 		for (i = 0; i < enic->rq_count; i++) {
2658 			netif_napi_add(netdev, &enic->napi[i],
2659 				enic_poll_msix_rq, NAPI_POLL_WEIGHT);
2660 		}
2661 		for (i = 0; i < enic->wq_count; i++)
2662 			netif_napi_add(netdev, &enic->napi[enic_cq_wq(enic, i)],
2663 				       enic_poll_msix_wq, NAPI_POLL_WEIGHT);
2664 		break;
2665 	}
2666 
2667 	return 0;
2668 
2669 err_out_free_vnic_resources:
2670 	enic_free_affinity_hint(enic);
2671 	enic_clear_intr_mode(enic);
2672 	enic_free_vnic_resources(enic);
2673 
2674 	return err;
2675 }
2676 
2677 static void enic_iounmap(struct enic *enic)
2678 {
2679 	unsigned int i;
2680 
2681 	for (i = 0; i < ARRAY_SIZE(enic->bar); i++)
2682 		if (enic->bar[i].vaddr)
2683 			iounmap(enic->bar[i].vaddr);
2684 }
2685 
2686 static int enic_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2687 {
2688 	struct device *dev = &pdev->dev;
2689 	struct net_device *netdev;
2690 	struct enic *enic;
2691 	int using_dac = 0;
2692 	unsigned int i;
2693 	int err;
2694 #ifdef CONFIG_PCI_IOV
2695 	int pos = 0;
2696 #endif
2697 	int num_pps = 1;
2698 
2699 	/* Allocate net device structure and initialize.  Private
2700 	 * instance data is initialized to zero.
2701 	 */
2702 
2703 	netdev = alloc_etherdev_mqs(sizeof(struct enic),
2704 				    ENIC_RQ_MAX, ENIC_WQ_MAX);
2705 	if (!netdev)
2706 		return -ENOMEM;
2707 
2708 	pci_set_drvdata(pdev, netdev);
2709 
2710 	SET_NETDEV_DEV(netdev, &pdev->dev);
2711 
2712 	enic = netdev_priv(netdev);
2713 	enic->netdev = netdev;
2714 	enic->pdev = pdev;
2715 
2716 	/* Setup PCI resources
2717 	 */
2718 
2719 	err = pci_enable_device_mem(pdev);
2720 	if (err) {
2721 		dev_err(dev, "Cannot enable PCI device, aborting\n");
2722 		goto err_out_free_netdev;
2723 	}
2724 
2725 	err = pci_request_regions(pdev, DRV_NAME);
2726 	if (err) {
2727 		dev_err(dev, "Cannot request PCI regions, aborting\n");
2728 		goto err_out_disable_device;
2729 	}
2730 
2731 	pci_set_master(pdev);
2732 
2733 	/* Query PCI controller on system for DMA addressing
2734 	 * limitation for the device.  Try 47-bit first, and
2735 	 * fail to 32-bit.
2736 	 */
2737 
2738 	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(47));
2739 	if (err) {
2740 		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2741 		if (err) {
2742 			dev_err(dev, "No usable DMA configuration, aborting\n");
2743 			goto err_out_release_regions;
2744 		}
2745 		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
2746 		if (err) {
2747 			dev_err(dev, "Unable to obtain %u-bit DMA "
2748 				"for consistent allocations, aborting\n", 32);
2749 			goto err_out_release_regions;
2750 		}
2751 	} else {
2752 		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(47));
2753 		if (err) {
2754 			dev_err(dev, "Unable to obtain %u-bit DMA "
2755 				"for consistent allocations, aborting\n", 47);
2756 			goto err_out_release_regions;
2757 		}
2758 		using_dac = 1;
2759 	}
2760 
2761 	/* Map vNIC resources from BAR0-5
2762 	 */
2763 
2764 	for (i = 0; i < ARRAY_SIZE(enic->bar); i++) {
2765 		if (!(pci_resource_flags(pdev, i) & IORESOURCE_MEM))
2766 			continue;
2767 		enic->bar[i].len = pci_resource_len(pdev, i);
2768 		enic->bar[i].vaddr = pci_iomap(pdev, i, enic->bar[i].len);
2769 		if (!enic->bar[i].vaddr) {
2770 			dev_err(dev, "Cannot memory-map BAR %d, aborting\n", i);
2771 			err = -ENODEV;
2772 			goto err_out_iounmap;
2773 		}
2774 		enic->bar[i].bus_addr = pci_resource_start(pdev, i);
2775 	}
2776 
2777 	/* Register vNIC device
2778 	 */
2779 
2780 	enic->vdev = vnic_dev_register(NULL, enic, pdev, enic->bar,
2781 		ARRAY_SIZE(enic->bar));
2782 	if (!enic->vdev) {
2783 		dev_err(dev, "vNIC registration failed, aborting\n");
2784 		err = -ENODEV;
2785 		goto err_out_iounmap;
2786 	}
2787 
2788 	err = vnic_devcmd_init(enic->vdev);
2789 
2790 	if (err)
2791 		goto err_out_vnic_unregister;
2792 
2793 #ifdef CONFIG_PCI_IOV
2794 	/* Get number of subvnics */
2795 	pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_SRIOV);
2796 	if (pos) {
2797 		pci_read_config_word(pdev, pos + PCI_SRIOV_TOTAL_VF,
2798 			&enic->num_vfs);
2799 		if (enic->num_vfs) {
2800 			err = pci_enable_sriov(pdev, enic->num_vfs);
2801 			if (err) {
2802 				dev_err(dev, "SRIOV enable failed, aborting."
2803 					" pci_enable_sriov() returned %d\n",
2804 					err);
2805 				goto err_out_vnic_unregister;
2806 			}
2807 			enic->priv_flags |= ENIC_SRIOV_ENABLED;
2808 			num_pps = enic->num_vfs;
2809 		}
2810 	}
2811 #endif
2812 
2813 	/* Allocate structure for port profiles */
2814 	enic->pp = kcalloc(num_pps, sizeof(*enic->pp), GFP_KERNEL);
2815 	if (!enic->pp) {
2816 		err = -ENOMEM;
2817 		goto err_out_disable_sriov_pp;
2818 	}
2819 
2820 	/* Issue device open to get device in known state
2821 	 */
2822 
2823 	err = enic_dev_open(enic);
2824 	if (err) {
2825 		dev_err(dev, "vNIC dev open failed, aborting\n");
2826 		goto err_out_disable_sriov;
2827 	}
2828 
2829 	/* Setup devcmd lock
2830 	 */
2831 
2832 	spin_lock_init(&enic->devcmd_lock);
2833 	spin_lock_init(&enic->enic_api_lock);
2834 
2835 	/*
2836 	 * Set ingress vlan rewrite mode before vnic initialization
2837 	 */
2838 
2839 	err = enic_dev_set_ig_vlan_rewrite_mode(enic);
2840 	if (err) {
2841 		dev_err(dev,
2842 			"Failed to set ingress vlan rewrite mode, aborting.\n");
2843 		goto err_out_dev_close;
2844 	}
2845 
2846 	/* Issue device init to initialize the vnic-to-switch link.
2847 	 * We'll start with carrier off and wait for link UP
2848 	 * notification later to turn on carrier.  We don't need
2849 	 * to wait here for the vnic-to-switch link initialization
2850 	 * to complete; link UP notification is the indication that
2851 	 * the process is complete.
2852 	 */
2853 
2854 	netif_carrier_off(netdev);
2855 
2856 	/* Do not call dev_init for a dynamic vnic.
2857 	 * For a dynamic vnic, init_prov_info will be
2858 	 * called later by an upper layer.
2859 	 */
2860 
2861 	if (!enic_is_dynamic(enic)) {
2862 		err = vnic_dev_init(enic->vdev, 0);
2863 		if (err) {
2864 			dev_err(dev, "vNIC dev init failed, aborting\n");
2865 			goto err_out_dev_close;
2866 		}
2867 	}
2868 
2869 	err = enic_dev_init(enic);
2870 	if (err) {
2871 		dev_err(dev, "Device initialization failed, aborting\n");
2872 		goto err_out_dev_close;
2873 	}
2874 
2875 	netif_set_real_num_tx_queues(netdev, enic->wq_count);
2876 	netif_set_real_num_rx_queues(netdev, enic->rq_count);
2877 
2878 	/* Setup notification timer, HW reset task, and wq locks
2879 	 */
2880 
2881 	timer_setup(&enic->notify_timer, enic_notify_timer, 0);
2882 
2883 	enic_rfs_flw_tbl_init(enic);
2884 	enic_set_rx_coal_setting(enic);
2885 	INIT_WORK(&enic->reset, enic_reset);
2886 	INIT_WORK(&enic->tx_hang_reset, enic_tx_hang_reset);
2887 	INIT_WORK(&enic->change_mtu_work, enic_change_mtu_work);
2888 
2889 	for (i = 0; i < enic->wq_count; i++)
2890 		spin_lock_init(&enic->wq_lock[i]);
2891 
2892 	/* Register net device
2893 	 */
2894 
2895 	enic->port_mtu = enic->config.mtu;
2896 
2897 	err = enic_set_mac_addr(netdev, enic->mac_addr);
2898 	if (err) {
2899 		dev_err(dev, "Invalid MAC address, aborting\n");
2900 		goto err_out_dev_deinit;
2901 	}
2902 
2903 	enic->tx_coalesce_usecs = enic->config.intr_timer_usec;
2904 	/* rx coalesce time already got initialized. This gets used
2905 	 * if adaptive coal is turned off
2906 	 */
2907 	enic->rx_coalesce_usecs = enic->tx_coalesce_usecs;
2908 
2909 	if (enic_is_dynamic(enic) || enic_is_sriov_vf(enic))
2910 		netdev->netdev_ops = &enic_netdev_dynamic_ops;
2911 	else
2912 		netdev->netdev_ops = &enic_netdev_ops;
2913 
2914 	netdev->watchdog_timeo = 2 * HZ;
2915 	enic_set_ethtool_ops(netdev);
2916 
2917 	netdev->features |= NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX;
2918 	if (ENIC_SETTING(enic, LOOP)) {
2919 		netdev->features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2920 		enic->loop_enable = 1;
2921 		enic->loop_tag = enic->config.loop_tag;
2922 		dev_info(dev, "loopback tag=0x%04x\n", enic->loop_tag);
2923 	}
2924 	if (ENIC_SETTING(enic, TXCSUM))
2925 		netdev->hw_features |= NETIF_F_SG | NETIF_F_HW_CSUM;
2926 	if (ENIC_SETTING(enic, TSO))
2927 		netdev->hw_features |= NETIF_F_TSO |
2928 			NETIF_F_TSO6 | NETIF_F_TSO_ECN;
2929 	if (ENIC_SETTING(enic, RSS))
2930 		netdev->hw_features |= NETIF_F_RXHASH;
2931 	if (ENIC_SETTING(enic, RXCSUM))
2932 		netdev->hw_features |= NETIF_F_RXCSUM;
2933 	if (ENIC_SETTING(enic, VXLAN)) {
2934 		u64 patch_level;
2935 		u64 a1 = 0;
2936 
2937 		netdev->hw_enc_features |= NETIF_F_RXCSUM		|
2938 					   NETIF_F_TSO			|
2939 					   NETIF_F_TSO6			|
2940 					   NETIF_F_TSO_ECN		|
2941 					   NETIF_F_GSO_UDP_TUNNEL	|
2942 					   NETIF_F_HW_CSUM		|
2943 					   NETIF_F_GSO_UDP_TUNNEL_CSUM;
2944 		netdev->hw_features |= netdev->hw_enc_features;
2945 		/* get bit mask from hw about supported offload bit level
2946 		 * BIT(0) = fw supports patch_level 0
2947 		 *	    fcoe bit = encap
2948 		 *	    fcoe_fc_crc_ok = outer csum ok
2949 		 * BIT(1) = always set by fw
2950 		 * BIT(2) = fw supports patch_level 2
2951 		 *	    BIT(0) in rss_hash = encap
2952 		 *	    BIT(1,2) in rss_hash = outer_ip_csum_ok/
2953 		 *				   outer_tcp_csum_ok
2954 		 * used in enic_rq_indicate_buf
2955 		 */
2956 		err = vnic_dev_get_supported_feature_ver(enic->vdev,
2957 							 VIC_FEATURE_VXLAN,
2958 							 &patch_level, &a1);
2959 		if (err)
2960 			patch_level = 0;
2961 		enic->vxlan.flags = (u8)a1;
2962 		/* mask bits that are supported by driver
2963 		 */
2964 		patch_level &= BIT_ULL(0) | BIT_ULL(2);
2965 		patch_level = fls(patch_level);
2966 		patch_level = patch_level ? patch_level - 1 : 0;
2967 		enic->vxlan.patch_level = patch_level;
2968 	}
2969 
2970 	netdev->features |= netdev->hw_features;
2971 	netdev->vlan_features |= netdev->features;
2972 
2973 #ifdef CONFIG_RFS_ACCEL
2974 	netdev->hw_features |= NETIF_F_NTUPLE;
2975 #endif
2976 
2977 	if (using_dac)
2978 		netdev->features |= NETIF_F_HIGHDMA;
2979 
2980 	netdev->priv_flags |= IFF_UNICAST_FLT;
2981 
2982 	/* MTU range: 68 - 9000 */
2983 	netdev->min_mtu = ENIC_MIN_MTU;
2984 	netdev->max_mtu = ENIC_MAX_MTU;
2985 	netdev->mtu	= enic->port_mtu;
2986 
2987 	err = register_netdev(netdev);
2988 	if (err) {
2989 		dev_err(dev, "Cannot register net device, aborting\n");
2990 		goto err_out_dev_deinit;
2991 	}
2992 	enic->rx_copybreak = RX_COPYBREAK_DEFAULT;
2993 
2994 	return 0;
2995 
2996 err_out_dev_deinit:
2997 	enic_dev_deinit(enic);
2998 err_out_dev_close:
2999 	vnic_dev_close(enic->vdev);
3000 err_out_disable_sriov:
3001 	kfree(enic->pp);
3002 err_out_disable_sriov_pp:
3003 #ifdef CONFIG_PCI_IOV
3004 	if (enic_sriov_enabled(enic)) {
3005 		pci_disable_sriov(pdev);
3006 		enic->priv_flags &= ~ENIC_SRIOV_ENABLED;
3007 	}
3008 #endif
3009 err_out_vnic_unregister:
3010 	vnic_dev_unregister(enic->vdev);
3011 err_out_iounmap:
3012 	enic_iounmap(enic);
3013 err_out_release_regions:
3014 	pci_release_regions(pdev);
3015 err_out_disable_device:
3016 	pci_disable_device(pdev);
3017 err_out_free_netdev:
3018 	free_netdev(netdev);
3019 
3020 	return err;
3021 }
3022 
3023 static void enic_remove(struct pci_dev *pdev)
3024 {
3025 	struct net_device *netdev = pci_get_drvdata(pdev);
3026 
3027 	if (netdev) {
3028 		struct enic *enic = netdev_priv(netdev);
3029 
3030 		cancel_work_sync(&enic->reset);
3031 		cancel_work_sync(&enic->change_mtu_work);
3032 		unregister_netdev(netdev);
3033 		enic_dev_deinit(enic);
3034 		vnic_dev_close(enic->vdev);
3035 #ifdef CONFIG_PCI_IOV
3036 		if (enic_sriov_enabled(enic)) {
3037 			pci_disable_sriov(pdev);
3038 			enic->priv_flags &= ~ENIC_SRIOV_ENABLED;
3039 		}
3040 #endif
3041 		kfree(enic->pp);
3042 		vnic_dev_unregister(enic->vdev);
3043 		enic_iounmap(enic);
3044 		pci_release_regions(pdev);
3045 		pci_disable_device(pdev);
3046 		free_netdev(netdev);
3047 	}
3048 }
3049 
3050 static struct pci_driver enic_driver = {
3051 	.name = DRV_NAME,
3052 	.id_table = enic_id_table,
3053 	.probe = enic_probe,
3054 	.remove = enic_remove,
3055 };
3056 
3057 static int __init enic_init_module(void)
3058 {
3059 	pr_info("%s, ver %s\n", DRV_DESCRIPTION, DRV_VERSION);
3060 
3061 	return pci_register_driver(&enic_driver);
3062 }
3063 
3064 static void __exit enic_cleanup_module(void)
3065 {
3066 	pci_unregister_driver(&enic_driver);
3067 }
3068 
3069 module_init(enic_init_module);
3070 module_exit(enic_cleanup_module);
3071