1 /*
2  * Linux driver for VMware's vmxnet3 ethernet NIC.
3  *
4  * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License as published by the
8  * Free Software Foundation; version 2 of the License and no later version.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13  * NON INFRINGEMENT. See the GNU General Public License for more
14  * details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19  *
20  * The full GNU General Public License is included in this distribution in
21  * the file called "COPYING".
22  *
23  * Maintained by: pv-drivers@vmware.com
24  *
25  */
26 
27 #include <linux/module.h>
28 #include <net/ip6_checksum.h>
29 
30 #include "vmxnet3_int.h"
31 
32 char vmxnet3_driver_name[] = "vmxnet3";
33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
34 
35 /*
36  * PCI Device ID Table
37  * Last entry must be all 0s
38  */
39 static const struct pci_device_id vmxnet3_pciid_table[] = {
40 	{PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
41 	{0}
42 };
43 
44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
45 
46 static int enable_mq = 1;
47 
48 static void
49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac);
50 
51 /*
52  *    Enable/Disable the given intr
53  */
54 static void
55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
56 {
57 	VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
58 }
59 
60 
61 static void
62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
63 {
64 	VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
65 }
66 
67 
68 /*
69  *    Enable/Disable all intrs used by the device
70  */
71 static void
72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
73 {
74 	int i;
75 
76 	for (i = 0; i < adapter->intr.num_intrs; i++)
77 		vmxnet3_enable_intr(adapter, i);
78 	adapter->shared->devRead.intrConf.intrCtrl &=
79 					cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
80 }
81 
82 
83 static void
84 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
85 {
86 	int i;
87 
88 	adapter->shared->devRead.intrConf.intrCtrl |=
89 					cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
90 	for (i = 0; i < adapter->intr.num_intrs; i++)
91 		vmxnet3_disable_intr(adapter, i);
92 }
93 
94 
95 static void
96 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
97 {
98 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
99 }
100 
101 
102 static bool
103 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
104 {
105 	return tq->stopped;
106 }
107 
108 
109 static void
110 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
111 {
112 	tq->stopped = false;
113 	netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
114 }
115 
116 
117 static void
118 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
119 {
120 	tq->stopped = false;
121 	netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
122 }
123 
124 
125 static void
126 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
127 {
128 	tq->stopped = true;
129 	tq->num_stop++;
130 	netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
131 }
132 
133 /* Check if capability is supported by UPT device or
134  * UPT is even requested
135  */
136 bool
137 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap)
138 {
139 	if (cap_supported & (1UL << VMXNET3_DCR_ERROR) ||
140 	    cap_supported & (1UL << cap)) {
141 		return true;
142 	}
143 
144 	return false;
145 }
146 
147 
148 /*
149  * Check the link state. This may start or stop the tx queue.
150  */
151 static void
152 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
153 {
154 	u32 ret;
155 	int i;
156 	unsigned long flags;
157 
158 	spin_lock_irqsave(&adapter->cmd_lock, flags);
159 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
160 	ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
161 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
162 
163 	adapter->link_speed = ret >> 16;
164 	if (ret & 1) { /* Link is up. */
165 		netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
166 			    adapter->link_speed);
167 		netif_carrier_on(adapter->netdev);
168 
169 		if (affectTxQueue) {
170 			for (i = 0; i < adapter->num_tx_queues; i++)
171 				vmxnet3_tq_start(&adapter->tx_queue[i],
172 						 adapter);
173 		}
174 	} else {
175 		netdev_info(adapter->netdev, "NIC Link is Down\n");
176 		netif_carrier_off(adapter->netdev);
177 
178 		if (affectTxQueue) {
179 			for (i = 0; i < adapter->num_tx_queues; i++)
180 				vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
181 		}
182 	}
183 }
184 
185 static void
186 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
187 {
188 	int i;
189 	unsigned long flags;
190 	u32 events = le32_to_cpu(adapter->shared->ecr);
191 	if (!events)
192 		return;
193 
194 	vmxnet3_ack_events(adapter, events);
195 
196 	/* Check if link state has changed */
197 	if (events & VMXNET3_ECR_LINK)
198 		vmxnet3_check_link(adapter, true);
199 
200 	/* Check if there is an error on xmit/recv queues */
201 	if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
202 		spin_lock_irqsave(&adapter->cmd_lock, flags);
203 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
204 				       VMXNET3_CMD_GET_QUEUE_STATUS);
205 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
206 
207 		for (i = 0; i < adapter->num_tx_queues; i++)
208 			if (adapter->tqd_start[i].status.stopped)
209 				dev_err(&adapter->netdev->dev,
210 					"%s: tq[%d] error 0x%x\n",
211 					adapter->netdev->name, i, le32_to_cpu(
212 					adapter->tqd_start[i].status.error));
213 		for (i = 0; i < adapter->num_rx_queues; i++)
214 			if (adapter->rqd_start[i].status.stopped)
215 				dev_err(&adapter->netdev->dev,
216 					"%s: rq[%d] error 0x%x\n",
217 					adapter->netdev->name, i,
218 					adapter->rqd_start[i].status.error);
219 
220 		schedule_work(&adapter->work);
221 	}
222 }
223 
224 #ifdef __BIG_ENDIAN_BITFIELD
225 /*
226  * The device expects the bitfields in shared structures to be written in
227  * little endian. When CPU is big endian, the following routines are used to
228  * correctly read and write into ABI.
229  * The general technique used here is : double word bitfields are defined in
230  * opposite order for big endian architecture. Then before reading them in
231  * driver the complete double word is translated using le32_to_cpu. Similarly
232  * After the driver writes into bitfields, cpu_to_le32 is used to translate the
233  * double words into required format.
234  * In order to avoid touching bits in shared structure more than once, temporary
235  * descriptors are used. These are passed as srcDesc to following functions.
236  */
237 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
238 				struct Vmxnet3_RxDesc *dstDesc)
239 {
240 	u32 *src = (u32 *)srcDesc + 2;
241 	u32 *dst = (u32 *)dstDesc + 2;
242 	dstDesc->addr = le64_to_cpu(srcDesc->addr);
243 	*dst = le32_to_cpu(*src);
244 	dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
245 }
246 
247 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
248 			       struct Vmxnet3_TxDesc *dstDesc)
249 {
250 	int i;
251 	u32 *src = (u32 *)(srcDesc + 1);
252 	u32 *dst = (u32 *)(dstDesc + 1);
253 
254 	/* Working backwards so that the gen bit is set at the end. */
255 	for (i = 2; i > 0; i--) {
256 		src--;
257 		dst--;
258 		*dst = cpu_to_le32(*src);
259 	}
260 }
261 
262 
263 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
264 				struct Vmxnet3_RxCompDesc *dstDesc)
265 {
266 	int i = 0;
267 	u32 *src = (u32 *)srcDesc;
268 	u32 *dst = (u32 *)dstDesc;
269 	for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
270 		*dst = le32_to_cpu(*src);
271 		src++;
272 		dst++;
273 	}
274 }
275 
276 
277 /* Used to read bitfield values from double words. */
278 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
279 {
280 	u32 temp = le32_to_cpu(*bitfield);
281 	u32 mask = ((1 << size) - 1) << pos;
282 	temp &= mask;
283 	temp >>= pos;
284 	return temp;
285 }
286 
287 
288 
289 #endif  /* __BIG_ENDIAN_BITFIELD */
290 
291 #ifdef __BIG_ENDIAN_BITFIELD
292 
293 #   define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
294 			txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
295 			VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
296 #   define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
297 			txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
298 			VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
299 #   define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
300 			VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
301 			VMXNET3_TCD_GEN_SIZE)
302 #   define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
303 			VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
304 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
305 			(dstrcd) = (tmp); \
306 			vmxnet3_RxCompToCPU((rcd), (tmp)); \
307 		} while (0)
308 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
309 			(dstrxd) = (tmp); \
310 			vmxnet3_RxDescToCPU((rxd), (tmp)); \
311 		} while (0)
312 
313 #else
314 
315 #   define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
316 #   define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
317 #   define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
318 #   define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
319 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
320 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
321 
322 #endif /* __BIG_ENDIAN_BITFIELD  */
323 
324 
325 static void
326 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
327 		     struct pci_dev *pdev)
328 {
329 	if (tbi->map_type == VMXNET3_MAP_SINGLE)
330 		dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
331 				 DMA_TO_DEVICE);
332 	else if (tbi->map_type == VMXNET3_MAP_PAGE)
333 		dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
334 			       DMA_TO_DEVICE);
335 	else
336 		BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
337 
338 	tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
339 }
340 
341 
342 static int
343 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
344 		  struct pci_dev *pdev,	struct vmxnet3_adapter *adapter)
345 {
346 	struct sk_buff *skb;
347 	int entries = 0;
348 
349 	/* no out of order completion */
350 	BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
351 	BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
352 
353 	skb = tq->buf_info[eop_idx].skb;
354 	BUG_ON(skb == NULL);
355 	tq->buf_info[eop_idx].skb = NULL;
356 
357 	VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
358 
359 	while (tq->tx_ring.next2comp != eop_idx) {
360 		vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
361 				     pdev);
362 
363 		/* update next2comp w/o tx_lock. Since we are marking more,
364 		 * instead of less, tx ring entries avail, the worst case is
365 		 * that the tx routine incorrectly re-queues a pkt due to
366 		 * insufficient tx ring entries.
367 		 */
368 		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
369 		entries++;
370 	}
371 
372 	dev_kfree_skb_any(skb);
373 	return entries;
374 }
375 
376 
377 static int
378 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
379 			struct vmxnet3_adapter *adapter)
380 {
381 	int completed = 0;
382 	union Vmxnet3_GenericDesc *gdesc;
383 
384 	gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
385 	while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
386 		/* Prevent any &gdesc->tcd field from being (speculatively)
387 		 * read before (&gdesc->tcd)->gen is read.
388 		 */
389 		dma_rmb();
390 
391 		completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
392 					       &gdesc->tcd), tq, adapter->pdev,
393 					       adapter);
394 
395 		vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
396 		gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
397 	}
398 
399 	if (completed) {
400 		spin_lock(&tq->tx_lock);
401 		if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
402 			     vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
403 			     VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
404 			     netif_carrier_ok(adapter->netdev))) {
405 			vmxnet3_tq_wake(tq, adapter);
406 		}
407 		spin_unlock(&tq->tx_lock);
408 	}
409 	return completed;
410 }
411 
412 
413 static void
414 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
415 		   struct vmxnet3_adapter *adapter)
416 {
417 	int i;
418 
419 	while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
420 		struct vmxnet3_tx_buf_info *tbi;
421 
422 		tbi = tq->buf_info + tq->tx_ring.next2comp;
423 
424 		vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
425 		if (tbi->skb) {
426 			dev_kfree_skb_any(tbi->skb);
427 			tbi->skb = NULL;
428 		}
429 		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
430 	}
431 
432 	/* sanity check, verify all buffers are indeed unmapped and freed */
433 	for (i = 0; i < tq->tx_ring.size; i++) {
434 		BUG_ON(tq->buf_info[i].skb != NULL ||
435 		       tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
436 	}
437 
438 	tq->tx_ring.gen = VMXNET3_INIT_GEN;
439 	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
440 
441 	tq->comp_ring.gen = VMXNET3_INIT_GEN;
442 	tq->comp_ring.next2proc = 0;
443 }
444 
445 
446 static void
447 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
448 		   struct vmxnet3_adapter *adapter)
449 {
450 	if (tq->tx_ring.base) {
451 		dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
452 				  sizeof(struct Vmxnet3_TxDesc),
453 				  tq->tx_ring.base, tq->tx_ring.basePA);
454 		tq->tx_ring.base = NULL;
455 	}
456 	if (tq->data_ring.base) {
457 		dma_free_coherent(&adapter->pdev->dev,
458 				  tq->data_ring.size * tq->txdata_desc_size,
459 				  tq->data_ring.base, tq->data_ring.basePA);
460 		tq->data_ring.base = NULL;
461 	}
462 	if (tq->comp_ring.base) {
463 		dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
464 				  sizeof(struct Vmxnet3_TxCompDesc),
465 				  tq->comp_ring.base, tq->comp_ring.basePA);
466 		tq->comp_ring.base = NULL;
467 	}
468 	kfree(tq->buf_info);
469 	tq->buf_info = NULL;
470 }
471 
472 
473 /* Destroy all tx queues */
474 void
475 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
476 {
477 	int i;
478 
479 	for (i = 0; i < adapter->num_tx_queues; i++)
480 		vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
481 }
482 
483 
484 static void
485 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
486 		struct vmxnet3_adapter *adapter)
487 {
488 	int i;
489 
490 	/* reset the tx ring contents to 0 and reset the tx ring states */
491 	memset(tq->tx_ring.base, 0, tq->tx_ring.size *
492 	       sizeof(struct Vmxnet3_TxDesc));
493 	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
494 	tq->tx_ring.gen = VMXNET3_INIT_GEN;
495 
496 	memset(tq->data_ring.base, 0,
497 	       tq->data_ring.size * tq->txdata_desc_size);
498 
499 	/* reset the tx comp ring contents to 0 and reset comp ring states */
500 	memset(tq->comp_ring.base, 0, tq->comp_ring.size *
501 	       sizeof(struct Vmxnet3_TxCompDesc));
502 	tq->comp_ring.next2proc = 0;
503 	tq->comp_ring.gen = VMXNET3_INIT_GEN;
504 
505 	/* reset the bookkeeping data */
506 	memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
507 	for (i = 0; i < tq->tx_ring.size; i++)
508 		tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
509 
510 	/* stats are not reset */
511 }
512 
513 
514 static int
515 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
516 		  struct vmxnet3_adapter *adapter)
517 {
518 	BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
519 	       tq->comp_ring.base || tq->buf_info);
520 
521 	tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
522 			tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
523 			&tq->tx_ring.basePA, GFP_KERNEL);
524 	if (!tq->tx_ring.base) {
525 		netdev_err(adapter->netdev, "failed to allocate tx ring\n");
526 		goto err;
527 	}
528 
529 	tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
530 			tq->data_ring.size * tq->txdata_desc_size,
531 			&tq->data_ring.basePA, GFP_KERNEL);
532 	if (!tq->data_ring.base) {
533 		netdev_err(adapter->netdev, "failed to allocate tx data ring\n");
534 		goto err;
535 	}
536 
537 	tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
538 			tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
539 			&tq->comp_ring.basePA, GFP_KERNEL);
540 	if (!tq->comp_ring.base) {
541 		netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
542 		goto err;
543 	}
544 
545 	tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]),
546 				    GFP_KERNEL,
547 				    dev_to_node(&adapter->pdev->dev));
548 	if (!tq->buf_info)
549 		goto err;
550 
551 	return 0;
552 
553 err:
554 	vmxnet3_tq_destroy(tq, adapter);
555 	return -ENOMEM;
556 }
557 
558 static void
559 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
560 {
561 	int i;
562 
563 	for (i = 0; i < adapter->num_tx_queues; i++)
564 		vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
565 }
566 
567 /*
568  *    starting from ring->next2fill, allocate rx buffers for the given ring
569  *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
570  *    are allocated or allocation fails
571  */
572 
573 static int
574 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
575 			int num_to_alloc, struct vmxnet3_adapter *adapter)
576 {
577 	int num_allocated = 0;
578 	struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
579 	struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
580 	u32 val;
581 
582 	while (num_allocated <= num_to_alloc) {
583 		struct vmxnet3_rx_buf_info *rbi;
584 		union Vmxnet3_GenericDesc *gd;
585 
586 		rbi = rbi_base + ring->next2fill;
587 		gd = ring->base + ring->next2fill;
588 		rbi->comp_state = VMXNET3_RXD_COMP_PENDING;
589 
590 		if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
591 			if (rbi->skb == NULL) {
592 				rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
593 								       rbi->len,
594 								       GFP_KERNEL);
595 				if (unlikely(rbi->skb == NULL)) {
596 					rq->stats.rx_buf_alloc_failure++;
597 					break;
598 				}
599 
600 				rbi->dma_addr = dma_map_single(
601 						&adapter->pdev->dev,
602 						rbi->skb->data, rbi->len,
603 						DMA_FROM_DEVICE);
604 				if (dma_mapping_error(&adapter->pdev->dev,
605 						      rbi->dma_addr)) {
606 					dev_kfree_skb_any(rbi->skb);
607 					rbi->skb = NULL;
608 					rq->stats.rx_buf_alloc_failure++;
609 					break;
610 				}
611 			} else {
612 				/* rx buffer skipped by the device */
613 			}
614 			val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
615 		} else {
616 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
617 			       rbi->len  != PAGE_SIZE);
618 
619 			if (rbi->page == NULL) {
620 				rbi->page = alloc_page(GFP_ATOMIC);
621 				if (unlikely(rbi->page == NULL)) {
622 					rq->stats.rx_buf_alloc_failure++;
623 					break;
624 				}
625 				rbi->dma_addr = dma_map_page(
626 						&adapter->pdev->dev,
627 						rbi->page, 0, PAGE_SIZE,
628 						DMA_FROM_DEVICE);
629 				if (dma_mapping_error(&adapter->pdev->dev,
630 						      rbi->dma_addr)) {
631 					put_page(rbi->page);
632 					rbi->page = NULL;
633 					rq->stats.rx_buf_alloc_failure++;
634 					break;
635 				}
636 			} else {
637 				/* rx buffers skipped by the device */
638 			}
639 			val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
640 		}
641 
642 		gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
643 		gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
644 					   | val | rbi->len);
645 
646 		/* Fill the last buffer but dont mark it ready, or else the
647 		 * device will think that the queue is full */
648 		if (num_allocated == num_to_alloc) {
649 			rbi->comp_state = VMXNET3_RXD_COMP_DONE;
650 			break;
651 		}
652 
653 		gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
654 		num_allocated++;
655 		vmxnet3_cmd_ring_adv_next2fill(ring);
656 	}
657 
658 	netdev_dbg(adapter->netdev,
659 		"alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
660 		num_allocated, ring->next2fill, ring->next2comp);
661 
662 	/* so that the device can distinguish a full ring and an empty ring */
663 	BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
664 
665 	return num_allocated;
666 }
667 
668 
669 static void
670 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
671 		    struct vmxnet3_rx_buf_info *rbi)
672 {
673 	skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
674 
675 	BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
676 
677 	__skb_frag_set_page(frag, rbi->page);
678 	skb_frag_off_set(frag, 0);
679 	skb_frag_size_set(frag, rcd->len);
680 	skb->data_len += rcd->len;
681 	skb->truesize += PAGE_SIZE;
682 	skb_shinfo(skb)->nr_frags++;
683 }
684 
685 
686 static int
687 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
688 		struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
689 		struct vmxnet3_adapter *adapter)
690 {
691 	u32 dw2, len;
692 	unsigned long buf_offset;
693 	int i;
694 	union Vmxnet3_GenericDesc *gdesc;
695 	struct vmxnet3_tx_buf_info *tbi = NULL;
696 
697 	BUG_ON(ctx->copy_size > skb_headlen(skb));
698 
699 	/* use the previous gen bit for the SOP desc */
700 	dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
701 
702 	ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
703 	gdesc = ctx->sop_txd; /* both loops below can be skipped */
704 
705 	/* no need to map the buffer if headers are copied */
706 	if (ctx->copy_size) {
707 		ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
708 					tq->tx_ring.next2fill *
709 					tq->txdata_desc_size);
710 		ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
711 		ctx->sop_txd->dword[3] = 0;
712 
713 		tbi = tq->buf_info + tq->tx_ring.next2fill;
714 		tbi->map_type = VMXNET3_MAP_NONE;
715 
716 		netdev_dbg(adapter->netdev,
717 			"txd[%u]: 0x%Lx 0x%x 0x%x\n",
718 			tq->tx_ring.next2fill,
719 			le64_to_cpu(ctx->sop_txd->txd.addr),
720 			ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
721 		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
722 
723 		/* use the right gen for non-SOP desc */
724 		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
725 	}
726 
727 	/* linear part can use multiple tx desc if it's big */
728 	len = skb_headlen(skb) - ctx->copy_size;
729 	buf_offset = ctx->copy_size;
730 	while (len) {
731 		u32 buf_size;
732 
733 		if (len < VMXNET3_MAX_TX_BUF_SIZE) {
734 			buf_size = len;
735 			dw2 |= len;
736 		} else {
737 			buf_size = VMXNET3_MAX_TX_BUF_SIZE;
738 			/* spec says that for TxDesc.len, 0 == 2^14 */
739 		}
740 
741 		tbi = tq->buf_info + tq->tx_ring.next2fill;
742 		tbi->map_type = VMXNET3_MAP_SINGLE;
743 		tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
744 				skb->data + buf_offset, buf_size,
745 				DMA_TO_DEVICE);
746 		if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
747 			return -EFAULT;
748 
749 		tbi->len = buf_size;
750 
751 		gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
752 		BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
753 
754 		gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
755 		gdesc->dword[2] = cpu_to_le32(dw2);
756 		gdesc->dword[3] = 0;
757 
758 		netdev_dbg(adapter->netdev,
759 			"txd[%u]: 0x%Lx 0x%x 0x%x\n",
760 			tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
761 			le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
762 		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
763 		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
764 
765 		len -= buf_size;
766 		buf_offset += buf_size;
767 	}
768 
769 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
770 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
771 		u32 buf_size;
772 
773 		buf_offset = 0;
774 		len = skb_frag_size(frag);
775 		while (len) {
776 			tbi = tq->buf_info + tq->tx_ring.next2fill;
777 			if (len < VMXNET3_MAX_TX_BUF_SIZE) {
778 				buf_size = len;
779 				dw2 |= len;
780 			} else {
781 				buf_size = VMXNET3_MAX_TX_BUF_SIZE;
782 				/* spec says that for TxDesc.len, 0 == 2^14 */
783 			}
784 			tbi->map_type = VMXNET3_MAP_PAGE;
785 			tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
786 							 buf_offset, buf_size,
787 							 DMA_TO_DEVICE);
788 			if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
789 				return -EFAULT;
790 
791 			tbi->len = buf_size;
792 
793 			gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
794 			BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
795 
796 			gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
797 			gdesc->dword[2] = cpu_to_le32(dw2);
798 			gdesc->dword[3] = 0;
799 
800 			netdev_dbg(adapter->netdev,
801 				"txd[%u]: 0x%llx %u %u\n",
802 				tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
803 				le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
804 			vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
805 			dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
806 
807 			len -= buf_size;
808 			buf_offset += buf_size;
809 		}
810 	}
811 
812 	ctx->eop_txd = gdesc;
813 
814 	/* set the last buf_info for the pkt */
815 	tbi->skb = skb;
816 	tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
817 
818 	return 0;
819 }
820 
821 
822 /* Init all tx queues */
823 static void
824 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
825 {
826 	int i;
827 
828 	for (i = 0; i < adapter->num_tx_queues; i++)
829 		vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
830 }
831 
832 
833 /*
834  *    parse relevant protocol headers:
835  *      For a tso pkt, relevant headers are L2/3/4 including options
836  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
837  *      if it's a TCP/UDP pkt
838  *
839  * Returns:
840  *    -1:  error happens during parsing
841  *     0:  protocol headers parsed, but too big to be copied
842  *     1:  protocol headers parsed and copied
843  *
844  * Other effects:
845  *    1. related *ctx fields are updated.
846  *    2. ctx->copy_size is # of bytes copied
847  *    3. the portion to be copied is guaranteed to be in the linear part
848  *
849  */
850 static int
851 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
852 		  struct vmxnet3_tx_ctx *ctx,
853 		  struct vmxnet3_adapter *adapter)
854 {
855 	u8 protocol = 0;
856 
857 	if (ctx->mss) {	/* TSO */
858 		if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) {
859 			ctx->l4_offset = skb_inner_transport_offset(skb);
860 			ctx->l4_hdr_size = inner_tcp_hdrlen(skb);
861 			ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size;
862 		} else {
863 			ctx->l4_offset = skb_transport_offset(skb);
864 			ctx->l4_hdr_size = tcp_hdrlen(skb);
865 			ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size;
866 		}
867 	} else {
868 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
869 			/* For encap packets, skb_checksum_start_offset refers
870 			 * to inner L4 offset. Thus, below works for encap as
871 			 * well as non-encap case
872 			 */
873 			ctx->l4_offset = skb_checksum_start_offset(skb);
874 
875 			if (VMXNET3_VERSION_GE_4(adapter) &&
876 			    skb->encapsulation) {
877 				struct iphdr *iph = inner_ip_hdr(skb);
878 
879 				if (iph->version == 4) {
880 					protocol = iph->protocol;
881 				} else {
882 					const struct ipv6hdr *ipv6h;
883 
884 					ipv6h = inner_ipv6_hdr(skb);
885 					protocol = ipv6h->nexthdr;
886 				}
887 			} else {
888 				if (ctx->ipv4) {
889 					const struct iphdr *iph = ip_hdr(skb);
890 
891 					protocol = iph->protocol;
892 				} else if (ctx->ipv6) {
893 					const struct ipv6hdr *ipv6h;
894 
895 					ipv6h = ipv6_hdr(skb);
896 					protocol = ipv6h->nexthdr;
897 				}
898 			}
899 
900 			switch (protocol) {
901 			case IPPROTO_TCP:
902 				ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) :
903 						   tcp_hdrlen(skb);
904 				break;
905 			case IPPROTO_UDP:
906 				ctx->l4_hdr_size = sizeof(struct udphdr);
907 				break;
908 			default:
909 				ctx->l4_hdr_size = 0;
910 				break;
911 			}
912 
913 			ctx->copy_size = min(ctx->l4_offset +
914 					 ctx->l4_hdr_size, skb->len);
915 		} else {
916 			ctx->l4_offset = 0;
917 			ctx->l4_hdr_size = 0;
918 			/* copy as much as allowed */
919 			ctx->copy_size = min_t(unsigned int,
920 					       tq->txdata_desc_size,
921 					       skb_headlen(skb));
922 		}
923 
924 		if (skb->len <= VMXNET3_HDR_COPY_SIZE)
925 			ctx->copy_size = skb->len;
926 
927 		/* make sure headers are accessible directly */
928 		if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
929 			goto err;
930 	}
931 
932 	if (unlikely(ctx->copy_size > tq->txdata_desc_size)) {
933 		tq->stats.oversized_hdr++;
934 		ctx->copy_size = 0;
935 		return 0;
936 	}
937 
938 	return 1;
939 err:
940 	return -1;
941 }
942 
943 /*
944  *    copy relevant protocol headers to the transmit ring:
945  *      For a tso pkt, relevant headers are L2/3/4 including options
946  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
947  *      if it's a TCP/UDP pkt
948  *
949  *
950  *    Note that this requires that vmxnet3_parse_hdr be called first to set the
951  *      appropriate bits in ctx first
952  */
953 static void
954 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
955 		 struct vmxnet3_tx_ctx *ctx,
956 		 struct vmxnet3_adapter *adapter)
957 {
958 	struct Vmxnet3_TxDataDesc *tdd;
959 
960 	tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base +
961 					    tq->tx_ring.next2fill *
962 					    tq->txdata_desc_size);
963 
964 	memcpy(tdd->data, skb->data, ctx->copy_size);
965 	netdev_dbg(adapter->netdev,
966 		"copy %u bytes to dataRing[%u]\n",
967 		ctx->copy_size, tq->tx_ring.next2fill);
968 }
969 
970 
971 static void
972 vmxnet3_prepare_inner_tso(struct sk_buff *skb,
973 			  struct vmxnet3_tx_ctx *ctx)
974 {
975 	struct tcphdr *tcph = inner_tcp_hdr(skb);
976 	struct iphdr *iph = inner_ip_hdr(skb);
977 
978 	if (iph->version == 4) {
979 		iph->check = 0;
980 		tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
981 						 IPPROTO_TCP, 0);
982 	} else {
983 		struct ipv6hdr *iph = inner_ipv6_hdr(skb);
984 
985 		tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
986 					       IPPROTO_TCP, 0);
987 	}
988 }
989 
990 static void
991 vmxnet3_prepare_tso(struct sk_buff *skb,
992 		    struct vmxnet3_tx_ctx *ctx)
993 {
994 	struct tcphdr *tcph = tcp_hdr(skb);
995 
996 	if (ctx->ipv4) {
997 		struct iphdr *iph = ip_hdr(skb);
998 
999 		iph->check = 0;
1000 		tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
1001 						 IPPROTO_TCP, 0);
1002 	} else if (ctx->ipv6) {
1003 		tcp_v6_gso_csum_prep(skb);
1004 	}
1005 }
1006 
1007 static int txd_estimate(const struct sk_buff *skb)
1008 {
1009 	int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1010 	int i;
1011 
1012 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1013 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1014 
1015 		count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
1016 	}
1017 	return count;
1018 }
1019 
1020 /*
1021  * Transmits a pkt thru a given tq
1022  * Returns:
1023  *    NETDEV_TX_OK:      descriptors are setup successfully
1024  *    NETDEV_TX_OK:      error occurred, the pkt is dropped
1025  *    NETDEV_TX_BUSY:    tx ring is full, queue is stopped
1026  *
1027  * Side-effects:
1028  *    1. tx ring may be changed
1029  *    2. tq stats may be updated accordingly
1030  *    3. shared->txNumDeferred may be updated
1031  */
1032 
1033 static int
1034 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
1035 		struct vmxnet3_adapter *adapter, struct net_device *netdev)
1036 {
1037 	int ret;
1038 	u32 count;
1039 	int num_pkts;
1040 	int tx_num_deferred;
1041 	unsigned long flags;
1042 	struct vmxnet3_tx_ctx ctx;
1043 	union Vmxnet3_GenericDesc *gdesc;
1044 #ifdef __BIG_ENDIAN_BITFIELD
1045 	/* Use temporary descriptor to avoid touching bits multiple times */
1046 	union Vmxnet3_GenericDesc tempTxDesc;
1047 #endif
1048 
1049 	count = txd_estimate(skb);
1050 
1051 	ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
1052 	ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6));
1053 
1054 	ctx.mss = skb_shinfo(skb)->gso_size;
1055 	if (ctx.mss) {
1056 		if (skb_header_cloned(skb)) {
1057 			if (unlikely(pskb_expand_head(skb, 0, 0,
1058 						      GFP_ATOMIC) != 0)) {
1059 				tq->stats.drop_tso++;
1060 				goto drop_pkt;
1061 			}
1062 			tq->stats.copy_skb_header++;
1063 		}
1064 		if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) {
1065 			/* tso pkts must not use more than
1066 			 * VMXNET3_MAX_TSO_TXD_PER_PKT entries
1067 			 */
1068 			if (skb_linearize(skb) != 0) {
1069 				tq->stats.drop_too_many_frags++;
1070 				goto drop_pkt;
1071 			}
1072 			tq->stats.linearized++;
1073 
1074 			/* recalculate the # of descriptors to use */
1075 			count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1076 			if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) {
1077 				tq->stats.drop_too_many_frags++;
1078 				goto drop_pkt;
1079 			}
1080 		}
1081 		if (skb->encapsulation) {
1082 			vmxnet3_prepare_inner_tso(skb, &ctx);
1083 		} else {
1084 			vmxnet3_prepare_tso(skb, &ctx);
1085 		}
1086 	} else {
1087 		if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
1088 
1089 			/* non-tso pkts must not use more than
1090 			 * VMXNET3_MAX_TXD_PER_PKT entries
1091 			 */
1092 			if (skb_linearize(skb) != 0) {
1093 				tq->stats.drop_too_many_frags++;
1094 				goto drop_pkt;
1095 			}
1096 			tq->stats.linearized++;
1097 
1098 			/* recalculate the # of descriptors to use */
1099 			count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1100 		}
1101 	}
1102 
1103 	ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter);
1104 	if (ret >= 0) {
1105 		BUG_ON(ret <= 0 && ctx.copy_size != 0);
1106 		/* hdrs parsed, check against other limits */
1107 		if (ctx.mss) {
1108 			if (unlikely(ctx.l4_offset + ctx.l4_hdr_size >
1109 				     VMXNET3_MAX_TX_BUF_SIZE)) {
1110 				tq->stats.drop_oversized_hdr++;
1111 				goto drop_pkt;
1112 			}
1113 		} else {
1114 			if (skb->ip_summed == CHECKSUM_PARTIAL) {
1115 				if (unlikely(ctx.l4_offset +
1116 					     skb->csum_offset >
1117 					     VMXNET3_MAX_CSUM_OFFSET)) {
1118 					tq->stats.drop_oversized_hdr++;
1119 					goto drop_pkt;
1120 				}
1121 			}
1122 		}
1123 	} else {
1124 		tq->stats.drop_hdr_inspect_err++;
1125 		goto drop_pkt;
1126 	}
1127 
1128 	spin_lock_irqsave(&tq->tx_lock, flags);
1129 
1130 	if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
1131 		tq->stats.tx_ring_full++;
1132 		netdev_dbg(adapter->netdev,
1133 			"tx queue stopped on %s, next2comp %u"
1134 			" next2fill %u\n", adapter->netdev->name,
1135 			tq->tx_ring.next2comp, tq->tx_ring.next2fill);
1136 
1137 		vmxnet3_tq_stop(tq, adapter);
1138 		spin_unlock_irqrestore(&tq->tx_lock, flags);
1139 		return NETDEV_TX_BUSY;
1140 	}
1141 
1142 
1143 	vmxnet3_copy_hdr(skb, tq, &ctx, adapter);
1144 
1145 	/* fill tx descs related to addr & len */
1146 	if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter))
1147 		goto unlock_drop_pkt;
1148 
1149 	/* setup the EOP desc */
1150 	ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1151 
1152 	/* setup the SOP desc */
1153 #ifdef __BIG_ENDIAN_BITFIELD
1154 	gdesc = &tempTxDesc;
1155 	gdesc->dword[2] = ctx.sop_txd->dword[2];
1156 	gdesc->dword[3] = ctx.sop_txd->dword[3];
1157 #else
1158 	gdesc = ctx.sop_txd;
1159 #endif
1160 	tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred);
1161 	if (ctx.mss) {
1162 		if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) {
1163 			gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
1164 			if (VMXNET3_VERSION_GE_7(adapter)) {
1165 				gdesc->txd.om = VMXNET3_OM_TSO;
1166 				gdesc->txd.ext1 = 1;
1167 			} else {
1168 				gdesc->txd.om = VMXNET3_OM_ENCAP;
1169 			}
1170 			gdesc->txd.msscof = ctx.mss;
1171 
1172 			if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
1173 				gdesc->txd.oco = 1;
1174 		} else {
1175 			gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
1176 			gdesc->txd.om = VMXNET3_OM_TSO;
1177 			gdesc->txd.msscof = ctx.mss;
1178 		}
1179 		num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss;
1180 	} else {
1181 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
1182 			if (VMXNET3_VERSION_GE_4(adapter) &&
1183 			    skb->encapsulation) {
1184 				gdesc->txd.hlen = ctx.l4_offset +
1185 						  ctx.l4_hdr_size;
1186 				if (VMXNET3_VERSION_GE_7(adapter)) {
1187 					gdesc->txd.om = VMXNET3_OM_CSUM;
1188 					gdesc->txd.msscof = ctx.l4_offset +
1189 							    skb->csum_offset;
1190 					gdesc->txd.ext1 = 1;
1191 				} else {
1192 					gdesc->txd.om = VMXNET3_OM_ENCAP;
1193 					gdesc->txd.msscof = 0;		/* Reserved */
1194 				}
1195 			} else {
1196 				gdesc->txd.hlen = ctx.l4_offset;
1197 				gdesc->txd.om = VMXNET3_OM_CSUM;
1198 				gdesc->txd.msscof = ctx.l4_offset +
1199 						    skb->csum_offset;
1200 			}
1201 		} else {
1202 			gdesc->txd.om = 0;
1203 			gdesc->txd.msscof = 0;
1204 		}
1205 		num_pkts = 1;
1206 	}
1207 	le32_add_cpu(&tq->shared->txNumDeferred, num_pkts);
1208 	tx_num_deferred += num_pkts;
1209 
1210 	if (skb_vlan_tag_present(skb)) {
1211 		gdesc->txd.ti = 1;
1212 		gdesc->txd.tci = skb_vlan_tag_get(skb);
1213 	}
1214 
1215 	/* Ensure that the write to (&gdesc->txd)->gen will be observed after
1216 	 * all other writes to &gdesc->txd.
1217 	 */
1218 	dma_wmb();
1219 
1220 	/* finally flips the GEN bit of the SOP desc. */
1221 	gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1222 						  VMXNET3_TXD_GEN);
1223 #ifdef __BIG_ENDIAN_BITFIELD
1224 	/* Finished updating in bitfields of Tx Desc, so write them in original
1225 	 * place.
1226 	 */
1227 	vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1228 			   (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1229 	gdesc = ctx.sop_txd;
1230 #endif
1231 	netdev_dbg(adapter->netdev,
1232 		"txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1233 		(u32)(ctx.sop_txd -
1234 		tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1235 		le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1236 
1237 	spin_unlock_irqrestore(&tq->tx_lock, flags);
1238 
1239 	if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) {
1240 		tq->shared->txNumDeferred = 0;
1241 		VMXNET3_WRITE_BAR0_REG(adapter,
1242 				       adapter->tx_prod_offset + tq->qid * 8,
1243 				       tq->tx_ring.next2fill);
1244 	}
1245 
1246 	return NETDEV_TX_OK;
1247 
1248 unlock_drop_pkt:
1249 	spin_unlock_irqrestore(&tq->tx_lock, flags);
1250 drop_pkt:
1251 	tq->stats.drop_total++;
1252 	dev_kfree_skb_any(skb);
1253 	return NETDEV_TX_OK;
1254 }
1255 
1256 
1257 static netdev_tx_t
1258 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1259 {
1260 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1261 
1262 	BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1263 	return vmxnet3_tq_xmit(skb,
1264 			       &adapter->tx_queue[skb->queue_mapping],
1265 			       adapter, netdev);
1266 }
1267 
1268 
1269 static void
1270 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1271 		struct sk_buff *skb,
1272 		union Vmxnet3_GenericDesc *gdesc)
1273 {
1274 	if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1275 		if (gdesc->rcd.v4 &&
1276 		    (le32_to_cpu(gdesc->dword[3]) &
1277 		     VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) {
1278 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1279 			WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
1280 				     !(le32_to_cpu(gdesc->dword[0]) &
1281 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1282 			WARN_ON_ONCE(gdesc->rcd.frg &&
1283 				     !(le32_to_cpu(gdesc->dword[0]) &
1284 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1285 		} else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) &
1286 					     (1 << VMXNET3_RCD_TUC_SHIFT))) {
1287 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1288 			WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
1289 				     !(le32_to_cpu(gdesc->dword[0]) &
1290 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1291 			WARN_ON_ONCE(gdesc->rcd.frg &&
1292 				     !(le32_to_cpu(gdesc->dword[0]) &
1293 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1294 		} else {
1295 			if (gdesc->rcd.csum) {
1296 				skb->csum = htons(gdesc->rcd.csum);
1297 				skb->ip_summed = CHECKSUM_PARTIAL;
1298 			} else {
1299 				skb_checksum_none_assert(skb);
1300 			}
1301 		}
1302 	} else {
1303 		skb_checksum_none_assert(skb);
1304 	}
1305 }
1306 
1307 
1308 static void
1309 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1310 		 struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
1311 {
1312 	rq->stats.drop_err++;
1313 	if (!rcd->fcs)
1314 		rq->stats.drop_fcs++;
1315 
1316 	rq->stats.drop_total++;
1317 
1318 	/*
1319 	 * We do not unmap and chain the rx buffer to the skb.
1320 	 * We basically pretend this buffer is not used and will be recycled
1321 	 * by vmxnet3_rq_alloc_rx_buf()
1322 	 */
1323 
1324 	/*
1325 	 * ctx->skb may be NULL if this is the first and the only one
1326 	 * desc for the pkt
1327 	 */
1328 	if (ctx->skb)
1329 		dev_kfree_skb_irq(ctx->skb);
1330 
1331 	ctx->skb = NULL;
1332 }
1333 
1334 
1335 static u32
1336 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
1337 		    union Vmxnet3_GenericDesc *gdesc)
1338 {
1339 	u32 hlen, maplen;
1340 	union {
1341 		void *ptr;
1342 		struct ethhdr *eth;
1343 		struct vlan_ethhdr *veth;
1344 		struct iphdr *ipv4;
1345 		struct ipv6hdr *ipv6;
1346 		struct tcphdr *tcp;
1347 	} hdr;
1348 	BUG_ON(gdesc->rcd.tcp == 0);
1349 
1350 	maplen = skb_headlen(skb);
1351 	if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen))
1352 		return 0;
1353 
1354 	if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
1355 	    skb->protocol == cpu_to_be16(ETH_P_8021AD))
1356 		hlen = sizeof(struct vlan_ethhdr);
1357 	else
1358 		hlen = sizeof(struct ethhdr);
1359 
1360 	hdr.eth = eth_hdr(skb);
1361 	if (gdesc->rcd.v4) {
1362 		BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) &&
1363 		       hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP));
1364 		hdr.ptr += hlen;
1365 		BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP);
1366 		hlen = hdr.ipv4->ihl << 2;
1367 		hdr.ptr += hdr.ipv4->ihl << 2;
1368 	} else if (gdesc->rcd.v6) {
1369 		BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) &&
1370 		       hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6));
1371 		hdr.ptr += hlen;
1372 		/* Use an estimated value, since we also need to handle
1373 		 * TSO case.
1374 		 */
1375 		if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1376 			return sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
1377 		hlen = sizeof(struct ipv6hdr);
1378 		hdr.ptr += sizeof(struct ipv6hdr);
1379 	} else {
1380 		/* Non-IP pkt, dont estimate header length */
1381 		return 0;
1382 	}
1383 
1384 	if (hlen + sizeof(struct tcphdr) > maplen)
1385 		return 0;
1386 
1387 	return (hlen + (hdr.tcp->doff << 2));
1388 }
1389 
1390 static int
1391 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1392 		       struct vmxnet3_adapter *adapter, int quota)
1393 {
1394 	u32 rxprod_reg[2] = {
1395 		adapter->rx_prod_offset, adapter->rx_prod2_offset
1396 	};
1397 	u32 num_pkts = 0;
1398 	bool skip_page_frags = false;
1399 	struct Vmxnet3_RxCompDesc *rcd;
1400 	struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1401 	u16 segCnt = 0, mss = 0;
1402 	int comp_offset, fill_offset;
1403 #ifdef __BIG_ENDIAN_BITFIELD
1404 	struct Vmxnet3_RxDesc rxCmdDesc;
1405 	struct Vmxnet3_RxCompDesc rxComp;
1406 #endif
1407 	vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1408 			  &rxComp);
1409 	while (rcd->gen == rq->comp_ring.gen) {
1410 		struct vmxnet3_rx_buf_info *rbi;
1411 		struct sk_buff *skb, *new_skb = NULL;
1412 		struct page *new_page = NULL;
1413 		dma_addr_t new_dma_addr;
1414 		int num_to_alloc;
1415 		struct Vmxnet3_RxDesc *rxd;
1416 		u32 idx, ring_idx;
1417 		struct vmxnet3_cmd_ring	*ring = NULL;
1418 		if (num_pkts >= quota) {
1419 			/* we may stop even before we see the EOP desc of
1420 			 * the current pkt
1421 			 */
1422 			break;
1423 		}
1424 
1425 		/* Prevent any rcd field from being (speculatively) read before
1426 		 * rcd->gen is read.
1427 		 */
1428 		dma_rmb();
1429 
1430 		BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 &&
1431 		       rcd->rqID != rq->dataRingQid);
1432 		idx = rcd->rxdIdx;
1433 		ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID);
1434 		ring = rq->rx_ring + ring_idx;
1435 		vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1436 				  &rxCmdDesc);
1437 		rbi = rq->buf_info[ring_idx] + idx;
1438 
1439 		BUG_ON(rxd->addr != rbi->dma_addr ||
1440 		       rxd->len != rbi->len);
1441 
1442 		if (unlikely(rcd->eop && rcd->err)) {
1443 			vmxnet3_rx_error(rq, rcd, ctx, adapter);
1444 			goto rcd_done;
1445 		}
1446 
1447 		if (rcd->sop) { /* first buf of the pkt */
1448 			bool rxDataRingUsed;
1449 			u16 len;
1450 
1451 			BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1452 			       (rcd->rqID != rq->qid &&
1453 				rcd->rqID != rq->dataRingQid));
1454 
1455 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1456 			BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1457 
1458 			if (unlikely(rcd->len == 0)) {
1459 				/* Pretend the rx buffer is skipped. */
1460 				BUG_ON(!(rcd->sop && rcd->eop));
1461 				netdev_dbg(adapter->netdev,
1462 					"rxRing[%u][%u] 0 length\n",
1463 					ring_idx, idx);
1464 				goto rcd_done;
1465 			}
1466 
1467 			skip_page_frags = false;
1468 			ctx->skb = rbi->skb;
1469 
1470 			rxDataRingUsed =
1471 				VMXNET3_RX_DATA_RING(adapter, rcd->rqID);
1472 			len = rxDataRingUsed ? rcd->len : rbi->len;
1473 			new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1474 							    len);
1475 			if (new_skb == NULL) {
1476 				/* Skb allocation failed, do not handover this
1477 				 * skb to stack. Reuse it. Drop the existing pkt
1478 				 */
1479 				rq->stats.rx_buf_alloc_failure++;
1480 				ctx->skb = NULL;
1481 				rq->stats.drop_total++;
1482 				skip_page_frags = true;
1483 				goto rcd_done;
1484 			}
1485 
1486 			if (rxDataRingUsed) {
1487 				size_t sz;
1488 
1489 				BUG_ON(rcd->len > rq->data_ring.desc_size);
1490 
1491 				ctx->skb = new_skb;
1492 				sz = rcd->rxdIdx * rq->data_ring.desc_size;
1493 				memcpy(new_skb->data,
1494 				       &rq->data_ring.base[sz], rcd->len);
1495 			} else {
1496 				ctx->skb = rbi->skb;
1497 
1498 				new_dma_addr =
1499 					dma_map_single(&adapter->pdev->dev,
1500 						       new_skb->data, rbi->len,
1501 						       DMA_FROM_DEVICE);
1502 				if (dma_mapping_error(&adapter->pdev->dev,
1503 						      new_dma_addr)) {
1504 					dev_kfree_skb(new_skb);
1505 					/* Skb allocation failed, do not
1506 					 * handover this skb to stack. Reuse
1507 					 * it. Drop the existing pkt.
1508 					 */
1509 					rq->stats.rx_buf_alloc_failure++;
1510 					ctx->skb = NULL;
1511 					rq->stats.drop_total++;
1512 					skip_page_frags = true;
1513 					goto rcd_done;
1514 				}
1515 
1516 				dma_unmap_single(&adapter->pdev->dev,
1517 						 rbi->dma_addr,
1518 						 rbi->len,
1519 						 DMA_FROM_DEVICE);
1520 
1521 				/* Immediate refill */
1522 				rbi->skb = new_skb;
1523 				rbi->dma_addr = new_dma_addr;
1524 				rxd->addr = cpu_to_le64(rbi->dma_addr);
1525 				rxd->len = rbi->len;
1526 			}
1527 
1528 #ifdef VMXNET3_RSS
1529 			if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1530 			    (adapter->netdev->features & NETIF_F_RXHASH)) {
1531 				enum pkt_hash_types hash_type;
1532 
1533 				switch (rcd->rssType) {
1534 				case VMXNET3_RCD_RSS_TYPE_IPV4:
1535 				case VMXNET3_RCD_RSS_TYPE_IPV6:
1536 					hash_type = PKT_HASH_TYPE_L3;
1537 					break;
1538 				case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
1539 				case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
1540 				case VMXNET3_RCD_RSS_TYPE_UDPIPV4:
1541 				case VMXNET3_RCD_RSS_TYPE_UDPIPV6:
1542 					hash_type = PKT_HASH_TYPE_L4;
1543 					break;
1544 				default:
1545 					hash_type = PKT_HASH_TYPE_L3;
1546 					break;
1547 				}
1548 				skb_set_hash(ctx->skb,
1549 					     le32_to_cpu(rcd->rssHash),
1550 					     hash_type);
1551 			}
1552 #endif
1553 			skb_record_rx_queue(ctx->skb, rq->qid);
1554 			skb_put(ctx->skb, rcd->len);
1555 
1556 			if (VMXNET3_VERSION_GE_2(adapter) &&
1557 			    rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
1558 				struct Vmxnet3_RxCompDescExt *rcdlro;
1559 				rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd;
1560 
1561 				segCnt = rcdlro->segCnt;
1562 				WARN_ON_ONCE(segCnt == 0);
1563 				mss = rcdlro->mss;
1564 				if (unlikely(segCnt <= 1))
1565 					segCnt = 0;
1566 			} else {
1567 				segCnt = 0;
1568 			}
1569 		} else {
1570 			BUG_ON(ctx->skb == NULL && !skip_page_frags);
1571 
1572 			/* non SOP buffer must be type 1 in most cases */
1573 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1574 			BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1575 
1576 			/* If an sop buffer was dropped, skip all
1577 			 * following non-sop fragments. They will be reused.
1578 			 */
1579 			if (skip_page_frags)
1580 				goto rcd_done;
1581 
1582 			if (rcd->len) {
1583 				new_page = alloc_page(GFP_ATOMIC);
1584 				/* Replacement page frag could not be allocated.
1585 				 * Reuse this page. Drop the pkt and free the
1586 				 * skb which contained this page as a frag. Skip
1587 				 * processing all the following non-sop frags.
1588 				 */
1589 				if (unlikely(!new_page)) {
1590 					rq->stats.rx_buf_alloc_failure++;
1591 					dev_kfree_skb(ctx->skb);
1592 					ctx->skb = NULL;
1593 					skip_page_frags = true;
1594 					goto rcd_done;
1595 				}
1596 				new_dma_addr = dma_map_page(&adapter->pdev->dev,
1597 							    new_page,
1598 							    0, PAGE_SIZE,
1599 							    DMA_FROM_DEVICE);
1600 				if (dma_mapping_error(&adapter->pdev->dev,
1601 						      new_dma_addr)) {
1602 					put_page(new_page);
1603 					rq->stats.rx_buf_alloc_failure++;
1604 					dev_kfree_skb(ctx->skb);
1605 					ctx->skb = NULL;
1606 					skip_page_frags = true;
1607 					goto rcd_done;
1608 				}
1609 
1610 				dma_unmap_page(&adapter->pdev->dev,
1611 					       rbi->dma_addr, rbi->len,
1612 					       DMA_FROM_DEVICE);
1613 
1614 				vmxnet3_append_frag(ctx->skb, rcd, rbi);
1615 
1616 				/* Immediate refill */
1617 				rbi->page = new_page;
1618 				rbi->dma_addr = new_dma_addr;
1619 				rxd->addr = cpu_to_le64(rbi->dma_addr);
1620 				rxd->len = rbi->len;
1621 			}
1622 		}
1623 
1624 
1625 		skb = ctx->skb;
1626 		if (rcd->eop) {
1627 			u32 mtu = adapter->netdev->mtu;
1628 			skb->len += skb->data_len;
1629 
1630 			vmxnet3_rx_csum(adapter, skb,
1631 					(union Vmxnet3_GenericDesc *)rcd);
1632 			skb->protocol = eth_type_trans(skb, adapter->netdev);
1633 			if (!rcd->tcp ||
1634 			    !(adapter->netdev->features & NETIF_F_LRO))
1635 				goto not_lro;
1636 
1637 			if (segCnt != 0 && mss != 0) {
1638 				skb_shinfo(skb)->gso_type = rcd->v4 ?
1639 					SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
1640 				skb_shinfo(skb)->gso_size = mss;
1641 				skb_shinfo(skb)->gso_segs = segCnt;
1642 			} else if (segCnt != 0 || skb->len > mtu) {
1643 				u32 hlen;
1644 
1645 				hlen = vmxnet3_get_hdr_len(adapter, skb,
1646 					(union Vmxnet3_GenericDesc *)rcd);
1647 				if (hlen == 0)
1648 					goto not_lro;
1649 
1650 				skb_shinfo(skb)->gso_type =
1651 					rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
1652 				if (segCnt != 0) {
1653 					skb_shinfo(skb)->gso_segs = segCnt;
1654 					skb_shinfo(skb)->gso_size =
1655 						DIV_ROUND_UP(skb->len -
1656 							hlen, segCnt);
1657 				} else {
1658 					skb_shinfo(skb)->gso_size = mtu - hlen;
1659 				}
1660 			}
1661 not_lro:
1662 			if (unlikely(rcd->ts))
1663 				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1664 
1665 			if (adapter->netdev->features & NETIF_F_LRO)
1666 				netif_receive_skb(skb);
1667 			else
1668 				napi_gro_receive(&rq->napi, skb);
1669 
1670 			ctx->skb = NULL;
1671 			num_pkts++;
1672 		}
1673 
1674 rcd_done:
1675 		/* device may have skipped some rx descs */
1676 		ring = rq->rx_ring + ring_idx;
1677 		rbi->comp_state = VMXNET3_RXD_COMP_DONE;
1678 
1679 		comp_offset = vmxnet3_cmd_ring_desc_avail(ring);
1680 		fill_offset = (idx > ring->next2fill ? 0 : ring->size) +
1681 			      idx - ring->next2fill - 1;
1682 		if (!ring->isOutOfOrder || fill_offset >= comp_offset)
1683 			ring->next2comp = idx;
1684 		num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1685 
1686 		/* Ensure that the writes to rxd->gen bits will be observed
1687 		 * after all other writes to rxd objects.
1688 		 */
1689 		dma_wmb();
1690 
1691 		while (num_to_alloc) {
1692 			rbi = rq->buf_info[ring_idx] + ring->next2fill;
1693 			if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP)))
1694 				goto refill_buf;
1695 			if (ring_idx == 0) {
1696 				/* ring0 Type1 buffers can get skipped; re-fill them */
1697 				if (rbi->buf_type != VMXNET3_RX_BUF_SKB)
1698 					goto refill_buf;
1699 			}
1700 			if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) {
1701 refill_buf:
1702 				vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1703 						  &rxCmdDesc);
1704 				WARN_ON(!rxd->addr);
1705 
1706 				/* Recv desc is ready to be used by the device */
1707 				rxd->gen = ring->gen;
1708 				vmxnet3_cmd_ring_adv_next2fill(ring);
1709 				rbi->comp_state = VMXNET3_RXD_COMP_PENDING;
1710 				num_to_alloc--;
1711 			} else {
1712 				/* rx completion hasn't occurred */
1713 				ring->isOutOfOrder = 1;
1714 				break;
1715 			}
1716 		}
1717 
1718 		if (num_to_alloc == 0) {
1719 			ring->isOutOfOrder = 0;
1720 		}
1721 
1722 		/* if needed, update the register */
1723 		if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) {
1724 			VMXNET3_WRITE_BAR0_REG(adapter,
1725 					       rxprod_reg[ring_idx] + rq->qid * 8,
1726 					       ring->next2fill);
1727 		}
1728 
1729 		vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1730 		vmxnet3_getRxComp(rcd,
1731 				  &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1732 	}
1733 
1734 	return num_pkts;
1735 }
1736 
1737 
1738 static void
1739 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1740 		   struct vmxnet3_adapter *adapter)
1741 {
1742 	u32 i, ring_idx;
1743 	struct Vmxnet3_RxDesc *rxd;
1744 
1745 	/* ring has already been cleaned up */
1746 	if (!rq->rx_ring[0].base)
1747 		return;
1748 
1749 	for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1750 		for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1751 #ifdef __BIG_ENDIAN_BITFIELD
1752 			struct Vmxnet3_RxDesc rxDesc;
1753 #endif
1754 			vmxnet3_getRxDesc(rxd,
1755 				&rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1756 
1757 			if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1758 					rq->buf_info[ring_idx][i].skb) {
1759 				dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1760 						 rxd->len, DMA_FROM_DEVICE);
1761 				dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1762 				rq->buf_info[ring_idx][i].skb = NULL;
1763 			} else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1764 					rq->buf_info[ring_idx][i].page) {
1765 				dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1766 					       rxd->len, DMA_FROM_DEVICE);
1767 				put_page(rq->buf_info[ring_idx][i].page);
1768 				rq->buf_info[ring_idx][i].page = NULL;
1769 			}
1770 		}
1771 
1772 		rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1773 		rq->rx_ring[ring_idx].next2fill =
1774 					rq->rx_ring[ring_idx].next2comp = 0;
1775 	}
1776 
1777 	rq->comp_ring.gen = VMXNET3_INIT_GEN;
1778 	rq->comp_ring.next2proc = 0;
1779 }
1780 
1781 
1782 static void
1783 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1784 {
1785 	int i;
1786 
1787 	for (i = 0; i < adapter->num_rx_queues; i++)
1788 		vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1789 }
1790 
1791 
1792 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1793 			       struct vmxnet3_adapter *adapter)
1794 {
1795 	int i;
1796 	int j;
1797 
1798 	/* all rx buffers must have already been freed */
1799 	for (i = 0; i < 2; i++) {
1800 		if (rq->buf_info[i]) {
1801 			for (j = 0; j < rq->rx_ring[i].size; j++)
1802 				BUG_ON(rq->buf_info[i][j].page != NULL);
1803 		}
1804 	}
1805 
1806 
1807 	for (i = 0; i < 2; i++) {
1808 		if (rq->rx_ring[i].base) {
1809 			dma_free_coherent(&adapter->pdev->dev,
1810 					  rq->rx_ring[i].size
1811 					  * sizeof(struct Vmxnet3_RxDesc),
1812 					  rq->rx_ring[i].base,
1813 					  rq->rx_ring[i].basePA);
1814 			rq->rx_ring[i].base = NULL;
1815 		}
1816 	}
1817 
1818 	if (rq->data_ring.base) {
1819 		dma_free_coherent(&adapter->pdev->dev,
1820 				  rq->rx_ring[0].size * rq->data_ring.desc_size,
1821 				  rq->data_ring.base, rq->data_ring.basePA);
1822 		rq->data_ring.base = NULL;
1823 	}
1824 
1825 	if (rq->comp_ring.base) {
1826 		dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1827 				  * sizeof(struct Vmxnet3_RxCompDesc),
1828 				  rq->comp_ring.base, rq->comp_ring.basePA);
1829 		rq->comp_ring.base = NULL;
1830 	}
1831 
1832 	kfree(rq->buf_info[0]);
1833 	rq->buf_info[0] = NULL;
1834 	rq->buf_info[1] = NULL;
1835 }
1836 
1837 static void
1838 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter)
1839 {
1840 	int i;
1841 
1842 	for (i = 0; i < adapter->num_rx_queues; i++) {
1843 		struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1844 
1845 		if (rq->data_ring.base) {
1846 			dma_free_coherent(&adapter->pdev->dev,
1847 					  (rq->rx_ring[0].size *
1848 					  rq->data_ring.desc_size),
1849 					  rq->data_ring.base,
1850 					  rq->data_ring.basePA);
1851 			rq->data_ring.base = NULL;
1852 			rq->data_ring.desc_size = 0;
1853 		}
1854 	}
1855 }
1856 
1857 static int
1858 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1859 		struct vmxnet3_adapter  *adapter)
1860 {
1861 	int i;
1862 
1863 	/* initialize buf_info */
1864 	for (i = 0; i < rq->rx_ring[0].size; i++) {
1865 
1866 		/* 1st buf for a pkt is skbuff */
1867 		if (i % adapter->rx_buf_per_pkt == 0) {
1868 			rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1869 			rq->buf_info[0][i].len = adapter->skb_buf_size;
1870 		} else { /* subsequent bufs for a pkt is frag */
1871 			rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1872 			rq->buf_info[0][i].len = PAGE_SIZE;
1873 		}
1874 	}
1875 	for (i = 0; i < rq->rx_ring[1].size; i++) {
1876 		rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1877 		rq->buf_info[1][i].len = PAGE_SIZE;
1878 	}
1879 
1880 	/* reset internal state and allocate buffers for both rings */
1881 	for (i = 0; i < 2; i++) {
1882 		rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1883 
1884 		memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1885 		       sizeof(struct Vmxnet3_RxDesc));
1886 		rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1887 		rq->rx_ring[i].isOutOfOrder = 0;
1888 	}
1889 	if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1890 				    adapter) == 0) {
1891 		/* at least has 1 rx buffer for the 1st ring */
1892 		return -ENOMEM;
1893 	}
1894 	vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1895 
1896 	/* reset the comp ring */
1897 	rq->comp_ring.next2proc = 0;
1898 	memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1899 	       sizeof(struct Vmxnet3_RxCompDesc));
1900 	rq->comp_ring.gen = VMXNET3_INIT_GEN;
1901 
1902 	/* reset rxctx */
1903 	rq->rx_ctx.skb = NULL;
1904 
1905 	/* stats are not reset */
1906 	return 0;
1907 }
1908 
1909 
1910 static int
1911 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1912 {
1913 	int i, err = 0;
1914 
1915 	for (i = 0; i < adapter->num_rx_queues; i++) {
1916 		err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1917 		if (unlikely(err)) {
1918 			dev_err(&adapter->netdev->dev, "%s: failed to "
1919 				"initialize rx queue%i\n",
1920 				adapter->netdev->name, i);
1921 			break;
1922 		}
1923 	}
1924 	return err;
1925 
1926 }
1927 
1928 
1929 static int
1930 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1931 {
1932 	int i;
1933 	size_t sz;
1934 	struct vmxnet3_rx_buf_info *bi;
1935 
1936 	for (i = 0; i < 2; i++) {
1937 
1938 		sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1939 		rq->rx_ring[i].base = dma_alloc_coherent(
1940 						&adapter->pdev->dev, sz,
1941 						&rq->rx_ring[i].basePA,
1942 						GFP_KERNEL);
1943 		if (!rq->rx_ring[i].base) {
1944 			netdev_err(adapter->netdev,
1945 				   "failed to allocate rx ring %d\n", i);
1946 			goto err;
1947 		}
1948 	}
1949 
1950 	if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) {
1951 		sz = rq->rx_ring[0].size * rq->data_ring.desc_size;
1952 		rq->data_ring.base =
1953 			dma_alloc_coherent(&adapter->pdev->dev, sz,
1954 					   &rq->data_ring.basePA,
1955 					   GFP_KERNEL);
1956 		if (!rq->data_ring.base) {
1957 			netdev_err(adapter->netdev,
1958 				   "rx data ring will be disabled\n");
1959 			adapter->rxdataring_enabled = false;
1960 		}
1961 	} else {
1962 		rq->data_ring.base = NULL;
1963 		rq->data_ring.desc_size = 0;
1964 	}
1965 
1966 	sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1967 	rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1968 						&rq->comp_ring.basePA,
1969 						GFP_KERNEL);
1970 	if (!rq->comp_ring.base) {
1971 		netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
1972 		goto err;
1973 	}
1974 
1975 	bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size,
1976 			  sizeof(rq->buf_info[0][0]), GFP_KERNEL,
1977 			  dev_to_node(&adapter->pdev->dev));
1978 	if (!bi)
1979 		goto err;
1980 
1981 	rq->buf_info[0] = bi;
1982 	rq->buf_info[1] = bi + rq->rx_ring[0].size;
1983 
1984 	return 0;
1985 
1986 err:
1987 	vmxnet3_rq_destroy(rq, adapter);
1988 	return -ENOMEM;
1989 }
1990 
1991 
1992 static int
1993 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
1994 {
1995 	int i, err = 0;
1996 
1997 	adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
1998 
1999 	for (i = 0; i < adapter->num_rx_queues; i++) {
2000 		err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
2001 		if (unlikely(err)) {
2002 			dev_err(&adapter->netdev->dev,
2003 				"%s: failed to create rx queue%i\n",
2004 				adapter->netdev->name, i);
2005 			goto err_out;
2006 		}
2007 	}
2008 
2009 	if (!adapter->rxdataring_enabled)
2010 		vmxnet3_rq_destroy_all_rxdataring(adapter);
2011 
2012 	return err;
2013 err_out:
2014 	vmxnet3_rq_destroy_all(adapter);
2015 	return err;
2016 
2017 }
2018 
2019 /* Multiple queue aware polling function for tx and rx */
2020 
2021 static int
2022 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
2023 {
2024 	int rcd_done = 0, i;
2025 	if (unlikely(adapter->shared->ecr))
2026 		vmxnet3_process_events(adapter);
2027 	for (i = 0; i < adapter->num_tx_queues; i++)
2028 		vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
2029 
2030 	for (i = 0; i < adapter->num_rx_queues; i++)
2031 		rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
2032 						   adapter, budget);
2033 	return rcd_done;
2034 }
2035 
2036 
2037 static int
2038 vmxnet3_poll(struct napi_struct *napi, int budget)
2039 {
2040 	struct vmxnet3_rx_queue *rx_queue = container_of(napi,
2041 					  struct vmxnet3_rx_queue, napi);
2042 	int rxd_done;
2043 
2044 	rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
2045 
2046 	if (rxd_done < budget) {
2047 		napi_complete_done(napi, rxd_done);
2048 		vmxnet3_enable_all_intrs(rx_queue->adapter);
2049 	}
2050 	return rxd_done;
2051 }
2052 
2053 /*
2054  * NAPI polling function for MSI-X mode with multiple Rx queues
2055  * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
2056  */
2057 
2058 static int
2059 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
2060 {
2061 	struct vmxnet3_rx_queue *rq = container_of(napi,
2062 						struct vmxnet3_rx_queue, napi);
2063 	struct vmxnet3_adapter *adapter = rq->adapter;
2064 	int rxd_done;
2065 
2066 	/* When sharing interrupt with corresponding tx queue, process
2067 	 * tx completions in that queue as well
2068 	 */
2069 	if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
2070 		struct vmxnet3_tx_queue *tq =
2071 				&adapter->tx_queue[rq - adapter->rx_queue];
2072 		vmxnet3_tq_tx_complete(tq, adapter);
2073 	}
2074 
2075 	rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
2076 
2077 	if (rxd_done < budget) {
2078 		napi_complete_done(napi, rxd_done);
2079 		vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
2080 	}
2081 	return rxd_done;
2082 }
2083 
2084 
2085 #ifdef CONFIG_PCI_MSI
2086 
2087 /*
2088  * Handle completion interrupts on tx queues
2089  * Returns whether or not the intr is handled
2090  */
2091 
2092 static irqreturn_t
2093 vmxnet3_msix_tx(int irq, void *data)
2094 {
2095 	struct vmxnet3_tx_queue *tq = data;
2096 	struct vmxnet3_adapter *adapter = tq->adapter;
2097 
2098 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2099 		vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
2100 
2101 	/* Handle the case where only one irq is allocate for all tx queues */
2102 	if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
2103 		int i;
2104 		for (i = 0; i < adapter->num_tx_queues; i++) {
2105 			struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
2106 			vmxnet3_tq_tx_complete(txq, adapter);
2107 		}
2108 	} else {
2109 		vmxnet3_tq_tx_complete(tq, adapter);
2110 	}
2111 	vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
2112 
2113 	return IRQ_HANDLED;
2114 }
2115 
2116 
2117 /*
2118  * Handle completion interrupts on rx queues. Returns whether or not the
2119  * intr is handled
2120  */
2121 
2122 static irqreturn_t
2123 vmxnet3_msix_rx(int irq, void *data)
2124 {
2125 	struct vmxnet3_rx_queue *rq = data;
2126 	struct vmxnet3_adapter *adapter = rq->adapter;
2127 
2128 	/* disable intr if needed */
2129 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2130 		vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
2131 	napi_schedule(&rq->napi);
2132 
2133 	return IRQ_HANDLED;
2134 }
2135 
2136 /*
2137  *----------------------------------------------------------------------------
2138  *
2139  * vmxnet3_msix_event --
2140  *
2141  *    vmxnet3 msix event intr handler
2142  *
2143  * Result:
2144  *    whether or not the intr is handled
2145  *
2146  *----------------------------------------------------------------------------
2147  */
2148 
2149 static irqreturn_t
2150 vmxnet3_msix_event(int irq, void *data)
2151 {
2152 	struct net_device *dev = data;
2153 	struct vmxnet3_adapter *adapter = netdev_priv(dev);
2154 
2155 	/* disable intr if needed */
2156 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2157 		vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
2158 
2159 	if (adapter->shared->ecr)
2160 		vmxnet3_process_events(adapter);
2161 
2162 	vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
2163 
2164 	return IRQ_HANDLED;
2165 }
2166 
2167 #endif /* CONFIG_PCI_MSI  */
2168 
2169 
2170 /* Interrupt handler for vmxnet3  */
2171 static irqreturn_t
2172 vmxnet3_intr(int irq, void *dev_id)
2173 {
2174 	struct net_device *dev = dev_id;
2175 	struct vmxnet3_adapter *adapter = netdev_priv(dev);
2176 
2177 	if (adapter->intr.type == VMXNET3_IT_INTX) {
2178 		u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
2179 		if (unlikely(icr == 0))
2180 			/* not ours */
2181 			return IRQ_NONE;
2182 	}
2183 
2184 
2185 	/* disable intr if needed */
2186 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2187 		vmxnet3_disable_all_intrs(adapter);
2188 
2189 	napi_schedule(&adapter->rx_queue[0].napi);
2190 
2191 	return IRQ_HANDLED;
2192 }
2193 
2194 #ifdef CONFIG_NET_POLL_CONTROLLER
2195 
2196 /* netpoll callback. */
2197 static void
2198 vmxnet3_netpoll(struct net_device *netdev)
2199 {
2200 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2201 
2202 	switch (adapter->intr.type) {
2203 #ifdef CONFIG_PCI_MSI
2204 	case VMXNET3_IT_MSIX: {
2205 		int i;
2206 		for (i = 0; i < adapter->num_rx_queues; i++)
2207 			vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
2208 		break;
2209 	}
2210 #endif
2211 	case VMXNET3_IT_MSI:
2212 	default:
2213 		vmxnet3_intr(0, adapter->netdev);
2214 		break;
2215 	}
2216 
2217 }
2218 #endif	/* CONFIG_NET_POLL_CONTROLLER */
2219 
2220 static int
2221 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
2222 {
2223 	struct vmxnet3_intr *intr = &adapter->intr;
2224 	int err = 0, i;
2225 	int vector = 0;
2226 
2227 #ifdef CONFIG_PCI_MSI
2228 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
2229 		for (i = 0; i < adapter->num_tx_queues; i++) {
2230 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
2231 				sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
2232 					adapter->netdev->name, vector);
2233 				err = request_irq(
2234 					      intr->msix_entries[vector].vector,
2235 					      vmxnet3_msix_tx, 0,
2236 					      adapter->tx_queue[i].name,
2237 					      &adapter->tx_queue[i]);
2238 			} else {
2239 				sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
2240 					adapter->netdev->name, vector);
2241 			}
2242 			if (err) {
2243 				dev_err(&adapter->netdev->dev,
2244 					"Failed to request irq for MSIX, %s, "
2245 					"error %d\n",
2246 					adapter->tx_queue[i].name, err);
2247 				return err;
2248 			}
2249 
2250 			/* Handle the case where only 1 MSIx was allocated for
2251 			 * all tx queues */
2252 			if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
2253 				for (; i < adapter->num_tx_queues; i++)
2254 					adapter->tx_queue[i].comp_ring.intr_idx
2255 								= vector;
2256 				vector++;
2257 				break;
2258 			} else {
2259 				adapter->tx_queue[i].comp_ring.intr_idx
2260 								= vector++;
2261 			}
2262 		}
2263 		if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
2264 			vector = 0;
2265 
2266 		for (i = 0; i < adapter->num_rx_queues; i++) {
2267 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
2268 				sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
2269 					adapter->netdev->name, vector);
2270 			else
2271 				sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
2272 					adapter->netdev->name, vector);
2273 			err = request_irq(intr->msix_entries[vector].vector,
2274 					  vmxnet3_msix_rx, 0,
2275 					  adapter->rx_queue[i].name,
2276 					  &(adapter->rx_queue[i]));
2277 			if (err) {
2278 				netdev_err(adapter->netdev,
2279 					   "Failed to request irq for MSIX, "
2280 					   "%s, error %d\n",
2281 					   adapter->rx_queue[i].name, err);
2282 				return err;
2283 			}
2284 
2285 			adapter->rx_queue[i].comp_ring.intr_idx = vector++;
2286 		}
2287 
2288 		sprintf(intr->event_msi_vector_name, "%s-event-%d",
2289 			adapter->netdev->name, vector);
2290 		err = request_irq(intr->msix_entries[vector].vector,
2291 				  vmxnet3_msix_event, 0,
2292 				  intr->event_msi_vector_name, adapter->netdev);
2293 		intr->event_intr_idx = vector;
2294 
2295 	} else if (intr->type == VMXNET3_IT_MSI) {
2296 		adapter->num_rx_queues = 1;
2297 		err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
2298 				  adapter->netdev->name, adapter->netdev);
2299 	} else {
2300 #endif
2301 		adapter->num_rx_queues = 1;
2302 		err = request_irq(adapter->pdev->irq, vmxnet3_intr,
2303 				  IRQF_SHARED, adapter->netdev->name,
2304 				  adapter->netdev);
2305 #ifdef CONFIG_PCI_MSI
2306 	}
2307 #endif
2308 	intr->num_intrs = vector + 1;
2309 	if (err) {
2310 		netdev_err(adapter->netdev,
2311 			   "Failed to request irq (intr type:%d), error %d\n",
2312 			   intr->type, err);
2313 	} else {
2314 		/* Number of rx queues will not change after this */
2315 		for (i = 0; i < adapter->num_rx_queues; i++) {
2316 			struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2317 			rq->qid = i;
2318 			rq->qid2 = i + adapter->num_rx_queues;
2319 			rq->dataRingQid = i + 2 * adapter->num_rx_queues;
2320 		}
2321 
2322 		/* init our intr settings */
2323 		for (i = 0; i < intr->num_intrs; i++)
2324 			intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
2325 		if (adapter->intr.type != VMXNET3_IT_MSIX) {
2326 			adapter->intr.event_intr_idx = 0;
2327 			for (i = 0; i < adapter->num_tx_queues; i++)
2328 				adapter->tx_queue[i].comp_ring.intr_idx = 0;
2329 			adapter->rx_queue[0].comp_ring.intr_idx = 0;
2330 		}
2331 
2332 		netdev_info(adapter->netdev,
2333 			    "intr type %u, mode %u, %u vectors allocated\n",
2334 			    intr->type, intr->mask_mode, intr->num_intrs);
2335 	}
2336 
2337 	return err;
2338 }
2339 
2340 
2341 static void
2342 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
2343 {
2344 	struct vmxnet3_intr *intr = &adapter->intr;
2345 	BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
2346 
2347 	switch (intr->type) {
2348 #ifdef CONFIG_PCI_MSI
2349 	case VMXNET3_IT_MSIX:
2350 	{
2351 		int i, vector = 0;
2352 
2353 		if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
2354 			for (i = 0; i < adapter->num_tx_queues; i++) {
2355 				free_irq(intr->msix_entries[vector++].vector,
2356 					 &(adapter->tx_queue[i]));
2357 				if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
2358 					break;
2359 			}
2360 		}
2361 
2362 		for (i = 0; i < adapter->num_rx_queues; i++) {
2363 			free_irq(intr->msix_entries[vector++].vector,
2364 				 &(adapter->rx_queue[i]));
2365 		}
2366 
2367 		free_irq(intr->msix_entries[vector].vector,
2368 			 adapter->netdev);
2369 		BUG_ON(vector >= intr->num_intrs);
2370 		break;
2371 	}
2372 #endif
2373 	case VMXNET3_IT_MSI:
2374 		free_irq(adapter->pdev->irq, adapter->netdev);
2375 		break;
2376 	case VMXNET3_IT_INTX:
2377 		free_irq(adapter->pdev->irq, adapter->netdev);
2378 		break;
2379 	default:
2380 		BUG();
2381 	}
2382 }
2383 
2384 
2385 static void
2386 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
2387 {
2388 	u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2389 	u16 vid;
2390 
2391 	/* allow untagged pkts */
2392 	VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
2393 
2394 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2395 		VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
2396 }
2397 
2398 
2399 static int
2400 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
2401 {
2402 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2403 
2404 	if (!(netdev->flags & IFF_PROMISC)) {
2405 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2406 		unsigned long flags;
2407 
2408 		VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
2409 		spin_lock_irqsave(&adapter->cmd_lock, flags);
2410 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2411 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2412 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2413 	}
2414 
2415 	set_bit(vid, adapter->active_vlans);
2416 
2417 	return 0;
2418 }
2419 
2420 
2421 static int
2422 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
2423 {
2424 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2425 
2426 	if (!(netdev->flags & IFF_PROMISC)) {
2427 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2428 		unsigned long flags;
2429 
2430 		VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
2431 		spin_lock_irqsave(&adapter->cmd_lock, flags);
2432 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2433 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2434 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2435 	}
2436 
2437 	clear_bit(vid, adapter->active_vlans);
2438 
2439 	return 0;
2440 }
2441 
2442 
2443 static u8 *
2444 vmxnet3_copy_mc(struct net_device *netdev)
2445 {
2446 	u8 *buf = NULL;
2447 	u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2448 
2449 	/* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2450 	if (sz <= 0xffff) {
2451 		/* We may be called with BH disabled */
2452 		buf = kmalloc(sz, GFP_ATOMIC);
2453 		if (buf) {
2454 			struct netdev_hw_addr *ha;
2455 			int i = 0;
2456 
2457 			netdev_for_each_mc_addr(ha, netdev)
2458 				memcpy(buf + i++ * ETH_ALEN, ha->addr,
2459 				       ETH_ALEN);
2460 		}
2461 	}
2462 	return buf;
2463 }
2464 
2465 
2466 static void
2467 vmxnet3_set_mc(struct net_device *netdev)
2468 {
2469 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2470 	unsigned long flags;
2471 	struct Vmxnet3_RxFilterConf *rxConf =
2472 					&adapter->shared->devRead.rxFilterConf;
2473 	u8 *new_table = NULL;
2474 	dma_addr_t new_table_pa = 0;
2475 	bool new_table_pa_valid = false;
2476 	u32 new_mode = VMXNET3_RXM_UCAST;
2477 
2478 	if (netdev->flags & IFF_PROMISC) {
2479 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2480 		memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2481 
2482 		new_mode |= VMXNET3_RXM_PROMISC;
2483 	} else {
2484 		vmxnet3_restore_vlan(adapter);
2485 	}
2486 
2487 	if (netdev->flags & IFF_BROADCAST)
2488 		new_mode |= VMXNET3_RXM_BCAST;
2489 
2490 	if (netdev->flags & IFF_ALLMULTI)
2491 		new_mode |= VMXNET3_RXM_ALL_MULTI;
2492 	else
2493 		if (!netdev_mc_empty(netdev)) {
2494 			new_table = vmxnet3_copy_mc(netdev);
2495 			if (new_table) {
2496 				size_t sz = netdev_mc_count(netdev) * ETH_ALEN;
2497 
2498 				rxConf->mfTableLen = cpu_to_le16(sz);
2499 				new_table_pa = dma_map_single(
2500 							&adapter->pdev->dev,
2501 							new_table,
2502 							sz,
2503 							DMA_TO_DEVICE);
2504 				if (!dma_mapping_error(&adapter->pdev->dev,
2505 						       new_table_pa)) {
2506 					new_mode |= VMXNET3_RXM_MCAST;
2507 					new_table_pa_valid = true;
2508 					rxConf->mfTablePA = cpu_to_le64(
2509 								new_table_pa);
2510 				}
2511 			}
2512 			if (!new_table_pa_valid) {
2513 				netdev_info(netdev,
2514 					    "failed to copy mcast list, setting ALL_MULTI\n");
2515 				new_mode |= VMXNET3_RXM_ALL_MULTI;
2516 			}
2517 		}
2518 
2519 	if (!(new_mode & VMXNET3_RXM_MCAST)) {
2520 		rxConf->mfTableLen = 0;
2521 		rxConf->mfTablePA = 0;
2522 	}
2523 
2524 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2525 	if (new_mode != rxConf->rxMode) {
2526 		rxConf->rxMode = cpu_to_le32(new_mode);
2527 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2528 				       VMXNET3_CMD_UPDATE_RX_MODE);
2529 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2530 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2531 	}
2532 
2533 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2534 			       VMXNET3_CMD_UPDATE_MAC_FILTERS);
2535 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2536 
2537 	if (new_table_pa_valid)
2538 		dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2539 				 rxConf->mfTableLen, DMA_TO_DEVICE);
2540 	kfree(new_table);
2541 }
2542 
2543 void
2544 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2545 {
2546 	int i;
2547 
2548 	for (i = 0; i < adapter->num_rx_queues; i++)
2549 		vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2550 }
2551 
2552 
2553 /*
2554  *   Set up driver_shared based on settings in adapter.
2555  */
2556 
2557 static void
2558 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2559 {
2560 	struct Vmxnet3_DriverShared *shared = adapter->shared;
2561 	struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2562 	struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt;
2563 	struct Vmxnet3_TxQueueConf *tqc;
2564 	struct Vmxnet3_RxQueueConf *rqc;
2565 	int i;
2566 
2567 	memset(shared, 0, sizeof(*shared));
2568 
2569 	/* driver settings */
2570 	shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2571 	devRead->misc.driverInfo.version = cpu_to_le32(
2572 						VMXNET3_DRIVER_VERSION_NUM);
2573 	devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2574 				VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2575 	devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2576 	*((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2577 				*((u32 *)&devRead->misc.driverInfo.gos));
2578 	devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2579 	devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2580 
2581 	devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2582 	devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2583 
2584 	/* set up feature flags */
2585 	if (adapter->netdev->features & NETIF_F_RXCSUM)
2586 		devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2587 
2588 	if (adapter->netdev->features & NETIF_F_LRO) {
2589 		devRead->misc.uptFeatures |= UPT1_F_LRO;
2590 		devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2591 	}
2592 	if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2593 		devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2594 
2595 	if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL |
2596 					 NETIF_F_GSO_UDP_TUNNEL_CSUM))
2597 		devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD;
2598 
2599 	devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2600 	devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2601 	devRead->misc.queueDescLen = cpu_to_le32(
2602 		adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2603 		adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2604 
2605 	/* tx queue settings */
2606 	devRead->misc.numTxQueues =  adapter->num_tx_queues;
2607 	for (i = 0; i < adapter->num_tx_queues; i++) {
2608 		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];
2609 		BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2610 		tqc = &adapter->tqd_start[i].conf;
2611 		tqc->txRingBasePA   = cpu_to_le64(tq->tx_ring.basePA);
2612 		tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2613 		tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2614 		tqc->ddPA           = cpu_to_le64(~0ULL);
2615 		tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);
2616 		tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);
2617 		tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size);
2618 		tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);
2619 		tqc->ddLen          = cpu_to_le32(0);
2620 		tqc->intrIdx        = tq->comp_ring.intr_idx;
2621 	}
2622 
2623 	/* rx queue settings */
2624 	devRead->misc.numRxQueues = adapter->num_rx_queues;
2625 	for (i = 0; i < adapter->num_rx_queues; i++) {
2626 		struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[i];
2627 		rqc = &adapter->rqd_start[i].conf;
2628 		rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2629 		rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2630 		rqc->compRingBasePA  = cpu_to_le64(rq->comp_ring.basePA);
2631 		rqc->ddPA            = cpu_to_le64(~0ULL);
2632 		rqc->rxRingSize[0]   = cpu_to_le32(rq->rx_ring[0].size);
2633 		rqc->rxRingSize[1]   = cpu_to_le32(rq->rx_ring[1].size);
2634 		rqc->compRingSize    = cpu_to_le32(rq->comp_ring.size);
2635 		rqc->ddLen           = cpu_to_le32(0);
2636 		rqc->intrIdx         = rq->comp_ring.intr_idx;
2637 		if (VMXNET3_VERSION_GE_3(adapter)) {
2638 			rqc->rxDataRingBasePA =
2639 				cpu_to_le64(rq->data_ring.basePA);
2640 			rqc->rxDataRingDescSize =
2641 				cpu_to_le16(rq->data_ring.desc_size);
2642 		}
2643 	}
2644 
2645 #ifdef VMXNET3_RSS
2646 	memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2647 
2648 	if (adapter->rss) {
2649 		struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2650 
2651 		devRead->misc.uptFeatures |= UPT1_F_RSS;
2652 		devRead->misc.numRxQueues = adapter->num_rx_queues;
2653 		rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2654 				    UPT1_RSS_HASH_TYPE_IPV4 |
2655 				    UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2656 				    UPT1_RSS_HASH_TYPE_IPV6;
2657 		rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2658 		rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2659 		rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2660 		netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey));
2661 
2662 		for (i = 0; i < rssConf->indTableSize; i++)
2663 			rssConf->indTable[i] = ethtool_rxfh_indir_default(
2664 				i, adapter->num_rx_queues);
2665 
2666 		devRead->rssConfDesc.confVer = 1;
2667 		devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2668 		devRead->rssConfDesc.confPA =
2669 			cpu_to_le64(adapter->rss_conf_pa);
2670 	}
2671 
2672 #endif /* VMXNET3_RSS */
2673 
2674 	/* intr settings */
2675 	if (!VMXNET3_VERSION_GE_6(adapter) ||
2676 	    !adapter->queuesExtEnabled) {
2677 		devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2678 					     VMXNET3_IMM_AUTO;
2679 		devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2680 		for (i = 0; i < adapter->intr.num_intrs; i++)
2681 			devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2682 
2683 		devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2684 		devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2685 	} else {
2686 		devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode ==
2687 						   VMXNET3_IMM_AUTO;
2688 		devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs;
2689 		for (i = 0; i < adapter->intr.num_intrs; i++)
2690 			devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i];
2691 
2692 		devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx;
2693 		devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2694 	}
2695 
2696 	/* rx filter settings */
2697 	devRead->rxFilterConf.rxMode = 0;
2698 	vmxnet3_restore_vlan(adapter);
2699 	vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2700 
2701 	/* the rest are already zeroed */
2702 }
2703 
2704 static void
2705 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter)
2706 {
2707 	struct Vmxnet3_DriverShared *shared = adapter->shared;
2708 	union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2709 	unsigned long flags;
2710 
2711 	if (!VMXNET3_VERSION_GE_7(adapter))
2712 		return;
2713 
2714 	cmdInfo->ringBufSize = adapter->ringBufSize;
2715 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2716 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2717 			       VMXNET3_CMD_SET_RING_BUFFER_SIZE);
2718 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2719 }
2720 
2721 static void
2722 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter)
2723 {
2724 	struct Vmxnet3_DriverShared *shared = adapter->shared;
2725 	union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2726 	unsigned long flags;
2727 
2728 	if (!VMXNET3_VERSION_GE_3(adapter))
2729 		return;
2730 
2731 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2732 	cmdInfo->varConf.confVer = 1;
2733 	cmdInfo->varConf.confLen =
2734 		cpu_to_le32(sizeof(*adapter->coal_conf));
2735 	cmdInfo->varConf.confPA  = cpu_to_le64(adapter->coal_conf_pa);
2736 
2737 	if (adapter->default_coal_mode) {
2738 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2739 				       VMXNET3_CMD_GET_COALESCE);
2740 	} else {
2741 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2742 				       VMXNET3_CMD_SET_COALESCE);
2743 	}
2744 
2745 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2746 }
2747 
2748 static void
2749 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter)
2750 {
2751 	struct Vmxnet3_DriverShared *shared = adapter->shared;
2752 	union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2753 	unsigned long flags;
2754 
2755 	if (!VMXNET3_VERSION_GE_4(adapter))
2756 		return;
2757 
2758 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2759 
2760 	if (adapter->default_rss_fields) {
2761 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2762 				       VMXNET3_CMD_GET_RSS_FIELDS);
2763 		adapter->rss_fields =
2764 			VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2765 	} else {
2766 		if (VMXNET3_VERSION_GE_7(adapter)) {
2767 			if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 ||
2768 			     adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) &&
2769 			    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2770 						       VMXNET3_CAP_UDP_RSS)) {
2771 				adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS;
2772 			} else {
2773 				adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS);
2774 			}
2775 
2776 			if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) &&
2777 			    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2778 						       VMXNET3_CAP_ESP_RSS_IPV4)) {
2779 				adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4;
2780 			} else {
2781 				adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4);
2782 			}
2783 
2784 			if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) &&
2785 			    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2786 						       VMXNET3_CAP_ESP_RSS_IPV6)) {
2787 				adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6;
2788 			} else {
2789 				adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6);
2790 			}
2791 
2792 			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
2793 			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
2794 			adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2795 		}
2796 		cmdInfo->setRssFields = adapter->rss_fields;
2797 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2798 				       VMXNET3_CMD_SET_RSS_FIELDS);
2799 		/* Not all requested RSS may get applied, so get and
2800 		 * cache what was actually applied.
2801 		 */
2802 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2803 				       VMXNET3_CMD_GET_RSS_FIELDS);
2804 		adapter->rss_fields =
2805 			VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2806 	}
2807 
2808 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2809 }
2810 
2811 int
2812 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2813 {
2814 	int err, i;
2815 	u32 ret;
2816 	unsigned long flags;
2817 
2818 	netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2819 		" ring sizes %u %u %u\n", adapter->netdev->name,
2820 		adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2821 		adapter->tx_queue[0].tx_ring.size,
2822 		adapter->rx_queue[0].rx_ring[0].size,
2823 		adapter->rx_queue[0].rx_ring[1].size);
2824 
2825 	vmxnet3_tq_init_all(adapter);
2826 	err = vmxnet3_rq_init_all(adapter);
2827 	if (err) {
2828 		netdev_err(adapter->netdev,
2829 			   "Failed to init rx queue error %d\n", err);
2830 		goto rq_err;
2831 	}
2832 
2833 	err = vmxnet3_request_irqs(adapter);
2834 	if (err) {
2835 		netdev_err(adapter->netdev,
2836 			   "Failed to setup irq for error %d\n", err);
2837 		goto irq_err;
2838 	}
2839 
2840 	vmxnet3_setup_driver_shared(adapter);
2841 
2842 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2843 			       adapter->shared_pa));
2844 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2845 			       adapter->shared_pa));
2846 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2847 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2848 			       VMXNET3_CMD_ACTIVATE_DEV);
2849 	ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2850 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2851 
2852 	if (ret != 0) {
2853 		netdev_err(adapter->netdev,
2854 			   "Failed to activate dev: error %u\n", ret);
2855 		err = -EINVAL;
2856 		goto activate_err;
2857 	}
2858 
2859 	vmxnet3_init_bufsize(adapter);
2860 	vmxnet3_init_coalesce(adapter);
2861 	vmxnet3_init_rssfields(adapter);
2862 
2863 	for (i = 0; i < adapter->num_rx_queues; i++) {
2864 		VMXNET3_WRITE_BAR0_REG(adapter,
2865 				adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN,
2866 				adapter->rx_queue[i].rx_ring[0].next2fill);
2867 		VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset +
2868 				(i * VMXNET3_REG_ALIGN)),
2869 				adapter->rx_queue[i].rx_ring[1].next2fill);
2870 	}
2871 
2872 	/* Apply the rx filter settins last. */
2873 	vmxnet3_set_mc(adapter->netdev);
2874 
2875 	/*
2876 	 * Check link state when first activating device. It will start the
2877 	 * tx queue if the link is up.
2878 	 */
2879 	vmxnet3_check_link(adapter, true);
2880 	netif_tx_wake_all_queues(adapter->netdev);
2881 	for (i = 0; i < adapter->num_rx_queues; i++)
2882 		napi_enable(&adapter->rx_queue[i].napi);
2883 	vmxnet3_enable_all_intrs(adapter);
2884 	clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2885 	return 0;
2886 
2887 activate_err:
2888 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2889 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2890 	vmxnet3_free_irqs(adapter);
2891 irq_err:
2892 rq_err:
2893 	/* free up buffers we allocated */
2894 	vmxnet3_rq_cleanup_all(adapter);
2895 	return err;
2896 }
2897 
2898 
2899 void
2900 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2901 {
2902 	unsigned long flags;
2903 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2904 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2905 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2906 }
2907 
2908 
2909 int
2910 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2911 {
2912 	int i;
2913 	unsigned long flags;
2914 	if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2915 		return 0;
2916 
2917 
2918 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2919 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2920 			       VMXNET3_CMD_QUIESCE_DEV);
2921 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2922 	vmxnet3_disable_all_intrs(adapter);
2923 
2924 	for (i = 0; i < adapter->num_rx_queues; i++)
2925 		napi_disable(&adapter->rx_queue[i].napi);
2926 	netif_tx_disable(adapter->netdev);
2927 	adapter->link_speed = 0;
2928 	netif_carrier_off(adapter->netdev);
2929 
2930 	vmxnet3_tq_cleanup_all(adapter);
2931 	vmxnet3_rq_cleanup_all(adapter);
2932 	vmxnet3_free_irqs(adapter);
2933 	return 0;
2934 }
2935 
2936 
2937 static void
2938 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac)
2939 {
2940 	u32 tmp;
2941 
2942 	tmp = *(u32 *)mac;
2943 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2944 
2945 	tmp = (mac[5] << 8) | mac[4];
2946 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2947 }
2948 
2949 
2950 static int
2951 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2952 {
2953 	struct sockaddr *addr = p;
2954 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2955 
2956 	dev_addr_set(netdev, addr->sa_data);
2957 	vmxnet3_write_mac_addr(adapter, addr->sa_data);
2958 
2959 	return 0;
2960 }
2961 
2962 
2963 /* ==================== initialization and cleanup routines ============ */
2964 
2965 static int
2966 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter)
2967 {
2968 	int err;
2969 	unsigned long mmio_start, mmio_len;
2970 	struct pci_dev *pdev = adapter->pdev;
2971 
2972 	err = pci_enable_device(pdev);
2973 	if (err) {
2974 		dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
2975 		return err;
2976 	}
2977 
2978 	err = pci_request_selected_regions(pdev, (1 << 2) - 1,
2979 					   vmxnet3_driver_name);
2980 	if (err) {
2981 		dev_err(&pdev->dev,
2982 			"Failed to request region for adapter: error %d\n", err);
2983 		goto err_enable_device;
2984 	}
2985 
2986 	pci_set_master(pdev);
2987 
2988 	mmio_start = pci_resource_start(pdev, 0);
2989 	mmio_len = pci_resource_len(pdev, 0);
2990 	adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
2991 	if (!adapter->hw_addr0) {
2992 		dev_err(&pdev->dev, "Failed to map bar0\n");
2993 		err = -EIO;
2994 		goto err_ioremap;
2995 	}
2996 
2997 	mmio_start = pci_resource_start(pdev, 1);
2998 	mmio_len = pci_resource_len(pdev, 1);
2999 	adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
3000 	if (!adapter->hw_addr1) {
3001 		dev_err(&pdev->dev, "Failed to map bar1\n");
3002 		err = -EIO;
3003 		goto err_bar1;
3004 	}
3005 	return 0;
3006 
3007 err_bar1:
3008 	iounmap(adapter->hw_addr0);
3009 err_ioremap:
3010 	pci_release_selected_regions(pdev, (1 << 2) - 1);
3011 err_enable_device:
3012 	pci_disable_device(pdev);
3013 	return err;
3014 }
3015 
3016 
3017 static void
3018 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
3019 {
3020 	BUG_ON(!adapter->pdev);
3021 
3022 	iounmap(adapter->hw_addr0);
3023 	iounmap(adapter->hw_addr1);
3024 	pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
3025 	pci_disable_device(adapter->pdev);
3026 }
3027 
3028 
3029 static void
3030 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
3031 {
3032 	size_t sz, i, ring0_size, ring1_size, comp_size;
3033 	/* With version7 ring1 will have only T0 buffers */
3034 	if (!VMXNET3_VERSION_GE_7(adapter)) {
3035 		if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
3036 					    VMXNET3_MAX_ETH_HDR_SIZE) {
3037 			adapter->skb_buf_size = adapter->netdev->mtu +
3038 						VMXNET3_MAX_ETH_HDR_SIZE;
3039 			if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
3040 				adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
3041 
3042 			adapter->rx_buf_per_pkt = 1;
3043 		} else {
3044 			adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
3045 			sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
3046 						    VMXNET3_MAX_ETH_HDR_SIZE;
3047 			adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
3048 		}
3049 	} else {
3050 		adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE,
3051 					    VMXNET3_MAX_SKB_BUF_SIZE);
3052 		adapter->rx_buf_per_pkt = 1;
3053 		adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size);
3054 		adapter->ringBufSize.ring1BufSizeType1 = 0;
3055 		adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE);
3056 	}
3057 
3058 	/*
3059 	 * for simplicity, force the ring0 size to be a multiple of
3060 	 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
3061 	 */
3062 	sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
3063 	ring0_size = adapter->rx_queue[0].rx_ring[0].size;
3064 	ring0_size = (ring0_size + sz - 1) / sz * sz;
3065 	ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
3066 			   sz * sz);
3067 	ring1_size = adapter->rx_queue[0].rx_ring[1].size;
3068 	ring1_size = (ring1_size + sz - 1) / sz * sz;
3069 	ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE /
3070 			   sz * sz);
3071 	/* For v7 and later, keep ring size power of 2 for UPT */
3072 	if (VMXNET3_VERSION_GE_7(adapter)) {
3073 		ring0_size = rounddown_pow_of_two(ring0_size);
3074 		ring1_size = rounddown_pow_of_two(ring1_size);
3075 	}
3076 	comp_size = ring0_size + ring1_size;
3077 
3078 	for (i = 0; i < adapter->num_rx_queues; i++) {
3079 		struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[i];
3080 
3081 		rq->rx_ring[0].size = ring0_size;
3082 		rq->rx_ring[1].size = ring1_size;
3083 		rq->comp_ring.size = comp_size;
3084 	}
3085 }
3086 
3087 
3088 int
3089 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
3090 		      u32 rx_ring_size, u32 rx_ring2_size,
3091 		      u16 txdata_desc_size, u16 rxdata_desc_size)
3092 {
3093 	int err = 0, i;
3094 
3095 	for (i = 0; i < adapter->num_tx_queues; i++) {
3096 		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];
3097 		tq->tx_ring.size   = tx_ring_size;
3098 		tq->data_ring.size = tx_ring_size;
3099 		tq->comp_ring.size = tx_ring_size;
3100 		tq->txdata_desc_size = txdata_desc_size;
3101 		tq->shared = &adapter->tqd_start[i].ctrl;
3102 		tq->stopped = true;
3103 		tq->adapter = adapter;
3104 		tq->qid = i;
3105 		err = vmxnet3_tq_create(tq, adapter);
3106 		/*
3107 		 * Too late to change num_tx_queues. We cannot do away with
3108 		 * lesser number of queues than what we asked for
3109 		 */
3110 		if (err)
3111 			goto queue_err;
3112 	}
3113 
3114 	adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
3115 	adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
3116 	vmxnet3_adjust_rx_ring_size(adapter);
3117 
3118 	adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
3119 	for (i = 0; i < adapter->num_rx_queues; i++) {
3120 		struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
3121 		/* qid and qid2 for rx queues will be assigned later when num
3122 		 * of rx queues is finalized after allocating intrs */
3123 		rq->shared = &adapter->rqd_start[i].ctrl;
3124 		rq->adapter = adapter;
3125 		rq->data_ring.desc_size = rxdata_desc_size;
3126 		err = vmxnet3_rq_create(rq, adapter);
3127 		if (err) {
3128 			if (i == 0) {
3129 				netdev_err(adapter->netdev,
3130 					   "Could not allocate any rx queues. "
3131 					   "Aborting.\n");
3132 				goto queue_err;
3133 			} else {
3134 				netdev_info(adapter->netdev,
3135 					    "Number of rx queues changed "
3136 					    "to : %d.\n", i);
3137 				adapter->num_rx_queues = i;
3138 				err = 0;
3139 				break;
3140 			}
3141 		}
3142 	}
3143 
3144 	if (!adapter->rxdataring_enabled)
3145 		vmxnet3_rq_destroy_all_rxdataring(adapter);
3146 
3147 	return err;
3148 queue_err:
3149 	vmxnet3_tq_destroy_all(adapter);
3150 	return err;
3151 }
3152 
3153 static int
3154 vmxnet3_open(struct net_device *netdev)
3155 {
3156 	struct vmxnet3_adapter *adapter;
3157 	int err, i;
3158 
3159 	adapter = netdev_priv(netdev);
3160 
3161 	for (i = 0; i < adapter->num_tx_queues; i++)
3162 		spin_lock_init(&adapter->tx_queue[i].tx_lock);
3163 
3164 	if (VMXNET3_VERSION_GE_3(adapter)) {
3165 		unsigned long flags;
3166 		u16 txdata_desc_size;
3167 
3168 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3169 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3170 				       VMXNET3_CMD_GET_TXDATA_DESC_SIZE);
3171 		txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter,
3172 							 VMXNET3_REG_CMD);
3173 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3174 
3175 		if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) ||
3176 		    (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) ||
3177 		    (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) {
3178 			adapter->txdata_desc_size =
3179 				sizeof(struct Vmxnet3_TxDataDesc);
3180 		} else {
3181 			adapter->txdata_desc_size = txdata_desc_size;
3182 		}
3183 	} else {
3184 		adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc);
3185 	}
3186 
3187 	err = vmxnet3_create_queues(adapter,
3188 				    adapter->tx_ring_size,
3189 				    adapter->rx_ring_size,
3190 				    adapter->rx_ring2_size,
3191 				    adapter->txdata_desc_size,
3192 				    adapter->rxdata_desc_size);
3193 	if (err)
3194 		goto queue_err;
3195 
3196 	err = vmxnet3_activate_dev(adapter);
3197 	if (err)
3198 		goto activate_err;
3199 
3200 	return 0;
3201 
3202 activate_err:
3203 	vmxnet3_rq_destroy_all(adapter);
3204 	vmxnet3_tq_destroy_all(adapter);
3205 queue_err:
3206 	return err;
3207 }
3208 
3209 
3210 static int
3211 vmxnet3_close(struct net_device *netdev)
3212 {
3213 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3214 
3215 	/*
3216 	 * Reset_work may be in the middle of resetting the device, wait for its
3217 	 * completion.
3218 	 */
3219 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3220 		usleep_range(1000, 2000);
3221 
3222 	vmxnet3_quiesce_dev(adapter);
3223 
3224 	vmxnet3_rq_destroy_all(adapter);
3225 	vmxnet3_tq_destroy_all(adapter);
3226 
3227 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3228 
3229 
3230 	return 0;
3231 }
3232 
3233 
3234 void
3235 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
3236 {
3237 	int i;
3238 
3239 	/*
3240 	 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
3241 	 * vmxnet3_close() will deadlock.
3242 	 */
3243 	BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
3244 
3245 	/* we need to enable NAPI, otherwise dev_close will deadlock */
3246 	for (i = 0; i < adapter->num_rx_queues; i++)
3247 		napi_enable(&adapter->rx_queue[i].napi);
3248 	/*
3249 	 * Need to clear the quiesce bit to ensure that vmxnet3_close
3250 	 * can quiesce the device properly
3251 	 */
3252 	clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3253 	dev_close(adapter->netdev);
3254 }
3255 
3256 
3257 static int
3258 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
3259 {
3260 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3261 	int err = 0;
3262 
3263 	netdev->mtu = new_mtu;
3264 
3265 	/*
3266 	 * Reset_work may be in the middle of resetting the device, wait for its
3267 	 * completion.
3268 	 */
3269 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3270 		usleep_range(1000, 2000);
3271 
3272 	if (netif_running(netdev)) {
3273 		vmxnet3_quiesce_dev(adapter);
3274 		vmxnet3_reset_dev(adapter);
3275 
3276 		/* we need to re-create the rx queue based on the new mtu */
3277 		vmxnet3_rq_destroy_all(adapter);
3278 		vmxnet3_adjust_rx_ring_size(adapter);
3279 		err = vmxnet3_rq_create_all(adapter);
3280 		if (err) {
3281 			netdev_err(netdev,
3282 				   "failed to re-create rx queues, "
3283 				   " error %d. Closing it.\n", err);
3284 			goto out;
3285 		}
3286 
3287 		err = vmxnet3_activate_dev(adapter);
3288 		if (err) {
3289 			netdev_err(netdev,
3290 				   "failed to re-activate, error %d. "
3291 				   "Closing it\n", err);
3292 			goto out;
3293 		}
3294 	}
3295 
3296 out:
3297 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3298 	if (err)
3299 		vmxnet3_force_close(adapter);
3300 
3301 	return err;
3302 }
3303 
3304 
3305 static void
3306 vmxnet3_declare_features(struct vmxnet3_adapter *adapter)
3307 {
3308 	struct net_device *netdev = adapter->netdev;
3309 
3310 	netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
3311 		NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
3312 		NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
3313 		NETIF_F_LRO | NETIF_F_HIGHDMA;
3314 
3315 	if (VMXNET3_VERSION_GE_4(adapter)) {
3316 		netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
3317 				NETIF_F_GSO_UDP_TUNNEL_CSUM;
3318 
3319 		netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM |
3320 			NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
3321 			NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
3322 			NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL |
3323 			NETIF_F_GSO_UDP_TUNNEL_CSUM;
3324 	}
3325 
3326 	if (VMXNET3_VERSION_GE_7(adapter)) {
3327 		unsigned long flags;
3328 
3329 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3330 					       VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) {
3331 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD;
3332 		}
3333 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3334 					       VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) {
3335 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD;
3336 		}
3337 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3338 					       VMXNET3_CAP_GENEVE_TSO)) {
3339 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO;
3340 		}
3341 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3342 					       VMXNET3_CAP_VXLAN_TSO)) {
3343 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO;
3344 		}
3345 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3346 					       VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) {
3347 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD;
3348 		}
3349 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3350 					       VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) {
3351 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD;
3352 		}
3353 
3354 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
3355 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3356 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
3357 		adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3358 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3359 
3360 		if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) &&
3361 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) &&
3362 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) &&
3363 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) {
3364 			netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL;
3365 			netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL;
3366 		}
3367 		if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) &&
3368 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) {
3369 			netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
3370 			netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
3371 		}
3372 	}
3373 
3374 	netdev->vlan_features = netdev->hw_features &
3375 				~(NETIF_F_HW_VLAN_CTAG_TX |
3376 				  NETIF_F_HW_VLAN_CTAG_RX);
3377 	netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
3378 }
3379 
3380 
3381 static void
3382 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
3383 {
3384 	u32 tmp;
3385 
3386 	tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
3387 	*(u32 *)mac = tmp;
3388 
3389 	tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
3390 	mac[4] = tmp & 0xff;
3391 	mac[5] = (tmp >> 8) & 0xff;
3392 }
3393 
3394 #ifdef CONFIG_PCI_MSI
3395 
3396 /*
3397  * Enable MSIx vectors.
3398  * Returns :
3399  *	VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
3400  *	 were enabled.
3401  *	number of vectors which were enabled otherwise (this number is greater
3402  *	 than VMXNET3_LINUX_MIN_MSIX_VECT)
3403  */
3404 
3405 static int
3406 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec)
3407 {
3408 	int ret = pci_enable_msix_range(adapter->pdev,
3409 					adapter->intr.msix_entries, nvec, nvec);
3410 
3411 	if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) {
3412 		dev_err(&adapter->netdev->dev,
3413 			"Failed to enable %d MSI-X, trying %d\n",
3414 			nvec, VMXNET3_LINUX_MIN_MSIX_VECT);
3415 
3416 		ret = pci_enable_msix_range(adapter->pdev,
3417 					    adapter->intr.msix_entries,
3418 					    VMXNET3_LINUX_MIN_MSIX_VECT,
3419 					    VMXNET3_LINUX_MIN_MSIX_VECT);
3420 	}
3421 
3422 	if (ret < 0) {
3423 		dev_err(&adapter->netdev->dev,
3424 			"Failed to enable MSI-X, error: %d\n", ret);
3425 	}
3426 
3427 	return ret;
3428 }
3429 
3430 
3431 #endif /* CONFIG_PCI_MSI */
3432 
3433 static void
3434 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
3435 {
3436 	u32 cfg;
3437 	unsigned long flags;
3438 
3439 	/* intr settings */
3440 	spin_lock_irqsave(&adapter->cmd_lock, flags);
3441 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3442 			       VMXNET3_CMD_GET_CONF_INTR);
3443 	cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3444 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3445 	adapter->intr.type = cfg & 0x3;
3446 	adapter->intr.mask_mode = (cfg >> 2) & 0x3;
3447 
3448 	if (adapter->intr.type == VMXNET3_IT_AUTO) {
3449 		adapter->intr.type = VMXNET3_IT_MSIX;
3450 	}
3451 
3452 #ifdef CONFIG_PCI_MSI
3453 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
3454 		int i, nvec, nvec_allocated;
3455 
3456 		nvec  = adapter->share_intr == VMXNET3_INTR_TXSHARE ?
3457 			1 : adapter->num_tx_queues;
3458 		nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ?
3459 			0 : adapter->num_rx_queues;
3460 		nvec += 1;	/* for link event */
3461 		nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ?
3462 		       nvec : VMXNET3_LINUX_MIN_MSIX_VECT;
3463 
3464 		for (i = 0; i < nvec; i++)
3465 			adapter->intr.msix_entries[i].entry = i;
3466 
3467 		nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec);
3468 		if (nvec_allocated < 0)
3469 			goto msix_err;
3470 
3471 		/* If we cannot allocate one MSIx vector per queue
3472 		 * then limit the number of rx queues to 1
3473 		 */
3474 		if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT &&
3475 		    nvec != VMXNET3_LINUX_MIN_MSIX_VECT) {
3476 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
3477 			    || adapter->num_rx_queues != 1) {
3478 				adapter->share_intr = VMXNET3_INTR_TXSHARE;
3479 				netdev_err(adapter->netdev,
3480 					   "Number of rx queues : 1\n");
3481 				adapter->num_rx_queues = 1;
3482 			}
3483 		}
3484 
3485 		adapter->intr.num_intrs = nvec_allocated;
3486 		return;
3487 
3488 msix_err:
3489 		/* If we cannot allocate MSIx vectors use only one rx queue */
3490 		dev_info(&adapter->pdev->dev,
3491 			 "Failed to enable MSI-X, error %d. "
3492 			 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated);
3493 
3494 		adapter->intr.type = VMXNET3_IT_MSI;
3495 	}
3496 
3497 	if (adapter->intr.type == VMXNET3_IT_MSI) {
3498 		if (!pci_enable_msi(adapter->pdev)) {
3499 			adapter->num_rx_queues = 1;
3500 			adapter->intr.num_intrs = 1;
3501 			return;
3502 		}
3503 	}
3504 #endif /* CONFIG_PCI_MSI */
3505 
3506 	adapter->num_rx_queues = 1;
3507 	dev_info(&adapter->netdev->dev,
3508 		 "Using INTx interrupt, #Rx queues: 1.\n");
3509 	adapter->intr.type = VMXNET3_IT_INTX;
3510 
3511 	/* INT-X related setting */
3512 	adapter->intr.num_intrs = 1;
3513 }
3514 
3515 
3516 static void
3517 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
3518 {
3519 	if (adapter->intr.type == VMXNET3_IT_MSIX)
3520 		pci_disable_msix(adapter->pdev);
3521 	else if (adapter->intr.type == VMXNET3_IT_MSI)
3522 		pci_disable_msi(adapter->pdev);
3523 	else
3524 		BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
3525 }
3526 
3527 
3528 static void
3529 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue)
3530 {
3531 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3532 	adapter->tx_timeout_count++;
3533 
3534 	netdev_err(adapter->netdev, "tx hang\n");
3535 	schedule_work(&adapter->work);
3536 }
3537 
3538 
3539 static void
3540 vmxnet3_reset_work(struct work_struct *data)
3541 {
3542 	struct vmxnet3_adapter *adapter;
3543 
3544 	adapter = container_of(data, struct vmxnet3_adapter, work);
3545 
3546 	/* if another thread is resetting the device, no need to proceed */
3547 	if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3548 		return;
3549 
3550 	/* if the device is closed, we must leave it alone */
3551 	rtnl_lock();
3552 	if (netif_running(adapter->netdev)) {
3553 		netdev_notice(adapter->netdev, "resetting\n");
3554 		vmxnet3_quiesce_dev(adapter);
3555 		vmxnet3_reset_dev(adapter);
3556 		vmxnet3_activate_dev(adapter);
3557 	} else {
3558 		netdev_info(adapter->netdev, "already closed\n");
3559 	}
3560 	rtnl_unlock();
3561 
3562 	netif_wake_queue(adapter->netdev);
3563 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3564 }
3565 
3566 
3567 static int
3568 vmxnet3_probe_device(struct pci_dev *pdev,
3569 		     const struct pci_device_id *id)
3570 {
3571 	static const struct net_device_ops vmxnet3_netdev_ops = {
3572 		.ndo_open = vmxnet3_open,
3573 		.ndo_stop = vmxnet3_close,
3574 		.ndo_start_xmit = vmxnet3_xmit_frame,
3575 		.ndo_set_mac_address = vmxnet3_set_mac_addr,
3576 		.ndo_change_mtu = vmxnet3_change_mtu,
3577 		.ndo_fix_features = vmxnet3_fix_features,
3578 		.ndo_set_features = vmxnet3_set_features,
3579 		.ndo_features_check = vmxnet3_features_check,
3580 		.ndo_get_stats64 = vmxnet3_get_stats64,
3581 		.ndo_tx_timeout = vmxnet3_tx_timeout,
3582 		.ndo_set_rx_mode = vmxnet3_set_mc,
3583 		.ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
3584 		.ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
3585 #ifdef CONFIG_NET_POLL_CONTROLLER
3586 		.ndo_poll_controller = vmxnet3_netpoll,
3587 #endif
3588 	};
3589 	int err;
3590 	u32 ver;
3591 	struct net_device *netdev;
3592 	struct vmxnet3_adapter *adapter;
3593 	u8 mac[ETH_ALEN];
3594 	int size;
3595 	int num_tx_queues;
3596 	int num_rx_queues;
3597 	int queues;
3598 	unsigned long flags;
3599 
3600 	if (!pci_msi_enabled())
3601 		enable_mq = 0;
3602 
3603 #ifdef VMXNET3_RSS
3604 	if (enable_mq)
3605 		num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3606 				    (int)num_online_cpus());
3607 	else
3608 #endif
3609 		num_rx_queues = 1;
3610 
3611 	if (enable_mq)
3612 		num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
3613 				    (int)num_online_cpus());
3614 	else
3615 		num_tx_queues = 1;
3616 
3617 	netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
3618 				   max(num_tx_queues, num_rx_queues));
3619 	if (!netdev)
3620 		return -ENOMEM;
3621 
3622 	pci_set_drvdata(pdev, netdev);
3623 	adapter = netdev_priv(netdev);
3624 	adapter->netdev = netdev;
3625 	adapter->pdev = pdev;
3626 
3627 	adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE;
3628 	adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE;
3629 	adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE;
3630 
3631 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3632 	if (err) {
3633 		dev_err(&pdev->dev, "dma_set_mask failed\n");
3634 		goto err_set_mask;
3635 	}
3636 
3637 	spin_lock_init(&adapter->cmd_lock);
3638 	adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
3639 					     sizeof(struct vmxnet3_adapter),
3640 					     DMA_TO_DEVICE);
3641 	if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) {
3642 		dev_err(&pdev->dev, "Failed to map dma\n");
3643 		err = -EFAULT;
3644 		goto err_set_mask;
3645 	}
3646 	adapter->shared = dma_alloc_coherent(
3647 				&adapter->pdev->dev,
3648 				sizeof(struct Vmxnet3_DriverShared),
3649 				&adapter->shared_pa, GFP_KERNEL);
3650 	if (!adapter->shared) {
3651 		dev_err(&pdev->dev, "Failed to allocate memory\n");
3652 		err = -ENOMEM;
3653 		goto err_alloc_shared;
3654 	}
3655 
3656 	err = vmxnet3_alloc_pci_resources(adapter);
3657 	if (err < 0)
3658 		goto err_alloc_pci;
3659 
3660 	ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3661 	if (ver & (1 << VMXNET3_REV_7)) {
3662 		VMXNET3_WRITE_BAR1_REG(adapter,
3663 				       VMXNET3_REG_VRRS,
3664 				       1 << VMXNET3_REV_7);
3665 		adapter->version = VMXNET3_REV_7 + 1;
3666 	} else if (ver & (1 << VMXNET3_REV_6)) {
3667 		VMXNET3_WRITE_BAR1_REG(adapter,
3668 				       VMXNET3_REG_VRRS,
3669 				       1 << VMXNET3_REV_6);
3670 		adapter->version = VMXNET3_REV_6 + 1;
3671 	} else if (ver & (1 << VMXNET3_REV_5)) {
3672 		VMXNET3_WRITE_BAR1_REG(adapter,
3673 				       VMXNET3_REG_VRRS,
3674 				       1 << VMXNET3_REV_5);
3675 		adapter->version = VMXNET3_REV_5 + 1;
3676 	} else if (ver & (1 << VMXNET3_REV_4)) {
3677 		VMXNET3_WRITE_BAR1_REG(adapter,
3678 				       VMXNET3_REG_VRRS,
3679 				       1 << VMXNET3_REV_4);
3680 		adapter->version = VMXNET3_REV_4 + 1;
3681 	} else if (ver & (1 << VMXNET3_REV_3)) {
3682 		VMXNET3_WRITE_BAR1_REG(adapter,
3683 				       VMXNET3_REG_VRRS,
3684 				       1 << VMXNET3_REV_3);
3685 		adapter->version = VMXNET3_REV_3 + 1;
3686 	} else if (ver & (1 << VMXNET3_REV_2)) {
3687 		VMXNET3_WRITE_BAR1_REG(adapter,
3688 				       VMXNET3_REG_VRRS,
3689 				       1 << VMXNET3_REV_2);
3690 		adapter->version = VMXNET3_REV_2 + 1;
3691 	} else if (ver & (1 << VMXNET3_REV_1)) {
3692 		VMXNET3_WRITE_BAR1_REG(adapter,
3693 				       VMXNET3_REG_VRRS,
3694 				       1 << VMXNET3_REV_1);
3695 		adapter->version = VMXNET3_REV_1 + 1;
3696 	} else {
3697 		dev_err(&pdev->dev,
3698 			"Incompatible h/w version (0x%x) for adapter\n", ver);
3699 		err = -EBUSY;
3700 		goto err_ver;
3701 	}
3702 	dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version);
3703 
3704 	ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3705 	if (ver & 1) {
3706 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3707 	} else {
3708 		dev_err(&pdev->dev,
3709 			"Incompatible upt version (0x%x) for adapter\n", ver);
3710 		err = -EBUSY;
3711 		goto err_ver;
3712 	}
3713 
3714 	if (VMXNET3_VERSION_GE_7(adapter)) {
3715 		adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR);
3716 		adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR);
3717 		if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) {
3718 			adapter->dev_caps[0] = adapter->devcap_supported[0] &
3719 							(1UL << VMXNET3_CAP_LARGE_BAR);
3720 		}
3721 		if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) &&
3722 		    adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) &&
3723 		    adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) {
3724 			adapter->dev_caps[0] |= adapter->devcap_supported[0] &
3725 						(1UL << VMXNET3_CAP_OOORX_COMP);
3726 		}
3727 		if (adapter->dev_caps[0])
3728 			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
3729 
3730 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3731 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
3732 		adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3733 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3734 	}
3735 
3736 	if (VMXNET3_VERSION_GE_7(adapter) &&
3737 	    adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) {
3738 		adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD;
3739 		adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD;
3740 		adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2;
3741 	} else {
3742 		adapter->tx_prod_offset = VMXNET3_REG_TXPROD;
3743 		adapter->rx_prod_offset = VMXNET3_REG_RXPROD;
3744 		adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2;
3745 	}
3746 
3747 	if (VMXNET3_VERSION_GE_6(adapter)) {
3748 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3749 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3750 				       VMXNET3_CMD_GET_MAX_QUEUES_CONF);
3751 		queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3752 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3753 		if (queues > 0) {
3754 			adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff));
3755 			adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff));
3756 		} else {
3757 			adapter->num_rx_queues = min(num_rx_queues,
3758 						     VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3759 			adapter->num_tx_queues = min(num_tx_queues,
3760 						     VMXNET3_DEVICE_DEFAULT_TX_QUEUES);
3761 		}
3762 		if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES ||
3763 		    adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) {
3764 			adapter->queuesExtEnabled = true;
3765 		} else {
3766 			adapter->queuesExtEnabled = false;
3767 		}
3768 	} else {
3769 		adapter->queuesExtEnabled = false;
3770 		num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3771 		num_tx_queues = rounddown_pow_of_two(num_tx_queues);
3772 		adapter->num_rx_queues = min(num_rx_queues,
3773 					     VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3774 		adapter->num_tx_queues = min(num_tx_queues,
3775 					     VMXNET3_DEVICE_DEFAULT_TX_QUEUES);
3776 	}
3777 	dev_info(&pdev->dev,
3778 		 "# of Tx queues : %d, # of Rx queues : %d\n",
3779 		 adapter->num_tx_queues, adapter->num_rx_queues);
3780 
3781 	adapter->rx_buf_per_pkt = 1;
3782 
3783 	size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3784 	size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
3785 	adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
3786 						&adapter->queue_desc_pa,
3787 						GFP_KERNEL);
3788 
3789 	if (!adapter->tqd_start) {
3790 		dev_err(&pdev->dev, "Failed to allocate memory\n");
3791 		err = -ENOMEM;
3792 		goto err_ver;
3793 	}
3794 	adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3795 							    adapter->num_tx_queues);
3796 
3797 	adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3798 					      sizeof(struct Vmxnet3_PMConf),
3799 					      &adapter->pm_conf_pa,
3800 					      GFP_KERNEL);
3801 	if (adapter->pm_conf == NULL) {
3802 		err = -ENOMEM;
3803 		goto err_alloc_pm;
3804 	}
3805 
3806 #ifdef VMXNET3_RSS
3807 
3808 	adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3809 					       sizeof(struct UPT1_RSSConf),
3810 					       &adapter->rss_conf_pa,
3811 					       GFP_KERNEL);
3812 	if (adapter->rss_conf == NULL) {
3813 		err = -ENOMEM;
3814 		goto err_alloc_rss;
3815 	}
3816 #endif /* VMXNET3_RSS */
3817 
3818 	if (VMXNET3_VERSION_GE_3(adapter)) {
3819 		adapter->coal_conf =
3820 			dma_alloc_coherent(&adapter->pdev->dev,
3821 					   sizeof(struct Vmxnet3_CoalesceScheme)
3822 					   ,
3823 					   &adapter->coal_conf_pa,
3824 					   GFP_KERNEL);
3825 		if (!adapter->coal_conf) {
3826 			err = -ENOMEM;
3827 			goto err_coal_conf;
3828 		}
3829 		adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED;
3830 		adapter->default_coal_mode = true;
3831 	}
3832 
3833 	if (VMXNET3_VERSION_GE_4(adapter)) {
3834 		adapter->default_rss_fields = true;
3835 		adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT;
3836 	}
3837 
3838 	SET_NETDEV_DEV(netdev, &pdev->dev);
3839 	vmxnet3_declare_features(adapter);
3840 
3841 	adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ?
3842 		VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
3843 
3844 	if (adapter->num_tx_queues == adapter->num_rx_queues)
3845 		adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3846 	else
3847 		adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3848 
3849 	vmxnet3_alloc_intr_resources(adapter);
3850 
3851 #ifdef VMXNET3_RSS
3852 	if (adapter->num_rx_queues > 1 &&
3853 	    adapter->intr.type == VMXNET3_IT_MSIX) {
3854 		adapter->rss = true;
3855 		netdev->hw_features |= NETIF_F_RXHASH;
3856 		netdev->features |= NETIF_F_RXHASH;
3857 		dev_dbg(&pdev->dev, "RSS is enabled.\n");
3858 	} else {
3859 		adapter->rss = false;
3860 	}
3861 #endif
3862 
3863 	vmxnet3_read_mac_addr(adapter, mac);
3864 	dev_addr_set(netdev, mac);
3865 
3866 	netdev->netdev_ops = &vmxnet3_netdev_ops;
3867 	vmxnet3_set_ethtool_ops(netdev);
3868 	netdev->watchdog_timeo = 5 * HZ;
3869 
3870 	/* MTU range: 60 - 9190 */
3871 	netdev->min_mtu = VMXNET3_MIN_MTU;
3872 	if (VMXNET3_VERSION_GE_6(adapter))
3873 		netdev->max_mtu = VMXNET3_V6_MAX_MTU;
3874 	else
3875 		netdev->max_mtu = VMXNET3_MAX_MTU;
3876 
3877 	INIT_WORK(&adapter->work, vmxnet3_reset_work);
3878 	set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3879 
3880 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
3881 		int i;
3882 		for (i = 0; i < adapter->num_rx_queues; i++) {
3883 			netif_napi_add(adapter->netdev,
3884 				       &adapter->rx_queue[i].napi,
3885 				       vmxnet3_poll_rx_only, 64);
3886 		}
3887 	} else {
3888 		netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3889 			       vmxnet3_poll, 64);
3890 	}
3891 
3892 	netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3893 	netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3894 
3895 	netif_carrier_off(netdev);
3896 	err = register_netdev(netdev);
3897 
3898 	if (err) {
3899 		dev_err(&pdev->dev, "Failed to register adapter\n");
3900 		goto err_register;
3901 	}
3902 
3903 	vmxnet3_check_link(adapter, false);
3904 	return 0;
3905 
3906 err_register:
3907 	if (VMXNET3_VERSION_GE_3(adapter)) {
3908 		dma_free_coherent(&adapter->pdev->dev,
3909 				  sizeof(struct Vmxnet3_CoalesceScheme),
3910 				  adapter->coal_conf, adapter->coal_conf_pa);
3911 	}
3912 	vmxnet3_free_intr_resources(adapter);
3913 err_coal_conf:
3914 #ifdef VMXNET3_RSS
3915 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3916 			  adapter->rss_conf, adapter->rss_conf_pa);
3917 err_alloc_rss:
3918 #endif
3919 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3920 			  adapter->pm_conf, adapter->pm_conf_pa);
3921 err_alloc_pm:
3922 	dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3923 			  adapter->queue_desc_pa);
3924 err_ver:
3925 	vmxnet3_free_pci_resources(adapter);
3926 err_alloc_pci:
3927 	dma_free_coherent(&adapter->pdev->dev,
3928 			  sizeof(struct Vmxnet3_DriverShared),
3929 			  adapter->shared, adapter->shared_pa);
3930 err_alloc_shared:
3931 	dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3932 			 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE);
3933 err_set_mask:
3934 	free_netdev(netdev);
3935 	return err;
3936 }
3937 
3938 
3939 static void
3940 vmxnet3_remove_device(struct pci_dev *pdev)
3941 {
3942 	struct net_device *netdev = pci_get_drvdata(pdev);
3943 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3944 	int size = 0;
3945 	int num_rx_queues, rx_queues;
3946 	unsigned long flags;
3947 
3948 #ifdef VMXNET3_RSS
3949 	if (enable_mq)
3950 		num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3951 				    (int)num_online_cpus());
3952 	else
3953 #endif
3954 		num_rx_queues = 1;
3955 	if (!VMXNET3_VERSION_GE_6(adapter)) {
3956 		num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3957 	}
3958 	if (VMXNET3_VERSION_GE_6(adapter)) {
3959 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3960 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3961 				       VMXNET3_CMD_GET_MAX_QUEUES_CONF);
3962 		rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3963 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3964 		if (rx_queues > 0)
3965 			rx_queues = (rx_queues >> 8) & 0xff;
3966 		else
3967 			rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3968 		num_rx_queues = min(num_rx_queues, rx_queues);
3969 	} else {
3970 		num_rx_queues = min(num_rx_queues,
3971 				    VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3972 	}
3973 
3974 	cancel_work_sync(&adapter->work);
3975 
3976 	unregister_netdev(netdev);
3977 
3978 	vmxnet3_free_intr_resources(adapter);
3979 	vmxnet3_free_pci_resources(adapter);
3980 	if (VMXNET3_VERSION_GE_3(adapter)) {
3981 		dma_free_coherent(&adapter->pdev->dev,
3982 				  sizeof(struct Vmxnet3_CoalesceScheme),
3983 				  adapter->coal_conf, adapter->coal_conf_pa);
3984 	}
3985 #ifdef VMXNET3_RSS
3986 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3987 			  adapter->rss_conf, adapter->rss_conf_pa);
3988 #endif
3989 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3990 			  adapter->pm_conf, adapter->pm_conf_pa);
3991 
3992 	size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3993 	size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
3994 	dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3995 			  adapter->queue_desc_pa);
3996 	dma_free_coherent(&adapter->pdev->dev,
3997 			  sizeof(struct Vmxnet3_DriverShared),
3998 			  adapter->shared, adapter->shared_pa);
3999 	dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
4000 			 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE);
4001 	free_netdev(netdev);
4002 }
4003 
4004 static void vmxnet3_shutdown_device(struct pci_dev *pdev)
4005 {
4006 	struct net_device *netdev = pci_get_drvdata(pdev);
4007 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4008 	unsigned long flags;
4009 
4010 	/* Reset_work may be in the middle of resetting the device, wait for its
4011 	 * completion.
4012 	 */
4013 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
4014 		usleep_range(1000, 2000);
4015 
4016 	if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED,
4017 			     &adapter->state)) {
4018 		clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
4019 		return;
4020 	}
4021 	spin_lock_irqsave(&adapter->cmd_lock, flags);
4022 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4023 			       VMXNET3_CMD_QUIESCE_DEV);
4024 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4025 	vmxnet3_disable_all_intrs(adapter);
4026 
4027 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
4028 }
4029 
4030 
4031 #ifdef CONFIG_PM
4032 
4033 static int
4034 vmxnet3_suspend(struct device *device)
4035 {
4036 	struct pci_dev *pdev = to_pci_dev(device);
4037 	struct net_device *netdev = pci_get_drvdata(pdev);
4038 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4039 	struct Vmxnet3_PMConf *pmConf;
4040 	struct ethhdr *ehdr;
4041 	struct arphdr *ahdr;
4042 	u8 *arpreq;
4043 	struct in_device *in_dev;
4044 	struct in_ifaddr *ifa;
4045 	unsigned long flags;
4046 	int i = 0;
4047 
4048 	if (!netif_running(netdev))
4049 		return 0;
4050 
4051 	for (i = 0; i < adapter->num_rx_queues; i++)
4052 		napi_disable(&adapter->rx_queue[i].napi);
4053 
4054 	vmxnet3_disable_all_intrs(adapter);
4055 	vmxnet3_free_irqs(adapter);
4056 	vmxnet3_free_intr_resources(adapter);
4057 
4058 	netif_device_detach(netdev);
4059 
4060 	/* Create wake-up filters. */
4061 	pmConf = adapter->pm_conf;
4062 	memset(pmConf, 0, sizeof(*pmConf));
4063 
4064 	if (adapter->wol & WAKE_UCAST) {
4065 		pmConf->filters[i].patternSize = ETH_ALEN;
4066 		pmConf->filters[i].maskSize = 1;
4067 		memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
4068 		pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
4069 
4070 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
4071 		i++;
4072 	}
4073 
4074 	if (adapter->wol & WAKE_ARP) {
4075 		rcu_read_lock();
4076 
4077 		in_dev = __in_dev_get_rcu(netdev);
4078 		if (!in_dev) {
4079 			rcu_read_unlock();
4080 			goto skip_arp;
4081 		}
4082 
4083 		ifa = rcu_dereference(in_dev->ifa_list);
4084 		if (!ifa) {
4085 			rcu_read_unlock();
4086 			goto skip_arp;
4087 		}
4088 
4089 		pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
4090 			sizeof(struct arphdr) +		/* ARP header */
4091 			2 * ETH_ALEN +		/* 2 Ethernet addresses*/
4092 			2 * sizeof(u32);	/*2 IPv4 addresses */
4093 		pmConf->filters[i].maskSize =
4094 			(pmConf->filters[i].patternSize - 1) / 8 + 1;
4095 
4096 		/* ETH_P_ARP in Ethernet header. */
4097 		ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
4098 		ehdr->h_proto = htons(ETH_P_ARP);
4099 
4100 		/* ARPOP_REQUEST in ARP header. */
4101 		ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
4102 		ahdr->ar_op = htons(ARPOP_REQUEST);
4103 		arpreq = (u8 *)(ahdr + 1);
4104 
4105 		/* The Unicast IPv4 address in 'tip' field. */
4106 		arpreq += 2 * ETH_ALEN + sizeof(u32);
4107 		*(__be32 *)arpreq = ifa->ifa_address;
4108 
4109 		rcu_read_unlock();
4110 
4111 		/* The mask for the relevant bits. */
4112 		pmConf->filters[i].mask[0] = 0x00;
4113 		pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
4114 		pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
4115 		pmConf->filters[i].mask[3] = 0x00;
4116 		pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
4117 		pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
4118 
4119 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
4120 		i++;
4121 	}
4122 
4123 skip_arp:
4124 	if (adapter->wol & WAKE_MAGIC)
4125 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
4126 
4127 	pmConf->numFilters = i;
4128 
4129 	adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
4130 	adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
4131 								  *pmConf));
4132 	adapter->shared->devRead.pmConfDesc.confPA =
4133 		cpu_to_le64(adapter->pm_conf_pa);
4134 
4135 	spin_lock_irqsave(&adapter->cmd_lock, flags);
4136 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4137 			       VMXNET3_CMD_UPDATE_PMCFG);
4138 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4139 
4140 	pci_save_state(pdev);
4141 	pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
4142 			adapter->wol);
4143 	pci_disable_device(pdev);
4144 	pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
4145 
4146 	return 0;
4147 }
4148 
4149 
4150 static int
4151 vmxnet3_resume(struct device *device)
4152 {
4153 	int err;
4154 	unsigned long flags;
4155 	struct pci_dev *pdev = to_pci_dev(device);
4156 	struct net_device *netdev = pci_get_drvdata(pdev);
4157 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4158 
4159 	if (!netif_running(netdev))
4160 		return 0;
4161 
4162 	pci_set_power_state(pdev, PCI_D0);
4163 	pci_restore_state(pdev);
4164 	err = pci_enable_device_mem(pdev);
4165 	if (err != 0)
4166 		return err;
4167 
4168 	pci_enable_wake(pdev, PCI_D0, 0);
4169 
4170 	vmxnet3_alloc_intr_resources(adapter);
4171 
4172 	/* During hibernate and suspend, device has to be reinitialized as the
4173 	 * device state need not be preserved.
4174 	 */
4175 
4176 	/* Need not check adapter state as other reset tasks cannot run during
4177 	 * device resume.
4178 	 */
4179 	spin_lock_irqsave(&adapter->cmd_lock, flags);
4180 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4181 			       VMXNET3_CMD_QUIESCE_DEV);
4182 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4183 	vmxnet3_tq_cleanup_all(adapter);
4184 	vmxnet3_rq_cleanup_all(adapter);
4185 
4186 	vmxnet3_reset_dev(adapter);
4187 	err = vmxnet3_activate_dev(adapter);
4188 	if (err != 0) {
4189 		netdev_err(netdev,
4190 			   "failed to re-activate on resume, error: %d", err);
4191 		vmxnet3_force_close(adapter);
4192 		return err;
4193 	}
4194 	netif_device_attach(netdev);
4195 
4196 	return 0;
4197 }
4198 
4199 static const struct dev_pm_ops vmxnet3_pm_ops = {
4200 	.suspend = vmxnet3_suspend,
4201 	.resume = vmxnet3_resume,
4202 	.freeze = vmxnet3_suspend,
4203 	.restore = vmxnet3_resume,
4204 };
4205 #endif
4206 
4207 static struct pci_driver vmxnet3_driver = {
4208 	.name		= vmxnet3_driver_name,
4209 	.id_table	= vmxnet3_pciid_table,
4210 	.probe		= vmxnet3_probe_device,
4211 	.remove		= vmxnet3_remove_device,
4212 	.shutdown	= vmxnet3_shutdown_device,
4213 #ifdef CONFIG_PM
4214 	.driver.pm	= &vmxnet3_pm_ops,
4215 #endif
4216 };
4217 
4218 
4219 static int __init
4220 vmxnet3_init_module(void)
4221 {
4222 	pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
4223 		VMXNET3_DRIVER_VERSION_REPORT);
4224 	return pci_register_driver(&vmxnet3_driver);
4225 }
4226 
4227 module_init(vmxnet3_init_module);
4228 
4229 
4230 static void
4231 vmxnet3_exit_module(void)
4232 {
4233 	pci_unregister_driver(&vmxnet3_driver);
4234 }
4235 
4236 module_exit(vmxnet3_exit_module);
4237 
4238 MODULE_AUTHOR("VMware, Inc.");
4239 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
4240 MODULE_LICENSE("GPL v2");
4241 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
4242