1 /*
2  * Linux driver for VMware's vmxnet3 ethernet NIC.
3  *
4  * Copyright (C) 2008-2022, VMware, Inc. All Rights Reserved.
5  *
6  * This program is free software; you can redistribute it and/or modify it
7  * under the terms of the GNU General Public License as published by the
8  * Free Software Foundation; version 2 of the License and no later version.
9  *
10  * This program is distributed in the hope that it will be useful, but
11  * WITHOUT ANY WARRANTY; without even the implied warranty of
12  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
13  * NON INFRINGEMENT. See the GNU General Public License for more
14  * details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
19  *
20  * The full GNU General Public License is included in this distribution in
21  * the file called "COPYING".
22  *
23  * Maintained by: pv-drivers@vmware.com
24  *
25  */
26 
27 #include <linux/module.h>
28 #include <net/ip6_checksum.h>
29 
30 #include "vmxnet3_int.h"
31 
32 char vmxnet3_driver_name[] = "vmxnet3";
33 #define VMXNET3_DRIVER_DESC "VMware vmxnet3 virtual NIC driver"
34 
35 /*
36  * PCI Device ID Table
37  * Last entry must be all 0s
38  */
39 static const struct pci_device_id vmxnet3_pciid_table[] = {
40 	{PCI_VDEVICE(VMWARE, PCI_DEVICE_ID_VMWARE_VMXNET3)},
41 	{0}
42 };
43 
44 MODULE_DEVICE_TABLE(pci, vmxnet3_pciid_table);
45 
46 static int enable_mq = 1;
47 
48 static void
49 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac);
50 
51 /*
52  *    Enable/Disable the given intr
53  */
54 static void
55 vmxnet3_enable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
56 {
57 	VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 0);
58 }
59 
60 
61 static void
62 vmxnet3_disable_intr(struct vmxnet3_adapter *adapter, unsigned intr_idx)
63 {
64 	VMXNET3_WRITE_BAR0_REG(adapter, VMXNET3_REG_IMR + intr_idx * 8, 1);
65 }
66 
67 
68 /*
69  *    Enable/Disable all intrs used by the device
70  */
71 static void
72 vmxnet3_enable_all_intrs(struct vmxnet3_adapter *adapter)
73 {
74 	int i;
75 
76 	for (i = 0; i < adapter->intr.num_intrs; i++)
77 		vmxnet3_enable_intr(adapter, i);
78 	if (!VMXNET3_VERSION_GE_6(adapter) ||
79 	    !adapter->queuesExtEnabled) {
80 		adapter->shared->devRead.intrConf.intrCtrl &=
81 					cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
82 	} else {
83 		adapter->shared->devReadExt.intrConfExt.intrCtrl &=
84 					cpu_to_le32(~VMXNET3_IC_DISABLE_ALL);
85 	}
86 }
87 
88 
89 static void
90 vmxnet3_disable_all_intrs(struct vmxnet3_adapter *adapter)
91 {
92 	int i;
93 
94 	if (!VMXNET3_VERSION_GE_6(adapter) ||
95 	    !adapter->queuesExtEnabled) {
96 		adapter->shared->devRead.intrConf.intrCtrl |=
97 					cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
98 	} else {
99 		adapter->shared->devReadExt.intrConfExt.intrCtrl |=
100 					cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
101 	}
102 	for (i = 0; i < adapter->intr.num_intrs; i++)
103 		vmxnet3_disable_intr(adapter, i);
104 }
105 
106 
107 static void
108 vmxnet3_ack_events(struct vmxnet3_adapter *adapter, u32 events)
109 {
110 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_ECR, events);
111 }
112 
113 
114 static bool
115 vmxnet3_tq_stopped(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
116 {
117 	return tq->stopped;
118 }
119 
120 
121 static void
122 vmxnet3_tq_start(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
123 {
124 	tq->stopped = false;
125 	netif_start_subqueue(adapter->netdev, tq - adapter->tx_queue);
126 }
127 
128 
129 static void
130 vmxnet3_tq_wake(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
131 {
132 	tq->stopped = false;
133 	netif_wake_subqueue(adapter->netdev, (tq - adapter->tx_queue));
134 }
135 
136 
137 static void
138 vmxnet3_tq_stop(struct vmxnet3_tx_queue *tq, struct vmxnet3_adapter *adapter)
139 {
140 	tq->stopped = true;
141 	tq->num_stop++;
142 	netif_stop_subqueue(adapter->netdev, (tq - adapter->tx_queue));
143 }
144 
145 /* Check if capability is supported by UPT device or
146  * UPT is even requested
147  */
148 bool
149 vmxnet3_check_ptcapability(u32 cap_supported, u32 cap)
150 {
151 	if (cap_supported & (1UL << VMXNET3_DCR_ERROR) ||
152 	    cap_supported & (1UL << cap)) {
153 		return true;
154 	}
155 
156 	return false;
157 }
158 
159 
160 /*
161  * Check the link state. This may start or stop the tx queue.
162  */
163 static void
164 vmxnet3_check_link(struct vmxnet3_adapter *adapter, bool affectTxQueue)
165 {
166 	u32 ret;
167 	int i;
168 	unsigned long flags;
169 
170 	spin_lock_irqsave(&adapter->cmd_lock, flags);
171 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_LINK);
172 	ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
173 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
174 
175 	adapter->link_speed = ret >> 16;
176 	if (ret & 1) { /* Link is up. */
177 		netdev_info(adapter->netdev, "NIC Link is Up %d Mbps\n",
178 			    adapter->link_speed);
179 		netif_carrier_on(adapter->netdev);
180 
181 		if (affectTxQueue) {
182 			for (i = 0; i < adapter->num_tx_queues; i++)
183 				vmxnet3_tq_start(&adapter->tx_queue[i],
184 						 adapter);
185 		}
186 	} else {
187 		netdev_info(adapter->netdev, "NIC Link is Down\n");
188 		netif_carrier_off(adapter->netdev);
189 
190 		if (affectTxQueue) {
191 			for (i = 0; i < adapter->num_tx_queues; i++)
192 				vmxnet3_tq_stop(&adapter->tx_queue[i], adapter);
193 		}
194 	}
195 }
196 
197 static void
198 vmxnet3_process_events(struct vmxnet3_adapter *adapter)
199 {
200 	int i;
201 	unsigned long flags;
202 	u32 events = le32_to_cpu(adapter->shared->ecr);
203 	if (!events)
204 		return;
205 
206 	vmxnet3_ack_events(adapter, events);
207 
208 	/* Check if link state has changed */
209 	if (events & VMXNET3_ECR_LINK)
210 		vmxnet3_check_link(adapter, true);
211 
212 	/* Check if there is an error on xmit/recv queues */
213 	if (events & (VMXNET3_ECR_TQERR | VMXNET3_ECR_RQERR)) {
214 		spin_lock_irqsave(&adapter->cmd_lock, flags);
215 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
216 				       VMXNET3_CMD_GET_QUEUE_STATUS);
217 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
218 
219 		for (i = 0; i < adapter->num_tx_queues; i++)
220 			if (adapter->tqd_start[i].status.stopped)
221 				dev_err(&adapter->netdev->dev,
222 					"%s: tq[%d] error 0x%x\n",
223 					adapter->netdev->name, i, le32_to_cpu(
224 					adapter->tqd_start[i].status.error));
225 		for (i = 0; i < adapter->num_rx_queues; i++)
226 			if (adapter->rqd_start[i].status.stopped)
227 				dev_err(&adapter->netdev->dev,
228 					"%s: rq[%d] error 0x%x\n",
229 					adapter->netdev->name, i,
230 					adapter->rqd_start[i].status.error);
231 
232 		schedule_work(&adapter->work);
233 	}
234 }
235 
236 #ifdef __BIG_ENDIAN_BITFIELD
237 /*
238  * The device expects the bitfields in shared structures to be written in
239  * little endian. When CPU is big endian, the following routines are used to
240  * correctly read and write into ABI.
241  * The general technique used here is : double word bitfields are defined in
242  * opposite order for big endian architecture. Then before reading them in
243  * driver the complete double word is translated using le32_to_cpu. Similarly
244  * After the driver writes into bitfields, cpu_to_le32 is used to translate the
245  * double words into required format.
246  * In order to avoid touching bits in shared structure more than once, temporary
247  * descriptors are used. These are passed as srcDesc to following functions.
248  */
249 static void vmxnet3_RxDescToCPU(const struct Vmxnet3_RxDesc *srcDesc,
250 				struct Vmxnet3_RxDesc *dstDesc)
251 {
252 	u32 *src = (u32 *)srcDesc + 2;
253 	u32 *dst = (u32 *)dstDesc + 2;
254 	dstDesc->addr = le64_to_cpu(srcDesc->addr);
255 	*dst = le32_to_cpu(*src);
256 	dstDesc->ext1 = le32_to_cpu(srcDesc->ext1);
257 }
258 
259 static void vmxnet3_TxDescToLe(const struct Vmxnet3_TxDesc *srcDesc,
260 			       struct Vmxnet3_TxDesc *dstDesc)
261 {
262 	int i;
263 	u32 *src = (u32 *)(srcDesc + 1);
264 	u32 *dst = (u32 *)(dstDesc + 1);
265 
266 	/* Working backwards so that the gen bit is set at the end. */
267 	for (i = 2; i > 0; i--) {
268 		src--;
269 		dst--;
270 		*dst = cpu_to_le32(*src);
271 	}
272 }
273 
274 
275 static void vmxnet3_RxCompToCPU(const struct Vmxnet3_RxCompDesc *srcDesc,
276 				struct Vmxnet3_RxCompDesc *dstDesc)
277 {
278 	int i = 0;
279 	u32 *src = (u32 *)srcDesc;
280 	u32 *dst = (u32 *)dstDesc;
281 	for (i = 0; i < sizeof(struct Vmxnet3_RxCompDesc) / sizeof(u32); i++) {
282 		*dst = le32_to_cpu(*src);
283 		src++;
284 		dst++;
285 	}
286 }
287 
288 
289 /* Used to read bitfield values from double words. */
290 static u32 get_bitfield32(const __le32 *bitfield, u32 pos, u32 size)
291 {
292 	u32 temp = le32_to_cpu(*bitfield);
293 	u32 mask = ((1 << size) - 1) << pos;
294 	temp &= mask;
295 	temp >>= pos;
296 	return temp;
297 }
298 
299 
300 
301 #endif  /* __BIG_ENDIAN_BITFIELD */
302 
303 #ifdef __BIG_ENDIAN_BITFIELD
304 
305 #   define VMXNET3_TXDESC_GET_GEN(txdesc) get_bitfield32(((const __le32 *) \
306 			txdesc) + VMXNET3_TXD_GEN_DWORD_SHIFT, \
307 			VMXNET3_TXD_GEN_SHIFT, VMXNET3_TXD_GEN_SIZE)
308 #   define VMXNET3_TXDESC_GET_EOP(txdesc) get_bitfield32(((const __le32 *) \
309 			txdesc) + VMXNET3_TXD_EOP_DWORD_SHIFT, \
310 			VMXNET3_TXD_EOP_SHIFT, VMXNET3_TXD_EOP_SIZE)
311 #   define VMXNET3_TCD_GET_GEN(tcd) get_bitfield32(((const __le32 *)tcd) + \
312 			VMXNET3_TCD_GEN_DWORD_SHIFT, VMXNET3_TCD_GEN_SHIFT, \
313 			VMXNET3_TCD_GEN_SIZE)
314 #   define VMXNET3_TCD_GET_TXIDX(tcd) get_bitfield32((const __le32 *)tcd, \
315 			VMXNET3_TCD_TXIDX_SHIFT, VMXNET3_TCD_TXIDX_SIZE)
316 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) do { \
317 			(dstrcd) = (tmp); \
318 			vmxnet3_RxCompToCPU((rcd), (tmp)); \
319 		} while (0)
320 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) do { \
321 			(dstrxd) = (tmp); \
322 			vmxnet3_RxDescToCPU((rxd), (tmp)); \
323 		} while (0)
324 
325 #else
326 
327 #   define VMXNET3_TXDESC_GET_GEN(txdesc) ((txdesc)->gen)
328 #   define VMXNET3_TXDESC_GET_EOP(txdesc) ((txdesc)->eop)
329 #   define VMXNET3_TCD_GET_GEN(tcd) ((tcd)->gen)
330 #   define VMXNET3_TCD_GET_TXIDX(tcd) ((tcd)->txdIdx)
331 #   define vmxnet3_getRxComp(dstrcd, rcd, tmp) (dstrcd) = (rcd)
332 #   define vmxnet3_getRxDesc(dstrxd, rxd, tmp) (dstrxd) = (rxd)
333 
334 #endif /* __BIG_ENDIAN_BITFIELD  */
335 
336 
337 static void
338 vmxnet3_unmap_tx_buf(struct vmxnet3_tx_buf_info *tbi,
339 		     struct pci_dev *pdev)
340 {
341 	if (tbi->map_type == VMXNET3_MAP_SINGLE)
342 		dma_unmap_single(&pdev->dev, tbi->dma_addr, tbi->len,
343 				 DMA_TO_DEVICE);
344 	else if (tbi->map_type == VMXNET3_MAP_PAGE)
345 		dma_unmap_page(&pdev->dev, tbi->dma_addr, tbi->len,
346 			       DMA_TO_DEVICE);
347 	else
348 		BUG_ON(tbi->map_type != VMXNET3_MAP_NONE);
349 
350 	tbi->map_type = VMXNET3_MAP_NONE; /* to help debugging */
351 }
352 
353 
354 static int
355 vmxnet3_unmap_pkt(u32 eop_idx, struct vmxnet3_tx_queue *tq,
356 		  struct pci_dev *pdev,	struct vmxnet3_adapter *adapter)
357 {
358 	struct sk_buff *skb;
359 	int entries = 0;
360 
361 	/* no out of order completion */
362 	BUG_ON(tq->buf_info[eop_idx].sop_idx != tq->tx_ring.next2comp);
363 	BUG_ON(VMXNET3_TXDESC_GET_EOP(&(tq->tx_ring.base[eop_idx].txd)) != 1);
364 
365 	skb = tq->buf_info[eop_idx].skb;
366 	BUG_ON(skb == NULL);
367 	tq->buf_info[eop_idx].skb = NULL;
368 
369 	VMXNET3_INC_RING_IDX_ONLY(eop_idx, tq->tx_ring.size);
370 
371 	while (tq->tx_ring.next2comp != eop_idx) {
372 		vmxnet3_unmap_tx_buf(tq->buf_info + tq->tx_ring.next2comp,
373 				     pdev);
374 
375 		/* update next2comp w/o tx_lock. Since we are marking more,
376 		 * instead of less, tx ring entries avail, the worst case is
377 		 * that the tx routine incorrectly re-queues a pkt due to
378 		 * insufficient tx ring entries.
379 		 */
380 		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
381 		entries++;
382 	}
383 
384 	dev_kfree_skb_any(skb);
385 	return entries;
386 }
387 
388 
389 static int
390 vmxnet3_tq_tx_complete(struct vmxnet3_tx_queue *tq,
391 			struct vmxnet3_adapter *adapter)
392 {
393 	int completed = 0;
394 	union Vmxnet3_GenericDesc *gdesc;
395 
396 	gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
397 	while (VMXNET3_TCD_GET_GEN(&gdesc->tcd) == tq->comp_ring.gen) {
398 		/* Prevent any &gdesc->tcd field from being (speculatively)
399 		 * read before (&gdesc->tcd)->gen is read.
400 		 */
401 		dma_rmb();
402 
403 		completed += vmxnet3_unmap_pkt(VMXNET3_TCD_GET_TXIDX(
404 					       &gdesc->tcd), tq, adapter->pdev,
405 					       adapter);
406 
407 		vmxnet3_comp_ring_adv_next2proc(&tq->comp_ring);
408 		gdesc = tq->comp_ring.base + tq->comp_ring.next2proc;
409 	}
410 
411 	if (completed) {
412 		spin_lock(&tq->tx_lock);
413 		if (unlikely(vmxnet3_tq_stopped(tq, adapter) &&
414 			     vmxnet3_cmd_ring_desc_avail(&tq->tx_ring) >
415 			     VMXNET3_WAKE_QUEUE_THRESHOLD(tq) &&
416 			     netif_carrier_ok(adapter->netdev))) {
417 			vmxnet3_tq_wake(tq, adapter);
418 		}
419 		spin_unlock(&tq->tx_lock);
420 	}
421 	return completed;
422 }
423 
424 
425 static void
426 vmxnet3_tq_cleanup(struct vmxnet3_tx_queue *tq,
427 		   struct vmxnet3_adapter *adapter)
428 {
429 	int i;
430 
431 	while (tq->tx_ring.next2comp != tq->tx_ring.next2fill) {
432 		struct vmxnet3_tx_buf_info *tbi;
433 
434 		tbi = tq->buf_info + tq->tx_ring.next2comp;
435 
436 		vmxnet3_unmap_tx_buf(tbi, adapter->pdev);
437 		if (tbi->skb) {
438 			dev_kfree_skb_any(tbi->skb);
439 			tbi->skb = NULL;
440 		}
441 		vmxnet3_cmd_ring_adv_next2comp(&tq->tx_ring);
442 	}
443 
444 	/* sanity check, verify all buffers are indeed unmapped and freed */
445 	for (i = 0; i < tq->tx_ring.size; i++) {
446 		BUG_ON(tq->buf_info[i].skb != NULL ||
447 		       tq->buf_info[i].map_type != VMXNET3_MAP_NONE);
448 	}
449 
450 	tq->tx_ring.gen = VMXNET3_INIT_GEN;
451 	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
452 
453 	tq->comp_ring.gen = VMXNET3_INIT_GEN;
454 	tq->comp_ring.next2proc = 0;
455 }
456 
457 
458 static void
459 vmxnet3_tq_destroy(struct vmxnet3_tx_queue *tq,
460 		   struct vmxnet3_adapter *adapter)
461 {
462 	if (tq->tx_ring.base) {
463 		dma_free_coherent(&adapter->pdev->dev, tq->tx_ring.size *
464 				  sizeof(struct Vmxnet3_TxDesc),
465 				  tq->tx_ring.base, tq->tx_ring.basePA);
466 		tq->tx_ring.base = NULL;
467 	}
468 	if (tq->data_ring.base) {
469 		dma_free_coherent(&adapter->pdev->dev,
470 				  tq->data_ring.size * tq->txdata_desc_size,
471 				  tq->data_ring.base, tq->data_ring.basePA);
472 		tq->data_ring.base = NULL;
473 	}
474 	if (tq->comp_ring.base) {
475 		dma_free_coherent(&adapter->pdev->dev, tq->comp_ring.size *
476 				  sizeof(struct Vmxnet3_TxCompDesc),
477 				  tq->comp_ring.base, tq->comp_ring.basePA);
478 		tq->comp_ring.base = NULL;
479 	}
480 	kfree(tq->buf_info);
481 	tq->buf_info = NULL;
482 }
483 
484 
485 /* Destroy all tx queues */
486 void
487 vmxnet3_tq_destroy_all(struct vmxnet3_adapter *adapter)
488 {
489 	int i;
490 
491 	for (i = 0; i < adapter->num_tx_queues; i++)
492 		vmxnet3_tq_destroy(&adapter->tx_queue[i], adapter);
493 }
494 
495 
496 static void
497 vmxnet3_tq_init(struct vmxnet3_tx_queue *tq,
498 		struct vmxnet3_adapter *adapter)
499 {
500 	int i;
501 
502 	/* reset the tx ring contents to 0 and reset the tx ring states */
503 	memset(tq->tx_ring.base, 0, tq->tx_ring.size *
504 	       sizeof(struct Vmxnet3_TxDesc));
505 	tq->tx_ring.next2fill = tq->tx_ring.next2comp = 0;
506 	tq->tx_ring.gen = VMXNET3_INIT_GEN;
507 
508 	memset(tq->data_ring.base, 0,
509 	       tq->data_ring.size * tq->txdata_desc_size);
510 
511 	/* reset the tx comp ring contents to 0 and reset comp ring states */
512 	memset(tq->comp_ring.base, 0, tq->comp_ring.size *
513 	       sizeof(struct Vmxnet3_TxCompDesc));
514 	tq->comp_ring.next2proc = 0;
515 	tq->comp_ring.gen = VMXNET3_INIT_GEN;
516 
517 	/* reset the bookkeeping data */
518 	memset(tq->buf_info, 0, sizeof(tq->buf_info[0]) * tq->tx_ring.size);
519 	for (i = 0; i < tq->tx_ring.size; i++)
520 		tq->buf_info[i].map_type = VMXNET3_MAP_NONE;
521 
522 	/* stats are not reset */
523 }
524 
525 
526 static int
527 vmxnet3_tq_create(struct vmxnet3_tx_queue *tq,
528 		  struct vmxnet3_adapter *adapter)
529 {
530 	BUG_ON(tq->tx_ring.base || tq->data_ring.base ||
531 	       tq->comp_ring.base || tq->buf_info);
532 
533 	tq->tx_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
534 			tq->tx_ring.size * sizeof(struct Vmxnet3_TxDesc),
535 			&tq->tx_ring.basePA, GFP_KERNEL);
536 	if (!tq->tx_ring.base) {
537 		netdev_err(adapter->netdev, "failed to allocate tx ring\n");
538 		goto err;
539 	}
540 
541 	tq->data_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
542 			tq->data_ring.size * tq->txdata_desc_size,
543 			&tq->data_ring.basePA, GFP_KERNEL);
544 	if (!tq->data_ring.base) {
545 		netdev_err(adapter->netdev, "failed to allocate tx data ring\n");
546 		goto err;
547 	}
548 
549 	tq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev,
550 			tq->comp_ring.size * sizeof(struct Vmxnet3_TxCompDesc),
551 			&tq->comp_ring.basePA, GFP_KERNEL);
552 	if (!tq->comp_ring.base) {
553 		netdev_err(adapter->netdev, "failed to allocate tx comp ring\n");
554 		goto err;
555 	}
556 
557 	tq->buf_info = kcalloc_node(tq->tx_ring.size, sizeof(tq->buf_info[0]),
558 				    GFP_KERNEL,
559 				    dev_to_node(&adapter->pdev->dev));
560 	if (!tq->buf_info)
561 		goto err;
562 
563 	return 0;
564 
565 err:
566 	vmxnet3_tq_destroy(tq, adapter);
567 	return -ENOMEM;
568 }
569 
570 static void
571 vmxnet3_tq_cleanup_all(struct vmxnet3_adapter *adapter)
572 {
573 	int i;
574 
575 	for (i = 0; i < adapter->num_tx_queues; i++)
576 		vmxnet3_tq_cleanup(&adapter->tx_queue[i], adapter);
577 }
578 
579 /*
580  *    starting from ring->next2fill, allocate rx buffers for the given ring
581  *    of the rx queue and update the rx desc. stop after @num_to_alloc buffers
582  *    are allocated or allocation fails
583  */
584 
585 static int
586 vmxnet3_rq_alloc_rx_buf(struct vmxnet3_rx_queue *rq, u32 ring_idx,
587 			int num_to_alloc, struct vmxnet3_adapter *adapter)
588 {
589 	int num_allocated = 0;
590 	struct vmxnet3_rx_buf_info *rbi_base = rq->buf_info[ring_idx];
591 	struct vmxnet3_cmd_ring *ring = &rq->rx_ring[ring_idx];
592 	u32 val;
593 
594 	while (num_allocated <= num_to_alloc) {
595 		struct vmxnet3_rx_buf_info *rbi;
596 		union Vmxnet3_GenericDesc *gd;
597 
598 		rbi = rbi_base + ring->next2fill;
599 		gd = ring->base + ring->next2fill;
600 		rbi->comp_state = VMXNET3_RXD_COMP_PENDING;
601 
602 		if (rbi->buf_type == VMXNET3_RX_BUF_SKB) {
603 			if (rbi->skb == NULL) {
604 				rbi->skb = __netdev_alloc_skb_ip_align(adapter->netdev,
605 								       rbi->len,
606 								       GFP_KERNEL);
607 				if (unlikely(rbi->skb == NULL)) {
608 					rq->stats.rx_buf_alloc_failure++;
609 					break;
610 				}
611 
612 				rbi->dma_addr = dma_map_single(
613 						&adapter->pdev->dev,
614 						rbi->skb->data, rbi->len,
615 						DMA_FROM_DEVICE);
616 				if (dma_mapping_error(&adapter->pdev->dev,
617 						      rbi->dma_addr)) {
618 					dev_kfree_skb_any(rbi->skb);
619 					rbi->skb = NULL;
620 					rq->stats.rx_buf_alloc_failure++;
621 					break;
622 				}
623 			} else {
624 				/* rx buffer skipped by the device */
625 			}
626 			val = VMXNET3_RXD_BTYPE_HEAD << VMXNET3_RXD_BTYPE_SHIFT;
627 		} else {
628 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE ||
629 			       rbi->len  != PAGE_SIZE);
630 
631 			if (rbi->page == NULL) {
632 				rbi->page = alloc_page(GFP_ATOMIC);
633 				if (unlikely(rbi->page == NULL)) {
634 					rq->stats.rx_buf_alloc_failure++;
635 					break;
636 				}
637 				rbi->dma_addr = dma_map_page(
638 						&adapter->pdev->dev,
639 						rbi->page, 0, PAGE_SIZE,
640 						DMA_FROM_DEVICE);
641 				if (dma_mapping_error(&adapter->pdev->dev,
642 						      rbi->dma_addr)) {
643 					put_page(rbi->page);
644 					rbi->page = NULL;
645 					rq->stats.rx_buf_alloc_failure++;
646 					break;
647 				}
648 			} else {
649 				/* rx buffers skipped by the device */
650 			}
651 			val = VMXNET3_RXD_BTYPE_BODY << VMXNET3_RXD_BTYPE_SHIFT;
652 		}
653 
654 		gd->rxd.addr = cpu_to_le64(rbi->dma_addr);
655 		gd->dword[2] = cpu_to_le32((!ring->gen << VMXNET3_RXD_GEN_SHIFT)
656 					   | val | rbi->len);
657 
658 		/* Fill the last buffer but dont mark it ready, or else the
659 		 * device will think that the queue is full */
660 		if (num_allocated == num_to_alloc) {
661 			rbi->comp_state = VMXNET3_RXD_COMP_DONE;
662 			break;
663 		}
664 
665 		gd->dword[2] |= cpu_to_le32(ring->gen << VMXNET3_RXD_GEN_SHIFT);
666 		num_allocated++;
667 		vmxnet3_cmd_ring_adv_next2fill(ring);
668 	}
669 
670 	netdev_dbg(adapter->netdev,
671 		"alloc_rx_buf: %d allocated, next2fill %u, next2comp %u\n",
672 		num_allocated, ring->next2fill, ring->next2comp);
673 
674 	/* so that the device can distinguish a full ring and an empty ring */
675 	BUG_ON(num_allocated != 0 && ring->next2fill == ring->next2comp);
676 
677 	return num_allocated;
678 }
679 
680 
681 static void
682 vmxnet3_append_frag(struct sk_buff *skb, struct Vmxnet3_RxCompDesc *rcd,
683 		    struct vmxnet3_rx_buf_info *rbi)
684 {
685 	skb_frag_t *frag = skb_shinfo(skb)->frags + skb_shinfo(skb)->nr_frags;
686 
687 	BUG_ON(skb_shinfo(skb)->nr_frags >= MAX_SKB_FRAGS);
688 
689 	__skb_frag_set_page(frag, rbi->page);
690 	skb_frag_off_set(frag, 0);
691 	skb_frag_size_set(frag, rcd->len);
692 	skb->data_len += rcd->len;
693 	skb->truesize += PAGE_SIZE;
694 	skb_shinfo(skb)->nr_frags++;
695 }
696 
697 
698 static int
699 vmxnet3_map_pkt(struct sk_buff *skb, struct vmxnet3_tx_ctx *ctx,
700 		struct vmxnet3_tx_queue *tq, struct pci_dev *pdev,
701 		struct vmxnet3_adapter *adapter)
702 {
703 	u32 dw2, len;
704 	unsigned long buf_offset;
705 	int i;
706 	union Vmxnet3_GenericDesc *gdesc;
707 	struct vmxnet3_tx_buf_info *tbi = NULL;
708 
709 	BUG_ON(ctx->copy_size > skb_headlen(skb));
710 
711 	/* use the previous gen bit for the SOP desc */
712 	dw2 = (tq->tx_ring.gen ^ 0x1) << VMXNET3_TXD_GEN_SHIFT;
713 
714 	ctx->sop_txd = tq->tx_ring.base + tq->tx_ring.next2fill;
715 	gdesc = ctx->sop_txd; /* both loops below can be skipped */
716 
717 	/* no need to map the buffer if headers are copied */
718 	if (ctx->copy_size) {
719 		ctx->sop_txd->txd.addr = cpu_to_le64(tq->data_ring.basePA +
720 					tq->tx_ring.next2fill *
721 					tq->txdata_desc_size);
722 		ctx->sop_txd->dword[2] = cpu_to_le32(dw2 | ctx->copy_size);
723 		ctx->sop_txd->dword[3] = 0;
724 
725 		tbi = tq->buf_info + tq->tx_ring.next2fill;
726 		tbi->map_type = VMXNET3_MAP_NONE;
727 
728 		netdev_dbg(adapter->netdev,
729 			"txd[%u]: 0x%Lx 0x%x 0x%x\n",
730 			tq->tx_ring.next2fill,
731 			le64_to_cpu(ctx->sop_txd->txd.addr),
732 			ctx->sop_txd->dword[2], ctx->sop_txd->dword[3]);
733 		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
734 
735 		/* use the right gen for non-SOP desc */
736 		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
737 	}
738 
739 	/* linear part can use multiple tx desc if it's big */
740 	len = skb_headlen(skb) - ctx->copy_size;
741 	buf_offset = ctx->copy_size;
742 	while (len) {
743 		u32 buf_size;
744 
745 		if (len < VMXNET3_MAX_TX_BUF_SIZE) {
746 			buf_size = len;
747 			dw2 |= len;
748 		} else {
749 			buf_size = VMXNET3_MAX_TX_BUF_SIZE;
750 			/* spec says that for TxDesc.len, 0 == 2^14 */
751 		}
752 
753 		tbi = tq->buf_info + tq->tx_ring.next2fill;
754 		tbi->map_type = VMXNET3_MAP_SINGLE;
755 		tbi->dma_addr = dma_map_single(&adapter->pdev->dev,
756 				skb->data + buf_offset, buf_size,
757 				DMA_TO_DEVICE);
758 		if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
759 			return -EFAULT;
760 
761 		tbi->len = buf_size;
762 
763 		gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
764 		BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
765 
766 		gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
767 		gdesc->dword[2] = cpu_to_le32(dw2);
768 		gdesc->dword[3] = 0;
769 
770 		netdev_dbg(adapter->netdev,
771 			"txd[%u]: 0x%Lx 0x%x 0x%x\n",
772 			tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
773 			le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
774 		vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
775 		dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
776 
777 		len -= buf_size;
778 		buf_offset += buf_size;
779 	}
780 
781 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
782 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
783 		u32 buf_size;
784 
785 		buf_offset = 0;
786 		len = skb_frag_size(frag);
787 		while (len) {
788 			tbi = tq->buf_info + tq->tx_ring.next2fill;
789 			if (len < VMXNET3_MAX_TX_BUF_SIZE) {
790 				buf_size = len;
791 				dw2 |= len;
792 			} else {
793 				buf_size = VMXNET3_MAX_TX_BUF_SIZE;
794 				/* spec says that for TxDesc.len, 0 == 2^14 */
795 			}
796 			tbi->map_type = VMXNET3_MAP_PAGE;
797 			tbi->dma_addr = skb_frag_dma_map(&adapter->pdev->dev, frag,
798 							 buf_offset, buf_size,
799 							 DMA_TO_DEVICE);
800 			if (dma_mapping_error(&adapter->pdev->dev, tbi->dma_addr))
801 				return -EFAULT;
802 
803 			tbi->len = buf_size;
804 
805 			gdesc = tq->tx_ring.base + tq->tx_ring.next2fill;
806 			BUG_ON(gdesc->txd.gen == tq->tx_ring.gen);
807 
808 			gdesc->txd.addr = cpu_to_le64(tbi->dma_addr);
809 			gdesc->dword[2] = cpu_to_le32(dw2);
810 			gdesc->dword[3] = 0;
811 
812 			netdev_dbg(adapter->netdev,
813 				"txd[%u]: 0x%llx %u %u\n",
814 				tq->tx_ring.next2fill, le64_to_cpu(gdesc->txd.addr),
815 				le32_to_cpu(gdesc->dword[2]), gdesc->dword[3]);
816 			vmxnet3_cmd_ring_adv_next2fill(&tq->tx_ring);
817 			dw2 = tq->tx_ring.gen << VMXNET3_TXD_GEN_SHIFT;
818 
819 			len -= buf_size;
820 			buf_offset += buf_size;
821 		}
822 	}
823 
824 	ctx->eop_txd = gdesc;
825 
826 	/* set the last buf_info for the pkt */
827 	tbi->skb = skb;
828 	tbi->sop_idx = ctx->sop_txd - tq->tx_ring.base;
829 
830 	return 0;
831 }
832 
833 
834 /* Init all tx queues */
835 static void
836 vmxnet3_tq_init_all(struct vmxnet3_adapter *adapter)
837 {
838 	int i;
839 
840 	for (i = 0; i < adapter->num_tx_queues; i++)
841 		vmxnet3_tq_init(&adapter->tx_queue[i], adapter);
842 }
843 
844 
845 /*
846  *    parse relevant protocol headers:
847  *      For a tso pkt, relevant headers are L2/3/4 including options
848  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
849  *      if it's a TCP/UDP pkt
850  *
851  * Returns:
852  *    -1:  error happens during parsing
853  *     0:  protocol headers parsed, but too big to be copied
854  *     1:  protocol headers parsed and copied
855  *
856  * Other effects:
857  *    1. related *ctx fields are updated.
858  *    2. ctx->copy_size is # of bytes copied
859  *    3. the portion to be copied is guaranteed to be in the linear part
860  *
861  */
862 static int
863 vmxnet3_parse_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
864 		  struct vmxnet3_tx_ctx *ctx,
865 		  struct vmxnet3_adapter *adapter)
866 {
867 	u8 protocol = 0;
868 
869 	if (ctx->mss) {	/* TSO */
870 		if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) {
871 			ctx->l4_offset = skb_inner_transport_offset(skb);
872 			ctx->l4_hdr_size = inner_tcp_hdrlen(skb);
873 			ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size;
874 		} else {
875 			ctx->l4_offset = skb_transport_offset(skb);
876 			ctx->l4_hdr_size = tcp_hdrlen(skb);
877 			ctx->copy_size = ctx->l4_offset + ctx->l4_hdr_size;
878 		}
879 	} else {
880 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
881 			/* For encap packets, skb_checksum_start_offset refers
882 			 * to inner L4 offset. Thus, below works for encap as
883 			 * well as non-encap case
884 			 */
885 			ctx->l4_offset = skb_checksum_start_offset(skb);
886 
887 			if (VMXNET3_VERSION_GE_4(adapter) &&
888 			    skb->encapsulation) {
889 				struct iphdr *iph = inner_ip_hdr(skb);
890 
891 				if (iph->version == 4) {
892 					protocol = iph->protocol;
893 				} else {
894 					const struct ipv6hdr *ipv6h;
895 
896 					ipv6h = inner_ipv6_hdr(skb);
897 					protocol = ipv6h->nexthdr;
898 				}
899 			} else {
900 				if (ctx->ipv4) {
901 					const struct iphdr *iph = ip_hdr(skb);
902 
903 					protocol = iph->protocol;
904 				} else if (ctx->ipv6) {
905 					const struct ipv6hdr *ipv6h;
906 
907 					ipv6h = ipv6_hdr(skb);
908 					protocol = ipv6h->nexthdr;
909 				}
910 			}
911 
912 			switch (protocol) {
913 			case IPPROTO_TCP:
914 				ctx->l4_hdr_size = skb->encapsulation ? inner_tcp_hdrlen(skb) :
915 						   tcp_hdrlen(skb);
916 				break;
917 			case IPPROTO_UDP:
918 				ctx->l4_hdr_size = sizeof(struct udphdr);
919 				break;
920 			default:
921 				ctx->l4_hdr_size = 0;
922 				break;
923 			}
924 
925 			ctx->copy_size = min(ctx->l4_offset +
926 					 ctx->l4_hdr_size, skb->len);
927 		} else {
928 			ctx->l4_offset = 0;
929 			ctx->l4_hdr_size = 0;
930 			/* copy as much as allowed */
931 			ctx->copy_size = min_t(unsigned int,
932 					       tq->txdata_desc_size,
933 					       skb_headlen(skb));
934 		}
935 
936 		if (skb->len <= VMXNET3_HDR_COPY_SIZE)
937 			ctx->copy_size = skb->len;
938 
939 		/* make sure headers are accessible directly */
940 		if (unlikely(!pskb_may_pull(skb, ctx->copy_size)))
941 			goto err;
942 	}
943 
944 	if (unlikely(ctx->copy_size > tq->txdata_desc_size)) {
945 		tq->stats.oversized_hdr++;
946 		ctx->copy_size = 0;
947 		return 0;
948 	}
949 
950 	return 1;
951 err:
952 	return -1;
953 }
954 
955 /*
956  *    copy relevant protocol headers to the transmit ring:
957  *      For a tso pkt, relevant headers are L2/3/4 including options
958  *      For a pkt requesting csum offloading, they are L2/3 and may include L4
959  *      if it's a TCP/UDP pkt
960  *
961  *
962  *    Note that this requires that vmxnet3_parse_hdr be called first to set the
963  *      appropriate bits in ctx first
964  */
965 static void
966 vmxnet3_copy_hdr(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
967 		 struct vmxnet3_tx_ctx *ctx,
968 		 struct vmxnet3_adapter *adapter)
969 {
970 	struct Vmxnet3_TxDataDesc *tdd;
971 
972 	tdd = (struct Vmxnet3_TxDataDesc *)((u8 *)tq->data_ring.base +
973 					    tq->tx_ring.next2fill *
974 					    tq->txdata_desc_size);
975 
976 	memcpy(tdd->data, skb->data, ctx->copy_size);
977 	netdev_dbg(adapter->netdev,
978 		"copy %u bytes to dataRing[%u]\n",
979 		ctx->copy_size, tq->tx_ring.next2fill);
980 }
981 
982 
983 static void
984 vmxnet3_prepare_inner_tso(struct sk_buff *skb,
985 			  struct vmxnet3_tx_ctx *ctx)
986 {
987 	struct tcphdr *tcph = inner_tcp_hdr(skb);
988 	struct iphdr *iph = inner_ip_hdr(skb);
989 
990 	if (iph->version == 4) {
991 		iph->check = 0;
992 		tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
993 						 IPPROTO_TCP, 0);
994 	} else {
995 		struct ipv6hdr *iph = inner_ipv6_hdr(skb);
996 
997 		tcph->check = ~csum_ipv6_magic(&iph->saddr, &iph->daddr, 0,
998 					       IPPROTO_TCP, 0);
999 	}
1000 }
1001 
1002 static void
1003 vmxnet3_prepare_tso(struct sk_buff *skb,
1004 		    struct vmxnet3_tx_ctx *ctx)
1005 {
1006 	struct tcphdr *tcph = tcp_hdr(skb);
1007 
1008 	if (ctx->ipv4) {
1009 		struct iphdr *iph = ip_hdr(skb);
1010 
1011 		iph->check = 0;
1012 		tcph->check = ~csum_tcpudp_magic(iph->saddr, iph->daddr, 0,
1013 						 IPPROTO_TCP, 0);
1014 	} else if (ctx->ipv6) {
1015 		tcp_v6_gso_csum_prep(skb);
1016 	}
1017 }
1018 
1019 static int txd_estimate(const struct sk_buff *skb)
1020 {
1021 	int count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1022 	int i;
1023 
1024 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
1025 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
1026 
1027 		count += VMXNET3_TXD_NEEDED(skb_frag_size(frag));
1028 	}
1029 	return count;
1030 }
1031 
1032 /*
1033  * Transmits a pkt thru a given tq
1034  * Returns:
1035  *    NETDEV_TX_OK:      descriptors are setup successfully
1036  *    NETDEV_TX_OK:      error occurred, the pkt is dropped
1037  *    NETDEV_TX_BUSY:    tx ring is full, queue is stopped
1038  *
1039  * Side-effects:
1040  *    1. tx ring may be changed
1041  *    2. tq stats may be updated accordingly
1042  *    3. shared->txNumDeferred may be updated
1043  */
1044 
1045 static int
1046 vmxnet3_tq_xmit(struct sk_buff *skb, struct vmxnet3_tx_queue *tq,
1047 		struct vmxnet3_adapter *adapter, struct net_device *netdev)
1048 {
1049 	int ret;
1050 	u32 count;
1051 	int num_pkts;
1052 	int tx_num_deferred;
1053 	unsigned long flags;
1054 	struct vmxnet3_tx_ctx ctx;
1055 	union Vmxnet3_GenericDesc *gdesc;
1056 #ifdef __BIG_ENDIAN_BITFIELD
1057 	/* Use temporary descriptor to avoid touching bits multiple times */
1058 	union Vmxnet3_GenericDesc tempTxDesc;
1059 #endif
1060 
1061 	count = txd_estimate(skb);
1062 
1063 	ctx.ipv4 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IP));
1064 	ctx.ipv6 = (vlan_get_protocol(skb) == cpu_to_be16(ETH_P_IPV6));
1065 
1066 	ctx.mss = skb_shinfo(skb)->gso_size;
1067 	if (ctx.mss) {
1068 		if (skb_header_cloned(skb)) {
1069 			if (unlikely(pskb_expand_head(skb, 0, 0,
1070 						      GFP_ATOMIC) != 0)) {
1071 				tq->stats.drop_tso++;
1072 				goto drop_pkt;
1073 			}
1074 			tq->stats.copy_skb_header++;
1075 		}
1076 		if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) {
1077 			/* tso pkts must not use more than
1078 			 * VMXNET3_MAX_TSO_TXD_PER_PKT entries
1079 			 */
1080 			if (skb_linearize(skb) != 0) {
1081 				tq->stats.drop_too_many_frags++;
1082 				goto drop_pkt;
1083 			}
1084 			tq->stats.linearized++;
1085 
1086 			/* recalculate the # of descriptors to use */
1087 			count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1088 			if (unlikely(count > VMXNET3_MAX_TSO_TXD_PER_PKT)) {
1089 				tq->stats.drop_too_many_frags++;
1090 				goto drop_pkt;
1091 			}
1092 		}
1093 		if (skb->encapsulation) {
1094 			vmxnet3_prepare_inner_tso(skb, &ctx);
1095 		} else {
1096 			vmxnet3_prepare_tso(skb, &ctx);
1097 		}
1098 	} else {
1099 		if (unlikely(count > VMXNET3_MAX_TXD_PER_PKT)) {
1100 
1101 			/* non-tso pkts must not use more than
1102 			 * VMXNET3_MAX_TXD_PER_PKT entries
1103 			 */
1104 			if (skb_linearize(skb) != 0) {
1105 				tq->stats.drop_too_many_frags++;
1106 				goto drop_pkt;
1107 			}
1108 			tq->stats.linearized++;
1109 
1110 			/* recalculate the # of descriptors to use */
1111 			count = VMXNET3_TXD_NEEDED(skb_headlen(skb)) + 1;
1112 		}
1113 	}
1114 
1115 	ret = vmxnet3_parse_hdr(skb, tq, &ctx, adapter);
1116 	if (ret >= 0) {
1117 		BUG_ON(ret <= 0 && ctx.copy_size != 0);
1118 		/* hdrs parsed, check against other limits */
1119 		if (ctx.mss) {
1120 			if (unlikely(ctx.l4_offset + ctx.l4_hdr_size >
1121 				     VMXNET3_MAX_TX_BUF_SIZE)) {
1122 				tq->stats.drop_oversized_hdr++;
1123 				goto drop_pkt;
1124 			}
1125 		} else {
1126 			if (skb->ip_summed == CHECKSUM_PARTIAL) {
1127 				if (unlikely(ctx.l4_offset +
1128 					     skb->csum_offset >
1129 					     VMXNET3_MAX_CSUM_OFFSET)) {
1130 					tq->stats.drop_oversized_hdr++;
1131 					goto drop_pkt;
1132 				}
1133 			}
1134 		}
1135 	} else {
1136 		tq->stats.drop_hdr_inspect_err++;
1137 		goto drop_pkt;
1138 	}
1139 
1140 	spin_lock_irqsave(&tq->tx_lock, flags);
1141 
1142 	if (count > vmxnet3_cmd_ring_desc_avail(&tq->tx_ring)) {
1143 		tq->stats.tx_ring_full++;
1144 		netdev_dbg(adapter->netdev,
1145 			"tx queue stopped on %s, next2comp %u"
1146 			" next2fill %u\n", adapter->netdev->name,
1147 			tq->tx_ring.next2comp, tq->tx_ring.next2fill);
1148 
1149 		vmxnet3_tq_stop(tq, adapter);
1150 		spin_unlock_irqrestore(&tq->tx_lock, flags);
1151 		return NETDEV_TX_BUSY;
1152 	}
1153 
1154 
1155 	vmxnet3_copy_hdr(skb, tq, &ctx, adapter);
1156 
1157 	/* fill tx descs related to addr & len */
1158 	if (vmxnet3_map_pkt(skb, &ctx, tq, adapter->pdev, adapter))
1159 		goto unlock_drop_pkt;
1160 
1161 	/* setup the EOP desc */
1162 	ctx.eop_txd->dword[3] = cpu_to_le32(VMXNET3_TXD_CQ | VMXNET3_TXD_EOP);
1163 
1164 	/* setup the SOP desc */
1165 #ifdef __BIG_ENDIAN_BITFIELD
1166 	gdesc = &tempTxDesc;
1167 	gdesc->dword[2] = ctx.sop_txd->dword[2];
1168 	gdesc->dword[3] = ctx.sop_txd->dword[3];
1169 #else
1170 	gdesc = ctx.sop_txd;
1171 #endif
1172 	tx_num_deferred = le32_to_cpu(tq->shared->txNumDeferred);
1173 	if (ctx.mss) {
1174 		if (VMXNET3_VERSION_GE_4(adapter) && skb->encapsulation) {
1175 			gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
1176 			if (VMXNET3_VERSION_GE_7(adapter)) {
1177 				gdesc->txd.om = VMXNET3_OM_TSO;
1178 				gdesc->txd.ext1 = 1;
1179 			} else {
1180 				gdesc->txd.om = VMXNET3_OM_ENCAP;
1181 			}
1182 			gdesc->txd.msscof = ctx.mss;
1183 
1184 			if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
1185 				gdesc->txd.oco = 1;
1186 		} else {
1187 			gdesc->txd.hlen = ctx.l4_offset + ctx.l4_hdr_size;
1188 			gdesc->txd.om = VMXNET3_OM_TSO;
1189 			gdesc->txd.msscof = ctx.mss;
1190 		}
1191 		num_pkts = (skb->len - gdesc->txd.hlen + ctx.mss - 1) / ctx.mss;
1192 	} else {
1193 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
1194 			if (VMXNET3_VERSION_GE_4(adapter) &&
1195 			    skb->encapsulation) {
1196 				gdesc->txd.hlen = ctx.l4_offset +
1197 						  ctx.l4_hdr_size;
1198 				if (VMXNET3_VERSION_GE_7(adapter)) {
1199 					gdesc->txd.om = VMXNET3_OM_CSUM;
1200 					gdesc->txd.msscof = ctx.l4_offset +
1201 							    skb->csum_offset;
1202 					gdesc->txd.ext1 = 1;
1203 				} else {
1204 					gdesc->txd.om = VMXNET3_OM_ENCAP;
1205 					gdesc->txd.msscof = 0;		/* Reserved */
1206 				}
1207 			} else {
1208 				gdesc->txd.hlen = ctx.l4_offset;
1209 				gdesc->txd.om = VMXNET3_OM_CSUM;
1210 				gdesc->txd.msscof = ctx.l4_offset +
1211 						    skb->csum_offset;
1212 			}
1213 		} else {
1214 			gdesc->txd.om = 0;
1215 			gdesc->txd.msscof = 0;
1216 		}
1217 		num_pkts = 1;
1218 	}
1219 	le32_add_cpu(&tq->shared->txNumDeferred, num_pkts);
1220 	tx_num_deferred += num_pkts;
1221 
1222 	if (skb_vlan_tag_present(skb)) {
1223 		gdesc->txd.ti = 1;
1224 		gdesc->txd.tci = skb_vlan_tag_get(skb);
1225 	}
1226 
1227 	/* Ensure that the write to (&gdesc->txd)->gen will be observed after
1228 	 * all other writes to &gdesc->txd.
1229 	 */
1230 	dma_wmb();
1231 
1232 	/* finally flips the GEN bit of the SOP desc. */
1233 	gdesc->dword[2] = cpu_to_le32(le32_to_cpu(gdesc->dword[2]) ^
1234 						  VMXNET3_TXD_GEN);
1235 #ifdef __BIG_ENDIAN_BITFIELD
1236 	/* Finished updating in bitfields of Tx Desc, so write them in original
1237 	 * place.
1238 	 */
1239 	vmxnet3_TxDescToLe((struct Vmxnet3_TxDesc *)gdesc,
1240 			   (struct Vmxnet3_TxDesc *)ctx.sop_txd);
1241 	gdesc = ctx.sop_txd;
1242 #endif
1243 	netdev_dbg(adapter->netdev,
1244 		"txd[%u]: SOP 0x%Lx 0x%x 0x%x\n",
1245 		(u32)(ctx.sop_txd -
1246 		tq->tx_ring.base), le64_to_cpu(gdesc->txd.addr),
1247 		le32_to_cpu(gdesc->dword[2]), le32_to_cpu(gdesc->dword[3]));
1248 
1249 	spin_unlock_irqrestore(&tq->tx_lock, flags);
1250 
1251 	if (tx_num_deferred >= le32_to_cpu(tq->shared->txThreshold)) {
1252 		tq->shared->txNumDeferred = 0;
1253 		VMXNET3_WRITE_BAR0_REG(adapter,
1254 				       adapter->tx_prod_offset + tq->qid * 8,
1255 				       tq->tx_ring.next2fill);
1256 	}
1257 
1258 	return NETDEV_TX_OK;
1259 
1260 unlock_drop_pkt:
1261 	spin_unlock_irqrestore(&tq->tx_lock, flags);
1262 drop_pkt:
1263 	tq->stats.drop_total++;
1264 	dev_kfree_skb_any(skb);
1265 	return NETDEV_TX_OK;
1266 }
1267 
1268 
1269 static netdev_tx_t
1270 vmxnet3_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
1271 {
1272 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
1273 
1274 	BUG_ON(skb->queue_mapping > adapter->num_tx_queues);
1275 	return vmxnet3_tq_xmit(skb,
1276 			       &adapter->tx_queue[skb->queue_mapping],
1277 			       adapter, netdev);
1278 }
1279 
1280 
1281 static void
1282 vmxnet3_rx_csum(struct vmxnet3_adapter *adapter,
1283 		struct sk_buff *skb,
1284 		union Vmxnet3_GenericDesc *gdesc)
1285 {
1286 	if (!gdesc->rcd.cnc && adapter->netdev->features & NETIF_F_RXCSUM) {
1287 		if (gdesc->rcd.v4 &&
1288 		    (le32_to_cpu(gdesc->dword[3]) &
1289 		     VMXNET3_RCD_CSUM_OK) == VMXNET3_RCD_CSUM_OK) {
1290 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1291 			if ((le32_to_cpu(gdesc->dword[0]) &
1292 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) {
1293 				skb->csum_level = 1;
1294 			}
1295 			WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
1296 				     !(le32_to_cpu(gdesc->dword[0]) &
1297 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1298 			WARN_ON_ONCE(gdesc->rcd.frg &&
1299 				     !(le32_to_cpu(gdesc->dword[0]) &
1300 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1301 		} else if (gdesc->rcd.v6 && (le32_to_cpu(gdesc->dword[3]) &
1302 					     (1 << VMXNET3_RCD_TUC_SHIFT))) {
1303 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1304 			if ((le32_to_cpu(gdesc->dword[0]) &
1305 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT))) {
1306 				skb->csum_level = 1;
1307 			}
1308 			WARN_ON_ONCE(!(gdesc->rcd.tcp || gdesc->rcd.udp) &&
1309 				     !(le32_to_cpu(gdesc->dword[0]) &
1310 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1311 			WARN_ON_ONCE(gdesc->rcd.frg &&
1312 				     !(le32_to_cpu(gdesc->dword[0]) &
1313 				     (1UL << VMXNET3_RCD_HDR_INNER_SHIFT)));
1314 		} else {
1315 			if (gdesc->rcd.csum) {
1316 				skb->csum = htons(gdesc->rcd.csum);
1317 				skb->ip_summed = CHECKSUM_PARTIAL;
1318 			} else {
1319 				skb_checksum_none_assert(skb);
1320 			}
1321 		}
1322 	} else {
1323 		skb_checksum_none_assert(skb);
1324 	}
1325 }
1326 
1327 
1328 static void
1329 vmxnet3_rx_error(struct vmxnet3_rx_queue *rq, struct Vmxnet3_RxCompDesc *rcd,
1330 		 struct vmxnet3_rx_ctx *ctx,  struct vmxnet3_adapter *adapter)
1331 {
1332 	rq->stats.drop_err++;
1333 	if (!rcd->fcs)
1334 		rq->stats.drop_fcs++;
1335 
1336 	rq->stats.drop_total++;
1337 
1338 	/*
1339 	 * We do not unmap and chain the rx buffer to the skb.
1340 	 * We basically pretend this buffer is not used and will be recycled
1341 	 * by vmxnet3_rq_alloc_rx_buf()
1342 	 */
1343 
1344 	/*
1345 	 * ctx->skb may be NULL if this is the first and the only one
1346 	 * desc for the pkt
1347 	 */
1348 	if (ctx->skb)
1349 		dev_kfree_skb_irq(ctx->skb);
1350 
1351 	ctx->skb = NULL;
1352 }
1353 
1354 
1355 static u32
1356 vmxnet3_get_hdr_len(struct vmxnet3_adapter *adapter, struct sk_buff *skb,
1357 		    union Vmxnet3_GenericDesc *gdesc)
1358 {
1359 	u32 hlen, maplen;
1360 	union {
1361 		void *ptr;
1362 		struct ethhdr *eth;
1363 		struct vlan_ethhdr *veth;
1364 		struct iphdr *ipv4;
1365 		struct ipv6hdr *ipv6;
1366 		struct tcphdr *tcp;
1367 	} hdr;
1368 	BUG_ON(gdesc->rcd.tcp == 0);
1369 
1370 	maplen = skb_headlen(skb);
1371 	if (unlikely(sizeof(struct iphdr) + sizeof(struct tcphdr) > maplen))
1372 		return 0;
1373 
1374 	if (skb->protocol == cpu_to_be16(ETH_P_8021Q) ||
1375 	    skb->protocol == cpu_to_be16(ETH_P_8021AD))
1376 		hlen = sizeof(struct vlan_ethhdr);
1377 	else
1378 		hlen = sizeof(struct ethhdr);
1379 
1380 	hdr.eth = eth_hdr(skb);
1381 	if (gdesc->rcd.v4) {
1382 		BUG_ON(hdr.eth->h_proto != htons(ETH_P_IP) &&
1383 		       hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IP));
1384 		hdr.ptr += hlen;
1385 		BUG_ON(hdr.ipv4->protocol != IPPROTO_TCP);
1386 		hlen = hdr.ipv4->ihl << 2;
1387 		hdr.ptr += hdr.ipv4->ihl << 2;
1388 	} else if (gdesc->rcd.v6) {
1389 		BUG_ON(hdr.eth->h_proto != htons(ETH_P_IPV6) &&
1390 		       hdr.veth->h_vlan_encapsulated_proto != htons(ETH_P_IPV6));
1391 		hdr.ptr += hlen;
1392 		/* Use an estimated value, since we also need to handle
1393 		 * TSO case.
1394 		 */
1395 		if (hdr.ipv6->nexthdr != IPPROTO_TCP)
1396 			return sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
1397 		hlen = sizeof(struct ipv6hdr);
1398 		hdr.ptr += sizeof(struct ipv6hdr);
1399 	} else {
1400 		/* Non-IP pkt, dont estimate header length */
1401 		return 0;
1402 	}
1403 
1404 	if (hlen + sizeof(struct tcphdr) > maplen)
1405 		return 0;
1406 
1407 	return (hlen + (hdr.tcp->doff << 2));
1408 }
1409 
1410 static int
1411 vmxnet3_rq_rx_complete(struct vmxnet3_rx_queue *rq,
1412 		       struct vmxnet3_adapter *adapter, int quota)
1413 {
1414 	u32 rxprod_reg[2] = {
1415 		adapter->rx_prod_offset, adapter->rx_prod2_offset
1416 	};
1417 	u32 num_pkts = 0;
1418 	bool skip_page_frags = false;
1419 	bool encap_lro = false;
1420 	struct Vmxnet3_RxCompDesc *rcd;
1421 	struct vmxnet3_rx_ctx *ctx = &rq->rx_ctx;
1422 	u16 segCnt = 0, mss = 0;
1423 	int comp_offset, fill_offset;
1424 #ifdef __BIG_ENDIAN_BITFIELD
1425 	struct Vmxnet3_RxDesc rxCmdDesc;
1426 	struct Vmxnet3_RxCompDesc rxComp;
1427 #endif
1428 	vmxnet3_getRxComp(rcd, &rq->comp_ring.base[rq->comp_ring.next2proc].rcd,
1429 			  &rxComp);
1430 	while (rcd->gen == rq->comp_ring.gen) {
1431 		struct vmxnet3_rx_buf_info *rbi;
1432 		struct sk_buff *skb, *new_skb = NULL;
1433 		struct page *new_page = NULL;
1434 		dma_addr_t new_dma_addr;
1435 		int num_to_alloc;
1436 		struct Vmxnet3_RxDesc *rxd;
1437 		u32 idx, ring_idx;
1438 		struct vmxnet3_cmd_ring	*ring = NULL;
1439 		if (num_pkts >= quota) {
1440 			/* we may stop even before we see the EOP desc of
1441 			 * the current pkt
1442 			 */
1443 			break;
1444 		}
1445 
1446 		/* Prevent any rcd field from being (speculatively) read before
1447 		 * rcd->gen is read.
1448 		 */
1449 		dma_rmb();
1450 
1451 		BUG_ON(rcd->rqID != rq->qid && rcd->rqID != rq->qid2 &&
1452 		       rcd->rqID != rq->dataRingQid);
1453 		idx = rcd->rxdIdx;
1454 		ring_idx = VMXNET3_GET_RING_IDX(adapter, rcd->rqID);
1455 		ring = rq->rx_ring + ring_idx;
1456 		vmxnet3_getRxDesc(rxd, &rq->rx_ring[ring_idx].base[idx].rxd,
1457 				  &rxCmdDesc);
1458 		rbi = rq->buf_info[ring_idx] + idx;
1459 
1460 		BUG_ON(rxd->addr != rbi->dma_addr ||
1461 		       rxd->len != rbi->len);
1462 
1463 		if (unlikely(rcd->eop && rcd->err)) {
1464 			vmxnet3_rx_error(rq, rcd, ctx, adapter);
1465 			goto rcd_done;
1466 		}
1467 
1468 		if (rcd->sop) { /* first buf of the pkt */
1469 			bool rxDataRingUsed;
1470 			u16 len;
1471 
1472 			BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_HEAD ||
1473 			       (rcd->rqID != rq->qid &&
1474 				rcd->rqID != rq->dataRingQid));
1475 
1476 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_SKB);
1477 			BUG_ON(ctx->skb != NULL || rbi->skb == NULL);
1478 
1479 			if (unlikely(rcd->len == 0)) {
1480 				/* Pretend the rx buffer is skipped. */
1481 				BUG_ON(!(rcd->sop && rcd->eop));
1482 				netdev_dbg(adapter->netdev,
1483 					"rxRing[%u][%u] 0 length\n",
1484 					ring_idx, idx);
1485 				goto rcd_done;
1486 			}
1487 
1488 			skip_page_frags = false;
1489 			ctx->skb = rbi->skb;
1490 
1491 			rxDataRingUsed =
1492 				VMXNET3_RX_DATA_RING(adapter, rcd->rqID);
1493 			len = rxDataRingUsed ? rcd->len : rbi->len;
1494 			new_skb = netdev_alloc_skb_ip_align(adapter->netdev,
1495 							    len);
1496 			if (new_skb == NULL) {
1497 				/* Skb allocation failed, do not handover this
1498 				 * skb to stack. Reuse it. Drop the existing pkt
1499 				 */
1500 				rq->stats.rx_buf_alloc_failure++;
1501 				ctx->skb = NULL;
1502 				rq->stats.drop_total++;
1503 				skip_page_frags = true;
1504 				goto rcd_done;
1505 			}
1506 
1507 			if (rxDataRingUsed && adapter->rxdataring_enabled) {
1508 				size_t sz;
1509 
1510 				BUG_ON(rcd->len > rq->data_ring.desc_size);
1511 
1512 				ctx->skb = new_skb;
1513 				sz = rcd->rxdIdx * rq->data_ring.desc_size;
1514 				memcpy(new_skb->data,
1515 				       &rq->data_ring.base[sz], rcd->len);
1516 			} else {
1517 				ctx->skb = rbi->skb;
1518 
1519 				new_dma_addr =
1520 					dma_map_single(&adapter->pdev->dev,
1521 						       new_skb->data, rbi->len,
1522 						       DMA_FROM_DEVICE);
1523 				if (dma_mapping_error(&adapter->pdev->dev,
1524 						      new_dma_addr)) {
1525 					dev_kfree_skb(new_skb);
1526 					/* Skb allocation failed, do not
1527 					 * handover this skb to stack. Reuse
1528 					 * it. Drop the existing pkt.
1529 					 */
1530 					rq->stats.rx_buf_alloc_failure++;
1531 					ctx->skb = NULL;
1532 					rq->stats.drop_total++;
1533 					skip_page_frags = true;
1534 					goto rcd_done;
1535 				}
1536 
1537 				dma_unmap_single(&adapter->pdev->dev,
1538 						 rbi->dma_addr,
1539 						 rbi->len,
1540 						 DMA_FROM_DEVICE);
1541 
1542 				/* Immediate refill */
1543 				rbi->skb = new_skb;
1544 				rbi->dma_addr = new_dma_addr;
1545 				rxd->addr = cpu_to_le64(rbi->dma_addr);
1546 				rxd->len = rbi->len;
1547 			}
1548 
1549 			skb_record_rx_queue(ctx->skb, rq->qid);
1550 			skb_put(ctx->skb, rcd->len);
1551 
1552 			if (VMXNET3_VERSION_GE_2(adapter) &&
1553 			    rcd->type == VMXNET3_CDTYPE_RXCOMP_LRO) {
1554 				struct Vmxnet3_RxCompDescExt *rcdlro;
1555 				union Vmxnet3_GenericDesc *gdesc;
1556 
1557 				rcdlro = (struct Vmxnet3_RxCompDescExt *)rcd;
1558 				gdesc = (union Vmxnet3_GenericDesc *)rcd;
1559 
1560 				segCnt = rcdlro->segCnt;
1561 				WARN_ON_ONCE(segCnt == 0);
1562 				mss = rcdlro->mss;
1563 				if (unlikely(segCnt <= 1))
1564 					segCnt = 0;
1565 				encap_lro = (le32_to_cpu(gdesc->dword[0]) &
1566 					(1UL << VMXNET3_RCD_HDR_INNER_SHIFT));
1567 			} else {
1568 				segCnt = 0;
1569 			}
1570 		} else {
1571 			BUG_ON(ctx->skb == NULL && !skip_page_frags);
1572 
1573 			/* non SOP buffer must be type 1 in most cases */
1574 			BUG_ON(rbi->buf_type != VMXNET3_RX_BUF_PAGE);
1575 			BUG_ON(rxd->btype != VMXNET3_RXD_BTYPE_BODY);
1576 
1577 			/* If an sop buffer was dropped, skip all
1578 			 * following non-sop fragments. They will be reused.
1579 			 */
1580 			if (skip_page_frags)
1581 				goto rcd_done;
1582 
1583 			if (rcd->len) {
1584 				new_page = alloc_page(GFP_ATOMIC);
1585 				/* Replacement page frag could not be allocated.
1586 				 * Reuse this page. Drop the pkt and free the
1587 				 * skb which contained this page as a frag. Skip
1588 				 * processing all the following non-sop frags.
1589 				 */
1590 				if (unlikely(!new_page)) {
1591 					rq->stats.rx_buf_alloc_failure++;
1592 					dev_kfree_skb(ctx->skb);
1593 					ctx->skb = NULL;
1594 					skip_page_frags = true;
1595 					goto rcd_done;
1596 				}
1597 				new_dma_addr = dma_map_page(&adapter->pdev->dev,
1598 							    new_page,
1599 							    0, PAGE_SIZE,
1600 							    DMA_FROM_DEVICE);
1601 				if (dma_mapping_error(&adapter->pdev->dev,
1602 						      new_dma_addr)) {
1603 					put_page(new_page);
1604 					rq->stats.rx_buf_alloc_failure++;
1605 					dev_kfree_skb(ctx->skb);
1606 					ctx->skb = NULL;
1607 					skip_page_frags = true;
1608 					goto rcd_done;
1609 				}
1610 
1611 				dma_unmap_page(&adapter->pdev->dev,
1612 					       rbi->dma_addr, rbi->len,
1613 					       DMA_FROM_DEVICE);
1614 
1615 				vmxnet3_append_frag(ctx->skb, rcd, rbi);
1616 
1617 				/* Immediate refill */
1618 				rbi->page = new_page;
1619 				rbi->dma_addr = new_dma_addr;
1620 				rxd->addr = cpu_to_le64(rbi->dma_addr);
1621 				rxd->len = rbi->len;
1622 			}
1623 		}
1624 
1625 
1626 		skb = ctx->skb;
1627 		if (rcd->eop) {
1628 			u32 mtu = adapter->netdev->mtu;
1629 			skb->len += skb->data_len;
1630 
1631 #ifdef VMXNET3_RSS
1632 			if (rcd->rssType != VMXNET3_RCD_RSS_TYPE_NONE &&
1633 			    (adapter->netdev->features & NETIF_F_RXHASH)) {
1634 				enum pkt_hash_types hash_type;
1635 
1636 				switch (rcd->rssType) {
1637 				case VMXNET3_RCD_RSS_TYPE_IPV4:
1638 				case VMXNET3_RCD_RSS_TYPE_IPV6:
1639 					hash_type = PKT_HASH_TYPE_L3;
1640 					break;
1641 				case VMXNET3_RCD_RSS_TYPE_TCPIPV4:
1642 				case VMXNET3_RCD_RSS_TYPE_TCPIPV6:
1643 				case VMXNET3_RCD_RSS_TYPE_UDPIPV4:
1644 				case VMXNET3_RCD_RSS_TYPE_UDPIPV6:
1645 					hash_type = PKT_HASH_TYPE_L4;
1646 					break;
1647 				default:
1648 					hash_type = PKT_HASH_TYPE_L3;
1649 					break;
1650 				}
1651 				skb_set_hash(skb,
1652 					     le32_to_cpu(rcd->rssHash),
1653 					     hash_type);
1654 			}
1655 #endif
1656 			vmxnet3_rx_csum(adapter, skb,
1657 					(union Vmxnet3_GenericDesc *)rcd);
1658 			skb->protocol = eth_type_trans(skb, adapter->netdev);
1659 			if ((!rcd->tcp && !encap_lro) ||
1660 			    !(adapter->netdev->features & NETIF_F_LRO))
1661 				goto not_lro;
1662 
1663 			if (segCnt != 0 && mss != 0) {
1664 				skb_shinfo(skb)->gso_type = rcd->v4 ?
1665 					SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
1666 				skb_shinfo(skb)->gso_size = mss;
1667 				skb_shinfo(skb)->gso_segs = segCnt;
1668 			} else if ((segCnt != 0 || skb->len > mtu) && !encap_lro) {
1669 				u32 hlen;
1670 
1671 				hlen = vmxnet3_get_hdr_len(adapter, skb,
1672 					(union Vmxnet3_GenericDesc *)rcd);
1673 				if (hlen == 0)
1674 					goto not_lro;
1675 
1676 				skb_shinfo(skb)->gso_type =
1677 					rcd->v4 ? SKB_GSO_TCPV4 : SKB_GSO_TCPV6;
1678 				if (segCnt != 0) {
1679 					skb_shinfo(skb)->gso_segs = segCnt;
1680 					skb_shinfo(skb)->gso_size =
1681 						DIV_ROUND_UP(skb->len -
1682 							hlen, segCnt);
1683 				} else {
1684 					skb_shinfo(skb)->gso_size = mtu - hlen;
1685 				}
1686 			}
1687 not_lro:
1688 			if (unlikely(rcd->ts))
1689 				__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), rcd->tci);
1690 
1691 			/* Use GRO callback if UPT is enabled */
1692 			if ((adapter->netdev->features & NETIF_F_LRO) &&
1693 			    !rq->shared->updateRxProd)
1694 				netif_receive_skb(skb);
1695 			else
1696 				napi_gro_receive(&rq->napi, skb);
1697 
1698 			ctx->skb = NULL;
1699 			encap_lro = false;
1700 			num_pkts++;
1701 		}
1702 
1703 rcd_done:
1704 		/* device may have skipped some rx descs */
1705 		ring = rq->rx_ring + ring_idx;
1706 		rbi->comp_state = VMXNET3_RXD_COMP_DONE;
1707 
1708 		comp_offset = vmxnet3_cmd_ring_desc_avail(ring);
1709 		fill_offset = (idx > ring->next2fill ? 0 : ring->size) +
1710 			      idx - ring->next2fill - 1;
1711 		if (!ring->isOutOfOrder || fill_offset >= comp_offset)
1712 			ring->next2comp = idx;
1713 		num_to_alloc = vmxnet3_cmd_ring_desc_avail(ring);
1714 
1715 		/* Ensure that the writes to rxd->gen bits will be observed
1716 		 * after all other writes to rxd objects.
1717 		 */
1718 		dma_wmb();
1719 
1720 		while (num_to_alloc) {
1721 			rbi = rq->buf_info[ring_idx] + ring->next2fill;
1722 			if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_OOORX_COMP)))
1723 				goto refill_buf;
1724 			if (ring_idx == 0) {
1725 				/* ring0 Type1 buffers can get skipped; re-fill them */
1726 				if (rbi->buf_type != VMXNET3_RX_BUF_SKB)
1727 					goto refill_buf;
1728 			}
1729 			if (rbi->comp_state == VMXNET3_RXD_COMP_DONE) {
1730 refill_buf:
1731 				vmxnet3_getRxDesc(rxd, &ring->base[ring->next2fill].rxd,
1732 						  &rxCmdDesc);
1733 				WARN_ON(!rxd->addr);
1734 
1735 				/* Recv desc is ready to be used by the device */
1736 				rxd->gen = ring->gen;
1737 				vmxnet3_cmd_ring_adv_next2fill(ring);
1738 				rbi->comp_state = VMXNET3_RXD_COMP_PENDING;
1739 				num_to_alloc--;
1740 			} else {
1741 				/* rx completion hasn't occurred */
1742 				ring->isOutOfOrder = 1;
1743 				break;
1744 			}
1745 		}
1746 
1747 		if (num_to_alloc == 0) {
1748 			ring->isOutOfOrder = 0;
1749 		}
1750 
1751 		/* if needed, update the register */
1752 		if (unlikely(rq->shared->updateRxProd) && (ring->next2fill & 0xf) == 0) {
1753 			VMXNET3_WRITE_BAR0_REG(adapter,
1754 					       rxprod_reg[ring_idx] + rq->qid * 8,
1755 					       ring->next2fill);
1756 		}
1757 
1758 		vmxnet3_comp_ring_adv_next2proc(&rq->comp_ring);
1759 		vmxnet3_getRxComp(rcd,
1760 				  &rq->comp_ring.base[rq->comp_ring.next2proc].rcd, &rxComp);
1761 	}
1762 
1763 	return num_pkts;
1764 }
1765 
1766 
1767 static void
1768 vmxnet3_rq_cleanup(struct vmxnet3_rx_queue *rq,
1769 		   struct vmxnet3_adapter *adapter)
1770 {
1771 	u32 i, ring_idx;
1772 	struct Vmxnet3_RxDesc *rxd;
1773 
1774 	/* ring has already been cleaned up */
1775 	if (!rq->rx_ring[0].base)
1776 		return;
1777 
1778 	for (ring_idx = 0; ring_idx < 2; ring_idx++) {
1779 		for (i = 0; i < rq->rx_ring[ring_idx].size; i++) {
1780 #ifdef __BIG_ENDIAN_BITFIELD
1781 			struct Vmxnet3_RxDesc rxDesc;
1782 #endif
1783 			vmxnet3_getRxDesc(rxd,
1784 				&rq->rx_ring[ring_idx].base[i].rxd, &rxDesc);
1785 
1786 			if (rxd->btype == VMXNET3_RXD_BTYPE_HEAD &&
1787 					rq->buf_info[ring_idx][i].skb) {
1788 				dma_unmap_single(&adapter->pdev->dev, rxd->addr,
1789 						 rxd->len, DMA_FROM_DEVICE);
1790 				dev_kfree_skb(rq->buf_info[ring_idx][i].skb);
1791 				rq->buf_info[ring_idx][i].skb = NULL;
1792 			} else if (rxd->btype == VMXNET3_RXD_BTYPE_BODY &&
1793 					rq->buf_info[ring_idx][i].page) {
1794 				dma_unmap_page(&adapter->pdev->dev, rxd->addr,
1795 					       rxd->len, DMA_FROM_DEVICE);
1796 				put_page(rq->buf_info[ring_idx][i].page);
1797 				rq->buf_info[ring_idx][i].page = NULL;
1798 			}
1799 		}
1800 
1801 		rq->rx_ring[ring_idx].gen = VMXNET3_INIT_GEN;
1802 		rq->rx_ring[ring_idx].next2fill =
1803 					rq->rx_ring[ring_idx].next2comp = 0;
1804 	}
1805 
1806 	rq->comp_ring.gen = VMXNET3_INIT_GEN;
1807 	rq->comp_ring.next2proc = 0;
1808 }
1809 
1810 
1811 static void
1812 vmxnet3_rq_cleanup_all(struct vmxnet3_adapter *adapter)
1813 {
1814 	int i;
1815 
1816 	for (i = 0; i < adapter->num_rx_queues; i++)
1817 		vmxnet3_rq_cleanup(&adapter->rx_queue[i], adapter);
1818 }
1819 
1820 
1821 static void vmxnet3_rq_destroy(struct vmxnet3_rx_queue *rq,
1822 			       struct vmxnet3_adapter *adapter)
1823 {
1824 	int i;
1825 	int j;
1826 
1827 	/* all rx buffers must have already been freed */
1828 	for (i = 0; i < 2; i++) {
1829 		if (rq->buf_info[i]) {
1830 			for (j = 0; j < rq->rx_ring[i].size; j++)
1831 				BUG_ON(rq->buf_info[i][j].page != NULL);
1832 		}
1833 	}
1834 
1835 
1836 	for (i = 0; i < 2; i++) {
1837 		if (rq->rx_ring[i].base) {
1838 			dma_free_coherent(&adapter->pdev->dev,
1839 					  rq->rx_ring[i].size
1840 					  * sizeof(struct Vmxnet3_RxDesc),
1841 					  rq->rx_ring[i].base,
1842 					  rq->rx_ring[i].basePA);
1843 			rq->rx_ring[i].base = NULL;
1844 		}
1845 	}
1846 
1847 	if (rq->data_ring.base) {
1848 		dma_free_coherent(&adapter->pdev->dev,
1849 				  rq->rx_ring[0].size * rq->data_ring.desc_size,
1850 				  rq->data_ring.base, rq->data_ring.basePA);
1851 		rq->data_ring.base = NULL;
1852 	}
1853 
1854 	if (rq->comp_ring.base) {
1855 		dma_free_coherent(&adapter->pdev->dev, rq->comp_ring.size
1856 				  * sizeof(struct Vmxnet3_RxCompDesc),
1857 				  rq->comp_ring.base, rq->comp_ring.basePA);
1858 		rq->comp_ring.base = NULL;
1859 	}
1860 
1861 	kfree(rq->buf_info[0]);
1862 	rq->buf_info[0] = NULL;
1863 	rq->buf_info[1] = NULL;
1864 }
1865 
1866 static void
1867 vmxnet3_rq_destroy_all_rxdataring(struct vmxnet3_adapter *adapter)
1868 {
1869 	int i;
1870 
1871 	for (i = 0; i < adapter->num_rx_queues; i++) {
1872 		struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
1873 
1874 		if (rq->data_ring.base) {
1875 			dma_free_coherent(&adapter->pdev->dev,
1876 					  (rq->rx_ring[0].size *
1877 					  rq->data_ring.desc_size),
1878 					  rq->data_ring.base,
1879 					  rq->data_ring.basePA);
1880 			rq->data_ring.base = NULL;
1881 			rq->data_ring.desc_size = 0;
1882 		}
1883 	}
1884 }
1885 
1886 static int
1887 vmxnet3_rq_init(struct vmxnet3_rx_queue *rq,
1888 		struct vmxnet3_adapter  *adapter)
1889 {
1890 	int i;
1891 
1892 	/* initialize buf_info */
1893 	for (i = 0; i < rq->rx_ring[0].size; i++) {
1894 
1895 		/* 1st buf for a pkt is skbuff */
1896 		if (i % adapter->rx_buf_per_pkt == 0) {
1897 			rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_SKB;
1898 			rq->buf_info[0][i].len = adapter->skb_buf_size;
1899 		} else { /* subsequent bufs for a pkt is frag */
1900 			rq->buf_info[0][i].buf_type = VMXNET3_RX_BUF_PAGE;
1901 			rq->buf_info[0][i].len = PAGE_SIZE;
1902 		}
1903 	}
1904 	for (i = 0; i < rq->rx_ring[1].size; i++) {
1905 		rq->buf_info[1][i].buf_type = VMXNET3_RX_BUF_PAGE;
1906 		rq->buf_info[1][i].len = PAGE_SIZE;
1907 	}
1908 
1909 	/* reset internal state and allocate buffers for both rings */
1910 	for (i = 0; i < 2; i++) {
1911 		rq->rx_ring[i].next2fill = rq->rx_ring[i].next2comp = 0;
1912 
1913 		memset(rq->rx_ring[i].base, 0, rq->rx_ring[i].size *
1914 		       sizeof(struct Vmxnet3_RxDesc));
1915 		rq->rx_ring[i].gen = VMXNET3_INIT_GEN;
1916 		rq->rx_ring[i].isOutOfOrder = 0;
1917 	}
1918 	if (vmxnet3_rq_alloc_rx_buf(rq, 0, rq->rx_ring[0].size - 1,
1919 				    adapter) == 0) {
1920 		/* at least has 1 rx buffer for the 1st ring */
1921 		return -ENOMEM;
1922 	}
1923 	vmxnet3_rq_alloc_rx_buf(rq, 1, rq->rx_ring[1].size - 1, adapter);
1924 
1925 	/* reset the comp ring */
1926 	rq->comp_ring.next2proc = 0;
1927 	memset(rq->comp_ring.base, 0, rq->comp_ring.size *
1928 	       sizeof(struct Vmxnet3_RxCompDesc));
1929 	rq->comp_ring.gen = VMXNET3_INIT_GEN;
1930 
1931 	/* reset rxctx */
1932 	rq->rx_ctx.skb = NULL;
1933 
1934 	/* stats are not reset */
1935 	return 0;
1936 }
1937 
1938 
1939 static int
1940 vmxnet3_rq_init_all(struct vmxnet3_adapter *adapter)
1941 {
1942 	int i, err = 0;
1943 
1944 	for (i = 0; i < adapter->num_rx_queues; i++) {
1945 		err = vmxnet3_rq_init(&adapter->rx_queue[i], adapter);
1946 		if (unlikely(err)) {
1947 			dev_err(&adapter->netdev->dev, "%s: failed to "
1948 				"initialize rx queue%i\n",
1949 				adapter->netdev->name, i);
1950 			break;
1951 		}
1952 	}
1953 	return err;
1954 
1955 }
1956 
1957 
1958 static int
1959 vmxnet3_rq_create(struct vmxnet3_rx_queue *rq, struct vmxnet3_adapter *adapter)
1960 {
1961 	int i;
1962 	size_t sz;
1963 	struct vmxnet3_rx_buf_info *bi;
1964 
1965 	for (i = 0; i < 2; i++) {
1966 
1967 		sz = rq->rx_ring[i].size * sizeof(struct Vmxnet3_RxDesc);
1968 		rq->rx_ring[i].base = dma_alloc_coherent(
1969 						&adapter->pdev->dev, sz,
1970 						&rq->rx_ring[i].basePA,
1971 						GFP_KERNEL);
1972 		if (!rq->rx_ring[i].base) {
1973 			netdev_err(adapter->netdev,
1974 				   "failed to allocate rx ring %d\n", i);
1975 			goto err;
1976 		}
1977 	}
1978 
1979 	if ((adapter->rxdataring_enabled) && (rq->data_ring.desc_size != 0)) {
1980 		sz = rq->rx_ring[0].size * rq->data_ring.desc_size;
1981 		rq->data_ring.base =
1982 			dma_alloc_coherent(&adapter->pdev->dev, sz,
1983 					   &rq->data_ring.basePA,
1984 					   GFP_KERNEL);
1985 		if (!rq->data_ring.base) {
1986 			netdev_err(adapter->netdev,
1987 				   "rx data ring will be disabled\n");
1988 			adapter->rxdataring_enabled = false;
1989 		}
1990 	} else {
1991 		rq->data_ring.base = NULL;
1992 		rq->data_ring.desc_size = 0;
1993 	}
1994 
1995 	sz = rq->comp_ring.size * sizeof(struct Vmxnet3_RxCompDesc);
1996 	rq->comp_ring.base = dma_alloc_coherent(&adapter->pdev->dev, sz,
1997 						&rq->comp_ring.basePA,
1998 						GFP_KERNEL);
1999 	if (!rq->comp_ring.base) {
2000 		netdev_err(adapter->netdev, "failed to allocate rx comp ring\n");
2001 		goto err;
2002 	}
2003 
2004 	bi = kcalloc_node(rq->rx_ring[0].size + rq->rx_ring[1].size,
2005 			  sizeof(rq->buf_info[0][0]), GFP_KERNEL,
2006 			  dev_to_node(&adapter->pdev->dev));
2007 	if (!bi)
2008 		goto err;
2009 
2010 	rq->buf_info[0] = bi;
2011 	rq->buf_info[1] = bi + rq->rx_ring[0].size;
2012 
2013 	return 0;
2014 
2015 err:
2016 	vmxnet3_rq_destroy(rq, adapter);
2017 	return -ENOMEM;
2018 }
2019 
2020 
2021 static int
2022 vmxnet3_rq_create_all(struct vmxnet3_adapter *adapter)
2023 {
2024 	int i, err = 0;
2025 
2026 	adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
2027 
2028 	for (i = 0; i < adapter->num_rx_queues; i++) {
2029 		err = vmxnet3_rq_create(&adapter->rx_queue[i], adapter);
2030 		if (unlikely(err)) {
2031 			dev_err(&adapter->netdev->dev,
2032 				"%s: failed to create rx queue%i\n",
2033 				adapter->netdev->name, i);
2034 			goto err_out;
2035 		}
2036 	}
2037 
2038 	if (!adapter->rxdataring_enabled)
2039 		vmxnet3_rq_destroy_all_rxdataring(adapter);
2040 
2041 	return err;
2042 err_out:
2043 	vmxnet3_rq_destroy_all(adapter);
2044 	return err;
2045 
2046 }
2047 
2048 /* Multiple queue aware polling function for tx and rx */
2049 
2050 static int
2051 vmxnet3_do_poll(struct vmxnet3_adapter *adapter, int budget)
2052 {
2053 	int rcd_done = 0, i;
2054 	if (unlikely(adapter->shared->ecr))
2055 		vmxnet3_process_events(adapter);
2056 	for (i = 0; i < adapter->num_tx_queues; i++)
2057 		vmxnet3_tq_tx_complete(&adapter->tx_queue[i], adapter);
2058 
2059 	for (i = 0; i < adapter->num_rx_queues; i++)
2060 		rcd_done += vmxnet3_rq_rx_complete(&adapter->rx_queue[i],
2061 						   adapter, budget);
2062 	return rcd_done;
2063 }
2064 
2065 
2066 static int
2067 vmxnet3_poll(struct napi_struct *napi, int budget)
2068 {
2069 	struct vmxnet3_rx_queue *rx_queue = container_of(napi,
2070 					  struct vmxnet3_rx_queue, napi);
2071 	int rxd_done;
2072 
2073 	rxd_done = vmxnet3_do_poll(rx_queue->adapter, budget);
2074 
2075 	if (rxd_done < budget) {
2076 		napi_complete_done(napi, rxd_done);
2077 		vmxnet3_enable_all_intrs(rx_queue->adapter);
2078 	}
2079 	return rxd_done;
2080 }
2081 
2082 /*
2083  * NAPI polling function for MSI-X mode with multiple Rx queues
2084  * Returns the # of the NAPI credit consumed (# of rx descriptors processed)
2085  */
2086 
2087 static int
2088 vmxnet3_poll_rx_only(struct napi_struct *napi, int budget)
2089 {
2090 	struct vmxnet3_rx_queue *rq = container_of(napi,
2091 						struct vmxnet3_rx_queue, napi);
2092 	struct vmxnet3_adapter *adapter = rq->adapter;
2093 	int rxd_done;
2094 
2095 	/* When sharing interrupt with corresponding tx queue, process
2096 	 * tx completions in that queue as well
2097 	 */
2098 	if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE) {
2099 		struct vmxnet3_tx_queue *tq =
2100 				&adapter->tx_queue[rq - adapter->rx_queue];
2101 		vmxnet3_tq_tx_complete(tq, adapter);
2102 	}
2103 
2104 	rxd_done = vmxnet3_rq_rx_complete(rq, adapter, budget);
2105 
2106 	if (rxd_done < budget) {
2107 		napi_complete_done(napi, rxd_done);
2108 		vmxnet3_enable_intr(adapter, rq->comp_ring.intr_idx);
2109 	}
2110 	return rxd_done;
2111 }
2112 
2113 
2114 #ifdef CONFIG_PCI_MSI
2115 
2116 /*
2117  * Handle completion interrupts on tx queues
2118  * Returns whether or not the intr is handled
2119  */
2120 
2121 static irqreturn_t
2122 vmxnet3_msix_tx(int irq, void *data)
2123 {
2124 	struct vmxnet3_tx_queue *tq = data;
2125 	struct vmxnet3_adapter *adapter = tq->adapter;
2126 
2127 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2128 		vmxnet3_disable_intr(adapter, tq->comp_ring.intr_idx);
2129 
2130 	/* Handle the case where only one irq is allocate for all tx queues */
2131 	if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
2132 		int i;
2133 		for (i = 0; i < adapter->num_tx_queues; i++) {
2134 			struct vmxnet3_tx_queue *txq = &adapter->tx_queue[i];
2135 			vmxnet3_tq_tx_complete(txq, adapter);
2136 		}
2137 	} else {
2138 		vmxnet3_tq_tx_complete(tq, adapter);
2139 	}
2140 	vmxnet3_enable_intr(adapter, tq->comp_ring.intr_idx);
2141 
2142 	return IRQ_HANDLED;
2143 }
2144 
2145 
2146 /*
2147  * Handle completion interrupts on rx queues. Returns whether or not the
2148  * intr is handled
2149  */
2150 
2151 static irqreturn_t
2152 vmxnet3_msix_rx(int irq, void *data)
2153 {
2154 	struct vmxnet3_rx_queue *rq = data;
2155 	struct vmxnet3_adapter *adapter = rq->adapter;
2156 
2157 	/* disable intr if needed */
2158 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2159 		vmxnet3_disable_intr(adapter, rq->comp_ring.intr_idx);
2160 	napi_schedule(&rq->napi);
2161 
2162 	return IRQ_HANDLED;
2163 }
2164 
2165 /*
2166  *----------------------------------------------------------------------------
2167  *
2168  * vmxnet3_msix_event --
2169  *
2170  *    vmxnet3 msix event intr handler
2171  *
2172  * Result:
2173  *    whether or not the intr is handled
2174  *
2175  *----------------------------------------------------------------------------
2176  */
2177 
2178 static irqreturn_t
2179 vmxnet3_msix_event(int irq, void *data)
2180 {
2181 	struct net_device *dev = data;
2182 	struct vmxnet3_adapter *adapter = netdev_priv(dev);
2183 
2184 	/* disable intr if needed */
2185 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2186 		vmxnet3_disable_intr(adapter, adapter->intr.event_intr_idx);
2187 
2188 	if (adapter->shared->ecr)
2189 		vmxnet3_process_events(adapter);
2190 
2191 	vmxnet3_enable_intr(adapter, adapter->intr.event_intr_idx);
2192 
2193 	return IRQ_HANDLED;
2194 }
2195 
2196 #endif /* CONFIG_PCI_MSI  */
2197 
2198 
2199 /* Interrupt handler for vmxnet3  */
2200 static irqreturn_t
2201 vmxnet3_intr(int irq, void *dev_id)
2202 {
2203 	struct net_device *dev = dev_id;
2204 	struct vmxnet3_adapter *adapter = netdev_priv(dev);
2205 
2206 	if (adapter->intr.type == VMXNET3_IT_INTX) {
2207 		u32 icr = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_ICR);
2208 		if (unlikely(icr == 0))
2209 			/* not ours */
2210 			return IRQ_NONE;
2211 	}
2212 
2213 
2214 	/* disable intr if needed */
2215 	if (adapter->intr.mask_mode == VMXNET3_IMM_ACTIVE)
2216 		vmxnet3_disable_all_intrs(adapter);
2217 
2218 	napi_schedule(&adapter->rx_queue[0].napi);
2219 
2220 	return IRQ_HANDLED;
2221 }
2222 
2223 #ifdef CONFIG_NET_POLL_CONTROLLER
2224 
2225 /* netpoll callback. */
2226 static void
2227 vmxnet3_netpoll(struct net_device *netdev)
2228 {
2229 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2230 
2231 	switch (adapter->intr.type) {
2232 #ifdef CONFIG_PCI_MSI
2233 	case VMXNET3_IT_MSIX: {
2234 		int i;
2235 		for (i = 0; i < adapter->num_rx_queues; i++)
2236 			vmxnet3_msix_rx(0, &adapter->rx_queue[i]);
2237 		break;
2238 	}
2239 #endif
2240 	case VMXNET3_IT_MSI:
2241 	default:
2242 		vmxnet3_intr(0, adapter->netdev);
2243 		break;
2244 	}
2245 
2246 }
2247 #endif	/* CONFIG_NET_POLL_CONTROLLER */
2248 
2249 static int
2250 vmxnet3_request_irqs(struct vmxnet3_adapter *adapter)
2251 {
2252 	struct vmxnet3_intr *intr = &adapter->intr;
2253 	int err = 0, i;
2254 	int vector = 0;
2255 
2256 #ifdef CONFIG_PCI_MSI
2257 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
2258 		for (i = 0; i < adapter->num_tx_queues; i++) {
2259 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
2260 				sprintf(adapter->tx_queue[i].name, "%s-tx-%d",
2261 					adapter->netdev->name, vector);
2262 				err = request_irq(
2263 					      intr->msix_entries[vector].vector,
2264 					      vmxnet3_msix_tx, 0,
2265 					      adapter->tx_queue[i].name,
2266 					      &adapter->tx_queue[i]);
2267 			} else {
2268 				sprintf(adapter->tx_queue[i].name, "%s-rxtx-%d",
2269 					adapter->netdev->name, vector);
2270 			}
2271 			if (err) {
2272 				dev_err(&adapter->netdev->dev,
2273 					"Failed to request irq for MSIX, %s, "
2274 					"error %d\n",
2275 					adapter->tx_queue[i].name, err);
2276 				return err;
2277 			}
2278 
2279 			/* Handle the case where only 1 MSIx was allocated for
2280 			 * all tx queues */
2281 			if (adapter->share_intr == VMXNET3_INTR_TXSHARE) {
2282 				for (; i < adapter->num_tx_queues; i++)
2283 					adapter->tx_queue[i].comp_ring.intr_idx
2284 								= vector;
2285 				vector++;
2286 				break;
2287 			} else {
2288 				adapter->tx_queue[i].comp_ring.intr_idx
2289 								= vector++;
2290 			}
2291 		}
2292 		if (adapter->share_intr == VMXNET3_INTR_BUDDYSHARE)
2293 			vector = 0;
2294 
2295 		for (i = 0; i < adapter->num_rx_queues; i++) {
2296 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE)
2297 				sprintf(adapter->rx_queue[i].name, "%s-rx-%d",
2298 					adapter->netdev->name, vector);
2299 			else
2300 				sprintf(adapter->rx_queue[i].name, "%s-rxtx-%d",
2301 					adapter->netdev->name, vector);
2302 			err = request_irq(intr->msix_entries[vector].vector,
2303 					  vmxnet3_msix_rx, 0,
2304 					  adapter->rx_queue[i].name,
2305 					  &(adapter->rx_queue[i]));
2306 			if (err) {
2307 				netdev_err(adapter->netdev,
2308 					   "Failed to request irq for MSIX, "
2309 					   "%s, error %d\n",
2310 					   adapter->rx_queue[i].name, err);
2311 				return err;
2312 			}
2313 
2314 			adapter->rx_queue[i].comp_ring.intr_idx = vector++;
2315 		}
2316 
2317 		sprintf(intr->event_msi_vector_name, "%s-event-%d",
2318 			adapter->netdev->name, vector);
2319 		err = request_irq(intr->msix_entries[vector].vector,
2320 				  vmxnet3_msix_event, 0,
2321 				  intr->event_msi_vector_name, adapter->netdev);
2322 		intr->event_intr_idx = vector;
2323 
2324 	} else if (intr->type == VMXNET3_IT_MSI) {
2325 		adapter->num_rx_queues = 1;
2326 		err = request_irq(adapter->pdev->irq, vmxnet3_intr, 0,
2327 				  adapter->netdev->name, adapter->netdev);
2328 	} else {
2329 #endif
2330 		adapter->num_rx_queues = 1;
2331 		err = request_irq(adapter->pdev->irq, vmxnet3_intr,
2332 				  IRQF_SHARED, adapter->netdev->name,
2333 				  adapter->netdev);
2334 #ifdef CONFIG_PCI_MSI
2335 	}
2336 #endif
2337 	intr->num_intrs = vector + 1;
2338 	if (err) {
2339 		netdev_err(adapter->netdev,
2340 			   "Failed to request irq (intr type:%d), error %d\n",
2341 			   intr->type, err);
2342 	} else {
2343 		/* Number of rx queues will not change after this */
2344 		for (i = 0; i < adapter->num_rx_queues; i++) {
2345 			struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
2346 			rq->qid = i;
2347 			rq->qid2 = i + adapter->num_rx_queues;
2348 			rq->dataRingQid = i + 2 * adapter->num_rx_queues;
2349 		}
2350 
2351 		/* init our intr settings */
2352 		for (i = 0; i < intr->num_intrs; i++)
2353 			intr->mod_levels[i] = UPT1_IML_ADAPTIVE;
2354 		if (adapter->intr.type != VMXNET3_IT_MSIX) {
2355 			adapter->intr.event_intr_idx = 0;
2356 			for (i = 0; i < adapter->num_tx_queues; i++)
2357 				adapter->tx_queue[i].comp_ring.intr_idx = 0;
2358 			adapter->rx_queue[0].comp_ring.intr_idx = 0;
2359 		}
2360 
2361 		netdev_info(adapter->netdev,
2362 			    "intr type %u, mode %u, %u vectors allocated\n",
2363 			    intr->type, intr->mask_mode, intr->num_intrs);
2364 	}
2365 
2366 	return err;
2367 }
2368 
2369 
2370 static void
2371 vmxnet3_free_irqs(struct vmxnet3_adapter *adapter)
2372 {
2373 	struct vmxnet3_intr *intr = &adapter->intr;
2374 	BUG_ON(intr->type == VMXNET3_IT_AUTO || intr->num_intrs <= 0);
2375 
2376 	switch (intr->type) {
2377 #ifdef CONFIG_PCI_MSI
2378 	case VMXNET3_IT_MSIX:
2379 	{
2380 		int i, vector = 0;
2381 
2382 		if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE) {
2383 			for (i = 0; i < adapter->num_tx_queues; i++) {
2384 				free_irq(intr->msix_entries[vector++].vector,
2385 					 &(adapter->tx_queue[i]));
2386 				if (adapter->share_intr == VMXNET3_INTR_TXSHARE)
2387 					break;
2388 			}
2389 		}
2390 
2391 		for (i = 0; i < adapter->num_rx_queues; i++) {
2392 			free_irq(intr->msix_entries[vector++].vector,
2393 				 &(adapter->rx_queue[i]));
2394 		}
2395 
2396 		free_irq(intr->msix_entries[vector].vector,
2397 			 adapter->netdev);
2398 		BUG_ON(vector >= intr->num_intrs);
2399 		break;
2400 	}
2401 #endif
2402 	case VMXNET3_IT_MSI:
2403 		free_irq(adapter->pdev->irq, adapter->netdev);
2404 		break;
2405 	case VMXNET3_IT_INTX:
2406 		free_irq(adapter->pdev->irq, adapter->netdev);
2407 		break;
2408 	default:
2409 		BUG();
2410 	}
2411 }
2412 
2413 
2414 static void
2415 vmxnet3_restore_vlan(struct vmxnet3_adapter *adapter)
2416 {
2417 	u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2418 	u16 vid;
2419 
2420 	/* allow untagged pkts */
2421 	VMXNET3_SET_VFTABLE_ENTRY(vfTable, 0);
2422 
2423 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
2424 		VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
2425 }
2426 
2427 
2428 static int
2429 vmxnet3_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid)
2430 {
2431 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2432 
2433 	if (!(netdev->flags & IFF_PROMISC)) {
2434 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2435 		unsigned long flags;
2436 
2437 		VMXNET3_SET_VFTABLE_ENTRY(vfTable, vid);
2438 		spin_lock_irqsave(&adapter->cmd_lock, flags);
2439 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2440 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2441 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2442 	}
2443 
2444 	set_bit(vid, adapter->active_vlans);
2445 
2446 	return 0;
2447 }
2448 
2449 
2450 static int
2451 vmxnet3_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid)
2452 {
2453 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2454 
2455 	if (!(netdev->flags & IFF_PROMISC)) {
2456 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2457 		unsigned long flags;
2458 
2459 		VMXNET3_CLEAR_VFTABLE_ENTRY(vfTable, vid);
2460 		spin_lock_irqsave(&adapter->cmd_lock, flags);
2461 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2462 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2463 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2464 	}
2465 
2466 	clear_bit(vid, adapter->active_vlans);
2467 
2468 	return 0;
2469 }
2470 
2471 
2472 static u8 *
2473 vmxnet3_copy_mc(struct net_device *netdev)
2474 {
2475 	u8 *buf = NULL;
2476 	u32 sz = netdev_mc_count(netdev) * ETH_ALEN;
2477 
2478 	/* struct Vmxnet3_RxFilterConf.mfTableLen is u16. */
2479 	if (sz <= 0xffff) {
2480 		/* We may be called with BH disabled */
2481 		buf = kmalloc(sz, GFP_ATOMIC);
2482 		if (buf) {
2483 			struct netdev_hw_addr *ha;
2484 			int i = 0;
2485 
2486 			netdev_for_each_mc_addr(ha, netdev)
2487 				memcpy(buf + i++ * ETH_ALEN, ha->addr,
2488 				       ETH_ALEN);
2489 		}
2490 	}
2491 	return buf;
2492 }
2493 
2494 
2495 static void
2496 vmxnet3_set_mc(struct net_device *netdev)
2497 {
2498 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2499 	unsigned long flags;
2500 	struct Vmxnet3_RxFilterConf *rxConf =
2501 					&adapter->shared->devRead.rxFilterConf;
2502 	u8 *new_table = NULL;
2503 	dma_addr_t new_table_pa = 0;
2504 	bool new_table_pa_valid = false;
2505 	u32 new_mode = VMXNET3_RXM_UCAST;
2506 
2507 	if (netdev->flags & IFF_PROMISC) {
2508 		u32 *vfTable = adapter->shared->devRead.rxFilterConf.vfTable;
2509 		memset(vfTable, 0, VMXNET3_VFT_SIZE * sizeof(*vfTable));
2510 
2511 		new_mode |= VMXNET3_RXM_PROMISC;
2512 	} else {
2513 		vmxnet3_restore_vlan(adapter);
2514 	}
2515 
2516 	if (netdev->flags & IFF_BROADCAST)
2517 		new_mode |= VMXNET3_RXM_BCAST;
2518 
2519 	if (netdev->flags & IFF_ALLMULTI)
2520 		new_mode |= VMXNET3_RXM_ALL_MULTI;
2521 	else
2522 		if (!netdev_mc_empty(netdev)) {
2523 			new_table = vmxnet3_copy_mc(netdev);
2524 			if (new_table) {
2525 				size_t sz = netdev_mc_count(netdev) * ETH_ALEN;
2526 
2527 				rxConf->mfTableLen = cpu_to_le16(sz);
2528 				new_table_pa = dma_map_single(
2529 							&adapter->pdev->dev,
2530 							new_table,
2531 							sz,
2532 							DMA_TO_DEVICE);
2533 				if (!dma_mapping_error(&adapter->pdev->dev,
2534 						       new_table_pa)) {
2535 					new_mode |= VMXNET3_RXM_MCAST;
2536 					new_table_pa_valid = true;
2537 					rxConf->mfTablePA = cpu_to_le64(
2538 								new_table_pa);
2539 				}
2540 			}
2541 			if (!new_table_pa_valid) {
2542 				netdev_info(netdev,
2543 					    "failed to copy mcast list, setting ALL_MULTI\n");
2544 				new_mode |= VMXNET3_RXM_ALL_MULTI;
2545 			}
2546 		}
2547 
2548 	if (!(new_mode & VMXNET3_RXM_MCAST)) {
2549 		rxConf->mfTableLen = 0;
2550 		rxConf->mfTablePA = 0;
2551 	}
2552 
2553 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2554 	if (new_mode != rxConf->rxMode) {
2555 		rxConf->rxMode = cpu_to_le32(new_mode);
2556 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2557 				       VMXNET3_CMD_UPDATE_RX_MODE);
2558 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2559 				       VMXNET3_CMD_UPDATE_VLAN_FILTERS);
2560 	}
2561 
2562 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2563 			       VMXNET3_CMD_UPDATE_MAC_FILTERS);
2564 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2565 
2566 	if (new_table_pa_valid)
2567 		dma_unmap_single(&adapter->pdev->dev, new_table_pa,
2568 				 rxConf->mfTableLen, DMA_TO_DEVICE);
2569 	kfree(new_table);
2570 }
2571 
2572 void
2573 vmxnet3_rq_destroy_all(struct vmxnet3_adapter *adapter)
2574 {
2575 	int i;
2576 
2577 	for (i = 0; i < adapter->num_rx_queues; i++)
2578 		vmxnet3_rq_destroy(&adapter->rx_queue[i], adapter);
2579 }
2580 
2581 
2582 /*
2583  *   Set up driver_shared based on settings in adapter.
2584  */
2585 
2586 static void
2587 vmxnet3_setup_driver_shared(struct vmxnet3_adapter *adapter)
2588 {
2589 	struct Vmxnet3_DriverShared *shared = adapter->shared;
2590 	struct Vmxnet3_DSDevRead *devRead = &shared->devRead;
2591 	struct Vmxnet3_DSDevReadExt *devReadExt = &shared->devReadExt;
2592 	struct Vmxnet3_TxQueueConf *tqc;
2593 	struct Vmxnet3_RxQueueConf *rqc;
2594 	int i;
2595 
2596 	memset(shared, 0, sizeof(*shared));
2597 
2598 	/* driver settings */
2599 	shared->magic = cpu_to_le32(VMXNET3_REV1_MAGIC);
2600 	devRead->misc.driverInfo.version = cpu_to_le32(
2601 						VMXNET3_DRIVER_VERSION_NUM);
2602 	devRead->misc.driverInfo.gos.gosBits = (sizeof(void *) == 4 ?
2603 				VMXNET3_GOS_BITS_32 : VMXNET3_GOS_BITS_64);
2604 	devRead->misc.driverInfo.gos.gosType = VMXNET3_GOS_TYPE_LINUX;
2605 	*((u32 *)&devRead->misc.driverInfo.gos) = cpu_to_le32(
2606 				*((u32 *)&devRead->misc.driverInfo.gos));
2607 	devRead->misc.driverInfo.vmxnet3RevSpt = cpu_to_le32(1);
2608 	devRead->misc.driverInfo.uptVerSpt = cpu_to_le32(1);
2609 
2610 	devRead->misc.ddPA = cpu_to_le64(adapter->adapter_pa);
2611 	devRead->misc.ddLen = cpu_to_le32(sizeof(struct vmxnet3_adapter));
2612 
2613 	/* set up feature flags */
2614 	if (adapter->netdev->features & NETIF_F_RXCSUM)
2615 		devRead->misc.uptFeatures |= UPT1_F_RXCSUM;
2616 
2617 	if (adapter->netdev->features & NETIF_F_LRO) {
2618 		devRead->misc.uptFeatures |= UPT1_F_LRO;
2619 		devRead->misc.maxNumRxSG = cpu_to_le16(1 + MAX_SKB_FRAGS);
2620 	}
2621 	if (adapter->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
2622 		devRead->misc.uptFeatures |= UPT1_F_RXVLAN;
2623 
2624 	if (adapter->netdev->features & (NETIF_F_GSO_UDP_TUNNEL |
2625 					 NETIF_F_GSO_UDP_TUNNEL_CSUM))
2626 		devRead->misc.uptFeatures |= UPT1_F_RXINNEROFLD;
2627 
2628 	devRead->misc.mtu = cpu_to_le32(adapter->netdev->mtu);
2629 	devRead->misc.queueDescPA = cpu_to_le64(adapter->queue_desc_pa);
2630 	devRead->misc.queueDescLen = cpu_to_le32(
2631 		adapter->num_tx_queues * sizeof(struct Vmxnet3_TxQueueDesc) +
2632 		adapter->num_rx_queues * sizeof(struct Vmxnet3_RxQueueDesc));
2633 
2634 	/* tx queue settings */
2635 	devRead->misc.numTxQueues =  adapter->num_tx_queues;
2636 	for (i = 0; i < adapter->num_tx_queues; i++) {
2637 		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];
2638 		BUG_ON(adapter->tx_queue[i].tx_ring.base == NULL);
2639 		tqc = &adapter->tqd_start[i].conf;
2640 		tqc->txRingBasePA   = cpu_to_le64(tq->tx_ring.basePA);
2641 		tqc->dataRingBasePA = cpu_to_le64(tq->data_ring.basePA);
2642 		tqc->compRingBasePA = cpu_to_le64(tq->comp_ring.basePA);
2643 		tqc->ddPA           = cpu_to_le64(~0ULL);
2644 		tqc->txRingSize     = cpu_to_le32(tq->tx_ring.size);
2645 		tqc->dataRingSize   = cpu_to_le32(tq->data_ring.size);
2646 		tqc->txDataRingDescSize = cpu_to_le32(tq->txdata_desc_size);
2647 		tqc->compRingSize   = cpu_to_le32(tq->comp_ring.size);
2648 		tqc->ddLen          = cpu_to_le32(0);
2649 		tqc->intrIdx        = tq->comp_ring.intr_idx;
2650 	}
2651 
2652 	/* rx queue settings */
2653 	devRead->misc.numRxQueues = adapter->num_rx_queues;
2654 	for (i = 0; i < adapter->num_rx_queues; i++) {
2655 		struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[i];
2656 		rqc = &adapter->rqd_start[i].conf;
2657 		rqc->rxRingBasePA[0] = cpu_to_le64(rq->rx_ring[0].basePA);
2658 		rqc->rxRingBasePA[1] = cpu_to_le64(rq->rx_ring[1].basePA);
2659 		rqc->compRingBasePA  = cpu_to_le64(rq->comp_ring.basePA);
2660 		rqc->ddPA            = cpu_to_le64(~0ULL);
2661 		rqc->rxRingSize[0]   = cpu_to_le32(rq->rx_ring[0].size);
2662 		rqc->rxRingSize[1]   = cpu_to_le32(rq->rx_ring[1].size);
2663 		rqc->compRingSize    = cpu_to_le32(rq->comp_ring.size);
2664 		rqc->ddLen           = cpu_to_le32(0);
2665 		rqc->intrIdx         = rq->comp_ring.intr_idx;
2666 		if (VMXNET3_VERSION_GE_3(adapter)) {
2667 			rqc->rxDataRingBasePA =
2668 				cpu_to_le64(rq->data_ring.basePA);
2669 			rqc->rxDataRingDescSize =
2670 				cpu_to_le16(rq->data_ring.desc_size);
2671 		}
2672 	}
2673 
2674 #ifdef VMXNET3_RSS
2675 	memset(adapter->rss_conf, 0, sizeof(*adapter->rss_conf));
2676 
2677 	if (adapter->rss) {
2678 		struct UPT1_RSSConf *rssConf = adapter->rss_conf;
2679 
2680 		devRead->misc.uptFeatures |= UPT1_F_RSS;
2681 		devRead->misc.numRxQueues = adapter->num_rx_queues;
2682 		rssConf->hashType = UPT1_RSS_HASH_TYPE_TCP_IPV4 |
2683 				    UPT1_RSS_HASH_TYPE_IPV4 |
2684 				    UPT1_RSS_HASH_TYPE_TCP_IPV6 |
2685 				    UPT1_RSS_HASH_TYPE_IPV6;
2686 		rssConf->hashFunc = UPT1_RSS_HASH_FUNC_TOEPLITZ;
2687 		rssConf->hashKeySize = UPT1_RSS_MAX_KEY_SIZE;
2688 		rssConf->indTableSize = VMXNET3_RSS_IND_TABLE_SIZE;
2689 		netdev_rss_key_fill(rssConf->hashKey, sizeof(rssConf->hashKey));
2690 
2691 		for (i = 0; i < rssConf->indTableSize; i++)
2692 			rssConf->indTable[i] = ethtool_rxfh_indir_default(
2693 				i, adapter->num_rx_queues);
2694 
2695 		devRead->rssConfDesc.confVer = 1;
2696 		devRead->rssConfDesc.confLen = cpu_to_le32(sizeof(*rssConf));
2697 		devRead->rssConfDesc.confPA =
2698 			cpu_to_le64(adapter->rss_conf_pa);
2699 	}
2700 
2701 #endif /* VMXNET3_RSS */
2702 
2703 	/* intr settings */
2704 	if (!VMXNET3_VERSION_GE_6(adapter) ||
2705 	    !adapter->queuesExtEnabled) {
2706 		devRead->intrConf.autoMask = adapter->intr.mask_mode ==
2707 					     VMXNET3_IMM_AUTO;
2708 		devRead->intrConf.numIntrs = adapter->intr.num_intrs;
2709 		for (i = 0; i < adapter->intr.num_intrs; i++)
2710 			devRead->intrConf.modLevels[i] = adapter->intr.mod_levels[i];
2711 
2712 		devRead->intrConf.eventIntrIdx = adapter->intr.event_intr_idx;
2713 		devRead->intrConf.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2714 	} else {
2715 		devReadExt->intrConfExt.autoMask = adapter->intr.mask_mode ==
2716 						   VMXNET3_IMM_AUTO;
2717 		devReadExt->intrConfExt.numIntrs = adapter->intr.num_intrs;
2718 		for (i = 0; i < adapter->intr.num_intrs; i++)
2719 			devReadExt->intrConfExt.modLevels[i] = adapter->intr.mod_levels[i];
2720 
2721 		devReadExt->intrConfExt.eventIntrIdx = adapter->intr.event_intr_idx;
2722 		devReadExt->intrConfExt.intrCtrl |= cpu_to_le32(VMXNET3_IC_DISABLE_ALL);
2723 	}
2724 
2725 	/* rx filter settings */
2726 	devRead->rxFilterConf.rxMode = 0;
2727 	vmxnet3_restore_vlan(adapter);
2728 	vmxnet3_write_mac_addr(adapter, adapter->netdev->dev_addr);
2729 
2730 	/* the rest are already zeroed */
2731 }
2732 
2733 static void
2734 vmxnet3_init_bufsize(struct vmxnet3_adapter *adapter)
2735 {
2736 	struct Vmxnet3_DriverShared *shared = adapter->shared;
2737 	union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2738 	unsigned long flags;
2739 
2740 	if (!VMXNET3_VERSION_GE_7(adapter))
2741 		return;
2742 
2743 	cmdInfo->ringBufSize = adapter->ringBufSize;
2744 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2745 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2746 			       VMXNET3_CMD_SET_RING_BUFFER_SIZE);
2747 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2748 }
2749 
2750 static void
2751 vmxnet3_init_coalesce(struct vmxnet3_adapter *adapter)
2752 {
2753 	struct Vmxnet3_DriverShared *shared = adapter->shared;
2754 	union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2755 	unsigned long flags;
2756 
2757 	if (!VMXNET3_VERSION_GE_3(adapter))
2758 		return;
2759 
2760 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2761 	cmdInfo->varConf.confVer = 1;
2762 	cmdInfo->varConf.confLen =
2763 		cpu_to_le32(sizeof(*adapter->coal_conf));
2764 	cmdInfo->varConf.confPA  = cpu_to_le64(adapter->coal_conf_pa);
2765 
2766 	if (adapter->default_coal_mode) {
2767 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2768 				       VMXNET3_CMD_GET_COALESCE);
2769 	} else {
2770 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2771 				       VMXNET3_CMD_SET_COALESCE);
2772 	}
2773 
2774 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2775 }
2776 
2777 static void
2778 vmxnet3_init_rssfields(struct vmxnet3_adapter *adapter)
2779 {
2780 	struct Vmxnet3_DriverShared *shared = adapter->shared;
2781 	union Vmxnet3_CmdInfo *cmdInfo = &shared->cu.cmdInfo;
2782 	unsigned long flags;
2783 
2784 	if (!VMXNET3_VERSION_GE_4(adapter))
2785 		return;
2786 
2787 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2788 
2789 	if (adapter->default_rss_fields) {
2790 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2791 				       VMXNET3_CMD_GET_RSS_FIELDS);
2792 		adapter->rss_fields =
2793 			VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2794 	} else {
2795 		if (VMXNET3_VERSION_GE_7(adapter)) {
2796 			if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP4 ||
2797 			     adapter->rss_fields & VMXNET3_RSS_FIELDS_UDPIP6) &&
2798 			    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2799 						       VMXNET3_CAP_UDP_RSS)) {
2800 				adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_UDP_RSS;
2801 			} else {
2802 				adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_UDP_RSS);
2803 			}
2804 
2805 			if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP4) &&
2806 			    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2807 						       VMXNET3_CAP_ESP_RSS_IPV4)) {
2808 				adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV4;
2809 			} else {
2810 				adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV4);
2811 			}
2812 
2813 			if ((adapter->rss_fields & VMXNET3_RSS_FIELDS_ESPIP6) &&
2814 			    vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
2815 						       VMXNET3_CAP_ESP_RSS_IPV6)) {
2816 				adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_ESP_RSS_IPV6;
2817 			} else {
2818 				adapter->dev_caps[0] &= ~(1UL << VMXNET3_CAP_ESP_RSS_IPV6);
2819 			}
2820 
2821 			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
2822 			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
2823 			adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2824 		}
2825 		cmdInfo->setRssFields = adapter->rss_fields;
2826 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2827 				       VMXNET3_CMD_SET_RSS_FIELDS);
2828 		/* Not all requested RSS may get applied, so get and
2829 		 * cache what was actually applied.
2830 		 */
2831 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2832 				       VMXNET3_CMD_GET_RSS_FIELDS);
2833 		adapter->rss_fields =
2834 			VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2835 	}
2836 
2837 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2838 }
2839 
2840 int
2841 vmxnet3_activate_dev(struct vmxnet3_adapter *adapter)
2842 {
2843 	int err, i;
2844 	u32 ret;
2845 	unsigned long flags;
2846 
2847 	netdev_dbg(adapter->netdev, "%s: skb_buf_size %d, rx_buf_per_pkt %d,"
2848 		" ring sizes %u %u %u\n", adapter->netdev->name,
2849 		adapter->skb_buf_size, adapter->rx_buf_per_pkt,
2850 		adapter->tx_queue[0].tx_ring.size,
2851 		adapter->rx_queue[0].rx_ring[0].size,
2852 		adapter->rx_queue[0].rx_ring[1].size);
2853 
2854 	vmxnet3_tq_init_all(adapter);
2855 	err = vmxnet3_rq_init_all(adapter);
2856 	if (err) {
2857 		netdev_err(adapter->netdev,
2858 			   "Failed to init rx queue error %d\n", err);
2859 		goto rq_err;
2860 	}
2861 
2862 	err = vmxnet3_request_irqs(adapter);
2863 	if (err) {
2864 		netdev_err(adapter->netdev,
2865 			   "Failed to setup irq for error %d\n", err);
2866 		goto irq_err;
2867 	}
2868 
2869 	vmxnet3_setup_driver_shared(adapter);
2870 
2871 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, VMXNET3_GET_ADDR_LO(
2872 			       adapter->shared_pa));
2873 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, VMXNET3_GET_ADDR_HI(
2874 			       adapter->shared_pa));
2875 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2876 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2877 			       VMXNET3_CMD_ACTIVATE_DEV);
2878 	ret = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
2879 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2880 
2881 	if (ret != 0) {
2882 		netdev_err(adapter->netdev,
2883 			   "Failed to activate dev: error %u\n", ret);
2884 		err = -EINVAL;
2885 		goto activate_err;
2886 	}
2887 
2888 	vmxnet3_init_bufsize(adapter);
2889 	vmxnet3_init_coalesce(adapter);
2890 	vmxnet3_init_rssfields(adapter);
2891 
2892 	for (i = 0; i < adapter->num_rx_queues; i++) {
2893 		VMXNET3_WRITE_BAR0_REG(adapter,
2894 				adapter->rx_prod_offset + i * VMXNET3_REG_ALIGN,
2895 				adapter->rx_queue[i].rx_ring[0].next2fill);
2896 		VMXNET3_WRITE_BAR0_REG(adapter, (adapter->rx_prod2_offset +
2897 				(i * VMXNET3_REG_ALIGN)),
2898 				adapter->rx_queue[i].rx_ring[1].next2fill);
2899 	}
2900 
2901 	/* Apply the rx filter settins last. */
2902 	vmxnet3_set_mc(adapter->netdev);
2903 
2904 	/*
2905 	 * Check link state when first activating device. It will start the
2906 	 * tx queue if the link is up.
2907 	 */
2908 	vmxnet3_check_link(adapter, true);
2909 	netif_tx_wake_all_queues(adapter->netdev);
2910 	for (i = 0; i < adapter->num_rx_queues; i++)
2911 		napi_enable(&adapter->rx_queue[i].napi);
2912 	vmxnet3_enable_all_intrs(adapter);
2913 	clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
2914 	return 0;
2915 
2916 activate_err:
2917 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAL, 0);
2918 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DSAH, 0);
2919 	vmxnet3_free_irqs(adapter);
2920 irq_err:
2921 rq_err:
2922 	/* free up buffers we allocated */
2923 	vmxnet3_rq_cleanup_all(adapter);
2924 	return err;
2925 }
2926 
2927 
2928 void
2929 vmxnet3_reset_dev(struct vmxnet3_adapter *adapter)
2930 {
2931 	unsigned long flags;
2932 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2933 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_RESET_DEV);
2934 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2935 }
2936 
2937 
2938 int
2939 vmxnet3_quiesce_dev(struct vmxnet3_adapter *adapter)
2940 {
2941 	int i;
2942 	unsigned long flags;
2943 	if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state))
2944 		return 0;
2945 
2946 
2947 	spin_lock_irqsave(&adapter->cmd_lock, flags);
2948 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
2949 			       VMXNET3_CMD_QUIESCE_DEV);
2950 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
2951 	vmxnet3_disable_all_intrs(adapter);
2952 
2953 	for (i = 0; i < adapter->num_rx_queues; i++)
2954 		napi_disable(&adapter->rx_queue[i].napi);
2955 	netif_tx_disable(adapter->netdev);
2956 	adapter->link_speed = 0;
2957 	netif_carrier_off(adapter->netdev);
2958 
2959 	vmxnet3_tq_cleanup_all(adapter);
2960 	vmxnet3_rq_cleanup_all(adapter);
2961 	vmxnet3_free_irqs(adapter);
2962 	return 0;
2963 }
2964 
2965 
2966 static void
2967 vmxnet3_write_mac_addr(struct vmxnet3_adapter *adapter, const u8 *mac)
2968 {
2969 	u32 tmp;
2970 
2971 	tmp = *(u32 *)mac;
2972 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACL, tmp);
2973 
2974 	tmp = (mac[5] << 8) | mac[4];
2975 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_MACH, tmp);
2976 }
2977 
2978 
2979 static int
2980 vmxnet3_set_mac_addr(struct net_device *netdev, void *p)
2981 {
2982 	struct sockaddr *addr = p;
2983 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
2984 
2985 	dev_addr_set(netdev, addr->sa_data);
2986 	vmxnet3_write_mac_addr(adapter, addr->sa_data);
2987 
2988 	return 0;
2989 }
2990 
2991 
2992 /* ==================== initialization and cleanup routines ============ */
2993 
2994 static int
2995 vmxnet3_alloc_pci_resources(struct vmxnet3_adapter *adapter)
2996 {
2997 	int err;
2998 	unsigned long mmio_start, mmio_len;
2999 	struct pci_dev *pdev = adapter->pdev;
3000 
3001 	err = pci_enable_device(pdev);
3002 	if (err) {
3003 		dev_err(&pdev->dev, "Failed to enable adapter: error %d\n", err);
3004 		return err;
3005 	}
3006 
3007 	err = pci_request_selected_regions(pdev, (1 << 2) - 1,
3008 					   vmxnet3_driver_name);
3009 	if (err) {
3010 		dev_err(&pdev->dev,
3011 			"Failed to request region for adapter: error %d\n", err);
3012 		goto err_enable_device;
3013 	}
3014 
3015 	pci_set_master(pdev);
3016 
3017 	mmio_start = pci_resource_start(pdev, 0);
3018 	mmio_len = pci_resource_len(pdev, 0);
3019 	adapter->hw_addr0 = ioremap(mmio_start, mmio_len);
3020 	if (!adapter->hw_addr0) {
3021 		dev_err(&pdev->dev, "Failed to map bar0\n");
3022 		err = -EIO;
3023 		goto err_ioremap;
3024 	}
3025 
3026 	mmio_start = pci_resource_start(pdev, 1);
3027 	mmio_len = pci_resource_len(pdev, 1);
3028 	adapter->hw_addr1 = ioremap(mmio_start, mmio_len);
3029 	if (!adapter->hw_addr1) {
3030 		dev_err(&pdev->dev, "Failed to map bar1\n");
3031 		err = -EIO;
3032 		goto err_bar1;
3033 	}
3034 	return 0;
3035 
3036 err_bar1:
3037 	iounmap(adapter->hw_addr0);
3038 err_ioremap:
3039 	pci_release_selected_regions(pdev, (1 << 2) - 1);
3040 err_enable_device:
3041 	pci_disable_device(pdev);
3042 	return err;
3043 }
3044 
3045 
3046 static void
3047 vmxnet3_free_pci_resources(struct vmxnet3_adapter *adapter)
3048 {
3049 	BUG_ON(!adapter->pdev);
3050 
3051 	iounmap(adapter->hw_addr0);
3052 	iounmap(adapter->hw_addr1);
3053 	pci_release_selected_regions(adapter->pdev, (1 << 2) - 1);
3054 	pci_disable_device(adapter->pdev);
3055 }
3056 
3057 
3058 static void
3059 vmxnet3_adjust_rx_ring_size(struct vmxnet3_adapter *adapter)
3060 {
3061 	size_t sz, i, ring0_size, ring1_size, comp_size;
3062 	/* With version7 ring1 will have only T0 buffers */
3063 	if (!VMXNET3_VERSION_GE_7(adapter)) {
3064 		if (adapter->netdev->mtu <= VMXNET3_MAX_SKB_BUF_SIZE -
3065 					    VMXNET3_MAX_ETH_HDR_SIZE) {
3066 			adapter->skb_buf_size = adapter->netdev->mtu +
3067 						VMXNET3_MAX_ETH_HDR_SIZE;
3068 			if (adapter->skb_buf_size < VMXNET3_MIN_T0_BUF_SIZE)
3069 				adapter->skb_buf_size = VMXNET3_MIN_T0_BUF_SIZE;
3070 
3071 			adapter->rx_buf_per_pkt = 1;
3072 		} else {
3073 			adapter->skb_buf_size = VMXNET3_MAX_SKB_BUF_SIZE;
3074 			sz = adapter->netdev->mtu - VMXNET3_MAX_SKB_BUF_SIZE +
3075 						    VMXNET3_MAX_ETH_HDR_SIZE;
3076 			adapter->rx_buf_per_pkt = 1 + (sz + PAGE_SIZE - 1) / PAGE_SIZE;
3077 		}
3078 	} else {
3079 		adapter->skb_buf_size = min((int)adapter->netdev->mtu + VMXNET3_MAX_ETH_HDR_SIZE,
3080 					    VMXNET3_MAX_SKB_BUF_SIZE);
3081 		adapter->rx_buf_per_pkt = 1;
3082 		adapter->ringBufSize.ring1BufSizeType0 = cpu_to_le16(adapter->skb_buf_size);
3083 		adapter->ringBufSize.ring1BufSizeType1 = 0;
3084 		adapter->ringBufSize.ring2BufSizeType1 = cpu_to_le16(PAGE_SIZE);
3085 	}
3086 
3087 	/*
3088 	 * for simplicity, force the ring0 size to be a multiple of
3089 	 * rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN
3090 	 */
3091 	sz = adapter->rx_buf_per_pkt * VMXNET3_RING_SIZE_ALIGN;
3092 	ring0_size = adapter->rx_queue[0].rx_ring[0].size;
3093 	ring0_size = (ring0_size + sz - 1) / sz * sz;
3094 	ring0_size = min_t(u32, ring0_size, VMXNET3_RX_RING_MAX_SIZE /
3095 			   sz * sz);
3096 	ring1_size = adapter->rx_queue[0].rx_ring[1].size;
3097 	ring1_size = (ring1_size + sz - 1) / sz * sz;
3098 	ring1_size = min_t(u32, ring1_size, VMXNET3_RX_RING2_MAX_SIZE /
3099 			   sz * sz);
3100 	/* For v7 and later, keep ring size power of 2 for UPT */
3101 	if (VMXNET3_VERSION_GE_7(adapter)) {
3102 		ring0_size = rounddown_pow_of_two(ring0_size);
3103 		ring1_size = rounddown_pow_of_two(ring1_size);
3104 	}
3105 	comp_size = ring0_size + ring1_size;
3106 
3107 	for (i = 0; i < adapter->num_rx_queues; i++) {
3108 		struct vmxnet3_rx_queue	*rq = &adapter->rx_queue[i];
3109 
3110 		rq->rx_ring[0].size = ring0_size;
3111 		rq->rx_ring[1].size = ring1_size;
3112 		rq->comp_ring.size = comp_size;
3113 	}
3114 }
3115 
3116 
3117 int
3118 vmxnet3_create_queues(struct vmxnet3_adapter *adapter, u32 tx_ring_size,
3119 		      u32 rx_ring_size, u32 rx_ring2_size,
3120 		      u16 txdata_desc_size, u16 rxdata_desc_size)
3121 {
3122 	int err = 0, i;
3123 
3124 	for (i = 0; i < adapter->num_tx_queues; i++) {
3125 		struct vmxnet3_tx_queue	*tq = &adapter->tx_queue[i];
3126 		tq->tx_ring.size   = tx_ring_size;
3127 		tq->data_ring.size = tx_ring_size;
3128 		tq->comp_ring.size = tx_ring_size;
3129 		tq->txdata_desc_size = txdata_desc_size;
3130 		tq->shared = &adapter->tqd_start[i].ctrl;
3131 		tq->stopped = true;
3132 		tq->adapter = adapter;
3133 		tq->qid = i;
3134 		err = vmxnet3_tq_create(tq, adapter);
3135 		/*
3136 		 * Too late to change num_tx_queues. We cannot do away with
3137 		 * lesser number of queues than what we asked for
3138 		 */
3139 		if (err)
3140 			goto queue_err;
3141 	}
3142 
3143 	adapter->rx_queue[0].rx_ring[0].size = rx_ring_size;
3144 	adapter->rx_queue[0].rx_ring[1].size = rx_ring2_size;
3145 	vmxnet3_adjust_rx_ring_size(adapter);
3146 
3147 	adapter->rxdataring_enabled = VMXNET3_VERSION_GE_3(adapter);
3148 	for (i = 0; i < adapter->num_rx_queues; i++) {
3149 		struct vmxnet3_rx_queue *rq = &adapter->rx_queue[i];
3150 		/* qid and qid2 for rx queues will be assigned later when num
3151 		 * of rx queues is finalized after allocating intrs */
3152 		rq->shared = &adapter->rqd_start[i].ctrl;
3153 		rq->adapter = adapter;
3154 		rq->data_ring.desc_size = rxdata_desc_size;
3155 		err = vmxnet3_rq_create(rq, adapter);
3156 		if (err) {
3157 			if (i == 0) {
3158 				netdev_err(adapter->netdev,
3159 					   "Could not allocate any rx queues. "
3160 					   "Aborting.\n");
3161 				goto queue_err;
3162 			} else {
3163 				netdev_info(adapter->netdev,
3164 					    "Number of rx queues changed "
3165 					    "to : %d.\n", i);
3166 				adapter->num_rx_queues = i;
3167 				err = 0;
3168 				break;
3169 			}
3170 		}
3171 	}
3172 
3173 	if (!adapter->rxdataring_enabled)
3174 		vmxnet3_rq_destroy_all_rxdataring(adapter);
3175 
3176 	return err;
3177 queue_err:
3178 	vmxnet3_tq_destroy_all(adapter);
3179 	return err;
3180 }
3181 
3182 static int
3183 vmxnet3_open(struct net_device *netdev)
3184 {
3185 	struct vmxnet3_adapter *adapter;
3186 	int err, i;
3187 
3188 	adapter = netdev_priv(netdev);
3189 
3190 	for (i = 0; i < adapter->num_tx_queues; i++)
3191 		spin_lock_init(&adapter->tx_queue[i].tx_lock);
3192 
3193 	if (VMXNET3_VERSION_GE_3(adapter)) {
3194 		unsigned long flags;
3195 		u16 txdata_desc_size;
3196 
3197 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3198 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3199 				       VMXNET3_CMD_GET_TXDATA_DESC_SIZE);
3200 		txdata_desc_size = VMXNET3_READ_BAR1_REG(adapter,
3201 							 VMXNET3_REG_CMD);
3202 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3203 
3204 		if ((txdata_desc_size < VMXNET3_TXDATA_DESC_MIN_SIZE) ||
3205 		    (txdata_desc_size > VMXNET3_TXDATA_DESC_MAX_SIZE) ||
3206 		    (txdata_desc_size & VMXNET3_TXDATA_DESC_SIZE_MASK)) {
3207 			adapter->txdata_desc_size =
3208 				sizeof(struct Vmxnet3_TxDataDesc);
3209 		} else {
3210 			adapter->txdata_desc_size = txdata_desc_size;
3211 		}
3212 	} else {
3213 		adapter->txdata_desc_size = sizeof(struct Vmxnet3_TxDataDesc);
3214 	}
3215 
3216 	err = vmxnet3_create_queues(adapter,
3217 				    adapter->tx_ring_size,
3218 				    adapter->rx_ring_size,
3219 				    adapter->rx_ring2_size,
3220 				    adapter->txdata_desc_size,
3221 				    adapter->rxdata_desc_size);
3222 	if (err)
3223 		goto queue_err;
3224 
3225 	err = vmxnet3_activate_dev(adapter);
3226 	if (err)
3227 		goto activate_err;
3228 
3229 	return 0;
3230 
3231 activate_err:
3232 	vmxnet3_rq_destroy_all(adapter);
3233 	vmxnet3_tq_destroy_all(adapter);
3234 queue_err:
3235 	return err;
3236 }
3237 
3238 
3239 static int
3240 vmxnet3_close(struct net_device *netdev)
3241 {
3242 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3243 
3244 	/*
3245 	 * Reset_work may be in the middle of resetting the device, wait for its
3246 	 * completion.
3247 	 */
3248 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3249 		usleep_range(1000, 2000);
3250 
3251 	vmxnet3_quiesce_dev(adapter);
3252 
3253 	vmxnet3_rq_destroy_all(adapter);
3254 	vmxnet3_tq_destroy_all(adapter);
3255 
3256 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3257 
3258 
3259 	return 0;
3260 }
3261 
3262 
3263 void
3264 vmxnet3_force_close(struct vmxnet3_adapter *adapter)
3265 {
3266 	int i;
3267 
3268 	/*
3269 	 * we must clear VMXNET3_STATE_BIT_RESETTING, otherwise
3270 	 * vmxnet3_close() will deadlock.
3271 	 */
3272 	BUG_ON(test_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state));
3273 
3274 	/* we need to enable NAPI, otherwise dev_close will deadlock */
3275 	for (i = 0; i < adapter->num_rx_queues; i++)
3276 		napi_enable(&adapter->rx_queue[i].napi);
3277 	/*
3278 	 * Need to clear the quiesce bit to ensure that vmxnet3_close
3279 	 * can quiesce the device properly
3280 	 */
3281 	clear_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3282 	dev_close(adapter->netdev);
3283 }
3284 
3285 
3286 static int
3287 vmxnet3_change_mtu(struct net_device *netdev, int new_mtu)
3288 {
3289 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3290 	int err = 0;
3291 
3292 	netdev->mtu = new_mtu;
3293 
3294 	/*
3295 	 * Reset_work may be in the middle of resetting the device, wait for its
3296 	 * completion.
3297 	 */
3298 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3299 		usleep_range(1000, 2000);
3300 
3301 	if (netif_running(netdev)) {
3302 		vmxnet3_quiesce_dev(adapter);
3303 		vmxnet3_reset_dev(adapter);
3304 
3305 		/* we need to re-create the rx queue based on the new mtu */
3306 		vmxnet3_rq_destroy_all(adapter);
3307 		vmxnet3_adjust_rx_ring_size(adapter);
3308 		err = vmxnet3_rq_create_all(adapter);
3309 		if (err) {
3310 			netdev_err(netdev,
3311 				   "failed to re-create rx queues, "
3312 				   " error %d. Closing it.\n", err);
3313 			goto out;
3314 		}
3315 
3316 		err = vmxnet3_activate_dev(adapter);
3317 		if (err) {
3318 			netdev_err(netdev,
3319 				   "failed to re-activate, error %d. "
3320 				   "Closing it\n", err);
3321 			goto out;
3322 		}
3323 	}
3324 
3325 out:
3326 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3327 	if (err)
3328 		vmxnet3_force_close(adapter);
3329 
3330 	return err;
3331 }
3332 
3333 
3334 static void
3335 vmxnet3_declare_features(struct vmxnet3_adapter *adapter)
3336 {
3337 	struct net_device *netdev = adapter->netdev;
3338 
3339 	netdev->hw_features = NETIF_F_SG | NETIF_F_RXCSUM |
3340 		NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
3341 		NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
3342 		NETIF_F_LRO | NETIF_F_HIGHDMA;
3343 
3344 	if (VMXNET3_VERSION_GE_4(adapter)) {
3345 		netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL |
3346 				NETIF_F_GSO_UDP_TUNNEL_CSUM;
3347 
3348 		netdev->hw_enc_features = NETIF_F_SG | NETIF_F_RXCSUM |
3349 			NETIF_F_HW_CSUM | NETIF_F_HW_VLAN_CTAG_TX |
3350 			NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_TSO | NETIF_F_TSO6 |
3351 			NETIF_F_LRO | NETIF_F_GSO_UDP_TUNNEL |
3352 			NETIF_F_GSO_UDP_TUNNEL_CSUM;
3353 	}
3354 
3355 	if (VMXNET3_VERSION_GE_7(adapter)) {
3356 		unsigned long flags;
3357 
3358 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3359 					       VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) {
3360 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD;
3361 		}
3362 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3363 					       VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) {
3364 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD;
3365 		}
3366 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3367 					       VMXNET3_CAP_GENEVE_TSO)) {
3368 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_TSO;
3369 		}
3370 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3371 					       VMXNET3_CAP_VXLAN_TSO)) {
3372 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_TSO;
3373 		}
3374 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3375 					       VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) {
3376 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD;
3377 		}
3378 		if (vmxnet3_check_ptcapability(adapter->ptcap_supported[0],
3379 					       VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD)) {
3380 			adapter->dev_caps[0] |= 1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD;
3381 		}
3382 
3383 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
3384 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3385 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
3386 		adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3387 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3388 
3389 		if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_CHECKSUM_OFFLOAD)) &&
3390 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_CHECKSUM_OFFLOAD)) &&
3391 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_TSO)) &&
3392 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_TSO))) {
3393 			netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL;
3394 			netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL;
3395 		}
3396 		if (!(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_GENEVE_OUTER_CHECKSUM_OFFLOAD)) &&
3397 		    !(adapter->dev_caps[0] & (1UL << VMXNET3_CAP_VXLAN_OUTER_CHECKSUM_OFFLOAD))) {
3398 			netdev->hw_enc_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
3399 			netdev->hw_features &= ~NETIF_F_GSO_UDP_TUNNEL_CSUM;
3400 		}
3401 	}
3402 
3403 	netdev->vlan_features = netdev->hw_features &
3404 				~(NETIF_F_HW_VLAN_CTAG_TX |
3405 				  NETIF_F_HW_VLAN_CTAG_RX);
3406 	netdev->features = netdev->hw_features | NETIF_F_HW_VLAN_CTAG_FILTER;
3407 }
3408 
3409 
3410 static void
3411 vmxnet3_read_mac_addr(struct vmxnet3_adapter *adapter, u8 *mac)
3412 {
3413 	u32 tmp;
3414 
3415 	tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACL);
3416 	*(u32 *)mac = tmp;
3417 
3418 	tmp = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_MACH);
3419 	mac[4] = tmp & 0xff;
3420 	mac[5] = (tmp >> 8) & 0xff;
3421 }
3422 
3423 #ifdef CONFIG_PCI_MSI
3424 
3425 /*
3426  * Enable MSIx vectors.
3427  * Returns :
3428  *	VMXNET3_LINUX_MIN_MSIX_VECT when only minimum number of vectors required
3429  *	 were enabled.
3430  *	number of vectors which were enabled otherwise (this number is greater
3431  *	 than VMXNET3_LINUX_MIN_MSIX_VECT)
3432  */
3433 
3434 static int
3435 vmxnet3_acquire_msix_vectors(struct vmxnet3_adapter *adapter, int nvec)
3436 {
3437 	int ret = pci_enable_msix_range(adapter->pdev,
3438 					adapter->intr.msix_entries, nvec, nvec);
3439 
3440 	if (ret == -ENOSPC && nvec > VMXNET3_LINUX_MIN_MSIX_VECT) {
3441 		dev_err(&adapter->netdev->dev,
3442 			"Failed to enable %d MSI-X, trying %d\n",
3443 			nvec, VMXNET3_LINUX_MIN_MSIX_VECT);
3444 
3445 		ret = pci_enable_msix_range(adapter->pdev,
3446 					    adapter->intr.msix_entries,
3447 					    VMXNET3_LINUX_MIN_MSIX_VECT,
3448 					    VMXNET3_LINUX_MIN_MSIX_VECT);
3449 	}
3450 
3451 	if (ret < 0) {
3452 		dev_err(&adapter->netdev->dev,
3453 			"Failed to enable MSI-X, error: %d\n", ret);
3454 	}
3455 
3456 	return ret;
3457 }
3458 
3459 
3460 #endif /* CONFIG_PCI_MSI */
3461 
3462 static void
3463 vmxnet3_alloc_intr_resources(struct vmxnet3_adapter *adapter)
3464 {
3465 	u32 cfg;
3466 	unsigned long flags;
3467 
3468 	/* intr settings */
3469 	spin_lock_irqsave(&adapter->cmd_lock, flags);
3470 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3471 			       VMXNET3_CMD_GET_CONF_INTR);
3472 	cfg = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3473 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3474 	adapter->intr.type = cfg & 0x3;
3475 	adapter->intr.mask_mode = (cfg >> 2) & 0x3;
3476 
3477 	if (adapter->intr.type == VMXNET3_IT_AUTO) {
3478 		adapter->intr.type = VMXNET3_IT_MSIX;
3479 	}
3480 
3481 #ifdef CONFIG_PCI_MSI
3482 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
3483 		int i, nvec, nvec_allocated;
3484 
3485 		nvec  = adapter->share_intr == VMXNET3_INTR_TXSHARE ?
3486 			1 : adapter->num_tx_queues;
3487 		nvec += adapter->share_intr == VMXNET3_INTR_BUDDYSHARE ?
3488 			0 : adapter->num_rx_queues;
3489 		nvec += 1;	/* for link event */
3490 		nvec = nvec > VMXNET3_LINUX_MIN_MSIX_VECT ?
3491 		       nvec : VMXNET3_LINUX_MIN_MSIX_VECT;
3492 
3493 		for (i = 0; i < nvec; i++)
3494 			adapter->intr.msix_entries[i].entry = i;
3495 
3496 		nvec_allocated = vmxnet3_acquire_msix_vectors(adapter, nvec);
3497 		if (nvec_allocated < 0)
3498 			goto msix_err;
3499 
3500 		/* If we cannot allocate one MSIx vector per queue
3501 		 * then limit the number of rx queues to 1
3502 		 */
3503 		if (nvec_allocated == VMXNET3_LINUX_MIN_MSIX_VECT &&
3504 		    nvec != VMXNET3_LINUX_MIN_MSIX_VECT) {
3505 			if (adapter->share_intr != VMXNET3_INTR_BUDDYSHARE
3506 			    || adapter->num_rx_queues != 1) {
3507 				adapter->share_intr = VMXNET3_INTR_TXSHARE;
3508 				netdev_err(adapter->netdev,
3509 					   "Number of rx queues : 1\n");
3510 				adapter->num_rx_queues = 1;
3511 			}
3512 		}
3513 
3514 		adapter->intr.num_intrs = nvec_allocated;
3515 		return;
3516 
3517 msix_err:
3518 		/* If we cannot allocate MSIx vectors use only one rx queue */
3519 		dev_info(&adapter->pdev->dev,
3520 			 "Failed to enable MSI-X, error %d. "
3521 			 "Limiting #rx queues to 1, try MSI.\n", nvec_allocated);
3522 
3523 		adapter->intr.type = VMXNET3_IT_MSI;
3524 	}
3525 
3526 	if (adapter->intr.type == VMXNET3_IT_MSI) {
3527 		if (!pci_enable_msi(adapter->pdev)) {
3528 			adapter->num_rx_queues = 1;
3529 			adapter->intr.num_intrs = 1;
3530 			return;
3531 		}
3532 	}
3533 #endif /* CONFIG_PCI_MSI */
3534 
3535 	adapter->num_rx_queues = 1;
3536 	dev_info(&adapter->netdev->dev,
3537 		 "Using INTx interrupt, #Rx queues: 1.\n");
3538 	adapter->intr.type = VMXNET3_IT_INTX;
3539 
3540 	/* INT-X related setting */
3541 	adapter->intr.num_intrs = 1;
3542 }
3543 
3544 
3545 static void
3546 vmxnet3_free_intr_resources(struct vmxnet3_adapter *adapter)
3547 {
3548 	if (adapter->intr.type == VMXNET3_IT_MSIX)
3549 		pci_disable_msix(adapter->pdev);
3550 	else if (adapter->intr.type == VMXNET3_IT_MSI)
3551 		pci_disable_msi(adapter->pdev);
3552 	else
3553 		BUG_ON(adapter->intr.type != VMXNET3_IT_INTX);
3554 }
3555 
3556 
3557 static void
3558 vmxnet3_tx_timeout(struct net_device *netdev, unsigned int txqueue)
3559 {
3560 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3561 	adapter->tx_timeout_count++;
3562 
3563 	netdev_err(adapter->netdev, "tx hang\n");
3564 	schedule_work(&adapter->work);
3565 }
3566 
3567 
3568 static void
3569 vmxnet3_reset_work(struct work_struct *data)
3570 {
3571 	struct vmxnet3_adapter *adapter;
3572 
3573 	adapter = container_of(data, struct vmxnet3_adapter, work);
3574 
3575 	/* if another thread is resetting the device, no need to proceed */
3576 	if (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
3577 		return;
3578 
3579 	/* if the device is closed, we must leave it alone */
3580 	rtnl_lock();
3581 	if (netif_running(adapter->netdev)) {
3582 		netdev_notice(adapter->netdev, "resetting\n");
3583 		vmxnet3_quiesce_dev(adapter);
3584 		vmxnet3_reset_dev(adapter);
3585 		vmxnet3_activate_dev(adapter);
3586 	} else {
3587 		netdev_info(adapter->netdev, "already closed\n");
3588 	}
3589 	rtnl_unlock();
3590 
3591 	netif_wake_queue(adapter->netdev);
3592 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
3593 }
3594 
3595 
3596 static int
3597 vmxnet3_probe_device(struct pci_dev *pdev,
3598 		     const struct pci_device_id *id)
3599 {
3600 	static const struct net_device_ops vmxnet3_netdev_ops = {
3601 		.ndo_open = vmxnet3_open,
3602 		.ndo_stop = vmxnet3_close,
3603 		.ndo_start_xmit = vmxnet3_xmit_frame,
3604 		.ndo_set_mac_address = vmxnet3_set_mac_addr,
3605 		.ndo_change_mtu = vmxnet3_change_mtu,
3606 		.ndo_fix_features = vmxnet3_fix_features,
3607 		.ndo_set_features = vmxnet3_set_features,
3608 		.ndo_features_check = vmxnet3_features_check,
3609 		.ndo_get_stats64 = vmxnet3_get_stats64,
3610 		.ndo_tx_timeout = vmxnet3_tx_timeout,
3611 		.ndo_set_rx_mode = vmxnet3_set_mc,
3612 		.ndo_vlan_rx_add_vid = vmxnet3_vlan_rx_add_vid,
3613 		.ndo_vlan_rx_kill_vid = vmxnet3_vlan_rx_kill_vid,
3614 #ifdef CONFIG_NET_POLL_CONTROLLER
3615 		.ndo_poll_controller = vmxnet3_netpoll,
3616 #endif
3617 	};
3618 	int err;
3619 	u32 ver;
3620 	struct net_device *netdev;
3621 	struct vmxnet3_adapter *adapter;
3622 	u8 mac[ETH_ALEN];
3623 	int size;
3624 	int num_tx_queues;
3625 	int num_rx_queues;
3626 	int queues;
3627 	unsigned long flags;
3628 
3629 	if (!pci_msi_enabled())
3630 		enable_mq = 0;
3631 
3632 #ifdef VMXNET3_RSS
3633 	if (enable_mq)
3634 		num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3635 				    (int)num_online_cpus());
3636 	else
3637 #endif
3638 		num_rx_queues = 1;
3639 
3640 	if (enable_mq)
3641 		num_tx_queues = min(VMXNET3_DEVICE_MAX_TX_QUEUES,
3642 				    (int)num_online_cpus());
3643 	else
3644 		num_tx_queues = 1;
3645 
3646 	netdev = alloc_etherdev_mq(sizeof(struct vmxnet3_adapter),
3647 				   max(num_tx_queues, num_rx_queues));
3648 	if (!netdev)
3649 		return -ENOMEM;
3650 
3651 	pci_set_drvdata(pdev, netdev);
3652 	adapter = netdev_priv(netdev);
3653 	adapter->netdev = netdev;
3654 	adapter->pdev = pdev;
3655 
3656 	adapter->tx_ring_size = VMXNET3_DEF_TX_RING_SIZE;
3657 	adapter->rx_ring_size = VMXNET3_DEF_RX_RING_SIZE;
3658 	adapter->rx_ring2_size = VMXNET3_DEF_RX_RING2_SIZE;
3659 
3660 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3661 	if (err) {
3662 		dev_err(&pdev->dev, "dma_set_mask failed\n");
3663 		goto err_set_mask;
3664 	}
3665 
3666 	spin_lock_init(&adapter->cmd_lock);
3667 	adapter->adapter_pa = dma_map_single(&adapter->pdev->dev, adapter,
3668 					     sizeof(struct vmxnet3_adapter),
3669 					     DMA_TO_DEVICE);
3670 	if (dma_mapping_error(&adapter->pdev->dev, adapter->adapter_pa)) {
3671 		dev_err(&pdev->dev, "Failed to map dma\n");
3672 		err = -EFAULT;
3673 		goto err_set_mask;
3674 	}
3675 	adapter->shared = dma_alloc_coherent(
3676 				&adapter->pdev->dev,
3677 				sizeof(struct Vmxnet3_DriverShared),
3678 				&adapter->shared_pa, GFP_KERNEL);
3679 	if (!adapter->shared) {
3680 		dev_err(&pdev->dev, "Failed to allocate memory\n");
3681 		err = -ENOMEM;
3682 		goto err_alloc_shared;
3683 	}
3684 
3685 	err = vmxnet3_alloc_pci_resources(adapter);
3686 	if (err < 0)
3687 		goto err_alloc_pci;
3688 
3689 	ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_VRRS);
3690 	if (ver & (1 << VMXNET3_REV_7)) {
3691 		VMXNET3_WRITE_BAR1_REG(adapter,
3692 				       VMXNET3_REG_VRRS,
3693 				       1 << VMXNET3_REV_7);
3694 		adapter->version = VMXNET3_REV_7 + 1;
3695 	} else if (ver & (1 << VMXNET3_REV_6)) {
3696 		VMXNET3_WRITE_BAR1_REG(adapter,
3697 				       VMXNET3_REG_VRRS,
3698 				       1 << VMXNET3_REV_6);
3699 		adapter->version = VMXNET3_REV_6 + 1;
3700 	} else if (ver & (1 << VMXNET3_REV_5)) {
3701 		VMXNET3_WRITE_BAR1_REG(adapter,
3702 				       VMXNET3_REG_VRRS,
3703 				       1 << VMXNET3_REV_5);
3704 		adapter->version = VMXNET3_REV_5 + 1;
3705 	} else if (ver & (1 << VMXNET3_REV_4)) {
3706 		VMXNET3_WRITE_BAR1_REG(adapter,
3707 				       VMXNET3_REG_VRRS,
3708 				       1 << VMXNET3_REV_4);
3709 		adapter->version = VMXNET3_REV_4 + 1;
3710 	} else if (ver & (1 << VMXNET3_REV_3)) {
3711 		VMXNET3_WRITE_BAR1_REG(adapter,
3712 				       VMXNET3_REG_VRRS,
3713 				       1 << VMXNET3_REV_3);
3714 		adapter->version = VMXNET3_REV_3 + 1;
3715 	} else if (ver & (1 << VMXNET3_REV_2)) {
3716 		VMXNET3_WRITE_BAR1_REG(adapter,
3717 				       VMXNET3_REG_VRRS,
3718 				       1 << VMXNET3_REV_2);
3719 		adapter->version = VMXNET3_REV_2 + 1;
3720 	} else if (ver & (1 << VMXNET3_REV_1)) {
3721 		VMXNET3_WRITE_BAR1_REG(adapter,
3722 				       VMXNET3_REG_VRRS,
3723 				       1 << VMXNET3_REV_1);
3724 		adapter->version = VMXNET3_REV_1 + 1;
3725 	} else {
3726 		dev_err(&pdev->dev,
3727 			"Incompatible h/w version (0x%x) for adapter\n", ver);
3728 		err = -EBUSY;
3729 		goto err_ver;
3730 	}
3731 	dev_dbg(&pdev->dev, "Using device version %d\n", adapter->version);
3732 
3733 	ver = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_UVRS);
3734 	if (ver & 1) {
3735 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_UVRS, 1);
3736 	} else {
3737 		dev_err(&pdev->dev,
3738 			"Incompatible upt version (0x%x) for adapter\n", ver);
3739 		err = -EBUSY;
3740 		goto err_ver;
3741 	}
3742 
3743 	if (VMXNET3_VERSION_GE_7(adapter)) {
3744 		adapter->devcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_DCR);
3745 		adapter->ptcap_supported[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_PTCR);
3746 		if (adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) {
3747 			adapter->dev_caps[0] = adapter->devcap_supported[0] &
3748 							(1UL << VMXNET3_CAP_LARGE_BAR);
3749 		}
3750 		if (!(adapter->ptcap_supported[0] & (1UL << VMXNET3_DCR_ERROR)) &&
3751 		    adapter->ptcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP) &&
3752 		    adapter->devcap_supported[0] & (1UL << VMXNET3_CAP_OOORX_COMP)) {
3753 			adapter->dev_caps[0] |= adapter->devcap_supported[0] &
3754 						(1UL << VMXNET3_CAP_OOORX_COMP);
3755 		}
3756 		if (adapter->dev_caps[0])
3757 			VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_DCR, adapter->dev_caps[0]);
3758 
3759 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3760 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD, VMXNET3_CMD_GET_DCR0_REG);
3761 		adapter->dev_caps[0] = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3762 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3763 	}
3764 
3765 	if (VMXNET3_VERSION_GE_7(adapter) &&
3766 	    adapter->dev_caps[0] & (1UL << VMXNET3_CAP_LARGE_BAR)) {
3767 		adapter->tx_prod_offset = VMXNET3_REG_LB_TXPROD;
3768 		adapter->rx_prod_offset = VMXNET3_REG_LB_RXPROD;
3769 		adapter->rx_prod2_offset = VMXNET3_REG_LB_RXPROD2;
3770 	} else {
3771 		adapter->tx_prod_offset = VMXNET3_REG_TXPROD;
3772 		adapter->rx_prod_offset = VMXNET3_REG_RXPROD;
3773 		adapter->rx_prod2_offset = VMXNET3_REG_RXPROD2;
3774 	}
3775 
3776 	if (VMXNET3_VERSION_GE_6(adapter)) {
3777 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3778 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3779 				       VMXNET3_CMD_GET_MAX_QUEUES_CONF);
3780 		queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3781 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3782 		if (queues > 0) {
3783 			adapter->num_rx_queues = min(num_rx_queues, ((queues >> 8) & 0xff));
3784 			adapter->num_tx_queues = min(num_tx_queues, (queues & 0xff));
3785 		} else {
3786 			adapter->num_rx_queues = min(num_rx_queues,
3787 						     VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3788 			adapter->num_tx_queues = min(num_tx_queues,
3789 						     VMXNET3_DEVICE_DEFAULT_TX_QUEUES);
3790 		}
3791 		if (adapter->num_rx_queues > VMXNET3_MAX_RX_QUEUES ||
3792 		    adapter->num_tx_queues > VMXNET3_MAX_TX_QUEUES) {
3793 			adapter->queuesExtEnabled = true;
3794 		} else {
3795 			adapter->queuesExtEnabled = false;
3796 		}
3797 	} else {
3798 		adapter->queuesExtEnabled = false;
3799 		num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3800 		num_tx_queues = rounddown_pow_of_two(num_tx_queues);
3801 		adapter->num_rx_queues = min(num_rx_queues,
3802 					     VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3803 		adapter->num_tx_queues = min(num_tx_queues,
3804 					     VMXNET3_DEVICE_DEFAULT_TX_QUEUES);
3805 	}
3806 	dev_info(&pdev->dev,
3807 		 "# of Tx queues : %d, # of Rx queues : %d\n",
3808 		 adapter->num_tx_queues, adapter->num_rx_queues);
3809 
3810 	adapter->rx_buf_per_pkt = 1;
3811 
3812 	size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
3813 	size += sizeof(struct Vmxnet3_RxQueueDesc) * adapter->num_rx_queues;
3814 	adapter->tqd_start = dma_alloc_coherent(&adapter->pdev->dev, size,
3815 						&adapter->queue_desc_pa,
3816 						GFP_KERNEL);
3817 
3818 	if (!adapter->tqd_start) {
3819 		dev_err(&pdev->dev, "Failed to allocate memory\n");
3820 		err = -ENOMEM;
3821 		goto err_ver;
3822 	}
3823 	adapter->rqd_start = (struct Vmxnet3_RxQueueDesc *)(adapter->tqd_start +
3824 							    adapter->num_tx_queues);
3825 
3826 	adapter->pm_conf = dma_alloc_coherent(&adapter->pdev->dev,
3827 					      sizeof(struct Vmxnet3_PMConf),
3828 					      &adapter->pm_conf_pa,
3829 					      GFP_KERNEL);
3830 	if (adapter->pm_conf == NULL) {
3831 		err = -ENOMEM;
3832 		goto err_alloc_pm;
3833 	}
3834 
3835 #ifdef VMXNET3_RSS
3836 
3837 	adapter->rss_conf = dma_alloc_coherent(&adapter->pdev->dev,
3838 					       sizeof(struct UPT1_RSSConf),
3839 					       &adapter->rss_conf_pa,
3840 					       GFP_KERNEL);
3841 	if (adapter->rss_conf == NULL) {
3842 		err = -ENOMEM;
3843 		goto err_alloc_rss;
3844 	}
3845 #endif /* VMXNET3_RSS */
3846 
3847 	if (VMXNET3_VERSION_GE_3(adapter)) {
3848 		adapter->coal_conf =
3849 			dma_alloc_coherent(&adapter->pdev->dev,
3850 					   sizeof(struct Vmxnet3_CoalesceScheme)
3851 					   ,
3852 					   &adapter->coal_conf_pa,
3853 					   GFP_KERNEL);
3854 		if (!adapter->coal_conf) {
3855 			err = -ENOMEM;
3856 			goto err_coal_conf;
3857 		}
3858 		adapter->coal_conf->coalMode = VMXNET3_COALESCE_DISABLED;
3859 		adapter->default_coal_mode = true;
3860 	}
3861 
3862 	if (VMXNET3_VERSION_GE_4(adapter)) {
3863 		adapter->default_rss_fields = true;
3864 		adapter->rss_fields = VMXNET3_RSS_FIELDS_DEFAULT;
3865 	}
3866 
3867 	SET_NETDEV_DEV(netdev, &pdev->dev);
3868 	vmxnet3_declare_features(adapter);
3869 
3870 	adapter->rxdata_desc_size = VMXNET3_VERSION_GE_3(adapter) ?
3871 		VMXNET3_DEF_RXDATA_DESC_SIZE : 0;
3872 
3873 	if (adapter->num_tx_queues == adapter->num_rx_queues)
3874 		adapter->share_intr = VMXNET3_INTR_BUDDYSHARE;
3875 	else
3876 		adapter->share_intr = VMXNET3_INTR_DONTSHARE;
3877 
3878 	vmxnet3_alloc_intr_resources(adapter);
3879 
3880 #ifdef VMXNET3_RSS
3881 	if (adapter->num_rx_queues > 1 &&
3882 	    adapter->intr.type == VMXNET3_IT_MSIX) {
3883 		adapter->rss = true;
3884 		netdev->hw_features |= NETIF_F_RXHASH;
3885 		netdev->features |= NETIF_F_RXHASH;
3886 		dev_dbg(&pdev->dev, "RSS is enabled.\n");
3887 	} else {
3888 		adapter->rss = false;
3889 	}
3890 #endif
3891 
3892 	vmxnet3_read_mac_addr(adapter, mac);
3893 	dev_addr_set(netdev, mac);
3894 
3895 	netdev->netdev_ops = &vmxnet3_netdev_ops;
3896 	vmxnet3_set_ethtool_ops(netdev);
3897 	netdev->watchdog_timeo = 5 * HZ;
3898 
3899 	/* MTU range: 60 - 9190 */
3900 	netdev->min_mtu = VMXNET3_MIN_MTU;
3901 	if (VMXNET3_VERSION_GE_6(adapter))
3902 		netdev->max_mtu = VMXNET3_V6_MAX_MTU;
3903 	else
3904 		netdev->max_mtu = VMXNET3_MAX_MTU;
3905 
3906 	INIT_WORK(&adapter->work, vmxnet3_reset_work);
3907 	set_bit(VMXNET3_STATE_BIT_QUIESCED, &adapter->state);
3908 
3909 	if (adapter->intr.type == VMXNET3_IT_MSIX) {
3910 		int i;
3911 		for (i = 0; i < adapter->num_rx_queues; i++) {
3912 			netif_napi_add(adapter->netdev,
3913 				       &adapter->rx_queue[i].napi,
3914 				       vmxnet3_poll_rx_only);
3915 		}
3916 	} else {
3917 		netif_napi_add(adapter->netdev, &adapter->rx_queue[0].napi,
3918 			       vmxnet3_poll);
3919 	}
3920 
3921 	netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
3922 	netif_set_real_num_rx_queues(adapter->netdev, adapter->num_rx_queues);
3923 
3924 	netif_carrier_off(netdev);
3925 	err = register_netdev(netdev);
3926 
3927 	if (err) {
3928 		dev_err(&pdev->dev, "Failed to register adapter\n");
3929 		goto err_register;
3930 	}
3931 
3932 	vmxnet3_check_link(adapter, false);
3933 	return 0;
3934 
3935 err_register:
3936 	if (VMXNET3_VERSION_GE_3(adapter)) {
3937 		dma_free_coherent(&adapter->pdev->dev,
3938 				  sizeof(struct Vmxnet3_CoalesceScheme),
3939 				  adapter->coal_conf, adapter->coal_conf_pa);
3940 	}
3941 	vmxnet3_free_intr_resources(adapter);
3942 err_coal_conf:
3943 #ifdef VMXNET3_RSS
3944 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
3945 			  adapter->rss_conf, adapter->rss_conf_pa);
3946 err_alloc_rss:
3947 #endif
3948 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
3949 			  adapter->pm_conf, adapter->pm_conf_pa);
3950 err_alloc_pm:
3951 	dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
3952 			  adapter->queue_desc_pa);
3953 err_ver:
3954 	vmxnet3_free_pci_resources(adapter);
3955 err_alloc_pci:
3956 	dma_free_coherent(&adapter->pdev->dev,
3957 			  sizeof(struct Vmxnet3_DriverShared),
3958 			  adapter->shared, adapter->shared_pa);
3959 err_alloc_shared:
3960 	dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
3961 			 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE);
3962 err_set_mask:
3963 	free_netdev(netdev);
3964 	return err;
3965 }
3966 
3967 
3968 static void
3969 vmxnet3_remove_device(struct pci_dev *pdev)
3970 {
3971 	struct net_device *netdev = pci_get_drvdata(pdev);
3972 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
3973 	int size = 0;
3974 	int num_rx_queues, rx_queues;
3975 	unsigned long flags;
3976 
3977 #ifdef VMXNET3_RSS
3978 	if (enable_mq)
3979 		num_rx_queues = min(VMXNET3_DEVICE_MAX_RX_QUEUES,
3980 				    (int)num_online_cpus());
3981 	else
3982 #endif
3983 		num_rx_queues = 1;
3984 	if (!VMXNET3_VERSION_GE_6(adapter)) {
3985 		num_rx_queues = rounddown_pow_of_two(num_rx_queues);
3986 	}
3987 	if (VMXNET3_VERSION_GE_6(adapter)) {
3988 		spin_lock_irqsave(&adapter->cmd_lock, flags);
3989 		VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
3990 				       VMXNET3_CMD_GET_MAX_QUEUES_CONF);
3991 		rx_queues = VMXNET3_READ_BAR1_REG(adapter, VMXNET3_REG_CMD);
3992 		spin_unlock_irqrestore(&adapter->cmd_lock, flags);
3993 		if (rx_queues > 0)
3994 			rx_queues = (rx_queues >> 8) & 0xff;
3995 		else
3996 			rx_queues = min(num_rx_queues, VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
3997 		num_rx_queues = min(num_rx_queues, rx_queues);
3998 	} else {
3999 		num_rx_queues = min(num_rx_queues,
4000 				    VMXNET3_DEVICE_DEFAULT_RX_QUEUES);
4001 	}
4002 
4003 	cancel_work_sync(&adapter->work);
4004 
4005 	unregister_netdev(netdev);
4006 
4007 	vmxnet3_free_intr_resources(adapter);
4008 	vmxnet3_free_pci_resources(adapter);
4009 	if (VMXNET3_VERSION_GE_3(adapter)) {
4010 		dma_free_coherent(&adapter->pdev->dev,
4011 				  sizeof(struct Vmxnet3_CoalesceScheme),
4012 				  adapter->coal_conf, adapter->coal_conf_pa);
4013 	}
4014 #ifdef VMXNET3_RSS
4015 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct UPT1_RSSConf),
4016 			  adapter->rss_conf, adapter->rss_conf_pa);
4017 #endif
4018 	dma_free_coherent(&adapter->pdev->dev, sizeof(struct Vmxnet3_PMConf),
4019 			  adapter->pm_conf, adapter->pm_conf_pa);
4020 
4021 	size = sizeof(struct Vmxnet3_TxQueueDesc) * adapter->num_tx_queues;
4022 	size += sizeof(struct Vmxnet3_RxQueueDesc) * num_rx_queues;
4023 	dma_free_coherent(&adapter->pdev->dev, size, adapter->tqd_start,
4024 			  adapter->queue_desc_pa);
4025 	dma_free_coherent(&adapter->pdev->dev,
4026 			  sizeof(struct Vmxnet3_DriverShared),
4027 			  adapter->shared, adapter->shared_pa);
4028 	dma_unmap_single(&adapter->pdev->dev, adapter->adapter_pa,
4029 			 sizeof(struct vmxnet3_adapter), DMA_TO_DEVICE);
4030 	free_netdev(netdev);
4031 }
4032 
4033 static void vmxnet3_shutdown_device(struct pci_dev *pdev)
4034 {
4035 	struct net_device *netdev = pci_get_drvdata(pdev);
4036 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4037 	unsigned long flags;
4038 
4039 	/* Reset_work may be in the middle of resetting the device, wait for its
4040 	 * completion.
4041 	 */
4042 	while (test_and_set_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state))
4043 		usleep_range(1000, 2000);
4044 
4045 	if (test_and_set_bit(VMXNET3_STATE_BIT_QUIESCED,
4046 			     &adapter->state)) {
4047 		clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
4048 		return;
4049 	}
4050 	spin_lock_irqsave(&adapter->cmd_lock, flags);
4051 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4052 			       VMXNET3_CMD_QUIESCE_DEV);
4053 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4054 	vmxnet3_disable_all_intrs(adapter);
4055 
4056 	clear_bit(VMXNET3_STATE_BIT_RESETTING, &adapter->state);
4057 }
4058 
4059 
4060 #ifdef CONFIG_PM
4061 
4062 static int
4063 vmxnet3_suspend(struct device *device)
4064 {
4065 	struct pci_dev *pdev = to_pci_dev(device);
4066 	struct net_device *netdev = pci_get_drvdata(pdev);
4067 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4068 	struct Vmxnet3_PMConf *pmConf;
4069 	struct ethhdr *ehdr;
4070 	struct arphdr *ahdr;
4071 	u8 *arpreq;
4072 	struct in_device *in_dev;
4073 	struct in_ifaddr *ifa;
4074 	unsigned long flags;
4075 	int i = 0;
4076 
4077 	if (!netif_running(netdev))
4078 		return 0;
4079 
4080 	for (i = 0; i < adapter->num_rx_queues; i++)
4081 		napi_disable(&adapter->rx_queue[i].napi);
4082 
4083 	vmxnet3_disable_all_intrs(adapter);
4084 	vmxnet3_free_irqs(adapter);
4085 	vmxnet3_free_intr_resources(adapter);
4086 
4087 	netif_device_detach(netdev);
4088 
4089 	/* Create wake-up filters. */
4090 	pmConf = adapter->pm_conf;
4091 	memset(pmConf, 0, sizeof(*pmConf));
4092 
4093 	if (adapter->wol & WAKE_UCAST) {
4094 		pmConf->filters[i].patternSize = ETH_ALEN;
4095 		pmConf->filters[i].maskSize = 1;
4096 		memcpy(pmConf->filters[i].pattern, netdev->dev_addr, ETH_ALEN);
4097 		pmConf->filters[i].mask[0] = 0x3F; /* LSB ETH_ALEN bits */
4098 
4099 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
4100 		i++;
4101 	}
4102 
4103 	if (adapter->wol & WAKE_ARP) {
4104 		rcu_read_lock();
4105 
4106 		in_dev = __in_dev_get_rcu(netdev);
4107 		if (!in_dev) {
4108 			rcu_read_unlock();
4109 			goto skip_arp;
4110 		}
4111 
4112 		ifa = rcu_dereference(in_dev->ifa_list);
4113 		if (!ifa) {
4114 			rcu_read_unlock();
4115 			goto skip_arp;
4116 		}
4117 
4118 		pmConf->filters[i].patternSize = ETH_HLEN + /* Ethernet header*/
4119 			sizeof(struct arphdr) +		/* ARP header */
4120 			2 * ETH_ALEN +		/* 2 Ethernet addresses*/
4121 			2 * sizeof(u32);	/*2 IPv4 addresses */
4122 		pmConf->filters[i].maskSize =
4123 			(pmConf->filters[i].patternSize - 1) / 8 + 1;
4124 
4125 		/* ETH_P_ARP in Ethernet header. */
4126 		ehdr = (struct ethhdr *)pmConf->filters[i].pattern;
4127 		ehdr->h_proto = htons(ETH_P_ARP);
4128 
4129 		/* ARPOP_REQUEST in ARP header. */
4130 		ahdr = (struct arphdr *)&pmConf->filters[i].pattern[ETH_HLEN];
4131 		ahdr->ar_op = htons(ARPOP_REQUEST);
4132 		arpreq = (u8 *)(ahdr + 1);
4133 
4134 		/* The Unicast IPv4 address in 'tip' field. */
4135 		arpreq += 2 * ETH_ALEN + sizeof(u32);
4136 		*(__be32 *)arpreq = ifa->ifa_address;
4137 
4138 		rcu_read_unlock();
4139 
4140 		/* The mask for the relevant bits. */
4141 		pmConf->filters[i].mask[0] = 0x00;
4142 		pmConf->filters[i].mask[1] = 0x30; /* ETH_P_ARP */
4143 		pmConf->filters[i].mask[2] = 0x30; /* ARPOP_REQUEST */
4144 		pmConf->filters[i].mask[3] = 0x00;
4145 		pmConf->filters[i].mask[4] = 0xC0; /* IPv4 TIP */
4146 		pmConf->filters[i].mask[5] = 0x03; /* IPv4 TIP */
4147 
4148 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_FILTER;
4149 		i++;
4150 	}
4151 
4152 skip_arp:
4153 	if (adapter->wol & WAKE_MAGIC)
4154 		pmConf->wakeUpEvents |= VMXNET3_PM_WAKEUP_MAGIC;
4155 
4156 	pmConf->numFilters = i;
4157 
4158 	adapter->shared->devRead.pmConfDesc.confVer = cpu_to_le32(1);
4159 	adapter->shared->devRead.pmConfDesc.confLen = cpu_to_le32(sizeof(
4160 								  *pmConf));
4161 	adapter->shared->devRead.pmConfDesc.confPA =
4162 		cpu_to_le64(adapter->pm_conf_pa);
4163 
4164 	spin_lock_irqsave(&adapter->cmd_lock, flags);
4165 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4166 			       VMXNET3_CMD_UPDATE_PMCFG);
4167 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4168 
4169 	pci_save_state(pdev);
4170 	pci_enable_wake(pdev, pci_choose_state(pdev, PMSG_SUSPEND),
4171 			adapter->wol);
4172 	pci_disable_device(pdev);
4173 	pci_set_power_state(pdev, pci_choose_state(pdev, PMSG_SUSPEND));
4174 
4175 	return 0;
4176 }
4177 
4178 
4179 static int
4180 vmxnet3_resume(struct device *device)
4181 {
4182 	int err;
4183 	unsigned long flags;
4184 	struct pci_dev *pdev = to_pci_dev(device);
4185 	struct net_device *netdev = pci_get_drvdata(pdev);
4186 	struct vmxnet3_adapter *adapter = netdev_priv(netdev);
4187 
4188 	if (!netif_running(netdev))
4189 		return 0;
4190 
4191 	pci_set_power_state(pdev, PCI_D0);
4192 	pci_restore_state(pdev);
4193 	err = pci_enable_device_mem(pdev);
4194 	if (err != 0)
4195 		return err;
4196 
4197 	pci_enable_wake(pdev, PCI_D0, 0);
4198 
4199 	vmxnet3_alloc_intr_resources(adapter);
4200 
4201 	/* During hibernate and suspend, device has to be reinitialized as the
4202 	 * device state need not be preserved.
4203 	 */
4204 
4205 	/* Need not check adapter state as other reset tasks cannot run during
4206 	 * device resume.
4207 	 */
4208 	spin_lock_irqsave(&adapter->cmd_lock, flags);
4209 	VMXNET3_WRITE_BAR1_REG(adapter, VMXNET3_REG_CMD,
4210 			       VMXNET3_CMD_QUIESCE_DEV);
4211 	spin_unlock_irqrestore(&adapter->cmd_lock, flags);
4212 	vmxnet3_tq_cleanup_all(adapter);
4213 	vmxnet3_rq_cleanup_all(adapter);
4214 
4215 	vmxnet3_reset_dev(adapter);
4216 	err = vmxnet3_activate_dev(adapter);
4217 	if (err != 0) {
4218 		netdev_err(netdev,
4219 			   "failed to re-activate on resume, error: %d", err);
4220 		vmxnet3_force_close(adapter);
4221 		return err;
4222 	}
4223 	netif_device_attach(netdev);
4224 
4225 	return 0;
4226 }
4227 
4228 static const struct dev_pm_ops vmxnet3_pm_ops = {
4229 	.suspend = vmxnet3_suspend,
4230 	.resume = vmxnet3_resume,
4231 	.freeze = vmxnet3_suspend,
4232 	.restore = vmxnet3_resume,
4233 };
4234 #endif
4235 
4236 static struct pci_driver vmxnet3_driver = {
4237 	.name		= vmxnet3_driver_name,
4238 	.id_table	= vmxnet3_pciid_table,
4239 	.probe		= vmxnet3_probe_device,
4240 	.remove		= vmxnet3_remove_device,
4241 	.shutdown	= vmxnet3_shutdown_device,
4242 #ifdef CONFIG_PM
4243 	.driver.pm	= &vmxnet3_pm_ops,
4244 #endif
4245 };
4246 
4247 
4248 static int __init
4249 vmxnet3_init_module(void)
4250 {
4251 	pr_info("%s - version %s\n", VMXNET3_DRIVER_DESC,
4252 		VMXNET3_DRIVER_VERSION_REPORT);
4253 	return pci_register_driver(&vmxnet3_driver);
4254 }
4255 
4256 module_init(vmxnet3_init_module);
4257 
4258 
4259 static void
4260 vmxnet3_exit_module(void)
4261 {
4262 	pci_unregister_driver(&vmxnet3_driver);
4263 }
4264 
4265 module_exit(vmxnet3_exit_module);
4266 
4267 MODULE_AUTHOR("VMware, Inc.");
4268 MODULE_DESCRIPTION(VMXNET3_DRIVER_DESC);
4269 MODULE_LICENSE("GPL v2");
4270 MODULE_VERSION(VMXNET3_DRIVER_VERSION_STRING);
4271