1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2009 - 2018 Intel Corporation. */
3 
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5 
6 #include <linux/module.h>
7 #include <linux/types.h>
8 #include <linux/init.h>
9 #include <linux/pci.h>
10 #include <linux/vmalloc.h>
11 #include <linux/pagemap.h>
12 #include <linux/delay.h>
13 #include <linux/netdevice.h>
14 #include <linux/tcp.h>
15 #include <linux/ipv6.h>
16 #include <linux/slab.h>
17 #include <net/checksum.h>
18 #include <net/ip6_checksum.h>
19 #include <linux/mii.h>
20 #include <linux/ethtool.h>
21 #include <linux/if_vlan.h>
22 #include <linux/prefetch.h>
23 #include <linux/sctp.h>
24 
25 #include "igbvf.h"
26 
27 char igbvf_driver_name[] = "igbvf";
28 static const char igbvf_driver_string[] =
29 		  "Intel(R) Gigabit Virtual Function Network Driver";
30 static const char igbvf_copyright[] =
31 		  "Copyright (c) 2009 - 2012 Intel Corporation.";
32 
33 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
34 static int debug = -1;
35 module_param(debug, int, 0);
36 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
37 
38 static int igbvf_poll(struct napi_struct *napi, int budget);
39 static void igbvf_reset(struct igbvf_adapter *);
40 static void igbvf_set_interrupt_capability(struct igbvf_adapter *);
41 static void igbvf_reset_interrupt_capability(struct igbvf_adapter *);
42 
43 static struct igbvf_info igbvf_vf_info = {
44 	.mac		= e1000_vfadapt,
45 	.flags		= 0,
46 	.pba		= 10,
47 	.init_ops	= e1000_init_function_pointers_vf,
48 };
49 
50 static struct igbvf_info igbvf_i350_vf_info = {
51 	.mac		= e1000_vfadapt_i350,
52 	.flags		= 0,
53 	.pba		= 10,
54 	.init_ops	= e1000_init_function_pointers_vf,
55 };
56 
57 static const struct igbvf_info *igbvf_info_tbl[] = {
58 	[board_vf]	= &igbvf_vf_info,
59 	[board_i350_vf]	= &igbvf_i350_vf_info,
60 };
61 
62 /**
63  * igbvf_desc_unused - calculate if we have unused descriptors
64  * @ring: address of receive ring structure
65  **/
66 static int igbvf_desc_unused(struct igbvf_ring *ring)
67 {
68 	if (ring->next_to_clean > ring->next_to_use)
69 		return ring->next_to_clean - ring->next_to_use - 1;
70 
71 	return ring->count + ring->next_to_clean - ring->next_to_use - 1;
72 }
73 
74 /**
75  * igbvf_receive_skb - helper function to handle Rx indications
76  * @adapter: board private structure
77  * @netdev: pointer to netdev struct
78  * @skb: skb to indicate to stack
79  * @status: descriptor status field as written by hardware
80  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
81  * @skb: pointer to sk_buff to be indicated to stack
82  **/
83 static void igbvf_receive_skb(struct igbvf_adapter *adapter,
84 			      struct net_device *netdev,
85 			      struct sk_buff *skb,
86 			      u32 status, __le16 vlan)
87 {
88 	u16 vid;
89 
90 	if (status & E1000_RXD_STAT_VP) {
91 		if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) &&
92 		    (status & E1000_RXDEXT_STATERR_LB))
93 			vid = be16_to_cpu((__force __be16)vlan) & E1000_RXD_SPC_VLAN_MASK;
94 		else
95 			vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
96 		if (test_bit(vid, adapter->active_vlans))
97 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
98 	}
99 
100 	napi_gro_receive(&adapter->rx_ring->napi, skb);
101 }
102 
103 static inline void igbvf_rx_checksum_adv(struct igbvf_adapter *adapter,
104 					 u32 status_err, struct sk_buff *skb)
105 {
106 	skb_checksum_none_assert(skb);
107 
108 	/* Ignore Checksum bit is set or checksum is disabled through ethtool */
109 	if ((status_err & E1000_RXD_STAT_IXSM) ||
110 	    (adapter->flags & IGBVF_FLAG_RX_CSUM_DISABLED))
111 		return;
112 
113 	/* TCP/UDP checksum error bit is set */
114 	if (status_err &
115 	    (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
116 		/* let the stack verify checksum errors */
117 		adapter->hw_csum_err++;
118 		return;
119 	}
120 
121 	/* It must be a TCP or UDP packet with a valid checksum */
122 	if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
123 		skb->ip_summed = CHECKSUM_UNNECESSARY;
124 
125 	adapter->hw_csum_good++;
126 }
127 
128 /**
129  * igbvf_alloc_rx_buffers - Replace used receive buffers; packet split
130  * @rx_ring: address of ring structure to repopulate
131  * @cleaned_count: number of buffers to repopulate
132  **/
133 static void igbvf_alloc_rx_buffers(struct igbvf_ring *rx_ring,
134 				   int cleaned_count)
135 {
136 	struct igbvf_adapter *adapter = rx_ring->adapter;
137 	struct net_device *netdev = adapter->netdev;
138 	struct pci_dev *pdev = adapter->pdev;
139 	union e1000_adv_rx_desc *rx_desc;
140 	struct igbvf_buffer *buffer_info;
141 	struct sk_buff *skb;
142 	unsigned int i;
143 	int bufsz;
144 
145 	i = rx_ring->next_to_use;
146 	buffer_info = &rx_ring->buffer_info[i];
147 
148 	if (adapter->rx_ps_hdr_size)
149 		bufsz = adapter->rx_ps_hdr_size;
150 	else
151 		bufsz = adapter->rx_buffer_len;
152 
153 	while (cleaned_count--) {
154 		rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
155 
156 		if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {
157 			if (!buffer_info->page) {
158 				buffer_info->page = alloc_page(GFP_ATOMIC);
159 				if (!buffer_info->page) {
160 					adapter->alloc_rx_buff_failed++;
161 					goto no_buffers;
162 				}
163 				buffer_info->page_offset = 0;
164 			} else {
165 				buffer_info->page_offset ^= PAGE_SIZE / 2;
166 			}
167 			buffer_info->page_dma =
168 				dma_map_page(&pdev->dev, buffer_info->page,
169 					     buffer_info->page_offset,
170 					     PAGE_SIZE / 2,
171 					     DMA_FROM_DEVICE);
172 			if (dma_mapping_error(&pdev->dev,
173 					      buffer_info->page_dma)) {
174 				__free_page(buffer_info->page);
175 				buffer_info->page = NULL;
176 				dev_err(&pdev->dev, "RX DMA map failed\n");
177 				break;
178 			}
179 		}
180 
181 		if (!buffer_info->skb) {
182 			skb = netdev_alloc_skb_ip_align(netdev, bufsz);
183 			if (!skb) {
184 				adapter->alloc_rx_buff_failed++;
185 				goto no_buffers;
186 			}
187 
188 			buffer_info->skb = skb;
189 			buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
190 							  bufsz,
191 							  DMA_FROM_DEVICE);
192 			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
193 				dev_kfree_skb(buffer_info->skb);
194 				buffer_info->skb = NULL;
195 				dev_err(&pdev->dev, "RX DMA map failed\n");
196 				goto no_buffers;
197 			}
198 		}
199 		/* Refresh the desc even if buffer_addrs didn't change because
200 		 * each write-back erases this info.
201 		 */
202 		if (adapter->rx_ps_hdr_size) {
203 			rx_desc->read.pkt_addr =
204 			     cpu_to_le64(buffer_info->page_dma);
205 			rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
206 		} else {
207 			rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
208 			rx_desc->read.hdr_addr = 0;
209 		}
210 
211 		i++;
212 		if (i == rx_ring->count)
213 			i = 0;
214 		buffer_info = &rx_ring->buffer_info[i];
215 	}
216 
217 no_buffers:
218 	if (rx_ring->next_to_use != i) {
219 		rx_ring->next_to_use = i;
220 		if (i == 0)
221 			i = (rx_ring->count - 1);
222 		else
223 			i--;
224 
225 		/* Force memory writes to complete before letting h/w
226 		 * know there are new descriptors to fetch.  (Only
227 		 * applicable for weak-ordered memory model archs,
228 		 * such as IA-64).
229 		*/
230 		wmb();
231 		writel(i, adapter->hw.hw_addr + rx_ring->tail);
232 	}
233 }
234 
235 /**
236  * igbvf_clean_rx_irq - Send received data up the network stack; legacy
237  * @adapter: board private structure
238  * @work_done: output parameter used to indicate completed work
239  * @work_to_do: input parameter setting limit of work
240  *
241  * the return value indicates whether actual cleaning was done, there
242  * is no guarantee that everything was cleaned
243  **/
244 static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,
245 			       int *work_done, int work_to_do)
246 {
247 	struct igbvf_ring *rx_ring = adapter->rx_ring;
248 	struct net_device *netdev = adapter->netdev;
249 	struct pci_dev *pdev = adapter->pdev;
250 	union e1000_adv_rx_desc *rx_desc, *next_rxd;
251 	struct igbvf_buffer *buffer_info, *next_buffer;
252 	struct sk_buff *skb;
253 	bool cleaned = false;
254 	int cleaned_count = 0;
255 	unsigned int total_bytes = 0, total_packets = 0;
256 	unsigned int i;
257 	u32 length, hlen, staterr;
258 
259 	i = rx_ring->next_to_clean;
260 	rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
261 	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
262 
263 	while (staterr & E1000_RXD_STAT_DD) {
264 		if (*work_done >= work_to_do)
265 			break;
266 		(*work_done)++;
267 		rmb(); /* read descriptor and rx_buffer_info after status DD */
268 
269 		buffer_info = &rx_ring->buffer_info[i];
270 
271 		/* HW will not DMA in data larger than the given buffer, even
272 		 * if it parses the (NFS, of course) header to be larger.  In
273 		 * that case, it fills the header buffer and spills the rest
274 		 * into the page.
275 		 */
276 		hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info)
277 		       & E1000_RXDADV_HDRBUFLEN_MASK) >>
278 		       E1000_RXDADV_HDRBUFLEN_SHIFT;
279 		if (hlen > adapter->rx_ps_hdr_size)
280 			hlen = adapter->rx_ps_hdr_size;
281 
282 		length = le16_to_cpu(rx_desc->wb.upper.length);
283 		cleaned = true;
284 		cleaned_count++;
285 
286 		skb = buffer_info->skb;
287 		prefetch(skb->data - NET_IP_ALIGN);
288 		buffer_info->skb = NULL;
289 		if (!adapter->rx_ps_hdr_size) {
290 			dma_unmap_single(&pdev->dev, buffer_info->dma,
291 					 adapter->rx_buffer_len,
292 					 DMA_FROM_DEVICE);
293 			buffer_info->dma = 0;
294 			skb_put(skb, length);
295 			goto send_up;
296 		}
297 
298 		if (!skb_shinfo(skb)->nr_frags) {
299 			dma_unmap_single(&pdev->dev, buffer_info->dma,
300 					 adapter->rx_ps_hdr_size,
301 					 DMA_FROM_DEVICE);
302 			buffer_info->dma = 0;
303 			skb_put(skb, hlen);
304 		}
305 
306 		if (length) {
307 			dma_unmap_page(&pdev->dev, buffer_info->page_dma,
308 				       PAGE_SIZE / 2,
309 				       DMA_FROM_DEVICE);
310 			buffer_info->page_dma = 0;
311 
312 			skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
313 					   buffer_info->page,
314 					   buffer_info->page_offset,
315 					   length);
316 
317 			if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||
318 			    (page_count(buffer_info->page) != 1))
319 				buffer_info->page = NULL;
320 			else
321 				get_page(buffer_info->page);
322 
323 			skb->len += length;
324 			skb->data_len += length;
325 			skb->truesize += PAGE_SIZE / 2;
326 		}
327 send_up:
328 		i++;
329 		if (i == rx_ring->count)
330 			i = 0;
331 		next_rxd = IGBVF_RX_DESC_ADV(*rx_ring, i);
332 		prefetch(next_rxd);
333 		next_buffer = &rx_ring->buffer_info[i];
334 
335 		if (!(staterr & E1000_RXD_STAT_EOP)) {
336 			buffer_info->skb = next_buffer->skb;
337 			buffer_info->dma = next_buffer->dma;
338 			next_buffer->skb = skb;
339 			next_buffer->dma = 0;
340 			goto next_desc;
341 		}
342 
343 		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
344 			dev_kfree_skb_irq(skb);
345 			goto next_desc;
346 		}
347 
348 		total_bytes += skb->len;
349 		total_packets++;
350 
351 		igbvf_rx_checksum_adv(adapter, staterr, skb);
352 
353 		skb->protocol = eth_type_trans(skb, netdev);
354 
355 		igbvf_receive_skb(adapter, netdev, skb, staterr,
356 				  rx_desc->wb.upper.vlan);
357 
358 next_desc:
359 		rx_desc->wb.upper.status_error = 0;
360 
361 		/* return some buffers to hardware, one at a time is too slow */
362 		if (cleaned_count >= IGBVF_RX_BUFFER_WRITE) {
363 			igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
364 			cleaned_count = 0;
365 		}
366 
367 		/* use prefetched values */
368 		rx_desc = next_rxd;
369 		buffer_info = next_buffer;
370 
371 		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
372 	}
373 
374 	rx_ring->next_to_clean = i;
375 	cleaned_count = igbvf_desc_unused(rx_ring);
376 
377 	if (cleaned_count)
378 		igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
379 
380 	adapter->total_rx_packets += total_packets;
381 	adapter->total_rx_bytes += total_bytes;
382 	netdev->stats.rx_bytes += total_bytes;
383 	netdev->stats.rx_packets += total_packets;
384 	return cleaned;
385 }
386 
387 static void igbvf_put_txbuf(struct igbvf_adapter *adapter,
388 			    struct igbvf_buffer *buffer_info)
389 {
390 	if (buffer_info->dma) {
391 		if (buffer_info->mapped_as_page)
392 			dma_unmap_page(&adapter->pdev->dev,
393 				       buffer_info->dma,
394 				       buffer_info->length,
395 				       DMA_TO_DEVICE);
396 		else
397 			dma_unmap_single(&adapter->pdev->dev,
398 					 buffer_info->dma,
399 					 buffer_info->length,
400 					 DMA_TO_DEVICE);
401 		buffer_info->dma = 0;
402 	}
403 	if (buffer_info->skb) {
404 		dev_kfree_skb_any(buffer_info->skb);
405 		buffer_info->skb = NULL;
406 	}
407 	buffer_info->time_stamp = 0;
408 }
409 
410 /**
411  * igbvf_setup_tx_resources - allocate Tx resources (Descriptors)
412  * @adapter: board private structure
413  * @tx_ring: ring being initialized
414  *
415  * Return 0 on success, negative on failure
416  **/
417 int igbvf_setup_tx_resources(struct igbvf_adapter *adapter,
418 			     struct igbvf_ring *tx_ring)
419 {
420 	struct pci_dev *pdev = adapter->pdev;
421 	int size;
422 
423 	size = sizeof(struct igbvf_buffer) * tx_ring->count;
424 	tx_ring->buffer_info = vzalloc(size);
425 	if (!tx_ring->buffer_info)
426 		goto err;
427 
428 	/* round up to nearest 4K */
429 	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
430 	tx_ring->size = ALIGN(tx_ring->size, 4096);
431 
432 	tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
433 					   &tx_ring->dma, GFP_KERNEL);
434 	if (!tx_ring->desc)
435 		goto err;
436 
437 	tx_ring->adapter = adapter;
438 	tx_ring->next_to_use = 0;
439 	tx_ring->next_to_clean = 0;
440 
441 	return 0;
442 err:
443 	vfree(tx_ring->buffer_info);
444 	dev_err(&adapter->pdev->dev,
445 		"Unable to allocate memory for the transmit descriptor ring\n");
446 	return -ENOMEM;
447 }
448 
449 /**
450  * igbvf_setup_rx_resources - allocate Rx resources (Descriptors)
451  * @adapter: board private structure
452  * @rx_ring: ring being initialized
453  *
454  * Returns 0 on success, negative on failure
455  **/
456 int igbvf_setup_rx_resources(struct igbvf_adapter *adapter,
457 			     struct igbvf_ring *rx_ring)
458 {
459 	struct pci_dev *pdev = adapter->pdev;
460 	int size, desc_len;
461 
462 	size = sizeof(struct igbvf_buffer) * rx_ring->count;
463 	rx_ring->buffer_info = vzalloc(size);
464 	if (!rx_ring->buffer_info)
465 		goto err;
466 
467 	desc_len = sizeof(union e1000_adv_rx_desc);
468 
469 	/* Round up to nearest 4K */
470 	rx_ring->size = rx_ring->count * desc_len;
471 	rx_ring->size = ALIGN(rx_ring->size, 4096);
472 
473 	rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
474 					   &rx_ring->dma, GFP_KERNEL);
475 	if (!rx_ring->desc)
476 		goto err;
477 
478 	rx_ring->next_to_clean = 0;
479 	rx_ring->next_to_use = 0;
480 
481 	rx_ring->adapter = adapter;
482 
483 	return 0;
484 
485 err:
486 	vfree(rx_ring->buffer_info);
487 	rx_ring->buffer_info = NULL;
488 	dev_err(&adapter->pdev->dev,
489 		"Unable to allocate memory for the receive descriptor ring\n");
490 	return -ENOMEM;
491 }
492 
493 /**
494  * igbvf_clean_tx_ring - Free Tx Buffers
495  * @tx_ring: ring to be cleaned
496  **/
497 static void igbvf_clean_tx_ring(struct igbvf_ring *tx_ring)
498 {
499 	struct igbvf_adapter *adapter = tx_ring->adapter;
500 	struct igbvf_buffer *buffer_info;
501 	unsigned long size;
502 	unsigned int i;
503 
504 	if (!tx_ring->buffer_info)
505 		return;
506 
507 	/* Free all the Tx ring sk_buffs */
508 	for (i = 0; i < tx_ring->count; i++) {
509 		buffer_info = &tx_ring->buffer_info[i];
510 		igbvf_put_txbuf(adapter, buffer_info);
511 	}
512 
513 	size = sizeof(struct igbvf_buffer) * tx_ring->count;
514 	memset(tx_ring->buffer_info, 0, size);
515 
516 	/* Zero out the descriptor ring */
517 	memset(tx_ring->desc, 0, tx_ring->size);
518 
519 	tx_ring->next_to_use = 0;
520 	tx_ring->next_to_clean = 0;
521 
522 	writel(0, adapter->hw.hw_addr + tx_ring->head);
523 	writel(0, adapter->hw.hw_addr + tx_ring->tail);
524 }
525 
526 /**
527  * igbvf_free_tx_resources - Free Tx Resources per Queue
528  * @tx_ring: ring to free resources from
529  *
530  * Free all transmit software resources
531  **/
532 void igbvf_free_tx_resources(struct igbvf_ring *tx_ring)
533 {
534 	struct pci_dev *pdev = tx_ring->adapter->pdev;
535 
536 	igbvf_clean_tx_ring(tx_ring);
537 
538 	vfree(tx_ring->buffer_info);
539 	tx_ring->buffer_info = NULL;
540 
541 	dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
542 			  tx_ring->dma);
543 
544 	tx_ring->desc = NULL;
545 }
546 
547 /**
548  * igbvf_clean_rx_ring - Free Rx Buffers per Queue
549  * @rx_ring: ring structure pointer to free buffers from
550  **/
551 static void igbvf_clean_rx_ring(struct igbvf_ring *rx_ring)
552 {
553 	struct igbvf_adapter *adapter = rx_ring->adapter;
554 	struct igbvf_buffer *buffer_info;
555 	struct pci_dev *pdev = adapter->pdev;
556 	unsigned long size;
557 	unsigned int i;
558 
559 	if (!rx_ring->buffer_info)
560 		return;
561 
562 	/* Free all the Rx ring sk_buffs */
563 	for (i = 0; i < rx_ring->count; i++) {
564 		buffer_info = &rx_ring->buffer_info[i];
565 		if (buffer_info->dma) {
566 			if (adapter->rx_ps_hdr_size) {
567 				dma_unmap_single(&pdev->dev, buffer_info->dma,
568 						 adapter->rx_ps_hdr_size,
569 						 DMA_FROM_DEVICE);
570 			} else {
571 				dma_unmap_single(&pdev->dev, buffer_info->dma,
572 						 adapter->rx_buffer_len,
573 						 DMA_FROM_DEVICE);
574 			}
575 			buffer_info->dma = 0;
576 		}
577 
578 		if (buffer_info->skb) {
579 			dev_kfree_skb(buffer_info->skb);
580 			buffer_info->skb = NULL;
581 		}
582 
583 		if (buffer_info->page) {
584 			if (buffer_info->page_dma)
585 				dma_unmap_page(&pdev->dev,
586 					       buffer_info->page_dma,
587 					       PAGE_SIZE / 2,
588 					       DMA_FROM_DEVICE);
589 			put_page(buffer_info->page);
590 			buffer_info->page = NULL;
591 			buffer_info->page_dma = 0;
592 			buffer_info->page_offset = 0;
593 		}
594 	}
595 
596 	size = sizeof(struct igbvf_buffer) * rx_ring->count;
597 	memset(rx_ring->buffer_info, 0, size);
598 
599 	/* Zero out the descriptor ring */
600 	memset(rx_ring->desc, 0, rx_ring->size);
601 
602 	rx_ring->next_to_clean = 0;
603 	rx_ring->next_to_use = 0;
604 
605 	writel(0, adapter->hw.hw_addr + rx_ring->head);
606 	writel(0, adapter->hw.hw_addr + rx_ring->tail);
607 }
608 
609 /**
610  * igbvf_free_rx_resources - Free Rx Resources
611  * @rx_ring: ring to clean the resources from
612  *
613  * Free all receive software resources
614  **/
615 
616 void igbvf_free_rx_resources(struct igbvf_ring *rx_ring)
617 {
618 	struct pci_dev *pdev = rx_ring->adapter->pdev;
619 
620 	igbvf_clean_rx_ring(rx_ring);
621 
622 	vfree(rx_ring->buffer_info);
623 	rx_ring->buffer_info = NULL;
624 
625 	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
626 			  rx_ring->dma);
627 	rx_ring->desc = NULL;
628 }
629 
630 /**
631  * igbvf_update_itr - update the dynamic ITR value based on statistics
632  * @adapter: pointer to adapter
633  * @itr_setting: current adapter->itr
634  * @packets: the number of packets during this measurement interval
635  * @bytes: the number of bytes during this measurement interval
636  *
637  * Stores a new ITR value based on packets and byte counts during the last
638  * interrupt.  The advantage of per interrupt computation is faster updates
639  * and more accurate ITR for the current traffic pattern.  Constants in this
640  * function were computed based on theoretical maximum wire speed and thresholds
641  * were set based on testing data as well as attempting to minimize response
642  * time while increasing bulk throughput.
643  **/
644 static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,
645 					   enum latency_range itr_setting,
646 					   int packets, int bytes)
647 {
648 	enum latency_range retval = itr_setting;
649 
650 	if (packets == 0)
651 		goto update_itr_done;
652 
653 	switch (itr_setting) {
654 	case lowest_latency:
655 		/* handle TSO and jumbo frames */
656 		if (bytes/packets > 8000)
657 			retval = bulk_latency;
658 		else if ((packets < 5) && (bytes > 512))
659 			retval = low_latency;
660 		break;
661 	case low_latency:  /* 50 usec aka 20000 ints/s */
662 		if (bytes > 10000) {
663 			/* this if handles the TSO accounting */
664 			if (bytes/packets > 8000)
665 				retval = bulk_latency;
666 			else if ((packets < 10) || ((bytes/packets) > 1200))
667 				retval = bulk_latency;
668 			else if ((packets > 35))
669 				retval = lowest_latency;
670 		} else if (bytes/packets > 2000) {
671 			retval = bulk_latency;
672 		} else if (packets <= 2 && bytes < 512) {
673 			retval = lowest_latency;
674 		}
675 		break;
676 	case bulk_latency: /* 250 usec aka 4000 ints/s */
677 		if (bytes > 25000) {
678 			if (packets > 35)
679 				retval = low_latency;
680 		} else if (bytes < 6000) {
681 			retval = low_latency;
682 		}
683 		break;
684 	default:
685 		break;
686 	}
687 
688 update_itr_done:
689 	return retval;
690 }
691 
692 static int igbvf_range_to_itr(enum latency_range current_range)
693 {
694 	int new_itr;
695 
696 	switch (current_range) {
697 	/* counts and packets in update_itr are dependent on these numbers */
698 	case lowest_latency:
699 		new_itr = IGBVF_70K_ITR;
700 		break;
701 	case low_latency:
702 		new_itr = IGBVF_20K_ITR;
703 		break;
704 	case bulk_latency:
705 		new_itr = IGBVF_4K_ITR;
706 		break;
707 	default:
708 		new_itr = IGBVF_START_ITR;
709 		break;
710 	}
711 	return new_itr;
712 }
713 
714 static void igbvf_set_itr(struct igbvf_adapter *adapter)
715 {
716 	u32 new_itr;
717 
718 	adapter->tx_ring->itr_range =
719 			igbvf_update_itr(adapter,
720 					 adapter->tx_ring->itr_val,
721 					 adapter->total_tx_packets,
722 					 adapter->total_tx_bytes);
723 
724 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
725 	if (adapter->requested_itr == 3 &&
726 	    adapter->tx_ring->itr_range == lowest_latency)
727 		adapter->tx_ring->itr_range = low_latency;
728 
729 	new_itr = igbvf_range_to_itr(adapter->tx_ring->itr_range);
730 
731 	if (new_itr != adapter->tx_ring->itr_val) {
732 		u32 current_itr = adapter->tx_ring->itr_val;
733 		/* this attempts to bias the interrupt rate towards Bulk
734 		 * by adding intermediate steps when interrupt rate is
735 		 * increasing
736 		 */
737 		new_itr = new_itr > current_itr ?
738 			  min(current_itr + (new_itr >> 2), new_itr) :
739 			  new_itr;
740 		adapter->tx_ring->itr_val = new_itr;
741 
742 		adapter->tx_ring->set_itr = 1;
743 	}
744 
745 	adapter->rx_ring->itr_range =
746 			igbvf_update_itr(adapter, adapter->rx_ring->itr_val,
747 					 adapter->total_rx_packets,
748 					 adapter->total_rx_bytes);
749 	if (adapter->requested_itr == 3 &&
750 	    adapter->rx_ring->itr_range == lowest_latency)
751 		adapter->rx_ring->itr_range = low_latency;
752 
753 	new_itr = igbvf_range_to_itr(adapter->rx_ring->itr_range);
754 
755 	if (new_itr != adapter->rx_ring->itr_val) {
756 		u32 current_itr = adapter->rx_ring->itr_val;
757 
758 		new_itr = new_itr > current_itr ?
759 			  min(current_itr + (new_itr >> 2), new_itr) :
760 			  new_itr;
761 		adapter->rx_ring->itr_val = new_itr;
762 
763 		adapter->rx_ring->set_itr = 1;
764 	}
765 }
766 
767 /**
768  * igbvf_clean_tx_irq - Reclaim resources after transmit completes
769  * @tx_ring: ring structure to clean descriptors from
770  *
771  * returns true if ring is completely cleaned
772  **/
773 static bool igbvf_clean_tx_irq(struct igbvf_ring *tx_ring)
774 {
775 	struct igbvf_adapter *adapter = tx_ring->adapter;
776 	struct net_device *netdev = adapter->netdev;
777 	struct igbvf_buffer *buffer_info;
778 	struct sk_buff *skb;
779 	union e1000_adv_tx_desc *tx_desc, *eop_desc;
780 	unsigned int total_bytes = 0, total_packets = 0;
781 	unsigned int i, count = 0;
782 	bool cleaned = false;
783 
784 	i = tx_ring->next_to_clean;
785 	buffer_info = &tx_ring->buffer_info[i];
786 	eop_desc = buffer_info->next_to_watch;
787 
788 	do {
789 		/* if next_to_watch is not set then there is no work pending */
790 		if (!eop_desc)
791 			break;
792 
793 		/* prevent any other reads prior to eop_desc */
794 		smp_rmb();
795 
796 		/* if DD is not set pending work has not been completed */
797 		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
798 			break;
799 
800 		/* clear next_to_watch to prevent false hangs */
801 		buffer_info->next_to_watch = NULL;
802 
803 		for (cleaned = false; !cleaned; count++) {
804 			tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
805 			cleaned = (tx_desc == eop_desc);
806 			skb = buffer_info->skb;
807 
808 			if (skb) {
809 				unsigned int segs, bytecount;
810 
811 				/* gso_segs is currently only valid for tcp */
812 				segs = skb_shinfo(skb)->gso_segs ?: 1;
813 				/* multiply data chunks by size of headers */
814 				bytecount = ((segs - 1) * skb_headlen(skb)) +
815 					    skb->len;
816 				total_packets += segs;
817 				total_bytes += bytecount;
818 			}
819 
820 			igbvf_put_txbuf(adapter, buffer_info);
821 			tx_desc->wb.status = 0;
822 
823 			i++;
824 			if (i == tx_ring->count)
825 				i = 0;
826 
827 			buffer_info = &tx_ring->buffer_info[i];
828 		}
829 
830 		eop_desc = buffer_info->next_to_watch;
831 	} while (count < tx_ring->count);
832 
833 	tx_ring->next_to_clean = i;
834 
835 	if (unlikely(count && netif_carrier_ok(netdev) &&
836 	    igbvf_desc_unused(tx_ring) >= IGBVF_TX_QUEUE_WAKE)) {
837 		/* Make sure that anybody stopping the queue after this
838 		 * sees the new next_to_clean.
839 		 */
840 		smp_mb();
841 		if (netif_queue_stopped(netdev) &&
842 		    !(test_bit(__IGBVF_DOWN, &adapter->state))) {
843 			netif_wake_queue(netdev);
844 			++adapter->restart_queue;
845 		}
846 	}
847 
848 	netdev->stats.tx_bytes += total_bytes;
849 	netdev->stats.tx_packets += total_packets;
850 	return count < tx_ring->count;
851 }
852 
853 static irqreturn_t igbvf_msix_other(int irq, void *data)
854 {
855 	struct net_device *netdev = data;
856 	struct igbvf_adapter *adapter = netdev_priv(netdev);
857 	struct e1000_hw *hw = &adapter->hw;
858 
859 	adapter->int_counter1++;
860 
861 	hw->mac.get_link_status = 1;
862 	if (!test_bit(__IGBVF_DOWN, &adapter->state))
863 		mod_timer(&adapter->watchdog_timer, jiffies + 1);
864 
865 	ew32(EIMS, adapter->eims_other);
866 
867 	return IRQ_HANDLED;
868 }
869 
870 static irqreturn_t igbvf_intr_msix_tx(int irq, void *data)
871 {
872 	struct net_device *netdev = data;
873 	struct igbvf_adapter *adapter = netdev_priv(netdev);
874 	struct e1000_hw *hw = &adapter->hw;
875 	struct igbvf_ring *tx_ring = adapter->tx_ring;
876 
877 	if (tx_ring->set_itr) {
878 		writel(tx_ring->itr_val,
879 		       adapter->hw.hw_addr + tx_ring->itr_register);
880 		adapter->tx_ring->set_itr = 0;
881 	}
882 
883 	adapter->total_tx_bytes = 0;
884 	adapter->total_tx_packets = 0;
885 
886 	/* auto mask will automatically re-enable the interrupt when we write
887 	 * EICS
888 	 */
889 	if (!igbvf_clean_tx_irq(tx_ring))
890 		/* Ring was not completely cleaned, so fire another interrupt */
891 		ew32(EICS, tx_ring->eims_value);
892 	else
893 		ew32(EIMS, tx_ring->eims_value);
894 
895 	return IRQ_HANDLED;
896 }
897 
898 static irqreturn_t igbvf_intr_msix_rx(int irq, void *data)
899 {
900 	struct net_device *netdev = data;
901 	struct igbvf_adapter *adapter = netdev_priv(netdev);
902 
903 	adapter->int_counter0++;
904 
905 	/* Write the ITR value calculated at the end of the
906 	 * previous interrupt.
907 	 */
908 	if (adapter->rx_ring->set_itr) {
909 		writel(adapter->rx_ring->itr_val,
910 		       adapter->hw.hw_addr + adapter->rx_ring->itr_register);
911 		adapter->rx_ring->set_itr = 0;
912 	}
913 
914 	if (napi_schedule_prep(&adapter->rx_ring->napi)) {
915 		adapter->total_rx_bytes = 0;
916 		adapter->total_rx_packets = 0;
917 		__napi_schedule(&adapter->rx_ring->napi);
918 	}
919 
920 	return IRQ_HANDLED;
921 }
922 
923 #define IGBVF_NO_QUEUE -1
924 
925 static void igbvf_assign_vector(struct igbvf_adapter *adapter, int rx_queue,
926 				int tx_queue, int msix_vector)
927 {
928 	struct e1000_hw *hw = &adapter->hw;
929 	u32 ivar, index;
930 
931 	/* 82576 uses a table-based method for assigning vectors.
932 	 * Each queue has a single entry in the table to which we write
933 	 * a vector number along with a "valid" bit.  Sadly, the layout
934 	 * of the table is somewhat counterintuitive.
935 	 */
936 	if (rx_queue > IGBVF_NO_QUEUE) {
937 		index = (rx_queue >> 1);
938 		ivar = array_er32(IVAR0, index);
939 		if (rx_queue & 0x1) {
940 			/* vector goes into third byte of register */
941 			ivar = ivar & 0xFF00FFFF;
942 			ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
943 		} else {
944 			/* vector goes into low byte of register */
945 			ivar = ivar & 0xFFFFFF00;
946 			ivar |= msix_vector | E1000_IVAR_VALID;
947 		}
948 		adapter->rx_ring[rx_queue].eims_value = BIT(msix_vector);
949 		array_ew32(IVAR0, index, ivar);
950 	}
951 	if (tx_queue > IGBVF_NO_QUEUE) {
952 		index = (tx_queue >> 1);
953 		ivar = array_er32(IVAR0, index);
954 		if (tx_queue & 0x1) {
955 			/* vector goes into high byte of register */
956 			ivar = ivar & 0x00FFFFFF;
957 			ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
958 		} else {
959 			/* vector goes into second byte of register */
960 			ivar = ivar & 0xFFFF00FF;
961 			ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
962 		}
963 		adapter->tx_ring[tx_queue].eims_value = BIT(msix_vector);
964 		array_ew32(IVAR0, index, ivar);
965 	}
966 }
967 
968 /**
969  * igbvf_configure_msix - Configure MSI-X hardware
970  * @adapter: board private structure
971  *
972  * igbvf_configure_msix sets up the hardware to properly
973  * generate MSI-X interrupts.
974  **/
975 static void igbvf_configure_msix(struct igbvf_adapter *adapter)
976 {
977 	u32 tmp;
978 	struct e1000_hw *hw = &adapter->hw;
979 	struct igbvf_ring *tx_ring = adapter->tx_ring;
980 	struct igbvf_ring *rx_ring = adapter->rx_ring;
981 	int vector = 0;
982 
983 	adapter->eims_enable_mask = 0;
984 
985 	igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, 0, vector++);
986 	adapter->eims_enable_mask |= tx_ring->eims_value;
987 	writel(tx_ring->itr_val, hw->hw_addr + tx_ring->itr_register);
988 	igbvf_assign_vector(adapter, 0, IGBVF_NO_QUEUE, vector++);
989 	adapter->eims_enable_mask |= rx_ring->eims_value;
990 	writel(rx_ring->itr_val, hw->hw_addr + rx_ring->itr_register);
991 
992 	/* set vector for other causes, i.e. link changes */
993 
994 	tmp = (vector++ | E1000_IVAR_VALID);
995 
996 	ew32(IVAR_MISC, tmp);
997 
998 	adapter->eims_enable_mask = GENMASK(vector - 1, 0);
999 	adapter->eims_other = BIT(vector - 1);
1000 	e1e_flush();
1001 }
1002 
1003 static void igbvf_reset_interrupt_capability(struct igbvf_adapter *adapter)
1004 {
1005 	if (adapter->msix_entries) {
1006 		pci_disable_msix(adapter->pdev);
1007 		kfree(adapter->msix_entries);
1008 		adapter->msix_entries = NULL;
1009 	}
1010 }
1011 
1012 /**
1013  * igbvf_set_interrupt_capability - set MSI or MSI-X if supported
1014  * @adapter: board private structure
1015  *
1016  * Attempt to configure interrupts using the best available
1017  * capabilities of the hardware and kernel.
1018  **/
1019 static void igbvf_set_interrupt_capability(struct igbvf_adapter *adapter)
1020 {
1021 	int err = -ENOMEM;
1022 	int i;
1023 
1024 	/* we allocate 3 vectors, 1 for Tx, 1 for Rx, one for PF messages */
1025 	adapter->msix_entries = kcalloc(3, sizeof(struct msix_entry),
1026 					GFP_KERNEL);
1027 	if (adapter->msix_entries) {
1028 		for (i = 0; i < 3; i++)
1029 			adapter->msix_entries[i].entry = i;
1030 
1031 		err = pci_enable_msix_range(adapter->pdev,
1032 					    adapter->msix_entries, 3, 3);
1033 	}
1034 
1035 	if (err < 0) {
1036 		/* MSI-X failed */
1037 		dev_err(&adapter->pdev->dev,
1038 			"Failed to initialize MSI-X interrupts.\n");
1039 		igbvf_reset_interrupt_capability(adapter);
1040 	}
1041 }
1042 
1043 /**
1044  * igbvf_request_msix - Initialize MSI-X interrupts
1045  * @adapter: board private structure
1046  *
1047  * igbvf_request_msix allocates MSI-X vectors and requests interrupts from the
1048  * kernel.
1049  **/
1050 static int igbvf_request_msix(struct igbvf_adapter *adapter)
1051 {
1052 	struct net_device *netdev = adapter->netdev;
1053 	int err = 0, vector = 0;
1054 
1055 	if (strlen(netdev->name) < (IFNAMSIZ - 5)) {
1056 		sprintf(adapter->tx_ring->name, "%s-tx-0", netdev->name);
1057 		sprintf(adapter->rx_ring->name, "%s-rx-0", netdev->name);
1058 	} else {
1059 		memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
1060 		memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
1061 	}
1062 
1063 	err = request_irq(adapter->msix_entries[vector].vector,
1064 			  igbvf_intr_msix_tx, 0, adapter->tx_ring->name,
1065 			  netdev);
1066 	if (err)
1067 		goto out;
1068 
1069 	adapter->tx_ring->itr_register = E1000_EITR(vector);
1070 	adapter->tx_ring->itr_val = adapter->current_itr;
1071 	vector++;
1072 
1073 	err = request_irq(adapter->msix_entries[vector].vector,
1074 			  igbvf_intr_msix_rx, 0, adapter->rx_ring->name,
1075 			  netdev);
1076 	if (err)
1077 		goto free_irq_tx;
1078 
1079 	adapter->rx_ring->itr_register = E1000_EITR(vector);
1080 	adapter->rx_ring->itr_val = adapter->current_itr;
1081 	vector++;
1082 
1083 	err = request_irq(adapter->msix_entries[vector].vector,
1084 			  igbvf_msix_other, 0, netdev->name, netdev);
1085 	if (err)
1086 		goto free_irq_rx;
1087 
1088 	igbvf_configure_msix(adapter);
1089 	return 0;
1090 free_irq_rx:
1091 	free_irq(adapter->msix_entries[--vector].vector, netdev);
1092 free_irq_tx:
1093 	free_irq(adapter->msix_entries[--vector].vector, netdev);
1094 out:
1095 	return err;
1096 }
1097 
1098 /**
1099  * igbvf_alloc_queues - Allocate memory for all rings
1100  * @adapter: board private structure to initialize
1101  **/
1102 static int igbvf_alloc_queues(struct igbvf_adapter *adapter)
1103 {
1104 	struct net_device *netdev = adapter->netdev;
1105 
1106 	adapter->tx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1107 	if (!adapter->tx_ring)
1108 		return -ENOMEM;
1109 
1110 	adapter->rx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1111 	if (!adapter->rx_ring) {
1112 		kfree(adapter->tx_ring);
1113 		return -ENOMEM;
1114 	}
1115 
1116 	netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll);
1117 
1118 	return 0;
1119 }
1120 
1121 /**
1122  * igbvf_request_irq - initialize interrupts
1123  * @adapter: board private structure
1124  *
1125  * Attempts to configure interrupts using the best available
1126  * capabilities of the hardware and kernel.
1127  **/
1128 static int igbvf_request_irq(struct igbvf_adapter *adapter)
1129 {
1130 	int err = -1;
1131 
1132 	/* igbvf supports msi-x only */
1133 	if (adapter->msix_entries)
1134 		err = igbvf_request_msix(adapter);
1135 
1136 	if (!err)
1137 		return err;
1138 
1139 	dev_err(&adapter->pdev->dev,
1140 		"Unable to allocate interrupt, Error: %d\n", err);
1141 
1142 	return err;
1143 }
1144 
1145 static void igbvf_free_irq(struct igbvf_adapter *adapter)
1146 {
1147 	struct net_device *netdev = adapter->netdev;
1148 	int vector;
1149 
1150 	if (adapter->msix_entries) {
1151 		for (vector = 0; vector < 3; vector++)
1152 			free_irq(adapter->msix_entries[vector].vector, netdev);
1153 	}
1154 }
1155 
1156 /**
1157  * igbvf_irq_disable - Mask off interrupt generation on the NIC
1158  * @adapter: board private structure
1159  **/
1160 static void igbvf_irq_disable(struct igbvf_adapter *adapter)
1161 {
1162 	struct e1000_hw *hw = &adapter->hw;
1163 
1164 	ew32(EIMC, ~0);
1165 
1166 	if (adapter->msix_entries)
1167 		ew32(EIAC, 0);
1168 }
1169 
1170 /**
1171  * igbvf_irq_enable - Enable default interrupt generation settings
1172  * @adapter: board private structure
1173  **/
1174 static void igbvf_irq_enable(struct igbvf_adapter *adapter)
1175 {
1176 	struct e1000_hw *hw = &adapter->hw;
1177 
1178 	ew32(EIAC, adapter->eims_enable_mask);
1179 	ew32(EIAM, adapter->eims_enable_mask);
1180 	ew32(EIMS, adapter->eims_enable_mask);
1181 }
1182 
1183 /**
1184  * igbvf_poll - NAPI Rx polling callback
1185  * @napi: struct associated with this polling callback
1186  * @budget: amount of packets driver is allowed to process this poll
1187  **/
1188 static int igbvf_poll(struct napi_struct *napi, int budget)
1189 {
1190 	struct igbvf_ring *rx_ring = container_of(napi, struct igbvf_ring, napi);
1191 	struct igbvf_adapter *adapter = rx_ring->adapter;
1192 	struct e1000_hw *hw = &adapter->hw;
1193 	int work_done = 0;
1194 
1195 	igbvf_clean_rx_irq(adapter, &work_done, budget);
1196 
1197 	if (work_done == budget)
1198 		return budget;
1199 
1200 	/* Exit the polling mode, but don't re-enable interrupts if stack might
1201 	 * poll us due to busy-polling
1202 	 */
1203 	if (likely(napi_complete_done(napi, work_done))) {
1204 		if (adapter->requested_itr & 3)
1205 			igbvf_set_itr(adapter);
1206 
1207 		if (!test_bit(__IGBVF_DOWN, &adapter->state))
1208 			ew32(EIMS, adapter->rx_ring->eims_value);
1209 	}
1210 
1211 	return work_done;
1212 }
1213 
1214 /**
1215  * igbvf_set_rlpml - set receive large packet maximum length
1216  * @adapter: board private structure
1217  *
1218  * Configure the maximum size of packets that will be received
1219  */
1220 static void igbvf_set_rlpml(struct igbvf_adapter *adapter)
1221 {
1222 	int max_frame_size;
1223 	struct e1000_hw *hw = &adapter->hw;
1224 
1225 	max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
1226 
1227 	spin_lock_bh(&hw->mbx_lock);
1228 
1229 	e1000_rlpml_set_vf(hw, max_frame_size);
1230 
1231 	spin_unlock_bh(&hw->mbx_lock);
1232 }
1233 
1234 static int igbvf_vlan_rx_add_vid(struct net_device *netdev,
1235 				 __be16 proto, u16 vid)
1236 {
1237 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1238 	struct e1000_hw *hw = &adapter->hw;
1239 
1240 	spin_lock_bh(&hw->mbx_lock);
1241 
1242 	if (hw->mac.ops.set_vfta(hw, vid, true)) {
1243 		dev_warn(&adapter->pdev->dev, "Vlan id %d\n is not added", vid);
1244 		spin_unlock_bh(&hw->mbx_lock);
1245 		return -EINVAL;
1246 	}
1247 
1248 	spin_unlock_bh(&hw->mbx_lock);
1249 
1250 	set_bit(vid, adapter->active_vlans);
1251 	return 0;
1252 }
1253 
1254 static int igbvf_vlan_rx_kill_vid(struct net_device *netdev,
1255 				  __be16 proto, u16 vid)
1256 {
1257 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1258 	struct e1000_hw *hw = &adapter->hw;
1259 
1260 	spin_lock_bh(&hw->mbx_lock);
1261 
1262 	if (hw->mac.ops.set_vfta(hw, vid, false)) {
1263 		dev_err(&adapter->pdev->dev,
1264 			"Failed to remove vlan id %d\n", vid);
1265 		spin_unlock_bh(&hw->mbx_lock);
1266 		return -EINVAL;
1267 	}
1268 
1269 	spin_unlock_bh(&hw->mbx_lock);
1270 
1271 	clear_bit(vid, adapter->active_vlans);
1272 	return 0;
1273 }
1274 
1275 static void igbvf_restore_vlan(struct igbvf_adapter *adapter)
1276 {
1277 	u16 vid;
1278 
1279 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1280 		igbvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
1281 }
1282 
1283 /**
1284  * igbvf_configure_tx - Configure Transmit Unit after Reset
1285  * @adapter: board private structure
1286  *
1287  * Configure the Tx unit of the MAC after a reset.
1288  **/
1289 static void igbvf_configure_tx(struct igbvf_adapter *adapter)
1290 {
1291 	struct e1000_hw *hw = &adapter->hw;
1292 	struct igbvf_ring *tx_ring = adapter->tx_ring;
1293 	u64 tdba;
1294 	u32 txdctl, dca_txctrl;
1295 
1296 	/* disable transmits */
1297 	txdctl = er32(TXDCTL(0));
1298 	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1299 	e1e_flush();
1300 	msleep(10);
1301 
1302 	/* Setup the HW Tx Head and Tail descriptor pointers */
1303 	ew32(TDLEN(0), tx_ring->count * sizeof(union e1000_adv_tx_desc));
1304 	tdba = tx_ring->dma;
1305 	ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
1306 	ew32(TDBAH(0), (tdba >> 32));
1307 	ew32(TDH(0), 0);
1308 	ew32(TDT(0), 0);
1309 	tx_ring->head = E1000_TDH(0);
1310 	tx_ring->tail = E1000_TDT(0);
1311 
1312 	/* Turn off Relaxed Ordering on head write-backs.  The writebacks
1313 	 * MUST be delivered in order or it will completely screw up
1314 	 * our bookkeeping.
1315 	 */
1316 	dca_txctrl = er32(DCA_TXCTRL(0));
1317 	dca_txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
1318 	ew32(DCA_TXCTRL(0), dca_txctrl);
1319 
1320 	/* enable transmits */
1321 	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
1322 	ew32(TXDCTL(0), txdctl);
1323 
1324 	/* Setup Transmit Descriptor Settings for eop descriptor */
1325 	adapter->txd_cmd = E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_IFCS;
1326 
1327 	/* enable Report Status bit */
1328 	adapter->txd_cmd |= E1000_ADVTXD_DCMD_RS;
1329 }
1330 
1331 /**
1332  * igbvf_setup_srrctl - configure the receive control registers
1333  * @adapter: Board private structure
1334  **/
1335 static void igbvf_setup_srrctl(struct igbvf_adapter *adapter)
1336 {
1337 	struct e1000_hw *hw = &adapter->hw;
1338 	u32 srrctl = 0;
1339 
1340 	srrctl &= ~(E1000_SRRCTL_DESCTYPE_MASK |
1341 		    E1000_SRRCTL_BSIZEHDR_MASK |
1342 		    E1000_SRRCTL_BSIZEPKT_MASK);
1343 
1344 	/* Enable queue drop to avoid head of line blocking */
1345 	srrctl |= E1000_SRRCTL_DROP_EN;
1346 
1347 	/* Setup buffer sizes */
1348 	srrctl |= ALIGN(adapter->rx_buffer_len, 1024) >>
1349 		  E1000_SRRCTL_BSIZEPKT_SHIFT;
1350 
1351 	if (adapter->rx_buffer_len < 2048) {
1352 		adapter->rx_ps_hdr_size = 0;
1353 		srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
1354 	} else {
1355 		adapter->rx_ps_hdr_size = 128;
1356 		srrctl |= adapter->rx_ps_hdr_size <<
1357 			  E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
1358 		srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1359 	}
1360 
1361 	ew32(SRRCTL(0), srrctl);
1362 }
1363 
1364 /**
1365  * igbvf_configure_rx - Configure Receive Unit after Reset
1366  * @adapter: board private structure
1367  *
1368  * Configure the Rx unit of the MAC after a reset.
1369  **/
1370 static void igbvf_configure_rx(struct igbvf_adapter *adapter)
1371 {
1372 	struct e1000_hw *hw = &adapter->hw;
1373 	struct igbvf_ring *rx_ring = adapter->rx_ring;
1374 	u64 rdba;
1375 	u32 rxdctl;
1376 
1377 	/* disable receives */
1378 	rxdctl = er32(RXDCTL(0));
1379 	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1380 	e1e_flush();
1381 	msleep(10);
1382 
1383 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
1384 	 * the Base and Length of the Rx Descriptor Ring
1385 	 */
1386 	rdba = rx_ring->dma;
1387 	ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
1388 	ew32(RDBAH(0), (rdba >> 32));
1389 	ew32(RDLEN(0), rx_ring->count * sizeof(union e1000_adv_rx_desc));
1390 	rx_ring->head = E1000_RDH(0);
1391 	rx_ring->tail = E1000_RDT(0);
1392 	ew32(RDH(0), 0);
1393 	ew32(RDT(0), 0);
1394 
1395 	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
1396 	rxdctl &= 0xFFF00000;
1397 	rxdctl |= IGBVF_RX_PTHRESH;
1398 	rxdctl |= IGBVF_RX_HTHRESH << 8;
1399 	rxdctl |= IGBVF_RX_WTHRESH << 16;
1400 
1401 	igbvf_set_rlpml(adapter);
1402 
1403 	/* enable receives */
1404 	ew32(RXDCTL(0), rxdctl);
1405 }
1406 
1407 /**
1408  * igbvf_set_multi - Multicast and Promiscuous mode set
1409  * @netdev: network interface device structure
1410  *
1411  * The set_multi entry point is called whenever the multicast address
1412  * list or the network interface flags are updated.  This routine is
1413  * responsible for configuring the hardware for proper multicast,
1414  * promiscuous mode, and all-multi behavior.
1415  **/
1416 static void igbvf_set_multi(struct net_device *netdev)
1417 {
1418 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1419 	struct e1000_hw *hw = &adapter->hw;
1420 	struct netdev_hw_addr *ha;
1421 	u8  *mta_list = NULL;
1422 	int i;
1423 
1424 	if (!netdev_mc_empty(netdev)) {
1425 		mta_list = kmalloc_array(netdev_mc_count(netdev), ETH_ALEN,
1426 					 GFP_ATOMIC);
1427 		if (!mta_list)
1428 			return;
1429 	}
1430 
1431 	/* prepare a packed array of only addresses. */
1432 	i = 0;
1433 	netdev_for_each_mc_addr(ha, netdev)
1434 		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
1435 
1436 	spin_lock_bh(&hw->mbx_lock);
1437 
1438 	hw->mac.ops.update_mc_addr_list(hw, mta_list, i, 0, 0);
1439 
1440 	spin_unlock_bh(&hw->mbx_lock);
1441 	kfree(mta_list);
1442 }
1443 
1444 /**
1445  * igbvf_set_uni - Configure unicast MAC filters
1446  * @netdev: network interface device structure
1447  *
1448  * This routine is responsible for configuring the hardware for proper
1449  * unicast filters.
1450  **/
1451 static int igbvf_set_uni(struct net_device *netdev)
1452 {
1453 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1454 	struct e1000_hw *hw = &adapter->hw;
1455 
1456 	if (netdev_uc_count(netdev) > IGBVF_MAX_MAC_FILTERS) {
1457 		pr_err("Too many unicast filters - No Space\n");
1458 		return -ENOSPC;
1459 	}
1460 
1461 	spin_lock_bh(&hw->mbx_lock);
1462 
1463 	/* Clear all unicast MAC filters */
1464 	hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_CLR, NULL);
1465 
1466 	spin_unlock_bh(&hw->mbx_lock);
1467 
1468 	if (!netdev_uc_empty(netdev)) {
1469 		struct netdev_hw_addr *ha;
1470 
1471 		/* Add MAC filters one by one */
1472 		netdev_for_each_uc_addr(ha, netdev) {
1473 			spin_lock_bh(&hw->mbx_lock);
1474 
1475 			hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_ADD,
1476 						ha->addr);
1477 
1478 			spin_unlock_bh(&hw->mbx_lock);
1479 			udelay(200);
1480 		}
1481 	}
1482 
1483 	return 0;
1484 }
1485 
1486 static void igbvf_set_rx_mode(struct net_device *netdev)
1487 {
1488 	igbvf_set_multi(netdev);
1489 	igbvf_set_uni(netdev);
1490 }
1491 
1492 /**
1493  * igbvf_configure - configure the hardware for Rx and Tx
1494  * @adapter: private board structure
1495  **/
1496 static void igbvf_configure(struct igbvf_adapter *adapter)
1497 {
1498 	igbvf_set_rx_mode(adapter->netdev);
1499 
1500 	igbvf_restore_vlan(adapter);
1501 
1502 	igbvf_configure_tx(adapter);
1503 	igbvf_setup_srrctl(adapter);
1504 	igbvf_configure_rx(adapter);
1505 	igbvf_alloc_rx_buffers(adapter->rx_ring,
1506 			       igbvf_desc_unused(adapter->rx_ring));
1507 }
1508 
1509 /* igbvf_reset - bring the hardware into a known good state
1510  * @adapter: private board structure
1511  *
1512  * This function boots the hardware and enables some settings that
1513  * require a configuration cycle of the hardware - those cannot be
1514  * set/changed during runtime. After reset the device needs to be
1515  * properly configured for Rx, Tx etc.
1516  */
1517 static void igbvf_reset(struct igbvf_adapter *adapter)
1518 {
1519 	struct e1000_mac_info *mac = &adapter->hw.mac;
1520 	struct net_device *netdev = adapter->netdev;
1521 	struct e1000_hw *hw = &adapter->hw;
1522 
1523 	spin_lock_bh(&hw->mbx_lock);
1524 
1525 	/* Allow time for pending master requests to run */
1526 	if (mac->ops.reset_hw(hw))
1527 		dev_info(&adapter->pdev->dev, "PF still resetting\n");
1528 
1529 	mac->ops.init_hw(hw);
1530 
1531 	spin_unlock_bh(&hw->mbx_lock);
1532 
1533 	if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1534 		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
1535 		memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1536 		       netdev->addr_len);
1537 	}
1538 
1539 	adapter->last_reset = jiffies;
1540 }
1541 
1542 int igbvf_up(struct igbvf_adapter *adapter)
1543 {
1544 	struct e1000_hw *hw = &adapter->hw;
1545 
1546 	/* hardware has been reset, we need to reload some things */
1547 	igbvf_configure(adapter);
1548 
1549 	clear_bit(__IGBVF_DOWN, &adapter->state);
1550 
1551 	napi_enable(&adapter->rx_ring->napi);
1552 	if (adapter->msix_entries)
1553 		igbvf_configure_msix(adapter);
1554 
1555 	/* Clear any pending interrupts. */
1556 	er32(EICR);
1557 	igbvf_irq_enable(adapter);
1558 
1559 	/* start the watchdog */
1560 	hw->mac.get_link_status = 1;
1561 	mod_timer(&adapter->watchdog_timer, jiffies + 1);
1562 
1563 	return 0;
1564 }
1565 
1566 void igbvf_down(struct igbvf_adapter *adapter)
1567 {
1568 	struct net_device *netdev = adapter->netdev;
1569 	struct e1000_hw *hw = &adapter->hw;
1570 	u32 rxdctl, txdctl;
1571 
1572 	/* signal that we're down so the interrupt handler does not
1573 	 * reschedule our watchdog timer
1574 	 */
1575 	set_bit(__IGBVF_DOWN, &adapter->state);
1576 
1577 	/* disable receives in the hardware */
1578 	rxdctl = er32(RXDCTL(0));
1579 	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1580 
1581 	netif_carrier_off(netdev);
1582 	netif_stop_queue(netdev);
1583 
1584 	/* disable transmits in the hardware */
1585 	txdctl = er32(TXDCTL(0));
1586 	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1587 
1588 	/* flush both disables and wait for them to finish */
1589 	e1e_flush();
1590 	msleep(10);
1591 
1592 	napi_disable(&adapter->rx_ring->napi);
1593 
1594 	igbvf_irq_disable(adapter);
1595 
1596 	del_timer_sync(&adapter->watchdog_timer);
1597 
1598 	/* record the stats before reset*/
1599 	igbvf_update_stats(adapter);
1600 
1601 	adapter->link_speed = 0;
1602 	adapter->link_duplex = 0;
1603 
1604 	igbvf_reset(adapter);
1605 	igbvf_clean_tx_ring(adapter->tx_ring);
1606 	igbvf_clean_rx_ring(adapter->rx_ring);
1607 }
1608 
1609 void igbvf_reinit_locked(struct igbvf_adapter *adapter)
1610 {
1611 	might_sleep();
1612 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
1613 		usleep_range(1000, 2000);
1614 	igbvf_down(adapter);
1615 	igbvf_up(adapter);
1616 	clear_bit(__IGBVF_RESETTING, &adapter->state);
1617 }
1618 
1619 /**
1620  * igbvf_sw_init - Initialize general software structures (struct igbvf_adapter)
1621  * @adapter: board private structure to initialize
1622  *
1623  * igbvf_sw_init initializes the Adapter private data structure.
1624  * Fields are initialized based on PCI device information and
1625  * OS network device settings (MTU size).
1626  **/
1627 static int igbvf_sw_init(struct igbvf_adapter *adapter)
1628 {
1629 	struct net_device *netdev = adapter->netdev;
1630 	s32 rc;
1631 
1632 	adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
1633 	adapter->rx_ps_hdr_size = 0;
1634 	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1635 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1636 
1637 	adapter->tx_int_delay = 8;
1638 	adapter->tx_abs_int_delay = 32;
1639 	adapter->rx_int_delay = 0;
1640 	adapter->rx_abs_int_delay = 8;
1641 	adapter->requested_itr = 3;
1642 	adapter->current_itr = IGBVF_START_ITR;
1643 
1644 	/* Set various function pointers */
1645 	adapter->ei->init_ops(&adapter->hw);
1646 
1647 	rc = adapter->hw.mac.ops.init_params(&adapter->hw);
1648 	if (rc)
1649 		return rc;
1650 
1651 	rc = adapter->hw.mbx.ops.init_params(&adapter->hw);
1652 	if (rc)
1653 		return rc;
1654 
1655 	igbvf_set_interrupt_capability(adapter);
1656 
1657 	if (igbvf_alloc_queues(adapter))
1658 		return -ENOMEM;
1659 
1660 	spin_lock_init(&adapter->tx_queue_lock);
1661 
1662 	/* Explicitly disable IRQ since the NIC can be in any state. */
1663 	igbvf_irq_disable(adapter);
1664 
1665 	spin_lock_init(&adapter->stats_lock);
1666 	spin_lock_init(&adapter->hw.mbx_lock);
1667 
1668 	set_bit(__IGBVF_DOWN, &adapter->state);
1669 	return 0;
1670 }
1671 
1672 static void igbvf_initialize_last_counter_stats(struct igbvf_adapter *adapter)
1673 {
1674 	struct e1000_hw *hw = &adapter->hw;
1675 
1676 	adapter->stats.last_gprc = er32(VFGPRC);
1677 	adapter->stats.last_gorc = er32(VFGORC);
1678 	adapter->stats.last_gptc = er32(VFGPTC);
1679 	adapter->stats.last_gotc = er32(VFGOTC);
1680 	adapter->stats.last_mprc = er32(VFMPRC);
1681 	adapter->stats.last_gotlbc = er32(VFGOTLBC);
1682 	adapter->stats.last_gptlbc = er32(VFGPTLBC);
1683 	adapter->stats.last_gorlbc = er32(VFGORLBC);
1684 	adapter->stats.last_gprlbc = er32(VFGPRLBC);
1685 
1686 	adapter->stats.base_gprc = er32(VFGPRC);
1687 	adapter->stats.base_gorc = er32(VFGORC);
1688 	adapter->stats.base_gptc = er32(VFGPTC);
1689 	adapter->stats.base_gotc = er32(VFGOTC);
1690 	adapter->stats.base_mprc = er32(VFMPRC);
1691 	adapter->stats.base_gotlbc = er32(VFGOTLBC);
1692 	adapter->stats.base_gptlbc = er32(VFGPTLBC);
1693 	adapter->stats.base_gorlbc = er32(VFGORLBC);
1694 	adapter->stats.base_gprlbc = er32(VFGPRLBC);
1695 }
1696 
1697 /**
1698  * igbvf_open - Called when a network interface is made active
1699  * @netdev: network interface device structure
1700  *
1701  * Returns 0 on success, negative value on failure
1702  *
1703  * The open entry point is called when a network interface is made
1704  * active by the system (IFF_UP).  At this point all resources needed
1705  * for transmit and receive operations are allocated, the interrupt
1706  * handler is registered with the OS, the watchdog timer is started,
1707  * and the stack is notified that the interface is ready.
1708  **/
1709 static int igbvf_open(struct net_device *netdev)
1710 {
1711 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1712 	struct e1000_hw *hw = &adapter->hw;
1713 	int err;
1714 
1715 	/* disallow open during test */
1716 	if (test_bit(__IGBVF_TESTING, &adapter->state))
1717 		return -EBUSY;
1718 
1719 	/* allocate transmit descriptors */
1720 	err = igbvf_setup_tx_resources(adapter, adapter->tx_ring);
1721 	if (err)
1722 		goto err_setup_tx;
1723 
1724 	/* allocate receive descriptors */
1725 	err = igbvf_setup_rx_resources(adapter, adapter->rx_ring);
1726 	if (err)
1727 		goto err_setup_rx;
1728 
1729 	/* before we allocate an interrupt, we must be ready to handle it.
1730 	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1731 	 * as soon as we call pci_request_irq, so we have to setup our
1732 	 * clean_rx handler before we do so.
1733 	 */
1734 	igbvf_configure(adapter);
1735 
1736 	err = igbvf_request_irq(adapter);
1737 	if (err)
1738 		goto err_req_irq;
1739 
1740 	/* From here on the code is the same as igbvf_up() */
1741 	clear_bit(__IGBVF_DOWN, &adapter->state);
1742 
1743 	napi_enable(&adapter->rx_ring->napi);
1744 
1745 	/* clear any pending interrupts */
1746 	er32(EICR);
1747 
1748 	igbvf_irq_enable(adapter);
1749 
1750 	/* start the watchdog */
1751 	hw->mac.get_link_status = 1;
1752 	mod_timer(&adapter->watchdog_timer, jiffies + 1);
1753 
1754 	return 0;
1755 
1756 err_req_irq:
1757 	igbvf_free_rx_resources(adapter->rx_ring);
1758 err_setup_rx:
1759 	igbvf_free_tx_resources(adapter->tx_ring);
1760 err_setup_tx:
1761 	igbvf_reset(adapter);
1762 
1763 	return err;
1764 }
1765 
1766 /**
1767  * igbvf_close - Disables a network interface
1768  * @netdev: network interface device structure
1769  *
1770  * Returns 0, this is not allowed to fail
1771  *
1772  * The close entry point is called when an interface is de-activated
1773  * by the OS.  The hardware is still under the drivers control, but
1774  * needs to be disabled.  A global MAC reset is issued to stop the
1775  * hardware, and all transmit and receive resources are freed.
1776  **/
1777 static int igbvf_close(struct net_device *netdev)
1778 {
1779 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1780 
1781 	WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
1782 	igbvf_down(adapter);
1783 
1784 	igbvf_free_irq(adapter);
1785 
1786 	igbvf_free_tx_resources(adapter->tx_ring);
1787 	igbvf_free_rx_resources(adapter->rx_ring);
1788 
1789 	return 0;
1790 }
1791 
1792 /**
1793  * igbvf_set_mac - Change the Ethernet Address of the NIC
1794  * @netdev: network interface device structure
1795  * @p: pointer to an address structure
1796  *
1797  * Returns 0 on success, negative on failure
1798  **/
1799 static int igbvf_set_mac(struct net_device *netdev, void *p)
1800 {
1801 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1802 	struct e1000_hw *hw = &adapter->hw;
1803 	struct sockaddr *addr = p;
1804 
1805 	if (!is_valid_ether_addr(addr->sa_data))
1806 		return -EADDRNOTAVAIL;
1807 
1808 	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
1809 
1810 	spin_lock_bh(&hw->mbx_lock);
1811 
1812 	hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
1813 
1814 	spin_unlock_bh(&hw->mbx_lock);
1815 
1816 	if (!ether_addr_equal(addr->sa_data, hw->mac.addr))
1817 		return -EADDRNOTAVAIL;
1818 
1819 	eth_hw_addr_set(netdev, addr->sa_data);
1820 
1821 	return 0;
1822 }
1823 
1824 #define UPDATE_VF_COUNTER(reg, name) \
1825 { \
1826 	u32 current_counter = er32(reg); \
1827 	if (current_counter < adapter->stats.last_##name) \
1828 		adapter->stats.name += 0x100000000LL; \
1829 	adapter->stats.last_##name = current_counter; \
1830 	adapter->stats.name &= 0xFFFFFFFF00000000LL; \
1831 	adapter->stats.name |= current_counter; \
1832 }
1833 
1834 /**
1835  * igbvf_update_stats - Update the board statistics counters
1836  * @adapter: board private structure
1837 **/
1838 void igbvf_update_stats(struct igbvf_adapter *adapter)
1839 {
1840 	struct e1000_hw *hw = &adapter->hw;
1841 	struct pci_dev *pdev = adapter->pdev;
1842 
1843 	/* Prevent stats update while adapter is being reset, link is down
1844 	 * or if the pci connection is down.
1845 	 */
1846 	if (adapter->link_speed == 0)
1847 		return;
1848 
1849 	if (test_bit(__IGBVF_RESETTING, &adapter->state))
1850 		return;
1851 
1852 	if (pci_channel_offline(pdev))
1853 		return;
1854 
1855 	UPDATE_VF_COUNTER(VFGPRC, gprc);
1856 	UPDATE_VF_COUNTER(VFGORC, gorc);
1857 	UPDATE_VF_COUNTER(VFGPTC, gptc);
1858 	UPDATE_VF_COUNTER(VFGOTC, gotc);
1859 	UPDATE_VF_COUNTER(VFMPRC, mprc);
1860 	UPDATE_VF_COUNTER(VFGOTLBC, gotlbc);
1861 	UPDATE_VF_COUNTER(VFGPTLBC, gptlbc);
1862 	UPDATE_VF_COUNTER(VFGORLBC, gorlbc);
1863 	UPDATE_VF_COUNTER(VFGPRLBC, gprlbc);
1864 
1865 	/* Fill out the OS statistics structure */
1866 	adapter->netdev->stats.multicast = adapter->stats.mprc;
1867 }
1868 
1869 static void igbvf_print_link_info(struct igbvf_adapter *adapter)
1870 {
1871 	dev_info(&adapter->pdev->dev, "Link is Up %d Mbps %s Duplex\n",
1872 		 adapter->link_speed,
1873 		 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
1874 }
1875 
1876 static bool igbvf_has_link(struct igbvf_adapter *adapter)
1877 {
1878 	struct e1000_hw *hw = &adapter->hw;
1879 	s32 ret_val = E1000_SUCCESS;
1880 	bool link_active;
1881 
1882 	/* If interface is down, stay link down */
1883 	if (test_bit(__IGBVF_DOWN, &adapter->state))
1884 		return false;
1885 
1886 	spin_lock_bh(&hw->mbx_lock);
1887 
1888 	ret_val = hw->mac.ops.check_for_link(hw);
1889 
1890 	spin_unlock_bh(&hw->mbx_lock);
1891 
1892 	link_active = !hw->mac.get_link_status;
1893 
1894 	/* if check for link returns error we will need to reset */
1895 	if (ret_val && time_after(jiffies, adapter->last_reset + (10 * HZ)))
1896 		schedule_work(&adapter->reset_task);
1897 
1898 	return link_active;
1899 }
1900 
1901 /**
1902  * igbvf_watchdog - Timer Call-back
1903  * @t: timer list pointer containing private struct
1904  **/
1905 static void igbvf_watchdog(struct timer_list *t)
1906 {
1907 	struct igbvf_adapter *adapter = from_timer(adapter, t, watchdog_timer);
1908 
1909 	/* Do the rest outside of interrupt context */
1910 	schedule_work(&adapter->watchdog_task);
1911 }
1912 
1913 static void igbvf_watchdog_task(struct work_struct *work)
1914 {
1915 	struct igbvf_adapter *adapter = container_of(work,
1916 						     struct igbvf_adapter,
1917 						     watchdog_task);
1918 	struct net_device *netdev = adapter->netdev;
1919 	struct e1000_mac_info *mac = &adapter->hw.mac;
1920 	struct igbvf_ring *tx_ring = adapter->tx_ring;
1921 	struct e1000_hw *hw = &adapter->hw;
1922 	u32 link;
1923 	int tx_pending = 0;
1924 
1925 	link = igbvf_has_link(adapter);
1926 
1927 	if (link) {
1928 		if (!netif_carrier_ok(netdev)) {
1929 			mac->ops.get_link_up_info(&adapter->hw,
1930 						  &adapter->link_speed,
1931 						  &adapter->link_duplex);
1932 			igbvf_print_link_info(adapter);
1933 
1934 			netif_carrier_on(netdev);
1935 			netif_wake_queue(netdev);
1936 		}
1937 	} else {
1938 		if (netif_carrier_ok(netdev)) {
1939 			adapter->link_speed = 0;
1940 			adapter->link_duplex = 0;
1941 			dev_info(&adapter->pdev->dev, "Link is Down\n");
1942 			netif_carrier_off(netdev);
1943 			netif_stop_queue(netdev);
1944 		}
1945 	}
1946 
1947 	if (netif_carrier_ok(netdev)) {
1948 		igbvf_update_stats(adapter);
1949 	} else {
1950 		tx_pending = (igbvf_desc_unused(tx_ring) + 1 <
1951 			      tx_ring->count);
1952 		if (tx_pending) {
1953 			/* We've lost link, so the controller stops DMA,
1954 			 * but we've got queued Tx work that's never going
1955 			 * to get done, so reset controller to flush Tx.
1956 			 * (Do the reset outside of interrupt context).
1957 			 */
1958 			adapter->tx_timeout_count++;
1959 			schedule_work(&adapter->reset_task);
1960 		}
1961 	}
1962 
1963 	/* Cause software interrupt to ensure Rx ring is cleaned */
1964 	ew32(EICS, adapter->rx_ring->eims_value);
1965 
1966 	/* Reset the timer */
1967 	if (!test_bit(__IGBVF_DOWN, &adapter->state))
1968 		mod_timer(&adapter->watchdog_timer,
1969 			  round_jiffies(jiffies + (2 * HZ)));
1970 }
1971 
1972 #define IGBVF_TX_FLAGS_CSUM		0x00000001
1973 #define IGBVF_TX_FLAGS_VLAN		0x00000002
1974 #define IGBVF_TX_FLAGS_TSO		0x00000004
1975 #define IGBVF_TX_FLAGS_IPV4		0x00000008
1976 #define IGBVF_TX_FLAGS_VLAN_MASK	0xffff0000
1977 #define IGBVF_TX_FLAGS_VLAN_SHIFT	16
1978 
1979 static void igbvf_tx_ctxtdesc(struct igbvf_ring *tx_ring, u32 vlan_macip_lens,
1980 			      u32 type_tucmd, u32 mss_l4len_idx)
1981 {
1982 	struct e1000_adv_tx_context_desc *context_desc;
1983 	struct igbvf_buffer *buffer_info;
1984 	u16 i = tx_ring->next_to_use;
1985 
1986 	context_desc = IGBVF_TX_CTXTDESC_ADV(*tx_ring, i);
1987 	buffer_info = &tx_ring->buffer_info[i];
1988 
1989 	i++;
1990 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1991 
1992 	/* set bits to identify this as an advanced context descriptor */
1993 	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
1994 
1995 	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
1996 	context_desc->seqnum_seed	= 0;
1997 	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
1998 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
1999 
2000 	buffer_info->time_stamp = jiffies;
2001 	buffer_info->dma = 0;
2002 }
2003 
2004 static int igbvf_tso(struct igbvf_ring *tx_ring,
2005 		     struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2006 {
2007 	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
2008 	union {
2009 		struct iphdr *v4;
2010 		struct ipv6hdr *v6;
2011 		unsigned char *hdr;
2012 	} ip;
2013 	union {
2014 		struct tcphdr *tcp;
2015 		unsigned char *hdr;
2016 	} l4;
2017 	u32 paylen, l4_offset;
2018 	int err;
2019 
2020 	if (skb->ip_summed != CHECKSUM_PARTIAL)
2021 		return 0;
2022 
2023 	if (!skb_is_gso(skb))
2024 		return 0;
2025 
2026 	err = skb_cow_head(skb, 0);
2027 	if (err < 0)
2028 		return err;
2029 
2030 	ip.hdr = skb_network_header(skb);
2031 	l4.hdr = skb_checksum_start(skb);
2032 
2033 	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2034 	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2035 
2036 	/* initialize outer IP header fields */
2037 	if (ip.v4->version == 4) {
2038 		unsigned char *csum_start = skb_checksum_start(skb);
2039 		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
2040 
2041 		/* IP header will have to cancel out any data that
2042 		 * is not a part of the outer IP header
2043 		 */
2044 		ip.v4->check = csum_fold(csum_partial(trans_start,
2045 						      csum_start - trans_start,
2046 						      0));
2047 		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
2048 
2049 		ip.v4->tot_len = 0;
2050 	} else {
2051 		ip.v6->payload_len = 0;
2052 	}
2053 
2054 	/* determine offset of inner transport header */
2055 	l4_offset = l4.hdr - skb->data;
2056 
2057 	/* compute length of segmentation header */
2058 	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
2059 
2060 	/* remove payload length from inner checksum */
2061 	paylen = skb->len - l4_offset;
2062 	csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
2063 
2064 	/* MSS L4LEN IDX */
2065 	mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
2066 	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
2067 
2068 	/* VLAN MACLEN IPLEN */
2069 	vlan_macip_lens = l4.hdr - ip.hdr;
2070 	vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
2071 	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2072 
2073 	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
2074 
2075 	return 1;
2076 }
2077 
2078 static bool igbvf_tx_csum(struct igbvf_ring *tx_ring, struct sk_buff *skb,
2079 			  u32 tx_flags, __be16 protocol)
2080 {
2081 	u32 vlan_macip_lens = 0;
2082 	u32 type_tucmd = 0;
2083 
2084 	if (skb->ip_summed != CHECKSUM_PARTIAL) {
2085 csum_failed:
2086 		if (!(tx_flags & IGBVF_TX_FLAGS_VLAN))
2087 			return false;
2088 		goto no_csum;
2089 	}
2090 
2091 	switch (skb->csum_offset) {
2092 	case offsetof(struct tcphdr, check):
2093 		type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2094 		fallthrough;
2095 	case offsetof(struct udphdr, check):
2096 		break;
2097 	case offsetof(struct sctphdr, checksum):
2098 		/* validate that this is actually an SCTP request */
2099 		if (skb_csum_is_sctp(skb)) {
2100 			type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
2101 			break;
2102 		}
2103 		fallthrough;
2104 	default:
2105 		skb_checksum_help(skb);
2106 		goto csum_failed;
2107 	}
2108 
2109 	vlan_macip_lens = skb_checksum_start_offset(skb) -
2110 			  skb_network_offset(skb);
2111 no_csum:
2112 	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
2113 	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2114 
2115 	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0);
2116 	return true;
2117 }
2118 
2119 static int igbvf_maybe_stop_tx(struct net_device *netdev, int size)
2120 {
2121 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2122 
2123 	/* there is enough descriptors then we don't need to worry  */
2124 	if (igbvf_desc_unused(adapter->tx_ring) >= size)
2125 		return 0;
2126 
2127 	netif_stop_queue(netdev);
2128 
2129 	/* Herbert's original patch had:
2130 	 *  smp_mb__after_netif_stop_queue();
2131 	 * but since that doesn't exist yet, just open code it.
2132 	 */
2133 	smp_mb();
2134 
2135 	/* We need to check again just in case room has been made available */
2136 	if (igbvf_desc_unused(adapter->tx_ring) < size)
2137 		return -EBUSY;
2138 
2139 	netif_wake_queue(netdev);
2140 
2141 	++adapter->restart_queue;
2142 	return 0;
2143 }
2144 
2145 #define IGBVF_MAX_TXD_PWR	16
2146 #define IGBVF_MAX_DATA_PER_TXD	(1u << IGBVF_MAX_TXD_PWR)
2147 
2148 static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter,
2149 				   struct igbvf_ring *tx_ring,
2150 				   struct sk_buff *skb)
2151 {
2152 	struct igbvf_buffer *buffer_info;
2153 	struct pci_dev *pdev = adapter->pdev;
2154 	unsigned int len = skb_headlen(skb);
2155 	unsigned int count = 0, i;
2156 	unsigned int f;
2157 
2158 	i = tx_ring->next_to_use;
2159 
2160 	buffer_info = &tx_ring->buffer_info[i];
2161 	BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2162 	buffer_info->length = len;
2163 	/* set time_stamp *before* dma to help avoid a possible race */
2164 	buffer_info->time_stamp = jiffies;
2165 	buffer_info->mapped_as_page = false;
2166 	buffer_info->dma = dma_map_single(&pdev->dev, skb->data, len,
2167 					  DMA_TO_DEVICE);
2168 	if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2169 		goto dma_error;
2170 
2171 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2172 		const skb_frag_t *frag;
2173 
2174 		count++;
2175 		i++;
2176 		if (i == tx_ring->count)
2177 			i = 0;
2178 
2179 		frag = &skb_shinfo(skb)->frags[f];
2180 		len = skb_frag_size(frag);
2181 
2182 		buffer_info = &tx_ring->buffer_info[i];
2183 		BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2184 		buffer_info->length = len;
2185 		buffer_info->time_stamp = jiffies;
2186 		buffer_info->mapped_as_page = true;
2187 		buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 0, len,
2188 						    DMA_TO_DEVICE);
2189 		if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2190 			goto dma_error;
2191 	}
2192 
2193 	tx_ring->buffer_info[i].skb = skb;
2194 
2195 	return ++count;
2196 
2197 dma_error:
2198 	dev_err(&pdev->dev, "TX DMA map failed\n");
2199 
2200 	/* clear timestamp and dma mappings for failed buffer_info mapping */
2201 	buffer_info->dma = 0;
2202 	buffer_info->time_stamp = 0;
2203 	buffer_info->length = 0;
2204 	buffer_info->mapped_as_page = false;
2205 	if (count)
2206 		count--;
2207 
2208 	/* clear timestamp and dma mappings for remaining portion of packet */
2209 	while (count--) {
2210 		if (i == 0)
2211 			i += tx_ring->count;
2212 		i--;
2213 		buffer_info = &tx_ring->buffer_info[i];
2214 		igbvf_put_txbuf(adapter, buffer_info);
2215 	}
2216 
2217 	return 0;
2218 }
2219 
2220 static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,
2221 				      struct igbvf_ring *tx_ring,
2222 				      int tx_flags, int count,
2223 				      unsigned int first, u32 paylen,
2224 				      u8 hdr_len)
2225 {
2226 	union e1000_adv_tx_desc *tx_desc = NULL;
2227 	struct igbvf_buffer *buffer_info;
2228 	u32 olinfo_status = 0, cmd_type_len;
2229 	unsigned int i;
2230 
2231 	cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
2232 			E1000_ADVTXD_DCMD_DEXT);
2233 
2234 	if (tx_flags & IGBVF_TX_FLAGS_VLAN)
2235 		cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2236 
2237 	if (tx_flags & IGBVF_TX_FLAGS_TSO) {
2238 		cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
2239 
2240 		/* insert tcp checksum */
2241 		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2242 
2243 		/* insert ip checksum */
2244 		if (tx_flags & IGBVF_TX_FLAGS_IPV4)
2245 			olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2246 
2247 	} else if (tx_flags & IGBVF_TX_FLAGS_CSUM) {
2248 		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2249 	}
2250 
2251 	olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
2252 
2253 	i = tx_ring->next_to_use;
2254 	while (count--) {
2255 		buffer_info = &tx_ring->buffer_info[i];
2256 		tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
2257 		tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
2258 		tx_desc->read.cmd_type_len =
2259 			 cpu_to_le32(cmd_type_len | buffer_info->length);
2260 		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2261 		i++;
2262 		if (i == tx_ring->count)
2263 			i = 0;
2264 	}
2265 
2266 	tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
2267 	/* Force memory writes to complete before letting h/w
2268 	 * know there are new descriptors to fetch.  (Only
2269 	 * applicable for weak-ordered memory model archs,
2270 	 * such as IA-64).
2271 	 */
2272 	wmb();
2273 
2274 	tx_ring->buffer_info[first].next_to_watch = tx_desc;
2275 	tx_ring->next_to_use = i;
2276 	writel(i, adapter->hw.hw_addr + tx_ring->tail);
2277 }
2278 
2279 static netdev_tx_t igbvf_xmit_frame_ring_adv(struct sk_buff *skb,
2280 					     struct net_device *netdev,
2281 					     struct igbvf_ring *tx_ring)
2282 {
2283 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2284 	unsigned int first, tx_flags = 0;
2285 	u8 hdr_len = 0;
2286 	int count = 0;
2287 	int tso = 0;
2288 	__be16 protocol = vlan_get_protocol(skb);
2289 
2290 	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2291 		dev_kfree_skb_any(skb);
2292 		return NETDEV_TX_OK;
2293 	}
2294 
2295 	if (skb->len <= 0) {
2296 		dev_kfree_skb_any(skb);
2297 		return NETDEV_TX_OK;
2298 	}
2299 
2300 	/* need: count + 4 desc gap to keep tail from touching
2301 	 *       + 2 desc gap to keep tail from touching head,
2302 	 *       + 1 desc for skb->data,
2303 	 *       + 1 desc for context descriptor,
2304 	 * head, otherwise try next time
2305 	 */
2306 	if (igbvf_maybe_stop_tx(netdev, skb_shinfo(skb)->nr_frags + 4)) {
2307 		/* this is a hard error */
2308 		return NETDEV_TX_BUSY;
2309 	}
2310 
2311 	if (skb_vlan_tag_present(skb)) {
2312 		tx_flags |= IGBVF_TX_FLAGS_VLAN;
2313 		tx_flags |= (skb_vlan_tag_get(skb) <<
2314 			     IGBVF_TX_FLAGS_VLAN_SHIFT);
2315 	}
2316 
2317 	if (protocol == htons(ETH_P_IP))
2318 		tx_flags |= IGBVF_TX_FLAGS_IPV4;
2319 
2320 	first = tx_ring->next_to_use;
2321 
2322 	tso = igbvf_tso(tx_ring, skb, tx_flags, &hdr_len);
2323 	if (unlikely(tso < 0)) {
2324 		dev_kfree_skb_any(skb);
2325 		return NETDEV_TX_OK;
2326 	}
2327 
2328 	if (tso)
2329 		tx_flags |= IGBVF_TX_FLAGS_TSO;
2330 	else if (igbvf_tx_csum(tx_ring, skb, tx_flags, protocol) &&
2331 		 (skb->ip_summed == CHECKSUM_PARTIAL))
2332 		tx_flags |= IGBVF_TX_FLAGS_CSUM;
2333 
2334 	/* count reflects descriptors mapped, if 0 then mapping error
2335 	 * has occurred and we need to rewind the descriptor queue
2336 	 */
2337 	count = igbvf_tx_map_adv(adapter, tx_ring, skb);
2338 
2339 	if (count) {
2340 		igbvf_tx_queue_adv(adapter, tx_ring, tx_flags, count,
2341 				   first, skb->len, hdr_len);
2342 		/* Make sure there is space in the ring for the next send. */
2343 		igbvf_maybe_stop_tx(netdev, MAX_SKB_FRAGS + 4);
2344 	} else {
2345 		dev_kfree_skb_any(skb);
2346 		tx_ring->buffer_info[first].time_stamp = 0;
2347 		tx_ring->next_to_use = first;
2348 	}
2349 
2350 	return NETDEV_TX_OK;
2351 }
2352 
2353 static netdev_tx_t igbvf_xmit_frame(struct sk_buff *skb,
2354 				    struct net_device *netdev)
2355 {
2356 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2357 	struct igbvf_ring *tx_ring;
2358 
2359 	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2360 		dev_kfree_skb_any(skb);
2361 		return NETDEV_TX_OK;
2362 	}
2363 
2364 	tx_ring = &adapter->tx_ring[0];
2365 
2366 	return igbvf_xmit_frame_ring_adv(skb, netdev, tx_ring);
2367 }
2368 
2369 /**
2370  * igbvf_tx_timeout - Respond to a Tx Hang
2371  * @netdev: network interface device structure
2372  * @txqueue: queue timing out (unused)
2373  **/
2374 static void igbvf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
2375 {
2376 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2377 
2378 	/* Do the reset outside of interrupt context */
2379 	adapter->tx_timeout_count++;
2380 	schedule_work(&adapter->reset_task);
2381 }
2382 
2383 static void igbvf_reset_task(struct work_struct *work)
2384 {
2385 	struct igbvf_adapter *adapter;
2386 
2387 	adapter = container_of(work, struct igbvf_adapter, reset_task);
2388 
2389 	igbvf_reinit_locked(adapter);
2390 }
2391 
2392 /**
2393  * igbvf_change_mtu - Change the Maximum Transfer Unit
2394  * @netdev: network interface device structure
2395  * @new_mtu: new value for maximum frame size
2396  *
2397  * Returns 0 on success, negative on failure
2398  **/
2399 static int igbvf_change_mtu(struct net_device *netdev, int new_mtu)
2400 {
2401 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2402 	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2403 
2404 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2405 		usleep_range(1000, 2000);
2406 	/* igbvf_down has a dependency on max_frame_size */
2407 	adapter->max_frame_size = max_frame;
2408 	if (netif_running(netdev))
2409 		igbvf_down(adapter);
2410 
2411 	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
2412 	 * means we reserve 2 more, this pushes us to allocate from the next
2413 	 * larger slab size.
2414 	 * i.e. RXBUFFER_2048 --> size-4096 slab
2415 	 * However with the new *_jumbo_rx* routines, jumbo receives will use
2416 	 * fragmented skbs
2417 	 */
2418 
2419 	if (max_frame <= 1024)
2420 		adapter->rx_buffer_len = 1024;
2421 	else if (max_frame <= 2048)
2422 		adapter->rx_buffer_len = 2048;
2423 	else
2424 #if (PAGE_SIZE / 2) > 16384
2425 		adapter->rx_buffer_len = 16384;
2426 #else
2427 		adapter->rx_buffer_len = PAGE_SIZE / 2;
2428 #endif
2429 
2430 	/* adjust allocation if LPE protects us, and we aren't using SBP */
2431 	if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
2432 	    (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
2433 		adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN +
2434 					 ETH_FCS_LEN;
2435 
2436 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
2437 		   netdev->mtu, new_mtu);
2438 	netdev->mtu = new_mtu;
2439 
2440 	if (netif_running(netdev))
2441 		igbvf_up(adapter);
2442 	else
2443 		igbvf_reset(adapter);
2444 
2445 	clear_bit(__IGBVF_RESETTING, &adapter->state);
2446 
2447 	return 0;
2448 }
2449 
2450 static int igbvf_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2451 {
2452 	switch (cmd) {
2453 	default:
2454 		return -EOPNOTSUPP;
2455 	}
2456 }
2457 
2458 static int igbvf_suspend(struct device *dev_d)
2459 {
2460 	struct net_device *netdev = dev_get_drvdata(dev_d);
2461 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2462 
2463 	netif_device_detach(netdev);
2464 
2465 	if (netif_running(netdev)) {
2466 		WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
2467 		igbvf_down(adapter);
2468 		igbvf_free_irq(adapter);
2469 	}
2470 
2471 	return 0;
2472 }
2473 
2474 static int __maybe_unused igbvf_resume(struct device *dev_d)
2475 {
2476 	struct pci_dev *pdev = to_pci_dev(dev_d);
2477 	struct net_device *netdev = pci_get_drvdata(pdev);
2478 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2479 	u32 err;
2480 
2481 	pci_set_master(pdev);
2482 
2483 	if (netif_running(netdev)) {
2484 		err = igbvf_request_irq(adapter);
2485 		if (err)
2486 			return err;
2487 	}
2488 
2489 	igbvf_reset(adapter);
2490 
2491 	if (netif_running(netdev))
2492 		igbvf_up(adapter);
2493 
2494 	netif_device_attach(netdev);
2495 
2496 	return 0;
2497 }
2498 
2499 static void igbvf_shutdown(struct pci_dev *pdev)
2500 {
2501 	igbvf_suspend(&pdev->dev);
2502 }
2503 
2504 #ifdef CONFIG_NET_POLL_CONTROLLER
2505 /* Polling 'interrupt' - used by things like netconsole to send skbs
2506  * without having to re-enable interrupts. It's not called while
2507  * the interrupt routine is executing.
2508  */
2509 static void igbvf_netpoll(struct net_device *netdev)
2510 {
2511 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2512 
2513 	disable_irq(adapter->pdev->irq);
2514 
2515 	igbvf_clean_tx_irq(adapter->tx_ring);
2516 
2517 	enable_irq(adapter->pdev->irq);
2518 }
2519 #endif
2520 
2521 /**
2522  * igbvf_io_error_detected - called when PCI error is detected
2523  * @pdev: Pointer to PCI device
2524  * @state: The current pci connection state
2525  *
2526  * This function is called after a PCI bus error affecting
2527  * this device has been detected.
2528  */
2529 static pci_ers_result_t igbvf_io_error_detected(struct pci_dev *pdev,
2530 						pci_channel_state_t state)
2531 {
2532 	struct net_device *netdev = pci_get_drvdata(pdev);
2533 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2534 
2535 	netif_device_detach(netdev);
2536 
2537 	if (state == pci_channel_io_perm_failure)
2538 		return PCI_ERS_RESULT_DISCONNECT;
2539 
2540 	if (netif_running(netdev))
2541 		igbvf_down(adapter);
2542 	pci_disable_device(pdev);
2543 
2544 	/* Request a slot reset. */
2545 	return PCI_ERS_RESULT_NEED_RESET;
2546 }
2547 
2548 /**
2549  * igbvf_io_slot_reset - called after the pci bus has been reset.
2550  * @pdev: Pointer to PCI device
2551  *
2552  * Restart the card from scratch, as if from a cold-boot. Implementation
2553  * resembles the first-half of the igbvf_resume routine.
2554  */
2555 static pci_ers_result_t igbvf_io_slot_reset(struct pci_dev *pdev)
2556 {
2557 	struct net_device *netdev = pci_get_drvdata(pdev);
2558 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2559 
2560 	if (pci_enable_device_mem(pdev)) {
2561 		dev_err(&pdev->dev,
2562 			"Cannot re-enable PCI device after reset.\n");
2563 		return PCI_ERS_RESULT_DISCONNECT;
2564 	}
2565 	pci_set_master(pdev);
2566 
2567 	igbvf_reset(adapter);
2568 
2569 	return PCI_ERS_RESULT_RECOVERED;
2570 }
2571 
2572 /**
2573  * igbvf_io_resume - called when traffic can start flowing again.
2574  * @pdev: Pointer to PCI device
2575  *
2576  * This callback is called when the error recovery driver tells us that
2577  * its OK to resume normal operation. Implementation resembles the
2578  * second-half of the igbvf_resume routine.
2579  */
2580 static void igbvf_io_resume(struct pci_dev *pdev)
2581 {
2582 	struct net_device *netdev = pci_get_drvdata(pdev);
2583 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2584 
2585 	if (netif_running(netdev)) {
2586 		if (igbvf_up(adapter)) {
2587 			dev_err(&pdev->dev,
2588 				"can't bring device back up after reset\n");
2589 			return;
2590 		}
2591 	}
2592 
2593 	netif_device_attach(netdev);
2594 }
2595 
2596 /**
2597  * igbvf_io_prepare - prepare device driver for PCI reset
2598  * @pdev: PCI device information struct
2599  */
2600 static void igbvf_io_prepare(struct pci_dev *pdev)
2601 {
2602 	struct net_device *netdev = pci_get_drvdata(pdev);
2603 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2604 
2605 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2606 		usleep_range(1000, 2000);
2607 	igbvf_down(adapter);
2608 }
2609 
2610 /**
2611  * igbvf_io_reset_done - PCI reset done, device driver reset can begin
2612  * @pdev: PCI device information struct
2613  */
2614 static void igbvf_io_reset_done(struct pci_dev *pdev)
2615 {
2616 	struct net_device *netdev = pci_get_drvdata(pdev);
2617 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2618 
2619 	igbvf_up(adapter);
2620 	clear_bit(__IGBVF_RESETTING, &adapter->state);
2621 }
2622 
2623 static void igbvf_print_device_info(struct igbvf_adapter *adapter)
2624 {
2625 	struct e1000_hw *hw = &adapter->hw;
2626 	struct net_device *netdev = adapter->netdev;
2627 	struct pci_dev *pdev = adapter->pdev;
2628 
2629 	if (hw->mac.type == e1000_vfadapt_i350)
2630 		dev_info(&pdev->dev, "Intel(R) I350 Virtual Function\n");
2631 	else
2632 		dev_info(&pdev->dev, "Intel(R) 82576 Virtual Function\n");
2633 	dev_info(&pdev->dev, "Address: %pM\n", netdev->dev_addr);
2634 }
2635 
2636 static int igbvf_set_features(struct net_device *netdev,
2637 			      netdev_features_t features)
2638 {
2639 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2640 
2641 	if (features & NETIF_F_RXCSUM)
2642 		adapter->flags &= ~IGBVF_FLAG_RX_CSUM_DISABLED;
2643 	else
2644 		adapter->flags |= IGBVF_FLAG_RX_CSUM_DISABLED;
2645 
2646 	return 0;
2647 }
2648 
2649 #define IGBVF_MAX_MAC_HDR_LEN		127
2650 #define IGBVF_MAX_NETWORK_HDR_LEN	511
2651 
2652 static netdev_features_t
2653 igbvf_features_check(struct sk_buff *skb, struct net_device *dev,
2654 		     netdev_features_t features)
2655 {
2656 	unsigned int network_hdr_len, mac_hdr_len;
2657 
2658 	/* Make certain the headers can be described by a context descriptor */
2659 	mac_hdr_len = skb_network_header(skb) - skb->data;
2660 	if (unlikely(mac_hdr_len > IGBVF_MAX_MAC_HDR_LEN))
2661 		return features & ~(NETIF_F_HW_CSUM |
2662 				    NETIF_F_SCTP_CRC |
2663 				    NETIF_F_HW_VLAN_CTAG_TX |
2664 				    NETIF_F_TSO |
2665 				    NETIF_F_TSO6);
2666 
2667 	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2668 	if (unlikely(network_hdr_len >  IGBVF_MAX_NETWORK_HDR_LEN))
2669 		return features & ~(NETIF_F_HW_CSUM |
2670 				    NETIF_F_SCTP_CRC |
2671 				    NETIF_F_TSO |
2672 				    NETIF_F_TSO6);
2673 
2674 	/* We can only support IPV4 TSO in tunnels if we can mangle the
2675 	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2676 	 */
2677 	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2678 		features &= ~NETIF_F_TSO;
2679 
2680 	return features;
2681 }
2682 
2683 static const struct net_device_ops igbvf_netdev_ops = {
2684 	.ndo_open		= igbvf_open,
2685 	.ndo_stop		= igbvf_close,
2686 	.ndo_start_xmit		= igbvf_xmit_frame,
2687 	.ndo_set_rx_mode	= igbvf_set_rx_mode,
2688 	.ndo_set_mac_address	= igbvf_set_mac,
2689 	.ndo_change_mtu		= igbvf_change_mtu,
2690 	.ndo_eth_ioctl		= igbvf_ioctl,
2691 	.ndo_tx_timeout		= igbvf_tx_timeout,
2692 	.ndo_vlan_rx_add_vid	= igbvf_vlan_rx_add_vid,
2693 	.ndo_vlan_rx_kill_vid	= igbvf_vlan_rx_kill_vid,
2694 #ifdef CONFIG_NET_POLL_CONTROLLER
2695 	.ndo_poll_controller	= igbvf_netpoll,
2696 #endif
2697 	.ndo_set_features	= igbvf_set_features,
2698 	.ndo_features_check	= igbvf_features_check,
2699 };
2700 
2701 /**
2702  * igbvf_probe - Device Initialization Routine
2703  * @pdev: PCI device information struct
2704  * @ent: entry in igbvf_pci_tbl
2705  *
2706  * Returns 0 on success, negative on failure
2707  *
2708  * igbvf_probe initializes an adapter identified by a pci_dev structure.
2709  * The OS initialization, configuring of the adapter private structure,
2710  * and a hardware reset occur.
2711  **/
2712 static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2713 {
2714 	struct net_device *netdev;
2715 	struct igbvf_adapter *adapter;
2716 	struct e1000_hw *hw;
2717 	const struct igbvf_info *ei = igbvf_info_tbl[ent->driver_data];
2718 	static int cards_found;
2719 	int err;
2720 
2721 	err = pci_enable_device_mem(pdev);
2722 	if (err)
2723 		return err;
2724 
2725 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2726 	if (err) {
2727 		dev_err(&pdev->dev,
2728 			"No usable DMA configuration, aborting\n");
2729 		goto err_dma;
2730 	}
2731 
2732 	err = pci_request_regions(pdev, igbvf_driver_name);
2733 	if (err)
2734 		goto err_pci_reg;
2735 
2736 	pci_set_master(pdev);
2737 
2738 	err = -ENOMEM;
2739 	netdev = alloc_etherdev(sizeof(struct igbvf_adapter));
2740 	if (!netdev)
2741 		goto err_alloc_etherdev;
2742 
2743 	SET_NETDEV_DEV(netdev, &pdev->dev);
2744 
2745 	pci_set_drvdata(pdev, netdev);
2746 	adapter = netdev_priv(netdev);
2747 	hw = &adapter->hw;
2748 	adapter->netdev = netdev;
2749 	adapter->pdev = pdev;
2750 	adapter->ei = ei;
2751 	adapter->pba = ei->pba;
2752 	adapter->flags = ei->flags;
2753 	adapter->hw.back = adapter;
2754 	adapter->hw.mac.type = ei->mac;
2755 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2756 
2757 	/* PCI config space info */
2758 
2759 	hw->vendor_id = pdev->vendor;
2760 	hw->device_id = pdev->device;
2761 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
2762 	hw->subsystem_device_id = pdev->subsystem_device;
2763 	hw->revision_id = pdev->revision;
2764 
2765 	err = -EIO;
2766 	adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0),
2767 				      pci_resource_len(pdev, 0));
2768 
2769 	if (!adapter->hw.hw_addr)
2770 		goto err_ioremap;
2771 
2772 	if (ei->get_variants) {
2773 		err = ei->get_variants(adapter);
2774 		if (err)
2775 			goto err_get_variants;
2776 	}
2777 
2778 	/* setup adapter struct */
2779 	err = igbvf_sw_init(adapter);
2780 	if (err)
2781 		goto err_sw_init;
2782 
2783 	/* construct the net_device struct */
2784 	netdev->netdev_ops = &igbvf_netdev_ops;
2785 
2786 	igbvf_set_ethtool_ops(netdev);
2787 	netdev->watchdog_timeo = 5 * HZ;
2788 	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
2789 
2790 	adapter->bd_number = cards_found++;
2791 
2792 	netdev->hw_features = NETIF_F_SG |
2793 			      NETIF_F_TSO |
2794 			      NETIF_F_TSO6 |
2795 			      NETIF_F_RXCSUM |
2796 			      NETIF_F_HW_CSUM |
2797 			      NETIF_F_SCTP_CRC;
2798 
2799 #define IGBVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
2800 				    NETIF_F_GSO_GRE_CSUM | \
2801 				    NETIF_F_GSO_IPXIP4 | \
2802 				    NETIF_F_GSO_IPXIP6 | \
2803 				    NETIF_F_GSO_UDP_TUNNEL | \
2804 				    NETIF_F_GSO_UDP_TUNNEL_CSUM)
2805 
2806 	netdev->gso_partial_features = IGBVF_GSO_PARTIAL_FEATURES;
2807 	netdev->hw_features |= NETIF_F_GSO_PARTIAL |
2808 			       IGBVF_GSO_PARTIAL_FEATURES;
2809 
2810 	netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;
2811 
2812 	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
2813 	netdev->mpls_features |= NETIF_F_HW_CSUM;
2814 	netdev->hw_enc_features |= netdev->vlan_features;
2815 
2816 	/* set this bit last since it cannot be part of vlan_features */
2817 	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2818 			    NETIF_F_HW_VLAN_CTAG_RX |
2819 			    NETIF_F_HW_VLAN_CTAG_TX;
2820 
2821 	/* MTU range: 68 - 9216 */
2822 	netdev->min_mtu = ETH_MIN_MTU;
2823 	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
2824 
2825 	spin_lock_bh(&hw->mbx_lock);
2826 
2827 	/*reset the controller to put the device in a known good state */
2828 	err = hw->mac.ops.reset_hw(hw);
2829 	if (err) {
2830 		dev_info(&pdev->dev,
2831 			 "PF still in reset state. Is the PF interface up?\n");
2832 	} else {
2833 		err = hw->mac.ops.read_mac_addr(hw);
2834 		if (err)
2835 			dev_info(&pdev->dev, "Error reading MAC address.\n");
2836 		else if (is_zero_ether_addr(adapter->hw.mac.addr))
2837 			dev_info(&pdev->dev,
2838 				 "MAC address not assigned by administrator.\n");
2839 		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2840 	}
2841 
2842 	spin_unlock_bh(&hw->mbx_lock);
2843 
2844 	if (!is_valid_ether_addr(netdev->dev_addr)) {
2845 		dev_info(&pdev->dev, "Assigning random MAC address.\n");
2846 		eth_hw_addr_random(netdev);
2847 		memcpy(adapter->hw.mac.addr, netdev->dev_addr,
2848 		       netdev->addr_len);
2849 	}
2850 
2851 	timer_setup(&adapter->watchdog_timer, igbvf_watchdog, 0);
2852 
2853 	INIT_WORK(&adapter->reset_task, igbvf_reset_task);
2854 	INIT_WORK(&adapter->watchdog_task, igbvf_watchdog_task);
2855 
2856 	/* ring size defaults */
2857 	adapter->rx_ring->count = 1024;
2858 	adapter->tx_ring->count = 1024;
2859 
2860 	/* reset the hardware with the new settings */
2861 	igbvf_reset(adapter);
2862 
2863 	/* set hardware-specific flags */
2864 	if (adapter->hw.mac.type == e1000_vfadapt_i350)
2865 		adapter->flags |= IGBVF_FLAG_RX_LB_VLAN_BSWAP;
2866 
2867 	strcpy(netdev->name, "eth%d");
2868 	err = register_netdev(netdev);
2869 	if (err)
2870 		goto err_hw_init;
2871 
2872 	/* tell the stack to leave us alone until igbvf_open() is called */
2873 	netif_carrier_off(netdev);
2874 	netif_stop_queue(netdev);
2875 
2876 	igbvf_print_device_info(adapter);
2877 
2878 	igbvf_initialize_last_counter_stats(adapter);
2879 
2880 	return 0;
2881 
2882 err_hw_init:
2883 	netif_napi_del(&adapter->rx_ring->napi);
2884 	kfree(adapter->tx_ring);
2885 	kfree(adapter->rx_ring);
2886 err_sw_init:
2887 	igbvf_reset_interrupt_capability(adapter);
2888 err_get_variants:
2889 	iounmap(adapter->hw.hw_addr);
2890 err_ioremap:
2891 	free_netdev(netdev);
2892 err_alloc_etherdev:
2893 	pci_release_regions(pdev);
2894 err_pci_reg:
2895 err_dma:
2896 	pci_disable_device(pdev);
2897 	return err;
2898 }
2899 
2900 /**
2901  * igbvf_remove - Device Removal Routine
2902  * @pdev: PCI device information struct
2903  *
2904  * igbvf_remove is called by the PCI subsystem to alert the driver
2905  * that it should release a PCI device.  The could be caused by a
2906  * Hot-Plug event, or because the driver is going to be removed from
2907  * memory.
2908  **/
2909 static void igbvf_remove(struct pci_dev *pdev)
2910 {
2911 	struct net_device *netdev = pci_get_drvdata(pdev);
2912 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2913 	struct e1000_hw *hw = &adapter->hw;
2914 
2915 	/* The watchdog timer may be rescheduled, so explicitly
2916 	 * disable it from being rescheduled.
2917 	 */
2918 	set_bit(__IGBVF_DOWN, &adapter->state);
2919 	del_timer_sync(&adapter->watchdog_timer);
2920 
2921 	cancel_work_sync(&adapter->reset_task);
2922 	cancel_work_sync(&adapter->watchdog_task);
2923 
2924 	unregister_netdev(netdev);
2925 
2926 	igbvf_reset_interrupt_capability(adapter);
2927 
2928 	/* it is important to delete the NAPI struct prior to freeing the
2929 	 * Rx ring so that you do not end up with null pointer refs
2930 	 */
2931 	netif_napi_del(&adapter->rx_ring->napi);
2932 	kfree(adapter->tx_ring);
2933 	kfree(adapter->rx_ring);
2934 
2935 	iounmap(hw->hw_addr);
2936 	if (hw->flash_address)
2937 		iounmap(hw->flash_address);
2938 	pci_release_regions(pdev);
2939 
2940 	free_netdev(netdev);
2941 
2942 	pci_disable_device(pdev);
2943 }
2944 
2945 /* PCI Error Recovery (ERS) */
2946 static const struct pci_error_handlers igbvf_err_handler = {
2947 	.error_detected = igbvf_io_error_detected,
2948 	.slot_reset = igbvf_io_slot_reset,
2949 	.resume = igbvf_io_resume,
2950 	.reset_prepare = igbvf_io_prepare,
2951 	.reset_done = igbvf_io_reset_done,
2952 };
2953 
2954 static const struct pci_device_id igbvf_pci_tbl[] = {
2955 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), board_vf },
2956 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), board_i350_vf },
2957 	{ } /* terminate list */
2958 };
2959 MODULE_DEVICE_TABLE(pci, igbvf_pci_tbl);
2960 
2961 static SIMPLE_DEV_PM_OPS(igbvf_pm_ops, igbvf_suspend, igbvf_resume);
2962 
2963 /* PCI Device API Driver */
2964 static struct pci_driver igbvf_driver = {
2965 	.name		= igbvf_driver_name,
2966 	.id_table	= igbvf_pci_tbl,
2967 	.probe		= igbvf_probe,
2968 	.remove		= igbvf_remove,
2969 	.driver.pm	= &igbvf_pm_ops,
2970 	.shutdown	= igbvf_shutdown,
2971 	.err_handler	= &igbvf_err_handler
2972 };
2973 
2974 /**
2975  * igbvf_init_module - Driver Registration Routine
2976  *
2977  * igbvf_init_module is the first routine called when the driver is
2978  * loaded. All it does is register with the PCI subsystem.
2979  **/
2980 static int __init igbvf_init_module(void)
2981 {
2982 	int ret;
2983 
2984 	pr_info("%s\n", igbvf_driver_string);
2985 	pr_info("%s\n", igbvf_copyright);
2986 
2987 	ret = pci_register_driver(&igbvf_driver);
2988 
2989 	return ret;
2990 }
2991 module_init(igbvf_init_module);
2992 
2993 /**
2994  * igbvf_exit_module - Driver Exit Cleanup Routine
2995  *
2996  * igbvf_exit_module is called just before the driver is removed
2997  * from memory.
2998  **/
2999 static void __exit igbvf_exit_module(void)
3000 {
3001 	pci_unregister_driver(&igbvf_driver);
3002 }
3003 module_exit(igbvf_exit_module);
3004 
3005 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
3006 MODULE_DESCRIPTION("Intel(R) Gigabit Virtual Function Network Driver");
3007 MODULE_LICENSE("GPL v2");
3008 
3009 /* netdev.c */
3010