1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2009 - 2018 Intel Corporation. */
3 
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5 
6 #include <linux/bitfield.h>
7 #include <linux/delay.h>
8 #include <linux/ethtool.h>
9 #include <linux/if_vlan.h>
10 #include <linux/init.h>
11 #include <linux/ipv6.h>
12 #include <linux/mii.h>
13 #include <linux/module.h>
14 #include <linux/netdevice.h>
15 #include <linux/pagemap.h>
16 #include <linux/pci.h>
17 #include <linux/prefetch.h>
18 #include <linux/sctp.h>
19 #include <linux/slab.h>
20 #include <linux/tcp.h>
21 #include <linux/types.h>
22 #include <linux/vmalloc.h>
23 #include <net/checksum.h>
24 #include <net/ip6_checksum.h>
25 #include "igbvf.h"
26 
27 char igbvf_driver_name[] = "igbvf";
28 static const char igbvf_driver_string[] =
29 		  "Intel(R) Gigabit Virtual Function Network Driver";
30 static const char igbvf_copyright[] =
31 		  "Copyright (c) 2009 - 2012 Intel Corporation.";
32 
33 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
34 static int debug = -1;
35 module_param(debug, int, 0);
36 MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
37 
38 static int igbvf_poll(struct napi_struct *napi, int budget);
39 static void igbvf_reset(struct igbvf_adapter *);
40 static void igbvf_set_interrupt_capability(struct igbvf_adapter *);
41 static void igbvf_reset_interrupt_capability(struct igbvf_adapter *);
42 
43 static struct igbvf_info igbvf_vf_info = {
44 	.mac		= e1000_vfadapt,
45 	.flags		= 0,
46 	.pba		= 10,
47 	.init_ops	= e1000_init_function_pointers_vf,
48 };
49 
50 static struct igbvf_info igbvf_i350_vf_info = {
51 	.mac		= e1000_vfadapt_i350,
52 	.flags		= 0,
53 	.pba		= 10,
54 	.init_ops	= e1000_init_function_pointers_vf,
55 };
56 
57 static const struct igbvf_info *igbvf_info_tbl[] = {
58 	[board_vf]	= &igbvf_vf_info,
59 	[board_i350_vf]	= &igbvf_i350_vf_info,
60 };
61 
62 /**
63  * igbvf_desc_unused - calculate if we have unused descriptors
64  * @ring: address of receive ring structure
65  **/
66 static int igbvf_desc_unused(struct igbvf_ring *ring)
67 {
68 	if (ring->next_to_clean > ring->next_to_use)
69 		return ring->next_to_clean - ring->next_to_use - 1;
70 
71 	return ring->count + ring->next_to_clean - ring->next_to_use - 1;
72 }
73 
74 /**
75  * igbvf_receive_skb - helper function to handle Rx indications
76  * @adapter: board private structure
77  * @netdev: pointer to netdev struct
78  * @skb: skb to indicate to stack
79  * @status: descriptor status field as written by hardware
80  * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
81  * @skb: pointer to sk_buff to be indicated to stack
82  **/
83 static void igbvf_receive_skb(struct igbvf_adapter *adapter,
84 			      struct net_device *netdev,
85 			      struct sk_buff *skb,
86 			      u32 status, __le16 vlan)
87 {
88 	u16 vid;
89 
90 	if (status & E1000_RXD_STAT_VP) {
91 		if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) &&
92 		    (status & E1000_RXDEXT_STATERR_LB))
93 			vid = be16_to_cpu((__force __be16)vlan) & E1000_RXD_SPC_VLAN_MASK;
94 		else
95 			vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
96 		if (test_bit(vid, adapter->active_vlans))
97 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
98 	}
99 
100 	napi_gro_receive(&adapter->rx_ring->napi, skb);
101 }
102 
103 static inline void igbvf_rx_checksum_adv(struct igbvf_adapter *adapter,
104 					 u32 status_err, struct sk_buff *skb)
105 {
106 	skb_checksum_none_assert(skb);
107 
108 	/* Ignore Checksum bit is set or checksum is disabled through ethtool */
109 	if ((status_err & E1000_RXD_STAT_IXSM) ||
110 	    (adapter->flags & IGBVF_FLAG_RX_CSUM_DISABLED))
111 		return;
112 
113 	/* TCP/UDP checksum error bit is set */
114 	if (status_err &
115 	    (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
116 		/* let the stack verify checksum errors */
117 		adapter->hw_csum_err++;
118 		return;
119 	}
120 
121 	/* It must be a TCP or UDP packet with a valid checksum */
122 	if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
123 		skb->ip_summed = CHECKSUM_UNNECESSARY;
124 
125 	adapter->hw_csum_good++;
126 }
127 
128 /**
129  * igbvf_alloc_rx_buffers - Replace used receive buffers; packet split
130  * @rx_ring: address of ring structure to repopulate
131  * @cleaned_count: number of buffers to repopulate
132  **/
133 static void igbvf_alloc_rx_buffers(struct igbvf_ring *rx_ring,
134 				   int cleaned_count)
135 {
136 	struct igbvf_adapter *adapter = rx_ring->adapter;
137 	struct net_device *netdev = adapter->netdev;
138 	struct pci_dev *pdev = adapter->pdev;
139 	union e1000_adv_rx_desc *rx_desc;
140 	struct igbvf_buffer *buffer_info;
141 	struct sk_buff *skb;
142 	unsigned int i;
143 	int bufsz;
144 
145 	i = rx_ring->next_to_use;
146 	buffer_info = &rx_ring->buffer_info[i];
147 
148 	if (adapter->rx_ps_hdr_size)
149 		bufsz = adapter->rx_ps_hdr_size;
150 	else
151 		bufsz = adapter->rx_buffer_len;
152 
153 	while (cleaned_count--) {
154 		rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
155 
156 		if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {
157 			if (!buffer_info->page) {
158 				buffer_info->page = alloc_page(GFP_ATOMIC);
159 				if (!buffer_info->page) {
160 					adapter->alloc_rx_buff_failed++;
161 					goto no_buffers;
162 				}
163 				buffer_info->page_offset = 0;
164 			} else {
165 				buffer_info->page_offset ^= PAGE_SIZE / 2;
166 			}
167 			buffer_info->page_dma =
168 				dma_map_page(&pdev->dev, buffer_info->page,
169 					     buffer_info->page_offset,
170 					     PAGE_SIZE / 2,
171 					     DMA_FROM_DEVICE);
172 			if (dma_mapping_error(&pdev->dev,
173 					      buffer_info->page_dma)) {
174 				__free_page(buffer_info->page);
175 				buffer_info->page = NULL;
176 				dev_err(&pdev->dev, "RX DMA map failed\n");
177 				break;
178 			}
179 		}
180 
181 		if (!buffer_info->skb) {
182 			skb = netdev_alloc_skb_ip_align(netdev, bufsz);
183 			if (!skb) {
184 				adapter->alloc_rx_buff_failed++;
185 				goto no_buffers;
186 			}
187 
188 			buffer_info->skb = skb;
189 			buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
190 							  bufsz,
191 							  DMA_FROM_DEVICE);
192 			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
193 				dev_kfree_skb(buffer_info->skb);
194 				buffer_info->skb = NULL;
195 				dev_err(&pdev->dev, "RX DMA map failed\n");
196 				goto no_buffers;
197 			}
198 		}
199 		/* Refresh the desc even if buffer_addrs didn't change because
200 		 * each write-back erases this info.
201 		 */
202 		if (adapter->rx_ps_hdr_size) {
203 			rx_desc->read.pkt_addr =
204 			     cpu_to_le64(buffer_info->page_dma);
205 			rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
206 		} else {
207 			rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
208 			rx_desc->read.hdr_addr = 0;
209 		}
210 
211 		i++;
212 		if (i == rx_ring->count)
213 			i = 0;
214 		buffer_info = &rx_ring->buffer_info[i];
215 	}
216 
217 no_buffers:
218 	if (rx_ring->next_to_use != i) {
219 		rx_ring->next_to_use = i;
220 		if (i == 0)
221 			i = (rx_ring->count - 1);
222 		else
223 			i--;
224 
225 		/* Force memory writes to complete before letting h/w
226 		 * know there are new descriptors to fetch.  (Only
227 		 * applicable for weak-ordered memory model archs,
228 		 * such as IA-64).
229 		*/
230 		wmb();
231 		writel(i, adapter->hw.hw_addr + rx_ring->tail);
232 	}
233 }
234 
235 /**
236  * igbvf_clean_rx_irq - Send received data up the network stack; legacy
237  * @adapter: board private structure
238  * @work_done: output parameter used to indicate completed work
239  * @work_to_do: input parameter setting limit of work
240  *
241  * the return value indicates whether actual cleaning was done, there
242  * is no guarantee that everything was cleaned
243  **/
244 static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,
245 			       int *work_done, int work_to_do)
246 {
247 	struct igbvf_ring *rx_ring = adapter->rx_ring;
248 	struct net_device *netdev = adapter->netdev;
249 	struct pci_dev *pdev = adapter->pdev;
250 	union e1000_adv_rx_desc *rx_desc, *next_rxd;
251 	struct igbvf_buffer *buffer_info, *next_buffer;
252 	struct sk_buff *skb;
253 	bool cleaned = false;
254 	int cleaned_count = 0;
255 	unsigned int total_bytes = 0, total_packets = 0;
256 	unsigned int i;
257 	u32 length, hlen, staterr;
258 
259 	i = rx_ring->next_to_clean;
260 	rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
261 	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
262 
263 	while (staterr & E1000_RXD_STAT_DD) {
264 		if (*work_done >= work_to_do)
265 			break;
266 		(*work_done)++;
267 		rmb(); /* read descriptor and rx_buffer_info after status DD */
268 
269 		buffer_info = &rx_ring->buffer_info[i];
270 
271 		/* HW will not DMA in data larger than the given buffer, even
272 		 * if it parses the (NFS, of course) header to be larger.  In
273 		 * that case, it fills the header buffer and spills the rest
274 		 * into the page.
275 		 */
276 		hlen = le16_get_bits(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info,
277 				     E1000_RXDADV_HDRBUFLEN_MASK);
278 		if (hlen > adapter->rx_ps_hdr_size)
279 			hlen = adapter->rx_ps_hdr_size;
280 
281 		length = le16_to_cpu(rx_desc->wb.upper.length);
282 		cleaned = true;
283 		cleaned_count++;
284 
285 		skb = buffer_info->skb;
286 		prefetch(skb->data - NET_IP_ALIGN);
287 		buffer_info->skb = NULL;
288 		if (!adapter->rx_ps_hdr_size) {
289 			dma_unmap_single(&pdev->dev, buffer_info->dma,
290 					 adapter->rx_buffer_len,
291 					 DMA_FROM_DEVICE);
292 			buffer_info->dma = 0;
293 			skb_put(skb, length);
294 			goto send_up;
295 		}
296 
297 		if (!skb_shinfo(skb)->nr_frags) {
298 			dma_unmap_single(&pdev->dev, buffer_info->dma,
299 					 adapter->rx_ps_hdr_size,
300 					 DMA_FROM_DEVICE);
301 			buffer_info->dma = 0;
302 			skb_put(skb, hlen);
303 		}
304 
305 		if (length) {
306 			dma_unmap_page(&pdev->dev, buffer_info->page_dma,
307 				       PAGE_SIZE / 2,
308 				       DMA_FROM_DEVICE);
309 			buffer_info->page_dma = 0;
310 
311 			skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
312 					   buffer_info->page,
313 					   buffer_info->page_offset,
314 					   length);
315 
316 			if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||
317 			    (page_count(buffer_info->page) != 1))
318 				buffer_info->page = NULL;
319 			else
320 				get_page(buffer_info->page);
321 
322 			skb->len += length;
323 			skb->data_len += length;
324 			skb->truesize += PAGE_SIZE / 2;
325 		}
326 send_up:
327 		i++;
328 		if (i == rx_ring->count)
329 			i = 0;
330 		next_rxd = IGBVF_RX_DESC_ADV(*rx_ring, i);
331 		prefetch(next_rxd);
332 		next_buffer = &rx_ring->buffer_info[i];
333 
334 		if (!(staterr & E1000_RXD_STAT_EOP)) {
335 			buffer_info->skb = next_buffer->skb;
336 			buffer_info->dma = next_buffer->dma;
337 			next_buffer->skb = skb;
338 			next_buffer->dma = 0;
339 			goto next_desc;
340 		}
341 
342 		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
343 			dev_kfree_skb_irq(skb);
344 			goto next_desc;
345 		}
346 
347 		total_bytes += skb->len;
348 		total_packets++;
349 
350 		igbvf_rx_checksum_adv(adapter, staterr, skb);
351 
352 		skb->protocol = eth_type_trans(skb, netdev);
353 
354 		igbvf_receive_skb(adapter, netdev, skb, staterr,
355 				  rx_desc->wb.upper.vlan);
356 
357 next_desc:
358 		rx_desc->wb.upper.status_error = 0;
359 
360 		/* return some buffers to hardware, one at a time is too slow */
361 		if (cleaned_count >= IGBVF_RX_BUFFER_WRITE) {
362 			igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
363 			cleaned_count = 0;
364 		}
365 
366 		/* use prefetched values */
367 		rx_desc = next_rxd;
368 		buffer_info = next_buffer;
369 
370 		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
371 	}
372 
373 	rx_ring->next_to_clean = i;
374 	cleaned_count = igbvf_desc_unused(rx_ring);
375 
376 	if (cleaned_count)
377 		igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
378 
379 	adapter->total_rx_packets += total_packets;
380 	adapter->total_rx_bytes += total_bytes;
381 	netdev->stats.rx_bytes += total_bytes;
382 	netdev->stats.rx_packets += total_packets;
383 	return cleaned;
384 }
385 
386 static void igbvf_put_txbuf(struct igbvf_adapter *adapter,
387 			    struct igbvf_buffer *buffer_info)
388 {
389 	if (buffer_info->dma) {
390 		if (buffer_info->mapped_as_page)
391 			dma_unmap_page(&adapter->pdev->dev,
392 				       buffer_info->dma,
393 				       buffer_info->length,
394 				       DMA_TO_DEVICE);
395 		else
396 			dma_unmap_single(&adapter->pdev->dev,
397 					 buffer_info->dma,
398 					 buffer_info->length,
399 					 DMA_TO_DEVICE);
400 		buffer_info->dma = 0;
401 	}
402 	if (buffer_info->skb) {
403 		dev_kfree_skb_any(buffer_info->skb);
404 		buffer_info->skb = NULL;
405 	}
406 	buffer_info->time_stamp = 0;
407 }
408 
409 /**
410  * igbvf_setup_tx_resources - allocate Tx resources (Descriptors)
411  * @adapter: board private structure
412  * @tx_ring: ring being initialized
413  *
414  * Return 0 on success, negative on failure
415  **/
416 int igbvf_setup_tx_resources(struct igbvf_adapter *adapter,
417 			     struct igbvf_ring *tx_ring)
418 {
419 	struct pci_dev *pdev = adapter->pdev;
420 	int size;
421 
422 	size = sizeof(struct igbvf_buffer) * tx_ring->count;
423 	tx_ring->buffer_info = vzalloc(size);
424 	if (!tx_ring->buffer_info)
425 		goto err;
426 
427 	/* round up to nearest 4K */
428 	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
429 	tx_ring->size = ALIGN(tx_ring->size, 4096);
430 
431 	tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
432 					   &tx_ring->dma, GFP_KERNEL);
433 	if (!tx_ring->desc)
434 		goto err;
435 
436 	tx_ring->adapter = adapter;
437 	tx_ring->next_to_use = 0;
438 	tx_ring->next_to_clean = 0;
439 
440 	return 0;
441 err:
442 	vfree(tx_ring->buffer_info);
443 	dev_err(&adapter->pdev->dev,
444 		"Unable to allocate memory for the transmit descriptor ring\n");
445 	return -ENOMEM;
446 }
447 
448 /**
449  * igbvf_setup_rx_resources - allocate Rx resources (Descriptors)
450  * @adapter: board private structure
451  * @rx_ring: ring being initialized
452  *
453  * Returns 0 on success, negative on failure
454  **/
455 int igbvf_setup_rx_resources(struct igbvf_adapter *adapter,
456 			     struct igbvf_ring *rx_ring)
457 {
458 	struct pci_dev *pdev = adapter->pdev;
459 	int size, desc_len;
460 
461 	size = sizeof(struct igbvf_buffer) * rx_ring->count;
462 	rx_ring->buffer_info = vzalloc(size);
463 	if (!rx_ring->buffer_info)
464 		goto err;
465 
466 	desc_len = sizeof(union e1000_adv_rx_desc);
467 
468 	/* Round up to nearest 4K */
469 	rx_ring->size = rx_ring->count * desc_len;
470 	rx_ring->size = ALIGN(rx_ring->size, 4096);
471 
472 	rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
473 					   &rx_ring->dma, GFP_KERNEL);
474 	if (!rx_ring->desc)
475 		goto err;
476 
477 	rx_ring->next_to_clean = 0;
478 	rx_ring->next_to_use = 0;
479 
480 	rx_ring->adapter = adapter;
481 
482 	return 0;
483 
484 err:
485 	vfree(rx_ring->buffer_info);
486 	rx_ring->buffer_info = NULL;
487 	dev_err(&adapter->pdev->dev,
488 		"Unable to allocate memory for the receive descriptor ring\n");
489 	return -ENOMEM;
490 }
491 
492 /**
493  * igbvf_clean_tx_ring - Free Tx Buffers
494  * @tx_ring: ring to be cleaned
495  **/
496 static void igbvf_clean_tx_ring(struct igbvf_ring *tx_ring)
497 {
498 	struct igbvf_adapter *adapter = tx_ring->adapter;
499 	struct igbvf_buffer *buffer_info;
500 	unsigned long size;
501 	unsigned int i;
502 
503 	if (!tx_ring->buffer_info)
504 		return;
505 
506 	/* Free all the Tx ring sk_buffs */
507 	for (i = 0; i < tx_ring->count; i++) {
508 		buffer_info = &tx_ring->buffer_info[i];
509 		igbvf_put_txbuf(adapter, buffer_info);
510 	}
511 
512 	size = sizeof(struct igbvf_buffer) * tx_ring->count;
513 	memset(tx_ring->buffer_info, 0, size);
514 
515 	/* Zero out the descriptor ring */
516 	memset(tx_ring->desc, 0, tx_ring->size);
517 
518 	tx_ring->next_to_use = 0;
519 	tx_ring->next_to_clean = 0;
520 
521 	writel(0, adapter->hw.hw_addr + tx_ring->head);
522 	writel(0, adapter->hw.hw_addr + tx_ring->tail);
523 }
524 
525 /**
526  * igbvf_free_tx_resources - Free Tx Resources per Queue
527  * @tx_ring: ring to free resources from
528  *
529  * Free all transmit software resources
530  **/
531 void igbvf_free_tx_resources(struct igbvf_ring *tx_ring)
532 {
533 	struct pci_dev *pdev = tx_ring->adapter->pdev;
534 
535 	igbvf_clean_tx_ring(tx_ring);
536 
537 	vfree(tx_ring->buffer_info);
538 	tx_ring->buffer_info = NULL;
539 
540 	dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
541 			  tx_ring->dma);
542 
543 	tx_ring->desc = NULL;
544 }
545 
546 /**
547  * igbvf_clean_rx_ring - Free Rx Buffers per Queue
548  * @rx_ring: ring structure pointer to free buffers from
549  **/
550 static void igbvf_clean_rx_ring(struct igbvf_ring *rx_ring)
551 {
552 	struct igbvf_adapter *adapter = rx_ring->adapter;
553 	struct igbvf_buffer *buffer_info;
554 	struct pci_dev *pdev = adapter->pdev;
555 	unsigned long size;
556 	unsigned int i;
557 
558 	if (!rx_ring->buffer_info)
559 		return;
560 
561 	/* Free all the Rx ring sk_buffs */
562 	for (i = 0; i < rx_ring->count; i++) {
563 		buffer_info = &rx_ring->buffer_info[i];
564 		if (buffer_info->dma) {
565 			if (adapter->rx_ps_hdr_size) {
566 				dma_unmap_single(&pdev->dev, buffer_info->dma,
567 						 adapter->rx_ps_hdr_size,
568 						 DMA_FROM_DEVICE);
569 			} else {
570 				dma_unmap_single(&pdev->dev, buffer_info->dma,
571 						 adapter->rx_buffer_len,
572 						 DMA_FROM_DEVICE);
573 			}
574 			buffer_info->dma = 0;
575 		}
576 
577 		if (buffer_info->skb) {
578 			dev_kfree_skb(buffer_info->skb);
579 			buffer_info->skb = NULL;
580 		}
581 
582 		if (buffer_info->page) {
583 			if (buffer_info->page_dma)
584 				dma_unmap_page(&pdev->dev,
585 					       buffer_info->page_dma,
586 					       PAGE_SIZE / 2,
587 					       DMA_FROM_DEVICE);
588 			put_page(buffer_info->page);
589 			buffer_info->page = NULL;
590 			buffer_info->page_dma = 0;
591 			buffer_info->page_offset = 0;
592 		}
593 	}
594 
595 	size = sizeof(struct igbvf_buffer) * rx_ring->count;
596 	memset(rx_ring->buffer_info, 0, size);
597 
598 	/* Zero out the descriptor ring */
599 	memset(rx_ring->desc, 0, rx_ring->size);
600 
601 	rx_ring->next_to_clean = 0;
602 	rx_ring->next_to_use = 0;
603 
604 	writel(0, adapter->hw.hw_addr + rx_ring->head);
605 	writel(0, adapter->hw.hw_addr + rx_ring->tail);
606 }
607 
608 /**
609  * igbvf_free_rx_resources - Free Rx Resources
610  * @rx_ring: ring to clean the resources from
611  *
612  * Free all receive software resources
613  **/
614 
615 void igbvf_free_rx_resources(struct igbvf_ring *rx_ring)
616 {
617 	struct pci_dev *pdev = rx_ring->adapter->pdev;
618 
619 	igbvf_clean_rx_ring(rx_ring);
620 
621 	vfree(rx_ring->buffer_info);
622 	rx_ring->buffer_info = NULL;
623 
624 	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
625 			  rx_ring->dma);
626 	rx_ring->desc = NULL;
627 }
628 
629 /**
630  * igbvf_update_itr - update the dynamic ITR value based on statistics
631  * @adapter: pointer to adapter
632  * @itr_setting: current adapter->itr
633  * @packets: the number of packets during this measurement interval
634  * @bytes: the number of bytes during this measurement interval
635  *
636  * Stores a new ITR value based on packets and byte counts during the last
637  * interrupt.  The advantage of per interrupt computation is faster updates
638  * and more accurate ITR for the current traffic pattern.  Constants in this
639  * function were computed based on theoretical maximum wire speed and thresholds
640  * were set based on testing data as well as attempting to minimize response
641  * time while increasing bulk throughput.
642  **/
643 static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,
644 					   enum latency_range itr_setting,
645 					   int packets, int bytes)
646 {
647 	enum latency_range retval = itr_setting;
648 
649 	if (packets == 0)
650 		goto update_itr_done;
651 
652 	switch (itr_setting) {
653 	case lowest_latency:
654 		/* handle TSO and jumbo frames */
655 		if (bytes/packets > 8000)
656 			retval = bulk_latency;
657 		else if ((packets < 5) && (bytes > 512))
658 			retval = low_latency;
659 		break;
660 	case low_latency:  /* 50 usec aka 20000 ints/s */
661 		if (bytes > 10000) {
662 			/* this if handles the TSO accounting */
663 			if (bytes/packets > 8000)
664 				retval = bulk_latency;
665 			else if ((packets < 10) || ((bytes/packets) > 1200))
666 				retval = bulk_latency;
667 			else if ((packets > 35))
668 				retval = lowest_latency;
669 		} else if (bytes/packets > 2000) {
670 			retval = bulk_latency;
671 		} else if (packets <= 2 && bytes < 512) {
672 			retval = lowest_latency;
673 		}
674 		break;
675 	case bulk_latency: /* 250 usec aka 4000 ints/s */
676 		if (bytes > 25000) {
677 			if (packets > 35)
678 				retval = low_latency;
679 		} else if (bytes < 6000) {
680 			retval = low_latency;
681 		}
682 		break;
683 	default:
684 		break;
685 	}
686 
687 update_itr_done:
688 	return retval;
689 }
690 
691 static int igbvf_range_to_itr(enum latency_range current_range)
692 {
693 	int new_itr;
694 
695 	switch (current_range) {
696 	/* counts and packets in update_itr are dependent on these numbers */
697 	case lowest_latency:
698 		new_itr = IGBVF_70K_ITR;
699 		break;
700 	case low_latency:
701 		new_itr = IGBVF_20K_ITR;
702 		break;
703 	case bulk_latency:
704 		new_itr = IGBVF_4K_ITR;
705 		break;
706 	default:
707 		new_itr = IGBVF_START_ITR;
708 		break;
709 	}
710 	return new_itr;
711 }
712 
713 static void igbvf_set_itr(struct igbvf_adapter *adapter)
714 {
715 	u32 new_itr;
716 
717 	adapter->tx_ring->itr_range =
718 			igbvf_update_itr(adapter,
719 					 adapter->tx_ring->itr_val,
720 					 adapter->total_tx_packets,
721 					 adapter->total_tx_bytes);
722 
723 	/* conservative mode (itr 3) eliminates the lowest_latency setting */
724 	if (adapter->requested_itr == 3 &&
725 	    adapter->tx_ring->itr_range == lowest_latency)
726 		adapter->tx_ring->itr_range = low_latency;
727 
728 	new_itr = igbvf_range_to_itr(adapter->tx_ring->itr_range);
729 
730 	if (new_itr != adapter->tx_ring->itr_val) {
731 		u32 current_itr = adapter->tx_ring->itr_val;
732 		/* this attempts to bias the interrupt rate towards Bulk
733 		 * by adding intermediate steps when interrupt rate is
734 		 * increasing
735 		 */
736 		new_itr = new_itr > current_itr ?
737 			  min(current_itr + (new_itr >> 2), new_itr) :
738 			  new_itr;
739 		adapter->tx_ring->itr_val = new_itr;
740 
741 		adapter->tx_ring->set_itr = 1;
742 	}
743 
744 	adapter->rx_ring->itr_range =
745 			igbvf_update_itr(adapter, adapter->rx_ring->itr_val,
746 					 adapter->total_rx_packets,
747 					 adapter->total_rx_bytes);
748 	if (adapter->requested_itr == 3 &&
749 	    adapter->rx_ring->itr_range == lowest_latency)
750 		adapter->rx_ring->itr_range = low_latency;
751 
752 	new_itr = igbvf_range_to_itr(adapter->rx_ring->itr_range);
753 
754 	if (new_itr != adapter->rx_ring->itr_val) {
755 		u32 current_itr = adapter->rx_ring->itr_val;
756 
757 		new_itr = new_itr > current_itr ?
758 			  min(current_itr + (new_itr >> 2), new_itr) :
759 			  new_itr;
760 		adapter->rx_ring->itr_val = new_itr;
761 
762 		adapter->rx_ring->set_itr = 1;
763 	}
764 }
765 
766 /**
767  * igbvf_clean_tx_irq - Reclaim resources after transmit completes
768  * @tx_ring: ring structure to clean descriptors from
769  *
770  * returns true if ring is completely cleaned
771  **/
772 static bool igbvf_clean_tx_irq(struct igbvf_ring *tx_ring)
773 {
774 	struct igbvf_adapter *adapter = tx_ring->adapter;
775 	struct net_device *netdev = adapter->netdev;
776 	struct igbvf_buffer *buffer_info;
777 	struct sk_buff *skb;
778 	union e1000_adv_tx_desc *tx_desc, *eop_desc;
779 	unsigned int total_bytes = 0, total_packets = 0;
780 	unsigned int i, count = 0;
781 	bool cleaned = false;
782 
783 	i = tx_ring->next_to_clean;
784 	buffer_info = &tx_ring->buffer_info[i];
785 	eop_desc = buffer_info->next_to_watch;
786 
787 	do {
788 		/* if next_to_watch is not set then there is no work pending */
789 		if (!eop_desc)
790 			break;
791 
792 		/* prevent any other reads prior to eop_desc */
793 		smp_rmb();
794 
795 		/* if DD is not set pending work has not been completed */
796 		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
797 			break;
798 
799 		/* clear next_to_watch to prevent false hangs */
800 		buffer_info->next_to_watch = NULL;
801 
802 		for (cleaned = false; !cleaned; count++) {
803 			tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
804 			cleaned = (tx_desc == eop_desc);
805 			skb = buffer_info->skb;
806 
807 			if (skb) {
808 				unsigned int segs, bytecount;
809 
810 				/* gso_segs is currently only valid for tcp */
811 				segs = skb_shinfo(skb)->gso_segs ?: 1;
812 				/* multiply data chunks by size of headers */
813 				bytecount = ((segs - 1) * skb_headlen(skb)) +
814 					    skb->len;
815 				total_packets += segs;
816 				total_bytes += bytecount;
817 			}
818 
819 			igbvf_put_txbuf(adapter, buffer_info);
820 			tx_desc->wb.status = 0;
821 
822 			i++;
823 			if (i == tx_ring->count)
824 				i = 0;
825 
826 			buffer_info = &tx_ring->buffer_info[i];
827 		}
828 
829 		eop_desc = buffer_info->next_to_watch;
830 	} while (count < tx_ring->count);
831 
832 	tx_ring->next_to_clean = i;
833 
834 	if (unlikely(count && netif_carrier_ok(netdev) &&
835 	    igbvf_desc_unused(tx_ring) >= IGBVF_TX_QUEUE_WAKE)) {
836 		/* Make sure that anybody stopping the queue after this
837 		 * sees the new next_to_clean.
838 		 */
839 		smp_mb();
840 		if (netif_queue_stopped(netdev) &&
841 		    !(test_bit(__IGBVF_DOWN, &adapter->state))) {
842 			netif_wake_queue(netdev);
843 			++adapter->restart_queue;
844 		}
845 	}
846 
847 	netdev->stats.tx_bytes += total_bytes;
848 	netdev->stats.tx_packets += total_packets;
849 	return count < tx_ring->count;
850 }
851 
852 static irqreturn_t igbvf_msix_other(int irq, void *data)
853 {
854 	struct net_device *netdev = data;
855 	struct igbvf_adapter *adapter = netdev_priv(netdev);
856 	struct e1000_hw *hw = &adapter->hw;
857 
858 	adapter->int_counter1++;
859 
860 	hw->mac.get_link_status = 1;
861 	if (!test_bit(__IGBVF_DOWN, &adapter->state))
862 		mod_timer(&adapter->watchdog_timer, jiffies + 1);
863 
864 	ew32(EIMS, adapter->eims_other);
865 
866 	return IRQ_HANDLED;
867 }
868 
869 static irqreturn_t igbvf_intr_msix_tx(int irq, void *data)
870 {
871 	struct net_device *netdev = data;
872 	struct igbvf_adapter *adapter = netdev_priv(netdev);
873 	struct e1000_hw *hw = &adapter->hw;
874 	struct igbvf_ring *tx_ring = adapter->tx_ring;
875 
876 	if (tx_ring->set_itr) {
877 		writel(tx_ring->itr_val,
878 		       adapter->hw.hw_addr + tx_ring->itr_register);
879 		adapter->tx_ring->set_itr = 0;
880 	}
881 
882 	adapter->total_tx_bytes = 0;
883 	adapter->total_tx_packets = 0;
884 
885 	/* auto mask will automatically re-enable the interrupt when we write
886 	 * EICS
887 	 */
888 	if (!igbvf_clean_tx_irq(tx_ring))
889 		/* Ring was not completely cleaned, so fire another interrupt */
890 		ew32(EICS, tx_ring->eims_value);
891 	else
892 		ew32(EIMS, tx_ring->eims_value);
893 
894 	return IRQ_HANDLED;
895 }
896 
897 static irqreturn_t igbvf_intr_msix_rx(int irq, void *data)
898 {
899 	struct net_device *netdev = data;
900 	struct igbvf_adapter *adapter = netdev_priv(netdev);
901 
902 	adapter->int_counter0++;
903 
904 	/* Write the ITR value calculated at the end of the
905 	 * previous interrupt.
906 	 */
907 	if (adapter->rx_ring->set_itr) {
908 		writel(adapter->rx_ring->itr_val,
909 		       adapter->hw.hw_addr + adapter->rx_ring->itr_register);
910 		adapter->rx_ring->set_itr = 0;
911 	}
912 
913 	if (napi_schedule_prep(&adapter->rx_ring->napi)) {
914 		adapter->total_rx_bytes = 0;
915 		adapter->total_rx_packets = 0;
916 		__napi_schedule(&adapter->rx_ring->napi);
917 	}
918 
919 	return IRQ_HANDLED;
920 }
921 
922 #define IGBVF_NO_QUEUE -1
923 
924 static void igbvf_assign_vector(struct igbvf_adapter *adapter, int rx_queue,
925 				int tx_queue, int msix_vector)
926 {
927 	struct e1000_hw *hw = &adapter->hw;
928 	u32 ivar, index;
929 
930 	/* 82576 uses a table-based method for assigning vectors.
931 	 * Each queue has a single entry in the table to which we write
932 	 * a vector number along with a "valid" bit.  Sadly, the layout
933 	 * of the table is somewhat counterintuitive.
934 	 */
935 	if (rx_queue > IGBVF_NO_QUEUE) {
936 		index = (rx_queue >> 1);
937 		ivar = array_er32(IVAR0, index);
938 		if (rx_queue & 0x1) {
939 			/* vector goes into third byte of register */
940 			ivar = ivar & 0xFF00FFFF;
941 			ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
942 		} else {
943 			/* vector goes into low byte of register */
944 			ivar = ivar & 0xFFFFFF00;
945 			ivar |= msix_vector | E1000_IVAR_VALID;
946 		}
947 		adapter->rx_ring[rx_queue].eims_value = BIT(msix_vector);
948 		array_ew32(IVAR0, index, ivar);
949 	}
950 	if (tx_queue > IGBVF_NO_QUEUE) {
951 		index = (tx_queue >> 1);
952 		ivar = array_er32(IVAR0, index);
953 		if (tx_queue & 0x1) {
954 			/* vector goes into high byte of register */
955 			ivar = ivar & 0x00FFFFFF;
956 			ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
957 		} else {
958 			/* vector goes into second byte of register */
959 			ivar = ivar & 0xFFFF00FF;
960 			ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
961 		}
962 		adapter->tx_ring[tx_queue].eims_value = BIT(msix_vector);
963 		array_ew32(IVAR0, index, ivar);
964 	}
965 }
966 
967 /**
968  * igbvf_configure_msix - Configure MSI-X hardware
969  * @adapter: board private structure
970  *
971  * igbvf_configure_msix sets up the hardware to properly
972  * generate MSI-X interrupts.
973  **/
974 static void igbvf_configure_msix(struct igbvf_adapter *adapter)
975 {
976 	u32 tmp;
977 	struct e1000_hw *hw = &adapter->hw;
978 	struct igbvf_ring *tx_ring = adapter->tx_ring;
979 	struct igbvf_ring *rx_ring = adapter->rx_ring;
980 	int vector = 0;
981 
982 	adapter->eims_enable_mask = 0;
983 
984 	igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, 0, vector++);
985 	adapter->eims_enable_mask |= tx_ring->eims_value;
986 	writel(tx_ring->itr_val, hw->hw_addr + tx_ring->itr_register);
987 	igbvf_assign_vector(adapter, 0, IGBVF_NO_QUEUE, vector++);
988 	adapter->eims_enable_mask |= rx_ring->eims_value;
989 	writel(rx_ring->itr_val, hw->hw_addr + rx_ring->itr_register);
990 
991 	/* set vector for other causes, i.e. link changes */
992 
993 	tmp = (vector++ | E1000_IVAR_VALID);
994 
995 	ew32(IVAR_MISC, tmp);
996 
997 	adapter->eims_enable_mask = GENMASK(vector - 1, 0);
998 	adapter->eims_other = BIT(vector - 1);
999 	e1e_flush();
1000 }
1001 
1002 static void igbvf_reset_interrupt_capability(struct igbvf_adapter *adapter)
1003 {
1004 	if (adapter->msix_entries) {
1005 		pci_disable_msix(adapter->pdev);
1006 		kfree(adapter->msix_entries);
1007 		adapter->msix_entries = NULL;
1008 	}
1009 }
1010 
1011 /**
1012  * igbvf_set_interrupt_capability - set MSI or MSI-X if supported
1013  * @adapter: board private structure
1014  *
1015  * Attempt to configure interrupts using the best available
1016  * capabilities of the hardware and kernel.
1017  **/
1018 static void igbvf_set_interrupt_capability(struct igbvf_adapter *adapter)
1019 {
1020 	int err = -ENOMEM;
1021 	int i;
1022 
1023 	/* we allocate 3 vectors, 1 for Tx, 1 for Rx, one for PF messages */
1024 	adapter->msix_entries = kcalloc(3, sizeof(struct msix_entry),
1025 					GFP_KERNEL);
1026 	if (adapter->msix_entries) {
1027 		for (i = 0; i < 3; i++)
1028 			adapter->msix_entries[i].entry = i;
1029 
1030 		err = pci_enable_msix_range(adapter->pdev,
1031 					    adapter->msix_entries, 3, 3);
1032 	}
1033 
1034 	if (err < 0) {
1035 		/* MSI-X failed */
1036 		dev_err(&adapter->pdev->dev,
1037 			"Failed to initialize MSI-X interrupts.\n");
1038 		igbvf_reset_interrupt_capability(adapter);
1039 	}
1040 }
1041 
1042 /**
1043  * igbvf_request_msix - Initialize MSI-X interrupts
1044  * @adapter: board private structure
1045  *
1046  * igbvf_request_msix allocates MSI-X vectors and requests interrupts from the
1047  * kernel.
1048  **/
1049 static int igbvf_request_msix(struct igbvf_adapter *adapter)
1050 {
1051 	struct net_device *netdev = adapter->netdev;
1052 	int err = 0, vector = 0;
1053 
1054 	if (strlen(netdev->name) < (IFNAMSIZ - 5)) {
1055 		sprintf(adapter->tx_ring->name, "%s-tx-0", netdev->name);
1056 		sprintf(adapter->rx_ring->name, "%s-rx-0", netdev->name);
1057 	} else {
1058 		memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
1059 		memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
1060 	}
1061 
1062 	err = request_irq(adapter->msix_entries[vector].vector,
1063 			  igbvf_intr_msix_tx, 0, adapter->tx_ring->name,
1064 			  netdev);
1065 	if (err)
1066 		goto out;
1067 
1068 	adapter->tx_ring->itr_register = E1000_EITR(vector);
1069 	adapter->tx_ring->itr_val = adapter->current_itr;
1070 	vector++;
1071 
1072 	err = request_irq(adapter->msix_entries[vector].vector,
1073 			  igbvf_intr_msix_rx, 0, adapter->rx_ring->name,
1074 			  netdev);
1075 	if (err)
1076 		goto free_irq_tx;
1077 
1078 	adapter->rx_ring->itr_register = E1000_EITR(vector);
1079 	adapter->rx_ring->itr_val = adapter->current_itr;
1080 	vector++;
1081 
1082 	err = request_irq(adapter->msix_entries[vector].vector,
1083 			  igbvf_msix_other, 0, netdev->name, netdev);
1084 	if (err)
1085 		goto free_irq_rx;
1086 
1087 	igbvf_configure_msix(adapter);
1088 	return 0;
1089 free_irq_rx:
1090 	free_irq(adapter->msix_entries[--vector].vector, netdev);
1091 free_irq_tx:
1092 	free_irq(adapter->msix_entries[--vector].vector, netdev);
1093 out:
1094 	return err;
1095 }
1096 
1097 /**
1098  * igbvf_alloc_queues - Allocate memory for all rings
1099  * @adapter: board private structure to initialize
1100  **/
1101 static int igbvf_alloc_queues(struct igbvf_adapter *adapter)
1102 {
1103 	struct net_device *netdev = adapter->netdev;
1104 
1105 	adapter->tx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1106 	if (!adapter->tx_ring)
1107 		return -ENOMEM;
1108 
1109 	adapter->rx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1110 	if (!adapter->rx_ring) {
1111 		kfree(adapter->tx_ring);
1112 		return -ENOMEM;
1113 	}
1114 
1115 	netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll);
1116 
1117 	return 0;
1118 }
1119 
1120 /**
1121  * igbvf_request_irq - initialize interrupts
1122  * @adapter: board private structure
1123  *
1124  * Attempts to configure interrupts using the best available
1125  * capabilities of the hardware and kernel.
1126  **/
1127 static int igbvf_request_irq(struct igbvf_adapter *adapter)
1128 {
1129 	int err = -1;
1130 
1131 	/* igbvf supports msi-x only */
1132 	if (adapter->msix_entries)
1133 		err = igbvf_request_msix(adapter);
1134 
1135 	if (!err)
1136 		return err;
1137 
1138 	dev_err(&adapter->pdev->dev,
1139 		"Unable to allocate interrupt, Error: %d\n", err);
1140 
1141 	return err;
1142 }
1143 
1144 static void igbvf_free_irq(struct igbvf_adapter *adapter)
1145 {
1146 	struct net_device *netdev = adapter->netdev;
1147 	int vector;
1148 
1149 	if (adapter->msix_entries) {
1150 		for (vector = 0; vector < 3; vector++)
1151 			free_irq(adapter->msix_entries[vector].vector, netdev);
1152 	}
1153 }
1154 
1155 /**
1156  * igbvf_irq_disable - Mask off interrupt generation on the NIC
1157  * @adapter: board private structure
1158  **/
1159 static void igbvf_irq_disable(struct igbvf_adapter *adapter)
1160 {
1161 	struct e1000_hw *hw = &adapter->hw;
1162 
1163 	ew32(EIMC, ~0);
1164 
1165 	if (adapter->msix_entries)
1166 		ew32(EIAC, 0);
1167 }
1168 
1169 /**
1170  * igbvf_irq_enable - Enable default interrupt generation settings
1171  * @adapter: board private structure
1172  **/
1173 static void igbvf_irq_enable(struct igbvf_adapter *adapter)
1174 {
1175 	struct e1000_hw *hw = &adapter->hw;
1176 
1177 	ew32(EIAC, adapter->eims_enable_mask);
1178 	ew32(EIAM, adapter->eims_enable_mask);
1179 	ew32(EIMS, adapter->eims_enable_mask);
1180 }
1181 
1182 /**
1183  * igbvf_poll - NAPI Rx polling callback
1184  * @napi: struct associated with this polling callback
1185  * @budget: amount of packets driver is allowed to process this poll
1186  **/
1187 static int igbvf_poll(struct napi_struct *napi, int budget)
1188 {
1189 	struct igbvf_ring *rx_ring = container_of(napi, struct igbvf_ring, napi);
1190 	struct igbvf_adapter *adapter = rx_ring->adapter;
1191 	struct e1000_hw *hw = &adapter->hw;
1192 	int work_done = 0;
1193 
1194 	igbvf_clean_rx_irq(adapter, &work_done, budget);
1195 
1196 	if (work_done == budget)
1197 		return budget;
1198 
1199 	/* Exit the polling mode, but don't re-enable interrupts if stack might
1200 	 * poll us due to busy-polling
1201 	 */
1202 	if (likely(napi_complete_done(napi, work_done))) {
1203 		if (adapter->requested_itr & 3)
1204 			igbvf_set_itr(adapter);
1205 
1206 		if (!test_bit(__IGBVF_DOWN, &adapter->state))
1207 			ew32(EIMS, adapter->rx_ring->eims_value);
1208 	}
1209 
1210 	return work_done;
1211 }
1212 
1213 /**
1214  * igbvf_set_rlpml - set receive large packet maximum length
1215  * @adapter: board private structure
1216  *
1217  * Configure the maximum size of packets that will be received
1218  */
1219 static void igbvf_set_rlpml(struct igbvf_adapter *adapter)
1220 {
1221 	int max_frame_size;
1222 	struct e1000_hw *hw = &adapter->hw;
1223 
1224 	max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
1225 
1226 	spin_lock_bh(&hw->mbx_lock);
1227 
1228 	e1000_rlpml_set_vf(hw, max_frame_size);
1229 
1230 	spin_unlock_bh(&hw->mbx_lock);
1231 }
1232 
1233 static int igbvf_vlan_rx_add_vid(struct net_device *netdev,
1234 				 __be16 proto, u16 vid)
1235 {
1236 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1237 	struct e1000_hw *hw = &adapter->hw;
1238 
1239 	spin_lock_bh(&hw->mbx_lock);
1240 
1241 	if (hw->mac.ops.set_vfta(hw, vid, true)) {
1242 		dev_warn(&adapter->pdev->dev, "Vlan id %d\n is not added", vid);
1243 		spin_unlock_bh(&hw->mbx_lock);
1244 		return -EINVAL;
1245 	}
1246 
1247 	spin_unlock_bh(&hw->mbx_lock);
1248 
1249 	set_bit(vid, adapter->active_vlans);
1250 	return 0;
1251 }
1252 
1253 static int igbvf_vlan_rx_kill_vid(struct net_device *netdev,
1254 				  __be16 proto, u16 vid)
1255 {
1256 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1257 	struct e1000_hw *hw = &adapter->hw;
1258 
1259 	spin_lock_bh(&hw->mbx_lock);
1260 
1261 	if (hw->mac.ops.set_vfta(hw, vid, false)) {
1262 		dev_err(&adapter->pdev->dev,
1263 			"Failed to remove vlan id %d\n", vid);
1264 		spin_unlock_bh(&hw->mbx_lock);
1265 		return -EINVAL;
1266 	}
1267 
1268 	spin_unlock_bh(&hw->mbx_lock);
1269 
1270 	clear_bit(vid, adapter->active_vlans);
1271 	return 0;
1272 }
1273 
1274 static void igbvf_restore_vlan(struct igbvf_adapter *adapter)
1275 {
1276 	u16 vid;
1277 
1278 	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1279 		igbvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
1280 }
1281 
1282 /**
1283  * igbvf_configure_tx - Configure Transmit Unit after Reset
1284  * @adapter: board private structure
1285  *
1286  * Configure the Tx unit of the MAC after a reset.
1287  **/
1288 static void igbvf_configure_tx(struct igbvf_adapter *adapter)
1289 {
1290 	struct e1000_hw *hw = &adapter->hw;
1291 	struct igbvf_ring *tx_ring = adapter->tx_ring;
1292 	u64 tdba;
1293 	u32 txdctl, dca_txctrl;
1294 
1295 	/* disable transmits */
1296 	txdctl = er32(TXDCTL(0));
1297 	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1298 	e1e_flush();
1299 	msleep(10);
1300 
1301 	/* Setup the HW Tx Head and Tail descriptor pointers */
1302 	ew32(TDLEN(0), tx_ring->count * sizeof(union e1000_adv_tx_desc));
1303 	tdba = tx_ring->dma;
1304 	ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
1305 	ew32(TDBAH(0), (tdba >> 32));
1306 	ew32(TDH(0), 0);
1307 	ew32(TDT(0), 0);
1308 	tx_ring->head = E1000_TDH(0);
1309 	tx_ring->tail = E1000_TDT(0);
1310 
1311 	/* Turn off Relaxed Ordering on head write-backs.  The writebacks
1312 	 * MUST be delivered in order or it will completely screw up
1313 	 * our bookkeeping.
1314 	 */
1315 	dca_txctrl = er32(DCA_TXCTRL(0));
1316 	dca_txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
1317 	ew32(DCA_TXCTRL(0), dca_txctrl);
1318 
1319 	/* enable transmits */
1320 	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
1321 	ew32(TXDCTL(0), txdctl);
1322 
1323 	/* Setup Transmit Descriptor Settings for eop descriptor */
1324 	adapter->txd_cmd = E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_IFCS;
1325 
1326 	/* enable Report Status bit */
1327 	adapter->txd_cmd |= E1000_ADVTXD_DCMD_RS;
1328 }
1329 
1330 /**
1331  * igbvf_setup_srrctl - configure the receive control registers
1332  * @adapter: Board private structure
1333  **/
1334 static void igbvf_setup_srrctl(struct igbvf_adapter *adapter)
1335 {
1336 	struct e1000_hw *hw = &adapter->hw;
1337 	u32 srrctl = 0;
1338 
1339 	srrctl &= ~(E1000_SRRCTL_DESCTYPE_MASK |
1340 		    E1000_SRRCTL_BSIZEHDR_MASK |
1341 		    E1000_SRRCTL_BSIZEPKT_MASK);
1342 
1343 	/* Enable queue drop to avoid head of line blocking */
1344 	srrctl |= E1000_SRRCTL_DROP_EN;
1345 
1346 	/* Setup buffer sizes */
1347 	srrctl |= ALIGN(adapter->rx_buffer_len, 1024) >>
1348 		  E1000_SRRCTL_BSIZEPKT_SHIFT;
1349 
1350 	if (adapter->rx_buffer_len < 2048) {
1351 		adapter->rx_ps_hdr_size = 0;
1352 		srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
1353 	} else {
1354 		adapter->rx_ps_hdr_size = 128;
1355 		srrctl |= adapter->rx_ps_hdr_size <<
1356 			  E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
1357 		srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1358 	}
1359 
1360 	ew32(SRRCTL(0), srrctl);
1361 }
1362 
1363 /**
1364  * igbvf_configure_rx - Configure Receive Unit after Reset
1365  * @adapter: board private structure
1366  *
1367  * Configure the Rx unit of the MAC after a reset.
1368  **/
1369 static void igbvf_configure_rx(struct igbvf_adapter *adapter)
1370 {
1371 	struct e1000_hw *hw = &adapter->hw;
1372 	struct igbvf_ring *rx_ring = adapter->rx_ring;
1373 	u64 rdba;
1374 	u32 rxdctl;
1375 
1376 	/* disable receives */
1377 	rxdctl = er32(RXDCTL(0));
1378 	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1379 	e1e_flush();
1380 	msleep(10);
1381 
1382 	/* Setup the HW Rx Head and Tail Descriptor Pointers and
1383 	 * the Base and Length of the Rx Descriptor Ring
1384 	 */
1385 	rdba = rx_ring->dma;
1386 	ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
1387 	ew32(RDBAH(0), (rdba >> 32));
1388 	ew32(RDLEN(0), rx_ring->count * sizeof(union e1000_adv_rx_desc));
1389 	rx_ring->head = E1000_RDH(0);
1390 	rx_ring->tail = E1000_RDT(0);
1391 	ew32(RDH(0), 0);
1392 	ew32(RDT(0), 0);
1393 
1394 	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
1395 	rxdctl &= 0xFFF00000;
1396 	rxdctl |= IGBVF_RX_PTHRESH;
1397 	rxdctl |= IGBVF_RX_HTHRESH << 8;
1398 	rxdctl |= IGBVF_RX_WTHRESH << 16;
1399 
1400 	igbvf_set_rlpml(adapter);
1401 
1402 	/* enable receives */
1403 	ew32(RXDCTL(0), rxdctl);
1404 }
1405 
1406 /**
1407  * igbvf_set_multi - Multicast and Promiscuous mode set
1408  * @netdev: network interface device structure
1409  *
1410  * The set_multi entry point is called whenever the multicast address
1411  * list or the network interface flags are updated.  This routine is
1412  * responsible for configuring the hardware for proper multicast,
1413  * promiscuous mode, and all-multi behavior.
1414  **/
1415 static void igbvf_set_multi(struct net_device *netdev)
1416 {
1417 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1418 	struct e1000_hw *hw = &adapter->hw;
1419 	struct netdev_hw_addr *ha;
1420 	u8  *mta_list = NULL;
1421 	int i;
1422 
1423 	if (!netdev_mc_empty(netdev)) {
1424 		mta_list = kmalloc_array(netdev_mc_count(netdev), ETH_ALEN,
1425 					 GFP_ATOMIC);
1426 		if (!mta_list)
1427 			return;
1428 	}
1429 
1430 	/* prepare a packed array of only addresses. */
1431 	i = 0;
1432 	netdev_for_each_mc_addr(ha, netdev)
1433 		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
1434 
1435 	spin_lock_bh(&hw->mbx_lock);
1436 
1437 	hw->mac.ops.update_mc_addr_list(hw, mta_list, i, 0, 0);
1438 
1439 	spin_unlock_bh(&hw->mbx_lock);
1440 	kfree(mta_list);
1441 }
1442 
1443 /**
1444  * igbvf_set_uni - Configure unicast MAC filters
1445  * @netdev: network interface device structure
1446  *
1447  * This routine is responsible for configuring the hardware for proper
1448  * unicast filters.
1449  **/
1450 static int igbvf_set_uni(struct net_device *netdev)
1451 {
1452 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1453 	struct e1000_hw *hw = &adapter->hw;
1454 
1455 	if (netdev_uc_count(netdev) > IGBVF_MAX_MAC_FILTERS) {
1456 		pr_err("Too many unicast filters - No Space\n");
1457 		return -ENOSPC;
1458 	}
1459 
1460 	spin_lock_bh(&hw->mbx_lock);
1461 
1462 	/* Clear all unicast MAC filters */
1463 	hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_CLR, NULL);
1464 
1465 	spin_unlock_bh(&hw->mbx_lock);
1466 
1467 	if (!netdev_uc_empty(netdev)) {
1468 		struct netdev_hw_addr *ha;
1469 
1470 		/* Add MAC filters one by one */
1471 		netdev_for_each_uc_addr(ha, netdev) {
1472 			spin_lock_bh(&hw->mbx_lock);
1473 
1474 			hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_ADD,
1475 						ha->addr);
1476 
1477 			spin_unlock_bh(&hw->mbx_lock);
1478 			udelay(200);
1479 		}
1480 	}
1481 
1482 	return 0;
1483 }
1484 
1485 static void igbvf_set_rx_mode(struct net_device *netdev)
1486 {
1487 	igbvf_set_multi(netdev);
1488 	igbvf_set_uni(netdev);
1489 }
1490 
1491 /**
1492  * igbvf_configure - configure the hardware for Rx and Tx
1493  * @adapter: private board structure
1494  **/
1495 static void igbvf_configure(struct igbvf_adapter *adapter)
1496 {
1497 	igbvf_set_rx_mode(adapter->netdev);
1498 
1499 	igbvf_restore_vlan(adapter);
1500 
1501 	igbvf_configure_tx(adapter);
1502 	igbvf_setup_srrctl(adapter);
1503 	igbvf_configure_rx(adapter);
1504 	igbvf_alloc_rx_buffers(adapter->rx_ring,
1505 			       igbvf_desc_unused(adapter->rx_ring));
1506 }
1507 
1508 /* igbvf_reset - bring the hardware into a known good state
1509  * @adapter: private board structure
1510  *
1511  * This function boots the hardware and enables some settings that
1512  * require a configuration cycle of the hardware - those cannot be
1513  * set/changed during runtime. After reset the device needs to be
1514  * properly configured for Rx, Tx etc.
1515  */
1516 static void igbvf_reset(struct igbvf_adapter *adapter)
1517 {
1518 	struct e1000_mac_info *mac = &adapter->hw.mac;
1519 	struct net_device *netdev = adapter->netdev;
1520 	struct e1000_hw *hw = &adapter->hw;
1521 
1522 	spin_lock_bh(&hw->mbx_lock);
1523 
1524 	/* Allow time for pending master requests to run */
1525 	if (mac->ops.reset_hw(hw))
1526 		dev_info(&adapter->pdev->dev, "PF still resetting\n");
1527 
1528 	mac->ops.init_hw(hw);
1529 
1530 	spin_unlock_bh(&hw->mbx_lock);
1531 
1532 	if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1533 		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
1534 		memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1535 		       netdev->addr_len);
1536 	}
1537 
1538 	adapter->last_reset = jiffies;
1539 }
1540 
1541 int igbvf_up(struct igbvf_adapter *adapter)
1542 {
1543 	struct e1000_hw *hw = &adapter->hw;
1544 
1545 	/* hardware has been reset, we need to reload some things */
1546 	igbvf_configure(adapter);
1547 
1548 	clear_bit(__IGBVF_DOWN, &adapter->state);
1549 
1550 	napi_enable(&adapter->rx_ring->napi);
1551 	if (adapter->msix_entries)
1552 		igbvf_configure_msix(adapter);
1553 
1554 	/* Clear any pending interrupts. */
1555 	er32(EICR);
1556 	igbvf_irq_enable(adapter);
1557 
1558 	/* start the watchdog */
1559 	hw->mac.get_link_status = 1;
1560 	mod_timer(&adapter->watchdog_timer, jiffies + 1);
1561 
1562 	return 0;
1563 }
1564 
1565 void igbvf_down(struct igbvf_adapter *adapter)
1566 {
1567 	struct net_device *netdev = adapter->netdev;
1568 	struct e1000_hw *hw = &adapter->hw;
1569 	u32 rxdctl, txdctl;
1570 
1571 	/* signal that we're down so the interrupt handler does not
1572 	 * reschedule our watchdog timer
1573 	 */
1574 	set_bit(__IGBVF_DOWN, &adapter->state);
1575 
1576 	/* disable receives in the hardware */
1577 	rxdctl = er32(RXDCTL(0));
1578 	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1579 
1580 	netif_carrier_off(netdev);
1581 	netif_stop_queue(netdev);
1582 
1583 	/* disable transmits in the hardware */
1584 	txdctl = er32(TXDCTL(0));
1585 	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1586 
1587 	/* flush both disables and wait for them to finish */
1588 	e1e_flush();
1589 	msleep(10);
1590 
1591 	napi_disable(&adapter->rx_ring->napi);
1592 
1593 	igbvf_irq_disable(adapter);
1594 
1595 	del_timer_sync(&adapter->watchdog_timer);
1596 
1597 	/* record the stats before reset*/
1598 	igbvf_update_stats(adapter);
1599 
1600 	adapter->link_speed = 0;
1601 	adapter->link_duplex = 0;
1602 
1603 	igbvf_reset(adapter);
1604 	igbvf_clean_tx_ring(adapter->tx_ring);
1605 	igbvf_clean_rx_ring(adapter->rx_ring);
1606 }
1607 
1608 void igbvf_reinit_locked(struct igbvf_adapter *adapter)
1609 {
1610 	might_sleep();
1611 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
1612 		usleep_range(1000, 2000);
1613 	igbvf_down(adapter);
1614 	igbvf_up(adapter);
1615 	clear_bit(__IGBVF_RESETTING, &adapter->state);
1616 }
1617 
1618 /**
1619  * igbvf_sw_init - Initialize general software structures (struct igbvf_adapter)
1620  * @adapter: board private structure to initialize
1621  *
1622  * igbvf_sw_init initializes the Adapter private data structure.
1623  * Fields are initialized based on PCI device information and
1624  * OS network device settings (MTU size).
1625  **/
1626 static int igbvf_sw_init(struct igbvf_adapter *adapter)
1627 {
1628 	struct net_device *netdev = adapter->netdev;
1629 	s32 rc;
1630 
1631 	adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
1632 	adapter->rx_ps_hdr_size = 0;
1633 	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1634 	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1635 
1636 	adapter->tx_int_delay = 8;
1637 	adapter->tx_abs_int_delay = 32;
1638 	adapter->rx_int_delay = 0;
1639 	adapter->rx_abs_int_delay = 8;
1640 	adapter->requested_itr = 3;
1641 	adapter->current_itr = IGBVF_START_ITR;
1642 
1643 	/* Set various function pointers */
1644 	adapter->ei->init_ops(&adapter->hw);
1645 
1646 	rc = adapter->hw.mac.ops.init_params(&adapter->hw);
1647 	if (rc)
1648 		return rc;
1649 
1650 	rc = adapter->hw.mbx.ops.init_params(&adapter->hw);
1651 	if (rc)
1652 		return rc;
1653 
1654 	igbvf_set_interrupt_capability(adapter);
1655 
1656 	if (igbvf_alloc_queues(adapter))
1657 		return -ENOMEM;
1658 
1659 	spin_lock_init(&adapter->tx_queue_lock);
1660 
1661 	/* Explicitly disable IRQ since the NIC can be in any state. */
1662 	igbvf_irq_disable(adapter);
1663 
1664 	spin_lock_init(&adapter->stats_lock);
1665 	spin_lock_init(&adapter->hw.mbx_lock);
1666 
1667 	set_bit(__IGBVF_DOWN, &adapter->state);
1668 	return 0;
1669 }
1670 
1671 static void igbvf_initialize_last_counter_stats(struct igbvf_adapter *adapter)
1672 {
1673 	struct e1000_hw *hw = &adapter->hw;
1674 
1675 	adapter->stats.last_gprc = er32(VFGPRC);
1676 	adapter->stats.last_gorc = er32(VFGORC);
1677 	adapter->stats.last_gptc = er32(VFGPTC);
1678 	adapter->stats.last_gotc = er32(VFGOTC);
1679 	adapter->stats.last_mprc = er32(VFMPRC);
1680 	adapter->stats.last_gotlbc = er32(VFGOTLBC);
1681 	adapter->stats.last_gptlbc = er32(VFGPTLBC);
1682 	adapter->stats.last_gorlbc = er32(VFGORLBC);
1683 	adapter->stats.last_gprlbc = er32(VFGPRLBC);
1684 
1685 	adapter->stats.base_gprc = er32(VFGPRC);
1686 	adapter->stats.base_gorc = er32(VFGORC);
1687 	adapter->stats.base_gptc = er32(VFGPTC);
1688 	adapter->stats.base_gotc = er32(VFGOTC);
1689 	adapter->stats.base_mprc = er32(VFMPRC);
1690 	adapter->stats.base_gotlbc = er32(VFGOTLBC);
1691 	adapter->stats.base_gptlbc = er32(VFGPTLBC);
1692 	adapter->stats.base_gorlbc = er32(VFGORLBC);
1693 	adapter->stats.base_gprlbc = er32(VFGPRLBC);
1694 }
1695 
1696 /**
1697  * igbvf_open - Called when a network interface is made active
1698  * @netdev: network interface device structure
1699  *
1700  * Returns 0 on success, negative value on failure
1701  *
1702  * The open entry point is called when a network interface is made
1703  * active by the system (IFF_UP).  At this point all resources needed
1704  * for transmit and receive operations are allocated, the interrupt
1705  * handler is registered with the OS, the watchdog timer is started,
1706  * and the stack is notified that the interface is ready.
1707  **/
1708 static int igbvf_open(struct net_device *netdev)
1709 {
1710 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1711 	struct e1000_hw *hw = &adapter->hw;
1712 	int err;
1713 
1714 	/* disallow open during test */
1715 	if (test_bit(__IGBVF_TESTING, &adapter->state))
1716 		return -EBUSY;
1717 
1718 	/* allocate transmit descriptors */
1719 	err = igbvf_setup_tx_resources(adapter, adapter->tx_ring);
1720 	if (err)
1721 		goto err_setup_tx;
1722 
1723 	/* allocate receive descriptors */
1724 	err = igbvf_setup_rx_resources(adapter, adapter->rx_ring);
1725 	if (err)
1726 		goto err_setup_rx;
1727 
1728 	/* before we allocate an interrupt, we must be ready to handle it.
1729 	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1730 	 * as soon as we call pci_request_irq, so we have to setup our
1731 	 * clean_rx handler before we do so.
1732 	 */
1733 	igbvf_configure(adapter);
1734 
1735 	err = igbvf_request_irq(adapter);
1736 	if (err)
1737 		goto err_req_irq;
1738 
1739 	/* From here on the code is the same as igbvf_up() */
1740 	clear_bit(__IGBVF_DOWN, &adapter->state);
1741 
1742 	napi_enable(&adapter->rx_ring->napi);
1743 
1744 	/* clear any pending interrupts */
1745 	er32(EICR);
1746 
1747 	igbvf_irq_enable(adapter);
1748 
1749 	/* start the watchdog */
1750 	hw->mac.get_link_status = 1;
1751 	mod_timer(&adapter->watchdog_timer, jiffies + 1);
1752 
1753 	return 0;
1754 
1755 err_req_irq:
1756 	igbvf_free_rx_resources(adapter->rx_ring);
1757 err_setup_rx:
1758 	igbvf_free_tx_resources(adapter->tx_ring);
1759 err_setup_tx:
1760 	igbvf_reset(adapter);
1761 
1762 	return err;
1763 }
1764 
1765 /**
1766  * igbvf_close - Disables a network interface
1767  * @netdev: network interface device structure
1768  *
1769  * Returns 0, this is not allowed to fail
1770  *
1771  * The close entry point is called when an interface is de-activated
1772  * by the OS.  The hardware is still under the drivers control, but
1773  * needs to be disabled.  A global MAC reset is issued to stop the
1774  * hardware, and all transmit and receive resources are freed.
1775  **/
1776 static int igbvf_close(struct net_device *netdev)
1777 {
1778 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1779 
1780 	WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
1781 	igbvf_down(adapter);
1782 
1783 	igbvf_free_irq(adapter);
1784 
1785 	igbvf_free_tx_resources(adapter->tx_ring);
1786 	igbvf_free_rx_resources(adapter->rx_ring);
1787 
1788 	return 0;
1789 }
1790 
1791 /**
1792  * igbvf_set_mac - Change the Ethernet Address of the NIC
1793  * @netdev: network interface device structure
1794  * @p: pointer to an address structure
1795  *
1796  * Returns 0 on success, negative on failure
1797  **/
1798 static int igbvf_set_mac(struct net_device *netdev, void *p)
1799 {
1800 	struct igbvf_adapter *adapter = netdev_priv(netdev);
1801 	struct e1000_hw *hw = &adapter->hw;
1802 	struct sockaddr *addr = p;
1803 
1804 	if (!is_valid_ether_addr(addr->sa_data))
1805 		return -EADDRNOTAVAIL;
1806 
1807 	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
1808 
1809 	spin_lock_bh(&hw->mbx_lock);
1810 
1811 	hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
1812 
1813 	spin_unlock_bh(&hw->mbx_lock);
1814 
1815 	if (!ether_addr_equal(addr->sa_data, hw->mac.addr))
1816 		return -EADDRNOTAVAIL;
1817 
1818 	eth_hw_addr_set(netdev, addr->sa_data);
1819 
1820 	return 0;
1821 }
1822 
1823 #define UPDATE_VF_COUNTER(reg, name) \
1824 { \
1825 	u32 current_counter = er32(reg); \
1826 	if (current_counter < adapter->stats.last_##name) \
1827 		adapter->stats.name += 0x100000000LL; \
1828 	adapter->stats.last_##name = current_counter; \
1829 	adapter->stats.name &= 0xFFFFFFFF00000000LL; \
1830 	adapter->stats.name |= current_counter; \
1831 }
1832 
1833 /**
1834  * igbvf_update_stats - Update the board statistics counters
1835  * @adapter: board private structure
1836 **/
1837 void igbvf_update_stats(struct igbvf_adapter *adapter)
1838 {
1839 	struct e1000_hw *hw = &adapter->hw;
1840 	struct pci_dev *pdev = adapter->pdev;
1841 
1842 	/* Prevent stats update while adapter is being reset, link is down
1843 	 * or if the pci connection is down.
1844 	 */
1845 	if (adapter->link_speed == 0)
1846 		return;
1847 
1848 	if (test_bit(__IGBVF_RESETTING, &adapter->state))
1849 		return;
1850 
1851 	if (pci_channel_offline(pdev))
1852 		return;
1853 
1854 	UPDATE_VF_COUNTER(VFGPRC, gprc);
1855 	UPDATE_VF_COUNTER(VFGORC, gorc);
1856 	UPDATE_VF_COUNTER(VFGPTC, gptc);
1857 	UPDATE_VF_COUNTER(VFGOTC, gotc);
1858 	UPDATE_VF_COUNTER(VFMPRC, mprc);
1859 	UPDATE_VF_COUNTER(VFGOTLBC, gotlbc);
1860 	UPDATE_VF_COUNTER(VFGPTLBC, gptlbc);
1861 	UPDATE_VF_COUNTER(VFGORLBC, gorlbc);
1862 	UPDATE_VF_COUNTER(VFGPRLBC, gprlbc);
1863 
1864 	/* Fill out the OS statistics structure */
1865 	adapter->netdev->stats.multicast = adapter->stats.mprc;
1866 }
1867 
1868 static void igbvf_print_link_info(struct igbvf_adapter *adapter)
1869 {
1870 	dev_info(&adapter->pdev->dev, "Link is Up %d Mbps %s Duplex\n",
1871 		 adapter->link_speed,
1872 		 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
1873 }
1874 
1875 static bool igbvf_has_link(struct igbvf_adapter *adapter)
1876 {
1877 	struct e1000_hw *hw = &adapter->hw;
1878 	s32 ret_val = E1000_SUCCESS;
1879 	bool link_active;
1880 
1881 	/* If interface is down, stay link down */
1882 	if (test_bit(__IGBVF_DOWN, &adapter->state))
1883 		return false;
1884 
1885 	spin_lock_bh(&hw->mbx_lock);
1886 
1887 	ret_val = hw->mac.ops.check_for_link(hw);
1888 
1889 	spin_unlock_bh(&hw->mbx_lock);
1890 
1891 	link_active = !hw->mac.get_link_status;
1892 
1893 	/* if check for link returns error we will need to reset */
1894 	if (ret_val && time_after(jiffies, adapter->last_reset + (10 * HZ)))
1895 		schedule_work(&adapter->reset_task);
1896 
1897 	return link_active;
1898 }
1899 
1900 /**
1901  * igbvf_watchdog - Timer Call-back
1902  * @t: timer list pointer containing private struct
1903  **/
1904 static void igbvf_watchdog(struct timer_list *t)
1905 {
1906 	struct igbvf_adapter *adapter = from_timer(adapter, t, watchdog_timer);
1907 
1908 	/* Do the rest outside of interrupt context */
1909 	schedule_work(&adapter->watchdog_task);
1910 }
1911 
1912 static void igbvf_watchdog_task(struct work_struct *work)
1913 {
1914 	struct igbvf_adapter *adapter = container_of(work,
1915 						     struct igbvf_adapter,
1916 						     watchdog_task);
1917 	struct net_device *netdev = adapter->netdev;
1918 	struct e1000_mac_info *mac = &adapter->hw.mac;
1919 	struct igbvf_ring *tx_ring = adapter->tx_ring;
1920 	struct e1000_hw *hw = &adapter->hw;
1921 	u32 link;
1922 	int tx_pending = 0;
1923 
1924 	link = igbvf_has_link(adapter);
1925 
1926 	if (link) {
1927 		if (!netif_carrier_ok(netdev)) {
1928 			mac->ops.get_link_up_info(&adapter->hw,
1929 						  &adapter->link_speed,
1930 						  &adapter->link_duplex);
1931 			igbvf_print_link_info(adapter);
1932 
1933 			netif_carrier_on(netdev);
1934 			netif_wake_queue(netdev);
1935 		}
1936 	} else {
1937 		if (netif_carrier_ok(netdev)) {
1938 			adapter->link_speed = 0;
1939 			adapter->link_duplex = 0;
1940 			dev_info(&adapter->pdev->dev, "Link is Down\n");
1941 			netif_carrier_off(netdev);
1942 			netif_stop_queue(netdev);
1943 		}
1944 	}
1945 
1946 	if (netif_carrier_ok(netdev)) {
1947 		igbvf_update_stats(adapter);
1948 	} else {
1949 		tx_pending = (igbvf_desc_unused(tx_ring) + 1 <
1950 			      tx_ring->count);
1951 		if (tx_pending) {
1952 			/* We've lost link, so the controller stops DMA,
1953 			 * but we've got queued Tx work that's never going
1954 			 * to get done, so reset controller to flush Tx.
1955 			 * (Do the reset outside of interrupt context).
1956 			 */
1957 			adapter->tx_timeout_count++;
1958 			schedule_work(&adapter->reset_task);
1959 		}
1960 	}
1961 
1962 	/* Cause software interrupt to ensure Rx ring is cleaned */
1963 	ew32(EICS, adapter->rx_ring->eims_value);
1964 
1965 	/* Reset the timer */
1966 	if (!test_bit(__IGBVF_DOWN, &adapter->state))
1967 		mod_timer(&adapter->watchdog_timer,
1968 			  round_jiffies(jiffies + (2 * HZ)));
1969 }
1970 
1971 #define IGBVF_TX_FLAGS_CSUM		0x00000001
1972 #define IGBVF_TX_FLAGS_VLAN		0x00000002
1973 #define IGBVF_TX_FLAGS_TSO		0x00000004
1974 #define IGBVF_TX_FLAGS_IPV4		0x00000008
1975 #define IGBVF_TX_FLAGS_VLAN_MASK	0xffff0000
1976 #define IGBVF_TX_FLAGS_VLAN_SHIFT	16
1977 
1978 static void igbvf_tx_ctxtdesc(struct igbvf_ring *tx_ring, u32 vlan_macip_lens,
1979 			      u32 type_tucmd, u32 mss_l4len_idx)
1980 {
1981 	struct e1000_adv_tx_context_desc *context_desc;
1982 	struct igbvf_buffer *buffer_info;
1983 	u16 i = tx_ring->next_to_use;
1984 
1985 	context_desc = IGBVF_TX_CTXTDESC_ADV(*tx_ring, i);
1986 	buffer_info = &tx_ring->buffer_info[i];
1987 
1988 	i++;
1989 	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1990 
1991 	/* set bits to identify this as an advanced context descriptor */
1992 	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
1993 
1994 	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
1995 	context_desc->seqnum_seed	= 0;
1996 	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
1997 	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
1998 
1999 	buffer_info->time_stamp = jiffies;
2000 	buffer_info->dma = 0;
2001 }
2002 
2003 static int igbvf_tso(struct igbvf_ring *tx_ring,
2004 		     struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2005 {
2006 	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
2007 	union {
2008 		struct iphdr *v4;
2009 		struct ipv6hdr *v6;
2010 		unsigned char *hdr;
2011 	} ip;
2012 	union {
2013 		struct tcphdr *tcp;
2014 		unsigned char *hdr;
2015 	} l4;
2016 	u32 paylen, l4_offset;
2017 	int err;
2018 
2019 	if (skb->ip_summed != CHECKSUM_PARTIAL)
2020 		return 0;
2021 
2022 	if (!skb_is_gso(skb))
2023 		return 0;
2024 
2025 	err = skb_cow_head(skb, 0);
2026 	if (err < 0)
2027 		return err;
2028 
2029 	ip.hdr = skb_network_header(skb);
2030 	l4.hdr = skb_checksum_start(skb);
2031 
2032 	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2033 	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2034 
2035 	/* initialize outer IP header fields */
2036 	if (ip.v4->version == 4) {
2037 		unsigned char *csum_start = skb_checksum_start(skb);
2038 		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
2039 
2040 		/* IP header will have to cancel out any data that
2041 		 * is not a part of the outer IP header
2042 		 */
2043 		ip.v4->check = csum_fold(csum_partial(trans_start,
2044 						      csum_start - trans_start,
2045 						      0));
2046 		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
2047 
2048 		ip.v4->tot_len = 0;
2049 	} else {
2050 		ip.v6->payload_len = 0;
2051 	}
2052 
2053 	/* determine offset of inner transport header */
2054 	l4_offset = l4.hdr - skb->data;
2055 
2056 	/* compute length of segmentation header */
2057 	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
2058 
2059 	/* remove payload length from inner checksum */
2060 	paylen = skb->len - l4_offset;
2061 	csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
2062 
2063 	/* MSS L4LEN IDX */
2064 	mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
2065 	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
2066 
2067 	/* VLAN MACLEN IPLEN */
2068 	vlan_macip_lens = l4.hdr - ip.hdr;
2069 	vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
2070 	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2071 
2072 	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
2073 
2074 	return 1;
2075 }
2076 
2077 static bool igbvf_tx_csum(struct igbvf_ring *tx_ring, struct sk_buff *skb,
2078 			  u32 tx_flags, __be16 protocol)
2079 {
2080 	u32 vlan_macip_lens = 0;
2081 	u32 type_tucmd = 0;
2082 
2083 	if (skb->ip_summed != CHECKSUM_PARTIAL) {
2084 csum_failed:
2085 		if (!(tx_flags & IGBVF_TX_FLAGS_VLAN))
2086 			return false;
2087 		goto no_csum;
2088 	}
2089 
2090 	switch (skb->csum_offset) {
2091 	case offsetof(struct tcphdr, check):
2092 		type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2093 		fallthrough;
2094 	case offsetof(struct udphdr, check):
2095 		break;
2096 	case offsetof(struct sctphdr, checksum):
2097 		/* validate that this is actually an SCTP request */
2098 		if (skb_csum_is_sctp(skb)) {
2099 			type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
2100 			break;
2101 		}
2102 		fallthrough;
2103 	default:
2104 		skb_checksum_help(skb);
2105 		goto csum_failed;
2106 	}
2107 
2108 	vlan_macip_lens = skb_checksum_start_offset(skb) -
2109 			  skb_network_offset(skb);
2110 no_csum:
2111 	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
2112 	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2113 
2114 	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0);
2115 	return true;
2116 }
2117 
2118 static int igbvf_maybe_stop_tx(struct net_device *netdev, int size)
2119 {
2120 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2121 
2122 	/* there is enough descriptors then we don't need to worry  */
2123 	if (igbvf_desc_unused(adapter->tx_ring) >= size)
2124 		return 0;
2125 
2126 	netif_stop_queue(netdev);
2127 
2128 	/* Herbert's original patch had:
2129 	 *  smp_mb__after_netif_stop_queue();
2130 	 * but since that doesn't exist yet, just open code it.
2131 	 */
2132 	smp_mb();
2133 
2134 	/* We need to check again just in case room has been made available */
2135 	if (igbvf_desc_unused(adapter->tx_ring) < size)
2136 		return -EBUSY;
2137 
2138 	netif_wake_queue(netdev);
2139 
2140 	++adapter->restart_queue;
2141 	return 0;
2142 }
2143 
2144 #define IGBVF_MAX_TXD_PWR	16
2145 #define IGBVF_MAX_DATA_PER_TXD	(1u << IGBVF_MAX_TXD_PWR)
2146 
2147 static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter,
2148 				   struct igbvf_ring *tx_ring,
2149 				   struct sk_buff *skb)
2150 {
2151 	struct igbvf_buffer *buffer_info;
2152 	struct pci_dev *pdev = adapter->pdev;
2153 	unsigned int len = skb_headlen(skb);
2154 	unsigned int count = 0, i;
2155 	unsigned int f;
2156 
2157 	i = tx_ring->next_to_use;
2158 
2159 	buffer_info = &tx_ring->buffer_info[i];
2160 	BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2161 	buffer_info->length = len;
2162 	/* set time_stamp *before* dma to help avoid a possible race */
2163 	buffer_info->time_stamp = jiffies;
2164 	buffer_info->mapped_as_page = false;
2165 	buffer_info->dma = dma_map_single(&pdev->dev, skb->data, len,
2166 					  DMA_TO_DEVICE);
2167 	if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2168 		goto dma_error;
2169 
2170 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2171 		const skb_frag_t *frag;
2172 
2173 		count++;
2174 		i++;
2175 		if (i == tx_ring->count)
2176 			i = 0;
2177 
2178 		frag = &skb_shinfo(skb)->frags[f];
2179 		len = skb_frag_size(frag);
2180 
2181 		buffer_info = &tx_ring->buffer_info[i];
2182 		BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2183 		buffer_info->length = len;
2184 		buffer_info->time_stamp = jiffies;
2185 		buffer_info->mapped_as_page = true;
2186 		buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 0, len,
2187 						    DMA_TO_DEVICE);
2188 		if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2189 			goto dma_error;
2190 	}
2191 
2192 	tx_ring->buffer_info[i].skb = skb;
2193 
2194 	return ++count;
2195 
2196 dma_error:
2197 	dev_err(&pdev->dev, "TX DMA map failed\n");
2198 
2199 	/* clear timestamp and dma mappings for failed buffer_info mapping */
2200 	buffer_info->dma = 0;
2201 	buffer_info->time_stamp = 0;
2202 	buffer_info->length = 0;
2203 	buffer_info->mapped_as_page = false;
2204 	if (count)
2205 		count--;
2206 
2207 	/* clear timestamp and dma mappings for remaining portion of packet */
2208 	while (count--) {
2209 		if (i == 0)
2210 			i += tx_ring->count;
2211 		i--;
2212 		buffer_info = &tx_ring->buffer_info[i];
2213 		igbvf_put_txbuf(adapter, buffer_info);
2214 	}
2215 
2216 	return 0;
2217 }
2218 
2219 static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,
2220 				      struct igbvf_ring *tx_ring,
2221 				      int tx_flags, int count,
2222 				      unsigned int first, u32 paylen,
2223 				      u8 hdr_len)
2224 {
2225 	union e1000_adv_tx_desc *tx_desc = NULL;
2226 	struct igbvf_buffer *buffer_info;
2227 	u32 olinfo_status = 0, cmd_type_len;
2228 	unsigned int i;
2229 
2230 	cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
2231 			E1000_ADVTXD_DCMD_DEXT);
2232 
2233 	if (tx_flags & IGBVF_TX_FLAGS_VLAN)
2234 		cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2235 
2236 	if (tx_flags & IGBVF_TX_FLAGS_TSO) {
2237 		cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
2238 
2239 		/* insert tcp checksum */
2240 		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2241 
2242 		/* insert ip checksum */
2243 		if (tx_flags & IGBVF_TX_FLAGS_IPV4)
2244 			olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2245 
2246 	} else if (tx_flags & IGBVF_TX_FLAGS_CSUM) {
2247 		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2248 	}
2249 
2250 	olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
2251 
2252 	i = tx_ring->next_to_use;
2253 	while (count--) {
2254 		buffer_info = &tx_ring->buffer_info[i];
2255 		tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
2256 		tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
2257 		tx_desc->read.cmd_type_len =
2258 			 cpu_to_le32(cmd_type_len | buffer_info->length);
2259 		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2260 		i++;
2261 		if (i == tx_ring->count)
2262 			i = 0;
2263 	}
2264 
2265 	tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
2266 	/* Force memory writes to complete before letting h/w
2267 	 * know there are new descriptors to fetch.  (Only
2268 	 * applicable for weak-ordered memory model archs,
2269 	 * such as IA-64).
2270 	 */
2271 	wmb();
2272 
2273 	tx_ring->buffer_info[first].next_to_watch = tx_desc;
2274 	tx_ring->next_to_use = i;
2275 	writel(i, adapter->hw.hw_addr + tx_ring->tail);
2276 }
2277 
2278 static netdev_tx_t igbvf_xmit_frame_ring_adv(struct sk_buff *skb,
2279 					     struct net_device *netdev,
2280 					     struct igbvf_ring *tx_ring)
2281 {
2282 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2283 	unsigned int first, tx_flags = 0;
2284 	u8 hdr_len = 0;
2285 	int count = 0;
2286 	int tso = 0;
2287 	__be16 protocol = vlan_get_protocol(skb);
2288 
2289 	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2290 		dev_kfree_skb_any(skb);
2291 		return NETDEV_TX_OK;
2292 	}
2293 
2294 	if (skb->len <= 0) {
2295 		dev_kfree_skb_any(skb);
2296 		return NETDEV_TX_OK;
2297 	}
2298 
2299 	/* need: count + 4 desc gap to keep tail from touching
2300 	 *       + 2 desc gap to keep tail from touching head,
2301 	 *       + 1 desc for skb->data,
2302 	 *       + 1 desc for context descriptor,
2303 	 * head, otherwise try next time
2304 	 */
2305 	if (igbvf_maybe_stop_tx(netdev, skb_shinfo(skb)->nr_frags + 4)) {
2306 		/* this is a hard error */
2307 		return NETDEV_TX_BUSY;
2308 	}
2309 
2310 	if (skb_vlan_tag_present(skb)) {
2311 		tx_flags |= IGBVF_TX_FLAGS_VLAN;
2312 		tx_flags |= (skb_vlan_tag_get(skb) <<
2313 			     IGBVF_TX_FLAGS_VLAN_SHIFT);
2314 	}
2315 
2316 	if (protocol == htons(ETH_P_IP))
2317 		tx_flags |= IGBVF_TX_FLAGS_IPV4;
2318 
2319 	first = tx_ring->next_to_use;
2320 
2321 	tso = igbvf_tso(tx_ring, skb, tx_flags, &hdr_len);
2322 	if (unlikely(tso < 0)) {
2323 		dev_kfree_skb_any(skb);
2324 		return NETDEV_TX_OK;
2325 	}
2326 
2327 	if (tso)
2328 		tx_flags |= IGBVF_TX_FLAGS_TSO;
2329 	else if (igbvf_tx_csum(tx_ring, skb, tx_flags, protocol) &&
2330 		 (skb->ip_summed == CHECKSUM_PARTIAL))
2331 		tx_flags |= IGBVF_TX_FLAGS_CSUM;
2332 
2333 	/* count reflects descriptors mapped, if 0 then mapping error
2334 	 * has occurred and we need to rewind the descriptor queue
2335 	 */
2336 	count = igbvf_tx_map_adv(adapter, tx_ring, skb);
2337 
2338 	if (count) {
2339 		igbvf_tx_queue_adv(adapter, tx_ring, tx_flags, count,
2340 				   first, skb->len, hdr_len);
2341 		/* Make sure there is space in the ring for the next send. */
2342 		igbvf_maybe_stop_tx(netdev, MAX_SKB_FRAGS + 4);
2343 	} else {
2344 		dev_kfree_skb_any(skb);
2345 		tx_ring->buffer_info[first].time_stamp = 0;
2346 		tx_ring->next_to_use = first;
2347 	}
2348 
2349 	return NETDEV_TX_OK;
2350 }
2351 
2352 static netdev_tx_t igbvf_xmit_frame(struct sk_buff *skb,
2353 				    struct net_device *netdev)
2354 {
2355 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2356 	struct igbvf_ring *tx_ring;
2357 
2358 	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2359 		dev_kfree_skb_any(skb);
2360 		return NETDEV_TX_OK;
2361 	}
2362 
2363 	tx_ring = &adapter->tx_ring[0];
2364 
2365 	return igbvf_xmit_frame_ring_adv(skb, netdev, tx_ring);
2366 }
2367 
2368 /**
2369  * igbvf_tx_timeout - Respond to a Tx Hang
2370  * @netdev: network interface device structure
2371  * @txqueue: queue timing out (unused)
2372  **/
2373 static void igbvf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
2374 {
2375 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2376 
2377 	/* Do the reset outside of interrupt context */
2378 	adapter->tx_timeout_count++;
2379 	schedule_work(&adapter->reset_task);
2380 }
2381 
2382 static void igbvf_reset_task(struct work_struct *work)
2383 {
2384 	struct igbvf_adapter *adapter;
2385 
2386 	adapter = container_of(work, struct igbvf_adapter, reset_task);
2387 
2388 	igbvf_reinit_locked(adapter);
2389 }
2390 
2391 /**
2392  * igbvf_change_mtu - Change the Maximum Transfer Unit
2393  * @netdev: network interface device structure
2394  * @new_mtu: new value for maximum frame size
2395  *
2396  * Returns 0 on success, negative on failure
2397  **/
2398 static int igbvf_change_mtu(struct net_device *netdev, int new_mtu)
2399 {
2400 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2401 	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2402 
2403 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2404 		usleep_range(1000, 2000);
2405 	/* igbvf_down has a dependency on max_frame_size */
2406 	adapter->max_frame_size = max_frame;
2407 	if (netif_running(netdev))
2408 		igbvf_down(adapter);
2409 
2410 	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
2411 	 * means we reserve 2 more, this pushes us to allocate from the next
2412 	 * larger slab size.
2413 	 * i.e. RXBUFFER_2048 --> size-4096 slab
2414 	 * However with the new *_jumbo_rx* routines, jumbo receives will use
2415 	 * fragmented skbs
2416 	 */
2417 
2418 	if (max_frame <= 1024)
2419 		adapter->rx_buffer_len = 1024;
2420 	else if (max_frame <= 2048)
2421 		adapter->rx_buffer_len = 2048;
2422 	else
2423 #if (PAGE_SIZE / 2) > 16384
2424 		adapter->rx_buffer_len = 16384;
2425 #else
2426 		adapter->rx_buffer_len = PAGE_SIZE / 2;
2427 #endif
2428 
2429 	/* adjust allocation if LPE protects us, and we aren't using SBP */
2430 	if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
2431 	    (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
2432 		adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN +
2433 					 ETH_FCS_LEN;
2434 
2435 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
2436 		   netdev->mtu, new_mtu);
2437 	netdev->mtu = new_mtu;
2438 
2439 	if (netif_running(netdev))
2440 		igbvf_up(adapter);
2441 	else
2442 		igbvf_reset(adapter);
2443 
2444 	clear_bit(__IGBVF_RESETTING, &adapter->state);
2445 
2446 	return 0;
2447 }
2448 
2449 static int igbvf_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2450 {
2451 	switch (cmd) {
2452 	default:
2453 		return -EOPNOTSUPP;
2454 	}
2455 }
2456 
2457 static int igbvf_suspend(struct device *dev_d)
2458 {
2459 	struct net_device *netdev = dev_get_drvdata(dev_d);
2460 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2461 
2462 	netif_device_detach(netdev);
2463 
2464 	if (netif_running(netdev)) {
2465 		WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
2466 		igbvf_down(adapter);
2467 		igbvf_free_irq(adapter);
2468 	}
2469 
2470 	return 0;
2471 }
2472 
2473 static int __maybe_unused igbvf_resume(struct device *dev_d)
2474 {
2475 	struct pci_dev *pdev = to_pci_dev(dev_d);
2476 	struct net_device *netdev = pci_get_drvdata(pdev);
2477 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2478 	u32 err;
2479 
2480 	pci_set_master(pdev);
2481 
2482 	if (netif_running(netdev)) {
2483 		err = igbvf_request_irq(adapter);
2484 		if (err)
2485 			return err;
2486 	}
2487 
2488 	igbvf_reset(adapter);
2489 
2490 	if (netif_running(netdev))
2491 		igbvf_up(adapter);
2492 
2493 	netif_device_attach(netdev);
2494 
2495 	return 0;
2496 }
2497 
2498 static void igbvf_shutdown(struct pci_dev *pdev)
2499 {
2500 	igbvf_suspend(&pdev->dev);
2501 }
2502 
2503 #ifdef CONFIG_NET_POLL_CONTROLLER
2504 /* Polling 'interrupt' - used by things like netconsole to send skbs
2505  * without having to re-enable interrupts. It's not called while
2506  * the interrupt routine is executing.
2507  */
2508 static void igbvf_netpoll(struct net_device *netdev)
2509 {
2510 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2511 
2512 	disable_irq(adapter->pdev->irq);
2513 
2514 	igbvf_clean_tx_irq(adapter->tx_ring);
2515 
2516 	enable_irq(adapter->pdev->irq);
2517 }
2518 #endif
2519 
2520 /**
2521  * igbvf_io_error_detected - called when PCI error is detected
2522  * @pdev: Pointer to PCI device
2523  * @state: The current pci connection state
2524  *
2525  * This function is called after a PCI bus error affecting
2526  * this device has been detected.
2527  */
2528 static pci_ers_result_t igbvf_io_error_detected(struct pci_dev *pdev,
2529 						pci_channel_state_t state)
2530 {
2531 	struct net_device *netdev = pci_get_drvdata(pdev);
2532 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2533 
2534 	netif_device_detach(netdev);
2535 
2536 	if (state == pci_channel_io_perm_failure)
2537 		return PCI_ERS_RESULT_DISCONNECT;
2538 
2539 	if (netif_running(netdev))
2540 		igbvf_down(adapter);
2541 	pci_disable_device(pdev);
2542 
2543 	/* Request a slot reset. */
2544 	return PCI_ERS_RESULT_NEED_RESET;
2545 }
2546 
2547 /**
2548  * igbvf_io_slot_reset - called after the pci bus has been reset.
2549  * @pdev: Pointer to PCI device
2550  *
2551  * Restart the card from scratch, as if from a cold-boot. Implementation
2552  * resembles the first-half of the igbvf_resume routine.
2553  */
2554 static pci_ers_result_t igbvf_io_slot_reset(struct pci_dev *pdev)
2555 {
2556 	struct net_device *netdev = pci_get_drvdata(pdev);
2557 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2558 
2559 	if (pci_enable_device_mem(pdev)) {
2560 		dev_err(&pdev->dev,
2561 			"Cannot re-enable PCI device after reset.\n");
2562 		return PCI_ERS_RESULT_DISCONNECT;
2563 	}
2564 	pci_set_master(pdev);
2565 
2566 	igbvf_reset(adapter);
2567 
2568 	return PCI_ERS_RESULT_RECOVERED;
2569 }
2570 
2571 /**
2572  * igbvf_io_resume - called when traffic can start flowing again.
2573  * @pdev: Pointer to PCI device
2574  *
2575  * This callback is called when the error recovery driver tells us that
2576  * its OK to resume normal operation. Implementation resembles the
2577  * second-half of the igbvf_resume routine.
2578  */
2579 static void igbvf_io_resume(struct pci_dev *pdev)
2580 {
2581 	struct net_device *netdev = pci_get_drvdata(pdev);
2582 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2583 
2584 	if (netif_running(netdev)) {
2585 		if (igbvf_up(adapter)) {
2586 			dev_err(&pdev->dev,
2587 				"can't bring device back up after reset\n");
2588 			return;
2589 		}
2590 	}
2591 
2592 	netif_device_attach(netdev);
2593 }
2594 
2595 /**
2596  * igbvf_io_prepare - prepare device driver for PCI reset
2597  * @pdev: PCI device information struct
2598  */
2599 static void igbvf_io_prepare(struct pci_dev *pdev)
2600 {
2601 	struct net_device *netdev = pci_get_drvdata(pdev);
2602 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2603 
2604 	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2605 		usleep_range(1000, 2000);
2606 	igbvf_down(adapter);
2607 }
2608 
2609 /**
2610  * igbvf_io_reset_done - PCI reset done, device driver reset can begin
2611  * @pdev: PCI device information struct
2612  */
2613 static void igbvf_io_reset_done(struct pci_dev *pdev)
2614 {
2615 	struct net_device *netdev = pci_get_drvdata(pdev);
2616 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2617 
2618 	igbvf_up(adapter);
2619 	clear_bit(__IGBVF_RESETTING, &adapter->state);
2620 }
2621 
2622 static void igbvf_print_device_info(struct igbvf_adapter *adapter)
2623 {
2624 	struct e1000_hw *hw = &adapter->hw;
2625 	struct net_device *netdev = adapter->netdev;
2626 	struct pci_dev *pdev = adapter->pdev;
2627 
2628 	if (hw->mac.type == e1000_vfadapt_i350)
2629 		dev_info(&pdev->dev, "Intel(R) I350 Virtual Function\n");
2630 	else
2631 		dev_info(&pdev->dev, "Intel(R) 82576 Virtual Function\n");
2632 	dev_info(&pdev->dev, "Address: %pM\n", netdev->dev_addr);
2633 }
2634 
2635 static int igbvf_set_features(struct net_device *netdev,
2636 			      netdev_features_t features)
2637 {
2638 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2639 
2640 	if (features & NETIF_F_RXCSUM)
2641 		adapter->flags &= ~IGBVF_FLAG_RX_CSUM_DISABLED;
2642 	else
2643 		adapter->flags |= IGBVF_FLAG_RX_CSUM_DISABLED;
2644 
2645 	return 0;
2646 }
2647 
2648 #define IGBVF_MAX_MAC_HDR_LEN		127
2649 #define IGBVF_MAX_NETWORK_HDR_LEN	511
2650 
2651 static netdev_features_t
2652 igbvf_features_check(struct sk_buff *skb, struct net_device *dev,
2653 		     netdev_features_t features)
2654 {
2655 	unsigned int network_hdr_len, mac_hdr_len;
2656 
2657 	/* Make certain the headers can be described by a context descriptor */
2658 	mac_hdr_len = skb_network_header(skb) - skb->data;
2659 	if (unlikely(mac_hdr_len > IGBVF_MAX_MAC_HDR_LEN))
2660 		return features & ~(NETIF_F_HW_CSUM |
2661 				    NETIF_F_SCTP_CRC |
2662 				    NETIF_F_HW_VLAN_CTAG_TX |
2663 				    NETIF_F_TSO |
2664 				    NETIF_F_TSO6);
2665 
2666 	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2667 	if (unlikely(network_hdr_len >  IGBVF_MAX_NETWORK_HDR_LEN))
2668 		return features & ~(NETIF_F_HW_CSUM |
2669 				    NETIF_F_SCTP_CRC |
2670 				    NETIF_F_TSO |
2671 				    NETIF_F_TSO6);
2672 
2673 	/* We can only support IPV4 TSO in tunnels if we can mangle the
2674 	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2675 	 */
2676 	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2677 		features &= ~NETIF_F_TSO;
2678 
2679 	return features;
2680 }
2681 
2682 static const struct net_device_ops igbvf_netdev_ops = {
2683 	.ndo_open		= igbvf_open,
2684 	.ndo_stop		= igbvf_close,
2685 	.ndo_start_xmit		= igbvf_xmit_frame,
2686 	.ndo_set_rx_mode	= igbvf_set_rx_mode,
2687 	.ndo_set_mac_address	= igbvf_set_mac,
2688 	.ndo_change_mtu		= igbvf_change_mtu,
2689 	.ndo_eth_ioctl		= igbvf_ioctl,
2690 	.ndo_tx_timeout		= igbvf_tx_timeout,
2691 	.ndo_vlan_rx_add_vid	= igbvf_vlan_rx_add_vid,
2692 	.ndo_vlan_rx_kill_vid	= igbvf_vlan_rx_kill_vid,
2693 #ifdef CONFIG_NET_POLL_CONTROLLER
2694 	.ndo_poll_controller	= igbvf_netpoll,
2695 #endif
2696 	.ndo_set_features	= igbvf_set_features,
2697 	.ndo_features_check	= igbvf_features_check,
2698 };
2699 
2700 /**
2701  * igbvf_probe - Device Initialization Routine
2702  * @pdev: PCI device information struct
2703  * @ent: entry in igbvf_pci_tbl
2704  *
2705  * Returns 0 on success, negative on failure
2706  *
2707  * igbvf_probe initializes an adapter identified by a pci_dev structure.
2708  * The OS initialization, configuring of the adapter private structure,
2709  * and a hardware reset occur.
2710  **/
2711 static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2712 {
2713 	struct net_device *netdev;
2714 	struct igbvf_adapter *adapter;
2715 	struct e1000_hw *hw;
2716 	const struct igbvf_info *ei = igbvf_info_tbl[ent->driver_data];
2717 	static int cards_found;
2718 	int err;
2719 
2720 	err = pci_enable_device_mem(pdev);
2721 	if (err)
2722 		return err;
2723 
2724 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2725 	if (err) {
2726 		dev_err(&pdev->dev,
2727 			"No usable DMA configuration, aborting\n");
2728 		goto err_dma;
2729 	}
2730 
2731 	err = pci_request_regions(pdev, igbvf_driver_name);
2732 	if (err)
2733 		goto err_pci_reg;
2734 
2735 	pci_set_master(pdev);
2736 
2737 	err = -ENOMEM;
2738 	netdev = alloc_etherdev(sizeof(struct igbvf_adapter));
2739 	if (!netdev)
2740 		goto err_alloc_etherdev;
2741 
2742 	SET_NETDEV_DEV(netdev, &pdev->dev);
2743 
2744 	pci_set_drvdata(pdev, netdev);
2745 	adapter = netdev_priv(netdev);
2746 	hw = &adapter->hw;
2747 	adapter->netdev = netdev;
2748 	adapter->pdev = pdev;
2749 	adapter->ei = ei;
2750 	adapter->pba = ei->pba;
2751 	adapter->flags = ei->flags;
2752 	adapter->hw.back = adapter;
2753 	adapter->hw.mac.type = ei->mac;
2754 	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2755 
2756 	/* PCI config space info */
2757 
2758 	hw->vendor_id = pdev->vendor;
2759 	hw->device_id = pdev->device;
2760 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
2761 	hw->subsystem_device_id = pdev->subsystem_device;
2762 	hw->revision_id = pdev->revision;
2763 
2764 	err = -EIO;
2765 	adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0),
2766 				      pci_resource_len(pdev, 0));
2767 
2768 	if (!adapter->hw.hw_addr)
2769 		goto err_ioremap;
2770 
2771 	if (ei->get_variants) {
2772 		err = ei->get_variants(adapter);
2773 		if (err)
2774 			goto err_get_variants;
2775 	}
2776 
2777 	/* setup adapter struct */
2778 	err = igbvf_sw_init(adapter);
2779 	if (err)
2780 		goto err_sw_init;
2781 
2782 	/* construct the net_device struct */
2783 	netdev->netdev_ops = &igbvf_netdev_ops;
2784 
2785 	igbvf_set_ethtool_ops(netdev);
2786 	netdev->watchdog_timeo = 5 * HZ;
2787 	strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
2788 
2789 	adapter->bd_number = cards_found++;
2790 
2791 	netdev->hw_features = NETIF_F_SG |
2792 			      NETIF_F_TSO |
2793 			      NETIF_F_TSO6 |
2794 			      NETIF_F_RXCSUM |
2795 			      NETIF_F_HW_CSUM |
2796 			      NETIF_F_SCTP_CRC;
2797 
2798 #define IGBVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
2799 				    NETIF_F_GSO_GRE_CSUM | \
2800 				    NETIF_F_GSO_IPXIP4 | \
2801 				    NETIF_F_GSO_IPXIP6 | \
2802 				    NETIF_F_GSO_UDP_TUNNEL | \
2803 				    NETIF_F_GSO_UDP_TUNNEL_CSUM)
2804 
2805 	netdev->gso_partial_features = IGBVF_GSO_PARTIAL_FEATURES;
2806 	netdev->hw_features |= NETIF_F_GSO_PARTIAL |
2807 			       IGBVF_GSO_PARTIAL_FEATURES;
2808 
2809 	netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;
2810 
2811 	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
2812 	netdev->mpls_features |= NETIF_F_HW_CSUM;
2813 	netdev->hw_enc_features |= netdev->vlan_features;
2814 
2815 	/* set this bit last since it cannot be part of vlan_features */
2816 	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2817 			    NETIF_F_HW_VLAN_CTAG_RX |
2818 			    NETIF_F_HW_VLAN_CTAG_TX;
2819 
2820 	/* MTU range: 68 - 9216 */
2821 	netdev->min_mtu = ETH_MIN_MTU;
2822 	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
2823 
2824 	spin_lock_bh(&hw->mbx_lock);
2825 
2826 	/*reset the controller to put the device in a known good state */
2827 	err = hw->mac.ops.reset_hw(hw);
2828 	if (err) {
2829 		dev_info(&pdev->dev,
2830 			 "PF still in reset state. Is the PF interface up?\n");
2831 	} else {
2832 		err = hw->mac.ops.read_mac_addr(hw);
2833 		if (err)
2834 			dev_info(&pdev->dev, "Error reading MAC address.\n");
2835 		else if (is_zero_ether_addr(adapter->hw.mac.addr))
2836 			dev_info(&pdev->dev,
2837 				 "MAC address not assigned by administrator.\n");
2838 		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2839 	}
2840 
2841 	spin_unlock_bh(&hw->mbx_lock);
2842 
2843 	if (!is_valid_ether_addr(netdev->dev_addr)) {
2844 		dev_info(&pdev->dev, "Assigning random MAC address.\n");
2845 		eth_hw_addr_random(netdev);
2846 		memcpy(adapter->hw.mac.addr, netdev->dev_addr,
2847 		       netdev->addr_len);
2848 	}
2849 
2850 	timer_setup(&adapter->watchdog_timer, igbvf_watchdog, 0);
2851 
2852 	INIT_WORK(&adapter->reset_task, igbvf_reset_task);
2853 	INIT_WORK(&adapter->watchdog_task, igbvf_watchdog_task);
2854 
2855 	/* ring size defaults */
2856 	adapter->rx_ring->count = 1024;
2857 	adapter->tx_ring->count = 1024;
2858 
2859 	/* reset the hardware with the new settings */
2860 	igbvf_reset(adapter);
2861 
2862 	/* set hardware-specific flags */
2863 	if (adapter->hw.mac.type == e1000_vfadapt_i350)
2864 		adapter->flags |= IGBVF_FLAG_RX_LB_VLAN_BSWAP;
2865 
2866 	strcpy(netdev->name, "eth%d");
2867 	err = register_netdev(netdev);
2868 	if (err)
2869 		goto err_hw_init;
2870 
2871 	/* tell the stack to leave us alone until igbvf_open() is called */
2872 	netif_carrier_off(netdev);
2873 	netif_stop_queue(netdev);
2874 
2875 	igbvf_print_device_info(adapter);
2876 
2877 	igbvf_initialize_last_counter_stats(adapter);
2878 
2879 	return 0;
2880 
2881 err_hw_init:
2882 	netif_napi_del(&adapter->rx_ring->napi);
2883 	kfree(adapter->tx_ring);
2884 	kfree(adapter->rx_ring);
2885 err_sw_init:
2886 	igbvf_reset_interrupt_capability(adapter);
2887 err_get_variants:
2888 	iounmap(adapter->hw.hw_addr);
2889 err_ioremap:
2890 	free_netdev(netdev);
2891 err_alloc_etherdev:
2892 	pci_release_regions(pdev);
2893 err_pci_reg:
2894 err_dma:
2895 	pci_disable_device(pdev);
2896 	return err;
2897 }
2898 
2899 /**
2900  * igbvf_remove - Device Removal Routine
2901  * @pdev: PCI device information struct
2902  *
2903  * igbvf_remove is called by the PCI subsystem to alert the driver
2904  * that it should release a PCI device.  The could be caused by a
2905  * Hot-Plug event, or because the driver is going to be removed from
2906  * memory.
2907  **/
2908 static void igbvf_remove(struct pci_dev *pdev)
2909 {
2910 	struct net_device *netdev = pci_get_drvdata(pdev);
2911 	struct igbvf_adapter *adapter = netdev_priv(netdev);
2912 	struct e1000_hw *hw = &adapter->hw;
2913 
2914 	/* The watchdog timer may be rescheduled, so explicitly
2915 	 * disable it from being rescheduled.
2916 	 */
2917 	set_bit(__IGBVF_DOWN, &adapter->state);
2918 	del_timer_sync(&adapter->watchdog_timer);
2919 
2920 	cancel_work_sync(&adapter->reset_task);
2921 	cancel_work_sync(&adapter->watchdog_task);
2922 
2923 	unregister_netdev(netdev);
2924 
2925 	igbvf_reset_interrupt_capability(adapter);
2926 
2927 	/* it is important to delete the NAPI struct prior to freeing the
2928 	 * Rx ring so that you do not end up with null pointer refs
2929 	 */
2930 	netif_napi_del(&adapter->rx_ring->napi);
2931 	kfree(adapter->tx_ring);
2932 	kfree(adapter->rx_ring);
2933 
2934 	iounmap(hw->hw_addr);
2935 	if (hw->flash_address)
2936 		iounmap(hw->flash_address);
2937 	pci_release_regions(pdev);
2938 
2939 	free_netdev(netdev);
2940 
2941 	pci_disable_device(pdev);
2942 }
2943 
2944 /* PCI Error Recovery (ERS) */
2945 static const struct pci_error_handlers igbvf_err_handler = {
2946 	.error_detected = igbvf_io_error_detected,
2947 	.slot_reset = igbvf_io_slot_reset,
2948 	.resume = igbvf_io_resume,
2949 	.reset_prepare = igbvf_io_prepare,
2950 	.reset_done = igbvf_io_reset_done,
2951 };
2952 
2953 static const struct pci_device_id igbvf_pci_tbl[] = {
2954 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), board_vf },
2955 	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), board_i350_vf },
2956 	{ } /* terminate list */
2957 };
2958 MODULE_DEVICE_TABLE(pci, igbvf_pci_tbl);
2959 
2960 static SIMPLE_DEV_PM_OPS(igbvf_pm_ops, igbvf_suspend, igbvf_resume);
2961 
2962 /* PCI Device API Driver */
2963 static struct pci_driver igbvf_driver = {
2964 	.name		= igbvf_driver_name,
2965 	.id_table	= igbvf_pci_tbl,
2966 	.probe		= igbvf_probe,
2967 	.remove		= igbvf_remove,
2968 	.driver.pm	= &igbvf_pm_ops,
2969 	.shutdown	= igbvf_shutdown,
2970 	.err_handler	= &igbvf_err_handler
2971 };
2972 
2973 /**
2974  * igbvf_init_module - Driver Registration Routine
2975  *
2976  * igbvf_init_module is the first routine called when the driver is
2977  * loaded. All it does is register with the PCI subsystem.
2978  **/
2979 static int __init igbvf_init_module(void)
2980 {
2981 	int ret;
2982 
2983 	pr_info("%s\n", igbvf_driver_string);
2984 	pr_info("%s\n", igbvf_copyright);
2985 
2986 	ret = pci_register_driver(&igbvf_driver);
2987 
2988 	return ret;
2989 }
2990 module_init(igbvf_init_module);
2991 
2992 /**
2993  * igbvf_exit_module - Driver Exit Cleanup Routine
2994  *
2995  * igbvf_exit_module is called just before the driver is removed
2996  * from memory.
2997  **/
2998 static void __exit igbvf_exit_module(void)
2999 {
3000 	pci_unregister_driver(&igbvf_driver);
3001 }
3002 module_exit(igbvf_exit_module);
3003 
3004 MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
3005 MODULE_DESCRIPTION("Intel(R) Gigabit Virtual Function Network Driver");
3006 MODULE_LICENSE("GPL v2");
3007 
3008 /* netdev.c */
3009