1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2019 Intel Corporation. */
3 
4 #include <linux/types.h>
5 #include <linux/module.h>
6 #include <net/ipv6.h>
7 #include <net/ip.h>
8 #include <net/tcp.h>
9 #include <linux/if_macvlan.h>
10 #include <linux/prefetch.h>
11 
12 #include "fm10k.h"
13 
14 #define DRV_VERSION	"0.26.1-k"
15 #define DRV_SUMMARY	"Intel(R) Ethernet Switch Host Interface Driver"
16 const char fm10k_driver_version[] = DRV_VERSION;
17 char fm10k_driver_name[] = "fm10k";
18 static const char fm10k_driver_string[] = DRV_SUMMARY;
19 static const char fm10k_copyright[] =
20 	"Copyright(c) 2013 - 2019 Intel Corporation.";
21 
22 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
23 MODULE_DESCRIPTION(DRV_SUMMARY);
24 MODULE_LICENSE("GPL v2");
25 MODULE_VERSION(DRV_VERSION);
26 
27 /* single workqueue for entire fm10k driver */
28 struct workqueue_struct *fm10k_workqueue;
29 
30 /**
31  * fm10k_init_module - Driver Registration Routine
32  *
33  * fm10k_init_module is the first routine called when the driver is
34  * loaded.  All it does is register with the PCI subsystem.
35  **/
36 static int __init fm10k_init_module(void)
37 {
38 	pr_info("%s - version %s\n", fm10k_driver_string, fm10k_driver_version);
39 	pr_info("%s\n", fm10k_copyright);
40 
41 	/* create driver workqueue */
42 	fm10k_workqueue = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0,
43 					  fm10k_driver_name);
44 	if (!fm10k_workqueue)
45 		return -ENOMEM;
46 
47 	fm10k_dbg_init();
48 
49 	return fm10k_register_pci_driver();
50 }
51 module_init(fm10k_init_module);
52 
53 /**
54  * fm10k_exit_module - Driver Exit Cleanup Routine
55  *
56  * fm10k_exit_module is called just before the driver is removed
57  * from memory.
58  **/
59 static void __exit fm10k_exit_module(void)
60 {
61 	fm10k_unregister_pci_driver();
62 
63 	fm10k_dbg_exit();
64 
65 	/* destroy driver workqueue */
66 	destroy_workqueue(fm10k_workqueue);
67 }
68 module_exit(fm10k_exit_module);
69 
70 static bool fm10k_alloc_mapped_page(struct fm10k_ring *rx_ring,
71 				    struct fm10k_rx_buffer *bi)
72 {
73 	struct page *page = bi->page;
74 	dma_addr_t dma;
75 
76 	/* Only page will be NULL if buffer was consumed */
77 	if (likely(page))
78 		return true;
79 
80 	/* alloc new page for storage */
81 	page = dev_alloc_page();
82 	if (unlikely(!page)) {
83 		rx_ring->rx_stats.alloc_failed++;
84 		return false;
85 	}
86 
87 	/* map page for use */
88 	dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
89 
90 	/* if mapping failed free memory back to system since
91 	 * there isn't much point in holding memory we can't use
92 	 */
93 	if (dma_mapping_error(rx_ring->dev, dma)) {
94 		__free_page(page);
95 
96 		rx_ring->rx_stats.alloc_failed++;
97 		return false;
98 	}
99 
100 	bi->dma = dma;
101 	bi->page = page;
102 	bi->page_offset = 0;
103 
104 	return true;
105 }
106 
107 /**
108  * fm10k_alloc_rx_buffers - Replace used receive buffers
109  * @rx_ring: ring to place buffers on
110  * @cleaned_count: number of buffers to replace
111  **/
112 void fm10k_alloc_rx_buffers(struct fm10k_ring *rx_ring, u16 cleaned_count)
113 {
114 	union fm10k_rx_desc *rx_desc;
115 	struct fm10k_rx_buffer *bi;
116 	u16 i = rx_ring->next_to_use;
117 
118 	/* nothing to do */
119 	if (!cleaned_count)
120 		return;
121 
122 	rx_desc = FM10K_RX_DESC(rx_ring, i);
123 	bi = &rx_ring->rx_buffer[i];
124 	i -= rx_ring->count;
125 
126 	do {
127 		if (!fm10k_alloc_mapped_page(rx_ring, bi))
128 			break;
129 
130 		/* Refresh the desc even if buffer_addrs didn't change
131 		 * because each write-back erases this info.
132 		 */
133 		rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
134 
135 		rx_desc++;
136 		bi++;
137 		i++;
138 		if (unlikely(!i)) {
139 			rx_desc = FM10K_RX_DESC(rx_ring, 0);
140 			bi = rx_ring->rx_buffer;
141 			i -= rx_ring->count;
142 		}
143 
144 		/* clear the status bits for the next_to_use descriptor */
145 		rx_desc->d.staterr = 0;
146 
147 		cleaned_count--;
148 	} while (cleaned_count);
149 
150 	i += rx_ring->count;
151 
152 	if (rx_ring->next_to_use != i) {
153 		/* record the next descriptor to use */
154 		rx_ring->next_to_use = i;
155 
156 		/* update next to alloc since we have filled the ring */
157 		rx_ring->next_to_alloc = i;
158 
159 		/* Force memory writes to complete before letting h/w
160 		 * know there are new descriptors to fetch.  (Only
161 		 * applicable for weak-ordered memory model archs,
162 		 * such as IA-64).
163 		 */
164 		wmb();
165 
166 		/* notify hardware of new descriptors */
167 		writel(i, rx_ring->tail);
168 	}
169 }
170 
171 /**
172  * fm10k_reuse_rx_page - page flip buffer and store it back on the ring
173  * @rx_ring: rx descriptor ring to store buffers on
174  * @old_buff: donor buffer to have page reused
175  *
176  * Synchronizes page for reuse by the interface
177  **/
178 static void fm10k_reuse_rx_page(struct fm10k_ring *rx_ring,
179 				struct fm10k_rx_buffer *old_buff)
180 {
181 	struct fm10k_rx_buffer *new_buff;
182 	u16 nta = rx_ring->next_to_alloc;
183 
184 	new_buff = &rx_ring->rx_buffer[nta];
185 
186 	/* update, and store next to alloc */
187 	nta++;
188 	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
189 
190 	/* transfer page from old buffer to new buffer */
191 	*new_buff = *old_buff;
192 
193 	/* sync the buffer for use by the device */
194 	dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma,
195 					 old_buff->page_offset,
196 					 FM10K_RX_BUFSZ,
197 					 DMA_FROM_DEVICE);
198 }
199 
200 static inline bool fm10k_page_is_reserved(struct page *page)
201 {
202 	return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
203 }
204 
205 static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer,
206 				    struct page *page,
207 				    unsigned int __maybe_unused truesize)
208 {
209 	/* avoid re-using remote pages */
210 	if (unlikely(fm10k_page_is_reserved(page)))
211 		return false;
212 
213 #if (PAGE_SIZE < 8192)
214 	/* if we are only owner of page we can reuse it */
215 	if (unlikely(page_count(page) != 1))
216 		return false;
217 
218 	/* flip page offset to other buffer */
219 	rx_buffer->page_offset ^= FM10K_RX_BUFSZ;
220 #else
221 	/* move offset up to the next cache line */
222 	rx_buffer->page_offset += truesize;
223 
224 	if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ))
225 		return false;
226 #endif
227 
228 	/* Even if we own the page, we are not allowed to use atomic_set()
229 	 * This would break get_page_unless_zero() users.
230 	 */
231 	page_ref_inc(page);
232 
233 	return true;
234 }
235 
236 /**
237  * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff
238  * @rx_buffer: buffer containing page to add
239  * @size: packet size from rx_desc
240  * @rx_desc: descriptor containing length of buffer written by hardware
241  * @skb: sk_buff to place the data into
242  *
243  * This function will add the data contained in rx_buffer->page to the skb.
244  * This is done either through a direct copy if the data in the buffer is
245  * less than the skb header size, otherwise it will just attach the page as
246  * a frag to the skb.
247  *
248  * The function will then update the page offset if necessary and return
249  * true if the buffer can be reused by the interface.
250  **/
251 static bool fm10k_add_rx_frag(struct fm10k_rx_buffer *rx_buffer,
252 			      unsigned int size,
253 			      union fm10k_rx_desc *rx_desc,
254 			      struct sk_buff *skb)
255 {
256 	struct page *page = rx_buffer->page;
257 	unsigned char *va = page_address(page) + rx_buffer->page_offset;
258 #if (PAGE_SIZE < 8192)
259 	unsigned int truesize = FM10K_RX_BUFSZ;
260 #else
261 	unsigned int truesize = ALIGN(size, 512);
262 #endif
263 	unsigned int pull_len;
264 
265 	if (unlikely(skb_is_nonlinear(skb)))
266 		goto add_tail_frag;
267 
268 	if (likely(size <= FM10K_RX_HDR_LEN)) {
269 		memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
270 
271 		/* page is not reserved, we can reuse buffer as-is */
272 		if (likely(!fm10k_page_is_reserved(page)))
273 			return true;
274 
275 		/* this page cannot be reused so discard it */
276 		__free_page(page);
277 		return false;
278 	}
279 
280 	/* we need the header to contain the greater of either ETH_HLEN or
281 	 * 60 bytes if the skb->len is less than 60 for skb_pad.
282 	 */
283 	pull_len = eth_get_headlen(skb->dev, va, FM10K_RX_HDR_LEN);
284 
285 	/* align pull length to size of long to optimize memcpy performance */
286 	memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));
287 
288 	/* update all of the pointers */
289 	va += pull_len;
290 	size -= pull_len;
291 
292 add_tail_frag:
293 	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
294 			(unsigned long)va & ~PAGE_MASK, size, truesize);
295 
296 	return fm10k_can_reuse_rx_page(rx_buffer, page, truesize);
297 }
298 
299 static struct sk_buff *fm10k_fetch_rx_buffer(struct fm10k_ring *rx_ring,
300 					     union fm10k_rx_desc *rx_desc,
301 					     struct sk_buff *skb)
302 {
303 	unsigned int size = le16_to_cpu(rx_desc->w.length);
304 	struct fm10k_rx_buffer *rx_buffer;
305 	struct page *page;
306 
307 	rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean];
308 	page = rx_buffer->page;
309 	prefetchw(page);
310 
311 	if (likely(!skb)) {
312 		void *page_addr = page_address(page) +
313 				  rx_buffer->page_offset;
314 
315 		/* prefetch first cache line of first page */
316 		prefetch(page_addr);
317 #if L1_CACHE_BYTES < 128
318 		prefetch((void *)((u8 *)page_addr + L1_CACHE_BYTES));
319 #endif
320 
321 		/* allocate a skb to store the frags */
322 		skb = napi_alloc_skb(&rx_ring->q_vector->napi,
323 				     FM10K_RX_HDR_LEN);
324 		if (unlikely(!skb)) {
325 			rx_ring->rx_stats.alloc_failed++;
326 			return NULL;
327 		}
328 
329 		/* we will be copying header into skb->data in
330 		 * pskb_may_pull so it is in our interest to prefetch
331 		 * it now to avoid a possible cache miss
332 		 */
333 		prefetchw(skb->data);
334 	}
335 
336 	/* we are reusing so sync this buffer for CPU use */
337 	dma_sync_single_range_for_cpu(rx_ring->dev,
338 				      rx_buffer->dma,
339 				      rx_buffer->page_offset,
340 				      size,
341 				      DMA_FROM_DEVICE);
342 
343 	/* pull page into skb */
344 	if (fm10k_add_rx_frag(rx_buffer, size, rx_desc, skb)) {
345 		/* hand second half of page back to the ring */
346 		fm10k_reuse_rx_page(rx_ring, rx_buffer);
347 	} else {
348 		/* we are not reusing the buffer so unmap it */
349 		dma_unmap_page(rx_ring->dev, rx_buffer->dma,
350 			       PAGE_SIZE, DMA_FROM_DEVICE);
351 	}
352 
353 	/* clear contents of rx_buffer */
354 	rx_buffer->page = NULL;
355 
356 	return skb;
357 }
358 
359 static inline void fm10k_rx_checksum(struct fm10k_ring *ring,
360 				     union fm10k_rx_desc *rx_desc,
361 				     struct sk_buff *skb)
362 {
363 	skb_checksum_none_assert(skb);
364 
365 	/* Rx checksum disabled via ethtool */
366 	if (!(ring->netdev->features & NETIF_F_RXCSUM))
367 		return;
368 
369 	/* TCP/UDP checksum error bit is set */
370 	if (fm10k_test_staterr(rx_desc,
371 			       FM10K_RXD_STATUS_L4E |
372 			       FM10K_RXD_STATUS_L4E2 |
373 			       FM10K_RXD_STATUS_IPE |
374 			       FM10K_RXD_STATUS_IPE2)) {
375 		ring->rx_stats.csum_err++;
376 		return;
377 	}
378 
379 	/* It must be a TCP or UDP packet with a valid checksum */
380 	if (fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS2))
381 		skb->encapsulation = true;
382 	else if (!fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS))
383 		return;
384 
385 	skb->ip_summed = CHECKSUM_UNNECESSARY;
386 
387 	ring->rx_stats.csum_good++;
388 }
389 
390 #define FM10K_RSS_L4_TYPES_MASK \
391 	(BIT(FM10K_RSSTYPE_IPV4_TCP) | \
392 	 BIT(FM10K_RSSTYPE_IPV4_UDP) | \
393 	 BIT(FM10K_RSSTYPE_IPV6_TCP) | \
394 	 BIT(FM10K_RSSTYPE_IPV6_UDP))
395 
396 static inline void fm10k_rx_hash(struct fm10k_ring *ring,
397 				 union fm10k_rx_desc *rx_desc,
398 				 struct sk_buff *skb)
399 {
400 	u16 rss_type;
401 
402 	if (!(ring->netdev->features & NETIF_F_RXHASH))
403 		return;
404 
405 	rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK;
406 	if (!rss_type)
407 		return;
408 
409 	skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss),
410 		     (BIT(rss_type) & FM10K_RSS_L4_TYPES_MASK) ?
411 		     PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
412 }
413 
414 static void fm10k_type_trans(struct fm10k_ring *rx_ring,
415 			     union fm10k_rx_desc __maybe_unused *rx_desc,
416 			     struct sk_buff *skb)
417 {
418 	struct net_device *dev = rx_ring->netdev;
419 	struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel);
420 
421 	/* check to see if DGLORT belongs to a MACVLAN */
422 	if (l2_accel) {
423 		u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1;
424 
425 		idx -= l2_accel->dglort;
426 		if (idx < l2_accel->size && l2_accel->macvlan[idx])
427 			dev = l2_accel->macvlan[idx];
428 		else
429 			l2_accel = NULL;
430 	}
431 
432 	/* Record Rx queue, or update macvlan statistics */
433 	if (!l2_accel)
434 		skb_record_rx_queue(skb, rx_ring->queue_index);
435 	else
436 		macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,
437 				 false);
438 
439 	skb->protocol = eth_type_trans(skb, dev);
440 }
441 
442 /**
443  * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor
444  * @rx_ring: rx descriptor ring packet is being transacted on
445  * @rx_desc: pointer to the EOP Rx descriptor
446  * @skb: pointer to current skb being populated
447  *
448  * This function checks the ring, descriptor, and packet information in
449  * order to populate the hash, checksum, VLAN, timestamp, protocol, and
450  * other fields within the skb.
451  **/
452 static unsigned int fm10k_process_skb_fields(struct fm10k_ring *rx_ring,
453 					     union fm10k_rx_desc *rx_desc,
454 					     struct sk_buff *skb)
455 {
456 	unsigned int len = skb->len;
457 
458 	fm10k_rx_hash(rx_ring, rx_desc, skb);
459 
460 	fm10k_rx_checksum(rx_ring, rx_desc, skb);
461 
462 	FM10K_CB(skb)->tstamp = rx_desc->q.timestamp;
463 
464 	FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan;
465 
466 	FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort;
467 
468 	if (rx_desc->w.vlan) {
469 		u16 vid = le16_to_cpu(rx_desc->w.vlan);
470 
471 		if ((vid & VLAN_VID_MASK) != rx_ring->vid)
472 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
473 		else if (vid & VLAN_PRIO_MASK)
474 			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
475 					       vid & VLAN_PRIO_MASK);
476 	}
477 
478 	fm10k_type_trans(rx_ring, rx_desc, skb);
479 
480 	return len;
481 }
482 
483 /**
484  * fm10k_is_non_eop - process handling of non-EOP buffers
485  * @rx_ring: Rx ring being processed
486  * @rx_desc: Rx descriptor for current buffer
487  *
488  * This function updates next to clean.  If the buffer is an EOP buffer
489  * this function exits returning false, otherwise it will place the
490  * sk_buff in the next buffer to be chained and return true indicating
491  * that this is in fact a non-EOP buffer.
492  **/
493 static bool fm10k_is_non_eop(struct fm10k_ring *rx_ring,
494 			     union fm10k_rx_desc *rx_desc)
495 {
496 	u32 ntc = rx_ring->next_to_clean + 1;
497 
498 	/* fetch, update, and store next to clean */
499 	ntc = (ntc < rx_ring->count) ? ntc : 0;
500 	rx_ring->next_to_clean = ntc;
501 
502 	prefetch(FM10K_RX_DESC(rx_ring, ntc));
503 
504 	if (likely(fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_EOP)))
505 		return false;
506 
507 	return true;
508 }
509 
510 /**
511  * fm10k_cleanup_headers - Correct corrupted or empty headers
512  * @rx_ring: rx descriptor ring packet is being transacted on
513  * @rx_desc: pointer to the EOP Rx descriptor
514  * @skb: pointer to current skb being fixed
515  *
516  * Address the case where we are pulling data in on pages only
517  * and as such no data is present in the skb header.
518  *
519  * In addition if skb is not at least 60 bytes we need to pad it so that
520  * it is large enough to qualify as a valid Ethernet frame.
521  *
522  * Returns true if an error was encountered and skb was freed.
523  **/
524 static bool fm10k_cleanup_headers(struct fm10k_ring *rx_ring,
525 				  union fm10k_rx_desc *rx_desc,
526 				  struct sk_buff *skb)
527 {
528 	if (unlikely((fm10k_test_staterr(rx_desc,
529 					 FM10K_RXD_STATUS_RXE)))) {
530 #define FM10K_TEST_RXD_BIT(rxd, bit) \
531 	((rxd)->w.csum_err & cpu_to_le16(bit))
532 		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_SWITCH_ERROR))
533 			rx_ring->rx_stats.switch_errors++;
534 		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_NO_DESCRIPTOR))
535 			rx_ring->rx_stats.drops++;
536 		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_PP_ERROR))
537 			rx_ring->rx_stats.pp_errors++;
538 		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_SWITCH_READY))
539 			rx_ring->rx_stats.link_errors++;
540 		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_TOO_BIG))
541 			rx_ring->rx_stats.length_errors++;
542 		dev_kfree_skb_any(skb);
543 		rx_ring->rx_stats.errors++;
544 		return true;
545 	}
546 
547 	/* if eth_skb_pad returns an error the skb was freed */
548 	if (eth_skb_pad(skb))
549 		return true;
550 
551 	return false;
552 }
553 
554 /**
555  * fm10k_receive_skb - helper function to handle rx indications
556  * @q_vector: structure containing interrupt and ring information
557  * @skb: packet to send up
558  **/
559 static void fm10k_receive_skb(struct fm10k_q_vector *q_vector,
560 			      struct sk_buff *skb)
561 {
562 	napi_gro_receive(&q_vector->napi, skb);
563 }
564 
565 static int fm10k_clean_rx_irq(struct fm10k_q_vector *q_vector,
566 			      struct fm10k_ring *rx_ring,
567 			      int budget)
568 {
569 	struct sk_buff *skb = rx_ring->skb;
570 	unsigned int total_bytes = 0, total_packets = 0;
571 	u16 cleaned_count = fm10k_desc_unused(rx_ring);
572 
573 	while (likely(total_packets < budget)) {
574 		union fm10k_rx_desc *rx_desc;
575 
576 		/* return some buffers to hardware, one at a time is too slow */
577 		if (cleaned_count >= FM10K_RX_BUFFER_WRITE) {
578 			fm10k_alloc_rx_buffers(rx_ring, cleaned_count);
579 			cleaned_count = 0;
580 		}
581 
582 		rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean);
583 
584 		if (!rx_desc->d.staterr)
585 			break;
586 
587 		/* This memory barrier is needed to keep us from reading
588 		 * any other fields out of the rx_desc until we know the
589 		 * descriptor has been written back
590 		 */
591 		dma_rmb();
592 
593 		/* retrieve a buffer from the ring */
594 		skb = fm10k_fetch_rx_buffer(rx_ring, rx_desc, skb);
595 
596 		/* exit if we failed to retrieve a buffer */
597 		if (!skb)
598 			break;
599 
600 		cleaned_count++;
601 
602 		/* fetch next buffer in frame if non-eop */
603 		if (fm10k_is_non_eop(rx_ring, rx_desc))
604 			continue;
605 
606 		/* verify the packet layout is correct */
607 		if (fm10k_cleanup_headers(rx_ring, rx_desc, skb)) {
608 			skb = NULL;
609 			continue;
610 		}
611 
612 		/* populate checksum, timestamp, VLAN, and protocol */
613 		total_bytes += fm10k_process_skb_fields(rx_ring, rx_desc, skb);
614 
615 		fm10k_receive_skb(q_vector, skb);
616 
617 		/* reset skb pointer */
618 		skb = NULL;
619 
620 		/* update budget accounting */
621 		total_packets++;
622 	}
623 
624 	/* place incomplete frames back on ring for completion */
625 	rx_ring->skb = skb;
626 
627 	u64_stats_update_begin(&rx_ring->syncp);
628 	rx_ring->stats.packets += total_packets;
629 	rx_ring->stats.bytes += total_bytes;
630 	u64_stats_update_end(&rx_ring->syncp);
631 	q_vector->rx.total_packets += total_packets;
632 	q_vector->rx.total_bytes += total_bytes;
633 
634 	return total_packets;
635 }
636 
637 #define VXLAN_HLEN (sizeof(struct udphdr) + 8)
638 static struct ethhdr *fm10k_port_is_vxlan(struct sk_buff *skb)
639 {
640 	struct fm10k_intfc *interface = netdev_priv(skb->dev);
641 	struct fm10k_udp_port *vxlan_port;
642 
643 	/* we can only offload a vxlan if we recognize it as such */
644 	vxlan_port = list_first_entry_or_null(&interface->vxlan_port,
645 					      struct fm10k_udp_port, list);
646 
647 	if (!vxlan_port)
648 		return NULL;
649 	if (vxlan_port->port != udp_hdr(skb)->dest)
650 		return NULL;
651 
652 	/* return offset of udp_hdr plus 8 bytes for VXLAN header */
653 	return (struct ethhdr *)(skb_transport_header(skb) + VXLAN_HLEN);
654 }
655 
656 #define FM10K_NVGRE_RESERVED0_FLAGS htons(0x9FFF)
657 #define NVGRE_TNI htons(0x2000)
658 struct fm10k_nvgre_hdr {
659 	__be16 flags;
660 	__be16 proto;
661 	__be32 tni;
662 };
663 
664 static struct ethhdr *fm10k_gre_is_nvgre(struct sk_buff *skb)
665 {
666 	struct fm10k_nvgre_hdr *nvgre_hdr;
667 	int hlen = ip_hdrlen(skb);
668 
669 	/* currently only IPv4 is supported due to hlen above */
670 	if (vlan_get_protocol(skb) != htons(ETH_P_IP))
671 		return NULL;
672 
673 	/* our transport header should be NVGRE */
674 	nvgre_hdr = (struct fm10k_nvgre_hdr *)(skb_network_header(skb) + hlen);
675 
676 	/* verify all reserved flags are 0 */
677 	if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS)
678 		return NULL;
679 
680 	/* report start of ethernet header */
681 	if (nvgre_hdr->flags & NVGRE_TNI)
682 		return (struct ethhdr *)(nvgre_hdr + 1);
683 
684 	return (struct ethhdr *)(&nvgre_hdr->tni);
685 }
686 
687 __be16 fm10k_tx_encap_offload(struct sk_buff *skb)
688 {
689 	u8 l4_hdr = 0, inner_l4_hdr = 0, inner_l4_hlen;
690 	struct ethhdr *eth_hdr;
691 
692 	if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
693 	    skb->inner_protocol != htons(ETH_P_TEB))
694 		return 0;
695 
696 	switch (vlan_get_protocol(skb)) {
697 	case htons(ETH_P_IP):
698 		l4_hdr = ip_hdr(skb)->protocol;
699 		break;
700 	case htons(ETH_P_IPV6):
701 		l4_hdr = ipv6_hdr(skb)->nexthdr;
702 		break;
703 	default:
704 		return 0;
705 	}
706 
707 	switch (l4_hdr) {
708 	case IPPROTO_UDP:
709 		eth_hdr = fm10k_port_is_vxlan(skb);
710 		break;
711 	case IPPROTO_GRE:
712 		eth_hdr = fm10k_gre_is_nvgre(skb);
713 		break;
714 	default:
715 		return 0;
716 	}
717 
718 	if (!eth_hdr)
719 		return 0;
720 
721 	switch (eth_hdr->h_proto) {
722 	case htons(ETH_P_IP):
723 		inner_l4_hdr = inner_ip_hdr(skb)->protocol;
724 		break;
725 	case htons(ETH_P_IPV6):
726 		inner_l4_hdr = inner_ipv6_hdr(skb)->nexthdr;
727 		break;
728 	default:
729 		return 0;
730 	}
731 
732 	switch (inner_l4_hdr) {
733 	case IPPROTO_TCP:
734 		inner_l4_hlen = inner_tcp_hdrlen(skb);
735 		break;
736 	case IPPROTO_UDP:
737 		inner_l4_hlen = 8;
738 		break;
739 	default:
740 		return 0;
741 	}
742 
743 	/* The hardware allows tunnel offloads only if the combined inner and
744 	 * outer header is 184 bytes or less
745 	 */
746 	if (skb_inner_transport_header(skb) + inner_l4_hlen -
747 	    skb_mac_header(skb) > FM10K_TUNNEL_HEADER_LENGTH)
748 		return 0;
749 
750 	return eth_hdr->h_proto;
751 }
752 
753 static int fm10k_tso(struct fm10k_ring *tx_ring,
754 		     struct fm10k_tx_buffer *first)
755 {
756 	struct sk_buff *skb = first->skb;
757 	struct fm10k_tx_desc *tx_desc;
758 	unsigned char *th;
759 	u8 hdrlen;
760 
761 	if (skb->ip_summed != CHECKSUM_PARTIAL)
762 		return 0;
763 
764 	if (!skb_is_gso(skb))
765 		return 0;
766 
767 	/* compute header lengths */
768 	if (skb->encapsulation) {
769 		if (!fm10k_tx_encap_offload(skb))
770 			goto err_vxlan;
771 		th = skb_inner_transport_header(skb);
772 	} else {
773 		th = skb_transport_header(skb);
774 	}
775 
776 	/* compute offset from SOF to transport header and add header len */
777 	hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2);
778 
779 	first->tx_flags |= FM10K_TX_FLAGS_CSUM;
780 
781 	/* update gso size and bytecount with header size */
782 	first->gso_segs = skb_shinfo(skb)->gso_segs;
783 	first->bytecount += (first->gso_segs - 1) * hdrlen;
784 
785 	/* populate Tx descriptor header size and mss */
786 	tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use);
787 	tx_desc->hdrlen = hdrlen;
788 	tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
789 
790 	return 1;
791 
792 err_vxlan:
793 	tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL;
794 	if (net_ratelimit())
795 		netdev_err(tx_ring->netdev,
796 			   "TSO requested for unsupported tunnel, disabling offload\n");
797 	return -1;
798 }
799 
800 static void fm10k_tx_csum(struct fm10k_ring *tx_ring,
801 			  struct fm10k_tx_buffer *first)
802 {
803 	struct sk_buff *skb = first->skb;
804 	struct fm10k_tx_desc *tx_desc;
805 	union {
806 		struct iphdr *ipv4;
807 		struct ipv6hdr *ipv6;
808 		u8 *raw;
809 	} network_hdr;
810 	u8 *transport_hdr;
811 	__be16 frag_off;
812 	__be16 protocol;
813 	u8 l4_hdr = 0;
814 
815 	if (skb->ip_summed != CHECKSUM_PARTIAL)
816 		goto no_csum;
817 
818 	if (skb->encapsulation) {
819 		protocol = fm10k_tx_encap_offload(skb);
820 		if (!protocol) {
821 			if (skb_checksum_help(skb)) {
822 				dev_warn(tx_ring->dev,
823 					 "failed to offload encap csum!\n");
824 				tx_ring->tx_stats.csum_err++;
825 			}
826 			goto no_csum;
827 		}
828 		network_hdr.raw = skb_inner_network_header(skb);
829 		transport_hdr = skb_inner_transport_header(skb);
830 	} else {
831 		protocol = vlan_get_protocol(skb);
832 		network_hdr.raw = skb_network_header(skb);
833 		transport_hdr = skb_transport_header(skb);
834 	}
835 
836 	switch (protocol) {
837 	case htons(ETH_P_IP):
838 		l4_hdr = network_hdr.ipv4->protocol;
839 		break;
840 	case htons(ETH_P_IPV6):
841 		l4_hdr = network_hdr.ipv6->nexthdr;
842 		if (likely((transport_hdr - network_hdr.raw) ==
843 			   sizeof(struct ipv6hdr)))
844 			break;
845 		ipv6_skip_exthdr(skb, network_hdr.raw - skb->data +
846 				      sizeof(struct ipv6hdr),
847 				 &l4_hdr, &frag_off);
848 		if (unlikely(frag_off))
849 			l4_hdr = NEXTHDR_FRAGMENT;
850 		break;
851 	default:
852 		break;
853 	}
854 
855 	switch (l4_hdr) {
856 	case IPPROTO_TCP:
857 	case IPPROTO_UDP:
858 		break;
859 	case IPPROTO_GRE:
860 		if (skb->encapsulation)
861 			break;
862 		/* fall through */
863 	default:
864 		if (unlikely(net_ratelimit())) {
865 			dev_warn(tx_ring->dev,
866 				 "partial checksum, version=%d l4 proto=%x\n",
867 				 protocol, l4_hdr);
868 		}
869 		skb_checksum_help(skb);
870 		tx_ring->tx_stats.csum_err++;
871 		goto no_csum;
872 	}
873 
874 	/* update TX checksum flag */
875 	first->tx_flags |= FM10K_TX_FLAGS_CSUM;
876 	tx_ring->tx_stats.csum_good++;
877 
878 no_csum:
879 	/* populate Tx descriptor header size and mss */
880 	tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use);
881 	tx_desc->hdrlen = 0;
882 	tx_desc->mss = 0;
883 }
884 
885 #define FM10K_SET_FLAG(_input, _flag, _result) \
886 	((_flag <= _result) ? \
887 	 ((u32)(_input & _flag) * (_result / _flag)) : \
888 	 ((u32)(_input & _flag) / (_flag / _result)))
889 
890 static u8 fm10k_tx_desc_flags(struct sk_buff *skb, u32 tx_flags)
891 {
892 	/* set type for advanced descriptor with frame checksum insertion */
893 	u32 desc_flags = 0;
894 
895 	/* set checksum offload bits */
896 	desc_flags |= FM10K_SET_FLAG(tx_flags, FM10K_TX_FLAGS_CSUM,
897 				     FM10K_TXD_FLAG_CSUM);
898 
899 	return desc_flags;
900 }
901 
902 static bool fm10k_tx_desc_push(struct fm10k_ring *tx_ring,
903 			       struct fm10k_tx_desc *tx_desc, u16 i,
904 			       dma_addr_t dma, unsigned int size, u8 desc_flags)
905 {
906 	/* set RS and INT for last frame in a cache line */
907 	if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0)
908 		desc_flags |= FM10K_TXD_FLAG_RS | FM10K_TXD_FLAG_INT;
909 
910 	/* record values to descriptor */
911 	tx_desc->buffer_addr = cpu_to_le64(dma);
912 	tx_desc->flags = desc_flags;
913 	tx_desc->buflen = cpu_to_le16(size);
914 
915 	/* return true if we just wrapped the ring */
916 	return i == tx_ring->count;
917 }
918 
919 static int __fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size)
920 {
921 	netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
922 
923 	/* Memory barrier before checking head and tail */
924 	smp_mb();
925 
926 	/* Check again in a case another CPU has just made room available */
927 	if (likely(fm10k_desc_unused(tx_ring) < size))
928 		return -EBUSY;
929 
930 	/* A reprieve! - use start_queue because it doesn't call schedule */
931 	netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
932 	++tx_ring->tx_stats.restart_queue;
933 	return 0;
934 }
935 
936 static inline int fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size)
937 {
938 	if (likely(fm10k_desc_unused(tx_ring) >= size))
939 		return 0;
940 	return __fm10k_maybe_stop_tx(tx_ring, size);
941 }
942 
943 static void fm10k_tx_map(struct fm10k_ring *tx_ring,
944 			 struct fm10k_tx_buffer *first)
945 {
946 	struct sk_buff *skb = first->skb;
947 	struct fm10k_tx_buffer *tx_buffer;
948 	struct fm10k_tx_desc *tx_desc;
949 	skb_frag_t *frag;
950 	unsigned char *data;
951 	dma_addr_t dma;
952 	unsigned int data_len, size;
953 	u32 tx_flags = first->tx_flags;
954 	u16 i = tx_ring->next_to_use;
955 	u8 flags = fm10k_tx_desc_flags(skb, tx_flags);
956 
957 	tx_desc = FM10K_TX_DESC(tx_ring, i);
958 
959 	/* add HW VLAN tag */
960 	if (skb_vlan_tag_present(skb))
961 		tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb));
962 	else
963 		tx_desc->vlan = 0;
964 
965 	size = skb_headlen(skb);
966 	data = skb->data;
967 
968 	dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
969 
970 	data_len = skb->data_len;
971 	tx_buffer = first;
972 
973 	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
974 		if (dma_mapping_error(tx_ring->dev, dma))
975 			goto dma_error;
976 
977 		/* record length, and DMA address */
978 		dma_unmap_len_set(tx_buffer, len, size);
979 		dma_unmap_addr_set(tx_buffer, dma, dma);
980 
981 		while (unlikely(size > FM10K_MAX_DATA_PER_TXD)) {
982 			if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, dma,
983 					       FM10K_MAX_DATA_PER_TXD, flags)) {
984 				tx_desc = FM10K_TX_DESC(tx_ring, 0);
985 				i = 0;
986 			}
987 
988 			dma += FM10K_MAX_DATA_PER_TXD;
989 			size -= FM10K_MAX_DATA_PER_TXD;
990 		}
991 
992 		if (likely(!data_len))
993 			break;
994 
995 		if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++,
996 				       dma, size, flags)) {
997 			tx_desc = FM10K_TX_DESC(tx_ring, 0);
998 			i = 0;
999 		}
1000 
1001 		size = skb_frag_size(frag);
1002 		data_len -= size;
1003 
1004 		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
1005 				       DMA_TO_DEVICE);
1006 
1007 		tx_buffer = &tx_ring->tx_buffer[i];
1008 	}
1009 
1010 	/* write last descriptor with LAST bit set */
1011 	flags |= FM10K_TXD_FLAG_LAST;
1012 
1013 	if (fm10k_tx_desc_push(tx_ring, tx_desc, i++, dma, size, flags))
1014 		i = 0;
1015 
1016 	/* record bytecount for BQL */
1017 	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1018 
1019 	/* record SW timestamp if HW timestamp is not available */
1020 	skb_tx_timestamp(first->skb);
1021 
1022 	/* Force memory writes to complete before letting h/w know there
1023 	 * are new descriptors to fetch.  (Only applicable for weak-ordered
1024 	 * memory model archs, such as IA-64).
1025 	 *
1026 	 * We also need this memory barrier to make certain all of the
1027 	 * status bits have been updated before next_to_watch is written.
1028 	 */
1029 	wmb();
1030 
1031 	/* set next_to_watch value indicating a packet is present */
1032 	first->next_to_watch = tx_desc;
1033 
1034 	tx_ring->next_to_use = i;
1035 
1036 	/* Make sure there is space in the ring for the next send. */
1037 	fm10k_maybe_stop_tx(tx_ring, DESC_NEEDED);
1038 
1039 	/* notify HW of packet */
1040 	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1041 		writel(i, tx_ring->tail);
1042 	}
1043 
1044 	return;
1045 dma_error:
1046 	dev_err(tx_ring->dev, "TX DMA map failed\n");
1047 
1048 	/* clear dma mappings for failed tx_buffer map */
1049 	for (;;) {
1050 		tx_buffer = &tx_ring->tx_buffer[i];
1051 		fm10k_unmap_and_free_tx_resource(tx_ring, tx_buffer);
1052 		if (tx_buffer == first)
1053 			break;
1054 		if (i == 0)
1055 			i = tx_ring->count;
1056 		i--;
1057 	}
1058 
1059 	tx_ring->next_to_use = i;
1060 }
1061 
1062 netdev_tx_t fm10k_xmit_frame_ring(struct sk_buff *skb,
1063 				  struct fm10k_ring *tx_ring)
1064 {
1065 	u16 count = TXD_USE_COUNT(skb_headlen(skb));
1066 	struct fm10k_tx_buffer *first;
1067 	unsigned short f;
1068 	u32 tx_flags = 0;
1069 	int tso;
1070 
1071 	/* need: 1 descriptor per page * PAGE_SIZE/FM10K_MAX_DATA_PER_TXD,
1072 	 *       + 1 desc for skb_headlen/FM10K_MAX_DATA_PER_TXD,
1073 	 *       + 2 desc gap to keep tail from touching head
1074 	 * otherwise try next time
1075 	 */
1076 	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
1077 		skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
1078 
1079 		count += TXD_USE_COUNT(skb_frag_size(frag));
1080 	}
1081 
1082 	if (fm10k_maybe_stop_tx(tx_ring, count + 3)) {
1083 		tx_ring->tx_stats.tx_busy++;
1084 		return NETDEV_TX_BUSY;
1085 	}
1086 
1087 	/* record the location of the first descriptor for this packet */
1088 	first = &tx_ring->tx_buffer[tx_ring->next_to_use];
1089 	first->skb = skb;
1090 	first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN);
1091 	first->gso_segs = 1;
1092 
1093 	/* record initial flags and protocol */
1094 	first->tx_flags = tx_flags;
1095 
1096 	tso = fm10k_tso(tx_ring, first);
1097 	if (tso < 0)
1098 		goto out_drop;
1099 	else if (!tso)
1100 		fm10k_tx_csum(tx_ring, first);
1101 
1102 	fm10k_tx_map(tx_ring, first);
1103 
1104 	return NETDEV_TX_OK;
1105 
1106 out_drop:
1107 	dev_kfree_skb_any(first->skb);
1108 	first->skb = NULL;
1109 
1110 	return NETDEV_TX_OK;
1111 }
1112 
1113 static u64 fm10k_get_tx_completed(struct fm10k_ring *ring)
1114 {
1115 	return ring->stats.packets;
1116 }
1117 
1118 /**
1119  * fm10k_get_tx_pending - how many Tx descriptors not processed
1120  * @ring: the ring structure
1121  * @in_sw: is tx_pending being checked in SW or in HW?
1122  */
1123 u64 fm10k_get_tx_pending(struct fm10k_ring *ring, bool in_sw)
1124 {
1125 	struct fm10k_intfc *interface = ring->q_vector->interface;
1126 	struct fm10k_hw *hw = &interface->hw;
1127 	u32 head, tail;
1128 
1129 	if (likely(in_sw)) {
1130 		head = ring->next_to_clean;
1131 		tail = ring->next_to_use;
1132 	} else {
1133 		head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx));
1134 		tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx));
1135 	}
1136 
1137 	return ((head <= tail) ? tail : tail + ring->count) - head;
1138 }
1139 
1140 bool fm10k_check_tx_hang(struct fm10k_ring *tx_ring)
1141 {
1142 	u32 tx_done = fm10k_get_tx_completed(tx_ring);
1143 	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
1144 	u32 tx_pending = fm10k_get_tx_pending(tx_ring, true);
1145 
1146 	clear_check_for_tx_hang(tx_ring);
1147 
1148 	/* Check for a hung queue, but be thorough. This verifies
1149 	 * that a transmit has been completed since the previous
1150 	 * check AND there is at least one packet pending. By
1151 	 * requiring this to fail twice we avoid races with
1152 	 * clearing the ARMED bit and conditions where we
1153 	 * run the check_tx_hang logic with a transmit completion
1154 	 * pending but without time to complete it yet.
1155 	 */
1156 	if (!tx_pending || (tx_done_old != tx_done)) {
1157 		/* update completed stats and continue */
1158 		tx_ring->tx_stats.tx_done_old = tx_done;
1159 		/* reset the countdown */
1160 		clear_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state);
1161 
1162 		return false;
1163 	}
1164 
1165 	/* make sure it is true for two checks in a row */
1166 	return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state);
1167 }
1168 
1169 /**
1170  * fm10k_tx_timeout_reset - initiate reset due to Tx timeout
1171  * @interface: driver private struct
1172  **/
1173 void fm10k_tx_timeout_reset(struct fm10k_intfc *interface)
1174 {
1175 	/* Do the reset outside of interrupt context */
1176 	if (!test_bit(__FM10K_DOWN, interface->state)) {
1177 		interface->tx_timeout_count++;
1178 		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1179 		fm10k_service_event_schedule(interface);
1180 	}
1181 }
1182 
1183 /**
1184  * fm10k_clean_tx_irq - Reclaim resources after transmit completes
1185  * @q_vector: structure containing interrupt and ring information
1186  * @tx_ring: tx ring to clean
1187  * @napi_budget: Used to determine if we are in netpoll
1188  **/
1189 static bool fm10k_clean_tx_irq(struct fm10k_q_vector *q_vector,
1190 			       struct fm10k_ring *tx_ring, int napi_budget)
1191 {
1192 	struct fm10k_intfc *interface = q_vector->interface;
1193 	struct fm10k_tx_buffer *tx_buffer;
1194 	struct fm10k_tx_desc *tx_desc;
1195 	unsigned int total_bytes = 0, total_packets = 0;
1196 	unsigned int budget = q_vector->tx.work_limit;
1197 	unsigned int i = tx_ring->next_to_clean;
1198 
1199 	if (test_bit(__FM10K_DOWN, interface->state))
1200 		return true;
1201 
1202 	tx_buffer = &tx_ring->tx_buffer[i];
1203 	tx_desc = FM10K_TX_DESC(tx_ring, i);
1204 	i -= tx_ring->count;
1205 
1206 	do {
1207 		struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch;
1208 
1209 		/* if next_to_watch is not set then there is no work pending */
1210 		if (!eop_desc)
1211 			break;
1212 
1213 		/* prevent any other reads prior to eop_desc */
1214 		smp_rmb();
1215 
1216 		/* if DD is not set pending work has not been completed */
1217 		if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE))
1218 			break;
1219 
1220 		/* clear next_to_watch to prevent false hangs */
1221 		tx_buffer->next_to_watch = NULL;
1222 
1223 		/* update the statistics for this packet */
1224 		total_bytes += tx_buffer->bytecount;
1225 		total_packets += tx_buffer->gso_segs;
1226 
1227 		/* free the skb */
1228 		napi_consume_skb(tx_buffer->skb, napi_budget);
1229 
1230 		/* unmap skb header data */
1231 		dma_unmap_single(tx_ring->dev,
1232 				 dma_unmap_addr(tx_buffer, dma),
1233 				 dma_unmap_len(tx_buffer, len),
1234 				 DMA_TO_DEVICE);
1235 
1236 		/* clear tx_buffer data */
1237 		tx_buffer->skb = NULL;
1238 		dma_unmap_len_set(tx_buffer, len, 0);
1239 
1240 		/* unmap remaining buffers */
1241 		while (tx_desc != eop_desc) {
1242 			tx_buffer++;
1243 			tx_desc++;
1244 			i++;
1245 			if (unlikely(!i)) {
1246 				i -= tx_ring->count;
1247 				tx_buffer = tx_ring->tx_buffer;
1248 				tx_desc = FM10K_TX_DESC(tx_ring, 0);
1249 			}
1250 
1251 			/* unmap any remaining paged data */
1252 			if (dma_unmap_len(tx_buffer, len)) {
1253 				dma_unmap_page(tx_ring->dev,
1254 					       dma_unmap_addr(tx_buffer, dma),
1255 					       dma_unmap_len(tx_buffer, len),
1256 					       DMA_TO_DEVICE);
1257 				dma_unmap_len_set(tx_buffer, len, 0);
1258 			}
1259 		}
1260 
1261 		/* move us one more past the eop_desc for start of next pkt */
1262 		tx_buffer++;
1263 		tx_desc++;
1264 		i++;
1265 		if (unlikely(!i)) {
1266 			i -= tx_ring->count;
1267 			tx_buffer = tx_ring->tx_buffer;
1268 			tx_desc = FM10K_TX_DESC(tx_ring, 0);
1269 		}
1270 
1271 		/* issue prefetch for next Tx descriptor */
1272 		prefetch(tx_desc);
1273 
1274 		/* update budget accounting */
1275 		budget--;
1276 	} while (likely(budget));
1277 
1278 	i += tx_ring->count;
1279 	tx_ring->next_to_clean = i;
1280 	u64_stats_update_begin(&tx_ring->syncp);
1281 	tx_ring->stats.bytes += total_bytes;
1282 	tx_ring->stats.packets += total_packets;
1283 	u64_stats_update_end(&tx_ring->syncp);
1284 	q_vector->tx.total_bytes += total_bytes;
1285 	q_vector->tx.total_packets += total_packets;
1286 
1287 	if (check_for_tx_hang(tx_ring) && fm10k_check_tx_hang(tx_ring)) {
1288 		/* schedule immediate reset if we believe we hung */
1289 		struct fm10k_hw *hw = &interface->hw;
1290 
1291 		netif_err(interface, drv, tx_ring->netdev,
1292 			  "Detected Tx Unit Hang\n"
1293 			  "  Tx Queue             <%d>\n"
1294 			  "  TDH, TDT             <%x>, <%x>\n"
1295 			  "  next_to_use          <%x>\n"
1296 			  "  next_to_clean        <%x>\n",
1297 			  tx_ring->queue_index,
1298 			  fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)),
1299 			  fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)),
1300 			  tx_ring->next_to_use, i);
1301 
1302 		netif_stop_subqueue(tx_ring->netdev,
1303 				    tx_ring->queue_index);
1304 
1305 		netif_info(interface, probe, tx_ring->netdev,
1306 			   "tx hang %d detected on queue %d, resetting interface\n",
1307 			   interface->tx_timeout_count + 1,
1308 			   tx_ring->queue_index);
1309 
1310 		fm10k_tx_timeout_reset(interface);
1311 
1312 		/* the netdev is about to reset, no point in enabling stuff */
1313 		return true;
1314 	}
1315 
1316 	/* notify netdev of completed buffers */
1317 	netdev_tx_completed_queue(txring_txq(tx_ring),
1318 				  total_packets, total_bytes);
1319 
1320 #define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2)
1321 	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
1322 		     (fm10k_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
1323 		/* Make sure that anybody stopping the queue after this
1324 		 * sees the new next_to_clean.
1325 		 */
1326 		smp_mb();
1327 		if (__netif_subqueue_stopped(tx_ring->netdev,
1328 					     tx_ring->queue_index) &&
1329 		    !test_bit(__FM10K_DOWN, interface->state)) {
1330 			netif_wake_subqueue(tx_ring->netdev,
1331 					    tx_ring->queue_index);
1332 			++tx_ring->tx_stats.restart_queue;
1333 		}
1334 	}
1335 
1336 	return !!budget;
1337 }
1338 
1339 /**
1340  * fm10k_update_itr - update the dynamic ITR value based on packet size
1341  *
1342  *      Stores a new ITR value based on strictly on packet size.  The
1343  *      divisors and thresholds used by this function were determined based
1344  *      on theoretical maximum wire speed and testing data, in order to
1345  *      minimize response time while increasing bulk throughput.
1346  *
1347  * @ring_container: Container for rings to have ITR updated
1348  **/
1349 static void fm10k_update_itr(struct fm10k_ring_container *ring_container)
1350 {
1351 	unsigned int avg_wire_size, packets, itr_round;
1352 
1353 	/* Only update ITR if we are using adaptive setting */
1354 	if (!ITR_IS_ADAPTIVE(ring_container->itr))
1355 		goto clear_counts;
1356 
1357 	packets = ring_container->total_packets;
1358 	if (!packets)
1359 		goto clear_counts;
1360 
1361 	avg_wire_size = ring_container->total_bytes / packets;
1362 
1363 	/* The following is a crude approximation of:
1364 	 *  wmem_default / (size + overhead) = desired_pkts_per_int
1365 	 *  rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
1366 	 *  (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
1367 	 *
1368 	 * Assuming wmem_default is 212992 and overhead is 640 bytes per
1369 	 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
1370 	 * formula down to
1371 	 *
1372 	 *  (34 * (size + 24)) / (size + 640) = ITR
1373 	 *
1374 	 * We first do some math on the packet size and then finally bitshift
1375 	 * by 8 after rounding up. We also have to account for PCIe link speed
1376 	 * difference as ITR scales based on this.
1377 	 */
1378 	if (avg_wire_size <= 360) {
1379 		/* Start at 250K ints/sec and gradually drop to 77K ints/sec */
1380 		avg_wire_size *= 8;
1381 		avg_wire_size += 376;
1382 	} else if (avg_wire_size <= 1152) {
1383 		/* 77K ints/sec to 45K ints/sec */
1384 		avg_wire_size *= 3;
1385 		avg_wire_size += 2176;
1386 	} else if (avg_wire_size <= 1920) {
1387 		/* 45K ints/sec to 38K ints/sec */
1388 		avg_wire_size += 4480;
1389 	} else {
1390 		/* plateau at a limit of 38K ints/sec */
1391 		avg_wire_size = 6656;
1392 	}
1393 
1394 	/* Perform final bitshift for division after rounding up to ensure
1395 	 * that the calculation will never get below a 1. The bit shift
1396 	 * accounts for changes in the ITR due to PCIe link speed.
1397 	 */
1398 	itr_round = READ_ONCE(ring_container->itr_scale) + 8;
1399 	avg_wire_size += BIT(itr_round) - 1;
1400 	avg_wire_size >>= itr_round;
1401 
1402 	/* write back value and retain adaptive flag */
1403 	ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE;
1404 
1405 clear_counts:
1406 	ring_container->total_bytes = 0;
1407 	ring_container->total_packets = 0;
1408 }
1409 
1410 static void fm10k_qv_enable(struct fm10k_q_vector *q_vector)
1411 {
1412 	/* Enable auto-mask and clear the current mask */
1413 	u32 itr = FM10K_ITR_ENABLE;
1414 
1415 	/* Update Tx ITR */
1416 	fm10k_update_itr(&q_vector->tx);
1417 
1418 	/* Update Rx ITR */
1419 	fm10k_update_itr(&q_vector->rx);
1420 
1421 	/* Store Tx itr in timer slot 0 */
1422 	itr |= (q_vector->tx.itr & FM10K_ITR_MAX);
1423 
1424 	/* Shift Rx itr to timer slot 1 */
1425 	itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT;
1426 
1427 	/* Write the final value to the ITR register */
1428 	writel(itr, q_vector->itr);
1429 }
1430 
1431 static int fm10k_poll(struct napi_struct *napi, int budget)
1432 {
1433 	struct fm10k_q_vector *q_vector =
1434 			       container_of(napi, struct fm10k_q_vector, napi);
1435 	struct fm10k_ring *ring;
1436 	int per_ring_budget, work_done = 0;
1437 	bool clean_complete = true;
1438 
1439 	fm10k_for_each_ring(ring, q_vector->tx) {
1440 		if (!fm10k_clean_tx_irq(q_vector, ring, budget))
1441 			clean_complete = false;
1442 	}
1443 
1444 	/* Handle case where we are called by netpoll with a budget of 0 */
1445 	if (budget <= 0)
1446 		return budget;
1447 
1448 	/* attempt to distribute budget to each queue fairly, but don't
1449 	 * allow the budget to go below 1 because we'll exit polling
1450 	 */
1451 	if (q_vector->rx.count > 1)
1452 		per_ring_budget = max(budget / q_vector->rx.count, 1);
1453 	else
1454 		per_ring_budget = budget;
1455 
1456 	fm10k_for_each_ring(ring, q_vector->rx) {
1457 		int work = fm10k_clean_rx_irq(q_vector, ring, per_ring_budget);
1458 
1459 		work_done += work;
1460 		if (work >= per_ring_budget)
1461 			clean_complete = false;
1462 	}
1463 
1464 	/* If all work not completed, return budget and keep polling */
1465 	if (!clean_complete)
1466 		return budget;
1467 
1468 	/* Exit the polling mode, but don't re-enable interrupts if stack might
1469 	 * poll us due to busy-polling
1470 	 */
1471 	if (likely(napi_complete_done(napi, work_done)))
1472 		fm10k_qv_enable(q_vector);
1473 
1474 	return min(work_done, budget - 1);
1475 }
1476 
1477 /**
1478  * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device
1479  * @interface: board private structure to initialize
1480  *
1481  * When QoS (Quality of Service) is enabled, allocate queues for
1482  * each traffic class.  If multiqueue isn't available,then abort QoS
1483  * initialization.
1484  *
1485  * This function handles all combinations of Qos and RSS.
1486  *
1487  **/
1488 static bool fm10k_set_qos_queues(struct fm10k_intfc *interface)
1489 {
1490 	struct net_device *dev = interface->netdev;
1491 	struct fm10k_ring_feature *f;
1492 	int rss_i, i;
1493 	int pcs;
1494 
1495 	/* Map queue offset and counts onto allocated tx queues */
1496 	pcs = netdev_get_num_tc(dev);
1497 
1498 	if (pcs <= 1)
1499 		return false;
1500 
1501 	/* set QoS mask and indices */
1502 	f = &interface->ring_feature[RING_F_QOS];
1503 	f->indices = pcs;
1504 	f->mask = BIT(fls(pcs - 1)) - 1;
1505 
1506 	/* determine the upper limit for our current DCB mode */
1507 	rss_i = interface->hw.mac.max_queues / pcs;
1508 	rss_i = BIT(fls(rss_i) - 1);
1509 
1510 	/* set RSS mask and indices */
1511 	f = &interface->ring_feature[RING_F_RSS];
1512 	rss_i = min_t(u16, rss_i, f->limit);
1513 	f->indices = rss_i;
1514 	f->mask = BIT(fls(rss_i - 1)) - 1;
1515 
1516 	/* configure pause class to queue mapping */
1517 	for (i = 0; i < pcs; i++)
1518 		netdev_set_tc_queue(dev, i, rss_i, rss_i * i);
1519 
1520 	interface->num_rx_queues = rss_i * pcs;
1521 	interface->num_tx_queues = rss_i * pcs;
1522 
1523 	return true;
1524 }
1525 
1526 /**
1527  * fm10k_set_rss_queues: Allocate queues for RSS
1528  * @interface: board private structure to initialize
1529  *
1530  * This is our "base" multiqueue mode.  RSS (Receive Side Scaling) will try
1531  * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1532  *
1533  **/
1534 static bool fm10k_set_rss_queues(struct fm10k_intfc *interface)
1535 {
1536 	struct fm10k_ring_feature *f;
1537 	u16 rss_i;
1538 
1539 	f = &interface->ring_feature[RING_F_RSS];
1540 	rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit);
1541 
1542 	/* record indices and power of 2 mask for RSS */
1543 	f->indices = rss_i;
1544 	f->mask = BIT(fls(rss_i - 1)) - 1;
1545 
1546 	interface->num_rx_queues = rss_i;
1547 	interface->num_tx_queues = rss_i;
1548 
1549 	return true;
1550 }
1551 
1552 /**
1553  * fm10k_set_num_queues: Allocate queues for device, feature dependent
1554  * @interface: board private structure to initialize
1555  *
1556  * This is the top level queue allocation routine.  The order here is very
1557  * important, starting with the "most" number of features turned on at once,
1558  * and ending with the smallest set of features.  This way large combinations
1559  * can be allocated if they're turned on, and smaller combinations are the
1560  * fallthrough conditions.
1561  *
1562  **/
1563 static void fm10k_set_num_queues(struct fm10k_intfc *interface)
1564 {
1565 	/* Attempt to setup QoS and RSS first */
1566 	if (fm10k_set_qos_queues(interface))
1567 		return;
1568 
1569 	/* If we don't have QoS, just fallback to only RSS. */
1570 	fm10k_set_rss_queues(interface);
1571 }
1572 
1573 /**
1574  * fm10k_reset_num_queues - Reset the number of queues to zero
1575  * @interface: board private structure
1576  *
1577  * This function should be called whenever we need to reset the number of
1578  * queues after an error condition.
1579  */
1580 static void fm10k_reset_num_queues(struct fm10k_intfc *interface)
1581 {
1582 	interface->num_tx_queues = 0;
1583 	interface->num_rx_queues = 0;
1584 	interface->num_q_vectors = 0;
1585 }
1586 
1587 /**
1588  * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector
1589  * @interface: board private structure to initialize
1590  * @v_count: q_vectors allocated on interface, used for ring interleaving
1591  * @v_idx: index of vector in interface struct
1592  * @txr_count: total number of Tx rings to allocate
1593  * @txr_idx: index of first Tx ring to allocate
1594  * @rxr_count: total number of Rx rings to allocate
1595  * @rxr_idx: index of first Rx ring to allocate
1596  *
1597  * We allocate one q_vector.  If allocation fails we return -ENOMEM.
1598  **/
1599 static int fm10k_alloc_q_vector(struct fm10k_intfc *interface,
1600 				unsigned int v_count, unsigned int v_idx,
1601 				unsigned int txr_count, unsigned int txr_idx,
1602 				unsigned int rxr_count, unsigned int rxr_idx)
1603 {
1604 	struct fm10k_q_vector *q_vector;
1605 	struct fm10k_ring *ring;
1606 	int ring_count;
1607 
1608 	ring_count = txr_count + rxr_count;
1609 
1610 	/* allocate q_vector and rings */
1611 	q_vector = kzalloc(struct_size(q_vector, ring, ring_count), GFP_KERNEL);
1612 	if (!q_vector)
1613 		return -ENOMEM;
1614 
1615 	/* initialize NAPI */
1616 	netif_napi_add(interface->netdev, &q_vector->napi,
1617 		       fm10k_poll, NAPI_POLL_WEIGHT);
1618 
1619 	/* tie q_vector and interface together */
1620 	interface->q_vector[v_idx] = q_vector;
1621 	q_vector->interface = interface;
1622 	q_vector->v_idx = v_idx;
1623 
1624 	/* initialize pointer to rings */
1625 	ring = q_vector->ring;
1626 
1627 	/* save Tx ring container info */
1628 	q_vector->tx.ring = ring;
1629 	q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK;
1630 	q_vector->tx.itr = interface->tx_itr;
1631 	q_vector->tx.itr_scale = interface->hw.mac.itr_scale;
1632 	q_vector->tx.count = txr_count;
1633 
1634 	while (txr_count) {
1635 		/* assign generic ring traits */
1636 		ring->dev = &interface->pdev->dev;
1637 		ring->netdev = interface->netdev;
1638 
1639 		/* configure backlink on ring */
1640 		ring->q_vector = q_vector;
1641 
1642 		/* apply Tx specific ring traits */
1643 		ring->count = interface->tx_ring_count;
1644 		ring->queue_index = txr_idx;
1645 
1646 		/* assign ring to interface */
1647 		interface->tx_ring[txr_idx] = ring;
1648 
1649 		/* update count and index */
1650 		txr_count--;
1651 		txr_idx += v_count;
1652 
1653 		/* push pointer to next ring */
1654 		ring++;
1655 	}
1656 
1657 	/* save Rx ring container info */
1658 	q_vector->rx.ring = ring;
1659 	q_vector->rx.itr = interface->rx_itr;
1660 	q_vector->rx.itr_scale = interface->hw.mac.itr_scale;
1661 	q_vector->rx.count = rxr_count;
1662 
1663 	while (rxr_count) {
1664 		/* assign generic ring traits */
1665 		ring->dev = &interface->pdev->dev;
1666 		ring->netdev = interface->netdev;
1667 		rcu_assign_pointer(ring->l2_accel, interface->l2_accel);
1668 
1669 		/* configure backlink on ring */
1670 		ring->q_vector = q_vector;
1671 
1672 		/* apply Rx specific ring traits */
1673 		ring->count = interface->rx_ring_count;
1674 		ring->queue_index = rxr_idx;
1675 
1676 		/* assign ring to interface */
1677 		interface->rx_ring[rxr_idx] = ring;
1678 
1679 		/* update count and index */
1680 		rxr_count--;
1681 		rxr_idx += v_count;
1682 
1683 		/* push pointer to next ring */
1684 		ring++;
1685 	}
1686 
1687 	fm10k_dbg_q_vector_init(q_vector);
1688 
1689 	return 0;
1690 }
1691 
1692 /**
1693  * fm10k_free_q_vector - Free memory allocated for specific interrupt vector
1694  * @interface: board private structure to initialize
1695  * @v_idx: Index of vector to be freed
1696  *
1697  * This function frees the memory allocated to the q_vector.  In addition if
1698  * NAPI is enabled it will delete any references to the NAPI struct prior
1699  * to freeing the q_vector.
1700  **/
1701 static void fm10k_free_q_vector(struct fm10k_intfc *interface, int v_idx)
1702 {
1703 	struct fm10k_q_vector *q_vector = interface->q_vector[v_idx];
1704 	struct fm10k_ring *ring;
1705 
1706 	fm10k_dbg_q_vector_exit(q_vector);
1707 
1708 	fm10k_for_each_ring(ring, q_vector->tx)
1709 		interface->tx_ring[ring->queue_index] = NULL;
1710 
1711 	fm10k_for_each_ring(ring, q_vector->rx)
1712 		interface->rx_ring[ring->queue_index] = NULL;
1713 
1714 	interface->q_vector[v_idx] = NULL;
1715 	netif_napi_del(&q_vector->napi);
1716 	kfree_rcu(q_vector, rcu);
1717 }
1718 
1719 /**
1720  * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors
1721  * @interface: board private structure to initialize
1722  *
1723  * We allocate one q_vector per queue interrupt.  If allocation fails we
1724  * return -ENOMEM.
1725  **/
1726 static int fm10k_alloc_q_vectors(struct fm10k_intfc *interface)
1727 {
1728 	unsigned int q_vectors = interface->num_q_vectors;
1729 	unsigned int rxr_remaining = interface->num_rx_queues;
1730 	unsigned int txr_remaining = interface->num_tx_queues;
1731 	unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1732 	int err;
1733 
1734 	if (q_vectors >= (rxr_remaining + txr_remaining)) {
1735 		for (; rxr_remaining; v_idx++) {
1736 			err = fm10k_alloc_q_vector(interface, q_vectors, v_idx,
1737 						   0, 0, 1, rxr_idx);
1738 			if (err)
1739 				goto err_out;
1740 
1741 			/* update counts and index */
1742 			rxr_remaining--;
1743 			rxr_idx++;
1744 		}
1745 	}
1746 
1747 	for (; v_idx < q_vectors; v_idx++) {
1748 		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1749 		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1750 
1751 		err = fm10k_alloc_q_vector(interface, q_vectors, v_idx,
1752 					   tqpv, txr_idx,
1753 					   rqpv, rxr_idx);
1754 
1755 		if (err)
1756 			goto err_out;
1757 
1758 		/* update counts and index */
1759 		rxr_remaining -= rqpv;
1760 		txr_remaining -= tqpv;
1761 		rxr_idx++;
1762 		txr_idx++;
1763 	}
1764 
1765 	return 0;
1766 
1767 err_out:
1768 	fm10k_reset_num_queues(interface);
1769 
1770 	while (v_idx--)
1771 		fm10k_free_q_vector(interface, v_idx);
1772 
1773 	return -ENOMEM;
1774 }
1775 
1776 /**
1777  * fm10k_free_q_vectors - Free memory allocated for interrupt vectors
1778  * @interface: board private structure to initialize
1779  *
1780  * This function frees the memory allocated to the q_vectors.  In addition if
1781  * NAPI is enabled it will delete any references to the NAPI struct prior
1782  * to freeing the q_vector.
1783  **/
1784 static void fm10k_free_q_vectors(struct fm10k_intfc *interface)
1785 {
1786 	int v_idx = interface->num_q_vectors;
1787 
1788 	fm10k_reset_num_queues(interface);
1789 
1790 	while (v_idx--)
1791 		fm10k_free_q_vector(interface, v_idx);
1792 }
1793 
1794 /**
1795  * f10k_reset_msix_capability - reset MSI-X capability
1796  * @interface: board private structure to initialize
1797  *
1798  * Reset the MSI-X capability back to its starting state
1799  **/
1800 static void fm10k_reset_msix_capability(struct fm10k_intfc *interface)
1801 {
1802 	pci_disable_msix(interface->pdev);
1803 	kfree(interface->msix_entries);
1804 	interface->msix_entries = NULL;
1805 }
1806 
1807 /**
1808  * f10k_init_msix_capability - configure MSI-X capability
1809  * @interface: board private structure to initialize
1810  *
1811  * Attempt to configure the interrupts using the best available
1812  * capabilities of the hardware and the kernel.
1813  **/
1814 static int fm10k_init_msix_capability(struct fm10k_intfc *interface)
1815 {
1816 	struct fm10k_hw *hw = &interface->hw;
1817 	int v_budget, vector;
1818 
1819 	/* It's easy to be greedy for MSI-X vectors, but it really
1820 	 * doesn't do us much good if we have a lot more vectors
1821 	 * than CPU's.  So let's be conservative and only ask for
1822 	 * (roughly) the same number of vectors as there are CPU's.
1823 	 * the default is to use pairs of vectors
1824 	 */
1825 	v_budget = max(interface->num_rx_queues, interface->num_tx_queues);
1826 	v_budget = min_t(u16, v_budget, num_online_cpus());
1827 
1828 	/* account for vectors not related to queues */
1829 	v_budget += NON_Q_VECTORS;
1830 
1831 	/* At the same time, hardware can only support a maximum of
1832 	 * hw.mac->max_msix_vectors vectors.  With features
1833 	 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx
1834 	 * descriptor queues supported by our device.  Thus, we cap it off in
1835 	 * those rare cases where the cpu count also exceeds our vector limit.
1836 	 */
1837 	v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors);
1838 
1839 	/* A failure in MSI-X entry allocation is fatal. */
1840 	interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
1841 					  GFP_KERNEL);
1842 	if (!interface->msix_entries)
1843 		return -ENOMEM;
1844 
1845 	/* populate entry values */
1846 	for (vector = 0; vector < v_budget; vector++)
1847 		interface->msix_entries[vector].entry = vector;
1848 
1849 	/* Attempt to enable MSI-X with requested value */
1850 	v_budget = pci_enable_msix_range(interface->pdev,
1851 					 interface->msix_entries,
1852 					 MIN_MSIX_COUNT(hw),
1853 					 v_budget);
1854 	if (v_budget < 0) {
1855 		kfree(interface->msix_entries);
1856 		interface->msix_entries = NULL;
1857 		return v_budget;
1858 	}
1859 
1860 	/* record the number of queues available for q_vectors */
1861 	interface->num_q_vectors = v_budget - NON_Q_VECTORS;
1862 
1863 	return 0;
1864 }
1865 
1866 /**
1867  * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS
1868  * @interface: Interface structure continaining rings and devices
1869  *
1870  * Cache the descriptor ring offsets for Qos
1871  **/
1872 static bool fm10k_cache_ring_qos(struct fm10k_intfc *interface)
1873 {
1874 	struct net_device *dev = interface->netdev;
1875 	int pc, offset, rss_i, i;
1876 	u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1;
1877 	u8 num_pcs = netdev_get_num_tc(dev);
1878 
1879 	if (num_pcs <= 1)
1880 		return false;
1881 
1882 	rss_i = interface->ring_feature[RING_F_RSS].indices;
1883 
1884 	for (pc = 0, offset = 0; pc < num_pcs; pc++, offset += rss_i) {
1885 		int q_idx = pc;
1886 
1887 		for (i = 0; i < rss_i; i++) {
1888 			interface->tx_ring[offset + i]->reg_idx = q_idx;
1889 			interface->tx_ring[offset + i]->qos_pc = pc;
1890 			interface->rx_ring[offset + i]->reg_idx = q_idx;
1891 			interface->rx_ring[offset + i]->qos_pc = pc;
1892 			q_idx += pc_stride;
1893 		}
1894 	}
1895 
1896 	return true;
1897 }
1898 
1899 /**
1900  * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS
1901  * @interface: Interface structure continaining rings and devices
1902  *
1903  * Cache the descriptor ring offsets for RSS
1904  **/
1905 static void fm10k_cache_ring_rss(struct fm10k_intfc *interface)
1906 {
1907 	int i;
1908 
1909 	for (i = 0; i < interface->num_rx_queues; i++)
1910 		interface->rx_ring[i]->reg_idx = i;
1911 
1912 	for (i = 0; i < interface->num_tx_queues; i++)
1913 		interface->tx_ring[i]->reg_idx = i;
1914 }
1915 
1916 /**
1917  * fm10k_assign_rings - Map rings to network devices
1918  * @interface: Interface structure containing rings and devices
1919  *
1920  * This function is meant to go though and configure both the network
1921  * devices so that they contain rings, and configure the rings so that
1922  * they function with their network devices.
1923  **/
1924 static void fm10k_assign_rings(struct fm10k_intfc *interface)
1925 {
1926 	if (fm10k_cache_ring_qos(interface))
1927 		return;
1928 
1929 	fm10k_cache_ring_rss(interface);
1930 }
1931 
1932 static void fm10k_init_reta(struct fm10k_intfc *interface)
1933 {
1934 	u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices;
1935 	u32 reta;
1936 
1937 	/* If the Rx flow indirection table has been configured manually, we
1938 	 * need to maintain it when possible.
1939 	 */
1940 	if (netif_is_rxfh_configured(interface->netdev)) {
1941 		for (i = FM10K_RETA_SIZE; i--;) {
1942 			reta = interface->reta[i];
1943 			if ((((reta << 24) >> 24) < rss_i) &&
1944 			    (((reta << 16) >> 24) < rss_i) &&
1945 			    (((reta <<  8) >> 24) < rss_i) &&
1946 			    (((reta)       >> 24) < rss_i))
1947 				continue;
1948 
1949 			/* this should never happen */
1950 			dev_err(&interface->pdev->dev,
1951 				"RSS indirection table assigned flows out of queue bounds. Reconfiguring.\n");
1952 			goto repopulate_reta;
1953 		}
1954 
1955 		/* do nothing if all of the elements are in bounds */
1956 		return;
1957 	}
1958 
1959 repopulate_reta:
1960 	fm10k_write_reta(interface, NULL);
1961 }
1962 
1963 /**
1964  * fm10k_init_queueing_scheme - Determine proper queueing scheme
1965  * @interface: board private structure to initialize
1966  *
1967  * We determine which queueing scheme to use based on...
1968  * - Hardware queue count (num_*_queues)
1969  *   - defined by miscellaneous hardware support/features (RSS, etc.)
1970  **/
1971 int fm10k_init_queueing_scheme(struct fm10k_intfc *interface)
1972 {
1973 	int err;
1974 
1975 	/* Number of supported queues */
1976 	fm10k_set_num_queues(interface);
1977 
1978 	/* Configure MSI-X capability */
1979 	err = fm10k_init_msix_capability(interface);
1980 	if (err) {
1981 		dev_err(&interface->pdev->dev,
1982 			"Unable to initialize MSI-X capability\n");
1983 		goto err_init_msix;
1984 	}
1985 
1986 	/* Allocate memory for queues */
1987 	err = fm10k_alloc_q_vectors(interface);
1988 	if (err) {
1989 		dev_err(&interface->pdev->dev,
1990 			"Unable to allocate queue vectors\n");
1991 		goto err_alloc_q_vectors;
1992 	}
1993 
1994 	/* Map rings to devices, and map devices to physical queues */
1995 	fm10k_assign_rings(interface);
1996 
1997 	/* Initialize RSS redirection table */
1998 	fm10k_init_reta(interface);
1999 
2000 	return 0;
2001 
2002 err_alloc_q_vectors:
2003 	fm10k_reset_msix_capability(interface);
2004 err_init_msix:
2005 	fm10k_reset_num_queues(interface);
2006 	return err;
2007 }
2008 
2009 /**
2010  * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings
2011  * @interface: board private structure to clear queueing scheme on
2012  *
2013  * We go through and clear queueing specific resources and reset the structure
2014  * to pre-load conditions
2015  **/
2016 void fm10k_clear_queueing_scheme(struct fm10k_intfc *interface)
2017 {
2018 	fm10k_free_q_vectors(interface);
2019 	fm10k_reset_msix_capability(interface);
2020 }
2021