1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
3 
4 #include "iavf.h"
5 #include "iavf_prototype.h"
6 #include "iavf_client.h"
7 /* All iavf tracepoints are defined by the include below, which must
8  * be included exactly once across the whole kernel with
9  * CREATE_TRACE_POINTS defined
10  */
11 #define CREATE_TRACE_POINTS
12 #include "iavf_trace.h"
13 
14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
16 static int iavf_close(struct net_device *netdev);
17 static void iavf_init_get_resources(struct iavf_adapter *adapter);
18 static int iavf_check_reset_complete(struct iavf_hw *hw);
19 
20 char iavf_driver_name[] = "iavf";
21 static const char iavf_driver_string[] =
22 	"Intel(R) Ethernet Adaptive Virtual Function Network Driver";
23 
24 static const char iavf_copyright[] =
25 	"Copyright (c) 2013 - 2018 Intel Corporation.";
26 
27 /* iavf_pci_tbl - PCI Device ID Table
28  *
29  * Wildcard entries (PCI_ANY_ID) should come last
30  * Last entry must be all 0s
31  *
32  * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
33  *   Class, Class Mask, private data (not used) }
34  */
35 static const struct pci_device_id iavf_pci_tbl[] = {
36 	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
37 	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
38 	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
39 	{PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
40 	/* required last entry */
41 	{0, }
42 };
43 
44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
45 
46 MODULE_ALIAS("i40evf");
47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
49 MODULE_LICENSE("GPL v2");
50 
51 static const struct net_device_ops iavf_netdev_ops;
52 struct workqueue_struct *iavf_wq;
53 
54 int iavf_status_to_errno(enum iavf_status status)
55 {
56 	switch (status) {
57 	case IAVF_SUCCESS:
58 		return 0;
59 	case IAVF_ERR_PARAM:
60 	case IAVF_ERR_MAC_TYPE:
61 	case IAVF_ERR_INVALID_MAC_ADDR:
62 	case IAVF_ERR_INVALID_LINK_SETTINGS:
63 	case IAVF_ERR_INVALID_PD_ID:
64 	case IAVF_ERR_INVALID_QP_ID:
65 	case IAVF_ERR_INVALID_CQ_ID:
66 	case IAVF_ERR_INVALID_CEQ_ID:
67 	case IAVF_ERR_INVALID_AEQ_ID:
68 	case IAVF_ERR_INVALID_SIZE:
69 	case IAVF_ERR_INVALID_ARP_INDEX:
70 	case IAVF_ERR_INVALID_FPM_FUNC_ID:
71 	case IAVF_ERR_QP_INVALID_MSG_SIZE:
72 	case IAVF_ERR_INVALID_FRAG_COUNT:
73 	case IAVF_ERR_INVALID_ALIGNMENT:
74 	case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:
75 	case IAVF_ERR_INVALID_IMM_DATA_SIZE:
76 	case IAVF_ERR_INVALID_VF_ID:
77 	case IAVF_ERR_INVALID_HMCFN_ID:
78 	case IAVF_ERR_INVALID_PBLE_INDEX:
79 	case IAVF_ERR_INVALID_SD_INDEX:
80 	case IAVF_ERR_INVALID_PAGE_DESC_INDEX:
81 	case IAVF_ERR_INVALID_SD_TYPE:
82 	case IAVF_ERR_INVALID_HMC_OBJ_INDEX:
83 	case IAVF_ERR_INVALID_HMC_OBJ_COUNT:
84 	case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:
85 		return -EINVAL;
86 	case IAVF_ERR_NVM:
87 	case IAVF_ERR_NVM_CHECKSUM:
88 	case IAVF_ERR_PHY:
89 	case IAVF_ERR_CONFIG:
90 	case IAVF_ERR_UNKNOWN_PHY:
91 	case IAVF_ERR_LINK_SETUP:
92 	case IAVF_ERR_ADAPTER_STOPPED:
93 	case IAVF_ERR_PRIMARY_REQUESTS_PENDING:
94 	case IAVF_ERR_AUTONEG_NOT_COMPLETE:
95 	case IAVF_ERR_RESET_FAILED:
96 	case IAVF_ERR_BAD_PTR:
97 	case IAVF_ERR_SWFW_SYNC:
98 	case IAVF_ERR_QP_TOOMANY_WRS_POSTED:
99 	case IAVF_ERR_QUEUE_EMPTY:
100 	case IAVF_ERR_FLUSHED_QUEUE:
101 	case IAVF_ERR_OPCODE_MISMATCH:
102 	case IAVF_ERR_CQP_COMPL_ERROR:
103 	case IAVF_ERR_BACKING_PAGE_ERROR:
104 	case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:
105 	case IAVF_ERR_MEMCPY_FAILED:
106 	case IAVF_ERR_SRQ_ENABLED:
107 	case IAVF_ERR_ADMIN_QUEUE_ERROR:
108 	case IAVF_ERR_ADMIN_QUEUE_FULL:
109 	case IAVF_ERR_BAD_IWARP_CQE:
110 	case IAVF_ERR_NVM_BLANK_MODE:
111 	case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:
112 	case IAVF_ERR_DIAG_TEST_FAILED:
113 	case IAVF_ERR_FIRMWARE_API_VERSION:
114 	case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
115 		return -EIO;
116 	case IAVF_ERR_DEVICE_NOT_SUPPORTED:
117 		return -ENODEV;
118 	case IAVF_ERR_NO_AVAILABLE_VSI:
119 	case IAVF_ERR_RING_FULL:
120 		return -ENOSPC;
121 	case IAVF_ERR_NO_MEMORY:
122 		return -ENOMEM;
123 	case IAVF_ERR_TIMEOUT:
124 	case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:
125 		return -ETIMEDOUT;
126 	case IAVF_ERR_NOT_IMPLEMENTED:
127 	case IAVF_NOT_SUPPORTED:
128 		return -EOPNOTSUPP;
129 	case IAVF_ERR_ADMIN_QUEUE_NO_WORK:
130 		return -EALREADY;
131 	case IAVF_ERR_NOT_READY:
132 		return -EBUSY;
133 	case IAVF_ERR_BUF_TOO_SHORT:
134 		return -EMSGSIZE;
135 	}
136 
137 	return -EIO;
138 }
139 
140 int virtchnl_status_to_errno(enum virtchnl_status_code v_status)
141 {
142 	switch (v_status) {
143 	case VIRTCHNL_STATUS_SUCCESS:
144 		return 0;
145 	case VIRTCHNL_STATUS_ERR_PARAM:
146 	case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:
147 		return -EINVAL;
148 	case VIRTCHNL_STATUS_ERR_NO_MEMORY:
149 		return -ENOMEM;
150 	case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:
151 	case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:
152 	case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR:
153 		return -EIO;
154 	case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED:
155 		return -EOPNOTSUPP;
156 	}
157 
158 	return -EIO;
159 }
160 
161 /**
162  * iavf_pdev_to_adapter - go from pci_dev to adapter
163  * @pdev: pci_dev pointer
164  */
165 static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)
166 {
167 	return netdev_priv(pci_get_drvdata(pdev));
168 }
169 
170 /**
171  * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
172  * @hw:   pointer to the HW structure
173  * @mem:  ptr to mem struct to fill out
174  * @size: size of memory requested
175  * @alignment: what to align the allocation to
176  **/
177 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
178 					 struct iavf_dma_mem *mem,
179 					 u64 size, u32 alignment)
180 {
181 	struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
182 
183 	if (!mem)
184 		return IAVF_ERR_PARAM;
185 
186 	mem->size = ALIGN(size, alignment);
187 	mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
188 				     (dma_addr_t *)&mem->pa, GFP_KERNEL);
189 	if (mem->va)
190 		return 0;
191 	else
192 		return IAVF_ERR_NO_MEMORY;
193 }
194 
195 /**
196  * iavf_free_dma_mem_d - OS specific memory free for shared code
197  * @hw:   pointer to the HW structure
198  * @mem:  ptr to mem struct to free
199  **/
200 enum iavf_status iavf_free_dma_mem_d(struct iavf_hw *hw,
201 				     struct iavf_dma_mem *mem)
202 {
203 	struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
204 
205 	if (!mem || !mem->va)
206 		return IAVF_ERR_PARAM;
207 	dma_free_coherent(&adapter->pdev->dev, mem->size,
208 			  mem->va, (dma_addr_t)mem->pa);
209 	return 0;
210 }
211 
212 /**
213  * iavf_allocate_virt_mem_d - OS specific memory alloc for shared code
214  * @hw:   pointer to the HW structure
215  * @mem:  ptr to mem struct to fill out
216  * @size: size of memory requested
217  **/
218 enum iavf_status iavf_allocate_virt_mem_d(struct iavf_hw *hw,
219 					  struct iavf_virt_mem *mem, u32 size)
220 {
221 	if (!mem)
222 		return IAVF_ERR_PARAM;
223 
224 	mem->size = size;
225 	mem->va = kzalloc(size, GFP_KERNEL);
226 
227 	if (mem->va)
228 		return 0;
229 	else
230 		return IAVF_ERR_NO_MEMORY;
231 }
232 
233 /**
234  * iavf_free_virt_mem_d - OS specific memory free for shared code
235  * @hw:   pointer to the HW structure
236  * @mem:  ptr to mem struct to free
237  **/
238 enum iavf_status iavf_free_virt_mem_d(struct iavf_hw *hw,
239 				      struct iavf_virt_mem *mem)
240 {
241 	if (!mem)
242 		return IAVF_ERR_PARAM;
243 
244 	/* it's ok to kfree a NULL pointer */
245 	kfree(mem->va);
246 
247 	return 0;
248 }
249 
250 /**
251  * iavf_lock_timeout - try to lock mutex but give up after timeout
252  * @lock: mutex that should be locked
253  * @msecs: timeout in msecs
254  *
255  * Returns 0 on success, negative on failure
256  **/
257 int iavf_lock_timeout(struct mutex *lock, unsigned int msecs)
258 {
259 	unsigned int wait, delay = 10;
260 
261 	for (wait = 0; wait < msecs; wait += delay) {
262 		if (mutex_trylock(lock))
263 			return 0;
264 
265 		msleep(delay);
266 	}
267 
268 	return -1;
269 }
270 
271 /**
272  * iavf_schedule_reset - Set the flags and schedule a reset event
273  * @adapter: board private structure
274  **/
275 void iavf_schedule_reset(struct iavf_adapter *adapter)
276 {
277 	if (!(adapter->flags &
278 	      (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
279 		adapter->flags |= IAVF_FLAG_RESET_NEEDED;
280 		queue_work(iavf_wq, &adapter->reset_task);
281 	}
282 }
283 
284 /**
285  * iavf_schedule_request_stats - Set the flags and schedule statistics request
286  * @adapter: board private structure
287  *
288  * Sets IAVF_FLAG_AQ_REQUEST_STATS flag so iavf_watchdog_task() will explicitly
289  * request and refresh ethtool stats
290  **/
291 void iavf_schedule_request_stats(struct iavf_adapter *adapter)
292 {
293 	adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_STATS;
294 	mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
295 }
296 
297 /**
298  * iavf_tx_timeout - Respond to a Tx Hang
299  * @netdev: network interface device structure
300  * @txqueue: queue number that is timing out
301  **/
302 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
303 {
304 	struct iavf_adapter *adapter = netdev_priv(netdev);
305 
306 	adapter->tx_timeout_count++;
307 	iavf_schedule_reset(adapter);
308 }
309 
310 /**
311  * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
312  * @adapter: board private structure
313  **/
314 static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
315 {
316 	struct iavf_hw *hw = &adapter->hw;
317 
318 	if (!adapter->msix_entries)
319 		return;
320 
321 	wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
322 
323 	iavf_flush(hw);
324 
325 	synchronize_irq(adapter->msix_entries[0].vector);
326 }
327 
328 /**
329  * iavf_misc_irq_enable - Enable default interrupt generation settings
330  * @adapter: board private structure
331  **/
332 static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
333 {
334 	struct iavf_hw *hw = &adapter->hw;
335 
336 	wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
337 				       IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
338 	wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
339 
340 	iavf_flush(hw);
341 }
342 
343 /**
344  * iavf_irq_disable - Mask off interrupt generation on the NIC
345  * @adapter: board private structure
346  **/
347 static void iavf_irq_disable(struct iavf_adapter *adapter)
348 {
349 	int i;
350 	struct iavf_hw *hw = &adapter->hw;
351 
352 	if (!adapter->msix_entries)
353 		return;
354 
355 	for (i = 1; i < adapter->num_msix_vectors; i++) {
356 		wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
357 		synchronize_irq(adapter->msix_entries[i].vector);
358 	}
359 	iavf_flush(hw);
360 }
361 
362 /**
363  * iavf_irq_enable_queues - Enable interrupt for specified queues
364  * @adapter: board private structure
365  * @mask: bitmap of queues to enable
366  **/
367 void iavf_irq_enable_queues(struct iavf_adapter *adapter, u32 mask)
368 {
369 	struct iavf_hw *hw = &adapter->hw;
370 	int i;
371 
372 	for (i = 1; i < adapter->num_msix_vectors; i++) {
373 		if (mask & BIT(i - 1)) {
374 			wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
375 			     IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
376 			     IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
377 		}
378 	}
379 }
380 
381 /**
382  * iavf_irq_enable - Enable default interrupt generation settings
383  * @adapter: board private structure
384  * @flush: boolean value whether to run rd32()
385  **/
386 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
387 {
388 	struct iavf_hw *hw = &adapter->hw;
389 
390 	iavf_misc_irq_enable(adapter);
391 	iavf_irq_enable_queues(adapter, ~0);
392 
393 	if (flush)
394 		iavf_flush(hw);
395 }
396 
397 /**
398  * iavf_msix_aq - Interrupt handler for vector 0
399  * @irq: interrupt number
400  * @data: pointer to netdev
401  **/
402 static irqreturn_t iavf_msix_aq(int irq, void *data)
403 {
404 	struct net_device *netdev = data;
405 	struct iavf_adapter *adapter = netdev_priv(netdev);
406 	struct iavf_hw *hw = &adapter->hw;
407 
408 	/* handle non-queue interrupts, these reads clear the registers */
409 	rd32(hw, IAVF_VFINT_ICR01);
410 	rd32(hw, IAVF_VFINT_ICR0_ENA1);
411 
412 	if (adapter->state != __IAVF_REMOVE)
413 		/* schedule work on the private workqueue */
414 		queue_work(iavf_wq, &adapter->adminq_task);
415 
416 	return IRQ_HANDLED;
417 }
418 
419 /**
420  * iavf_msix_clean_rings - MSIX mode Interrupt Handler
421  * @irq: interrupt number
422  * @data: pointer to a q_vector
423  **/
424 static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
425 {
426 	struct iavf_q_vector *q_vector = data;
427 
428 	if (!q_vector->tx.ring && !q_vector->rx.ring)
429 		return IRQ_HANDLED;
430 
431 	napi_schedule_irqoff(&q_vector->napi);
432 
433 	return IRQ_HANDLED;
434 }
435 
436 /**
437  * iavf_map_vector_to_rxq - associate irqs with rx queues
438  * @adapter: board private structure
439  * @v_idx: interrupt number
440  * @r_idx: queue number
441  **/
442 static void
443 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
444 {
445 	struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
446 	struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
447 	struct iavf_hw *hw = &adapter->hw;
448 
449 	rx_ring->q_vector = q_vector;
450 	rx_ring->next = q_vector->rx.ring;
451 	rx_ring->vsi = &adapter->vsi;
452 	q_vector->rx.ring = rx_ring;
453 	q_vector->rx.count++;
454 	q_vector->rx.next_update = jiffies + 1;
455 	q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
456 	q_vector->ring_mask |= BIT(r_idx);
457 	wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
458 	     q_vector->rx.current_itr >> 1);
459 	q_vector->rx.current_itr = q_vector->rx.target_itr;
460 }
461 
462 /**
463  * iavf_map_vector_to_txq - associate irqs with tx queues
464  * @adapter: board private structure
465  * @v_idx: interrupt number
466  * @t_idx: queue number
467  **/
468 static void
469 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
470 {
471 	struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
472 	struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
473 	struct iavf_hw *hw = &adapter->hw;
474 
475 	tx_ring->q_vector = q_vector;
476 	tx_ring->next = q_vector->tx.ring;
477 	tx_ring->vsi = &adapter->vsi;
478 	q_vector->tx.ring = tx_ring;
479 	q_vector->tx.count++;
480 	q_vector->tx.next_update = jiffies + 1;
481 	q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
482 	q_vector->num_ringpairs++;
483 	wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
484 	     q_vector->tx.target_itr >> 1);
485 	q_vector->tx.current_itr = q_vector->tx.target_itr;
486 }
487 
488 /**
489  * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
490  * @adapter: board private structure to initialize
491  *
492  * This function maps descriptor rings to the queue-specific vectors
493  * we were allotted through the MSI-X enabling code.  Ideally, we'd have
494  * one vector per ring/queue, but on a constrained vector budget, we
495  * group the rings as "efficiently" as possible.  You would add new
496  * mapping configurations in here.
497  **/
498 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
499 {
500 	int rings_remaining = adapter->num_active_queues;
501 	int ridx = 0, vidx = 0;
502 	int q_vectors;
503 
504 	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
505 
506 	for (; ridx < rings_remaining; ridx++) {
507 		iavf_map_vector_to_rxq(adapter, vidx, ridx);
508 		iavf_map_vector_to_txq(adapter, vidx, ridx);
509 
510 		/* In the case where we have more queues than vectors, continue
511 		 * round-robin on vectors until all queues are mapped.
512 		 */
513 		if (++vidx >= q_vectors)
514 			vidx = 0;
515 	}
516 
517 	adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
518 }
519 
520 /**
521  * iavf_irq_affinity_notify - Callback for affinity changes
522  * @notify: context as to what irq was changed
523  * @mask: the new affinity mask
524  *
525  * This is a callback function used by the irq_set_affinity_notifier function
526  * so that we may register to receive changes to the irq affinity masks.
527  **/
528 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
529 				     const cpumask_t *mask)
530 {
531 	struct iavf_q_vector *q_vector =
532 		container_of(notify, struct iavf_q_vector, affinity_notify);
533 
534 	cpumask_copy(&q_vector->affinity_mask, mask);
535 }
536 
537 /**
538  * iavf_irq_affinity_release - Callback for affinity notifier release
539  * @ref: internal core kernel usage
540  *
541  * This is a callback function used by the irq_set_affinity_notifier function
542  * to inform the current notification subscriber that they will no longer
543  * receive notifications.
544  **/
545 static void iavf_irq_affinity_release(struct kref *ref) {}
546 
547 /**
548  * iavf_request_traffic_irqs - Initialize MSI-X interrupts
549  * @adapter: board private structure
550  * @basename: device basename
551  *
552  * Allocates MSI-X vectors for tx and rx handling, and requests
553  * interrupts from the kernel.
554  **/
555 static int
556 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
557 {
558 	unsigned int vector, q_vectors;
559 	unsigned int rx_int_idx = 0, tx_int_idx = 0;
560 	int irq_num, err;
561 	int cpu;
562 
563 	iavf_irq_disable(adapter);
564 	/* Decrement for Other and TCP Timer vectors */
565 	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
566 
567 	for (vector = 0; vector < q_vectors; vector++) {
568 		struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
569 
570 		irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
571 
572 		if (q_vector->tx.ring && q_vector->rx.ring) {
573 			snprintf(q_vector->name, sizeof(q_vector->name),
574 				 "iavf-%s-TxRx-%u", basename, rx_int_idx++);
575 			tx_int_idx++;
576 		} else if (q_vector->rx.ring) {
577 			snprintf(q_vector->name, sizeof(q_vector->name),
578 				 "iavf-%s-rx-%u", basename, rx_int_idx++);
579 		} else if (q_vector->tx.ring) {
580 			snprintf(q_vector->name, sizeof(q_vector->name),
581 				 "iavf-%s-tx-%u", basename, tx_int_idx++);
582 		} else {
583 			/* skip this unused q_vector */
584 			continue;
585 		}
586 		err = request_irq(irq_num,
587 				  iavf_msix_clean_rings,
588 				  0,
589 				  q_vector->name,
590 				  q_vector);
591 		if (err) {
592 			dev_info(&adapter->pdev->dev,
593 				 "Request_irq failed, error: %d\n", err);
594 			goto free_queue_irqs;
595 		}
596 		/* register for affinity change notifications */
597 		q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
598 		q_vector->affinity_notify.release =
599 						   iavf_irq_affinity_release;
600 		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
601 		/* Spread the IRQ affinity hints across online CPUs. Note that
602 		 * get_cpu_mask returns a mask with a permanent lifetime so
603 		 * it's safe to use as a hint for irq_update_affinity_hint.
604 		 */
605 		cpu = cpumask_local_spread(q_vector->v_idx, -1);
606 		irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
607 	}
608 
609 	return 0;
610 
611 free_queue_irqs:
612 	while (vector) {
613 		vector--;
614 		irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
615 		irq_set_affinity_notifier(irq_num, NULL);
616 		irq_update_affinity_hint(irq_num, NULL);
617 		free_irq(irq_num, &adapter->q_vectors[vector]);
618 	}
619 	return err;
620 }
621 
622 /**
623  * iavf_request_misc_irq - Initialize MSI-X interrupts
624  * @adapter: board private structure
625  *
626  * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
627  * vector is only for the admin queue, and stays active even when the netdev
628  * is closed.
629  **/
630 static int iavf_request_misc_irq(struct iavf_adapter *adapter)
631 {
632 	struct net_device *netdev = adapter->netdev;
633 	int err;
634 
635 	snprintf(adapter->misc_vector_name,
636 		 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
637 		 dev_name(&adapter->pdev->dev));
638 	err = request_irq(adapter->msix_entries[0].vector,
639 			  &iavf_msix_aq, 0,
640 			  adapter->misc_vector_name, netdev);
641 	if (err) {
642 		dev_err(&adapter->pdev->dev,
643 			"request_irq for %s failed: %d\n",
644 			adapter->misc_vector_name, err);
645 		free_irq(adapter->msix_entries[0].vector, netdev);
646 	}
647 	return err;
648 }
649 
650 /**
651  * iavf_free_traffic_irqs - Free MSI-X interrupts
652  * @adapter: board private structure
653  *
654  * Frees all MSI-X vectors other than 0.
655  **/
656 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
657 {
658 	int vector, irq_num, q_vectors;
659 
660 	if (!adapter->msix_entries)
661 		return;
662 
663 	q_vectors = adapter->num_msix_vectors - NONQ_VECS;
664 
665 	for (vector = 0; vector < q_vectors; vector++) {
666 		irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
667 		irq_set_affinity_notifier(irq_num, NULL);
668 		irq_update_affinity_hint(irq_num, NULL);
669 		free_irq(irq_num, &adapter->q_vectors[vector]);
670 	}
671 }
672 
673 /**
674  * iavf_free_misc_irq - Free MSI-X miscellaneous vector
675  * @adapter: board private structure
676  *
677  * Frees MSI-X vector 0.
678  **/
679 static void iavf_free_misc_irq(struct iavf_adapter *adapter)
680 {
681 	struct net_device *netdev = adapter->netdev;
682 
683 	if (!adapter->msix_entries)
684 		return;
685 
686 	free_irq(adapter->msix_entries[0].vector, netdev);
687 }
688 
689 /**
690  * iavf_configure_tx - Configure Transmit Unit after Reset
691  * @adapter: board private structure
692  *
693  * Configure the Tx unit of the MAC after a reset.
694  **/
695 static void iavf_configure_tx(struct iavf_adapter *adapter)
696 {
697 	struct iavf_hw *hw = &adapter->hw;
698 	int i;
699 
700 	for (i = 0; i < adapter->num_active_queues; i++)
701 		adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
702 }
703 
704 /**
705  * iavf_configure_rx - Configure Receive Unit after Reset
706  * @adapter: board private structure
707  *
708  * Configure the Rx unit of the MAC after a reset.
709  **/
710 static void iavf_configure_rx(struct iavf_adapter *adapter)
711 {
712 	unsigned int rx_buf_len = IAVF_RXBUFFER_2048;
713 	struct iavf_hw *hw = &adapter->hw;
714 	int i;
715 
716 	/* Legacy Rx will always default to a 2048 buffer size. */
717 #if (PAGE_SIZE < 8192)
718 	if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
719 		struct net_device *netdev = adapter->netdev;
720 
721 		/* For jumbo frames on systems with 4K pages we have to use
722 		 * an order 1 page, so we might as well increase the size
723 		 * of our Rx buffer to make better use of the available space
724 		 */
725 		rx_buf_len = IAVF_RXBUFFER_3072;
726 
727 		/* We use a 1536 buffer size for configurations with
728 		 * standard Ethernet mtu.  On x86 this gives us enough room
729 		 * for shared info and 192 bytes of padding.
730 		 */
731 		if (!IAVF_2K_TOO_SMALL_WITH_PADDING &&
732 		    (netdev->mtu <= ETH_DATA_LEN))
733 			rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
734 	}
735 #endif
736 
737 	for (i = 0; i < adapter->num_active_queues; i++) {
738 		adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
739 		adapter->rx_rings[i].rx_buf_len = rx_buf_len;
740 
741 		if (adapter->flags & IAVF_FLAG_LEGACY_RX)
742 			clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
743 		else
744 			set_ring_build_skb_enabled(&adapter->rx_rings[i]);
745 	}
746 }
747 
748 /**
749  * iavf_find_vlan - Search filter list for specific vlan filter
750  * @adapter: board private structure
751  * @vlan: vlan tag
752  *
753  * Returns ptr to the filter object or NULL. Must be called while holding the
754  * mac_vlan_list_lock.
755  **/
756 static struct
757 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter,
758 				 struct iavf_vlan vlan)
759 {
760 	struct iavf_vlan_filter *f;
761 
762 	list_for_each_entry(f, &adapter->vlan_filter_list, list) {
763 		if (f->vlan.vid == vlan.vid &&
764 		    f->vlan.tpid == vlan.tpid)
765 			return f;
766 	}
767 
768 	return NULL;
769 }
770 
771 /**
772  * iavf_add_vlan - Add a vlan filter to the list
773  * @adapter: board private structure
774  * @vlan: VLAN tag
775  *
776  * Returns ptr to the filter object or NULL when no memory available.
777  **/
778 static struct
779 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
780 				struct iavf_vlan vlan)
781 {
782 	struct iavf_vlan_filter *f = NULL;
783 
784 	spin_lock_bh(&adapter->mac_vlan_list_lock);
785 
786 	f = iavf_find_vlan(adapter, vlan);
787 	if (!f) {
788 		f = kzalloc(sizeof(*f), GFP_ATOMIC);
789 		if (!f)
790 			goto clearout;
791 
792 		f->vlan = vlan;
793 
794 		list_add_tail(&f->list, &adapter->vlan_filter_list);
795 		f->add = true;
796 		adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
797 	}
798 
799 clearout:
800 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
801 	return f;
802 }
803 
804 /**
805  * iavf_del_vlan - Remove a vlan filter from the list
806  * @adapter: board private structure
807  * @vlan: VLAN tag
808  **/
809 static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)
810 {
811 	struct iavf_vlan_filter *f;
812 
813 	spin_lock_bh(&adapter->mac_vlan_list_lock);
814 
815 	f = iavf_find_vlan(adapter, vlan);
816 	if (f) {
817 		f->remove = true;
818 		adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
819 	}
820 
821 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
822 }
823 
824 /**
825  * iavf_restore_filters
826  * @adapter: board private structure
827  *
828  * Restore existing non MAC filters when VF netdev comes back up
829  **/
830 static void iavf_restore_filters(struct iavf_adapter *adapter)
831 {
832 	u16 vid;
833 
834 	/* re-add all VLAN filters */
835 	for_each_set_bit(vid, adapter->vsi.active_cvlans, VLAN_N_VID)
836 		iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021Q));
837 
838 	for_each_set_bit(vid, adapter->vsi.active_svlans, VLAN_N_VID)
839 		iavf_add_vlan(adapter, IAVF_VLAN(vid, ETH_P_8021AD));
840 }
841 
842 /**
843  * iavf_get_num_vlans_added - get number of VLANs added
844  * @adapter: board private structure
845  */
846 u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter)
847 {
848 	return bitmap_weight(adapter->vsi.active_cvlans, VLAN_N_VID) +
849 		bitmap_weight(adapter->vsi.active_svlans, VLAN_N_VID);
850 }
851 
852 /**
853  * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF
854  * @adapter: board private structure
855  *
856  * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN,
857  * do not impose a limit as that maintains current behavior and for
858  * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF.
859  **/
860 static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter)
861 {
862 	/* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has
863 	 * never been a limit on the VF driver side
864 	 */
865 	if (VLAN_ALLOWED(adapter))
866 		return VLAN_N_VID;
867 	else if (VLAN_V2_ALLOWED(adapter))
868 		return adapter->vlan_v2_caps.filtering.max_filters;
869 
870 	return 0;
871 }
872 
873 /**
874  * iavf_max_vlans_added - check if maximum VLANs allowed already exist
875  * @adapter: board private structure
876  **/
877 static bool iavf_max_vlans_added(struct iavf_adapter *adapter)
878 {
879 	if (iavf_get_num_vlans_added(adapter) <
880 	    iavf_get_max_vlans_allowed(adapter))
881 		return false;
882 
883 	return true;
884 }
885 
886 /**
887  * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
888  * @netdev: network device struct
889  * @proto: unused protocol data
890  * @vid: VLAN tag
891  **/
892 static int iavf_vlan_rx_add_vid(struct net_device *netdev,
893 				__always_unused __be16 proto, u16 vid)
894 {
895 	struct iavf_adapter *adapter = netdev_priv(netdev);
896 
897 	if (!VLAN_FILTERING_ALLOWED(adapter))
898 		return -EIO;
899 
900 	if (iavf_max_vlans_added(adapter)) {
901 		netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n",
902 			   iavf_get_max_vlans_allowed(adapter));
903 		return -EIO;
904 	}
905 
906 	if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))))
907 		return -ENOMEM;
908 
909 	return 0;
910 }
911 
912 /**
913  * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
914  * @netdev: network device struct
915  * @proto: unused protocol data
916  * @vid: VLAN tag
917  **/
918 static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
919 				 __always_unused __be16 proto, u16 vid)
920 {
921 	struct iavf_adapter *adapter = netdev_priv(netdev);
922 
923 	iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)));
924 	if (proto == cpu_to_be16(ETH_P_8021Q))
925 		clear_bit(vid, adapter->vsi.active_cvlans);
926 	else
927 		clear_bit(vid, adapter->vsi.active_svlans);
928 
929 	return 0;
930 }
931 
932 /**
933  * iavf_find_filter - Search filter list for specific mac filter
934  * @adapter: board private structure
935  * @macaddr: the MAC address
936  *
937  * Returns ptr to the filter object or NULL. Must be called while holding the
938  * mac_vlan_list_lock.
939  **/
940 static struct
941 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
942 				  const u8 *macaddr)
943 {
944 	struct iavf_mac_filter *f;
945 
946 	if (!macaddr)
947 		return NULL;
948 
949 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
950 		if (ether_addr_equal(macaddr, f->macaddr))
951 			return f;
952 	}
953 	return NULL;
954 }
955 
956 /**
957  * iavf_add_filter - Add a mac filter to the filter list
958  * @adapter: board private structure
959  * @macaddr: the MAC address
960  *
961  * Returns ptr to the filter object or NULL when no memory available.
962  **/
963 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
964 					const u8 *macaddr)
965 {
966 	struct iavf_mac_filter *f;
967 
968 	if (!macaddr)
969 		return NULL;
970 
971 	f = iavf_find_filter(adapter, macaddr);
972 	if (!f) {
973 		f = kzalloc(sizeof(*f), GFP_ATOMIC);
974 		if (!f)
975 			return f;
976 
977 		ether_addr_copy(f->macaddr, macaddr);
978 
979 		list_add_tail(&f->list, &adapter->mac_filter_list);
980 		f->add = true;
981 		f->add_handled = false;
982 		f->is_new_mac = true;
983 		f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr);
984 		adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
985 	} else {
986 		f->remove = false;
987 	}
988 
989 	return f;
990 }
991 
992 /**
993  * iavf_replace_primary_mac - Replace current primary address
994  * @adapter: board private structure
995  * @new_mac: new MAC address to be applied
996  *
997  * Replace current dev_addr and send request to PF for removal of previous
998  * primary MAC address filter and addition of new primary MAC filter.
999  * Return 0 for success, -ENOMEM for failure.
1000  *
1001  * Do not call this with mac_vlan_list_lock!
1002  **/
1003 int iavf_replace_primary_mac(struct iavf_adapter *adapter,
1004 			     const u8 *new_mac)
1005 {
1006 	struct iavf_hw *hw = &adapter->hw;
1007 	struct iavf_mac_filter *f;
1008 
1009 	spin_lock_bh(&adapter->mac_vlan_list_lock);
1010 
1011 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
1012 		f->is_primary = false;
1013 	}
1014 
1015 	f = iavf_find_filter(adapter, hw->mac.addr);
1016 	if (f) {
1017 		f->remove = true;
1018 		adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1019 	}
1020 
1021 	f = iavf_add_filter(adapter, new_mac);
1022 
1023 	if (f) {
1024 		/* Always send the request to add if changing primary MAC
1025 		 * even if filter is already present on the list
1026 		 */
1027 		f->is_primary = true;
1028 		f->add = true;
1029 		adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
1030 		ether_addr_copy(hw->mac.addr, new_mac);
1031 	}
1032 
1033 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
1034 
1035 	/* schedule the watchdog task to immediately process the request */
1036 	if (f) {
1037 		queue_work(iavf_wq, &adapter->watchdog_task.work);
1038 		return 0;
1039 	}
1040 	return -ENOMEM;
1041 }
1042 
1043 /**
1044  * iavf_is_mac_set_handled - wait for a response to set MAC from PF
1045  * @netdev: network interface device structure
1046  * @macaddr: MAC address to set
1047  *
1048  * Returns true on success, false on failure
1049  */
1050 static bool iavf_is_mac_set_handled(struct net_device *netdev,
1051 				    const u8 *macaddr)
1052 {
1053 	struct iavf_adapter *adapter = netdev_priv(netdev);
1054 	struct iavf_mac_filter *f;
1055 	bool ret = false;
1056 
1057 	spin_lock_bh(&adapter->mac_vlan_list_lock);
1058 
1059 	f = iavf_find_filter(adapter, macaddr);
1060 
1061 	if (!f || (!f->add && f->add_handled))
1062 		ret = true;
1063 
1064 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
1065 
1066 	return ret;
1067 }
1068 
1069 /**
1070  * iavf_set_mac - NDO callback to set port MAC address
1071  * @netdev: network interface device structure
1072  * @p: pointer to an address structure
1073  *
1074  * Returns 0 on success, negative on failure
1075  */
1076 static int iavf_set_mac(struct net_device *netdev, void *p)
1077 {
1078 	struct iavf_adapter *adapter = netdev_priv(netdev);
1079 	struct sockaddr *addr = p;
1080 	bool handle_mac = iavf_is_mac_set_handled(netdev, addr->sa_data);
1081 	int ret;
1082 
1083 	if (!is_valid_ether_addr(addr->sa_data))
1084 		return -EADDRNOTAVAIL;
1085 
1086 	ret = iavf_replace_primary_mac(adapter, addr->sa_data);
1087 
1088 	if (ret)
1089 		return ret;
1090 
1091 	/* If this is an initial set MAC during VF spawn do not wait */
1092 	if (adapter->flags & IAVF_FLAG_INITIAL_MAC_SET) {
1093 		adapter->flags &= ~IAVF_FLAG_INITIAL_MAC_SET;
1094 		return 0;
1095 	}
1096 
1097 	if (handle_mac)
1098 		goto done;
1099 
1100 	ret = wait_event_interruptible_timeout(adapter->vc_waitqueue, false, msecs_to_jiffies(2500));
1101 
1102 	/* If ret < 0 then it means wait was interrupted.
1103 	 * If ret == 0 then it means we got a timeout.
1104 	 * else it means we got response for set MAC from PF,
1105 	 * check if netdev MAC was updated to requested MAC,
1106 	 * if yes then set MAC succeeded otherwise it failed return -EACCES
1107 	 */
1108 	if (ret < 0)
1109 		return ret;
1110 
1111 	if (!ret)
1112 		return -EAGAIN;
1113 
1114 done:
1115 	if (!ether_addr_equal(netdev->dev_addr, addr->sa_data))
1116 		return -EACCES;
1117 
1118 	return 0;
1119 }
1120 
1121 /**
1122  * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
1123  * @netdev: the netdevice
1124  * @addr: address to add
1125  *
1126  * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1127  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1128  */
1129 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
1130 {
1131 	struct iavf_adapter *adapter = netdev_priv(netdev);
1132 
1133 	if (iavf_add_filter(adapter, addr))
1134 		return 0;
1135 	else
1136 		return -ENOMEM;
1137 }
1138 
1139 /**
1140  * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1141  * @netdev: the netdevice
1142  * @addr: address to add
1143  *
1144  * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1145  * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1146  */
1147 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
1148 {
1149 	struct iavf_adapter *adapter = netdev_priv(netdev);
1150 	struct iavf_mac_filter *f;
1151 
1152 	/* Under some circumstances, we might receive a request to delete
1153 	 * our own device address from our uc list. Because we store the
1154 	 * device address in the VSI's MAC/VLAN filter list, we need to ignore
1155 	 * such requests and not delete our device address from this list.
1156 	 */
1157 	if (ether_addr_equal(addr, netdev->dev_addr))
1158 		return 0;
1159 
1160 	f = iavf_find_filter(adapter, addr);
1161 	if (f) {
1162 		f->remove = true;
1163 		adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1164 	}
1165 	return 0;
1166 }
1167 
1168 /**
1169  * iavf_set_rx_mode - NDO callback to set the netdev filters
1170  * @netdev: network interface device structure
1171  **/
1172 static void iavf_set_rx_mode(struct net_device *netdev)
1173 {
1174 	struct iavf_adapter *adapter = netdev_priv(netdev);
1175 
1176 	spin_lock_bh(&adapter->mac_vlan_list_lock);
1177 	__dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1178 	__dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1179 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
1180 
1181 	if (netdev->flags & IFF_PROMISC &&
1182 	    !(adapter->flags & IAVF_FLAG_PROMISC_ON))
1183 		adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_PROMISC;
1184 	else if (!(netdev->flags & IFF_PROMISC) &&
1185 		 adapter->flags & IAVF_FLAG_PROMISC_ON)
1186 		adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_PROMISC;
1187 
1188 	if (netdev->flags & IFF_ALLMULTI &&
1189 	    !(adapter->flags & IAVF_FLAG_ALLMULTI_ON))
1190 		adapter->aq_required |= IAVF_FLAG_AQ_REQUEST_ALLMULTI;
1191 	else if (!(netdev->flags & IFF_ALLMULTI) &&
1192 		 adapter->flags & IAVF_FLAG_ALLMULTI_ON)
1193 		adapter->aq_required |= IAVF_FLAG_AQ_RELEASE_ALLMULTI;
1194 }
1195 
1196 /**
1197  * iavf_napi_enable_all - enable NAPI on all queue vectors
1198  * @adapter: board private structure
1199  **/
1200 static void iavf_napi_enable_all(struct iavf_adapter *adapter)
1201 {
1202 	int q_idx;
1203 	struct iavf_q_vector *q_vector;
1204 	int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1205 
1206 	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1207 		struct napi_struct *napi;
1208 
1209 		q_vector = &adapter->q_vectors[q_idx];
1210 		napi = &q_vector->napi;
1211 		napi_enable(napi);
1212 	}
1213 }
1214 
1215 /**
1216  * iavf_napi_disable_all - disable NAPI on all queue vectors
1217  * @adapter: board private structure
1218  **/
1219 static void iavf_napi_disable_all(struct iavf_adapter *adapter)
1220 {
1221 	int q_idx;
1222 	struct iavf_q_vector *q_vector;
1223 	int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1224 
1225 	for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1226 		q_vector = &adapter->q_vectors[q_idx];
1227 		napi_disable(&q_vector->napi);
1228 	}
1229 }
1230 
1231 /**
1232  * iavf_configure - set up transmit and receive data structures
1233  * @adapter: board private structure
1234  **/
1235 static void iavf_configure(struct iavf_adapter *adapter)
1236 {
1237 	struct net_device *netdev = adapter->netdev;
1238 	int i;
1239 
1240 	iavf_set_rx_mode(netdev);
1241 
1242 	iavf_configure_tx(adapter);
1243 	iavf_configure_rx(adapter);
1244 	adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
1245 
1246 	for (i = 0; i < adapter->num_active_queues; i++) {
1247 		struct iavf_ring *ring = &adapter->rx_rings[i];
1248 
1249 		iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));
1250 	}
1251 }
1252 
1253 /**
1254  * iavf_up_complete - Finish the last steps of bringing up a connection
1255  * @adapter: board private structure
1256  *
1257  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1258  **/
1259 static void iavf_up_complete(struct iavf_adapter *adapter)
1260 {
1261 	iavf_change_state(adapter, __IAVF_RUNNING);
1262 	clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1263 
1264 	iavf_napi_enable_all(adapter);
1265 
1266 	adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
1267 	if (CLIENT_ENABLED(adapter))
1268 		adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN;
1269 	mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
1270 }
1271 
1272 /**
1273  * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF
1274  * yet and mark other to be removed.
1275  * @adapter: board private structure
1276  **/
1277 static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
1278 {
1279 	struct iavf_vlan_filter *vlf, *vlftmp;
1280 	struct iavf_mac_filter *f, *ftmp;
1281 
1282 	spin_lock_bh(&adapter->mac_vlan_list_lock);
1283 	/* clear the sync flag on all filters */
1284 	__dev_uc_unsync(adapter->netdev, NULL);
1285 	__dev_mc_unsync(adapter->netdev, NULL);
1286 
1287 	/* remove all MAC filters */
1288 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list,
1289 				 list) {
1290 		if (f->add) {
1291 			list_del(&f->list);
1292 			kfree(f);
1293 		} else {
1294 			f->remove = true;
1295 		}
1296 	}
1297 
1298 	/* remove all VLAN filters */
1299 	list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
1300 				 list) {
1301 		if (vlf->add) {
1302 			list_del(&vlf->list);
1303 			kfree(vlf);
1304 		} else {
1305 			vlf->remove = true;
1306 		}
1307 	}
1308 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
1309 }
1310 
1311 /**
1312  * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and
1313  * mark other to be removed.
1314  * @adapter: board private structure
1315  **/
1316 static void iavf_clear_cloud_filters(struct iavf_adapter *adapter)
1317 {
1318 	struct iavf_cloud_filter *cf, *cftmp;
1319 
1320 	/* remove all cloud filters */
1321 	spin_lock_bh(&adapter->cloud_filter_list_lock);
1322 	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
1323 				 list) {
1324 		if (cf->add) {
1325 			list_del(&cf->list);
1326 			kfree(cf);
1327 			adapter->num_cloud_filters--;
1328 		} else {
1329 			cf->del = true;
1330 		}
1331 	}
1332 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
1333 }
1334 
1335 /**
1336  * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark
1337  * other to be removed.
1338  * @adapter: board private structure
1339  **/
1340 static void iavf_clear_fdir_filters(struct iavf_adapter *adapter)
1341 {
1342 	struct iavf_fdir_fltr *fdir, *fdirtmp;
1343 
1344 	/* remove all Flow Director filters */
1345 	spin_lock_bh(&adapter->fdir_fltr_lock);
1346 	list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head,
1347 				 list) {
1348 		if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {
1349 			list_del(&fdir->list);
1350 			kfree(fdir);
1351 			adapter->fdir_active_fltr--;
1352 		} else {
1353 			fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
1354 		}
1355 	}
1356 	spin_unlock_bh(&adapter->fdir_fltr_lock);
1357 }
1358 
1359 /**
1360  * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark
1361  * other to be removed.
1362  * @adapter: board private structure
1363  **/
1364 static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter)
1365 {
1366 	struct iavf_adv_rss *rss, *rsstmp;
1367 
1368 	/* remove all advance RSS configuration */
1369 	spin_lock_bh(&adapter->adv_rss_lock);
1370 	list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
1371 				 list) {
1372 		if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {
1373 			list_del(&rss->list);
1374 			kfree(rss);
1375 		} else {
1376 			rss->state = IAVF_ADV_RSS_DEL_REQUEST;
1377 		}
1378 	}
1379 	spin_unlock_bh(&adapter->adv_rss_lock);
1380 }
1381 
1382 /**
1383  * iavf_down - Shutdown the connection processing
1384  * @adapter: board private structure
1385  *
1386  * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1387  **/
1388 void iavf_down(struct iavf_adapter *adapter)
1389 {
1390 	struct net_device *netdev = adapter->netdev;
1391 
1392 	if (adapter->state <= __IAVF_DOWN_PENDING)
1393 		return;
1394 
1395 	netif_carrier_off(netdev);
1396 	netif_tx_disable(netdev);
1397 	adapter->link_up = false;
1398 	iavf_napi_disable_all(adapter);
1399 	iavf_irq_disable(adapter);
1400 
1401 	iavf_clear_mac_vlan_filters(adapter);
1402 	iavf_clear_cloud_filters(adapter);
1403 	iavf_clear_fdir_filters(adapter);
1404 	iavf_clear_adv_rss_conf(adapter);
1405 
1406 	if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)) {
1407 		/* cancel any current operation */
1408 		adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1409 		/* Schedule operations to close down the HW. Don't wait
1410 		 * here for this to complete. The watchdog is still running
1411 		 * and it will take care of this.
1412 		 */
1413 		if (!list_empty(&adapter->mac_filter_list))
1414 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1415 		if (!list_empty(&adapter->vlan_filter_list))
1416 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1417 		if (!list_empty(&adapter->cloud_filter_list))
1418 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1419 		if (!list_empty(&adapter->fdir_list_head))
1420 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
1421 		if (!list_empty(&adapter->adv_rss_list_head))
1422 			adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
1423 		adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
1424 	}
1425 
1426 	mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
1427 }
1428 
1429 /**
1430  * iavf_acquire_msix_vectors - Setup the MSIX capability
1431  * @adapter: board private structure
1432  * @vectors: number of vectors to request
1433  *
1434  * Work with the OS to set up the MSIX vectors needed.
1435  *
1436  * Returns 0 on success, negative on failure
1437  **/
1438 static int
1439 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1440 {
1441 	int err, vector_threshold;
1442 
1443 	/* We'll want at least 3 (vector_threshold):
1444 	 * 0) Other (Admin Queue and link, mostly)
1445 	 * 1) TxQ[0] Cleanup
1446 	 * 2) RxQ[0] Cleanup
1447 	 */
1448 	vector_threshold = MIN_MSIX_COUNT;
1449 
1450 	/* The more we get, the more we will assign to Tx/Rx Cleanup
1451 	 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1452 	 * Right now, we simply care about how many we'll get; we'll
1453 	 * set them up later while requesting irq's.
1454 	 */
1455 	err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1456 				    vector_threshold, vectors);
1457 	if (err < 0) {
1458 		dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1459 		kfree(adapter->msix_entries);
1460 		adapter->msix_entries = NULL;
1461 		return err;
1462 	}
1463 
1464 	/* Adjust for only the vectors we'll use, which is minimum
1465 	 * of max_msix_q_vectors + NONQ_VECS, or the number of
1466 	 * vectors we were allocated.
1467 	 */
1468 	adapter->num_msix_vectors = err;
1469 	return 0;
1470 }
1471 
1472 /**
1473  * iavf_free_queues - Free memory for all rings
1474  * @adapter: board private structure to initialize
1475  *
1476  * Free all of the memory associated with queue pairs.
1477  **/
1478 static void iavf_free_queues(struct iavf_adapter *adapter)
1479 {
1480 	if (!adapter->vsi_res)
1481 		return;
1482 	adapter->num_active_queues = 0;
1483 	kfree(adapter->tx_rings);
1484 	adapter->tx_rings = NULL;
1485 	kfree(adapter->rx_rings);
1486 	adapter->rx_rings = NULL;
1487 }
1488 
1489 /**
1490  * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload
1491  * @adapter: board private structure
1492  *
1493  * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or
1494  * stripped in certain descriptor fields. Instead of checking the offload
1495  * capability bits in the hot path, cache the location the ring specific
1496  * flags.
1497  */
1498 void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter)
1499 {
1500 	int i;
1501 
1502 	for (i = 0; i < adapter->num_active_queues; i++) {
1503 		struct iavf_ring *tx_ring = &adapter->tx_rings[i];
1504 		struct iavf_ring *rx_ring = &adapter->rx_rings[i];
1505 
1506 		/* prevent multiple L2TAG bits being set after VFR */
1507 		tx_ring->flags &=
1508 			~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1509 			  IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2);
1510 		rx_ring->flags &=
1511 			~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1512 			  IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2);
1513 
1514 		if (VLAN_ALLOWED(adapter)) {
1515 			tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1516 			rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1517 		} else if (VLAN_V2_ALLOWED(adapter)) {
1518 			struct virtchnl_vlan_supported_caps *stripping_support;
1519 			struct virtchnl_vlan_supported_caps *insertion_support;
1520 
1521 			stripping_support =
1522 				&adapter->vlan_v2_caps.offloads.stripping_support;
1523 			insertion_support =
1524 				&adapter->vlan_v2_caps.offloads.insertion_support;
1525 
1526 			if (stripping_support->outer) {
1527 				if (stripping_support->outer &
1528 				    VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1529 					rx_ring->flags |=
1530 						IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1531 				else if (stripping_support->outer &
1532 					 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1533 					rx_ring->flags |=
1534 						IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1535 			} else if (stripping_support->inner) {
1536 				if (stripping_support->inner &
1537 				    VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1538 					rx_ring->flags |=
1539 						IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1540 				else if (stripping_support->inner &
1541 					 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1542 					rx_ring->flags |=
1543 						IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1544 			}
1545 
1546 			if (insertion_support->outer) {
1547 				if (insertion_support->outer &
1548 				    VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1549 					tx_ring->flags |=
1550 						IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1551 				else if (insertion_support->outer &
1552 					 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1553 					tx_ring->flags |=
1554 						IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1555 			} else if (insertion_support->inner) {
1556 				if (insertion_support->inner &
1557 				    VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1558 					tx_ring->flags |=
1559 						IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1560 				else if (insertion_support->inner &
1561 					 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1562 					tx_ring->flags |=
1563 						IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1564 			}
1565 		}
1566 	}
1567 }
1568 
1569 /**
1570  * iavf_alloc_queues - Allocate memory for all rings
1571  * @adapter: board private structure to initialize
1572  *
1573  * We allocate one ring per queue at run-time since we don't know the
1574  * number of queues at compile-time.  The polling_netdev array is
1575  * intended for Multiqueue, but should work fine with a single queue.
1576  **/
1577 static int iavf_alloc_queues(struct iavf_adapter *adapter)
1578 {
1579 	int i, num_active_queues;
1580 
1581 	/* If we're in reset reallocating queues we don't actually know yet for
1582 	 * certain the PF gave us the number of queues we asked for but we'll
1583 	 * assume it did.  Once basic reset is finished we'll confirm once we
1584 	 * start negotiating config with PF.
1585 	 */
1586 	if (adapter->num_req_queues)
1587 		num_active_queues = adapter->num_req_queues;
1588 	else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1589 		 adapter->num_tc)
1590 		num_active_queues = adapter->ch_config.total_qps;
1591 	else
1592 		num_active_queues = min_t(int,
1593 					  adapter->vsi_res->num_queue_pairs,
1594 					  (int)(num_online_cpus()));
1595 
1596 
1597 	adapter->tx_rings = kcalloc(num_active_queues,
1598 				    sizeof(struct iavf_ring), GFP_KERNEL);
1599 	if (!adapter->tx_rings)
1600 		goto err_out;
1601 	adapter->rx_rings = kcalloc(num_active_queues,
1602 				    sizeof(struct iavf_ring), GFP_KERNEL);
1603 	if (!adapter->rx_rings)
1604 		goto err_out;
1605 
1606 	for (i = 0; i < num_active_queues; i++) {
1607 		struct iavf_ring *tx_ring;
1608 		struct iavf_ring *rx_ring;
1609 
1610 		tx_ring = &adapter->tx_rings[i];
1611 
1612 		tx_ring->queue_index = i;
1613 		tx_ring->netdev = adapter->netdev;
1614 		tx_ring->dev = &adapter->pdev->dev;
1615 		tx_ring->count = adapter->tx_desc_count;
1616 		tx_ring->itr_setting = IAVF_ITR_TX_DEF;
1617 		if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1618 			tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
1619 
1620 		rx_ring = &adapter->rx_rings[i];
1621 		rx_ring->queue_index = i;
1622 		rx_ring->netdev = adapter->netdev;
1623 		rx_ring->dev = &adapter->pdev->dev;
1624 		rx_ring->count = adapter->rx_desc_count;
1625 		rx_ring->itr_setting = IAVF_ITR_RX_DEF;
1626 	}
1627 
1628 	adapter->num_active_queues = num_active_queues;
1629 
1630 	iavf_set_queue_vlan_tag_loc(adapter);
1631 
1632 	return 0;
1633 
1634 err_out:
1635 	iavf_free_queues(adapter);
1636 	return -ENOMEM;
1637 }
1638 
1639 /**
1640  * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1641  * @adapter: board private structure to initialize
1642  *
1643  * Attempt to configure the interrupts using the best available
1644  * capabilities of the hardware and the kernel.
1645  **/
1646 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1647 {
1648 	int vector, v_budget;
1649 	int pairs = 0;
1650 	int err = 0;
1651 
1652 	if (!adapter->vsi_res) {
1653 		err = -EIO;
1654 		goto out;
1655 	}
1656 	pairs = adapter->num_active_queues;
1657 
1658 	/* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1659 	 * us much good if we have more vectors than CPUs. However, we already
1660 	 * limit the total number of queues by the number of CPUs so we do not
1661 	 * need any further limiting here.
1662 	 */
1663 	v_budget = min_t(int, pairs + NONQ_VECS,
1664 			 (int)adapter->vf_res->max_vectors);
1665 
1666 	adapter->msix_entries = kcalloc(v_budget,
1667 					sizeof(struct msix_entry), GFP_KERNEL);
1668 	if (!adapter->msix_entries) {
1669 		err = -ENOMEM;
1670 		goto out;
1671 	}
1672 
1673 	for (vector = 0; vector < v_budget; vector++)
1674 		adapter->msix_entries[vector].entry = vector;
1675 
1676 	err = iavf_acquire_msix_vectors(adapter, v_budget);
1677 
1678 out:
1679 	netif_set_real_num_rx_queues(adapter->netdev, pairs);
1680 	netif_set_real_num_tx_queues(adapter->netdev, pairs);
1681 	return err;
1682 }
1683 
1684 /**
1685  * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
1686  * @adapter: board private structure
1687  *
1688  * Return 0 on success, negative on failure
1689  **/
1690 static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1691 {
1692 	struct iavf_aqc_get_set_rss_key_data *rss_key =
1693 		(struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
1694 	struct iavf_hw *hw = &adapter->hw;
1695 	enum iavf_status status;
1696 
1697 	if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1698 		/* bail because we already have a command pending */
1699 		dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1700 			adapter->current_op);
1701 		return -EBUSY;
1702 	}
1703 
1704 	status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1705 	if (status) {
1706 		dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1707 			iavf_stat_str(hw, status),
1708 			iavf_aq_str(hw, hw->aq.asq_last_status));
1709 		return iavf_status_to_errno(status);
1710 
1711 	}
1712 
1713 	status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1714 				     adapter->rss_lut, adapter->rss_lut_size);
1715 	if (status) {
1716 		dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1717 			iavf_stat_str(hw, status),
1718 			iavf_aq_str(hw, hw->aq.asq_last_status));
1719 		return iavf_status_to_errno(status);
1720 	}
1721 
1722 	return 0;
1723 
1724 }
1725 
1726 /**
1727  * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1728  * @adapter: board private structure
1729  *
1730  * Returns 0 on success, negative on failure
1731  **/
1732 static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1733 {
1734 	struct iavf_hw *hw = &adapter->hw;
1735 	u32 *dw;
1736 	u16 i;
1737 
1738 	dw = (u32 *)adapter->rss_key;
1739 	for (i = 0; i <= adapter->rss_key_size / 4; i++)
1740 		wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
1741 
1742 	dw = (u32 *)adapter->rss_lut;
1743 	for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1744 		wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
1745 
1746 	iavf_flush(hw);
1747 
1748 	return 0;
1749 }
1750 
1751 /**
1752  * iavf_config_rss - Configure RSS keys and lut
1753  * @adapter: board private structure
1754  *
1755  * Returns 0 on success, negative on failure
1756  **/
1757 int iavf_config_rss(struct iavf_adapter *adapter)
1758 {
1759 
1760 	if (RSS_PF(adapter)) {
1761 		adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1762 					IAVF_FLAG_AQ_SET_RSS_KEY;
1763 		return 0;
1764 	} else if (RSS_AQ(adapter)) {
1765 		return iavf_config_rss_aq(adapter);
1766 	} else {
1767 		return iavf_config_rss_reg(adapter);
1768 	}
1769 }
1770 
1771 /**
1772  * iavf_fill_rss_lut - Fill the lut with default values
1773  * @adapter: board private structure
1774  **/
1775 static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1776 {
1777 	u16 i;
1778 
1779 	for (i = 0; i < adapter->rss_lut_size; i++)
1780 		adapter->rss_lut[i] = i % adapter->num_active_queues;
1781 }
1782 
1783 /**
1784  * iavf_init_rss - Prepare for RSS
1785  * @adapter: board private structure
1786  *
1787  * Return 0 on success, negative on failure
1788  **/
1789 static int iavf_init_rss(struct iavf_adapter *adapter)
1790 {
1791 	struct iavf_hw *hw = &adapter->hw;
1792 
1793 	if (!RSS_PF(adapter)) {
1794 		/* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1795 		if (adapter->vf_res->vf_cap_flags &
1796 		    VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1797 			adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
1798 		else
1799 			adapter->hena = IAVF_DEFAULT_RSS_HENA;
1800 
1801 		wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
1802 		wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1803 	}
1804 
1805 	iavf_fill_rss_lut(adapter);
1806 	netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1807 
1808 	return iavf_config_rss(adapter);
1809 }
1810 
1811 /**
1812  * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1813  * @adapter: board private structure to initialize
1814  *
1815  * We allocate one q_vector per queue interrupt.  If allocation fails we
1816  * return -ENOMEM.
1817  **/
1818 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1819 {
1820 	int q_idx = 0, num_q_vectors;
1821 	struct iavf_q_vector *q_vector;
1822 
1823 	num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1824 	adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1825 				     GFP_KERNEL);
1826 	if (!adapter->q_vectors)
1827 		return -ENOMEM;
1828 
1829 	for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1830 		q_vector = &adapter->q_vectors[q_idx];
1831 		q_vector->adapter = adapter;
1832 		q_vector->vsi = &adapter->vsi;
1833 		q_vector->v_idx = q_idx;
1834 		q_vector->reg_idx = q_idx;
1835 		cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1836 		netif_napi_add(adapter->netdev, &q_vector->napi,
1837 			       iavf_napi_poll, NAPI_POLL_WEIGHT);
1838 	}
1839 
1840 	return 0;
1841 }
1842 
1843 /**
1844  * iavf_free_q_vectors - Free memory allocated for interrupt vectors
1845  * @adapter: board private structure to initialize
1846  *
1847  * This function frees the memory allocated to the q_vectors.  In addition if
1848  * NAPI is enabled it will delete any references to the NAPI struct prior
1849  * to freeing the q_vector.
1850  **/
1851 static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1852 {
1853 	int q_idx, num_q_vectors;
1854 	int napi_vectors;
1855 
1856 	if (!adapter->q_vectors)
1857 		return;
1858 
1859 	num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1860 	napi_vectors = adapter->num_active_queues;
1861 
1862 	for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1863 		struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
1864 
1865 		if (q_idx < napi_vectors)
1866 			netif_napi_del(&q_vector->napi);
1867 	}
1868 	kfree(adapter->q_vectors);
1869 	adapter->q_vectors = NULL;
1870 }
1871 
1872 /**
1873  * iavf_reset_interrupt_capability - Reset MSIX setup
1874  * @adapter: board private structure
1875  *
1876  **/
1877 void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1878 {
1879 	if (!adapter->msix_entries)
1880 		return;
1881 
1882 	pci_disable_msix(adapter->pdev);
1883 	kfree(adapter->msix_entries);
1884 	adapter->msix_entries = NULL;
1885 }
1886 
1887 /**
1888  * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1889  * @adapter: board private structure to initialize
1890  *
1891  **/
1892 int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1893 {
1894 	int err;
1895 
1896 	err = iavf_alloc_queues(adapter);
1897 	if (err) {
1898 		dev_err(&adapter->pdev->dev,
1899 			"Unable to allocate memory for queues\n");
1900 		goto err_alloc_queues;
1901 	}
1902 
1903 	rtnl_lock();
1904 	err = iavf_set_interrupt_capability(adapter);
1905 	rtnl_unlock();
1906 	if (err) {
1907 		dev_err(&adapter->pdev->dev,
1908 			"Unable to setup interrupt capabilities\n");
1909 		goto err_set_interrupt;
1910 	}
1911 
1912 	err = iavf_alloc_q_vectors(adapter);
1913 	if (err) {
1914 		dev_err(&adapter->pdev->dev,
1915 			"Unable to allocate memory for queue vectors\n");
1916 		goto err_alloc_q_vectors;
1917 	}
1918 
1919 	/* If we've made it so far while ADq flag being ON, then we haven't
1920 	 * bailed out anywhere in middle. And ADq isn't just enabled but actual
1921 	 * resources have been allocated in the reset path.
1922 	 * Now we can truly claim that ADq is enabled.
1923 	 */
1924 	if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1925 	    adapter->num_tc)
1926 		dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1927 			 adapter->num_tc);
1928 
1929 	dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1930 		 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1931 		 adapter->num_active_queues);
1932 
1933 	return 0;
1934 err_alloc_q_vectors:
1935 	iavf_reset_interrupt_capability(adapter);
1936 err_set_interrupt:
1937 	iavf_free_queues(adapter);
1938 err_alloc_queues:
1939 	return err;
1940 }
1941 
1942 /**
1943  * iavf_free_rss - Free memory used by RSS structs
1944  * @adapter: board private structure
1945  **/
1946 static void iavf_free_rss(struct iavf_adapter *adapter)
1947 {
1948 	kfree(adapter->rss_key);
1949 	adapter->rss_key = NULL;
1950 
1951 	kfree(adapter->rss_lut);
1952 	adapter->rss_lut = NULL;
1953 }
1954 
1955 /**
1956  * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1957  * @adapter: board private structure
1958  *
1959  * Returns 0 on success, negative on failure
1960  **/
1961 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter)
1962 {
1963 	struct net_device *netdev = adapter->netdev;
1964 	int err;
1965 
1966 	if (netif_running(netdev))
1967 		iavf_free_traffic_irqs(adapter);
1968 	iavf_free_misc_irq(adapter);
1969 	iavf_reset_interrupt_capability(adapter);
1970 	iavf_free_q_vectors(adapter);
1971 	iavf_free_queues(adapter);
1972 
1973 	err =  iavf_init_interrupt_scheme(adapter);
1974 	if (err)
1975 		goto err;
1976 
1977 	netif_tx_stop_all_queues(netdev);
1978 
1979 	err = iavf_request_misc_irq(adapter);
1980 	if (err)
1981 		goto err;
1982 
1983 	set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1984 
1985 	iavf_map_rings_to_vectors(adapter);
1986 err:
1987 	return err;
1988 }
1989 
1990 /**
1991  * iavf_process_aq_command - process aq_required flags
1992  * and sends aq command
1993  * @adapter: pointer to iavf adapter structure
1994  *
1995  * Returns 0 on success
1996  * Returns error code if no command was sent
1997  * or error code if the command failed.
1998  **/
1999 static int iavf_process_aq_command(struct iavf_adapter *adapter)
2000 {
2001 	if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
2002 		return iavf_send_vf_config_msg(adapter);
2003 	if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS)
2004 		return iavf_send_vf_offload_vlan_v2_msg(adapter);
2005 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
2006 		iavf_disable_queues(adapter);
2007 		return 0;
2008 	}
2009 
2010 	if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
2011 		iavf_map_queues(adapter);
2012 		return 0;
2013 	}
2014 
2015 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
2016 		iavf_add_ether_addrs(adapter);
2017 		return 0;
2018 	}
2019 
2020 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
2021 		iavf_add_vlans(adapter);
2022 		return 0;
2023 	}
2024 
2025 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
2026 		iavf_del_ether_addrs(adapter);
2027 		return 0;
2028 	}
2029 
2030 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
2031 		iavf_del_vlans(adapter);
2032 		return 0;
2033 	}
2034 
2035 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
2036 		iavf_enable_vlan_stripping(adapter);
2037 		return 0;
2038 	}
2039 
2040 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
2041 		iavf_disable_vlan_stripping(adapter);
2042 		return 0;
2043 	}
2044 
2045 	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
2046 		iavf_configure_queues(adapter);
2047 		return 0;
2048 	}
2049 
2050 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
2051 		iavf_enable_queues(adapter);
2052 		return 0;
2053 	}
2054 
2055 	if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
2056 		/* This message goes straight to the firmware, not the
2057 		 * PF, so we don't have to set current_op as we will
2058 		 * not get a response through the ARQ.
2059 		 */
2060 		adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
2061 		return 0;
2062 	}
2063 	if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
2064 		iavf_get_hena(adapter);
2065 		return 0;
2066 	}
2067 	if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
2068 		iavf_set_hena(adapter);
2069 		return 0;
2070 	}
2071 	if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
2072 		iavf_set_rss_key(adapter);
2073 		return 0;
2074 	}
2075 	if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
2076 		iavf_set_rss_lut(adapter);
2077 		return 0;
2078 	}
2079 
2080 	if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_PROMISC) {
2081 		iavf_set_promiscuous(adapter, FLAG_VF_UNICAST_PROMISC |
2082 				       FLAG_VF_MULTICAST_PROMISC);
2083 		return 0;
2084 	}
2085 
2086 	if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_ALLMULTI) {
2087 		iavf_set_promiscuous(adapter, FLAG_VF_MULTICAST_PROMISC);
2088 		return 0;
2089 	}
2090 	if ((adapter->aq_required & IAVF_FLAG_AQ_RELEASE_PROMISC) ||
2091 	    (adapter->aq_required & IAVF_FLAG_AQ_RELEASE_ALLMULTI)) {
2092 		iavf_set_promiscuous(adapter, 0);
2093 		return 0;
2094 	}
2095 
2096 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
2097 		iavf_enable_channels(adapter);
2098 		return 0;
2099 	}
2100 
2101 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
2102 		iavf_disable_channels(adapter);
2103 		return 0;
2104 	}
2105 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2106 		iavf_add_cloud_filter(adapter);
2107 		return 0;
2108 	}
2109 
2110 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2111 		iavf_del_cloud_filter(adapter);
2112 		return 0;
2113 	}
2114 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2115 		iavf_del_cloud_filter(adapter);
2116 		return 0;
2117 	}
2118 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2119 		iavf_add_cloud_filter(adapter);
2120 		return 0;
2121 	}
2122 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
2123 		iavf_add_fdir_filter(adapter);
2124 		return IAVF_SUCCESS;
2125 	}
2126 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
2127 		iavf_del_fdir_filter(adapter);
2128 		return IAVF_SUCCESS;
2129 	}
2130 	if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
2131 		iavf_add_adv_rss_cfg(adapter);
2132 		return 0;
2133 	}
2134 	if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
2135 		iavf_del_adv_rss_cfg(adapter);
2136 		return 0;
2137 	}
2138 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) {
2139 		iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2140 		return 0;
2141 	}
2142 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) {
2143 		iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2144 		return 0;
2145 	}
2146 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) {
2147 		iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2148 		return 0;
2149 	}
2150 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) {
2151 		iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2152 		return 0;
2153 	}
2154 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) {
2155 		iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2156 		return 0;
2157 	}
2158 	if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) {
2159 		iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2160 		return 0;
2161 	}
2162 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) {
2163 		iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2164 		return 0;
2165 	}
2166 	if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) {
2167 		iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2168 		return 0;
2169 	}
2170 
2171 	if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {
2172 		iavf_request_stats(adapter);
2173 		return 0;
2174 	}
2175 
2176 	return -EAGAIN;
2177 }
2178 
2179 /**
2180  * iavf_set_vlan_offload_features - set VLAN offload configuration
2181  * @adapter: board private structure
2182  * @prev_features: previous features used for comparison
2183  * @features: updated features used for configuration
2184  *
2185  * Set the aq_required bit(s) based on the requested features passed in to
2186  * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule
2187  * the watchdog if any changes are requested to expedite the request via
2188  * virtchnl.
2189  **/
2190 void
2191 iavf_set_vlan_offload_features(struct iavf_adapter *adapter,
2192 			       netdev_features_t prev_features,
2193 			       netdev_features_t features)
2194 {
2195 	bool enable_stripping = true, enable_insertion = true;
2196 	u16 vlan_ethertype = 0;
2197 	u64 aq_required = 0;
2198 
2199 	/* keep cases separate because one ethertype for offloads can be
2200 	 * disabled at the same time as another is disabled, so check for an
2201 	 * enabled ethertype first, then check for disabled. Default to
2202 	 * ETH_P_8021Q so an ethertype is specified if disabling insertion and
2203 	 * stripping.
2204 	 */
2205 	if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2206 		vlan_ethertype = ETH_P_8021AD;
2207 	else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2208 		vlan_ethertype = ETH_P_8021Q;
2209 	else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2210 		vlan_ethertype = ETH_P_8021AD;
2211 	else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2212 		vlan_ethertype = ETH_P_8021Q;
2213 	else
2214 		vlan_ethertype = ETH_P_8021Q;
2215 
2216 	if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
2217 		enable_stripping = false;
2218 	if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
2219 		enable_insertion = false;
2220 
2221 	if (VLAN_ALLOWED(adapter)) {
2222 		/* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN
2223 		 * stripping via virtchnl. VLAN insertion can be toggled on the
2224 		 * netdev, but it doesn't require a virtchnl message
2225 		 */
2226 		if (enable_stripping)
2227 			aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
2228 		else
2229 			aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
2230 
2231 	} else if (VLAN_V2_ALLOWED(adapter)) {
2232 		switch (vlan_ethertype) {
2233 		case ETH_P_8021Q:
2234 			if (enable_stripping)
2235 				aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;
2236 			else
2237 				aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;
2238 
2239 			if (enable_insertion)
2240 				aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;
2241 			else
2242 				aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;
2243 			break;
2244 		case ETH_P_8021AD:
2245 			if (enable_stripping)
2246 				aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;
2247 			else
2248 				aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;
2249 
2250 			if (enable_insertion)
2251 				aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;
2252 			else
2253 				aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;
2254 			break;
2255 		}
2256 	}
2257 
2258 	if (aq_required) {
2259 		adapter->aq_required |= aq_required;
2260 		mod_delayed_work(iavf_wq, &adapter->watchdog_task, 0);
2261 	}
2262 }
2263 
2264 /**
2265  * iavf_startup - first step of driver startup
2266  * @adapter: board private structure
2267  *
2268  * Function process __IAVF_STARTUP driver state.
2269  * When success the state is changed to __IAVF_INIT_VERSION_CHECK
2270  * when fails the state is changed to __IAVF_INIT_FAILED
2271  **/
2272 static void iavf_startup(struct iavf_adapter *adapter)
2273 {
2274 	struct pci_dev *pdev = adapter->pdev;
2275 	struct iavf_hw *hw = &adapter->hw;
2276 	enum iavf_status status;
2277 	int ret;
2278 
2279 	WARN_ON(adapter->state != __IAVF_STARTUP);
2280 
2281 	/* driver loaded, probe complete */
2282 	adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2283 	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2284 	status = iavf_set_mac_type(hw);
2285 	if (status) {
2286 		dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", status);
2287 		goto err;
2288 	}
2289 
2290 	ret = iavf_check_reset_complete(hw);
2291 	if (ret) {
2292 		dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
2293 			 ret);
2294 		goto err;
2295 	}
2296 	hw->aq.num_arq_entries = IAVF_AQ_LEN;
2297 	hw->aq.num_asq_entries = IAVF_AQ_LEN;
2298 	hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2299 	hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2300 
2301 	status = iavf_init_adminq(hw);
2302 	if (status) {
2303 		dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
2304 			status);
2305 		goto err;
2306 	}
2307 	ret = iavf_send_api_ver(adapter);
2308 	if (ret) {
2309 		dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret);
2310 		iavf_shutdown_adminq(hw);
2311 		goto err;
2312 	}
2313 	iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK);
2314 	return;
2315 err:
2316 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2317 }
2318 
2319 /**
2320  * iavf_init_version_check - second step of driver startup
2321  * @adapter: board private structure
2322  *
2323  * Function process __IAVF_INIT_VERSION_CHECK driver state.
2324  * When success the state is changed to __IAVF_INIT_GET_RESOURCES
2325  * when fails the state is changed to __IAVF_INIT_FAILED
2326  **/
2327 static void iavf_init_version_check(struct iavf_adapter *adapter)
2328 {
2329 	struct pci_dev *pdev = adapter->pdev;
2330 	struct iavf_hw *hw = &adapter->hw;
2331 	int err = -EAGAIN;
2332 
2333 	WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
2334 
2335 	if (!iavf_asq_done(hw)) {
2336 		dev_err(&pdev->dev, "Admin queue command never completed\n");
2337 		iavf_shutdown_adminq(hw);
2338 		iavf_change_state(adapter, __IAVF_STARTUP);
2339 		goto err;
2340 	}
2341 
2342 	/* aq msg sent, awaiting reply */
2343 	err = iavf_verify_api_ver(adapter);
2344 	if (err) {
2345 		if (err == -EALREADY)
2346 			err = iavf_send_api_ver(adapter);
2347 		else
2348 			dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
2349 				adapter->pf_version.major,
2350 				adapter->pf_version.minor,
2351 				VIRTCHNL_VERSION_MAJOR,
2352 				VIRTCHNL_VERSION_MINOR);
2353 		goto err;
2354 	}
2355 	err = iavf_send_vf_config_msg(adapter);
2356 	if (err) {
2357 		dev_err(&pdev->dev, "Unable to send config request (%d)\n",
2358 			err);
2359 		goto err;
2360 	}
2361 	iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES);
2362 	return;
2363 err:
2364 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2365 }
2366 
2367 /**
2368  * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES
2369  * @adapter: board private structure
2370  */
2371 int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter)
2372 {
2373 	int i, num_req_queues = adapter->num_req_queues;
2374 	struct iavf_vsi *vsi = &adapter->vsi;
2375 
2376 	for (i = 0; i < adapter->vf_res->num_vsis; i++) {
2377 		if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
2378 			adapter->vsi_res = &adapter->vf_res->vsi_res[i];
2379 	}
2380 	if (!adapter->vsi_res) {
2381 		dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
2382 		return -ENODEV;
2383 	}
2384 
2385 	if (num_req_queues &&
2386 	    num_req_queues > adapter->vsi_res->num_queue_pairs) {
2387 		/* Problem.  The PF gave us fewer queues than what we had
2388 		 * negotiated in our request.  Need a reset to see if we can't
2389 		 * get back to a working state.
2390 		 */
2391 		dev_err(&adapter->pdev->dev,
2392 			"Requested %d queues, but PF only gave us %d.\n",
2393 			num_req_queues,
2394 			adapter->vsi_res->num_queue_pairs);
2395 		adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED;
2396 		adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
2397 		iavf_schedule_reset(adapter);
2398 
2399 		return -EAGAIN;
2400 	}
2401 	adapter->num_req_queues = 0;
2402 	adapter->vsi.id = adapter->vsi_res->vsi_id;
2403 
2404 	adapter->vsi.back = adapter;
2405 	adapter->vsi.base_vector = 1;
2406 	vsi->netdev = adapter->netdev;
2407 	vsi->qs_handle = adapter->vsi_res->qset_handle;
2408 	if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
2409 		adapter->rss_key_size = adapter->vf_res->rss_key_size;
2410 		adapter->rss_lut_size = adapter->vf_res->rss_lut_size;
2411 	} else {
2412 		adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
2413 		adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
2414 	}
2415 
2416 	return 0;
2417 }
2418 
2419 /**
2420  * iavf_init_get_resources - third step of driver startup
2421  * @adapter: board private structure
2422  *
2423  * Function process __IAVF_INIT_GET_RESOURCES driver state and
2424  * finishes driver initialization procedure.
2425  * When success the state is changed to __IAVF_DOWN
2426  * when fails the state is changed to __IAVF_INIT_FAILED
2427  **/
2428 static void iavf_init_get_resources(struct iavf_adapter *adapter)
2429 {
2430 	struct pci_dev *pdev = adapter->pdev;
2431 	struct iavf_hw *hw = &adapter->hw;
2432 	int err;
2433 
2434 	WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
2435 	/* aq msg sent, awaiting reply */
2436 	if (!adapter->vf_res) {
2437 		adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
2438 					  GFP_KERNEL);
2439 		if (!adapter->vf_res) {
2440 			err = -ENOMEM;
2441 			goto err;
2442 		}
2443 	}
2444 	err = iavf_get_vf_config(adapter);
2445 	if (err == -EALREADY) {
2446 		err = iavf_send_vf_config_msg(adapter);
2447 		goto err;
2448 	} else if (err == -EINVAL) {
2449 		/* We only get -EINVAL if the device is in a very bad
2450 		 * state or if we've been disabled for previous bad
2451 		 * behavior. Either way, we're done now.
2452 		 */
2453 		iavf_shutdown_adminq(hw);
2454 		dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
2455 		return;
2456 	}
2457 	if (err) {
2458 		dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
2459 		goto err_alloc;
2460 	}
2461 
2462 	err = iavf_parse_vf_resource_msg(adapter);
2463 	if (err) {
2464 		dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n",
2465 			err);
2466 		goto err_alloc;
2467 	}
2468 	/* Some features require additional messages to negotiate extended
2469 	 * capabilities. These are processed in sequence by the
2470 	 * __IAVF_INIT_EXTENDED_CAPS driver state.
2471 	 */
2472 	adapter->extended_caps = IAVF_EXTENDED_CAPS;
2473 
2474 	iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS);
2475 	return;
2476 
2477 err_alloc:
2478 	kfree(adapter->vf_res);
2479 	adapter->vf_res = NULL;
2480 err:
2481 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2482 }
2483 
2484 /**
2485  * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2486  * @adapter: board private structure
2487  *
2488  * Function processes send of the extended VLAN V2 capability message to the
2489  * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent,
2490  * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2.
2491  */
2492 static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2493 {
2494 	int ret;
2495 
2496 	WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2));
2497 
2498 	ret = iavf_send_vf_offload_vlan_v2_msg(adapter);
2499 	if (ret && ret == -EOPNOTSUPP) {
2500 		/* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case,
2501 		 * we did not send the capability exchange message and do not
2502 		 * expect a response.
2503 		 */
2504 		adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2505 	}
2506 
2507 	/* We sent the message, so move on to the next step */
2508 	adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2509 }
2510 
2511 /**
2512  * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2513  * @adapter: board private structure
2514  *
2515  * Function processes receipt of the extended VLAN V2 capability message from
2516  * the PF.
2517  **/
2518 static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2519 {
2520 	int ret;
2521 
2522 	WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2));
2523 
2524 	memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps));
2525 
2526 	ret = iavf_get_vf_vlan_v2_caps(adapter);
2527 	if (ret)
2528 		goto err;
2529 
2530 	/* We've processed receipt of the VLAN V2 caps message */
2531 	adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2532 	return;
2533 err:
2534 	/* We didn't receive a reply. Make sure we try sending again when
2535 	 * __IAVF_INIT_FAILED attempts to recover.
2536 	 */
2537 	adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2538 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2539 }
2540 
2541 /**
2542  * iavf_init_process_extended_caps - Part of driver startup
2543  * @adapter: board private structure
2544  *
2545  * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state
2546  * handles negotiating capabilities for features which require an additional
2547  * message.
2548  *
2549  * Once all extended capabilities exchanges are finished, the driver will
2550  * transition into __IAVF_INIT_CONFIG_ADAPTER.
2551  */
2552 static void iavf_init_process_extended_caps(struct iavf_adapter *adapter)
2553 {
2554 	WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS);
2555 
2556 	/* Process capability exchange for VLAN V2 */
2557 	if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) {
2558 		iavf_init_send_offload_vlan_v2_caps(adapter);
2559 		return;
2560 	} else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) {
2561 		iavf_init_recv_offload_vlan_v2_caps(adapter);
2562 		return;
2563 	}
2564 
2565 	/* When we reach here, no further extended capabilities exchanges are
2566 	 * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER
2567 	 */
2568 	iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER);
2569 }
2570 
2571 /**
2572  * iavf_init_config_adapter - last part of driver startup
2573  * @adapter: board private structure
2574  *
2575  * After all the supported capabilities are negotiated, then the
2576  * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization.
2577  */
2578 static void iavf_init_config_adapter(struct iavf_adapter *adapter)
2579 {
2580 	struct net_device *netdev = adapter->netdev;
2581 	struct pci_dev *pdev = adapter->pdev;
2582 	int err;
2583 
2584 	WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER);
2585 
2586 	if (iavf_process_config(adapter))
2587 		goto err;
2588 
2589 	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2590 
2591 	adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
2592 
2593 	netdev->netdev_ops = &iavf_netdev_ops;
2594 	iavf_set_ethtool_ops(netdev);
2595 	netdev->watchdog_timeo = 5 * HZ;
2596 
2597 	/* MTU range: 68 - 9710 */
2598 	netdev->min_mtu = ETH_MIN_MTU;
2599 	netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD;
2600 
2601 	if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
2602 		dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
2603 			 adapter->hw.mac.addr);
2604 		eth_hw_addr_random(netdev);
2605 		ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
2606 	} else {
2607 		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2608 		ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
2609 	}
2610 
2611 	adapter->flags |= IAVF_FLAG_INITIAL_MAC_SET;
2612 
2613 	adapter->tx_desc_count = IAVF_DEFAULT_TXD;
2614 	adapter->rx_desc_count = IAVF_DEFAULT_RXD;
2615 	err = iavf_init_interrupt_scheme(adapter);
2616 	if (err)
2617 		goto err_sw_init;
2618 	iavf_map_rings_to_vectors(adapter);
2619 	if (adapter->vf_res->vf_cap_flags &
2620 		VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
2621 		adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
2622 
2623 	err = iavf_request_misc_irq(adapter);
2624 	if (err)
2625 		goto err_sw_init;
2626 
2627 	netif_carrier_off(netdev);
2628 	adapter->link_up = false;
2629 
2630 	/* set the semaphore to prevent any callbacks after device registration
2631 	 * up to time when state of driver will be set to __IAVF_DOWN
2632 	 */
2633 	rtnl_lock();
2634 	if (!adapter->netdev_registered) {
2635 		err = register_netdevice(netdev);
2636 		if (err) {
2637 			rtnl_unlock();
2638 			goto err_register;
2639 		}
2640 	}
2641 
2642 	adapter->netdev_registered = true;
2643 
2644 	netif_tx_stop_all_queues(netdev);
2645 	if (CLIENT_ALLOWED(adapter)) {
2646 		err = iavf_lan_add_device(adapter);
2647 		if (err)
2648 			dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
2649 				 err);
2650 	}
2651 	dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
2652 	if (netdev->features & NETIF_F_GRO)
2653 		dev_info(&pdev->dev, "GRO is enabled\n");
2654 
2655 	iavf_change_state(adapter, __IAVF_DOWN);
2656 	set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2657 	rtnl_unlock();
2658 
2659 	iavf_misc_irq_enable(adapter);
2660 	wake_up(&adapter->down_waitqueue);
2661 
2662 	adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
2663 	adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
2664 	if (!adapter->rss_key || !adapter->rss_lut) {
2665 		err = -ENOMEM;
2666 		goto err_mem;
2667 	}
2668 	if (RSS_AQ(adapter))
2669 		adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2670 	else
2671 		iavf_init_rss(adapter);
2672 
2673 	if (VLAN_V2_ALLOWED(adapter))
2674 		/* request initial VLAN offload settings */
2675 		iavf_set_vlan_offload_features(adapter, 0, netdev->features);
2676 
2677 	return;
2678 err_mem:
2679 	iavf_free_rss(adapter);
2680 err_register:
2681 	iavf_free_misc_irq(adapter);
2682 err_sw_init:
2683 	iavf_reset_interrupt_capability(adapter);
2684 err:
2685 	iavf_change_state(adapter, __IAVF_INIT_FAILED);
2686 }
2687 
2688 /**
2689  * iavf_watchdog_task - Periodic call-back task
2690  * @work: pointer to work_struct
2691  **/
2692 static void iavf_watchdog_task(struct work_struct *work)
2693 {
2694 	struct iavf_adapter *adapter = container_of(work,
2695 						    struct iavf_adapter,
2696 						    watchdog_task.work);
2697 	struct iavf_hw *hw = &adapter->hw;
2698 	u32 reg_val;
2699 
2700 	if (!mutex_trylock(&adapter->crit_lock)) {
2701 		if (adapter->state == __IAVF_REMOVE)
2702 			return;
2703 
2704 		goto restart_watchdog;
2705 	}
2706 
2707 	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2708 		iavf_change_state(adapter, __IAVF_COMM_FAILED);
2709 
2710 	if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
2711 		adapter->aq_required = 0;
2712 		adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2713 		mutex_unlock(&adapter->crit_lock);
2714 		queue_work(iavf_wq, &adapter->reset_task);
2715 		return;
2716 	}
2717 
2718 	switch (adapter->state) {
2719 	case __IAVF_STARTUP:
2720 		iavf_startup(adapter);
2721 		mutex_unlock(&adapter->crit_lock);
2722 		queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2723 				   msecs_to_jiffies(30));
2724 		return;
2725 	case __IAVF_INIT_VERSION_CHECK:
2726 		iavf_init_version_check(adapter);
2727 		mutex_unlock(&adapter->crit_lock);
2728 		queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2729 				   msecs_to_jiffies(30));
2730 		return;
2731 	case __IAVF_INIT_GET_RESOURCES:
2732 		iavf_init_get_resources(adapter);
2733 		mutex_unlock(&adapter->crit_lock);
2734 		queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2735 				   msecs_to_jiffies(1));
2736 		return;
2737 	case __IAVF_INIT_EXTENDED_CAPS:
2738 		iavf_init_process_extended_caps(adapter);
2739 		mutex_unlock(&adapter->crit_lock);
2740 		queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2741 				   msecs_to_jiffies(1));
2742 		return;
2743 	case __IAVF_INIT_CONFIG_ADAPTER:
2744 		iavf_init_config_adapter(adapter);
2745 		mutex_unlock(&adapter->crit_lock);
2746 		queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2747 				   msecs_to_jiffies(1));
2748 		return;
2749 	case __IAVF_INIT_FAILED:
2750 		if (test_bit(__IAVF_IN_REMOVE_TASK,
2751 			     &adapter->crit_section)) {
2752 			/* Do not update the state and do not reschedule
2753 			 * watchdog task, iavf_remove should handle this state
2754 			 * as it can loop forever
2755 			 */
2756 			mutex_unlock(&adapter->crit_lock);
2757 			return;
2758 		}
2759 		if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
2760 			dev_err(&adapter->pdev->dev,
2761 				"Failed to communicate with PF; waiting before retry\n");
2762 			adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2763 			iavf_shutdown_adminq(hw);
2764 			mutex_unlock(&adapter->crit_lock);
2765 			queue_delayed_work(iavf_wq,
2766 					   &adapter->watchdog_task, (5 * HZ));
2767 			return;
2768 		}
2769 		/* Try again from failed step*/
2770 		iavf_change_state(adapter, adapter->last_state);
2771 		mutex_unlock(&adapter->crit_lock);
2772 		queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ);
2773 		return;
2774 	case __IAVF_COMM_FAILED:
2775 		if (test_bit(__IAVF_IN_REMOVE_TASK,
2776 			     &adapter->crit_section)) {
2777 			/* Set state to __IAVF_INIT_FAILED and perform remove
2778 			 * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task
2779 			 * doesn't bring the state back to __IAVF_COMM_FAILED.
2780 			 */
2781 			iavf_change_state(adapter, __IAVF_INIT_FAILED);
2782 			adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2783 			mutex_unlock(&adapter->crit_lock);
2784 			return;
2785 		}
2786 		reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2787 			  IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2788 		if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
2789 		    reg_val == VIRTCHNL_VFR_COMPLETED) {
2790 			/* A chance for redemption! */
2791 			dev_err(&adapter->pdev->dev,
2792 				"Hardware came out of reset. Attempting reinit.\n");
2793 			/* When init task contacts the PF and
2794 			 * gets everything set up again, it'll restart the
2795 			 * watchdog for us. Down, boy. Sit. Stay. Woof.
2796 			 */
2797 			iavf_change_state(adapter, __IAVF_STARTUP);
2798 			adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2799 		}
2800 		adapter->aq_required = 0;
2801 		adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2802 		mutex_unlock(&adapter->crit_lock);
2803 		queue_delayed_work(iavf_wq,
2804 				   &adapter->watchdog_task,
2805 				   msecs_to_jiffies(10));
2806 		return;
2807 	case __IAVF_RESETTING:
2808 		mutex_unlock(&adapter->crit_lock);
2809 		queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
2810 		return;
2811 	case __IAVF_DOWN:
2812 	case __IAVF_DOWN_PENDING:
2813 	case __IAVF_TESTING:
2814 	case __IAVF_RUNNING:
2815 		if (adapter->current_op) {
2816 			if (!iavf_asq_done(hw)) {
2817 				dev_dbg(&adapter->pdev->dev,
2818 					"Admin queue timeout\n");
2819 				iavf_send_api_ver(adapter);
2820 			}
2821 		} else {
2822 			int ret = iavf_process_aq_command(adapter);
2823 
2824 			/* An error will be returned if no commands were
2825 			 * processed; use this opportunity to update stats
2826 			 * if the error isn't -ENOTSUPP
2827 			 */
2828 			if (ret && ret != -EOPNOTSUPP &&
2829 			    adapter->state == __IAVF_RUNNING)
2830 				iavf_request_stats(adapter);
2831 		}
2832 		if (adapter->state == __IAVF_RUNNING)
2833 			iavf_detect_recover_hung(&adapter->vsi);
2834 		break;
2835 	case __IAVF_REMOVE:
2836 	default:
2837 		mutex_unlock(&adapter->crit_lock);
2838 		return;
2839 	}
2840 
2841 	/* check for hw reset */
2842 	reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2843 	if (!reg_val) {
2844 		adapter->flags |= IAVF_FLAG_RESET_PENDING;
2845 		adapter->aq_required = 0;
2846 		adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2847 		dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
2848 		queue_work(iavf_wq, &adapter->reset_task);
2849 		mutex_unlock(&adapter->crit_lock);
2850 		queue_delayed_work(iavf_wq,
2851 				   &adapter->watchdog_task, HZ * 2);
2852 		return;
2853 	}
2854 
2855 	schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
2856 	mutex_unlock(&adapter->crit_lock);
2857 restart_watchdog:
2858 	if (adapter->state >= __IAVF_DOWN)
2859 		queue_work(iavf_wq, &adapter->adminq_task);
2860 	if (adapter->aq_required)
2861 		queue_delayed_work(iavf_wq, &adapter->watchdog_task,
2862 				   msecs_to_jiffies(20));
2863 	else
2864 		queue_delayed_work(iavf_wq, &adapter->watchdog_task, HZ * 2);
2865 }
2866 
2867 /**
2868  * iavf_disable_vf - disable VF
2869  * @adapter: board private structure
2870  *
2871  * Set communication failed flag and free all resources.
2872  * NOTE: This function is expected to be called with crit_lock being held.
2873  **/
2874 static void iavf_disable_vf(struct iavf_adapter *adapter)
2875 {
2876 	struct iavf_mac_filter *f, *ftmp;
2877 	struct iavf_vlan_filter *fv, *fvtmp;
2878 	struct iavf_cloud_filter *cf, *cftmp;
2879 
2880 	adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2881 
2882 	/* We don't use netif_running() because it may be true prior to
2883 	 * ndo_open() returning, so we can't assume it means all our open
2884 	 * tasks have finished, since we're not holding the rtnl_lock here.
2885 	 */
2886 	if (adapter->state == __IAVF_RUNNING) {
2887 		set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2888 		netif_carrier_off(adapter->netdev);
2889 		netif_tx_disable(adapter->netdev);
2890 		adapter->link_up = false;
2891 		iavf_napi_disable_all(adapter);
2892 		iavf_irq_disable(adapter);
2893 		iavf_free_traffic_irqs(adapter);
2894 		iavf_free_all_tx_resources(adapter);
2895 		iavf_free_all_rx_resources(adapter);
2896 	}
2897 
2898 	spin_lock_bh(&adapter->mac_vlan_list_lock);
2899 
2900 	/* Delete all of the filters */
2901 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2902 		list_del(&f->list);
2903 		kfree(f);
2904 	}
2905 
2906 	list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
2907 		list_del(&fv->list);
2908 		kfree(fv);
2909 	}
2910 
2911 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
2912 
2913 	spin_lock_bh(&adapter->cloud_filter_list_lock);
2914 	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
2915 		list_del(&cf->list);
2916 		kfree(cf);
2917 		adapter->num_cloud_filters--;
2918 	}
2919 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
2920 
2921 	iavf_free_misc_irq(adapter);
2922 	iavf_reset_interrupt_capability(adapter);
2923 	iavf_free_q_vectors(adapter);
2924 	iavf_free_queues(adapter);
2925 	memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
2926 	iavf_shutdown_adminq(&adapter->hw);
2927 	adapter->netdev->flags &= ~IFF_UP;
2928 	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2929 	iavf_change_state(adapter, __IAVF_DOWN);
2930 	wake_up(&adapter->down_waitqueue);
2931 	dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
2932 }
2933 
2934 /**
2935  * iavf_reset_task - Call-back task to handle hardware reset
2936  * @work: pointer to work_struct
2937  *
2938  * During reset we need to shut down and reinitialize the admin queue
2939  * before we can use it to communicate with the PF again. We also clear
2940  * and reinit the rings because that context is lost as well.
2941  **/
2942 static void iavf_reset_task(struct work_struct *work)
2943 {
2944 	struct iavf_adapter *adapter = container_of(work,
2945 						      struct iavf_adapter,
2946 						      reset_task);
2947 	struct virtchnl_vf_resource *vfres = adapter->vf_res;
2948 	struct net_device *netdev = adapter->netdev;
2949 	struct iavf_hw *hw = &adapter->hw;
2950 	struct iavf_mac_filter *f, *ftmp;
2951 	struct iavf_cloud_filter *cf;
2952 	enum iavf_status status;
2953 	u32 reg_val;
2954 	int i = 0, err;
2955 	bool running;
2956 
2957 	/* Detach interface to avoid subsequent NDO callbacks */
2958 	rtnl_lock();
2959 	netif_device_detach(netdev);
2960 	rtnl_unlock();
2961 
2962 	/* When device is being removed it doesn't make sense to run the reset
2963 	 * task, just return in such a case.
2964 	 */
2965 	if (!mutex_trylock(&adapter->crit_lock)) {
2966 		if (adapter->state != __IAVF_REMOVE)
2967 			queue_work(iavf_wq, &adapter->reset_task);
2968 
2969 		goto reset_finish;
2970 	}
2971 
2972 	while (!mutex_trylock(&adapter->client_lock))
2973 		usleep_range(500, 1000);
2974 	if (CLIENT_ENABLED(adapter)) {
2975 		adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN |
2976 				    IAVF_FLAG_CLIENT_NEEDS_CLOSE |
2977 				    IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
2978 				    IAVF_FLAG_SERVICE_CLIENT_REQUESTED);
2979 		cancel_delayed_work_sync(&adapter->client_task);
2980 		iavf_notify_client_close(&adapter->vsi, true);
2981 	}
2982 	iavf_misc_irq_disable(adapter);
2983 	if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
2984 		adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
2985 		/* Restart the AQ here. If we have been reset but didn't
2986 		 * detect it, or if the PF had to reinit, our AQ will be hosed.
2987 		 */
2988 		iavf_shutdown_adminq(hw);
2989 		iavf_init_adminq(hw);
2990 		iavf_request_reset(adapter);
2991 	}
2992 	adapter->flags |= IAVF_FLAG_RESET_PENDING;
2993 
2994 	/* poll until we see the reset actually happen */
2995 	for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
2996 		reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
2997 			  IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2998 		if (!reg_val)
2999 			break;
3000 		usleep_range(5000, 10000);
3001 	}
3002 	if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
3003 		dev_info(&adapter->pdev->dev, "Never saw reset\n");
3004 		goto continue_reset; /* act like the reset happened */
3005 	}
3006 
3007 	/* wait until the reset is complete and the PF is responding to us */
3008 	for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
3009 		/* sleep first to make sure a minimum wait time is met */
3010 		msleep(IAVF_RESET_WAIT_MS);
3011 
3012 		reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
3013 			  IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
3014 		if (reg_val == VIRTCHNL_VFR_VFACTIVE)
3015 			break;
3016 	}
3017 
3018 	pci_set_master(adapter->pdev);
3019 	pci_restore_msi_state(adapter->pdev);
3020 
3021 	if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
3022 		dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
3023 			reg_val);
3024 		iavf_disable_vf(adapter);
3025 		mutex_unlock(&adapter->client_lock);
3026 		mutex_unlock(&adapter->crit_lock);
3027 		return; /* Do not attempt to reinit. It's dead, Jim. */
3028 	}
3029 
3030 continue_reset:
3031 	/* We don't use netif_running() because it may be true prior to
3032 	 * ndo_open() returning, so we can't assume it means all our open
3033 	 * tasks have finished, since we're not holding the rtnl_lock here.
3034 	 */
3035 	running = adapter->state == __IAVF_RUNNING;
3036 
3037 	if (running) {
3038 		netif_carrier_off(netdev);
3039 		adapter->link_up = false;
3040 		iavf_napi_disable_all(adapter);
3041 	}
3042 	iavf_irq_disable(adapter);
3043 
3044 	iavf_change_state(adapter, __IAVF_RESETTING);
3045 	adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
3046 
3047 	/* free the Tx/Rx rings and descriptors, might be better to just
3048 	 * re-use them sometime in the future
3049 	 */
3050 	iavf_free_all_rx_resources(adapter);
3051 	iavf_free_all_tx_resources(adapter);
3052 
3053 	adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
3054 	/* kill and reinit the admin queue */
3055 	iavf_shutdown_adminq(hw);
3056 	adapter->current_op = VIRTCHNL_OP_UNKNOWN;
3057 	status = iavf_init_adminq(hw);
3058 	if (status) {
3059 		dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
3060 			 status);
3061 		goto reset_err;
3062 	}
3063 	adapter->aq_required = 0;
3064 
3065 	if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3066 	    (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3067 		err = iavf_reinit_interrupt_scheme(adapter);
3068 		if (err)
3069 			goto reset_err;
3070 	}
3071 
3072 	if (RSS_AQ(adapter)) {
3073 		adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
3074 	} else {
3075 		err = iavf_init_rss(adapter);
3076 		if (err)
3077 			goto reset_err;
3078 	}
3079 
3080 	adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
3081 	/* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been
3082 	 * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here,
3083 	 * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until
3084 	 * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have
3085 	 * been successfully sent and negotiated
3086 	 */
3087 	adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;
3088 	adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
3089 
3090 	spin_lock_bh(&adapter->mac_vlan_list_lock);
3091 
3092 	/* Delete filter for the current MAC address, it could have
3093 	 * been changed by the PF via administratively set MAC.
3094 	 * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
3095 	 */
3096 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3097 		if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {
3098 			list_del(&f->list);
3099 			kfree(f);
3100 		}
3101 	}
3102 	/* re-add all MAC filters */
3103 	list_for_each_entry(f, &adapter->mac_filter_list, list) {
3104 		f->add = true;
3105 	}
3106 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
3107 
3108 	/* check if TCs are running and re-add all cloud filters */
3109 	spin_lock_bh(&adapter->cloud_filter_list_lock);
3110 	if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
3111 	    adapter->num_tc) {
3112 		list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
3113 			cf->add = true;
3114 		}
3115 	}
3116 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
3117 
3118 	adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
3119 	adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
3120 	iavf_misc_irq_enable(adapter);
3121 
3122 	bitmap_clear(adapter->vsi.active_cvlans, 0, VLAN_N_VID);
3123 	bitmap_clear(adapter->vsi.active_svlans, 0, VLAN_N_VID);
3124 
3125 	mod_delayed_work(iavf_wq, &adapter->watchdog_task, 2);
3126 
3127 	/* We were running when the reset started, so we need to restore some
3128 	 * state here.
3129 	 */
3130 	if (running) {
3131 		/* allocate transmit descriptors */
3132 		err = iavf_setup_all_tx_resources(adapter);
3133 		if (err)
3134 			goto reset_err;
3135 
3136 		/* allocate receive descriptors */
3137 		err = iavf_setup_all_rx_resources(adapter);
3138 		if (err)
3139 			goto reset_err;
3140 
3141 		if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3142 		    (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3143 			err = iavf_request_traffic_irqs(adapter, netdev->name);
3144 			if (err)
3145 				goto reset_err;
3146 
3147 			adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED;
3148 		}
3149 
3150 		iavf_configure(adapter);
3151 
3152 		/* iavf_up_complete() will switch device back
3153 		 * to __IAVF_RUNNING
3154 		 */
3155 		iavf_up_complete(adapter);
3156 
3157 		iavf_irq_enable(adapter, true);
3158 	} else {
3159 		iavf_change_state(adapter, __IAVF_DOWN);
3160 		wake_up(&adapter->down_waitqueue);
3161 	}
3162 
3163 	adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
3164 
3165 	mutex_unlock(&adapter->client_lock);
3166 	mutex_unlock(&adapter->crit_lock);
3167 
3168 	goto reset_finish;
3169 reset_err:
3170 	if (running) {
3171 		set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
3172 		iavf_free_traffic_irqs(adapter);
3173 	}
3174 	iavf_disable_vf(adapter);
3175 
3176 	mutex_unlock(&adapter->client_lock);
3177 	mutex_unlock(&adapter->crit_lock);
3178 	dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
3179 reset_finish:
3180 	rtnl_lock();
3181 	netif_device_attach(netdev);
3182 	rtnl_unlock();
3183 }
3184 
3185 /**
3186  * iavf_adminq_task - worker thread to clean the admin queue
3187  * @work: pointer to work_struct containing our data
3188  **/
3189 static void iavf_adminq_task(struct work_struct *work)
3190 {
3191 	struct iavf_adapter *adapter =
3192 		container_of(work, struct iavf_adapter, adminq_task);
3193 	struct iavf_hw *hw = &adapter->hw;
3194 	struct iavf_arq_event_info event;
3195 	enum virtchnl_ops v_op;
3196 	enum iavf_status ret, v_ret;
3197 	u32 val, oldval;
3198 	u16 pending;
3199 
3200 	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
3201 		goto out;
3202 
3203 	if (!mutex_trylock(&adapter->crit_lock)) {
3204 		if (adapter->state == __IAVF_REMOVE)
3205 			return;
3206 
3207 		queue_work(iavf_wq, &adapter->adminq_task);
3208 		goto out;
3209 	}
3210 
3211 	event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
3212 	event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
3213 	if (!event.msg_buf)
3214 		goto out;
3215 
3216 	do {
3217 		ret = iavf_clean_arq_element(hw, &event, &pending);
3218 		v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
3219 		v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
3220 
3221 		if (ret || !v_op)
3222 			break; /* No event to process or error cleaning ARQ */
3223 
3224 		iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
3225 					 event.msg_len);
3226 		if (pending != 0)
3227 			memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
3228 	} while (pending);
3229 	mutex_unlock(&adapter->crit_lock);
3230 
3231 	if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES)) {
3232 		if (adapter->netdev_registered ||
3233 		    !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
3234 			struct net_device *netdev = adapter->netdev;
3235 
3236 			rtnl_lock();
3237 			netdev_update_features(netdev);
3238 			rtnl_unlock();
3239 			/* Request VLAN offload settings */
3240 			if (VLAN_V2_ALLOWED(adapter))
3241 				iavf_set_vlan_offload_features
3242 					(adapter, 0, netdev->features);
3243 
3244 			iavf_set_queue_vlan_tag_loc(adapter);
3245 		}
3246 
3247 		adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;
3248 	}
3249 	if ((adapter->flags &
3250 	     (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED)) ||
3251 	    adapter->state == __IAVF_RESETTING)
3252 		goto freedom;
3253 
3254 	/* check for error indications */
3255 	val = rd32(hw, hw->aq.arq.len);
3256 	if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */
3257 		goto freedom;
3258 	oldval = val;
3259 	if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
3260 		dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
3261 		val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
3262 	}
3263 	if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
3264 		dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
3265 		val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
3266 	}
3267 	if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
3268 		dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
3269 		val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
3270 	}
3271 	if (oldval != val)
3272 		wr32(hw, hw->aq.arq.len, val);
3273 
3274 	val = rd32(hw, hw->aq.asq.len);
3275 	oldval = val;
3276 	if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
3277 		dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
3278 		val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
3279 	}
3280 	if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
3281 		dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
3282 		val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
3283 	}
3284 	if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
3285 		dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
3286 		val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
3287 	}
3288 	if (oldval != val)
3289 		wr32(hw, hw->aq.asq.len, val);
3290 
3291 freedom:
3292 	kfree(event.msg_buf);
3293 out:
3294 	/* re-enable Admin queue interrupt cause */
3295 	iavf_misc_irq_enable(adapter);
3296 }
3297 
3298 /**
3299  * iavf_client_task - worker thread to perform client work
3300  * @work: pointer to work_struct containing our data
3301  *
3302  * This task handles client interactions. Because client calls can be
3303  * reentrant, we can't handle them in the watchdog.
3304  **/
3305 static void iavf_client_task(struct work_struct *work)
3306 {
3307 	struct iavf_adapter *adapter =
3308 		container_of(work, struct iavf_adapter, client_task.work);
3309 
3310 	/* If we can't get the client bit, just give up. We'll be rescheduled
3311 	 * later.
3312 	 */
3313 
3314 	if (!mutex_trylock(&adapter->client_lock))
3315 		return;
3316 
3317 	if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) {
3318 		iavf_client_subtask(adapter);
3319 		adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
3320 		goto out;
3321 	}
3322 	if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
3323 		iavf_notify_client_l2_params(&adapter->vsi);
3324 		adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
3325 		goto out;
3326 	}
3327 	if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) {
3328 		iavf_notify_client_close(&adapter->vsi, false);
3329 		adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE;
3330 		goto out;
3331 	}
3332 	if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) {
3333 		iavf_notify_client_open(&adapter->vsi);
3334 		adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN;
3335 	}
3336 out:
3337 	mutex_unlock(&adapter->client_lock);
3338 }
3339 
3340 /**
3341  * iavf_free_all_tx_resources - Free Tx Resources for All Queues
3342  * @adapter: board private structure
3343  *
3344  * Free all transmit software resources
3345  **/
3346 void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
3347 {
3348 	int i;
3349 
3350 	if (!adapter->tx_rings)
3351 		return;
3352 
3353 	for (i = 0; i < adapter->num_active_queues; i++)
3354 		if (adapter->tx_rings[i].desc)
3355 			iavf_free_tx_resources(&adapter->tx_rings[i]);
3356 }
3357 
3358 /**
3359  * iavf_setup_all_tx_resources - allocate all queues Tx resources
3360  * @adapter: board private structure
3361  *
3362  * If this function returns with an error, then it's possible one or
3363  * more of the rings is populated (while the rest are not).  It is the
3364  * callers duty to clean those orphaned rings.
3365  *
3366  * Return 0 on success, negative on failure
3367  **/
3368 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
3369 {
3370 	int i, err = 0;
3371 
3372 	for (i = 0; i < adapter->num_active_queues; i++) {
3373 		adapter->tx_rings[i].count = adapter->tx_desc_count;
3374 		err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
3375 		if (!err)
3376 			continue;
3377 		dev_err(&adapter->pdev->dev,
3378 			"Allocation for Tx Queue %u failed\n", i);
3379 		break;
3380 	}
3381 
3382 	return err;
3383 }
3384 
3385 /**
3386  * iavf_setup_all_rx_resources - allocate all queues Rx resources
3387  * @adapter: board private structure
3388  *
3389  * If this function returns with an error, then it's possible one or
3390  * more of the rings is populated (while the rest are not).  It is the
3391  * callers duty to clean those orphaned rings.
3392  *
3393  * Return 0 on success, negative on failure
3394  **/
3395 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
3396 {
3397 	int i, err = 0;
3398 
3399 	for (i = 0; i < adapter->num_active_queues; i++) {
3400 		adapter->rx_rings[i].count = adapter->rx_desc_count;
3401 		err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
3402 		if (!err)
3403 			continue;
3404 		dev_err(&adapter->pdev->dev,
3405 			"Allocation for Rx Queue %u failed\n", i);
3406 		break;
3407 	}
3408 	return err;
3409 }
3410 
3411 /**
3412  * iavf_free_all_rx_resources - Free Rx Resources for All Queues
3413  * @adapter: board private structure
3414  *
3415  * Free all receive software resources
3416  **/
3417 void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
3418 {
3419 	int i;
3420 
3421 	if (!adapter->rx_rings)
3422 		return;
3423 
3424 	for (i = 0; i < adapter->num_active_queues; i++)
3425 		if (adapter->rx_rings[i].desc)
3426 			iavf_free_rx_resources(&adapter->rx_rings[i]);
3427 }
3428 
3429 /**
3430  * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
3431  * @adapter: board private structure
3432  * @max_tx_rate: max Tx bw for a tc
3433  **/
3434 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
3435 				      u64 max_tx_rate)
3436 {
3437 	int speed = 0, ret = 0;
3438 
3439 	if (ADV_LINK_SUPPORT(adapter)) {
3440 		if (adapter->link_speed_mbps < U32_MAX) {
3441 			speed = adapter->link_speed_mbps;
3442 			goto validate_bw;
3443 		} else {
3444 			dev_err(&adapter->pdev->dev, "Unknown link speed\n");
3445 			return -EINVAL;
3446 		}
3447 	}
3448 
3449 	switch (adapter->link_speed) {
3450 	case VIRTCHNL_LINK_SPEED_40GB:
3451 		speed = SPEED_40000;
3452 		break;
3453 	case VIRTCHNL_LINK_SPEED_25GB:
3454 		speed = SPEED_25000;
3455 		break;
3456 	case VIRTCHNL_LINK_SPEED_20GB:
3457 		speed = SPEED_20000;
3458 		break;
3459 	case VIRTCHNL_LINK_SPEED_10GB:
3460 		speed = SPEED_10000;
3461 		break;
3462 	case VIRTCHNL_LINK_SPEED_5GB:
3463 		speed = SPEED_5000;
3464 		break;
3465 	case VIRTCHNL_LINK_SPEED_2_5GB:
3466 		speed = SPEED_2500;
3467 		break;
3468 	case VIRTCHNL_LINK_SPEED_1GB:
3469 		speed = SPEED_1000;
3470 		break;
3471 	case VIRTCHNL_LINK_SPEED_100MB:
3472 		speed = SPEED_100;
3473 		break;
3474 	default:
3475 		break;
3476 	}
3477 
3478 validate_bw:
3479 	if (max_tx_rate > speed) {
3480 		dev_err(&adapter->pdev->dev,
3481 			"Invalid tx rate specified\n");
3482 		ret = -EINVAL;
3483 	}
3484 
3485 	return ret;
3486 }
3487 
3488 /**
3489  * iavf_validate_ch_config - validate queue mapping info
3490  * @adapter: board private structure
3491  * @mqprio_qopt: queue parameters
3492  *
3493  * This function validates if the config provided by the user to
3494  * configure queue channels is valid or not. Returns 0 on a valid
3495  * config.
3496  **/
3497 static int iavf_validate_ch_config(struct iavf_adapter *adapter,
3498 				   struct tc_mqprio_qopt_offload *mqprio_qopt)
3499 {
3500 	u64 total_max_rate = 0;
3501 	u32 tx_rate_rem = 0;
3502 	int i, num_qps = 0;
3503 	u64 tx_rate = 0;
3504 	int ret = 0;
3505 
3506 	if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
3507 	    mqprio_qopt->qopt.num_tc < 1)
3508 		return -EINVAL;
3509 
3510 	for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
3511 		if (!mqprio_qopt->qopt.count[i] ||
3512 		    mqprio_qopt->qopt.offset[i] != num_qps)
3513 			return -EINVAL;
3514 		if (mqprio_qopt->min_rate[i]) {
3515 			dev_err(&adapter->pdev->dev,
3516 				"Invalid min tx rate (greater than 0) specified for TC%d\n",
3517 				i);
3518 			return -EINVAL;
3519 		}
3520 
3521 		/* convert to Mbps */
3522 		tx_rate = div_u64(mqprio_qopt->max_rate[i],
3523 				  IAVF_MBPS_DIVISOR);
3524 
3525 		if (mqprio_qopt->max_rate[i] &&
3526 		    tx_rate < IAVF_MBPS_QUANTA) {
3527 			dev_err(&adapter->pdev->dev,
3528 				"Invalid max tx rate for TC%d, minimum %dMbps\n",
3529 				i, IAVF_MBPS_QUANTA);
3530 			return -EINVAL;
3531 		}
3532 
3533 		(void)div_u64_rem(tx_rate, IAVF_MBPS_QUANTA, &tx_rate_rem);
3534 
3535 		if (tx_rate_rem != 0) {
3536 			dev_err(&adapter->pdev->dev,
3537 				"Invalid max tx rate for TC%d, not divisible by %d\n",
3538 				i, IAVF_MBPS_QUANTA);
3539 			return -EINVAL;
3540 		}
3541 
3542 		total_max_rate += tx_rate;
3543 		num_qps += mqprio_qopt->qopt.count[i];
3544 	}
3545 	if (num_qps > adapter->num_active_queues) {
3546 		dev_err(&adapter->pdev->dev,
3547 			"Cannot support requested number of queues\n");
3548 		return -EINVAL;
3549 	}
3550 
3551 	ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
3552 	return ret;
3553 }
3554 
3555 /**
3556  * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
3557  * @adapter: board private structure
3558  **/
3559 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
3560 {
3561 	struct iavf_cloud_filter *cf, *cftmp;
3562 
3563 	spin_lock_bh(&adapter->cloud_filter_list_lock);
3564 	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
3565 				 list) {
3566 		list_del(&cf->list);
3567 		kfree(cf);
3568 		adapter->num_cloud_filters--;
3569 	}
3570 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
3571 }
3572 
3573 /**
3574  * __iavf_setup_tc - configure multiple traffic classes
3575  * @netdev: network interface device structure
3576  * @type_data: tc offload data
3577  *
3578  * This function processes the config information provided by the
3579  * user to configure traffic classes/queue channels and packages the
3580  * information to request the PF to setup traffic classes.
3581  *
3582  * Returns 0 on success.
3583  **/
3584 static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
3585 {
3586 	struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
3587 	struct iavf_adapter *adapter = netdev_priv(netdev);
3588 	struct virtchnl_vf_resource *vfres = adapter->vf_res;
3589 	u8 num_tc = 0, total_qps = 0;
3590 	int ret = 0, netdev_tc = 0;
3591 	u64 max_tx_rate;
3592 	u16 mode;
3593 	int i;
3594 
3595 	num_tc = mqprio_qopt->qopt.num_tc;
3596 	mode = mqprio_qopt->mode;
3597 
3598 	/* delete queue_channel */
3599 	if (!mqprio_qopt->qopt.hw) {
3600 		if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
3601 			/* reset the tc configuration */
3602 			netdev_reset_tc(netdev);
3603 			adapter->num_tc = 0;
3604 			netif_tx_stop_all_queues(netdev);
3605 			netif_tx_disable(netdev);
3606 			iavf_del_all_cloud_filters(adapter);
3607 			adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
3608 			total_qps = adapter->orig_num_active_queues;
3609 			goto exit;
3610 		} else {
3611 			return -EINVAL;
3612 		}
3613 	}
3614 
3615 	/* add queue channel */
3616 	if (mode == TC_MQPRIO_MODE_CHANNEL) {
3617 		if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
3618 			dev_err(&adapter->pdev->dev, "ADq not supported\n");
3619 			return -EOPNOTSUPP;
3620 		}
3621 		if (adapter->ch_config.state != __IAVF_TC_INVALID) {
3622 			dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
3623 			return -EINVAL;
3624 		}
3625 
3626 		ret = iavf_validate_ch_config(adapter, mqprio_qopt);
3627 		if (ret)
3628 			return ret;
3629 		/* Return if same TC config is requested */
3630 		if (adapter->num_tc == num_tc)
3631 			return 0;
3632 		adapter->num_tc = num_tc;
3633 
3634 		for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3635 			if (i < num_tc) {
3636 				adapter->ch_config.ch_info[i].count =
3637 					mqprio_qopt->qopt.count[i];
3638 				adapter->ch_config.ch_info[i].offset =
3639 					mqprio_qopt->qopt.offset[i];
3640 				total_qps += mqprio_qopt->qopt.count[i];
3641 				max_tx_rate = mqprio_qopt->max_rate[i];
3642 				/* convert to Mbps */
3643 				max_tx_rate = div_u64(max_tx_rate,
3644 						      IAVF_MBPS_DIVISOR);
3645 				adapter->ch_config.ch_info[i].max_tx_rate =
3646 					max_tx_rate;
3647 			} else {
3648 				adapter->ch_config.ch_info[i].count = 1;
3649 				adapter->ch_config.ch_info[i].offset = 0;
3650 			}
3651 		}
3652 
3653 		/* Take snapshot of original config such as "num_active_queues"
3654 		 * It is used later when delete ADQ flow is exercised, so that
3655 		 * once delete ADQ flow completes, VF shall go back to its
3656 		 * original queue configuration
3657 		 */
3658 
3659 		adapter->orig_num_active_queues = adapter->num_active_queues;
3660 
3661 		/* Store queue info based on TC so that VF gets configured
3662 		 * with correct number of queues when VF completes ADQ config
3663 		 * flow
3664 		 */
3665 		adapter->ch_config.total_qps = total_qps;
3666 
3667 		netif_tx_stop_all_queues(netdev);
3668 		netif_tx_disable(netdev);
3669 		adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
3670 		netdev_reset_tc(netdev);
3671 		/* Report the tc mapping up the stack */
3672 		netdev_set_num_tc(adapter->netdev, num_tc);
3673 		for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3674 			u16 qcount = mqprio_qopt->qopt.count[i];
3675 			u16 qoffset = mqprio_qopt->qopt.offset[i];
3676 
3677 			if (i < num_tc)
3678 				netdev_set_tc_queue(netdev, netdev_tc++, qcount,
3679 						    qoffset);
3680 		}
3681 	}
3682 exit:
3683 	if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
3684 		return 0;
3685 
3686 	netif_set_real_num_rx_queues(netdev, total_qps);
3687 	netif_set_real_num_tx_queues(netdev, total_qps);
3688 
3689 	return ret;
3690 }
3691 
3692 /**
3693  * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
3694  * @adapter: board private structure
3695  * @f: pointer to struct flow_cls_offload
3696  * @filter: pointer to cloud filter structure
3697  */
3698 static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
3699 				 struct flow_cls_offload *f,
3700 				 struct iavf_cloud_filter *filter)
3701 {
3702 	struct flow_rule *rule = flow_cls_offload_flow_rule(f);
3703 	struct flow_dissector *dissector = rule->match.dissector;
3704 	u16 n_proto_mask = 0;
3705 	u16 n_proto_key = 0;
3706 	u8 field_flags = 0;
3707 	u16 addr_type = 0;
3708 	u16 n_proto = 0;
3709 	int i = 0;
3710 	struct virtchnl_filter *vf = &filter->f;
3711 
3712 	if (dissector->used_keys &
3713 	    ~(BIT(FLOW_DISSECTOR_KEY_CONTROL) |
3714 	      BIT(FLOW_DISSECTOR_KEY_BASIC) |
3715 	      BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
3716 	      BIT(FLOW_DISSECTOR_KEY_VLAN) |
3717 	      BIT(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
3718 	      BIT(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
3719 	      BIT(FLOW_DISSECTOR_KEY_PORTS) |
3720 	      BIT(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
3721 		dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%x\n",
3722 			dissector->used_keys);
3723 		return -EOPNOTSUPP;
3724 	}
3725 
3726 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
3727 		struct flow_match_enc_keyid match;
3728 
3729 		flow_rule_match_enc_keyid(rule, &match);
3730 		if (match.mask->keyid != 0)
3731 			field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
3732 	}
3733 
3734 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
3735 		struct flow_match_basic match;
3736 
3737 		flow_rule_match_basic(rule, &match);
3738 		n_proto_key = ntohs(match.key->n_proto);
3739 		n_proto_mask = ntohs(match.mask->n_proto);
3740 
3741 		if (n_proto_key == ETH_P_ALL) {
3742 			n_proto_key = 0;
3743 			n_proto_mask = 0;
3744 		}
3745 		n_proto = n_proto_key & n_proto_mask;
3746 		if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
3747 			return -EINVAL;
3748 		if (n_proto == ETH_P_IPV6) {
3749 			/* specify flow type as TCP IPv6 */
3750 			vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
3751 		}
3752 
3753 		if (match.key->ip_proto != IPPROTO_TCP) {
3754 			dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
3755 			return -EINVAL;
3756 		}
3757 	}
3758 
3759 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
3760 		struct flow_match_eth_addrs match;
3761 
3762 		flow_rule_match_eth_addrs(rule, &match);
3763 
3764 		/* use is_broadcast and is_zero to check for all 0xf or 0 */
3765 		if (!is_zero_ether_addr(match.mask->dst)) {
3766 			if (is_broadcast_ether_addr(match.mask->dst)) {
3767 				field_flags |= IAVF_CLOUD_FIELD_OMAC;
3768 			} else {
3769 				dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
3770 					match.mask->dst);
3771 				return -EINVAL;
3772 			}
3773 		}
3774 
3775 		if (!is_zero_ether_addr(match.mask->src)) {
3776 			if (is_broadcast_ether_addr(match.mask->src)) {
3777 				field_flags |= IAVF_CLOUD_FIELD_IMAC;
3778 			} else {
3779 				dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
3780 					match.mask->src);
3781 				return -EINVAL;
3782 			}
3783 		}
3784 
3785 		if (!is_zero_ether_addr(match.key->dst))
3786 			if (is_valid_ether_addr(match.key->dst) ||
3787 			    is_multicast_ether_addr(match.key->dst)) {
3788 				/* set the mask if a valid dst_mac address */
3789 				for (i = 0; i < ETH_ALEN; i++)
3790 					vf->mask.tcp_spec.dst_mac[i] |= 0xff;
3791 				ether_addr_copy(vf->data.tcp_spec.dst_mac,
3792 						match.key->dst);
3793 			}
3794 
3795 		if (!is_zero_ether_addr(match.key->src))
3796 			if (is_valid_ether_addr(match.key->src) ||
3797 			    is_multicast_ether_addr(match.key->src)) {
3798 				/* set the mask if a valid dst_mac address */
3799 				for (i = 0; i < ETH_ALEN; i++)
3800 					vf->mask.tcp_spec.src_mac[i] |= 0xff;
3801 				ether_addr_copy(vf->data.tcp_spec.src_mac,
3802 						match.key->src);
3803 		}
3804 	}
3805 
3806 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
3807 		struct flow_match_vlan match;
3808 
3809 		flow_rule_match_vlan(rule, &match);
3810 		if (match.mask->vlan_id) {
3811 			if (match.mask->vlan_id == VLAN_VID_MASK) {
3812 				field_flags |= IAVF_CLOUD_FIELD_IVLAN;
3813 			} else {
3814 				dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
3815 					match.mask->vlan_id);
3816 				return -EINVAL;
3817 			}
3818 		}
3819 		vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
3820 		vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
3821 	}
3822 
3823 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
3824 		struct flow_match_control match;
3825 
3826 		flow_rule_match_control(rule, &match);
3827 		addr_type = match.key->addr_type;
3828 	}
3829 
3830 	if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
3831 		struct flow_match_ipv4_addrs match;
3832 
3833 		flow_rule_match_ipv4_addrs(rule, &match);
3834 		if (match.mask->dst) {
3835 			if (match.mask->dst == cpu_to_be32(0xffffffff)) {
3836 				field_flags |= IAVF_CLOUD_FIELD_IIP;
3837 			} else {
3838 				dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
3839 					be32_to_cpu(match.mask->dst));
3840 				return -EINVAL;
3841 			}
3842 		}
3843 
3844 		if (match.mask->src) {
3845 			if (match.mask->src == cpu_to_be32(0xffffffff)) {
3846 				field_flags |= IAVF_CLOUD_FIELD_IIP;
3847 			} else {
3848 				dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
3849 					be32_to_cpu(match.mask->dst));
3850 				return -EINVAL;
3851 			}
3852 		}
3853 
3854 		if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
3855 			dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
3856 			return -EINVAL;
3857 		}
3858 		if (match.key->dst) {
3859 			vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
3860 			vf->data.tcp_spec.dst_ip[0] = match.key->dst;
3861 		}
3862 		if (match.key->src) {
3863 			vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
3864 			vf->data.tcp_spec.src_ip[0] = match.key->src;
3865 		}
3866 	}
3867 
3868 	if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
3869 		struct flow_match_ipv6_addrs match;
3870 
3871 		flow_rule_match_ipv6_addrs(rule, &match);
3872 
3873 		/* validate mask, make sure it is not IPV6_ADDR_ANY */
3874 		if (ipv6_addr_any(&match.mask->dst)) {
3875 			dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
3876 				IPV6_ADDR_ANY);
3877 			return -EINVAL;
3878 		}
3879 
3880 		/* src and dest IPv6 address should not be LOOPBACK
3881 		 * (0:0:0:0:0:0:0:1) which can be represented as ::1
3882 		 */
3883 		if (ipv6_addr_loopback(&match.key->dst) ||
3884 		    ipv6_addr_loopback(&match.key->src)) {
3885 			dev_err(&adapter->pdev->dev,
3886 				"ipv6 addr should not be loopback\n");
3887 			return -EINVAL;
3888 		}
3889 		if (!ipv6_addr_any(&match.mask->dst) ||
3890 		    !ipv6_addr_any(&match.mask->src))
3891 			field_flags |= IAVF_CLOUD_FIELD_IIP;
3892 
3893 		for (i = 0; i < 4; i++)
3894 			vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
3895 		memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
3896 		       sizeof(vf->data.tcp_spec.dst_ip));
3897 		for (i = 0; i < 4; i++)
3898 			vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
3899 		memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
3900 		       sizeof(vf->data.tcp_spec.src_ip));
3901 	}
3902 	if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
3903 		struct flow_match_ports match;
3904 
3905 		flow_rule_match_ports(rule, &match);
3906 		if (match.mask->src) {
3907 			if (match.mask->src == cpu_to_be16(0xffff)) {
3908 				field_flags |= IAVF_CLOUD_FIELD_IIP;
3909 			} else {
3910 				dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
3911 					be16_to_cpu(match.mask->src));
3912 				return -EINVAL;
3913 			}
3914 		}
3915 
3916 		if (match.mask->dst) {
3917 			if (match.mask->dst == cpu_to_be16(0xffff)) {
3918 				field_flags |= IAVF_CLOUD_FIELD_IIP;
3919 			} else {
3920 				dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
3921 					be16_to_cpu(match.mask->dst));
3922 				return -EINVAL;
3923 			}
3924 		}
3925 		if (match.key->dst) {
3926 			vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
3927 			vf->data.tcp_spec.dst_port = match.key->dst;
3928 		}
3929 
3930 		if (match.key->src) {
3931 			vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
3932 			vf->data.tcp_spec.src_port = match.key->src;
3933 		}
3934 	}
3935 	vf->field_flags = field_flags;
3936 
3937 	return 0;
3938 }
3939 
3940 /**
3941  * iavf_handle_tclass - Forward to a traffic class on the device
3942  * @adapter: board private structure
3943  * @tc: traffic class index on the device
3944  * @filter: pointer to cloud filter structure
3945  */
3946 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
3947 			      struct iavf_cloud_filter *filter)
3948 {
3949 	if (tc == 0)
3950 		return 0;
3951 	if (tc < adapter->num_tc) {
3952 		if (!filter->f.data.tcp_spec.dst_port) {
3953 			dev_err(&adapter->pdev->dev,
3954 				"Specify destination port to redirect to traffic class other than TC0\n");
3955 			return -EINVAL;
3956 		}
3957 	}
3958 	/* redirect to a traffic class on the same device */
3959 	filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
3960 	filter->f.action_meta = tc;
3961 	return 0;
3962 }
3963 
3964 /**
3965  * iavf_find_cf - Find the cloud filter in the list
3966  * @adapter: Board private structure
3967  * @cookie: filter specific cookie
3968  *
3969  * Returns ptr to the filter object or NULL. Must be called while holding the
3970  * cloud_filter_list_lock.
3971  */
3972 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
3973 					      unsigned long *cookie)
3974 {
3975 	struct iavf_cloud_filter *filter = NULL;
3976 
3977 	if (!cookie)
3978 		return NULL;
3979 
3980 	list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
3981 		if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
3982 			return filter;
3983 	}
3984 	return NULL;
3985 }
3986 
3987 /**
3988  * iavf_configure_clsflower - Add tc flower filters
3989  * @adapter: board private structure
3990  * @cls_flower: Pointer to struct flow_cls_offload
3991  */
3992 static int iavf_configure_clsflower(struct iavf_adapter *adapter,
3993 				    struct flow_cls_offload *cls_flower)
3994 {
3995 	int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
3996 	struct iavf_cloud_filter *filter = NULL;
3997 	int err = -EINVAL, count = 50;
3998 
3999 	if (tc < 0) {
4000 		dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
4001 		return -EINVAL;
4002 	}
4003 
4004 	filter = kzalloc(sizeof(*filter), GFP_KERNEL);
4005 	if (!filter)
4006 		return -ENOMEM;
4007 
4008 	while (!mutex_trylock(&adapter->crit_lock)) {
4009 		if (--count == 0) {
4010 			kfree(filter);
4011 			return err;
4012 		}
4013 		udelay(1);
4014 	}
4015 
4016 	filter->cookie = cls_flower->cookie;
4017 
4018 	/* bail out here if filter already exists */
4019 	spin_lock_bh(&adapter->cloud_filter_list_lock);
4020 	if (iavf_find_cf(adapter, &cls_flower->cookie)) {
4021 		dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n");
4022 		err = -EEXIST;
4023 		goto spin_unlock;
4024 	}
4025 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
4026 
4027 	/* set the mask to all zeroes to begin with */
4028 	memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
4029 	/* start out with flow type and eth type IPv4 to begin with */
4030 	filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
4031 	err = iavf_parse_cls_flower(adapter, cls_flower, filter);
4032 	if (err)
4033 		goto err;
4034 
4035 	err = iavf_handle_tclass(adapter, tc, filter);
4036 	if (err)
4037 		goto err;
4038 
4039 	/* add filter to the list */
4040 	spin_lock_bh(&adapter->cloud_filter_list_lock);
4041 	list_add_tail(&filter->list, &adapter->cloud_filter_list);
4042 	adapter->num_cloud_filters++;
4043 	filter->add = true;
4044 	adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
4045 spin_unlock:
4046 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
4047 err:
4048 	if (err)
4049 		kfree(filter);
4050 
4051 	mutex_unlock(&adapter->crit_lock);
4052 	return err;
4053 }
4054 
4055 /**
4056  * iavf_delete_clsflower - Remove tc flower filters
4057  * @adapter: board private structure
4058  * @cls_flower: Pointer to struct flow_cls_offload
4059  */
4060 static int iavf_delete_clsflower(struct iavf_adapter *adapter,
4061 				 struct flow_cls_offload *cls_flower)
4062 {
4063 	struct iavf_cloud_filter *filter = NULL;
4064 	int err = 0;
4065 
4066 	spin_lock_bh(&adapter->cloud_filter_list_lock);
4067 	filter = iavf_find_cf(adapter, &cls_flower->cookie);
4068 	if (filter) {
4069 		filter->del = true;
4070 		adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
4071 	} else {
4072 		err = -EINVAL;
4073 	}
4074 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
4075 
4076 	return err;
4077 }
4078 
4079 /**
4080  * iavf_setup_tc_cls_flower - flower classifier offloads
4081  * @adapter: board private structure
4082  * @cls_flower: pointer to flow_cls_offload struct with flow info
4083  */
4084 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
4085 				    struct flow_cls_offload *cls_flower)
4086 {
4087 	switch (cls_flower->command) {
4088 	case FLOW_CLS_REPLACE:
4089 		return iavf_configure_clsflower(adapter, cls_flower);
4090 	case FLOW_CLS_DESTROY:
4091 		return iavf_delete_clsflower(adapter, cls_flower);
4092 	case FLOW_CLS_STATS:
4093 		return -EOPNOTSUPP;
4094 	default:
4095 		return -EOPNOTSUPP;
4096 	}
4097 }
4098 
4099 /**
4100  * iavf_setup_tc_block_cb - block callback for tc
4101  * @type: type of offload
4102  * @type_data: offload data
4103  * @cb_priv:
4104  *
4105  * This function is the block callback for traffic classes
4106  **/
4107 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4108 				  void *cb_priv)
4109 {
4110 	struct iavf_adapter *adapter = cb_priv;
4111 
4112 	if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
4113 		return -EOPNOTSUPP;
4114 
4115 	switch (type) {
4116 	case TC_SETUP_CLSFLOWER:
4117 		return iavf_setup_tc_cls_flower(cb_priv, type_data);
4118 	default:
4119 		return -EOPNOTSUPP;
4120 	}
4121 }
4122 
4123 static LIST_HEAD(iavf_block_cb_list);
4124 
4125 /**
4126  * iavf_setup_tc - configure multiple traffic classes
4127  * @netdev: network interface device structure
4128  * @type: type of offload
4129  * @type_data: tc offload data
4130  *
4131  * This function is the callback to ndo_setup_tc in the
4132  * netdev_ops.
4133  *
4134  * Returns 0 on success
4135  **/
4136 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
4137 			 void *type_data)
4138 {
4139 	struct iavf_adapter *adapter = netdev_priv(netdev);
4140 
4141 	switch (type) {
4142 	case TC_SETUP_QDISC_MQPRIO:
4143 		return __iavf_setup_tc(netdev, type_data);
4144 	case TC_SETUP_BLOCK:
4145 		return flow_block_cb_setup_simple(type_data,
4146 						  &iavf_block_cb_list,
4147 						  iavf_setup_tc_block_cb,
4148 						  adapter, adapter, true);
4149 	default:
4150 		return -EOPNOTSUPP;
4151 	}
4152 }
4153 
4154 /**
4155  * iavf_open - Called when a network interface is made active
4156  * @netdev: network interface device structure
4157  *
4158  * Returns 0 on success, negative value on failure
4159  *
4160  * The open entry point is called when a network interface is made
4161  * active by the system (IFF_UP).  At this point all resources needed
4162  * for transmit and receive operations are allocated, the interrupt
4163  * handler is registered with the OS, the watchdog is started,
4164  * and the stack is notified that the interface is ready.
4165  **/
4166 static int iavf_open(struct net_device *netdev)
4167 {
4168 	struct iavf_adapter *adapter = netdev_priv(netdev);
4169 	int err;
4170 
4171 	if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
4172 		dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
4173 		return -EIO;
4174 	}
4175 
4176 	while (!mutex_trylock(&adapter->crit_lock)) {
4177 		/* If we are in __IAVF_INIT_CONFIG_ADAPTER state the crit_lock
4178 		 * is already taken and iavf_open is called from an upper
4179 		 * device's notifier reacting on NETDEV_REGISTER event.
4180 		 * We have to leave here to avoid dead lock.
4181 		 */
4182 		if (adapter->state == __IAVF_INIT_CONFIG_ADAPTER)
4183 			return -EBUSY;
4184 
4185 		usleep_range(500, 1000);
4186 	}
4187 
4188 	if (adapter->state != __IAVF_DOWN) {
4189 		err = -EBUSY;
4190 		goto err_unlock;
4191 	}
4192 
4193 	if (adapter->state == __IAVF_RUNNING &&
4194 	    !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {
4195 		dev_dbg(&adapter->pdev->dev, "VF is already open.\n");
4196 		err = 0;
4197 		goto err_unlock;
4198 	}
4199 
4200 	/* allocate transmit descriptors */
4201 	err = iavf_setup_all_tx_resources(adapter);
4202 	if (err)
4203 		goto err_setup_tx;
4204 
4205 	/* allocate receive descriptors */
4206 	err = iavf_setup_all_rx_resources(adapter);
4207 	if (err)
4208 		goto err_setup_rx;
4209 
4210 	/* clear any pending interrupts, may auto mask */
4211 	err = iavf_request_traffic_irqs(adapter, netdev->name);
4212 	if (err)
4213 		goto err_req_irq;
4214 
4215 	spin_lock_bh(&adapter->mac_vlan_list_lock);
4216 
4217 	iavf_add_filter(adapter, adapter->hw.mac.addr);
4218 
4219 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
4220 
4221 	/* Restore VLAN filters that were removed with IFF_DOWN */
4222 	iavf_restore_filters(adapter);
4223 
4224 	iavf_configure(adapter);
4225 
4226 	iavf_up_complete(adapter);
4227 
4228 	iavf_irq_enable(adapter, true);
4229 
4230 	mutex_unlock(&adapter->crit_lock);
4231 
4232 	return 0;
4233 
4234 err_req_irq:
4235 	iavf_down(adapter);
4236 	iavf_free_traffic_irqs(adapter);
4237 err_setup_rx:
4238 	iavf_free_all_rx_resources(adapter);
4239 err_setup_tx:
4240 	iavf_free_all_tx_resources(adapter);
4241 err_unlock:
4242 	mutex_unlock(&adapter->crit_lock);
4243 
4244 	return err;
4245 }
4246 
4247 /**
4248  * iavf_close - Disables a network interface
4249  * @netdev: network interface device structure
4250  *
4251  * Returns 0, this is not allowed to fail
4252  *
4253  * The close entry point is called when an interface is de-activated
4254  * by the OS.  The hardware is still under the drivers control, but
4255  * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
4256  * are freed, along with all transmit and receive resources.
4257  **/
4258 static int iavf_close(struct net_device *netdev)
4259 {
4260 	struct iavf_adapter *adapter = netdev_priv(netdev);
4261 	u64 aq_to_restore;
4262 	int status;
4263 
4264 	mutex_lock(&adapter->crit_lock);
4265 
4266 	if (adapter->state <= __IAVF_DOWN_PENDING) {
4267 		mutex_unlock(&adapter->crit_lock);
4268 		return 0;
4269 	}
4270 
4271 	set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
4272 	if (CLIENT_ENABLED(adapter))
4273 		adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
4274 	/* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before
4275 	 * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl
4276 	 * deadlock with adminq_task() until iavf_close timeouts. We must send
4277 	 * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make
4278 	 * disable queues possible for vf. Give only necessary flags to
4279 	 * iavf_down and save other to set them right before iavf_close()
4280 	 * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and
4281 	 * iavf will be in DOWN state.
4282 	 */
4283 	aq_to_restore = adapter->aq_required;
4284 	adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG;
4285 
4286 	/* Remove flags which we do not want to send after close or we want to
4287 	 * send before disable queues.
4288 	 */
4289 	aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG		|
4290 			   IAVF_FLAG_AQ_ENABLE_QUEUES		|
4291 			   IAVF_FLAG_AQ_CONFIGURE_QUEUES	|
4292 			   IAVF_FLAG_AQ_ADD_VLAN_FILTER		|
4293 			   IAVF_FLAG_AQ_ADD_MAC_FILTER		|
4294 			   IAVF_FLAG_AQ_ADD_CLOUD_FILTER	|
4295 			   IAVF_FLAG_AQ_ADD_FDIR_FILTER		|
4296 			   IAVF_FLAG_AQ_ADD_ADV_RSS_CFG);
4297 
4298 	iavf_down(adapter);
4299 	iavf_change_state(adapter, __IAVF_DOWN_PENDING);
4300 	iavf_free_traffic_irqs(adapter);
4301 
4302 	mutex_unlock(&adapter->crit_lock);
4303 
4304 	/* We explicitly don't free resources here because the hardware is
4305 	 * still active and can DMA into memory. Resources are cleared in
4306 	 * iavf_virtchnl_completion() after we get confirmation from the PF
4307 	 * driver that the rings have been stopped.
4308 	 *
4309 	 * Also, we wait for state to transition to __IAVF_DOWN before
4310 	 * returning. State change occurs in iavf_virtchnl_completion() after
4311 	 * VF resources are released (which occurs after PF driver processes and
4312 	 * responds to admin queue commands).
4313 	 */
4314 
4315 	status = wait_event_timeout(adapter->down_waitqueue,
4316 				    adapter->state == __IAVF_DOWN,
4317 				    msecs_to_jiffies(500));
4318 	if (!status)
4319 		netdev_warn(netdev, "Device resources not yet released\n");
4320 
4321 	mutex_lock(&adapter->crit_lock);
4322 	adapter->aq_required |= aq_to_restore;
4323 	mutex_unlock(&adapter->crit_lock);
4324 	return 0;
4325 }
4326 
4327 /**
4328  * iavf_change_mtu - Change the Maximum Transfer Unit
4329  * @netdev: network interface device structure
4330  * @new_mtu: new value for maximum frame size
4331  *
4332  * Returns 0 on success, negative on failure
4333  **/
4334 static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
4335 {
4336 	struct iavf_adapter *adapter = netdev_priv(netdev);
4337 
4338 	netdev_dbg(netdev, "changing MTU from %d to %d\n",
4339 		   netdev->mtu, new_mtu);
4340 	netdev->mtu = new_mtu;
4341 	if (CLIENT_ENABLED(adapter)) {
4342 		iavf_notify_client_l2_params(&adapter->vsi);
4343 		adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
4344 	}
4345 
4346 	if (netif_running(netdev)) {
4347 		adapter->flags |= IAVF_FLAG_RESET_NEEDED;
4348 		queue_work(iavf_wq, &adapter->reset_task);
4349 	}
4350 
4351 	return 0;
4352 }
4353 
4354 #define NETIF_VLAN_OFFLOAD_FEATURES	(NETIF_F_HW_VLAN_CTAG_RX | \
4355 					 NETIF_F_HW_VLAN_CTAG_TX | \
4356 					 NETIF_F_HW_VLAN_STAG_RX | \
4357 					 NETIF_F_HW_VLAN_STAG_TX)
4358 
4359 /**
4360  * iavf_set_features - set the netdev feature flags
4361  * @netdev: ptr to the netdev being adjusted
4362  * @features: the feature set that the stack is suggesting
4363  * Note: expects to be called while under rtnl_lock()
4364  **/
4365 static int iavf_set_features(struct net_device *netdev,
4366 			     netdev_features_t features)
4367 {
4368 	struct iavf_adapter *adapter = netdev_priv(netdev);
4369 
4370 	/* trigger update on any VLAN feature change */
4371 	if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^
4372 	    (features & NETIF_VLAN_OFFLOAD_FEATURES))
4373 		iavf_set_vlan_offload_features(adapter, netdev->features,
4374 					       features);
4375 
4376 	return 0;
4377 }
4378 
4379 /**
4380  * iavf_features_check - Validate encapsulated packet conforms to limits
4381  * @skb: skb buff
4382  * @dev: This physical port's netdev
4383  * @features: Offload features that the stack believes apply
4384  **/
4385 static netdev_features_t iavf_features_check(struct sk_buff *skb,
4386 					     struct net_device *dev,
4387 					     netdev_features_t features)
4388 {
4389 	size_t len;
4390 
4391 	/* No point in doing any of this if neither checksum nor GSO are
4392 	 * being requested for this frame.  We can rule out both by just
4393 	 * checking for CHECKSUM_PARTIAL
4394 	 */
4395 	if (skb->ip_summed != CHECKSUM_PARTIAL)
4396 		return features;
4397 
4398 	/* We cannot support GSO if the MSS is going to be less than
4399 	 * 64 bytes.  If it is then we need to drop support for GSO.
4400 	 */
4401 	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
4402 		features &= ~NETIF_F_GSO_MASK;
4403 
4404 	/* MACLEN can support at most 63 words */
4405 	len = skb_network_header(skb) - skb->data;
4406 	if (len & ~(63 * 2))
4407 		goto out_err;
4408 
4409 	/* IPLEN and EIPLEN can support at most 127 dwords */
4410 	len = skb_transport_header(skb) - skb_network_header(skb);
4411 	if (len & ~(127 * 4))
4412 		goto out_err;
4413 
4414 	if (skb->encapsulation) {
4415 		/* L4TUNLEN can support 127 words */
4416 		len = skb_inner_network_header(skb) - skb_transport_header(skb);
4417 		if (len & ~(127 * 2))
4418 			goto out_err;
4419 
4420 		/* IPLEN can support at most 127 dwords */
4421 		len = skb_inner_transport_header(skb) -
4422 		      skb_inner_network_header(skb);
4423 		if (len & ~(127 * 4))
4424 			goto out_err;
4425 	}
4426 
4427 	/* No need to validate L4LEN as TCP is the only protocol with a
4428 	 * flexible value and we support all possible values supported
4429 	 * by TCP, which is at most 15 dwords
4430 	 */
4431 
4432 	return features;
4433 out_err:
4434 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4435 }
4436 
4437 /**
4438  * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off
4439  * @adapter: board private structure
4440  *
4441  * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4442  * were negotiated determine the VLAN features that can be toggled on and off.
4443  **/
4444 static netdev_features_t
4445 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter)
4446 {
4447 	netdev_features_t hw_features = 0;
4448 
4449 	if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4450 		return hw_features;
4451 
4452 	/* Enable VLAN features if supported */
4453 	if (VLAN_ALLOWED(adapter)) {
4454 		hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
4455 				NETIF_F_HW_VLAN_CTAG_RX);
4456 	} else if (VLAN_V2_ALLOWED(adapter)) {
4457 		struct virtchnl_vlan_caps *vlan_v2_caps =
4458 			&adapter->vlan_v2_caps;
4459 		struct virtchnl_vlan_supported_caps *stripping_support =
4460 			&vlan_v2_caps->offloads.stripping_support;
4461 		struct virtchnl_vlan_supported_caps *insertion_support =
4462 			&vlan_v2_caps->offloads.insertion_support;
4463 
4464 		if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4465 		    stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4466 			if (stripping_support->outer &
4467 			    VIRTCHNL_VLAN_ETHERTYPE_8100)
4468 				hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4469 			if (stripping_support->outer &
4470 			    VIRTCHNL_VLAN_ETHERTYPE_88A8)
4471 				hw_features |= NETIF_F_HW_VLAN_STAG_RX;
4472 		} else if (stripping_support->inner !=
4473 			   VIRTCHNL_VLAN_UNSUPPORTED &&
4474 			   stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4475 			if (stripping_support->inner &
4476 			    VIRTCHNL_VLAN_ETHERTYPE_8100)
4477 				hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4478 		}
4479 
4480 		if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4481 		    insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4482 			if (insertion_support->outer &
4483 			    VIRTCHNL_VLAN_ETHERTYPE_8100)
4484 				hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4485 			if (insertion_support->outer &
4486 			    VIRTCHNL_VLAN_ETHERTYPE_88A8)
4487 				hw_features |= NETIF_F_HW_VLAN_STAG_TX;
4488 		} else if (insertion_support->inner &&
4489 			   insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4490 			if (insertion_support->inner &
4491 			    VIRTCHNL_VLAN_ETHERTYPE_8100)
4492 				hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4493 		}
4494 	}
4495 
4496 	return hw_features;
4497 }
4498 
4499 /**
4500  * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures
4501  * @adapter: board private structure
4502  *
4503  * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4504  * were negotiated determine the VLAN features that are enabled by default.
4505  **/
4506 static netdev_features_t
4507 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter)
4508 {
4509 	netdev_features_t features = 0;
4510 
4511 	if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4512 		return features;
4513 
4514 	if (VLAN_ALLOWED(adapter)) {
4515 		features |= NETIF_F_HW_VLAN_CTAG_FILTER |
4516 			NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX;
4517 	} else if (VLAN_V2_ALLOWED(adapter)) {
4518 		struct virtchnl_vlan_caps *vlan_v2_caps =
4519 			&adapter->vlan_v2_caps;
4520 		struct virtchnl_vlan_supported_caps *filtering_support =
4521 			&vlan_v2_caps->filtering.filtering_support;
4522 		struct virtchnl_vlan_supported_caps *stripping_support =
4523 			&vlan_v2_caps->offloads.stripping_support;
4524 		struct virtchnl_vlan_supported_caps *insertion_support =
4525 			&vlan_v2_caps->offloads.insertion_support;
4526 		u32 ethertype_init;
4527 
4528 		/* give priority to outer stripping and don't support both outer
4529 		 * and inner stripping
4530 		 */
4531 		ethertype_init = vlan_v2_caps->offloads.ethertype_init;
4532 		if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4533 			if (stripping_support->outer &
4534 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4535 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4536 				features |= NETIF_F_HW_VLAN_CTAG_RX;
4537 			else if (stripping_support->outer &
4538 				 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4539 				 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4540 				features |= NETIF_F_HW_VLAN_STAG_RX;
4541 		} else if (stripping_support->inner !=
4542 			   VIRTCHNL_VLAN_UNSUPPORTED) {
4543 			if (stripping_support->inner &
4544 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4545 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4546 				features |= NETIF_F_HW_VLAN_CTAG_RX;
4547 		}
4548 
4549 		/* give priority to outer insertion and don't support both outer
4550 		 * and inner insertion
4551 		 */
4552 		if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4553 			if (insertion_support->outer &
4554 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4555 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4556 				features |= NETIF_F_HW_VLAN_CTAG_TX;
4557 			else if (insertion_support->outer &
4558 				 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4559 				 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4560 				features |= NETIF_F_HW_VLAN_STAG_TX;
4561 		} else if (insertion_support->inner !=
4562 			   VIRTCHNL_VLAN_UNSUPPORTED) {
4563 			if (insertion_support->inner &
4564 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4565 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4566 				features |= NETIF_F_HW_VLAN_CTAG_TX;
4567 		}
4568 
4569 		/* give priority to outer filtering and don't bother if both
4570 		 * outer and inner filtering are enabled
4571 		 */
4572 		ethertype_init = vlan_v2_caps->filtering.ethertype_init;
4573 		if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4574 			if (filtering_support->outer &
4575 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4576 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4577 				features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4578 			if (filtering_support->outer &
4579 			    VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4580 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4581 				features |= NETIF_F_HW_VLAN_STAG_FILTER;
4582 		} else if (filtering_support->inner !=
4583 			   VIRTCHNL_VLAN_UNSUPPORTED) {
4584 			if (filtering_support->inner &
4585 			    VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4586 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4587 				features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4588 			if (filtering_support->inner &
4589 			    VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4590 			    ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4591 				features |= NETIF_F_HW_VLAN_STAG_FILTER;
4592 		}
4593 	}
4594 
4595 	return features;
4596 }
4597 
4598 #define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \
4599 	(!(((requested) & (feature_bit)) && \
4600 	   !((allowed) & (feature_bit))))
4601 
4602 /**
4603  * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support
4604  * @adapter: board private structure
4605  * @requested_features: stack requested NETDEV features
4606  **/
4607 static netdev_features_t
4608 iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter,
4609 			      netdev_features_t requested_features)
4610 {
4611 	netdev_features_t allowed_features;
4612 
4613 	allowed_features = iavf_get_netdev_vlan_hw_features(adapter) |
4614 		iavf_get_netdev_vlan_features(adapter);
4615 
4616 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4617 					      allowed_features,
4618 					      NETIF_F_HW_VLAN_CTAG_TX))
4619 		requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX;
4620 
4621 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4622 					      allowed_features,
4623 					      NETIF_F_HW_VLAN_CTAG_RX))
4624 		requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4625 
4626 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4627 					      allowed_features,
4628 					      NETIF_F_HW_VLAN_STAG_TX))
4629 		requested_features &= ~NETIF_F_HW_VLAN_STAG_TX;
4630 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4631 					      allowed_features,
4632 					      NETIF_F_HW_VLAN_STAG_RX))
4633 		requested_features &= ~NETIF_F_HW_VLAN_STAG_RX;
4634 
4635 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4636 					      allowed_features,
4637 					      NETIF_F_HW_VLAN_CTAG_FILTER))
4638 		requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4639 
4640 	if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4641 					      allowed_features,
4642 					      NETIF_F_HW_VLAN_STAG_FILTER))
4643 		requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER;
4644 
4645 	if ((requested_features &
4646 	     (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
4647 	    (requested_features &
4648 	     (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) &&
4649 	    adapter->vlan_v2_caps.offloads.ethertype_match ==
4650 	    VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) {
4651 		netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
4652 		requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX |
4653 					NETIF_F_HW_VLAN_STAG_TX);
4654 	}
4655 
4656 	return requested_features;
4657 }
4658 
4659 /**
4660  * iavf_fix_features - fix up the netdev feature bits
4661  * @netdev: our net device
4662  * @features: desired feature bits
4663  *
4664  * Returns fixed-up features bits
4665  **/
4666 static netdev_features_t iavf_fix_features(struct net_device *netdev,
4667 					   netdev_features_t features)
4668 {
4669 	struct iavf_adapter *adapter = netdev_priv(netdev);
4670 
4671 	return iavf_fix_netdev_vlan_features(adapter, features);
4672 }
4673 
4674 static const struct net_device_ops iavf_netdev_ops = {
4675 	.ndo_open		= iavf_open,
4676 	.ndo_stop		= iavf_close,
4677 	.ndo_start_xmit		= iavf_xmit_frame,
4678 	.ndo_set_rx_mode	= iavf_set_rx_mode,
4679 	.ndo_validate_addr	= eth_validate_addr,
4680 	.ndo_set_mac_address	= iavf_set_mac,
4681 	.ndo_change_mtu		= iavf_change_mtu,
4682 	.ndo_tx_timeout		= iavf_tx_timeout,
4683 	.ndo_vlan_rx_add_vid	= iavf_vlan_rx_add_vid,
4684 	.ndo_vlan_rx_kill_vid	= iavf_vlan_rx_kill_vid,
4685 	.ndo_features_check	= iavf_features_check,
4686 	.ndo_fix_features	= iavf_fix_features,
4687 	.ndo_set_features	= iavf_set_features,
4688 	.ndo_setup_tc		= iavf_setup_tc,
4689 };
4690 
4691 /**
4692  * iavf_check_reset_complete - check that VF reset is complete
4693  * @hw: pointer to hw struct
4694  *
4695  * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
4696  **/
4697 static int iavf_check_reset_complete(struct iavf_hw *hw)
4698 {
4699 	u32 rstat;
4700 	int i;
4701 
4702 	for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
4703 		rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
4704 			     IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
4705 		if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
4706 		    (rstat == VIRTCHNL_VFR_COMPLETED))
4707 			return 0;
4708 		usleep_range(10, 20);
4709 	}
4710 	return -EBUSY;
4711 }
4712 
4713 /**
4714  * iavf_process_config - Process the config information we got from the PF
4715  * @adapter: board private structure
4716  *
4717  * Verify that we have a valid config struct, and set up our netdev features
4718  * and our VSI struct.
4719  **/
4720 int iavf_process_config(struct iavf_adapter *adapter)
4721 {
4722 	struct virtchnl_vf_resource *vfres = adapter->vf_res;
4723 	netdev_features_t hw_vlan_features, vlan_features;
4724 	struct net_device *netdev = adapter->netdev;
4725 	netdev_features_t hw_enc_features;
4726 	netdev_features_t hw_features;
4727 
4728 	hw_enc_features = NETIF_F_SG			|
4729 			  NETIF_F_IP_CSUM		|
4730 			  NETIF_F_IPV6_CSUM		|
4731 			  NETIF_F_HIGHDMA		|
4732 			  NETIF_F_SOFT_FEATURES	|
4733 			  NETIF_F_TSO			|
4734 			  NETIF_F_TSO_ECN		|
4735 			  NETIF_F_TSO6			|
4736 			  NETIF_F_SCTP_CRC		|
4737 			  NETIF_F_RXHASH		|
4738 			  NETIF_F_RXCSUM		|
4739 			  0;
4740 
4741 	/* advertise to stack only if offloads for encapsulated packets is
4742 	 * supported
4743 	 */
4744 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
4745 		hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL	|
4746 				   NETIF_F_GSO_GRE		|
4747 				   NETIF_F_GSO_GRE_CSUM		|
4748 				   NETIF_F_GSO_IPXIP4		|
4749 				   NETIF_F_GSO_IPXIP6		|
4750 				   NETIF_F_GSO_UDP_TUNNEL_CSUM	|
4751 				   NETIF_F_GSO_PARTIAL		|
4752 				   0;
4753 
4754 		if (!(vfres->vf_cap_flags &
4755 		      VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
4756 			netdev->gso_partial_features |=
4757 				NETIF_F_GSO_UDP_TUNNEL_CSUM;
4758 
4759 		netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
4760 		netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
4761 		netdev->hw_enc_features |= hw_enc_features;
4762 	}
4763 	/* record features VLANs can make use of */
4764 	netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
4765 
4766 	/* Write features and hw_features separately to avoid polluting
4767 	 * with, or dropping, features that are set when we registered.
4768 	 */
4769 	hw_features = hw_enc_features;
4770 
4771 	/* get HW VLAN features that can be toggled */
4772 	hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter);
4773 
4774 	/* Enable cloud filter if ADQ is supported */
4775 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
4776 		hw_features |= NETIF_F_HW_TC;
4777 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
4778 		hw_features |= NETIF_F_GSO_UDP_L4;
4779 
4780 	netdev->hw_features |= hw_features | hw_vlan_features;
4781 	vlan_features = iavf_get_netdev_vlan_features(adapter);
4782 
4783 	netdev->features |= hw_features | vlan_features;
4784 
4785 	if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
4786 		netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4787 
4788 	netdev->priv_flags |= IFF_UNICAST_FLT;
4789 
4790 	/* Do not turn on offloads when they are requested to be turned off.
4791 	 * TSO needs minimum 576 bytes to work correctly.
4792 	 */
4793 	if (netdev->wanted_features) {
4794 		if (!(netdev->wanted_features & NETIF_F_TSO) ||
4795 		    netdev->mtu < 576)
4796 			netdev->features &= ~NETIF_F_TSO;
4797 		if (!(netdev->wanted_features & NETIF_F_TSO6) ||
4798 		    netdev->mtu < 576)
4799 			netdev->features &= ~NETIF_F_TSO6;
4800 		if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
4801 			netdev->features &= ~NETIF_F_TSO_ECN;
4802 		if (!(netdev->wanted_features & NETIF_F_GRO))
4803 			netdev->features &= ~NETIF_F_GRO;
4804 		if (!(netdev->wanted_features & NETIF_F_GSO))
4805 			netdev->features &= ~NETIF_F_GSO;
4806 	}
4807 
4808 	return 0;
4809 }
4810 
4811 /**
4812  * iavf_shutdown - Shutdown the device in preparation for a reboot
4813  * @pdev: pci device structure
4814  **/
4815 static void iavf_shutdown(struct pci_dev *pdev)
4816 {
4817 	struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
4818 	struct net_device *netdev = adapter->netdev;
4819 
4820 	netif_device_detach(netdev);
4821 
4822 	if (netif_running(netdev))
4823 		iavf_close(netdev);
4824 
4825 	if (iavf_lock_timeout(&adapter->crit_lock, 5000))
4826 		dev_warn(&adapter->pdev->dev, "failed to acquire crit_lock in %s\n", __FUNCTION__);
4827 	/* Prevent the watchdog from running. */
4828 	iavf_change_state(adapter, __IAVF_REMOVE);
4829 	adapter->aq_required = 0;
4830 	mutex_unlock(&adapter->crit_lock);
4831 
4832 #ifdef CONFIG_PM
4833 	pci_save_state(pdev);
4834 
4835 #endif
4836 	pci_disable_device(pdev);
4837 }
4838 
4839 /**
4840  * iavf_probe - Device Initialization Routine
4841  * @pdev: PCI device information struct
4842  * @ent: entry in iavf_pci_tbl
4843  *
4844  * Returns 0 on success, negative on failure
4845  *
4846  * iavf_probe initializes an adapter identified by a pci_dev structure.
4847  * The OS initialization, configuring of the adapter private structure,
4848  * and a hardware reset occur.
4849  **/
4850 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4851 {
4852 	struct net_device *netdev;
4853 	struct iavf_adapter *adapter = NULL;
4854 	struct iavf_hw *hw = NULL;
4855 	int err;
4856 
4857 	err = pci_enable_device(pdev);
4858 	if (err)
4859 		return err;
4860 
4861 	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4862 	if (err) {
4863 		dev_err(&pdev->dev,
4864 			"DMA configuration failed: 0x%x\n", err);
4865 		goto err_dma;
4866 	}
4867 
4868 	err = pci_request_regions(pdev, iavf_driver_name);
4869 	if (err) {
4870 		dev_err(&pdev->dev,
4871 			"pci_request_regions failed 0x%x\n", err);
4872 		goto err_pci_reg;
4873 	}
4874 
4875 	pci_enable_pcie_error_reporting(pdev);
4876 
4877 	pci_set_master(pdev);
4878 
4879 	netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
4880 				   IAVF_MAX_REQ_QUEUES);
4881 	if (!netdev) {
4882 		err = -ENOMEM;
4883 		goto err_alloc_etherdev;
4884 	}
4885 
4886 	SET_NETDEV_DEV(netdev, &pdev->dev);
4887 
4888 	pci_set_drvdata(pdev, netdev);
4889 	adapter = netdev_priv(netdev);
4890 
4891 	adapter->netdev = netdev;
4892 	adapter->pdev = pdev;
4893 
4894 	hw = &adapter->hw;
4895 	hw->back = adapter;
4896 
4897 	adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
4898 	iavf_change_state(adapter, __IAVF_STARTUP);
4899 
4900 	/* Call save state here because it relies on the adapter struct. */
4901 	pci_save_state(pdev);
4902 
4903 	hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
4904 			      pci_resource_len(pdev, 0));
4905 	if (!hw->hw_addr) {
4906 		err = -EIO;
4907 		goto err_ioremap;
4908 	}
4909 	hw->vendor_id = pdev->vendor;
4910 	hw->device_id = pdev->device;
4911 	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
4912 	hw->subsystem_vendor_id = pdev->subsystem_vendor;
4913 	hw->subsystem_device_id = pdev->subsystem_device;
4914 	hw->bus.device = PCI_SLOT(pdev->devfn);
4915 	hw->bus.func = PCI_FUNC(pdev->devfn);
4916 	hw->bus.bus_id = pdev->bus->number;
4917 
4918 	/* set up the locks for the AQ, do this only once in probe
4919 	 * and destroy them only once in remove
4920 	 */
4921 	mutex_init(&adapter->crit_lock);
4922 	mutex_init(&adapter->client_lock);
4923 	mutex_init(&hw->aq.asq_mutex);
4924 	mutex_init(&hw->aq.arq_mutex);
4925 
4926 	spin_lock_init(&adapter->mac_vlan_list_lock);
4927 	spin_lock_init(&adapter->cloud_filter_list_lock);
4928 	spin_lock_init(&adapter->fdir_fltr_lock);
4929 	spin_lock_init(&adapter->adv_rss_lock);
4930 
4931 	INIT_LIST_HEAD(&adapter->mac_filter_list);
4932 	INIT_LIST_HEAD(&adapter->vlan_filter_list);
4933 	INIT_LIST_HEAD(&adapter->cloud_filter_list);
4934 	INIT_LIST_HEAD(&adapter->fdir_list_head);
4935 	INIT_LIST_HEAD(&adapter->adv_rss_list_head);
4936 
4937 	INIT_WORK(&adapter->reset_task, iavf_reset_task);
4938 	INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
4939 	INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
4940 	INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
4941 	queue_delayed_work(iavf_wq, &adapter->watchdog_task,
4942 			   msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
4943 
4944 	/* Setup the wait queue for indicating transition to down status */
4945 	init_waitqueue_head(&adapter->down_waitqueue);
4946 
4947 	/* Setup the wait queue for indicating virtchannel events */
4948 	init_waitqueue_head(&adapter->vc_waitqueue);
4949 
4950 	return 0;
4951 
4952 err_ioremap:
4953 	free_netdev(netdev);
4954 err_alloc_etherdev:
4955 	pci_disable_pcie_error_reporting(pdev);
4956 	pci_release_regions(pdev);
4957 err_pci_reg:
4958 err_dma:
4959 	pci_disable_device(pdev);
4960 	return err;
4961 }
4962 
4963 /**
4964  * iavf_suspend - Power management suspend routine
4965  * @dev_d: device info pointer
4966  *
4967  * Called when the system (VM) is entering sleep/suspend.
4968  **/
4969 static int __maybe_unused iavf_suspend(struct device *dev_d)
4970 {
4971 	struct net_device *netdev = dev_get_drvdata(dev_d);
4972 	struct iavf_adapter *adapter = netdev_priv(netdev);
4973 
4974 	netif_device_detach(netdev);
4975 
4976 	while (!mutex_trylock(&adapter->crit_lock))
4977 		usleep_range(500, 1000);
4978 
4979 	if (netif_running(netdev)) {
4980 		rtnl_lock();
4981 		iavf_down(adapter);
4982 		rtnl_unlock();
4983 	}
4984 	iavf_free_misc_irq(adapter);
4985 	iavf_reset_interrupt_capability(adapter);
4986 
4987 	mutex_unlock(&adapter->crit_lock);
4988 
4989 	return 0;
4990 }
4991 
4992 /**
4993  * iavf_resume - Power management resume routine
4994  * @dev_d: device info pointer
4995  *
4996  * Called when the system (VM) is resumed from sleep/suspend.
4997  **/
4998 static int __maybe_unused iavf_resume(struct device *dev_d)
4999 {
5000 	struct pci_dev *pdev = to_pci_dev(dev_d);
5001 	struct iavf_adapter *adapter;
5002 	u32 err;
5003 
5004 	adapter = iavf_pdev_to_adapter(pdev);
5005 
5006 	pci_set_master(pdev);
5007 
5008 	rtnl_lock();
5009 	err = iavf_set_interrupt_capability(adapter);
5010 	if (err) {
5011 		rtnl_unlock();
5012 		dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
5013 		return err;
5014 	}
5015 	err = iavf_request_misc_irq(adapter);
5016 	rtnl_unlock();
5017 	if (err) {
5018 		dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
5019 		return err;
5020 	}
5021 
5022 	queue_work(iavf_wq, &adapter->reset_task);
5023 
5024 	netif_device_attach(adapter->netdev);
5025 
5026 	return err;
5027 }
5028 
5029 /**
5030  * iavf_remove - Device Removal Routine
5031  * @pdev: PCI device information struct
5032  *
5033  * iavf_remove is called by the PCI subsystem to alert the driver
5034  * that it should release a PCI device.  The could be caused by a
5035  * Hot-Plug event, or because the driver is going to be removed from
5036  * memory.
5037  **/
5038 static void iavf_remove(struct pci_dev *pdev)
5039 {
5040 	struct iavf_adapter *adapter = iavf_pdev_to_adapter(pdev);
5041 	struct net_device *netdev = adapter->netdev;
5042 	struct iavf_fdir_fltr *fdir, *fdirtmp;
5043 	struct iavf_vlan_filter *vlf, *vlftmp;
5044 	struct iavf_adv_rss *rss, *rsstmp;
5045 	struct iavf_mac_filter *f, *ftmp;
5046 	struct iavf_cloud_filter *cf, *cftmp;
5047 	struct iavf_hw *hw = &adapter->hw;
5048 	int err;
5049 
5050 	/* When reboot/shutdown is in progress no need to do anything
5051 	 * as the adapter is already REMOVE state that was set during
5052 	 * iavf_shutdown() callback.
5053 	 */
5054 	if (adapter->state == __IAVF_REMOVE)
5055 		return;
5056 
5057 	set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section);
5058 	/* Wait until port initialization is complete.
5059 	 * There are flows where register/unregister netdev may race.
5060 	 */
5061 	while (1) {
5062 		mutex_lock(&adapter->crit_lock);
5063 		if (adapter->state == __IAVF_RUNNING ||
5064 		    adapter->state == __IAVF_DOWN ||
5065 		    adapter->state == __IAVF_INIT_FAILED) {
5066 			mutex_unlock(&adapter->crit_lock);
5067 			break;
5068 		}
5069 
5070 		mutex_unlock(&adapter->crit_lock);
5071 		usleep_range(500, 1000);
5072 	}
5073 	cancel_delayed_work_sync(&adapter->watchdog_task);
5074 
5075 	if (adapter->netdev_registered) {
5076 		rtnl_lock();
5077 		unregister_netdevice(netdev);
5078 		adapter->netdev_registered = false;
5079 		rtnl_unlock();
5080 	}
5081 	if (CLIENT_ALLOWED(adapter)) {
5082 		err = iavf_lan_del_device(adapter);
5083 		if (err)
5084 			dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
5085 				 err);
5086 	}
5087 
5088 	mutex_lock(&adapter->crit_lock);
5089 	dev_info(&adapter->pdev->dev, "Remove device\n");
5090 	iavf_change_state(adapter, __IAVF_REMOVE);
5091 
5092 	iavf_request_reset(adapter);
5093 	msleep(50);
5094 	/* If the FW isn't responding, kick it once, but only once. */
5095 	if (!iavf_asq_done(hw)) {
5096 		iavf_request_reset(adapter);
5097 		msleep(50);
5098 	}
5099 
5100 	iavf_misc_irq_disable(adapter);
5101 	/* Shut down all the garbage mashers on the detention level */
5102 	cancel_work_sync(&adapter->reset_task);
5103 	cancel_delayed_work_sync(&adapter->watchdog_task);
5104 	cancel_work_sync(&adapter->adminq_task);
5105 	cancel_delayed_work_sync(&adapter->client_task);
5106 
5107 	adapter->aq_required = 0;
5108 	adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
5109 
5110 	iavf_free_all_tx_resources(adapter);
5111 	iavf_free_all_rx_resources(adapter);
5112 	iavf_free_misc_irq(adapter);
5113 
5114 	iavf_reset_interrupt_capability(adapter);
5115 	iavf_free_q_vectors(adapter);
5116 
5117 	iavf_free_rss(adapter);
5118 
5119 	if (hw->aq.asq.count)
5120 		iavf_shutdown_adminq(hw);
5121 
5122 	/* destroy the locks only once, here */
5123 	mutex_destroy(&hw->aq.arq_mutex);
5124 	mutex_destroy(&hw->aq.asq_mutex);
5125 	mutex_destroy(&adapter->client_lock);
5126 	mutex_unlock(&adapter->crit_lock);
5127 	mutex_destroy(&adapter->crit_lock);
5128 
5129 	iounmap(hw->hw_addr);
5130 	pci_release_regions(pdev);
5131 	iavf_free_queues(adapter);
5132 	kfree(adapter->vf_res);
5133 	spin_lock_bh(&adapter->mac_vlan_list_lock);
5134 	/* If we got removed before an up/down sequence, we've got a filter
5135 	 * hanging out there that we need to get rid of.
5136 	 */
5137 	list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
5138 		list_del(&f->list);
5139 		kfree(f);
5140 	}
5141 	list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
5142 				 list) {
5143 		list_del(&vlf->list);
5144 		kfree(vlf);
5145 	}
5146 
5147 	spin_unlock_bh(&adapter->mac_vlan_list_lock);
5148 
5149 	spin_lock_bh(&adapter->cloud_filter_list_lock);
5150 	list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
5151 		list_del(&cf->list);
5152 		kfree(cf);
5153 	}
5154 	spin_unlock_bh(&adapter->cloud_filter_list_lock);
5155 
5156 	spin_lock_bh(&adapter->fdir_fltr_lock);
5157 	list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
5158 		list_del(&fdir->list);
5159 		kfree(fdir);
5160 	}
5161 	spin_unlock_bh(&adapter->fdir_fltr_lock);
5162 
5163 	spin_lock_bh(&adapter->adv_rss_lock);
5164 	list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
5165 				 list) {
5166 		list_del(&rss->list);
5167 		kfree(rss);
5168 	}
5169 	spin_unlock_bh(&adapter->adv_rss_lock);
5170 
5171 	free_netdev(netdev);
5172 
5173 	pci_disable_pcie_error_reporting(pdev);
5174 
5175 	pci_disable_device(pdev);
5176 }
5177 
5178 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
5179 
5180 static struct pci_driver iavf_driver = {
5181 	.name      = iavf_driver_name,
5182 	.id_table  = iavf_pci_tbl,
5183 	.probe     = iavf_probe,
5184 	.remove    = iavf_remove,
5185 	.driver.pm = &iavf_pm_ops,
5186 	.shutdown  = iavf_shutdown,
5187 };
5188 
5189 /**
5190  * iavf_init_module - Driver Registration Routine
5191  *
5192  * iavf_init_module is the first routine called when the driver is
5193  * loaded. All it does is register with the PCI subsystem.
5194  **/
5195 static int __init iavf_init_module(void)
5196 {
5197 	pr_info("iavf: %s\n", iavf_driver_string);
5198 
5199 	pr_info("%s\n", iavf_copyright);
5200 
5201 	iavf_wq = alloc_workqueue("%s", WQ_UNBOUND | WQ_MEM_RECLAIM, 1,
5202 				  iavf_driver_name);
5203 	if (!iavf_wq) {
5204 		pr_err("%s: Failed to create workqueue\n", iavf_driver_name);
5205 		return -ENOMEM;
5206 	}
5207 	return pci_register_driver(&iavf_driver);
5208 }
5209 
5210 module_init(iavf_init_module);
5211 
5212 /**
5213  * iavf_exit_module - Driver Exit Cleanup Routine
5214  *
5215  * iavf_exit_module is called just before the driver is removed
5216  * from memory.
5217  **/
5218 static void __exit iavf_exit_module(void)
5219 {
5220 	pci_unregister_driver(&iavf_driver);
5221 	destroy_workqueue(iavf_wq);
5222 }
5223 
5224 module_exit(iavf_exit_module);
5225 
5226 /* iavf_main.c */
5227