1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
3
4 #include "iavf.h"
5 #include "iavf_prototype.h"
6 #include "iavf_client.h"
7 /* All iavf tracepoints are defined by the include below, which must
8 * be included exactly once across the whole kernel with
9 * CREATE_TRACE_POINTS defined
10 */
11 #define CREATE_TRACE_POINTS
12 #include "iavf_trace.h"
13
14 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter);
15 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter);
16 static int iavf_close(struct net_device *netdev);
17 static void iavf_init_get_resources(struct iavf_adapter *adapter);
18 static int iavf_check_reset_complete(struct iavf_hw *hw);
19
20 char iavf_driver_name[] = "iavf";
21 static const char iavf_driver_string[] =
22 "Intel(R) Ethernet Adaptive Virtual Function Network Driver";
23
24 static const char iavf_copyright[] =
25 "Copyright (c) 2013 - 2018 Intel Corporation.";
26
27 /* iavf_pci_tbl - PCI Device ID Table
28 *
29 * Wildcard entries (PCI_ANY_ID) should come last
30 * Last entry must be all 0s
31 *
32 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
33 * Class, Class Mask, private data (not used) }
34 */
35 static const struct pci_device_id iavf_pci_tbl[] = {
36 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF), 0},
37 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_VF_HV), 0},
38 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_X722_VF), 0},
39 {PCI_VDEVICE(INTEL, IAVF_DEV_ID_ADAPTIVE_VF), 0},
40 /* required last entry */
41 {0, }
42 };
43
44 MODULE_DEVICE_TABLE(pci, iavf_pci_tbl);
45
46 MODULE_ALIAS("i40evf");
47 MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
48 MODULE_DESCRIPTION("Intel(R) Ethernet Adaptive Virtual Function Network Driver");
49 MODULE_LICENSE("GPL v2");
50
51 static const struct net_device_ops iavf_netdev_ops;
52
iavf_status_to_errno(enum iavf_status status)53 int iavf_status_to_errno(enum iavf_status status)
54 {
55 switch (status) {
56 case IAVF_SUCCESS:
57 return 0;
58 case IAVF_ERR_PARAM:
59 case IAVF_ERR_MAC_TYPE:
60 case IAVF_ERR_INVALID_MAC_ADDR:
61 case IAVF_ERR_INVALID_LINK_SETTINGS:
62 case IAVF_ERR_INVALID_PD_ID:
63 case IAVF_ERR_INVALID_QP_ID:
64 case IAVF_ERR_INVALID_CQ_ID:
65 case IAVF_ERR_INVALID_CEQ_ID:
66 case IAVF_ERR_INVALID_AEQ_ID:
67 case IAVF_ERR_INVALID_SIZE:
68 case IAVF_ERR_INVALID_ARP_INDEX:
69 case IAVF_ERR_INVALID_FPM_FUNC_ID:
70 case IAVF_ERR_QP_INVALID_MSG_SIZE:
71 case IAVF_ERR_INVALID_FRAG_COUNT:
72 case IAVF_ERR_INVALID_ALIGNMENT:
73 case IAVF_ERR_INVALID_PUSH_PAGE_INDEX:
74 case IAVF_ERR_INVALID_IMM_DATA_SIZE:
75 case IAVF_ERR_INVALID_VF_ID:
76 case IAVF_ERR_INVALID_HMCFN_ID:
77 case IAVF_ERR_INVALID_PBLE_INDEX:
78 case IAVF_ERR_INVALID_SD_INDEX:
79 case IAVF_ERR_INVALID_PAGE_DESC_INDEX:
80 case IAVF_ERR_INVALID_SD_TYPE:
81 case IAVF_ERR_INVALID_HMC_OBJ_INDEX:
82 case IAVF_ERR_INVALID_HMC_OBJ_COUNT:
83 case IAVF_ERR_INVALID_SRQ_ARM_LIMIT:
84 return -EINVAL;
85 case IAVF_ERR_NVM:
86 case IAVF_ERR_NVM_CHECKSUM:
87 case IAVF_ERR_PHY:
88 case IAVF_ERR_CONFIG:
89 case IAVF_ERR_UNKNOWN_PHY:
90 case IAVF_ERR_LINK_SETUP:
91 case IAVF_ERR_ADAPTER_STOPPED:
92 case IAVF_ERR_PRIMARY_REQUESTS_PENDING:
93 case IAVF_ERR_AUTONEG_NOT_COMPLETE:
94 case IAVF_ERR_RESET_FAILED:
95 case IAVF_ERR_BAD_PTR:
96 case IAVF_ERR_SWFW_SYNC:
97 case IAVF_ERR_QP_TOOMANY_WRS_POSTED:
98 case IAVF_ERR_QUEUE_EMPTY:
99 case IAVF_ERR_FLUSHED_QUEUE:
100 case IAVF_ERR_OPCODE_MISMATCH:
101 case IAVF_ERR_CQP_COMPL_ERROR:
102 case IAVF_ERR_BACKING_PAGE_ERROR:
103 case IAVF_ERR_NO_PBLCHUNKS_AVAILABLE:
104 case IAVF_ERR_MEMCPY_FAILED:
105 case IAVF_ERR_SRQ_ENABLED:
106 case IAVF_ERR_ADMIN_QUEUE_ERROR:
107 case IAVF_ERR_ADMIN_QUEUE_FULL:
108 case IAVF_ERR_BAD_RDMA_CQE:
109 case IAVF_ERR_NVM_BLANK_MODE:
110 case IAVF_ERR_PE_DOORBELL_NOT_ENABLED:
111 case IAVF_ERR_DIAG_TEST_FAILED:
112 case IAVF_ERR_FIRMWARE_API_VERSION:
113 case IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR:
114 return -EIO;
115 case IAVF_ERR_DEVICE_NOT_SUPPORTED:
116 return -ENODEV;
117 case IAVF_ERR_NO_AVAILABLE_VSI:
118 case IAVF_ERR_RING_FULL:
119 return -ENOSPC;
120 case IAVF_ERR_NO_MEMORY:
121 return -ENOMEM;
122 case IAVF_ERR_TIMEOUT:
123 case IAVF_ERR_ADMIN_QUEUE_TIMEOUT:
124 return -ETIMEDOUT;
125 case IAVF_ERR_NOT_IMPLEMENTED:
126 case IAVF_NOT_SUPPORTED:
127 return -EOPNOTSUPP;
128 case IAVF_ERR_ADMIN_QUEUE_NO_WORK:
129 return -EALREADY;
130 case IAVF_ERR_NOT_READY:
131 return -EBUSY;
132 case IAVF_ERR_BUF_TOO_SHORT:
133 return -EMSGSIZE;
134 }
135
136 return -EIO;
137 }
138
virtchnl_status_to_errno(enum virtchnl_status_code v_status)139 int virtchnl_status_to_errno(enum virtchnl_status_code v_status)
140 {
141 switch (v_status) {
142 case VIRTCHNL_STATUS_SUCCESS:
143 return 0;
144 case VIRTCHNL_STATUS_ERR_PARAM:
145 case VIRTCHNL_STATUS_ERR_INVALID_VF_ID:
146 return -EINVAL;
147 case VIRTCHNL_STATUS_ERR_NO_MEMORY:
148 return -ENOMEM;
149 case VIRTCHNL_STATUS_ERR_OPCODE_MISMATCH:
150 case VIRTCHNL_STATUS_ERR_CQP_COMPL_ERROR:
151 case VIRTCHNL_STATUS_ERR_ADMIN_QUEUE_ERROR:
152 return -EIO;
153 case VIRTCHNL_STATUS_ERR_NOT_SUPPORTED:
154 return -EOPNOTSUPP;
155 }
156
157 return -EIO;
158 }
159
160 /**
161 * iavf_pdev_to_adapter - go from pci_dev to adapter
162 * @pdev: pci_dev pointer
163 */
iavf_pdev_to_adapter(struct pci_dev * pdev)164 static struct iavf_adapter *iavf_pdev_to_adapter(struct pci_dev *pdev)
165 {
166 return netdev_priv(pci_get_drvdata(pdev));
167 }
168
169 /**
170 * iavf_is_reset_in_progress - Check if a reset is in progress
171 * @adapter: board private structure
172 */
iavf_is_reset_in_progress(struct iavf_adapter * adapter)173 static bool iavf_is_reset_in_progress(struct iavf_adapter *adapter)
174 {
175 if (adapter->state == __IAVF_RESETTING ||
176 adapter->flags & (IAVF_FLAG_RESET_PENDING |
177 IAVF_FLAG_RESET_NEEDED))
178 return true;
179
180 return false;
181 }
182
183 /**
184 * iavf_wait_for_reset - Wait for reset to finish.
185 * @adapter: board private structure
186 *
187 * Returns 0 if reset finished successfully, negative on timeout or interrupt.
188 */
iavf_wait_for_reset(struct iavf_adapter * adapter)189 int iavf_wait_for_reset(struct iavf_adapter *adapter)
190 {
191 int ret = wait_event_interruptible_timeout(adapter->reset_waitqueue,
192 !iavf_is_reset_in_progress(adapter),
193 msecs_to_jiffies(5000));
194
195 /* If ret < 0 then it means wait was interrupted.
196 * If ret == 0 then it means we got a timeout while waiting
197 * for reset to finish.
198 * If ret > 0 it means reset has finished.
199 */
200 if (ret > 0)
201 return 0;
202 else if (ret < 0)
203 return -EINTR;
204 else
205 return -EBUSY;
206 }
207
208 /**
209 * iavf_allocate_dma_mem_d - OS specific memory alloc for shared code
210 * @hw: pointer to the HW structure
211 * @mem: ptr to mem struct to fill out
212 * @size: size of memory requested
213 * @alignment: what to align the allocation to
214 **/
iavf_allocate_dma_mem_d(struct iavf_hw * hw,struct iavf_dma_mem * mem,u64 size,u32 alignment)215 enum iavf_status iavf_allocate_dma_mem_d(struct iavf_hw *hw,
216 struct iavf_dma_mem *mem,
217 u64 size, u32 alignment)
218 {
219 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
220
221 if (!mem)
222 return IAVF_ERR_PARAM;
223
224 mem->size = ALIGN(size, alignment);
225 mem->va = dma_alloc_coherent(&adapter->pdev->dev, mem->size,
226 (dma_addr_t *)&mem->pa, GFP_KERNEL);
227 if (mem->va)
228 return 0;
229 else
230 return IAVF_ERR_NO_MEMORY;
231 }
232
233 /**
234 * iavf_free_dma_mem - wrapper for DMA memory freeing
235 * @hw: pointer to the HW structure
236 * @mem: ptr to mem struct to free
237 **/
iavf_free_dma_mem(struct iavf_hw * hw,struct iavf_dma_mem * mem)238 enum iavf_status iavf_free_dma_mem(struct iavf_hw *hw, struct iavf_dma_mem *mem)
239 {
240 struct iavf_adapter *adapter = (struct iavf_adapter *)hw->back;
241
242 if (!mem || !mem->va)
243 return IAVF_ERR_PARAM;
244 dma_free_coherent(&adapter->pdev->dev, mem->size,
245 mem->va, (dma_addr_t)mem->pa);
246 return 0;
247 }
248
249 /**
250 * iavf_allocate_virt_mem - virt memory alloc wrapper
251 * @hw: pointer to the HW structure
252 * @mem: ptr to mem struct to fill out
253 * @size: size of memory requested
254 **/
iavf_allocate_virt_mem(struct iavf_hw * hw,struct iavf_virt_mem * mem,u32 size)255 enum iavf_status iavf_allocate_virt_mem(struct iavf_hw *hw,
256 struct iavf_virt_mem *mem, u32 size)
257 {
258 if (!mem)
259 return IAVF_ERR_PARAM;
260
261 mem->size = size;
262 mem->va = kzalloc(size, GFP_KERNEL);
263
264 if (mem->va)
265 return 0;
266 else
267 return IAVF_ERR_NO_MEMORY;
268 }
269
270 /**
271 * iavf_free_virt_mem - virt memory free wrapper
272 * @hw: pointer to the HW structure
273 * @mem: ptr to mem struct to free
274 **/
iavf_free_virt_mem(struct iavf_hw * hw,struct iavf_virt_mem * mem)275 void iavf_free_virt_mem(struct iavf_hw *hw, struct iavf_virt_mem *mem)
276 {
277 kfree(mem->va);
278 }
279
280 /**
281 * iavf_schedule_reset - Set the flags and schedule a reset event
282 * @adapter: board private structure
283 * @flags: IAVF_FLAG_RESET_PENDING or IAVF_FLAG_RESET_NEEDED
284 **/
iavf_schedule_reset(struct iavf_adapter * adapter,u64 flags)285 void iavf_schedule_reset(struct iavf_adapter *adapter, u64 flags)
286 {
287 if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section) &&
288 !(adapter->flags &
289 (IAVF_FLAG_RESET_PENDING | IAVF_FLAG_RESET_NEEDED))) {
290 adapter->flags |= flags;
291 queue_work(adapter->wq, &adapter->reset_task);
292 }
293 }
294
295 /**
296 * iavf_schedule_aq_request - Set the flags and schedule aq request
297 * @adapter: board private structure
298 * @flags: requested aq flags
299 **/
iavf_schedule_aq_request(struct iavf_adapter * adapter,u64 flags)300 void iavf_schedule_aq_request(struct iavf_adapter *adapter, u64 flags)
301 {
302 adapter->aq_required |= flags;
303 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
304 }
305
306 /**
307 * iavf_tx_timeout - Respond to a Tx Hang
308 * @netdev: network interface device structure
309 * @txqueue: queue number that is timing out
310 **/
iavf_tx_timeout(struct net_device * netdev,unsigned int txqueue)311 static void iavf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
312 {
313 struct iavf_adapter *adapter = netdev_priv(netdev);
314
315 adapter->tx_timeout_count++;
316 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
317 }
318
319 /**
320 * iavf_misc_irq_disable - Mask off interrupt generation on the NIC
321 * @adapter: board private structure
322 **/
iavf_misc_irq_disable(struct iavf_adapter * adapter)323 static void iavf_misc_irq_disable(struct iavf_adapter *adapter)
324 {
325 struct iavf_hw *hw = &adapter->hw;
326
327 if (!adapter->msix_entries)
328 return;
329
330 wr32(hw, IAVF_VFINT_DYN_CTL01, 0);
331
332 iavf_flush(hw);
333
334 synchronize_irq(adapter->msix_entries[0].vector);
335 }
336
337 /**
338 * iavf_misc_irq_enable - Enable default interrupt generation settings
339 * @adapter: board private structure
340 **/
iavf_misc_irq_enable(struct iavf_adapter * adapter)341 static void iavf_misc_irq_enable(struct iavf_adapter *adapter)
342 {
343 struct iavf_hw *hw = &adapter->hw;
344
345 wr32(hw, IAVF_VFINT_DYN_CTL01, IAVF_VFINT_DYN_CTL01_INTENA_MASK |
346 IAVF_VFINT_DYN_CTL01_ITR_INDX_MASK);
347 wr32(hw, IAVF_VFINT_ICR0_ENA1, IAVF_VFINT_ICR0_ENA1_ADMINQ_MASK);
348
349 iavf_flush(hw);
350 }
351
352 /**
353 * iavf_irq_disable - Mask off interrupt generation on the NIC
354 * @adapter: board private structure
355 **/
iavf_irq_disable(struct iavf_adapter * adapter)356 static void iavf_irq_disable(struct iavf_adapter *adapter)
357 {
358 int i;
359 struct iavf_hw *hw = &adapter->hw;
360
361 if (!adapter->msix_entries)
362 return;
363
364 for (i = 1; i < adapter->num_msix_vectors; i++) {
365 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1), 0);
366 synchronize_irq(adapter->msix_entries[i].vector);
367 }
368 iavf_flush(hw);
369 }
370
371 /**
372 * iavf_irq_enable_queues - Enable interrupt for all queues
373 * @adapter: board private structure
374 **/
iavf_irq_enable_queues(struct iavf_adapter * adapter)375 static void iavf_irq_enable_queues(struct iavf_adapter *adapter)
376 {
377 struct iavf_hw *hw = &adapter->hw;
378 int i;
379
380 for (i = 1; i < adapter->num_msix_vectors; i++) {
381 wr32(hw, IAVF_VFINT_DYN_CTLN1(i - 1),
382 IAVF_VFINT_DYN_CTLN1_INTENA_MASK |
383 IAVF_VFINT_DYN_CTLN1_ITR_INDX_MASK);
384 }
385 }
386
387 /**
388 * iavf_irq_enable - Enable default interrupt generation settings
389 * @adapter: board private structure
390 * @flush: boolean value whether to run rd32()
391 **/
iavf_irq_enable(struct iavf_adapter * adapter,bool flush)392 void iavf_irq_enable(struct iavf_adapter *adapter, bool flush)
393 {
394 struct iavf_hw *hw = &adapter->hw;
395
396 iavf_misc_irq_enable(adapter);
397 iavf_irq_enable_queues(adapter);
398
399 if (flush)
400 iavf_flush(hw);
401 }
402
403 /**
404 * iavf_msix_aq - Interrupt handler for vector 0
405 * @irq: interrupt number
406 * @data: pointer to netdev
407 **/
iavf_msix_aq(int irq,void * data)408 static irqreturn_t iavf_msix_aq(int irq, void *data)
409 {
410 struct net_device *netdev = data;
411 struct iavf_adapter *adapter = netdev_priv(netdev);
412 struct iavf_hw *hw = &adapter->hw;
413
414 /* handle non-queue interrupts, these reads clear the registers */
415 rd32(hw, IAVF_VFINT_ICR01);
416 rd32(hw, IAVF_VFINT_ICR0_ENA1);
417
418 if (adapter->state != __IAVF_REMOVE)
419 /* schedule work on the private workqueue */
420 queue_work(adapter->wq, &adapter->adminq_task);
421
422 return IRQ_HANDLED;
423 }
424
425 /**
426 * iavf_msix_clean_rings - MSIX mode Interrupt Handler
427 * @irq: interrupt number
428 * @data: pointer to a q_vector
429 **/
iavf_msix_clean_rings(int irq,void * data)430 static irqreturn_t iavf_msix_clean_rings(int irq, void *data)
431 {
432 struct iavf_q_vector *q_vector = data;
433
434 if (!q_vector->tx.ring && !q_vector->rx.ring)
435 return IRQ_HANDLED;
436
437 napi_schedule_irqoff(&q_vector->napi);
438
439 return IRQ_HANDLED;
440 }
441
442 /**
443 * iavf_map_vector_to_rxq - associate irqs with rx queues
444 * @adapter: board private structure
445 * @v_idx: interrupt number
446 * @r_idx: queue number
447 **/
448 static void
iavf_map_vector_to_rxq(struct iavf_adapter * adapter,int v_idx,int r_idx)449 iavf_map_vector_to_rxq(struct iavf_adapter *adapter, int v_idx, int r_idx)
450 {
451 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
452 struct iavf_ring *rx_ring = &adapter->rx_rings[r_idx];
453 struct iavf_hw *hw = &adapter->hw;
454
455 rx_ring->q_vector = q_vector;
456 rx_ring->next = q_vector->rx.ring;
457 rx_ring->vsi = &adapter->vsi;
458 q_vector->rx.ring = rx_ring;
459 q_vector->rx.count++;
460 q_vector->rx.next_update = jiffies + 1;
461 q_vector->rx.target_itr = ITR_TO_REG(rx_ring->itr_setting);
462 q_vector->ring_mask |= BIT(r_idx);
463 wr32(hw, IAVF_VFINT_ITRN1(IAVF_RX_ITR, q_vector->reg_idx),
464 q_vector->rx.current_itr >> 1);
465 q_vector->rx.current_itr = q_vector->rx.target_itr;
466 }
467
468 /**
469 * iavf_map_vector_to_txq - associate irqs with tx queues
470 * @adapter: board private structure
471 * @v_idx: interrupt number
472 * @t_idx: queue number
473 **/
474 static void
iavf_map_vector_to_txq(struct iavf_adapter * adapter,int v_idx,int t_idx)475 iavf_map_vector_to_txq(struct iavf_adapter *adapter, int v_idx, int t_idx)
476 {
477 struct iavf_q_vector *q_vector = &adapter->q_vectors[v_idx];
478 struct iavf_ring *tx_ring = &adapter->tx_rings[t_idx];
479 struct iavf_hw *hw = &adapter->hw;
480
481 tx_ring->q_vector = q_vector;
482 tx_ring->next = q_vector->tx.ring;
483 tx_ring->vsi = &adapter->vsi;
484 q_vector->tx.ring = tx_ring;
485 q_vector->tx.count++;
486 q_vector->tx.next_update = jiffies + 1;
487 q_vector->tx.target_itr = ITR_TO_REG(tx_ring->itr_setting);
488 q_vector->num_ringpairs++;
489 wr32(hw, IAVF_VFINT_ITRN1(IAVF_TX_ITR, q_vector->reg_idx),
490 q_vector->tx.target_itr >> 1);
491 q_vector->tx.current_itr = q_vector->tx.target_itr;
492 }
493
494 /**
495 * iavf_map_rings_to_vectors - Maps descriptor rings to vectors
496 * @adapter: board private structure to initialize
497 *
498 * This function maps descriptor rings to the queue-specific vectors
499 * we were allotted through the MSI-X enabling code. Ideally, we'd have
500 * one vector per ring/queue, but on a constrained vector budget, we
501 * group the rings as "efficiently" as possible. You would add new
502 * mapping configurations in here.
503 **/
iavf_map_rings_to_vectors(struct iavf_adapter * adapter)504 static void iavf_map_rings_to_vectors(struct iavf_adapter *adapter)
505 {
506 int rings_remaining = adapter->num_active_queues;
507 int ridx = 0, vidx = 0;
508 int q_vectors;
509
510 q_vectors = adapter->num_msix_vectors - NONQ_VECS;
511
512 for (; ridx < rings_remaining; ridx++) {
513 iavf_map_vector_to_rxq(adapter, vidx, ridx);
514 iavf_map_vector_to_txq(adapter, vidx, ridx);
515
516 /* In the case where we have more queues than vectors, continue
517 * round-robin on vectors until all queues are mapped.
518 */
519 if (++vidx >= q_vectors)
520 vidx = 0;
521 }
522
523 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
524 }
525
526 /**
527 * iavf_irq_affinity_notify - Callback for affinity changes
528 * @notify: context as to what irq was changed
529 * @mask: the new affinity mask
530 *
531 * This is a callback function used by the irq_set_affinity_notifier function
532 * so that we may register to receive changes to the irq affinity masks.
533 **/
iavf_irq_affinity_notify(struct irq_affinity_notify * notify,const cpumask_t * mask)534 static void iavf_irq_affinity_notify(struct irq_affinity_notify *notify,
535 const cpumask_t *mask)
536 {
537 struct iavf_q_vector *q_vector =
538 container_of(notify, struct iavf_q_vector, affinity_notify);
539
540 cpumask_copy(&q_vector->affinity_mask, mask);
541 }
542
543 /**
544 * iavf_irq_affinity_release - Callback for affinity notifier release
545 * @ref: internal core kernel usage
546 *
547 * This is a callback function used by the irq_set_affinity_notifier function
548 * to inform the current notification subscriber that they will no longer
549 * receive notifications.
550 **/
iavf_irq_affinity_release(struct kref * ref)551 static void iavf_irq_affinity_release(struct kref *ref) {}
552
553 /**
554 * iavf_request_traffic_irqs - Initialize MSI-X interrupts
555 * @adapter: board private structure
556 * @basename: device basename
557 *
558 * Allocates MSI-X vectors for tx and rx handling, and requests
559 * interrupts from the kernel.
560 **/
561 static int
iavf_request_traffic_irqs(struct iavf_adapter * adapter,char * basename)562 iavf_request_traffic_irqs(struct iavf_adapter *adapter, char *basename)
563 {
564 unsigned int vector, q_vectors;
565 unsigned int rx_int_idx = 0, tx_int_idx = 0;
566 int irq_num, err;
567 int cpu;
568
569 iavf_irq_disable(adapter);
570 /* Decrement for Other and TCP Timer vectors */
571 q_vectors = adapter->num_msix_vectors - NONQ_VECS;
572
573 for (vector = 0; vector < q_vectors; vector++) {
574 struct iavf_q_vector *q_vector = &adapter->q_vectors[vector];
575
576 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
577
578 if (q_vector->tx.ring && q_vector->rx.ring) {
579 snprintf(q_vector->name, sizeof(q_vector->name),
580 "iavf-%s-TxRx-%u", basename, rx_int_idx++);
581 tx_int_idx++;
582 } else if (q_vector->rx.ring) {
583 snprintf(q_vector->name, sizeof(q_vector->name),
584 "iavf-%s-rx-%u", basename, rx_int_idx++);
585 } else if (q_vector->tx.ring) {
586 snprintf(q_vector->name, sizeof(q_vector->name),
587 "iavf-%s-tx-%u", basename, tx_int_idx++);
588 } else {
589 /* skip this unused q_vector */
590 continue;
591 }
592 err = request_irq(irq_num,
593 iavf_msix_clean_rings,
594 0,
595 q_vector->name,
596 q_vector);
597 if (err) {
598 dev_info(&adapter->pdev->dev,
599 "Request_irq failed, error: %d\n", err);
600 goto free_queue_irqs;
601 }
602 /* register for affinity change notifications */
603 q_vector->affinity_notify.notify = iavf_irq_affinity_notify;
604 q_vector->affinity_notify.release =
605 iavf_irq_affinity_release;
606 irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
607 /* Spread the IRQ affinity hints across online CPUs. Note that
608 * get_cpu_mask returns a mask with a permanent lifetime so
609 * it's safe to use as a hint for irq_update_affinity_hint.
610 */
611 cpu = cpumask_local_spread(q_vector->v_idx, -1);
612 irq_update_affinity_hint(irq_num, get_cpu_mask(cpu));
613 }
614
615 return 0;
616
617 free_queue_irqs:
618 while (vector) {
619 vector--;
620 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
621 irq_set_affinity_notifier(irq_num, NULL);
622 irq_update_affinity_hint(irq_num, NULL);
623 free_irq(irq_num, &adapter->q_vectors[vector]);
624 }
625 return err;
626 }
627
628 /**
629 * iavf_request_misc_irq - Initialize MSI-X interrupts
630 * @adapter: board private structure
631 *
632 * Allocates MSI-X vector 0 and requests interrupts from the kernel. This
633 * vector is only for the admin queue, and stays active even when the netdev
634 * is closed.
635 **/
iavf_request_misc_irq(struct iavf_adapter * adapter)636 static int iavf_request_misc_irq(struct iavf_adapter *adapter)
637 {
638 struct net_device *netdev = adapter->netdev;
639 int err;
640
641 snprintf(adapter->misc_vector_name,
642 sizeof(adapter->misc_vector_name) - 1, "iavf-%s:mbx",
643 dev_name(&adapter->pdev->dev));
644 err = request_irq(adapter->msix_entries[0].vector,
645 &iavf_msix_aq, 0,
646 adapter->misc_vector_name, netdev);
647 if (err) {
648 dev_err(&adapter->pdev->dev,
649 "request_irq for %s failed: %d\n",
650 adapter->misc_vector_name, err);
651 free_irq(adapter->msix_entries[0].vector, netdev);
652 }
653 return err;
654 }
655
656 /**
657 * iavf_free_traffic_irqs - Free MSI-X interrupts
658 * @adapter: board private structure
659 *
660 * Frees all MSI-X vectors other than 0.
661 **/
iavf_free_traffic_irqs(struct iavf_adapter * adapter)662 static void iavf_free_traffic_irqs(struct iavf_adapter *adapter)
663 {
664 int vector, irq_num, q_vectors;
665
666 if (!adapter->msix_entries)
667 return;
668
669 q_vectors = adapter->num_msix_vectors - NONQ_VECS;
670
671 for (vector = 0; vector < q_vectors; vector++) {
672 irq_num = adapter->msix_entries[vector + NONQ_VECS].vector;
673 irq_set_affinity_notifier(irq_num, NULL);
674 irq_update_affinity_hint(irq_num, NULL);
675 free_irq(irq_num, &adapter->q_vectors[vector]);
676 }
677 }
678
679 /**
680 * iavf_free_misc_irq - Free MSI-X miscellaneous vector
681 * @adapter: board private structure
682 *
683 * Frees MSI-X vector 0.
684 **/
iavf_free_misc_irq(struct iavf_adapter * adapter)685 static void iavf_free_misc_irq(struct iavf_adapter *adapter)
686 {
687 struct net_device *netdev = adapter->netdev;
688
689 if (!adapter->msix_entries)
690 return;
691
692 free_irq(adapter->msix_entries[0].vector, netdev);
693 }
694
695 /**
696 * iavf_configure_tx - Configure Transmit Unit after Reset
697 * @adapter: board private structure
698 *
699 * Configure the Tx unit of the MAC after a reset.
700 **/
iavf_configure_tx(struct iavf_adapter * adapter)701 static void iavf_configure_tx(struct iavf_adapter *adapter)
702 {
703 struct iavf_hw *hw = &adapter->hw;
704 int i;
705
706 for (i = 0; i < adapter->num_active_queues; i++)
707 adapter->tx_rings[i].tail = hw->hw_addr + IAVF_QTX_TAIL1(i);
708 }
709
710 /**
711 * iavf_configure_rx - Configure Receive Unit after Reset
712 * @adapter: board private structure
713 *
714 * Configure the Rx unit of the MAC after a reset.
715 **/
iavf_configure_rx(struct iavf_adapter * adapter)716 static void iavf_configure_rx(struct iavf_adapter *adapter)
717 {
718 unsigned int rx_buf_len = IAVF_RXBUFFER_2048;
719 struct iavf_hw *hw = &adapter->hw;
720 int i;
721
722 /* Legacy Rx will always default to a 2048 buffer size. */
723 #if (PAGE_SIZE < 8192)
724 if (!(adapter->flags & IAVF_FLAG_LEGACY_RX)) {
725 struct net_device *netdev = adapter->netdev;
726
727 /* For jumbo frames on systems with 4K pages we have to use
728 * an order 1 page, so we might as well increase the size
729 * of our Rx buffer to make better use of the available space
730 */
731 rx_buf_len = IAVF_RXBUFFER_3072;
732
733 /* We use a 1536 buffer size for configurations with
734 * standard Ethernet mtu. On x86 this gives us enough room
735 * for shared info and 192 bytes of padding.
736 */
737 if (!IAVF_2K_TOO_SMALL_WITH_PADDING &&
738 (netdev->mtu <= ETH_DATA_LEN))
739 rx_buf_len = IAVF_RXBUFFER_1536 - NET_IP_ALIGN;
740 }
741 #endif
742
743 for (i = 0; i < adapter->num_active_queues; i++) {
744 adapter->rx_rings[i].tail = hw->hw_addr + IAVF_QRX_TAIL1(i);
745 adapter->rx_rings[i].rx_buf_len = rx_buf_len;
746
747 if (adapter->flags & IAVF_FLAG_LEGACY_RX)
748 clear_ring_build_skb_enabled(&adapter->rx_rings[i]);
749 else
750 set_ring_build_skb_enabled(&adapter->rx_rings[i]);
751 }
752 }
753
754 /**
755 * iavf_find_vlan - Search filter list for specific vlan filter
756 * @adapter: board private structure
757 * @vlan: vlan tag
758 *
759 * Returns ptr to the filter object or NULL. Must be called while holding the
760 * mac_vlan_list_lock.
761 **/
762 static struct
iavf_find_vlan(struct iavf_adapter * adapter,struct iavf_vlan vlan)763 iavf_vlan_filter *iavf_find_vlan(struct iavf_adapter *adapter,
764 struct iavf_vlan vlan)
765 {
766 struct iavf_vlan_filter *f;
767
768 list_for_each_entry(f, &adapter->vlan_filter_list, list) {
769 if (f->vlan.vid == vlan.vid &&
770 f->vlan.tpid == vlan.tpid)
771 return f;
772 }
773
774 return NULL;
775 }
776
777 /**
778 * iavf_add_vlan - Add a vlan filter to the list
779 * @adapter: board private structure
780 * @vlan: VLAN tag
781 *
782 * Returns ptr to the filter object or NULL when no memory available.
783 **/
784 static struct
iavf_add_vlan(struct iavf_adapter * adapter,struct iavf_vlan vlan)785 iavf_vlan_filter *iavf_add_vlan(struct iavf_adapter *adapter,
786 struct iavf_vlan vlan)
787 {
788 struct iavf_vlan_filter *f = NULL;
789
790 spin_lock_bh(&adapter->mac_vlan_list_lock);
791
792 f = iavf_find_vlan(adapter, vlan);
793 if (!f) {
794 f = kzalloc(sizeof(*f), GFP_ATOMIC);
795 if (!f)
796 goto clearout;
797
798 f->vlan = vlan;
799
800 list_add_tail(&f->list, &adapter->vlan_filter_list);
801 f->state = IAVF_VLAN_ADD;
802 adapter->num_vlan_filters++;
803 iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_VLAN_FILTER);
804 } else if (f->state == IAVF_VLAN_REMOVE) {
805 /* IAVF_VLAN_REMOVE means that VLAN wasn't yet removed.
806 * We can safely only change the state here.
807 */
808 f->state = IAVF_VLAN_ACTIVE;
809 }
810
811 clearout:
812 spin_unlock_bh(&adapter->mac_vlan_list_lock);
813 return f;
814 }
815
816 /**
817 * iavf_del_vlan - Remove a vlan filter from the list
818 * @adapter: board private structure
819 * @vlan: VLAN tag
820 **/
iavf_del_vlan(struct iavf_adapter * adapter,struct iavf_vlan vlan)821 static void iavf_del_vlan(struct iavf_adapter *adapter, struct iavf_vlan vlan)
822 {
823 struct iavf_vlan_filter *f;
824
825 spin_lock_bh(&adapter->mac_vlan_list_lock);
826
827 f = iavf_find_vlan(adapter, vlan);
828 if (f) {
829 /* IAVF_ADD_VLAN means that VLAN wasn't even added yet.
830 * Remove it from the list.
831 */
832 if (f->state == IAVF_VLAN_ADD) {
833 list_del(&f->list);
834 kfree(f);
835 adapter->num_vlan_filters--;
836 } else {
837 f->state = IAVF_VLAN_REMOVE;
838 iavf_schedule_aq_request(adapter,
839 IAVF_FLAG_AQ_DEL_VLAN_FILTER);
840 }
841 }
842
843 spin_unlock_bh(&adapter->mac_vlan_list_lock);
844 }
845
846 /**
847 * iavf_restore_filters
848 * @adapter: board private structure
849 *
850 * Restore existing non MAC filters when VF netdev comes back up
851 **/
iavf_restore_filters(struct iavf_adapter * adapter)852 static void iavf_restore_filters(struct iavf_adapter *adapter)
853 {
854 struct iavf_vlan_filter *f;
855
856 /* re-add all VLAN filters */
857 spin_lock_bh(&adapter->mac_vlan_list_lock);
858
859 list_for_each_entry(f, &adapter->vlan_filter_list, list) {
860 if (f->state == IAVF_VLAN_INACTIVE)
861 f->state = IAVF_VLAN_ADD;
862 }
863
864 spin_unlock_bh(&adapter->mac_vlan_list_lock);
865 adapter->aq_required |= IAVF_FLAG_AQ_ADD_VLAN_FILTER;
866 }
867
868 /**
869 * iavf_get_num_vlans_added - get number of VLANs added
870 * @adapter: board private structure
871 */
iavf_get_num_vlans_added(struct iavf_adapter * adapter)872 u16 iavf_get_num_vlans_added(struct iavf_adapter *adapter)
873 {
874 return adapter->num_vlan_filters;
875 }
876
877 /**
878 * iavf_get_max_vlans_allowed - get maximum VLANs allowed for this VF
879 * @adapter: board private structure
880 *
881 * This depends on the negotiated VLAN capability. For VIRTCHNL_VF_OFFLOAD_VLAN,
882 * do not impose a limit as that maintains current behavior and for
883 * VIRTCHNL_VF_OFFLOAD_VLAN_V2, use the maximum allowed sent from the PF.
884 **/
iavf_get_max_vlans_allowed(struct iavf_adapter * adapter)885 static u16 iavf_get_max_vlans_allowed(struct iavf_adapter *adapter)
886 {
887 /* don't impose any limit for VIRTCHNL_VF_OFFLOAD_VLAN since there has
888 * never been a limit on the VF driver side
889 */
890 if (VLAN_ALLOWED(adapter))
891 return VLAN_N_VID;
892 else if (VLAN_V2_ALLOWED(adapter))
893 return adapter->vlan_v2_caps.filtering.max_filters;
894
895 return 0;
896 }
897
898 /**
899 * iavf_max_vlans_added - check if maximum VLANs allowed already exist
900 * @adapter: board private structure
901 **/
iavf_max_vlans_added(struct iavf_adapter * adapter)902 static bool iavf_max_vlans_added(struct iavf_adapter *adapter)
903 {
904 if (iavf_get_num_vlans_added(adapter) <
905 iavf_get_max_vlans_allowed(adapter))
906 return false;
907
908 return true;
909 }
910
911 /**
912 * iavf_vlan_rx_add_vid - Add a VLAN filter to a device
913 * @netdev: network device struct
914 * @proto: unused protocol data
915 * @vid: VLAN tag
916 **/
iavf_vlan_rx_add_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)917 static int iavf_vlan_rx_add_vid(struct net_device *netdev,
918 __always_unused __be16 proto, u16 vid)
919 {
920 struct iavf_adapter *adapter = netdev_priv(netdev);
921
922 /* Do not track VLAN 0 filter, always added by the PF on VF init */
923 if (!vid)
924 return 0;
925
926 if (!VLAN_FILTERING_ALLOWED(adapter))
927 return -EIO;
928
929 if (iavf_max_vlans_added(adapter)) {
930 netdev_err(netdev, "Max allowed VLAN filters %u. Remove existing VLANs or disable filtering via Ethtool if supported.\n",
931 iavf_get_max_vlans_allowed(adapter));
932 return -EIO;
933 }
934
935 if (!iavf_add_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto))))
936 return -ENOMEM;
937
938 return 0;
939 }
940
941 /**
942 * iavf_vlan_rx_kill_vid - Remove a VLAN filter from a device
943 * @netdev: network device struct
944 * @proto: unused protocol data
945 * @vid: VLAN tag
946 **/
iavf_vlan_rx_kill_vid(struct net_device * netdev,__always_unused __be16 proto,u16 vid)947 static int iavf_vlan_rx_kill_vid(struct net_device *netdev,
948 __always_unused __be16 proto, u16 vid)
949 {
950 struct iavf_adapter *adapter = netdev_priv(netdev);
951
952 /* We do not track VLAN 0 filter */
953 if (!vid)
954 return 0;
955
956 iavf_del_vlan(adapter, IAVF_VLAN(vid, be16_to_cpu(proto)));
957 return 0;
958 }
959
960 /**
961 * iavf_find_filter - Search filter list for specific mac filter
962 * @adapter: board private structure
963 * @macaddr: the MAC address
964 *
965 * Returns ptr to the filter object or NULL. Must be called while holding the
966 * mac_vlan_list_lock.
967 **/
968 static struct
iavf_find_filter(struct iavf_adapter * adapter,const u8 * macaddr)969 iavf_mac_filter *iavf_find_filter(struct iavf_adapter *adapter,
970 const u8 *macaddr)
971 {
972 struct iavf_mac_filter *f;
973
974 if (!macaddr)
975 return NULL;
976
977 list_for_each_entry(f, &adapter->mac_filter_list, list) {
978 if (ether_addr_equal(macaddr, f->macaddr))
979 return f;
980 }
981 return NULL;
982 }
983
984 /**
985 * iavf_add_filter - Add a mac filter to the filter list
986 * @adapter: board private structure
987 * @macaddr: the MAC address
988 *
989 * Returns ptr to the filter object or NULL when no memory available.
990 **/
iavf_add_filter(struct iavf_adapter * adapter,const u8 * macaddr)991 struct iavf_mac_filter *iavf_add_filter(struct iavf_adapter *adapter,
992 const u8 *macaddr)
993 {
994 struct iavf_mac_filter *f;
995
996 if (!macaddr)
997 return NULL;
998
999 f = iavf_find_filter(adapter, macaddr);
1000 if (!f) {
1001 f = kzalloc(sizeof(*f), GFP_ATOMIC);
1002 if (!f)
1003 return f;
1004
1005 ether_addr_copy(f->macaddr, macaddr);
1006
1007 list_add_tail(&f->list, &adapter->mac_filter_list);
1008 f->add = true;
1009 f->add_handled = false;
1010 f->is_new_mac = true;
1011 f->is_primary = ether_addr_equal(macaddr, adapter->hw.mac.addr);
1012 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
1013 } else {
1014 f->remove = false;
1015 }
1016
1017 return f;
1018 }
1019
1020 /**
1021 * iavf_replace_primary_mac - Replace current primary address
1022 * @adapter: board private structure
1023 * @new_mac: new MAC address to be applied
1024 *
1025 * Replace current dev_addr and send request to PF for removal of previous
1026 * primary MAC address filter and addition of new primary MAC filter.
1027 * Return 0 for success, -ENOMEM for failure.
1028 *
1029 * Do not call this with mac_vlan_list_lock!
1030 **/
iavf_replace_primary_mac(struct iavf_adapter * adapter,const u8 * new_mac)1031 static int iavf_replace_primary_mac(struct iavf_adapter *adapter,
1032 const u8 *new_mac)
1033 {
1034 struct iavf_hw *hw = &adapter->hw;
1035 struct iavf_mac_filter *new_f;
1036 struct iavf_mac_filter *old_f;
1037
1038 spin_lock_bh(&adapter->mac_vlan_list_lock);
1039
1040 new_f = iavf_add_filter(adapter, new_mac);
1041 if (!new_f) {
1042 spin_unlock_bh(&adapter->mac_vlan_list_lock);
1043 return -ENOMEM;
1044 }
1045
1046 old_f = iavf_find_filter(adapter, hw->mac.addr);
1047 if (old_f) {
1048 old_f->is_primary = false;
1049 old_f->remove = true;
1050 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1051 }
1052 /* Always send the request to add if changing primary MAC,
1053 * even if filter is already present on the list
1054 */
1055 new_f->is_primary = true;
1056 new_f->add = true;
1057 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
1058 ether_addr_copy(hw->mac.addr, new_mac);
1059
1060 spin_unlock_bh(&adapter->mac_vlan_list_lock);
1061
1062 /* schedule the watchdog task to immediately process the request */
1063 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
1064 return 0;
1065 }
1066
1067 /**
1068 * iavf_is_mac_set_handled - wait for a response to set MAC from PF
1069 * @netdev: network interface device structure
1070 * @macaddr: MAC address to set
1071 *
1072 * Returns true on success, false on failure
1073 */
iavf_is_mac_set_handled(struct net_device * netdev,const u8 * macaddr)1074 static bool iavf_is_mac_set_handled(struct net_device *netdev,
1075 const u8 *macaddr)
1076 {
1077 struct iavf_adapter *adapter = netdev_priv(netdev);
1078 struct iavf_mac_filter *f;
1079 bool ret = false;
1080
1081 spin_lock_bh(&adapter->mac_vlan_list_lock);
1082
1083 f = iavf_find_filter(adapter, macaddr);
1084
1085 if (!f || (!f->add && f->add_handled))
1086 ret = true;
1087
1088 spin_unlock_bh(&adapter->mac_vlan_list_lock);
1089
1090 return ret;
1091 }
1092
1093 /**
1094 * iavf_set_mac - NDO callback to set port MAC address
1095 * @netdev: network interface device structure
1096 * @p: pointer to an address structure
1097 *
1098 * Returns 0 on success, negative on failure
1099 */
iavf_set_mac(struct net_device * netdev,void * p)1100 static int iavf_set_mac(struct net_device *netdev, void *p)
1101 {
1102 struct iavf_adapter *adapter = netdev_priv(netdev);
1103 struct sockaddr *addr = p;
1104 int ret;
1105
1106 if (!is_valid_ether_addr(addr->sa_data))
1107 return -EADDRNOTAVAIL;
1108
1109 ret = iavf_replace_primary_mac(adapter, addr->sa_data);
1110
1111 if (ret)
1112 return ret;
1113
1114 ret = wait_event_interruptible_timeout(adapter->vc_waitqueue,
1115 iavf_is_mac_set_handled(netdev, addr->sa_data),
1116 msecs_to_jiffies(2500));
1117
1118 /* If ret < 0 then it means wait was interrupted.
1119 * If ret == 0 then it means we got a timeout.
1120 * else it means we got response for set MAC from PF,
1121 * check if netdev MAC was updated to requested MAC,
1122 * if yes then set MAC succeeded otherwise it failed return -EACCES
1123 */
1124 if (ret < 0)
1125 return ret;
1126
1127 if (!ret)
1128 return -EAGAIN;
1129
1130 if (!ether_addr_equal(netdev->dev_addr, addr->sa_data))
1131 return -EACCES;
1132
1133 return 0;
1134 }
1135
1136 /**
1137 * iavf_addr_sync - Callback for dev_(mc|uc)_sync to add address
1138 * @netdev: the netdevice
1139 * @addr: address to add
1140 *
1141 * Called by __dev_(mc|uc)_sync when an address needs to be added. We call
1142 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1143 */
iavf_addr_sync(struct net_device * netdev,const u8 * addr)1144 static int iavf_addr_sync(struct net_device *netdev, const u8 *addr)
1145 {
1146 struct iavf_adapter *adapter = netdev_priv(netdev);
1147
1148 if (iavf_add_filter(adapter, addr))
1149 return 0;
1150 else
1151 return -ENOMEM;
1152 }
1153
1154 /**
1155 * iavf_addr_unsync - Callback for dev_(mc|uc)_sync to remove address
1156 * @netdev: the netdevice
1157 * @addr: address to add
1158 *
1159 * Called by __dev_(mc|uc)_sync when an address needs to be removed. We call
1160 * __dev_(uc|mc)_sync from .set_rx_mode and guarantee to hold the hash lock.
1161 */
iavf_addr_unsync(struct net_device * netdev,const u8 * addr)1162 static int iavf_addr_unsync(struct net_device *netdev, const u8 *addr)
1163 {
1164 struct iavf_adapter *adapter = netdev_priv(netdev);
1165 struct iavf_mac_filter *f;
1166
1167 /* Under some circumstances, we might receive a request to delete
1168 * our own device address from our uc list. Because we store the
1169 * device address in the VSI's MAC/VLAN filter list, we need to ignore
1170 * such requests and not delete our device address from this list.
1171 */
1172 if (ether_addr_equal(addr, netdev->dev_addr))
1173 return 0;
1174
1175 f = iavf_find_filter(adapter, addr);
1176 if (f) {
1177 f->remove = true;
1178 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1179 }
1180 return 0;
1181 }
1182
1183 /**
1184 * iavf_promiscuous_mode_changed - check if promiscuous mode bits changed
1185 * @adapter: device specific adapter
1186 */
iavf_promiscuous_mode_changed(struct iavf_adapter * adapter)1187 bool iavf_promiscuous_mode_changed(struct iavf_adapter *adapter)
1188 {
1189 return (adapter->current_netdev_promisc_flags ^ adapter->netdev->flags) &
1190 (IFF_PROMISC | IFF_ALLMULTI);
1191 }
1192
1193 /**
1194 * iavf_set_rx_mode - NDO callback to set the netdev filters
1195 * @netdev: network interface device structure
1196 **/
iavf_set_rx_mode(struct net_device * netdev)1197 static void iavf_set_rx_mode(struct net_device *netdev)
1198 {
1199 struct iavf_adapter *adapter = netdev_priv(netdev);
1200
1201 spin_lock_bh(&adapter->mac_vlan_list_lock);
1202 __dev_uc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1203 __dev_mc_sync(netdev, iavf_addr_sync, iavf_addr_unsync);
1204 spin_unlock_bh(&adapter->mac_vlan_list_lock);
1205
1206 spin_lock_bh(&adapter->current_netdev_promisc_flags_lock);
1207 if (iavf_promiscuous_mode_changed(adapter))
1208 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE;
1209 spin_unlock_bh(&adapter->current_netdev_promisc_flags_lock);
1210 }
1211
1212 /**
1213 * iavf_napi_enable_all - enable NAPI on all queue vectors
1214 * @adapter: board private structure
1215 **/
iavf_napi_enable_all(struct iavf_adapter * adapter)1216 static void iavf_napi_enable_all(struct iavf_adapter *adapter)
1217 {
1218 int q_idx;
1219 struct iavf_q_vector *q_vector;
1220 int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1221
1222 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1223 struct napi_struct *napi;
1224
1225 q_vector = &adapter->q_vectors[q_idx];
1226 napi = &q_vector->napi;
1227 napi_enable(napi);
1228 }
1229 }
1230
1231 /**
1232 * iavf_napi_disable_all - disable NAPI on all queue vectors
1233 * @adapter: board private structure
1234 **/
iavf_napi_disable_all(struct iavf_adapter * adapter)1235 static void iavf_napi_disable_all(struct iavf_adapter *adapter)
1236 {
1237 int q_idx;
1238 struct iavf_q_vector *q_vector;
1239 int q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1240
1241 for (q_idx = 0; q_idx < q_vectors; q_idx++) {
1242 q_vector = &adapter->q_vectors[q_idx];
1243 napi_disable(&q_vector->napi);
1244 }
1245 }
1246
1247 /**
1248 * iavf_configure - set up transmit and receive data structures
1249 * @adapter: board private structure
1250 **/
iavf_configure(struct iavf_adapter * adapter)1251 static void iavf_configure(struct iavf_adapter *adapter)
1252 {
1253 struct net_device *netdev = adapter->netdev;
1254 int i;
1255
1256 iavf_set_rx_mode(netdev);
1257
1258 iavf_configure_tx(adapter);
1259 iavf_configure_rx(adapter);
1260 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_QUEUES;
1261
1262 for (i = 0; i < adapter->num_active_queues; i++) {
1263 struct iavf_ring *ring = &adapter->rx_rings[i];
1264
1265 iavf_alloc_rx_buffers(ring, IAVF_DESC_UNUSED(ring));
1266 }
1267 }
1268
1269 /**
1270 * iavf_up_complete - Finish the last steps of bringing up a connection
1271 * @adapter: board private structure
1272 *
1273 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1274 **/
iavf_up_complete(struct iavf_adapter * adapter)1275 static void iavf_up_complete(struct iavf_adapter *adapter)
1276 {
1277 iavf_change_state(adapter, __IAVF_RUNNING);
1278 clear_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1279
1280 iavf_napi_enable_all(adapter);
1281
1282 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_QUEUES;
1283 if (CLIENT_ENABLED(adapter))
1284 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_OPEN;
1285 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
1286 }
1287
1288 /**
1289 * iavf_clear_mac_vlan_filters - Remove mac and vlan filters not sent to PF
1290 * yet and mark other to be removed.
1291 * @adapter: board private structure
1292 **/
iavf_clear_mac_vlan_filters(struct iavf_adapter * adapter)1293 static void iavf_clear_mac_vlan_filters(struct iavf_adapter *adapter)
1294 {
1295 struct iavf_vlan_filter *vlf, *vlftmp;
1296 struct iavf_mac_filter *f, *ftmp;
1297
1298 spin_lock_bh(&adapter->mac_vlan_list_lock);
1299 /* clear the sync flag on all filters */
1300 __dev_uc_unsync(adapter->netdev, NULL);
1301 __dev_mc_unsync(adapter->netdev, NULL);
1302
1303 /* remove all MAC filters */
1304 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list,
1305 list) {
1306 if (f->add) {
1307 list_del(&f->list);
1308 kfree(f);
1309 } else {
1310 f->remove = true;
1311 }
1312 }
1313
1314 /* disable all VLAN filters */
1315 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
1316 list)
1317 vlf->state = IAVF_VLAN_DISABLE;
1318
1319 spin_unlock_bh(&adapter->mac_vlan_list_lock);
1320 }
1321
1322 /**
1323 * iavf_clear_cloud_filters - Remove cloud filters not sent to PF yet and
1324 * mark other to be removed.
1325 * @adapter: board private structure
1326 **/
iavf_clear_cloud_filters(struct iavf_adapter * adapter)1327 static void iavf_clear_cloud_filters(struct iavf_adapter *adapter)
1328 {
1329 struct iavf_cloud_filter *cf, *cftmp;
1330
1331 /* remove all cloud filters */
1332 spin_lock_bh(&adapter->cloud_filter_list_lock);
1333 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
1334 list) {
1335 if (cf->add) {
1336 list_del(&cf->list);
1337 kfree(cf);
1338 adapter->num_cloud_filters--;
1339 } else {
1340 cf->del = true;
1341 }
1342 }
1343 spin_unlock_bh(&adapter->cloud_filter_list_lock);
1344 }
1345
1346 /**
1347 * iavf_clear_fdir_filters - Remove fdir filters not sent to PF yet and mark
1348 * other to be removed.
1349 * @adapter: board private structure
1350 **/
iavf_clear_fdir_filters(struct iavf_adapter * adapter)1351 static void iavf_clear_fdir_filters(struct iavf_adapter *adapter)
1352 {
1353 struct iavf_fdir_fltr *fdir;
1354
1355 /* remove all Flow Director filters */
1356 spin_lock_bh(&adapter->fdir_fltr_lock);
1357 list_for_each_entry(fdir, &adapter->fdir_list_head, list) {
1358 if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST) {
1359 /* Cancel a request, keep filter as inactive */
1360 fdir->state = IAVF_FDIR_FLTR_INACTIVE;
1361 } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING ||
1362 fdir->state == IAVF_FDIR_FLTR_ACTIVE) {
1363 /* Disable filters which are active or have a pending
1364 * request to PF to be added
1365 */
1366 fdir->state = IAVF_FDIR_FLTR_DIS_REQUEST;
1367 }
1368 }
1369 spin_unlock_bh(&adapter->fdir_fltr_lock);
1370 }
1371
1372 /**
1373 * iavf_clear_adv_rss_conf - Remove adv rss conf not sent to PF yet and mark
1374 * other to be removed.
1375 * @adapter: board private structure
1376 **/
iavf_clear_adv_rss_conf(struct iavf_adapter * adapter)1377 static void iavf_clear_adv_rss_conf(struct iavf_adapter *adapter)
1378 {
1379 struct iavf_adv_rss *rss, *rsstmp;
1380
1381 /* remove all advance RSS configuration */
1382 spin_lock_bh(&adapter->adv_rss_lock);
1383 list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
1384 list) {
1385 if (rss->state == IAVF_ADV_RSS_ADD_REQUEST) {
1386 list_del(&rss->list);
1387 kfree(rss);
1388 } else {
1389 rss->state = IAVF_ADV_RSS_DEL_REQUEST;
1390 }
1391 }
1392 spin_unlock_bh(&adapter->adv_rss_lock);
1393 }
1394
1395 /**
1396 * iavf_down - Shutdown the connection processing
1397 * @adapter: board private structure
1398 *
1399 * Expects to be called while holding the __IAVF_IN_CRITICAL_TASK bit lock.
1400 **/
iavf_down(struct iavf_adapter * adapter)1401 void iavf_down(struct iavf_adapter *adapter)
1402 {
1403 struct net_device *netdev = adapter->netdev;
1404
1405 if (adapter->state <= __IAVF_DOWN_PENDING)
1406 return;
1407
1408 netif_carrier_off(netdev);
1409 netif_tx_disable(netdev);
1410 adapter->link_up = false;
1411 iavf_napi_disable_all(adapter);
1412 iavf_irq_disable(adapter);
1413
1414 iavf_clear_mac_vlan_filters(adapter);
1415 iavf_clear_cloud_filters(adapter);
1416 iavf_clear_fdir_filters(adapter);
1417 iavf_clear_adv_rss_conf(adapter);
1418
1419 if (!(adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) &&
1420 !(test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))) {
1421 /* cancel any current operation */
1422 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
1423 /* Schedule operations to close down the HW. Don't wait
1424 * here for this to complete. The watchdog is still running
1425 * and it will take care of this.
1426 */
1427 if (!list_empty(&adapter->mac_filter_list))
1428 adapter->aq_required |= IAVF_FLAG_AQ_DEL_MAC_FILTER;
1429 if (!list_empty(&adapter->vlan_filter_list))
1430 adapter->aq_required |= IAVF_FLAG_AQ_DEL_VLAN_FILTER;
1431 if (!list_empty(&adapter->cloud_filter_list))
1432 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
1433 if (!list_empty(&adapter->fdir_list_head))
1434 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
1435 if (!list_empty(&adapter->adv_rss_list_head))
1436 adapter->aq_required |= IAVF_FLAG_AQ_DEL_ADV_RSS_CFG;
1437 }
1438
1439 adapter->aq_required |= IAVF_FLAG_AQ_DISABLE_QUEUES;
1440 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
1441 }
1442
1443 /**
1444 * iavf_acquire_msix_vectors - Setup the MSIX capability
1445 * @adapter: board private structure
1446 * @vectors: number of vectors to request
1447 *
1448 * Work with the OS to set up the MSIX vectors needed.
1449 *
1450 * Returns 0 on success, negative on failure
1451 **/
1452 static int
iavf_acquire_msix_vectors(struct iavf_adapter * adapter,int vectors)1453 iavf_acquire_msix_vectors(struct iavf_adapter *adapter, int vectors)
1454 {
1455 int err, vector_threshold;
1456
1457 /* We'll want at least 3 (vector_threshold):
1458 * 0) Other (Admin Queue and link, mostly)
1459 * 1) TxQ[0] Cleanup
1460 * 2) RxQ[0] Cleanup
1461 */
1462 vector_threshold = MIN_MSIX_COUNT;
1463
1464 /* The more we get, the more we will assign to Tx/Rx Cleanup
1465 * for the separate queues...where Rx Cleanup >= Tx Cleanup.
1466 * Right now, we simply care about how many we'll get; we'll
1467 * set them up later while requesting irq's.
1468 */
1469 err = pci_enable_msix_range(adapter->pdev, adapter->msix_entries,
1470 vector_threshold, vectors);
1471 if (err < 0) {
1472 dev_err(&adapter->pdev->dev, "Unable to allocate MSI-X interrupts\n");
1473 kfree(adapter->msix_entries);
1474 adapter->msix_entries = NULL;
1475 return err;
1476 }
1477
1478 /* Adjust for only the vectors we'll use, which is minimum
1479 * of max_msix_q_vectors + NONQ_VECS, or the number of
1480 * vectors we were allocated.
1481 */
1482 adapter->num_msix_vectors = err;
1483 return 0;
1484 }
1485
1486 /**
1487 * iavf_free_queues - Free memory for all rings
1488 * @adapter: board private structure to initialize
1489 *
1490 * Free all of the memory associated with queue pairs.
1491 **/
iavf_free_queues(struct iavf_adapter * adapter)1492 static void iavf_free_queues(struct iavf_adapter *adapter)
1493 {
1494 if (!adapter->vsi_res)
1495 return;
1496 adapter->num_active_queues = 0;
1497 kfree(adapter->tx_rings);
1498 adapter->tx_rings = NULL;
1499 kfree(adapter->rx_rings);
1500 adapter->rx_rings = NULL;
1501 }
1502
1503 /**
1504 * iavf_set_queue_vlan_tag_loc - set location for VLAN tag offload
1505 * @adapter: board private structure
1506 *
1507 * Based on negotiated capabilities, the VLAN tag needs to be inserted and/or
1508 * stripped in certain descriptor fields. Instead of checking the offload
1509 * capability bits in the hot path, cache the location the ring specific
1510 * flags.
1511 */
iavf_set_queue_vlan_tag_loc(struct iavf_adapter * adapter)1512 void iavf_set_queue_vlan_tag_loc(struct iavf_adapter *adapter)
1513 {
1514 int i;
1515
1516 for (i = 0; i < adapter->num_active_queues; i++) {
1517 struct iavf_ring *tx_ring = &adapter->tx_rings[i];
1518 struct iavf_ring *rx_ring = &adapter->rx_rings[i];
1519
1520 /* prevent multiple L2TAG bits being set after VFR */
1521 tx_ring->flags &=
1522 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1523 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2);
1524 rx_ring->flags &=
1525 ~(IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1 |
1526 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2);
1527
1528 if (VLAN_ALLOWED(adapter)) {
1529 tx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1530 rx_ring->flags |= IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1531 } else if (VLAN_V2_ALLOWED(adapter)) {
1532 struct virtchnl_vlan_supported_caps *stripping_support;
1533 struct virtchnl_vlan_supported_caps *insertion_support;
1534
1535 stripping_support =
1536 &adapter->vlan_v2_caps.offloads.stripping_support;
1537 insertion_support =
1538 &adapter->vlan_v2_caps.offloads.insertion_support;
1539
1540 if (stripping_support->outer) {
1541 if (stripping_support->outer &
1542 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1543 rx_ring->flags |=
1544 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1545 else if (stripping_support->outer &
1546 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1547 rx_ring->flags |=
1548 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1549 } else if (stripping_support->inner) {
1550 if (stripping_support->inner &
1551 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1552 rx_ring->flags |=
1553 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1554 else if (stripping_support->inner &
1555 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2_2)
1556 rx_ring->flags |=
1557 IAVF_RXR_FLAGS_VLAN_TAG_LOC_L2TAG2_2;
1558 }
1559
1560 if (insertion_support->outer) {
1561 if (insertion_support->outer &
1562 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1563 tx_ring->flags |=
1564 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1565 else if (insertion_support->outer &
1566 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1567 tx_ring->flags |=
1568 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1569 } else if (insertion_support->inner) {
1570 if (insertion_support->inner &
1571 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG1)
1572 tx_ring->flags |=
1573 IAVF_TXRX_FLAGS_VLAN_TAG_LOC_L2TAG1;
1574 else if (insertion_support->inner &
1575 VIRTCHNL_VLAN_TAG_LOCATION_L2TAG2)
1576 tx_ring->flags |=
1577 IAVF_TXR_FLAGS_VLAN_TAG_LOC_L2TAG2;
1578 }
1579 }
1580 }
1581 }
1582
1583 /**
1584 * iavf_alloc_queues - Allocate memory for all rings
1585 * @adapter: board private structure to initialize
1586 *
1587 * We allocate one ring per queue at run-time since we don't know the
1588 * number of queues at compile-time. The polling_netdev array is
1589 * intended for Multiqueue, but should work fine with a single queue.
1590 **/
iavf_alloc_queues(struct iavf_adapter * adapter)1591 static int iavf_alloc_queues(struct iavf_adapter *adapter)
1592 {
1593 int i, num_active_queues;
1594
1595 /* If we're in reset reallocating queues we don't actually know yet for
1596 * certain the PF gave us the number of queues we asked for but we'll
1597 * assume it did. Once basic reset is finished we'll confirm once we
1598 * start negotiating config with PF.
1599 */
1600 if (adapter->num_req_queues)
1601 num_active_queues = adapter->num_req_queues;
1602 else if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1603 adapter->num_tc)
1604 num_active_queues = adapter->ch_config.total_qps;
1605 else
1606 num_active_queues = min_t(int,
1607 adapter->vsi_res->num_queue_pairs,
1608 (int)(num_online_cpus()));
1609
1610
1611 adapter->tx_rings = kcalloc(num_active_queues,
1612 sizeof(struct iavf_ring), GFP_KERNEL);
1613 if (!adapter->tx_rings)
1614 goto err_out;
1615 adapter->rx_rings = kcalloc(num_active_queues,
1616 sizeof(struct iavf_ring), GFP_KERNEL);
1617 if (!adapter->rx_rings)
1618 goto err_out;
1619
1620 for (i = 0; i < num_active_queues; i++) {
1621 struct iavf_ring *tx_ring;
1622 struct iavf_ring *rx_ring;
1623
1624 tx_ring = &adapter->tx_rings[i];
1625
1626 tx_ring->queue_index = i;
1627 tx_ring->netdev = adapter->netdev;
1628 tx_ring->dev = &adapter->pdev->dev;
1629 tx_ring->count = adapter->tx_desc_count;
1630 tx_ring->itr_setting = IAVF_ITR_TX_DEF;
1631 if (adapter->flags & IAVF_FLAG_WB_ON_ITR_CAPABLE)
1632 tx_ring->flags |= IAVF_TXR_FLAGS_WB_ON_ITR;
1633
1634 rx_ring = &adapter->rx_rings[i];
1635 rx_ring->queue_index = i;
1636 rx_ring->netdev = adapter->netdev;
1637 rx_ring->dev = &adapter->pdev->dev;
1638 rx_ring->count = adapter->rx_desc_count;
1639 rx_ring->itr_setting = IAVF_ITR_RX_DEF;
1640 }
1641
1642 adapter->num_active_queues = num_active_queues;
1643
1644 iavf_set_queue_vlan_tag_loc(adapter);
1645
1646 return 0;
1647
1648 err_out:
1649 iavf_free_queues(adapter);
1650 return -ENOMEM;
1651 }
1652
1653 /**
1654 * iavf_set_interrupt_capability - set MSI-X or FAIL if not supported
1655 * @adapter: board private structure to initialize
1656 *
1657 * Attempt to configure the interrupts using the best available
1658 * capabilities of the hardware and the kernel.
1659 **/
iavf_set_interrupt_capability(struct iavf_adapter * adapter)1660 static int iavf_set_interrupt_capability(struct iavf_adapter *adapter)
1661 {
1662 int vector, v_budget;
1663 int pairs = 0;
1664 int err = 0;
1665
1666 if (!adapter->vsi_res) {
1667 err = -EIO;
1668 goto out;
1669 }
1670 pairs = adapter->num_active_queues;
1671
1672 /* It's easy to be greedy for MSI-X vectors, but it really doesn't do
1673 * us much good if we have more vectors than CPUs. However, we already
1674 * limit the total number of queues by the number of CPUs so we do not
1675 * need any further limiting here.
1676 */
1677 v_budget = min_t(int, pairs + NONQ_VECS,
1678 (int)adapter->vf_res->max_vectors);
1679
1680 adapter->msix_entries = kcalloc(v_budget,
1681 sizeof(struct msix_entry), GFP_KERNEL);
1682 if (!adapter->msix_entries) {
1683 err = -ENOMEM;
1684 goto out;
1685 }
1686
1687 for (vector = 0; vector < v_budget; vector++)
1688 adapter->msix_entries[vector].entry = vector;
1689
1690 err = iavf_acquire_msix_vectors(adapter, v_budget);
1691 if (!err)
1692 iavf_schedule_finish_config(adapter);
1693
1694 out:
1695 return err;
1696 }
1697
1698 /**
1699 * iavf_config_rss_aq - Configure RSS keys and lut by using AQ commands
1700 * @adapter: board private structure
1701 *
1702 * Return 0 on success, negative on failure
1703 **/
iavf_config_rss_aq(struct iavf_adapter * adapter)1704 static int iavf_config_rss_aq(struct iavf_adapter *adapter)
1705 {
1706 struct iavf_aqc_get_set_rss_key_data *rss_key =
1707 (struct iavf_aqc_get_set_rss_key_data *)adapter->rss_key;
1708 struct iavf_hw *hw = &adapter->hw;
1709 enum iavf_status status;
1710
1711 if (adapter->current_op != VIRTCHNL_OP_UNKNOWN) {
1712 /* bail because we already have a command pending */
1713 dev_err(&adapter->pdev->dev, "Cannot configure RSS, command %d pending\n",
1714 adapter->current_op);
1715 return -EBUSY;
1716 }
1717
1718 status = iavf_aq_set_rss_key(hw, adapter->vsi.id, rss_key);
1719 if (status) {
1720 dev_err(&adapter->pdev->dev, "Cannot set RSS key, err %s aq_err %s\n",
1721 iavf_stat_str(hw, status),
1722 iavf_aq_str(hw, hw->aq.asq_last_status));
1723 return iavf_status_to_errno(status);
1724
1725 }
1726
1727 status = iavf_aq_set_rss_lut(hw, adapter->vsi.id, false,
1728 adapter->rss_lut, adapter->rss_lut_size);
1729 if (status) {
1730 dev_err(&adapter->pdev->dev, "Cannot set RSS lut, err %s aq_err %s\n",
1731 iavf_stat_str(hw, status),
1732 iavf_aq_str(hw, hw->aq.asq_last_status));
1733 return iavf_status_to_errno(status);
1734 }
1735
1736 return 0;
1737
1738 }
1739
1740 /**
1741 * iavf_config_rss_reg - Configure RSS keys and lut by writing registers
1742 * @adapter: board private structure
1743 *
1744 * Returns 0 on success, negative on failure
1745 **/
iavf_config_rss_reg(struct iavf_adapter * adapter)1746 static int iavf_config_rss_reg(struct iavf_adapter *adapter)
1747 {
1748 struct iavf_hw *hw = &adapter->hw;
1749 u32 *dw;
1750 u16 i;
1751
1752 dw = (u32 *)adapter->rss_key;
1753 for (i = 0; i <= adapter->rss_key_size / 4; i++)
1754 wr32(hw, IAVF_VFQF_HKEY(i), dw[i]);
1755
1756 dw = (u32 *)adapter->rss_lut;
1757 for (i = 0; i <= adapter->rss_lut_size / 4; i++)
1758 wr32(hw, IAVF_VFQF_HLUT(i), dw[i]);
1759
1760 iavf_flush(hw);
1761
1762 return 0;
1763 }
1764
1765 /**
1766 * iavf_config_rss - Configure RSS keys and lut
1767 * @adapter: board private structure
1768 *
1769 * Returns 0 on success, negative on failure
1770 **/
iavf_config_rss(struct iavf_adapter * adapter)1771 int iavf_config_rss(struct iavf_adapter *adapter)
1772 {
1773
1774 if (RSS_PF(adapter)) {
1775 adapter->aq_required |= IAVF_FLAG_AQ_SET_RSS_LUT |
1776 IAVF_FLAG_AQ_SET_RSS_KEY;
1777 return 0;
1778 } else if (RSS_AQ(adapter)) {
1779 return iavf_config_rss_aq(adapter);
1780 } else {
1781 return iavf_config_rss_reg(adapter);
1782 }
1783 }
1784
1785 /**
1786 * iavf_fill_rss_lut - Fill the lut with default values
1787 * @adapter: board private structure
1788 **/
iavf_fill_rss_lut(struct iavf_adapter * adapter)1789 static void iavf_fill_rss_lut(struct iavf_adapter *adapter)
1790 {
1791 u16 i;
1792
1793 for (i = 0; i < adapter->rss_lut_size; i++)
1794 adapter->rss_lut[i] = i % adapter->num_active_queues;
1795 }
1796
1797 /**
1798 * iavf_init_rss - Prepare for RSS
1799 * @adapter: board private structure
1800 *
1801 * Return 0 on success, negative on failure
1802 **/
iavf_init_rss(struct iavf_adapter * adapter)1803 static int iavf_init_rss(struct iavf_adapter *adapter)
1804 {
1805 struct iavf_hw *hw = &adapter->hw;
1806
1807 if (!RSS_PF(adapter)) {
1808 /* Enable PCTYPES for RSS, TCP/UDP with IPv4/IPv6 */
1809 if (adapter->vf_res->vf_cap_flags &
1810 VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2)
1811 adapter->hena = IAVF_DEFAULT_RSS_HENA_EXPANDED;
1812 else
1813 adapter->hena = IAVF_DEFAULT_RSS_HENA;
1814
1815 wr32(hw, IAVF_VFQF_HENA(0), (u32)adapter->hena);
1816 wr32(hw, IAVF_VFQF_HENA(1), (u32)(adapter->hena >> 32));
1817 }
1818
1819 iavf_fill_rss_lut(adapter);
1820 netdev_rss_key_fill((void *)adapter->rss_key, adapter->rss_key_size);
1821
1822 return iavf_config_rss(adapter);
1823 }
1824
1825 /**
1826 * iavf_alloc_q_vectors - Allocate memory for interrupt vectors
1827 * @adapter: board private structure to initialize
1828 *
1829 * We allocate one q_vector per queue interrupt. If allocation fails we
1830 * return -ENOMEM.
1831 **/
iavf_alloc_q_vectors(struct iavf_adapter * adapter)1832 static int iavf_alloc_q_vectors(struct iavf_adapter *adapter)
1833 {
1834 int q_idx = 0, num_q_vectors;
1835 struct iavf_q_vector *q_vector;
1836
1837 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1838 adapter->q_vectors = kcalloc(num_q_vectors, sizeof(*q_vector),
1839 GFP_KERNEL);
1840 if (!adapter->q_vectors)
1841 return -ENOMEM;
1842
1843 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1844 q_vector = &adapter->q_vectors[q_idx];
1845 q_vector->adapter = adapter;
1846 q_vector->vsi = &adapter->vsi;
1847 q_vector->v_idx = q_idx;
1848 q_vector->reg_idx = q_idx;
1849 cpumask_copy(&q_vector->affinity_mask, cpu_possible_mask);
1850 netif_napi_add(adapter->netdev, &q_vector->napi,
1851 iavf_napi_poll);
1852 }
1853
1854 return 0;
1855 }
1856
1857 /**
1858 * iavf_free_q_vectors - Free memory allocated for interrupt vectors
1859 * @adapter: board private structure to initialize
1860 *
1861 * This function frees the memory allocated to the q_vectors. In addition if
1862 * NAPI is enabled it will delete any references to the NAPI struct prior
1863 * to freeing the q_vector.
1864 **/
iavf_free_q_vectors(struct iavf_adapter * adapter)1865 static void iavf_free_q_vectors(struct iavf_adapter *adapter)
1866 {
1867 int q_idx, num_q_vectors;
1868
1869 if (!adapter->q_vectors)
1870 return;
1871
1872 num_q_vectors = adapter->num_msix_vectors - NONQ_VECS;
1873
1874 for (q_idx = 0; q_idx < num_q_vectors; q_idx++) {
1875 struct iavf_q_vector *q_vector = &adapter->q_vectors[q_idx];
1876
1877 netif_napi_del(&q_vector->napi);
1878 }
1879 kfree(adapter->q_vectors);
1880 adapter->q_vectors = NULL;
1881 }
1882
1883 /**
1884 * iavf_reset_interrupt_capability - Reset MSIX setup
1885 * @adapter: board private structure
1886 *
1887 **/
iavf_reset_interrupt_capability(struct iavf_adapter * adapter)1888 static void iavf_reset_interrupt_capability(struct iavf_adapter *adapter)
1889 {
1890 if (!adapter->msix_entries)
1891 return;
1892
1893 pci_disable_msix(adapter->pdev);
1894 kfree(adapter->msix_entries);
1895 adapter->msix_entries = NULL;
1896 }
1897
1898 /**
1899 * iavf_init_interrupt_scheme - Determine if MSIX is supported and init
1900 * @adapter: board private structure to initialize
1901 *
1902 **/
iavf_init_interrupt_scheme(struct iavf_adapter * adapter)1903 static int iavf_init_interrupt_scheme(struct iavf_adapter *adapter)
1904 {
1905 int err;
1906
1907 err = iavf_alloc_queues(adapter);
1908 if (err) {
1909 dev_err(&adapter->pdev->dev,
1910 "Unable to allocate memory for queues\n");
1911 goto err_alloc_queues;
1912 }
1913
1914 err = iavf_set_interrupt_capability(adapter);
1915 if (err) {
1916 dev_err(&adapter->pdev->dev,
1917 "Unable to setup interrupt capabilities\n");
1918 goto err_set_interrupt;
1919 }
1920
1921 err = iavf_alloc_q_vectors(adapter);
1922 if (err) {
1923 dev_err(&adapter->pdev->dev,
1924 "Unable to allocate memory for queue vectors\n");
1925 goto err_alloc_q_vectors;
1926 }
1927
1928 /* If we've made it so far while ADq flag being ON, then we haven't
1929 * bailed out anywhere in middle. And ADq isn't just enabled but actual
1930 * resources have been allocated in the reset path.
1931 * Now we can truly claim that ADq is enabled.
1932 */
1933 if ((adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
1934 adapter->num_tc)
1935 dev_info(&adapter->pdev->dev, "ADq Enabled, %u TCs created",
1936 adapter->num_tc);
1937
1938 dev_info(&adapter->pdev->dev, "Multiqueue %s: Queue pair count = %u",
1939 (adapter->num_active_queues > 1) ? "Enabled" : "Disabled",
1940 adapter->num_active_queues);
1941
1942 return 0;
1943 err_alloc_q_vectors:
1944 iavf_reset_interrupt_capability(adapter);
1945 err_set_interrupt:
1946 iavf_free_queues(adapter);
1947 err_alloc_queues:
1948 return err;
1949 }
1950
1951 /**
1952 * iavf_free_rss - Free memory used by RSS structs
1953 * @adapter: board private structure
1954 **/
iavf_free_rss(struct iavf_adapter * adapter)1955 static void iavf_free_rss(struct iavf_adapter *adapter)
1956 {
1957 kfree(adapter->rss_key);
1958 adapter->rss_key = NULL;
1959
1960 kfree(adapter->rss_lut);
1961 adapter->rss_lut = NULL;
1962 }
1963
1964 /**
1965 * iavf_reinit_interrupt_scheme - Reallocate queues and vectors
1966 * @adapter: board private structure
1967 * @running: true if adapter->state == __IAVF_RUNNING
1968 *
1969 * Returns 0 on success, negative on failure
1970 **/
iavf_reinit_interrupt_scheme(struct iavf_adapter * adapter,bool running)1971 static int iavf_reinit_interrupt_scheme(struct iavf_adapter *adapter, bool running)
1972 {
1973 struct net_device *netdev = adapter->netdev;
1974 int err;
1975
1976 if (running)
1977 iavf_free_traffic_irqs(adapter);
1978 iavf_free_misc_irq(adapter);
1979 iavf_reset_interrupt_capability(adapter);
1980 iavf_free_q_vectors(adapter);
1981 iavf_free_queues(adapter);
1982
1983 err = iavf_init_interrupt_scheme(adapter);
1984 if (err)
1985 goto err;
1986
1987 netif_tx_stop_all_queues(netdev);
1988
1989 err = iavf_request_misc_irq(adapter);
1990 if (err)
1991 goto err;
1992
1993 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
1994
1995 iavf_map_rings_to_vectors(adapter);
1996 err:
1997 return err;
1998 }
1999
2000 /**
2001 * iavf_finish_config - do all netdev work that needs RTNL
2002 * @work: our work_struct
2003 *
2004 * Do work that needs both RTNL and crit_lock.
2005 **/
iavf_finish_config(struct work_struct * work)2006 static void iavf_finish_config(struct work_struct *work)
2007 {
2008 struct iavf_adapter *adapter;
2009 int pairs, err;
2010
2011 adapter = container_of(work, struct iavf_adapter, finish_config);
2012
2013 /* Always take RTNL first to prevent circular lock dependency */
2014 rtnl_lock();
2015 mutex_lock(&adapter->crit_lock);
2016
2017 if ((adapter->flags & IAVF_FLAG_SETUP_NETDEV_FEATURES) &&
2018 adapter->netdev_registered &&
2019 !test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section)) {
2020 netdev_update_features(adapter->netdev);
2021 adapter->flags &= ~IAVF_FLAG_SETUP_NETDEV_FEATURES;
2022 }
2023
2024 switch (adapter->state) {
2025 case __IAVF_DOWN:
2026 if (!adapter->netdev_registered) {
2027 err = register_netdevice(adapter->netdev);
2028 if (err) {
2029 dev_err(&adapter->pdev->dev, "Unable to register netdev (%d)\n",
2030 err);
2031
2032 /* go back and try again.*/
2033 iavf_free_rss(adapter);
2034 iavf_free_misc_irq(adapter);
2035 iavf_reset_interrupt_capability(adapter);
2036 iavf_change_state(adapter,
2037 __IAVF_INIT_CONFIG_ADAPTER);
2038 goto out;
2039 }
2040 adapter->netdev_registered = true;
2041 }
2042
2043 /* Set the real number of queues when reset occurs while
2044 * state == __IAVF_DOWN
2045 */
2046 fallthrough;
2047 case __IAVF_RUNNING:
2048 pairs = adapter->num_active_queues;
2049 netif_set_real_num_rx_queues(adapter->netdev, pairs);
2050 netif_set_real_num_tx_queues(adapter->netdev, pairs);
2051 break;
2052
2053 default:
2054 break;
2055 }
2056
2057 out:
2058 mutex_unlock(&adapter->crit_lock);
2059 rtnl_unlock();
2060 }
2061
2062 /**
2063 * iavf_schedule_finish_config - Set the flags and schedule a reset event
2064 * @adapter: board private structure
2065 **/
iavf_schedule_finish_config(struct iavf_adapter * adapter)2066 void iavf_schedule_finish_config(struct iavf_adapter *adapter)
2067 {
2068 if (!test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
2069 queue_work(adapter->wq, &adapter->finish_config);
2070 }
2071
2072 /**
2073 * iavf_process_aq_command - process aq_required flags
2074 * and sends aq command
2075 * @adapter: pointer to iavf adapter structure
2076 *
2077 * Returns 0 on success
2078 * Returns error code if no command was sent
2079 * or error code if the command failed.
2080 **/
iavf_process_aq_command(struct iavf_adapter * adapter)2081 static int iavf_process_aq_command(struct iavf_adapter *adapter)
2082 {
2083 if (adapter->aq_required & IAVF_FLAG_AQ_GET_CONFIG)
2084 return iavf_send_vf_config_msg(adapter);
2085 if (adapter->aq_required & IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS)
2086 return iavf_send_vf_offload_vlan_v2_msg(adapter);
2087 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_QUEUES) {
2088 iavf_disable_queues(adapter);
2089 return 0;
2090 }
2091
2092 if (adapter->aq_required & IAVF_FLAG_AQ_MAP_VECTORS) {
2093 iavf_map_queues(adapter);
2094 return 0;
2095 }
2096
2097 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_MAC_FILTER) {
2098 iavf_add_ether_addrs(adapter);
2099 return 0;
2100 }
2101
2102 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_VLAN_FILTER) {
2103 iavf_add_vlans(adapter);
2104 return 0;
2105 }
2106
2107 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_MAC_FILTER) {
2108 iavf_del_ether_addrs(adapter);
2109 return 0;
2110 }
2111
2112 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_VLAN_FILTER) {
2113 iavf_del_vlans(adapter);
2114 return 0;
2115 }
2116
2117 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING) {
2118 iavf_enable_vlan_stripping(adapter);
2119 return 0;
2120 }
2121
2122 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING) {
2123 iavf_disable_vlan_stripping(adapter);
2124 return 0;
2125 }
2126
2127 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_QUEUES) {
2128 iavf_configure_queues(adapter);
2129 return 0;
2130 }
2131
2132 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_QUEUES) {
2133 iavf_enable_queues(adapter);
2134 return 0;
2135 }
2136
2137 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_RSS) {
2138 /* This message goes straight to the firmware, not the
2139 * PF, so we don't have to set current_op as we will
2140 * not get a response through the ARQ.
2141 */
2142 adapter->aq_required &= ~IAVF_FLAG_AQ_CONFIGURE_RSS;
2143 return 0;
2144 }
2145 if (adapter->aq_required & IAVF_FLAG_AQ_GET_HENA) {
2146 iavf_get_hena(adapter);
2147 return 0;
2148 }
2149 if (adapter->aq_required & IAVF_FLAG_AQ_SET_HENA) {
2150 iavf_set_hena(adapter);
2151 return 0;
2152 }
2153 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_KEY) {
2154 iavf_set_rss_key(adapter);
2155 return 0;
2156 }
2157 if (adapter->aq_required & IAVF_FLAG_AQ_SET_RSS_LUT) {
2158 iavf_set_rss_lut(adapter);
2159 return 0;
2160 }
2161
2162 if (adapter->aq_required & IAVF_FLAG_AQ_CONFIGURE_PROMISC_MODE) {
2163 iavf_set_promiscuous(adapter);
2164 return 0;
2165 }
2166
2167 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CHANNELS) {
2168 iavf_enable_channels(adapter);
2169 return 0;
2170 }
2171
2172 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CHANNELS) {
2173 iavf_disable_channels(adapter);
2174 return 0;
2175 }
2176 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2177 iavf_add_cloud_filter(adapter);
2178 return 0;
2179 }
2180
2181 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2182 iavf_del_cloud_filter(adapter);
2183 return 0;
2184 }
2185 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_CLOUD_FILTER) {
2186 iavf_del_cloud_filter(adapter);
2187 return 0;
2188 }
2189 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_CLOUD_FILTER) {
2190 iavf_add_cloud_filter(adapter);
2191 return 0;
2192 }
2193 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_FDIR_FILTER) {
2194 iavf_add_fdir_filter(adapter);
2195 return IAVF_SUCCESS;
2196 }
2197 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_FDIR_FILTER) {
2198 iavf_del_fdir_filter(adapter);
2199 return IAVF_SUCCESS;
2200 }
2201 if (adapter->aq_required & IAVF_FLAG_AQ_ADD_ADV_RSS_CFG) {
2202 iavf_add_adv_rss_cfg(adapter);
2203 return 0;
2204 }
2205 if (adapter->aq_required & IAVF_FLAG_AQ_DEL_ADV_RSS_CFG) {
2206 iavf_del_adv_rss_cfg(adapter);
2207 return 0;
2208 }
2209 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING) {
2210 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2211 return 0;
2212 }
2213 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING) {
2214 iavf_disable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2215 return 0;
2216 }
2217 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING) {
2218 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021Q);
2219 return 0;
2220 }
2221 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING) {
2222 iavf_enable_vlan_stripping_v2(adapter, ETH_P_8021AD);
2223 return 0;
2224 }
2225 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION) {
2226 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2227 return 0;
2228 }
2229 if (adapter->aq_required & IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION) {
2230 iavf_disable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2231 return 0;
2232 }
2233 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION) {
2234 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021Q);
2235 return 0;
2236 }
2237 if (adapter->aq_required & IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION) {
2238 iavf_enable_vlan_insertion_v2(adapter, ETH_P_8021AD);
2239 return 0;
2240 }
2241
2242 if (adapter->aq_required & IAVF_FLAG_AQ_REQUEST_STATS) {
2243 iavf_request_stats(adapter);
2244 return 0;
2245 }
2246
2247 return -EAGAIN;
2248 }
2249
2250 /**
2251 * iavf_set_vlan_offload_features - set VLAN offload configuration
2252 * @adapter: board private structure
2253 * @prev_features: previous features used for comparison
2254 * @features: updated features used for configuration
2255 *
2256 * Set the aq_required bit(s) based on the requested features passed in to
2257 * configure VLAN stripping and/or VLAN insertion if supported. Also, schedule
2258 * the watchdog if any changes are requested to expedite the request via
2259 * virtchnl.
2260 **/
2261 static void
iavf_set_vlan_offload_features(struct iavf_adapter * adapter,netdev_features_t prev_features,netdev_features_t features)2262 iavf_set_vlan_offload_features(struct iavf_adapter *adapter,
2263 netdev_features_t prev_features,
2264 netdev_features_t features)
2265 {
2266 bool enable_stripping = true, enable_insertion = true;
2267 u16 vlan_ethertype = 0;
2268 u64 aq_required = 0;
2269
2270 /* keep cases separate because one ethertype for offloads can be
2271 * disabled at the same time as another is disabled, so check for an
2272 * enabled ethertype first, then check for disabled. Default to
2273 * ETH_P_8021Q so an ethertype is specified if disabling insertion and
2274 * stripping.
2275 */
2276 if (features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2277 vlan_ethertype = ETH_P_8021AD;
2278 else if (features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2279 vlan_ethertype = ETH_P_8021Q;
2280 else if (prev_features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX))
2281 vlan_ethertype = ETH_P_8021AD;
2282 else if (prev_features & (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX))
2283 vlan_ethertype = ETH_P_8021Q;
2284 else
2285 vlan_ethertype = ETH_P_8021Q;
2286
2287 if (!(features & (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_CTAG_RX)))
2288 enable_stripping = false;
2289 if (!(features & (NETIF_F_HW_VLAN_STAG_TX | NETIF_F_HW_VLAN_CTAG_TX)))
2290 enable_insertion = false;
2291
2292 if (VLAN_ALLOWED(adapter)) {
2293 /* VIRTCHNL_VF_OFFLOAD_VLAN only has support for toggling VLAN
2294 * stripping via virtchnl. VLAN insertion can be toggled on the
2295 * netdev, but it doesn't require a virtchnl message
2296 */
2297 if (enable_stripping)
2298 aq_required |= IAVF_FLAG_AQ_ENABLE_VLAN_STRIPPING;
2299 else
2300 aq_required |= IAVF_FLAG_AQ_DISABLE_VLAN_STRIPPING;
2301
2302 } else if (VLAN_V2_ALLOWED(adapter)) {
2303 switch (vlan_ethertype) {
2304 case ETH_P_8021Q:
2305 if (enable_stripping)
2306 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_STRIPPING;
2307 else
2308 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_STRIPPING;
2309
2310 if (enable_insertion)
2311 aq_required |= IAVF_FLAG_AQ_ENABLE_CTAG_VLAN_INSERTION;
2312 else
2313 aq_required |= IAVF_FLAG_AQ_DISABLE_CTAG_VLAN_INSERTION;
2314 break;
2315 case ETH_P_8021AD:
2316 if (enable_stripping)
2317 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_STRIPPING;
2318 else
2319 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_STRIPPING;
2320
2321 if (enable_insertion)
2322 aq_required |= IAVF_FLAG_AQ_ENABLE_STAG_VLAN_INSERTION;
2323 else
2324 aq_required |= IAVF_FLAG_AQ_DISABLE_STAG_VLAN_INSERTION;
2325 break;
2326 }
2327 }
2328
2329 if (aq_required) {
2330 adapter->aq_required |= aq_required;
2331 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
2332 }
2333 }
2334
2335 /**
2336 * iavf_startup - first step of driver startup
2337 * @adapter: board private structure
2338 *
2339 * Function process __IAVF_STARTUP driver state.
2340 * When success the state is changed to __IAVF_INIT_VERSION_CHECK
2341 * when fails the state is changed to __IAVF_INIT_FAILED
2342 **/
iavf_startup(struct iavf_adapter * adapter)2343 static void iavf_startup(struct iavf_adapter *adapter)
2344 {
2345 struct pci_dev *pdev = adapter->pdev;
2346 struct iavf_hw *hw = &adapter->hw;
2347 enum iavf_status status;
2348 int ret;
2349
2350 WARN_ON(adapter->state != __IAVF_STARTUP);
2351
2352 /* driver loaded, probe complete */
2353 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2354 adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2355 status = iavf_set_mac_type(hw);
2356 if (status) {
2357 dev_err(&pdev->dev, "Failed to set MAC type (%d)\n", status);
2358 goto err;
2359 }
2360
2361 ret = iavf_check_reset_complete(hw);
2362 if (ret) {
2363 dev_info(&pdev->dev, "Device is still in reset (%d), retrying\n",
2364 ret);
2365 goto err;
2366 }
2367 hw->aq.num_arq_entries = IAVF_AQ_LEN;
2368 hw->aq.num_asq_entries = IAVF_AQ_LEN;
2369 hw->aq.arq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2370 hw->aq.asq_buf_size = IAVF_MAX_AQ_BUF_SIZE;
2371
2372 status = iavf_init_adminq(hw);
2373 if (status) {
2374 dev_err(&pdev->dev, "Failed to init Admin Queue (%d)\n",
2375 status);
2376 goto err;
2377 }
2378 ret = iavf_send_api_ver(adapter);
2379 if (ret) {
2380 dev_err(&pdev->dev, "Unable to send to PF (%d)\n", ret);
2381 iavf_shutdown_adminq(hw);
2382 goto err;
2383 }
2384 iavf_change_state(adapter, __IAVF_INIT_VERSION_CHECK);
2385 return;
2386 err:
2387 iavf_change_state(adapter, __IAVF_INIT_FAILED);
2388 }
2389
2390 /**
2391 * iavf_init_version_check - second step of driver startup
2392 * @adapter: board private structure
2393 *
2394 * Function process __IAVF_INIT_VERSION_CHECK driver state.
2395 * When success the state is changed to __IAVF_INIT_GET_RESOURCES
2396 * when fails the state is changed to __IAVF_INIT_FAILED
2397 **/
iavf_init_version_check(struct iavf_adapter * adapter)2398 static void iavf_init_version_check(struct iavf_adapter *adapter)
2399 {
2400 struct pci_dev *pdev = adapter->pdev;
2401 struct iavf_hw *hw = &adapter->hw;
2402 int err = -EAGAIN;
2403
2404 WARN_ON(adapter->state != __IAVF_INIT_VERSION_CHECK);
2405
2406 if (!iavf_asq_done(hw)) {
2407 dev_err(&pdev->dev, "Admin queue command never completed\n");
2408 iavf_shutdown_adminq(hw);
2409 iavf_change_state(adapter, __IAVF_STARTUP);
2410 goto err;
2411 }
2412
2413 /* aq msg sent, awaiting reply */
2414 err = iavf_verify_api_ver(adapter);
2415 if (err) {
2416 if (err == -EALREADY)
2417 err = iavf_send_api_ver(adapter);
2418 else
2419 dev_err(&pdev->dev, "Unsupported PF API version %d.%d, expected %d.%d\n",
2420 adapter->pf_version.major,
2421 adapter->pf_version.minor,
2422 VIRTCHNL_VERSION_MAJOR,
2423 VIRTCHNL_VERSION_MINOR);
2424 goto err;
2425 }
2426 err = iavf_send_vf_config_msg(adapter);
2427 if (err) {
2428 dev_err(&pdev->dev, "Unable to send config request (%d)\n",
2429 err);
2430 goto err;
2431 }
2432 iavf_change_state(adapter, __IAVF_INIT_GET_RESOURCES);
2433 return;
2434 err:
2435 iavf_change_state(adapter, __IAVF_INIT_FAILED);
2436 }
2437
2438 /**
2439 * iavf_parse_vf_resource_msg - parse response from VIRTCHNL_OP_GET_VF_RESOURCES
2440 * @adapter: board private structure
2441 */
iavf_parse_vf_resource_msg(struct iavf_adapter * adapter)2442 int iavf_parse_vf_resource_msg(struct iavf_adapter *adapter)
2443 {
2444 int i, num_req_queues = adapter->num_req_queues;
2445 struct iavf_vsi *vsi = &adapter->vsi;
2446
2447 for (i = 0; i < adapter->vf_res->num_vsis; i++) {
2448 if (adapter->vf_res->vsi_res[i].vsi_type == VIRTCHNL_VSI_SRIOV)
2449 adapter->vsi_res = &adapter->vf_res->vsi_res[i];
2450 }
2451 if (!adapter->vsi_res) {
2452 dev_err(&adapter->pdev->dev, "No LAN VSI found\n");
2453 return -ENODEV;
2454 }
2455
2456 if (num_req_queues &&
2457 num_req_queues > adapter->vsi_res->num_queue_pairs) {
2458 /* Problem. The PF gave us fewer queues than what we had
2459 * negotiated in our request. Need a reset to see if we can't
2460 * get back to a working state.
2461 */
2462 dev_err(&adapter->pdev->dev,
2463 "Requested %d queues, but PF only gave us %d.\n",
2464 num_req_queues,
2465 adapter->vsi_res->num_queue_pairs);
2466 adapter->flags |= IAVF_FLAG_REINIT_MSIX_NEEDED;
2467 adapter->num_req_queues = adapter->vsi_res->num_queue_pairs;
2468 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
2469
2470 return -EAGAIN;
2471 }
2472 adapter->num_req_queues = 0;
2473 adapter->vsi.id = adapter->vsi_res->vsi_id;
2474
2475 adapter->vsi.back = adapter;
2476 adapter->vsi.base_vector = 1;
2477 vsi->netdev = adapter->netdev;
2478 vsi->qs_handle = adapter->vsi_res->qset_handle;
2479 if (adapter->vf_res->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_RSS_PF) {
2480 adapter->rss_key_size = adapter->vf_res->rss_key_size;
2481 adapter->rss_lut_size = adapter->vf_res->rss_lut_size;
2482 } else {
2483 adapter->rss_key_size = IAVF_HKEY_ARRAY_SIZE;
2484 adapter->rss_lut_size = IAVF_HLUT_ARRAY_SIZE;
2485 }
2486
2487 return 0;
2488 }
2489
2490 /**
2491 * iavf_init_get_resources - third step of driver startup
2492 * @adapter: board private structure
2493 *
2494 * Function process __IAVF_INIT_GET_RESOURCES driver state and
2495 * finishes driver initialization procedure.
2496 * When success the state is changed to __IAVF_DOWN
2497 * when fails the state is changed to __IAVF_INIT_FAILED
2498 **/
iavf_init_get_resources(struct iavf_adapter * adapter)2499 static void iavf_init_get_resources(struct iavf_adapter *adapter)
2500 {
2501 struct pci_dev *pdev = adapter->pdev;
2502 struct iavf_hw *hw = &adapter->hw;
2503 int err;
2504
2505 WARN_ON(adapter->state != __IAVF_INIT_GET_RESOURCES);
2506 /* aq msg sent, awaiting reply */
2507 if (!adapter->vf_res) {
2508 adapter->vf_res = kzalloc(IAVF_VIRTCHNL_VF_RESOURCE_SIZE,
2509 GFP_KERNEL);
2510 if (!adapter->vf_res) {
2511 err = -ENOMEM;
2512 goto err;
2513 }
2514 }
2515 err = iavf_get_vf_config(adapter);
2516 if (err == -EALREADY) {
2517 err = iavf_send_vf_config_msg(adapter);
2518 goto err;
2519 } else if (err == -EINVAL) {
2520 /* We only get -EINVAL if the device is in a very bad
2521 * state or if we've been disabled for previous bad
2522 * behavior. Either way, we're done now.
2523 */
2524 iavf_shutdown_adminq(hw);
2525 dev_err(&pdev->dev, "Unable to get VF config due to PF error condition, not retrying\n");
2526 return;
2527 }
2528 if (err) {
2529 dev_err(&pdev->dev, "Unable to get VF config (%d)\n", err);
2530 goto err_alloc;
2531 }
2532
2533 err = iavf_parse_vf_resource_msg(adapter);
2534 if (err) {
2535 dev_err(&pdev->dev, "Failed to parse VF resource message from PF (%d)\n",
2536 err);
2537 goto err_alloc;
2538 }
2539 /* Some features require additional messages to negotiate extended
2540 * capabilities. These are processed in sequence by the
2541 * __IAVF_INIT_EXTENDED_CAPS driver state.
2542 */
2543 adapter->extended_caps = IAVF_EXTENDED_CAPS;
2544
2545 iavf_change_state(adapter, __IAVF_INIT_EXTENDED_CAPS);
2546 return;
2547
2548 err_alloc:
2549 kfree(adapter->vf_res);
2550 adapter->vf_res = NULL;
2551 err:
2552 iavf_change_state(adapter, __IAVF_INIT_FAILED);
2553 }
2554
2555 /**
2556 * iavf_init_send_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2557 * @adapter: board private structure
2558 *
2559 * Function processes send of the extended VLAN V2 capability message to the
2560 * PF. Must clear IAVF_EXTENDED_CAP_RECV_VLAN_V2 if the message is not sent,
2561 * e.g. due to PF not negotiating VIRTCHNL_VF_OFFLOAD_VLAN_V2.
2562 */
iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter * adapter)2563 static void iavf_init_send_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2564 {
2565 int ret;
2566
2567 WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2));
2568
2569 ret = iavf_send_vf_offload_vlan_v2_msg(adapter);
2570 if (ret && ret == -EOPNOTSUPP) {
2571 /* PF does not support VIRTCHNL_VF_OFFLOAD_V2. In this case,
2572 * we did not send the capability exchange message and do not
2573 * expect a response.
2574 */
2575 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2576 }
2577
2578 /* We sent the message, so move on to the next step */
2579 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2580 }
2581
2582 /**
2583 * iavf_init_recv_offload_vlan_v2_caps - part of initializing VLAN V2 caps
2584 * @adapter: board private structure
2585 *
2586 * Function processes receipt of the extended VLAN V2 capability message from
2587 * the PF.
2588 **/
iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter * adapter)2589 static void iavf_init_recv_offload_vlan_v2_caps(struct iavf_adapter *adapter)
2590 {
2591 int ret;
2592
2593 WARN_ON(!(adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2));
2594
2595 memset(&adapter->vlan_v2_caps, 0, sizeof(adapter->vlan_v2_caps));
2596
2597 ret = iavf_get_vf_vlan_v2_caps(adapter);
2598 if (ret)
2599 goto err;
2600
2601 /* We've processed receipt of the VLAN V2 caps message */
2602 adapter->extended_caps &= ~IAVF_EXTENDED_CAP_RECV_VLAN_V2;
2603 return;
2604 err:
2605 /* We didn't receive a reply. Make sure we try sending again when
2606 * __IAVF_INIT_FAILED attempts to recover.
2607 */
2608 adapter->extended_caps |= IAVF_EXTENDED_CAP_SEND_VLAN_V2;
2609 iavf_change_state(adapter, __IAVF_INIT_FAILED);
2610 }
2611
2612 /**
2613 * iavf_init_process_extended_caps - Part of driver startup
2614 * @adapter: board private structure
2615 *
2616 * Function processes __IAVF_INIT_EXTENDED_CAPS driver state. This state
2617 * handles negotiating capabilities for features which require an additional
2618 * message.
2619 *
2620 * Once all extended capabilities exchanges are finished, the driver will
2621 * transition into __IAVF_INIT_CONFIG_ADAPTER.
2622 */
iavf_init_process_extended_caps(struct iavf_adapter * adapter)2623 static void iavf_init_process_extended_caps(struct iavf_adapter *adapter)
2624 {
2625 WARN_ON(adapter->state != __IAVF_INIT_EXTENDED_CAPS);
2626
2627 /* Process capability exchange for VLAN V2 */
2628 if (adapter->extended_caps & IAVF_EXTENDED_CAP_SEND_VLAN_V2) {
2629 iavf_init_send_offload_vlan_v2_caps(adapter);
2630 return;
2631 } else if (adapter->extended_caps & IAVF_EXTENDED_CAP_RECV_VLAN_V2) {
2632 iavf_init_recv_offload_vlan_v2_caps(adapter);
2633 return;
2634 }
2635
2636 /* When we reach here, no further extended capabilities exchanges are
2637 * necessary, so we finally transition into __IAVF_INIT_CONFIG_ADAPTER
2638 */
2639 iavf_change_state(adapter, __IAVF_INIT_CONFIG_ADAPTER);
2640 }
2641
2642 /**
2643 * iavf_init_config_adapter - last part of driver startup
2644 * @adapter: board private structure
2645 *
2646 * After all the supported capabilities are negotiated, then the
2647 * __IAVF_INIT_CONFIG_ADAPTER state will finish driver initialization.
2648 */
iavf_init_config_adapter(struct iavf_adapter * adapter)2649 static void iavf_init_config_adapter(struct iavf_adapter *adapter)
2650 {
2651 struct net_device *netdev = adapter->netdev;
2652 struct pci_dev *pdev = adapter->pdev;
2653 int err;
2654
2655 WARN_ON(adapter->state != __IAVF_INIT_CONFIG_ADAPTER);
2656
2657 if (iavf_process_config(adapter))
2658 goto err;
2659
2660 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2661
2662 adapter->flags |= IAVF_FLAG_RX_CSUM_ENABLED;
2663
2664 netdev->netdev_ops = &iavf_netdev_ops;
2665 iavf_set_ethtool_ops(netdev);
2666 netdev->watchdog_timeo = 5 * HZ;
2667
2668 /* MTU range: 68 - 9710 */
2669 netdev->min_mtu = ETH_MIN_MTU;
2670 netdev->max_mtu = IAVF_MAX_RXBUFFER - IAVF_PACKET_HDR_PAD;
2671
2672 if (!is_valid_ether_addr(adapter->hw.mac.addr)) {
2673 dev_info(&pdev->dev, "Invalid MAC address %pM, using random\n",
2674 adapter->hw.mac.addr);
2675 eth_hw_addr_random(netdev);
2676 ether_addr_copy(adapter->hw.mac.addr, netdev->dev_addr);
2677 } else {
2678 eth_hw_addr_set(netdev, adapter->hw.mac.addr);
2679 ether_addr_copy(netdev->perm_addr, adapter->hw.mac.addr);
2680 }
2681
2682 adapter->tx_desc_count = IAVF_DEFAULT_TXD;
2683 adapter->rx_desc_count = IAVF_DEFAULT_RXD;
2684 err = iavf_init_interrupt_scheme(adapter);
2685 if (err)
2686 goto err_sw_init;
2687 iavf_map_rings_to_vectors(adapter);
2688 if (adapter->vf_res->vf_cap_flags &
2689 VIRTCHNL_VF_OFFLOAD_WB_ON_ITR)
2690 adapter->flags |= IAVF_FLAG_WB_ON_ITR_CAPABLE;
2691
2692 err = iavf_request_misc_irq(adapter);
2693 if (err)
2694 goto err_sw_init;
2695
2696 netif_carrier_off(netdev);
2697 adapter->link_up = false;
2698 netif_tx_stop_all_queues(netdev);
2699
2700 if (CLIENT_ALLOWED(adapter)) {
2701 err = iavf_lan_add_device(adapter);
2702 if (err)
2703 dev_info(&pdev->dev, "Failed to add VF to client API service list: %d\n",
2704 err);
2705 }
2706 dev_info(&pdev->dev, "MAC address: %pM\n", adapter->hw.mac.addr);
2707 if (netdev->features & NETIF_F_GRO)
2708 dev_info(&pdev->dev, "GRO is enabled\n");
2709
2710 iavf_change_state(adapter, __IAVF_DOWN);
2711 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2712
2713 iavf_misc_irq_enable(adapter);
2714 wake_up(&adapter->down_waitqueue);
2715
2716 adapter->rss_key = kzalloc(adapter->rss_key_size, GFP_KERNEL);
2717 adapter->rss_lut = kzalloc(adapter->rss_lut_size, GFP_KERNEL);
2718 if (!adapter->rss_key || !adapter->rss_lut) {
2719 err = -ENOMEM;
2720 goto err_mem;
2721 }
2722 if (RSS_AQ(adapter))
2723 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
2724 else
2725 iavf_init_rss(adapter);
2726
2727 if (VLAN_V2_ALLOWED(adapter))
2728 /* request initial VLAN offload settings */
2729 iavf_set_vlan_offload_features(adapter, 0, netdev->features);
2730
2731 iavf_schedule_finish_config(adapter);
2732 return;
2733
2734 err_mem:
2735 iavf_free_rss(adapter);
2736 iavf_free_misc_irq(adapter);
2737 err_sw_init:
2738 iavf_reset_interrupt_capability(adapter);
2739 err:
2740 iavf_change_state(adapter, __IAVF_INIT_FAILED);
2741 }
2742
2743 /**
2744 * iavf_watchdog_task - Periodic call-back task
2745 * @work: pointer to work_struct
2746 **/
iavf_watchdog_task(struct work_struct * work)2747 static void iavf_watchdog_task(struct work_struct *work)
2748 {
2749 struct iavf_adapter *adapter = container_of(work,
2750 struct iavf_adapter,
2751 watchdog_task.work);
2752 struct iavf_hw *hw = &adapter->hw;
2753 u32 reg_val;
2754
2755 if (!mutex_trylock(&adapter->crit_lock)) {
2756 if (adapter->state == __IAVF_REMOVE)
2757 return;
2758
2759 goto restart_watchdog;
2760 }
2761
2762 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
2763 iavf_change_state(adapter, __IAVF_COMM_FAILED);
2764
2765 switch (adapter->state) {
2766 case __IAVF_STARTUP:
2767 iavf_startup(adapter);
2768 mutex_unlock(&adapter->crit_lock);
2769 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2770 msecs_to_jiffies(30));
2771 return;
2772 case __IAVF_INIT_VERSION_CHECK:
2773 iavf_init_version_check(adapter);
2774 mutex_unlock(&adapter->crit_lock);
2775 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2776 msecs_to_jiffies(30));
2777 return;
2778 case __IAVF_INIT_GET_RESOURCES:
2779 iavf_init_get_resources(adapter);
2780 mutex_unlock(&adapter->crit_lock);
2781 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2782 msecs_to_jiffies(1));
2783 return;
2784 case __IAVF_INIT_EXTENDED_CAPS:
2785 iavf_init_process_extended_caps(adapter);
2786 mutex_unlock(&adapter->crit_lock);
2787 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2788 msecs_to_jiffies(1));
2789 return;
2790 case __IAVF_INIT_CONFIG_ADAPTER:
2791 iavf_init_config_adapter(adapter);
2792 mutex_unlock(&adapter->crit_lock);
2793 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2794 msecs_to_jiffies(1));
2795 return;
2796 case __IAVF_INIT_FAILED:
2797 if (test_bit(__IAVF_IN_REMOVE_TASK,
2798 &adapter->crit_section)) {
2799 /* Do not update the state and do not reschedule
2800 * watchdog task, iavf_remove should handle this state
2801 * as it can loop forever
2802 */
2803 mutex_unlock(&adapter->crit_lock);
2804 return;
2805 }
2806 if (++adapter->aq_wait_count > IAVF_AQ_MAX_ERR) {
2807 dev_err(&adapter->pdev->dev,
2808 "Failed to communicate with PF; waiting before retry\n");
2809 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2810 iavf_shutdown_adminq(hw);
2811 mutex_unlock(&adapter->crit_lock);
2812 queue_delayed_work(adapter->wq,
2813 &adapter->watchdog_task, (5 * HZ));
2814 return;
2815 }
2816 /* Try again from failed step*/
2817 iavf_change_state(adapter, adapter->last_state);
2818 mutex_unlock(&adapter->crit_lock);
2819 queue_delayed_work(adapter->wq, &adapter->watchdog_task, HZ);
2820 return;
2821 case __IAVF_COMM_FAILED:
2822 if (test_bit(__IAVF_IN_REMOVE_TASK,
2823 &adapter->crit_section)) {
2824 /* Set state to __IAVF_INIT_FAILED and perform remove
2825 * steps. Remove IAVF_FLAG_PF_COMMS_FAILED so the task
2826 * doesn't bring the state back to __IAVF_COMM_FAILED.
2827 */
2828 iavf_change_state(adapter, __IAVF_INIT_FAILED);
2829 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2830 mutex_unlock(&adapter->crit_lock);
2831 return;
2832 }
2833 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
2834 IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
2835 if (reg_val == VIRTCHNL_VFR_VFACTIVE ||
2836 reg_val == VIRTCHNL_VFR_COMPLETED) {
2837 /* A chance for redemption! */
2838 dev_err(&adapter->pdev->dev,
2839 "Hardware came out of reset. Attempting reinit.\n");
2840 /* When init task contacts the PF and
2841 * gets everything set up again, it'll restart the
2842 * watchdog for us. Down, boy. Sit. Stay. Woof.
2843 */
2844 iavf_change_state(adapter, __IAVF_STARTUP);
2845 adapter->flags &= ~IAVF_FLAG_PF_COMMS_FAILED;
2846 }
2847 adapter->aq_required = 0;
2848 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2849 mutex_unlock(&adapter->crit_lock);
2850 queue_delayed_work(adapter->wq,
2851 &adapter->watchdog_task,
2852 msecs_to_jiffies(10));
2853 return;
2854 case __IAVF_RESETTING:
2855 mutex_unlock(&adapter->crit_lock);
2856 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2857 HZ * 2);
2858 return;
2859 case __IAVF_DOWN:
2860 case __IAVF_DOWN_PENDING:
2861 case __IAVF_TESTING:
2862 case __IAVF_RUNNING:
2863 if (adapter->current_op) {
2864 if (!iavf_asq_done(hw)) {
2865 dev_dbg(&adapter->pdev->dev,
2866 "Admin queue timeout\n");
2867 iavf_send_api_ver(adapter);
2868 }
2869 } else {
2870 int ret = iavf_process_aq_command(adapter);
2871
2872 /* An error will be returned if no commands were
2873 * processed; use this opportunity to update stats
2874 * if the error isn't -ENOTSUPP
2875 */
2876 if (ret && ret != -EOPNOTSUPP &&
2877 adapter->state == __IAVF_RUNNING)
2878 iavf_request_stats(adapter);
2879 }
2880 if (adapter->state == __IAVF_RUNNING)
2881 iavf_detect_recover_hung(&adapter->vsi);
2882 break;
2883 case __IAVF_REMOVE:
2884 default:
2885 mutex_unlock(&adapter->crit_lock);
2886 return;
2887 }
2888
2889 /* check for hw reset */
2890 reg_val = rd32(hw, IAVF_VF_ARQLEN1) & IAVF_VF_ARQLEN1_ARQENABLE_MASK;
2891 if (!reg_val) {
2892 adapter->aq_required = 0;
2893 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
2894 dev_err(&adapter->pdev->dev, "Hardware reset detected\n");
2895 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_PENDING);
2896 mutex_unlock(&adapter->crit_lock);
2897 queue_delayed_work(adapter->wq,
2898 &adapter->watchdog_task, HZ * 2);
2899 return;
2900 }
2901
2902 schedule_delayed_work(&adapter->client_task, msecs_to_jiffies(5));
2903 mutex_unlock(&adapter->crit_lock);
2904 restart_watchdog:
2905 if (adapter->state >= __IAVF_DOWN)
2906 queue_work(adapter->wq, &adapter->adminq_task);
2907 if (adapter->aq_required)
2908 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2909 msecs_to_jiffies(20));
2910 else
2911 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
2912 HZ * 2);
2913 }
2914
2915 /**
2916 * iavf_disable_vf - disable VF
2917 * @adapter: board private structure
2918 *
2919 * Set communication failed flag and free all resources.
2920 * NOTE: This function is expected to be called with crit_lock being held.
2921 **/
iavf_disable_vf(struct iavf_adapter * adapter)2922 static void iavf_disable_vf(struct iavf_adapter *adapter)
2923 {
2924 struct iavf_mac_filter *f, *ftmp;
2925 struct iavf_vlan_filter *fv, *fvtmp;
2926 struct iavf_cloud_filter *cf, *cftmp;
2927
2928 adapter->flags |= IAVF_FLAG_PF_COMMS_FAILED;
2929
2930 /* We don't use netif_running() because it may be true prior to
2931 * ndo_open() returning, so we can't assume it means all our open
2932 * tasks have finished, since we're not holding the rtnl_lock here.
2933 */
2934 if (adapter->state == __IAVF_RUNNING) {
2935 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
2936 netif_carrier_off(adapter->netdev);
2937 netif_tx_disable(adapter->netdev);
2938 adapter->link_up = false;
2939 iavf_napi_disable_all(adapter);
2940 iavf_irq_disable(adapter);
2941 iavf_free_traffic_irqs(adapter);
2942 iavf_free_all_tx_resources(adapter);
2943 iavf_free_all_rx_resources(adapter);
2944 }
2945
2946 spin_lock_bh(&adapter->mac_vlan_list_lock);
2947
2948 /* Delete all of the filters */
2949 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
2950 list_del(&f->list);
2951 kfree(f);
2952 }
2953
2954 list_for_each_entry_safe(fv, fvtmp, &adapter->vlan_filter_list, list) {
2955 list_del(&fv->list);
2956 kfree(fv);
2957 }
2958 adapter->num_vlan_filters = 0;
2959
2960 spin_unlock_bh(&adapter->mac_vlan_list_lock);
2961
2962 spin_lock_bh(&adapter->cloud_filter_list_lock);
2963 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
2964 list_del(&cf->list);
2965 kfree(cf);
2966 adapter->num_cloud_filters--;
2967 }
2968 spin_unlock_bh(&adapter->cloud_filter_list_lock);
2969
2970 iavf_free_misc_irq(adapter);
2971 iavf_reset_interrupt_capability(adapter);
2972 iavf_free_q_vectors(adapter);
2973 iavf_free_queues(adapter);
2974 memset(adapter->vf_res, 0, IAVF_VIRTCHNL_VF_RESOURCE_SIZE);
2975 iavf_shutdown_adminq(&adapter->hw);
2976 adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
2977 iavf_change_state(adapter, __IAVF_DOWN);
2978 wake_up(&adapter->down_waitqueue);
2979 dev_info(&adapter->pdev->dev, "Reset task did not complete, VF disabled\n");
2980 }
2981
2982 /**
2983 * iavf_reset_task - Call-back task to handle hardware reset
2984 * @work: pointer to work_struct
2985 *
2986 * During reset we need to shut down and reinitialize the admin queue
2987 * before we can use it to communicate with the PF again. We also clear
2988 * and reinit the rings because that context is lost as well.
2989 **/
iavf_reset_task(struct work_struct * work)2990 static void iavf_reset_task(struct work_struct *work)
2991 {
2992 struct iavf_adapter *adapter = container_of(work,
2993 struct iavf_adapter,
2994 reset_task);
2995 struct virtchnl_vf_resource *vfres = adapter->vf_res;
2996 struct net_device *netdev = adapter->netdev;
2997 struct iavf_hw *hw = &adapter->hw;
2998 struct iavf_mac_filter *f, *ftmp;
2999 struct iavf_cloud_filter *cf;
3000 enum iavf_status status;
3001 u32 reg_val;
3002 int i = 0, err;
3003 bool running;
3004
3005 /* When device is being removed it doesn't make sense to run the reset
3006 * task, just return in such a case.
3007 */
3008 if (!mutex_trylock(&adapter->crit_lock)) {
3009 if (adapter->state != __IAVF_REMOVE)
3010 queue_work(adapter->wq, &adapter->reset_task);
3011
3012 return;
3013 }
3014
3015 while (!mutex_trylock(&adapter->client_lock))
3016 usleep_range(500, 1000);
3017 if (CLIENT_ENABLED(adapter)) {
3018 adapter->flags &= ~(IAVF_FLAG_CLIENT_NEEDS_OPEN |
3019 IAVF_FLAG_CLIENT_NEEDS_CLOSE |
3020 IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS |
3021 IAVF_FLAG_SERVICE_CLIENT_REQUESTED);
3022 cancel_delayed_work_sync(&adapter->client_task);
3023 iavf_notify_client_close(&adapter->vsi, true);
3024 }
3025 iavf_misc_irq_disable(adapter);
3026 if (adapter->flags & IAVF_FLAG_RESET_NEEDED) {
3027 adapter->flags &= ~IAVF_FLAG_RESET_NEEDED;
3028 /* Restart the AQ here. If we have been reset but didn't
3029 * detect it, or if the PF had to reinit, our AQ will be hosed.
3030 */
3031 iavf_shutdown_adminq(hw);
3032 iavf_init_adminq(hw);
3033 iavf_request_reset(adapter);
3034 }
3035 adapter->flags |= IAVF_FLAG_RESET_PENDING;
3036
3037 /* poll until we see the reset actually happen */
3038 for (i = 0; i < IAVF_RESET_WAIT_DETECTED_COUNT; i++) {
3039 reg_val = rd32(hw, IAVF_VF_ARQLEN1) &
3040 IAVF_VF_ARQLEN1_ARQENABLE_MASK;
3041 if (!reg_val)
3042 break;
3043 usleep_range(5000, 10000);
3044 }
3045 if (i == IAVF_RESET_WAIT_DETECTED_COUNT) {
3046 dev_info(&adapter->pdev->dev, "Never saw reset\n");
3047 goto continue_reset; /* act like the reset happened */
3048 }
3049
3050 /* wait until the reset is complete and the PF is responding to us */
3051 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
3052 /* sleep first to make sure a minimum wait time is met */
3053 msleep(IAVF_RESET_WAIT_MS);
3054
3055 reg_val = rd32(hw, IAVF_VFGEN_RSTAT) &
3056 IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
3057 if (reg_val == VIRTCHNL_VFR_VFACTIVE)
3058 break;
3059 }
3060
3061 pci_set_master(adapter->pdev);
3062 pci_restore_msi_state(adapter->pdev);
3063
3064 if (i == IAVF_RESET_WAIT_COMPLETE_COUNT) {
3065 dev_err(&adapter->pdev->dev, "Reset never finished (%x)\n",
3066 reg_val);
3067 iavf_disable_vf(adapter);
3068 mutex_unlock(&adapter->client_lock);
3069 mutex_unlock(&adapter->crit_lock);
3070 return; /* Do not attempt to reinit. It's dead, Jim. */
3071 }
3072
3073 continue_reset:
3074 /* We don't use netif_running() because it may be true prior to
3075 * ndo_open() returning, so we can't assume it means all our open
3076 * tasks have finished, since we're not holding the rtnl_lock here.
3077 */
3078 running = adapter->state == __IAVF_RUNNING;
3079
3080 if (running) {
3081 netif_carrier_off(netdev);
3082 netif_tx_stop_all_queues(netdev);
3083 adapter->link_up = false;
3084 iavf_napi_disable_all(adapter);
3085 }
3086 iavf_irq_disable(adapter);
3087
3088 iavf_change_state(adapter, __IAVF_RESETTING);
3089 adapter->flags &= ~IAVF_FLAG_RESET_PENDING;
3090
3091 /* free the Tx/Rx rings and descriptors, might be better to just
3092 * re-use them sometime in the future
3093 */
3094 iavf_free_all_rx_resources(adapter);
3095 iavf_free_all_tx_resources(adapter);
3096
3097 adapter->flags |= IAVF_FLAG_QUEUES_DISABLED;
3098 /* kill and reinit the admin queue */
3099 iavf_shutdown_adminq(hw);
3100 adapter->current_op = VIRTCHNL_OP_UNKNOWN;
3101 status = iavf_init_adminq(hw);
3102 if (status) {
3103 dev_info(&adapter->pdev->dev, "Failed to init adminq: %d\n",
3104 status);
3105 goto reset_err;
3106 }
3107 adapter->aq_required = 0;
3108
3109 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3110 (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3111 err = iavf_reinit_interrupt_scheme(adapter, running);
3112 if (err)
3113 goto reset_err;
3114 }
3115
3116 if (RSS_AQ(adapter)) {
3117 adapter->aq_required |= IAVF_FLAG_AQ_CONFIGURE_RSS;
3118 } else {
3119 err = iavf_init_rss(adapter);
3120 if (err)
3121 goto reset_err;
3122 }
3123
3124 adapter->aq_required |= IAVF_FLAG_AQ_GET_CONFIG;
3125 /* always set since VIRTCHNL_OP_GET_VF_RESOURCES has not been
3126 * sent/received yet, so VLAN_V2_ALLOWED() cannot is not reliable here,
3127 * however the VIRTCHNL_OP_GET_OFFLOAD_VLAN_V2_CAPS won't be sent until
3128 * VIRTCHNL_OP_GET_VF_RESOURCES and VIRTCHNL_VF_OFFLOAD_VLAN_V2 have
3129 * been successfully sent and negotiated
3130 */
3131 adapter->aq_required |= IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS;
3132 adapter->aq_required |= IAVF_FLAG_AQ_MAP_VECTORS;
3133
3134 spin_lock_bh(&adapter->mac_vlan_list_lock);
3135
3136 /* Delete filter for the current MAC address, it could have
3137 * been changed by the PF via administratively set MAC.
3138 * Will be re-added via VIRTCHNL_OP_GET_VF_RESOURCES.
3139 */
3140 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
3141 if (ether_addr_equal(f->macaddr, adapter->hw.mac.addr)) {
3142 list_del(&f->list);
3143 kfree(f);
3144 }
3145 }
3146 /* re-add all MAC filters */
3147 list_for_each_entry(f, &adapter->mac_filter_list, list) {
3148 f->add = true;
3149 }
3150 spin_unlock_bh(&adapter->mac_vlan_list_lock);
3151
3152 /* check if TCs are running and re-add all cloud filters */
3153 spin_lock_bh(&adapter->cloud_filter_list_lock);
3154 if ((vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) &&
3155 adapter->num_tc) {
3156 list_for_each_entry(cf, &adapter->cloud_filter_list, list) {
3157 cf->add = true;
3158 }
3159 }
3160 spin_unlock_bh(&adapter->cloud_filter_list_lock);
3161
3162 adapter->aq_required |= IAVF_FLAG_AQ_ADD_MAC_FILTER;
3163 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
3164 iavf_misc_irq_enable(adapter);
3165
3166 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 2);
3167
3168 /* We were running when the reset started, so we need to restore some
3169 * state here.
3170 */
3171 if (running) {
3172 /* allocate transmit descriptors */
3173 err = iavf_setup_all_tx_resources(adapter);
3174 if (err)
3175 goto reset_err;
3176
3177 /* allocate receive descriptors */
3178 err = iavf_setup_all_rx_resources(adapter);
3179 if (err)
3180 goto reset_err;
3181
3182 if ((adapter->flags & IAVF_FLAG_REINIT_MSIX_NEEDED) ||
3183 (adapter->flags & IAVF_FLAG_REINIT_ITR_NEEDED)) {
3184 err = iavf_request_traffic_irqs(adapter, netdev->name);
3185 if (err)
3186 goto reset_err;
3187
3188 adapter->flags &= ~IAVF_FLAG_REINIT_MSIX_NEEDED;
3189 }
3190
3191 iavf_configure(adapter);
3192
3193 /* iavf_up_complete() will switch device back
3194 * to __IAVF_RUNNING
3195 */
3196 iavf_up_complete(adapter);
3197
3198 iavf_irq_enable(adapter, true);
3199 } else {
3200 iavf_change_state(adapter, __IAVF_DOWN);
3201 wake_up(&adapter->down_waitqueue);
3202 }
3203
3204 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
3205
3206 wake_up(&adapter->reset_waitqueue);
3207 mutex_unlock(&adapter->client_lock);
3208 mutex_unlock(&adapter->crit_lock);
3209
3210 return;
3211 reset_err:
3212 if (running) {
3213 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
3214 iavf_free_traffic_irqs(adapter);
3215 }
3216 iavf_disable_vf(adapter);
3217
3218 mutex_unlock(&adapter->client_lock);
3219 mutex_unlock(&adapter->crit_lock);
3220 dev_err(&adapter->pdev->dev, "failed to allocate resources during reinit\n");
3221 }
3222
3223 /**
3224 * iavf_adminq_task - worker thread to clean the admin queue
3225 * @work: pointer to work_struct containing our data
3226 **/
iavf_adminq_task(struct work_struct * work)3227 static void iavf_adminq_task(struct work_struct *work)
3228 {
3229 struct iavf_adapter *adapter =
3230 container_of(work, struct iavf_adapter, adminq_task);
3231 struct iavf_hw *hw = &adapter->hw;
3232 struct iavf_arq_event_info event;
3233 enum virtchnl_ops v_op;
3234 enum iavf_status ret, v_ret;
3235 u32 val, oldval;
3236 u16 pending;
3237
3238 if (!mutex_trylock(&adapter->crit_lock)) {
3239 if (adapter->state == __IAVF_REMOVE)
3240 return;
3241
3242 queue_work(adapter->wq, &adapter->adminq_task);
3243 goto out;
3244 }
3245
3246 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED)
3247 goto unlock;
3248
3249 event.buf_len = IAVF_MAX_AQ_BUF_SIZE;
3250 event.msg_buf = kzalloc(event.buf_len, GFP_KERNEL);
3251 if (!event.msg_buf)
3252 goto unlock;
3253
3254 do {
3255 ret = iavf_clean_arq_element(hw, &event, &pending);
3256 v_op = (enum virtchnl_ops)le32_to_cpu(event.desc.cookie_high);
3257 v_ret = (enum iavf_status)le32_to_cpu(event.desc.cookie_low);
3258
3259 if (ret || !v_op)
3260 break; /* No event to process or error cleaning ARQ */
3261
3262 iavf_virtchnl_completion(adapter, v_op, v_ret, event.msg_buf,
3263 event.msg_len);
3264 if (pending != 0)
3265 memset(event.msg_buf, 0, IAVF_MAX_AQ_BUF_SIZE);
3266 } while (pending);
3267
3268 if (iavf_is_reset_in_progress(adapter))
3269 goto freedom;
3270
3271 /* check for error indications */
3272 val = rd32(hw, hw->aq.arq.len);
3273 if (val == 0xdeadbeef || val == 0xffffffff) /* device in reset */
3274 goto freedom;
3275 oldval = val;
3276 if (val & IAVF_VF_ARQLEN1_ARQVFE_MASK) {
3277 dev_info(&adapter->pdev->dev, "ARQ VF Error detected\n");
3278 val &= ~IAVF_VF_ARQLEN1_ARQVFE_MASK;
3279 }
3280 if (val & IAVF_VF_ARQLEN1_ARQOVFL_MASK) {
3281 dev_info(&adapter->pdev->dev, "ARQ Overflow Error detected\n");
3282 val &= ~IAVF_VF_ARQLEN1_ARQOVFL_MASK;
3283 }
3284 if (val & IAVF_VF_ARQLEN1_ARQCRIT_MASK) {
3285 dev_info(&adapter->pdev->dev, "ARQ Critical Error detected\n");
3286 val &= ~IAVF_VF_ARQLEN1_ARQCRIT_MASK;
3287 }
3288 if (oldval != val)
3289 wr32(hw, hw->aq.arq.len, val);
3290
3291 val = rd32(hw, hw->aq.asq.len);
3292 oldval = val;
3293 if (val & IAVF_VF_ATQLEN1_ATQVFE_MASK) {
3294 dev_info(&adapter->pdev->dev, "ASQ VF Error detected\n");
3295 val &= ~IAVF_VF_ATQLEN1_ATQVFE_MASK;
3296 }
3297 if (val & IAVF_VF_ATQLEN1_ATQOVFL_MASK) {
3298 dev_info(&adapter->pdev->dev, "ASQ Overflow Error detected\n");
3299 val &= ~IAVF_VF_ATQLEN1_ATQOVFL_MASK;
3300 }
3301 if (val & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
3302 dev_info(&adapter->pdev->dev, "ASQ Critical Error detected\n");
3303 val &= ~IAVF_VF_ATQLEN1_ATQCRIT_MASK;
3304 }
3305 if (oldval != val)
3306 wr32(hw, hw->aq.asq.len, val);
3307
3308 freedom:
3309 kfree(event.msg_buf);
3310 unlock:
3311 mutex_unlock(&adapter->crit_lock);
3312 out:
3313 /* re-enable Admin queue interrupt cause */
3314 iavf_misc_irq_enable(adapter);
3315 }
3316
3317 /**
3318 * iavf_client_task - worker thread to perform client work
3319 * @work: pointer to work_struct containing our data
3320 *
3321 * This task handles client interactions. Because client calls can be
3322 * reentrant, we can't handle them in the watchdog.
3323 **/
iavf_client_task(struct work_struct * work)3324 static void iavf_client_task(struct work_struct *work)
3325 {
3326 struct iavf_adapter *adapter =
3327 container_of(work, struct iavf_adapter, client_task.work);
3328
3329 /* If we can't get the client bit, just give up. We'll be rescheduled
3330 * later.
3331 */
3332
3333 if (!mutex_trylock(&adapter->client_lock))
3334 return;
3335
3336 if (adapter->flags & IAVF_FLAG_SERVICE_CLIENT_REQUESTED) {
3337 iavf_client_subtask(adapter);
3338 adapter->flags &= ~IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
3339 goto out;
3340 }
3341 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS) {
3342 iavf_notify_client_l2_params(&adapter->vsi);
3343 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_L2_PARAMS;
3344 goto out;
3345 }
3346 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_CLOSE) {
3347 iavf_notify_client_close(&adapter->vsi, false);
3348 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_CLOSE;
3349 goto out;
3350 }
3351 if (adapter->flags & IAVF_FLAG_CLIENT_NEEDS_OPEN) {
3352 iavf_notify_client_open(&adapter->vsi);
3353 adapter->flags &= ~IAVF_FLAG_CLIENT_NEEDS_OPEN;
3354 }
3355 out:
3356 mutex_unlock(&adapter->client_lock);
3357 }
3358
3359 /**
3360 * iavf_free_all_tx_resources - Free Tx Resources for All Queues
3361 * @adapter: board private structure
3362 *
3363 * Free all transmit software resources
3364 **/
iavf_free_all_tx_resources(struct iavf_adapter * adapter)3365 void iavf_free_all_tx_resources(struct iavf_adapter *adapter)
3366 {
3367 int i;
3368
3369 if (!adapter->tx_rings)
3370 return;
3371
3372 for (i = 0; i < adapter->num_active_queues; i++)
3373 if (adapter->tx_rings[i].desc)
3374 iavf_free_tx_resources(&adapter->tx_rings[i]);
3375 }
3376
3377 /**
3378 * iavf_setup_all_tx_resources - allocate all queues Tx resources
3379 * @adapter: board private structure
3380 *
3381 * If this function returns with an error, then it's possible one or
3382 * more of the rings is populated (while the rest are not). It is the
3383 * callers duty to clean those orphaned rings.
3384 *
3385 * Return 0 on success, negative on failure
3386 **/
iavf_setup_all_tx_resources(struct iavf_adapter * adapter)3387 static int iavf_setup_all_tx_resources(struct iavf_adapter *adapter)
3388 {
3389 int i, err = 0;
3390
3391 for (i = 0; i < adapter->num_active_queues; i++) {
3392 adapter->tx_rings[i].count = adapter->tx_desc_count;
3393 err = iavf_setup_tx_descriptors(&adapter->tx_rings[i]);
3394 if (!err)
3395 continue;
3396 dev_err(&adapter->pdev->dev,
3397 "Allocation for Tx Queue %u failed\n", i);
3398 break;
3399 }
3400
3401 return err;
3402 }
3403
3404 /**
3405 * iavf_setup_all_rx_resources - allocate all queues Rx resources
3406 * @adapter: board private structure
3407 *
3408 * If this function returns with an error, then it's possible one or
3409 * more of the rings is populated (while the rest are not). It is the
3410 * callers duty to clean those orphaned rings.
3411 *
3412 * Return 0 on success, negative on failure
3413 **/
iavf_setup_all_rx_resources(struct iavf_adapter * adapter)3414 static int iavf_setup_all_rx_resources(struct iavf_adapter *adapter)
3415 {
3416 int i, err = 0;
3417
3418 for (i = 0; i < adapter->num_active_queues; i++) {
3419 adapter->rx_rings[i].count = adapter->rx_desc_count;
3420 err = iavf_setup_rx_descriptors(&adapter->rx_rings[i]);
3421 if (!err)
3422 continue;
3423 dev_err(&adapter->pdev->dev,
3424 "Allocation for Rx Queue %u failed\n", i);
3425 break;
3426 }
3427 return err;
3428 }
3429
3430 /**
3431 * iavf_free_all_rx_resources - Free Rx Resources for All Queues
3432 * @adapter: board private structure
3433 *
3434 * Free all receive software resources
3435 **/
iavf_free_all_rx_resources(struct iavf_adapter * adapter)3436 void iavf_free_all_rx_resources(struct iavf_adapter *adapter)
3437 {
3438 int i;
3439
3440 if (!adapter->rx_rings)
3441 return;
3442
3443 for (i = 0; i < adapter->num_active_queues; i++)
3444 if (adapter->rx_rings[i].desc)
3445 iavf_free_rx_resources(&adapter->rx_rings[i]);
3446 }
3447
3448 /**
3449 * iavf_validate_tx_bandwidth - validate the max Tx bandwidth
3450 * @adapter: board private structure
3451 * @max_tx_rate: max Tx bw for a tc
3452 **/
iavf_validate_tx_bandwidth(struct iavf_adapter * adapter,u64 max_tx_rate)3453 static int iavf_validate_tx_bandwidth(struct iavf_adapter *adapter,
3454 u64 max_tx_rate)
3455 {
3456 int speed = 0, ret = 0;
3457
3458 if (ADV_LINK_SUPPORT(adapter)) {
3459 if (adapter->link_speed_mbps < U32_MAX) {
3460 speed = adapter->link_speed_mbps;
3461 goto validate_bw;
3462 } else {
3463 dev_err(&adapter->pdev->dev, "Unknown link speed\n");
3464 return -EINVAL;
3465 }
3466 }
3467
3468 switch (adapter->link_speed) {
3469 case VIRTCHNL_LINK_SPEED_40GB:
3470 speed = SPEED_40000;
3471 break;
3472 case VIRTCHNL_LINK_SPEED_25GB:
3473 speed = SPEED_25000;
3474 break;
3475 case VIRTCHNL_LINK_SPEED_20GB:
3476 speed = SPEED_20000;
3477 break;
3478 case VIRTCHNL_LINK_SPEED_10GB:
3479 speed = SPEED_10000;
3480 break;
3481 case VIRTCHNL_LINK_SPEED_5GB:
3482 speed = SPEED_5000;
3483 break;
3484 case VIRTCHNL_LINK_SPEED_2_5GB:
3485 speed = SPEED_2500;
3486 break;
3487 case VIRTCHNL_LINK_SPEED_1GB:
3488 speed = SPEED_1000;
3489 break;
3490 case VIRTCHNL_LINK_SPEED_100MB:
3491 speed = SPEED_100;
3492 break;
3493 default:
3494 break;
3495 }
3496
3497 validate_bw:
3498 if (max_tx_rate > speed) {
3499 dev_err(&adapter->pdev->dev,
3500 "Invalid tx rate specified\n");
3501 ret = -EINVAL;
3502 }
3503
3504 return ret;
3505 }
3506
3507 /**
3508 * iavf_validate_ch_config - validate queue mapping info
3509 * @adapter: board private structure
3510 * @mqprio_qopt: queue parameters
3511 *
3512 * This function validates if the config provided by the user to
3513 * configure queue channels is valid or not. Returns 0 on a valid
3514 * config.
3515 **/
iavf_validate_ch_config(struct iavf_adapter * adapter,struct tc_mqprio_qopt_offload * mqprio_qopt)3516 static int iavf_validate_ch_config(struct iavf_adapter *adapter,
3517 struct tc_mqprio_qopt_offload *mqprio_qopt)
3518 {
3519 u64 total_max_rate = 0;
3520 u32 tx_rate_rem = 0;
3521 int i, num_qps = 0;
3522 u64 tx_rate = 0;
3523 int ret = 0;
3524
3525 if (mqprio_qopt->qopt.num_tc > IAVF_MAX_TRAFFIC_CLASS ||
3526 mqprio_qopt->qopt.num_tc < 1)
3527 return -EINVAL;
3528
3529 for (i = 0; i <= mqprio_qopt->qopt.num_tc - 1; i++) {
3530 if (!mqprio_qopt->qopt.count[i] ||
3531 mqprio_qopt->qopt.offset[i] != num_qps)
3532 return -EINVAL;
3533 if (mqprio_qopt->min_rate[i]) {
3534 dev_err(&adapter->pdev->dev,
3535 "Invalid min tx rate (greater than 0) specified for TC%d\n",
3536 i);
3537 return -EINVAL;
3538 }
3539
3540 /* convert to Mbps */
3541 tx_rate = div_u64(mqprio_qopt->max_rate[i],
3542 IAVF_MBPS_DIVISOR);
3543
3544 if (mqprio_qopt->max_rate[i] &&
3545 tx_rate < IAVF_MBPS_QUANTA) {
3546 dev_err(&adapter->pdev->dev,
3547 "Invalid max tx rate for TC%d, minimum %dMbps\n",
3548 i, IAVF_MBPS_QUANTA);
3549 return -EINVAL;
3550 }
3551
3552 (void)div_u64_rem(tx_rate, IAVF_MBPS_QUANTA, &tx_rate_rem);
3553
3554 if (tx_rate_rem != 0) {
3555 dev_err(&adapter->pdev->dev,
3556 "Invalid max tx rate for TC%d, not divisible by %d\n",
3557 i, IAVF_MBPS_QUANTA);
3558 return -EINVAL;
3559 }
3560
3561 total_max_rate += tx_rate;
3562 num_qps += mqprio_qopt->qopt.count[i];
3563 }
3564 if (num_qps > adapter->num_active_queues) {
3565 dev_err(&adapter->pdev->dev,
3566 "Cannot support requested number of queues\n");
3567 return -EINVAL;
3568 }
3569
3570 ret = iavf_validate_tx_bandwidth(adapter, total_max_rate);
3571 return ret;
3572 }
3573
3574 /**
3575 * iavf_del_all_cloud_filters - delete all cloud filters on the traffic classes
3576 * @adapter: board private structure
3577 **/
iavf_del_all_cloud_filters(struct iavf_adapter * adapter)3578 static void iavf_del_all_cloud_filters(struct iavf_adapter *adapter)
3579 {
3580 struct iavf_cloud_filter *cf, *cftmp;
3581
3582 spin_lock_bh(&adapter->cloud_filter_list_lock);
3583 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list,
3584 list) {
3585 list_del(&cf->list);
3586 kfree(cf);
3587 adapter->num_cloud_filters--;
3588 }
3589 spin_unlock_bh(&adapter->cloud_filter_list_lock);
3590 }
3591
3592 /**
3593 * iavf_is_tc_config_same - Compare the mqprio TC config with the
3594 * TC config already configured on this adapter.
3595 * @adapter: board private structure
3596 * @mqprio_qopt: TC config received from kernel.
3597 *
3598 * This function compares the TC config received from the kernel
3599 * with the config already configured on the adapter.
3600 *
3601 * Return: True if configuration is same, false otherwise.
3602 **/
iavf_is_tc_config_same(struct iavf_adapter * adapter,struct tc_mqprio_qopt * mqprio_qopt)3603 static bool iavf_is_tc_config_same(struct iavf_adapter *adapter,
3604 struct tc_mqprio_qopt *mqprio_qopt)
3605 {
3606 struct virtchnl_channel_info *ch = &adapter->ch_config.ch_info[0];
3607 int i;
3608
3609 if (adapter->num_tc != mqprio_qopt->num_tc)
3610 return false;
3611
3612 for (i = 0; i < adapter->num_tc; i++) {
3613 if (ch[i].count != mqprio_qopt->count[i] ||
3614 ch[i].offset != mqprio_qopt->offset[i])
3615 return false;
3616 }
3617 return true;
3618 }
3619
3620 /**
3621 * __iavf_setup_tc - configure multiple traffic classes
3622 * @netdev: network interface device structure
3623 * @type_data: tc offload data
3624 *
3625 * This function processes the config information provided by the
3626 * user to configure traffic classes/queue channels and packages the
3627 * information to request the PF to setup traffic classes.
3628 *
3629 * Returns 0 on success.
3630 **/
__iavf_setup_tc(struct net_device * netdev,void * type_data)3631 static int __iavf_setup_tc(struct net_device *netdev, void *type_data)
3632 {
3633 struct tc_mqprio_qopt_offload *mqprio_qopt = type_data;
3634 struct iavf_adapter *adapter = netdev_priv(netdev);
3635 struct virtchnl_vf_resource *vfres = adapter->vf_res;
3636 u8 num_tc = 0, total_qps = 0;
3637 int ret = 0, netdev_tc = 0;
3638 u64 max_tx_rate;
3639 u16 mode;
3640 int i;
3641
3642 num_tc = mqprio_qopt->qopt.num_tc;
3643 mode = mqprio_qopt->mode;
3644
3645 /* delete queue_channel */
3646 if (!mqprio_qopt->qopt.hw) {
3647 if (adapter->ch_config.state == __IAVF_TC_RUNNING) {
3648 /* reset the tc configuration */
3649 netdev_reset_tc(netdev);
3650 adapter->num_tc = 0;
3651 netif_tx_stop_all_queues(netdev);
3652 netif_tx_disable(netdev);
3653 iavf_del_all_cloud_filters(adapter);
3654 adapter->aq_required = IAVF_FLAG_AQ_DISABLE_CHANNELS;
3655 total_qps = adapter->orig_num_active_queues;
3656 goto exit;
3657 } else {
3658 return -EINVAL;
3659 }
3660 }
3661
3662 /* add queue channel */
3663 if (mode == TC_MQPRIO_MODE_CHANNEL) {
3664 if (!(vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)) {
3665 dev_err(&adapter->pdev->dev, "ADq not supported\n");
3666 return -EOPNOTSUPP;
3667 }
3668 if (adapter->ch_config.state != __IAVF_TC_INVALID) {
3669 dev_err(&adapter->pdev->dev, "TC configuration already exists\n");
3670 return -EINVAL;
3671 }
3672
3673 ret = iavf_validate_ch_config(adapter, mqprio_qopt);
3674 if (ret)
3675 return ret;
3676 /* Return if same TC config is requested */
3677 if (iavf_is_tc_config_same(adapter, &mqprio_qopt->qopt))
3678 return 0;
3679 adapter->num_tc = num_tc;
3680
3681 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3682 if (i < num_tc) {
3683 adapter->ch_config.ch_info[i].count =
3684 mqprio_qopt->qopt.count[i];
3685 adapter->ch_config.ch_info[i].offset =
3686 mqprio_qopt->qopt.offset[i];
3687 total_qps += mqprio_qopt->qopt.count[i];
3688 max_tx_rate = mqprio_qopt->max_rate[i];
3689 /* convert to Mbps */
3690 max_tx_rate = div_u64(max_tx_rate,
3691 IAVF_MBPS_DIVISOR);
3692 adapter->ch_config.ch_info[i].max_tx_rate =
3693 max_tx_rate;
3694 } else {
3695 adapter->ch_config.ch_info[i].count = 1;
3696 adapter->ch_config.ch_info[i].offset = 0;
3697 }
3698 }
3699
3700 /* Take snapshot of original config such as "num_active_queues"
3701 * It is used later when delete ADQ flow is exercised, so that
3702 * once delete ADQ flow completes, VF shall go back to its
3703 * original queue configuration
3704 */
3705
3706 adapter->orig_num_active_queues = adapter->num_active_queues;
3707
3708 /* Store queue info based on TC so that VF gets configured
3709 * with correct number of queues when VF completes ADQ config
3710 * flow
3711 */
3712 adapter->ch_config.total_qps = total_qps;
3713
3714 netif_tx_stop_all_queues(netdev);
3715 netif_tx_disable(netdev);
3716 adapter->aq_required |= IAVF_FLAG_AQ_ENABLE_CHANNELS;
3717 netdev_reset_tc(netdev);
3718 /* Report the tc mapping up the stack */
3719 netdev_set_num_tc(adapter->netdev, num_tc);
3720 for (i = 0; i < IAVF_MAX_TRAFFIC_CLASS; i++) {
3721 u16 qcount = mqprio_qopt->qopt.count[i];
3722 u16 qoffset = mqprio_qopt->qopt.offset[i];
3723
3724 if (i < num_tc)
3725 netdev_set_tc_queue(netdev, netdev_tc++, qcount,
3726 qoffset);
3727 }
3728 }
3729 exit:
3730 if (test_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
3731 return 0;
3732
3733 netif_set_real_num_rx_queues(netdev, total_qps);
3734 netif_set_real_num_tx_queues(netdev, total_qps);
3735
3736 return ret;
3737 }
3738
3739 /**
3740 * iavf_parse_cls_flower - Parse tc flower filters provided by kernel
3741 * @adapter: board private structure
3742 * @f: pointer to struct flow_cls_offload
3743 * @filter: pointer to cloud filter structure
3744 */
iavf_parse_cls_flower(struct iavf_adapter * adapter,struct flow_cls_offload * f,struct iavf_cloud_filter * filter)3745 static int iavf_parse_cls_flower(struct iavf_adapter *adapter,
3746 struct flow_cls_offload *f,
3747 struct iavf_cloud_filter *filter)
3748 {
3749 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
3750 struct flow_dissector *dissector = rule->match.dissector;
3751 u16 n_proto_mask = 0;
3752 u16 n_proto_key = 0;
3753 u8 field_flags = 0;
3754 u16 addr_type = 0;
3755 u16 n_proto = 0;
3756 int i = 0;
3757 struct virtchnl_filter *vf = &filter->f;
3758
3759 if (dissector->used_keys &
3760 ~(BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
3761 BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
3762 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
3763 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN) |
3764 BIT_ULL(FLOW_DISSECTOR_KEY_IPV4_ADDRS) |
3765 BIT_ULL(FLOW_DISSECTOR_KEY_IPV6_ADDRS) |
3766 BIT_ULL(FLOW_DISSECTOR_KEY_PORTS) |
3767 BIT_ULL(FLOW_DISSECTOR_KEY_ENC_KEYID))) {
3768 dev_err(&adapter->pdev->dev, "Unsupported key used: 0x%llx\n",
3769 dissector->used_keys);
3770 return -EOPNOTSUPP;
3771 }
3772
3773 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_KEYID)) {
3774 struct flow_match_enc_keyid match;
3775
3776 flow_rule_match_enc_keyid(rule, &match);
3777 if (match.mask->keyid != 0)
3778 field_flags |= IAVF_CLOUD_FIELD_TEN_ID;
3779 }
3780
3781 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
3782 struct flow_match_basic match;
3783
3784 flow_rule_match_basic(rule, &match);
3785 n_proto_key = ntohs(match.key->n_proto);
3786 n_proto_mask = ntohs(match.mask->n_proto);
3787
3788 if (n_proto_key == ETH_P_ALL) {
3789 n_proto_key = 0;
3790 n_proto_mask = 0;
3791 }
3792 n_proto = n_proto_key & n_proto_mask;
3793 if (n_proto != ETH_P_IP && n_proto != ETH_P_IPV6)
3794 return -EINVAL;
3795 if (n_proto == ETH_P_IPV6) {
3796 /* specify flow type as TCP IPv6 */
3797 vf->flow_type = VIRTCHNL_TCP_V6_FLOW;
3798 }
3799
3800 if (match.key->ip_proto != IPPROTO_TCP) {
3801 dev_info(&adapter->pdev->dev, "Only TCP transport is supported\n");
3802 return -EINVAL;
3803 }
3804 }
3805
3806 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
3807 struct flow_match_eth_addrs match;
3808
3809 flow_rule_match_eth_addrs(rule, &match);
3810
3811 /* use is_broadcast and is_zero to check for all 0xf or 0 */
3812 if (!is_zero_ether_addr(match.mask->dst)) {
3813 if (is_broadcast_ether_addr(match.mask->dst)) {
3814 field_flags |= IAVF_CLOUD_FIELD_OMAC;
3815 } else {
3816 dev_err(&adapter->pdev->dev, "Bad ether dest mask %pM\n",
3817 match.mask->dst);
3818 return -EINVAL;
3819 }
3820 }
3821
3822 if (!is_zero_ether_addr(match.mask->src)) {
3823 if (is_broadcast_ether_addr(match.mask->src)) {
3824 field_flags |= IAVF_CLOUD_FIELD_IMAC;
3825 } else {
3826 dev_err(&adapter->pdev->dev, "Bad ether src mask %pM\n",
3827 match.mask->src);
3828 return -EINVAL;
3829 }
3830 }
3831
3832 if (!is_zero_ether_addr(match.key->dst))
3833 if (is_valid_ether_addr(match.key->dst) ||
3834 is_multicast_ether_addr(match.key->dst)) {
3835 /* set the mask if a valid dst_mac address */
3836 for (i = 0; i < ETH_ALEN; i++)
3837 vf->mask.tcp_spec.dst_mac[i] |= 0xff;
3838 ether_addr_copy(vf->data.tcp_spec.dst_mac,
3839 match.key->dst);
3840 }
3841
3842 if (!is_zero_ether_addr(match.key->src))
3843 if (is_valid_ether_addr(match.key->src) ||
3844 is_multicast_ether_addr(match.key->src)) {
3845 /* set the mask if a valid dst_mac address */
3846 for (i = 0; i < ETH_ALEN; i++)
3847 vf->mask.tcp_spec.src_mac[i] |= 0xff;
3848 ether_addr_copy(vf->data.tcp_spec.src_mac,
3849 match.key->src);
3850 }
3851 }
3852
3853 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
3854 struct flow_match_vlan match;
3855
3856 flow_rule_match_vlan(rule, &match);
3857 if (match.mask->vlan_id) {
3858 if (match.mask->vlan_id == VLAN_VID_MASK) {
3859 field_flags |= IAVF_CLOUD_FIELD_IVLAN;
3860 } else {
3861 dev_err(&adapter->pdev->dev, "Bad vlan mask %u\n",
3862 match.mask->vlan_id);
3863 return -EINVAL;
3864 }
3865 }
3866 vf->mask.tcp_spec.vlan_id |= cpu_to_be16(0xffff);
3867 vf->data.tcp_spec.vlan_id = cpu_to_be16(match.key->vlan_id);
3868 }
3869
3870 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_CONTROL)) {
3871 struct flow_match_control match;
3872
3873 flow_rule_match_control(rule, &match);
3874 addr_type = match.key->addr_type;
3875 }
3876
3877 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
3878 struct flow_match_ipv4_addrs match;
3879
3880 flow_rule_match_ipv4_addrs(rule, &match);
3881 if (match.mask->dst) {
3882 if (match.mask->dst == cpu_to_be32(0xffffffff)) {
3883 field_flags |= IAVF_CLOUD_FIELD_IIP;
3884 } else {
3885 dev_err(&adapter->pdev->dev, "Bad ip dst mask 0x%08x\n",
3886 be32_to_cpu(match.mask->dst));
3887 return -EINVAL;
3888 }
3889 }
3890
3891 if (match.mask->src) {
3892 if (match.mask->src == cpu_to_be32(0xffffffff)) {
3893 field_flags |= IAVF_CLOUD_FIELD_IIP;
3894 } else {
3895 dev_err(&adapter->pdev->dev, "Bad ip src mask 0x%08x\n",
3896 be32_to_cpu(match.mask->src));
3897 return -EINVAL;
3898 }
3899 }
3900
3901 if (field_flags & IAVF_CLOUD_FIELD_TEN_ID) {
3902 dev_info(&adapter->pdev->dev, "Tenant id not allowed for ip filter\n");
3903 return -EINVAL;
3904 }
3905 if (match.key->dst) {
3906 vf->mask.tcp_spec.dst_ip[0] |= cpu_to_be32(0xffffffff);
3907 vf->data.tcp_spec.dst_ip[0] = match.key->dst;
3908 }
3909 if (match.key->src) {
3910 vf->mask.tcp_spec.src_ip[0] |= cpu_to_be32(0xffffffff);
3911 vf->data.tcp_spec.src_ip[0] = match.key->src;
3912 }
3913 }
3914
3915 if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
3916 struct flow_match_ipv6_addrs match;
3917
3918 flow_rule_match_ipv6_addrs(rule, &match);
3919
3920 /* validate mask, make sure it is not IPV6_ADDR_ANY */
3921 if (ipv6_addr_any(&match.mask->dst)) {
3922 dev_err(&adapter->pdev->dev, "Bad ipv6 dst mask 0x%02x\n",
3923 IPV6_ADDR_ANY);
3924 return -EINVAL;
3925 }
3926
3927 /* src and dest IPv6 address should not be LOOPBACK
3928 * (0:0:0:0:0:0:0:1) which can be represented as ::1
3929 */
3930 if (ipv6_addr_loopback(&match.key->dst) ||
3931 ipv6_addr_loopback(&match.key->src)) {
3932 dev_err(&adapter->pdev->dev,
3933 "ipv6 addr should not be loopback\n");
3934 return -EINVAL;
3935 }
3936 if (!ipv6_addr_any(&match.mask->dst) ||
3937 !ipv6_addr_any(&match.mask->src))
3938 field_flags |= IAVF_CLOUD_FIELD_IIP;
3939
3940 for (i = 0; i < 4; i++)
3941 vf->mask.tcp_spec.dst_ip[i] |= cpu_to_be32(0xffffffff);
3942 memcpy(&vf->data.tcp_spec.dst_ip, &match.key->dst.s6_addr32,
3943 sizeof(vf->data.tcp_spec.dst_ip));
3944 for (i = 0; i < 4; i++)
3945 vf->mask.tcp_spec.src_ip[i] |= cpu_to_be32(0xffffffff);
3946 memcpy(&vf->data.tcp_spec.src_ip, &match.key->src.s6_addr32,
3947 sizeof(vf->data.tcp_spec.src_ip));
3948 }
3949 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_PORTS)) {
3950 struct flow_match_ports match;
3951
3952 flow_rule_match_ports(rule, &match);
3953 if (match.mask->src) {
3954 if (match.mask->src == cpu_to_be16(0xffff)) {
3955 field_flags |= IAVF_CLOUD_FIELD_IIP;
3956 } else {
3957 dev_err(&adapter->pdev->dev, "Bad src port mask %u\n",
3958 be16_to_cpu(match.mask->src));
3959 return -EINVAL;
3960 }
3961 }
3962
3963 if (match.mask->dst) {
3964 if (match.mask->dst == cpu_to_be16(0xffff)) {
3965 field_flags |= IAVF_CLOUD_FIELD_IIP;
3966 } else {
3967 dev_err(&adapter->pdev->dev, "Bad dst port mask %u\n",
3968 be16_to_cpu(match.mask->dst));
3969 return -EINVAL;
3970 }
3971 }
3972 if (match.key->dst) {
3973 vf->mask.tcp_spec.dst_port |= cpu_to_be16(0xffff);
3974 vf->data.tcp_spec.dst_port = match.key->dst;
3975 }
3976
3977 if (match.key->src) {
3978 vf->mask.tcp_spec.src_port |= cpu_to_be16(0xffff);
3979 vf->data.tcp_spec.src_port = match.key->src;
3980 }
3981 }
3982 vf->field_flags = field_flags;
3983
3984 return 0;
3985 }
3986
3987 /**
3988 * iavf_handle_tclass - Forward to a traffic class on the device
3989 * @adapter: board private structure
3990 * @tc: traffic class index on the device
3991 * @filter: pointer to cloud filter structure
3992 */
iavf_handle_tclass(struct iavf_adapter * adapter,u32 tc,struct iavf_cloud_filter * filter)3993 static int iavf_handle_tclass(struct iavf_adapter *adapter, u32 tc,
3994 struct iavf_cloud_filter *filter)
3995 {
3996 if (tc == 0)
3997 return 0;
3998 if (tc < adapter->num_tc) {
3999 if (!filter->f.data.tcp_spec.dst_port) {
4000 dev_err(&adapter->pdev->dev,
4001 "Specify destination port to redirect to traffic class other than TC0\n");
4002 return -EINVAL;
4003 }
4004 }
4005 /* redirect to a traffic class on the same device */
4006 filter->f.action = VIRTCHNL_ACTION_TC_REDIRECT;
4007 filter->f.action_meta = tc;
4008 return 0;
4009 }
4010
4011 /**
4012 * iavf_find_cf - Find the cloud filter in the list
4013 * @adapter: Board private structure
4014 * @cookie: filter specific cookie
4015 *
4016 * Returns ptr to the filter object or NULL. Must be called while holding the
4017 * cloud_filter_list_lock.
4018 */
iavf_find_cf(struct iavf_adapter * adapter,unsigned long * cookie)4019 static struct iavf_cloud_filter *iavf_find_cf(struct iavf_adapter *adapter,
4020 unsigned long *cookie)
4021 {
4022 struct iavf_cloud_filter *filter = NULL;
4023
4024 if (!cookie)
4025 return NULL;
4026
4027 list_for_each_entry(filter, &adapter->cloud_filter_list, list) {
4028 if (!memcmp(cookie, &filter->cookie, sizeof(filter->cookie)))
4029 return filter;
4030 }
4031 return NULL;
4032 }
4033
4034 /**
4035 * iavf_configure_clsflower - Add tc flower filters
4036 * @adapter: board private structure
4037 * @cls_flower: Pointer to struct flow_cls_offload
4038 */
iavf_configure_clsflower(struct iavf_adapter * adapter,struct flow_cls_offload * cls_flower)4039 static int iavf_configure_clsflower(struct iavf_adapter *adapter,
4040 struct flow_cls_offload *cls_flower)
4041 {
4042 int tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
4043 struct iavf_cloud_filter *filter = NULL;
4044 int err = -EINVAL, count = 50;
4045
4046 if (tc < 0) {
4047 dev_err(&adapter->pdev->dev, "Invalid traffic class\n");
4048 return -EINVAL;
4049 }
4050
4051 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
4052 if (!filter)
4053 return -ENOMEM;
4054
4055 while (!mutex_trylock(&adapter->crit_lock)) {
4056 if (--count == 0) {
4057 kfree(filter);
4058 return err;
4059 }
4060 udelay(1);
4061 }
4062
4063 filter->cookie = cls_flower->cookie;
4064
4065 /* bail out here if filter already exists */
4066 spin_lock_bh(&adapter->cloud_filter_list_lock);
4067 if (iavf_find_cf(adapter, &cls_flower->cookie)) {
4068 dev_err(&adapter->pdev->dev, "Failed to add TC Flower filter, it already exists\n");
4069 err = -EEXIST;
4070 goto spin_unlock;
4071 }
4072 spin_unlock_bh(&adapter->cloud_filter_list_lock);
4073
4074 /* set the mask to all zeroes to begin with */
4075 memset(&filter->f.mask.tcp_spec, 0, sizeof(struct virtchnl_l4_spec));
4076 /* start out with flow type and eth type IPv4 to begin with */
4077 filter->f.flow_type = VIRTCHNL_TCP_V4_FLOW;
4078 err = iavf_parse_cls_flower(adapter, cls_flower, filter);
4079 if (err)
4080 goto err;
4081
4082 err = iavf_handle_tclass(adapter, tc, filter);
4083 if (err)
4084 goto err;
4085
4086 /* add filter to the list */
4087 spin_lock_bh(&adapter->cloud_filter_list_lock);
4088 list_add_tail(&filter->list, &adapter->cloud_filter_list);
4089 adapter->num_cloud_filters++;
4090 filter->add = true;
4091 adapter->aq_required |= IAVF_FLAG_AQ_ADD_CLOUD_FILTER;
4092 spin_unlock:
4093 spin_unlock_bh(&adapter->cloud_filter_list_lock);
4094 err:
4095 if (err)
4096 kfree(filter);
4097
4098 mutex_unlock(&adapter->crit_lock);
4099 return err;
4100 }
4101
4102 /**
4103 * iavf_delete_clsflower - Remove tc flower filters
4104 * @adapter: board private structure
4105 * @cls_flower: Pointer to struct flow_cls_offload
4106 */
iavf_delete_clsflower(struct iavf_adapter * adapter,struct flow_cls_offload * cls_flower)4107 static int iavf_delete_clsflower(struct iavf_adapter *adapter,
4108 struct flow_cls_offload *cls_flower)
4109 {
4110 struct iavf_cloud_filter *filter = NULL;
4111 int err = 0;
4112
4113 spin_lock_bh(&adapter->cloud_filter_list_lock);
4114 filter = iavf_find_cf(adapter, &cls_flower->cookie);
4115 if (filter) {
4116 filter->del = true;
4117 adapter->aq_required |= IAVF_FLAG_AQ_DEL_CLOUD_FILTER;
4118 } else {
4119 err = -EINVAL;
4120 }
4121 spin_unlock_bh(&adapter->cloud_filter_list_lock);
4122
4123 return err;
4124 }
4125
4126 /**
4127 * iavf_setup_tc_cls_flower - flower classifier offloads
4128 * @adapter: board private structure
4129 * @cls_flower: pointer to flow_cls_offload struct with flow info
4130 */
iavf_setup_tc_cls_flower(struct iavf_adapter * adapter,struct flow_cls_offload * cls_flower)4131 static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter,
4132 struct flow_cls_offload *cls_flower)
4133 {
4134 switch (cls_flower->command) {
4135 case FLOW_CLS_REPLACE:
4136 return iavf_configure_clsflower(adapter, cls_flower);
4137 case FLOW_CLS_DESTROY:
4138 return iavf_delete_clsflower(adapter, cls_flower);
4139 case FLOW_CLS_STATS:
4140 return -EOPNOTSUPP;
4141 default:
4142 return -EOPNOTSUPP;
4143 }
4144 }
4145
4146 /**
4147 * iavf_setup_tc_block_cb - block callback for tc
4148 * @type: type of offload
4149 * @type_data: offload data
4150 * @cb_priv:
4151 *
4152 * This function is the block callback for traffic classes
4153 **/
iavf_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)4154 static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4155 void *cb_priv)
4156 {
4157 struct iavf_adapter *adapter = cb_priv;
4158
4159 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
4160 return -EOPNOTSUPP;
4161
4162 switch (type) {
4163 case TC_SETUP_CLSFLOWER:
4164 return iavf_setup_tc_cls_flower(cb_priv, type_data);
4165 default:
4166 return -EOPNOTSUPP;
4167 }
4168 }
4169
4170 static LIST_HEAD(iavf_block_cb_list);
4171
4172 /**
4173 * iavf_setup_tc - configure multiple traffic classes
4174 * @netdev: network interface device structure
4175 * @type: type of offload
4176 * @type_data: tc offload data
4177 *
4178 * This function is the callback to ndo_setup_tc in the
4179 * netdev_ops.
4180 *
4181 * Returns 0 on success
4182 **/
iavf_setup_tc(struct net_device * netdev,enum tc_setup_type type,void * type_data)4183 static int iavf_setup_tc(struct net_device *netdev, enum tc_setup_type type,
4184 void *type_data)
4185 {
4186 struct iavf_adapter *adapter = netdev_priv(netdev);
4187
4188 switch (type) {
4189 case TC_SETUP_QDISC_MQPRIO:
4190 return __iavf_setup_tc(netdev, type_data);
4191 case TC_SETUP_BLOCK:
4192 return flow_block_cb_setup_simple(type_data,
4193 &iavf_block_cb_list,
4194 iavf_setup_tc_block_cb,
4195 adapter, adapter, true);
4196 default:
4197 return -EOPNOTSUPP;
4198 }
4199 }
4200
4201 /**
4202 * iavf_restore_fdir_filters
4203 * @adapter: board private structure
4204 *
4205 * Restore existing FDIR filters when VF netdev comes back up.
4206 **/
iavf_restore_fdir_filters(struct iavf_adapter * adapter)4207 static void iavf_restore_fdir_filters(struct iavf_adapter *adapter)
4208 {
4209 struct iavf_fdir_fltr *f;
4210
4211 spin_lock_bh(&adapter->fdir_fltr_lock);
4212 list_for_each_entry(f, &adapter->fdir_list_head, list) {
4213 if (f->state == IAVF_FDIR_FLTR_DIS_REQUEST) {
4214 /* Cancel a request, keep filter as active */
4215 f->state = IAVF_FDIR_FLTR_ACTIVE;
4216 } else if (f->state == IAVF_FDIR_FLTR_DIS_PENDING ||
4217 f->state == IAVF_FDIR_FLTR_INACTIVE) {
4218 /* Add filters which are inactive or have a pending
4219 * request to PF to be deleted
4220 */
4221 f->state = IAVF_FDIR_FLTR_ADD_REQUEST;
4222 adapter->aq_required |= IAVF_FLAG_AQ_ADD_FDIR_FILTER;
4223 }
4224 }
4225 spin_unlock_bh(&adapter->fdir_fltr_lock);
4226 }
4227
4228 /**
4229 * iavf_open - Called when a network interface is made active
4230 * @netdev: network interface device structure
4231 *
4232 * Returns 0 on success, negative value on failure
4233 *
4234 * The open entry point is called when a network interface is made
4235 * active by the system (IFF_UP). At this point all resources needed
4236 * for transmit and receive operations are allocated, the interrupt
4237 * handler is registered with the OS, the watchdog is started,
4238 * and the stack is notified that the interface is ready.
4239 **/
iavf_open(struct net_device * netdev)4240 static int iavf_open(struct net_device *netdev)
4241 {
4242 struct iavf_adapter *adapter = netdev_priv(netdev);
4243 int err;
4244
4245 if (adapter->flags & IAVF_FLAG_PF_COMMS_FAILED) {
4246 dev_err(&adapter->pdev->dev, "Unable to open device due to PF driver failure.\n");
4247 return -EIO;
4248 }
4249
4250 while (!mutex_trylock(&adapter->crit_lock)) {
4251 /* If we are in __IAVF_INIT_CONFIG_ADAPTER state the crit_lock
4252 * is already taken and iavf_open is called from an upper
4253 * device's notifier reacting on NETDEV_REGISTER event.
4254 * We have to leave here to avoid dead lock.
4255 */
4256 if (adapter->state == __IAVF_INIT_CONFIG_ADAPTER)
4257 return -EBUSY;
4258
4259 usleep_range(500, 1000);
4260 }
4261
4262 if (adapter->state != __IAVF_DOWN) {
4263 err = -EBUSY;
4264 goto err_unlock;
4265 }
4266
4267 if (adapter->state == __IAVF_RUNNING &&
4268 !test_bit(__IAVF_VSI_DOWN, adapter->vsi.state)) {
4269 dev_dbg(&adapter->pdev->dev, "VF is already open.\n");
4270 err = 0;
4271 goto err_unlock;
4272 }
4273
4274 /* allocate transmit descriptors */
4275 err = iavf_setup_all_tx_resources(adapter);
4276 if (err)
4277 goto err_setup_tx;
4278
4279 /* allocate receive descriptors */
4280 err = iavf_setup_all_rx_resources(adapter);
4281 if (err)
4282 goto err_setup_rx;
4283
4284 /* clear any pending interrupts, may auto mask */
4285 err = iavf_request_traffic_irqs(adapter, netdev->name);
4286 if (err)
4287 goto err_req_irq;
4288
4289 spin_lock_bh(&adapter->mac_vlan_list_lock);
4290
4291 iavf_add_filter(adapter, adapter->hw.mac.addr);
4292
4293 spin_unlock_bh(&adapter->mac_vlan_list_lock);
4294
4295 /* Restore filters that were removed with IFF_DOWN */
4296 iavf_restore_filters(adapter);
4297 iavf_restore_fdir_filters(adapter);
4298
4299 iavf_configure(adapter);
4300
4301 iavf_up_complete(adapter);
4302
4303 iavf_irq_enable(adapter, true);
4304
4305 mutex_unlock(&adapter->crit_lock);
4306
4307 return 0;
4308
4309 err_req_irq:
4310 iavf_down(adapter);
4311 iavf_free_traffic_irqs(adapter);
4312 err_setup_rx:
4313 iavf_free_all_rx_resources(adapter);
4314 err_setup_tx:
4315 iavf_free_all_tx_resources(adapter);
4316 err_unlock:
4317 mutex_unlock(&adapter->crit_lock);
4318
4319 return err;
4320 }
4321
4322 /**
4323 * iavf_close - Disables a network interface
4324 * @netdev: network interface device structure
4325 *
4326 * Returns 0, this is not allowed to fail
4327 *
4328 * The close entry point is called when an interface is de-activated
4329 * by the OS. The hardware is still under the drivers control, but
4330 * needs to be disabled. All IRQs except vector 0 (reserved for admin queue)
4331 * are freed, along with all transmit and receive resources.
4332 **/
iavf_close(struct net_device * netdev)4333 static int iavf_close(struct net_device *netdev)
4334 {
4335 struct iavf_adapter *adapter = netdev_priv(netdev);
4336 u64 aq_to_restore;
4337 int status;
4338
4339 mutex_lock(&adapter->crit_lock);
4340
4341 if (adapter->state <= __IAVF_DOWN_PENDING) {
4342 mutex_unlock(&adapter->crit_lock);
4343 return 0;
4344 }
4345
4346 set_bit(__IAVF_VSI_DOWN, adapter->vsi.state);
4347 if (CLIENT_ENABLED(adapter))
4348 adapter->flags |= IAVF_FLAG_CLIENT_NEEDS_CLOSE;
4349 /* We cannot send IAVF_FLAG_AQ_GET_OFFLOAD_VLAN_V2_CAPS before
4350 * IAVF_FLAG_AQ_DISABLE_QUEUES because in such case there is rtnl
4351 * deadlock with adminq_task() until iavf_close timeouts. We must send
4352 * IAVF_FLAG_AQ_GET_CONFIG before IAVF_FLAG_AQ_DISABLE_QUEUES to make
4353 * disable queues possible for vf. Give only necessary flags to
4354 * iavf_down and save other to set them right before iavf_close()
4355 * returns, when IAVF_FLAG_AQ_DISABLE_QUEUES will be already sent and
4356 * iavf will be in DOWN state.
4357 */
4358 aq_to_restore = adapter->aq_required;
4359 adapter->aq_required &= IAVF_FLAG_AQ_GET_CONFIG;
4360
4361 /* Remove flags which we do not want to send after close or we want to
4362 * send before disable queues.
4363 */
4364 aq_to_restore &= ~(IAVF_FLAG_AQ_GET_CONFIG |
4365 IAVF_FLAG_AQ_ENABLE_QUEUES |
4366 IAVF_FLAG_AQ_CONFIGURE_QUEUES |
4367 IAVF_FLAG_AQ_ADD_VLAN_FILTER |
4368 IAVF_FLAG_AQ_ADD_MAC_FILTER |
4369 IAVF_FLAG_AQ_ADD_CLOUD_FILTER |
4370 IAVF_FLAG_AQ_ADD_FDIR_FILTER |
4371 IAVF_FLAG_AQ_ADD_ADV_RSS_CFG);
4372
4373 iavf_down(adapter);
4374 iavf_change_state(adapter, __IAVF_DOWN_PENDING);
4375 iavf_free_traffic_irqs(adapter);
4376
4377 mutex_unlock(&adapter->crit_lock);
4378
4379 /* We explicitly don't free resources here because the hardware is
4380 * still active and can DMA into memory. Resources are cleared in
4381 * iavf_virtchnl_completion() after we get confirmation from the PF
4382 * driver that the rings have been stopped.
4383 *
4384 * Also, we wait for state to transition to __IAVF_DOWN before
4385 * returning. State change occurs in iavf_virtchnl_completion() after
4386 * VF resources are released (which occurs after PF driver processes and
4387 * responds to admin queue commands).
4388 */
4389
4390 status = wait_event_timeout(adapter->down_waitqueue,
4391 adapter->state == __IAVF_DOWN,
4392 msecs_to_jiffies(500));
4393 if (!status)
4394 netdev_warn(netdev, "Device resources not yet released\n");
4395
4396 mutex_lock(&adapter->crit_lock);
4397 adapter->aq_required |= aq_to_restore;
4398 mutex_unlock(&adapter->crit_lock);
4399 return 0;
4400 }
4401
4402 /**
4403 * iavf_change_mtu - Change the Maximum Transfer Unit
4404 * @netdev: network interface device structure
4405 * @new_mtu: new value for maximum frame size
4406 *
4407 * Returns 0 on success, negative on failure
4408 **/
iavf_change_mtu(struct net_device * netdev,int new_mtu)4409 static int iavf_change_mtu(struct net_device *netdev, int new_mtu)
4410 {
4411 struct iavf_adapter *adapter = netdev_priv(netdev);
4412 int ret = 0;
4413
4414 netdev_dbg(netdev, "changing MTU from %d to %d\n",
4415 netdev->mtu, new_mtu);
4416 netdev->mtu = new_mtu;
4417 if (CLIENT_ENABLED(adapter)) {
4418 iavf_notify_client_l2_params(&adapter->vsi);
4419 adapter->flags |= IAVF_FLAG_SERVICE_CLIENT_REQUESTED;
4420 }
4421
4422 if (netif_running(netdev)) {
4423 iavf_schedule_reset(adapter, IAVF_FLAG_RESET_NEEDED);
4424 ret = iavf_wait_for_reset(adapter);
4425 if (ret < 0)
4426 netdev_warn(netdev, "MTU change interrupted waiting for reset");
4427 else if (ret)
4428 netdev_warn(netdev, "MTU change timed out waiting for reset");
4429 }
4430
4431 return ret;
4432 }
4433
4434 /**
4435 * iavf_disable_fdir - disable Flow Director and clear existing filters
4436 * @adapter: board private structure
4437 **/
iavf_disable_fdir(struct iavf_adapter * adapter)4438 static void iavf_disable_fdir(struct iavf_adapter *adapter)
4439 {
4440 struct iavf_fdir_fltr *fdir, *fdirtmp;
4441 bool del_filters = false;
4442
4443 adapter->flags &= ~IAVF_FLAG_FDIR_ENABLED;
4444
4445 /* remove all Flow Director filters */
4446 spin_lock_bh(&adapter->fdir_fltr_lock);
4447 list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head,
4448 list) {
4449 if (fdir->state == IAVF_FDIR_FLTR_ADD_REQUEST ||
4450 fdir->state == IAVF_FDIR_FLTR_INACTIVE) {
4451 /* Delete filters not registered in PF */
4452 list_del(&fdir->list);
4453 kfree(fdir);
4454 adapter->fdir_active_fltr--;
4455 } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING ||
4456 fdir->state == IAVF_FDIR_FLTR_DIS_REQUEST ||
4457 fdir->state == IAVF_FDIR_FLTR_ACTIVE) {
4458 /* Filters registered in PF, schedule their deletion */
4459 fdir->state = IAVF_FDIR_FLTR_DEL_REQUEST;
4460 del_filters = true;
4461 } else if (fdir->state == IAVF_FDIR_FLTR_DIS_PENDING) {
4462 /* Request to delete filter already sent to PF, change
4463 * state to DEL_PENDING to delete filter after PF's
4464 * response, not set as INACTIVE
4465 */
4466 fdir->state = IAVF_FDIR_FLTR_DEL_PENDING;
4467 }
4468 }
4469 spin_unlock_bh(&adapter->fdir_fltr_lock);
4470
4471 if (del_filters) {
4472 adapter->aq_required |= IAVF_FLAG_AQ_DEL_FDIR_FILTER;
4473 mod_delayed_work(adapter->wq, &adapter->watchdog_task, 0);
4474 }
4475 }
4476
4477 #define NETIF_VLAN_OFFLOAD_FEATURES (NETIF_F_HW_VLAN_CTAG_RX | \
4478 NETIF_F_HW_VLAN_CTAG_TX | \
4479 NETIF_F_HW_VLAN_STAG_RX | \
4480 NETIF_F_HW_VLAN_STAG_TX)
4481
4482 /**
4483 * iavf_set_features - set the netdev feature flags
4484 * @netdev: ptr to the netdev being adjusted
4485 * @features: the feature set that the stack is suggesting
4486 * Note: expects to be called while under rtnl_lock()
4487 **/
iavf_set_features(struct net_device * netdev,netdev_features_t features)4488 static int iavf_set_features(struct net_device *netdev,
4489 netdev_features_t features)
4490 {
4491 struct iavf_adapter *adapter = netdev_priv(netdev);
4492
4493 /* trigger update on any VLAN feature change */
4494 if ((netdev->features & NETIF_VLAN_OFFLOAD_FEATURES) ^
4495 (features & NETIF_VLAN_OFFLOAD_FEATURES))
4496 iavf_set_vlan_offload_features(adapter, netdev->features,
4497 features);
4498
4499 if ((netdev->features & NETIF_F_NTUPLE) ^ (features & NETIF_F_NTUPLE)) {
4500 if (features & NETIF_F_NTUPLE)
4501 adapter->flags |= IAVF_FLAG_FDIR_ENABLED;
4502 else
4503 iavf_disable_fdir(adapter);
4504 }
4505
4506 return 0;
4507 }
4508
4509 /**
4510 * iavf_features_check - Validate encapsulated packet conforms to limits
4511 * @skb: skb buff
4512 * @dev: This physical port's netdev
4513 * @features: Offload features that the stack believes apply
4514 **/
iavf_features_check(struct sk_buff * skb,struct net_device * dev,netdev_features_t features)4515 static netdev_features_t iavf_features_check(struct sk_buff *skb,
4516 struct net_device *dev,
4517 netdev_features_t features)
4518 {
4519 size_t len;
4520
4521 /* No point in doing any of this if neither checksum nor GSO are
4522 * being requested for this frame. We can rule out both by just
4523 * checking for CHECKSUM_PARTIAL
4524 */
4525 if (skb->ip_summed != CHECKSUM_PARTIAL)
4526 return features;
4527
4528 /* We cannot support GSO if the MSS is going to be less than
4529 * 64 bytes. If it is then we need to drop support for GSO.
4530 */
4531 if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
4532 features &= ~NETIF_F_GSO_MASK;
4533
4534 /* MACLEN can support at most 63 words */
4535 len = skb_network_header(skb) - skb->data;
4536 if (len & ~(63 * 2))
4537 goto out_err;
4538
4539 /* IPLEN and EIPLEN can support at most 127 dwords */
4540 len = skb_transport_header(skb) - skb_network_header(skb);
4541 if (len & ~(127 * 4))
4542 goto out_err;
4543
4544 if (skb->encapsulation) {
4545 /* L4TUNLEN can support 127 words */
4546 len = skb_inner_network_header(skb) - skb_transport_header(skb);
4547 if (len & ~(127 * 2))
4548 goto out_err;
4549
4550 /* IPLEN can support at most 127 dwords */
4551 len = skb_inner_transport_header(skb) -
4552 skb_inner_network_header(skb);
4553 if (len & ~(127 * 4))
4554 goto out_err;
4555 }
4556
4557 /* No need to validate L4LEN as TCP is the only protocol with a
4558 * flexible value and we support all possible values supported
4559 * by TCP, which is at most 15 dwords
4560 */
4561
4562 return features;
4563 out_err:
4564 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4565 }
4566
4567 /**
4568 * iavf_get_netdev_vlan_hw_features - get NETDEV VLAN features that can toggle on/off
4569 * @adapter: board private structure
4570 *
4571 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4572 * were negotiated determine the VLAN features that can be toggled on and off.
4573 **/
4574 static netdev_features_t
iavf_get_netdev_vlan_hw_features(struct iavf_adapter * adapter)4575 iavf_get_netdev_vlan_hw_features(struct iavf_adapter *adapter)
4576 {
4577 netdev_features_t hw_features = 0;
4578
4579 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4580 return hw_features;
4581
4582 /* Enable VLAN features if supported */
4583 if (VLAN_ALLOWED(adapter)) {
4584 hw_features |= (NETIF_F_HW_VLAN_CTAG_TX |
4585 NETIF_F_HW_VLAN_CTAG_RX);
4586 } else if (VLAN_V2_ALLOWED(adapter)) {
4587 struct virtchnl_vlan_caps *vlan_v2_caps =
4588 &adapter->vlan_v2_caps;
4589 struct virtchnl_vlan_supported_caps *stripping_support =
4590 &vlan_v2_caps->offloads.stripping_support;
4591 struct virtchnl_vlan_supported_caps *insertion_support =
4592 &vlan_v2_caps->offloads.insertion_support;
4593
4594 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4595 stripping_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4596 if (stripping_support->outer &
4597 VIRTCHNL_VLAN_ETHERTYPE_8100)
4598 hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4599 if (stripping_support->outer &
4600 VIRTCHNL_VLAN_ETHERTYPE_88A8)
4601 hw_features |= NETIF_F_HW_VLAN_STAG_RX;
4602 } else if (stripping_support->inner !=
4603 VIRTCHNL_VLAN_UNSUPPORTED &&
4604 stripping_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4605 if (stripping_support->inner &
4606 VIRTCHNL_VLAN_ETHERTYPE_8100)
4607 hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
4608 }
4609
4610 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED &&
4611 insertion_support->outer & VIRTCHNL_VLAN_TOGGLE) {
4612 if (insertion_support->outer &
4613 VIRTCHNL_VLAN_ETHERTYPE_8100)
4614 hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4615 if (insertion_support->outer &
4616 VIRTCHNL_VLAN_ETHERTYPE_88A8)
4617 hw_features |= NETIF_F_HW_VLAN_STAG_TX;
4618 } else if (insertion_support->inner &&
4619 insertion_support->inner & VIRTCHNL_VLAN_TOGGLE) {
4620 if (insertion_support->inner &
4621 VIRTCHNL_VLAN_ETHERTYPE_8100)
4622 hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
4623 }
4624 }
4625
4626 return hw_features;
4627 }
4628
4629 /**
4630 * iavf_get_netdev_vlan_features - get the enabled NETDEV VLAN fetures
4631 * @adapter: board private structure
4632 *
4633 * Depending on whether VIRTHCNL_VF_OFFLOAD_VLAN or VIRTCHNL_VF_OFFLOAD_VLAN_V2
4634 * were negotiated determine the VLAN features that are enabled by default.
4635 **/
4636 static netdev_features_t
iavf_get_netdev_vlan_features(struct iavf_adapter * adapter)4637 iavf_get_netdev_vlan_features(struct iavf_adapter *adapter)
4638 {
4639 netdev_features_t features = 0;
4640
4641 if (!adapter->vf_res || !adapter->vf_res->vf_cap_flags)
4642 return features;
4643
4644 if (VLAN_ALLOWED(adapter)) {
4645 features |= NETIF_F_HW_VLAN_CTAG_FILTER |
4646 NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX;
4647 } else if (VLAN_V2_ALLOWED(adapter)) {
4648 struct virtchnl_vlan_caps *vlan_v2_caps =
4649 &adapter->vlan_v2_caps;
4650 struct virtchnl_vlan_supported_caps *filtering_support =
4651 &vlan_v2_caps->filtering.filtering_support;
4652 struct virtchnl_vlan_supported_caps *stripping_support =
4653 &vlan_v2_caps->offloads.stripping_support;
4654 struct virtchnl_vlan_supported_caps *insertion_support =
4655 &vlan_v2_caps->offloads.insertion_support;
4656 u32 ethertype_init;
4657
4658 /* give priority to outer stripping and don't support both outer
4659 * and inner stripping
4660 */
4661 ethertype_init = vlan_v2_caps->offloads.ethertype_init;
4662 if (stripping_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4663 if (stripping_support->outer &
4664 VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4665 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4666 features |= NETIF_F_HW_VLAN_CTAG_RX;
4667 else if (stripping_support->outer &
4668 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4669 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4670 features |= NETIF_F_HW_VLAN_STAG_RX;
4671 } else if (stripping_support->inner !=
4672 VIRTCHNL_VLAN_UNSUPPORTED) {
4673 if (stripping_support->inner &
4674 VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4675 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4676 features |= NETIF_F_HW_VLAN_CTAG_RX;
4677 }
4678
4679 /* give priority to outer insertion and don't support both outer
4680 * and inner insertion
4681 */
4682 if (insertion_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4683 if (insertion_support->outer &
4684 VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4685 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4686 features |= NETIF_F_HW_VLAN_CTAG_TX;
4687 else if (insertion_support->outer &
4688 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4689 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4690 features |= NETIF_F_HW_VLAN_STAG_TX;
4691 } else if (insertion_support->inner !=
4692 VIRTCHNL_VLAN_UNSUPPORTED) {
4693 if (insertion_support->inner &
4694 VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4695 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4696 features |= NETIF_F_HW_VLAN_CTAG_TX;
4697 }
4698
4699 /* give priority to outer filtering and don't bother if both
4700 * outer and inner filtering are enabled
4701 */
4702 ethertype_init = vlan_v2_caps->filtering.ethertype_init;
4703 if (filtering_support->outer != VIRTCHNL_VLAN_UNSUPPORTED) {
4704 if (filtering_support->outer &
4705 VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4706 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4707 features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4708 if (filtering_support->outer &
4709 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4710 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4711 features |= NETIF_F_HW_VLAN_STAG_FILTER;
4712 } else if (filtering_support->inner !=
4713 VIRTCHNL_VLAN_UNSUPPORTED) {
4714 if (filtering_support->inner &
4715 VIRTCHNL_VLAN_ETHERTYPE_8100 &&
4716 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_8100)
4717 features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4718 if (filtering_support->inner &
4719 VIRTCHNL_VLAN_ETHERTYPE_88A8 &&
4720 ethertype_init & VIRTCHNL_VLAN_ETHERTYPE_88A8)
4721 features |= NETIF_F_HW_VLAN_STAG_FILTER;
4722 }
4723 }
4724
4725 return features;
4726 }
4727
4728 #define IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested, allowed, feature_bit) \
4729 (!(((requested) & (feature_bit)) && \
4730 !((allowed) & (feature_bit))))
4731
4732 /**
4733 * iavf_fix_netdev_vlan_features - fix NETDEV VLAN features based on support
4734 * @adapter: board private structure
4735 * @requested_features: stack requested NETDEV features
4736 **/
4737 static netdev_features_t
iavf_fix_netdev_vlan_features(struct iavf_adapter * adapter,netdev_features_t requested_features)4738 iavf_fix_netdev_vlan_features(struct iavf_adapter *adapter,
4739 netdev_features_t requested_features)
4740 {
4741 netdev_features_t allowed_features;
4742
4743 allowed_features = iavf_get_netdev_vlan_hw_features(adapter) |
4744 iavf_get_netdev_vlan_features(adapter);
4745
4746 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4747 allowed_features,
4748 NETIF_F_HW_VLAN_CTAG_TX))
4749 requested_features &= ~NETIF_F_HW_VLAN_CTAG_TX;
4750
4751 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4752 allowed_features,
4753 NETIF_F_HW_VLAN_CTAG_RX))
4754 requested_features &= ~NETIF_F_HW_VLAN_CTAG_RX;
4755
4756 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4757 allowed_features,
4758 NETIF_F_HW_VLAN_STAG_TX))
4759 requested_features &= ~NETIF_F_HW_VLAN_STAG_TX;
4760 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4761 allowed_features,
4762 NETIF_F_HW_VLAN_STAG_RX))
4763 requested_features &= ~NETIF_F_HW_VLAN_STAG_RX;
4764
4765 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4766 allowed_features,
4767 NETIF_F_HW_VLAN_CTAG_FILTER))
4768 requested_features &= ~NETIF_F_HW_VLAN_CTAG_FILTER;
4769
4770 if (!IAVF_NETDEV_VLAN_FEATURE_ALLOWED(requested_features,
4771 allowed_features,
4772 NETIF_F_HW_VLAN_STAG_FILTER))
4773 requested_features &= ~NETIF_F_HW_VLAN_STAG_FILTER;
4774
4775 if ((requested_features &
4776 (NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_CTAG_TX)) &&
4777 (requested_features &
4778 (NETIF_F_HW_VLAN_STAG_RX | NETIF_F_HW_VLAN_STAG_TX)) &&
4779 adapter->vlan_v2_caps.offloads.ethertype_match ==
4780 VIRTCHNL_ETHERTYPE_STRIPPING_MATCHES_INSERTION) {
4781 netdev_warn(adapter->netdev, "cannot support CTAG and STAG VLAN stripping and/or insertion simultaneously since CTAG and STAG offloads are mutually exclusive, clearing STAG offload settings\n");
4782 requested_features &= ~(NETIF_F_HW_VLAN_STAG_RX |
4783 NETIF_F_HW_VLAN_STAG_TX);
4784 }
4785
4786 return requested_features;
4787 }
4788
4789 /**
4790 * iavf_fix_features - fix up the netdev feature bits
4791 * @netdev: our net device
4792 * @features: desired feature bits
4793 *
4794 * Returns fixed-up features bits
4795 **/
iavf_fix_features(struct net_device * netdev,netdev_features_t features)4796 static netdev_features_t iavf_fix_features(struct net_device *netdev,
4797 netdev_features_t features)
4798 {
4799 struct iavf_adapter *adapter = netdev_priv(netdev);
4800
4801 if (!FDIR_FLTR_SUPPORT(adapter))
4802 features &= ~NETIF_F_NTUPLE;
4803
4804 return iavf_fix_netdev_vlan_features(adapter, features);
4805 }
4806
4807 static const struct net_device_ops iavf_netdev_ops = {
4808 .ndo_open = iavf_open,
4809 .ndo_stop = iavf_close,
4810 .ndo_start_xmit = iavf_xmit_frame,
4811 .ndo_set_rx_mode = iavf_set_rx_mode,
4812 .ndo_validate_addr = eth_validate_addr,
4813 .ndo_set_mac_address = iavf_set_mac,
4814 .ndo_change_mtu = iavf_change_mtu,
4815 .ndo_tx_timeout = iavf_tx_timeout,
4816 .ndo_vlan_rx_add_vid = iavf_vlan_rx_add_vid,
4817 .ndo_vlan_rx_kill_vid = iavf_vlan_rx_kill_vid,
4818 .ndo_features_check = iavf_features_check,
4819 .ndo_fix_features = iavf_fix_features,
4820 .ndo_set_features = iavf_set_features,
4821 .ndo_setup_tc = iavf_setup_tc,
4822 };
4823
4824 /**
4825 * iavf_check_reset_complete - check that VF reset is complete
4826 * @hw: pointer to hw struct
4827 *
4828 * Returns 0 if device is ready to use, or -EBUSY if it's in reset.
4829 **/
iavf_check_reset_complete(struct iavf_hw * hw)4830 static int iavf_check_reset_complete(struct iavf_hw *hw)
4831 {
4832 u32 rstat;
4833 int i;
4834
4835 for (i = 0; i < IAVF_RESET_WAIT_COMPLETE_COUNT; i++) {
4836 rstat = rd32(hw, IAVF_VFGEN_RSTAT) &
4837 IAVF_VFGEN_RSTAT_VFR_STATE_MASK;
4838 if ((rstat == VIRTCHNL_VFR_VFACTIVE) ||
4839 (rstat == VIRTCHNL_VFR_COMPLETED))
4840 return 0;
4841 usleep_range(10, 20);
4842 }
4843 return -EBUSY;
4844 }
4845
4846 /**
4847 * iavf_process_config - Process the config information we got from the PF
4848 * @adapter: board private structure
4849 *
4850 * Verify that we have a valid config struct, and set up our netdev features
4851 * and our VSI struct.
4852 **/
iavf_process_config(struct iavf_adapter * adapter)4853 int iavf_process_config(struct iavf_adapter *adapter)
4854 {
4855 struct virtchnl_vf_resource *vfres = adapter->vf_res;
4856 netdev_features_t hw_vlan_features, vlan_features;
4857 struct net_device *netdev = adapter->netdev;
4858 netdev_features_t hw_enc_features;
4859 netdev_features_t hw_features;
4860
4861 hw_enc_features = NETIF_F_SG |
4862 NETIF_F_IP_CSUM |
4863 NETIF_F_IPV6_CSUM |
4864 NETIF_F_HIGHDMA |
4865 NETIF_F_SOFT_FEATURES |
4866 NETIF_F_TSO |
4867 NETIF_F_TSO_ECN |
4868 NETIF_F_TSO6 |
4869 NETIF_F_SCTP_CRC |
4870 NETIF_F_RXHASH |
4871 NETIF_F_RXCSUM |
4872 0;
4873
4874 /* advertise to stack only if offloads for encapsulated packets is
4875 * supported
4876 */
4877 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ENCAP) {
4878 hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
4879 NETIF_F_GSO_GRE |
4880 NETIF_F_GSO_GRE_CSUM |
4881 NETIF_F_GSO_IPXIP4 |
4882 NETIF_F_GSO_IPXIP6 |
4883 NETIF_F_GSO_UDP_TUNNEL_CSUM |
4884 NETIF_F_GSO_PARTIAL |
4885 0;
4886
4887 if (!(vfres->vf_cap_flags &
4888 VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM))
4889 netdev->gso_partial_features |=
4890 NETIF_F_GSO_UDP_TUNNEL_CSUM;
4891
4892 netdev->gso_partial_features |= NETIF_F_GSO_GRE_CSUM;
4893 netdev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
4894 netdev->hw_enc_features |= hw_enc_features;
4895 }
4896 /* record features VLANs can make use of */
4897 netdev->vlan_features |= hw_enc_features | NETIF_F_TSO_MANGLEID;
4898
4899 /* Write features and hw_features separately to avoid polluting
4900 * with, or dropping, features that are set when we registered.
4901 */
4902 hw_features = hw_enc_features;
4903
4904 /* get HW VLAN features that can be toggled */
4905 hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter);
4906
4907 /* Enable cloud filter if ADQ is supported */
4908 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ)
4909 hw_features |= NETIF_F_HW_TC;
4910 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO)
4911 hw_features |= NETIF_F_GSO_UDP_L4;
4912
4913 netdev->hw_features |= hw_features | hw_vlan_features;
4914 vlan_features = iavf_get_netdev_vlan_features(adapter);
4915
4916 netdev->features |= hw_features | vlan_features;
4917
4918 if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_VLAN)
4919 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4920
4921 if (FDIR_FLTR_SUPPORT(adapter)) {
4922 netdev->hw_features |= NETIF_F_NTUPLE;
4923 netdev->features |= NETIF_F_NTUPLE;
4924 adapter->flags |= IAVF_FLAG_FDIR_ENABLED;
4925 }
4926
4927 netdev->priv_flags |= IFF_UNICAST_FLT;
4928
4929 /* Do not turn on offloads when they are requested to be turned off.
4930 * TSO needs minimum 576 bytes to work correctly.
4931 */
4932 if (netdev->wanted_features) {
4933 if (!(netdev->wanted_features & NETIF_F_TSO) ||
4934 netdev->mtu < 576)
4935 netdev->features &= ~NETIF_F_TSO;
4936 if (!(netdev->wanted_features & NETIF_F_TSO6) ||
4937 netdev->mtu < 576)
4938 netdev->features &= ~NETIF_F_TSO6;
4939 if (!(netdev->wanted_features & NETIF_F_TSO_ECN))
4940 netdev->features &= ~NETIF_F_TSO_ECN;
4941 if (!(netdev->wanted_features & NETIF_F_GRO))
4942 netdev->features &= ~NETIF_F_GRO;
4943 if (!(netdev->wanted_features & NETIF_F_GSO))
4944 netdev->features &= ~NETIF_F_GSO;
4945 }
4946
4947 return 0;
4948 }
4949
4950 /**
4951 * iavf_probe - Device Initialization Routine
4952 * @pdev: PCI device information struct
4953 * @ent: entry in iavf_pci_tbl
4954 *
4955 * Returns 0 on success, negative on failure
4956 *
4957 * iavf_probe initializes an adapter identified by a pci_dev structure.
4958 * The OS initialization, configuring of the adapter private structure,
4959 * and a hardware reset occur.
4960 **/
iavf_probe(struct pci_dev * pdev,const struct pci_device_id * ent)4961 static int iavf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4962 {
4963 struct net_device *netdev;
4964 struct iavf_adapter *adapter = NULL;
4965 struct iavf_hw *hw = NULL;
4966 int err;
4967
4968 err = pci_enable_device(pdev);
4969 if (err)
4970 return err;
4971
4972 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
4973 if (err) {
4974 dev_err(&pdev->dev,
4975 "DMA configuration failed: 0x%x\n", err);
4976 goto err_dma;
4977 }
4978
4979 err = pci_request_regions(pdev, iavf_driver_name);
4980 if (err) {
4981 dev_err(&pdev->dev,
4982 "pci_request_regions failed 0x%x\n", err);
4983 goto err_pci_reg;
4984 }
4985
4986 pci_set_master(pdev);
4987
4988 netdev = alloc_etherdev_mq(sizeof(struct iavf_adapter),
4989 IAVF_MAX_REQ_QUEUES);
4990 if (!netdev) {
4991 err = -ENOMEM;
4992 goto err_alloc_etherdev;
4993 }
4994
4995 SET_NETDEV_DEV(netdev, &pdev->dev);
4996
4997 pci_set_drvdata(pdev, netdev);
4998 adapter = netdev_priv(netdev);
4999
5000 adapter->netdev = netdev;
5001 adapter->pdev = pdev;
5002
5003 hw = &adapter->hw;
5004 hw->back = adapter;
5005
5006 adapter->wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM,
5007 iavf_driver_name);
5008 if (!adapter->wq) {
5009 err = -ENOMEM;
5010 goto err_alloc_wq;
5011 }
5012
5013 adapter->msg_enable = BIT(DEFAULT_DEBUG_LEVEL_SHIFT) - 1;
5014 iavf_change_state(adapter, __IAVF_STARTUP);
5015
5016 /* Call save state here because it relies on the adapter struct. */
5017 pci_save_state(pdev);
5018
5019 hw->hw_addr = ioremap(pci_resource_start(pdev, 0),
5020 pci_resource_len(pdev, 0));
5021 if (!hw->hw_addr) {
5022 err = -EIO;
5023 goto err_ioremap;
5024 }
5025 hw->vendor_id = pdev->vendor;
5026 hw->device_id = pdev->device;
5027 pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
5028 hw->subsystem_vendor_id = pdev->subsystem_vendor;
5029 hw->subsystem_device_id = pdev->subsystem_device;
5030 hw->bus.device = PCI_SLOT(pdev->devfn);
5031 hw->bus.func = PCI_FUNC(pdev->devfn);
5032 hw->bus.bus_id = pdev->bus->number;
5033
5034 /* set up the locks for the AQ, do this only once in probe
5035 * and destroy them only once in remove
5036 */
5037 mutex_init(&adapter->crit_lock);
5038 mutex_init(&adapter->client_lock);
5039 mutex_init(&hw->aq.asq_mutex);
5040 mutex_init(&hw->aq.arq_mutex);
5041
5042 spin_lock_init(&adapter->mac_vlan_list_lock);
5043 spin_lock_init(&adapter->cloud_filter_list_lock);
5044 spin_lock_init(&adapter->fdir_fltr_lock);
5045 spin_lock_init(&adapter->adv_rss_lock);
5046 spin_lock_init(&adapter->current_netdev_promisc_flags_lock);
5047
5048 INIT_LIST_HEAD(&adapter->mac_filter_list);
5049 INIT_LIST_HEAD(&adapter->vlan_filter_list);
5050 INIT_LIST_HEAD(&adapter->cloud_filter_list);
5051 INIT_LIST_HEAD(&adapter->fdir_list_head);
5052 INIT_LIST_HEAD(&adapter->adv_rss_list_head);
5053
5054 INIT_WORK(&adapter->reset_task, iavf_reset_task);
5055 INIT_WORK(&adapter->adminq_task, iavf_adminq_task);
5056 INIT_WORK(&adapter->finish_config, iavf_finish_config);
5057 INIT_DELAYED_WORK(&adapter->watchdog_task, iavf_watchdog_task);
5058 INIT_DELAYED_WORK(&adapter->client_task, iavf_client_task);
5059
5060 /* Setup the wait queue for indicating transition to down status */
5061 init_waitqueue_head(&adapter->down_waitqueue);
5062
5063 /* Setup the wait queue for indicating transition to running state */
5064 init_waitqueue_head(&adapter->reset_waitqueue);
5065
5066 /* Setup the wait queue for indicating virtchannel events */
5067 init_waitqueue_head(&adapter->vc_waitqueue);
5068
5069 queue_delayed_work(adapter->wq, &adapter->watchdog_task,
5070 msecs_to_jiffies(5 * (pdev->devfn & 0x07)));
5071 /* Initialization goes on in the work. Do not add more of it below. */
5072 return 0;
5073
5074 err_ioremap:
5075 destroy_workqueue(adapter->wq);
5076 err_alloc_wq:
5077 free_netdev(netdev);
5078 err_alloc_etherdev:
5079 pci_release_regions(pdev);
5080 err_pci_reg:
5081 err_dma:
5082 pci_disable_device(pdev);
5083 return err;
5084 }
5085
5086 /**
5087 * iavf_suspend - Power management suspend routine
5088 * @dev_d: device info pointer
5089 *
5090 * Called when the system (VM) is entering sleep/suspend.
5091 **/
iavf_suspend(struct device * dev_d)5092 static int __maybe_unused iavf_suspend(struct device *dev_d)
5093 {
5094 struct net_device *netdev = dev_get_drvdata(dev_d);
5095 struct iavf_adapter *adapter = netdev_priv(netdev);
5096
5097 netif_device_detach(netdev);
5098
5099 while (!mutex_trylock(&adapter->crit_lock))
5100 usleep_range(500, 1000);
5101
5102 if (netif_running(netdev)) {
5103 rtnl_lock();
5104 iavf_down(adapter);
5105 rtnl_unlock();
5106 }
5107 iavf_free_misc_irq(adapter);
5108 iavf_reset_interrupt_capability(adapter);
5109
5110 mutex_unlock(&adapter->crit_lock);
5111
5112 return 0;
5113 }
5114
5115 /**
5116 * iavf_resume - Power management resume routine
5117 * @dev_d: device info pointer
5118 *
5119 * Called when the system (VM) is resumed from sleep/suspend.
5120 **/
iavf_resume(struct device * dev_d)5121 static int __maybe_unused iavf_resume(struct device *dev_d)
5122 {
5123 struct pci_dev *pdev = to_pci_dev(dev_d);
5124 struct iavf_adapter *adapter;
5125 u32 err;
5126
5127 adapter = iavf_pdev_to_adapter(pdev);
5128
5129 pci_set_master(pdev);
5130
5131 rtnl_lock();
5132 err = iavf_set_interrupt_capability(adapter);
5133 if (err) {
5134 rtnl_unlock();
5135 dev_err(&pdev->dev, "Cannot enable MSI-X interrupts.\n");
5136 return err;
5137 }
5138 err = iavf_request_misc_irq(adapter);
5139 rtnl_unlock();
5140 if (err) {
5141 dev_err(&pdev->dev, "Cannot get interrupt vector.\n");
5142 return err;
5143 }
5144
5145 queue_work(adapter->wq, &adapter->reset_task);
5146
5147 netif_device_attach(adapter->netdev);
5148
5149 return err;
5150 }
5151
5152 /**
5153 * iavf_remove - Device Removal Routine
5154 * @pdev: PCI device information struct
5155 *
5156 * iavf_remove is called by the PCI subsystem to alert the driver
5157 * that it should release a PCI device. The could be caused by a
5158 * Hot-Plug event, or because the driver is going to be removed from
5159 * memory.
5160 **/
iavf_remove(struct pci_dev * pdev)5161 static void iavf_remove(struct pci_dev *pdev)
5162 {
5163 struct iavf_fdir_fltr *fdir, *fdirtmp;
5164 struct iavf_vlan_filter *vlf, *vlftmp;
5165 struct iavf_cloud_filter *cf, *cftmp;
5166 struct iavf_adv_rss *rss, *rsstmp;
5167 struct iavf_mac_filter *f, *ftmp;
5168 struct iavf_adapter *adapter;
5169 struct net_device *netdev;
5170 struct iavf_hw *hw;
5171 int err;
5172
5173 /* Don't proceed with remove if netdev is already freed */
5174 netdev = pci_get_drvdata(pdev);
5175 if (!netdev)
5176 return;
5177
5178 adapter = iavf_pdev_to_adapter(pdev);
5179 hw = &adapter->hw;
5180
5181 if (test_and_set_bit(__IAVF_IN_REMOVE_TASK, &adapter->crit_section))
5182 return;
5183
5184 /* Wait until port initialization is complete.
5185 * There are flows where register/unregister netdev may race.
5186 */
5187 while (1) {
5188 mutex_lock(&adapter->crit_lock);
5189 if (adapter->state == __IAVF_RUNNING ||
5190 adapter->state == __IAVF_DOWN ||
5191 adapter->state == __IAVF_INIT_FAILED) {
5192 mutex_unlock(&adapter->crit_lock);
5193 break;
5194 }
5195 /* Simply return if we already went through iavf_shutdown */
5196 if (adapter->state == __IAVF_REMOVE) {
5197 mutex_unlock(&adapter->crit_lock);
5198 return;
5199 }
5200
5201 mutex_unlock(&adapter->crit_lock);
5202 usleep_range(500, 1000);
5203 }
5204 cancel_delayed_work_sync(&adapter->watchdog_task);
5205 cancel_work_sync(&adapter->finish_config);
5206
5207 rtnl_lock();
5208 if (adapter->netdev_registered) {
5209 unregister_netdevice(netdev);
5210 adapter->netdev_registered = false;
5211 }
5212 rtnl_unlock();
5213
5214 if (CLIENT_ALLOWED(adapter)) {
5215 err = iavf_lan_del_device(adapter);
5216 if (err)
5217 dev_warn(&pdev->dev, "Failed to delete client device: %d\n",
5218 err);
5219 }
5220
5221 mutex_lock(&adapter->crit_lock);
5222 dev_info(&adapter->pdev->dev, "Removing device\n");
5223 iavf_change_state(adapter, __IAVF_REMOVE);
5224
5225 iavf_request_reset(adapter);
5226 msleep(50);
5227 /* If the FW isn't responding, kick it once, but only once. */
5228 if (!iavf_asq_done(hw)) {
5229 iavf_request_reset(adapter);
5230 msleep(50);
5231 }
5232
5233 iavf_misc_irq_disable(adapter);
5234 /* Shut down all the garbage mashers on the detention level */
5235 cancel_work_sync(&adapter->reset_task);
5236 cancel_delayed_work_sync(&adapter->watchdog_task);
5237 cancel_work_sync(&adapter->adminq_task);
5238 cancel_delayed_work_sync(&adapter->client_task);
5239
5240 adapter->aq_required = 0;
5241 adapter->flags &= ~IAVF_FLAG_REINIT_ITR_NEEDED;
5242
5243 iavf_free_all_tx_resources(adapter);
5244 iavf_free_all_rx_resources(adapter);
5245 iavf_free_misc_irq(adapter);
5246
5247 iavf_reset_interrupt_capability(adapter);
5248 iavf_free_q_vectors(adapter);
5249
5250 iavf_free_rss(adapter);
5251
5252 if (hw->aq.asq.count)
5253 iavf_shutdown_adminq(hw);
5254
5255 /* destroy the locks only once, here */
5256 mutex_destroy(&hw->aq.arq_mutex);
5257 mutex_destroy(&hw->aq.asq_mutex);
5258 mutex_destroy(&adapter->client_lock);
5259 mutex_unlock(&adapter->crit_lock);
5260 mutex_destroy(&adapter->crit_lock);
5261
5262 iounmap(hw->hw_addr);
5263 pci_release_regions(pdev);
5264 iavf_free_queues(adapter);
5265 kfree(adapter->vf_res);
5266 spin_lock_bh(&adapter->mac_vlan_list_lock);
5267 /* If we got removed before an up/down sequence, we've got a filter
5268 * hanging out there that we need to get rid of.
5269 */
5270 list_for_each_entry_safe(f, ftmp, &adapter->mac_filter_list, list) {
5271 list_del(&f->list);
5272 kfree(f);
5273 }
5274 list_for_each_entry_safe(vlf, vlftmp, &adapter->vlan_filter_list,
5275 list) {
5276 list_del(&vlf->list);
5277 kfree(vlf);
5278 }
5279
5280 spin_unlock_bh(&adapter->mac_vlan_list_lock);
5281
5282 spin_lock_bh(&adapter->cloud_filter_list_lock);
5283 list_for_each_entry_safe(cf, cftmp, &adapter->cloud_filter_list, list) {
5284 list_del(&cf->list);
5285 kfree(cf);
5286 }
5287 spin_unlock_bh(&adapter->cloud_filter_list_lock);
5288
5289 spin_lock_bh(&adapter->fdir_fltr_lock);
5290 list_for_each_entry_safe(fdir, fdirtmp, &adapter->fdir_list_head, list) {
5291 list_del(&fdir->list);
5292 kfree(fdir);
5293 }
5294 spin_unlock_bh(&adapter->fdir_fltr_lock);
5295
5296 spin_lock_bh(&adapter->adv_rss_lock);
5297 list_for_each_entry_safe(rss, rsstmp, &adapter->adv_rss_list_head,
5298 list) {
5299 list_del(&rss->list);
5300 kfree(rss);
5301 }
5302 spin_unlock_bh(&adapter->adv_rss_lock);
5303
5304 destroy_workqueue(adapter->wq);
5305
5306 pci_set_drvdata(pdev, NULL);
5307
5308 free_netdev(netdev);
5309
5310 pci_disable_device(pdev);
5311 }
5312
5313 /**
5314 * iavf_shutdown - Shutdown the device in preparation for a reboot
5315 * @pdev: pci device structure
5316 **/
iavf_shutdown(struct pci_dev * pdev)5317 static void iavf_shutdown(struct pci_dev *pdev)
5318 {
5319 iavf_remove(pdev);
5320
5321 if (system_state == SYSTEM_POWER_OFF)
5322 pci_set_power_state(pdev, PCI_D3hot);
5323 }
5324
5325 static SIMPLE_DEV_PM_OPS(iavf_pm_ops, iavf_suspend, iavf_resume);
5326
5327 static struct pci_driver iavf_driver = {
5328 .name = iavf_driver_name,
5329 .id_table = iavf_pci_tbl,
5330 .probe = iavf_probe,
5331 .remove = iavf_remove,
5332 .driver.pm = &iavf_pm_ops,
5333 .shutdown = iavf_shutdown,
5334 };
5335
5336 /**
5337 * iavf_init_module - Driver Registration Routine
5338 *
5339 * iavf_init_module is the first routine called when the driver is
5340 * loaded. All it does is register with the PCI subsystem.
5341 **/
iavf_init_module(void)5342 static int __init iavf_init_module(void)
5343 {
5344 pr_info("iavf: %s\n", iavf_driver_string);
5345
5346 pr_info("%s\n", iavf_copyright);
5347
5348 return pci_register_driver(&iavf_driver);
5349 }
5350
5351 module_init(iavf_init_module);
5352
5353 /**
5354 * iavf_exit_module - Driver Exit Cleanup Routine
5355 *
5356 * iavf_exit_module is called just before the driver is removed
5357 * from memory.
5358 **/
iavf_exit_module(void)5359 static void __exit iavf_exit_module(void)
5360 {
5361 pci_unregister_driver(&iavf_driver);
5362 }
5363
5364 module_exit(iavf_exit_module);
5365
5366 /* iavf_main.c */
5367