1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
4  */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #ifdef CONFIG_RFS_ACCEL
9 #include <linux/cpu_rmap.h>
10 #endif /* CONFIG_RFS_ACCEL */
11 #include <linux/ethtool.h>
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/numa.h>
15 #include <linux/pci.h>
16 #include <linux/utsname.h>
17 #include <linux/version.h>
18 #include <linux/vmalloc.h>
19 #include <net/ip.h>
20 
21 #include "ena_netdev.h"
22 #include <linux/bpf_trace.h>
23 #include "ena_pci_id_tbl.h"
24 
25 MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
26 MODULE_DESCRIPTION(DEVICE_NAME);
27 MODULE_LICENSE("GPL");
28 
29 /* Time in jiffies before concluding the transmitter is hung. */
30 #define TX_TIMEOUT  (5 * HZ)
31 
32 #define ENA_MAX_RINGS min_t(unsigned int, ENA_MAX_NUM_IO_QUEUES, num_possible_cpus())
33 
34 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \
35 		NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)
36 
37 static struct ena_aenq_handlers aenq_handlers;
38 
39 static struct workqueue_struct *ena_wq;
40 
41 MODULE_DEVICE_TABLE(pci, ena_pci_tbl);
42 
43 static int ena_rss_init_default(struct ena_adapter *adapter);
44 static void check_for_admin_com_state(struct ena_adapter *adapter);
45 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful);
46 static int ena_restore_device(struct ena_adapter *adapter);
47 
48 static void ena_init_io_rings(struct ena_adapter *adapter,
49 			      int first_index, int count);
50 static void ena_init_napi_in_range(struct ena_adapter *adapter, int first_index,
51 				   int count);
52 static void ena_del_napi_in_range(struct ena_adapter *adapter, int first_index,
53 				  int count);
54 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid);
55 static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
56 					   int first_index,
57 					   int count);
58 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid);
59 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid);
60 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget);
61 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter);
62 static void ena_free_all_io_tx_resources(struct ena_adapter *adapter);
63 static void ena_napi_disable_in_range(struct ena_adapter *adapter,
64 				      int first_index, int count);
65 static void ena_napi_enable_in_range(struct ena_adapter *adapter,
66 				     int first_index, int count);
67 static int ena_up(struct ena_adapter *adapter);
68 static void ena_down(struct ena_adapter *adapter);
69 static void ena_unmask_interrupt(struct ena_ring *tx_ring,
70 				 struct ena_ring *rx_ring);
71 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
72 				      struct ena_ring *rx_ring);
73 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
74 			      struct ena_tx_buffer *tx_info);
75 static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
76 					    int first_index, int count);
77 
78 /* Increase a stat by cnt while holding syncp seqlock on 32bit machines */
79 static void ena_increase_stat(u64 *statp, u64 cnt,
80 			      struct u64_stats_sync *syncp)
81 {
82 	u64_stats_update_begin(syncp);
83 	(*statp) += cnt;
84 	u64_stats_update_end(syncp);
85 }
86 
87 static void ena_ring_tx_doorbell(struct ena_ring *tx_ring)
88 {
89 	ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
90 	ena_increase_stat(&tx_ring->tx_stats.doorbells, 1, &tx_ring->syncp);
91 }
92 
93 static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue)
94 {
95 	struct ena_adapter *adapter = netdev_priv(dev);
96 
97 	/* Change the state of the device to trigger reset
98 	 * Check that we are not in the middle or a trigger already
99 	 */
100 
101 	if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
102 		return;
103 
104 	ena_reset_device(adapter, ENA_REGS_RESET_OS_NETDEV_WD);
105 	ena_increase_stat(&adapter->dev_stats.tx_timeout, 1, &adapter->syncp);
106 
107 	netif_err(adapter, tx_err, dev, "Transmit time out\n");
108 }
109 
110 static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)
111 {
112 	int i;
113 
114 	for (i = 0; i < adapter->num_io_queues; i++)
115 		adapter->rx_ring[i].mtu = mtu;
116 }
117 
118 static int ena_change_mtu(struct net_device *dev, int new_mtu)
119 {
120 	struct ena_adapter *adapter = netdev_priv(dev);
121 	int ret;
122 
123 	ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
124 	if (!ret) {
125 		netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu);
126 		update_rx_ring_mtu(adapter, new_mtu);
127 		dev->mtu = new_mtu;
128 	} else {
129 		netif_err(adapter, drv, dev, "Failed to set MTU to %d\n",
130 			  new_mtu);
131 	}
132 
133 	return ret;
134 }
135 
136 static int ena_xmit_common(struct net_device *dev,
137 			   struct ena_ring *ring,
138 			   struct ena_tx_buffer *tx_info,
139 			   struct ena_com_tx_ctx *ena_tx_ctx,
140 			   u16 next_to_use,
141 			   u32 bytes)
142 {
143 	struct ena_adapter *adapter = netdev_priv(dev);
144 	int rc, nb_hw_desc;
145 
146 	if (unlikely(ena_com_is_doorbell_needed(ring->ena_com_io_sq,
147 						ena_tx_ctx))) {
148 		netif_dbg(adapter, tx_queued, dev,
149 			  "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
150 			  ring->qid);
151 		ena_ring_tx_doorbell(ring);
152 	}
153 
154 	/* prepare the packet's descriptors to dma engine */
155 	rc = ena_com_prepare_tx(ring->ena_com_io_sq, ena_tx_ctx,
156 				&nb_hw_desc);
157 
158 	/* In case there isn't enough space in the queue for the packet,
159 	 * we simply drop it. All other failure reasons of
160 	 * ena_com_prepare_tx() are fatal and therefore require a device reset.
161 	 */
162 	if (unlikely(rc)) {
163 		netif_err(adapter, tx_queued, dev,
164 			  "Failed to prepare tx bufs\n");
165 		ena_increase_stat(&ring->tx_stats.prepare_ctx_err, 1,
166 				  &ring->syncp);
167 		if (rc != -ENOMEM)
168 			ena_reset_device(adapter,
169 					 ENA_REGS_RESET_DRIVER_INVALID_STATE);
170 		return rc;
171 	}
172 
173 	u64_stats_update_begin(&ring->syncp);
174 	ring->tx_stats.cnt++;
175 	ring->tx_stats.bytes += bytes;
176 	u64_stats_update_end(&ring->syncp);
177 
178 	tx_info->tx_descs = nb_hw_desc;
179 	tx_info->last_jiffies = jiffies;
180 	tx_info->print_once = 0;
181 
182 	ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
183 						 ring->ring_size);
184 	return 0;
185 }
186 
187 /* This is the XDP napi callback. XDP queues use a separate napi callback
188  * than Rx/Tx queues.
189  */
190 static int ena_xdp_io_poll(struct napi_struct *napi, int budget)
191 {
192 	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
193 	u32 xdp_work_done, xdp_budget;
194 	struct ena_ring *xdp_ring;
195 	int napi_comp_call = 0;
196 	int ret;
197 
198 	xdp_ring = ena_napi->xdp_ring;
199 
200 	xdp_budget = budget;
201 
202 	if (!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags) ||
203 	    test_bit(ENA_FLAG_TRIGGER_RESET, &xdp_ring->adapter->flags)) {
204 		napi_complete_done(napi, 0);
205 		return 0;
206 	}
207 
208 	xdp_work_done = ena_clean_xdp_irq(xdp_ring, xdp_budget);
209 
210 	/* If the device is about to reset or down, avoid unmask
211 	 * the interrupt and return 0 so NAPI won't reschedule
212 	 */
213 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags))) {
214 		napi_complete_done(napi, 0);
215 		ret = 0;
216 	} else if (xdp_budget > xdp_work_done) {
217 		napi_comp_call = 1;
218 		if (napi_complete_done(napi, xdp_work_done))
219 			ena_unmask_interrupt(xdp_ring, NULL);
220 		ena_update_ring_numa_node(xdp_ring, NULL);
221 		ret = xdp_work_done;
222 	} else {
223 		ret = xdp_budget;
224 	}
225 
226 	u64_stats_update_begin(&xdp_ring->syncp);
227 	xdp_ring->tx_stats.napi_comp += napi_comp_call;
228 	xdp_ring->tx_stats.tx_poll++;
229 	u64_stats_update_end(&xdp_ring->syncp);
230 	xdp_ring->tx_stats.last_napi_jiffies = jiffies;
231 
232 	return ret;
233 }
234 
235 static int ena_xdp_tx_map_frame(struct ena_ring *xdp_ring,
236 				struct ena_tx_buffer *tx_info,
237 				struct xdp_frame *xdpf,
238 				struct ena_com_tx_ctx *ena_tx_ctx)
239 {
240 	struct ena_adapter *adapter = xdp_ring->adapter;
241 	struct ena_com_buf *ena_buf;
242 	int push_len = 0;
243 	dma_addr_t dma;
244 	void *data;
245 	u32 size;
246 
247 	tx_info->xdpf = xdpf;
248 	data = tx_info->xdpf->data;
249 	size = tx_info->xdpf->len;
250 
251 	if (xdp_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
252 		/* Designate part of the packet for LLQ */
253 		push_len = min_t(u32, size, xdp_ring->tx_max_header_size);
254 
255 		ena_tx_ctx->push_header = data;
256 
257 		size -= push_len;
258 		data += push_len;
259 	}
260 
261 	ena_tx_ctx->header_len = push_len;
262 
263 	if (size > 0) {
264 		dma = dma_map_single(xdp_ring->dev,
265 				     data,
266 				     size,
267 				     DMA_TO_DEVICE);
268 		if (unlikely(dma_mapping_error(xdp_ring->dev, dma)))
269 			goto error_report_dma_error;
270 
271 		tx_info->map_linear_data = 0;
272 
273 		ena_buf = tx_info->bufs;
274 		ena_buf->paddr = dma;
275 		ena_buf->len = size;
276 
277 		ena_tx_ctx->ena_bufs = ena_buf;
278 		ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1;
279 	}
280 
281 	return 0;
282 
283 error_report_dma_error:
284 	ena_increase_stat(&xdp_ring->tx_stats.dma_mapping_err, 1,
285 			  &xdp_ring->syncp);
286 	netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n");
287 
288 	return -EINVAL;
289 }
290 
291 static int ena_xdp_xmit_frame(struct ena_ring *xdp_ring,
292 			      struct net_device *dev,
293 			      struct xdp_frame *xdpf,
294 			      int flags)
295 {
296 	struct ena_com_tx_ctx ena_tx_ctx = {};
297 	struct ena_tx_buffer *tx_info;
298 	u16 next_to_use, req_id;
299 	int rc;
300 
301 	next_to_use = xdp_ring->next_to_use;
302 	req_id = xdp_ring->free_ids[next_to_use];
303 	tx_info = &xdp_ring->tx_buffer_info[req_id];
304 	tx_info->num_of_bufs = 0;
305 
306 	rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &ena_tx_ctx);
307 	if (unlikely(rc))
308 		return rc;
309 
310 	ena_tx_ctx.req_id = req_id;
311 
312 	rc = ena_xmit_common(dev,
313 			     xdp_ring,
314 			     tx_info,
315 			     &ena_tx_ctx,
316 			     next_to_use,
317 			     xdpf->len);
318 	if (rc)
319 		goto error_unmap_dma;
320 
321 	/* trigger the dma engine. ena_ring_tx_doorbell()
322 	 * calls a memory barrier inside it.
323 	 */
324 	if (flags & XDP_XMIT_FLUSH)
325 		ena_ring_tx_doorbell(xdp_ring);
326 
327 	return rc;
328 
329 error_unmap_dma:
330 	ena_unmap_tx_buff(xdp_ring, tx_info);
331 	tx_info->xdpf = NULL;
332 	return rc;
333 }
334 
335 static int ena_xdp_xmit(struct net_device *dev, int n,
336 			struct xdp_frame **frames, u32 flags)
337 {
338 	struct ena_adapter *adapter = netdev_priv(dev);
339 	struct ena_ring *xdp_ring;
340 	int qid, i, nxmit = 0;
341 
342 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
343 		return -EINVAL;
344 
345 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
346 		return -ENETDOWN;
347 
348 	/* We assume that all rings have the same XDP program */
349 	if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog))
350 		return -ENXIO;
351 
352 	qid = smp_processor_id() % adapter->xdp_num_queues;
353 	qid += adapter->xdp_first_ring;
354 	xdp_ring = &adapter->tx_ring[qid];
355 
356 	/* Other CPU ids might try to send thorugh this queue */
357 	spin_lock(&xdp_ring->xdp_tx_lock);
358 
359 	for (i = 0; i < n; i++) {
360 		if (ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0))
361 			break;
362 		nxmit++;
363 	}
364 
365 	/* Ring doorbell to make device aware of the packets */
366 	if (flags & XDP_XMIT_FLUSH)
367 		ena_ring_tx_doorbell(xdp_ring);
368 
369 	spin_unlock(&xdp_ring->xdp_tx_lock);
370 
371 	/* Return number of packets sent */
372 	return nxmit;
373 }
374 
375 static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp)
376 {
377 	struct bpf_prog *xdp_prog;
378 	struct ena_ring *xdp_ring;
379 	u32 verdict = XDP_PASS;
380 	struct xdp_frame *xdpf;
381 	u64 *xdp_stat;
382 
383 	xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog);
384 
385 	if (!xdp_prog)
386 		goto out;
387 
388 	verdict = bpf_prog_run_xdp(xdp_prog, xdp);
389 
390 	switch (verdict) {
391 	case XDP_TX:
392 		xdpf = xdp_convert_buff_to_frame(xdp);
393 		if (unlikely(!xdpf)) {
394 			trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
395 			xdp_stat = &rx_ring->rx_stats.xdp_aborted;
396 			verdict = XDP_ABORTED;
397 			break;
398 		}
399 
400 		/* Find xmit queue */
401 		xdp_ring = rx_ring->xdp_ring;
402 
403 		/* The XDP queues are shared between XDP_TX and XDP_REDIRECT */
404 		spin_lock(&xdp_ring->xdp_tx_lock);
405 
406 		if (ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf,
407 				       XDP_XMIT_FLUSH))
408 			xdp_return_frame(xdpf);
409 
410 		spin_unlock(&xdp_ring->xdp_tx_lock);
411 		xdp_stat = &rx_ring->rx_stats.xdp_tx;
412 		break;
413 	case XDP_REDIRECT:
414 		if (likely(!xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))) {
415 			xdp_stat = &rx_ring->rx_stats.xdp_redirect;
416 			break;
417 		}
418 		trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
419 		xdp_stat = &rx_ring->rx_stats.xdp_aborted;
420 		verdict = XDP_ABORTED;
421 		break;
422 	case XDP_ABORTED:
423 		trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
424 		xdp_stat = &rx_ring->rx_stats.xdp_aborted;
425 		break;
426 	case XDP_DROP:
427 		xdp_stat = &rx_ring->rx_stats.xdp_drop;
428 		break;
429 	case XDP_PASS:
430 		xdp_stat = &rx_ring->rx_stats.xdp_pass;
431 		break;
432 	default:
433 		bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict);
434 		xdp_stat = &rx_ring->rx_stats.xdp_invalid;
435 	}
436 
437 	ena_increase_stat(xdp_stat, 1, &rx_ring->syncp);
438 out:
439 	return verdict;
440 }
441 
442 static void ena_init_all_xdp_queues(struct ena_adapter *adapter)
443 {
444 	adapter->xdp_first_ring = adapter->num_io_queues;
445 	adapter->xdp_num_queues = adapter->num_io_queues;
446 
447 	ena_init_io_rings(adapter,
448 			  adapter->xdp_first_ring,
449 			  adapter->xdp_num_queues);
450 }
451 
452 static int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter)
453 {
454 	int rc = 0;
455 
456 	rc = ena_setup_tx_resources_in_range(adapter, adapter->xdp_first_ring,
457 					     adapter->xdp_num_queues);
458 	if (rc)
459 		goto setup_err;
460 
461 	rc = ena_create_io_tx_queues_in_range(adapter,
462 					      adapter->xdp_first_ring,
463 					      adapter->xdp_num_queues);
464 	if (rc)
465 		goto create_err;
466 
467 	return 0;
468 
469 create_err:
470 	ena_free_all_io_tx_resources(adapter);
471 setup_err:
472 	return rc;
473 }
474 
475 /* Provides a way for both kernel and bpf-prog to know
476  * more about the RX-queue a given XDP frame arrived on.
477  */
478 static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)
479 {
480 	int rc;
481 
482 	rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0);
483 
484 	if (rc) {
485 		netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
486 			  "Failed to register xdp rx queue info. RX queue num %d rc: %d\n",
487 			  rx_ring->qid, rc);
488 		goto err;
489 	}
490 
491 	rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED,
492 					NULL);
493 
494 	if (rc) {
495 		netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
496 			  "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n",
497 			  rx_ring->qid, rc);
498 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
499 	}
500 
501 err:
502 	return rc;
503 }
504 
505 static void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring)
506 {
507 	xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq);
508 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
509 }
510 
511 static void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,
512 						 struct bpf_prog *prog,
513 						 int first, int count)
514 {
515 	struct ena_ring *rx_ring;
516 	int i = 0;
517 
518 	for (i = first; i < count; i++) {
519 		rx_ring = &adapter->rx_ring[i];
520 		xchg(&rx_ring->xdp_bpf_prog, prog);
521 		if (prog) {
522 			ena_xdp_register_rxq_info(rx_ring);
523 			rx_ring->rx_headroom = XDP_PACKET_HEADROOM;
524 		} else {
525 			ena_xdp_unregister_rxq_info(rx_ring);
526 			rx_ring->rx_headroom = NET_SKB_PAD;
527 		}
528 	}
529 }
530 
531 static void ena_xdp_exchange_program(struct ena_adapter *adapter,
532 				     struct bpf_prog *prog)
533 {
534 	struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog);
535 
536 	ena_xdp_exchange_program_rx_in_range(adapter,
537 					     prog,
538 					     0,
539 					     adapter->num_io_queues);
540 
541 	if (old_bpf_prog)
542 		bpf_prog_put(old_bpf_prog);
543 }
544 
545 static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter)
546 {
547 	bool was_up;
548 	int rc;
549 
550 	was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
551 
552 	if (was_up)
553 		ena_down(adapter);
554 
555 	adapter->xdp_first_ring = 0;
556 	adapter->xdp_num_queues = 0;
557 	ena_xdp_exchange_program(adapter, NULL);
558 	if (was_up) {
559 		rc = ena_up(adapter);
560 		if (rc)
561 			return rc;
562 	}
563 	return 0;
564 }
565 
566 static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf)
567 {
568 	struct ena_adapter *adapter = netdev_priv(netdev);
569 	struct bpf_prog *prog = bpf->prog;
570 	struct bpf_prog *old_bpf_prog;
571 	int rc, prev_mtu;
572 	bool is_up;
573 
574 	is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
575 	rc = ena_xdp_allowed(adapter);
576 	if (rc == ENA_XDP_ALLOWED) {
577 		old_bpf_prog = adapter->xdp_bpf_prog;
578 		if (prog) {
579 			if (!is_up) {
580 				ena_init_all_xdp_queues(adapter);
581 			} else if (!old_bpf_prog) {
582 				ena_down(adapter);
583 				ena_init_all_xdp_queues(adapter);
584 			}
585 			ena_xdp_exchange_program(adapter, prog);
586 
587 			if (is_up && !old_bpf_prog) {
588 				rc = ena_up(adapter);
589 				if (rc)
590 					return rc;
591 			}
592 		} else if (old_bpf_prog) {
593 			rc = ena_destroy_and_free_all_xdp_queues(adapter);
594 			if (rc)
595 				return rc;
596 		}
597 
598 		prev_mtu = netdev->max_mtu;
599 		netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu;
600 
601 		if (!old_bpf_prog)
602 			netif_info(adapter, drv, adapter->netdev,
603 				   "XDP program is set, changing the max_mtu from %d to %d",
604 				   prev_mtu, netdev->max_mtu);
605 
606 	} else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) {
607 		netif_err(adapter, drv, adapter->netdev,
608 			  "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on",
609 			  netdev->mtu, ENA_XDP_MAX_MTU);
610 		NL_SET_ERR_MSG_MOD(bpf->extack,
611 				   "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info");
612 		return -EINVAL;
613 	} else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) {
614 		netif_err(adapter, drv, adapter->netdev,
615 			  "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n",
616 			  adapter->num_io_queues, adapter->max_num_io_queues);
617 		NL_SET_ERR_MSG_MOD(bpf->extack,
618 				   "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info");
619 		return -EINVAL;
620 	}
621 
622 	return 0;
623 }
624 
625 /* This is the main xdp callback, it's used by the kernel to set/unset the xdp
626  * program as well as to query the current xdp program id.
627  */
628 static int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
629 {
630 	switch (bpf->command) {
631 	case XDP_SETUP_PROG:
632 		return ena_xdp_set(netdev, bpf);
633 	default:
634 		return -EINVAL;
635 	}
636 	return 0;
637 }
638 
639 static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)
640 {
641 #ifdef CONFIG_RFS_ACCEL
642 	u32 i;
643 	int rc;
644 
645 	adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_io_queues);
646 	if (!adapter->netdev->rx_cpu_rmap)
647 		return -ENOMEM;
648 	for (i = 0; i < adapter->num_io_queues; i++) {
649 		int irq_idx = ENA_IO_IRQ_IDX(i);
650 
651 		rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
652 				      pci_irq_vector(adapter->pdev, irq_idx));
653 		if (rc) {
654 			free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
655 			adapter->netdev->rx_cpu_rmap = NULL;
656 			return rc;
657 		}
658 	}
659 #endif /* CONFIG_RFS_ACCEL */
660 	return 0;
661 }
662 
663 static void ena_init_io_rings_common(struct ena_adapter *adapter,
664 				     struct ena_ring *ring, u16 qid)
665 {
666 	ring->qid = qid;
667 	ring->pdev = adapter->pdev;
668 	ring->dev = &adapter->pdev->dev;
669 	ring->netdev = adapter->netdev;
670 	ring->napi = &adapter->ena_napi[qid].napi;
671 	ring->adapter = adapter;
672 	ring->ena_dev = adapter->ena_dev;
673 	ring->per_napi_packets = 0;
674 	ring->cpu = 0;
675 	ring->no_interrupt_event_cnt = 0;
676 	u64_stats_init(&ring->syncp);
677 }
678 
679 static void ena_init_io_rings(struct ena_adapter *adapter,
680 			      int first_index, int count)
681 {
682 	struct ena_com_dev *ena_dev;
683 	struct ena_ring *txr, *rxr;
684 	int i;
685 
686 	ena_dev = adapter->ena_dev;
687 
688 	for (i = first_index; i < first_index + count; i++) {
689 		txr = &adapter->tx_ring[i];
690 		rxr = &adapter->rx_ring[i];
691 
692 		/* TX common ring state */
693 		ena_init_io_rings_common(adapter, txr, i);
694 
695 		/* TX specific ring state */
696 		txr->ring_size = adapter->requested_tx_ring_size;
697 		txr->tx_max_header_size = ena_dev->tx_max_header_size;
698 		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
699 		txr->sgl_size = adapter->max_tx_sgl_size;
700 		txr->smoothed_interval =
701 			ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
702 		txr->disable_meta_caching = adapter->disable_meta_caching;
703 		spin_lock_init(&txr->xdp_tx_lock);
704 
705 		/* Don't init RX queues for xdp queues */
706 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
707 			/* RX common ring state */
708 			ena_init_io_rings_common(adapter, rxr, i);
709 
710 			/* RX specific ring state */
711 			rxr->ring_size = adapter->requested_rx_ring_size;
712 			rxr->rx_copybreak = adapter->rx_copybreak;
713 			rxr->sgl_size = adapter->max_rx_sgl_size;
714 			rxr->smoothed_interval =
715 				ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
716 			rxr->empty_rx_queue = 0;
717 			rxr->rx_headroom = NET_SKB_PAD;
718 			adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
719 			rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];
720 		}
721 	}
722 }
723 
724 /* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors)
725  * @adapter: network interface device structure
726  * @qid: queue index
727  *
728  * Return 0 on success, negative on failure
729  */
730 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
731 {
732 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
733 	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
734 	int size, i, node;
735 
736 	if (tx_ring->tx_buffer_info) {
737 		netif_err(adapter, ifup,
738 			  adapter->netdev, "tx_buffer_info info is not NULL");
739 		return -EEXIST;
740 	}
741 
742 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
743 	node = cpu_to_node(ena_irq->cpu);
744 
745 	tx_ring->tx_buffer_info = vzalloc_node(size, node);
746 	if (!tx_ring->tx_buffer_info) {
747 		tx_ring->tx_buffer_info = vzalloc(size);
748 		if (!tx_ring->tx_buffer_info)
749 			goto err_tx_buffer_info;
750 	}
751 
752 	size = sizeof(u16) * tx_ring->ring_size;
753 	tx_ring->free_ids = vzalloc_node(size, node);
754 	if (!tx_ring->free_ids) {
755 		tx_ring->free_ids = vzalloc(size);
756 		if (!tx_ring->free_ids)
757 			goto err_tx_free_ids;
758 	}
759 
760 	size = tx_ring->tx_max_header_size;
761 	tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);
762 	if (!tx_ring->push_buf_intermediate_buf) {
763 		tx_ring->push_buf_intermediate_buf = vzalloc(size);
764 		if (!tx_ring->push_buf_intermediate_buf)
765 			goto err_push_buf_intermediate_buf;
766 	}
767 
768 	/* Req id ring for TX out of order completions */
769 	for (i = 0; i < tx_ring->ring_size; i++)
770 		tx_ring->free_ids[i] = i;
771 
772 	/* Reset tx statistics */
773 	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
774 
775 	tx_ring->next_to_use = 0;
776 	tx_ring->next_to_clean = 0;
777 	tx_ring->cpu = ena_irq->cpu;
778 	return 0;
779 
780 err_push_buf_intermediate_buf:
781 	vfree(tx_ring->free_ids);
782 	tx_ring->free_ids = NULL;
783 err_tx_free_ids:
784 	vfree(tx_ring->tx_buffer_info);
785 	tx_ring->tx_buffer_info = NULL;
786 err_tx_buffer_info:
787 	return -ENOMEM;
788 }
789 
790 /* ena_free_tx_resources - Free I/O Tx Resources per Queue
791  * @adapter: network interface device structure
792  * @qid: queue index
793  *
794  * Free all transmit software resources
795  */
796 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)
797 {
798 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
799 
800 	vfree(tx_ring->tx_buffer_info);
801 	tx_ring->tx_buffer_info = NULL;
802 
803 	vfree(tx_ring->free_ids);
804 	tx_ring->free_ids = NULL;
805 
806 	vfree(tx_ring->push_buf_intermediate_buf);
807 	tx_ring->push_buf_intermediate_buf = NULL;
808 }
809 
810 static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
811 					   int first_index,
812 					   int count)
813 {
814 	int i, rc = 0;
815 
816 	for (i = first_index; i < first_index + count; i++) {
817 		rc = ena_setup_tx_resources(adapter, i);
818 		if (rc)
819 			goto err_setup_tx;
820 	}
821 
822 	return 0;
823 
824 err_setup_tx:
825 
826 	netif_err(adapter, ifup, adapter->netdev,
827 		  "Tx queue %d: allocation failed\n", i);
828 
829 	/* rewind the index freeing the rings as we go */
830 	while (first_index < i--)
831 		ena_free_tx_resources(adapter, i);
832 	return rc;
833 }
834 
835 static void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter,
836 						  int first_index, int count)
837 {
838 	int i;
839 
840 	for (i = first_index; i < first_index + count; i++)
841 		ena_free_tx_resources(adapter, i);
842 }
843 
844 /* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues
845  * @adapter: board private structure
846  *
847  * Free all transmit software resources
848  */
849 static void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
850 {
851 	ena_free_all_io_tx_resources_in_range(adapter,
852 					      0,
853 					      adapter->xdp_num_queues +
854 					      adapter->num_io_queues);
855 }
856 
857 /* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
858  * @adapter: network interface device structure
859  * @qid: queue index
860  *
861  * Returns 0 on success, negative on failure
862  */
863 static int ena_setup_rx_resources(struct ena_adapter *adapter,
864 				  u32 qid)
865 {
866 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
867 	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
868 	int size, node, i;
869 
870 	if (rx_ring->rx_buffer_info) {
871 		netif_err(adapter, ifup, adapter->netdev,
872 			  "rx_buffer_info is not NULL");
873 		return -EEXIST;
874 	}
875 
876 	/* alloc extra element so in rx path
877 	 * we can always prefetch rx_info + 1
878 	 */
879 	size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1);
880 	node = cpu_to_node(ena_irq->cpu);
881 
882 	rx_ring->rx_buffer_info = vzalloc_node(size, node);
883 	if (!rx_ring->rx_buffer_info) {
884 		rx_ring->rx_buffer_info = vzalloc(size);
885 		if (!rx_ring->rx_buffer_info)
886 			return -ENOMEM;
887 	}
888 
889 	size = sizeof(u16) * rx_ring->ring_size;
890 	rx_ring->free_ids = vzalloc_node(size, node);
891 	if (!rx_ring->free_ids) {
892 		rx_ring->free_ids = vzalloc(size);
893 		if (!rx_ring->free_ids) {
894 			vfree(rx_ring->rx_buffer_info);
895 			rx_ring->rx_buffer_info = NULL;
896 			return -ENOMEM;
897 		}
898 	}
899 
900 	/* Req id ring for receiving RX pkts out of order */
901 	for (i = 0; i < rx_ring->ring_size; i++)
902 		rx_ring->free_ids[i] = i;
903 
904 	/* Reset rx statistics */
905 	memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
906 
907 	rx_ring->next_to_clean = 0;
908 	rx_ring->next_to_use = 0;
909 	rx_ring->cpu = ena_irq->cpu;
910 
911 	return 0;
912 }
913 
914 /* ena_free_rx_resources - Free I/O Rx Resources
915  * @adapter: network interface device structure
916  * @qid: queue index
917  *
918  * Free all receive software resources
919  */
920 static void ena_free_rx_resources(struct ena_adapter *adapter,
921 				  u32 qid)
922 {
923 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
924 
925 	vfree(rx_ring->rx_buffer_info);
926 	rx_ring->rx_buffer_info = NULL;
927 
928 	vfree(rx_ring->free_ids);
929 	rx_ring->free_ids = NULL;
930 }
931 
932 /* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
933  * @adapter: board private structure
934  *
935  * Return 0 on success, negative on failure
936  */
937 static int ena_setup_all_rx_resources(struct ena_adapter *adapter)
938 {
939 	int i, rc = 0;
940 
941 	for (i = 0; i < adapter->num_io_queues; i++) {
942 		rc = ena_setup_rx_resources(adapter, i);
943 		if (rc)
944 			goto err_setup_rx;
945 	}
946 
947 	return 0;
948 
949 err_setup_rx:
950 
951 	netif_err(adapter, ifup, adapter->netdev,
952 		  "Rx queue %d: allocation failed\n", i);
953 
954 	/* rewind the index freeing the rings as we go */
955 	while (i--)
956 		ena_free_rx_resources(adapter, i);
957 	return rc;
958 }
959 
960 /* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues
961  * @adapter: board private structure
962  *
963  * Free all receive software resources
964  */
965 static void ena_free_all_io_rx_resources(struct ena_adapter *adapter)
966 {
967 	int i;
968 
969 	for (i = 0; i < adapter->num_io_queues; i++)
970 		ena_free_rx_resources(adapter, i);
971 }
972 
973 static struct page *ena_alloc_map_page(struct ena_ring *rx_ring,
974 				       dma_addr_t *dma)
975 {
976 	struct page *page;
977 
978 	/* This would allocate the page on the same NUMA node the executing code
979 	 * is running on.
980 	 */
981 	page = dev_alloc_page();
982 	if (!page) {
983 		ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1,
984 				  &rx_ring->syncp);
985 		return ERR_PTR(-ENOSPC);
986 	}
987 
988 	/* To enable NIC-side port-mirroring, AKA SPAN port,
989 	 * we make the buffer readable from the nic as well
990 	 */
991 	*dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE,
992 			    DMA_BIDIRECTIONAL);
993 	if (unlikely(dma_mapping_error(rx_ring->dev, *dma))) {
994 		ena_increase_stat(&rx_ring->rx_stats.dma_mapping_err, 1,
995 				  &rx_ring->syncp);
996 		__free_page(page);
997 		return ERR_PTR(-EIO);
998 	}
999 
1000 	return page;
1001 }
1002 
1003 static int ena_alloc_rx_buffer(struct ena_ring *rx_ring,
1004 			       struct ena_rx_buffer *rx_info)
1005 {
1006 	int headroom = rx_ring->rx_headroom;
1007 	struct ena_com_buf *ena_buf;
1008 	struct page *page;
1009 	dma_addr_t dma;
1010 	int tailroom;
1011 
1012 	/* restore page offset value in case it has been changed by device */
1013 	rx_info->page_offset = headroom;
1014 
1015 	/* if previous allocated page is not used */
1016 	if (unlikely(rx_info->page))
1017 		return 0;
1018 
1019 	/* We handle DMA here */
1020 	page = ena_alloc_map_page(rx_ring, &dma);
1021 	if (unlikely(IS_ERR(page)))
1022 		return PTR_ERR(page);
1023 
1024 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1025 		  "Allocate page %p, rx_info %p\n", page, rx_info);
1026 
1027 	tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1028 
1029 	rx_info->page = page;
1030 	ena_buf = &rx_info->ena_buf;
1031 	ena_buf->paddr = dma + headroom;
1032 	ena_buf->len = ENA_PAGE_SIZE - headroom - tailroom;
1033 
1034 	return 0;
1035 }
1036 
1037 static void ena_unmap_rx_buff(struct ena_ring *rx_ring,
1038 			      struct ena_rx_buffer *rx_info)
1039 {
1040 	struct ena_com_buf *ena_buf = &rx_info->ena_buf;
1041 
1042 	dma_unmap_page(rx_ring->dev, ena_buf->paddr - rx_ring->rx_headroom,
1043 		       ENA_PAGE_SIZE,
1044 		       DMA_BIDIRECTIONAL);
1045 }
1046 
1047 static void ena_free_rx_page(struct ena_ring *rx_ring,
1048 			     struct ena_rx_buffer *rx_info)
1049 {
1050 	struct page *page = rx_info->page;
1051 
1052 	if (unlikely(!page)) {
1053 		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
1054 			   "Trying to free unallocated buffer\n");
1055 		return;
1056 	}
1057 
1058 	ena_unmap_rx_buff(rx_ring, rx_info);
1059 
1060 	__free_page(page);
1061 	rx_info->page = NULL;
1062 }
1063 
1064 static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
1065 {
1066 	u16 next_to_use, req_id;
1067 	u32 i;
1068 	int rc;
1069 
1070 	next_to_use = rx_ring->next_to_use;
1071 
1072 	for (i = 0; i < num; i++) {
1073 		struct ena_rx_buffer *rx_info;
1074 
1075 		req_id = rx_ring->free_ids[next_to_use];
1076 
1077 		rx_info = &rx_ring->rx_buffer_info[req_id];
1078 
1079 		rc = ena_alloc_rx_buffer(rx_ring, rx_info);
1080 		if (unlikely(rc < 0)) {
1081 			netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
1082 				   "Failed to allocate buffer for rx queue %d\n",
1083 				   rx_ring->qid);
1084 			break;
1085 		}
1086 		rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1087 						&rx_info->ena_buf,
1088 						req_id);
1089 		if (unlikely(rc)) {
1090 			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
1091 				   "Failed to add buffer for rx queue %d\n",
1092 				   rx_ring->qid);
1093 			break;
1094 		}
1095 		next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1096 						   rx_ring->ring_size);
1097 	}
1098 
1099 	if (unlikely(i < num)) {
1100 		ena_increase_stat(&rx_ring->rx_stats.refil_partial, 1,
1101 				  &rx_ring->syncp);
1102 		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
1103 			   "Refilled rx qid %d with only %d buffers (from %d)\n",
1104 			   rx_ring->qid, i, num);
1105 	}
1106 
1107 	/* ena_com_write_sq_doorbell issues a wmb() */
1108 	if (likely(i))
1109 		ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
1110 
1111 	rx_ring->next_to_use = next_to_use;
1112 
1113 	return i;
1114 }
1115 
1116 static void ena_free_rx_bufs(struct ena_adapter *adapter,
1117 			     u32 qid)
1118 {
1119 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
1120 	u32 i;
1121 
1122 	for (i = 0; i < rx_ring->ring_size; i++) {
1123 		struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
1124 
1125 		if (rx_info->page)
1126 			ena_free_rx_page(rx_ring, rx_info);
1127 	}
1128 }
1129 
1130 /* ena_refill_all_rx_bufs - allocate all queues Rx buffers
1131  * @adapter: board private structure
1132  */
1133 static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)
1134 {
1135 	struct ena_ring *rx_ring;
1136 	int i, rc, bufs_num;
1137 
1138 	for (i = 0; i < adapter->num_io_queues; i++) {
1139 		rx_ring = &adapter->rx_ring[i];
1140 		bufs_num = rx_ring->ring_size - 1;
1141 		rc = ena_refill_rx_bufs(rx_ring, bufs_num);
1142 
1143 		if (unlikely(rc != bufs_num))
1144 			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
1145 				   "Refilling Queue %d failed. allocated %d buffers from: %d\n",
1146 				   i, rc, bufs_num);
1147 	}
1148 }
1149 
1150 static void ena_free_all_rx_bufs(struct ena_adapter *adapter)
1151 {
1152 	int i;
1153 
1154 	for (i = 0; i < adapter->num_io_queues; i++)
1155 		ena_free_rx_bufs(adapter, i);
1156 }
1157 
1158 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
1159 			      struct ena_tx_buffer *tx_info)
1160 {
1161 	struct ena_com_buf *ena_buf;
1162 	u32 cnt;
1163 	int i;
1164 
1165 	ena_buf = tx_info->bufs;
1166 	cnt = tx_info->num_of_bufs;
1167 
1168 	if (unlikely(!cnt))
1169 		return;
1170 
1171 	if (tx_info->map_linear_data) {
1172 		dma_unmap_single(tx_ring->dev,
1173 				 dma_unmap_addr(ena_buf, paddr),
1174 				 dma_unmap_len(ena_buf, len),
1175 				 DMA_TO_DEVICE);
1176 		ena_buf++;
1177 		cnt--;
1178 	}
1179 
1180 	/* unmap remaining mapped pages */
1181 	for (i = 0; i < cnt; i++) {
1182 		dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
1183 			       dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
1184 		ena_buf++;
1185 	}
1186 }
1187 
1188 /* ena_free_tx_bufs - Free Tx Buffers per Queue
1189  * @tx_ring: TX ring for which buffers be freed
1190  */
1191 static void ena_free_tx_bufs(struct ena_ring *tx_ring)
1192 {
1193 	bool print_once = true;
1194 	u32 i;
1195 
1196 	for (i = 0; i < tx_ring->ring_size; i++) {
1197 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
1198 
1199 		if (!tx_info->skb)
1200 			continue;
1201 
1202 		if (print_once) {
1203 			netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,
1204 				     "Free uncompleted tx skb qid %d idx 0x%x\n",
1205 				     tx_ring->qid, i);
1206 			print_once = false;
1207 		} else {
1208 			netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,
1209 				  "Free uncompleted tx skb qid %d idx 0x%x\n",
1210 				  tx_ring->qid, i);
1211 		}
1212 
1213 		ena_unmap_tx_buff(tx_ring, tx_info);
1214 
1215 		dev_kfree_skb_any(tx_info->skb);
1216 	}
1217 	netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
1218 						  tx_ring->qid));
1219 }
1220 
1221 static void ena_free_all_tx_bufs(struct ena_adapter *adapter)
1222 {
1223 	struct ena_ring *tx_ring;
1224 	int i;
1225 
1226 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
1227 		tx_ring = &adapter->tx_ring[i];
1228 		ena_free_tx_bufs(tx_ring);
1229 	}
1230 }
1231 
1232 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter)
1233 {
1234 	u16 ena_qid;
1235 	int i;
1236 
1237 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
1238 		ena_qid = ENA_IO_TXQ_IDX(i);
1239 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1240 	}
1241 }
1242 
1243 static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)
1244 {
1245 	u16 ena_qid;
1246 	int i;
1247 
1248 	for (i = 0; i < adapter->num_io_queues; i++) {
1249 		ena_qid = ENA_IO_RXQ_IDX(i);
1250 		cancel_work_sync(&adapter->ena_napi[i].dim.work);
1251 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1252 	}
1253 }
1254 
1255 static void ena_destroy_all_io_queues(struct ena_adapter *adapter)
1256 {
1257 	ena_destroy_all_tx_queues(adapter);
1258 	ena_destroy_all_rx_queues(adapter);
1259 }
1260 
1261 static int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,
1262 				 struct ena_tx_buffer *tx_info, bool is_xdp)
1263 {
1264 	if (tx_info)
1265 		netif_err(ring->adapter,
1266 			  tx_done,
1267 			  ring->netdev,
1268 			  "tx_info doesn't have valid %s. qid %u req_id %u",
1269 			   is_xdp ? "xdp frame" : "skb", ring->qid, req_id);
1270 	else
1271 		netif_err(ring->adapter,
1272 			  tx_done,
1273 			  ring->netdev,
1274 			  "Invalid req_id %u in qid %u\n",
1275 			  req_id, ring->qid);
1276 
1277 	ena_increase_stat(&ring->tx_stats.bad_req_id, 1, &ring->syncp);
1278 	ena_reset_device(ring->adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
1279 
1280 	return -EFAULT;
1281 }
1282 
1283 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
1284 {
1285 	struct ena_tx_buffer *tx_info;
1286 
1287 	tx_info = &tx_ring->tx_buffer_info[req_id];
1288 	if (likely(tx_info->skb))
1289 		return 0;
1290 
1291 	return handle_invalid_req_id(tx_ring, req_id, tx_info, false);
1292 }
1293 
1294 static int validate_xdp_req_id(struct ena_ring *xdp_ring, u16 req_id)
1295 {
1296 	struct ena_tx_buffer *tx_info;
1297 
1298 	tx_info = &xdp_ring->tx_buffer_info[req_id];
1299 	if (likely(tx_info->xdpf))
1300 		return 0;
1301 
1302 	return handle_invalid_req_id(xdp_ring, req_id, tx_info, true);
1303 }
1304 
1305 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
1306 {
1307 	struct netdev_queue *txq;
1308 	bool above_thresh;
1309 	u32 tx_bytes = 0;
1310 	u32 total_done = 0;
1311 	u16 next_to_clean;
1312 	u16 req_id;
1313 	int tx_pkts = 0;
1314 	int rc;
1315 
1316 	next_to_clean = tx_ring->next_to_clean;
1317 	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);
1318 
1319 	while (tx_pkts < budget) {
1320 		struct ena_tx_buffer *tx_info;
1321 		struct sk_buff *skb;
1322 
1323 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
1324 						&req_id);
1325 		if (rc) {
1326 			if (unlikely(rc == -EINVAL))
1327 				handle_invalid_req_id(tx_ring, req_id, NULL,
1328 						      false);
1329 			break;
1330 		}
1331 
1332 		/* validate that the request id points to a valid skb */
1333 		rc = validate_tx_req_id(tx_ring, req_id);
1334 		if (rc)
1335 			break;
1336 
1337 		tx_info = &tx_ring->tx_buffer_info[req_id];
1338 		skb = tx_info->skb;
1339 
1340 		/* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */
1341 		prefetch(&skb->end);
1342 
1343 		tx_info->skb = NULL;
1344 		tx_info->last_jiffies = 0;
1345 
1346 		ena_unmap_tx_buff(tx_ring, tx_info);
1347 
1348 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
1349 			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,
1350 			  skb);
1351 
1352 		tx_bytes += skb->len;
1353 		dev_kfree_skb(skb);
1354 		tx_pkts++;
1355 		total_done += tx_info->tx_descs;
1356 
1357 		tx_ring->free_ids[next_to_clean] = req_id;
1358 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
1359 						     tx_ring->ring_size);
1360 	}
1361 
1362 	tx_ring->next_to_clean = next_to_clean;
1363 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
1364 	ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
1365 
1366 	netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1367 
1368 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
1369 		  "tx_poll: q %d done. total pkts: %d\n",
1370 		  tx_ring->qid, tx_pkts);
1371 
1372 	/* need to make the rings circular update visible to
1373 	 * ena_start_xmit() before checking for netif_queue_stopped().
1374 	 */
1375 	smp_mb();
1376 
1377 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1378 						    ENA_TX_WAKEUP_THRESH);
1379 	if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {
1380 		__netif_tx_lock(txq, smp_processor_id());
1381 		above_thresh =
1382 			ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1383 						     ENA_TX_WAKEUP_THRESH);
1384 		if (netif_tx_queue_stopped(txq) && above_thresh &&
1385 		    test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
1386 			netif_tx_wake_queue(txq);
1387 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
1388 					  &tx_ring->syncp);
1389 		}
1390 		__netif_tx_unlock(txq);
1391 	}
1392 
1393 	return tx_pkts;
1394 }
1395 
1396 static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag)
1397 {
1398 	struct sk_buff *skb;
1399 
1400 	if (!first_frag)
1401 		skb = napi_alloc_skb(rx_ring->napi, rx_ring->rx_copybreak);
1402 	else
1403 		skb = napi_build_skb(first_frag, ENA_PAGE_SIZE);
1404 
1405 	if (unlikely(!skb)) {
1406 		ena_increase_stat(&rx_ring->rx_stats.skb_alloc_fail, 1,
1407 				  &rx_ring->syncp);
1408 
1409 		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1410 			  "Failed to allocate skb. first_frag %s\n",
1411 			  first_frag ? "provided" : "not provided");
1412 		return NULL;
1413 	}
1414 
1415 	return skb;
1416 }
1417 
1418 static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
1419 				  struct ena_com_rx_buf_info *ena_bufs,
1420 				  u32 descs,
1421 				  u16 *next_to_clean)
1422 {
1423 	struct ena_rx_buffer *rx_info;
1424 	struct ena_adapter *adapter;
1425 	u16 len, req_id, buf = 0;
1426 	struct sk_buff *skb;
1427 	void *page_addr;
1428 	u32 page_offset;
1429 	void *data_addr;
1430 
1431 	len = ena_bufs[buf].len;
1432 	req_id = ena_bufs[buf].req_id;
1433 
1434 	rx_info = &rx_ring->rx_buffer_info[req_id];
1435 
1436 	if (unlikely(!rx_info->page)) {
1437 		adapter = rx_ring->adapter;
1438 		netif_err(adapter, rx_err, rx_ring->netdev,
1439 			  "Page is NULL. qid %u req_id %u\n", rx_ring->qid, req_id);
1440 		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp);
1441 		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
1442 		return NULL;
1443 	}
1444 
1445 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1446 		  "rx_info %p page %p\n",
1447 		  rx_info, rx_info->page);
1448 
1449 	/* save virt address of first buffer */
1450 	page_addr = page_address(rx_info->page);
1451 	page_offset = rx_info->page_offset;
1452 	data_addr = page_addr + page_offset;
1453 
1454 	prefetch(data_addr);
1455 
1456 	if (len <= rx_ring->rx_copybreak) {
1457 		skb = ena_alloc_skb(rx_ring, NULL);
1458 		if (unlikely(!skb))
1459 			return NULL;
1460 
1461 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1462 			  "RX allocated small packet. len %d. data_len %d\n",
1463 			  skb->len, skb->data_len);
1464 
1465 		/* sync this buffer for CPU use */
1466 		dma_sync_single_for_cpu(rx_ring->dev,
1467 					dma_unmap_addr(&rx_info->ena_buf, paddr),
1468 					len,
1469 					DMA_FROM_DEVICE);
1470 		skb_copy_to_linear_data(skb, data_addr, len);
1471 		dma_sync_single_for_device(rx_ring->dev,
1472 					   dma_unmap_addr(&rx_info->ena_buf, paddr),
1473 					   len,
1474 					   DMA_FROM_DEVICE);
1475 
1476 		skb_put(skb, len);
1477 		skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1478 		rx_ring->free_ids[*next_to_clean] = req_id;
1479 		*next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
1480 						     rx_ring->ring_size);
1481 		return skb;
1482 	}
1483 
1484 	ena_unmap_rx_buff(rx_ring, rx_info);
1485 
1486 	skb = ena_alloc_skb(rx_ring, page_addr);
1487 	if (unlikely(!skb))
1488 		return NULL;
1489 
1490 	/* Populate skb's linear part */
1491 	skb_reserve(skb, page_offset);
1492 	skb_put(skb, len);
1493 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1494 
1495 	do {
1496 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1497 			  "RX skb updated. len %d. data_len %d\n",
1498 			  skb->len, skb->data_len);
1499 
1500 		rx_info->page = NULL;
1501 
1502 		rx_ring->free_ids[*next_to_clean] = req_id;
1503 		*next_to_clean =
1504 			ENA_RX_RING_IDX_NEXT(*next_to_clean,
1505 					     rx_ring->ring_size);
1506 		if (likely(--descs == 0))
1507 			break;
1508 
1509 		buf++;
1510 		len = ena_bufs[buf].len;
1511 		req_id = ena_bufs[buf].req_id;
1512 
1513 		rx_info = &rx_ring->rx_buffer_info[req_id];
1514 
1515 		ena_unmap_rx_buff(rx_ring, rx_info);
1516 
1517 		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,
1518 				rx_info->page_offset, len, ENA_PAGE_SIZE);
1519 
1520 	} while (1);
1521 
1522 	return skb;
1523 }
1524 
1525 /* ena_rx_checksum - indicate in skb if hw indicated a good cksum
1526  * @adapter: structure containing adapter specific data
1527  * @ena_rx_ctx: received packet context/metadata
1528  * @skb: skb currently being received and modified
1529  */
1530 static void ena_rx_checksum(struct ena_ring *rx_ring,
1531 				   struct ena_com_rx_ctx *ena_rx_ctx,
1532 				   struct sk_buff *skb)
1533 {
1534 	/* Rx csum disabled */
1535 	if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) {
1536 		skb->ip_summed = CHECKSUM_NONE;
1537 		return;
1538 	}
1539 
1540 	/* For fragmented packets the checksum isn't valid */
1541 	if (ena_rx_ctx->frag) {
1542 		skb->ip_summed = CHECKSUM_NONE;
1543 		return;
1544 	}
1545 
1546 	/* if IP and error */
1547 	if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
1548 		     (ena_rx_ctx->l3_csum_err))) {
1549 		/* ipv4 checksum error */
1550 		skb->ip_summed = CHECKSUM_NONE;
1551 		ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,
1552 				  &rx_ring->syncp);
1553 		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1554 			  "RX IPv4 header checksum error\n");
1555 		return;
1556 	}
1557 
1558 	/* if TCP/UDP */
1559 	if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1560 		   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) {
1561 		if (unlikely(ena_rx_ctx->l4_csum_err)) {
1562 			/* TCP/UDP checksum error */
1563 			ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,
1564 					  &rx_ring->syncp);
1565 			netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1566 				  "RX L4 checksum error\n");
1567 			skb->ip_summed = CHECKSUM_NONE;
1568 			return;
1569 		}
1570 
1571 		if (likely(ena_rx_ctx->l4_csum_checked)) {
1572 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1573 			ena_increase_stat(&rx_ring->rx_stats.csum_good, 1,
1574 					  &rx_ring->syncp);
1575 		} else {
1576 			ena_increase_stat(&rx_ring->rx_stats.csum_unchecked, 1,
1577 					  &rx_ring->syncp);
1578 			skb->ip_summed = CHECKSUM_NONE;
1579 		}
1580 	} else {
1581 		skb->ip_summed = CHECKSUM_NONE;
1582 		return;
1583 	}
1584 
1585 }
1586 
1587 static void ena_set_rx_hash(struct ena_ring *rx_ring,
1588 			    struct ena_com_rx_ctx *ena_rx_ctx,
1589 			    struct sk_buff *skb)
1590 {
1591 	enum pkt_hash_types hash_type;
1592 
1593 	if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) {
1594 		if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1595 			   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)))
1596 
1597 			hash_type = PKT_HASH_TYPE_L4;
1598 		else
1599 			hash_type = PKT_HASH_TYPE_NONE;
1600 
1601 		/* Override hash type if the packet is fragmented */
1602 		if (ena_rx_ctx->frag)
1603 			hash_type = PKT_HASH_TYPE_NONE;
1604 
1605 		skb_set_hash(skb, ena_rx_ctx->hash, hash_type);
1606 	}
1607 }
1608 
1609 static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp)
1610 {
1611 	struct ena_rx_buffer *rx_info;
1612 	int ret;
1613 
1614 	rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1615 	xdp_prepare_buff(xdp, page_address(rx_info->page),
1616 			 rx_info->page_offset,
1617 			 rx_ring->ena_bufs[0].len, false);
1618 	/* If for some reason we received a bigger packet than
1619 	 * we expect, then we simply drop it
1620 	 */
1621 	if (unlikely(rx_ring->ena_bufs[0].len > ENA_XDP_MAX_MTU))
1622 		return XDP_DROP;
1623 
1624 	ret = ena_xdp_execute(rx_ring, xdp);
1625 
1626 	/* The xdp program might expand the headers */
1627 	if (ret == XDP_PASS) {
1628 		rx_info->page_offset = xdp->data - xdp->data_hard_start;
1629 		rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data;
1630 	}
1631 
1632 	return ret;
1633 }
1634 /* ena_clean_rx_irq - Cleanup RX irq
1635  * @rx_ring: RX ring to clean
1636  * @napi: napi handler
1637  * @budget: how many packets driver is allowed to clean
1638  *
1639  * Returns the number of cleaned buffers.
1640  */
1641 static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
1642 			    u32 budget)
1643 {
1644 	u16 next_to_clean = rx_ring->next_to_clean;
1645 	struct ena_com_rx_ctx ena_rx_ctx;
1646 	struct ena_rx_buffer *rx_info;
1647 	struct ena_adapter *adapter;
1648 	u32 res_budget, work_done;
1649 	int rx_copybreak_pkt = 0;
1650 	int refill_threshold;
1651 	struct sk_buff *skb;
1652 	int refill_required;
1653 	struct xdp_buff xdp;
1654 	int xdp_flags = 0;
1655 	int total_len = 0;
1656 	int xdp_verdict;
1657 	int rc = 0;
1658 	int i;
1659 
1660 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1661 		  "%s qid %d\n", __func__, rx_ring->qid);
1662 	res_budget = budget;
1663 	xdp_init_buff(&xdp, ENA_PAGE_SIZE, &rx_ring->xdp_rxq);
1664 
1665 	do {
1666 		xdp_verdict = XDP_PASS;
1667 		skb = NULL;
1668 		ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
1669 		ena_rx_ctx.max_bufs = rx_ring->sgl_size;
1670 		ena_rx_ctx.descs = 0;
1671 		ena_rx_ctx.pkt_offset = 0;
1672 		rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
1673 				    rx_ring->ena_com_io_sq,
1674 				    &ena_rx_ctx);
1675 		if (unlikely(rc))
1676 			goto error;
1677 
1678 		if (unlikely(ena_rx_ctx.descs == 0))
1679 			break;
1680 
1681 		/* First descriptor might have an offset set by the device */
1682 		rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1683 		rx_info->page_offset += ena_rx_ctx.pkt_offset;
1684 
1685 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1686 			  "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
1687 			  rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
1688 			  ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
1689 
1690 		if (ena_xdp_present_ring(rx_ring))
1691 			xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp);
1692 
1693 		/* allocate skb and fill it */
1694 		if (xdp_verdict == XDP_PASS)
1695 			skb = ena_rx_skb(rx_ring,
1696 					 rx_ring->ena_bufs,
1697 					 ena_rx_ctx.descs,
1698 					 &next_to_clean);
1699 
1700 		if (unlikely(!skb)) {
1701 			for (i = 0; i < ena_rx_ctx.descs; i++) {
1702 				int req_id = rx_ring->ena_bufs[i].req_id;
1703 
1704 				rx_ring->free_ids[next_to_clean] = req_id;
1705 				next_to_clean =
1706 					ENA_RX_RING_IDX_NEXT(next_to_clean,
1707 							     rx_ring->ring_size);
1708 
1709 				/* Packets was passed for transmission, unmap it
1710 				 * from RX side.
1711 				 */
1712 				if (xdp_verdict == XDP_TX || xdp_verdict == XDP_REDIRECT) {
1713 					ena_unmap_rx_buff(rx_ring,
1714 							  &rx_ring->rx_buffer_info[req_id]);
1715 					rx_ring->rx_buffer_info[req_id].page = NULL;
1716 				}
1717 			}
1718 			if (xdp_verdict != XDP_PASS) {
1719 				xdp_flags |= xdp_verdict;
1720 				res_budget--;
1721 				continue;
1722 			}
1723 			break;
1724 		}
1725 
1726 		ena_rx_checksum(rx_ring, &ena_rx_ctx, skb);
1727 
1728 		ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb);
1729 
1730 		skb_record_rx_queue(skb, rx_ring->qid);
1731 
1732 		if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak)
1733 			rx_copybreak_pkt++;
1734 
1735 		total_len += skb->len;
1736 
1737 		napi_gro_receive(napi, skb);
1738 
1739 		res_budget--;
1740 	} while (likely(res_budget));
1741 
1742 	work_done = budget - res_budget;
1743 	rx_ring->per_napi_packets += work_done;
1744 	u64_stats_update_begin(&rx_ring->syncp);
1745 	rx_ring->rx_stats.bytes += total_len;
1746 	rx_ring->rx_stats.cnt += work_done;
1747 	rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt;
1748 	u64_stats_update_end(&rx_ring->syncp);
1749 
1750 	rx_ring->next_to_clean = next_to_clean;
1751 
1752 	refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
1753 	refill_threshold =
1754 		min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
1755 		      ENA_RX_REFILL_THRESH_PACKET);
1756 
1757 	/* Optimization, try to batch new rx buffers */
1758 	if (refill_required > refill_threshold) {
1759 		ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
1760 		ena_refill_rx_bufs(rx_ring, refill_required);
1761 	}
1762 
1763 	if (xdp_flags & XDP_REDIRECT)
1764 		xdp_do_flush_map();
1765 
1766 	return work_done;
1767 
1768 error:
1769 	adapter = netdev_priv(rx_ring->netdev);
1770 
1771 	if (rc == -ENOSPC) {
1772 		ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1,
1773 				  &rx_ring->syncp);
1774 		ena_reset_device(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS);
1775 	} else {
1776 		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1,
1777 				  &rx_ring->syncp);
1778 		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
1779 	}
1780 	return 0;
1781 }
1782 
1783 static void ena_dim_work(struct work_struct *w)
1784 {
1785 	struct dim *dim = container_of(w, struct dim, work);
1786 	struct dim_cq_moder cur_moder =
1787 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1788 	struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim);
1789 
1790 	ena_napi->rx_ring->smoothed_interval = cur_moder.usec;
1791 	dim->state = DIM_START_MEASURE;
1792 }
1793 
1794 static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi)
1795 {
1796 	struct dim_sample dim_sample;
1797 	struct ena_ring *rx_ring = ena_napi->rx_ring;
1798 
1799 	if (!rx_ring->per_napi_packets)
1800 		return;
1801 
1802 	rx_ring->non_empty_napi_events++;
1803 
1804 	dim_update_sample(rx_ring->non_empty_napi_events,
1805 			  rx_ring->rx_stats.cnt,
1806 			  rx_ring->rx_stats.bytes,
1807 			  &dim_sample);
1808 
1809 	net_dim(&ena_napi->dim, dim_sample);
1810 
1811 	rx_ring->per_napi_packets = 0;
1812 }
1813 
1814 static void ena_unmask_interrupt(struct ena_ring *tx_ring,
1815 					struct ena_ring *rx_ring)
1816 {
1817 	struct ena_eth_io_intr_reg intr_reg;
1818 	u32 rx_interval = 0;
1819 	/* Rx ring can be NULL when for XDP tx queues which don't have an
1820 	 * accompanying rx_ring pair.
1821 	 */
1822 	if (rx_ring)
1823 		rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ?
1824 			rx_ring->smoothed_interval :
1825 			ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev);
1826 
1827 	/* Update intr register: rx intr delay,
1828 	 * tx intr delay and interrupt unmask
1829 	 */
1830 	ena_com_update_intr_reg(&intr_reg,
1831 				rx_interval,
1832 				tx_ring->smoothed_interval,
1833 				true);
1834 
1835 	ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1,
1836 			  &tx_ring->syncp);
1837 
1838 	/* It is a shared MSI-X.
1839 	 * Tx and Rx CQ have pointer to it.
1840 	 * So we use one of them to reach the intr reg
1841 	 * The Tx ring is used because the rx_ring is NULL for XDP queues
1842 	 */
1843 	ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);
1844 }
1845 
1846 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
1847 					     struct ena_ring *rx_ring)
1848 {
1849 	int cpu = get_cpu();
1850 	int numa_node;
1851 
1852 	/* Check only one ring since the 2 rings are running on the same cpu */
1853 	if (likely(tx_ring->cpu == cpu))
1854 		goto out;
1855 
1856 	numa_node = cpu_to_node(cpu);
1857 	put_cpu();
1858 
1859 	if (numa_node != NUMA_NO_NODE) {
1860 		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
1861 		if (rx_ring)
1862 			ena_com_update_numa_node(rx_ring->ena_com_io_cq,
1863 						 numa_node);
1864 	}
1865 
1866 	tx_ring->cpu = cpu;
1867 	if (rx_ring)
1868 		rx_ring->cpu = cpu;
1869 
1870 	return;
1871 out:
1872 	put_cpu();
1873 }
1874 
1875 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget)
1876 {
1877 	u32 total_done = 0;
1878 	u16 next_to_clean;
1879 	u32 tx_bytes = 0;
1880 	int tx_pkts = 0;
1881 	u16 req_id;
1882 	int rc;
1883 
1884 	if (unlikely(!xdp_ring))
1885 		return 0;
1886 	next_to_clean = xdp_ring->next_to_clean;
1887 
1888 	while (tx_pkts < budget) {
1889 		struct ena_tx_buffer *tx_info;
1890 		struct xdp_frame *xdpf;
1891 
1892 		rc = ena_com_tx_comp_req_id_get(xdp_ring->ena_com_io_cq,
1893 						&req_id);
1894 		if (rc) {
1895 			if (unlikely(rc == -EINVAL))
1896 				handle_invalid_req_id(xdp_ring, req_id, NULL,
1897 						      true);
1898 			break;
1899 		}
1900 
1901 		/* validate that the request id points to a valid xdp_frame */
1902 		rc = validate_xdp_req_id(xdp_ring, req_id);
1903 		if (rc)
1904 			break;
1905 
1906 		tx_info = &xdp_ring->tx_buffer_info[req_id];
1907 		xdpf = tx_info->xdpf;
1908 
1909 		tx_info->xdpf = NULL;
1910 		tx_info->last_jiffies = 0;
1911 		ena_unmap_tx_buff(xdp_ring, tx_info);
1912 
1913 		netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,
1914 			  "tx_poll: q %d skb %p completed\n", xdp_ring->qid,
1915 			  xdpf);
1916 
1917 		tx_bytes += xdpf->len;
1918 		tx_pkts++;
1919 		total_done += tx_info->tx_descs;
1920 
1921 		xdp_return_frame(xdpf);
1922 		xdp_ring->free_ids[next_to_clean] = req_id;
1923 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
1924 						     xdp_ring->ring_size);
1925 	}
1926 
1927 	xdp_ring->next_to_clean = next_to_clean;
1928 	ena_com_comp_ack(xdp_ring->ena_com_io_sq, total_done);
1929 	ena_com_update_dev_comp_head(xdp_ring->ena_com_io_cq);
1930 
1931 	netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,
1932 		  "tx_poll: q %d done. total pkts: %d\n",
1933 		  xdp_ring->qid, tx_pkts);
1934 
1935 	return tx_pkts;
1936 }
1937 
1938 static int ena_io_poll(struct napi_struct *napi, int budget)
1939 {
1940 	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
1941 	struct ena_ring *tx_ring, *rx_ring;
1942 	int tx_work_done;
1943 	int rx_work_done = 0;
1944 	int tx_budget;
1945 	int napi_comp_call = 0;
1946 	int ret;
1947 
1948 	tx_ring = ena_napi->tx_ring;
1949 	rx_ring = ena_napi->rx_ring;
1950 
1951 	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
1952 
1953 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1954 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
1955 		napi_complete_done(napi, 0);
1956 		return 0;
1957 	}
1958 
1959 	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
1960 	/* On netpoll the budget is zero and the handler should only clean the
1961 	 * tx completions.
1962 	 */
1963 	if (likely(budget))
1964 		rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);
1965 
1966 	/* If the device is about to reset or down, avoid unmask
1967 	 * the interrupt and return 0 so NAPI won't reschedule
1968 	 */
1969 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1970 		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {
1971 		napi_complete_done(napi, 0);
1972 		ret = 0;
1973 
1974 	} else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
1975 		napi_comp_call = 1;
1976 
1977 		/* Update numa and unmask the interrupt only when schedule
1978 		 * from the interrupt context (vs from sk_busy_loop)
1979 		 */
1980 		if (napi_complete_done(napi, rx_work_done) &&
1981 		    READ_ONCE(ena_napi->interrupts_masked)) {
1982 			smp_rmb(); /* make sure interrupts_masked is read */
1983 			WRITE_ONCE(ena_napi->interrupts_masked, false);
1984 			/* We apply adaptive moderation on Rx path only.
1985 			 * Tx uses static interrupt moderation.
1986 			 */
1987 			if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
1988 				ena_adjust_adaptive_rx_intr_moderation(ena_napi);
1989 
1990 			ena_unmask_interrupt(tx_ring, rx_ring);
1991 		}
1992 
1993 		ena_update_ring_numa_node(tx_ring, rx_ring);
1994 
1995 		ret = rx_work_done;
1996 	} else {
1997 		ret = budget;
1998 	}
1999 
2000 	u64_stats_update_begin(&tx_ring->syncp);
2001 	tx_ring->tx_stats.napi_comp += napi_comp_call;
2002 	tx_ring->tx_stats.tx_poll++;
2003 	u64_stats_update_end(&tx_ring->syncp);
2004 
2005 	tx_ring->tx_stats.last_napi_jiffies = jiffies;
2006 
2007 	return ret;
2008 }
2009 
2010 static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data)
2011 {
2012 	struct ena_adapter *adapter = (struct ena_adapter *)data;
2013 
2014 	ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
2015 
2016 	/* Don't call the aenq handler before probe is done */
2017 	if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)))
2018 		ena_com_aenq_intr_handler(adapter->ena_dev, data);
2019 
2020 	return IRQ_HANDLED;
2021 }
2022 
2023 /* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx
2024  * @irq: interrupt number
2025  * @data: pointer to a network interface private napi device structure
2026  */
2027 static irqreturn_t ena_intr_msix_io(int irq, void *data)
2028 {
2029 	struct ena_napi *ena_napi = data;
2030 
2031 	/* Used to check HW health */
2032 	WRITE_ONCE(ena_napi->first_interrupt, true);
2033 
2034 	WRITE_ONCE(ena_napi->interrupts_masked, true);
2035 	smp_wmb(); /* write interrupts_masked before calling napi */
2036 
2037 	napi_schedule_irqoff(&ena_napi->napi);
2038 
2039 	return IRQ_HANDLED;
2040 }
2041 
2042 /* Reserve a single MSI-X vector for management (admin + aenq).
2043  * plus reserve one vector for each potential io queue.
2044  * the number of potential io queues is the minimum of what the device
2045  * supports and the number of vCPUs.
2046  */
2047 static int ena_enable_msix(struct ena_adapter *adapter)
2048 {
2049 	int msix_vecs, irq_cnt;
2050 
2051 	if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
2052 		netif_err(adapter, probe, adapter->netdev,
2053 			  "Error, MSI-X is already enabled\n");
2054 		return -EPERM;
2055 	}
2056 
2057 	/* Reserved the max msix vectors we might need */
2058 	msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);
2059 	netif_dbg(adapter, probe, adapter->netdev,
2060 		  "Trying to enable MSI-X, vectors %d\n", msix_vecs);
2061 
2062 	irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,
2063 					msix_vecs, PCI_IRQ_MSIX);
2064 
2065 	if (irq_cnt < 0) {
2066 		netif_err(adapter, probe, adapter->netdev,
2067 			  "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt);
2068 		return -ENOSPC;
2069 	}
2070 
2071 	if (irq_cnt != msix_vecs) {
2072 		netif_notice(adapter, probe, adapter->netdev,
2073 			     "Enable only %d MSI-X (out of %d), reduce the number of queues\n",
2074 			     irq_cnt, msix_vecs);
2075 		adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
2076 	}
2077 
2078 	if (ena_init_rx_cpu_rmap(adapter))
2079 		netif_warn(adapter, probe, adapter->netdev,
2080 			   "Failed to map IRQs to CPUs\n");
2081 
2082 	adapter->msix_vecs = irq_cnt;
2083 	set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);
2084 
2085 	return 0;
2086 }
2087 
2088 static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)
2089 {
2090 	u32 cpu;
2091 
2092 	snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
2093 		 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
2094 		 pci_name(adapter->pdev));
2095 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler =
2096 		ena_intr_msix_mgmnt;
2097 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
2098 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
2099 		pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);
2100 	cpu = cpumask_first(cpu_online_mask);
2101 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
2102 	cpumask_set_cpu(cpu,
2103 			&adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask);
2104 }
2105 
2106 static void ena_setup_io_intr(struct ena_adapter *adapter)
2107 {
2108 	struct net_device *netdev;
2109 	int irq_idx, i, cpu;
2110 	int io_queue_count;
2111 
2112 	netdev = adapter->netdev;
2113 	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2114 
2115 	for (i = 0; i < io_queue_count; i++) {
2116 		irq_idx = ENA_IO_IRQ_IDX(i);
2117 		cpu = i % num_online_cpus();
2118 
2119 		snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
2120 			 "%s-Tx-Rx-%d", netdev->name, i);
2121 		adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
2122 		adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
2123 		adapter->irq_tbl[irq_idx].vector =
2124 			pci_irq_vector(adapter->pdev, irq_idx);
2125 		adapter->irq_tbl[irq_idx].cpu = cpu;
2126 
2127 		cpumask_set_cpu(cpu,
2128 				&adapter->irq_tbl[irq_idx].affinity_hint_mask);
2129 	}
2130 }
2131 
2132 static int ena_request_mgmnt_irq(struct ena_adapter *adapter)
2133 {
2134 	unsigned long flags = 0;
2135 	struct ena_irq *irq;
2136 	int rc;
2137 
2138 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2139 	rc = request_irq(irq->vector, irq->handler, flags, irq->name,
2140 			 irq->data);
2141 	if (rc) {
2142 		netif_err(adapter, probe, adapter->netdev,
2143 			  "Failed to request admin irq\n");
2144 		return rc;
2145 	}
2146 
2147 	netif_dbg(adapter, probe, adapter->netdev,
2148 		  "Set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
2149 		  irq->affinity_hint_mask.bits[0], irq->vector);
2150 
2151 	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
2152 
2153 	return rc;
2154 }
2155 
2156 static int ena_request_io_irq(struct ena_adapter *adapter)
2157 {
2158 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2159 	unsigned long flags = 0;
2160 	struct ena_irq *irq;
2161 	int rc = 0, i, k;
2162 
2163 	if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
2164 		netif_err(adapter, ifup, adapter->netdev,
2165 			  "Failed to request I/O IRQ: MSI-X is not enabled\n");
2166 		return -EINVAL;
2167 	}
2168 
2169 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
2170 		irq = &adapter->irq_tbl[i];
2171 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
2172 				 irq->data);
2173 		if (rc) {
2174 			netif_err(adapter, ifup, adapter->netdev,
2175 				  "Failed to request I/O IRQ. index %d rc %d\n",
2176 				   i, rc);
2177 			goto err;
2178 		}
2179 
2180 		netif_dbg(adapter, ifup, adapter->netdev,
2181 			  "Set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
2182 			  i, irq->affinity_hint_mask.bits[0], irq->vector);
2183 
2184 		irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
2185 	}
2186 
2187 	return rc;
2188 
2189 err:
2190 	for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) {
2191 		irq = &adapter->irq_tbl[k];
2192 		free_irq(irq->vector, irq->data);
2193 	}
2194 
2195 	return rc;
2196 }
2197 
2198 static void ena_free_mgmnt_irq(struct ena_adapter *adapter)
2199 {
2200 	struct ena_irq *irq;
2201 
2202 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2203 	synchronize_irq(irq->vector);
2204 	irq_set_affinity_hint(irq->vector, NULL);
2205 	free_irq(irq->vector, irq->data);
2206 }
2207 
2208 static void ena_free_io_irq(struct ena_adapter *adapter)
2209 {
2210 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2211 	struct ena_irq *irq;
2212 	int i;
2213 
2214 #ifdef CONFIG_RFS_ACCEL
2215 	if (adapter->msix_vecs >= 1) {
2216 		free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
2217 		adapter->netdev->rx_cpu_rmap = NULL;
2218 	}
2219 #endif /* CONFIG_RFS_ACCEL */
2220 
2221 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
2222 		irq = &adapter->irq_tbl[i];
2223 		irq_set_affinity_hint(irq->vector, NULL);
2224 		free_irq(irq->vector, irq->data);
2225 	}
2226 }
2227 
2228 static void ena_disable_msix(struct ena_adapter *adapter)
2229 {
2230 	if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))
2231 		pci_free_irq_vectors(adapter->pdev);
2232 }
2233 
2234 static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
2235 {
2236 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2237 	int i;
2238 
2239 	if (!netif_running(adapter->netdev))
2240 		return;
2241 
2242 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++)
2243 		synchronize_irq(adapter->irq_tbl[i].vector);
2244 }
2245 
2246 static void ena_del_napi_in_range(struct ena_adapter *adapter,
2247 				  int first_index,
2248 				  int count)
2249 {
2250 	int i;
2251 
2252 	for (i = first_index; i < first_index + count; i++) {
2253 		netif_napi_del(&adapter->ena_napi[i].napi);
2254 
2255 		WARN_ON(!ENA_IS_XDP_INDEX(adapter, i) &&
2256 			adapter->ena_napi[i].xdp_ring);
2257 	}
2258 }
2259 
2260 static void ena_init_napi_in_range(struct ena_adapter *adapter,
2261 				   int first_index, int count)
2262 {
2263 	int i;
2264 
2265 	for (i = first_index; i < first_index + count; i++) {
2266 		struct ena_napi *napi = &adapter->ena_napi[i];
2267 
2268 		netif_napi_add(adapter->netdev, &napi->napi,
2269 			       ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll);
2270 
2271 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
2272 			napi->rx_ring = &adapter->rx_ring[i];
2273 			napi->tx_ring = &adapter->tx_ring[i];
2274 		} else {
2275 			napi->xdp_ring = &adapter->tx_ring[i];
2276 		}
2277 		napi->qid = i;
2278 	}
2279 }
2280 
2281 static void ena_napi_disable_in_range(struct ena_adapter *adapter,
2282 				      int first_index,
2283 				      int count)
2284 {
2285 	int i;
2286 
2287 	for (i = first_index; i < first_index + count; i++)
2288 		napi_disable(&adapter->ena_napi[i].napi);
2289 }
2290 
2291 static void ena_napi_enable_in_range(struct ena_adapter *adapter,
2292 				     int first_index,
2293 				     int count)
2294 {
2295 	int i;
2296 
2297 	for (i = first_index; i < first_index + count; i++)
2298 		napi_enable(&adapter->ena_napi[i].napi);
2299 }
2300 
2301 /* Configure the Rx forwarding */
2302 static int ena_rss_configure(struct ena_adapter *adapter)
2303 {
2304 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2305 	int rc;
2306 
2307 	/* In case the RSS table wasn't initialized by probe */
2308 	if (!ena_dev->rss.tbl_log_size) {
2309 		rc = ena_rss_init_default(adapter);
2310 		if (rc && (rc != -EOPNOTSUPP)) {
2311 			netif_err(adapter, ifup, adapter->netdev,
2312 				  "Failed to init RSS rc: %d\n", rc);
2313 			return rc;
2314 		}
2315 	}
2316 
2317 	/* Set indirect table */
2318 	rc = ena_com_indirect_table_set(ena_dev);
2319 	if (unlikely(rc && rc != -EOPNOTSUPP))
2320 		return rc;
2321 
2322 	/* Configure hash function (if supported) */
2323 	rc = ena_com_set_hash_function(ena_dev);
2324 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
2325 		return rc;
2326 
2327 	/* Configure hash inputs (if supported) */
2328 	rc = ena_com_set_hash_ctrl(ena_dev);
2329 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
2330 		return rc;
2331 
2332 	return 0;
2333 }
2334 
2335 static int ena_up_complete(struct ena_adapter *adapter)
2336 {
2337 	int rc;
2338 
2339 	rc = ena_rss_configure(adapter);
2340 	if (rc)
2341 		return rc;
2342 
2343 	ena_change_mtu(adapter->netdev, adapter->netdev->mtu);
2344 
2345 	ena_refill_all_rx_bufs(adapter);
2346 
2347 	/* enable transmits */
2348 	netif_tx_start_all_queues(adapter->netdev);
2349 
2350 	ena_napi_enable_in_range(adapter,
2351 				 0,
2352 				 adapter->xdp_num_queues + adapter->num_io_queues);
2353 
2354 	return 0;
2355 }
2356 
2357 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
2358 {
2359 	struct ena_com_create_io_ctx ctx;
2360 	struct ena_com_dev *ena_dev;
2361 	struct ena_ring *tx_ring;
2362 	u32 msix_vector;
2363 	u16 ena_qid;
2364 	int rc;
2365 
2366 	ena_dev = adapter->ena_dev;
2367 
2368 	tx_ring = &adapter->tx_ring[qid];
2369 	msix_vector = ENA_IO_IRQ_IDX(qid);
2370 	ena_qid = ENA_IO_TXQ_IDX(qid);
2371 
2372 	memset(&ctx, 0x0, sizeof(ctx));
2373 
2374 	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
2375 	ctx.qid = ena_qid;
2376 	ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
2377 	ctx.msix_vector = msix_vector;
2378 	ctx.queue_size = tx_ring->ring_size;
2379 	ctx.numa_node = cpu_to_node(tx_ring->cpu);
2380 
2381 	rc = ena_com_create_io_queue(ena_dev, &ctx);
2382 	if (rc) {
2383 		netif_err(adapter, ifup, adapter->netdev,
2384 			  "Failed to create I/O TX queue num %d rc: %d\n",
2385 			  qid, rc);
2386 		return rc;
2387 	}
2388 
2389 	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
2390 				     &tx_ring->ena_com_io_sq,
2391 				     &tx_ring->ena_com_io_cq);
2392 	if (rc) {
2393 		netif_err(adapter, ifup, adapter->netdev,
2394 			  "Failed to get TX queue handlers. TX queue num %d rc: %d\n",
2395 			  qid, rc);
2396 		ena_com_destroy_io_queue(ena_dev, ena_qid);
2397 		return rc;
2398 	}
2399 
2400 	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
2401 	return rc;
2402 }
2403 
2404 static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
2405 					    int first_index, int count)
2406 {
2407 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2408 	int rc, i;
2409 
2410 	for (i = first_index; i < first_index + count; i++) {
2411 		rc = ena_create_io_tx_queue(adapter, i);
2412 		if (rc)
2413 			goto create_err;
2414 	}
2415 
2416 	return 0;
2417 
2418 create_err:
2419 	while (i-- > first_index)
2420 		ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
2421 
2422 	return rc;
2423 }
2424 
2425 static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
2426 {
2427 	struct ena_com_dev *ena_dev;
2428 	struct ena_com_create_io_ctx ctx;
2429 	struct ena_ring *rx_ring;
2430 	u32 msix_vector;
2431 	u16 ena_qid;
2432 	int rc;
2433 
2434 	ena_dev = adapter->ena_dev;
2435 
2436 	rx_ring = &adapter->rx_ring[qid];
2437 	msix_vector = ENA_IO_IRQ_IDX(qid);
2438 	ena_qid = ENA_IO_RXQ_IDX(qid);
2439 
2440 	memset(&ctx, 0x0, sizeof(ctx));
2441 
2442 	ctx.qid = ena_qid;
2443 	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
2444 	ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2445 	ctx.msix_vector = msix_vector;
2446 	ctx.queue_size = rx_ring->ring_size;
2447 	ctx.numa_node = cpu_to_node(rx_ring->cpu);
2448 
2449 	rc = ena_com_create_io_queue(ena_dev, &ctx);
2450 	if (rc) {
2451 		netif_err(adapter, ifup, adapter->netdev,
2452 			  "Failed to create I/O RX queue num %d rc: %d\n",
2453 			  qid, rc);
2454 		return rc;
2455 	}
2456 
2457 	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
2458 				     &rx_ring->ena_com_io_sq,
2459 				     &rx_ring->ena_com_io_cq);
2460 	if (rc) {
2461 		netif_err(adapter, ifup, adapter->netdev,
2462 			  "Failed to get RX queue handlers. RX queue num %d rc: %d\n",
2463 			  qid, rc);
2464 		goto err;
2465 	}
2466 
2467 	ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
2468 
2469 	return rc;
2470 err:
2471 	ena_com_destroy_io_queue(ena_dev, ena_qid);
2472 	return rc;
2473 }
2474 
2475 static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)
2476 {
2477 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2478 	int rc, i;
2479 
2480 	for (i = 0; i < adapter->num_io_queues; i++) {
2481 		rc = ena_create_io_rx_queue(adapter, i);
2482 		if (rc)
2483 			goto create_err;
2484 		INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work);
2485 	}
2486 
2487 	return 0;
2488 
2489 create_err:
2490 	while (i--) {
2491 		cancel_work_sync(&adapter->ena_napi[i].dim.work);
2492 		ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
2493 	}
2494 
2495 	return rc;
2496 }
2497 
2498 static void set_io_rings_size(struct ena_adapter *adapter,
2499 			      int new_tx_size,
2500 			      int new_rx_size)
2501 {
2502 	int i;
2503 
2504 	for (i = 0; i < adapter->num_io_queues; i++) {
2505 		adapter->tx_ring[i].ring_size = new_tx_size;
2506 		adapter->rx_ring[i].ring_size = new_rx_size;
2507 	}
2508 }
2509 
2510 /* This function allows queue allocation to backoff when the system is
2511  * low on memory. If there is not enough memory to allocate io queues
2512  * the driver will try to allocate smaller queues.
2513  *
2514  * The backoff algorithm is as follows:
2515  *  1. Try to allocate TX and RX and if successful.
2516  *  1.1. return success
2517  *
2518  *  2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same).
2519  *
2520  *  3. If TX or RX is smaller than 256
2521  *  3.1. return failure.
2522  *  4. else
2523  *  4.1. go back to 1.
2524  */
2525 static int create_queues_with_size_backoff(struct ena_adapter *adapter)
2526 {
2527 	int rc, cur_rx_ring_size, cur_tx_ring_size;
2528 	int new_rx_ring_size, new_tx_ring_size;
2529 
2530 	/* current queue sizes might be set to smaller than the requested
2531 	 * ones due to past queue allocation failures.
2532 	 */
2533 	set_io_rings_size(adapter, adapter->requested_tx_ring_size,
2534 			  adapter->requested_rx_ring_size);
2535 
2536 	while (1) {
2537 		if (ena_xdp_present(adapter)) {
2538 			rc = ena_setup_and_create_all_xdp_queues(adapter);
2539 
2540 			if (rc)
2541 				goto err_setup_tx;
2542 		}
2543 		rc = ena_setup_tx_resources_in_range(adapter,
2544 						     0,
2545 						     adapter->num_io_queues);
2546 		if (rc)
2547 			goto err_setup_tx;
2548 
2549 		rc = ena_create_io_tx_queues_in_range(adapter,
2550 						      0,
2551 						      adapter->num_io_queues);
2552 		if (rc)
2553 			goto err_create_tx_queues;
2554 
2555 		rc = ena_setup_all_rx_resources(adapter);
2556 		if (rc)
2557 			goto err_setup_rx;
2558 
2559 		rc = ena_create_all_io_rx_queues(adapter);
2560 		if (rc)
2561 			goto err_create_rx_queues;
2562 
2563 		return 0;
2564 
2565 err_create_rx_queues:
2566 		ena_free_all_io_rx_resources(adapter);
2567 err_setup_rx:
2568 		ena_destroy_all_tx_queues(adapter);
2569 err_create_tx_queues:
2570 		ena_free_all_io_tx_resources(adapter);
2571 err_setup_tx:
2572 		if (rc != -ENOMEM) {
2573 			netif_err(adapter, ifup, adapter->netdev,
2574 				  "Queue creation failed with error code %d\n",
2575 				  rc);
2576 			return rc;
2577 		}
2578 
2579 		cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2580 		cur_rx_ring_size = adapter->rx_ring[0].ring_size;
2581 
2582 		netif_err(adapter, ifup, adapter->netdev,
2583 			  "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
2584 			  cur_tx_ring_size, cur_rx_ring_size);
2585 
2586 		new_tx_ring_size = cur_tx_ring_size;
2587 		new_rx_ring_size = cur_rx_ring_size;
2588 
2589 		/* Decrease the size of the larger queue, or
2590 		 * decrease both if they are the same size.
2591 		 */
2592 		if (cur_rx_ring_size <= cur_tx_ring_size)
2593 			new_tx_ring_size = cur_tx_ring_size / 2;
2594 		if (cur_rx_ring_size >= cur_tx_ring_size)
2595 			new_rx_ring_size = cur_rx_ring_size / 2;
2596 
2597 		if (new_tx_ring_size < ENA_MIN_RING_SIZE ||
2598 		    new_rx_ring_size < ENA_MIN_RING_SIZE) {
2599 			netif_err(adapter, ifup, adapter->netdev,
2600 				  "Queue creation failed with the smallest possible queue size of %d for both queues. Not retrying with smaller queues\n",
2601 				  ENA_MIN_RING_SIZE);
2602 			return rc;
2603 		}
2604 
2605 		netif_err(adapter, ifup, adapter->netdev,
2606 			  "Retrying queue creation with sizes TX=%d, RX=%d\n",
2607 			  new_tx_ring_size,
2608 			  new_rx_ring_size);
2609 
2610 		set_io_rings_size(adapter, new_tx_ring_size,
2611 				  new_rx_ring_size);
2612 	}
2613 }
2614 
2615 static int ena_up(struct ena_adapter *adapter)
2616 {
2617 	int io_queue_count, rc, i;
2618 
2619 	netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
2620 
2621 	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2622 	ena_setup_io_intr(adapter);
2623 
2624 	/* napi poll functions should be initialized before running
2625 	 * request_irq(), to handle a rare condition where there is a pending
2626 	 * interrupt, causing the ISR to fire immediately while the poll
2627 	 * function wasn't set yet, causing a null dereference
2628 	 */
2629 	ena_init_napi_in_range(adapter, 0, io_queue_count);
2630 
2631 	rc = ena_request_io_irq(adapter);
2632 	if (rc)
2633 		goto err_req_irq;
2634 
2635 	rc = create_queues_with_size_backoff(adapter);
2636 	if (rc)
2637 		goto err_create_queues_with_backoff;
2638 
2639 	rc = ena_up_complete(adapter);
2640 	if (rc)
2641 		goto err_up;
2642 
2643 	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
2644 		netif_carrier_on(adapter->netdev);
2645 
2646 	ena_increase_stat(&adapter->dev_stats.interface_up, 1,
2647 			  &adapter->syncp);
2648 
2649 	set_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2650 
2651 	/* Enable completion queues interrupt */
2652 	for (i = 0; i < adapter->num_io_queues; i++)
2653 		ena_unmask_interrupt(&adapter->tx_ring[i],
2654 				     &adapter->rx_ring[i]);
2655 
2656 	/* schedule napi in case we had pending packets
2657 	 * from the last time we disable napi
2658 	 */
2659 	for (i = 0; i < io_queue_count; i++)
2660 		napi_schedule(&adapter->ena_napi[i].napi);
2661 
2662 	return rc;
2663 
2664 err_up:
2665 	ena_destroy_all_tx_queues(adapter);
2666 	ena_free_all_io_tx_resources(adapter);
2667 	ena_destroy_all_rx_queues(adapter);
2668 	ena_free_all_io_rx_resources(adapter);
2669 err_create_queues_with_backoff:
2670 	ena_free_io_irq(adapter);
2671 err_req_irq:
2672 	ena_del_napi_in_range(adapter, 0, io_queue_count);
2673 
2674 	return rc;
2675 }
2676 
2677 static void ena_down(struct ena_adapter *adapter)
2678 {
2679 	int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2680 
2681 	netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__);
2682 
2683 	clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2684 
2685 	ena_increase_stat(&adapter->dev_stats.interface_down, 1,
2686 			  &adapter->syncp);
2687 
2688 	netif_carrier_off(adapter->netdev);
2689 	netif_tx_disable(adapter->netdev);
2690 
2691 	/* After this point the napi handler won't enable the tx queue */
2692 	ena_napi_disable_in_range(adapter, 0, io_queue_count);
2693 
2694 	/* After destroy the queue there won't be any new interrupts */
2695 
2696 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {
2697 		int rc;
2698 
2699 		rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
2700 		if (rc)
2701 			netif_err(adapter, ifdown, adapter->netdev,
2702 				  "Device reset failed\n");
2703 		/* stop submitting admin commands on a device that was reset */
2704 		ena_com_set_admin_running_state(adapter->ena_dev, false);
2705 	}
2706 
2707 	ena_destroy_all_io_queues(adapter);
2708 
2709 	ena_disable_io_intr_sync(adapter);
2710 	ena_free_io_irq(adapter);
2711 	ena_del_napi_in_range(adapter, 0, io_queue_count);
2712 
2713 	ena_free_all_tx_bufs(adapter);
2714 	ena_free_all_rx_bufs(adapter);
2715 	ena_free_all_io_tx_resources(adapter);
2716 	ena_free_all_io_rx_resources(adapter);
2717 }
2718 
2719 /* ena_open - Called when a network interface is made active
2720  * @netdev: network interface device structure
2721  *
2722  * Returns 0 on success, negative value on failure
2723  *
2724  * The open entry point is called when a network interface is made
2725  * active by the system (IFF_UP).  At this point all resources needed
2726  * for transmit and receive operations are allocated, the interrupt
2727  * handler is registered with the OS, the watchdog timer is started,
2728  * and the stack is notified that the interface is ready.
2729  */
2730 static int ena_open(struct net_device *netdev)
2731 {
2732 	struct ena_adapter *adapter = netdev_priv(netdev);
2733 	int rc;
2734 
2735 	/* Notify the stack of the actual queue counts. */
2736 	rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues);
2737 	if (rc) {
2738 		netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");
2739 		return rc;
2740 	}
2741 
2742 	rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues);
2743 	if (rc) {
2744 		netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");
2745 		return rc;
2746 	}
2747 
2748 	rc = ena_up(adapter);
2749 	if (rc)
2750 		return rc;
2751 
2752 	return rc;
2753 }
2754 
2755 /* ena_close - Disables a network interface
2756  * @netdev: network interface device structure
2757  *
2758  * Returns 0, this is not allowed to fail
2759  *
2760  * The close entry point is called when an interface is de-activated
2761  * by the OS.  The hardware is still under the drivers control, but
2762  * needs to be disabled.  A global MAC reset is issued to stop the
2763  * hardware, and all transmit and receive resources are freed.
2764  */
2765 static int ena_close(struct net_device *netdev)
2766 {
2767 	struct ena_adapter *adapter = netdev_priv(netdev);
2768 
2769 	netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);
2770 
2771 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
2772 		return 0;
2773 
2774 	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2775 		ena_down(adapter);
2776 
2777 	/* Check for device status and issue reset if needed*/
2778 	check_for_admin_com_state(adapter);
2779 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
2780 		netif_err(adapter, ifdown, adapter->netdev,
2781 			  "Destroy failure, restarting device\n");
2782 		ena_dump_stats_to_dmesg(adapter);
2783 		/* rtnl lock already obtained in dev_ioctl() layer */
2784 		ena_destroy_device(adapter, false);
2785 		ena_restore_device(adapter);
2786 	}
2787 
2788 	return 0;
2789 }
2790 
2791 int ena_update_queue_sizes(struct ena_adapter *adapter,
2792 			   u32 new_tx_size,
2793 			   u32 new_rx_size)
2794 {
2795 	bool dev_was_up;
2796 
2797 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2798 	ena_close(adapter->netdev);
2799 	adapter->requested_tx_ring_size = new_tx_size;
2800 	adapter->requested_rx_ring_size = new_rx_size;
2801 	ena_init_io_rings(adapter,
2802 			  0,
2803 			  adapter->xdp_num_queues +
2804 			  adapter->num_io_queues);
2805 	return dev_was_up ? ena_up(adapter) : 0;
2806 }
2807 
2808 int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count)
2809 {
2810 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2811 	int prev_channel_count;
2812 	bool dev_was_up;
2813 
2814 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2815 	ena_close(adapter->netdev);
2816 	prev_channel_count = adapter->num_io_queues;
2817 	adapter->num_io_queues = new_channel_count;
2818 	if (ena_xdp_present(adapter) &&
2819 	    ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) {
2820 		adapter->xdp_first_ring = new_channel_count;
2821 		adapter->xdp_num_queues = new_channel_count;
2822 		if (prev_channel_count > new_channel_count)
2823 			ena_xdp_exchange_program_rx_in_range(adapter,
2824 							     NULL,
2825 							     new_channel_count,
2826 							     prev_channel_count);
2827 		else
2828 			ena_xdp_exchange_program_rx_in_range(adapter,
2829 							     adapter->xdp_bpf_prog,
2830 							     prev_channel_count,
2831 							     new_channel_count);
2832 	}
2833 
2834 	/* We need to destroy the rss table so that the indirection
2835 	 * table will be reinitialized by ena_up()
2836 	 */
2837 	ena_com_rss_destroy(ena_dev);
2838 	ena_init_io_rings(adapter,
2839 			  0,
2840 			  adapter->xdp_num_queues +
2841 			  adapter->num_io_queues);
2842 	return dev_was_up ? ena_open(adapter->netdev) : 0;
2843 }
2844 
2845 static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx,
2846 			struct sk_buff *skb,
2847 			bool disable_meta_caching)
2848 {
2849 	u32 mss = skb_shinfo(skb)->gso_size;
2850 	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
2851 	u8 l4_protocol = 0;
2852 
2853 	if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) {
2854 		ena_tx_ctx->l4_csum_enable = 1;
2855 		if (mss) {
2856 			ena_tx_ctx->tso_enable = 1;
2857 			ena_meta->l4_hdr_len = tcp_hdr(skb)->doff;
2858 			ena_tx_ctx->l4_csum_partial = 0;
2859 		} else {
2860 			ena_tx_ctx->tso_enable = 0;
2861 			ena_meta->l4_hdr_len = 0;
2862 			ena_tx_ctx->l4_csum_partial = 1;
2863 		}
2864 
2865 		switch (ip_hdr(skb)->version) {
2866 		case IPVERSION:
2867 			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
2868 			if (ip_hdr(skb)->frag_off & htons(IP_DF))
2869 				ena_tx_ctx->df = 1;
2870 			if (mss)
2871 				ena_tx_ctx->l3_csum_enable = 1;
2872 			l4_protocol = ip_hdr(skb)->protocol;
2873 			break;
2874 		case 6:
2875 			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
2876 			l4_protocol = ipv6_hdr(skb)->nexthdr;
2877 			break;
2878 		default:
2879 			break;
2880 		}
2881 
2882 		if (l4_protocol == IPPROTO_TCP)
2883 			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
2884 		else
2885 			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
2886 
2887 		ena_meta->mss = mss;
2888 		ena_meta->l3_hdr_len = skb_network_header_len(skb);
2889 		ena_meta->l3_hdr_offset = skb_network_offset(skb);
2890 		ena_tx_ctx->meta_valid = 1;
2891 	} else if (disable_meta_caching) {
2892 		memset(ena_meta, 0, sizeof(*ena_meta));
2893 		ena_tx_ctx->meta_valid = 1;
2894 	} else {
2895 		ena_tx_ctx->meta_valid = 0;
2896 	}
2897 }
2898 
2899 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
2900 				       struct sk_buff *skb)
2901 {
2902 	int num_frags, header_len, rc;
2903 
2904 	num_frags = skb_shinfo(skb)->nr_frags;
2905 	header_len = skb_headlen(skb);
2906 
2907 	if (num_frags < tx_ring->sgl_size)
2908 		return 0;
2909 
2910 	if ((num_frags == tx_ring->sgl_size) &&
2911 	    (header_len < tx_ring->tx_max_header_size))
2912 		return 0;
2913 
2914 	ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp);
2915 
2916 	rc = skb_linearize(skb);
2917 	if (unlikely(rc)) {
2918 		ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1,
2919 				  &tx_ring->syncp);
2920 	}
2921 
2922 	return rc;
2923 }
2924 
2925 static int ena_tx_map_skb(struct ena_ring *tx_ring,
2926 			  struct ena_tx_buffer *tx_info,
2927 			  struct sk_buff *skb,
2928 			  void **push_hdr,
2929 			  u16 *header_len)
2930 {
2931 	struct ena_adapter *adapter = tx_ring->adapter;
2932 	struct ena_com_buf *ena_buf;
2933 	dma_addr_t dma;
2934 	u32 skb_head_len, frag_len, last_frag;
2935 	u16 push_len = 0;
2936 	u16 delta = 0;
2937 	int i = 0;
2938 
2939 	skb_head_len = skb_headlen(skb);
2940 	tx_info->skb = skb;
2941 	ena_buf = tx_info->bufs;
2942 
2943 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2944 		/* When the device is LLQ mode, the driver will copy
2945 		 * the header into the device memory space.
2946 		 * the ena_com layer assume the header is in a linear
2947 		 * memory space.
2948 		 * This assumption might be wrong since part of the header
2949 		 * can be in the fragmented buffers.
2950 		 * Use skb_header_pointer to make sure the header is in a
2951 		 * linear memory space.
2952 		 */
2953 
2954 		push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);
2955 		*push_hdr = skb_header_pointer(skb, 0, push_len,
2956 					       tx_ring->push_buf_intermediate_buf);
2957 		*header_len = push_len;
2958 		if (unlikely(skb->data != *push_hdr)) {
2959 			ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1,
2960 					  &tx_ring->syncp);
2961 
2962 			delta = push_len - skb_head_len;
2963 		}
2964 	} else {
2965 		*push_hdr = NULL;
2966 		*header_len = min_t(u32, skb_head_len,
2967 				    tx_ring->tx_max_header_size);
2968 	}
2969 
2970 	netif_dbg(adapter, tx_queued, adapter->netdev,
2971 		  "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
2972 		  *push_hdr, push_len);
2973 
2974 	if (skb_head_len > push_len) {
2975 		dma = dma_map_single(tx_ring->dev, skb->data + push_len,
2976 				     skb_head_len - push_len, DMA_TO_DEVICE);
2977 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
2978 			goto error_report_dma_error;
2979 
2980 		ena_buf->paddr = dma;
2981 		ena_buf->len = skb_head_len - push_len;
2982 
2983 		ena_buf++;
2984 		tx_info->num_of_bufs++;
2985 		tx_info->map_linear_data = 1;
2986 	} else {
2987 		tx_info->map_linear_data = 0;
2988 	}
2989 
2990 	last_frag = skb_shinfo(skb)->nr_frags;
2991 
2992 	for (i = 0; i < last_frag; i++) {
2993 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
2994 
2995 		frag_len = skb_frag_size(frag);
2996 
2997 		if (unlikely(delta >= frag_len)) {
2998 			delta -= frag_len;
2999 			continue;
3000 		}
3001 
3002 		dma = skb_frag_dma_map(tx_ring->dev, frag, delta,
3003 				       frag_len - delta, DMA_TO_DEVICE);
3004 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
3005 			goto error_report_dma_error;
3006 
3007 		ena_buf->paddr = dma;
3008 		ena_buf->len = frag_len - delta;
3009 		ena_buf++;
3010 		tx_info->num_of_bufs++;
3011 		delta = 0;
3012 	}
3013 
3014 	return 0;
3015 
3016 error_report_dma_error:
3017 	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,
3018 			  &tx_ring->syncp);
3019 	netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map skb\n");
3020 
3021 	tx_info->skb = NULL;
3022 
3023 	tx_info->num_of_bufs += i;
3024 	ena_unmap_tx_buff(tx_ring, tx_info);
3025 
3026 	return -EINVAL;
3027 }
3028 
3029 /* Called with netif_tx_lock. */
3030 static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
3031 {
3032 	struct ena_adapter *adapter = netdev_priv(dev);
3033 	struct ena_tx_buffer *tx_info;
3034 	struct ena_com_tx_ctx ena_tx_ctx;
3035 	struct ena_ring *tx_ring;
3036 	struct netdev_queue *txq;
3037 	void *push_hdr;
3038 	u16 next_to_use, req_id, header_len;
3039 	int qid, rc;
3040 
3041 	netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
3042 	/*  Determine which tx ring we will be placed on */
3043 	qid = skb_get_queue_mapping(skb);
3044 	tx_ring = &adapter->tx_ring[qid];
3045 	txq = netdev_get_tx_queue(dev, qid);
3046 
3047 	rc = ena_check_and_linearize_skb(tx_ring, skb);
3048 	if (unlikely(rc))
3049 		goto error_drop_packet;
3050 
3051 	skb_tx_timestamp(skb);
3052 
3053 	next_to_use = tx_ring->next_to_use;
3054 	req_id = tx_ring->free_ids[next_to_use];
3055 	tx_info = &tx_ring->tx_buffer_info[req_id];
3056 	tx_info->num_of_bufs = 0;
3057 
3058 	WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
3059 
3060 	rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);
3061 	if (unlikely(rc))
3062 		goto error_drop_packet;
3063 
3064 	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
3065 	ena_tx_ctx.ena_bufs = tx_info->bufs;
3066 	ena_tx_ctx.push_header = push_hdr;
3067 	ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
3068 	ena_tx_ctx.req_id = req_id;
3069 	ena_tx_ctx.header_len = header_len;
3070 
3071 	/* set flags and meta data */
3072 	ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);
3073 
3074 	rc = ena_xmit_common(dev,
3075 			     tx_ring,
3076 			     tx_info,
3077 			     &ena_tx_ctx,
3078 			     next_to_use,
3079 			     skb->len);
3080 	if (rc)
3081 		goto error_unmap_dma;
3082 
3083 	netdev_tx_sent_queue(txq, skb->len);
3084 
3085 	/* stop the queue when no more space available, the packet can have up
3086 	 * to sgl_size + 2. one for the meta descriptor and one for header
3087 	 * (if the header is larger than tx_max_header_size).
3088 	 */
3089 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
3090 						   tx_ring->sgl_size + 2))) {
3091 		netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",
3092 			  __func__, qid);
3093 
3094 		netif_tx_stop_queue(txq);
3095 		ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1,
3096 				  &tx_ring->syncp);
3097 
3098 		/* There is a rare condition where this function decide to
3099 		 * stop the queue but meanwhile clean_tx_irq updates
3100 		 * next_to_completion and terminates.
3101 		 * The queue will remain stopped forever.
3102 		 * To solve this issue add a mb() to make sure that
3103 		 * netif_tx_stop_queue() write is vissible before checking if
3104 		 * there is additional space in the queue.
3105 		 */
3106 		smp_mb();
3107 
3108 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
3109 						 ENA_TX_WAKEUP_THRESH)) {
3110 			netif_tx_wake_queue(txq);
3111 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
3112 					  &tx_ring->syncp);
3113 		}
3114 	}
3115 
3116 	if (netif_xmit_stopped(txq) || !netdev_xmit_more())
3117 		/* trigger the dma engine. ena_ring_tx_doorbell()
3118 		 * calls a memory barrier inside it.
3119 		 */
3120 		ena_ring_tx_doorbell(tx_ring);
3121 
3122 	return NETDEV_TX_OK;
3123 
3124 error_unmap_dma:
3125 	ena_unmap_tx_buff(tx_ring, tx_info);
3126 	tx_info->skb = NULL;
3127 
3128 error_drop_packet:
3129 	dev_kfree_skb(skb);
3130 	return NETDEV_TX_OK;
3131 }
3132 
3133 static u16 ena_select_queue(struct net_device *dev, struct sk_buff *skb,
3134 			    struct net_device *sb_dev)
3135 {
3136 	u16 qid;
3137 	/* we suspect that this is good for in--kernel network services that
3138 	 * want to loop incoming skb rx to tx in normal user generated traffic,
3139 	 * most probably we will not get to this
3140 	 */
3141 	if (skb_rx_queue_recorded(skb))
3142 		qid = skb_get_rx_queue(skb);
3143 	else
3144 		qid = netdev_pick_tx(dev, skb, NULL);
3145 
3146 	return qid;
3147 }
3148 
3149 static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
3150 {
3151 	struct device *dev = &pdev->dev;
3152 	struct ena_admin_host_info *host_info;
3153 	int rc;
3154 
3155 	/* Allocate only the host info */
3156 	rc = ena_com_allocate_host_info(ena_dev);
3157 	if (rc) {
3158 		dev_err(dev, "Cannot allocate host info\n");
3159 		return;
3160 	}
3161 
3162 	host_info = ena_dev->host_attr.host_info;
3163 
3164 	host_info->bdf = (pdev->bus->number << 8) | pdev->devfn;
3165 	host_info->os_type = ENA_ADMIN_OS_LINUX;
3166 	host_info->kernel_ver = LINUX_VERSION_CODE;
3167 	strscpy(host_info->kernel_ver_str, utsname()->version,
3168 		sizeof(host_info->kernel_ver_str) - 1);
3169 	host_info->os_dist = 0;
3170 	strncpy(host_info->os_dist_str, utsname()->release,
3171 		sizeof(host_info->os_dist_str) - 1);
3172 	host_info->driver_version =
3173 		(DRV_MODULE_GEN_MAJOR) |
3174 		(DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
3175 		(DRV_MODULE_GEN_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) |
3176 		("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);
3177 	host_info->num_cpus = num_online_cpus();
3178 
3179 	host_info->driver_supported_features =
3180 		ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |
3181 		ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK |
3182 		ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK |
3183 		ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK;
3184 
3185 	rc = ena_com_set_host_attributes(ena_dev);
3186 	if (rc) {
3187 		if (rc == -EOPNOTSUPP)
3188 			dev_warn(dev, "Cannot set host attributes\n");
3189 		else
3190 			dev_err(dev, "Cannot set host attributes\n");
3191 
3192 		goto err;
3193 	}
3194 
3195 	return;
3196 
3197 err:
3198 	ena_com_delete_host_info(ena_dev);
3199 }
3200 
3201 static void ena_config_debug_area(struct ena_adapter *adapter)
3202 {
3203 	u32 debug_area_size;
3204 	int rc, ss_count;
3205 
3206 	ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS);
3207 	if (ss_count <= 0) {
3208 		netif_err(adapter, drv, adapter->netdev,
3209 			  "SS count is negative\n");
3210 		return;
3211 	}
3212 
3213 	/* allocate 32 bytes for each string and 64bit for the value */
3214 	debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;
3215 
3216 	rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);
3217 	if (rc) {
3218 		netif_err(adapter, drv, adapter->netdev,
3219 			  "Cannot allocate debug area\n");
3220 		return;
3221 	}
3222 
3223 	rc = ena_com_set_host_attributes(adapter->ena_dev);
3224 	if (rc) {
3225 		if (rc == -EOPNOTSUPP)
3226 			netif_warn(adapter, drv, adapter->netdev,
3227 				   "Cannot set host attributes\n");
3228 		else
3229 			netif_err(adapter, drv, adapter->netdev,
3230 				  "Cannot set host attributes\n");
3231 		goto err;
3232 	}
3233 
3234 	return;
3235 err:
3236 	ena_com_delete_debug_area(adapter->ena_dev);
3237 }
3238 
3239 int ena_update_hw_stats(struct ena_adapter *adapter)
3240 {
3241 	int rc;
3242 
3243 	rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_stats);
3244 	if (rc) {
3245 		netdev_err(adapter->netdev, "Failed to get ENI stats\n");
3246 		return rc;
3247 	}
3248 
3249 	return 0;
3250 }
3251 
3252 static void ena_get_stats64(struct net_device *netdev,
3253 			    struct rtnl_link_stats64 *stats)
3254 {
3255 	struct ena_adapter *adapter = netdev_priv(netdev);
3256 	struct ena_ring *rx_ring, *tx_ring;
3257 	unsigned int start;
3258 	u64 rx_drops;
3259 	u64 tx_drops;
3260 	int i;
3261 
3262 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3263 		return;
3264 
3265 	for (i = 0; i < adapter->num_io_queues; i++) {
3266 		u64 bytes, packets;
3267 
3268 		tx_ring = &adapter->tx_ring[i];
3269 
3270 		do {
3271 			start = u64_stats_fetch_begin(&tx_ring->syncp);
3272 			packets = tx_ring->tx_stats.cnt;
3273 			bytes = tx_ring->tx_stats.bytes;
3274 		} while (u64_stats_fetch_retry(&tx_ring->syncp, start));
3275 
3276 		stats->tx_packets += packets;
3277 		stats->tx_bytes += bytes;
3278 
3279 		rx_ring = &adapter->rx_ring[i];
3280 
3281 		do {
3282 			start = u64_stats_fetch_begin(&rx_ring->syncp);
3283 			packets = rx_ring->rx_stats.cnt;
3284 			bytes = rx_ring->rx_stats.bytes;
3285 		} while (u64_stats_fetch_retry(&rx_ring->syncp, start));
3286 
3287 		stats->rx_packets += packets;
3288 		stats->rx_bytes += bytes;
3289 	}
3290 
3291 	do {
3292 		start = u64_stats_fetch_begin(&adapter->syncp);
3293 		rx_drops = adapter->dev_stats.rx_drops;
3294 		tx_drops = adapter->dev_stats.tx_drops;
3295 	} while (u64_stats_fetch_retry(&adapter->syncp, start));
3296 
3297 	stats->rx_dropped = rx_drops;
3298 	stats->tx_dropped = tx_drops;
3299 
3300 	stats->multicast = 0;
3301 	stats->collisions = 0;
3302 
3303 	stats->rx_length_errors = 0;
3304 	stats->rx_crc_errors = 0;
3305 	stats->rx_frame_errors = 0;
3306 	stats->rx_fifo_errors = 0;
3307 	stats->rx_missed_errors = 0;
3308 	stats->tx_window_errors = 0;
3309 
3310 	stats->rx_errors = 0;
3311 	stats->tx_errors = 0;
3312 }
3313 
3314 static const struct net_device_ops ena_netdev_ops = {
3315 	.ndo_open		= ena_open,
3316 	.ndo_stop		= ena_close,
3317 	.ndo_start_xmit		= ena_start_xmit,
3318 	.ndo_select_queue	= ena_select_queue,
3319 	.ndo_get_stats64	= ena_get_stats64,
3320 	.ndo_tx_timeout		= ena_tx_timeout,
3321 	.ndo_change_mtu		= ena_change_mtu,
3322 	.ndo_set_mac_address	= NULL,
3323 	.ndo_validate_addr	= eth_validate_addr,
3324 	.ndo_bpf		= ena_xdp,
3325 	.ndo_xdp_xmit		= ena_xdp_xmit,
3326 };
3327 
3328 static int ena_device_validate_params(struct ena_adapter *adapter,
3329 				      struct ena_com_dev_get_features_ctx *get_feat_ctx)
3330 {
3331 	struct net_device *netdev = adapter->netdev;
3332 	int rc;
3333 
3334 	rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr,
3335 			      adapter->mac_addr);
3336 	if (!rc) {
3337 		netif_err(adapter, drv, netdev,
3338 			  "Error, mac address are different\n");
3339 		return -EINVAL;
3340 	}
3341 
3342 	if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {
3343 		netif_err(adapter, drv, netdev,
3344 			  "Error, device max mtu is smaller than netdev MTU\n");
3345 		return -EINVAL;
3346 	}
3347 
3348 	return 0;
3349 }
3350 
3351 static void set_default_llq_configurations(struct ena_llq_configurations *llq_config)
3352 {
3353 	llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
3354 	llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
3355 	llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
3356 	llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
3357 	llq_config->llq_ring_entry_size_value = 128;
3358 }
3359 
3360 static int ena_set_queues_placement_policy(struct pci_dev *pdev,
3361 					   struct ena_com_dev *ena_dev,
3362 					   struct ena_admin_feature_llq_desc *llq,
3363 					   struct ena_llq_configurations *llq_default_configurations)
3364 {
3365 	int rc;
3366 	u32 llq_feature_mask;
3367 
3368 	llq_feature_mask = 1 << ENA_ADMIN_LLQ;
3369 	if (!(ena_dev->supported_features & llq_feature_mask)) {
3370 		dev_warn(&pdev->dev,
3371 			"LLQ is not supported Fallback to host mode policy.\n");
3372 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3373 		return 0;
3374 	}
3375 
3376 	rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
3377 	if (unlikely(rc)) {
3378 		dev_err(&pdev->dev,
3379 			"Failed to configure the device mode.  Fallback to host mode policy.\n");
3380 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3381 	}
3382 
3383 	return 0;
3384 }
3385 
3386 static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev,
3387 			       int bars)
3388 {
3389 	bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR));
3390 
3391 	if (!has_mem_bar) {
3392 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
3393 			dev_err(&pdev->dev,
3394 				"ENA device does not expose LLQ bar. Fallback to host mode policy.\n");
3395 			ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3396 		}
3397 
3398 		return 0;
3399 	}
3400 
3401 	ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
3402 					   pci_resource_start(pdev, ENA_MEM_BAR),
3403 					   pci_resource_len(pdev, ENA_MEM_BAR));
3404 
3405 	if (!ena_dev->mem_bar)
3406 		return -EFAULT;
3407 
3408 	return 0;
3409 }
3410 
3411 static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
3412 			   struct ena_com_dev_get_features_ctx *get_feat_ctx,
3413 			   bool *wd_state)
3414 {
3415 	struct ena_llq_configurations llq_config;
3416 	struct device *dev = &pdev->dev;
3417 	bool readless_supported;
3418 	u32 aenq_groups;
3419 	int dma_width;
3420 	int rc;
3421 
3422 	rc = ena_com_mmio_reg_read_request_init(ena_dev);
3423 	if (rc) {
3424 		dev_err(dev, "Failed to init mmio read less\n");
3425 		return rc;
3426 	}
3427 
3428 	/* The PCIe configuration space revision id indicate if mmio reg
3429 	 * read is disabled
3430 	 */
3431 	readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
3432 	ena_com_set_mmio_read_mode(ena_dev, readless_supported);
3433 
3434 	rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
3435 	if (rc) {
3436 		dev_err(dev, "Can not reset device\n");
3437 		goto err_mmio_read_less;
3438 	}
3439 
3440 	rc = ena_com_validate_version(ena_dev);
3441 	if (rc) {
3442 		dev_err(dev, "Device version is too low\n");
3443 		goto err_mmio_read_less;
3444 	}
3445 
3446 	dma_width = ena_com_get_dma_width(ena_dev);
3447 	if (dma_width < 0) {
3448 		dev_err(dev, "Invalid dma width value %d", dma_width);
3449 		rc = dma_width;
3450 		goto err_mmio_read_less;
3451 	}
3452 
3453 	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(dma_width));
3454 	if (rc) {
3455 		dev_err(dev, "dma_set_mask_and_coherent failed %d\n", rc);
3456 		goto err_mmio_read_less;
3457 	}
3458 
3459 	/* ENA admin level init */
3460 	rc = ena_com_admin_init(ena_dev, &aenq_handlers);
3461 	if (rc) {
3462 		dev_err(dev,
3463 			"Can not initialize ena admin queue with device\n");
3464 		goto err_mmio_read_less;
3465 	}
3466 
3467 	/* To enable the msix interrupts the driver needs to know the number
3468 	 * of queues. So the driver uses polling mode to retrieve this
3469 	 * information
3470 	 */
3471 	ena_com_set_admin_polling_mode(ena_dev, true);
3472 
3473 	ena_config_host_info(ena_dev, pdev);
3474 
3475 	/* Get Device Attributes*/
3476 	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3477 	if (rc) {
3478 		dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc);
3479 		goto err_admin_init;
3480 	}
3481 
3482 	/* Try to turn all the available aenq groups */
3483 	aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
3484 		BIT(ENA_ADMIN_FATAL_ERROR) |
3485 		BIT(ENA_ADMIN_WARNING) |
3486 		BIT(ENA_ADMIN_NOTIFICATION) |
3487 		BIT(ENA_ADMIN_KEEP_ALIVE);
3488 
3489 	aenq_groups &= get_feat_ctx->aenq.supported_groups;
3490 
3491 	rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3492 	if (rc) {
3493 		dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc);
3494 		goto err_admin_init;
3495 	}
3496 
3497 	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3498 
3499 	set_default_llq_configurations(&llq_config);
3500 
3501 	rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
3502 					     &llq_config);
3503 	if (rc) {
3504 		dev_err(dev, "ENA device init failed\n");
3505 		goto err_admin_init;
3506 	}
3507 
3508 	return 0;
3509 
3510 err_admin_init:
3511 	ena_com_delete_host_info(ena_dev);
3512 	ena_com_admin_destroy(ena_dev);
3513 err_mmio_read_less:
3514 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3515 
3516 	return rc;
3517 }
3518 
3519 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)
3520 {
3521 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3522 	struct device *dev = &adapter->pdev->dev;
3523 	int rc;
3524 
3525 	rc = ena_enable_msix(adapter);
3526 	if (rc) {
3527 		dev_err(dev, "Can not reserve msix vectors\n");
3528 		return rc;
3529 	}
3530 
3531 	ena_setup_mgmnt_intr(adapter);
3532 
3533 	rc = ena_request_mgmnt_irq(adapter);
3534 	if (rc) {
3535 		dev_err(dev, "Can not setup management interrupts\n");
3536 		goto err_disable_msix;
3537 	}
3538 
3539 	ena_com_set_admin_polling_mode(ena_dev, false);
3540 
3541 	ena_com_admin_aenq_enable(ena_dev);
3542 
3543 	return 0;
3544 
3545 err_disable_msix:
3546 	ena_disable_msix(adapter);
3547 
3548 	return rc;
3549 }
3550 
3551 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3552 {
3553 	struct net_device *netdev = adapter->netdev;
3554 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3555 	bool dev_up;
3556 
3557 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
3558 		return;
3559 
3560 	netif_carrier_off(netdev);
3561 
3562 	del_timer_sync(&adapter->timer_service);
3563 
3564 	dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
3565 	adapter->dev_up_before_reset = dev_up;
3566 	if (!graceful)
3567 		ena_com_set_admin_running_state(ena_dev, false);
3568 
3569 	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3570 		ena_down(adapter);
3571 
3572 	/* Stop the device from sending AENQ events (in case reset flag is set
3573 	 *  and device is up, ena_down() already reset the device.
3574 	 */
3575 	if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up))
3576 		ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
3577 
3578 	ena_free_mgmnt_irq(adapter);
3579 
3580 	ena_disable_msix(adapter);
3581 
3582 	ena_com_abort_admin_commands(ena_dev);
3583 
3584 	ena_com_wait_for_abort_completion(ena_dev);
3585 
3586 	ena_com_admin_destroy(ena_dev);
3587 
3588 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3589 
3590 	/* return reset reason to default value */
3591 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3592 
3593 	clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3594 	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3595 }
3596 
3597 static int ena_restore_device(struct ena_adapter *adapter)
3598 {
3599 	struct ena_com_dev_get_features_ctx get_feat_ctx;
3600 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3601 	struct pci_dev *pdev = adapter->pdev;
3602 	bool wd_state;
3603 	int rc;
3604 
3605 	set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3606 	rc = ena_device_init(ena_dev, adapter->pdev, &get_feat_ctx, &wd_state);
3607 	if (rc) {
3608 		dev_err(&pdev->dev, "Can not initialize device\n");
3609 		goto err;
3610 	}
3611 	adapter->wd_state = wd_state;
3612 
3613 	rc = ena_device_validate_params(adapter, &get_feat_ctx);
3614 	if (rc) {
3615 		dev_err(&pdev->dev, "Validation of device parameters failed\n");
3616 		goto err_device_destroy;
3617 	}
3618 
3619 	rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3620 	if (rc) {
3621 		dev_err(&pdev->dev, "Enable MSI-X failed\n");
3622 		goto err_device_destroy;
3623 	}
3624 	/* If the interface was up before the reset bring it up */
3625 	if (adapter->dev_up_before_reset) {
3626 		rc = ena_up(adapter);
3627 		if (rc) {
3628 			dev_err(&pdev->dev, "Failed to create I/O queues\n");
3629 			goto err_disable_msix;
3630 		}
3631 	}
3632 
3633 	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3634 
3635 	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3636 	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
3637 		netif_carrier_on(adapter->netdev);
3638 
3639 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3640 	adapter->last_keep_alive_jiffies = jiffies;
3641 
3642 	return rc;
3643 err_disable_msix:
3644 	ena_free_mgmnt_irq(adapter);
3645 	ena_disable_msix(adapter);
3646 err_device_destroy:
3647 	ena_com_abort_admin_commands(ena_dev);
3648 	ena_com_wait_for_abort_completion(ena_dev);
3649 	ena_com_admin_destroy(ena_dev);
3650 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3651 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3652 err:
3653 	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3654 	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3655 	dev_err(&pdev->dev,
3656 		"Reset attempt failed. Can not reset the device\n");
3657 
3658 	return rc;
3659 }
3660 
3661 static void ena_fw_reset_device(struct work_struct *work)
3662 {
3663 	struct ena_adapter *adapter =
3664 		container_of(work, struct ena_adapter, reset_task);
3665 
3666 	rtnl_lock();
3667 
3668 	if (likely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3669 		ena_destroy_device(adapter, false);
3670 		ena_restore_device(adapter);
3671 
3672 		dev_err(&adapter->pdev->dev, "Device reset completed successfully\n");
3673 	}
3674 
3675 	rtnl_unlock();
3676 }
3677 
3678 static int check_for_rx_interrupt_queue(struct ena_adapter *adapter,
3679 					struct ena_ring *rx_ring)
3680 {
3681 	struct ena_napi *ena_napi = container_of(rx_ring->napi, struct ena_napi, napi);
3682 
3683 	if (likely(READ_ONCE(ena_napi->first_interrupt)))
3684 		return 0;
3685 
3686 	if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
3687 		return 0;
3688 
3689 	rx_ring->no_interrupt_event_cnt++;
3690 
3691 	if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) {
3692 		netif_err(adapter, rx_err, adapter->netdev,
3693 			  "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",
3694 			  rx_ring->qid);
3695 
3696 		ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3697 		return -EIO;
3698 	}
3699 
3700 	return 0;
3701 }
3702 
3703 static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
3704 					  struct ena_ring *tx_ring)
3705 {
3706 	struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi);
3707 	unsigned int time_since_last_napi;
3708 	unsigned int missing_tx_comp_to;
3709 	bool is_tx_comp_time_expired;
3710 	struct ena_tx_buffer *tx_buf;
3711 	unsigned long last_jiffies;
3712 	u32 missed_tx = 0;
3713 	int i, rc = 0;
3714 
3715 	for (i = 0; i < tx_ring->ring_size; i++) {
3716 		tx_buf = &tx_ring->tx_buffer_info[i];
3717 		last_jiffies = tx_buf->last_jiffies;
3718 
3719 		if (last_jiffies == 0)
3720 			/* no pending Tx at this location */
3721 			continue;
3722 
3723 		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +
3724 			 2 * adapter->missing_tx_completion_to);
3725 
3726 		if (unlikely(!READ_ONCE(ena_napi->first_interrupt) && is_tx_comp_time_expired)) {
3727 			/* If after graceful period interrupt is still not
3728 			 * received, we schedule a reset
3729 			 */
3730 			netif_err(adapter, tx_err, adapter->netdev,
3731 				  "Potential MSIX issue on Tx side Queue = %d. Reset the device\n",
3732 				  tx_ring->qid);
3733 			ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3734 			return -EIO;
3735 		}
3736 
3737 		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +
3738 			adapter->missing_tx_completion_to);
3739 
3740 		if (unlikely(is_tx_comp_time_expired)) {
3741 			if (!tx_buf->print_once) {
3742 				time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);
3743 				missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to);
3744 				netif_notice(adapter, tx_err, adapter->netdev,
3745 					     "Found a Tx that wasn't completed on time, qid %d, index %d. %u usecs have passed since last napi execution. Missing Tx timeout value %u msecs\n",
3746 					     tx_ring->qid, i, time_since_last_napi, missing_tx_comp_to);
3747 			}
3748 
3749 			tx_buf->print_once = 1;
3750 			missed_tx++;
3751 		}
3752 	}
3753 
3754 	if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {
3755 		netif_err(adapter, tx_err, adapter->netdev,
3756 			  "The number of lost tx completions is above the threshold (%d > %d). Reset the device\n",
3757 			  missed_tx,
3758 			  adapter->missing_tx_completion_threshold);
3759 		ena_reset_device(adapter, ENA_REGS_RESET_MISS_TX_CMPL);
3760 		rc = -EIO;
3761 	}
3762 
3763 	ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx,
3764 			  &tx_ring->syncp);
3765 
3766 	return rc;
3767 }
3768 
3769 static void check_for_missing_completions(struct ena_adapter *adapter)
3770 {
3771 	struct ena_ring *tx_ring;
3772 	struct ena_ring *rx_ring;
3773 	int i, budget, rc;
3774 	int io_queue_count;
3775 
3776 	io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues;
3777 	/* Make sure the driver doesn't turn the device in other process */
3778 	smp_rmb();
3779 
3780 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3781 		return;
3782 
3783 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3784 		return;
3785 
3786 	if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT)
3787 		return;
3788 
3789 	budget = ENA_MONITORED_TX_QUEUES;
3790 
3791 	for (i = adapter->last_monitored_tx_qid; i < io_queue_count; i++) {
3792 		tx_ring = &adapter->tx_ring[i];
3793 		rx_ring = &adapter->rx_ring[i];
3794 
3795 		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
3796 		if (unlikely(rc))
3797 			return;
3798 
3799 		rc =  !ENA_IS_XDP_INDEX(adapter, i) ?
3800 			check_for_rx_interrupt_queue(adapter, rx_ring) : 0;
3801 		if (unlikely(rc))
3802 			return;
3803 
3804 		budget--;
3805 		if (!budget)
3806 			break;
3807 	}
3808 
3809 	adapter->last_monitored_tx_qid = i % io_queue_count;
3810 }
3811 
3812 /* trigger napi schedule after 2 consecutive detections */
3813 #define EMPTY_RX_REFILL 2
3814 /* For the rare case where the device runs out of Rx descriptors and the
3815  * napi handler failed to refill new Rx descriptors (due to a lack of memory
3816  * for example).
3817  * This case will lead to a deadlock:
3818  * The device won't send interrupts since all the new Rx packets will be dropped
3819  * The napi handler won't allocate new Rx descriptors so the device will be
3820  * able to send new packets.
3821  *
3822  * This scenario can happen when the kernel's vm.min_free_kbytes is too small.
3823  * It is recommended to have at least 512MB, with a minimum of 128MB for
3824  * constrained environment).
3825  *
3826  * When such a situation is detected - Reschedule napi
3827  */
3828 static void check_for_empty_rx_ring(struct ena_adapter *adapter)
3829 {
3830 	struct ena_ring *rx_ring;
3831 	int i, refill_required;
3832 
3833 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3834 		return;
3835 
3836 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3837 		return;
3838 
3839 	for (i = 0; i < adapter->num_io_queues; i++) {
3840 		rx_ring = &adapter->rx_ring[i];
3841 
3842 		refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
3843 		if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
3844 			rx_ring->empty_rx_queue++;
3845 
3846 			if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
3847 				ena_increase_stat(&rx_ring->rx_stats.empty_rx_ring, 1,
3848 						  &rx_ring->syncp);
3849 
3850 				netif_err(adapter, drv, adapter->netdev,
3851 					  "Trigger refill for ring %d\n", i);
3852 
3853 				napi_schedule(rx_ring->napi);
3854 				rx_ring->empty_rx_queue = 0;
3855 			}
3856 		} else {
3857 			rx_ring->empty_rx_queue = 0;
3858 		}
3859 	}
3860 }
3861 
3862 /* Check for keep alive expiration */
3863 static void check_for_missing_keep_alive(struct ena_adapter *adapter)
3864 {
3865 	unsigned long keep_alive_expired;
3866 
3867 	if (!adapter->wd_state)
3868 		return;
3869 
3870 	if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3871 		return;
3872 
3873 	keep_alive_expired = adapter->last_keep_alive_jiffies +
3874 			     adapter->keep_alive_timeout;
3875 	if (unlikely(time_is_before_jiffies(keep_alive_expired))) {
3876 		netif_err(adapter, drv, adapter->netdev,
3877 			  "Keep alive watchdog timeout.\n");
3878 		ena_increase_stat(&adapter->dev_stats.wd_expired, 1,
3879 				  &adapter->syncp);
3880 		ena_reset_device(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO);
3881 	}
3882 }
3883 
3884 static void check_for_admin_com_state(struct ena_adapter *adapter)
3885 {
3886 	if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {
3887 		netif_err(adapter, drv, adapter->netdev,
3888 			  "ENA admin queue is not in running state!\n");
3889 		ena_increase_stat(&adapter->dev_stats.admin_q_pause, 1,
3890 				  &adapter->syncp);
3891 		ena_reset_device(adapter, ENA_REGS_RESET_ADMIN_TO);
3892 	}
3893 }
3894 
3895 static void ena_update_hints(struct ena_adapter *adapter,
3896 			     struct ena_admin_ena_hw_hints *hints)
3897 {
3898 	struct net_device *netdev = adapter->netdev;
3899 
3900 	if (hints->admin_completion_tx_timeout)
3901 		adapter->ena_dev->admin_queue.completion_timeout =
3902 			hints->admin_completion_tx_timeout * 1000;
3903 
3904 	if (hints->mmio_read_timeout)
3905 		/* convert to usec */
3906 		adapter->ena_dev->mmio_read.reg_read_to =
3907 			hints->mmio_read_timeout * 1000;
3908 
3909 	if (hints->missed_tx_completion_count_threshold_to_reset)
3910 		adapter->missing_tx_completion_threshold =
3911 			hints->missed_tx_completion_count_threshold_to_reset;
3912 
3913 	if (hints->missing_tx_completion_timeout) {
3914 		if (hints->missing_tx_completion_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3915 			adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT;
3916 		else
3917 			adapter->missing_tx_completion_to =
3918 				msecs_to_jiffies(hints->missing_tx_completion_timeout);
3919 	}
3920 
3921 	if (hints->netdev_wd_timeout)
3922 		netdev->watchdog_timeo = msecs_to_jiffies(hints->netdev_wd_timeout);
3923 
3924 	if (hints->driver_watchdog_timeout) {
3925 		if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3926 			adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3927 		else
3928 			adapter->keep_alive_timeout =
3929 				msecs_to_jiffies(hints->driver_watchdog_timeout);
3930 	}
3931 }
3932 
3933 static void ena_update_host_info(struct ena_admin_host_info *host_info,
3934 				 struct net_device *netdev)
3935 {
3936 	host_info->supported_network_features[0] =
3937 		netdev->features & GENMASK_ULL(31, 0);
3938 	host_info->supported_network_features[1] =
3939 		(netdev->features & GENMASK_ULL(63, 32)) >> 32;
3940 }
3941 
3942 static void ena_timer_service(struct timer_list *t)
3943 {
3944 	struct ena_adapter *adapter = from_timer(adapter, t, timer_service);
3945 	u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr;
3946 	struct ena_admin_host_info *host_info =
3947 		adapter->ena_dev->host_attr.host_info;
3948 
3949 	check_for_missing_keep_alive(adapter);
3950 
3951 	check_for_admin_com_state(adapter);
3952 
3953 	check_for_missing_completions(adapter);
3954 
3955 	check_for_empty_rx_ring(adapter);
3956 
3957 	if (debug_area)
3958 		ena_dump_stats_to_buf(adapter, debug_area);
3959 
3960 	if (host_info)
3961 		ena_update_host_info(host_info, adapter->netdev);
3962 
3963 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3964 		netif_err(adapter, drv, adapter->netdev,
3965 			  "Trigger reset is on\n");
3966 		ena_dump_stats_to_dmesg(adapter);
3967 		queue_work(ena_wq, &adapter->reset_task);
3968 		return;
3969 	}
3970 
3971 	/* Reset the timer */
3972 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3973 }
3974 
3975 static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev,
3976 				     struct ena_com_dev *ena_dev,
3977 				     struct ena_com_dev_get_features_ctx *get_feat_ctx)
3978 {
3979 	u32 io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
3980 
3981 	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
3982 		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
3983 			&get_feat_ctx->max_queue_ext.max_queue_ext;
3984 		io_rx_num = min_t(u32, max_queue_ext->max_rx_sq_num,
3985 				  max_queue_ext->max_rx_cq_num);
3986 
3987 		io_tx_sq_num = max_queue_ext->max_tx_sq_num;
3988 		io_tx_cq_num = max_queue_ext->max_tx_cq_num;
3989 	} else {
3990 		struct ena_admin_queue_feature_desc *max_queues =
3991 			&get_feat_ctx->max_queues;
3992 		io_tx_sq_num = max_queues->max_sq_num;
3993 		io_tx_cq_num = max_queues->max_cq_num;
3994 		io_rx_num = min_t(u32, io_tx_sq_num, io_tx_cq_num);
3995 	}
3996 
3997 	/* In case of LLQ use the llq fields for the tx SQ/CQ */
3998 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
3999 		io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
4000 
4001 	max_num_io_queues = min_t(u32, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
4002 	max_num_io_queues = min_t(u32, max_num_io_queues, io_rx_num);
4003 	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_sq_num);
4004 	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num);
4005 	/* 1 IRQ for mgmnt and 1 IRQs for each IO direction */
4006 	max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1);
4007 
4008 	return max_num_io_queues;
4009 }
4010 
4011 static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
4012 				 struct net_device *netdev)
4013 {
4014 	netdev_features_t dev_features = 0;
4015 
4016 	/* Set offload features */
4017 	if (feat->offload.tx &
4018 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
4019 		dev_features |= NETIF_F_IP_CSUM;
4020 
4021 	if (feat->offload.tx &
4022 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
4023 		dev_features |= NETIF_F_IPV6_CSUM;
4024 
4025 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
4026 		dev_features |= NETIF_F_TSO;
4027 
4028 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
4029 		dev_features |= NETIF_F_TSO6;
4030 
4031 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)
4032 		dev_features |= NETIF_F_TSO_ECN;
4033 
4034 	if (feat->offload.rx_supported &
4035 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
4036 		dev_features |= NETIF_F_RXCSUM;
4037 
4038 	if (feat->offload.rx_supported &
4039 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
4040 		dev_features |= NETIF_F_RXCSUM;
4041 
4042 	netdev->features =
4043 		dev_features |
4044 		NETIF_F_SG |
4045 		NETIF_F_RXHASH |
4046 		NETIF_F_HIGHDMA;
4047 
4048 	netdev->hw_features |= netdev->features;
4049 	netdev->vlan_features |= netdev->features;
4050 }
4051 
4052 static void ena_set_conf_feat_params(struct ena_adapter *adapter,
4053 				     struct ena_com_dev_get_features_ctx *feat)
4054 {
4055 	struct net_device *netdev = adapter->netdev;
4056 
4057 	/* Copy mac address */
4058 	if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) {
4059 		eth_hw_addr_random(netdev);
4060 		ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
4061 	} else {
4062 		ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr);
4063 		eth_hw_addr_set(netdev, adapter->mac_addr);
4064 	}
4065 
4066 	/* Set offload features */
4067 	ena_set_dev_offloads(feat, netdev);
4068 
4069 	adapter->max_mtu = feat->dev_attr.max_mtu;
4070 	netdev->max_mtu = adapter->max_mtu;
4071 	netdev->min_mtu = ENA_MIN_MTU;
4072 }
4073 
4074 static int ena_rss_init_default(struct ena_adapter *adapter)
4075 {
4076 	struct ena_com_dev *ena_dev = adapter->ena_dev;
4077 	struct device *dev = &adapter->pdev->dev;
4078 	int rc, i;
4079 	u32 val;
4080 
4081 	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
4082 	if (unlikely(rc)) {
4083 		dev_err(dev, "Cannot init indirect table\n");
4084 		goto err_rss_init;
4085 	}
4086 
4087 	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
4088 		val = ethtool_rxfh_indir_default(i, adapter->num_io_queues);
4089 		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
4090 						       ENA_IO_RXQ_IDX(val));
4091 		if (unlikely(rc)) {
4092 			dev_err(dev, "Cannot fill indirect table\n");
4093 			goto err_fill_indir;
4094 		}
4095 	}
4096 
4097 	rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL,
4098 					ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
4099 	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
4100 		dev_err(dev, "Cannot fill hash function\n");
4101 		goto err_fill_indir;
4102 	}
4103 
4104 	rc = ena_com_set_default_hash_ctrl(ena_dev);
4105 	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
4106 		dev_err(dev, "Cannot fill hash control\n");
4107 		goto err_fill_indir;
4108 	}
4109 
4110 	return 0;
4111 
4112 err_fill_indir:
4113 	ena_com_rss_destroy(ena_dev);
4114 err_rss_init:
4115 
4116 	return rc;
4117 }
4118 
4119 static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
4120 {
4121 	int release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
4122 
4123 	pci_release_selected_regions(pdev, release_bars);
4124 }
4125 
4126 
4127 static void ena_calc_io_queue_size(struct ena_adapter *adapter,
4128 				   struct ena_com_dev_get_features_ctx *get_feat_ctx)
4129 {
4130 	struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq;
4131 	struct ena_com_dev *ena_dev = adapter->ena_dev;
4132 	u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;
4133 	u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;
4134 	u32 max_tx_queue_size;
4135 	u32 max_rx_queue_size;
4136 
4137 	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
4138 		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
4139 			&get_feat_ctx->max_queue_ext.max_queue_ext;
4140 		max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,
4141 					  max_queue_ext->max_rx_sq_depth);
4142 		max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
4143 
4144 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4145 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
4146 						  llq->max_llq_depth);
4147 		else
4148 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
4149 						  max_queue_ext->max_tx_sq_depth);
4150 
4151 		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4152 						 max_queue_ext->max_per_packet_tx_descs);
4153 		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4154 						 max_queue_ext->max_per_packet_rx_descs);
4155 	} else {
4156 		struct ena_admin_queue_feature_desc *max_queues =
4157 			&get_feat_ctx->max_queues;
4158 		max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,
4159 					  max_queues->max_sq_depth);
4160 		max_tx_queue_size = max_queues->max_cq_depth;
4161 
4162 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4163 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
4164 						  llq->max_llq_depth);
4165 		else
4166 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
4167 						  max_queues->max_sq_depth);
4168 
4169 		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4170 						 max_queues->max_packet_tx_descs);
4171 		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4172 						 max_queues->max_packet_rx_descs);
4173 	}
4174 
4175 	max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);
4176 	max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);
4177 
4178 	tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
4179 				  max_tx_queue_size);
4180 	rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
4181 				  max_rx_queue_size);
4182 
4183 	tx_queue_size = rounddown_pow_of_two(tx_queue_size);
4184 	rx_queue_size = rounddown_pow_of_two(rx_queue_size);
4185 
4186 	adapter->max_tx_ring_size  = max_tx_queue_size;
4187 	adapter->max_rx_ring_size = max_rx_queue_size;
4188 	adapter->requested_tx_ring_size = tx_queue_size;
4189 	adapter->requested_rx_ring_size = rx_queue_size;
4190 }
4191 
4192 /* ena_probe - Device Initialization Routine
4193  * @pdev: PCI device information struct
4194  * @ent: entry in ena_pci_tbl
4195  *
4196  * Returns 0 on success, negative on failure
4197  *
4198  * ena_probe initializes an adapter identified by a pci_dev structure.
4199  * The OS initialization, configuring of the adapter private structure,
4200  * and a hardware reset occur.
4201  */
4202 static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4203 {
4204 	struct ena_com_dev_get_features_ctx get_feat_ctx;
4205 	struct ena_com_dev *ena_dev = NULL;
4206 	struct ena_adapter *adapter;
4207 	struct net_device *netdev;
4208 	static int adapters_found;
4209 	u32 max_num_io_queues;
4210 	bool wd_state;
4211 	int bars, rc;
4212 
4213 	dev_dbg(&pdev->dev, "%s\n", __func__);
4214 
4215 	rc = pci_enable_device_mem(pdev);
4216 	if (rc) {
4217 		dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
4218 		return rc;
4219 	}
4220 
4221 	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(ENA_MAX_PHYS_ADDR_SIZE_BITS));
4222 	if (rc) {
4223 		dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", rc);
4224 		goto err_disable_device;
4225 	}
4226 
4227 	pci_set_master(pdev);
4228 
4229 	ena_dev = vzalloc(sizeof(*ena_dev));
4230 	if (!ena_dev) {
4231 		rc = -ENOMEM;
4232 		goto err_disable_device;
4233 	}
4234 
4235 	bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
4236 	rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
4237 	if (rc) {
4238 		dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
4239 			rc);
4240 		goto err_free_ena_dev;
4241 	}
4242 
4243 	ena_dev->reg_bar = devm_ioremap(&pdev->dev,
4244 					pci_resource_start(pdev, ENA_REG_BAR),
4245 					pci_resource_len(pdev, ENA_REG_BAR));
4246 	if (!ena_dev->reg_bar) {
4247 		dev_err(&pdev->dev, "Failed to remap regs bar\n");
4248 		rc = -EFAULT;
4249 		goto err_free_region;
4250 	}
4251 
4252 	ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;
4253 
4254 	ena_dev->dmadev = &pdev->dev;
4255 
4256 	netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), ENA_MAX_RINGS);
4257 	if (!netdev) {
4258 		dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");
4259 		rc = -ENOMEM;
4260 		goto err_free_region;
4261 	}
4262 
4263 	SET_NETDEV_DEV(netdev, &pdev->dev);
4264 	adapter = netdev_priv(netdev);
4265 	adapter->ena_dev = ena_dev;
4266 	adapter->netdev = netdev;
4267 	adapter->pdev = pdev;
4268 	adapter->msg_enable = DEFAULT_MSG_ENABLE;
4269 
4270 	ena_dev->net_device = netdev;
4271 
4272 	pci_set_drvdata(pdev, adapter);
4273 
4274 	rc = ena_device_init(ena_dev, pdev, &get_feat_ctx, &wd_state);
4275 	if (rc) {
4276 		dev_err(&pdev->dev, "ENA device init failed\n");
4277 		if (rc == -ETIME)
4278 			rc = -EPROBE_DEFER;
4279 		goto err_netdev_destroy;
4280 	}
4281 
4282 	rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);
4283 	if (rc) {
4284 		dev_err(&pdev->dev, "ENA llq bar mapping failed\n");
4285 		goto err_device_destroy;
4286 	}
4287 
4288 	/* Initial TX and RX interrupt delay. Assumes 1 usec granularity.
4289 	 * Updated during device initialization with the real granularity
4290 	 */
4291 	ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
4292 	ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;
4293 	ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
4294 	max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx);
4295 	ena_calc_io_queue_size(adapter, &get_feat_ctx);
4296 	if (unlikely(!max_num_io_queues)) {
4297 		rc = -EFAULT;
4298 		goto err_device_destroy;
4299 	}
4300 
4301 	ena_set_conf_feat_params(adapter, &get_feat_ctx);
4302 
4303 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
4304 
4305 	adapter->num_io_queues = max_num_io_queues;
4306 	adapter->max_num_io_queues = max_num_io_queues;
4307 	adapter->last_monitored_tx_qid = 0;
4308 
4309 	adapter->xdp_first_ring = 0;
4310 	adapter->xdp_num_queues = 0;
4311 
4312 	adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
4313 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4314 		adapter->disable_meta_caching =
4315 			!!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &
4316 			   BIT(ENA_ADMIN_DISABLE_META_CACHING));
4317 
4318 	adapter->wd_state = wd_state;
4319 
4320 	snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
4321 
4322 	rc = ena_com_init_interrupt_moderation(adapter->ena_dev);
4323 	if (rc) {
4324 		dev_err(&pdev->dev,
4325 			"Failed to query interrupt moderation feature\n");
4326 		goto err_device_destroy;
4327 	}
4328 	ena_init_io_rings(adapter,
4329 			  0,
4330 			  adapter->xdp_num_queues +
4331 			  adapter->num_io_queues);
4332 
4333 	netdev->netdev_ops = &ena_netdev_ops;
4334 	netdev->watchdog_timeo = TX_TIMEOUT;
4335 	ena_set_ethtool_ops(netdev);
4336 
4337 	netdev->priv_flags |= IFF_UNICAST_FLT;
4338 
4339 	u64_stats_init(&adapter->syncp);
4340 
4341 	rc = ena_enable_msix_and_set_admin_interrupts(adapter);
4342 	if (rc) {
4343 		dev_err(&pdev->dev,
4344 			"Failed to enable and set the admin interrupts\n");
4345 		goto err_worker_destroy;
4346 	}
4347 	rc = ena_rss_init_default(adapter);
4348 	if (rc && (rc != -EOPNOTSUPP)) {
4349 		dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
4350 		goto err_free_msix;
4351 	}
4352 
4353 	ena_config_debug_area(adapter);
4354 
4355 	memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);
4356 
4357 	netif_carrier_off(netdev);
4358 
4359 	rc = register_netdev(netdev);
4360 	if (rc) {
4361 		dev_err(&pdev->dev, "Cannot register net device\n");
4362 		goto err_rss;
4363 	}
4364 
4365 	INIT_WORK(&adapter->reset_task, ena_fw_reset_device);
4366 
4367 	adapter->last_keep_alive_jiffies = jiffies;
4368 	adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;
4369 	adapter->missing_tx_completion_to = TX_TIMEOUT;
4370 	adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS;
4371 
4372 	ena_update_hints(adapter, &get_feat_ctx.hw_hints);
4373 
4374 	timer_setup(&adapter->timer_service, ena_timer_service, 0);
4375 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
4376 
4377 	dev_info(&pdev->dev,
4378 		 "%s found at mem %lx, mac addr %pM\n",
4379 		 DEVICE_NAME, (long)pci_resource_start(pdev, 0),
4380 		 netdev->dev_addr);
4381 
4382 	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
4383 
4384 	adapters_found++;
4385 
4386 	return 0;
4387 
4388 err_rss:
4389 	ena_com_delete_debug_area(ena_dev);
4390 	ena_com_rss_destroy(ena_dev);
4391 err_free_msix:
4392 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
4393 	/* stop submitting admin commands on a device that was reset */
4394 	ena_com_set_admin_running_state(ena_dev, false);
4395 	ena_free_mgmnt_irq(adapter);
4396 	ena_disable_msix(adapter);
4397 err_worker_destroy:
4398 	del_timer(&adapter->timer_service);
4399 err_device_destroy:
4400 	ena_com_delete_host_info(ena_dev);
4401 	ena_com_admin_destroy(ena_dev);
4402 err_netdev_destroy:
4403 	free_netdev(netdev);
4404 err_free_region:
4405 	ena_release_bars(ena_dev, pdev);
4406 err_free_ena_dev:
4407 	vfree(ena_dev);
4408 err_disable_device:
4409 	pci_disable_device(pdev);
4410 	return rc;
4411 }
4412 
4413 /*****************************************************************************/
4414 
4415 /* __ena_shutoff - Helper used in both PCI remove/shutdown routines
4416  * @pdev: PCI device information struct
4417  * @shutdown: Is it a shutdown operation? If false, means it is a removal
4418  *
4419  * __ena_shutoff is a helper routine that does the real work on shutdown and
4420  * removal paths; the difference between those paths is with regards to whether
4421  * dettach or unregister the netdevice.
4422  */
4423 static void __ena_shutoff(struct pci_dev *pdev, bool shutdown)
4424 {
4425 	struct ena_adapter *adapter = pci_get_drvdata(pdev);
4426 	struct ena_com_dev *ena_dev;
4427 	struct net_device *netdev;
4428 
4429 	ena_dev = adapter->ena_dev;
4430 	netdev = adapter->netdev;
4431 
4432 #ifdef CONFIG_RFS_ACCEL
4433 	if ((adapter->msix_vecs >= 1) && (netdev->rx_cpu_rmap)) {
4434 		free_irq_cpu_rmap(netdev->rx_cpu_rmap);
4435 		netdev->rx_cpu_rmap = NULL;
4436 	}
4437 #endif /* CONFIG_RFS_ACCEL */
4438 
4439 	/* Make sure timer and reset routine won't be called after
4440 	 * freeing device resources.
4441 	 */
4442 	del_timer_sync(&adapter->timer_service);
4443 	cancel_work_sync(&adapter->reset_task);
4444 
4445 	rtnl_lock(); /* lock released inside the below if-else block */
4446 	adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN;
4447 	ena_destroy_device(adapter, true);
4448 	if (shutdown) {
4449 		netif_device_detach(netdev);
4450 		dev_close(netdev);
4451 		rtnl_unlock();
4452 	} else {
4453 		rtnl_unlock();
4454 		unregister_netdev(netdev);
4455 		free_netdev(netdev);
4456 	}
4457 
4458 	ena_com_rss_destroy(ena_dev);
4459 
4460 	ena_com_delete_debug_area(ena_dev);
4461 
4462 	ena_com_delete_host_info(ena_dev);
4463 
4464 	ena_release_bars(ena_dev, pdev);
4465 
4466 	pci_disable_device(pdev);
4467 
4468 	vfree(ena_dev);
4469 }
4470 
4471 /* ena_remove - Device Removal Routine
4472  * @pdev: PCI device information struct
4473  *
4474  * ena_remove is called by the PCI subsystem to alert the driver
4475  * that it should release a PCI device.
4476  */
4477 
4478 static void ena_remove(struct pci_dev *pdev)
4479 {
4480 	__ena_shutoff(pdev, false);
4481 }
4482 
4483 /* ena_shutdown - Device Shutdown Routine
4484  * @pdev: PCI device information struct
4485  *
4486  * ena_shutdown is called by the PCI subsystem to alert the driver that
4487  * a shutdown/reboot (or kexec) is happening and device must be disabled.
4488  */
4489 
4490 static void ena_shutdown(struct pci_dev *pdev)
4491 {
4492 	__ena_shutoff(pdev, true);
4493 }
4494 
4495 /* ena_suspend - PM suspend callback
4496  * @dev_d: Device information struct
4497  */
4498 static int __maybe_unused ena_suspend(struct device *dev_d)
4499 {
4500 	struct pci_dev *pdev = to_pci_dev(dev_d);
4501 	struct ena_adapter *adapter = pci_get_drvdata(pdev);
4502 
4503 	ena_increase_stat(&adapter->dev_stats.suspend, 1, &adapter->syncp);
4504 
4505 	rtnl_lock();
4506 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
4507 		dev_err(&pdev->dev,
4508 			"Ignoring device reset request as the device is being suspended\n");
4509 		clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
4510 	}
4511 	ena_destroy_device(adapter, true);
4512 	rtnl_unlock();
4513 	return 0;
4514 }
4515 
4516 /* ena_resume - PM resume callback
4517  * @dev_d: Device information struct
4518  */
4519 static int __maybe_unused ena_resume(struct device *dev_d)
4520 {
4521 	struct ena_adapter *adapter = dev_get_drvdata(dev_d);
4522 	int rc;
4523 
4524 	ena_increase_stat(&adapter->dev_stats.resume, 1, &adapter->syncp);
4525 
4526 	rtnl_lock();
4527 	rc = ena_restore_device(adapter);
4528 	rtnl_unlock();
4529 	return rc;
4530 }
4531 
4532 static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume);
4533 
4534 static struct pci_driver ena_pci_driver = {
4535 	.name		= DRV_MODULE_NAME,
4536 	.id_table	= ena_pci_tbl,
4537 	.probe		= ena_probe,
4538 	.remove		= ena_remove,
4539 	.shutdown	= ena_shutdown,
4540 	.driver.pm	= &ena_pm_ops,
4541 	.sriov_configure = pci_sriov_configure_simple,
4542 };
4543 
4544 static int __init ena_init(void)
4545 {
4546 	int ret;
4547 
4548 	ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);
4549 	if (!ena_wq) {
4550 		pr_err("Failed to create workqueue\n");
4551 		return -ENOMEM;
4552 	}
4553 
4554 	ret = pci_register_driver(&ena_pci_driver);
4555 	if (ret)
4556 		destroy_workqueue(ena_wq);
4557 
4558 	return ret;
4559 }
4560 
4561 static void __exit ena_cleanup(void)
4562 {
4563 	pci_unregister_driver(&ena_pci_driver);
4564 
4565 	if (ena_wq) {
4566 		destroy_workqueue(ena_wq);
4567 		ena_wq = NULL;
4568 	}
4569 }
4570 
4571 /******************************************************************************
4572  ******************************** AENQ Handlers *******************************
4573  *****************************************************************************/
4574 /* ena_update_on_link_change:
4575  * Notify the network interface about the change in link status
4576  */
4577 static void ena_update_on_link_change(void *adapter_data,
4578 				      struct ena_admin_aenq_entry *aenq_e)
4579 {
4580 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4581 	struct ena_admin_aenq_link_change_desc *aenq_desc =
4582 		(struct ena_admin_aenq_link_change_desc *)aenq_e;
4583 	int status = aenq_desc->flags &
4584 		ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
4585 
4586 	if (status) {
4587 		netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
4588 		set_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4589 		if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags))
4590 			netif_carrier_on(adapter->netdev);
4591 	} else {
4592 		clear_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4593 		netif_carrier_off(adapter->netdev);
4594 	}
4595 }
4596 
4597 static void ena_keep_alive_wd(void *adapter_data,
4598 			      struct ena_admin_aenq_entry *aenq_e)
4599 {
4600 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4601 	struct ena_admin_aenq_keep_alive_desc *desc;
4602 	u64 rx_drops;
4603 	u64 tx_drops;
4604 
4605 	desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
4606 	adapter->last_keep_alive_jiffies = jiffies;
4607 
4608 	rx_drops = ((u64)desc->rx_drops_high << 32) | desc->rx_drops_low;
4609 	tx_drops = ((u64)desc->tx_drops_high << 32) | desc->tx_drops_low;
4610 
4611 	u64_stats_update_begin(&adapter->syncp);
4612 	/* These stats are accumulated by the device, so the counters indicate
4613 	 * all drops since last reset.
4614 	 */
4615 	adapter->dev_stats.rx_drops = rx_drops;
4616 	adapter->dev_stats.tx_drops = tx_drops;
4617 	u64_stats_update_end(&adapter->syncp);
4618 }
4619 
4620 static void ena_notification(void *adapter_data,
4621 			     struct ena_admin_aenq_entry *aenq_e)
4622 {
4623 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4624 	struct ena_admin_ena_hw_hints *hints;
4625 
4626 	WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
4627 	     "Invalid group(%x) expected %x\n",
4628 	     aenq_e->aenq_common_desc.group,
4629 	     ENA_ADMIN_NOTIFICATION);
4630 
4631 	switch (aenq_e->aenq_common_desc.syndrome) {
4632 	case ENA_ADMIN_UPDATE_HINTS:
4633 		hints = (struct ena_admin_ena_hw_hints *)
4634 			(&aenq_e->inline_data_w4);
4635 		ena_update_hints(adapter, hints);
4636 		break;
4637 	default:
4638 		netif_err(adapter, drv, adapter->netdev,
4639 			  "Invalid aenq notification link state %d\n",
4640 			  aenq_e->aenq_common_desc.syndrome);
4641 	}
4642 }
4643 
4644 /* This handler will called for unknown event group or unimplemented handlers*/
4645 static void unimplemented_aenq_handler(void *data,
4646 				       struct ena_admin_aenq_entry *aenq_e)
4647 {
4648 	struct ena_adapter *adapter = (struct ena_adapter *)data;
4649 
4650 	netif_err(adapter, drv, adapter->netdev,
4651 		  "Unknown event was received or event with unimplemented handler\n");
4652 }
4653 
4654 static struct ena_aenq_handlers aenq_handlers = {
4655 	.handlers = {
4656 		[ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
4657 		[ENA_ADMIN_NOTIFICATION] = ena_notification,
4658 		[ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
4659 	},
4660 	.unimplemented_handler = unimplemented_aenq_handler
4661 };
4662 
4663 module_init(ena_init);
4664 module_exit(ena_cleanup);
4665