1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright 2015-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
4  */
5 
6 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7 
8 #ifdef CONFIG_RFS_ACCEL
9 #include <linux/cpu_rmap.h>
10 #endif /* CONFIG_RFS_ACCEL */
11 #include <linux/ethtool.h>
12 #include <linux/kernel.h>
13 #include <linux/module.h>
14 #include <linux/numa.h>
15 #include <linux/pci.h>
16 #include <linux/utsname.h>
17 #include <linux/version.h>
18 #include <linux/vmalloc.h>
19 #include <net/ip.h>
20 
21 #include "ena_netdev.h"
22 #include <linux/bpf_trace.h>
23 #include "ena_pci_id_tbl.h"
24 
25 MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
26 MODULE_DESCRIPTION(DEVICE_NAME);
27 MODULE_LICENSE("GPL");
28 
29 /* Time in jiffies before concluding the transmitter is hung. */
30 #define TX_TIMEOUT  (5 * HZ)
31 
32 #define ENA_MAX_RINGS min_t(unsigned int, ENA_MAX_NUM_IO_QUEUES, num_possible_cpus())
33 
34 #define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_IFUP | \
35 		NETIF_MSG_TX_DONE | NETIF_MSG_TX_ERR | NETIF_MSG_RX_ERR)
36 
37 static struct ena_aenq_handlers aenq_handlers;
38 
39 static struct workqueue_struct *ena_wq;
40 
41 MODULE_DEVICE_TABLE(pci, ena_pci_tbl);
42 
43 static int ena_rss_init_default(struct ena_adapter *adapter);
44 static void check_for_admin_com_state(struct ena_adapter *adapter);
45 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful);
46 static int ena_restore_device(struct ena_adapter *adapter);
47 
48 static void ena_init_io_rings(struct ena_adapter *adapter,
49 			      int first_index, int count);
50 static void ena_init_napi_in_range(struct ena_adapter *adapter, int first_index,
51 				   int count);
52 static void ena_del_napi_in_range(struct ena_adapter *adapter, int first_index,
53 				  int count);
54 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid);
55 static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
56 					   int first_index,
57 					   int count);
58 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid);
59 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid);
60 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget);
61 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter);
62 static void ena_free_all_io_tx_resources(struct ena_adapter *adapter);
63 static void ena_napi_disable_in_range(struct ena_adapter *adapter,
64 				      int first_index, int count);
65 static void ena_napi_enable_in_range(struct ena_adapter *adapter,
66 				     int first_index, int count);
67 static int ena_up(struct ena_adapter *adapter);
68 static void ena_down(struct ena_adapter *adapter);
69 static void ena_unmask_interrupt(struct ena_ring *tx_ring,
70 				 struct ena_ring *rx_ring);
71 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
72 				      struct ena_ring *rx_ring);
73 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
74 			      struct ena_tx_buffer *tx_info);
75 static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
76 					    int first_index, int count);
77 
78 /* Increase a stat by cnt while holding syncp seqlock on 32bit machines */
79 static void ena_increase_stat(u64 *statp, u64 cnt,
80 			      struct u64_stats_sync *syncp)
81 {
82 	u64_stats_update_begin(syncp);
83 	(*statp) += cnt;
84 	u64_stats_update_end(syncp);
85 }
86 
87 static void ena_ring_tx_doorbell(struct ena_ring *tx_ring)
88 {
89 	ena_com_write_sq_doorbell(tx_ring->ena_com_io_sq);
90 	ena_increase_stat(&tx_ring->tx_stats.doorbells, 1, &tx_ring->syncp);
91 }
92 
93 static void ena_tx_timeout(struct net_device *dev, unsigned int txqueue)
94 {
95 	struct ena_adapter *adapter = netdev_priv(dev);
96 
97 	/* Change the state of the device to trigger reset
98 	 * Check that we are not in the middle or a trigger already
99 	 */
100 
101 	if (test_and_set_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
102 		return;
103 
104 	ena_reset_device(adapter, ENA_REGS_RESET_OS_NETDEV_WD);
105 	ena_increase_stat(&adapter->dev_stats.tx_timeout, 1, &adapter->syncp);
106 
107 	netif_err(adapter, tx_err, dev, "Transmit time out\n");
108 }
109 
110 static void update_rx_ring_mtu(struct ena_adapter *adapter, int mtu)
111 {
112 	int i;
113 
114 	for (i = 0; i < adapter->num_io_queues; i++)
115 		adapter->rx_ring[i].mtu = mtu;
116 }
117 
118 static int ena_change_mtu(struct net_device *dev, int new_mtu)
119 {
120 	struct ena_adapter *adapter = netdev_priv(dev);
121 	int ret;
122 
123 	ret = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
124 	if (!ret) {
125 		netif_dbg(adapter, drv, dev, "Set MTU to %d\n", new_mtu);
126 		update_rx_ring_mtu(adapter, new_mtu);
127 		dev->mtu = new_mtu;
128 	} else {
129 		netif_err(adapter, drv, dev, "Failed to set MTU to %d\n",
130 			  new_mtu);
131 	}
132 
133 	return ret;
134 }
135 
136 static int ena_xmit_common(struct net_device *dev,
137 			   struct ena_ring *ring,
138 			   struct ena_tx_buffer *tx_info,
139 			   struct ena_com_tx_ctx *ena_tx_ctx,
140 			   u16 next_to_use,
141 			   u32 bytes)
142 {
143 	struct ena_adapter *adapter = netdev_priv(dev);
144 	int rc, nb_hw_desc;
145 
146 	if (unlikely(ena_com_is_doorbell_needed(ring->ena_com_io_sq,
147 						ena_tx_ctx))) {
148 		netif_dbg(adapter, tx_queued, dev,
149 			  "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
150 			  ring->qid);
151 		ena_ring_tx_doorbell(ring);
152 	}
153 
154 	/* prepare the packet's descriptors to dma engine */
155 	rc = ena_com_prepare_tx(ring->ena_com_io_sq, ena_tx_ctx,
156 				&nb_hw_desc);
157 
158 	/* In case there isn't enough space in the queue for the packet,
159 	 * we simply drop it. All other failure reasons of
160 	 * ena_com_prepare_tx() are fatal and therefore require a device reset.
161 	 */
162 	if (unlikely(rc)) {
163 		netif_err(adapter, tx_queued, dev,
164 			  "Failed to prepare tx bufs\n");
165 		ena_increase_stat(&ring->tx_stats.prepare_ctx_err, 1,
166 				  &ring->syncp);
167 		if (rc != -ENOMEM)
168 			ena_reset_device(adapter,
169 					 ENA_REGS_RESET_DRIVER_INVALID_STATE);
170 		return rc;
171 	}
172 
173 	u64_stats_update_begin(&ring->syncp);
174 	ring->tx_stats.cnt++;
175 	ring->tx_stats.bytes += bytes;
176 	u64_stats_update_end(&ring->syncp);
177 
178 	tx_info->tx_descs = nb_hw_desc;
179 	tx_info->last_jiffies = jiffies;
180 	tx_info->print_once = 0;
181 
182 	ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
183 						 ring->ring_size);
184 	return 0;
185 }
186 
187 /* This is the XDP napi callback. XDP queues use a separate napi callback
188  * than Rx/Tx queues.
189  */
190 static int ena_xdp_io_poll(struct napi_struct *napi, int budget)
191 {
192 	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
193 	u32 xdp_work_done, xdp_budget;
194 	struct ena_ring *xdp_ring;
195 	int napi_comp_call = 0;
196 	int ret;
197 
198 	xdp_ring = ena_napi->xdp_ring;
199 
200 	xdp_budget = budget;
201 
202 	if (!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags) ||
203 	    test_bit(ENA_FLAG_TRIGGER_RESET, &xdp_ring->adapter->flags)) {
204 		napi_complete_done(napi, 0);
205 		return 0;
206 	}
207 
208 	xdp_work_done = ena_clean_xdp_irq(xdp_ring, xdp_budget);
209 
210 	/* If the device is about to reset or down, avoid unmask
211 	 * the interrupt and return 0 so NAPI won't reschedule
212 	 */
213 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &xdp_ring->adapter->flags))) {
214 		napi_complete_done(napi, 0);
215 		ret = 0;
216 	} else if (xdp_budget > xdp_work_done) {
217 		napi_comp_call = 1;
218 		if (napi_complete_done(napi, xdp_work_done))
219 			ena_unmask_interrupt(xdp_ring, NULL);
220 		ena_update_ring_numa_node(xdp_ring, NULL);
221 		ret = xdp_work_done;
222 	} else {
223 		ret = xdp_budget;
224 	}
225 
226 	u64_stats_update_begin(&xdp_ring->syncp);
227 	xdp_ring->tx_stats.napi_comp += napi_comp_call;
228 	xdp_ring->tx_stats.tx_poll++;
229 	u64_stats_update_end(&xdp_ring->syncp);
230 	xdp_ring->tx_stats.last_napi_jiffies = jiffies;
231 
232 	return ret;
233 }
234 
235 static int ena_xdp_tx_map_frame(struct ena_ring *xdp_ring,
236 				struct ena_tx_buffer *tx_info,
237 				struct xdp_frame *xdpf,
238 				struct ena_com_tx_ctx *ena_tx_ctx)
239 {
240 	struct ena_adapter *adapter = xdp_ring->adapter;
241 	struct ena_com_buf *ena_buf;
242 	int push_len = 0;
243 	dma_addr_t dma;
244 	void *data;
245 	u32 size;
246 
247 	tx_info->xdpf = xdpf;
248 	data = tx_info->xdpf->data;
249 	size = tx_info->xdpf->len;
250 
251 	if (xdp_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
252 		/* Designate part of the packet for LLQ */
253 		push_len = min_t(u32, size, xdp_ring->tx_max_header_size);
254 
255 		ena_tx_ctx->push_header = data;
256 
257 		size -= push_len;
258 		data += push_len;
259 	}
260 
261 	ena_tx_ctx->header_len = push_len;
262 
263 	if (size > 0) {
264 		dma = dma_map_single(xdp_ring->dev,
265 				     data,
266 				     size,
267 				     DMA_TO_DEVICE);
268 		if (unlikely(dma_mapping_error(xdp_ring->dev, dma)))
269 			goto error_report_dma_error;
270 
271 		tx_info->map_linear_data = 0;
272 
273 		ena_buf = tx_info->bufs;
274 		ena_buf->paddr = dma;
275 		ena_buf->len = size;
276 
277 		ena_tx_ctx->ena_bufs = ena_buf;
278 		ena_tx_ctx->num_bufs = tx_info->num_of_bufs = 1;
279 	}
280 
281 	return 0;
282 
283 error_report_dma_error:
284 	ena_increase_stat(&xdp_ring->tx_stats.dma_mapping_err, 1,
285 			  &xdp_ring->syncp);
286 	netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map xdp buff\n");
287 
288 	return -EINVAL;
289 }
290 
291 static int ena_xdp_xmit_frame(struct ena_ring *xdp_ring,
292 			      struct net_device *dev,
293 			      struct xdp_frame *xdpf,
294 			      int flags)
295 {
296 	struct ena_com_tx_ctx ena_tx_ctx = {};
297 	struct ena_tx_buffer *tx_info;
298 	u16 next_to_use, req_id;
299 	int rc;
300 
301 	next_to_use = xdp_ring->next_to_use;
302 	req_id = xdp_ring->free_ids[next_to_use];
303 	tx_info = &xdp_ring->tx_buffer_info[req_id];
304 	tx_info->num_of_bufs = 0;
305 
306 	rc = ena_xdp_tx_map_frame(xdp_ring, tx_info, xdpf, &ena_tx_ctx);
307 	if (unlikely(rc))
308 		return rc;
309 
310 	ena_tx_ctx.req_id = req_id;
311 
312 	rc = ena_xmit_common(dev,
313 			     xdp_ring,
314 			     tx_info,
315 			     &ena_tx_ctx,
316 			     next_to_use,
317 			     xdpf->len);
318 	if (rc)
319 		goto error_unmap_dma;
320 
321 	/* trigger the dma engine. ena_ring_tx_doorbell()
322 	 * calls a memory barrier inside it.
323 	 */
324 	if (flags & XDP_XMIT_FLUSH)
325 		ena_ring_tx_doorbell(xdp_ring);
326 
327 	return rc;
328 
329 error_unmap_dma:
330 	ena_unmap_tx_buff(xdp_ring, tx_info);
331 	tx_info->xdpf = NULL;
332 	return rc;
333 }
334 
335 static int ena_xdp_xmit(struct net_device *dev, int n,
336 			struct xdp_frame **frames, u32 flags)
337 {
338 	struct ena_adapter *adapter = netdev_priv(dev);
339 	struct ena_ring *xdp_ring;
340 	int qid, i, nxmit = 0;
341 
342 	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
343 		return -EINVAL;
344 
345 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
346 		return -ENETDOWN;
347 
348 	/* We assume that all rings have the same XDP program */
349 	if (!READ_ONCE(adapter->rx_ring->xdp_bpf_prog))
350 		return -ENXIO;
351 
352 	qid = smp_processor_id() % adapter->xdp_num_queues;
353 	qid += adapter->xdp_first_ring;
354 	xdp_ring = &adapter->tx_ring[qid];
355 
356 	/* Other CPU ids might try to send thorugh this queue */
357 	spin_lock(&xdp_ring->xdp_tx_lock);
358 
359 	for (i = 0; i < n; i++) {
360 		if (ena_xdp_xmit_frame(xdp_ring, dev, frames[i], 0))
361 			break;
362 		nxmit++;
363 	}
364 
365 	/* Ring doorbell to make device aware of the packets */
366 	if (flags & XDP_XMIT_FLUSH)
367 		ena_ring_tx_doorbell(xdp_ring);
368 
369 	spin_unlock(&xdp_ring->xdp_tx_lock);
370 
371 	/* Return number of packets sent */
372 	return nxmit;
373 }
374 
375 static int ena_xdp_execute(struct ena_ring *rx_ring, struct xdp_buff *xdp)
376 {
377 	u32 verdict = ENA_XDP_PASS;
378 	struct bpf_prog *xdp_prog;
379 	struct ena_ring *xdp_ring;
380 	struct xdp_frame *xdpf;
381 	u64 *xdp_stat;
382 
383 	xdp_prog = READ_ONCE(rx_ring->xdp_bpf_prog);
384 
385 	if (!xdp_prog)
386 		goto out;
387 
388 	verdict = bpf_prog_run_xdp(xdp_prog, xdp);
389 
390 	switch (verdict) {
391 	case XDP_TX:
392 		xdpf = xdp_convert_buff_to_frame(xdp);
393 		if (unlikely(!xdpf)) {
394 			trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
395 			xdp_stat = &rx_ring->rx_stats.xdp_aborted;
396 			verdict = ENA_XDP_DROP;
397 			break;
398 		}
399 
400 		/* Find xmit queue */
401 		xdp_ring = rx_ring->xdp_ring;
402 
403 		/* The XDP queues are shared between XDP_TX and XDP_REDIRECT */
404 		spin_lock(&xdp_ring->xdp_tx_lock);
405 
406 		if (ena_xdp_xmit_frame(xdp_ring, rx_ring->netdev, xdpf,
407 				       XDP_XMIT_FLUSH))
408 			xdp_return_frame(xdpf);
409 
410 		spin_unlock(&xdp_ring->xdp_tx_lock);
411 		xdp_stat = &rx_ring->rx_stats.xdp_tx;
412 		verdict = ENA_XDP_TX;
413 		break;
414 	case XDP_REDIRECT:
415 		if (likely(!xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog))) {
416 			xdp_stat = &rx_ring->rx_stats.xdp_redirect;
417 			verdict = ENA_XDP_REDIRECT;
418 			break;
419 		}
420 		trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
421 		xdp_stat = &rx_ring->rx_stats.xdp_aborted;
422 		verdict = ENA_XDP_DROP;
423 		break;
424 	case XDP_ABORTED:
425 		trace_xdp_exception(rx_ring->netdev, xdp_prog, verdict);
426 		xdp_stat = &rx_ring->rx_stats.xdp_aborted;
427 		verdict = ENA_XDP_DROP;
428 		break;
429 	case XDP_DROP:
430 		xdp_stat = &rx_ring->rx_stats.xdp_drop;
431 		verdict = ENA_XDP_DROP;
432 		break;
433 	case XDP_PASS:
434 		xdp_stat = &rx_ring->rx_stats.xdp_pass;
435 		verdict = ENA_XDP_PASS;
436 		break;
437 	default:
438 		bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, verdict);
439 		xdp_stat = &rx_ring->rx_stats.xdp_invalid;
440 		verdict = ENA_XDP_DROP;
441 	}
442 
443 	ena_increase_stat(xdp_stat, 1, &rx_ring->syncp);
444 out:
445 	return verdict;
446 }
447 
448 static void ena_init_all_xdp_queues(struct ena_adapter *adapter)
449 {
450 	adapter->xdp_first_ring = adapter->num_io_queues;
451 	adapter->xdp_num_queues = adapter->num_io_queues;
452 
453 	ena_init_io_rings(adapter,
454 			  adapter->xdp_first_ring,
455 			  adapter->xdp_num_queues);
456 }
457 
458 static int ena_setup_and_create_all_xdp_queues(struct ena_adapter *adapter)
459 {
460 	int rc = 0;
461 
462 	rc = ena_setup_tx_resources_in_range(adapter, adapter->xdp_first_ring,
463 					     adapter->xdp_num_queues);
464 	if (rc)
465 		goto setup_err;
466 
467 	rc = ena_create_io_tx_queues_in_range(adapter,
468 					      adapter->xdp_first_ring,
469 					      adapter->xdp_num_queues);
470 	if (rc)
471 		goto create_err;
472 
473 	return 0;
474 
475 create_err:
476 	ena_free_all_io_tx_resources(adapter);
477 setup_err:
478 	return rc;
479 }
480 
481 /* Provides a way for both kernel and bpf-prog to know
482  * more about the RX-queue a given XDP frame arrived on.
483  */
484 static int ena_xdp_register_rxq_info(struct ena_ring *rx_ring)
485 {
486 	int rc;
487 
488 	rc = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->qid, 0);
489 
490 	if (rc) {
491 		netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
492 			  "Failed to register xdp rx queue info. RX queue num %d rc: %d\n",
493 			  rx_ring->qid, rc);
494 		goto err;
495 	}
496 
497 	rc = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, MEM_TYPE_PAGE_SHARED,
498 					NULL);
499 
500 	if (rc) {
501 		netif_err(rx_ring->adapter, ifup, rx_ring->netdev,
502 			  "Failed to register xdp rx queue info memory model. RX queue num %d rc: %d\n",
503 			  rx_ring->qid, rc);
504 		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
505 	}
506 
507 err:
508 	return rc;
509 }
510 
511 static void ena_xdp_unregister_rxq_info(struct ena_ring *rx_ring)
512 {
513 	xdp_rxq_info_unreg_mem_model(&rx_ring->xdp_rxq);
514 	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
515 }
516 
517 static void ena_xdp_exchange_program_rx_in_range(struct ena_adapter *adapter,
518 						 struct bpf_prog *prog,
519 						 int first, int count)
520 {
521 	struct bpf_prog *old_bpf_prog;
522 	struct ena_ring *rx_ring;
523 	int i = 0;
524 
525 	for (i = first; i < count; i++) {
526 		rx_ring = &adapter->rx_ring[i];
527 		old_bpf_prog = xchg(&rx_ring->xdp_bpf_prog, prog);
528 
529 		if (!old_bpf_prog && prog) {
530 			ena_xdp_register_rxq_info(rx_ring);
531 			rx_ring->rx_headroom = XDP_PACKET_HEADROOM;
532 		} else if (old_bpf_prog && !prog) {
533 			ena_xdp_unregister_rxq_info(rx_ring);
534 			rx_ring->rx_headroom = NET_SKB_PAD;
535 		}
536 	}
537 }
538 
539 static void ena_xdp_exchange_program(struct ena_adapter *adapter,
540 				     struct bpf_prog *prog)
541 {
542 	struct bpf_prog *old_bpf_prog = xchg(&adapter->xdp_bpf_prog, prog);
543 
544 	ena_xdp_exchange_program_rx_in_range(adapter,
545 					     prog,
546 					     0,
547 					     adapter->num_io_queues);
548 
549 	if (old_bpf_prog)
550 		bpf_prog_put(old_bpf_prog);
551 }
552 
553 static int ena_destroy_and_free_all_xdp_queues(struct ena_adapter *adapter)
554 {
555 	bool was_up;
556 	int rc;
557 
558 	was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
559 
560 	if (was_up)
561 		ena_down(adapter);
562 
563 	adapter->xdp_first_ring = 0;
564 	adapter->xdp_num_queues = 0;
565 	ena_xdp_exchange_program(adapter, NULL);
566 	if (was_up) {
567 		rc = ena_up(adapter);
568 		if (rc)
569 			return rc;
570 	}
571 	return 0;
572 }
573 
574 static int ena_xdp_set(struct net_device *netdev, struct netdev_bpf *bpf)
575 {
576 	struct ena_adapter *adapter = netdev_priv(netdev);
577 	struct bpf_prog *prog = bpf->prog;
578 	struct bpf_prog *old_bpf_prog;
579 	int rc, prev_mtu;
580 	bool is_up;
581 
582 	is_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
583 	rc = ena_xdp_allowed(adapter);
584 	if (rc == ENA_XDP_ALLOWED) {
585 		old_bpf_prog = adapter->xdp_bpf_prog;
586 		if (prog) {
587 			if (!is_up) {
588 				ena_init_all_xdp_queues(adapter);
589 			} else if (!old_bpf_prog) {
590 				ena_down(adapter);
591 				ena_init_all_xdp_queues(adapter);
592 			}
593 			ena_xdp_exchange_program(adapter, prog);
594 
595 			if (is_up && !old_bpf_prog) {
596 				rc = ena_up(adapter);
597 				if (rc)
598 					return rc;
599 			}
600 			xdp_features_set_redirect_target(netdev, false);
601 		} else if (old_bpf_prog) {
602 			xdp_features_clear_redirect_target(netdev);
603 			rc = ena_destroy_and_free_all_xdp_queues(adapter);
604 			if (rc)
605 				return rc;
606 		}
607 
608 		prev_mtu = netdev->max_mtu;
609 		netdev->max_mtu = prog ? ENA_XDP_MAX_MTU : adapter->max_mtu;
610 
611 		if (!old_bpf_prog)
612 			netif_info(adapter, drv, adapter->netdev,
613 				   "XDP program is set, changing the max_mtu from %d to %d",
614 				   prev_mtu, netdev->max_mtu);
615 
616 	} else if (rc == ENA_XDP_CURRENT_MTU_TOO_LARGE) {
617 		netif_err(adapter, drv, adapter->netdev,
618 			  "Failed to set xdp program, the current MTU (%d) is larger than the maximum allowed MTU (%lu) while xdp is on",
619 			  netdev->mtu, ENA_XDP_MAX_MTU);
620 		NL_SET_ERR_MSG_MOD(bpf->extack,
621 				   "Failed to set xdp program, the current MTU is larger than the maximum allowed MTU. Check the dmesg for more info");
622 		return -EINVAL;
623 	} else if (rc == ENA_XDP_NO_ENOUGH_QUEUES) {
624 		netif_err(adapter, drv, adapter->netdev,
625 			  "Failed to set xdp program, the Rx/Tx channel count should be at most half of the maximum allowed channel count. The current queue count (%d), the maximal queue count (%d)\n",
626 			  adapter->num_io_queues, adapter->max_num_io_queues);
627 		NL_SET_ERR_MSG_MOD(bpf->extack,
628 				   "Failed to set xdp program, there is no enough space for allocating XDP queues, Check the dmesg for more info");
629 		return -EINVAL;
630 	}
631 
632 	return 0;
633 }
634 
635 /* This is the main xdp callback, it's used by the kernel to set/unset the xdp
636  * program as well as to query the current xdp program id.
637  */
638 static int ena_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
639 {
640 	switch (bpf->command) {
641 	case XDP_SETUP_PROG:
642 		return ena_xdp_set(netdev, bpf);
643 	default:
644 		return -EINVAL;
645 	}
646 	return 0;
647 }
648 
649 static int ena_init_rx_cpu_rmap(struct ena_adapter *adapter)
650 {
651 #ifdef CONFIG_RFS_ACCEL
652 	u32 i;
653 	int rc;
654 
655 	adapter->netdev->rx_cpu_rmap = alloc_irq_cpu_rmap(adapter->num_io_queues);
656 	if (!adapter->netdev->rx_cpu_rmap)
657 		return -ENOMEM;
658 	for (i = 0; i < adapter->num_io_queues; i++) {
659 		int irq_idx = ENA_IO_IRQ_IDX(i);
660 
661 		rc = irq_cpu_rmap_add(adapter->netdev->rx_cpu_rmap,
662 				      pci_irq_vector(adapter->pdev, irq_idx));
663 		if (rc) {
664 			free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
665 			adapter->netdev->rx_cpu_rmap = NULL;
666 			return rc;
667 		}
668 	}
669 #endif /* CONFIG_RFS_ACCEL */
670 	return 0;
671 }
672 
673 static void ena_init_io_rings_common(struct ena_adapter *adapter,
674 				     struct ena_ring *ring, u16 qid)
675 {
676 	ring->qid = qid;
677 	ring->pdev = adapter->pdev;
678 	ring->dev = &adapter->pdev->dev;
679 	ring->netdev = adapter->netdev;
680 	ring->napi = &adapter->ena_napi[qid].napi;
681 	ring->adapter = adapter;
682 	ring->ena_dev = adapter->ena_dev;
683 	ring->per_napi_packets = 0;
684 	ring->cpu = 0;
685 	ring->numa_node = 0;
686 	ring->no_interrupt_event_cnt = 0;
687 	u64_stats_init(&ring->syncp);
688 }
689 
690 static void ena_init_io_rings(struct ena_adapter *adapter,
691 			      int first_index, int count)
692 {
693 	struct ena_com_dev *ena_dev;
694 	struct ena_ring *txr, *rxr;
695 	int i;
696 
697 	ena_dev = adapter->ena_dev;
698 
699 	for (i = first_index; i < first_index + count; i++) {
700 		txr = &adapter->tx_ring[i];
701 		rxr = &adapter->rx_ring[i];
702 
703 		/* TX common ring state */
704 		ena_init_io_rings_common(adapter, txr, i);
705 
706 		/* TX specific ring state */
707 		txr->ring_size = adapter->requested_tx_ring_size;
708 		txr->tx_max_header_size = ena_dev->tx_max_header_size;
709 		txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
710 		txr->sgl_size = adapter->max_tx_sgl_size;
711 		txr->smoothed_interval =
712 			ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
713 		txr->disable_meta_caching = adapter->disable_meta_caching;
714 		spin_lock_init(&txr->xdp_tx_lock);
715 
716 		/* Don't init RX queues for xdp queues */
717 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
718 			/* RX common ring state */
719 			ena_init_io_rings_common(adapter, rxr, i);
720 
721 			/* RX specific ring state */
722 			rxr->ring_size = adapter->requested_rx_ring_size;
723 			rxr->rx_copybreak = adapter->rx_copybreak;
724 			rxr->sgl_size = adapter->max_rx_sgl_size;
725 			rxr->smoothed_interval =
726 				ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
727 			rxr->empty_rx_queue = 0;
728 			rxr->rx_headroom = NET_SKB_PAD;
729 			adapter->ena_napi[i].dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
730 			rxr->xdp_ring = &adapter->tx_ring[i + adapter->num_io_queues];
731 		}
732 	}
733 }
734 
735 /* ena_setup_tx_resources - allocate I/O Tx resources (Descriptors)
736  * @adapter: network interface device structure
737  * @qid: queue index
738  *
739  * Return 0 on success, negative on failure
740  */
741 static int ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
742 {
743 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
744 	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
745 	int size, i, node;
746 
747 	if (tx_ring->tx_buffer_info) {
748 		netif_err(adapter, ifup,
749 			  adapter->netdev, "tx_buffer_info info is not NULL");
750 		return -EEXIST;
751 	}
752 
753 	size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
754 	node = cpu_to_node(ena_irq->cpu);
755 
756 	tx_ring->tx_buffer_info = vzalloc_node(size, node);
757 	if (!tx_ring->tx_buffer_info) {
758 		tx_ring->tx_buffer_info = vzalloc(size);
759 		if (!tx_ring->tx_buffer_info)
760 			goto err_tx_buffer_info;
761 	}
762 
763 	size = sizeof(u16) * tx_ring->ring_size;
764 	tx_ring->free_ids = vzalloc_node(size, node);
765 	if (!tx_ring->free_ids) {
766 		tx_ring->free_ids = vzalloc(size);
767 		if (!tx_ring->free_ids)
768 			goto err_tx_free_ids;
769 	}
770 
771 	size = tx_ring->tx_max_header_size;
772 	tx_ring->push_buf_intermediate_buf = vzalloc_node(size, node);
773 	if (!tx_ring->push_buf_intermediate_buf) {
774 		tx_ring->push_buf_intermediate_buf = vzalloc(size);
775 		if (!tx_ring->push_buf_intermediate_buf)
776 			goto err_push_buf_intermediate_buf;
777 	}
778 
779 	/* Req id ring for TX out of order completions */
780 	for (i = 0; i < tx_ring->ring_size; i++)
781 		tx_ring->free_ids[i] = i;
782 
783 	/* Reset tx statistics */
784 	memset(&tx_ring->tx_stats, 0x0, sizeof(tx_ring->tx_stats));
785 
786 	tx_ring->next_to_use = 0;
787 	tx_ring->next_to_clean = 0;
788 	tx_ring->cpu = ena_irq->cpu;
789 	tx_ring->numa_node = node;
790 	return 0;
791 
792 err_push_buf_intermediate_buf:
793 	vfree(tx_ring->free_ids);
794 	tx_ring->free_ids = NULL;
795 err_tx_free_ids:
796 	vfree(tx_ring->tx_buffer_info);
797 	tx_ring->tx_buffer_info = NULL;
798 err_tx_buffer_info:
799 	return -ENOMEM;
800 }
801 
802 /* ena_free_tx_resources - Free I/O Tx Resources per Queue
803  * @adapter: network interface device structure
804  * @qid: queue index
805  *
806  * Free all transmit software resources
807  */
808 static void ena_free_tx_resources(struct ena_adapter *adapter, int qid)
809 {
810 	struct ena_ring *tx_ring = &adapter->tx_ring[qid];
811 
812 	vfree(tx_ring->tx_buffer_info);
813 	tx_ring->tx_buffer_info = NULL;
814 
815 	vfree(tx_ring->free_ids);
816 	tx_ring->free_ids = NULL;
817 
818 	vfree(tx_ring->push_buf_intermediate_buf);
819 	tx_ring->push_buf_intermediate_buf = NULL;
820 }
821 
822 static int ena_setup_tx_resources_in_range(struct ena_adapter *adapter,
823 					   int first_index,
824 					   int count)
825 {
826 	int i, rc = 0;
827 
828 	for (i = first_index; i < first_index + count; i++) {
829 		rc = ena_setup_tx_resources(adapter, i);
830 		if (rc)
831 			goto err_setup_tx;
832 	}
833 
834 	return 0;
835 
836 err_setup_tx:
837 
838 	netif_err(adapter, ifup, adapter->netdev,
839 		  "Tx queue %d: allocation failed\n", i);
840 
841 	/* rewind the index freeing the rings as we go */
842 	while (first_index < i--)
843 		ena_free_tx_resources(adapter, i);
844 	return rc;
845 }
846 
847 static void ena_free_all_io_tx_resources_in_range(struct ena_adapter *adapter,
848 						  int first_index, int count)
849 {
850 	int i;
851 
852 	for (i = first_index; i < first_index + count; i++)
853 		ena_free_tx_resources(adapter, i);
854 }
855 
856 /* ena_free_all_io_tx_resources - Free I/O Tx Resources for All Queues
857  * @adapter: board private structure
858  *
859  * Free all transmit software resources
860  */
861 static void ena_free_all_io_tx_resources(struct ena_adapter *adapter)
862 {
863 	ena_free_all_io_tx_resources_in_range(adapter,
864 					      0,
865 					      adapter->xdp_num_queues +
866 					      adapter->num_io_queues);
867 }
868 
869 /* ena_setup_rx_resources - allocate I/O Rx resources (Descriptors)
870  * @adapter: network interface device structure
871  * @qid: queue index
872  *
873  * Returns 0 on success, negative on failure
874  */
875 static int ena_setup_rx_resources(struct ena_adapter *adapter,
876 				  u32 qid)
877 {
878 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
879 	struct ena_irq *ena_irq = &adapter->irq_tbl[ENA_IO_IRQ_IDX(qid)];
880 	int size, node, i;
881 
882 	if (rx_ring->rx_buffer_info) {
883 		netif_err(adapter, ifup, adapter->netdev,
884 			  "rx_buffer_info is not NULL");
885 		return -EEXIST;
886 	}
887 
888 	/* alloc extra element so in rx path
889 	 * we can always prefetch rx_info + 1
890 	 */
891 	size = sizeof(struct ena_rx_buffer) * (rx_ring->ring_size + 1);
892 	node = cpu_to_node(ena_irq->cpu);
893 
894 	rx_ring->rx_buffer_info = vzalloc_node(size, node);
895 	if (!rx_ring->rx_buffer_info) {
896 		rx_ring->rx_buffer_info = vzalloc(size);
897 		if (!rx_ring->rx_buffer_info)
898 			return -ENOMEM;
899 	}
900 
901 	size = sizeof(u16) * rx_ring->ring_size;
902 	rx_ring->free_ids = vzalloc_node(size, node);
903 	if (!rx_ring->free_ids) {
904 		rx_ring->free_ids = vzalloc(size);
905 		if (!rx_ring->free_ids) {
906 			vfree(rx_ring->rx_buffer_info);
907 			rx_ring->rx_buffer_info = NULL;
908 			return -ENOMEM;
909 		}
910 	}
911 
912 	/* Req id ring for receiving RX pkts out of order */
913 	for (i = 0; i < rx_ring->ring_size; i++)
914 		rx_ring->free_ids[i] = i;
915 
916 	/* Reset rx statistics */
917 	memset(&rx_ring->rx_stats, 0x0, sizeof(rx_ring->rx_stats));
918 
919 	rx_ring->next_to_clean = 0;
920 	rx_ring->next_to_use = 0;
921 	rx_ring->cpu = ena_irq->cpu;
922 	rx_ring->numa_node = node;
923 
924 	return 0;
925 }
926 
927 /* ena_free_rx_resources - Free I/O Rx Resources
928  * @adapter: network interface device structure
929  * @qid: queue index
930  *
931  * Free all receive software resources
932  */
933 static void ena_free_rx_resources(struct ena_adapter *adapter,
934 				  u32 qid)
935 {
936 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
937 
938 	vfree(rx_ring->rx_buffer_info);
939 	rx_ring->rx_buffer_info = NULL;
940 
941 	vfree(rx_ring->free_ids);
942 	rx_ring->free_ids = NULL;
943 }
944 
945 /* ena_setup_all_rx_resources - allocate I/O Rx queues resources for all queues
946  * @adapter: board private structure
947  *
948  * Return 0 on success, negative on failure
949  */
950 static int ena_setup_all_rx_resources(struct ena_adapter *adapter)
951 {
952 	int i, rc = 0;
953 
954 	for (i = 0; i < adapter->num_io_queues; i++) {
955 		rc = ena_setup_rx_resources(adapter, i);
956 		if (rc)
957 			goto err_setup_rx;
958 	}
959 
960 	return 0;
961 
962 err_setup_rx:
963 
964 	netif_err(adapter, ifup, adapter->netdev,
965 		  "Rx queue %d: allocation failed\n", i);
966 
967 	/* rewind the index freeing the rings as we go */
968 	while (i--)
969 		ena_free_rx_resources(adapter, i);
970 	return rc;
971 }
972 
973 /* ena_free_all_io_rx_resources - Free I/O Rx Resources for All Queues
974  * @adapter: board private structure
975  *
976  * Free all receive software resources
977  */
978 static void ena_free_all_io_rx_resources(struct ena_adapter *adapter)
979 {
980 	int i;
981 
982 	for (i = 0; i < adapter->num_io_queues; i++)
983 		ena_free_rx_resources(adapter, i);
984 }
985 
986 static struct page *ena_alloc_map_page(struct ena_ring *rx_ring,
987 				       dma_addr_t *dma)
988 {
989 	struct page *page;
990 
991 	/* This would allocate the page on the same NUMA node the executing code
992 	 * is running on.
993 	 */
994 	page = dev_alloc_page();
995 	if (!page) {
996 		ena_increase_stat(&rx_ring->rx_stats.page_alloc_fail, 1,
997 				  &rx_ring->syncp);
998 		return ERR_PTR(-ENOSPC);
999 	}
1000 
1001 	/* To enable NIC-side port-mirroring, AKA SPAN port,
1002 	 * we make the buffer readable from the nic as well
1003 	 */
1004 	*dma = dma_map_page(rx_ring->dev, page, 0, ENA_PAGE_SIZE,
1005 			    DMA_BIDIRECTIONAL);
1006 	if (unlikely(dma_mapping_error(rx_ring->dev, *dma))) {
1007 		ena_increase_stat(&rx_ring->rx_stats.dma_mapping_err, 1,
1008 				  &rx_ring->syncp);
1009 		__free_page(page);
1010 		return ERR_PTR(-EIO);
1011 	}
1012 
1013 	return page;
1014 }
1015 
1016 static int ena_alloc_rx_buffer(struct ena_ring *rx_ring,
1017 			       struct ena_rx_buffer *rx_info)
1018 {
1019 	int headroom = rx_ring->rx_headroom;
1020 	struct ena_com_buf *ena_buf;
1021 	struct page *page;
1022 	dma_addr_t dma;
1023 	int tailroom;
1024 
1025 	/* restore page offset value in case it has been changed by device */
1026 	rx_info->page_offset = headroom;
1027 
1028 	/* if previous allocated page is not used */
1029 	if (unlikely(rx_info->page))
1030 		return 0;
1031 
1032 	/* We handle DMA here */
1033 	page = ena_alloc_map_page(rx_ring, &dma);
1034 	if (unlikely(IS_ERR(page)))
1035 		return PTR_ERR(page);
1036 
1037 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1038 		  "Allocate page %p, rx_info %p\n", page, rx_info);
1039 
1040 	tailroom = SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
1041 
1042 	rx_info->page = page;
1043 	ena_buf = &rx_info->ena_buf;
1044 	ena_buf->paddr = dma + headroom;
1045 	ena_buf->len = ENA_PAGE_SIZE - headroom - tailroom;
1046 
1047 	return 0;
1048 }
1049 
1050 static void ena_unmap_rx_buff(struct ena_ring *rx_ring,
1051 			      struct ena_rx_buffer *rx_info)
1052 {
1053 	struct ena_com_buf *ena_buf = &rx_info->ena_buf;
1054 
1055 	dma_unmap_page(rx_ring->dev, ena_buf->paddr - rx_ring->rx_headroom,
1056 		       ENA_PAGE_SIZE,
1057 		       DMA_BIDIRECTIONAL);
1058 }
1059 
1060 static void ena_free_rx_page(struct ena_ring *rx_ring,
1061 			     struct ena_rx_buffer *rx_info)
1062 {
1063 	struct page *page = rx_info->page;
1064 
1065 	if (unlikely(!page)) {
1066 		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
1067 			   "Trying to free unallocated buffer\n");
1068 		return;
1069 	}
1070 
1071 	ena_unmap_rx_buff(rx_ring, rx_info);
1072 
1073 	__free_page(page);
1074 	rx_info->page = NULL;
1075 }
1076 
1077 static int ena_refill_rx_bufs(struct ena_ring *rx_ring, u32 num)
1078 {
1079 	u16 next_to_use, req_id;
1080 	u32 i;
1081 	int rc;
1082 
1083 	next_to_use = rx_ring->next_to_use;
1084 
1085 	for (i = 0; i < num; i++) {
1086 		struct ena_rx_buffer *rx_info;
1087 
1088 		req_id = rx_ring->free_ids[next_to_use];
1089 
1090 		rx_info = &rx_ring->rx_buffer_info[req_id];
1091 
1092 		rc = ena_alloc_rx_buffer(rx_ring, rx_info);
1093 		if (unlikely(rc < 0)) {
1094 			netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
1095 				   "Failed to allocate buffer for rx queue %d\n",
1096 				   rx_ring->qid);
1097 			break;
1098 		}
1099 		rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1100 						&rx_info->ena_buf,
1101 						req_id);
1102 		if (unlikely(rc)) {
1103 			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
1104 				   "Failed to add buffer for rx queue %d\n",
1105 				   rx_ring->qid);
1106 			break;
1107 		}
1108 		next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1109 						   rx_ring->ring_size);
1110 	}
1111 
1112 	if (unlikely(i < num)) {
1113 		ena_increase_stat(&rx_ring->rx_stats.refil_partial, 1,
1114 				  &rx_ring->syncp);
1115 		netif_warn(rx_ring->adapter, rx_err, rx_ring->netdev,
1116 			   "Refilled rx qid %d with only %d buffers (from %d)\n",
1117 			   rx_ring->qid, i, num);
1118 	}
1119 
1120 	/* ena_com_write_sq_doorbell issues a wmb() */
1121 	if (likely(i))
1122 		ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
1123 
1124 	rx_ring->next_to_use = next_to_use;
1125 
1126 	return i;
1127 }
1128 
1129 static void ena_free_rx_bufs(struct ena_adapter *adapter,
1130 			     u32 qid)
1131 {
1132 	struct ena_ring *rx_ring = &adapter->rx_ring[qid];
1133 	u32 i;
1134 
1135 	for (i = 0; i < rx_ring->ring_size; i++) {
1136 		struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
1137 
1138 		if (rx_info->page)
1139 			ena_free_rx_page(rx_ring, rx_info);
1140 	}
1141 }
1142 
1143 /* ena_refill_all_rx_bufs - allocate all queues Rx buffers
1144  * @adapter: board private structure
1145  */
1146 static void ena_refill_all_rx_bufs(struct ena_adapter *adapter)
1147 {
1148 	struct ena_ring *rx_ring;
1149 	int i, rc, bufs_num;
1150 
1151 	for (i = 0; i < adapter->num_io_queues; i++) {
1152 		rx_ring = &adapter->rx_ring[i];
1153 		bufs_num = rx_ring->ring_size - 1;
1154 		rc = ena_refill_rx_bufs(rx_ring, bufs_num);
1155 
1156 		if (unlikely(rc != bufs_num))
1157 			netif_warn(rx_ring->adapter, rx_status, rx_ring->netdev,
1158 				   "Refilling Queue %d failed. allocated %d buffers from: %d\n",
1159 				   i, rc, bufs_num);
1160 	}
1161 }
1162 
1163 static void ena_free_all_rx_bufs(struct ena_adapter *adapter)
1164 {
1165 	int i;
1166 
1167 	for (i = 0; i < adapter->num_io_queues; i++)
1168 		ena_free_rx_bufs(adapter, i);
1169 }
1170 
1171 static void ena_unmap_tx_buff(struct ena_ring *tx_ring,
1172 			      struct ena_tx_buffer *tx_info)
1173 {
1174 	struct ena_com_buf *ena_buf;
1175 	u32 cnt;
1176 	int i;
1177 
1178 	ena_buf = tx_info->bufs;
1179 	cnt = tx_info->num_of_bufs;
1180 
1181 	if (unlikely(!cnt))
1182 		return;
1183 
1184 	if (tx_info->map_linear_data) {
1185 		dma_unmap_single(tx_ring->dev,
1186 				 dma_unmap_addr(ena_buf, paddr),
1187 				 dma_unmap_len(ena_buf, len),
1188 				 DMA_TO_DEVICE);
1189 		ena_buf++;
1190 		cnt--;
1191 	}
1192 
1193 	/* unmap remaining mapped pages */
1194 	for (i = 0; i < cnt; i++) {
1195 		dma_unmap_page(tx_ring->dev, dma_unmap_addr(ena_buf, paddr),
1196 			       dma_unmap_len(ena_buf, len), DMA_TO_DEVICE);
1197 		ena_buf++;
1198 	}
1199 }
1200 
1201 /* ena_free_tx_bufs - Free Tx Buffers per Queue
1202  * @tx_ring: TX ring for which buffers be freed
1203  */
1204 static void ena_free_tx_bufs(struct ena_ring *tx_ring)
1205 {
1206 	bool print_once = true;
1207 	u32 i;
1208 
1209 	for (i = 0; i < tx_ring->ring_size; i++) {
1210 		struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
1211 
1212 		if (!tx_info->skb)
1213 			continue;
1214 
1215 		if (print_once) {
1216 			netif_notice(tx_ring->adapter, ifdown, tx_ring->netdev,
1217 				     "Free uncompleted tx skb qid %d idx 0x%x\n",
1218 				     tx_ring->qid, i);
1219 			print_once = false;
1220 		} else {
1221 			netif_dbg(tx_ring->adapter, ifdown, tx_ring->netdev,
1222 				  "Free uncompleted tx skb qid %d idx 0x%x\n",
1223 				  tx_ring->qid, i);
1224 		}
1225 
1226 		ena_unmap_tx_buff(tx_ring, tx_info);
1227 
1228 		dev_kfree_skb_any(tx_info->skb);
1229 	}
1230 	netdev_tx_reset_queue(netdev_get_tx_queue(tx_ring->netdev,
1231 						  tx_ring->qid));
1232 }
1233 
1234 static void ena_free_all_tx_bufs(struct ena_adapter *adapter)
1235 {
1236 	struct ena_ring *tx_ring;
1237 	int i;
1238 
1239 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
1240 		tx_ring = &adapter->tx_ring[i];
1241 		ena_free_tx_bufs(tx_ring);
1242 	}
1243 }
1244 
1245 static void ena_destroy_all_tx_queues(struct ena_adapter *adapter)
1246 {
1247 	u16 ena_qid;
1248 	int i;
1249 
1250 	for (i = 0; i < adapter->num_io_queues + adapter->xdp_num_queues; i++) {
1251 		ena_qid = ENA_IO_TXQ_IDX(i);
1252 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1253 	}
1254 }
1255 
1256 static void ena_destroy_all_rx_queues(struct ena_adapter *adapter)
1257 {
1258 	u16 ena_qid;
1259 	int i;
1260 
1261 	for (i = 0; i < adapter->num_io_queues; i++) {
1262 		ena_qid = ENA_IO_RXQ_IDX(i);
1263 		cancel_work_sync(&adapter->ena_napi[i].dim.work);
1264 		ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1265 	}
1266 }
1267 
1268 static void ena_destroy_all_io_queues(struct ena_adapter *adapter)
1269 {
1270 	ena_destroy_all_tx_queues(adapter);
1271 	ena_destroy_all_rx_queues(adapter);
1272 }
1273 
1274 static int handle_invalid_req_id(struct ena_ring *ring, u16 req_id,
1275 				 struct ena_tx_buffer *tx_info, bool is_xdp)
1276 {
1277 	if (tx_info)
1278 		netif_err(ring->adapter,
1279 			  tx_done,
1280 			  ring->netdev,
1281 			  "tx_info doesn't have valid %s. qid %u req_id %u",
1282 			   is_xdp ? "xdp frame" : "skb", ring->qid, req_id);
1283 	else
1284 		netif_err(ring->adapter,
1285 			  tx_done,
1286 			  ring->netdev,
1287 			  "Invalid req_id %u in qid %u\n",
1288 			  req_id, ring->qid);
1289 
1290 	ena_increase_stat(&ring->tx_stats.bad_req_id, 1, &ring->syncp);
1291 	ena_reset_device(ring->adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
1292 
1293 	return -EFAULT;
1294 }
1295 
1296 static int validate_tx_req_id(struct ena_ring *tx_ring, u16 req_id)
1297 {
1298 	struct ena_tx_buffer *tx_info;
1299 
1300 	tx_info = &tx_ring->tx_buffer_info[req_id];
1301 	if (likely(tx_info->skb))
1302 		return 0;
1303 
1304 	return handle_invalid_req_id(tx_ring, req_id, tx_info, false);
1305 }
1306 
1307 static int validate_xdp_req_id(struct ena_ring *xdp_ring, u16 req_id)
1308 {
1309 	struct ena_tx_buffer *tx_info;
1310 
1311 	tx_info = &xdp_ring->tx_buffer_info[req_id];
1312 	if (likely(tx_info->xdpf))
1313 		return 0;
1314 
1315 	return handle_invalid_req_id(xdp_ring, req_id, tx_info, true);
1316 }
1317 
1318 static int ena_clean_tx_irq(struct ena_ring *tx_ring, u32 budget)
1319 {
1320 	struct netdev_queue *txq;
1321 	bool above_thresh;
1322 	u32 tx_bytes = 0;
1323 	u32 total_done = 0;
1324 	u16 next_to_clean;
1325 	u16 req_id;
1326 	int tx_pkts = 0;
1327 	int rc;
1328 
1329 	next_to_clean = tx_ring->next_to_clean;
1330 	txq = netdev_get_tx_queue(tx_ring->netdev, tx_ring->qid);
1331 
1332 	while (tx_pkts < budget) {
1333 		struct ena_tx_buffer *tx_info;
1334 		struct sk_buff *skb;
1335 
1336 		rc = ena_com_tx_comp_req_id_get(tx_ring->ena_com_io_cq,
1337 						&req_id);
1338 		if (rc) {
1339 			if (unlikely(rc == -EINVAL))
1340 				handle_invalid_req_id(tx_ring, req_id, NULL,
1341 						      false);
1342 			break;
1343 		}
1344 
1345 		/* validate that the request id points to a valid skb */
1346 		rc = validate_tx_req_id(tx_ring, req_id);
1347 		if (rc)
1348 			break;
1349 
1350 		tx_info = &tx_ring->tx_buffer_info[req_id];
1351 		skb = tx_info->skb;
1352 
1353 		/* prefetch skb_end_pointer() to speedup skb_shinfo(skb) */
1354 		prefetch(&skb->end);
1355 
1356 		tx_info->skb = NULL;
1357 		tx_info->last_jiffies = 0;
1358 
1359 		ena_unmap_tx_buff(tx_ring, tx_info);
1360 
1361 		netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
1362 			  "tx_poll: q %d skb %p completed\n", tx_ring->qid,
1363 			  skb);
1364 
1365 		tx_bytes += skb->len;
1366 		dev_kfree_skb(skb);
1367 		tx_pkts++;
1368 		total_done += tx_info->tx_descs;
1369 
1370 		tx_ring->free_ids[next_to_clean] = req_id;
1371 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
1372 						     tx_ring->ring_size);
1373 	}
1374 
1375 	tx_ring->next_to_clean = next_to_clean;
1376 	ena_com_comp_ack(tx_ring->ena_com_io_sq, total_done);
1377 	ena_com_update_dev_comp_head(tx_ring->ena_com_io_cq);
1378 
1379 	netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
1380 
1381 	netif_dbg(tx_ring->adapter, tx_done, tx_ring->netdev,
1382 		  "tx_poll: q %d done. total pkts: %d\n",
1383 		  tx_ring->qid, tx_pkts);
1384 
1385 	/* need to make the rings circular update visible to
1386 	 * ena_start_xmit() before checking for netif_queue_stopped().
1387 	 */
1388 	smp_mb();
1389 
1390 	above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1391 						    ENA_TX_WAKEUP_THRESH);
1392 	if (unlikely(netif_tx_queue_stopped(txq) && above_thresh)) {
1393 		__netif_tx_lock(txq, smp_processor_id());
1394 		above_thresh =
1395 			ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1396 						     ENA_TX_WAKEUP_THRESH);
1397 		if (netif_tx_queue_stopped(txq) && above_thresh &&
1398 		    test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags)) {
1399 			netif_tx_wake_queue(txq);
1400 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
1401 					  &tx_ring->syncp);
1402 		}
1403 		__netif_tx_unlock(txq);
1404 	}
1405 
1406 	return tx_pkts;
1407 }
1408 
1409 static struct sk_buff *ena_alloc_skb(struct ena_ring *rx_ring, void *first_frag)
1410 {
1411 	struct sk_buff *skb;
1412 
1413 	if (!first_frag)
1414 		skb = napi_alloc_skb(rx_ring->napi, rx_ring->rx_copybreak);
1415 	else
1416 		skb = napi_build_skb(first_frag, ENA_PAGE_SIZE);
1417 
1418 	if (unlikely(!skb)) {
1419 		ena_increase_stat(&rx_ring->rx_stats.skb_alloc_fail, 1,
1420 				  &rx_ring->syncp);
1421 
1422 		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1423 			  "Failed to allocate skb. first_frag %s\n",
1424 			  first_frag ? "provided" : "not provided");
1425 		return NULL;
1426 	}
1427 
1428 	return skb;
1429 }
1430 
1431 static struct sk_buff *ena_rx_skb(struct ena_ring *rx_ring,
1432 				  struct ena_com_rx_buf_info *ena_bufs,
1433 				  u32 descs,
1434 				  u16 *next_to_clean)
1435 {
1436 	struct ena_rx_buffer *rx_info;
1437 	struct ena_adapter *adapter;
1438 	u16 len, req_id, buf = 0;
1439 	struct sk_buff *skb;
1440 	void *page_addr;
1441 	u32 page_offset;
1442 	void *data_addr;
1443 
1444 	len = ena_bufs[buf].len;
1445 	req_id = ena_bufs[buf].req_id;
1446 
1447 	rx_info = &rx_ring->rx_buffer_info[req_id];
1448 
1449 	if (unlikely(!rx_info->page)) {
1450 		adapter = rx_ring->adapter;
1451 		netif_err(adapter, rx_err, rx_ring->netdev,
1452 			  "Page is NULL. qid %u req_id %u\n", rx_ring->qid, req_id);
1453 		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1, &rx_ring->syncp);
1454 		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
1455 		return NULL;
1456 	}
1457 
1458 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1459 		  "rx_info %p page %p\n",
1460 		  rx_info, rx_info->page);
1461 
1462 	/* save virt address of first buffer */
1463 	page_addr = page_address(rx_info->page);
1464 	page_offset = rx_info->page_offset;
1465 	data_addr = page_addr + page_offset;
1466 
1467 	prefetch(data_addr);
1468 
1469 	if (len <= rx_ring->rx_copybreak) {
1470 		skb = ena_alloc_skb(rx_ring, NULL);
1471 		if (unlikely(!skb))
1472 			return NULL;
1473 
1474 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1475 			  "RX allocated small packet. len %d. data_len %d\n",
1476 			  skb->len, skb->data_len);
1477 
1478 		/* sync this buffer for CPU use */
1479 		dma_sync_single_for_cpu(rx_ring->dev,
1480 					dma_unmap_addr(&rx_info->ena_buf, paddr),
1481 					len,
1482 					DMA_FROM_DEVICE);
1483 		skb_copy_to_linear_data(skb, data_addr, len);
1484 		dma_sync_single_for_device(rx_ring->dev,
1485 					   dma_unmap_addr(&rx_info->ena_buf, paddr),
1486 					   len,
1487 					   DMA_FROM_DEVICE);
1488 
1489 		skb_put(skb, len);
1490 		skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1491 		rx_ring->free_ids[*next_to_clean] = req_id;
1492 		*next_to_clean = ENA_RX_RING_IDX_ADD(*next_to_clean, descs,
1493 						     rx_ring->ring_size);
1494 		return skb;
1495 	}
1496 
1497 	ena_unmap_rx_buff(rx_ring, rx_info);
1498 
1499 	skb = ena_alloc_skb(rx_ring, page_addr);
1500 	if (unlikely(!skb))
1501 		return NULL;
1502 
1503 	/* Populate skb's linear part */
1504 	skb_reserve(skb, page_offset);
1505 	skb_put(skb, len);
1506 	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1507 
1508 	do {
1509 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1510 			  "RX skb updated. len %d. data_len %d\n",
1511 			  skb->len, skb->data_len);
1512 
1513 		rx_info->page = NULL;
1514 
1515 		rx_ring->free_ids[*next_to_clean] = req_id;
1516 		*next_to_clean =
1517 			ENA_RX_RING_IDX_NEXT(*next_to_clean,
1518 					     rx_ring->ring_size);
1519 		if (likely(--descs == 0))
1520 			break;
1521 
1522 		buf++;
1523 		len = ena_bufs[buf].len;
1524 		req_id = ena_bufs[buf].req_id;
1525 
1526 		rx_info = &rx_ring->rx_buffer_info[req_id];
1527 
1528 		ena_unmap_rx_buff(rx_ring, rx_info);
1529 
1530 		skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_info->page,
1531 				rx_info->page_offset, len, ENA_PAGE_SIZE);
1532 
1533 	} while (1);
1534 
1535 	return skb;
1536 }
1537 
1538 /* ena_rx_checksum - indicate in skb if hw indicated a good cksum
1539  * @adapter: structure containing adapter specific data
1540  * @ena_rx_ctx: received packet context/metadata
1541  * @skb: skb currently being received and modified
1542  */
1543 static void ena_rx_checksum(struct ena_ring *rx_ring,
1544 				   struct ena_com_rx_ctx *ena_rx_ctx,
1545 				   struct sk_buff *skb)
1546 {
1547 	/* Rx csum disabled */
1548 	if (unlikely(!(rx_ring->netdev->features & NETIF_F_RXCSUM))) {
1549 		skb->ip_summed = CHECKSUM_NONE;
1550 		return;
1551 	}
1552 
1553 	/* For fragmented packets the checksum isn't valid */
1554 	if (ena_rx_ctx->frag) {
1555 		skb->ip_summed = CHECKSUM_NONE;
1556 		return;
1557 	}
1558 
1559 	/* if IP and error */
1560 	if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
1561 		     (ena_rx_ctx->l3_csum_err))) {
1562 		/* ipv4 checksum error */
1563 		skb->ip_summed = CHECKSUM_NONE;
1564 		ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,
1565 				  &rx_ring->syncp);
1566 		netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1567 			  "RX IPv4 header checksum error\n");
1568 		return;
1569 	}
1570 
1571 	/* if TCP/UDP */
1572 	if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1573 		   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP))) {
1574 		if (unlikely(ena_rx_ctx->l4_csum_err)) {
1575 			/* TCP/UDP checksum error */
1576 			ena_increase_stat(&rx_ring->rx_stats.csum_bad, 1,
1577 					  &rx_ring->syncp);
1578 			netif_dbg(rx_ring->adapter, rx_err, rx_ring->netdev,
1579 				  "RX L4 checksum error\n");
1580 			skb->ip_summed = CHECKSUM_NONE;
1581 			return;
1582 		}
1583 
1584 		if (likely(ena_rx_ctx->l4_csum_checked)) {
1585 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1586 			ena_increase_stat(&rx_ring->rx_stats.csum_good, 1,
1587 					  &rx_ring->syncp);
1588 		} else {
1589 			ena_increase_stat(&rx_ring->rx_stats.csum_unchecked, 1,
1590 					  &rx_ring->syncp);
1591 			skb->ip_summed = CHECKSUM_NONE;
1592 		}
1593 	} else {
1594 		skb->ip_summed = CHECKSUM_NONE;
1595 		return;
1596 	}
1597 
1598 }
1599 
1600 static void ena_set_rx_hash(struct ena_ring *rx_ring,
1601 			    struct ena_com_rx_ctx *ena_rx_ctx,
1602 			    struct sk_buff *skb)
1603 {
1604 	enum pkt_hash_types hash_type;
1605 
1606 	if (likely(rx_ring->netdev->features & NETIF_F_RXHASH)) {
1607 		if (likely((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1608 			   (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)))
1609 
1610 			hash_type = PKT_HASH_TYPE_L4;
1611 		else
1612 			hash_type = PKT_HASH_TYPE_NONE;
1613 
1614 		/* Override hash type if the packet is fragmented */
1615 		if (ena_rx_ctx->frag)
1616 			hash_type = PKT_HASH_TYPE_NONE;
1617 
1618 		skb_set_hash(skb, ena_rx_ctx->hash, hash_type);
1619 	}
1620 }
1621 
1622 static int ena_xdp_handle_buff(struct ena_ring *rx_ring, struct xdp_buff *xdp)
1623 {
1624 	struct ena_rx_buffer *rx_info;
1625 	int ret;
1626 
1627 	rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1628 	xdp_prepare_buff(xdp, page_address(rx_info->page),
1629 			 rx_info->page_offset,
1630 			 rx_ring->ena_bufs[0].len, false);
1631 	/* If for some reason we received a bigger packet than
1632 	 * we expect, then we simply drop it
1633 	 */
1634 	if (unlikely(rx_ring->ena_bufs[0].len > ENA_XDP_MAX_MTU))
1635 		return ENA_XDP_DROP;
1636 
1637 	ret = ena_xdp_execute(rx_ring, xdp);
1638 
1639 	/* The xdp program might expand the headers */
1640 	if (ret == ENA_XDP_PASS) {
1641 		rx_info->page_offset = xdp->data - xdp->data_hard_start;
1642 		rx_ring->ena_bufs[0].len = xdp->data_end - xdp->data;
1643 	}
1644 
1645 	return ret;
1646 }
1647 /* ena_clean_rx_irq - Cleanup RX irq
1648  * @rx_ring: RX ring to clean
1649  * @napi: napi handler
1650  * @budget: how many packets driver is allowed to clean
1651  *
1652  * Returns the number of cleaned buffers.
1653  */
1654 static int ena_clean_rx_irq(struct ena_ring *rx_ring, struct napi_struct *napi,
1655 			    u32 budget)
1656 {
1657 	u16 next_to_clean = rx_ring->next_to_clean;
1658 	struct ena_com_rx_ctx ena_rx_ctx;
1659 	struct ena_rx_buffer *rx_info;
1660 	struct ena_adapter *adapter;
1661 	u32 res_budget, work_done;
1662 	int rx_copybreak_pkt = 0;
1663 	int refill_threshold;
1664 	struct sk_buff *skb;
1665 	int refill_required;
1666 	struct xdp_buff xdp;
1667 	int xdp_flags = 0;
1668 	int total_len = 0;
1669 	int xdp_verdict;
1670 	int rc = 0;
1671 	int i;
1672 
1673 	netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1674 		  "%s qid %d\n", __func__, rx_ring->qid);
1675 	res_budget = budget;
1676 	xdp_init_buff(&xdp, ENA_PAGE_SIZE, &rx_ring->xdp_rxq);
1677 
1678 	do {
1679 		xdp_verdict = ENA_XDP_PASS;
1680 		skb = NULL;
1681 		ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
1682 		ena_rx_ctx.max_bufs = rx_ring->sgl_size;
1683 		ena_rx_ctx.descs = 0;
1684 		ena_rx_ctx.pkt_offset = 0;
1685 		rc = ena_com_rx_pkt(rx_ring->ena_com_io_cq,
1686 				    rx_ring->ena_com_io_sq,
1687 				    &ena_rx_ctx);
1688 		if (unlikely(rc))
1689 			goto error;
1690 
1691 		if (unlikely(ena_rx_ctx.descs == 0))
1692 			break;
1693 
1694 		/* First descriptor might have an offset set by the device */
1695 		rx_info = &rx_ring->rx_buffer_info[rx_ring->ena_bufs[0].req_id];
1696 		rx_info->page_offset += ena_rx_ctx.pkt_offset;
1697 
1698 		netif_dbg(rx_ring->adapter, rx_status, rx_ring->netdev,
1699 			  "rx_poll: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
1700 			  rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
1701 			  ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
1702 
1703 		if (ena_xdp_present_ring(rx_ring))
1704 			xdp_verdict = ena_xdp_handle_buff(rx_ring, &xdp);
1705 
1706 		/* allocate skb and fill it */
1707 		if (xdp_verdict == ENA_XDP_PASS)
1708 			skb = ena_rx_skb(rx_ring,
1709 					 rx_ring->ena_bufs,
1710 					 ena_rx_ctx.descs,
1711 					 &next_to_clean);
1712 
1713 		if (unlikely(!skb)) {
1714 			for (i = 0; i < ena_rx_ctx.descs; i++) {
1715 				int req_id = rx_ring->ena_bufs[i].req_id;
1716 
1717 				rx_ring->free_ids[next_to_clean] = req_id;
1718 				next_to_clean =
1719 					ENA_RX_RING_IDX_NEXT(next_to_clean,
1720 							     rx_ring->ring_size);
1721 
1722 				/* Packets was passed for transmission, unmap it
1723 				 * from RX side.
1724 				 */
1725 				if (xdp_verdict & ENA_XDP_FORWARDED) {
1726 					ena_unmap_rx_buff(rx_ring,
1727 							  &rx_ring->rx_buffer_info[req_id]);
1728 					rx_ring->rx_buffer_info[req_id].page = NULL;
1729 				}
1730 			}
1731 			if (xdp_verdict != ENA_XDP_PASS) {
1732 				xdp_flags |= xdp_verdict;
1733 				total_len += ena_rx_ctx.ena_bufs[0].len;
1734 				res_budget--;
1735 				continue;
1736 			}
1737 			break;
1738 		}
1739 
1740 		ena_rx_checksum(rx_ring, &ena_rx_ctx, skb);
1741 
1742 		ena_set_rx_hash(rx_ring, &ena_rx_ctx, skb);
1743 
1744 		skb_record_rx_queue(skb, rx_ring->qid);
1745 
1746 		if (rx_ring->ena_bufs[0].len <= rx_ring->rx_copybreak)
1747 			rx_copybreak_pkt++;
1748 
1749 		total_len += skb->len;
1750 
1751 		napi_gro_receive(napi, skb);
1752 
1753 		res_budget--;
1754 	} while (likely(res_budget));
1755 
1756 	work_done = budget - res_budget;
1757 	rx_ring->per_napi_packets += work_done;
1758 	u64_stats_update_begin(&rx_ring->syncp);
1759 	rx_ring->rx_stats.bytes += total_len;
1760 	rx_ring->rx_stats.cnt += work_done;
1761 	rx_ring->rx_stats.rx_copybreak_pkt += rx_copybreak_pkt;
1762 	u64_stats_update_end(&rx_ring->syncp);
1763 
1764 	rx_ring->next_to_clean = next_to_clean;
1765 
1766 	refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
1767 	refill_threshold =
1768 		min_t(int, rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
1769 		      ENA_RX_REFILL_THRESH_PACKET);
1770 
1771 	/* Optimization, try to batch new rx buffers */
1772 	if (refill_required > refill_threshold) {
1773 		ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
1774 		ena_refill_rx_bufs(rx_ring, refill_required);
1775 	}
1776 
1777 	if (xdp_flags & ENA_XDP_REDIRECT)
1778 		xdp_do_flush_map();
1779 
1780 	return work_done;
1781 
1782 error:
1783 	adapter = netdev_priv(rx_ring->netdev);
1784 
1785 	if (rc == -ENOSPC) {
1786 		ena_increase_stat(&rx_ring->rx_stats.bad_desc_num, 1,
1787 				  &rx_ring->syncp);
1788 		ena_reset_device(adapter, ENA_REGS_RESET_TOO_MANY_RX_DESCS);
1789 	} else {
1790 		ena_increase_stat(&rx_ring->rx_stats.bad_req_id, 1,
1791 				  &rx_ring->syncp);
1792 		ena_reset_device(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
1793 	}
1794 	return 0;
1795 }
1796 
1797 static void ena_dim_work(struct work_struct *w)
1798 {
1799 	struct dim *dim = container_of(w, struct dim, work);
1800 	struct dim_cq_moder cur_moder =
1801 		net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
1802 	struct ena_napi *ena_napi = container_of(dim, struct ena_napi, dim);
1803 
1804 	ena_napi->rx_ring->smoothed_interval = cur_moder.usec;
1805 	dim->state = DIM_START_MEASURE;
1806 }
1807 
1808 static void ena_adjust_adaptive_rx_intr_moderation(struct ena_napi *ena_napi)
1809 {
1810 	struct dim_sample dim_sample;
1811 	struct ena_ring *rx_ring = ena_napi->rx_ring;
1812 
1813 	if (!rx_ring->per_napi_packets)
1814 		return;
1815 
1816 	rx_ring->non_empty_napi_events++;
1817 
1818 	dim_update_sample(rx_ring->non_empty_napi_events,
1819 			  rx_ring->rx_stats.cnt,
1820 			  rx_ring->rx_stats.bytes,
1821 			  &dim_sample);
1822 
1823 	net_dim(&ena_napi->dim, dim_sample);
1824 
1825 	rx_ring->per_napi_packets = 0;
1826 }
1827 
1828 static void ena_unmask_interrupt(struct ena_ring *tx_ring,
1829 					struct ena_ring *rx_ring)
1830 {
1831 	u32 rx_interval = tx_ring->smoothed_interval;
1832 	struct ena_eth_io_intr_reg intr_reg;
1833 
1834 	/* Rx ring can be NULL when for XDP tx queues which don't have an
1835 	 * accompanying rx_ring pair.
1836 	 */
1837 	if (rx_ring)
1838 		rx_interval = ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev) ?
1839 			rx_ring->smoothed_interval :
1840 			ena_com_get_nonadaptive_moderation_interval_rx(rx_ring->ena_dev);
1841 
1842 	/* Update intr register: rx intr delay,
1843 	 * tx intr delay and interrupt unmask
1844 	 */
1845 	ena_com_update_intr_reg(&intr_reg,
1846 				rx_interval,
1847 				tx_ring->smoothed_interval,
1848 				true);
1849 
1850 	ena_increase_stat(&tx_ring->tx_stats.unmask_interrupt, 1,
1851 			  &tx_ring->syncp);
1852 
1853 	/* It is a shared MSI-X.
1854 	 * Tx and Rx CQ have pointer to it.
1855 	 * So we use one of them to reach the intr reg
1856 	 * The Tx ring is used because the rx_ring is NULL for XDP queues
1857 	 */
1858 	ena_com_unmask_intr(tx_ring->ena_com_io_cq, &intr_reg);
1859 }
1860 
1861 static void ena_update_ring_numa_node(struct ena_ring *tx_ring,
1862 					     struct ena_ring *rx_ring)
1863 {
1864 	int cpu = get_cpu();
1865 	int numa_node;
1866 
1867 	/* Check only one ring since the 2 rings are running on the same cpu */
1868 	if (likely(tx_ring->cpu == cpu))
1869 		goto out;
1870 
1871 	tx_ring->cpu = cpu;
1872 	if (rx_ring)
1873 		rx_ring->cpu = cpu;
1874 
1875 	numa_node = cpu_to_node(cpu);
1876 
1877 	if (likely(tx_ring->numa_node == numa_node))
1878 		goto out;
1879 
1880 	put_cpu();
1881 
1882 	if (numa_node != NUMA_NO_NODE) {
1883 		ena_com_update_numa_node(tx_ring->ena_com_io_cq, numa_node);
1884 		tx_ring->numa_node = numa_node;
1885 		if (rx_ring) {
1886 			rx_ring->numa_node = numa_node;
1887 			ena_com_update_numa_node(rx_ring->ena_com_io_cq,
1888 						 numa_node);
1889 		}
1890 	}
1891 
1892 	return;
1893 out:
1894 	put_cpu();
1895 }
1896 
1897 static int ena_clean_xdp_irq(struct ena_ring *xdp_ring, u32 budget)
1898 {
1899 	u32 total_done = 0;
1900 	u16 next_to_clean;
1901 	u32 tx_bytes = 0;
1902 	int tx_pkts = 0;
1903 	u16 req_id;
1904 	int rc;
1905 
1906 	if (unlikely(!xdp_ring))
1907 		return 0;
1908 	next_to_clean = xdp_ring->next_to_clean;
1909 
1910 	while (tx_pkts < budget) {
1911 		struct ena_tx_buffer *tx_info;
1912 		struct xdp_frame *xdpf;
1913 
1914 		rc = ena_com_tx_comp_req_id_get(xdp_ring->ena_com_io_cq,
1915 						&req_id);
1916 		if (rc) {
1917 			if (unlikely(rc == -EINVAL))
1918 				handle_invalid_req_id(xdp_ring, req_id, NULL,
1919 						      true);
1920 			break;
1921 		}
1922 
1923 		/* validate that the request id points to a valid xdp_frame */
1924 		rc = validate_xdp_req_id(xdp_ring, req_id);
1925 		if (rc)
1926 			break;
1927 
1928 		tx_info = &xdp_ring->tx_buffer_info[req_id];
1929 		xdpf = tx_info->xdpf;
1930 
1931 		tx_info->xdpf = NULL;
1932 		tx_info->last_jiffies = 0;
1933 		ena_unmap_tx_buff(xdp_ring, tx_info);
1934 
1935 		netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,
1936 			  "tx_poll: q %d skb %p completed\n", xdp_ring->qid,
1937 			  xdpf);
1938 
1939 		tx_bytes += xdpf->len;
1940 		tx_pkts++;
1941 		total_done += tx_info->tx_descs;
1942 
1943 		xdp_return_frame(xdpf);
1944 		xdp_ring->free_ids[next_to_clean] = req_id;
1945 		next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
1946 						     xdp_ring->ring_size);
1947 	}
1948 
1949 	xdp_ring->next_to_clean = next_to_clean;
1950 	ena_com_comp_ack(xdp_ring->ena_com_io_sq, total_done);
1951 	ena_com_update_dev_comp_head(xdp_ring->ena_com_io_cq);
1952 
1953 	netif_dbg(xdp_ring->adapter, tx_done, xdp_ring->netdev,
1954 		  "tx_poll: q %d done. total pkts: %d\n",
1955 		  xdp_ring->qid, tx_pkts);
1956 
1957 	return tx_pkts;
1958 }
1959 
1960 static int ena_io_poll(struct napi_struct *napi, int budget)
1961 {
1962 	struct ena_napi *ena_napi = container_of(napi, struct ena_napi, napi);
1963 	struct ena_ring *tx_ring, *rx_ring;
1964 	int tx_work_done;
1965 	int rx_work_done = 0;
1966 	int tx_budget;
1967 	int napi_comp_call = 0;
1968 	int ret;
1969 
1970 	tx_ring = ena_napi->tx_ring;
1971 	rx_ring = ena_napi->rx_ring;
1972 
1973 	tx_budget = tx_ring->ring_size / ENA_TX_POLL_BUDGET_DIVIDER;
1974 
1975 	if (!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1976 	    test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags)) {
1977 		napi_complete_done(napi, 0);
1978 		return 0;
1979 	}
1980 
1981 	tx_work_done = ena_clean_tx_irq(tx_ring, tx_budget);
1982 	/* On netpoll the budget is zero and the handler should only clean the
1983 	 * tx completions.
1984 	 */
1985 	if (likely(budget))
1986 		rx_work_done = ena_clean_rx_irq(rx_ring, napi, budget);
1987 
1988 	/* If the device is about to reset or down, avoid unmask
1989 	 * the interrupt and return 0 so NAPI won't reschedule
1990 	 */
1991 	if (unlikely(!test_bit(ENA_FLAG_DEV_UP, &tx_ring->adapter->flags) ||
1992 		     test_bit(ENA_FLAG_TRIGGER_RESET, &tx_ring->adapter->flags))) {
1993 		napi_complete_done(napi, 0);
1994 		ret = 0;
1995 
1996 	} else if ((budget > rx_work_done) && (tx_budget > tx_work_done)) {
1997 		napi_comp_call = 1;
1998 
1999 		/* Update numa and unmask the interrupt only when schedule
2000 		 * from the interrupt context (vs from sk_busy_loop)
2001 		 */
2002 		if (napi_complete_done(napi, rx_work_done) &&
2003 		    READ_ONCE(ena_napi->interrupts_masked)) {
2004 			smp_rmb(); /* make sure interrupts_masked is read */
2005 			WRITE_ONCE(ena_napi->interrupts_masked, false);
2006 			/* We apply adaptive moderation on Rx path only.
2007 			 * Tx uses static interrupt moderation.
2008 			 */
2009 			if (ena_com_get_adaptive_moderation_enabled(rx_ring->ena_dev))
2010 				ena_adjust_adaptive_rx_intr_moderation(ena_napi);
2011 
2012 			ena_update_ring_numa_node(tx_ring, rx_ring);
2013 			ena_unmask_interrupt(tx_ring, rx_ring);
2014 		}
2015 
2016 		ret = rx_work_done;
2017 	} else {
2018 		ret = budget;
2019 	}
2020 
2021 	u64_stats_update_begin(&tx_ring->syncp);
2022 	tx_ring->tx_stats.napi_comp += napi_comp_call;
2023 	tx_ring->tx_stats.tx_poll++;
2024 	u64_stats_update_end(&tx_ring->syncp);
2025 
2026 	tx_ring->tx_stats.last_napi_jiffies = jiffies;
2027 
2028 	return ret;
2029 }
2030 
2031 static irqreturn_t ena_intr_msix_mgmnt(int irq, void *data)
2032 {
2033 	struct ena_adapter *adapter = (struct ena_adapter *)data;
2034 
2035 	ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
2036 
2037 	/* Don't call the aenq handler before probe is done */
2038 	if (likely(test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags)))
2039 		ena_com_aenq_intr_handler(adapter->ena_dev, data);
2040 
2041 	return IRQ_HANDLED;
2042 }
2043 
2044 /* ena_intr_msix_io - MSI-X Interrupt Handler for Tx/Rx
2045  * @irq: interrupt number
2046  * @data: pointer to a network interface private napi device structure
2047  */
2048 static irqreturn_t ena_intr_msix_io(int irq, void *data)
2049 {
2050 	struct ena_napi *ena_napi = data;
2051 
2052 	/* Used to check HW health */
2053 	WRITE_ONCE(ena_napi->first_interrupt, true);
2054 
2055 	WRITE_ONCE(ena_napi->interrupts_masked, true);
2056 	smp_wmb(); /* write interrupts_masked before calling napi */
2057 
2058 	napi_schedule_irqoff(&ena_napi->napi);
2059 
2060 	return IRQ_HANDLED;
2061 }
2062 
2063 /* Reserve a single MSI-X vector for management (admin + aenq).
2064  * plus reserve one vector for each potential io queue.
2065  * the number of potential io queues is the minimum of what the device
2066  * supports and the number of vCPUs.
2067  */
2068 static int ena_enable_msix(struct ena_adapter *adapter)
2069 {
2070 	int msix_vecs, irq_cnt;
2071 
2072 	if (test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
2073 		netif_err(adapter, probe, adapter->netdev,
2074 			  "Error, MSI-X is already enabled\n");
2075 		return -EPERM;
2076 	}
2077 
2078 	/* Reserved the max msix vectors we might need */
2079 	msix_vecs = ENA_MAX_MSIX_VEC(adapter->max_num_io_queues);
2080 	netif_dbg(adapter, probe, adapter->netdev,
2081 		  "Trying to enable MSI-X, vectors %d\n", msix_vecs);
2082 
2083 	irq_cnt = pci_alloc_irq_vectors(adapter->pdev, ENA_MIN_MSIX_VEC,
2084 					msix_vecs, PCI_IRQ_MSIX);
2085 
2086 	if (irq_cnt < 0) {
2087 		netif_err(adapter, probe, adapter->netdev,
2088 			  "Failed to enable MSI-X. irq_cnt %d\n", irq_cnt);
2089 		return -ENOSPC;
2090 	}
2091 
2092 	if (irq_cnt != msix_vecs) {
2093 		netif_notice(adapter, probe, adapter->netdev,
2094 			     "Enable only %d MSI-X (out of %d), reduce the number of queues\n",
2095 			     irq_cnt, msix_vecs);
2096 		adapter->num_io_queues = irq_cnt - ENA_ADMIN_MSIX_VEC;
2097 	}
2098 
2099 	if (ena_init_rx_cpu_rmap(adapter))
2100 		netif_warn(adapter, probe, adapter->netdev,
2101 			   "Failed to map IRQs to CPUs\n");
2102 
2103 	adapter->msix_vecs = irq_cnt;
2104 	set_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags);
2105 
2106 	return 0;
2107 }
2108 
2109 static void ena_setup_mgmnt_intr(struct ena_adapter *adapter)
2110 {
2111 	u32 cpu;
2112 
2113 	snprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
2114 		 ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
2115 		 pci_name(adapter->pdev));
2116 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler =
2117 		ena_intr_msix_mgmnt;
2118 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
2119 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
2120 		pci_irq_vector(adapter->pdev, ENA_MGMNT_IRQ_IDX);
2121 	cpu = cpumask_first(cpu_online_mask);
2122 	adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].cpu = cpu;
2123 	cpumask_set_cpu(cpu,
2124 			&adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].affinity_hint_mask);
2125 }
2126 
2127 static void ena_setup_io_intr(struct ena_adapter *adapter)
2128 {
2129 	struct net_device *netdev;
2130 	int irq_idx, i, cpu;
2131 	int io_queue_count;
2132 
2133 	netdev = adapter->netdev;
2134 	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2135 
2136 	for (i = 0; i < io_queue_count; i++) {
2137 		irq_idx = ENA_IO_IRQ_IDX(i);
2138 		cpu = i % num_online_cpus();
2139 
2140 		snprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
2141 			 "%s-Tx-Rx-%d", netdev->name, i);
2142 		adapter->irq_tbl[irq_idx].handler = ena_intr_msix_io;
2143 		adapter->irq_tbl[irq_idx].data = &adapter->ena_napi[i];
2144 		adapter->irq_tbl[irq_idx].vector =
2145 			pci_irq_vector(adapter->pdev, irq_idx);
2146 		adapter->irq_tbl[irq_idx].cpu = cpu;
2147 
2148 		cpumask_set_cpu(cpu,
2149 				&adapter->irq_tbl[irq_idx].affinity_hint_mask);
2150 	}
2151 }
2152 
2153 static int ena_request_mgmnt_irq(struct ena_adapter *adapter)
2154 {
2155 	unsigned long flags = 0;
2156 	struct ena_irq *irq;
2157 	int rc;
2158 
2159 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2160 	rc = request_irq(irq->vector, irq->handler, flags, irq->name,
2161 			 irq->data);
2162 	if (rc) {
2163 		netif_err(adapter, probe, adapter->netdev,
2164 			  "Failed to request admin irq\n");
2165 		return rc;
2166 	}
2167 
2168 	netif_dbg(adapter, probe, adapter->netdev,
2169 		  "Set affinity hint of mgmnt irq.to 0x%lx (irq vector: %d)\n",
2170 		  irq->affinity_hint_mask.bits[0], irq->vector);
2171 
2172 	irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
2173 
2174 	return rc;
2175 }
2176 
2177 static int ena_request_io_irq(struct ena_adapter *adapter)
2178 {
2179 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2180 	unsigned long flags = 0;
2181 	struct ena_irq *irq;
2182 	int rc = 0, i, k;
2183 
2184 	if (!test_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags)) {
2185 		netif_err(adapter, ifup, adapter->netdev,
2186 			  "Failed to request I/O IRQ: MSI-X is not enabled\n");
2187 		return -EINVAL;
2188 	}
2189 
2190 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
2191 		irq = &adapter->irq_tbl[i];
2192 		rc = request_irq(irq->vector, irq->handler, flags, irq->name,
2193 				 irq->data);
2194 		if (rc) {
2195 			netif_err(adapter, ifup, adapter->netdev,
2196 				  "Failed to request I/O IRQ. index %d rc %d\n",
2197 				   i, rc);
2198 			goto err;
2199 		}
2200 
2201 		netif_dbg(adapter, ifup, adapter->netdev,
2202 			  "Set affinity hint of irq. index %d to 0x%lx (irq vector: %d)\n",
2203 			  i, irq->affinity_hint_mask.bits[0], irq->vector);
2204 
2205 		irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask);
2206 	}
2207 
2208 	return rc;
2209 
2210 err:
2211 	for (k = ENA_IO_IRQ_FIRST_IDX; k < i; k++) {
2212 		irq = &adapter->irq_tbl[k];
2213 		free_irq(irq->vector, irq->data);
2214 	}
2215 
2216 	return rc;
2217 }
2218 
2219 static void ena_free_mgmnt_irq(struct ena_adapter *adapter)
2220 {
2221 	struct ena_irq *irq;
2222 
2223 	irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2224 	synchronize_irq(irq->vector);
2225 	irq_set_affinity_hint(irq->vector, NULL);
2226 	free_irq(irq->vector, irq->data);
2227 }
2228 
2229 static void ena_free_io_irq(struct ena_adapter *adapter)
2230 {
2231 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2232 	struct ena_irq *irq;
2233 	int i;
2234 
2235 #ifdef CONFIG_RFS_ACCEL
2236 	if (adapter->msix_vecs >= 1) {
2237 		free_irq_cpu_rmap(adapter->netdev->rx_cpu_rmap);
2238 		adapter->netdev->rx_cpu_rmap = NULL;
2239 	}
2240 #endif /* CONFIG_RFS_ACCEL */
2241 
2242 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++) {
2243 		irq = &adapter->irq_tbl[i];
2244 		irq_set_affinity_hint(irq->vector, NULL);
2245 		free_irq(irq->vector, irq->data);
2246 	}
2247 }
2248 
2249 static void ena_disable_msix(struct ena_adapter *adapter)
2250 {
2251 	if (test_and_clear_bit(ENA_FLAG_MSIX_ENABLED, &adapter->flags))
2252 		pci_free_irq_vectors(adapter->pdev);
2253 }
2254 
2255 static void ena_disable_io_intr_sync(struct ena_adapter *adapter)
2256 {
2257 	u32 io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2258 	int i;
2259 
2260 	if (!netif_running(adapter->netdev))
2261 		return;
2262 
2263 	for (i = ENA_IO_IRQ_FIRST_IDX; i < ENA_MAX_MSIX_VEC(io_queue_count); i++)
2264 		synchronize_irq(adapter->irq_tbl[i].vector);
2265 }
2266 
2267 static void ena_del_napi_in_range(struct ena_adapter *adapter,
2268 				  int first_index,
2269 				  int count)
2270 {
2271 	int i;
2272 
2273 	for (i = first_index; i < first_index + count; i++) {
2274 		netif_napi_del(&adapter->ena_napi[i].napi);
2275 
2276 		WARN_ON(!ENA_IS_XDP_INDEX(adapter, i) &&
2277 			adapter->ena_napi[i].xdp_ring);
2278 	}
2279 }
2280 
2281 static void ena_init_napi_in_range(struct ena_adapter *adapter,
2282 				   int first_index, int count)
2283 {
2284 	int i;
2285 
2286 	for (i = first_index; i < first_index + count; i++) {
2287 		struct ena_napi *napi = &adapter->ena_napi[i];
2288 
2289 		netif_napi_add(adapter->netdev, &napi->napi,
2290 			       ENA_IS_XDP_INDEX(adapter, i) ? ena_xdp_io_poll : ena_io_poll);
2291 
2292 		if (!ENA_IS_XDP_INDEX(adapter, i)) {
2293 			napi->rx_ring = &adapter->rx_ring[i];
2294 			napi->tx_ring = &adapter->tx_ring[i];
2295 		} else {
2296 			napi->xdp_ring = &adapter->tx_ring[i];
2297 		}
2298 		napi->qid = i;
2299 	}
2300 }
2301 
2302 static void ena_napi_disable_in_range(struct ena_adapter *adapter,
2303 				      int first_index,
2304 				      int count)
2305 {
2306 	int i;
2307 
2308 	for (i = first_index; i < first_index + count; i++)
2309 		napi_disable(&adapter->ena_napi[i].napi);
2310 }
2311 
2312 static void ena_napi_enable_in_range(struct ena_adapter *adapter,
2313 				     int first_index,
2314 				     int count)
2315 {
2316 	int i;
2317 
2318 	for (i = first_index; i < first_index + count; i++)
2319 		napi_enable(&adapter->ena_napi[i].napi);
2320 }
2321 
2322 /* Configure the Rx forwarding */
2323 static int ena_rss_configure(struct ena_adapter *adapter)
2324 {
2325 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2326 	int rc;
2327 
2328 	/* In case the RSS table wasn't initialized by probe */
2329 	if (!ena_dev->rss.tbl_log_size) {
2330 		rc = ena_rss_init_default(adapter);
2331 		if (rc && (rc != -EOPNOTSUPP)) {
2332 			netif_err(adapter, ifup, adapter->netdev,
2333 				  "Failed to init RSS rc: %d\n", rc);
2334 			return rc;
2335 		}
2336 	}
2337 
2338 	/* Set indirect table */
2339 	rc = ena_com_indirect_table_set(ena_dev);
2340 	if (unlikely(rc && rc != -EOPNOTSUPP))
2341 		return rc;
2342 
2343 	/* Configure hash function (if supported) */
2344 	rc = ena_com_set_hash_function(ena_dev);
2345 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
2346 		return rc;
2347 
2348 	/* Configure hash inputs (if supported) */
2349 	rc = ena_com_set_hash_ctrl(ena_dev);
2350 	if (unlikely(rc && (rc != -EOPNOTSUPP)))
2351 		return rc;
2352 
2353 	return 0;
2354 }
2355 
2356 static int ena_up_complete(struct ena_adapter *adapter)
2357 {
2358 	int rc;
2359 
2360 	rc = ena_rss_configure(adapter);
2361 	if (rc)
2362 		return rc;
2363 
2364 	ena_change_mtu(adapter->netdev, adapter->netdev->mtu);
2365 
2366 	ena_refill_all_rx_bufs(adapter);
2367 
2368 	/* enable transmits */
2369 	netif_tx_start_all_queues(adapter->netdev);
2370 
2371 	ena_napi_enable_in_range(adapter,
2372 				 0,
2373 				 adapter->xdp_num_queues + adapter->num_io_queues);
2374 
2375 	return 0;
2376 }
2377 
2378 static int ena_create_io_tx_queue(struct ena_adapter *adapter, int qid)
2379 {
2380 	struct ena_com_create_io_ctx ctx;
2381 	struct ena_com_dev *ena_dev;
2382 	struct ena_ring *tx_ring;
2383 	u32 msix_vector;
2384 	u16 ena_qid;
2385 	int rc;
2386 
2387 	ena_dev = adapter->ena_dev;
2388 
2389 	tx_ring = &adapter->tx_ring[qid];
2390 	msix_vector = ENA_IO_IRQ_IDX(qid);
2391 	ena_qid = ENA_IO_TXQ_IDX(qid);
2392 
2393 	memset(&ctx, 0x0, sizeof(ctx));
2394 
2395 	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
2396 	ctx.qid = ena_qid;
2397 	ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
2398 	ctx.msix_vector = msix_vector;
2399 	ctx.queue_size = tx_ring->ring_size;
2400 	ctx.numa_node = tx_ring->numa_node;
2401 
2402 	rc = ena_com_create_io_queue(ena_dev, &ctx);
2403 	if (rc) {
2404 		netif_err(adapter, ifup, adapter->netdev,
2405 			  "Failed to create I/O TX queue num %d rc: %d\n",
2406 			  qid, rc);
2407 		return rc;
2408 	}
2409 
2410 	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
2411 				     &tx_ring->ena_com_io_sq,
2412 				     &tx_ring->ena_com_io_cq);
2413 	if (rc) {
2414 		netif_err(adapter, ifup, adapter->netdev,
2415 			  "Failed to get TX queue handlers. TX queue num %d rc: %d\n",
2416 			  qid, rc);
2417 		ena_com_destroy_io_queue(ena_dev, ena_qid);
2418 		return rc;
2419 	}
2420 
2421 	ena_com_update_numa_node(tx_ring->ena_com_io_cq, ctx.numa_node);
2422 	return rc;
2423 }
2424 
2425 static int ena_create_io_tx_queues_in_range(struct ena_adapter *adapter,
2426 					    int first_index, int count)
2427 {
2428 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2429 	int rc, i;
2430 
2431 	for (i = first_index; i < first_index + count; i++) {
2432 		rc = ena_create_io_tx_queue(adapter, i);
2433 		if (rc)
2434 			goto create_err;
2435 	}
2436 
2437 	return 0;
2438 
2439 create_err:
2440 	while (i-- > first_index)
2441 		ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
2442 
2443 	return rc;
2444 }
2445 
2446 static int ena_create_io_rx_queue(struct ena_adapter *adapter, int qid)
2447 {
2448 	struct ena_com_dev *ena_dev;
2449 	struct ena_com_create_io_ctx ctx;
2450 	struct ena_ring *rx_ring;
2451 	u32 msix_vector;
2452 	u16 ena_qid;
2453 	int rc;
2454 
2455 	ena_dev = adapter->ena_dev;
2456 
2457 	rx_ring = &adapter->rx_ring[qid];
2458 	msix_vector = ENA_IO_IRQ_IDX(qid);
2459 	ena_qid = ENA_IO_RXQ_IDX(qid);
2460 
2461 	memset(&ctx, 0x0, sizeof(ctx));
2462 
2463 	ctx.qid = ena_qid;
2464 	ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
2465 	ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
2466 	ctx.msix_vector = msix_vector;
2467 	ctx.queue_size = rx_ring->ring_size;
2468 	ctx.numa_node = rx_ring->numa_node;
2469 
2470 	rc = ena_com_create_io_queue(ena_dev, &ctx);
2471 	if (rc) {
2472 		netif_err(adapter, ifup, adapter->netdev,
2473 			  "Failed to create I/O RX queue num %d rc: %d\n",
2474 			  qid, rc);
2475 		return rc;
2476 	}
2477 
2478 	rc = ena_com_get_io_handlers(ena_dev, ena_qid,
2479 				     &rx_ring->ena_com_io_sq,
2480 				     &rx_ring->ena_com_io_cq);
2481 	if (rc) {
2482 		netif_err(adapter, ifup, adapter->netdev,
2483 			  "Failed to get RX queue handlers. RX queue num %d rc: %d\n",
2484 			  qid, rc);
2485 		goto err;
2486 	}
2487 
2488 	ena_com_update_numa_node(rx_ring->ena_com_io_cq, ctx.numa_node);
2489 
2490 	return rc;
2491 err:
2492 	ena_com_destroy_io_queue(ena_dev, ena_qid);
2493 	return rc;
2494 }
2495 
2496 static int ena_create_all_io_rx_queues(struct ena_adapter *adapter)
2497 {
2498 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2499 	int rc, i;
2500 
2501 	for (i = 0; i < adapter->num_io_queues; i++) {
2502 		rc = ena_create_io_rx_queue(adapter, i);
2503 		if (rc)
2504 			goto create_err;
2505 		INIT_WORK(&adapter->ena_napi[i].dim.work, ena_dim_work);
2506 	}
2507 
2508 	return 0;
2509 
2510 create_err:
2511 	while (i--) {
2512 		cancel_work_sync(&adapter->ena_napi[i].dim.work);
2513 		ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
2514 	}
2515 
2516 	return rc;
2517 }
2518 
2519 static void set_io_rings_size(struct ena_adapter *adapter,
2520 			      int new_tx_size,
2521 			      int new_rx_size)
2522 {
2523 	int i;
2524 
2525 	for (i = 0; i < adapter->num_io_queues; i++) {
2526 		adapter->tx_ring[i].ring_size = new_tx_size;
2527 		adapter->rx_ring[i].ring_size = new_rx_size;
2528 	}
2529 }
2530 
2531 /* This function allows queue allocation to backoff when the system is
2532  * low on memory. If there is not enough memory to allocate io queues
2533  * the driver will try to allocate smaller queues.
2534  *
2535  * The backoff algorithm is as follows:
2536  *  1. Try to allocate TX and RX and if successful.
2537  *  1.1. return success
2538  *
2539  *  2. Divide by 2 the size of the larger of RX and TX queues (or both if their size is the same).
2540  *
2541  *  3. If TX or RX is smaller than 256
2542  *  3.1. return failure.
2543  *  4. else
2544  *  4.1. go back to 1.
2545  */
2546 static int create_queues_with_size_backoff(struct ena_adapter *adapter)
2547 {
2548 	int rc, cur_rx_ring_size, cur_tx_ring_size;
2549 	int new_rx_ring_size, new_tx_ring_size;
2550 
2551 	/* current queue sizes might be set to smaller than the requested
2552 	 * ones due to past queue allocation failures.
2553 	 */
2554 	set_io_rings_size(adapter, adapter->requested_tx_ring_size,
2555 			  adapter->requested_rx_ring_size);
2556 
2557 	while (1) {
2558 		if (ena_xdp_present(adapter)) {
2559 			rc = ena_setup_and_create_all_xdp_queues(adapter);
2560 
2561 			if (rc)
2562 				goto err_setup_tx;
2563 		}
2564 		rc = ena_setup_tx_resources_in_range(adapter,
2565 						     0,
2566 						     adapter->num_io_queues);
2567 		if (rc)
2568 			goto err_setup_tx;
2569 
2570 		rc = ena_create_io_tx_queues_in_range(adapter,
2571 						      0,
2572 						      adapter->num_io_queues);
2573 		if (rc)
2574 			goto err_create_tx_queues;
2575 
2576 		rc = ena_setup_all_rx_resources(adapter);
2577 		if (rc)
2578 			goto err_setup_rx;
2579 
2580 		rc = ena_create_all_io_rx_queues(adapter);
2581 		if (rc)
2582 			goto err_create_rx_queues;
2583 
2584 		return 0;
2585 
2586 err_create_rx_queues:
2587 		ena_free_all_io_rx_resources(adapter);
2588 err_setup_rx:
2589 		ena_destroy_all_tx_queues(adapter);
2590 err_create_tx_queues:
2591 		ena_free_all_io_tx_resources(adapter);
2592 err_setup_tx:
2593 		if (rc != -ENOMEM) {
2594 			netif_err(adapter, ifup, adapter->netdev,
2595 				  "Queue creation failed with error code %d\n",
2596 				  rc);
2597 			return rc;
2598 		}
2599 
2600 		cur_tx_ring_size = adapter->tx_ring[0].ring_size;
2601 		cur_rx_ring_size = adapter->rx_ring[0].ring_size;
2602 
2603 		netif_err(adapter, ifup, adapter->netdev,
2604 			  "Not enough memory to create queues with sizes TX=%d, RX=%d\n",
2605 			  cur_tx_ring_size, cur_rx_ring_size);
2606 
2607 		new_tx_ring_size = cur_tx_ring_size;
2608 		new_rx_ring_size = cur_rx_ring_size;
2609 
2610 		/* Decrease the size of the larger queue, or
2611 		 * decrease both if they are the same size.
2612 		 */
2613 		if (cur_rx_ring_size <= cur_tx_ring_size)
2614 			new_tx_ring_size = cur_tx_ring_size / 2;
2615 		if (cur_rx_ring_size >= cur_tx_ring_size)
2616 			new_rx_ring_size = cur_rx_ring_size / 2;
2617 
2618 		if (new_tx_ring_size < ENA_MIN_RING_SIZE ||
2619 		    new_rx_ring_size < ENA_MIN_RING_SIZE) {
2620 			netif_err(adapter, ifup, adapter->netdev,
2621 				  "Queue creation failed with the smallest possible queue size of %d for both queues. Not retrying with smaller queues\n",
2622 				  ENA_MIN_RING_SIZE);
2623 			return rc;
2624 		}
2625 
2626 		netif_err(adapter, ifup, adapter->netdev,
2627 			  "Retrying queue creation with sizes TX=%d, RX=%d\n",
2628 			  new_tx_ring_size,
2629 			  new_rx_ring_size);
2630 
2631 		set_io_rings_size(adapter, new_tx_ring_size,
2632 				  new_rx_ring_size);
2633 	}
2634 }
2635 
2636 static int ena_up(struct ena_adapter *adapter)
2637 {
2638 	int io_queue_count, rc, i;
2639 
2640 	netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
2641 
2642 	io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2643 	ena_setup_io_intr(adapter);
2644 
2645 	/* napi poll functions should be initialized before running
2646 	 * request_irq(), to handle a rare condition where there is a pending
2647 	 * interrupt, causing the ISR to fire immediately while the poll
2648 	 * function wasn't set yet, causing a null dereference
2649 	 */
2650 	ena_init_napi_in_range(adapter, 0, io_queue_count);
2651 
2652 	rc = ena_request_io_irq(adapter);
2653 	if (rc)
2654 		goto err_req_irq;
2655 
2656 	rc = create_queues_with_size_backoff(adapter);
2657 	if (rc)
2658 		goto err_create_queues_with_backoff;
2659 
2660 	rc = ena_up_complete(adapter);
2661 	if (rc)
2662 		goto err_up;
2663 
2664 	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
2665 		netif_carrier_on(adapter->netdev);
2666 
2667 	ena_increase_stat(&adapter->dev_stats.interface_up, 1,
2668 			  &adapter->syncp);
2669 
2670 	set_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2671 
2672 	/* Enable completion queues interrupt */
2673 	for (i = 0; i < adapter->num_io_queues; i++)
2674 		ena_unmask_interrupt(&adapter->tx_ring[i],
2675 				     &adapter->rx_ring[i]);
2676 
2677 	/* schedule napi in case we had pending packets
2678 	 * from the last time we disable napi
2679 	 */
2680 	for (i = 0; i < io_queue_count; i++)
2681 		napi_schedule(&adapter->ena_napi[i].napi);
2682 
2683 	return rc;
2684 
2685 err_up:
2686 	ena_destroy_all_tx_queues(adapter);
2687 	ena_free_all_io_tx_resources(adapter);
2688 	ena_destroy_all_rx_queues(adapter);
2689 	ena_free_all_io_rx_resources(adapter);
2690 err_create_queues_with_backoff:
2691 	ena_free_io_irq(adapter);
2692 err_req_irq:
2693 	ena_del_napi_in_range(adapter, 0, io_queue_count);
2694 
2695 	return rc;
2696 }
2697 
2698 static void ena_down(struct ena_adapter *adapter)
2699 {
2700 	int io_queue_count = adapter->num_io_queues + adapter->xdp_num_queues;
2701 
2702 	netif_info(adapter, ifdown, adapter->netdev, "%s\n", __func__);
2703 
2704 	clear_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2705 
2706 	ena_increase_stat(&adapter->dev_stats.interface_down, 1,
2707 			  &adapter->syncp);
2708 
2709 	netif_carrier_off(adapter->netdev);
2710 	netif_tx_disable(adapter->netdev);
2711 
2712 	/* After this point the napi handler won't enable the tx queue */
2713 	ena_napi_disable_in_range(adapter, 0, io_queue_count);
2714 
2715 	/* After destroy the queue there won't be any new interrupts */
2716 
2717 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags)) {
2718 		int rc;
2719 
2720 		rc = ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
2721 		if (rc)
2722 			netif_err(adapter, ifdown, adapter->netdev,
2723 				  "Device reset failed\n");
2724 		/* stop submitting admin commands on a device that was reset */
2725 		ena_com_set_admin_running_state(adapter->ena_dev, false);
2726 	}
2727 
2728 	ena_destroy_all_io_queues(adapter);
2729 
2730 	ena_disable_io_intr_sync(adapter);
2731 	ena_free_io_irq(adapter);
2732 	ena_del_napi_in_range(adapter, 0, io_queue_count);
2733 
2734 	ena_free_all_tx_bufs(adapter);
2735 	ena_free_all_rx_bufs(adapter);
2736 	ena_free_all_io_tx_resources(adapter);
2737 	ena_free_all_io_rx_resources(adapter);
2738 }
2739 
2740 /* ena_open - Called when a network interface is made active
2741  * @netdev: network interface device structure
2742  *
2743  * Returns 0 on success, negative value on failure
2744  *
2745  * The open entry point is called when a network interface is made
2746  * active by the system (IFF_UP).  At this point all resources needed
2747  * for transmit and receive operations are allocated, the interrupt
2748  * handler is registered with the OS, the watchdog timer is started,
2749  * and the stack is notified that the interface is ready.
2750  */
2751 static int ena_open(struct net_device *netdev)
2752 {
2753 	struct ena_adapter *adapter = netdev_priv(netdev);
2754 	int rc;
2755 
2756 	/* Notify the stack of the actual queue counts. */
2757 	rc = netif_set_real_num_tx_queues(netdev, adapter->num_io_queues);
2758 	if (rc) {
2759 		netif_err(adapter, ifup, netdev, "Can't set num tx queues\n");
2760 		return rc;
2761 	}
2762 
2763 	rc = netif_set_real_num_rx_queues(netdev, adapter->num_io_queues);
2764 	if (rc) {
2765 		netif_err(adapter, ifup, netdev, "Can't set num rx queues\n");
2766 		return rc;
2767 	}
2768 
2769 	rc = ena_up(adapter);
2770 	if (rc)
2771 		return rc;
2772 
2773 	return rc;
2774 }
2775 
2776 /* ena_close - Disables a network interface
2777  * @netdev: network interface device structure
2778  *
2779  * Returns 0, this is not allowed to fail
2780  *
2781  * The close entry point is called when an interface is de-activated
2782  * by the OS.  The hardware is still under the drivers control, but
2783  * needs to be disabled.  A global MAC reset is issued to stop the
2784  * hardware, and all transmit and receive resources are freed.
2785  */
2786 static int ena_close(struct net_device *netdev)
2787 {
2788 	struct ena_adapter *adapter = netdev_priv(netdev);
2789 
2790 	netif_dbg(adapter, ifdown, netdev, "%s\n", __func__);
2791 
2792 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
2793 		return 0;
2794 
2795 	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
2796 		ena_down(adapter);
2797 
2798 	/* Check for device status and issue reset if needed*/
2799 	check_for_admin_com_state(adapter);
2800 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
2801 		netif_err(adapter, ifdown, adapter->netdev,
2802 			  "Destroy failure, restarting device\n");
2803 		ena_dump_stats_to_dmesg(adapter);
2804 		/* rtnl lock already obtained in dev_ioctl() layer */
2805 		ena_destroy_device(adapter, false);
2806 		ena_restore_device(adapter);
2807 	}
2808 
2809 	return 0;
2810 }
2811 
2812 int ena_update_queue_sizes(struct ena_adapter *adapter,
2813 			   u32 new_tx_size,
2814 			   u32 new_rx_size)
2815 {
2816 	bool dev_was_up;
2817 
2818 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2819 	ena_close(adapter->netdev);
2820 	adapter->requested_tx_ring_size = new_tx_size;
2821 	adapter->requested_rx_ring_size = new_rx_size;
2822 	ena_init_io_rings(adapter,
2823 			  0,
2824 			  adapter->xdp_num_queues +
2825 			  adapter->num_io_queues);
2826 	return dev_was_up ? ena_up(adapter) : 0;
2827 }
2828 
2829 int ena_set_rx_copybreak(struct ena_adapter *adapter, u32 rx_copybreak)
2830 {
2831 	struct ena_ring *rx_ring;
2832 	int i;
2833 
2834 	if (rx_copybreak > min_t(u16, adapter->netdev->mtu, ENA_PAGE_SIZE))
2835 		return -EINVAL;
2836 
2837 	adapter->rx_copybreak = rx_copybreak;
2838 
2839 	for (i = 0; i < adapter->num_io_queues; i++) {
2840 		rx_ring = &adapter->rx_ring[i];
2841 		rx_ring->rx_copybreak = rx_copybreak;
2842 	}
2843 
2844 	return 0;
2845 }
2846 
2847 int ena_update_queue_count(struct ena_adapter *adapter, u32 new_channel_count)
2848 {
2849 	struct ena_com_dev *ena_dev = adapter->ena_dev;
2850 	int prev_channel_count;
2851 	bool dev_was_up;
2852 
2853 	dev_was_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
2854 	ena_close(adapter->netdev);
2855 	prev_channel_count = adapter->num_io_queues;
2856 	adapter->num_io_queues = new_channel_count;
2857 	if (ena_xdp_present(adapter) &&
2858 	    ena_xdp_allowed(adapter) == ENA_XDP_ALLOWED) {
2859 		adapter->xdp_first_ring = new_channel_count;
2860 		adapter->xdp_num_queues = new_channel_count;
2861 		if (prev_channel_count > new_channel_count)
2862 			ena_xdp_exchange_program_rx_in_range(adapter,
2863 							     NULL,
2864 							     new_channel_count,
2865 							     prev_channel_count);
2866 		else
2867 			ena_xdp_exchange_program_rx_in_range(adapter,
2868 							     adapter->xdp_bpf_prog,
2869 							     prev_channel_count,
2870 							     new_channel_count);
2871 	}
2872 
2873 	/* We need to destroy the rss table so that the indirection
2874 	 * table will be reinitialized by ena_up()
2875 	 */
2876 	ena_com_rss_destroy(ena_dev);
2877 	ena_init_io_rings(adapter,
2878 			  0,
2879 			  adapter->xdp_num_queues +
2880 			  adapter->num_io_queues);
2881 	return dev_was_up ? ena_open(adapter->netdev) : 0;
2882 }
2883 
2884 static void ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx,
2885 			struct sk_buff *skb,
2886 			bool disable_meta_caching)
2887 {
2888 	u32 mss = skb_shinfo(skb)->gso_size;
2889 	struct ena_com_tx_meta *ena_meta = &ena_tx_ctx->ena_meta;
2890 	u8 l4_protocol = 0;
2891 
2892 	if ((skb->ip_summed == CHECKSUM_PARTIAL) || mss) {
2893 		ena_tx_ctx->l4_csum_enable = 1;
2894 		if (mss) {
2895 			ena_tx_ctx->tso_enable = 1;
2896 			ena_meta->l4_hdr_len = tcp_hdr(skb)->doff;
2897 			ena_tx_ctx->l4_csum_partial = 0;
2898 		} else {
2899 			ena_tx_ctx->tso_enable = 0;
2900 			ena_meta->l4_hdr_len = 0;
2901 			ena_tx_ctx->l4_csum_partial = 1;
2902 		}
2903 
2904 		switch (ip_hdr(skb)->version) {
2905 		case IPVERSION:
2906 			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
2907 			if (ip_hdr(skb)->frag_off & htons(IP_DF))
2908 				ena_tx_ctx->df = 1;
2909 			if (mss)
2910 				ena_tx_ctx->l3_csum_enable = 1;
2911 			l4_protocol = ip_hdr(skb)->protocol;
2912 			break;
2913 		case 6:
2914 			ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
2915 			l4_protocol = ipv6_hdr(skb)->nexthdr;
2916 			break;
2917 		default:
2918 			break;
2919 		}
2920 
2921 		if (l4_protocol == IPPROTO_TCP)
2922 			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
2923 		else
2924 			ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
2925 
2926 		ena_meta->mss = mss;
2927 		ena_meta->l3_hdr_len = skb_network_header_len(skb);
2928 		ena_meta->l3_hdr_offset = skb_network_offset(skb);
2929 		ena_tx_ctx->meta_valid = 1;
2930 	} else if (disable_meta_caching) {
2931 		memset(ena_meta, 0, sizeof(*ena_meta));
2932 		ena_tx_ctx->meta_valid = 1;
2933 	} else {
2934 		ena_tx_ctx->meta_valid = 0;
2935 	}
2936 }
2937 
2938 static int ena_check_and_linearize_skb(struct ena_ring *tx_ring,
2939 				       struct sk_buff *skb)
2940 {
2941 	int num_frags, header_len, rc;
2942 
2943 	num_frags = skb_shinfo(skb)->nr_frags;
2944 	header_len = skb_headlen(skb);
2945 
2946 	if (num_frags < tx_ring->sgl_size)
2947 		return 0;
2948 
2949 	if ((num_frags == tx_ring->sgl_size) &&
2950 	    (header_len < tx_ring->tx_max_header_size))
2951 		return 0;
2952 
2953 	ena_increase_stat(&tx_ring->tx_stats.linearize, 1, &tx_ring->syncp);
2954 
2955 	rc = skb_linearize(skb);
2956 	if (unlikely(rc)) {
2957 		ena_increase_stat(&tx_ring->tx_stats.linearize_failed, 1,
2958 				  &tx_ring->syncp);
2959 	}
2960 
2961 	return rc;
2962 }
2963 
2964 static int ena_tx_map_skb(struct ena_ring *tx_ring,
2965 			  struct ena_tx_buffer *tx_info,
2966 			  struct sk_buff *skb,
2967 			  void **push_hdr,
2968 			  u16 *header_len)
2969 {
2970 	struct ena_adapter *adapter = tx_ring->adapter;
2971 	struct ena_com_buf *ena_buf;
2972 	dma_addr_t dma;
2973 	u32 skb_head_len, frag_len, last_frag;
2974 	u16 push_len = 0;
2975 	u16 delta = 0;
2976 	int i = 0;
2977 
2978 	skb_head_len = skb_headlen(skb);
2979 	tx_info->skb = skb;
2980 	ena_buf = tx_info->bufs;
2981 
2982 	if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
2983 		/* When the device is LLQ mode, the driver will copy
2984 		 * the header into the device memory space.
2985 		 * the ena_com layer assume the header is in a linear
2986 		 * memory space.
2987 		 * This assumption might be wrong since part of the header
2988 		 * can be in the fragmented buffers.
2989 		 * Use skb_header_pointer to make sure the header is in a
2990 		 * linear memory space.
2991 		 */
2992 
2993 		push_len = min_t(u32, skb->len, tx_ring->tx_max_header_size);
2994 		*push_hdr = skb_header_pointer(skb, 0, push_len,
2995 					       tx_ring->push_buf_intermediate_buf);
2996 		*header_len = push_len;
2997 		if (unlikely(skb->data != *push_hdr)) {
2998 			ena_increase_stat(&tx_ring->tx_stats.llq_buffer_copy, 1,
2999 					  &tx_ring->syncp);
3000 
3001 			delta = push_len - skb_head_len;
3002 		}
3003 	} else {
3004 		*push_hdr = NULL;
3005 		*header_len = min_t(u32, skb_head_len,
3006 				    tx_ring->tx_max_header_size);
3007 	}
3008 
3009 	netif_dbg(adapter, tx_queued, adapter->netdev,
3010 		  "skb: %p header_buf->vaddr: %p push_len: %d\n", skb,
3011 		  *push_hdr, push_len);
3012 
3013 	if (skb_head_len > push_len) {
3014 		dma = dma_map_single(tx_ring->dev, skb->data + push_len,
3015 				     skb_head_len - push_len, DMA_TO_DEVICE);
3016 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
3017 			goto error_report_dma_error;
3018 
3019 		ena_buf->paddr = dma;
3020 		ena_buf->len = skb_head_len - push_len;
3021 
3022 		ena_buf++;
3023 		tx_info->num_of_bufs++;
3024 		tx_info->map_linear_data = 1;
3025 	} else {
3026 		tx_info->map_linear_data = 0;
3027 	}
3028 
3029 	last_frag = skb_shinfo(skb)->nr_frags;
3030 
3031 	for (i = 0; i < last_frag; i++) {
3032 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3033 
3034 		frag_len = skb_frag_size(frag);
3035 
3036 		if (unlikely(delta >= frag_len)) {
3037 			delta -= frag_len;
3038 			continue;
3039 		}
3040 
3041 		dma = skb_frag_dma_map(tx_ring->dev, frag, delta,
3042 				       frag_len - delta, DMA_TO_DEVICE);
3043 		if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
3044 			goto error_report_dma_error;
3045 
3046 		ena_buf->paddr = dma;
3047 		ena_buf->len = frag_len - delta;
3048 		ena_buf++;
3049 		tx_info->num_of_bufs++;
3050 		delta = 0;
3051 	}
3052 
3053 	return 0;
3054 
3055 error_report_dma_error:
3056 	ena_increase_stat(&tx_ring->tx_stats.dma_mapping_err, 1,
3057 			  &tx_ring->syncp);
3058 	netif_warn(adapter, tx_queued, adapter->netdev, "Failed to map skb\n");
3059 
3060 	tx_info->skb = NULL;
3061 
3062 	tx_info->num_of_bufs += i;
3063 	ena_unmap_tx_buff(tx_ring, tx_info);
3064 
3065 	return -EINVAL;
3066 }
3067 
3068 /* Called with netif_tx_lock. */
3069 static netdev_tx_t ena_start_xmit(struct sk_buff *skb, struct net_device *dev)
3070 {
3071 	struct ena_adapter *adapter = netdev_priv(dev);
3072 	struct ena_tx_buffer *tx_info;
3073 	struct ena_com_tx_ctx ena_tx_ctx;
3074 	struct ena_ring *tx_ring;
3075 	struct netdev_queue *txq;
3076 	void *push_hdr;
3077 	u16 next_to_use, req_id, header_len;
3078 	int qid, rc;
3079 
3080 	netif_dbg(adapter, tx_queued, dev, "%s skb %p\n", __func__, skb);
3081 	/*  Determine which tx ring we will be placed on */
3082 	qid = skb_get_queue_mapping(skb);
3083 	tx_ring = &adapter->tx_ring[qid];
3084 	txq = netdev_get_tx_queue(dev, qid);
3085 
3086 	rc = ena_check_and_linearize_skb(tx_ring, skb);
3087 	if (unlikely(rc))
3088 		goto error_drop_packet;
3089 
3090 	skb_tx_timestamp(skb);
3091 
3092 	next_to_use = tx_ring->next_to_use;
3093 	req_id = tx_ring->free_ids[next_to_use];
3094 	tx_info = &tx_ring->tx_buffer_info[req_id];
3095 	tx_info->num_of_bufs = 0;
3096 
3097 	WARN(tx_info->skb, "SKB isn't NULL req_id %d\n", req_id);
3098 
3099 	rc = ena_tx_map_skb(tx_ring, tx_info, skb, &push_hdr, &header_len);
3100 	if (unlikely(rc))
3101 		goto error_drop_packet;
3102 
3103 	memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
3104 	ena_tx_ctx.ena_bufs = tx_info->bufs;
3105 	ena_tx_ctx.push_header = push_hdr;
3106 	ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
3107 	ena_tx_ctx.req_id = req_id;
3108 	ena_tx_ctx.header_len = header_len;
3109 
3110 	/* set flags and meta data */
3111 	ena_tx_csum(&ena_tx_ctx, skb, tx_ring->disable_meta_caching);
3112 
3113 	rc = ena_xmit_common(dev,
3114 			     tx_ring,
3115 			     tx_info,
3116 			     &ena_tx_ctx,
3117 			     next_to_use,
3118 			     skb->len);
3119 	if (rc)
3120 		goto error_unmap_dma;
3121 
3122 	netdev_tx_sent_queue(txq, skb->len);
3123 
3124 	/* stop the queue when no more space available, the packet can have up
3125 	 * to sgl_size + 2. one for the meta descriptor and one for header
3126 	 * (if the header is larger than tx_max_header_size).
3127 	 */
3128 	if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
3129 						   tx_ring->sgl_size + 2))) {
3130 		netif_dbg(adapter, tx_queued, dev, "%s stop queue %d\n",
3131 			  __func__, qid);
3132 
3133 		netif_tx_stop_queue(txq);
3134 		ena_increase_stat(&tx_ring->tx_stats.queue_stop, 1,
3135 				  &tx_ring->syncp);
3136 
3137 		/* There is a rare condition where this function decide to
3138 		 * stop the queue but meanwhile clean_tx_irq updates
3139 		 * next_to_completion and terminates.
3140 		 * The queue will remain stopped forever.
3141 		 * To solve this issue add a mb() to make sure that
3142 		 * netif_tx_stop_queue() write is vissible before checking if
3143 		 * there is additional space in the queue.
3144 		 */
3145 		smp_mb();
3146 
3147 		if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
3148 						 ENA_TX_WAKEUP_THRESH)) {
3149 			netif_tx_wake_queue(txq);
3150 			ena_increase_stat(&tx_ring->tx_stats.queue_wakeup, 1,
3151 					  &tx_ring->syncp);
3152 		}
3153 	}
3154 
3155 	if (netif_xmit_stopped(txq) || !netdev_xmit_more())
3156 		/* trigger the dma engine. ena_ring_tx_doorbell()
3157 		 * calls a memory barrier inside it.
3158 		 */
3159 		ena_ring_tx_doorbell(tx_ring);
3160 
3161 	return NETDEV_TX_OK;
3162 
3163 error_unmap_dma:
3164 	ena_unmap_tx_buff(tx_ring, tx_info);
3165 	tx_info->skb = NULL;
3166 
3167 error_drop_packet:
3168 	dev_kfree_skb(skb);
3169 	return NETDEV_TX_OK;
3170 }
3171 
3172 static u16 ena_select_queue(struct net_device *dev, struct sk_buff *skb,
3173 			    struct net_device *sb_dev)
3174 {
3175 	u16 qid;
3176 	/* we suspect that this is good for in--kernel network services that
3177 	 * want to loop incoming skb rx to tx in normal user generated traffic,
3178 	 * most probably we will not get to this
3179 	 */
3180 	if (skb_rx_queue_recorded(skb))
3181 		qid = skb_get_rx_queue(skb);
3182 	else
3183 		qid = netdev_pick_tx(dev, skb, NULL);
3184 
3185 	return qid;
3186 }
3187 
3188 static void ena_config_host_info(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
3189 {
3190 	struct device *dev = &pdev->dev;
3191 	struct ena_admin_host_info *host_info;
3192 	int rc;
3193 
3194 	/* Allocate only the host info */
3195 	rc = ena_com_allocate_host_info(ena_dev);
3196 	if (rc) {
3197 		dev_err(dev, "Cannot allocate host info\n");
3198 		return;
3199 	}
3200 
3201 	host_info = ena_dev->host_attr.host_info;
3202 
3203 	host_info->bdf = (pdev->bus->number << 8) | pdev->devfn;
3204 	host_info->os_type = ENA_ADMIN_OS_LINUX;
3205 	host_info->kernel_ver = LINUX_VERSION_CODE;
3206 	strscpy(host_info->kernel_ver_str, utsname()->version,
3207 		sizeof(host_info->kernel_ver_str) - 1);
3208 	host_info->os_dist = 0;
3209 	strncpy(host_info->os_dist_str, utsname()->release,
3210 		sizeof(host_info->os_dist_str) - 1);
3211 	host_info->driver_version =
3212 		(DRV_MODULE_GEN_MAJOR) |
3213 		(DRV_MODULE_GEN_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
3214 		(DRV_MODULE_GEN_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT) |
3215 		("K"[0] << ENA_ADMIN_HOST_INFO_MODULE_TYPE_SHIFT);
3216 	host_info->num_cpus = num_online_cpus();
3217 
3218 	host_info->driver_supported_features =
3219 		ENA_ADMIN_HOST_INFO_RX_OFFSET_MASK |
3220 		ENA_ADMIN_HOST_INFO_INTERRUPT_MODERATION_MASK |
3221 		ENA_ADMIN_HOST_INFO_RX_BUF_MIRRORING_MASK |
3222 		ENA_ADMIN_HOST_INFO_RSS_CONFIGURABLE_FUNCTION_KEY_MASK;
3223 
3224 	rc = ena_com_set_host_attributes(ena_dev);
3225 	if (rc) {
3226 		if (rc == -EOPNOTSUPP)
3227 			dev_warn(dev, "Cannot set host attributes\n");
3228 		else
3229 			dev_err(dev, "Cannot set host attributes\n");
3230 
3231 		goto err;
3232 	}
3233 
3234 	return;
3235 
3236 err:
3237 	ena_com_delete_host_info(ena_dev);
3238 }
3239 
3240 static void ena_config_debug_area(struct ena_adapter *adapter)
3241 {
3242 	u32 debug_area_size;
3243 	int rc, ss_count;
3244 
3245 	ss_count = ena_get_sset_count(adapter->netdev, ETH_SS_STATS);
3246 	if (ss_count <= 0) {
3247 		netif_err(adapter, drv, adapter->netdev,
3248 			  "SS count is negative\n");
3249 		return;
3250 	}
3251 
3252 	/* allocate 32 bytes for each string and 64bit for the value */
3253 	debug_area_size = ss_count * ETH_GSTRING_LEN + sizeof(u64) * ss_count;
3254 
3255 	rc = ena_com_allocate_debug_area(adapter->ena_dev, debug_area_size);
3256 	if (rc) {
3257 		netif_err(adapter, drv, adapter->netdev,
3258 			  "Cannot allocate debug area\n");
3259 		return;
3260 	}
3261 
3262 	rc = ena_com_set_host_attributes(adapter->ena_dev);
3263 	if (rc) {
3264 		if (rc == -EOPNOTSUPP)
3265 			netif_warn(adapter, drv, adapter->netdev,
3266 				   "Cannot set host attributes\n");
3267 		else
3268 			netif_err(adapter, drv, adapter->netdev,
3269 				  "Cannot set host attributes\n");
3270 		goto err;
3271 	}
3272 
3273 	return;
3274 err:
3275 	ena_com_delete_debug_area(adapter->ena_dev);
3276 }
3277 
3278 int ena_update_hw_stats(struct ena_adapter *adapter)
3279 {
3280 	int rc;
3281 
3282 	rc = ena_com_get_eni_stats(adapter->ena_dev, &adapter->eni_stats);
3283 	if (rc) {
3284 		netdev_err(adapter->netdev, "Failed to get ENI stats\n");
3285 		return rc;
3286 	}
3287 
3288 	return 0;
3289 }
3290 
3291 static void ena_get_stats64(struct net_device *netdev,
3292 			    struct rtnl_link_stats64 *stats)
3293 {
3294 	struct ena_adapter *adapter = netdev_priv(netdev);
3295 	struct ena_ring *rx_ring, *tx_ring;
3296 	unsigned int start;
3297 	u64 rx_drops;
3298 	u64 tx_drops;
3299 	int i;
3300 
3301 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3302 		return;
3303 
3304 	for (i = 0; i < adapter->num_io_queues; i++) {
3305 		u64 bytes, packets;
3306 
3307 		tx_ring = &adapter->tx_ring[i];
3308 
3309 		do {
3310 			start = u64_stats_fetch_begin(&tx_ring->syncp);
3311 			packets = tx_ring->tx_stats.cnt;
3312 			bytes = tx_ring->tx_stats.bytes;
3313 		} while (u64_stats_fetch_retry(&tx_ring->syncp, start));
3314 
3315 		stats->tx_packets += packets;
3316 		stats->tx_bytes += bytes;
3317 
3318 		rx_ring = &adapter->rx_ring[i];
3319 
3320 		do {
3321 			start = u64_stats_fetch_begin(&rx_ring->syncp);
3322 			packets = rx_ring->rx_stats.cnt;
3323 			bytes = rx_ring->rx_stats.bytes;
3324 		} while (u64_stats_fetch_retry(&rx_ring->syncp, start));
3325 
3326 		stats->rx_packets += packets;
3327 		stats->rx_bytes += bytes;
3328 	}
3329 
3330 	do {
3331 		start = u64_stats_fetch_begin(&adapter->syncp);
3332 		rx_drops = adapter->dev_stats.rx_drops;
3333 		tx_drops = adapter->dev_stats.tx_drops;
3334 	} while (u64_stats_fetch_retry(&adapter->syncp, start));
3335 
3336 	stats->rx_dropped = rx_drops;
3337 	stats->tx_dropped = tx_drops;
3338 
3339 	stats->multicast = 0;
3340 	stats->collisions = 0;
3341 
3342 	stats->rx_length_errors = 0;
3343 	stats->rx_crc_errors = 0;
3344 	stats->rx_frame_errors = 0;
3345 	stats->rx_fifo_errors = 0;
3346 	stats->rx_missed_errors = 0;
3347 	stats->tx_window_errors = 0;
3348 
3349 	stats->rx_errors = 0;
3350 	stats->tx_errors = 0;
3351 }
3352 
3353 static const struct net_device_ops ena_netdev_ops = {
3354 	.ndo_open		= ena_open,
3355 	.ndo_stop		= ena_close,
3356 	.ndo_start_xmit		= ena_start_xmit,
3357 	.ndo_select_queue	= ena_select_queue,
3358 	.ndo_get_stats64	= ena_get_stats64,
3359 	.ndo_tx_timeout		= ena_tx_timeout,
3360 	.ndo_change_mtu		= ena_change_mtu,
3361 	.ndo_set_mac_address	= NULL,
3362 	.ndo_validate_addr	= eth_validate_addr,
3363 	.ndo_bpf		= ena_xdp,
3364 	.ndo_xdp_xmit		= ena_xdp_xmit,
3365 };
3366 
3367 static int ena_device_validate_params(struct ena_adapter *adapter,
3368 				      struct ena_com_dev_get_features_ctx *get_feat_ctx)
3369 {
3370 	struct net_device *netdev = adapter->netdev;
3371 	int rc;
3372 
3373 	rc = ether_addr_equal(get_feat_ctx->dev_attr.mac_addr,
3374 			      adapter->mac_addr);
3375 	if (!rc) {
3376 		netif_err(adapter, drv, netdev,
3377 			  "Error, mac address are different\n");
3378 		return -EINVAL;
3379 	}
3380 
3381 	if (get_feat_ctx->dev_attr.max_mtu < netdev->mtu) {
3382 		netif_err(adapter, drv, netdev,
3383 			  "Error, device max mtu is smaller than netdev MTU\n");
3384 		return -EINVAL;
3385 	}
3386 
3387 	return 0;
3388 }
3389 
3390 static void set_default_llq_configurations(struct ena_llq_configurations *llq_config)
3391 {
3392 	llq_config->llq_header_location = ENA_ADMIN_INLINE_HEADER;
3393 	llq_config->llq_stride_ctrl = ENA_ADMIN_MULTIPLE_DESCS_PER_ENTRY;
3394 	llq_config->llq_num_decs_before_header = ENA_ADMIN_LLQ_NUM_DESCS_BEFORE_HEADER_2;
3395 	llq_config->llq_ring_entry_size = ENA_ADMIN_LIST_ENTRY_SIZE_128B;
3396 	llq_config->llq_ring_entry_size_value = 128;
3397 }
3398 
3399 static int ena_set_queues_placement_policy(struct pci_dev *pdev,
3400 					   struct ena_com_dev *ena_dev,
3401 					   struct ena_admin_feature_llq_desc *llq,
3402 					   struct ena_llq_configurations *llq_default_configurations)
3403 {
3404 	int rc;
3405 	u32 llq_feature_mask;
3406 
3407 	llq_feature_mask = 1 << ENA_ADMIN_LLQ;
3408 	if (!(ena_dev->supported_features & llq_feature_mask)) {
3409 		dev_warn(&pdev->dev,
3410 			"LLQ is not supported Fallback to host mode policy.\n");
3411 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3412 		return 0;
3413 	}
3414 
3415 	rc = ena_com_config_dev_mode(ena_dev, llq, llq_default_configurations);
3416 	if (unlikely(rc)) {
3417 		dev_err(&pdev->dev,
3418 			"Failed to configure the device mode.  Fallback to host mode policy.\n");
3419 		ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3420 	}
3421 
3422 	return 0;
3423 }
3424 
3425 static int ena_map_llq_mem_bar(struct pci_dev *pdev, struct ena_com_dev *ena_dev,
3426 			       int bars)
3427 {
3428 	bool has_mem_bar = !!(bars & BIT(ENA_MEM_BAR));
3429 
3430 	if (!has_mem_bar) {
3431 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
3432 			dev_err(&pdev->dev,
3433 				"ENA device does not expose LLQ bar. Fallback to host mode policy.\n");
3434 			ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3435 		}
3436 
3437 		return 0;
3438 	}
3439 
3440 	ena_dev->mem_bar = devm_ioremap_wc(&pdev->dev,
3441 					   pci_resource_start(pdev, ENA_MEM_BAR),
3442 					   pci_resource_len(pdev, ENA_MEM_BAR));
3443 
3444 	if (!ena_dev->mem_bar)
3445 		return -EFAULT;
3446 
3447 	return 0;
3448 }
3449 
3450 static int ena_device_init(struct ena_com_dev *ena_dev, struct pci_dev *pdev,
3451 			   struct ena_com_dev_get_features_ctx *get_feat_ctx,
3452 			   bool *wd_state)
3453 {
3454 	struct ena_llq_configurations llq_config;
3455 	struct device *dev = &pdev->dev;
3456 	bool readless_supported;
3457 	u32 aenq_groups;
3458 	int dma_width;
3459 	int rc;
3460 
3461 	rc = ena_com_mmio_reg_read_request_init(ena_dev);
3462 	if (rc) {
3463 		dev_err(dev, "Failed to init mmio read less\n");
3464 		return rc;
3465 	}
3466 
3467 	/* The PCIe configuration space revision id indicate if mmio reg
3468 	 * read is disabled
3469 	 */
3470 	readless_supported = !(pdev->revision & ENA_MMIO_DISABLE_REG_READ);
3471 	ena_com_set_mmio_read_mode(ena_dev, readless_supported);
3472 
3473 	rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
3474 	if (rc) {
3475 		dev_err(dev, "Can not reset device\n");
3476 		goto err_mmio_read_less;
3477 	}
3478 
3479 	rc = ena_com_validate_version(ena_dev);
3480 	if (rc) {
3481 		dev_err(dev, "Device version is too low\n");
3482 		goto err_mmio_read_less;
3483 	}
3484 
3485 	dma_width = ena_com_get_dma_width(ena_dev);
3486 	if (dma_width < 0) {
3487 		dev_err(dev, "Invalid dma width value %d", dma_width);
3488 		rc = dma_width;
3489 		goto err_mmio_read_less;
3490 	}
3491 
3492 	rc = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(dma_width));
3493 	if (rc) {
3494 		dev_err(dev, "dma_set_mask_and_coherent failed %d\n", rc);
3495 		goto err_mmio_read_less;
3496 	}
3497 
3498 	/* ENA admin level init */
3499 	rc = ena_com_admin_init(ena_dev, &aenq_handlers);
3500 	if (rc) {
3501 		dev_err(dev,
3502 			"Can not initialize ena admin queue with device\n");
3503 		goto err_mmio_read_less;
3504 	}
3505 
3506 	/* To enable the msix interrupts the driver needs to know the number
3507 	 * of queues. So the driver uses polling mode to retrieve this
3508 	 * information
3509 	 */
3510 	ena_com_set_admin_polling_mode(ena_dev, true);
3511 
3512 	ena_config_host_info(ena_dev, pdev);
3513 
3514 	/* Get Device Attributes*/
3515 	rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3516 	if (rc) {
3517 		dev_err(dev, "Cannot get attribute for ena device rc=%d\n", rc);
3518 		goto err_admin_init;
3519 	}
3520 
3521 	/* Try to turn all the available aenq groups */
3522 	aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) |
3523 		BIT(ENA_ADMIN_FATAL_ERROR) |
3524 		BIT(ENA_ADMIN_WARNING) |
3525 		BIT(ENA_ADMIN_NOTIFICATION) |
3526 		BIT(ENA_ADMIN_KEEP_ALIVE);
3527 
3528 	aenq_groups &= get_feat_ctx->aenq.supported_groups;
3529 
3530 	rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3531 	if (rc) {
3532 		dev_err(dev, "Cannot configure aenq groups rc= %d\n", rc);
3533 		goto err_admin_init;
3534 	}
3535 
3536 	*wd_state = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3537 
3538 	set_default_llq_configurations(&llq_config);
3539 
3540 	rc = ena_set_queues_placement_policy(pdev, ena_dev, &get_feat_ctx->llq,
3541 					     &llq_config);
3542 	if (rc) {
3543 		dev_err(dev, "ENA device init failed\n");
3544 		goto err_admin_init;
3545 	}
3546 
3547 	return 0;
3548 
3549 err_admin_init:
3550 	ena_com_delete_host_info(ena_dev);
3551 	ena_com_admin_destroy(ena_dev);
3552 err_mmio_read_less:
3553 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3554 
3555 	return rc;
3556 }
3557 
3558 static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter)
3559 {
3560 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3561 	struct device *dev = &adapter->pdev->dev;
3562 	int rc;
3563 
3564 	rc = ena_enable_msix(adapter);
3565 	if (rc) {
3566 		dev_err(dev, "Can not reserve msix vectors\n");
3567 		return rc;
3568 	}
3569 
3570 	ena_setup_mgmnt_intr(adapter);
3571 
3572 	rc = ena_request_mgmnt_irq(adapter);
3573 	if (rc) {
3574 		dev_err(dev, "Can not setup management interrupts\n");
3575 		goto err_disable_msix;
3576 	}
3577 
3578 	ena_com_set_admin_polling_mode(ena_dev, false);
3579 
3580 	ena_com_admin_aenq_enable(ena_dev);
3581 
3582 	return 0;
3583 
3584 err_disable_msix:
3585 	ena_disable_msix(adapter);
3586 
3587 	return rc;
3588 }
3589 
3590 static void ena_destroy_device(struct ena_adapter *adapter, bool graceful)
3591 {
3592 	struct net_device *netdev = adapter->netdev;
3593 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3594 	bool dev_up;
3595 
3596 	if (!test_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags))
3597 		return;
3598 
3599 	netif_carrier_off(netdev);
3600 
3601 	del_timer_sync(&adapter->timer_service);
3602 
3603 	dev_up = test_bit(ENA_FLAG_DEV_UP, &adapter->flags);
3604 	adapter->dev_up_before_reset = dev_up;
3605 	if (!graceful)
3606 		ena_com_set_admin_running_state(ena_dev, false);
3607 
3608 	if (test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3609 		ena_down(adapter);
3610 
3611 	/* Stop the device from sending AENQ events (in case reset flag is set
3612 	 *  and device is up, ena_down() already reset the device.
3613 	 */
3614 	if (!(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags) && dev_up))
3615 		ena_com_dev_reset(adapter->ena_dev, adapter->reset_reason);
3616 
3617 	ena_free_mgmnt_irq(adapter);
3618 
3619 	ena_disable_msix(adapter);
3620 
3621 	ena_com_abort_admin_commands(ena_dev);
3622 
3623 	ena_com_wait_for_abort_completion(ena_dev);
3624 
3625 	ena_com_admin_destroy(ena_dev);
3626 
3627 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3628 
3629 	/* return reset reason to default value */
3630 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3631 
3632 	clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
3633 	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3634 }
3635 
3636 static int ena_restore_device(struct ena_adapter *adapter)
3637 {
3638 	struct ena_com_dev_get_features_ctx get_feat_ctx;
3639 	struct ena_com_dev *ena_dev = adapter->ena_dev;
3640 	struct pci_dev *pdev = adapter->pdev;
3641 	bool wd_state;
3642 	int rc;
3643 
3644 	set_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3645 	rc = ena_device_init(ena_dev, adapter->pdev, &get_feat_ctx, &wd_state);
3646 	if (rc) {
3647 		dev_err(&pdev->dev, "Can not initialize device\n");
3648 		goto err;
3649 	}
3650 	adapter->wd_state = wd_state;
3651 
3652 	rc = ena_device_validate_params(adapter, &get_feat_ctx);
3653 	if (rc) {
3654 		dev_err(&pdev->dev, "Validation of device parameters failed\n");
3655 		goto err_device_destroy;
3656 	}
3657 
3658 	rc = ena_enable_msix_and_set_admin_interrupts(adapter);
3659 	if (rc) {
3660 		dev_err(&pdev->dev, "Enable MSI-X failed\n");
3661 		goto err_device_destroy;
3662 	}
3663 	/* If the interface was up before the reset bring it up */
3664 	if (adapter->dev_up_before_reset) {
3665 		rc = ena_up(adapter);
3666 		if (rc) {
3667 			dev_err(&pdev->dev, "Failed to create I/O queues\n");
3668 			goto err_disable_msix;
3669 		}
3670 	}
3671 
3672 	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3673 
3674 	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3675 	if (test_bit(ENA_FLAG_LINK_UP, &adapter->flags))
3676 		netif_carrier_on(adapter->netdev);
3677 
3678 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
3679 	adapter->last_keep_alive_jiffies = jiffies;
3680 
3681 	return rc;
3682 err_disable_msix:
3683 	ena_free_mgmnt_irq(adapter);
3684 	ena_disable_msix(adapter);
3685 err_device_destroy:
3686 	ena_com_abort_admin_commands(ena_dev);
3687 	ena_com_wait_for_abort_completion(ena_dev);
3688 	ena_com_admin_destroy(ena_dev);
3689 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_DRIVER_INVALID_STATE);
3690 	ena_com_mmio_reg_read_request_destroy(ena_dev);
3691 err:
3692 	clear_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
3693 	clear_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags);
3694 	dev_err(&pdev->dev,
3695 		"Reset attempt failed. Can not reset the device\n");
3696 
3697 	return rc;
3698 }
3699 
3700 static void ena_fw_reset_device(struct work_struct *work)
3701 {
3702 	struct ena_adapter *adapter =
3703 		container_of(work, struct ena_adapter, reset_task);
3704 
3705 	rtnl_lock();
3706 
3707 	if (likely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
3708 		ena_destroy_device(adapter, false);
3709 		ena_restore_device(adapter);
3710 
3711 		dev_err(&adapter->pdev->dev, "Device reset completed successfully\n");
3712 	}
3713 
3714 	rtnl_unlock();
3715 }
3716 
3717 static int check_for_rx_interrupt_queue(struct ena_adapter *adapter,
3718 					struct ena_ring *rx_ring)
3719 {
3720 	struct ena_napi *ena_napi = container_of(rx_ring->napi, struct ena_napi, napi);
3721 
3722 	if (likely(READ_ONCE(ena_napi->first_interrupt)))
3723 		return 0;
3724 
3725 	if (ena_com_cq_empty(rx_ring->ena_com_io_cq))
3726 		return 0;
3727 
3728 	rx_ring->no_interrupt_event_cnt++;
3729 
3730 	if (rx_ring->no_interrupt_event_cnt == ENA_MAX_NO_INTERRUPT_ITERATIONS) {
3731 		netif_err(adapter, rx_err, adapter->netdev,
3732 			  "Potential MSIX issue on Rx side Queue = %d. Reset the device\n",
3733 			  rx_ring->qid);
3734 
3735 		ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3736 		return -EIO;
3737 	}
3738 
3739 	return 0;
3740 }
3741 
3742 static int check_missing_comp_in_tx_queue(struct ena_adapter *adapter,
3743 					  struct ena_ring *tx_ring)
3744 {
3745 	struct ena_napi *ena_napi = container_of(tx_ring->napi, struct ena_napi, napi);
3746 	unsigned int time_since_last_napi;
3747 	unsigned int missing_tx_comp_to;
3748 	bool is_tx_comp_time_expired;
3749 	struct ena_tx_buffer *tx_buf;
3750 	unsigned long last_jiffies;
3751 	u32 missed_tx = 0;
3752 	int i, rc = 0;
3753 
3754 	for (i = 0; i < tx_ring->ring_size; i++) {
3755 		tx_buf = &tx_ring->tx_buffer_info[i];
3756 		last_jiffies = tx_buf->last_jiffies;
3757 
3758 		if (last_jiffies == 0)
3759 			/* no pending Tx at this location */
3760 			continue;
3761 
3762 		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +
3763 			 2 * adapter->missing_tx_completion_to);
3764 
3765 		if (unlikely(!READ_ONCE(ena_napi->first_interrupt) && is_tx_comp_time_expired)) {
3766 			/* If after graceful period interrupt is still not
3767 			 * received, we schedule a reset
3768 			 */
3769 			netif_err(adapter, tx_err, adapter->netdev,
3770 				  "Potential MSIX issue on Tx side Queue = %d. Reset the device\n",
3771 				  tx_ring->qid);
3772 			ena_reset_device(adapter, ENA_REGS_RESET_MISS_INTERRUPT);
3773 			return -EIO;
3774 		}
3775 
3776 		is_tx_comp_time_expired = time_is_before_jiffies(last_jiffies +
3777 			adapter->missing_tx_completion_to);
3778 
3779 		if (unlikely(is_tx_comp_time_expired)) {
3780 			if (!tx_buf->print_once) {
3781 				time_since_last_napi = jiffies_to_usecs(jiffies - tx_ring->tx_stats.last_napi_jiffies);
3782 				missing_tx_comp_to = jiffies_to_msecs(adapter->missing_tx_completion_to);
3783 				netif_notice(adapter, tx_err, adapter->netdev,
3784 					     "Found a Tx that wasn't completed on time, qid %d, index %d. %u usecs have passed since last napi execution. Missing Tx timeout value %u msecs\n",
3785 					     tx_ring->qid, i, time_since_last_napi, missing_tx_comp_to);
3786 			}
3787 
3788 			tx_buf->print_once = 1;
3789 			missed_tx++;
3790 		}
3791 	}
3792 
3793 	if (unlikely(missed_tx > adapter->missing_tx_completion_threshold)) {
3794 		netif_err(adapter, tx_err, adapter->netdev,
3795 			  "The number of lost tx completions is above the threshold (%d > %d). Reset the device\n",
3796 			  missed_tx,
3797 			  adapter->missing_tx_completion_threshold);
3798 		ena_reset_device(adapter, ENA_REGS_RESET_MISS_TX_CMPL);
3799 		rc = -EIO;
3800 	}
3801 
3802 	ena_increase_stat(&tx_ring->tx_stats.missed_tx, missed_tx,
3803 			  &tx_ring->syncp);
3804 
3805 	return rc;
3806 }
3807 
3808 static void check_for_missing_completions(struct ena_adapter *adapter)
3809 {
3810 	struct ena_ring *tx_ring;
3811 	struct ena_ring *rx_ring;
3812 	int i, budget, rc;
3813 	int io_queue_count;
3814 
3815 	io_queue_count = adapter->xdp_num_queues + adapter->num_io_queues;
3816 	/* Make sure the driver doesn't turn the device in other process */
3817 	smp_rmb();
3818 
3819 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3820 		return;
3821 
3822 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3823 		return;
3824 
3825 	if (adapter->missing_tx_completion_to == ENA_HW_HINTS_NO_TIMEOUT)
3826 		return;
3827 
3828 	budget = ENA_MONITORED_TX_QUEUES;
3829 
3830 	for (i = adapter->last_monitored_tx_qid; i < io_queue_count; i++) {
3831 		tx_ring = &adapter->tx_ring[i];
3832 		rx_ring = &adapter->rx_ring[i];
3833 
3834 		rc = check_missing_comp_in_tx_queue(adapter, tx_ring);
3835 		if (unlikely(rc))
3836 			return;
3837 
3838 		rc =  !ENA_IS_XDP_INDEX(adapter, i) ?
3839 			check_for_rx_interrupt_queue(adapter, rx_ring) : 0;
3840 		if (unlikely(rc))
3841 			return;
3842 
3843 		budget--;
3844 		if (!budget)
3845 			break;
3846 	}
3847 
3848 	adapter->last_monitored_tx_qid = i % io_queue_count;
3849 }
3850 
3851 /* trigger napi schedule after 2 consecutive detections */
3852 #define EMPTY_RX_REFILL 2
3853 /* For the rare case where the device runs out of Rx descriptors and the
3854  * napi handler failed to refill new Rx descriptors (due to a lack of memory
3855  * for example).
3856  * This case will lead to a deadlock:
3857  * The device won't send interrupts since all the new Rx packets will be dropped
3858  * The napi handler won't allocate new Rx descriptors so the device will be
3859  * able to send new packets.
3860  *
3861  * This scenario can happen when the kernel's vm.min_free_kbytes is too small.
3862  * It is recommended to have at least 512MB, with a minimum of 128MB for
3863  * constrained environment).
3864  *
3865  * When such a situation is detected - Reschedule napi
3866  */
3867 static void check_for_empty_rx_ring(struct ena_adapter *adapter)
3868 {
3869 	struct ena_ring *rx_ring;
3870 	int i, refill_required;
3871 
3872 	if (!test_bit(ENA_FLAG_DEV_UP, &adapter->flags))
3873 		return;
3874 
3875 	if (test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))
3876 		return;
3877 
3878 	for (i = 0; i < adapter->num_io_queues; i++) {
3879 		rx_ring = &adapter->rx_ring[i];
3880 
3881 		refill_required = ena_com_free_q_entries(rx_ring->ena_com_io_sq);
3882 		if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
3883 			rx_ring->empty_rx_queue++;
3884 
3885 			if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
3886 				ena_increase_stat(&rx_ring->rx_stats.empty_rx_ring, 1,
3887 						  &rx_ring->syncp);
3888 
3889 				netif_err(adapter, drv, adapter->netdev,
3890 					  "Trigger refill for ring %d\n", i);
3891 
3892 				napi_schedule(rx_ring->napi);
3893 				rx_ring->empty_rx_queue = 0;
3894 			}
3895 		} else {
3896 			rx_ring->empty_rx_queue = 0;
3897 		}
3898 	}
3899 }
3900 
3901 /* Check for keep alive expiration */
3902 static void check_for_missing_keep_alive(struct ena_adapter *adapter)
3903 {
3904 	unsigned long keep_alive_expired;
3905 
3906 	if (!adapter->wd_state)
3907 		return;
3908 
3909 	if (adapter->keep_alive_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3910 		return;
3911 
3912 	keep_alive_expired = adapter->last_keep_alive_jiffies +
3913 			     adapter->keep_alive_timeout;
3914 	if (unlikely(time_is_before_jiffies(keep_alive_expired))) {
3915 		netif_err(adapter, drv, adapter->netdev,
3916 			  "Keep alive watchdog timeout.\n");
3917 		ena_increase_stat(&adapter->dev_stats.wd_expired, 1,
3918 				  &adapter->syncp);
3919 		ena_reset_device(adapter, ENA_REGS_RESET_KEEP_ALIVE_TO);
3920 	}
3921 }
3922 
3923 static void check_for_admin_com_state(struct ena_adapter *adapter)
3924 {
3925 	if (unlikely(!ena_com_get_admin_running_state(adapter->ena_dev))) {
3926 		netif_err(adapter, drv, adapter->netdev,
3927 			  "ENA admin queue is not in running state!\n");
3928 		ena_increase_stat(&adapter->dev_stats.admin_q_pause, 1,
3929 				  &adapter->syncp);
3930 		ena_reset_device(adapter, ENA_REGS_RESET_ADMIN_TO);
3931 	}
3932 }
3933 
3934 static void ena_update_hints(struct ena_adapter *adapter,
3935 			     struct ena_admin_ena_hw_hints *hints)
3936 {
3937 	struct net_device *netdev = adapter->netdev;
3938 
3939 	if (hints->admin_completion_tx_timeout)
3940 		adapter->ena_dev->admin_queue.completion_timeout =
3941 			hints->admin_completion_tx_timeout * 1000;
3942 
3943 	if (hints->mmio_read_timeout)
3944 		/* convert to usec */
3945 		adapter->ena_dev->mmio_read.reg_read_to =
3946 			hints->mmio_read_timeout * 1000;
3947 
3948 	if (hints->missed_tx_completion_count_threshold_to_reset)
3949 		adapter->missing_tx_completion_threshold =
3950 			hints->missed_tx_completion_count_threshold_to_reset;
3951 
3952 	if (hints->missing_tx_completion_timeout) {
3953 		if (hints->missing_tx_completion_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3954 			adapter->missing_tx_completion_to = ENA_HW_HINTS_NO_TIMEOUT;
3955 		else
3956 			adapter->missing_tx_completion_to =
3957 				msecs_to_jiffies(hints->missing_tx_completion_timeout);
3958 	}
3959 
3960 	if (hints->netdev_wd_timeout)
3961 		netdev->watchdog_timeo = msecs_to_jiffies(hints->netdev_wd_timeout);
3962 
3963 	if (hints->driver_watchdog_timeout) {
3964 		if (hints->driver_watchdog_timeout == ENA_HW_HINTS_NO_TIMEOUT)
3965 			adapter->keep_alive_timeout = ENA_HW_HINTS_NO_TIMEOUT;
3966 		else
3967 			adapter->keep_alive_timeout =
3968 				msecs_to_jiffies(hints->driver_watchdog_timeout);
3969 	}
3970 }
3971 
3972 static void ena_update_host_info(struct ena_admin_host_info *host_info,
3973 				 struct net_device *netdev)
3974 {
3975 	host_info->supported_network_features[0] =
3976 		netdev->features & GENMASK_ULL(31, 0);
3977 	host_info->supported_network_features[1] =
3978 		(netdev->features & GENMASK_ULL(63, 32)) >> 32;
3979 }
3980 
3981 static void ena_timer_service(struct timer_list *t)
3982 {
3983 	struct ena_adapter *adapter = from_timer(adapter, t, timer_service);
3984 	u8 *debug_area = adapter->ena_dev->host_attr.debug_area_virt_addr;
3985 	struct ena_admin_host_info *host_info =
3986 		adapter->ena_dev->host_attr.host_info;
3987 
3988 	check_for_missing_keep_alive(adapter);
3989 
3990 	check_for_admin_com_state(adapter);
3991 
3992 	check_for_missing_completions(adapter);
3993 
3994 	check_for_empty_rx_ring(adapter);
3995 
3996 	if (debug_area)
3997 		ena_dump_stats_to_buf(adapter, debug_area);
3998 
3999 	if (host_info)
4000 		ena_update_host_info(host_info, adapter->netdev);
4001 
4002 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
4003 		netif_err(adapter, drv, adapter->netdev,
4004 			  "Trigger reset is on\n");
4005 		ena_dump_stats_to_dmesg(adapter);
4006 		queue_work(ena_wq, &adapter->reset_task);
4007 		return;
4008 	}
4009 
4010 	/* Reset the timer */
4011 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
4012 }
4013 
4014 static u32 ena_calc_max_io_queue_num(struct pci_dev *pdev,
4015 				     struct ena_com_dev *ena_dev,
4016 				     struct ena_com_dev_get_features_ctx *get_feat_ctx)
4017 {
4018 	u32 io_tx_sq_num, io_tx_cq_num, io_rx_num, max_num_io_queues;
4019 
4020 	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
4021 		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
4022 			&get_feat_ctx->max_queue_ext.max_queue_ext;
4023 		io_rx_num = min_t(u32, max_queue_ext->max_rx_sq_num,
4024 				  max_queue_ext->max_rx_cq_num);
4025 
4026 		io_tx_sq_num = max_queue_ext->max_tx_sq_num;
4027 		io_tx_cq_num = max_queue_ext->max_tx_cq_num;
4028 	} else {
4029 		struct ena_admin_queue_feature_desc *max_queues =
4030 			&get_feat_ctx->max_queues;
4031 		io_tx_sq_num = max_queues->max_sq_num;
4032 		io_tx_cq_num = max_queues->max_cq_num;
4033 		io_rx_num = min_t(u32, io_tx_sq_num, io_tx_cq_num);
4034 	}
4035 
4036 	/* In case of LLQ use the llq fields for the tx SQ/CQ */
4037 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4038 		io_tx_sq_num = get_feat_ctx->llq.max_llq_num;
4039 
4040 	max_num_io_queues = min_t(u32, num_online_cpus(), ENA_MAX_NUM_IO_QUEUES);
4041 	max_num_io_queues = min_t(u32, max_num_io_queues, io_rx_num);
4042 	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_sq_num);
4043 	max_num_io_queues = min_t(u32, max_num_io_queues, io_tx_cq_num);
4044 	/* 1 IRQ for mgmnt and 1 IRQs for each IO direction */
4045 	max_num_io_queues = min_t(u32, max_num_io_queues, pci_msix_vec_count(pdev) - 1);
4046 
4047 	return max_num_io_queues;
4048 }
4049 
4050 static void ena_set_dev_offloads(struct ena_com_dev_get_features_ctx *feat,
4051 				 struct net_device *netdev)
4052 {
4053 	netdev_features_t dev_features = 0;
4054 
4055 	/* Set offload features */
4056 	if (feat->offload.tx &
4057 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)
4058 		dev_features |= NETIF_F_IP_CSUM;
4059 
4060 	if (feat->offload.tx &
4061 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)
4062 		dev_features |= NETIF_F_IPV6_CSUM;
4063 
4064 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK)
4065 		dev_features |= NETIF_F_TSO;
4066 
4067 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK)
4068 		dev_features |= NETIF_F_TSO6;
4069 
4070 	if (feat->offload.tx & ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_ECN_MASK)
4071 		dev_features |= NETIF_F_TSO_ECN;
4072 
4073 	if (feat->offload.rx_supported &
4074 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK)
4075 		dev_features |= NETIF_F_RXCSUM;
4076 
4077 	if (feat->offload.rx_supported &
4078 		ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK)
4079 		dev_features |= NETIF_F_RXCSUM;
4080 
4081 	netdev->features =
4082 		dev_features |
4083 		NETIF_F_SG |
4084 		NETIF_F_RXHASH |
4085 		NETIF_F_HIGHDMA;
4086 
4087 	netdev->hw_features |= netdev->features;
4088 	netdev->vlan_features |= netdev->features;
4089 }
4090 
4091 static void ena_set_conf_feat_params(struct ena_adapter *adapter,
4092 				     struct ena_com_dev_get_features_ctx *feat)
4093 {
4094 	struct net_device *netdev = adapter->netdev;
4095 
4096 	/* Copy mac address */
4097 	if (!is_valid_ether_addr(feat->dev_attr.mac_addr)) {
4098 		eth_hw_addr_random(netdev);
4099 		ether_addr_copy(adapter->mac_addr, netdev->dev_addr);
4100 	} else {
4101 		ether_addr_copy(adapter->mac_addr, feat->dev_attr.mac_addr);
4102 		eth_hw_addr_set(netdev, adapter->mac_addr);
4103 	}
4104 
4105 	/* Set offload features */
4106 	ena_set_dev_offloads(feat, netdev);
4107 
4108 	adapter->max_mtu = feat->dev_attr.max_mtu;
4109 	netdev->max_mtu = adapter->max_mtu;
4110 	netdev->min_mtu = ENA_MIN_MTU;
4111 }
4112 
4113 static int ena_rss_init_default(struct ena_adapter *adapter)
4114 {
4115 	struct ena_com_dev *ena_dev = adapter->ena_dev;
4116 	struct device *dev = &adapter->pdev->dev;
4117 	int rc, i;
4118 	u32 val;
4119 
4120 	rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
4121 	if (unlikely(rc)) {
4122 		dev_err(dev, "Cannot init indirect table\n");
4123 		goto err_rss_init;
4124 	}
4125 
4126 	for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
4127 		val = ethtool_rxfh_indir_default(i, adapter->num_io_queues);
4128 		rc = ena_com_indirect_table_fill_entry(ena_dev, i,
4129 						       ENA_IO_RXQ_IDX(val));
4130 		if (unlikely(rc)) {
4131 			dev_err(dev, "Cannot fill indirect table\n");
4132 			goto err_fill_indir;
4133 		}
4134 	}
4135 
4136 	rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_TOEPLITZ, NULL,
4137 					ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
4138 	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
4139 		dev_err(dev, "Cannot fill hash function\n");
4140 		goto err_fill_indir;
4141 	}
4142 
4143 	rc = ena_com_set_default_hash_ctrl(ena_dev);
4144 	if (unlikely(rc && (rc != -EOPNOTSUPP))) {
4145 		dev_err(dev, "Cannot fill hash control\n");
4146 		goto err_fill_indir;
4147 	}
4148 
4149 	return 0;
4150 
4151 err_fill_indir:
4152 	ena_com_rss_destroy(ena_dev);
4153 err_rss_init:
4154 
4155 	return rc;
4156 }
4157 
4158 static void ena_release_bars(struct ena_com_dev *ena_dev, struct pci_dev *pdev)
4159 {
4160 	int release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
4161 
4162 	pci_release_selected_regions(pdev, release_bars);
4163 }
4164 
4165 
4166 static void ena_calc_io_queue_size(struct ena_adapter *adapter,
4167 				   struct ena_com_dev_get_features_ctx *get_feat_ctx)
4168 {
4169 	struct ena_admin_feature_llq_desc *llq = &get_feat_ctx->llq;
4170 	struct ena_com_dev *ena_dev = adapter->ena_dev;
4171 	u32 tx_queue_size = ENA_DEFAULT_RING_SIZE;
4172 	u32 rx_queue_size = ENA_DEFAULT_RING_SIZE;
4173 	u32 max_tx_queue_size;
4174 	u32 max_rx_queue_size;
4175 
4176 	if (ena_dev->supported_features & BIT(ENA_ADMIN_MAX_QUEUES_EXT)) {
4177 		struct ena_admin_queue_ext_feature_fields *max_queue_ext =
4178 			&get_feat_ctx->max_queue_ext.max_queue_ext;
4179 		max_rx_queue_size = min_t(u32, max_queue_ext->max_rx_cq_depth,
4180 					  max_queue_ext->max_rx_sq_depth);
4181 		max_tx_queue_size = max_queue_ext->max_tx_cq_depth;
4182 
4183 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4184 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
4185 						  llq->max_llq_depth);
4186 		else
4187 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
4188 						  max_queue_ext->max_tx_sq_depth);
4189 
4190 		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4191 						 max_queue_ext->max_per_packet_tx_descs);
4192 		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4193 						 max_queue_ext->max_per_packet_rx_descs);
4194 	} else {
4195 		struct ena_admin_queue_feature_desc *max_queues =
4196 			&get_feat_ctx->max_queues;
4197 		max_rx_queue_size = min_t(u32, max_queues->max_cq_depth,
4198 					  max_queues->max_sq_depth);
4199 		max_tx_queue_size = max_queues->max_cq_depth;
4200 
4201 		if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4202 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
4203 						  llq->max_llq_depth);
4204 		else
4205 			max_tx_queue_size = min_t(u32, max_tx_queue_size,
4206 						  max_queues->max_sq_depth);
4207 
4208 		adapter->max_tx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4209 						 max_queues->max_packet_tx_descs);
4210 		adapter->max_rx_sgl_size = min_t(u16, ENA_PKT_MAX_BUFS,
4211 						 max_queues->max_packet_rx_descs);
4212 	}
4213 
4214 	max_tx_queue_size = rounddown_pow_of_two(max_tx_queue_size);
4215 	max_rx_queue_size = rounddown_pow_of_two(max_rx_queue_size);
4216 
4217 	tx_queue_size = clamp_val(tx_queue_size, ENA_MIN_RING_SIZE,
4218 				  max_tx_queue_size);
4219 	rx_queue_size = clamp_val(rx_queue_size, ENA_MIN_RING_SIZE,
4220 				  max_rx_queue_size);
4221 
4222 	tx_queue_size = rounddown_pow_of_two(tx_queue_size);
4223 	rx_queue_size = rounddown_pow_of_two(rx_queue_size);
4224 
4225 	adapter->max_tx_ring_size  = max_tx_queue_size;
4226 	adapter->max_rx_ring_size = max_rx_queue_size;
4227 	adapter->requested_tx_ring_size = tx_queue_size;
4228 	adapter->requested_rx_ring_size = rx_queue_size;
4229 }
4230 
4231 /* ena_probe - Device Initialization Routine
4232  * @pdev: PCI device information struct
4233  * @ent: entry in ena_pci_tbl
4234  *
4235  * Returns 0 on success, negative on failure
4236  *
4237  * ena_probe initializes an adapter identified by a pci_dev structure.
4238  * The OS initialization, configuring of the adapter private structure,
4239  * and a hardware reset occur.
4240  */
4241 static int ena_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
4242 {
4243 	struct ena_com_dev_get_features_ctx get_feat_ctx;
4244 	struct ena_com_dev *ena_dev = NULL;
4245 	struct ena_adapter *adapter;
4246 	struct net_device *netdev;
4247 	static int adapters_found;
4248 	u32 max_num_io_queues;
4249 	bool wd_state;
4250 	int bars, rc;
4251 
4252 	dev_dbg(&pdev->dev, "%s\n", __func__);
4253 
4254 	rc = pci_enable_device_mem(pdev);
4255 	if (rc) {
4256 		dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
4257 		return rc;
4258 	}
4259 
4260 	rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(ENA_MAX_PHYS_ADDR_SIZE_BITS));
4261 	if (rc) {
4262 		dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", rc);
4263 		goto err_disable_device;
4264 	}
4265 
4266 	pci_set_master(pdev);
4267 
4268 	ena_dev = vzalloc(sizeof(*ena_dev));
4269 	if (!ena_dev) {
4270 		rc = -ENOMEM;
4271 		goto err_disable_device;
4272 	}
4273 
4274 	bars = pci_select_bars(pdev, IORESOURCE_MEM) & ENA_BAR_MASK;
4275 	rc = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
4276 	if (rc) {
4277 		dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
4278 			rc);
4279 		goto err_free_ena_dev;
4280 	}
4281 
4282 	ena_dev->reg_bar = devm_ioremap(&pdev->dev,
4283 					pci_resource_start(pdev, ENA_REG_BAR),
4284 					pci_resource_len(pdev, ENA_REG_BAR));
4285 	if (!ena_dev->reg_bar) {
4286 		dev_err(&pdev->dev, "Failed to remap regs bar\n");
4287 		rc = -EFAULT;
4288 		goto err_free_region;
4289 	}
4290 
4291 	ena_dev->ena_min_poll_delay_us = ENA_ADMIN_POLL_DELAY_US;
4292 
4293 	ena_dev->dmadev = &pdev->dev;
4294 
4295 	netdev = alloc_etherdev_mq(sizeof(struct ena_adapter), ENA_MAX_RINGS);
4296 	if (!netdev) {
4297 		dev_err(&pdev->dev, "alloc_etherdev_mq failed\n");
4298 		rc = -ENOMEM;
4299 		goto err_free_region;
4300 	}
4301 
4302 	SET_NETDEV_DEV(netdev, &pdev->dev);
4303 	adapter = netdev_priv(netdev);
4304 	adapter->ena_dev = ena_dev;
4305 	adapter->netdev = netdev;
4306 	adapter->pdev = pdev;
4307 	adapter->msg_enable = DEFAULT_MSG_ENABLE;
4308 
4309 	ena_dev->net_device = netdev;
4310 
4311 	pci_set_drvdata(pdev, adapter);
4312 
4313 	rc = ena_device_init(ena_dev, pdev, &get_feat_ctx, &wd_state);
4314 	if (rc) {
4315 		dev_err(&pdev->dev, "ENA device init failed\n");
4316 		if (rc == -ETIME)
4317 			rc = -EPROBE_DEFER;
4318 		goto err_netdev_destroy;
4319 	}
4320 
4321 	rc = ena_map_llq_mem_bar(pdev, ena_dev, bars);
4322 	if (rc) {
4323 		dev_err(&pdev->dev, "ENA llq bar mapping failed\n");
4324 		goto err_device_destroy;
4325 	}
4326 
4327 	/* Initial TX and RX interrupt delay. Assumes 1 usec granularity.
4328 	 * Updated during device initialization with the real granularity
4329 	 */
4330 	ena_dev->intr_moder_tx_interval = ENA_INTR_INITIAL_TX_INTERVAL_USECS;
4331 	ena_dev->intr_moder_rx_interval = ENA_INTR_INITIAL_RX_INTERVAL_USECS;
4332 	ena_dev->intr_delay_resolution = ENA_DEFAULT_INTR_DELAY_RESOLUTION;
4333 	max_num_io_queues = ena_calc_max_io_queue_num(pdev, ena_dev, &get_feat_ctx);
4334 	ena_calc_io_queue_size(adapter, &get_feat_ctx);
4335 	if (unlikely(!max_num_io_queues)) {
4336 		rc = -EFAULT;
4337 		goto err_device_destroy;
4338 	}
4339 
4340 	ena_set_conf_feat_params(adapter, &get_feat_ctx);
4341 
4342 	adapter->reset_reason = ENA_REGS_RESET_NORMAL;
4343 
4344 	adapter->num_io_queues = max_num_io_queues;
4345 	adapter->max_num_io_queues = max_num_io_queues;
4346 	adapter->last_monitored_tx_qid = 0;
4347 
4348 	adapter->xdp_first_ring = 0;
4349 	adapter->xdp_num_queues = 0;
4350 
4351 	adapter->rx_copybreak = ENA_DEFAULT_RX_COPYBREAK;
4352 	if (ena_dev->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV)
4353 		adapter->disable_meta_caching =
4354 			!!(get_feat_ctx.llq.accel_mode.u.get.supported_flags &
4355 			   BIT(ENA_ADMIN_DISABLE_META_CACHING));
4356 
4357 	adapter->wd_state = wd_state;
4358 
4359 	snprintf(adapter->name, ENA_NAME_MAX_LEN, "ena_%d", adapters_found);
4360 
4361 	rc = ena_com_init_interrupt_moderation(adapter->ena_dev);
4362 	if (rc) {
4363 		dev_err(&pdev->dev,
4364 			"Failed to query interrupt moderation feature\n");
4365 		goto err_device_destroy;
4366 	}
4367 	ena_init_io_rings(adapter,
4368 			  0,
4369 			  adapter->xdp_num_queues +
4370 			  adapter->num_io_queues);
4371 
4372 	netdev->netdev_ops = &ena_netdev_ops;
4373 	netdev->watchdog_timeo = TX_TIMEOUT;
4374 	ena_set_ethtool_ops(netdev);
4375 
4376 	netdev->priv_flags |= IFF_UNICAST_FLT;
4377 
4378 	u64_stats_init(&adapter->syncp);
4379 
4380 	rc = ena_enable_msix_and_set_admin_interrupts(adapter);
4381 	if (rc) {
4382 		dev_err(&pdev->dev,
4383 			"Failed to enable and set the admin interrupts\n");
4384 		goto err_worker_destroy;
4385 	}
4386 	rc = ena_rss_init_default(adapter);
4387 	if (rc && (rc != -EOPNOTSUPP)) {
4388 		dev_err(&pdev->dev, "Cannot init RSS rc: %d\n", rc);
4389 		goto err_free_msix;
4390 	}
4391 
4392 	ena_config_debug_area(adapter);
4393 
4394 	if (ena_xdp_legal_queue_count(adapter, adapter->num_io_queues))
4395 		netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
4396 				       NETDEV_XDP_ACT_REDIRECT;
4397 
4398 	memcpy(adapter->netdev->perm_addr, adapter->mac_addr, netdev->addr_len);
4399 
4400 	netif_carrier_off(netdev);
4401 
4402 	rc = register_netdev(netdev);
4403 	if (rc) {
4404 		dev_err(&pdev->dev, "Cannot register net device\n");
4405 		goto err_rss;
4406 	}
4407 
4408 	INIT_WORK(&adapter->reset_task, ena_fw_reset_device);
4409 
4410 	adapter->last_keep_alive_jiffies = jiffies;
4411 	adapter->keep_alive_timeout = ENA_DEVICE_KALIVE_TIMEOUT;
4412 	adapter->missing_tx_completion_to = TX_TIMEOUT;
4413 	adapter->missing_tx_completion_threshold = MAX_NUM_OF_TIMEOUTED_PACKETS;
4414 
4415 	ena_update_hints(adapter, &get_feat_ctx.hw_hints);
4416 
4417 	timer_setup(&adapter->timer_service, ena_timer_service, 0);
4418 	mod_timer(&adapter->timer_service, round_jiffies(jiffies + HZ));
4419 
4420 	dev_info(&pdev->dev,
4421 		 "%s found at mem %lx, mac addr %pM\n",
4422 		 DEVICE_NAME, (long)pci_resource_start(pdev, 0),
4423 		 netdev->dev_addr);
4424 
4425 	set_bit(ENA_FLAG_DEVICE_RUNNING, &adapter->flags);
4426 
4427 	adapters_found++;
4428 
4429 	return 0;
4430 
4431 err_rss:
4432 	ena_com_delete_debug_area(ena_dev);
4433 	ena_com_rss_destroy(ena_dev);
4434 err_free_msix:
4435 	ena_com_dev_reset(ena_dev, ENA_REGS_RESET_INIT_ERR);
4436 	/* stop submitting admin commands on a device that was reset */
4437 	ena_com_set_admin_running_state(ena_dev, false);
4438 	ena_free_mgmnt_irq(adapter);
4439 	ena_disable_msix(adapter);
4440 err_worker_destroy:
4441 	del_timer(&adapter->timer_service);
4442 err_device_destroy:
4443 	ena_com_delete_host_info(ena_dev);
4444 	ena_com_admin_destroy(ena_dev);
4445 err_netdev_destroy:
4446 	free_netdev(netdev);
4447 err_free_region:
4448 	ena_release_bars(ena_dev, pdev);
4449 err_free_ena_dev:
4450 	vfree(ena_dev);
4451 err_disable_device:
4452 	pci_disable_device(pdev);
4453 	return rc;
4454 }
4455 
4456 /*****************************************************************************/
4457 
4458 /* __ena_shutoff - Helper used in both PCI remove/shutdown routines
4459  * @pdev: PCI device information struct
4460  * @shutdown: Is it a shutdown operation? If false, means it is a removal
4461  *
4462  * __ena_shutoff is a helper routine that does the real work on shutdown and
4463  * removal paths; the difference between those paths is with regards to whether
4464  * dettach or unregister the netdevice.
4465  */
4466 static void __ena_shutoff(struct pci_dev *pdev, bool shutdown)
4467 {
4468 	struct ena_adapter *adapter = pci_get_drvdata(pdev);
4469 	struct ena_com_dev *ena_dev;
4470 	struct net_device *netdev;
4471 
4472 	ena_dev = adapter->ena_dev;
4473 	netdev = adapter->netdev;
4474 
4475 #ifdef CONFIG_RFS_ACCEL
4476 	if ((adapter->msix_vecs >= 1) && (netdev->rx_cpu_rmap)) {
4477 		free_irq_cpu_rmap(netdev->rx_cpu_rmap);
4478 		netdev->rx_cpu_rmap = NULL;
4479 	}
4480 #endif /* CONFIG_RFS_ACCEL */
4481 
4482 	/* Make sure timer and reset routine won't be called after
4483 	 * freeing device resources.
4484 	 */
4485 	del_timer_sync(&adapter->timer_service);
4486 	cancel_work_sync(&adapter->reset_task);
4487 
4488 	rtnl_lock(); /* lock released inside the below if-else block */
4489 	adapter->reset_reason = ENA_REGS_RESET_SHUTDOWN;
4490 	ena_destroy_device(adapter, true);
4491 	if (shutdown) {
4492 		netif_device_detach(netdev);
4493 		dev_close(netdev);
4494 		rtnl_unlock();
4495 	} else {
4496 		rtnl_unlock();
4497 		unregister_netdev(netdev);
4498 		free_netdev(netdev);
4499 	}
4500 
4501 	ena_com_rss_destroy(ena_dev);
4502 
4503 	ena_com_delete_debug_area(ena_dev);
4504 
4505 	ena_com_delete_host_info(ena_dev);
4506 
4507 	ena_release_bars(ena_dev, pdev);
4508 
4509 	pci_disable_device(pdev);
4510 
4511 	vfree(ena_dev);
4512 }
4513 
4514 /* ena_remove - Device Removal Routine
4515  * @pdev: PCI device information struct
4516  *
4517  * ena_remove is called by the PCI subsystem to alert the driver
4518  * that it should release a PCI device.
4519  */
4520 
4521 static void ena_remove(struct pci_dev *pdev)
4522 {
4523 	__ena_shutoff(pdev, false);
4524 }
4525 
4526 /* ena_shutdown - Device Shutdown Routine
4527  * @pdev: PCI device information struct
4528  *
4529  * ena_shutdown is called by the PCI subsystem to alert the driver that
4530  * a shutdown/reboot (or kexec) is happening and device must be disabled.
4531  */
4532 
4533 static void ena_shutdown(struct pci_dev *pdev)
4534 {
4535 	__ena_shutoff(pdev, true);
4536 }
4537 
4538 /* ena_suspend - PM suspend callback
4539  * @dev_d: Device information struct
4540  */
4541 static int __maybe_unused ena_suspend(struct device *dev_d)
4542 {
4543 	struct pci_dev *pdev = to_pci_dev(dev_d);
4544 	struct ena_adapter *adapter = pci_get_drvdata(pdev);
4545 
4546 	ena_increase_stat(&adapter->dev_stats.suspend, 1, &adapter->syncp);
4547 
4548 	rtnl_lock();
4549 	if (unlikely(test_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags))) {
4550 		dev_err(&pdev->dev,
4551 			"Ignoring device reset request as the device is being suspended\n");
4552 		clear_bit(ENA_FLAG_TRIGGER_RESET, &adapter->flags);
4553 	}
4554 	ena_destroy_device(adapter, true);
4555 	rtnl_unlock();
4556 	return 0;
4557 }
4558 
4559 /* ena_resume - PM resume callback
4560  * @dev_d: Device information struct
4561  */
4562 static int __maybe_unused ena_resume(struct device *dev_d)
4563 {
4564 	struct ena_adapter *adapter = dev_get_drvdata(dev_d);
4565 	int rc;
4566 
4567 	ena_increase_stat(&adapter->dev_stats.resume, 1, &adapter->syncp);
4568 
4569 	rtnl_lock();
4570 	rc = ena_restore_device(adapter);
4571 	rtnl_unlock();
4572 	return rc;
4573 }
4574 
4575 static SIMPLE_DEV_PM_OPS(ena_pm_ops, ena_suspend, ena_resume);
4576 
4577 static struct pci_driver ena_pci_driver = {
4578 	.name		= DRV_MODULE_NAME,
4579 	.id_table	= ena_pci_tbl,
4580 	.probe		= ena_probe,
4581 	.remove		= ena_remove,
4582 	.shutdown	= ena_shutdown,
4583 	.driver.pm	= &ena_pm_ops,
4584 	.sriov_configure = pci_sriov_configure_simple,
4585 };
4586 
4587 static int __init ena_init(void)
4588 {
4589 	int ret;
4590 
4591 	ena_wq = create_singlethread_workqueue(DRV_MODULE_NAME);
4592 	if (!ena_wq) {
4593 		pr_err("Failed to create workqueue\n");
4594 		return -ENOMEM;
4595 	}
4596 
4597 	ret = pci_register_driver(&ena_pci_driver);
4598 	if (ret)
4599 		destroy_workqueue(ena_wq);
4600 
4601 	return ret;
4602 }
4603 
4604 static void __exit ena_cleanup(void)
4605 {
4606 	pci_unregister_driver(&ena_pci_driver);
4607 
4608 	if (ena_wq) {
4609 		destroy_workqueue(ena_wq);
4610 		ena_wq = NULL;
4611 	}
4612 }
4613 
4614 /******************************************************************************
4615  ******************************** AENQ Handlers *******************************
4616  *****************************************************************************/
4617 /* ena_update_on_link_change:
4618  * Notify the network interface about the change in link status
4619  */
4620 static void ena_update_on_link_change(void *adapter_data,
4621 				      struct ena_admin_aenq_entry *aenq_e)
4622 {
4623 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4624 	struct ena_admin_aenq_link_change_desc *aenq_desc =
4625 		(struct ena_admin_aenq_link_change_desc *)aenq_e;
4626 	int status = aenq_desc->flags &
4627 		ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
4628 
4629 	if (status) {
4630 		netif_dbg(adapter, ifup, adapter->netdev, "%s\n", __func__);
4631 		set_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4632 		if (!test_bit(ENA_FLAG_ONGOING_RESET, &adapter->flags))
4633 			netif_carrier_on(adapter->netdev);
4634 	} else {
4635 		clear_bit(ENA_FLAG_LINK_UP, &adapter->flags);
4636 		netif_carrier_off(adapter->netdev);
4637 	}
4638 }
4639 
4640 static void ena_keep_alive_wd(void *adapter_data,
4641 			      struct ena_admin_aenq_entry *aenq_e)
4642 {
4643 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4644 	struct ena_admin_aenq_keep_alive_desc *desc;
4645 	u64 rx_drops;
4646 	u64 tx_drops;
4647 
4648 	desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
4649 	adapter->last_keep_alive_jiffies = jiffies;
4650 
4651 	rx_drops = ((u64)desc->rx_drops_high << 32) | desc->rx_drops_low;
4652 	tx_drops = ((u64)desc->tx_drops_high << 32) | desc->tx_drops_low;
4653 
4654 	u64_stats_update_begin(&adapter->syncp);
4655 	/* These stats are accumulated by the device, so the counters indicate
4656 	 * all drops since last reset.
4657 	 */
4658 	adapter->dev_stats.rx_drops = rx_drops;
4659 	adapter->dev_stats.tx_drops = tx_drops;
4660 	u64_stats_update_end(&adapter->syncp);
4661 }
4662 
4663 static void ena_notification(void *adapter_data,
4664 			     struct ena_admin_aenq_entry *aenq_e)
4665 {
4666 	struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
4667 	struct ena_admin_ena_hw_hints *hints;
4668 
4669 	WARN(aenq_e->aenq_common_desc.group != ENA_ADMIN_NOTIFICATION,
4670 	     "Invalid group(%x) expected %x\n",
4671 	     aenq_e->aenq_common_desc.group,
4672 	     ENA_ADMIN_NOTIFICATION);
4673 
4674 	switch (aenq_e->aenq_common_desc.syndrome) {
4675 	case ENA_ADMIN_UPDATE_HINTS:
4676 		hints = (struct ena_admin_ena_hw_hints *)
4677 			(&aenq_e->inline_data_w4);
4678 		ena_update_hints(adapter, hints);
4679 		break;
4680 	default:
4681 		netif_err(adapter, drv, adapter->netdev,
4682 			  "Invalid aenq notification link state %d\n",
4683 			  aenq_e->aenq_common_desc.syndrome);
4684 	}
4685 }
4686 
4687 /* This handler will called for unknown event group or unimplemented handlers*/
4688 static void unimplemented_aenq_handler(void *data,
4689 				       struct ena_admin_aenq_entry *aenq_e)
4690 {
4691 	struct ena_adapter *adapter = (struct ena_adapter *)data;
4692 
4693 	netif_err(adapter, drv, adapter->netdev,
4694 		  "Unknown event was received or event with unimplemented handler\n");
4695 }
4696 
4697 static struct ena_aenq_handlers aenq_handlers = {
4698 	.handlers = {
4699 		[ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
4700 		[ENA_ADMIN_NOTIFICATION] = ena_notification,
4701 		[ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
4702 	},
4703 	.unimplemented_handler = unimplemented_aenq_handler
4704 };
4705 
4706 module_init(ena_init);
4707 module_exit(ena_cleanup);
4708